From 20c40298e609fcee9364506b3b430a06b2e61035 Mon Sep 17 00:00:00 2001 From: Pearl Dsilva Date: Wed, 10 Apr 2024 09:47:56 -0400 Subject: [PATCH] CKS Enhancements - Phase 2: Add and Remove external nodes to and from a kubernetes cluster --------- Co-authored-by: nvazquez --- .../kubernetes/cluster/KubernetesCluster.java | 15 + .../com/cloud/network/NetworkService.java | 2 + .../cloud/template/TemplateApiService.java | 19 +- .../api/ApiCommandResourceType.java | 3 +- .../apache/cloudstack/api/ApiConstants.java | 3 + .../api/command/user/iso/DetachIsoCmd.java | 2 +- .../user/template/UpdateTemplateCmd.java | 9 + .../response/KubernetesUserVmResponse.java | 24 ++ .../cloud/agent/api/HandleCksIsoCommand.java | 34 ++ .../resource/virtualnetwork/VRScripts.java | 3 + .../VirtualRoutingResource.java | 12 + .../cloud/network/dao/FirewallRulesDao.java | 2 + .../network/dao/FirewallRulesDaoImpl.java | 17 + .../rules/dao/PortForwardingRulesDao.java | 2 + .../rules/dao/PortForwardingRulesDaoImpl.java | 11 + .../META-INF/db/schema-41810to41900.sql | 1 + .../cluster/KubernetesClusterEventTypes.java | 2 + .../cluster/KubernetesClusterManagerImpl.java | 107 ++++++- .../cluster/KubernetesClusterService.java | 18 ++ .../cluster/KubernetesClusterVmMapVO.java | 11 + .../KubernetesClusterActionWorker.java | 272 ++++++++++++++-- .../KubernetesClusterAddWorker.java | 301 ++++++++++++++++++ .../KubernetesClusterDestroyWorker.java | 2 +- .../KubernetesClusterRemoveWorker.java | 166 ++++++++++ ...esClusterResourceModifierActionWorker.java | 133 ++------ .../KubernetesClusterScaleWorker.java | 14 +- .../KubernetesClusterStartWorker.java | 4 +- .../AddNodesToKubernetesClusterCmd.java | 132 ++++++++ .../RemoveNodesFromKubernetesClusterCmd.java | 109 +++++++ .../response/KubernetesClusterResponse.java | 18 +- .../src/main/resources/conf/k8s-node.yml | 13 + .../resources/script/remove-node-from-cluster | 27 ++ .../main/resources/script/validate-cks-node | 45 +++ .../com/cloud/network/NetworkServiceImpl.java | 21 ++ .../network/router/CommandSetupHelper.java | 8 + .../cloud/template/TemplateManagerImpl.java | 61 ++-- .../com/cloud/vpc/MockNetworkManagerImpl.java | 5 + systemvm/debian/opt/cloud/bin/cks_iso.sh | 34 ++ .../debian/opt/cloud/bin/cs/CsGuestNetwork.py | 3 +- .../ubuntu/22.04/scripts/setup_template.sh | 27 ++ ui/public/locales/en.json | 18 +- ui/src/config/section/compute.js | 20 ++ ui/src/config/section/image.js | 2 +- ui/src/utils/plugins.js | 18 +- ui/src/views/compute/KubernetesAddNodes.vue | 177 ++++++++++ .../views/compute/KubernetesRemoveNodes.vue | 151 +++++++++ ui/src/views/compute/RegisterUserData.vue | 9 +- .../views/image/RegisterOrUploadTemplate.vue | 2 +- ui/src/views/image/UpdateTemplate.vue | 8 +- 49 files changed, 1918 insertions(+), 179 deletions(-) rename {plugins/integrations/kubernetes-service => api}/src/main/java/com/cloud/kubernetes/cluster/KubernetesCluster.java (86%) create mode 100644 api/src/main/java/org/apache/cloudstack/api/response/KubernetesUserVmResponse.java create mode 100644 core/src/main/java/com/cloud/agent/api/HandleCksIsoCommand.java create mode 100644 plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterAddWorker.java create mode 100644 plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterRemoveWorker.java create mode 100644 plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/AddNodesToKubernetesClusterCmd.java create mode 100644 plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/RemoveNodesFromKubernetesClusterCmd.java create mode 100644 plugins/integrations/kubernetes-service/src/main/resources/script/remove-node-from-cluster create mode 100644 plugins/integrations/kubernetes-service/src/main/resources/script/validate-cks-node create mode 100644 systemvm/debian/opt/cloud/bin/cks_iso.sh create mode 100644 tools/appliance/cks/ubuntu/22.04/scripts/setup_template.sh create mode 100644 ui/src/views/compute/KubernetesAddNodes.vue create mode 100644 ui/src/views/compute/KubernetesRemoveNodes.vue diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesCluster.java b/api/src/main/java/com/cloud/kubernetes/cluster/KubernetesCluster.java similarity index 86% rename from plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesCluster.java rename to api/src/main/java/com/cloud/kubernetes/cluster/KubernetesCluster.java index d4545589ced..c01d8f82a26 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesCluster.java +++ b/api/src/main/java/com/cloud/kubernetes/cluster/KubernetesCluster.java @@ -44,6 +44,8 @@ public interface KubernetesCluster extends ControlledEntity, com.cloud.utils.fsm AutoscaleRequested, ScaleUpRequested, ScaleDownRequested, + AddNodeRequested, + RemoveNodeRequested, UpgradeRequested, OperationSucceeded, OperationFailed, @@ -59,6 +61,8 @@ public interface KubernetesCluster extends ControlledEntity, com.cloud.utils.fsm Stopped("All resources for the Kubernetes cluster are destroyed, Kubernetes cluster may still have ephemeral resource like persistent volumes provisioned"), Scaling("Transient state in which resources are either getting scaled up/down"), Upgrading("Transient state in which cluster is getting upgraded"), + Importing("Transient state in which additional nodes are added as worker nodes to a cluster"), + RemovingNodes("Transient state in which additional nodes are removed from a cluster"), Alert("State to represent Kubernetes clusters which are not in expected desired state (operationally in active control place, stopped cluster VM's etc)."), Recovering("State in which Kubernetes cluster is recovering from alert state"), Destroyed("End state of Kubernetes cluster in which all resources are destroyed, cluster will not be usable further"), @@ -96,6 +100,17 @@ public interface KubernetesCluster extends ControlledEntity, com.cloud.utils.fsm s_fsm.addTransition(State.Upgrading, Event.OperationSucceeded, State.Running); s_fsm.addTransition(State.Upgrading, Event.OperationFailed, State.Alert); + s_fsm.addTransition(State.Running, Event.AddNodeRequested, State.Importing); + s_fsm.addTransition(State.Alert, Event.AddNodeRequested, State.Importing); + s_fsm.addTransition(State.Importing, Event.OperationSucceeded, State.Running); + s_fsm.addTransition(State.Importing, Event.OperationFailed, State.Running); + s_fsm.addTransition(State.Alert, Event.OperationSucceeded, State.Running); + + s_fsm.addTransition(State.Running, Event.RemoveNodeRequested, State.RemovingNodes); + s_fsm.addTransition(State.Alert, Event.RemoveNodeRequested, State.RemovingNodes); + s_fsm.addTransition(State.RemovingNodes, Event.OperationSucceeded, State.Running); + s_fsm.addTransition(State.RemovingNodes, Event.OperationFailed, State.Running); + s_fsm.addTransition(State.Alert, Event.RecoveryRequested, State.Recovering); s_fsm.addTransition(State.Recovering, Event.OperationSucceeded, State.Running); s_fsm.addTransition(State.Recovering, Event.OperationFailed, State.Alert); diff --git a/api/src/main/java/com/cloud/network/NetworkService.java b/api/src/main/java/com/cloud/network/NetworkService.java index 51799e25cda..02ca13cb9a4 100644 --- a/api/src/main/java/com/cloud/network/NetworkService.java +++ b/api/src/main/java/com/cloud/network/NetworkService.java @@ -263,4 +263,6 @@ public interface NetworkService { InternalLoadBalancerElementService getInternalLoadBalancerElementByNetworkServiceProviderId(long networkProviderId); InternalLoadBalancerElementService getInternalLoadBalancerElementById(long providerId); List getInternalLoadBalancerElements(); + + boolean handleCksIsoOnNetworkVirtualRouter(Long virtualRouterId, boolean mount) throws ResourceUnavailableException; } diff --git a/api/src/main/java/com/cloud/template/TemplateApiService.java b/api/src/main/java/com/cloud/template/TemplateApiService.java index 5b494c308c3..6138f24c92b 100644 --- a/api/src/main/java/com/cloud/template/TemplateApiService.java +++ b/api/src/main/java/com/cloud/template/TemplateApiService.java @@ -58,10 +58,23 @@ public interface TemplateApiService { VirtualMachineTemplate prepareTemplate(long templateId, long zoneId, Long storageId); + /** + * Detach ISO from VM + * @param vmId id of the VM + * @param isoId id of the ISO (when passed). If it is not passed, it will get it from user_vm table + * @param extraParams forced, isVirtualRouter + * @return true when operation succeeds, false if not + */ + boolean detachIso(long vmId, Long isoId, Boolean... extraParams); - boolean detachIso(long vmId, boolean forced); - - boolean attachIso(long isoId, long vmId, boolean forced); + /** + * Attach ISO to a VM + * @param isoId id of the ISO to attach + * @param vmId id of the VM to attach the ISO to + * @param extraParams: forced, isVirtualRouter + * @return true when operation succeeds, false if not + */ + boolean attachIso(long isoId, long vmId, Boolean... extraParams); /** * Deletes a template diff --git a/api/src/main/java/org/apache/cloudstack/api/ApiCommandResourceType.java b/api/src/main/java/org/apache/cloudstack/api/ApiCommandResourceType.java index aafc039b36b..38efa428726 100644 --- a/api/src/main/java/org/apache/cloudstack/api/ApiCommandResourceType.java +++ b/api/src/main/java/org/apache/cloudstack/api/ApiCommandResourceType.java @@ -81,7 +81,8 @@ public enum ApiCommandResourceType { ManagementServer(org.apache.cloudstack.management.ManagementServerHost.class), ObjectStore(org.apache.cloudstack.storage.object.ObjectStore.class), Bucket(org.apache.cloudstack.storage.object.Bucket.class), - QuotaTariff(org.apache.cloudstack.quota.QuotaTariff.class); + QuotaTariff(org.apache.cloudstack.quota.QuotaTariff.class), + KubernetesCluster(com.cloud.kubernetes.cluster.KubernetesCluster.class); private final Class clazz; diff --git a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java index b148d0ccbea..72b2d466dd3 100644 --- a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java +++ b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java @@ -305,6 +305,7 @@ public class ApiConstants { public static final String MIGRATIONS = "migrations"; public static final String MEMORY = "memory"; public static final String MODE = "mode"; + public static final String MOUNT_CKS_ISO_ON_VR = "mountcksisoonvr"; public static final String NSX_MODE = "nsxmode"; public static final String NSX_ENABLED = "isnsxenabled"; public static final String NAME = "name"; @@ -1051,6 +1052,8 @@ public class ApiConstants { public static final String NODE_IDS = "nodeids"; public static final String CONTROL_NODES = "controlnodes"; public static final String ETCD_NODES = "etcdnodes"; + public static final String EXTERNAL_NODES = "externalnodes"; + public static final String IS_EXTERNAL_NODE = "isexternalnode"; public static final String MIN_SEMANTIC_VERSION = "minimumsemanticversion"; public static final String MIN_KUBERNETES_VERSION_ID = "minimumkubernetesversionid"; public static final String NODE_ROOT_DISK_SIZE = "noderootdisksize"; diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/iso/DetachIsoCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/iso/DetachIsoCmd.java index 292e1c6f099..78f1a4bcdee 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/iso/DetachIsoCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/iso/DetachIsoCmd.java @@ -104,7 +104,7 @@ public class DetachIsoCmd extends BaseAsyncCmd implements UserCmd { @Override public void execute() { - boolean result = _templateService.detachIso(virtualMachineId, isForced()); + boolean result = _templateService.detachIso(virtualMachineId, null, isForced()); if (result) { UserVm userVm = _entityMgr.findById(UserVm.class, virtualMachineId); UserVmResponse response = _responseGenerator.createUserVmResponse(getResponseView(), "virtualmachine", userVm).get(0); diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/template/UpdateTemplateCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/template/UpdateTemplateCmd.java index dbbd771293a..93c860a7bf8 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/template/UpdateTemplateCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/template/UpdateTemplateCmd.java @@ -46,6 +46,11 @@ public class UpdateTemplateCmd extends BaseUpdateTemplateOrIsoCmd implements Use @Parameter(name = ApiConstants.TEMPLATE_TAG, type = CommandType.STRING, description = "the tag for this template.", since = "4.20.0") private String templateTag; + @Parameter(name = ApiConstants.FOR_CKS, type = CommandType.BOOLEAN, + description = "indicates that the template can be used for deployment of CKS clusters", + since = "4.20.0") + private Boolean forCks; + ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// ///////////////////////////////////////////////////// @@ -63,6 +68,10 @@ public class UpdateTemplateCmd extends BaseUpdateTemplateOrIsoCmd implements Use return templateTag; } + public boolean getForCks() { + return Boolean.TRUE.equals(forCks); + } + ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// diff --git a/api/src/main/java/org/apache/cloudstack/api/response/KubernetesUserVmResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/KubernetesUserVmResponse.java new file mode 100644 index 00000000000..61edbafd48f --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/response/KubernetesUserVmResponse.java @@ -0,0 +1,24 @@ +package org.apache.cloudstack.api.response; + +import com.cloud.network.router.VirtualRouter; +import com.cloud.serializer.Param; +import com.cloud.uservm.UserVm; +import com.cloud.vm.VirtualMachine; +import com.google.gson.annotations.SerializedName; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.EntityReference; + +@EntityReference(value = {VirtualMachine.class, UserVm.class, VirtualRouter.class}) +public class KubernetesUserVmResponse extends UserVmResponse { + @SerializedName(ApiConstants.IS_EXTERNAL_NODE) + @Param(description = "If the VM is an externally added node") + private boolean isExternalNode; + + public boolean isExternalNode() { + return isExternalNode; + } + + public void setExternalNode(boolean externalNode) { + isExternalNode = externalNode; + } +} diff --git a/core/src/main/java/com/cloud/agent/api/HandleCksIsoCommand.java b/core/src/main/java/com/cloud/agent/api/HandleCksIsoCommand.java new file mode 100644 index 00000000000..16942bb05d4 --- /dev/null +++ b/core/src/main/java/com/cloud/agent/api/HandleCksIsoCommand.java @@ -0,0 +1,34 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// +package com.cloud.agent.api; + +import com.cloud.agent.api.routing.NetworkElementCommand; + +public class HandleCksIsoCommand extends NetworkElementCommand { + + private boolean mountCksIso; + + public HandleCksIsoCommand(boolean mountCksIso) { + this.mountCksIso = mountCksIso; + } + + public boolean isMountCksIso() { + return mountCksIso; + } +} diff --git a/core/src/main/java/com/cloud/agent/resource/virtualnetwork/VRScripts.java b/core/src/main/java/com/cloud/agent/resource/virtualnetwork/VRScripts.java index ebe5e9a7ec9..1396a2aa002 100644 --- a/core/src/main/java/com/cloud/agent/resource/virtualnetwork/VRScripts.java +++ b/core/src/main/java/com/cloud/agent/resource/virtualnetwork/VRScripts.java @@ -81,4 +81,7 @@ public class VRScripts { public static final String VR_UPDATE_INTERFACE_CONFIG = "update_interface_config.sh"; public static final String ROUTER_FILESYSTEM_WRITABLE_CHECK = "filesystem_writable_check.py"; + + // CKS ISO mount + public static final String CKS_ISO_MOUNT_SERVE = "cks_iso.sh"; } diff --git a/core/src/main/java/com/cloud/agent/resource/virtualnetwork/VirtualRoutingResource.java b/core/src/main/java/com/cloud/agent/resource/virtualnetwork/VirtualRoutingResource.java index 3c86b3a0dcc..cbce8cbc39d 100644 --- a/core/src/main/java/com/cloud/agent/resource/virtualnetwork/VirtualRoutingResource.java +++ b/core/src/main/java/com/cloud/agent/resource/virtualnetwork/VirtualRoutingResource.java @@ -34,6 +34,7 @@ import java.util.concurrent.locks.ReentrantLock; import javax.naming.ConfigurationException; +import com.cloud.agent.api.HandleCksIsoCommand; import com.cloud.agent.api.routing.UpdateNetworkCommand; import com.cloud.agent.api.to.IpAddressTO; import com.cloud.network.router.VirtualRouter; @@ -144,6 +145,10 @@ public class VirtualRoutingResource { return execute((UpdateNetworkCommand) cmd); } + if (cmd instanceof HandleCksIsoCommand) { + return execute((HandleCksIsoCommand) cmd); + } + if (_vrAggregateCommandsSet.containsKey(routerName)) { _vrAggregateCommandsSet.get(routerName).add(cmd); aggregated = true; @@ -171,6 +176,13 @@ public class VirtualRoutingResource { } } + protected Answer execute(final HandleCksIsoCommand cmd) { + String routerIp = getRouterSshControlIp(cmd); + s_logger.info("Attempting to mount CKS ISO on Virtual Router"); + ExecutionResult result = _vrDeployer.executeInVR(routerIp, VRScripts.CKS_ISO_MOUNT_SERVE, String.valueOf(cmd.isMountCksIso())); + return new Answer(cmd, result.isSuccess(), result.getDetails()); + } + private Answer execute(final SetupKeyStoreCommand cmd) { final String args = String.format("/usr/local/cloud/systemvm/conf/agent.properties " + "/usr/local/cloud/systemvm/conf/%s " + diff --git a/engine/schema/src/main/java/com/cloud/network/dao/FirewallRulesDao.java b/engine/schema/src/main/java/com/cloud/network/dao/FirewallRulesDao.java index 21200dbf9b5..0c59d10c80b 100644 --- a/engine/schema/src/main/java/com/cloud/network/dao/FirewallRulesDao.java +++ b/engine/schema/src/main/java/com/cloud/network/dao/FirewallRulesDao.java @@ -72,4 +72,6 @@ public interface FirewallRulesDao extends GenericDao { void loadSourceCidrs(FirewallRuleVO rule); void loadDestinationCidrs(FirewallRuleVO rule); + + FirewallRuleVO findByNetworkIdAndPorts(long networkId, int startPort, int endPort); } diff --git a/engine/schema/src/main/java/com/cloud/network/dao/FirewallRulesDaoImpl.java b/engine/schema/src/main/java/com/cloud/network/dao/FirewallRulesDaoImpl.java index 3ac860b08c5..78d86ced32f 100644 --- a/engine/schema/src/main/java/com/cloud/network/dao/FirewallRulesDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/network/dao/FirewallRulesDaoImpl.java @@ -48,6 +48,7 @@ public class FirewallRulesDaoImpl extends GenericDaoBase i protected final SearchBuilder NotRevokedSearch; protected final SearchBuilder ReleaseSearch; protected SearchBuilder VmSearch; + protected SearchBuilder FirewallByPortsAndNetwork; protected final SearchBuilder SystemRuleSearch; protected final GenericSearchBuilder RulesByIpCount; @@ -104,6 +105,12 @@ public class FirewallRulesDaoImpl extends GenericDaoBase i RulesByIpCount.and("ipAddressId", RulesByIpCount.entity().getSourceIpAddressId(), Op.EQ); RulesByIpCount.and("state", RulesByIpCount.entity().getState(), Op.EQ); RulesByIpCount.done(); + + FirewallByPortsAndNetwork = createSearchBuilder(); + FirewallByPortsAndNetwork.and("networkId", FirewallByPortsAndNetwork.entity().getNetworkId(), Op.EQ); + FirewallByPortsAndNetwork.and("sourcePortStart", FirewallByPortsAndNetwork.entity().getSourcePortStart(), Op.EQ); + FirewallByPortsAndNetwork.and("sourcePortEnd", FirewallByPortsAndNetwork.entity().getSourcePortEnd(), Op.EQ); + FirewallByPortsAndNetwork.done(); } @Override @@ -386,4 +393,14 @@ public class FirewallRulesDaoImpl extends GenericDaoBase i rule.setDestinationCidrsList(destCidrs); } + @Override + public FirewallRuleVO findByNetworkIdAndPorts(long networkId, int startPort, int endPort) { + SearchCriteria sc = FirewallByPortsAndNetwork.create(); + sc.setParameters("networkId", networkId); + sc.setParameters("sourcePortStart", startPort); + sc.setParameters("sourcePortEnd", endPort); + + return findOneBy(sc); + } + } diff --git a/engine/schema/src/main/java/com/cloud/network/rules/dao/PortForwardingRulesDao.java b/engine/schema/src/main/java/com/cloud/network/rules/dao/PortForwardingRulesDao.java index b89d04ad15a..622665fc003 100644 --- a/engine/schema/src/main/java/com/cloud/network/rules/dao/PortForwardingRulesDao.java +++ b/engine/schema/src/main/java/com/cloud/network/rules/dao/PortForwardingRulesDao.java @@ -47,4 +47,6 @@ public interface PortForwardingRulesDao extends GenericDao listByNetworkAndDestIpAddr(String ip4Address, long networkId); + + PortForwardingRuleVO findByNetworkAndPorts(long networkId, int startPort, int endPort); } diff --git a/engine/schema/src/main/java/com/cloud/network/rules/dao/PortForwardingRulesDaoImpl.java b/engine/schema/src/main/java/com/cloud/network/rules/dao/PortForwardingRulesDaoImpl.java index 29cba516d72..2653515596b 100644 --- a/engine/schema/src/main/java/com/cloud/network/rules/dao/PortForwardingRulesDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/network/rules/dao/PortForwardingRulesDaoImpl.java @@ -54,6 +54,8 @@ public class PortForwardingRulesDaoImpl extends GenericDaoBase sc = AllFieldsSearch.create(); + sc.setParameters("networkId", networkId); + sc.setParameters("sourcePortStart", startPort); + sc.setParameters("sourcePortEnd", endPort); + return findOneBy(sc); + } } diff --git a/engine/schema/src/main/resources/META-INF/db/schema-41810to41900.sql b/engine/schema/src/main/resources/META-INF/db/schema-41810to41900.sql index 1fe43a20a5c..c1eff3f8a2d 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-41810to41900.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-41810to41900.sql @@ -336,6 +336,7 @@ CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.kubernetes_cluster','control_templat CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.kubernetes_cluster','worker_template_id', 'bigint unsigned COMMENT "template id to be used for Worker Node(s)"'); CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.kubernetes_cluster','etcd_template_id', 'bigint unsigned COMMENT "template id to be used for etcd Nodes"'); CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.kubernetes_cluster_vm_map','etcd_node', 'tinyint(1) unsigned NOT NULL DEFAULT 0 COMMENT "indicates if the VM is an etcd node"'); +CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.kubernetes_cluster_vm_map','external_node', 'tinyint(1) unsigned NOT NULL DEFAULT 0 COMMENT "indicates if the node was imported into the Kubernetes cluster"'); ALTER TABLE `cloud`.`kubernetes_cluster` ADD CONSTRAINT `fk_cluster__control_service_offering_id` FOREIGN KEY `fk_cluster__control_service_offering_id`(`control_service_offering_id`) REFERENCES `service_offering`(`id`) ON DELETE CASCADE; ALTER TABLE `cloud`.`kubernetes_cluster` ADD CONSTRAINT `fk_cluster__worker_service_offering_id` FOREIGN KEY `fk_cluster__worker_service_offering_id`(`worker_service_offering_id`) REFERENCES `service_offering`(`id`) ON DELETE CASCADE; diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterEventTypes.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterEventTypes.java index a947e4273be..486a093e4ad 100755 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterEventTypes.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterEventTypes.java @@ -23,4 +23,6 @@ public class KubernetesClusterEventTypes { public static final String EVENT_KUBERNETES_CLUSTER_STOP = "KUBERNETES.CLUSTER.STOP"; public static final String EVENT_KUBERNETES_CLUSTER_SCALE = "KUBERNETES.CLUSTER.SCALE"; public static final String EVENT_KUBERNETES_CLUSTER_UPGRADE = "KUBERNETES.CLUSTER.UPGRADE"; + public static final String EVENT_KUBERNETES_CLUSTER_NODES_ADD = "KUBERNETES.CLUSTER.NODES.ADD"; + public static final String EVENT_KUBERNETES_CLUSTER_NODES_REMOVE = "KUBERNETES.CLUSTER.NODES.REMOVE"; } diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java index 41bd63ad7ee..6d4b013298a 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java @@ -23,10 +23,12 @@ import static com.cloud.kubernetes.cluster.KubernetesClusterHelper.KubernetesClu import static com.cloud.utils.NumbersUtil.toHumanReadableSize; import static com.cloud.vm.UserVmManager.AllowUserExpungeRecoverVm; +import java.lang.reflect.InvocationTargetException; import java.net.MalformedURLException; import java.net.URL; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.Date; import java.util.EnumSet; import java.util.HashMap; @@ -45,23 +47,33 @@ import javax.inject.Inject; import javax.naming.ConfigurationException; import com.cloud.kubernetes.cluster.KubernetesClusterHelper.KubernetesClusterNodeType; +import com.cloud.kubernetes.cluster.actionworkers.KubernetesClusterRemoveWorker; import com.cloud.network.dao.NsxProviderDao; import com.cloud.network.element.NsxProviderVO; +import com.cloud.kubernetes.cluster.actionworkers.KubernetesClusterAddWorker; +import com.cloud.template.TemplateApiService; import com.cloud.uservm.UserVm; +import com.cloud.vm.NicVO; import com.cloud.vm.UserVmService; +import com.cloud.vm.dao.NicDao; +import com.cloud.vm.dao.UserVmDetailsDao; import org.apache.cloudstack.acl.ControlledEntity; import org.apache.cloudstack.acl.SecurityChecker; import org.apache.cloudstack.annotation.AnnotationService; import org.apache.cloudstack.annotation.dao.AnnotationDao; import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.ApiConstants.VMDetails; +import org.apache.cloudstack.api.ApiErrorCode; import org.apache.cloudstack.api.BaseCmd; import org.apache.cloudstack.api.ResponseObject.ResponseView; +import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.command.user.kubernetes.cluster.AddNodesToKubernetesClusterCmd; import org.apache.cloudstack.api.command.user.kubernetes.cluster.AddVirtualMachinesToKubernetesClusterCmd; import org.apache.cloudstack.api.command.user.kubernetes.cluster.CreateKubernetesClusterCmd; import org.apache.cloudstack.api.command.user.kubernetes.cluster.DeleteKubernetesClusterCmd; import org.apache.cloudstack.api.command.user.kubernetes.cluster.GetKubernetesClusterConfigCmd; import org.apache.cloudstack.api.command.user.kubernetes.cluster.ListKubernetesClustersCmd; +import org.apache.cloudstack.api.command.user.kubernetes.cluster.RemoveNodesFromKubernetesClusterCmd; import org.apache.cloudstack.api.command.user.kubernetes.cluster.RemoveVirtualMachinesFromKubernetesClusterCmd; import org.apache.cloudstack.api.command.user.kubernetes.cluster.ScaleKubernetesClusterCmd; import org.apache.cloudstack.api.command.user.kubernetes.cluster.StartKubernetesClusterCmd; @@ -69,6 +81,7 @@ import org.apache.cloudstack.api.command.user.kubernetes.cluster.StopKubernetesC import org.apache.cloudstack.api.command.user.kubernetes.cluster.UpgradeKubernetesClusterCmd; import org.apache.cloudstack.api.response.KubernetesClusterConfigResponse; import org.apache.cloudstack.api.response.KubernetesClusterResponse; +import org.apache.cloudstack.api.response.KubernetesUserVmResponse; import org.apache.cloudstack.api.response.ListResponse; import org.apache.cloudstack.api.response.RemoveVirtualMachinesFromKubernetesClusterResponse; import org.apache.cloudstack.api.response.UserVmResponse; @@ -77,6 +90,7 @@ import org.apache.cloudstack.context.CallContext; import org.apache.cloudstack.engine.orchestration.service.NetworkOrchestrationService; import org.apache.cloudstack.framework.config.ConfigKey; import org.apache.cloudstack.managed.context.ManagedContextRunnable; +import org.apache.commons.beanutils.BeanUtils; import org.apache.commons.codec.binary.Base64; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.collections.MapUtils; @@ -234,6 +248,8 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne @Inject protected UserDao userDao; @Inject + protected UserVmDetailsDao userVmDetailsDao; + @Inject protected VMInstanceDao vmInstanceDao; @Inject protected UserVmJoinDao userVmJoinDao; @@ -271,9 +287,12 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne public NetworkHelper networkHelper; @Inject private NsxProviderDao nsxProviderDao; - + @Inject + private NicDao nicDao; @Inject private UserVmService userVmService; + @Inject + private TemplateApiService templateService; private void logMessage(final Level logLevel, final String message, final Exception e) { if (logLevel == Level.WARN) { @@ -662,7 +681,7 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne } } - List vmResponses = new ArrayList(); + List vmResponses = new ArrayList<>(); List vmList = kubernetesClusterVmMapDao.listByClusterId(kubernetesCluster.getId()); ResponseView respView = ResponseView.Restricted; Account caller = CallContext.current().getCallingAccount(); @@ -676,9 +695,17 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne if (userVM != null) { UserVmResponse vmResponse = ApiDBUtils.newUserVmResponse(respView, responseName, userVM, EnumSet.of(VMDetails.nics), caller); - vmResponses.add(vmResponse); + KubernetesUserVmResponse kubernetesUserVmResponse = new KubernetesUserVmResponse(); + try { + BeanUtils.copyProperties(kubernetesUserVmResponse, vmResponse); + } catch (IllegalAccessException | InvocationTargetException e) { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to generate zone metrics response"); + } + kubernetesUserVmResponse.setExternalNode(vmMapVO.isExternalNode()); + vmResponses.add(kubernetesUserVmResponse); } } + response.setExternalNodes(vmList.stream().filter(KubernetesClusterVmMapVO::isEtcdNode).count()); } response.setHasAnnotation(annotationDao.hasAnnotations(kubernetesCluster.getUuid(), AnnotationService.EntityType.KUBERNETES_CLUSTER.name(), accountService.isRootAdmin(caller.getId()))); @@ -776,7 +803,9 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne BaseCmd.getCommandNameByClass(ScaleKubernetesClusterCmd.class), BaseCmd.getCommandNameByClass(StartKubernetesClusterCmd.class), BaseCmd.getCommandNameByClass(StopKubernetesClusterCmd.class), - BaseCmd.getCommandNameByClass(UpgradeKubernetesClusterCmd.class) + BaseCmd.getCommandNameByClass(UpgradeKubernetesClusterCmd.class), + BaseCmd.getCommandNameByClass(AddNodesToKubernetesClusterCmd.class), + BaseCmd.getCommandNameByClass(RemoveNodesFromKubernetesClusterCmd.class) ).contains(cmdName); case ExternalManaged: return Arrays.asList( @@ -924,7 +953,7 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne } catch (InvalidParameterValueException e) { String msg = String.format("Given service offering ID: %s for %s nodes is not suitable for the Kubernetes cluster version %s - %s", serviceOffering, key, clusterKubernetesVersion, e.getMessage()); - LOGGER.error(msg); + logger.error(msg); throw new InvalidParameterValueException(msg); } } @@ -1824,6 +1853,71 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne return true; } + @Override + public boolean addNodesToKubernetesCluster(AddNodesToKubernetesClusterCmd cmd) { + KubernetesClusterVO kubernetesCluster = validateCluster(cmd.getClusterId()); + long networkId = kubernetesCluster.getNetworkId(); + NetworkVO networkVO = networkDao.findById(networkId); + List validNodeIds = validateNodes(cmd.getNodeIds(), networkId, networkVO.getName(), kubernetesCluster, false); + if (validNodeIds.isEmpty()) { + throw new CloudRuntimeException("No valid nodes found to be added to the Kubernetes cluster"); + } + KubernetesClusterAddWorker addWorker = new KubernetesClusterAddWorker(kubernetesCluster, KubernetesClusterManagerImpl.this); + addWorker = ComponentContext.inject(addWorker); + return addWorker.addNodesToCluster(validNodeIds, cmd.isMountCksIsoOnVr()); + } + + @Override + public boolean removeNodesFromKubernetesCluster(RemoveNodesFromKubernetesClusterCmd cmd) throws Exception { + KubernetesClusterVO kubernetesCluster = validateCluster(cmd.getClusterId()); + List validNodeIds = validateNodes(cmd.getNodeIds(), null, null, kubernetesCluster, true); + if (validNodeIds.isEmpty()) { + throw new CloudRuntimeException("No valid nodes found to be removed from the Kubernetes cluster"); + } + KubernetesClusterRemoveWorker removeWorker = new KubernetesClusterRemoveWorker(kubernetesCluster, KubernetesClusterManagerImpl.this); + removeWorker = ComponentContext.inject(removeWorker); + return removeWorker.removeNodesFromCluster(validNodeIds); + } + + private KubernetesClusterVO validateCluster(long clusterId) { + KubernetesClusterVO kubernetesCluster = kubernetesClusterDao.findById(clusterId); + if (kubernetesCluster == null) { + throw new InvalidParameterValueException("Invalid Kubernetes cluster ID specified"); + } + return kubernetesCluster; + } + + private List validateNodes(List nodeIds, Long networkId, String networkName, KubernetesCluster cluster, boolean removeNodes) { + List validNodeIds = new ArrayList<>(nodeIds); + for (Long id : nodeIds) { + VMInstanceVO node = vmInstanceDao.findById(id); + if (Objects.isNull(node)) { + logger.error(String.format("Failed to find node (physical or virtual machine) with ID: %s", id)); + validNodeIds.remove(id); + } else if (!removeNodes) { + VMTemplateVO template = templateDao.findById(node.getTemplateId()); + if (Objects.isNull(template)) { + logger.error((String.format("Failed to find template with ID: %s", id))); + validNodeIds.remove(id); + } else if (!template.isForCks()) { + logger.error(String.format("Node: %s is deployed with a template that is not marked to be used for CKS", node.getId())); + validNodeIds.remove(id); + } + NicVO nicVO = nicDao.findDefaultNicForVM(id); + if (networkId != nicVO.getNetworkId()) { + logger.error(String.format("Node: %s does not have its default NIC in the kubernetes cluster network: %s", node.getId(), networkName)); + validNodeIds.remove(id); + } + List clusterVmMapVO = kubernetesClusterVmMapDao.listByClusterIdAndVmIdsIn(cluster.getId(), Collections.singletonList(id)); + if (Objects.nonNull(clusterVmMapVO) && !clusterVmMapVO.isEmpty()) { + logger.warn(String.format("Node: %s is already part of the cluster %s", node.getId(), cluster.getName())); + validNodeIds.remove(id); + } + } + } + return validNodeIds; + } + @Override public List removeVmsFromCluster(RemoveVirtualMachinesFromKubernetesClusterCmd cmd) { if (!KubernetesServiceEnabled.value()) { @@ -1898,6 +1992,8 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne cmdList.add(UpgradeKubernetesClusterCmd.class); cmdList.add(AddVirtualMachinesToKubernetesClusterCmd.class); cmdList.add(RemoveVirtualMachinesFromKubernetesClusterCmd.class); + cmdList.add(AddNodesToKubernetesClusterCmd.class); + cmdList.add(RemoveNodesFromKubernetesClusterCmd.class); return cmdList; } @@ -2190,6 +2286,7 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne KubernetesClusterScaleTimeout, KubernetesClusterUpgradeTimeout, KubernetesClusterUpgradeRetries, + KubernetesClusterAddNodeTimeout, KubernetesClusterExperimentalFeaturesEnabled, KubernetesMaxClusterSize }; diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterService.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterService.java index 39b926537f5..9512a0563cb 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterService.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterService.java @@ -16,11 +16,13 @@ // under the License. package com.cloud.kubernetes.cluster; +import org.apache.cloudstack.api.command.user.kubernetes.cluster.AddNodesToKubernetesClusterCmd; import org.apache.cloudstack.api.command.user.kubernetes.cluster.AddVirtualMachinesToKubernetesClusterCmd; import org.apache.cloudstack.api.command.user.kubernetes.cluster.CreateKubernetesClusterCmd; import org.apache.cloudstack.api.command.user.kubernetes.cluster.DeleteKubernetesClusterCmd; import org.apache.cloudstack.api.command.user.kubernetes.cluster.GetKubernetesClusterConfigCmd; import org.apache.cloudstack.api.command.user.kubernetes.cluster.ListKubernetesClustersCmd; +import org.apache.cloudstack.api.command.user.kubernetes.cluster.RemoveNodesFromKubernetesClusterCmd; import org.apache.cloudstack.api.command.user.kubernetes.cluster.RemoveVirtualMachinesFromKubernetesClusterCmd; import org.apache.cloudstack.api.command.user.kubernetes.cluster.ScaleKubernetesClusterCmd; import org.apache.cloudstack.api.command.user.kubernetes.cluster.StopKubernetesClusterCmd; @@ -78,6 +80,18 @@ public interface KubernetesClusterService extends PluggableService, Configurable "The number of retries if fail to upgrade kubernetes cluster due to some reasons (e.g. drain node, etcdserver leader changed)", true, KubernetesServiceEnabled.key()); + static final ConfigKey KubernetesClusterAddNodeTimeout = new ConfigKey("Advanced", Long.class, + "cloud.kubernetes.cluster.add.node.timeout", + "3600", + "Timeout interval (in seconds) in which an external node(VM / baremetal host) addition to a cluster should be completed", + true, + KubernetesServiceEnabled.key()); + static final ConfigKey KubernetesClusterRemoveNodeTimeout = new ConfigKey("Advanced", Long.class, + "cloud.kubernetes.cluster.add.node.timeout", + "900", + "Timeout interval (in seconds) in which an external node(VM / baremetal host) removal from a cluster should be completed", + true, + KubernetesServiceEnabled.key()); static final ConfigKey KubernetesClusterExperimentalFeaturesEnabled = new ConfigKey("Advanced", Boolean.class, "cloud.kubernetes.cluster.experimental.features.enabled", "false", @@ -118,5 +132,9 @@ public interface KubernetesClusterService extends PluggableService, Configurable boolean addVmsToCluster(AddVirtualMachinesToKubernetesClusterCmd cmd); + boolean addNodesToKubernetesCluster(AddNodesToKubernetesClusterCmd cmd); + + boolean removeNodesFromKubernetesCluster(RemoveNodesFromKubernetesClusterCmd cmd) throws Exception; + List removeVmsFromCluster(RemoveVirtualMachinesFromKubernetesClusterCmd cmd); } diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterVmMapVO.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterVmMapVO.java index 565d7233f3d..804d5b016f8 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterVmMapVO.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterVmMapVO.java @@ -45,6 +45,9 @@ public class KubernetesClusterVmMapVO implements KubernetesClusterVmMap { @Column(name = "etcd_node") boolean etcdNode; + @Column(name = "external_node") + boolean externalNode; + public KubernetesClusterVmMapVO() { } @@ -94,4 +97,12 @@ public class KubernetesClusterVmMapVO implements KubernetesClusterVmMap { public void setEtcdNode(boolean etcdNode) { this.etcdNode = etcdNode; } + + public boolean isExternalNode() { + return externalNode; + } + + public void setExternalNode(boolean externalNode) { + this.externalNode = externalNode; + } } diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java index 58ec18848a1..39d9261b5c4 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java @@ -1,19 +1,4 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. + package com.cloud.kubernetes.cluster.actionworkers; @@ -21,13 +6,17 @@ import java.io.BufferedWriter; import java.io.File; import java.io.FileWriter; import java.io.IOException; +import java.lang.reflect.Field; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.HashMap; import java.util.HashSet; import java.util.List; +import java.util.Map; import java.util.Objects; import java.util.Set; +import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; import javax.inject.Inject; @@ -36,12 +25,32 @@ import org.apache.logging.log4j.Level; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.LogManager; import com.cloud.kubernetes.cluster.KubernetesClusterHelper.KubernetesClusterNodeType; +import com.cloud.network.dao.NetworkVO; import com.cloud.offering.ServiceOffering; +import com.cloud.exception.ManagementServerException; +import com.cloud.exception.NetworkRuleConflictException; +import com.cloud.kubernetes.cluster.utils.KubernetesClusterUtil; +import com.cloud.network.firewall.FirewallService; +import com.cloud.network.rules.FirewallRule; +import com.cloud.network.rules.PortForwardingRuleVO; +import com.cloud.network.rules.RulesService; +import com.cloud.network.rules.dao.PortForwardingRulesDao; +import com.cloud.user.SSHKeyPairVO; +import com.cloud.utils.component.ComponentContext; +import com.cloud.utils.db.TransactionCallbackWithException; +import com.cloud.utils.net.Ip; +import com.cloud.vm.Nic; +import com.cloud.vm.NicVO; +import com.cloud.vm.VirtualMachine; +import com.cloud.vm.dao.NicDao; +import com.cloud.vm.UserVmManager; import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.command.user.firewall.CreateFirewallRuleCmd; import org.apache.cloudstack.ca.CAManager; import org.apache.cloudstack.config.ApiServiceConfiguration; import org.apache.cloudstack.engine.orchestration.service.NetworkOrchestrationService; import org.apache.cloudstack.framework.config.dao.ConfigurationDao; +import org.apache.commons.codec.binary.Base64; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.StringUtils; @@ -151,6 +160,8 @@ public class KubernetesClusterActionWorker { @Inject protected UserVmService userVmService; @Inject + protected UserVmManager userVmManager; + @Inject protected VlanDao vlanDao; @Inject protected VirtualMachineManager itMgr; @@ -160,6 +171,14 @@ public class KubernetesClusterActionWorker { public ProjectService projectService; @Inject public VpcService vpcService; + @Inject + public PortForwardingRulesDao portForwardingRulesDao; + @Inject + protected RulesService rulesService; + @Inject + protected FirewallService firewallService; + @Inject + private NicDao nicDao; protected KubernetesClusterDao kubernetesClusterDao; protected KubernetesClusterVmMapDao kubernetesClusterVmMapDao; @@ -180,6 +199,8 @@ public class KubernetesClusterActionWorker { protected final String deploySecretsScriptFilename = "deploy-cloudstack-secret"; protected final String deployProviderScriptFilename = "deploy-provider"; protected final String autoscaleScriptFilename = "autoscale-kube-cluster"; + protected final String validateNodeScript = "validate-cks-node"; + protected final String removeNodeFromClusterScript = "remove-node-from-cluster"; protected final String scriptPath = "/opt/bin/"; protected File deploySecretsScriptFile; protected File deployProviderScriptFile; @@ -318,11 +339,12 @@ public class KubernetesClusterActionWorker { return new File(keyFile); } - protected KubernetesClusterVmMapVO addKubernetesClusterVm(final long kubernetesClusterId, final long vmId, boolean isControlNode) { + protected KubernetesClusterVmMapVO addKubernetesClusterVm(final long kubernetesClusterId, final long vmId, boolean isControlNode, boolean isExternalNode) { return Transaction.execute(new TransactionCallback() { @Override public KubernetesClusterVmMapVO doInTransaction(TransactionStatus status) { KubernetesClusterVmMapVO newClusterVmMap = new KubernetesClusterVmMapVO(kubernetesClusterId, vmId, isControlNode); + newClusterVmMap.setExternalNode(isExternalNode); kubernetesClusterVmMapDao.persist(newClusterVmMap); return newClusterVmMap; } @@ -377,6 +399,22 @@ public class KubernetesClusterActionWorker { return address; } + protected IpAddress getPublicIp(Network network) throws ManagementServerException { + if (network.getVpcId() != null) { + IpAddress publicIp = getVpcTierKubernetesPublicIp(network); + if (publicIp == null) { + throw new ManagementServerException(String.format("No public IP addresses found for VPC tier : %s, Kubernetes cluster : %s", network.getName(), kubernetesCluster.getName())); + } + return publicIp; + } + IpAddress publicIp = getNetworkSourceNatIp(network); + if (publicIp == null) { + throw new ManagementServerException(String.format("No source NAT IP addresses found for network : %s, Kubernetes cluster : %s", + network.getName(), kubernetesCluster.getName())); + } + return publicIp; + } + protected IpAddress acquireVpcTierKubernetesPublicIp(Network network) throws InsufficientAddressCapacityException, ResourceAllocationException, ResourceUnavailableException { IpAddress ip = networkService.allocateIP(owner, kubernetesCluster.getZoneId(), network.getId(), null, null); @@ -503,7 +541,7 @@ public class KubernetesClusterActionWorker { for (UserVm vm : clusterVMs) { boolean result = false; try { - result = templateService.detachIso(vm.getId(), true); + result = templateService.detachIso(vm.getId(), null, true); } catch (CloudRuntimeException ex) { logger.warn(String.format("Failed to detach binaries ISO from VM : %s in the Kubernetes cluster : %s ", vm.getDisplayName(), kubernetesCluster.getName()), ex); } @@ -613,12 +651,15 @@ public class KubernetesClusterActionWorker { copyScriptFile(nodeAddress, sshPort, autoscaleScriptFile, autoscaleScriptFilename); } - protected void copyScriptFile(String nodeAddress, final int sshPort, File file, String desitnation) { + protected void copyScriptFile(String nodeAddress, final int sshPort, File file, String destination) { try { + if (Objects.isNull(sshKeyFile)) { + sshKeyFile = getManagementServerSshPublicKeyFile(); + } SshHelper.scpTo(nodeAddress, sshPort, getControlNodeLoginUser(), sshKeyFile, null, - "~/", file.getAbsolutePath(), "0755"); - String cmdStr = String.format("sudo mv ~/%s %s/%s", file.getName(), scriptPath, desitnation); - SshHelper.sshExecute(publicIpAddress, sshPort, getControlNodeLoginUser(), sshKeyFile, null, + "~/", file.getAbsolutePath(), "0755", 20000, 30 * 60 * 1000); + String cmdStr = String.format("sudo mv ~/%s %s/%s", file.getName(), scriptPath, destination); + SshHelper.sshExecute(nodeAddress, sshPort, getControlNodeLoginUser(), sshKeyFile, null, cmdStr, 10000, 10000, 10 * 60 * 1000); } catch (Exception e) { throw new CloudRuntimeException(e); @@ -729,4 +770,191 @@ public class KubernetesClusterActionWorker { } return false; } + + protected void provisionPublicIpPortForwardingRule(IpAddress publicIp, Network network, Account account, + final long vmId, final int sourcePort, final int destPort) throws NetworkRuleConflictException, ResourceUnavailableException { + final long publicIpId = publicIp.getId(); + final long networkId = network.getId(); + final long accountId = account.getId(); + final long domainId = account.getDomainId(); + Nic vmNic = networkModel.getNicInNetwork(vmId, networkId); + final Ip vmIp = new Ip(vmNic.getIPv4Address()); + PortForwardingRuleVO pfRule = Transaction.execute((TransactionCallbackWithException) status -> { + PortForwardingRuleVO newRule = + new PortForwardingRuleVO(null, publicIpId, + sourcePort, sourcePort, + vmIp, + destPort, destPort, + "tcp", networkId, accountId, domainId, vmId); + newRule.setDisplay(true); + newRule.setState(FirewallRule.State.Add); + newRule = portForwardingRulesDao.persist(newRule); + return newRule; + }); + rulesService.applyPortForwardingRules(publicIp.getId(), account); + if (logger.isInfoEnabled()) { + logger.info(String.format("Provisioned SSH port forwarding rule: %s from port %d to %d on %s to the VM IP : %s in Kubernetes cluster : %s", pfRule.getUuid(), sourcePort, destPort, publicIp.getAddress().addr(), vmIp.toString(), kubernetesCluster.getName())); + } + } + + public String getKubernetesNodeConfig(final String joinIp, final boolean ejectIso) throws IOException { + String k8sNodeConfig = readResourceFile("/conf/k8s-node.yml"); + final String sshPubKey = "{{ k8s.ssh.pub.key }}"; + final String joinIpKey = "{{ k8s_control_node.join_ip }}"; + final String clusterTokenKey = "{{ k8s_control_node.cluster.token }}"; + final String ejectIsoKey = "{{ k8s.eject.iso }}"; + final String routerIpKey = "{{ k8s.vr.iso.mounted.ip }}"; + NicVO routerNicOnNetwork = getVirtualRouterNicOnKubernetesClusterNetwork(kubernetesCluster); + String routerIp = routerNicOnNetwork.getIPv4Address(); + String pubKey = "- \"" + configurationDao.getValue("ssh.publickey") + "\""; +// if (Objects.isNull(owner)) { +// owner = accountDao.findById(kubernetesCluster.getAccountId()); +// } + String sshKeyPair = kubernetesCluster.getKeyPair(); + if (StringUtils.isNotEmpty(sshKeyPair)) { + SSHKeyPairVO sshkp = sshKeyPairDao.findByName(owner.getAccountId(), owner.getDomainId(), sshKeyPair); + if (sshkp != null) { + pubKey += "\n - \"" + sshkp.getPublicKey() + "\""; + } + } + k8sNodeConfig = k8sNodeConfig.replace(sshPubKey, pubKey); + k8sNodeConfig = k8sNodeConfig.replace(joinIpKey, joinIp); + k8sNodeConfig = k8sNodeConfig.replace(clusterTokenKey, KubernetesClusterUtil.generateClusterToken(kubernetesCluster)); + k8sNodeConfig = k8sNodeConfig.replace(ejectIsoKey, String.valueOf(ejectIso)); + k8sNodeConfig = k8sNodeConfig.replace(routerIpKey, routerIp); + + k8sNodeConfig = updateKubeConfigWithRegistryDetails(k8sNodeConfig); + + return k8sNodeConfig; + } + + protected String updateKubeConfigWithRegistryDetails(String k8sConfig) { + /* genarate /etc/containerd/config.toml file on the nodes only if Kubernetes cluster is created to + * use docker private registry */ + String registryUsername = null; + String registryPassword = null; + String registryUrl = null; + + List details = kubernetesClusterDetailsDao.listDetails(kubernetesCluster.getId()); + for (KubernetesClusterDetailsVO detail : details) { + if (detail.getName().equals(ApiConstants.DOCKER_REGISTRY_USER_NAME)) { + registryUsername = detail.getValue(); + } + if (detail.getName().equals(ApiConstants.DOCKER_REGISTRY_PASSWORD)) { + registryPassword = detail.getValue(); + } + if (detail.getName().equals(ApiConstants.DOCKER_REGISTRY_URL)) { + registryUrl = detail.getValue(); + } + } + + if (StringUtils.isNoneEmpty(registryUsername, registryPassword, registryUrl)) { + // Update runcmd in the cloud-init configuration to run a script that updates the containerd config with provided registry details + String runCmd = "- bash -x /opt/bin/setup-containerd"; + + String registryEp = registryUrl.split("://")[1]; + k8sConfig = k8sConfig.replace("- containerd config default > /etc/containerd/config.toml", runCmd); + final String registryUrlKey = "{{registry.url}}"; + final String registryUrlEpKey = "{{registry.url.endpoint}}"; + final String registryAuthKey = "{{registry.token}}"; + final String registryUname = "{{registry.username}}"; + final String registryPsswd = "{{registry.password}}"; + + final String usernamePasswordKey = registryUsername + ":" + registryPassword; + String base64Auth = Base64.encodeBase64String(usernamePasswordKey.getBytes(com.cloud.utils.StringUtils.getPreferredCharset())); + k8sConfig = k8sConfig.replace(registryUrlKey, registryUrl); + k8sConfig = k8sConfig.replace(registryUrlEpKey, registryEp); + k8sConfig = k8sConfig.replace(registryUname, registryUsername); + k8sConfig = k8sConfig.replace(registryPsswd, registryPassword); + k8sConfig = k8sConfig.replace(registryAuthKey, base64Auth); + } + return k8sConfig; + } + + public Map addFirewallRulesForNodes(IpAddress publicIp, int size) throws ManagementServerException { + Map vmIdPortMap = new HashMap<>(); + try { + List clusterVmList = kubernetesClusterVmMapDao.listByClusterId(kubernetesCluster.getId()); + List externalNodes = clusterVmList.stream().filter(KubernetesClusterVmMapVO::isExternalNode).collect(Collectors.toList()); + int endPort = (CLUSTER_NODES_DEFAULT_START_SSH_PORT + clusterVmList.size() - externalNodes.size() - 1); + provisionFirewallRules(publicIp, owner, CLUSTER_NODES_DEFAULT_START_SSH_PORT, endPort); + if (logger.isInfoEnabled()) { + logger.info(String.format("Provisioned firewall rule to open up port %d to %d on %s for Kubernetes cluster : %s", CLUSTER_NODES_DEFAULT_START_SSH_PORT, endPort, publicIp.getAddress().addr(), kubernetesCluster.getName())); + } + if (!externalNodes.isEmpty()) { + AtomicInteger additionalNodes = new AtomicInteger(1); + externalNodes.forEach(externalNode -> { + int port = endPort + additionalNodes.get(); + try { + provisionFirewallRules(publicIp, owner, port, port); + vmIdPortMap.put(externalNode.getVmId(), port); + } catch (NoSuchFieldException | IllegalAccessException | ResourceUnavailableException | NetworkRuleConflictException e) { + throw new CloudRuntimeException(String.format("Failed to provision firewall rules for SSH access for the Kubernetes cluster : %s", kubernetesCluster.getName()), e); + } + additionalNodes.addAndGet(1); + }); + } + } catch (NoSuchFieldException | IllegalAccessException | ResourceUnavailableException | NetworkRuleConflictException e) { + throw new ManagementServerException(String.format("Failed to provision firewall rules for SSH access for the Kubernetes cluster : %s", kubernetesCluster.getName()), e); + } + return vmIdPortMap; + } + + protected void provisionFirewallRules(final IpAddress publicIp, final Account account, int startPort, int endPort) throws NoSuchFieldException, + IllegalAccessException, ResourceUnavailableException, NetworkRuleConflictException { + List sourceCidrList = new ArrayList(); + sourceCidrList.add("0.0.0.0/0"); + + CreateFirewallRuleCmd rule = new CreateFirewallRuleCmd(); + rule = ComponentContext.inject(rule); + + Field addressField = rule.getClass().getDeclaredField("ipAddressId"); + addressField.setAccessible(true); + addressField.set(rule, publicIp.getId()); + + Field protocolField = rule.getClass().getDeclaredField("protocol"); + protocolField.setAccessible(true); + protocolField.set(rule, "TCP"); + + Field startPortField = rule.getClass().getDeclaredField("publicStartPort"); + startPortField.setAccessible(true); + startPortField.set(rule, startPort); + + Field endPortField = rule.getClass().getDeclaredField("publicEndPort"); + endPortField.setAccessible(true); + endPortField.set(rule, endPort); + + Field cidrField = rule.getClass().getDeclaredField("cidrlist"); + cidrField.setAccessible(true); + cidrField.set(rule, sourceCidrList); + + firewallService.createIngressFirewallRule(rule); + firewallService.applyIngressFwRules(publicIp.getId(), account); + } + + protected NicVO getVirtualRouterNicOnKubernetesClusterNetwork(KubernetesCluster kubernetesCluster) { + long networkId = kubernetesCluster.getNetworkId(); + NetworkVO kubernetesClusterNetwork = networkDao.findById(networkId); + if (kubernetesClusterNetwork == null) { + logAndThrow(Level.ERROR, String.format("Cannot find network %s set on Kubernetes Cluster %s", networkId, kubernetesCluster.getName())); + } + NicVO routerNicOnNetwork = nicDao.findByNetworkIdAndType(networkId, VirtualMachine.Type.DomainRouter); + if (routerNicOnNetwork == null) { + logAndThrow(Level.ERROR, String.format("Cannot find a Virtual Router on Kubernetes Cluster %s network %s", kubernetesCluster.getName(), kubernetesClusterNetwork.getName())); + } + return routerNicOnNetwork; + } + + protected Map getVmPortMap() { + List clusterVmList = kubernetesClusterVmMapDao.listByClusterId(kubernetesCluster.getId()); + List externalNodes = clusterVmList.stream().filter(KubernetesClusterVmMapVO::isExternalNode).collect(Collectors.toList()); + Map vmIdPortMap = new HashMap<>(); + int defaultNodesCount = clusterVmList.size() - externalNodes.size(); + AtomicInteger i = new AtomicInteger(0); + externalNodes.forEach(node -> { + vmIdPortMap.put(node.getVmId(), CLUSTER_NODES_DEFAULT_START_SSH_PORT + defaultNodesCount + i.get()); + i.addAndGet(1); + }); + return vmIdPortMap; + } } diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterAddWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterAddWorker.java new file mode 100644 index 00000000000..0f59e6bcf45 --- /dev/null +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterAddWorker.java @@ -0,0 +1,301 @@ +package com.cloud.kubernetes.cluster.actionworkers; + +import com.cloud.event.ActionEventUtils; +import com.cloud.event.EventVO; +import com.cloud.exception.InsufficientCapacityException; +import com.cloud.exception.ManagementServerException; +import com.cloud.exception.NetworkRuleConflictException; +import com.cloud.exception.ResourceUnavailableException; +import com.cloud.hypervisor.Hypervisor; +import com.cloud.kubernetes.cluster.KubernetesCluster; +import com.cloud.kubernetes.cluster.KubernetesClusterEventTypes; +import com.cloud.kubernetes.cluster.KubernetesClusterManagerImpl; +import com.cloud.kubernetes.cluster.KubernetesClusterService; +import com.cloud.kubernetes.cluster.KubernetesClusterVO; +import com.cloud.kubernetes.cluster.utils.KubernetesClusterUtil; +import com.cloud.network.IpAddress; +import com.cloud.network.Network; +import com.cloud.network.dao.FirewallRulesDao; +import com.cloud.network.rules.FirewallRuleVO; +import com.cloud.network.rules.PortForwardingRuleVO; +import com.cloud.service.ServiceOfferingVO; +import com.cloud.user.Account; +import com.cloud.uservm.UserVm; +import com.cloud.utils.Pair; +import com.cloud.utils.Ternary; +import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.utils.ssh.SshHelper; +import com.cloud.vm.UserVmVO; +import org.apache.cloudstack.api.ApiCommandResourceType; +import org.apache.cloudstack.api.BaseCmd; +import org.apache.cloudstack.api.command.user.vm.RebootVMCmd; +import org.apache.cloudstack.context.CallContext; +import org.apache.commons.codec.binary.Base64; +import org.apache.log4j.Level; + +import javax.inject.Inject; +import java.io.File; +import java.io.IOException; +import java.lang.reflect.Field; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import java.util.stream.Collectors; + +public class KubernetesClusterAddWorker extends KubernetesClusterActionWorker { + + @Inject + private FirewallRulesDao firewallRulesDao; + private long addNodeTimeoutTime; + + List finalNodeIds = new ArrayList<>(); + + public KubernetesClusterAddWorker(KubernetesCluster kubernetesCluster, KubernetesClusterManagerImpl clusterManager) { + super(kubernetesCluster, clusterManager); + } + + public boolean addNodesToCluster(List nodeIds, boolean mountCksIsoOnVr) throws CloudRuntimeException { + try { + init(); + addNodeTimeoutTime = System.currentTimeMillis() + KubernetesClusterService.KubernetesClusterAddNodeTimeout.value() * 1000; + Long networkId = kubernetesCluster.getNetworkId(); + Network network = networkDao.findById(networkId); + if (Objects.isNull(network)) { + throw new CloudRuntimeException(String.format("Failed to find network with id: %s", networkId)); + } + templateDao.findById(kubernetesCluster.getTemplateId()); + IpAddress publicIp = null; + try { + publicIp = getPublicIp(network); + } catch (ManagementServerException e) { + throw new CloudRuntimeException(String.format("Failed to retrieve public IP for the network: %s ", network.getName())); + } + attachCksIsoForNodesAdditionToCluster(nodeIds, kubernetesCluster.getId(), mountCksIsoOnVr); + stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.AddNodeRequested); + Ternary nodesAddedAndMemory = importNodeToCluster(nodeIds, network, publicIp, mountCksIsoOnVr); + int nodesAdded = nodesAddedAndMemory.first(); + updateKubernetesCluster(kubernetesCluster.getId(), nodesAddedAndMemory); + if (nodeIds.size() != nodesAdded) { + String msg = String.format("Not every node was added to the CKS cluster %s, nodes added: %s out of %s", kubernetesCluster.getUuid(), nodesAdded, nodeIds.size()); + LOGGER.info(msg); + detachCksIsoFromNodesAddedToCluster(nodeIds, kubernetesCluster.getId(), mountCksIsoOnVr); + stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); + ActionEventUtils.onCompletedActionEvent(CallContext.current().getCallingUserId(), CallContext.current().getCallingAccountId(), + EventVO.LEVEL_ERROR, KubernetesClusterEventTypes.EVENT_KUBERNETES_CLUSTER_NODES_ADD, + msg, kubernetesCluster.getId(), ApiCommandResourceType.KubernetesCluster.toString(), 0); + return false; + } + Pair publicIpSshPort = getKubernetesClusterServerIpSshPort(null); + KubernetesClusterUtil.validateKubernetesClusterReadyNodesCount(kubernetesCluster, publicIpSshPort.first(), publicIpSshPort.second(), + getControlNodeLoginUser(), sshKeyFile, addNodeTimeoutTime, 15000); + detachCksIsoFromNodesAddedToCluster(nodeIds, kubernetesCluster.getId(), mountCksIsoOnVr); + stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationSucceeded); + String description = String.format("Successfully added %s nodes to Kubernetes Cluster %s", nodesAdded, kubernetesCluster.getUuid()); + ActionEventUtils.onCompletedActionEvent(CallContext.current().getCallingUserId(), CallContext.current().getCallingAccountId(), + EventVO.LEVEL_INFO, KubernetesClusterEventTypes.EVENT_KUBERNETES_CLUSTER_NODES_ADD, + description, kubernetesCluster.getId(), ApiCommandResourceType.KubernetesCluster.toString(), 0); + return true; + } catch (Exception e) { + stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); + throw new CloudRuntimeException(e); + } + } + + private void detachCksIsoFromNodesAddedToCluster(List nodeIds, long kubernetesClusterId, boolean mountCksIsoOnVr) { + if (mountCksIsoOnVr) { + detachIsoOnVirtualRouter(kubernetesClusterId); + } else { + LOGGER.info("Detaching CKS ISO from the nodes"); + List vms = nodeIds.stream().map(nodeId -> userVmDao.findById(nodeId)).collect(Collectors.toList()); + detachIsoKubernetesVMs(vms); + } + } + + public void detachIsoOnVirtualRouter(Long kubernetesClusterId) { + KubernetesClusterVO kubernetesCluster = kubernetesClusterDao.findById(kubernetesClusterId); + Long virtualRouterId = getVirtualRouterNicOnKubernetesClusterNetwork(kubernetesCluster).getInstanceId(); + long isoId = kubernetesSupportedVersionDao.findById(kubernetesCluster.getKubernetesVersionId()).getIsoId(); + + try { + networkService.handleCksIsoOnNetworkVirtualRouter(virtualRouterId, false); + } catch (ResourceUnavailableException e) { + String err = String.format("Error trying to handle ISO %s on virtual router %s", isoId, virtualRouterId); + LOGGER.error(err); + throw new CloudRuntimeException(err); + } + + try { + templateService.detachIso(virtualRouterId, isoId, true, true); + } catch (CloudRuntimeException e) { + String err = String.format("Error trying to detach ISO %s from virtual router %s", isoId, virtualRouterId); + LOGGER.error(err, e); + } + } + + public void attachCksIsoForNodesAdditionToCluster(List nodeIds, Long kubernetesClusterId, boolean mountCksIsoOnVr) { + if (mountCksIsoOnVr) { + attachAndServeIsoOnVirtualRouter(kubernetesClusterId); + } else { + LOGGER.info("Attaching CKS ISO to the nodes"); + List vms = nodeIds.stream().map(nodeId -> userVmDao.findById(nodeId)).collect(Collectors.toList()); + attachIsoKubernetesVMs(vms); + } + } + + public void attachAndServeIsoOnVirtualRouter(Long kubernetesClusterId) { + KubernetesClusterVO kubernetesCluster = kubernetesClusterDao.findById(kubernetesClusterId); + Long virtualRouterId = getVirtualRouterNicOnKubernetesClusterNetwork(kubernetesCluster).getInstanceId(); + long isoId = kubernetesSupportedVersionDao.findById(kubernetesCluster.getKubernetesVersionId()).getIsoId(); + + try { + templateService.attachIso(isoId, virtualRouterId, true, true); + } catch (CloudRuntimeException e) { + String err = String.format("Error trying to attach ISO %s to virtual router %s", isoId, virtualRouterId); + LOGGER.error(err); + throw new CloudRuntimeException(err); + } + + try { + networkService.handleCksIsoOnNetworkVirtualRouter(virtualRouterId, true); + } catch (ResourceUnavailableException e) { + String err = String.format("Error trying to handle ISO %s on virtual router %s", isoId, virtualRouterId); + LOGGER.error(err); + throw new CloudRuntimeException(err); + } + } + + private Ternary importNodeToCluster(List nodeIds, Network network, IpAddress publicIp, boolean mountCksIsoOnVr) { + int nodeIndex = 0; + Long additionalMemory = 0L; + Long additionalCores = 0L; + for (Long nodeId : nodeIds) { + UserVmVO vm = userVmDao.findById(nodeId); + String k8sControlNodeConfig = null; + try { + k8sControlNodeConfig = getKubernetesNodeConfig(publicIp.getAddress().addr(), Hypervisor.HypervisorType.VMware.equals(clusterTemplate.getHypervisorType())); + } catch (IOException e) { + logAndThrow(Level.ERROR, "Failed to read Kubernetes control node configuration file", e); + } + if (Objects.isNull(k8sControlNodeConfig)) { + logAndThrow(Level.ERROR, "Error generating worker node configuration"); + } + String base64UserData = Base64.encodeBase64String(k8sControlNodeConfig.getBytes(com.cloud.utils.StringUtils.getPreferredCharset())); + + Pair result = validateAndSetupNode(network, publicIp, owner, nodeId, nodeIndex, base64UserData); + if (Boolean.TRUE.equals(result.first())) { + ServiceOfferingVO offeringVO = serviceOfferingDao.findById(vm.getId(), vm.getServiceOfferingId()); + additionalMemory += offeringVO.getRamSize(); + additionalCores += offeringVO.getCpu(); + String msg = String.format("VM %s added as a node on the Kubernetes Cluster %s", vm.getUuid(), kubernetesCluster.getUuid()); + ActionEventUtils.onCompletedActionEvent(CallContext.current().getCallingUserId(), CallContext.current().getCallingAccountId(), + EventVO.LEVEL_INFO, KubernetesClusterEventTypes.EVENT_KUBERNETES_CLUSTER_NODES_ADD, + msg, vm.getId(), ApiCommandResourceType.VirtualMachine.toString(), 0); + } + if (Boolean.FALSE.equals(result.first())) { + LOGGER.error(String.format("Failed to add node %s [%s] to Kubernetes cluster : %s", vm.getName(), vm.getUuid(), kubernetesCluster.getName())); + } + if (System.currentTimeMillis() > addNodeTimeoutTime) { + LOGGER.error(String.format("Failed to add node %s to Kubernetes cluster : %s", nodeId, kubernetesCluster.getName())); + } + nodeIndex = result.second(); + } + return new Ternary<>(nodeIndex, additionalMemory, additionalCores); + } + + private Pair validateAndSetupNode(Network network, IpAddress publicIp, Account account, + Long nodeId, int nodeIndex, String base64UserData) { + int startSshPortNumber = KubernetesClusterActionWorker.CLUSTER_NODES_DEFAULT_START_SSH_PORT + (int) kubernetesCluster.getTotalNodeCount(); + int sshStartPort = startSshPortNumber + nodeIndex; + try { + if (Objects.isNull(network.getVpcId())) { + provisionFirewallRules(publicIp, owner, sshStartPort, sshStartPort); + } + provisionPublicIpPortForwardingRule(publicIp, network, account, nodeId, sshStartPort, DEFAULT_SSH_PORT); + boolean isCompatible = validateNodeCompatibility(publicIp, nodeId, sshStartPort); + if (!isCompatible) { + revertNetworkRules(network, nodeId, sshStartPort); + return new Pair<>(false, nodeIndex); + } + + userVmManager.updateVirtualMachine(nodeId, null, null, null, null, + null, base64UserData, null, null, null, + BaseCmd.HTTPMethod.POST, null, null, null, null, null); + + RebootVMCmd rebootVMCmd = new RebootVMCmd(); + Field idField = rebootVMCmd.getClass().getDeclaredField("id"); + idField.setAccessible(true); + idField.set(rebootVMCmd, nodeId); + userVmService.rebootVirtualMachine(rebootVMCmd); + finalNodeIds.add(nodeId); + } catch (ResourceUnavailableException | NetworkRuleConflictException | NoSuchFieldException | + InsufficientCapacityException | IllegalAccessException e) { + LOGGER.error(String.format("Failed to activate API port forwarding rules for the Kubernetes cluster : %s", kubernetesCluster.getName())); + // remove added Firewall and PF rules + revertNetworkRules(network, nodeId, sshStartPort); + return new Pair<>( false, nodeIndex); + } catch (Exception e) { + throw new CloudRuntimeException(e); + } + return new Pair<>(true, ++nodeIndex); + } + + private void updateKubernetesCluster(long clusterId, Ternary additionalNodesDetails) { + int additionalNodeCount = additionalNodesDetails.first(); + KubernetesClusterVO kubernetesClusterVO = kubernetesClusterDao.findById(clusterId); + kubernetesClusterVO.setNodeCount(kubernetesClusterVO.getNodeCount() + additionalNodeCount); + kubernetesClusterVO.setMemory(kubernetesClusterVO.getMemory() + additionalNodesDetails.second()); + kubernetesClusterVO.setCores(kubernetesClusterVO.getCores() + additionalNodesDetails.third()); + kubernetesClusterDao.update(clusterId, kubernetesClusterVO); + kubernetesCluster = kubernetesClusterVO; + + finalNodeIds.forEach(id -> addKubernetesClusterVm(clusterId, id, false, true)); + } + + + private boolean validateNodeCompatibility(IpAddress publicIp, long nodeId, int nodeSshPort) throws CloudRuntimeException { + File pkFile = getManagementServerSshPublicKeyFile(); + try { + File validateNodeScriptFile = retrieveScriptFile(validateNodeScript); + Thread.sleep(15*1000); + copyScriptFile(publicIp.getAddress().addr(), nodeSshPort, validateNodeScriptFile, validateNodeScript); + String command = String.format("%s%s", scriptPath, validateNodeScript); + Pair result = SshHelper.sshExecute(publicIp.getAddress().addr(), nodeSshPort, getControlNodeLoginUser(), + pkFile, null, command, 10000, 10000, 10 * 60 * 1000); + if (Boolean.FALSE.equals(result.first())) { + LOGGER.error(String.format("Node with ID: %s cannot be added as a worker node as it does not have " + + "the following dependencies: %s ", nodeId, result.second())); + return false; + } + } catch (Exception e) { + LOGGER.error(String.format("Failed to validate node with ID: %s", nodeId), e); + return false; + } + UserVmVO userVm = userVmDao.findById(nodeId); + cleanupCloudInitSemFolder(userVm, publicIp, pkFile, nodeSshPort); + return true; + } + + private void cleanupCloudInitSemFolder(UserVm userVm, IpAddress publicIp, File pkFile, int nodeSshPort) { + try { + String command = String.format("sudo rm -rf /var/lib/cloud/instances/%s/sem/*", userVm.getUuid()); + Pair result = SshHelper.sshExecute(publicIp.getAddress().addr(), nodeSshPort, getControlNodeLoginUser(), + pkFile, null, command, 10000, 10000, 10 * 60 * 1000); + if (Boolean.FALSE.equals(result.first())) { + LOGGER.error(String.format("Failed to cleanup previous applied userdata on node: %s; This may hamper to addition of the node to the cluster ", userVm.getName())); + } + } catch (Exception e) { + LOGGER.error(String.format("Failed to cleanup previous applied userdata on node: %s; This may hamper to addition of the node to the cluster ", userVm.getName()), e); + } + } + + private void revertNetworkRules(Network network, long vmId, int port) { + FirewallRuleVO ruleVO = firewallRulesDao.findByNetworkIdAndPorts(network.getId(), port, port); + if (Objects.isNull(network.getVpcId())) { + firewallService.revokeIngressFirewallRule(ruleVO.getId(), true); + } + List pfRules = portForwardingRulesDao.listByVm(vmId); + for (PortForwardingRuleVO pfRule : pfRules) { + rulesService.revokePortForwardingRule(pfRule.getId(), true); + } + } +} diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterDestroyWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterDestroyWorker.java index 6e87d2071cd..f8ed5df0cd0 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterDestroyWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterDestroyWorker.java @@ -151,7 +151,7 @@ public class KubernetesClusterDestroyWorker extends KubernetesClusterResourceMod if (firewallRule == null) { logMessage(Level.WARN, "Firewall rule for API access can't be removed", null); } - firewallRule = removeSshFirewallRule(publicIp); + firewallRule = removeSshFirewallRule(publicIp, network.getId()); if (firewallRule == null) { logMessage(Level.WARN, "Firewall rule for SSH access can't be removed", null); } diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterRemoveWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterRemoveWorker.java new file mode 100644 index 00000000000..bb783f1a087 --- /dev/null +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterRemoveWorker.java @@ -0,0 +1,166 @@ +package com.cloud.kubernetes.cluster.actionworkers; + +import com.cloud.event.ActionEventUtils; +import com.cloud.event.EventVO; +import com.cloud.exception.ManagementServerException; +import com.cloud.kubernetes.cluster.KubernetesCluster; +import com.cloud.kubernetes.cluster.KubernetesClusterEventTypes; +import com.cloud.kubernetes.cluster.KubernetesClusterManagerImpl; +import com.cloud.kubernetes.cluster.KubernetesClusterService; +import com.cloud.kubernetes.cluster.KubernetesClusterVO; +import com.cloud.network.IpAddress; +import com.cloud.network.Network; +import com.cloud.network.dao.FirewallRulesDao; +import com.cloud.network.rules.FirewallRuleVO; +import com.cloud.network.rules.PortForwardingRuleVO; +import com.cloud.service.ServiceOfferingVO; +import com.cloud.utils.Pair; +import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.utils.ssh.SshHelper; +import com.cloud.vm.UserVmVO; +import org.apache.cloudstack.api.ApiCommandResourceType; +import org.apache.cloudstack.context.CallContext; + +import javax.inject.Inject; +import java.io.File; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import java.util.Optional; + +public class KubernetesClusterRemoveWorker extends KubernetesClusterActionWorker { + + @Inject + private FirewallRulesDao firewallRulesDao; + + private long removeNodeTimeoutTime; + + public KubernetesClusterRemoveWorker(KubernetesCluster kubernetesCluster, KubernetesClusterManagerImpl clusterManager) { + super(kubernetesCluster, clusterManager); + } + + public boolean removeNodesFromCluster(List nodeIds) { + init(); + removeNodeTimeoutTime = System.currentTimeMillis() + KubernetesClusterService.KubernetesClusterAddNodeTimeout.value() * 1000; + Long networkId = kubernetesCluster.getNetworkId(); + Network network = networkDao.findById(networkId); + if (Objects.isNull(network)) { + throw new CloudRuntimeException(String.format("Failed to find network with id: %s", networkId)); + } + IpAddress publicIp = null; + try { + publicIp = getPublicIp(network); + } catch (ManagementServerException e) { + throw new CloudRuntimeException(String.format("Failed to retrieve public IP for the network: %s ", network.getName())); + } + stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.RemoveNodeRequested); + boolean result = removeNodesFromCluster(nodeIds, network, publicIp); + if (!result) { + stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); + } else { + stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationSucceeded); + } + String description = String.format("Successfully removed %s nodes from the Kubernetes Cluster %s", nodeIds.size(), kubernetesCluster.getUuid()); + ActionEventUtils.onCompletedActionEvent(CallContext.current().getCallingUserId(), CallContext.current().getCallingAccountId(), + EventVO.LEVEL_INFO, KubernetesClusterEventTypes.EVENT_KUBERNETES_CLUSTER_NODES_REMOVE, + description, kubernetesCluster.getId(), ApiCommandResourceType.KubernetesCluster.toString(), 0); + return result; + } + + private boolean removeNodesFromCluster(List nodeIds, Network network, IpAddress publicIp) { + boolean result = true; + List removedNodeIds = new ArrayList<>(); + long removedMemory = 0L; + long removedCores = 0L; + for (Long nodeId : nodeIds) { + UserVmVO vm = userVmDao.findById(nodeId); + if (vm == null) { + LOGGER.debug(String.format("Couldn't find a VM with ID %s, skipping removal from Kubernetes cluster", nodeId)); + continue; + } + try { + removeNodeVmFromCluster(nodeId, vm.getDisplayName(), publicIp.getAddress().addr()); + result &= removeNodePortForwardingRules(nodeId, network, vm); + if (System.currentTimeMillis() > removeNodeTimeoutTime) { + LOGGER.error(String.format("Removal of node %s from Kubernetes cluster %s timed out", vm.getName(), kubernetesCluster.getName())); + result = false; + continue; + } + ServiceOfferingVO offeringVO = serviceOfferingDao.findById(vm.getId(), vm.getServiceOfferingId()); + removedNodeIds.add(nodeId); + removedMemory += offeringVO.getRamSize(); + removedCores += offeringVO.getCpu(); + String description = String.format("Successfully removed the node %s from Kubernetes cluster %s", vm.getUuid(), kubernetesCluster.getUuid()); + LOGGER.info(description); + ActionEventUtils.onCompletedActionEvent(CallContext.current().getCallingUserId(), CallContext.current().getCallingAccountId(), + EventVO.LEVEL_INFO, KubernetesClusterEventTypes.EVENT_KUBERNETES_CLUSTER_NODES_REMOVE, + description, vm.getId(), ApiCommandResourceType.VirtualMachine.toString(), 0); + } catch (Exception e) { + String err = String.format("Error trying to remove node %s from Kubernetes Cluster %s: %s", vm.getUuid(), kubernetesCluster.getUuid(), e.getMessage()); + LOGGER.error(err, e); + result = false; + } + } + updateKubernetesCluster(kubernetesCluster.getId(), removedNodeIds, removedMemory, removedCores); + return result; + } + + protected boolean removeNodePortForwardingRules(Long nodeId, Network network, UserVmVO vm) { + List pfRules = portForwardingRulesDao.listByVm(nodeId); + boolean result = true; + for (PortForwardingRuleVO pfRule : pfRules) { + try { + result &= rulesService.revokePortForwardingRule(pfRule.getId(), true); + if (Objects.isNull(network.getVpcId())) { + FirewallRuleVO ruleVO = firewallRulesDao.findByNetworkIdAndPorts(network.getId(), pfRule.getSourcePortStart(), pfRule.getSourcePortEnd()); + result &= firewallService.revokeIngressFirewallRule(ruleVO.getId(), true); + } + } catch (Exception e) { + String err = String.format("Failed to cleanup network rules for node %s, due to: %s", vm.getName(), e.getMessage()); + LOGGER.error(err, e); + } + } + return result; + } + + private void removeNodeVmFromCluster(Long nodeId, String nodeName, String publicIp) throws Exception { + File removeNodeScriptFile = retrieveScriptFile(removeNodeFromClusterScript); + copyScriptFile(publicIp, CLUSTER_NODES_DEFAULT_START_SSH_PORT, removeNodeScriptFile, removeNodeFromClusterScript); + File pkFile = getManagementServerSshPublicKeyFile(); + String command = String.format("%s%s %s %s %s", scriptPath, removeNodeFromClusterScript, nodeName, "control", "remove"); + Pair result = SshHelper.sshExecute(publicIp, CLUSTER_NODES_DEFAULT_START_SSH_PORT, getControlNodeLoginUser(), + pkFile, null, command, 10000, 10000, 10 * 60 * 1000); + if (Boolean.FALSE.equals(result.first())) { + LOGGER.error(String.format("Node: %s failed to be gracefully drained as a worker node from cluster %s ", nodeName, kubernetesCluster.getName())); + } + List nodePfRules = portForwardingRulesDao.listByVm(nodeId); + Optional nodeSshPort = nodePfRules.stream().filter(rule -> rule.getDestinationPortStart() == DEFAULT_SSH_PORT + && rule.getVirtualMachineId() == nodeId && rule.getSourcePortStart() >= CLUSTER_NODES_DEFAULT_START_SSH_PORT).findFirst(); + if (nodeSshPort.isPresent()) { + copyScriptFile(publicIp, nodeSshPort.get().getSourcePortStart(), removeNodeScriptFile, removeNodeFromClusterScript); + command = String.format("sudo %s%s %s %s %s", scriptPath, removeNodeFromClusterScript, nodeName, "worker", "remove"); + result = SshHelper.sshExecute(publicIp, nodeSshPort.get().getSourcePortStart(), getControlNodeLoginUser(), + pkFile, null, command, 10000, 10000, 10 * 60 * 1000); + if (Boolean.FALSE.equals(result.first())) { + LOGGER.error(String.format("Failed to reset node: %s from cluster %s ", nodeName, kubernetesCluster.getName())); + } + command = String.format("%s%s %s %s %s", scriptPath, removeNodeFromClusterScript, nodeName, "control", "delete"); + result = SshHelper.sshExecute(publicIp, CLUSTER_NODES_DEFAULT_START_SSH_PORT, getControlNodeLoginUser(), + pkFile, null, command, 10000, 10000, 10 * 60 * 1000); + if (Boolean.FALSE.equals(result.first())) { + LOGGER.error(String.format("Node: %s failed to be gracefully delete node from cluster %s ", nodeName, kubernetesCluster.getName())); + } + + } + } + + private void updateKubernetesCluster(long clusterId, List nodesRemoved, long deallocatedRam, long deallocatedCores) { + KubernetesClusterVO kubernetesClusterVO = kubernetesClusterDao.findById(clusterId); + kubernetesClusterVO.setNodeCount(kubernetesClusterVO.getNodeCount() - nodesRemoved.size()); + kubernetesClusterVO.setMemory(kubernetesClusterVO.getMemory() - deallocatedRam); + kubernetesClusterVO.setCores(kubernetesClusterVO.getCores() - deallocatedCores); + kubernetesClusterDao.update(clusterId, kubernetesClusterVO); + + nodesRemoved.forEach(id -> kubernetesClusterVmMapDao.removeByClusterIdAndVmIdsIn(clusterId, nodesRemoved)); + } +} diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java index afc7382eae8..6657d4268ec 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java @@ -21,6 +21,7 @@ import static com.cloud.kubernetes.cluster.KubernetesClusterHelper.KubernetesClu import static com.cloud.kubernetes.cluster.KubernetesClusterHelper.KubernetesClusterNodeType.ETCD; import static com.cloud.kubernetes.cluster.KubernetesClusterHelper.KubernetesClusterNodeType.WORKER; import static com.cloud.utils.NumbersUtil.toHumanReadableSize; +import static com.cloud.utils.db.Transaction.execute; import java.io.File; import java.io.IOException; @@ -29,6 +30,7 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.concurrent.ConcurrentHashMap; import java.util.stream.Collectors; @@ -36,9 +38,12 @@ import javax.inject.Inject; import com.cloud.kubernetes.cluster.KubernetesClusterHelper.KubernetesClusterNodeType; import com.cloud.network.rules.FirewallManager; +import com.cloud.network.rules.RulesService; +import com.cloud.network.rules.dao.PortForwardingRulesDao; import com.cloud.offering.NetworkOffering; import com.cloud.offerings.dao.NetworkOfferingDao; -import org.apache.cloudstack.api.ApiConstants; +import com.cloud.utils.db.TransactionCallbackWithException; +import com.cloud.utils.net.Ip; import org.apache.cloudstack.api.BaseCmd; import org.apache.cloudstack.api.command.user.firewall.CreateFirewallRuleCmd; import org.apache.cloudstack.api.command.user.network.CreateNetworkACLCmd; @@ -69,23 +74,18 @@ import com.cloud.host.HostVO; import com.cloud.host.dao.HostDao; import com.cloud.hypervisor.Hypervisor; import com.cloud.kubernetes.cluster.KubernetesCluster; -import com.cloud.kubernetes.cluster.KubernetesClusterDetailsVO; import com.cloud.kubernetes.cluster.KubernetesClusterManagerImpl; import com.cloud.kubernetes.cluster.KubernetesClusterVO; -import com.cloud.kubernetes.cluster.utils.KubernetesClusterUtil; import com.cloud.network.IpAddress; import com.cloud.network.Network; import com.cloud.network.dao.FirewallRulesDao; import com.cloud.network.dao.LoadBalancerDao; import com.cloud.network.dao.LoadBalancerVO; -import com.cloud.network.firewall.FirewallService; import com.cloud.network.lb.LoadBalancingRulesService; import com.cloud.network.rules.FirewallRule; import com.cloud.network.rules.FirewallRuleVO; import com.cloud.network.rules.LoadBalancer; import com.cloud.network.rules.PortForwardingRuleVO; -import com.cloud.network.rules.RulesService; -import com.cloud.network.rules.dao.PortForwardingRulesDao; import com.cloud.network.vpc.NetworkACL; import com.cloud.network.vpc.NetworkACLItem; import com.cloud.network.vpc.NetworkACLItemDao; @@ -99,16 +99,12 @@ import com.cloud.storage.VolumeVO; import com.cloud.storage.dao.LaunchPermissionDao; import com.cloud.storage.dao.VolumeDao; import com.cloud.user.Account; -import com.cloud.user.SSHKeyPairVO; import com.cloud.uservm.UserVm; import com.cloud.utils.Pair; import com.cloud.utils.component.ComponentContext; -import com.cloud.utils.db.Transaction; import com.cloud.utils.db.TransactionCallback; -import com.cloud.utils.db.TransactionCallbackWithException; import com.cloud.utils.db.TransactionStatus; import com.cloud.utils.exception.CloudRuntimeException; -import com.cloud.utils.net.Ip; import com.cloud.utils.net.NetUtils; import com.cloud.utils.ssh.SshHelper; import com.cloud.vm.Nic; @@ -131,8 +127,6 @@ public class KubernetesClusterResourceModifierActionWorker extends KubernetesClu @Inject protected FirewallRulesDao firewallRulesDao; @Inject - protected FirewallService firewallService; - @Inject protected NetworkACLService networkACLService; @Inject protected NetworkACLItemDao networkACLItemDao; @@ -172,72 +166,6 @@ public class KubernetesClusterResourceModifierActionWorker extends KubernetesClu kubernetesClusterNodeNamePrefix = getKubernetesClusterNodeNamePrefix(); } - private String getKubernetesNodeConfig(final String joinIp, final boolean ejectIso) throws IOException { - String k8sNodeConfig = readResourceFile("/conf/k8s-node.yml"); - final String sshPubKey = "{{ k8s.ssh.pub.key }}"; - final String joinIpKey = "{{ k8s_control_node.join_ip }}"; - final String clusterTokenKey = "{{ k8s_control_node.cluster.token }}"; - final String ejectIsoKey = "{{ k8s.eject.iso }}"; - String pubKey = "- \"" + configurationDao.getValue("ssh.publickey") + "\""; - String sshKeyPair = kubernetesCluster.getKeyPair(); - if (StringUtils.isNotEmpty(sshKeyPair)) { - SSHKeyPairVO sshkp = sshKeyPairDao.findByName(owner.getAccountId(), owner.getDomainId(), sshKeyPair); - if (sshkp != null) { - pubKey += "\n - \"" + sshkp.getPublicKey() + "\""; - } - } - k8sNodeConfig = k8sNodeConfig.replace(sshPubKey, pubKey); - k8sNodeConfig = k8sNodeConfig.replace(joinIpKey, joinIp); - k8sNodeConfig = k8sNodeConfig.replace(clusterTokenKey, KubernetesClusterUtil.generateClusterToken(kubernetesCluster)); - k8sNodeConfig = k8sNodeConfig.replace(ejectIsoKey, String.valueOf(ejectIso)); - - k8sNodeConfig = updateKubeConfigWithRegistryDetails(k8sNodeConfig); - - return k8sNodeConfig; - } - - protected String updateKubeConfigWithRegistryDetails(String k8sConfig) { - /* genarate /etc/containerd/config.toml file on the nodes only if Kubernetes cluster is created to - * use docker private registry */ - String registryUsername = null; - String registryPassword = null; - String registryUrl = null; - - List details = kubernetesClusterDetailsDao.listDetails(kubernetesCluster.getId()); - for (KubernetesClusterDetailsVO detail : details) { - if (detail.getName().equals(ApiConstants.DOCKER_REGISTRY_USER_NAME)) { - registryUsername = detail.getValue(); - } - if (detail.getName().equals(ApiConstants.DOCKER_REGISTRY_PASSWORD)) { - registryPassword = detail.getValue(); - } - if (detail.getName().equals(ApiConstants.DOCKER_REGISTRY_URL)) { - registryUrl = detail.getValue(); - } - } - - if (StringUtils.isNoneEmpty(registryUsername, registryPassword, registryUrl)) { - // Update runcmd in the cloud-init configuration to run a script that updates the containerd config with provided registry details - String runCmd = "- bash -x /opt/bin/setup-containerd"; - - String registryEp = registryUrl.split("://")[1]; - k8sConfig = k8sConfig.replace("- containerd config default > /etc/containerd/config.toml", runCmd); - final String registryUrlKey = "{{registry.url}}"; - final String registryUrlEpKey = "{{registry.url.endpoint}}"; - final String registryAuthKey = "{{registry.token}}"; - final String registryUname = "{{registry.username}}"; - final String registryPsswd = "{{registry.password}}"; - - final String usernamePasswordKey = registryUsername + ":" + registryPassword; - String base64Auth = Base64.encodeBase64String(usernamePasswordKey.getBytes(com.cloud.utils.StringUtils.getPreferredCharset())); - k8sConfig = k8sConfig.replace(registryUrlKey, registryUrl); - k8sConfig = k8sConfig.replace(registryUrlEpKey, registryEp); - k8sConfig = k8sConfig.replace(registryUname, registryUsername); - k8sConfig = k8sConfig.replace(registryPsswd, registryPassword); - k8sConfig = k8sConfig.replace(registryAuthKey, base64Auth); - } - return k8sConfig; - } protected DeployDestination plan(final long nodesCount, final DataCenter zone, final ServiceOffering offering) throws InsufficientServerCapacityException { final int cpu_requested = offering.getCpu() * offering.getSpeed(); final long ram_requested = offering.getRamSize() * 1024L * 1024L; @@ -357,7 +285,7 @@ public class KubernetesClusterResourceModifierActionWorker extends KubernetesClu List nodes = new ArrayList<>(); for (int i = offset + 1; i <= nodeCount; i++) { UserVm vm = createKubernetesNode(publicIpAddress); - addKubernetesClusterVm(kubernetesCluster.getId(), vm.getId(), false); + addKubernetesClusterVm(kubernetesCluster.getId(), vm.getId(), false, false); if (kubernetesCluster.getNodeRootDiskSize() > 0) { resizeNodeVolume(vm); } @@ -469,7 +397,7 @@ public class KubernetesClusterResourceModifierActionWorker extends KubernetesClu final long domainId = account.getDomainId(); Nic vmNic = networkModel.getNicInNetwork(vmId, networkId); final Ip vmIp = new Ip(vmNic.getIPv4Address()); - PortForwardingRuleVO pfRule = Transaction.execute((TransactionCallbackWithException) status -> { + PortForwardingRuleVO pfRule = execute((TransactionCallbackWithException) status -> { PortForwardingRuleVO newRule = new PortForwardingRuleVO(null, publicIpId, sourcePort, sourcePort, @@ -501,11 +429,18 @@ public class KubernetesClusterResourceModifierActionWorker extends KubernetesClu * @throws NetworkRuleConflictException */ protected void provisionSshPortForwardingRules(IpAddress publicIp, Network network, Account account, - List clusterVMIds) throws ResourceUnavailableException, + List clusterVMIds, Map vmIdPortMap) throws ResourceUnavailableException, NetworkRuleConflictException { if (!CollectionUtils.isEmpty(clusterVMIds)) { - for (int i = 0; i < clusterVMIds.size(); ++i) { - provisionPublicIpPortForwardingRule(publicIp, network, account, clusterVMIds.get(i), CLUSTER_NODES_DEFAULT_START_SSH_PORT + i, DEFAULT_SSH_PORT); + int defaultNodesCount = clusterVMIds.size() - vmIdPortMap.size(); + int sourcePort = CLUSTER_NODES_DEFAULT_START_SSH_PORT; + for (int i = 0; i < defaultNodesCount; ++i) { + sourcePort = CLUSTER_NODES_DEFAULT_START_SSH_PORT + i; + provisionPublicIpPortForwardingRule(publicIp, network, account, clusterVMIds.get(i), sourcePort, DEFAULT_SSH_PORT); + } + for (int i = defaultNodesCount; i < clusterVMIds.size(); ++i) { + sourcePort += 1; + provisionPublicIpPortForwardingRule(publicIp, network, account, clusterVMIds.get(i), sourcePort, DEFAULT_SSH_PORT); } } } @@ -524,14 +459,14 @@ public class KubernetesClusterResourceModifierActionWorker extends KubernetesClu return rule; } - protected FirewallRule removeSshFirewallRule(final IpAddress publicIp) { + protected FirewallRule removeSshFirewallRule(final IpAddress publicIp, final long networkId) { FirewallRule rule = null; List firewallRules = firewallRulesDao.listByIpAndPurposeAndNotRevoked(publicIp.getId(), FirewallRule.Purpose.Firewall); for (FirewallRuleVO firewallRule : firewallRules) { - if (firewallRule.getSourcePortStart() == CLUSTER_NODES_DEFAULT_START_SSH_PORT) { + PortForwardingRuleVO pfRule = portForwardingRulesDao.findByNetworkAndPorts(networkId, firewallRule.getSourcePortStart(), firewallRule.getSourcePortEnd()); + if (firewallRule.getSourcePortStart() == CLUSTER_NODES_DEFAULT_START_SSH_PORT || (Objects.nonNull(pfRule) && pfRule.getDestinationPortStart() == DEFAULT_SSH_PORT) ) { rule = firewallRule; firewallService.revokeIngressFwRule(firewallRule.getId(), true); - break; } } return rule; @@ -546,7 +481,7 @@ public class KubernetesClusterResourceModifierActionWorker extends KubernetesClu for (PortForwardingRuleVO pfRule : pfRules) { if (pfRule.getVirtualMachineId() == vmId) { portForwardingRulesDao.remove(pfRule.getId()); - LOGGER.trace("Marking PF rule " + pfRule + " with Revoke state"); + logger.trace("Marking PF rule " + pfRule + " with Revoke state"); pfRule.setState(FirewallRule.State.Revoke); revokedRules.add(pfRule); break; @@ -643,19 +578,11 @@ public class KubernetesClusterResourceModifierActionWorker extends KubernetesClu lbService.assignToLoadBalancer(lb.getId(), null, vmIdIpMap, false); } - protected void createFirewallRules(IpAddress publicIp, List clusterVMIds, boolean apiRule) throws ManagementServerException { + protected Map createFirewallRules(IpAddress publicIp, List clusterVMIds, boolean apiRule) throws ManagementServerException { // Firewall rule for SSH access on each node VM - try { - int endPort = CLUSTER_NODES_DEFAULT_START_SSH_PORT + clusterVMIds.size() - 1; - provisionFirewallRules(publicIp, owner, CLUSTER_NODES_DEFAULT_START_SSH_PORT, endPort); - if (logger.isInfoEnabled()) { - logger.info(String.format("Provisioned firewall rule to open up port %d to %d on %s for Kubernetes cluster : %s", CLUSTER_NODES_DEFAULT_START_SSH_PORT, endPort, publicIp.getAddress().addr(), kubernetesCluster.getName())); - } - } catch (NoSuchFieldException | IllegalAccessException | ResourceUnavailableException | NetworkRuleConflictException e) { - throw new ManagementServerException(String.format("Failed to provision firewall rules for SSH access for the Kubernetes cluster : %s", kubernetesCluster.getName()), e); - } + Map vmIdPortMap = addFirewallRulesForNodes(publicIp, clusterVMIds.size()); if (!apiRule) { - return; + return vmIdPortMap; } // Firewall rule for API access for control node VMs try { @@ -667,6 +594,7 @@ public class KubernetesClusterResourceModifierActionWorker extends KubernetesClu } catch (NoSuchFieldException | IllegalAccessException | ResourceUnavailableException | NetworkRuleConflictException e) { throw new ManagementServerException(String.format("Failed to provision firewall rules for API access for the Kubernetes cluster : %s", kubernetesCluster.getName()), e); } + return vmIdPortMap; } /** @@ -681,11 +609,11 @@ public class KubernetesClusterResourceModifierActionWorker extends KubernetesClu * @throws ManagementServerException */ protected void setupKubernetesClusterIsolatedNetworkRules(IpAddress publicIp, Network network, List clusterVMIds, boolean apiRule) throws ManagementServerException { - createFirewallRules(publicIp, clusterVMIds, apiRule); + Map vmIdPortMap = createFirewallRules(publicIp, clusterVMIds, apiRule); // Port forwarding rule for SSH access on each node VM try { - provisionSshPortForwardingRules(publicIp, network, owner, clusterVMIds); + provisionSshPortForwardingRules(publicIp, network, owner, clusterVMIds, vmIdPortMap); } catch (ResourceUnavailableException | NetworkRuleConflictException e) { throw new ManagementServerException(String.format("Failed to activate SSH port forwarding rules for the Kubernetes cluster : %s", kubernetesCluster.getName()), e); } @@ -775,7 +703,8 @@ public class KubernetesClusterResourceModifierActionWorker extends KubernetesClu // Add port forwarding rule for SSH access on each node VM try { - provisionSshPortForwardingRules(publicIp, network, owner, clusterVMIds); + Map vmIdPortMap = getVmPortMap(); + provisionSshPortForwardingRules(publicIp, network, owner, clusterVMIds, vmIdPortMap); } catch (ResourceUnavailableException | NetworkRuleConflictException e) { throw new ManagementServerException(String.format("Failed to activate SSH port forwarding rules for the Kubernetes cluster : %s", kubernetesCluster.getName()), e); } @@ -802,7 +731,7 @@ public class KubernetesClusterResourceModifierActionWorker extends KubernetesClu final KubernetesClusterNodeType nodeType, final boolean updateNodeOffering, final boolean updateClusterOffering) { - return Transaction.execute(new TransactionCallback() { + return execute(new TransactionCallback() { @Override public KubernetesClusterVO doInTransaction(TransactionStatus status) { KubernetesClusterVO updatedCluster = kubernetesClusterDao.findById(kubernetesCluster.getId()); diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java index 60085e0c909..1e672c7a2ff 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java @@ -134,7 +134,7 @@ public class KubernetesClusterScaleWorker extends KubernetesClusterResourceModif } // Remove existing SSH firewall rules - FirewallRule firewallRule = removeSshFirewallRule(publicIp); + FirewallRule firewallRule = removeSshFirewallRule(publicIp, network.getId()); if (firewallRule == null) { throw new ManagementServerException("Firewall rule for node SSH access can't be provisioned"); } @@ -159,7 +159,8 @@ public class KubernetesClusterScaleWorker extends KubernetesClusterResourceModif } // Add port forwarding rule for SSH access on each node VM try { - provisionSshPortForwardingRules(publicIp, network, owner, clusterVMIds); + Map vmIdPortMap = getVmPortMap(); + provisionSshPortForwardingRules(publicIp, network, owner, clusterVMIds, vmIdPortMap); } catch (ResourceUnavailableException | NetworkRuleConflictException e) { throw new ManagementServerException(String.format("Failed to activate SSH port forwarding rules for the Kubernetes cluster : %s", kubernetesCluster.getName()), e); } @@ -416,10 +417,11 @@ public class KubernetesClusterScaleWorker extends KubernetesClusterResourceModif } List vmList; if (this.nodeIds != null) { - vmList = getKubernetesClusterVMMapsForNodes(this.nodeIds); + vmList = getKubernetesClusterVMMapsForNodes(this.nodeIds).stream().filter(vm -> !vm.isExternalNode()).collect(Collectors.toList()); } else { vmList = getKubernetesClusterVMMaps(); - vmList = vmList.subList((int) (kubernetesCluster.getControlNodeCount() + clusterSize), vmList.size()); + vmList = vmList.stream().filter(vm -> !vm.isExternalNode()).collect(Collectors.toList()); + vmList = vmList.subList((int) (kubernetesCluster.getControlNodeCount() + clusterSize - 1), vmList.size()); } Collections.reverse(vmList); removeNodesFromCluster(vmList); @@ -441,7 +443,9 @@ public class KubernetesClusterScaleWorker extends KubernetesClusterResourceModif logTransitStateToFailedIfNeededAndThrow(Level.ERROR, String.format("Scaling failed for Kubernetes cluster : %s, unable to provision node VM in the cluster", kubernetesCluster.getName()), e); } try { - List clusterVMIds = getKubernetesClusterVMMaps().stream().map(KubernetesClusterVmMapVO::getVmId).collect(Collectors.toList()); + List externalNodeIds = getKubernetesClusterVMMaps().stream().filter(KubernetesClusterVmMapVO::isExternalNode).map(KubernetesClusterVmMapVO::getVmId).collect(Collectors.toList()); + List clusterVMIds = getKubernetesClusterVMMaps().stream().filter(vm -> !vm.isExternalNode()).map(KubernetesClusterVmMapVO::getVmId).collect(Collectors.toList()); + clusterVMIds.addAll(externalNodeIds); scaleKubernetesClusterNetworkRules(clusterVMIds); } catch (ManagementServerException e) { logTransitStateToFailedIfNeededAndThrow(Level.ERROR, String.format("Scaling failed for Kubernetes cluster : %s, unable to update network rules", kubernetesCluster.getName()), e); diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java index 9f83d0ed0b9..97bd51fb780 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java @@ -317,7 +317,7 @@ public class KubernetesClusterStartWorker extends KubernetesClusterResourceModif ManagementServerException, InsufficientCapacityException, ResourceUnavailableException { UserVm k8sControlVM = null; k8sControlVM = createKubernetesControlNode(network, publicIpAddress); - addKubernetesClusterVm(kubernetesCluster.getId(), k8sControlVM.getId(), true); + addKubernetesClusterVm(kubernetesCluster.getId(), k8sControlVM.getId(), true, false); if (kubernetesCluster.getNodeRootDiskSize() > 0) { resizeNodeVolume(k8sControlVM); } @@ -339,7 +339,7 @@ public class KubernetesClusterStartWorker extends KubernetesClusterResourceModif for (int i = 1; i < kubernetesCluster.getControlNodeCount(); i++) { UserVm vm = null; vm = createKubernetesAdditionalControlNode(publicIpAddress, i); - addKubernetesClusterVm(kubernetesCluster.getId(), vm.getId(), true); + addKubernetesClusterVm(kubernetesCluster.getId(), vm.getId(), true, false); if (kubernetesCluster.getNodeRootDiskSize() > 0) { resizeNodeVolume(vm); } diff --git a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/AddNodesToKubernetesClusterCmd.java b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/AddNodesToKubernetesClusterCmd.java new file mode 100644 index 00000000000..3392928d225 --- /dev/null +++ b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/AddNodesToKubernetesClusterCmd.java @@ -0,0 +1,132 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.command.user.kubernetes.cluster; + +import com.cloud.kubernetes.cluster.KubernetesClusterEventTypes; +import com.cloud.kubernetes.cluster.KubernetesClusterService; + +import com.cloud.utils.exception.CloudRuntimeException; +import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiCommandResourceType; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.ApiErrorCode; +import org.apache.cloudstack.api.BaseAsyncCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.response.KubernetesClusterResponse; +import org.apache.cloudstack.api.response.UserVmResponse; +import org.apache.cloudstack.context.CallContext; +import org.apache.log4j.Logger; + +import javax.inject.Inject; + +import java.util.List; + +@APICommand(name = "addNodesToKubernetesCluster", + description = "Add nodes as workers to an existing CKS cluster. ", + responseObject = KubernetesClusterResponse.class, + since = "4.20.0", + authorized = {RoleType.Admin, RoleType.ResourceAdmin, RoleType.DomainAdmin, RoleType.User}) +public class AddNodesToKubernetesClusterCmd extends BaseAsyncCmd { + + @Inject + public KubernetesClusterService kubernetesClusterService; + + public static final Logger LOGGER = Logger.getLogger(AddNodesToKubernetesClusterCmd.class.getName()); + + @Parameter(name = ApiConstants.NODE_IDS, + type = CommandType.LIST, + collectionType = CommandType.UUID, + entityType= UserVmResponse.class, + description = "comma separated list of (external) node (physical or virtual machines) IDs that need to be" + + "added as worker nodes to an existing managed Kubernetes cluster (CKS)", + required = true, + since = "4.20.0") + private List nodeIds; + + @Parameter(name = ApiConstants.ID, type = CommandType.UUID, required = true, + entityType = KubernetesClusterResponse.class, + description = "the ID of the Kubernetes cluster", since = "4.20.0") + private Long clusterId; + + @Parameter(name = ApiConstants.MOUNT_CKS_ISO_ON_VR, type = CommandType.BOOLEAN, + description = "(optional) Vmware only, uses the CKS cluster network VR to mount the CKS ISO", + since = "4.20.0") + private Boolean mountCksIsoOnVr; + + + ///////////////////////////////////////////////////// + /////////////////// Accessors /////////////////////// + ///////////////////////////////////////////////////// + + public List getNodeIds() { + return nodeIds; + } + + public Long getClusterId() { + return clusterId; + } + + public boolean isMountCksIsoOnVr() { + return mountCksIsoOnVr != null && mountCksIsoOnVr; + } + + ///////////////////////////////////////////////////// + /////////////// API Implementation/////////////////// + ///////////////////////////////////////////////////// + + @Override + public String getEventType() { + return KubernetesClusterEventTypes.EVENT_KUBERNETES_CLUSTER_NODES_ADD; + } + + @Override + public String getEventDescription() { + return String.format("Adding %s nodes to the Kubernetes cluster with ID: %s", nodeIds.size(), clusterId); + } + + @Override + public void execute() { + try { + if (!kubernetesClusterService.addNodesToKubernetesCluster(this)) { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, String.format("Failed to add node(s) Kubernetes cluster ID: %d", getClusterId())); + } + final KubernetesClusterResponse response = kubernetesClusterService.createKubernetesClusterResponse(getClusterId()); + response.setResponseName(getCommandName()); + setResponseObject(response); + } catch (Exception e) { + throw new CloudRuntimeException(String.format("Failed to add nodes to cluster due to: %s", e.getLocalizedMessage()), e); + } + } + + @Override + public long getEntityOwnerId() { + return CallContext.current().getCallingAccount().getId(); + } + + @Override + public ApiCommandResourceType getApiResourceType() { + return ApiCommandResourceType.KubernetesCluster; + } + + @Override + public Long getApiResourceId() { + return getClusterId(); + } + +} diff --git a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/RemoveNodesFromKubernetesClusterCmd.java b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/RemoveNodesFromKubernetesClusterCmd.java new file mode 100644 index 00000000000..c0b569778fe --- /dev/null +++ b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/RemoveNodesFromKubernetesClusterCmd.java @@ -0,0 +1,109 @@ +package org.apache.cloudstack.api.command.user.kubernetes.cluster; + +import com.cloud.exception.ConcurrentOperationException; +import com.cloud.exception.InsufficientCapacityException; +import com.cloud.exception.NetworkRuleConflictException; +import com.cloud.exception.ResourceAllocationException; +import com.cloud.exception.ResourceUnavailableException; +import com.cloud.kubernetes.cluster.KubernetesClusterEventTypes; +import com.cloud.kubernetes.cluster.KubernetesClusterService; +import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiCommandResourceType; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.ApiErrorCode; +import org.apache.cloudstack.api.BaseAsyncCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.response.KubernetesClusterResponse; +import org.apache.cloudstack.api.response.UserVmResponse; +import org.apache.cloudstack.context.CallContext; +import org.apache.log4j.Logger; + +import javax.inject.Inject; +import java.util.List; + +@APICommand(name = "removeNodesFromKubernetesCluster", + description = "Removes external nodes from a CKS cluster. ", + responseObject = KubernetesClusterResponse.class, + since = "4.20.0", + authorized = {RoleType.Admin, RoleType.ResourceAdmin, RoleType.DomainAdmin, RoleType.User}) +public class RemoveNodesFromKubernetesClusterCmd extends BaseAsyncCmd { + + @Inject + public KubernetesClusterService kubernetesClusterService; + + protected static final Logger LOGGER = Logger.getLogger(RemoveNodesFromKubernetesClusterCmd.class); + + @Parameter(name = ApiConstants.NODE_IDS, + type = CommandType.LIST, + collectionType = CommandType.UUID, + entityType= UserVmResponse.class, + description = "comma separated list of node (physical or virtual machines) IDs that need to be" + + "removed from the Kubernetes cluster (CKS)", + required = true, + since = "4.20.0") + private List nodeIds; + + @Parameter(name = ApiConstants.ID, type = CommandType.UUID, required = true, + entityType = KubernetesClusterResponse.class, + description = "the ID of the Kubernetes cluster", since = "4.20.0") + private Long clusterId; + + ///////////////////////////////////////////////////// + /////////////////// Accessors /////////////////////// + ///////////////////////////////////////////////////// + + public List getNodeIds() { + return nodeIds; + } + + public Long getClusterId() { + return clusterId; + } + + ///////////////////////////////////////////////////// + /////////////// API Implementation/////////////////// + ///////////////////////////////////////////////////// + + @Override + public String getEventType() { + return KubernetesClusterEventTypes.EVENT_KUBERNETES_CLUSTER_NODES_REMOVE; + } + + @Override + public String getEventDescription() { + return String.format("Removing %s nodes from the Kubernetes Cluster with ID: %s", nodeIds.size(), clusterId); + } + + @Override + public void execute() throws ResourceUnavailableException, InsufficientCapacityException, ServerApiException, ConcurrentOperationException, ResourceAllocationException, NetworkRuleConflictException { + try { + if (!kubernetesClusterService.removeNodesFromKubernetesCluster(this)) { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, String.format("Failed to remove node(s) from Kubernetes cluster ID: %d", getClusterId())); + } + final KubernetesClusterResponse response = kubernetesClusterService.createKubernetesClusterResponse(getClusterId()); + response.setResponseName(getCommandName()); + setResponseObject(response); + } catch (Exception e) { + String err = String.format("Failed to remove node(s) from Kubernetes cluster ID: %d due to: %s", getClusterId(), e.getMessage()); + LOGGER.error(err, e); + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, err); + } + } + + @Override + public long getEntityOwnerId() { + return CallContext.current().getCallingAccount().getId(); + } + + @Override + public ApiCommandResourceType getApiResourceType() { + return ApiCommandResourceType.KubernetesCluster; + } + + @Override + public Long getApiResourceId() { + return getClusterId(); + } +} diff --git a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/response/KubernetesClusterResponse.java b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/response/KubernetesClusterResponse.java index c14fd9812f5..48f5bab471b 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/response/KubernetesClusterResponse.java +++ b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/response/KubernetesClusterResponse.java @@ -86,6 +86,10 @@ public class KubernetesClusterResponse extends BaseResponseWithAnnotations imple @Param(description = "the number of the etcd nodes on the Kubernetes cluster") private Long etcdNodes; + @SerializedName(ApiConstants.EXTERNAL_NODES) + @Param(description = "the number of the externally added worker nodes to the Kubernetes cluster") + private Long externalNodes; + @SerializedName(ApiConstants.TEMPLATE_ID) @Param(description = "the ID of the template of the Kubernetes cluster") private String templateId; @@ -165,7 +169,7 @@ public class KubernetesClusterResponse extends BaseResponseWithAnnotations imple @SerializedName(ApiConstants.VIRTUAL_MACHINES) @Param(description = "the list of virtualmachine associated with this Kubernetes cluster") - private List virtualMachines; + private List virtualMachines; @SerializedName(ApiConstants.IP_ADDRESS) @Param(description = "Public IP Address of the cluster") @@ -443,11 +447,19 @@ public class KubernetesClusterResponse extends BaseResponseWithAnnotations imple this.etcdNodes = etcdNodes; } - public void setVirtualMachines(List virtualMachines) { + public Long getExternalNodes() { + return externalNodes; + } + + public void setExternalNodes(Long externalNodes) { + this.externalNodes = externalNodes; + } + + public void setVirtualMachines(List virtualMachines) { this.virtualMachines = virtualMachines; } - public List getVirtualMachines() { + public List getVirtualMachines() { return virtualMachines; } diff --git a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-node.yml b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-node.yml index de1f4c9ffc7..0dba6fa7f5b 100644 --- a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-node.yml +++ b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-node.yml @@ -76,6 +76,19 @@ write_files: fi fi done <<< "$output" + else + ### Download from VR ### + ROUTER_IP="{{ k8s.vr.iso.mounted.ip }}" + echo "Downloading CKS binaries from the VR $ROUTER_IP" + if [ ! -d "${ISO_MOUNT_DIR}" ]; then + mkdir "${ISO_MOUNT_DIR}" + fi + ### Download from ROUTER_IP/cks-iso into ISO_MOUNT_DIR + AUX_DOWNLOAD_DIR=/aux-dwnld + mkdir -p $AUX_DOWNLOAD_DIR + wget -r -R "index.html*" $ROUTER_IP/cks-iso -P $AUX_DOWNLOAD_DIR + mv $AUX_DOWNLOAD_DIR/$ROUTER_IP/cks-iso/* $ISO_MOUNT_DIR + rm -rf $AUX_DOWNLOAD_DIR fi if [ -d "$BINARIES_DIR" ]; then break diff --git a/plugins/integrations/kubernetes-service/src/main/resources/script/remove-node-from-cluster b/plugins/integrations/kubernetes-service/src/main/resources/script/remove-node-from-cluster new file mode 100644 index 00000000000..1d2788bdd87 --- /dev/null +++ b/plugins/integrations/kubernetes-service/src/main/resources/script/remove-node-from-cluster @@ -0,0 +1,27 @@ +#!/bin/bash + +export PATH=$PATH:/opt/bin +node_name=$1 +node_type=$2 +operation=$3 + +if [ $operation == "remove" ]; then + if [ $node_type == "control" ]; then + # get the specific node + kubectl get nodes $node_name >/dev/null 2>&1 + if [[ $(echo $?) -eq 1 ]]; then + echo "No node with name $node_name present in the cluster, exiting..." + exit 0 + else + # Drain the node + kubectl drain $node_name --delete-local-data --force --ignore-daemonsets + fi + else + kubeadm reset -f + fi +else + sudo mkdir -p /home/cloud/.kube + sudo cp /root/.kube/config /home/cloud/.kube/ + sudo chown -R cloud:cloud /home/cloud/.kube + kubectl delete node $node_name +fi \ No newline at end of file diff --git a/plugins/integrations/kubernetes-service/src/main/resources/script/validate-cks-node b/plugins/integrations/kubernetes-service/src/main/resources/script/validate-cks-node new file mode 100644 index 00000000000..a6d38414b0a --- /dev/null +++ b/plugins/integrations/kubernetes-service/src/main/resources/script/validate-cks-node @@ -0,0 +1,45 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +OS=`awk -F= '/^NAME/{print $2}' /etc/os-release` +REQUIRED_PACKAGES=(cloud-init cloud-guest-utils conntrack apt-transport-https ca-certificates curl gnupg gnupg-agent \ + software-properties-common gnupg lsb-release python3-json-pointer python3-jsonschema cloud-init containerd.io) +declare -a MISSING_PACKAGES +if [[ $OS == *"Ubuntu"* || $OS == *"Debian"* ]]; then + for package in ${REQUIRED_PACKAGES[@]}; do + dpkg -s $package >/dev/null 2>&1 + if [ $? -eq 1 ]; then + MISSING_PACKAGES+="$package" + fi + done +else + for package in ${REQUIRED_PACKAGES[@]}; do + rpm -qa | grep $package >/dev/null 2>&1 + if [ $? -eq 1 ]; then + MISSING_PACKAGES[${#MISSING_PACKAGES[@]}]=$package + fi + done +fi + +echo ${#MISSING_PACKAGES[@]} +if (( ${#MISSING_PACKAGES[@]} )); then + echo "Following packages are missing in the node template: ${MISSING_PACKAGES[@]}" + exit 1 +else + echo 0 +fi \ No newline at end of file diff --git a/server/src/main/java/com/cloud/network/NetworkServiceImpl.java b/server/src/main/java/com/cloud/network/NetworkServiceImpl.java index 1314d7dd574..e1fc14aeb72 100644 --- a/server/src/main/java/com/cloud/network/NetworkServiceImpl.java +++ b/server/src/main/java/com/cloud/network/NetworkServiceImpl.java @@ -6144,6 +6144,27 @@ public class NetworkServiceImpl extends ManagerBase implements NetworkService, C return new ArrayList<>(this.internalLoadBalancerElementServiceMap.values()); } + @Override + public boolean handleCksIsoOnNetworkVirtualRouter(Long virtualRouterId, boolean mount) throws ResourceUnavailableException { + DomainRouterVO router = routerDao.findById(virtualRouterId); + if (router == null) { + String err = String.format("Cannot find VR with ID %s", virtualRouterId); + s_logger.error(err); + throw new CloudRuntimeException(err); + } + Commands commands = new Commands(Command.OnError.Stop); + commandSetupHelper.createHandleCksIsoCommand(router, mount, commands); + if (!networkHelper.sendCommandsToRouter(router, commands)) { + throw new CloudRuntimeException(String.format("Unable to send commands to virtual router: %s", router.getHostId())); + } + Answer answer = commands.getAnswer("handleCksIso"); + if (answer == null || !answer.getResult()) { + s_logger.error(String.format("Could not handle the CKS ISO properly: %s", answer.getDetails())); + return false; + } + return true; + } + /** * Retrieves the active quarantine for the given public IP address. It can find by the ID of the quarantine or the address of the public IP. * @throws CloudRuntimeException if it does not find an active quarantine for the given public IP. diff --git a/server/src/main/java/com/cloud/network/router/CommandSetupHelper.java b/server/src/main/java/com/cloud/network/router/CommandSetupHelper.java index ce5024a5e1b..69192100514 100644 --- a/server/src/main/java/com/cloud/network/router/CommandSetupHelper.java +++ b/server/src/main/java/com/cloud/network/router/CommandSetupHelper.java @@ -28,6 +28,7 @@ import java.util.Set; import javax.inject.Inject; +import com.cloud.agent.api.HandleCksIsoCommand; import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.engine.orchestration.service.NetworkOrchestrationService; import org.apache.cloudstack.framework.config.dao.ConfigurationDao; @@ -1404,4 +1405,11 @@ public class CommandSetupHelper { cmds.addCommand("updateNetwork", cmd); } } + + public void createHandleCksIsoCommand(final VirtualRouter router, final boolean mount, Commands cmds) { + HandleCksIsoCommand command = new HandleCksIsoCommand(mount); + command.setAccessDetail(NetworkElementCommand.ROUTER_IP, _routerControlHelper.getRouterControlIp(router.getId())); + command.setAccessDetail(NetworkElementCommand.ROUTER_NAME, router.getInstanceName()); + cmds.addCommand("handleCksIso", command); + } } diff --git a/server/src/main/java/com/cloud/template/TemplateManagerImpl.java b/server/src/main/java/com/cloud/template/TemplateManagerImpl.java index c4692ce62d6..7c15c8e369f 100755 --- a/server/src/main/java/com/cloud/template/TemplateManagerImpl.java +++ b/server/src/main/java/com/cloud/template/TemplateManagerImpl.java @@ -34,6 +34,7 @@ import java.util.stream.Collectors; import javax.inject.Inject; import javax.naming.ConfigurationException; +import com.cloud.vm.VirtualMachine; import org.apache.cloudstack.acl.SecurityChecker.AccessType; import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.BaseCmd; @@ -1141,35 +1142,33 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, @Override @ActionEvent(eventType = EventTypes.EVENT_ISO_DETACH, eventDescription = "detaching ISO", async = true) - public boolean detachIso(long vmId, boolean forced) { + public boolean detachIso(long vmId, Long isoParamId, Boolean... extraParams) { Account caller = CallContext.current().getCallingAccount(); Long userId = CallContext.current().getCallingUserId(); - // Verify input parameters - UserVmVO vmInstanceCheck = _userVmDao.findById(vmId); - if (vmInstanceCheck == null) { - throw new InvalidParameterValueException("Unable to find a virtual machine with id " + vmId); - } + boolean forced = extraParams != null && extraParams.length > 0 ? extraParams[0] : false; + boolean isVirtualRouter = extraParams != null && extraParams.length > 1 ? extraParams[1] : false; - UserVm userVM = _userVmDao.findById(vmId); - if (userVM == null) { + // Verify input parameters + VirtualMachine virtualMachine = !isVirtualRouter ? _userVmDao.findById(vmId) : _vmInstanceDao.findById(vmId); + if (virtualMachine == null || (isVirtualRouter && virtualMachine.getType() != VirtualMachine.Type.DomainRouter)) { throw new InvalidParameterValueException("Please specify a valid VM."); } - _accountMgr.checkAccess(caller, null, true, userVM); + _accountMgr.checkAccess(caller, null, true, virtualMachine); - Long isoId = userVM.getIsoId(); + Long isoId = !isVirtualRouter ? ((UserVm) virtualMachine).getIsoId() : isoParamId; if (isoId == null) { throw new InvalidParameterValueException("The specified VM has no ISO attached to it."); } - CallContext.current().setEventDetails("Vm Id: " + userVM.getUuid() + " ISO Id: " + isoId); + CallContext.current().setEventDetails("Vm Id: " + virtualMachine.getUuid() + " ISO Id: " + isoId); - State vmState = userVM.getState(); + State vmState = virtualMachine.getState(); if (vmState != State.Running && vmState != State.Stopped) { throw new InvalidParameterValueException("Please specify a VM that is either Stopped or Running."); } - boolean result = attachISOToVM(vmId, userId, isoId, false, forced); // attach=false + boolean result = attachISOToVM(vmId, userId, isoId, false, forced, isVirtualRouter); // attach=false // => detach if (result) { return result; @@ -1180,14 +1179,26 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, @Override @ActionEvent(eventType = EventTypes.EVENT_ISO_ATTACH, eventDescription = "attaching ISO", async = true) - public boolean attachIso(long isoId, long vmId, boolean forced) { + public boolean attachIso(long isoId, long vmId, Boolean... extraParams) { Account caller = CallContext.current().getCallingAccount(); Long userId = CallContext.current().getCallingUserId(); + boolean forced = extraParams != null && extraParams.length > 0 ? extraParams[0] : false; + boolean isVirtualRouter = extraParams != null && extraParams.length > 1 ? extraParams[1] : false; + // Verify input parameters - UserVmVO vm = _userVmDao.findById(vmId); + VirtualMachine vm = _userVmDao.findById(vmId); if (vm == null) { - throw new InvalidParameterValueException("Unable to find a virtual machine with id " + vmId); + if (isVirtualRouter) { + vm = _vmInstanceDao.findById(vmId); + if (vm == null) { + throw new InvalidParameterValueException("Unable to find a virtual machine with id " + vmId); + } else if (vm.getType() != VirtualMachine.Type.DomainRouter) { + throw new InvalidParameterValueException("Unable to find a virtual router with id " + vmId); + } + } else { + throw new InvalidParameterValueException("Unable to find a virtual machine with id " + vmId); + } } VMTemplateVO iso = _tmpltDao.findById(isoId); @@ -1223,7 +1234,7 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, if (VMWARE_TOOLS_ISO.equals(iso.getUniqueName()) && vm.getHypervisorType() != Hypervisor.HypervisorType.VMware) { throw new InvalidParameterValueException("Cannot attach VMware tools drivers to incompatible hypervisor " + vm.getHypervisorType()); } - boolean result = attachISOToVM(vmId, userId, isoId, true, forced); + boolean result = attachISOToVM(vmId, userId, isoId, true, forced, isVirtualRouter); if (result) { return result; } else { @@ -1262,10 +1273,10 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, } } - private boolean attachISOToVM(long vmId, long isoId, boolean attach, boolean forced) { - UserVmVO vm = _userVmDao.findById(vmId); + private boolean attachISOToVM(long vmId, long isoId, boolean attach, boolean forced, boolean isVirtualRouter) { + VirtualMachine vm = !isVirtualRouter ? _userVmDao.findById(vmId) : _vmInstanceDao.findById(vmId); - if (vm == null) { + if (vm == null || (isVirtualRouter && vm.getType() != VirtualMachine.Type.DomainRouter)) { return false; } else if (vm.getState() != State.Running) { return true; @@ -1304,16 +1315,16 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, return (a != null && a.getResult()); } - private boolean attachISOToVM(long vmId, long userId, long isoId, boolean attach, boolean forced) { + private boolean attachISOToVM(long vmId, long userId, long isoId, boolean attach, boolean forced, boolean isVirtualRouter) { UserVmVO vm = _userVmDao.findById(vmId); VMTemplateVO iso = _tmpltDao.findById(isoId); - boolean success = attachISOToVM(vmId, isoId, attach, forced); - if (success && attach) { + boolean success = attachISOToVM(vmId, isoId, attach, forced, isVirtualRouter); + if (success && attach && !isVirtualRouter) { vm.setIsoId(iso.getId()); _userVmDao.update(vmId, vm); } - if (success && !attach) { + if (success && !attach && !isVirtualRouter) { vm.setIsoId(null); _userVmDao.update(vmId, vm); } @@ -2113,6 +2124,7 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, Map details = cmd.getDetails(); Account account = CallContext.current().getCallingAccount(); boolean cleanupDetails = cmd.isCleanupDetails(); + boolean forCks = cmd instanceof UpdateTemplateCmd && ((UpdateTemplateCmd) cmd).getForCks(); // verify that template exists VMTemplateVO template = _tmpltDao.findById(id); @@ -2260,6 +2272,7 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, template.setDetails(details); _tmpltDao.saveDetails(template); } + template.setForCks(forCks); _tmpltDao.update(id, template); diff --git a/server/src/test/java/com/cloud/vpc/MockNetworkManagerImpl.java b/server/src/test/java/com/cloud/vpc/MockNetworkManagerImpl.java index 106fc7fa543..4eaf31e6bb8 100644 --- a/server/src/test/java/com/cloud/vpc/MockNetworkManagerImpl.java +++ b/server/src/test/java/com/cloud/vpc/MockNetworkManagerImpl.java @@ -1108,4 +1108,9 @@ public class MockNetworkManagerImpl extends ManagerBase implements NetworkOrches public List getInternalLoadBalancerElements() { return null; } + + @Override + public boolean handleCksIsoOnNetworkVirtualRouter(Long virtualRouterId, boolean mount) { + return false; + } } diff --git a/systemvm/debian/opt/cloud/bin/cks_iso.sh b/systemvm/debian/opt/cloud/bin/cks_iso.sh new file mode 100644 index 00000000000..7cbcbc040d5 --- /dev/null +++ b/systemvm/debian/opt/cloud/bin/cks_iso.sh @@ -0,0 +1,34 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +BASE_DIR=/var/www/html +CKS_ISO_DIR=$BASE_DIR/cks-iso +if [ "$1" == "true" ] +then + mkdir -p $CKS_ISO_DIR + echo "Options +Indexes" > $BASE_DIR/.htaccess + echo "Mounting CKS ISO into $CKS_ISO_DIR" + mount /dev/cdrom $CKS_ISO_DIR +else + echo "Unmounting CKS ISO from $CKS_ISO_DIR" + umount $CKS_ISO_DIR + echo "Options -Indexes" > $BASE_DIR/.htaccess + rm -rf $CKS_ISO_DIR +fi +echo "Restarting apache2 service" +service apache2 restart diff --git a/systemvm/debian/opt/cloud/bin/cs/CsGuestNetwork.py b/systemvm/debian/opt/cloud/bin/cs/CsGuestNetwork.py index 41b8b644186..fe8737208c4 100755 --- a/systemvm/debian/opt/cloud/bin/cs/CsGuestNetwork.py +++ b/systemvm/debian/opt/cloud/bin/cs/CsGuestNetwork.py @@ -15,7 +15,6 @@ # specific language governing permissions and limitations # under the License. from merge import DataBag -from . import CsHelper class CsGuestNetwork: @@ -40,7 +39,7 @@ class CsGuestNetwork: return self.config.get_dns() dns = [] - if 'router_guest_gateway' in self.data and not self.config.use_extdns() and 'is_vr_guest_gateway' not in self.data: + if 'router_guest_gateway' in self.data and not self.config.use_extdns() and ('is_vr_guest_gateway' not in self.data or not self.data['is_vr_guest_gateway']): dns.append(self.data['router_guest_gateway']) if 'dns' in self.data: diff --git a/tools/appliance/cks/ubuntu/22.04/scripts/setup_template.sh b/tools/appliance/cks/ubuntu/22.04/scripts/setup_template.sh new file mode 100644 index 00000000000..0326b128daa --- /dev/null +++ b/tools/appliance/cks/ubuntu/22.04/scripts/setup_template.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +function create_user() { + username=$1 + password=$2 + + # Create the user with the specified username + sudo useradd -m -s /bin/bash $username + + # Set the user's password + echo "$username:$password" | sudo chpasswd + + echo "User '$username' has been created with the password '$password'" +} + +sudo mkdir -p /opt/bin +create_user cloud password + +echo $SSHKEY +if [[ ! -z "$SSHKEY" ]]; then + mkdir -p /home/cloud/.ssh/ + mkdir .ssh + echo $SSHKEY > ~/.ssh/authorized_keys +else + echo "Please place Management server public key in the variables" + exit 1 +fi diff --git a/ui/public/locales/en.json b/ui/public/locales/en.json index a49b1dee583..7e52486aa3d 100644 --- a/ui/public/locales/en.json +++ b/ui/public/locales/en.json @@ -47,6 +47,8 @@ "label.acquiring.ip": "Acquiring IP", "label.associated.resource": "Associated resource", "label.action": "Action", +"label.action.add.nodes.to.kubernetes.cluster": "Add nodes to Kubernetes cluster", +"label.action.remove.nodes.from.kubernetes.cluster": "Remove nodes from Kubernetes cluster", "label.action.attach.disk": "Attach disk", "label.action.attach.iso": "Attach ISO", "label.action.bulk.delete.egress.firewall.rules": "Bulk delete egress firewall rules", @@ -245,6 +247,7 @@ "label.add.list.name": "ACL List name", "label.add.logical.router": "Add Logical Router to this Network", "label.add.more": "Add more", +"label.add.nodes": "Add Nodes to Kubernetes Cluster", "label.add.netscaler.device": "Add Netscaler device", "label.add.network": "Add Network", "label.add.network.acl": "Add Network ACL", @@ -922,7 +925,7 @@ "label.fix.errors": "Fix errors", "label.fixed": "Fixed offering", "label.for": "for", -"label.for.cks": "For CKS", +"label.forcks": "For CKS", "label.forbidden": "Forbidden", "label.forced": "Force", "label.force.stop": "Force stop", @@ -1119,6 +1122,7 @@ "label.ipv6.subnets": "IPv6 Subnets", "label.ip.addresses": "IP Addresses", "label.iqn": "Target IQN", +"label.is.base64.encoded": "Base64 encoded", "label.is.in.progress": "is in progress", "label.is.shared": "Is shared", "label.is2faenabled": "Is 2FA enabled", @@ -1176,11 +1180,13 @@ "label.kubernetes": "Kubernetes", "label.kubernetes.access.details": "The kubernetes nodes can be accessed via ssh using:
ssh -i [ssh_key] -p [port_number] cloud@[public_ip_address]

where,
ssh_key: points to the ssh private key file corresponding to the key that was associated while creating the Kubernetes cluster. If no ssh key was provided during Kubernetes cluster creation, use the ssh private key of the management server.
port_number: can be obtained from the Port Forwarding Tab (Public Port column)", "label.kubernetes.cluster": "Kubernetes cluster", +"label.kubernetes.cluster.add.nodes.to.cluster": "Add nodes to Kubernetes cluster", "label.kubernetes.cluster.create": "Create Kubernetes cluster", "label.kubernetes.cluster.delete": "Delete Kubernetes cluster", "label.kubernetes.cluster.scale": "Scale Kubernetes cluster", "label.kubernetes.cluster.start": "Start Kubernetes cluster", "label.kubernetes.cluster.stop": "Stop Kubernetes cluster", +"label.kubernetes.cluster.remove.nodes.from.cluster": "Remove nodes from Kubernetes cluster", "label.kubernetes.cluster.upgrade": "Upgrade Kubernetes cluster", "label.kubernetes.dashboard": "Kubernetes dashboard UI", "label.kubernetes.dashboard.create.token": "Create token for Kubernetes dashboard", @@ -1374,6 +1380,7 @@ "label.monitor.url": "URL Path", "label.monthly": "Monthly", "label.more.access.dashboard.ui": "More about accessing dashboard UI", +"label.mount.cks.iso.on.vr": "Mount and serve CKS ISO on the CKS cluster's network Virtual Router", "label.move.down.row": "Move down one row", "label.move.to.bottom": "Move to bottom", "label.move.to.top": "Move to top", @@ -1741,6 +1748,7 @@ "label.remove.logical.router": "Remove logical router", "label.remove.network.offering": "Remove Network offering", "label.remove.network.route.table": "Remove Tungsten Fabric Network routing table", +"label.remove.nodes": "Remove nodes from Kubernetes cluster", "label.remove.pf": "Remove port forwarding rule", "label.remove.policy": "Remove policy", "label.remove.project.account": "Remove Account from project", @@ -2353,6 +2361,8 @@ "label.vnmc": "VNMC", "label.volgroup": "Volume group", "label.volume": "Volume", +"label.vms.empty": "No VMs available to be added to the Kubernetes cluster", +"label.vms.remove.empty": "No external VMs present in the Kubernetes cluster to be removed", "label.volume.empty": "No data volumes attached to this Instance", "label.volume.volumefileupload.description": "Click or drag file to this area to upload.", "label.volume.encryption.support": "Volume Encryption Supported", @@ -2590,6 +2600,8 @@ "message.adding.host": "Adding host", "message.adding.netscaler.device": "Adding Netscaler device", "message.adding.netscaler.provider": "Adding Netscaler provider", +"message.adding.nodes.to.cluster": "Adding nodes to Kubernetes cluster", +"message.removing.nodes.from.cluster": "Removing nodes from Kubernetes cluster", "message.advanced.security.group": "Choose this if you wish to use security groups to provide guest Instance isolation.", "message.allowed": "Allowed", "message.alert.show.all.stats.data": "This may return a lot of data depending on VM statistics and retention settings", @@ -3008,10 +3020,12 @@ "message.ip.v6.prefix.delete": "IPv6 prefix deleted", "message.iso.desc": "Disc image containing data or bootable media for OS.", "message.kubeconfig.cluster.not.available": "Kubernetes cluster kubeconfig not available currently.", +"message.kubernetes.cluster.add.nodes": "Please confirm that you want to add the following nodes to the cluster", "message.kubernetes.cluster.delete": "Please confirm that you want to destroy the cluster.", "message.kubernetes.cluster.scale": "Please select desired cluster configuration.", "message.kubernetes.cluster.start": "Please confirm that you want to start the cluster.", "message.kubernetes.cluster.stop": "Please confirm that you want to stop the cluster.", +"message.kubernetes.cluster.remove.nodes": "Please confirm that you want to remove the following nodes from the cluster", "message.kubernetes.cluster.upgrade": "Please select new Kubernetes version.", "message.kubernetes.version.delete": "Please confirm that you want to delete this Kubernetes version.", "message.l2.network.unsupported.for.nsx": "L2 networks aren't supported for NSX enabled zones", @@ -3189,6 +3203,8 @@ "message.success.add.network.acl": "Successfully added Network ACL list", "message.success.add.network.static.route": "Successfully added Network Static Route", "message.success.add.network.permissions": "Successfully added Network permissions", +"message.success.add.nodes.to.cluster": "Successfully added nodes to Kubernetes cluster", +"message.success.remove.nodes.from.cluster": "Successfully removed nodes from Kubernetes cluster", "message.success.add.physical.network": "Successfully added Physical Network", "message.success.add.object.storage": "Successfully added Object Storage", "message.success.add.policy.rule": "Successfully added Policy rule", diff --git a/ui/src/config/section/compute.js b/ui/src/config/section/compute.js index 7a0644ba98f..a952f515b95 100644 --- a/ui/src/config/section/compute.js +++ b/ui/src/config/section/compute.js @@ -601,6 +601,26 @@ export default { popup: true, component: shallowRef(defineAsyncComponent(() => import('@/views/compute/UpgradeKubernetesCluster.vue'))) }, + { + api: 'addNodesToKubernetesCluster', + icon: 'plus-outlined', + label: 'label.kubernetes.cluster.add.nodes.to.cluster', + message: 'message.kubernetes.cluster.add.nodes', + dataView: true, + show: (record) => { return ['Running', 'Alert'].includes(record.state) && record.clustertype === 'CloudManaged' }, + popup: true, + component: shallowRef(defineAsyncComponent(() => import('@/views/compute/KubernetesAddNodes.vue'))) + }, + { + api: 'removeNodesFromKubernetesCluster', + icon: 'minus-outlined', + label: 'label.kubernetes.cluster.remove.nodes.from.cluster', + message: 'message.kubernetes.cluster.remove.nodes', + dataView: true, + show: (record) => { return ['Running', 'Alert'].includes(record.state) && record.clustertype === 'CloudManaged' && (record.virtualmachines.filter(vm => vm.isexternalnode) || []).length > 0 }, + popup: true, + component: shallowRef(defineAsyncComponent(() => import('@/views/compute/KubernetesRemoveNodes.vue'))) + }, { api: 'deleteKubernetesCluster', icon: 'delete-outlined', diff --git a/ui/src/config/section/image.js b/ui/src/config/section/image.js index e6095c4aba1..7ac200eed2b 100644 --- a/ui/src/config/section/image.js +++ b/ui/src/config/section/image.js @@ -59,7 +59,7 @@ export default { details: () => { var fields = ['name', 'id', 'displaytext', 'checksum', 'hypervisor', 'format', 'ostypename', 'size', 'physicalsize', 'isready', 'passwordenabled', 'crossZones', 'templatetype', 'directdownload', 'deployasis', 'ispublic', 'isfeatured', 'isextractable', 'isdynamicallyscalable', 'crosszones', 'type', - 'account', 'domain', 'created', 'userdatadetails', 'userdatapolicy'] + 'account', 'domain', 'created', 'userdatadetails', 'userdatapolicy', 'forcks'] if (['Admin'].includes(store.getters.userInfo.roletype)) { fields.push('templatetag', 'templatetype', 'url') } diff --git a/ui/src/utils/plugins.js b/ui/src/utils/plugins.js index 3e829bdb8dd..67dfa8366fe 100644 --- a/ui/src/utils/plugins.js +++ b/ui/src/utils/plugins.js @@ -390,6 +390,8 @@ export const resourceTypePlugin = { return 'publicip' case 'NetworkAcl': return 'acllist' + case 'KubernetesCluster': + return 'kubernetes' case 'SystemVm': case 'PhysicalNetwork': case 'Backup': @@ -427,6 +429,9 @@ export const resourceTypePlugin = { var routePath = this.$getRouteFromResourceType(resourceType) if (!routePath) return '' var route = this.$router.resolve('/' + routePath) + if (routePath === 'kubernetes') { + return route?.meta?.icon[0] + } return route?.meta?.icon || '' } } @@ -488,6 +493,15 @@ export const fileSizeUtilPlugin = { } } +function isBase64 (str) { + try { + const decoded = new TextDecoder().decode(Uint8Array.from(atob(str), c => c.charCodeAt(0))) + return btoa(decoded) === str + } catch (err) { + return false + } +} + export const genericUtilPlugin = { install (app) { app.config.globalProperties.$isValidUuid = function (uuid) { @@ -496,8 +510,8 @@ export const genericUtilPlugin = { } app.config.globalProperties.$toBase64AndURIEncoded = function (text) { - const base64regex = /^([0-9a-zA-Z+/]{4})*(([0-9a-zA-Z+/]{2}==)|([0-9a-zA-Z+/]{3}=))?$/ - if (base64regex.test(text)) { + console.log(isBase64(text)) + if (isBase64(text)) { return text } return encodeURIComponent(btoa(unescape(encodeURIComponent(text)))) diff --git a/ui/src/views/compute/KubernetesAddNodes.vue b/ui/src/views/compute/KubernetesAddNodes.vue new file mode 100644 index 00000000000..d6d8dc72731 --- /dev/null +++ b/ui/src/views/compute/KubernetesAddNodes.vue @@ -0,0 +1,177 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + + + + diff --git a/ui/src/views/compute/KubernetesRemoveNodes.vue b/ui/src/views/compute/KubernetesRemoveNodes.vue new file mode 100644 index 00000000000..57d6409b784 --- /dev/null +++ b/ui/src/views/compute/KubernetesRemoveNodes.vue @@ -0,0 +1,151 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + + + + diff --git a/ui/src/views/compute/RegisterUserData.vue b/ui/src/views/compute/RegisterUserData.vue index 990e59ff277..9e4b3623f6f 100644 --- a/ui/src/views/compute/RegisterUserData.vue +++ b/ui/src/views/compute/RegisterUserData.vue @@ -43,6 +43,9 @@ v-model:value="form.userdata" :placeholder="apiParams.userdata.description"/> + + + + + + +