From 2a1de76517d33b3d5d01eb51365d794733a90aad Mon Sep 17 00:00:00 2001 From: Pearl Dsilva Date: Tue, 25 Jun 2024 20:53:33 -0400 Subject: [PATCH] Add support for using different CNI plugins with CKS * Add support for using different CNI plugins with CKS * remove unused import * Add UI support and list cni config API * necessary UI changes * add license * changes to support external cni * UI changes * Fix NPE on restarting VPC with additional public IPs * fix merge conflict * add asnumber to create k8s svc layer * support cni framework to use as-numbers * update code * condition to ignore undefined jinja template variables --- .../main/java/com/cloud/event/EventTypes.java | 1 + .../kubernetes/cluster/KubernetesCluster.java | 2 + .../com/cloud/server/ManagementService.java | 11 +- .../main/java/com/cloud/user/UserData.java | 1 + .../apache/cloudstack/api/ApiConstants.java | 3 + .../userdata/BaseRegisterUserDataCmd.java | 87 ++++++++ .../userdata/ListCniConfigurationCmd.java | 58 ++++++ .../user/userdata/ListUserDataCmd.java | 2 +- .../userdata/RegisterCniConfigurationCmd.java | 76 +++++++ .../user/userdata/RegisterUserDataCmd.java | 79 +------- .../user/userdata/ListUserDataCmdTest.java | 4 +- .../upgrade/dao/Upgrade41910to42000.java | 94 --------- .../upgrade/dao/Upgrade42010to42100.java | 97 +++++++++ .../main/java/com/cloud/user/UserDataVO.java | 10 + .../META-INF/db/schema-41910to42000.sql | 23 --- .../META-INF/db/schema-42010to42100.sql | 34 ++++ .../cluster/KubernetesClusterManagerImpl.java | 24 ++- .../cluster/KubernetesClusterService.java | 2 + .../cluster/KubernetesClusterVO.java | 23 +++ .../KubernetesClusterActionWorker.java | 40 ++-- .../KubernetesClusterDestroyWorker.java | 18 ++ .../KubernetesClusterStartWorker.java | 31 ++- .../KubernetesClusterUpgradeWorker.java | 5 +- .../cluster/CreateKubernetesClusterCmd.java | 18 ++ .../main/resources/conf/k8s-control-node.yml | 8 +- .../resources/script/upgrade-kubernetes.sh | 10 +- .../java/com/cloud/api/ApiResponseHelper.java | 10 +- .../com/cloud/network/NetworkServiceImpl.java | 31 +++ .../cloud/server/ManagementServerImpl.java | 35 +++- .../server/ManagementServerImplTest.java | 6 +- ui/public/locales/en.json | 6 +- ui/src/components/view/DetailsTab.vue | 7 +- ui/src/config/section/compute.js | 77 ++++++++ ui/src/views/AutogenView.vue | 6 + .../views/compute/CreateKubernetesCluster.vue | 187 +++++++++++++++++- ui/src/views/compute/RegisterUserData.vue | 53 +++-- .../compute/wizard/UserDataSelection.vue | 6 + 37 files changed, 927 insertions(+), 258 deletions(-) create mode 100644 api/src/main/java/org/apache/cloudstack/api/command/user/userdata/BaseRegisterUserDataCmd.java create mode 100644 api/src/main/java/org/apache/cloudstack/api/command/user/userdata/ListCniConfigurationCmd.java create mode 100644 api/src/main/java/org/apache/cloudstack/api/command/user/userdata/RegisterCniConfigurationCmd.java diff --git a/api/src/main/java/com/cloud/event/EventTypes.java b/api/src/main/java/com/cloud/event/EventTypes.java index 81ed185dae5..d73bd056022 100644 --- a/api/src/main/java/com/cloud/event/EventTypes.java +++ b/api/src/main/java/com/cloud/event/EventTypes.java @@ -289,6 +289,7 @@ public class EventTypes { //registering userdata events public static final String EVENT_REGISTER_USER_DATA = "REGISTER.USER.DATA"; + public static final String EVENT_REGISTER_CNI_CONFIG = "REGISTER.CNI.CONFIG"; //register for user API and secret keys public static final String EVENT_REGISTER_FOR_SECRET_API_KEY = "REGISTER.USER.KEY"; diff --git a/api/src/main/java/com/cloud/kubernetes/cluster/KubernetesCluster.java b/api/src/main/java/com/cloud/kubernetes/cluster/KubernetesCluster.java index c01d8f82a26..c0eb7812988 100644 --- a/api/src/main/java/com/cloud/kubernetes/cluster/KubernetesCluster.java +++ b/api/src/main/java/com/cloud/kubernetes/cluster/KubernetesCluster.java @@ -164,4 +164,6 @@ public interface KubernetesCluster extends ControlledEntity, com.cloud.utils.fsm Long getWorkerTemplateId(); Long getEtcdTemplateId(); Long getEtcdNodeCount(); + Long getCniConfigId(); + String getCniConfigDetails(); } diff --git a/api/src/main/java/com/cloud/server/ManagementService.java b/api/src/main/java/com/cloud/server/ManagementService.java index 18f3e901cd9..377fbfb5f07 100644 --- a/api/src/main/java/com/cloud/server/ManagementService.java +++ b/api/src/main/java/com/cloud/server/ManagementService.java @@ -59,9 +59,9 @@ import org.apache.cloudstack.api.command.user.ssh.CreateSSHKeyPairCmd; import org.apache.cloudstack.api.command.user.ssh.DeleteSSHKeyPairCmd; import org.apache.cloudstack.api.command.user.ssh.ListSSHKeyPairsCmd; import org.apache.cloudstack.api.command.user.ssh.RegisterSSHKeyPairCmd; +import org.apache.cloudstack.api.command.user.userdata.BaseRegisterUserDataCmd; import org.apache.cloudstack.api.command.user.userdata.DeleteUserDataCmd; import org.apache.cloudstack.api.command.user.userdata.ListUserDataCmd; -import org.apache.cloudstack.api.command.user.userdata.RegisterUserDataCmd; import org.apache.cloudstack.api.command.user.vm.GetVMPasswordCmd; import org.apache.cloudstack.api.command.user.vmgroup.UpdateVMGroupCmd; import org.apache.cloudstack.config.Configuration; @@ -360,17 +360,16 @@ public interface ManagementService { * The api command class. * @return The list of userdatas found. */ - Pair, Integer> listUserDatas(ListUserDataCmd cmd); + Pair, Integer> listUserDatas(ListUserDataCmd cmd, boolean forCks); /** * Registers a userdata. * - * @param cmd - * The api command class. + * @param cmd The api command class. + * @param forCks * @return A VO with the registered userdata. */ - UserData registerUserData(RegisterUserDataCmd cmd); - + UserData registerUserData(BaseRegisterUserDataCmd cmd); /** * Deletes a userdata. * diff --git a/api/src/main/java/com/cloud/user/UserData.java b/api/src/main/java/com/cloud/user/UserData.java index fa0c50473c0..13a3c74f367 100644 --- a/api/src/main/java/com/cloud/user/UserData.java +++ b/api/src/main/java/com/cloud/user/UserData.java @@ -29,4 +29,5 @@ public interface UserData extends ControlledEntity, InternalIdentity, Identity { String getUserData(); String getParams(); + boolean isForCks(); } diff --git a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java index 77373aaf10b..4381447a4bf 100644 --- a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java +++ b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java @@ -112,6 +112,9 @@ public class ApiConstants { public static final String CN = "cn"; public static final String COMMAND = "command"; public static final String CMD_EVENT_TYPE = "cmdeventtype"; + public static final String CNI_CONFIG = "cniconfig"; + public static final String CNI_CONFIG_ID = "cniconfigurationid"; + public static final String CNI_CONFIG_DETAILS = "cniconfigdetails"; public static final String COMPONENT = "component"; public static final String CPU_CORE_PER_SOCKET = "cpucorepersocket"; public static final String CPU_NUMBER = "cpunumber"; diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/userdata/BaseRegisterUserDataCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/userdata/BaseRegisterUserDataCmd.java new file mode 100644 index 00000000000..32c1eaf42f8 --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/userdata/BaseRegisterUserDataCmd.java @@ -0,0 +1,87 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.command.user.userdata; + +import com.cloud.exception.InvalidParameterValueException; +import com.cloud.network.NetworkModel; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.BaseCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.response.DomainResponse; +import org.apache.cloudstack.api.response.ProjectResponse; +import org.apache.commons.lang3.StringUtils; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +public abstract class BaseRegisterUserDataCmd extends BaseCmd { + + @Parameter(name = ApiConstants.NAME, type = CommandType.STRING, required = true, description = "Name of the userdata") + private String name; + + //Owner information + @Parameter(name = ApiConstants.ACCOUNT, type = CommandType.STRING, description = "an optional account for the userdata. Must be used with domainId.") + private String accountName; + + @Parameter(name = ApiConstants.DOMAIN_ID, + type = CommandType.UUID, + entityType = DomainResponse.class, + description = "an optional domainId for the userdata. If the account parameter is used, domainId must also be used.") + private Long domainId; + + @Parameter(name = ApiConstants.PROJECT_ID, type = CommandType.UUID, entityType = ProjectResponse.class, description = "an optional project for the userdata") + private Long projectId; + + @Parameter(name = ApiConstants.PARAMS, type = CommandType.STRING, description = "comma separated list of variables declared in userdata content") + private String params; + + ///////////////////////////////////////////////////// + /////////////////// Accessors /////////////////////// + ///////////////////////////////////////////////////// + + public String getName() { + return name; + } + + public String getAccountName() { + return accountName; + } + + public Long getDomainId() { + return domainId; + } + + public Long getProjectId() { + return projectId; + } + + public String getParams() { + checkForVRMetadataFileNames(params); + return params; + } + + public void checkForVRMetadataFileNames(String params) { + if (StringUtils.isNotEmpty(params)) { + List keyValuePairs = new ArrayList<>(Arrays.asList(params.split(","))); + keyValuePairs.retainAll(NetworkModel.metadataFileNames); + if (!keyValuePairs.isEmpty()) { + throw new InvalidParameterValueException(String.format("Params passed here have a few virtual router metadata file names %s", keyValuePairs)); + } + } + } +} diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/userdata/ListCniConfigurationCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/userdata/ListCniConfigurationCmd.java new file mode 100644 index 00000000000..ecd769eb5a1 --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/userdata/ListCniConfigurationCmd.java @@ -0,0 +1,58 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.command.user.userdata; + +import com.cloud.user.UserData; +import com.cloud.utils.Pair; +import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.response.ListResponse; +import org.apache.cloudstack.api.response.UserDataResponse; +import org.apache.log4j.Logger; + +import java.util.ArrayList; +import java.util.List; + +@APICommand(name = "listCniConfiguration", description = "List userdata for CNI plugins", responseObject = UserDataResponse.class, entityType = {UserData.class}, + requestHasSensitiveInfo = false, responseHasSensitiveInfo = false, since = "4.20", + authorized = {RoleType.Admin, RoleType.ResourceAdmin, RoleType.DomainAdmin, RoleType.User}) +public class ListCniConfigurationCmd extends ListUserDataCmd { + public static final Logger s_logger = Logger.getLogger(ListCniConfigurationCmd.class.getName()); + + ///////////////////////////////////////////////////// + /////////////// API Implementation/////////////////// + ///////////////////////////////////////////////////// + + @Override + public void execute() { + Pair, Integer> resultList = _mgr.listUserDatas(this, true); + List responses = new ArrayList<>(); + for (UserData result : resultList.first()) { + UserDataResponse r = _responseGenerator.createUserDataResponse(result); + r.setObjectName(ApiConstants.CNI_CONFIG); + responses.add(r); + } + + ListResponse response = new ListResponse<>(); + response.setResponses(responses, resultList.second()); + response.setResponseName(getCommandName()); + setResponseObject(response); + } + + +} diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/userdata/ListUserDataCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/userdata/ListUserDataCmd.java index 64ab3ec3d70..16bf1e5c1e4 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/userdata/ListUserDataCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/userdata/ListUserDataCmd.java @@ -61,7 +61,7 @@ public class ListUserDataCmd extends BaseListProjectAndAccountResourcesCmd { @Override public void execute() { - Pair, Integer> resultList = _mgr.listUserDatas(this); + Pair, Integer> resultList = _mgr.listUserDatas(this, false); List responses = new ArrayList<>(); for (UserData result : resultList.first()) { UserDataResponse r = _responseGenerator.createUserDataResponse(result); diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/userdata/RegisterCniConfigurationCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/userdata/RegisterCniConfigurationCmd.java new file mode 100644 index 00000000000..0a5f687f392 --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/userdata/RegisterCniConfigurationCmd.java @@ -0,0 +1,76 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.command.user.userdata; + +import com.cloud.user.UserData; +import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.response.SuccessResponse; +import org.apache.cloudstack.api.response.UserDataResponse; +import org.apache.cloudstack.context.CallContext; +import org.apache.log4j.Logger; + +@APICommand(name = "registerCniConfiguration", + description = "Register a CNI Configuration to be used with CKS cluster", + since = "4.19.0", + responseObject = SuccessResponse.class, + requestHasSensitiveInfo = false, + responseHasSensitiveInfo = false, + authorized = {RoleType.Admin, RoleType.ResourceAdmin, RoleType.DomainAdmin, RoleType.User}) +public class RegisterCniConfigurationCmd extends BaseRegisterUserDataCmd { + public static final Logger s_logger = Logger.getLogger(RegisterCniConfigurationCmd.class.getName()); + + ///////////////////////////////////////////////////// + //////////////// API parameters ///////////////////// + ///////////////////////////////////////////////////// + + @Parameter(name = ApiConstants.CNI_CONFIG, type = CommandType.STRING, description = "CNI Configuration content to be registered as User data", length = 1048576) + private String cniConfig; + + ///////////////////////////////////////////////////// + /////////////////// Accessors /////////////////////// + ///////////////////////////////////////////////////// + + public String getCniConfig() { + return cniConfig; + } + + ///////////////////////////////////////////////////// + /////////////// API Implementation/////////////////// + ///////////////////////////////////////////////////// + + @Override + public void execute() { + UserData result = _mgr.registerUserData(this); + UserDataResponse response = _responseGenerator.createUserDataResponse(result); + response.setResponseName(getCommandName()); + response.setObjectName(ApiConstants.CNI_CONFIG); + setResponseObject(response); + } + + @Override + public long getEntityOwnerId() { + Long accountId = _accountService.finalyzeAccountId(getAccountName(), getDomainId(), getProjectId(), true); + if (accountId == null) { + return CallContext.current().getCallingAccount().getId(); + } + + return accountId; + } +} diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/userdata/RegisterUserDataCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/userdata/RegisterUserDataCmd.java index 41d865d678c..e2160d1418b 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/userdata/RegisterUserDataCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/userdata/RegisterUserDataCmd.java @@ -16,30 +16,20 @@ // under the License. package org.apache.cloudstack.api.command.user.userdata; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; - import org.apache.cloudstack.acl.RoleType; import org.apache.cloudstack.api.APICommand; import org.apache.cloudstack.api.ApiConstants; -import org.apache.cloudstack.api.BaseCmd; import org.apache.cloudstack.api.Parameter; import org.apache.cloudstack.api.ServerApiException; -import org.apache.cloudstack.api.response.DomainResponse; -import org.apache.cloudstack.api.response.ProjectResponse; import org.apache.cloudstack.api.response.SuccessResponse; import org.apache.cloudstack.api.response.UserDataResponse; import org.apache.cloudstack.context.CallContext; -import org.apache.commons.lang3.StringUtils; import com.cloud.exception.ConcurrentOperationException; import com.cloud.exception.InsufficientCapacityException; -import com.cloud.exception.InvalidParameterValueException; import com.cloud.exception.NetworkRuleConflictException; import com.cloud.exception.ResourceAllocationException; import com.cloud.exception.ResourceUnavailableException; -import com.cloud.network.NetworkModel; import com.cloud.user.UserData; @APICommand(name = "registerUserData", @@ -49,89 +39,28 @@ import com.cloud.user.UserData; requestHasSensitiveInfo = false, responseHasSensitiveInfo = false, authorized = {RoleType.Admin, RoleType.ResourceAdmin, RoleType.DomainAdmin, RoleType.User}) -public class RegisterUserDataCmd extends BaseCmd { +public class RegisterUserDataCmd extends BaseRegisterUserDataCmd { ///////////////////////////////////////////////////// //////////////// API parameters ///////////////////// ///////////////////////////////////////////////////// - @Parameter(name = ApiConstants.NAME, type = CommandType.STRING, required = true, description = "Name of the userdata") - private String name; + @Parameter(name = ApiConstants.USER_DATA, type = CommandType.STRING, required = true, description = "Userdata content", length = 1048576) + protected String userData; - //Owner information - @Parameter(name = ApiConstants.ACCOUNT, type = CommandType.STRING, description = "an optional account for the userdata. Must be used with domainId.") - private String accountName; - - @Parameter(name = ApiConstants.DOMAIN_ID, - type = CommandType.UUID, - entityType = DomainResponse.class, - description = "an optional domainId for the userdata. If the account parameter is used, domainId must also be used.") - private Long domainId; - - @Parameter(name = ApiConstants.PROJECT_ID, type = CommandType.UUID, entityType = ProjectResponse.class, description = "an optional project for the userdata") - private Long projectId; - - @Parameter(name = ApiConstants.USER_DATA, - type = CommandType.STRING, - required = true, - description = "Base64 encoded userdata content. " + - "Using HTTP GET (via querystring), you can send up to 4KB of data after base64 encoding. " + - "Using HTTP POST (via POST body), you can send up to 1MB of data after base64 encoding. " + - "You also need to change vm.userdata.max.length value", - length = 1048576) - private String userData; - - @Parameter(name = ApiConstants.PARAMS, type = CommandType.STRING, description = "comma separated list of variables declared in userdata content") - private String params; - - - ///////////////////////////////////////////////////// - /////////////////// Accessors /////////////////////// - ///////////////////////////////////////////////////// - - public String getName() { - return name; - } - - public String getAccountName() { - return accountName; - } - - public Long getDomainId() { - return domainId; - } - - public Long getProjectId() { - return projectId; - } public String getUserData() { return userData; } - public String getParams() { - checkForVRMetadataFileNames(params); - return params; - } - - public void checkForVRMetadataFileNames(String params) { - if (StringUtils.isNotEmpty(params)) { - List keyValuePairs = new ArrayList<>(Arrays.asList(params.split(","))); - keyValuePairs.retainAll(NetworkModel.metadataFileNames); - if (!keyValuePairs.isEmpty()) { - throw new InvalidParameterValueException(String.format("Params passed here have a few virtual router metadata file names %s", keyValuePairs)); - } - } - } - ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// @Override public long getEntityOwnerId() { - Long accountId = _accountService.finalyzeAccountId(accountName, domainId, projectId, true); + Long accountId = _accountService.finalyzeAccountId(getAccountName(), getDomainId(), getProjectId(), true); if (accountId == null) { return CallContext.current().getCallingAccount().getId(); } diff --git a/api/src/test/java/org/apache/cloudstack/api/command/user/userdata/ListUserDataCmdTest.java b/api/src/test/java/org/apache/cloudstack/api/command/user/userdata/ListUserDataCmdTest.java index 3f47a078445..8b7db292462 100644 --- a/api/src/test/java/org/apache/cloudstack/api/command/user/userdata/ListUserDataCmdTest.java +++ b/api/src/test/java/org/apache/cloudstack/api/command/user/userdata/ListUserDataCmdTest.java @@ -68,7 +68,7 @@ public class ListUserDataCmdTest { Pair, Integer> result = new Pair, Integer>(userDataList, 1); UserDataResponse userDataResponse = Mockito.mock(UserDataResponse.class); - Mockito.when(_mgr.listUserDatas(cmd)).thenReturn(result); + Mockito.when(_mgr.listUserDatas(cmd, false)).thenReturn(result); Mockito.when(_responseGenerator.createUserDataResponse(userData)).thenReturn(userDataResponse); cmd.execute(); @@ -82,7 +82,7 @@ public class ListUserDataCmdTest { List userDataList = new ArrayList(); Pair, Integer> result = new Pair, Integer>(userDataList, 0); - Mockito.when(_mgr.listUserDatas(cmd)).thenReturn(result); + Mockito.when(_mgr.listUserDatas(cmd, false)).thenReturn(result); cmd.execute(); diff --git a/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade41910to42000.java b/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade41910to42000.java index 8745ae18034..4c26c6bde50 100644 --- a/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade41910to42000.java +++ b/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade41910to42000.java @@ -21,10 +21,6 @@ import java.sql.Connection; import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; import com.cloud.upgrade.SystemVmTemplateRegistration; import com.cloud.utils.exception.CloudRuntimeException; @@ -61,7 +57,6 @@ public class Upgrade41910to42000 extends DbUpgradeAbstractImpl implements DbUpgr @Override public void performDataMigration(Connection conn) { - updateKubernetesClusterNodeVersions(conn); checkAndUpdateAffinityGroupNameCharSetToUtf8mb4(conn); } @@ -91,95 +86,6 @@ public class Upgrade41910to42000 extends DbUpgradeAbstractImpl implements DbUpgr } } - private Map getKubernetesClusterIdsAndVersion(Connection conn) { - String listKubernetesClusters = "SELECT c.id, v.semantic_version FROM `cloud`.`kubernetes_cluster` c JOIN `cloud`.`kubernetes_supported_version` v ON (c.kubernetes_version_id = v.id) WHERE c.removed is NULL;"; - Map clusterAndVersion = new HashMap<>(); - try { - PreparedStatement pstmt = conn.prepareStatement(listKubernetesClusters); - ResultSet rs = pstmt.executeQuery(); - while (rs.next()) { - clusterAndVersion.put(rs.getLong(1), rs.getString(2)); - } - rs.close(); - pstmt.close(); - } catch (SQLException e) { - String errMsg = String.format("Failed to get all the kubernetes cluster ids due to: %s", e.getMessage()); - logger.error(errMsg); - throw new CloudRuntimeException(errMsg, e); - } - return clusterAndVersion; - } - - private List getKubernetesClusterVmMapIds(Connection conn, Long cksClusterId) { - List kubernetesClusterVmIds = new ArrayList<>(); - String getKubernetesClustersVmMap = "SELECT id FROM `cloud`.`kubernetes_cluster_vm_map` WHERE cluster_id = %s;"; - try { - PreparedStatement pstmt = conn.prepareStatement(String.format(getKubernetesClustersVmMap, cksClusterId)); - ResultSet rs = pstmt.executeQuery(); - while (rs.next()) { - kubernetesClusterVmIds.add(rs.getLong(1)); - } - rs.close(); - pstmt.close(); - } catch (SQLException e) { - String errMsg = String.format("Failed to get the kubernetes cluster vm map IDs for kubernetes cluster with id: %s," + - " due to: %s", cksClusterId, e.getMessage()); - logger.error(errMsg, e); - throw new CloudRuntimeException(errMsg, e); - } - return kubernetesClusterVmIds; - } - - private void updateKubernetesNodeVersion(Connection conn, List kubernetesClusterVmIds, Long cksClusterId, String cksVersion) { - String updateKubernetesNodeVersion = "UPDATE `cloud`.`kubernetes_cluster_vm_map` set kubernetes_node_version = ? WHERE id = ?;"; - for (Long nodeVmId : kubernetesClusterVmIds) { - try { - PreparedStatement pstmt = conn.prepareStatement(updateKubernetesNodeVersion); - pstmt.setString(1, cksVersion); - pstmt.setLong(2, nodeVmId); - pstmt.executeUpdate(); - pstmt.close(); - } catch (Exception e) { - String errMsg = String.format("Failed to update the node version for kubernetes cluster nodes for the" + - " kubernetes cluster with id: %s," + - " due to: %s", cksClusterId, e.getMessage()); - logger.error(errMsg, e); - throw new CloudRuntimeException(errMsg, e); - } - } - } - - private void updateKubernetesNodeVersions(Connection conn, Map clusterAndVersion) { - List kubernetesClusterVmIds; - for (Map.Entry clusterVersionEntry : clusterAndVersion.entrySet()) { - try { - Long cksClusterId = clusterVersionEntry.getKey(); - String cksVersion = clusterVersionEntry.getValue(); - logger.debug(String.format("Adding CKS version %s to existing CKS cluster %s nodes", cksVersion, cksClusterId)); - kubernetesClusterVmIds = getKubernetesClusterVmMapIds(conn, cksClusterId); - updateKubernetesNodeVersion(conn, kubernetesClusterVmIds, cksClusterId, cksVersion); - } catch (Exception e) { - String errMsg = String.format("Failed to update the node version for kubernetes cluster nodes for the" + - " kubernetes cluster with id: %s," + - " due to: %s", clusterVersionEntry.getKey(), e.getMessage()); - logger.error(errMsg, e); - throw new CloudRuntimeException(errMsg, e); - } - } - } - - private void updateKubernetesClusterNodeVersions(Connection conn) { - //get list of all non removed kubernetes clusters - try { - Map clusterAndVersion = getKubernetesClusterIdsAndVersion(conn); - updateKubernetesNodeVersions(conn, clusterAndVersion); - } catch (Exception e) { - String errMsg = "Failed to update kubernetes cluster nodes version"; - logger.error(errMsg); - throw new CloudRuntimeException(errMsg, e); - } - } - private void checkAndUpdateAffinityGroupNameCharSetToUtf8mb4(Connection conn) { logger.debug("Check and update char set for affinity group name to utf8mb4"); try { diff --git a/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade42010to42100.java b/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade42010to42100.java index 06a68ec3d8b..eb557da58d8 100644 --- a/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade42010to42100.java +++ b/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade42010to42100.java @@ -21,6 +21,13 @@ import com.cloud.utils.exception.CloudRuntimeException; import java.io.InputStream; import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; public class Upgrade42010to42100 extends DbUpgradeAbstractImpl implements DbUpgrade, DbUpgradeSystemVmTemplate { private SystemVmTemplateRegistration systemVmTemplateRegistration; @@ -53,6 +60,7 @@ public class Upgrade42010to42100 extends DbUpgradeAbstractImpl implements DbUpgr @Override public void performDataMigration(Connection conn) { + updateKubernetesClusterNodeVersions(conn); } @Override @@ -80,4 +88,93 @@ public class Upgrade42010to42100 extends DbUpgradeAbstractImpl implements DbUpgr throw new CloudRuntimeException("Failed to find / register SystemVM template(s)"); } } + + private void updateKubernetesClusterNodeVersions(Connection conn) { + //get list of all non removed kubernetes clusters + try { + Map clusterAndVersion = getKubernetesClusterIdsAndVersion(conn); + updateKubernetesNodeVersions(conn, clusterAndVersion); + } catch (Exception e) { + String errMsg = "Failed to update kubernetes cluster nodes version"; + logger.error(errMsg); + throw new CloudRuntimeException(errMsg, e); + } + } + + private Map getKubernetesClusterIdsAndVersion(Connection conn) { + String listKubernetesClusters = "SELECT c.id, v.semantic_version FROM `cloud`.`kubernetes_cluster` c JOIN `cloud`.`kubernetes_supported_version` v ON (c.kubernetes_version_id = v.id) WHERE c.removed is NULL;"; + Map clusterAndVersion = new HashMap<>(); + try { + PreparedStatement pstmt = conn.prepareStatement(listKubernetesClusters); + ResultSet rs = pstmt.executeQuery(); + while (rs.next()) { + clusterAndVersion.put(rs.getLong(1), rs.getString(2)); + } + rs.close(); + pstmt.close(); + } catch (SQLException e) { + String errMsg = String.format("Failed to get all the kubernetes cluster ids due to: %s", e.getMessage()); + logger.error(errMsg); + throw new CloudRuntimeException(errMsg, e); + } + return clusterAndVersion; + } + + private void updateKubernetesNodeVersions(Connection conn, Map clusterAndVersion) { + List kubernetesClusterVmIds; + for (Map.Entry clusterVersionEntry : clusterAndVersion.entrySet()) { + try { + Long cksClusterId = clusterVersionEntry.getKey(); + String cksVersion = clusterVersionEntry.getValue(); + logger.debug(String.format("Adding CKS version %s to existing CKS cluster %s nodes", cksVersion, cksClusterId)); + kubernetesClusterVmIds = getKubernetesClusterVmMapIds(conn, cksClusterId); + updateKubernetesNodeVersion(conn, kubernetesClusterVmIds, cksClusterId, cksVersion); + } catch (Exception e) { + String errMsg = String.format("Failed to update the node version for kubernetes cluster nodes for the" + + " kubernetes cluster with id: %s," + + " due to: %s", clusterVersionEntry.getKey(), e.getMessage()); + logger.error(errMsg, e); + throw new CloudRuntimeException(errMsg, e); + } + } + } + + private List getKubernetesClusterVmMapIds(Connection conn, Long cksClusterId) { + List kubernetesClusterVmIds = new ArrayList<>(); + String getKubernetesClustersVmMap = "SELECT id FROM `cloud`.`kubernetes_cluster_vm_map` WHERE cluster_id = %s;"; + try { + PreparedStatement pstmt = conn.prepareStatement(String.format(getKubernetesClustersVmMap, cksClusterId)); + ResultSet rs = pstmt.executeQuery(); + while (rs.next()) { + kubernetesClusterVmIds.add(rs.getLong(1)); + } + rs.close(); + pstmt.close(); + } catch (SQLException e) { + String errMsg = String.format("Failed to get the kubernetes cluster vm map IDs for kubernetes cluster with id: %s," + + " due to: %s", cksClusterId, e.getMessage()); + logger.error(errMsg, e); + throw new CloudRuntimeException(errMsg, e); + } + return kubernetesClusterVmIds; + } + + private void updateKubernetesNodeVersion(Connection conn, List kubernetesClusterVmIds, Long cksClusterId, String cksVersion) { + String updateKubernetesNodeVersion = "UPDATE `cloud`.`kubernetes_cluster_vm_map` set kubernetes_node_version = ? WHERE id = ?;"; + for (Long nodeVmId : kubernetesClusterVmIds) { + try { + PreparedStatement pstmt = conn.prepareStatement(updateKubernetesNodeVersion); + pstmt.setString(1, cksVersion); + pstmt.setLong(2, nodeVmId); + pstmt.executeUpdate(); + pstmt.close(); + } catch (Exception e) { + String errMsg = String.format("Failed to update the node version for kubernetes cluster nodes for the" + + " kubernetes cluster with id: %s," + + " due to: %s", cksClusterId, e.getMessage()); + logger.error(errMsg, e); + throw new CloudRuntimeException(errMsg, e); + } + } + } } diff --git a/engine/schema/src/main/java/com/cloud/user/UserDataVO.java b/engine/schema/src/main/java/com/cloud/user/UserDataVO.java index a8e48ad22b1..e8864976069 100644 --- a/engine/schema/src/main/java/com/cloud/user/UserDataVO.java +++ b/engine/schema/src/main/java/com/cloud/user/UserDataVO.java @@ -65,6 +65,9 @@ public class UserDataVO implements UserData { @Column(name = GenericDao.REMOVED_COLUMN) private Date removed; + @Column(name = "for_cks") + private boolean forCks; + @Override public long getDomainId() { return domainId; @@ -105,6 +108,11 @@ public class UserDataVO implements UserData { return params; } + @Override + public boolean isForCks() { + return forCks; + } + public void setAccountId(long accountId) { this.accountId = accountId; } @@ -132,4 +140,6 @@ public class UserDataVO implements UserData { public Date getRemoved() { return removed; } + + public void setForCks(boolean forCks) { this.forCks = forCks; } } diff --git a/engine/schema/src/main/resources/META-INF/db/schema-41910to42000.sql b/engine/schema/src/main/resources/META-INF/db/schema-41910to42000.sql index 2ab86ea7d89..c36b71c2f25 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-41910to42000.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-41910to42000.sql @@ -425,26 +425,3 @@ INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid, hypervisor_type, hypervi CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.vm_instance', 'delete_protection', 'boolean DEFAULT FALSE COMMENT "delete protection for vm" '); CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.volumes', 'delete_protection', 'boolean DEFAULT FALSE COMMENT "delete protection for volumes" '); - --- Add for_cks column to the vm_template table -CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.vm_template','for_cks', 'int(1) unsigned DEFAULT "0" COMMENT "if true, the template can be used for CKS cluster deployment"'); - --- Add support for different node types service offerings on CKS clusters -CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.kubernetes_cluster','control_service_offering_id', 'bigint unsigned COMMENT "service offering ID for Control Node(s)"'); -CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.kubernetes_cluster','worker_service_offering_id', 'bigint unsigned COMMENT "service offering ID for Worker Node(s)"'); -CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.kubernetes_cluster','etcd_service_offering_id', 'bigint unsigned COMMENT "service offering ID for etcd Nodes"'); -CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.kubernetes_cluster','etcd_node_count', 'bigint unsigned COMMENT "number of etcd nodes to be deployed for the Kubernetes cluster"'); -CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.kubernetes_cluster','control_template_id', 'bigint unsigned COMMENT "template id to be used for Control Node(s)"'); -CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.kubernetes_cluster','worker_template_id', 'bigint unsigned COMMENT "template id to be used for Worker Node(s)"'); -CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.kubernetes_cluster','etcd_template_id', 'bigint unsigned COMMENT "template id to be used for etcd Nodes"'); -CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.kubernetes_cluster_vm_map','etcd_node', 'tinyint(1) unsigned NOT NULL DEFAULT 0 COMMENT "indicates if the VM is an etcd node"'); -CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.kubernetes_cluster_vm_map','external_node', 'tinyint(1) unsigned NOT NULL DEFAULT 0 COMMENT "indicates if the node was imported into the Kubernetes cluster"'); -CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.kubernetes_cluster_vm_map','manual_upgrade', 'tinyint(1) unsigned NOT NULL DEFAULT 0 COMMENT "indicates if the node is marked for manual upgrade and excluded from the Kubernetes cluster upgrade operation"'); -CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.kubernetes_cluster_vm_map','kubernetes_node_version', 'varchar(40) COMMENT "version of k8s the cluster node is on"'); - -ALTER TABLE `cloud`.`kubernetes_cluster` ADD CONSTRAINT `fk_cluster__control_service_offering_id` FOREIGN KEY `fk_cluster__control_service_offering_id`(`control_service_offering_id`) REFERENCES `service_offering`(`id`) ON DELETE CASCADE; -ALTER TABLE `cloud`.`kubernetes_cluster` ADD CONSTRAINT `fk_cluster__worker_service_offering_id` FOREIGN KEY `fk_cluster__worker_service_offering_id`(`worker_service_offering_id`) REFERENCES `service_offering`(`id`) ON DELETE CASCADE; -ALTER TABLE `cloud`.`kubernetes_cluster` ADD CONSTRAINT `fk_cluster__etcd_service_offering_id` FOREIGN KEY `fk_cluster__etcd_service_offering_id`(`etcd_service_offering_id`) REFERENCES `service_offering`(`id`) ON DELETE CASCADE; -ALTER TABLE `cloud`.`kubernetes_cluster` ADD CONSTRAINT `fk_cluster__control_template_id` FOREIGN KEY `fk_cluster__control_template_id`(`control_template_id`) REFERENCES `vm_template`(`id`) ON DELETE CASCADE; -ALTER TABLE `cloud`.`kubernetes_cluster` ADD CONSTRAINT `fk_cluster__worker_template_id` FOREIGN KEY `fk_cluster__worker_template_id`(`worker_template_id`) REFERENCES `vm_template`(`id`) ON DELETE CASCADE; -ALTER TABLE `cloud`.`kubernetes_cluster` ADD CONSTRAINT `fk_cluster__etcd_template_id` FOREIGN KEY `fk_cluster__etcd_template_id`(`etcd_template_id`) REFERENCES `vm_template`(`id`) ON DELETE CASCADE; diff --git a/engine/schema/src/main/resources/META-INF/db/schema-42010to42100.sql b/engine/schema/src/main/resources/META-INF/db/schema-42010to42100.sql index 91223bab798..4f4b2f2a994 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-42010to42100.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-42010to42100.sql @@ -24,3 +24,37 @@ CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.console_session', 'console_endpoint_ -- Add client_address column to cloud.console_session table CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.console_session', 'client_address', 'VARCHAR(45)'); + +----------------------------------------------------------- +-- CKS Enhancements: +----------------------------------------------------------- +-- Add for_cks column to the vm_template table +CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.vm_template','for_cks', 'int(1) unsigned DEFAULT "0" COMMENT "if true, the template can be used for CKS cluster deployment"'); + +-- Add support for different node types service offerings on CKS clusters +CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.kubernetes_cluster','control_service_offering_id', 'bigint unsigned COMMENT "service offering ID for Control Node(s)"'); +CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.kubernetes_cluster','worker_service_offering_id', 'bigint unsigned COMMENT "service offering ID for Worker Node(s)"'); +CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.kubernetes_cluster','etcd_service_offering_id', 'bigint unsigned COMMENT "service offering ID for etcd Nodes"'); +CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.kubernetes_cluster','etcd_node_count', 'bigint unsigned COMMENT "number of etcd nodes to be deployed for the Kubernetes cluster"'); +CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.kubernetes_cluster','control_template_id', 'bigint unsigned COMMENT "template id to be used for Control Node(s)"'); +CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.kubernetes_cluster','worker_template_id', 'bigint unsigned COMMENT "template id to be used for Worker Node(s)"'); +CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.kubernetes_cluster','etcd_template_id', 'bigint unsigned COMMENT "template id to be used for etcd Nodes"'); +CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.kubernetes_cluster','cni_config_id', 'bigint unsigned COMMENT "userdata id representing the associated cni configuration"'); +CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.kubernetes_cluster','cni_config_details', 'varchar(4096) DEFAULT NULL COMMENT "userdata details representing the values required for the cni configuration associated"'); +CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.kubernetes_cluster_vm_map','etcd_node', 'tinyint(1) unsigned NOT NULL DEFAULT 0 COMMENT "indicates if the VM is an etcd node"'); +CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.kubernetes_cluster_vm_map','external_node', 'tinyint(1) unsigned NOT NULL DEFAULT 0 COMMENT "indicates if the node was imported into the Kubernetes cluster"'); +CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.kubernetes_cluster_vm_map','manual_upgrade', 'tinyint(1) unsigned NOT NULL DEFAULT 0 COMMENT "indicates if the node is marked for manual upgrade and excluded from the Kubernetes cluster upgrade operation"'); +CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.kubernetes_cluster_vm_map','kubernetes_node_version', 'varchar(40) COMMENT "version of k8s the cluster node is on"'); + +ALTER TABLE `cloud`.`kubernetes_cluster` ADD CONSTRAINT `fk_cluster__control_service_offering_id` FOREIGN KEY `fk_cluster__control_service_offering_id`(`control_service_offering_id`) REFERENCES `service_offering`(`id`) ON DELETE CASCADE; +ALTER TABLE `cloud`.`kubernetes_cluster` ADD CONSTRAINT `fk_cluster__worker_service_offering_id` FOREIGN KEY `fk_cluster__worker_service_offering_id`(`worker_service_offering_id`) REFERENCES `service_offering`(`id`) ON DELETE CASCADE; +ALTER TABLE `cloud`.`kubernetes_cluster` ADD CONSTRAINT `fk_cluster__etcd_service_offering_id` FOREIGN KEY `fk_cluster__etcd_service_offering_id`(`etcd_service_offering_id`) REFERENCES `service_offering`(`id`) ON DELETE CASCADE; +ALTER TABLE `cloud`.`kubernetes_cluster` ADD CONSTRAINT `fk_cluster__control_template_id` FOREIGN KEY `fk_cluster__control_template_id`(`control_template_id`) REFERENCES `vm_template`(`id`) ON DELETE CASCADE; +ALTER TABLE `cloud`.`kubernetes_cluster` ADD CONSTRAINT `fk_cluster__worker_template_id` FOREIGN KEY `fk_cluster__worker_template_id`(`worker_template_id`) REFERENCES `vm_template`(`id`) ON DELETE CASCADE; +ALTER TABLE `cloud`.`kubernetes_cluster` ADD CONSTRAINT `fk_cluster__etcd_template_id` FOREIGN KEY `fk_cluster__etcd_template_id`(`etcd_template_id`) REFERENCES `vm_template`(`id`) ON DELETE CASCADE; + +-- Add for_cks column to the user_data table to represent CNI Configuration stored as userdata +CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.user_data','for_cks', 'int(1) unsigned DEFAULT "0" COMMENT "if true, the userdata represent CNI configuration meant for CKS use only"'); +----------------------------------------------------------- +-- END - CKS Enhancements +----------------------------------------------------------- diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java index 17533ebd3af..ab8fe3e4308 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java @@ -47,13 +47,16 @@ import java.util.stream.Collectors; import javax.inject.Inject; import javax.naming.ConfigurationException; +import com.cloud.dc.ASNumberVO; import com.cloud.dc.DedicatedResourceVO; +import com.cloud.dc.dao.ASNumberDao; import com.cloud.dc.dao.DedicatedResourceDao; import com.cloud.exception.ManagementServerException; import com.cloud.exception.ResourceUnavailableException; import com.cloud.host.Host; import com.cloud.kubernetes.cluster.KubernetesServiceHelper.KubernetesClusterNodeType; import com.cloud.kubernetes.cluster.actionworkers.KubernetesClusterRemoveWorker; +import com.cloud.network.NetworkServiceImpl; import com.cloud.network.dao.NsxProviderDao; import com.cloud.network.element.NsxProviderVO; import com.cloud.kubernetes.cluster.actionworkers.KubernetesClusterAddWorker; @@ -316,6 +319,8 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne private PortForwardingRulesDao pfRuleDao; @Inject RoutedIpv4Manager routedIpv4Manager; + @Inject + private ASNumberDao asNumberDao; private void logMessage(final Level logLevel, final String message, final Exception e) { if (logLevel == Level.WARN) { @@ -1037,7 +1042,7 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne } private Network getKubernetesClusterNetworkIfMissing(final String clusterName, final DataCenter zone, final Account owner, final int controlNodesCount, - final int nodesCount, final String externalLoadBalancerIpAddress, final Long networkId) throws CloudRuntimeException { + final int nodesCount, final String externalLoadBalancerIpAddress, final Long networkId, final Long asNumber) throws CloudRuntimeException { Network network = null; if (networkId != null) { network = networkDao.findById(networkId); @@ -1070,6 +1075,10 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne network = networkService.createGuestNetwork(networkOffering.getId(), clusterName + "-network", owner.getAccountName() + "-network", owner, physicalNetwork, zone.getId(), ControlledEntity.ACLType.Account); + ASNumberVO asNumberVO = NetworkServiceImpl.checkAndSelectASNumber(asNumber, zone, networkOffering, asNumberDao); + if (Objects.nonNull(asNumber)) { + NetworkServiceImpl.allocateASNumber(asNumberVO, network, asNumberDao); + } } catch (ConcurrentOperationException | InsufficientCapacityException | ResourceAllocationException e) { logAndThrow(Level.ERROR, String.format("Unable to create network for the Kubernetes cluster: %s", clusterName)); } finally { @@ -1433,6 +1442,7 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne final Account owner = accountService.getActiveAccountById(cmd.getEntityOwnerId()); final KubernetesSupportedVersion clusterKubernetesVersion = kubernetesSupportedVersionDao.findById(cmd.getKubernetesVersionId()); final Hypervisor.HypervisorType hypervisor = cmd.getHypervisorType(); + final Long asNumber = cmd.getAsNumber(); Map serviceOfferingNodeTypeMap = cmd.getServiceOfferingNodeTypeMap(); Long defaultServiceOfferingId = cmd.getServiceOfferingId(); @@ -1457,7 +1467,7 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne final VMTemplateVO controlNodeTemplate = getKubernetesServiceTemplate(zone, hypervisorType, templateNodeTypeMap, CONTROL); final VMTemplateVO workerNodeTemplate = getKubernetesServiceTemplate(zone, hypervisorType, templateNodeTypeMap, WORKER); final VMTemplateVO etcdNodeTemplate = getKubernetesServiceTemplate(zone, hypervisorType, templateNodeTypeMap, ETCD); - final Network defaultNetwork = getKubernetesClusterNetworkIfMissing(cmd.getName(), zone, owner, (int)controlNodeCount, (int)clusterSize, cmd.getExternalLoadBalancerIpAddress(), cmd.getNetworkId()); + final Network defaultNetwork = getKubernetesClusterNetworkIfMissing(cmd.getName(), zone, owner, (int)controlNodeCount, (int)clusterSize, cmd.getExternalLoadBalancerIpAddress(), cmd.getNetworkId(), asNumber); final SecurityGroup finalSecurityGroup = securityGroup; final KubernetesClusterVO cluster = Transaction.execute(new TransactionCallback() { @Override @@ -1472,6 +1482,12 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne defaultNetwork.getId(), owner.getDomainId(), owner.getAccountId(), controlNodeCount, clusterSize, KubernetesCluster.State.Created, cmd.getSSHKeyPairName(), cores, memory, cmd.getNodeRootDiskSize(), "", KubernetesCluster.ClusterType.CloudManaged); + newCluster.setCniConfigId(cmd.getCniConfigId()); + String cniConfigDetails = null; + if (MapUtils.isNotEmpty(cmd.getCniConfigDetails())) { + cniConfigDetails = cmd.getCniConfigDetails().toString(); + } + newCluster.setCniConfigDetails(cniConfigDetails); if (serviceOfferingNodeTypeMap.containsKey(WORKER.name())) { newCluster.setWorkerServiceOfferingId(serviceOfferingNodeTypeMap.get(WORKER.name())); } @@ -1635,7 +1651,9 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne * @return * @throws CloudRuntimeException */ - protected boolean startKubernetesCluster(long kubernetesClusterId, Long domainId, String accountName, Long asNumber, boolean onCreate) throws CloudRuntimeException, ManagementServerException, ResourceUnavailableException, InsufficientCapacityException { + @Override + public boolean startKubernetesCluster(long kubernetesClusterId, Long domainId, String accountName, Long asNumber, boolean onCreate) + throws CloudRuntimeException, ManagementServerException, ResourceUnavailableException, InsufficientCapacityException { if (!KubernetesServiceEnabled.value()) { logAndThrow(Level.ERROR, "Kubernetes Service plugin is disabled"); } diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterService.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterService.java index 2c05f103ad7..ad7aef3250b 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterService.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterService.java @@ -151,6 +151,8 @@ public interface KubernetesClusterService extends PluggableService, Configurable void startKubernetesCluster(StartKubernetesClusterCmd cmd) throws CloudRuntimeException, ManagementServerException, ResourceUnavailableException, InsufficientCapacityException; + boolean startKubernetesCluster(long kubernetesClusterId, Long domainId, String accountName, Long asNumber, boolean onCreate) throws CloudRuntimeException, ManagementServerException, ResourceUnavailableException, InsufficientCapacityException; + boolean stopKubernetesCluster(StopKubernetesClusterCmd cmd) throws CloudRuntimeException; boolean deleteKubernetesCluster(DeleteKubernetesClusterCmd cmd) throws CloudRuntimeException; diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterVO.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterVO.java index 0b4f7f60ac2..ef23ec7477c 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterVO.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterVO.java @@ -138,6 +138,12 @@ public class KubernetesClusterVO implements KubernetesCluster { @Column(name = "etcd_template_id") private Long etcdTemplateId; + @Column(name = "cni_config_id", nullable = true) + private Long cniConfigId = null; + + @Column(name = "cni_config_details", updatable = true, length = 4096) + private String cniConfigDetails; + @Override public long getId() { return id; @@ -483,4 +489,21 @@ public class KubernetesClusterVO implements KubernetesCluster { public void setControlTemplateId(Long controlTemplateId) { this.controlTemplateId = controlTemplateId; } + + public Long getCniConfigId() { + return cniConfigId; + } + + public void setCniConfigId(Long cniConfigId) { + this.cniConfigId = cniConfigId; + } + + public String getCniConfigDetails() { + return cniConfigDetails; + } + + public void setCniConfigDetails(String cniConfigDetails) { + this.cniConfigDetails = cniConfigDetails; + } + } diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java index 98825c1329c..83cd52684ca 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java @@ -51,6 +51,7 @@ import com.cloud.network.rules.PortForwardingRuleVO; import com.cloud.network.rules.RulesService; import com.cloud.network.rules.dao.PortForwardingRulesDao; import com.cloud.user.SSHKeyPairVO; +import com.cloud.user.dao.UserDataDao; import com.cloud.utils.component.ComponentContext; import com.cloud.utils.db.TransactionCallbackWithException; import com.cloud.utils.net.Ip; @@ -69,6 +70,7 @@ import org.apache.cloudstack.config.ApiServiceConfiguration; import org.apache.cloudstack.context.CallContext; import org.apache.cloudstack.engine.orchestration.service.NetworkOrchestrationService; import org.apache.cloudstack.framework.config.dao.ConfigurationDao; +import org.apache.cloudstack.userdata.UserDataManager; import org.apache.commons.codec.binary.Base64; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.io.IOUtils; @@ -179,6 +181,10 @@ public class KubernetesClusterActionWorker { @Inject protected UserVmService userVmService; @Inject + protected UserDataManager userDataManager; + @Inject + protected UserDataDao userDataDao; + @Inject protected UserVmManager userVmManager; @Inject protected VlanDao vlanDao; @@ -725,20 +731,30 @@ public class KubernetesClusterActionWorker { String command = String.format("sudo /opt/bin/kubectl annotate node %s cluster-autoscaler.kubernetes.io/scale-down-disabled=true ; ", name); commands.append(command); } - try { - File pkFile = getManagementServerSshPublicKeyFile(); - Pair publicIpSshPort = getKubernetesClusterServerIpSshPort(null); - publicIpAddress = publicIpSshPort.first(); - sshPort = publicIpSshPort.second(); + int retryCounter = 0; + while (retryCounter < 3) { + retryCounter++; + try { + File pkFile = getManagementServerSshPublicKeyFile(); + Pair publicIpSshPort = getKubernetesClusterServerIpSshPort(null); + publicIpAddress = publicIpSshPort.first(); + sshPort = publicIpSshPort.second(); - Pair result = SshHelper.sshExecute(publicIpAddress, sshPort, getControlNodeLoginUser(), - pkFile, null, commands.toString(), 10000, 10000, 60000); - return result.first(); - } catch (Exception e) { - String msg = String.format("Failed to taint control nodes on : %s : %s", kubernetesCluster.getName(), e.getMessage()); - logMessage(Level.ERROR, msg, e); - return false; + Pair result = SshHelper.sshExecute(publicIpAddress, sshPort, getControlNodeLoginUser(), + pkFile, null, commands.toString(), 10000, 10000, 60000); + return result.first(); + } catch (Exception e) { + String msg = String.format("Failed to taint control nodes on : %s : %s", kubernetesCluster.getName(), e.getMessage()); + logMessage(Level.ERROR, msg, e); + } + try { + Thread.sleep(5 * 1000L); + } catch (InterruptedException ie) { + LOGGER.error(String.format("Error while attempting to taint nodes on Kubernetes cluster: %s", kubernetesCluster.getName()), ie); + } + retryCounter++; } + return false; } protected boolean deployProvider() { diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterDestroyWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterDestroyWorker.java index 80741343d30..9500f24020b 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterDestroyWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterDestroyWorker.java @@ -23,6 +23,10 @@ import java.util.stream.Collectors; import javax.inject.Inject; +import com.cloud.dc.ASNumberVO; +import com.cloud.dc.BGPService; +import com.cloud.dc.DataCenter; +import com.cloud.dc.dao.ASNumberDao; import org.apache.cloudstack.annotation.AnnotationService; import org.apache.cloudstack.annotation.dao.AnnotationDao; import org.apache.cloudstack.api.ApiCommandResourceType; @@ -63,6 +67,10 @@ public class KubernetesClusterDestroyWorker extends KubernetesClusterResourceMod protected AccountManager accountManager; @Inject private AnnotationDao annotationDao; + @Inject + private ASNumberDao asNumberDao; + @Inject + private BGPService bgpService; private List clusterVMs; @@ -130,6 +138,7 @@ public class KubernetesClusterDestroyWorker extends KubernetesClusterResourceMod Account owner = accountManager.getAccount(network.getAccountId()); User callerUser = accountManager.getActiveUser(CallContext.current().getCallingUserId()); ReservationContext context = new ReservationContextImpl(null, null, callerUser, owner); + releaseASNumber(kubernetesCluster.getZoneId(), kubernetesCluster.getNetworkId()); boolean networkDestroyed = networkMgr.destroyNetwork(kubernetesCluster.getNetworkId(), context, true); if (!networkDestroyed) { String msg = String.format("Failed to destroy network : %s as part of Kubernetes cluster : %s cleanup", network.getName(), kubernetesCluster.getName()); @@ -143,6 +152,15 @@ public class KubernetesClusterDestroyWorker extends KubernetesClusterResourceMod } } + private void releaseASNumber(Long zoneId, long networkId) { + DataCenter zone = dataCenterDao.findById(zoneId); + ASNumberVO asNumber = asNumberDao.findByZoneAndNetworkId(zone.getId(), networkId); + if (asNumber != null) { + LOGGER.debug(String.format("Releasing AS number %s from network %s", asNumber.getAsNumber(), networkId)); + bgpService.releaseASNumber(zone.getId(), asNumber.getAsNumber()); + } + } + protected void deleteKubernetesClusterIsolatedNetworkRules(Network network, List removedVmIds) throws ManagementServerException { IpAddress publicIp = getNetworkSourceNatIp(network); if (publicIp == null) { diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java index 44b1bda3aa4..408e2599a1e 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java @@ -36,6 +36,7 @@ import com.cloud.exception.NetworkRuleConflictException; import com.cloud.exception.PermissionDeniedException; import com.cloud.network.vpc.NetworkACL; import com.cloud.storage.VMTemplateVO; +import com.cloud.user.UserDataVO; import org.apache.cloudstack.api.BaseCmd; import org.apache.cloudstack.api.InternalIdentity; import org.apache.cloudstack.framework.ca.Certificate; @@ -140,7 +141,7 @@ public class KubernetesClusterStartWorker extends KubernetesClusterResourceModif private Pair getKubernetesControlNodeConfig(final String controlNodeIp, final String serverIp, final List etcdIps, final String hostName, final boolean haSupported, - final boolean ejectIso) throws IOException { + final boolean ejectIso, final boolean externalCni) throws IOException { String k8sControlNodeConfig = readResourceFile("/conf/k8s-control-node.yml"); final String apiServerCert = "{{ k8s_control_node.apiserver.crt }}"; final String apiServerKey = "{{ k8s_control_node.apiserver.key }}"; @@ -157,6 +158,7 @@ public class KubernetesClusterStartWorker extends KubernetesClusterResourceModif final String k8sApiPort = "{{ k8s.api_server_port }}"; final String certSans = "{{ k8s_control.server_ips }}"; final String k8sCertificate = "{{ k8s_control.certificate_key }}"; + final String externalCniPlugin = "{{ k8s.external.cni.plugin }}"; final List addresses = new ArrayList<>(); addresses.add(controlNodeIp); @@ -207,6 +209,7 @@ public class KubernetesClusterStartWorker extends KubernetesClusterResourceModif k8sControlNodeConfig = k8sControlNodeConfig.replace(k8sApiPort, String.valueOf(CLUSTER_API_PORT)); k8sControlNodeConfig = k8sControlNodeConfig.replace(certSans, String.format("- %s", serverIp)); k8sControlNodeConfig = k8sControlNodeConfig.replace(k8sCertificate, KubernetesClusterUtil.generateClusterHACertificateKey(kubernetesCluster)); + k8sControlNodeConfig = k8sControlNodeConfig.replace(externalCniPlugin, String.valueOf(externalCni)); k8sControlNodeConfig = updateKubeConfigWithRegistryDetails(k8sControlNodeConfig); @@ -238,34 +241,47 @@ public class KubernetesClusterStartWorker extends KubernetesClusterResourceModif String suffix = Long.toHexString(System.currentTimeMillis()); String hostName = String.format("%s-control-%s", kubernetesClusterNodeNamePrefix, suffix); boolean haSupported = isKubernetesVersionSupportsHA(); + Long userDataId = kubernetesCluster.getCniConfigId(); Pair k8sControlNodeConfigAndControlIp = new Pair<>(null, null); try { - k8sControlNodeConfigAndControlIp = getKubernetesControlNodeConfig(controlNodeIp, serverIp, etcdIps, hostName, haSupported, Hypervisor.HypervisorType.VMware.equals(clusterTemplate.getHypervisorType())); + k8sControlNodeConfigAndControlIp = getKubernetesControlNodeConfig(controlNodeIp, serverIp, etcdIps, hostName, haSupported, Hypervisor.HypervisorType.VMware.equals(clusterTemplate.getHypervisorType()), Objects.nonNull(userDataId)); } catch (IOException e) { logAndThrow(Level.ERROR, "Failed to read Kubernetes control node configuration file", e); } String k8sControlNodeConfig = k8sControlNodeConfigAndControlIp.first(); String base64UserData = Base64.encodeBase64String(k8sControlNodeConfig.getBytes(com.cloud.utils.StringUtils.getPreferredCharset())); + if (Objects.nonNull(userDataId)) { + logger.info("concatenating userdata"); + UserDataVO cniConfigVo = userDataDao.findById(userDataId); + String cniConfig = new String(Base64.decodeBase64(cniConfigVo.getUserData())); + if (Objects.nonNull(asNumber)) { + cniConfig = substituteASNumber(cniConfig, asNumber); + cniConfig = Base64.encodeBase64String(cniConfig.getBytes(com.cloud.utils.StringUtils.getPreferredCharset())); + } + base64UserData = userDataManager.concatenateUserData(base64UserData, cniConfig, null); + } + List keypairs = new ArrayList(); if (StringUtils.isNotBlank(kubernetesCluster.getKeyPair())) { keypairs.add(kubernetesCluster.getKeyPair()); } Long affinityGroupId = getExplicitAffinityGroup(domainId, accountId); + String userDataDetails = kubernetesCluster.getCniConfigDetails(); if (kubernetesCluster.getSecurityGroupId() != null && networkModel.checkSecurityGroupSupportForNetwork(owner, zone, networkIds, List.of(kubernetesCluster.getSecurityGroupId()))) { List securityGroupIds = new ArrayList<>(); securityGroupIds.add(kubernetesCluster.getSecurityGroupId()); controlVm = userVmService.createAdvancedSecurityGroupVirtualMachine(zone, serviceOffering, controlNodeTemplate, networkIds, securityGroupIds, owner, - hostName, hostName, null, null, null, Hypervisor.HypervisorType.None, BaseCmd.HTTPMethod.POST,base64UserData, null, null, keypairs, + hostName, hostName, null, null, null, Hypervisor.HypervisorType.None, BaseCmd.HTTPMethod.POST,base64UserData, userDataId, userDataDetails, keypairs, requestedIps, addrs, null, null, Objects.nonNull(affinityGroupId) ? Collections.singletonList(affinityGroupId) : null, customParameterMap, null, null, null, null, true, null, UserVmManager.CKS_NODE); } else { controlVm = userVmService.createAdvancedVirtualMachine(zone, serviceOffering, controlNodeTemplate, networkIds, owner, hostName, hostName, null, null, null, - Hypervisor.HypervisorType.None, BaseCmd.HTTPMethod.POST, base64UserData, null, null, keypairs, + Hypervisor.HypervisorType.None, BaseCmd.HTTPMethod.POST, base64UserData, userDataId, userDataDetails, keypairs, requestedIps, addrs, null, null, Objects.nonNull(affinityGroupId) ? Collections.singletonList(affinityGroupId) : null, customParameterMap, null, null, null, null, true, UserVmManager.CKS_NODE, null); } @@ -275,6 +291,13 @@ public class KubernetesClusterStartWorker extends KubernetesClusterResourceModif return controlVm; } + private String substituteASNumber(String cniConfig, Long asNumber) { + final String asNumberKey = "{{ AS_NUMBER }}"; + cniConfig = cniConfig.replace(asNumberKey, String.valueOf(asNumber)); + return cniConfig; + + } + private String getKubernetesAdditionalControlNodeConfig(final String joinIp, final boolean ejectIso) throws IOException { String k8sControlNodeConfig = readResourceFile("/conf/k8s-control-node-add.yml"); final String joinIpKey = "{{ k8s_control_node.join_ip }}"; diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterUpgradeWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterUpgradeWorker.java index d8ef426d3f0..c2468c2d220 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterUpgradeWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterUpgradeWorker.java @@ -20,6 +20,7 @@ package com.cloud.kubernetes.cluster.actionworkers; import java.io.File; import java.util.ArrayList; import java.util.List; +import java.util.Objects; import java.util.stream.Collectors; import com.cloud.kubernetes.cluster.KubernetesClusterVmMapVO; @@ -67,12 +68,12 @@ public class KubernetesClusterUpgradeWorker extends KubernetesClusterActionWorke String nodeAddress = (index > 0 && sshPort == 22) ? vm.getPrivateIpAddress() : publicIpAddress; SshHelper.scpTo(nodeAddress, nodeSshPort, getControlNodeLoginUser(), sshKeyFile, null, "~/", upgradeScriptFile.getAbsolutePath(), "0755"); - String cmdStr = String.format("sudo ./%s %s %s %s %s", + String cmdStr = String.format("sudo ./%s %s %s %s %s %s", upgradeScriptFile.getName(), upgradeVersion.getSemanticVersion(), index == 0 ? "true" : "false", KubernetesVersionManagerImpl.compareSemanticVersions(upgradeVersion.getSemanticVersion(), "1.15.0") < 0 ? "true" : "false", - Hypervisor.HypervisorType.VMware.equals(vm.getHypervisorType())); + Hypervisor.HypervisorType.VMware.equals(vm.getHypervisorType()), Objects.isNull(kubernetesCluster.getCniConfigId())); return SshHelper.sshExecute(nodeAddress, nodeSshPort, getControlNodeLoginUser(), sshKeyFile, null, cmdStr, 10000, 10000, 10 * 60 * 1000); diff --git a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/CreateKubernetesClusterCmd.java b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/CreateKubernetesClusterCmd.java index 93b2afed7d4..abd5bd013aa 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/CreateKubernetesClusterCmd.java +++ b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/CreateKubernetesClusterCmd.java @@ -52,6 +52,7 @@ import org.apache.cloudstack.api.response.KubernetesSupportedVersionResponse; import org.apache.cloudstack.api.response.NetworkResponse; import org.apache.cloudstack.api.response.ProjectResponse; import org.apache.cloudstack.api.response.ServiceOfferingResponse; +import org.apache.cloudstack.api.response.UserDataResponse; import org.apache.cloudstack.api.response.ZoneResponse; import org.apache.cloudstack.context.CallContext; import org.apache.cloudstack.framework.config.dao.ConfigurationDao; @@ -193,6 +194,15 @@ public class CreateKubernetesClusterCmd extends BaseAsyncCreateCmd { @Parameter(name = ApiConstants.HYPERVISOR, type = CommandType.STRING, description = "the hypervisor on which the CKS cluster is to be deployed. This is required if the zone in which the CKS cluster is being deployed has clusters with different hypervisor types.") private String hypervisor; + @Parameter(name = ApiConstants.CNI_CONFIG_ID, type = CommandType.UUID, entityType = UserDataResponse.class, description = "the ID of the Userdata", since = "4.19.0") + private Long cniConfigId; + + @Parameter(name = ApiConstants.CNI_CONFIG_DETAILS, type = CommandType.MAP, + description = "used to specify the parameters values for the variables in userdata. " + + "Example: cniconfigdetails[0].key=accesskey&cniconfigdetails[0].value=s389ddssaa&" + + "cniconfigdetails[1].key=secretkey&cniconfigdetails[1].value=8dshfsss", since = "4.19.0") + private Map cniConfigDetails; + @Parameter(name=ApiConstants.AS_NUMBER, type=CommandType.LONG, description="the AS Number of the network") private Long asNumber; @@ -349,6 +359,14 @@ public class CreateKubernetesClusterCmd extends BaseAsyncCreateCmd { return asNumber; } + public Map getCniConfigDetails() { + return convertDetailsToMap(cniConfigDetails); + } + + public Long getCniConfigId() { + return cniConfigId; + } + ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// diff --git a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node.yml b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node.yml index 41668aab6da..42924a10fc0 100644 --- a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node.yml +++ b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node.yml @@ -1,3 +1,4 @@ +## template: jinja #cloud-config # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file @@ -279,6 +280,7 @@ write_files: fi EXTERNAL_ETCD_NODES={{ etcd.unstacked_etcd }} + EXTERNAL_CNI_PLUGIN={{ k8s.external.cni.plugin }} MAX_SETUP_CRUCIAL_CMD_ATTEMPTS=3 crucial_cmd_attempts=1 while true; do @@ -320,7 +322,9 @@ write_files: if [ -d "$K8S_CONFIG_SCRIPTS_COPY_DIR" ]; then ### Network, dashboard configs available offline ### echo "Offline configs are available!" - /opt/bin/kubectl apply -f ${K8S_CONFIG_SCRIPTS_COPY_DIR}/network.yaml + if [[ ${EXTERNAL_CNI_PLUGIN} == false ]]; then + /opt/bin/kubectl apply -f ${K8S_CONFIG_SCRIPTS_COPY_DIR}/network.yaml + fi /opt/bin/kubectl apply -f ${K8S_CONFIG_SCRIPTS_COPY_DIR}/dashboard.yaml rm -rf "${K8S_CONFIG_SCRIPTS_COPY_DIR}" else @@ -335,6 +339,7 @@ write_files: sudo touch /home/cloud/success echo "true" > /home/cloud/success +{% if registry is defined %} - path: /opt/bin/setup-containerd permissions: '0755' owner: root:root @@ -352,6 +357,7 @@ write_files: echo "Restarting containerd service" systemctl daemon-reload systemctl restart containerd +{% endif %} - path: /etc/systemd/system/deploy-kube-system.service permissions: '0755' diff --git a/plugins/integrations/kubernetes-service/src/main/resources/script/upgrade-kubernetes.sh b/plugins/integrations/kubernetes-service/src/main/resources/script/upgrade-kubernetes.sh index 480b002ef17..a947d508436 100755 --- a/plugins/integrations/kubernetes-service/src/main/resources/script/upgrade-kubernetes.sh +++ b/plugins/integrations/kubernetes-service/src/main/resources/script/upgrade-kubernetes.sh @@ -18,7 +18,7 @@ # Version 1.14 and below needs extra flags with kubeadm upgrade node if [ $# -lt 4 ]; then - echo "Invalid input. Valid usage: ./upgrade-kubernetes.sh UPGRADE_VERSION IS_CONTROL_NODE IS_OLD_VERSION IS_EJECT_ISO" + echo "Invalid input. Valid usage: ./upgrade-kubernetes.sh UPGRADE_VERSION IS_CONTROL_NODE IS_OLD_VERSION IS_EJECT_ISO IS_EXTERNAL_CNI" echo "eg: ./upgrade-kubernetes.sh 1.16.3 true false false" exit 1 fi @@ -35,6 +35,10 @@ EJECT_ISO_FROM_OS=false if [ $# -gt 3 ]; then EJECT_ISO_FROM_OS="${4}" fi +EXTERNAL_CNI=false +if [ $# -gt 4 ]; then + EXTERNAL_CNI="${5}" +fi export PATH=$PATH:/opt/bin if [[ "$PATH" != *:/usr/sbin && "$PATH" != *:/usr/sbin:* ]]; then @@ -144,7 +148,9 @@ if [ -d "$BINARIES_DIR" ]; then systemctl restart kubelet if [ "${IS_MAIN_CONTROL}" == 'true' ]; then - /opt/bin/kubectl apply -f ${BINARIES_DIR}/network.yaml + if [[ ${EXTERNAL_CNI} == true ]]; then + /opt/bin/kubectl apply -f ${BINARIES_DIR}/network.yaml + fi /opt/bin/kubectl apply -f ${BINARIES_DIR}/dashboard.yaml fi diff --git a/server/src/main/java/com/cloud/api/ApiResponseHelper.java b/server/src/main/java/com/cloud/api/ApiResponseHelper.java index 810f0abd7e0..81757721a3f 100644 --- a/server/src/main/java/com/cloud/api/ApiResponseHelper.java +++ b/server/src/main/java/com/cloud/api/ApiResponseHelper.java @@ -5411,9 +5411,13 @@ public class ApiResponseHelper implements ResponseGenerator { response.setZoneName(zone.getName()); response.setAsNumber(asn.getAsNumber()); ASNumberRangeVO range = asNumberRangeDao.findById(asn.getAsNumberRangeId()); - response.setAsNumberRangeId(range.getUuid()); - String rangeText = String.format("%s-%s", range.getStartASNumber(), range.getEndASNumber()); - response.setAsNumberRange(rangeText); + if (Objects.nonNull(range)) { + response.setAsNumberRangeId(range.getUuid()); + String rangeText = String.format("%s-%s", range.getStartASNumber(), range.getEndASNumber()); + response.setAsNumberRange(rangeText); + } else { + s_logger.info("is null for as number: "+ asn.getAsNumber()); + } response.setAllocated(asn.getAllocatedTime()); response.setAllocationState(asn.isAllocated() ? "Allocated" : "Free"); if (asn.getVpcId() != null) { diff --git a/server/src/main/java/com/cloud/network/NetworkServiceImpl.java b/server/src/main/java/com/cloud/network/NetworkServiceImpl.java index 0d4e3fdedc2..ea51b03602f 100644 --- a/server/src/main/java/com/cloud/network/NetworkServiceImpl.java +++ b/server/src/main/java/com/cloud/network/NetworkServiceImpl.java @@ -41,7 +41,9 @@ import javax.inject.Inject; import javax.naming.ConfigurationException; import com.cloud.bgp.BGPService; +import com.cloud.dc.ASNumberVO; import com.cloud.dc.VlanDetailsVO; +import com.cloud.dc.dao.ASNumberDao; import com.cloud.dc.dao.VlanDetailsDao; import com.cloud.network.dao.NsxProviderDao; import com.cloud.network.dao.PublicIpQuarantineDao; @@ -421,6 +423,8 @@ public class NetworkServiceImpl extends ManagerBase implements NetworkService, C RoutedIpv4Manager routedIpv4Manager; @Inject private BGPService bgpService; + @Inject + private ASNumberDao asNumberDao; List internalLoadBalancerElementServices = new ArrayList<>(); Map internalLoadBalancerElementServiceMap = new HashMap<>(); @@ -1726,6 +1730,9 @@ public class NetworkServiceImpl extends ManagerBase implements NetworkService, C routedIpv4Manager.validateBgpPeers(owner, zone.getId(), bgpPeerIds); } + // Check AS number if provided + ASNumberVO asNumberVO = checkAndSelectASNumber(asNumber, zone, ntwkOff, asNumberDao); + if (ipv4) { // For non-root admins check cidr limit - if it's allowed by global config value if (!_accountMgr.isRootAdmin(caller.getId()) && cidr != null) { @@ -1836,6 +1843,10 @@ public class NetworkServiceImpl extends ManagerBase implements NetworkService, C routedIpv4Manager.persistBgpPeersForGuestNetwork(network.getId(), bgpPeerIds); } + if (asNumberVO != null) { + bgpService.allocateASNumber(zone.getId(), asNumberVO.getAsNumber(), network.getId(), network.getVpcId()); + } + // if the network offering has persistent set to true, implement the network if (ntwkOff.isPersistent()) { return implementedNetworkInCreation(caller, zone, network); @@ -1847,6 +1858,26 @@ public class NetworkServiceImpl extends ManagerBase implements NetworkService, C return !networkOffering.isForVpc() && NetworkOffering.RoutingMode.Dynamic == networkOffering.getRoutingMode(); } + public static ASNumberVO checkAndSelectASNumber(Long asNumber, DataCenter zone, NetworkOffering networkOffering, ASNumberDao asNumberDao) { + if (NetworkOffering.RoutingMode.Dynamic != networkOffering.getRoutingMode()) { + return null; + } + return BooleanUtils.toBoolean(networkOffering.isSpecifyAsNumber()) ? + validateASNumber(asNumber, asNumberDao) : + asNumberDao.findOneByAllocationStateAndZone(zone.getId(), false); + } + + protected static ASNumberVO validateASNumber(Long asNumber, ASNumberDao asNumberDao) { + if (asNumber == null) { + return null; + } + ASNumberVO asNumberVO = asNumberDao.findByAsNumber(asNumber); + if (asNumberVO == null || asNumberVO.isAllocated()) { + throw new InvalidParameterValueException(String.format("The AS Number %s is already allocated, please select a free AS Number", asNumber)); + } + return asNumberVO; + } + private void validateNetworkCreationSupported(long zoneId, String zoneName, GuestType guestType) { NsxProviderVO nsxProviderVO = nsxProviderDao.findByZoneId(zoneId); if (Objects.nonNull(nsxProviderVO) && GuestType.L2.equals(guestType)) { diff --git a/server/src/main/java/com/cloud/server/ManagementServerImpl.java b/server/src/main/java/com/cloud/server/ManagementServerImpl.java index 2062ee1e94d..73a466e3aaa 100644 --- a/server/src/main/java/com/cloud/server/ManagementServerImpl.java +++ b/server/src/main/java/com/cloud/server/ManagementServerImpl.java @@ -519,9 +519,12 @@ import org.apache.cloudstack.api.command.user.template.ListTemplatesCmd; import org.apache.cloudstack.api.command.user.template.RegisterTemplateCmd; import org.apache.cloudstack.api.command.user.template.UpdateTemplateCmd; import org.apache.cloudstack.api.command.user.template.UpdateTemplatePermissionsCmd; +import org.apache.cloudstack.api.command.user.userdata.BaseRegisterUserDataCmd; import org.apache.cloudstack.api.command.user.userdata.DeleteUserDataCmd; import org.apache.cloudstack.api.command.user.userdata.LinkUserDataToTemplateCmd; +import org.apache.cloudstack.api.command.user.userdata.ListCniConfigurationCmd; import org.apache.cloudstack.api.command.user.userdata.ListUserDataCmd; +import org.apache.cloudstack.api.command.user.userdata.RegisterCniConfigurationCmd; import org.apache.cloudstack.api.command.user.userdata.RegisterUserDataCmd; import org.apache.cloudstack.api.command.user.vm.AddIpToVmNicCmd; import org.apache.cloudstack.api.command.user.vm.AddNicToVMCmd; @@ -4030,6 +4033,8 @@ public class ManagementServerImpl extends ManagerBase implements ManagementServe cmdList.add(DeleteUserDataCmd.class); cmdList.add(ListUserDataCmd.class); cmdList.add(LinkUserDataToTemplateCmd.class); + cmdList.add(RegisterCniConfigurationCmd.class); + cmdList.add(ListCniConfigurationCmd.class); //object store APIs cmdList.add(AddObjectStoragePoolCmd.class); @@ -4829,7 +4834,7 @@ public class ManagementServerImpl extends ManagerBase implements ManagementServe } @Override - public Pair, Integer> listUserDatas(final ListUserDataCmd cmd) { + public Pair, Integer> listUserDatas(final ListUserDataCmd cmd, final boolean forCks) { final Long id = cmd.getId(); final String name = cmd.getName(); final String keyword = cmd.getKeyword(); @@ -4849,6 +4854,8 @@ public class ManagementServerImpl extends ManagerBase implements ManagementServe sb.and("id", sb.entity().getId(), SearchCriteria.Op.EQ); sb.and("name", sb.entity().getName(), SearchCriteria.Op.EQ); sb.and("keyword", sb.entity().getName(), SearchCriteria.Op.LIKE); + sb.and("name", sb.entity().getName(), SearchCriteria.Op.EQ); + sb.and("forCks", sb.entity().isForCks(), SearchCriteria.Op.EQ); final SearchCriteria sc = sb.create(); _accountMgr.buildACLSearchCriteria(sc, domainId, isRecursive, permittedAccounts, listProjectResourcesCriteria); @@ -4864,24 +4871,33 @@ public class ManagementServerImpl extends ManagerBase implements ManagementServe sc.setParameters("keyword", "%" + keyword + "%"); } + sc.setParameters("forCks", forCks); + final Pair, Integer> result = userDataDao.searchAndCount(sc, searchFilter); return new Pair<>(result.first(), result.second()); } @Override @ActionEvent(eventType = EventTypes.EVENT_REGISTER_USER_DATA, eventDescription = "registering userdata", async = true) - public UserData registerUserData(final RegisterUserDataCmd cmd) { + public UserData registerUserData(final BaseRegisterUserDataCmd cmd) { final Account owner = getOwner(cmd); checkForUserDataByName(cmd, owner); - checkForUserData(cmd, owner); - final String name = cmd.getName(); - String userdata = cmd.getUserData(); + + boolean forCks = false; + String userdata = null; + if (cmd instanceof RegisterUserDataCmd) { + userdata = ((RegisterUserDataCmd) cmd).getUserData(); + checkForUserData(((RegisterUserDataCmd) cmd), owner); + } else { + userdata = ((RegisterCniConfigurationCmd) cmd).getCniConfig(); + forCks = true; + } final String params = cmd.getParams(); userdata = userDataManager.validateUserData(userdata, cmd.getHttpMethod()); - return createAndSaveUserData(name, userdata, params, owner); + return createAndSaveUserData(name, userdata, params, owner, forCks); } /** @@ -4901,7 +4917,7 @@ public class ManagementServerImpl extends ManagerBase implements ManagementServe * @param owner * @throws InvalidParameterValueException */ - private void checkForUserDataByName(final RegisterUserDataCmd cmd, final Account owner) throws InvalidParameterValueException { + private void checkForUserDataByName(final BaseRegisterUserDataCmd cmd, final Account owner) throws InvalidParameterValueException { final UserDataVO userData = userDataDao.findByName(owner.getAccountId(), owner.getDomainId(), cmd.getName()); if (userData != null) { throw new InvalidParameterValueException(String.format("A userdata with name %s already exists for this account.", cmd.getName())); @@ -4970,7 +4986,7 @@ public class ManagementServerImpl extends ManagerBase implements ManagementServe * @param cmd * @return Account */ - protected Account getOwner(final RegisterUserDataCmd cmd) { + protected Account getOwner(final BaseRegisterUserDataCmd cmd) { final Account caller = getCaller(); return _accountMgr.finalizeOwner(caller, cmd.getAccountName(), cmd.getDomainId(), cmd.getProjectId()); } @@ -4998,7 +5014,7 @@ public class ManagementServerImpl extends ManagerBase implements ManagementServe return newPair; } - private UserData createAndSaveUserData(final String name, final String userdata, final String params, final Account owner) { + private UserData createAndSaveUserData(final String name, final String userdata, final String params, final Account owner, final boolean isForCks) { final UserDataVO userDataVO = new UserDataVO(); userDataVO.setAccountId(owner.getAccountId()); @@ -5006,6 +5022,7 @@ public class ManagementServerImpl extends ManagerBase implements ManagementServe userDataVO.setName(name); userDataVO.setUserData(userdata); userDataVO.setParams(params); + userDataVO.setForCks(isForCks); userDataDao.persist(userDataVO); diff --git a/server/src/test/java/com/cloud/server/ManagementServerImplTest.java b/server/src/test/java/com/cloud/server/ManagementServerImplTest.java index b26cd455cfb..df94a7d2157 100644 --- a/server/src/test/java/com/cloud/server/ManagementServerImplTest.java +++ b/server/src/test/java/com/cloud/server/ManagementServerImplTest.java @@ -479,7 +479,7 @@ public class ManagementServerImplTest { Pair, Integer> result = new Pair(userDataList, 1); when(_userDataDao.searchAndCount(nullable(SearchCriteria.class), nullable(Filter.class))).thenReturn(result); - Pair, Integer> userdataResultList = spy.listUserDatas(cmd); + Pair, Integer> userdataResultList = spy.listUserDatas(cmd, false); Assert.assertEquals(userdataResultList.first().get(0), userDataList.get(0)); } @@ -512,7 +512,7 @@ public class ManagementServerImplTest { Pair, Integer> result = new Pair(userDataList, 1); when(_userDataDao.searchAndCount(nullable(SearchCriteria.class), nullable(Filter.class))).thenReturn(result); - Pair, Integer> userdataResultList = spy.listUserDatas(cmd); + Pair, Integer> userdataResultList = spy.listUserDatas(cmd, false); Assert.assertEquals(userdataResultList.first().get(0), userDataList.get(0)); } @@ -545,7 +545,7 @@ public class ManagementServerImplTest { Pair, Integer> result = new Pair(userDataList, 1); when(_userDataDao.searchAndCount(nullable(SearchCriteria.class), nullable(Filter.class))).thenReturn(result); - Pair, Integer> userdataResultList = spy.listUserDatas(cmd); + Pair, Integer> userdataResultList = spy.listUserDatas(cmd, false); Assert.assertEquals(userdataResultList.first().get(0), userDataList.get(0)); } diff --git a/ui/public/locales/en.json b/ui/public/locales/en.json index 243a071f05b..9ab009bae25 100644 --- a/ui/public/locales/en.json +++ b/ui/public/locales/en.json @@ -1874,7 +1874,8 @@ "label.region": "Region", "label.register.oauth": "Register OAuth", "label.register.template": "Register Template", -"label.register.user.data": "Register a userdata", +"label.register.user.data": "Register User Data", +"label.register.cni.config": "Register CNI Configuration", "label.reinstall.vm": "Reinstall Instance", "label.reject": "Reject", "label.related": "Related", @@ -2648,6 +2649,8 @@ "label.bucket.policy": "Bucket Policy", "label.usersecretkey": "Secret Key", "label.create.bucket": "Create Bucket", +"label.cniconfiguration": "CNI Configuration", +"label.cniconfigparams": "CNI Configuration parameters", "message.acquire.ip.failed": "Failed to acquire IP.", "message.action.acquire.ip": "Please confirm that you want to acquire new IP.", "message.action.cancel.maintenance": "Your host has been successfully canceled for maintenance. This process can take up to several minutes.", @@ -3020,6 +3023,7 @@ "message.desc.reset.ssh.key.pair": "Please specify a ssh key pair that you would like to add to this Instance.", "message.desc.secondary.storage": "Each zone must have at least one NFS or secondary storage server. We will add the first one now. Secondary storage stores Instance Templates, ISO images, and Instance disk volume Snapshots. This server must be available to all hosts in the zone.

Provide the IP address and exported path.", "message.desc.register.user.data": "Please fill in the following data to register a User data.", +"message.desc.register.cni.config": "Please fill in the following data to register CNI Configuration as user data.", "message.desc.registered.user.data": "Registered a User Data.", "message.desc.zone": "A zone is the largest organizational unit in CloudStack, and it typically corresponds to a single datacenter. Zones provide physical isolation and redundancy. A zone consists of one or more pods (each of which contains hosts and primary storage servers) and a secondary storage server which is shared by all pods in the zone.", "message.desc.zone.edge": "A zone is the largest organizational unit in CloudStack, and it typically corresponds to a single datacenter. Zones provide physical isolation and redundancy. An edge zone consists of one or more hosts (each of which provides local storage as primary storage servers). Only shared and L2 Networks can be deployed in such zones and functionalities that require secondary storages are not supported.", diff --git a/ui/src/components/view/DetailsTab.vue b/ui/src/components/view/DetailsTab.vue index 3622f87e67d..71bb544f079 100644 --- a/ui/src/components/view/DetailsTab.vue +++ b/ui/src/components/view/DetailsTab.vue @@ -44,7 +44,9 @@