From 10c77c88c8a929238e755f24682dd0c0e2aea28c Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Tue, 24 Jan 2023 16:11:32 +0100 Subject: [PATCH] CKS: fix upgrade of HA cluster (#7118) --- .../KubernetesClusterUpgradeWorker.java | 2 +- .../cluster/utils/KubernetesClusterUtil.java | 60 +++++++++++-------- .../utils/KubernetesClusterUtilTest.java | 49 ++++----------- 3 files changed, 47 insertions(+), 64 deletions(-) diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterUpgradeWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterUpgradeWorker.java index 1cc525c49e0..9b7b6ca47c2 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterUpgradeWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterUpgradeWorker.java @@ -122,7 +122,7 @@ public class KubernetesClusterUpgradeWorker extends KubernetesClusterActionWorke logTransitStateDetachIsoAndThrow(Level.ERROR, String.format("Failed to upgrade Kubernetes cluster : %s, unable to get control Kubernetes node on VM : %s in ready state", kubernetesCluster.getName(), vm.getDisplayName()), kubernetesCluster, clusterVMs, KubernetesCluster.Event.OperationFailed, null); } } - if (!KubernetesClusterUtil.clusterNodeVersionMatches(upgradeVersion.getSemanticVersion(), i==0, publicIpAddress, sshPort, getControlNodeLoginUser(), getManagementServerSshPublicKeyFile(), hostName)) { + if (!KubernetesClusterUtil.clusterNodeVersionMatches(upgradeVersion.getSemanticVersion(), publicIpAddress, sshPort, getControlNodeLoginUser(), getManagementServerSshPublicKeyFile(), hostName, upgradeTimeoutTime, 15000)) { logTransitStateDetachIsoAndThrow(Level.ERROR, String.format("Failed to upgrade Kubernetes cluster : %s, unable to get Kubernetes node on VM : %s upgraded to version %s", kubernetesCluster.getName(), vm.getDisplayName(), upgradeVersion.getSemanticVersion()), kubernetesCluster, clusterVMs, KubernetesCluster.Event.OperationFailed, null); } if (LOGGER.isInfoEnabled()) { diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterUtil.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterUtil.java index 99da4435c8d..e1210a607e6 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterUtil.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterUtil.java @@ -45,13 +45,14 @@ public class KubernetesClusterUtil { protected static final Logger LOGGER = Logger.getLogger(KubernetesClusterUtil.class); - public static final String CLUSTER_NODE_VERSION_COMMAND = "sudo /opt/bin/kubectl version --short"; + public static final String CLUSTER_NODE_READY_COMMAND = "sudo /opt/bin/kubectl get nodes | awk '{if ($1 == \"%s\" && $2 == \"Ready\") print $1}'"; + public static final String CLUSTER_NODE_VERSION_COMMAND = "sudo /opt/bin/kubectl get nodes | awk '{if ($1 == \"%s\") print $5}'"; public static boolean isKubernetesClusterNodeReady(final KubernetesCluster kubernetesCluster, String ipAddress, int port, String user, File sshKeyFile, String nodeName) throws Exception { Pair result = SshHelper.sshExecute(ipAddress, port, user, sshKeyFile, null, - String.format("sudo /opt/bin/kubectl get nodes | awk '{if ($1 == \"%s\" && $2 == \"Ready\") print $1}'", nodeName.toLowerCase()), + String.format(CLUSTER_NODE_READY_COMMAND, nodeName.toLowerCase()), 10000, 10000, 20000); if (result.first() && nodeName.equals(result.second().trim())) { return true; @@ -327,37 +328,44 @@ public class KubernetesClusterUtil { return token.toString().substring(0, 64); } - public static boolean clusterNodeVersionMatches(final String version, boolean isControlNode, + public static boolean clusterNodeVersionMatches(final String version, final String ipAddress, final int port, final String user, final File sshKeyFile, - final String hostName) { - Pair result = null; - try { - result = SshHelper.sshExecute( - ipAddress, port, - user, sshKeyFile, null, - CLUSTER_NODE_VERSION_COMMAND, - 10000, 10000, 20000); - } catch (Exception e) { + final String hostName, + final long timeoutTime, final long waitDuration) { + int retry = 10; + while (System.currentTimeMillis() < timeoutTime && retry-- > 0) { if (LOGGER.isDebugEnabled()) { - LOGGER.debug(String.format("Failed to retrieve Kubernetes version from cluster node : %s due to exception", hostName), e); + LOGGER.debug(String.format("Checking if the Kubernetes version of cluster node %s is %s", hostName, version)); + } + try { + Pair result = SshHelper.sshExecute( + ipAddress, port, + user, sshKeyFile, null, + String.format(CLUSTER_NODE_VERSION_COMMAND, hostName.toLowerCase()), + 10000, 10000, 20000); + if (clusterNodeVersionMatches(result, version)) { + return true; + } + } catch (Exception e) { + if (LOGGER.isDebugEnabled()) { + LOGGER.debug(String.format("Failed to retrieve Kubernetes version from cluster node : %s due to exception", hostName), e); + } + } + try { + Thread.sleep(waitDuration); + } catch (InterruptedException ex) { + LOGGER.warn(String.format("Error while waiting during Kubernetes version check of cluster node : %s", hostName), ex); } - return false; } - if (Boolean.FALSE.equals(result.first()) || StringUtils.isBlank(result.second())) { + return false; + } + + protected static boolean clusterNodeVersionMatches(final Pair result, final String version) { + if (result == null || Boolean.FALSE.equals(result.first()) || StringUtils.isBlank(result.second())) { return false; } String response = result.second(); - boolean clientVersionPresent = false; - boolean serverVersionPresent = false; - for (String line : response.split("\n")) { - if (line.contains("Client Version") && line.contains(String.format("v%s", version))) { - clientVersionPresent = true; - } - if (isControlNode && line.contains("Server Version") && line.contains(String.format("v%s", version))) { - serverVersionPresent = true; - } - } - return clientVersionPresent && (!isControlNode || serverVersionPresent); + return response.contains(String.format("v%s", version)); } } diff --git a/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterUtilTest.java b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterUtilTest.java index 475eeba7fc8..53bc1a3c0f6 100644 --- a/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterUtilTest.java +++ b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterUtilTest.java @@ -39,54 +39,29 @@ public class KubernetesClusterUtilTest { String hostName = "host"; private void mockSshHelperExecuteThrowAndTestVersionMatch() { - try { - Mockito.when(SshHelper.sshExecute(ipAddress, port, user, sshKeyFile, null, KubernetesClusterUtil.CLUSTER_NODE_VERSION_COMMAND, 10000, 10000, 20000)).thenThrow(Exception.class); - } catch (Exception e) { - Assert.fail(String.format("Exception: %s", e.getMessage())); - } - Assert.assertFalse(KubernetesClusterUtil.clusterNodeVersionMatches("1.24.0", false, ipAddress, port, user, sshKeyFile, hostName)); + Pair resultPair = null; + boolean result = KubernetesClusterUtil.clusterNodeVersionMatches(resultPair, "1.24.0"); + Assert.assertFalse(result); } - private void mockSshHelperExecuteAndTestVersionMatch(boolean status, String response, boolean isControlNode, boolean expectedResult) { - try { - Mockito.when(SshHelper.sshExecute(ipAddress, port, user, sshKeyFile, null, KubernetesClusterUtil.CLUSTER_NODE_VERSION_COMMAND, 10000, 10000, 20000)).thenReturn(new Pair<>(status, response)); - } catch (Exception e) { - Assert.fail(String.format("Exception: %s", e.getMessage())); - } - boolean result = KubernetesClusterUtil.clusterNodeVersionMatches("1.24.0", isControlNode, ipAddress, port, user, sshKeyFile, hostName); + private void mockSshHelperExecuteAndTestVersionMatch(boolean status, String response, boolean expectedResult) { + Pair resultPair = new Pair<>(status, response); + boolean result = KubernetesClusterUtil.clusterNodeVersionMatches(resultPair, "1.24.0"); Assert.assertEquals(expectedResult, result); } @Test public void testClusterNodeVersionMatches() { PowerMockito.mockStatic(SshHelper.class); - String v1233WorkerNodeOutput = "Client Version: v1.23.3\n" + - "The connection to the server localhost:8080 was refused - did you specify the right host or port?"; - String v1240WorkerNodeOutput = "Client Version: v1.24.0\n" + - "Kustomize Version: v4.5.4\n" + - "The connection to the server localhost:8080 was refused - did you specify the right host or port?"; - String v1240ControlNodeOutput = "Client Version: v1.24.0\n" + - "Kustomize Version: v4.5.4\n" + - "Server Version: v1.24.0"; - mockSshHelperExecuteAndTestVersionMatch(true, v1240WorkerNodeOutput, false, true); + String v1233WorkerNodeOutput = "v1.23.3"; + String v1240WorkerNodeOutput = "v1.24.0"; + mockSshHelperExecuteAndTestVersionMatch(true, v1240WorkerNodeOutput, true); - mockSshHelperExecuteAndTestVersionMatch(true, v1240ControlNodeOutput, true, true); + mockSshHelperExecuteAndTestVersionMatch(true, v1233WorkerNodeOutput, false); - mockSshHelperExecuteAndTestVersionMatch(true, v1240WorkerNodeOutput, true, false); + mockSshHelperExecuteAndTestVersionMatch(false, v1240WorkerNodeOutput, false); - mockSshHelperExecuteAndTestVersionMatch(false, v1240WorkerNodeOutput, false, false); - - mockSshHelperExecuteAndTestVersionMatch(true, v1233WorkerNodeOutput, false, false); - - mockSshHelperExecuteAndTestVersionMatch(true, "Client Version: v1.24.0\n" + - "Kustomize Version: v4.5.4\n" + - "Server Version: v1.23.0", true, false); - - mockSshHelperExecuteAndTestVersionMatch(true, null, false, false); - - mockSshHelperExecuteAndTestVersionMatch(false, "-\n-", false, false); - - mockSshHelperExecuteAndTestVersionMatch(false, "1.24.0", false, false); + mockSshHelperExecuteAndTestVersionMatch(false, v1233WorkerNodeOutput, false); mockSshHelperExecuteThrowAndTestVersionMatch(); }