etan39
/
cloudstack
mirror of https://github.com/apache/cloudstack.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Pre-seed images on worker nodes before control plane upgrade for air-gapped envs

This commit is contained in:
Pearl Dsilva 2026-06-16 22:34:50 -04:00
parent 8936e4c535
commit 220e4b2e59
2 changed files with 55 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

45
plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterUpgradeWorker.java
View File

@ -63,6 +63,20 @@ public class KubernetesClusterUpgradeWorker extends KubernetesClusterActionWorke
upgradeScriptFile = retrieveScriptFile(upgradeScriptFilename);
}
private Pair<Boolean, String> preseedImagesOnVM(final UserVm vm, final int index) throws Exception {
int nodeSshPort = sshPort == 22 ? sshPort : sshPort + index;
String nodeAddress = (index > 0 && sshPort == 22) ? vm.getPrivateIpAddress() : publicIpAddress;
SshHelper.scpTo(nodeAddress, nodeSshPort, getControlNodeLoginUser(), sshKeyFile, null,
"~/", upgradeScriptFile.getAbsolutePath(), "0755");
String cmdStr = String.format("sudo ./%s %s false false %s %s true",
upgradeScriptFile.getName(),
upgradeVersion.getSemanticVersion(),
Hypervisor.HypervisorType.VMware.equals(vm.getHypervisorType()),
Objects.isNull(kubernetesCluster.getCniConfigId()));
return SshHelper.sshExecute(nodeAddress, nodeSshPort, getControlNodeLoginUser(), sshKeyFile, null,
cmdStr, 10000, 10000, 5 * 60 * 1000);
}
private Pair<Boolean, String> runInstallScriptOnVM(final UserVm vm, final int index) throws Exception {
int nodeSshPort = sshPort == 22 ? sshPort : sshPort + index;
String nodeAddress = (index > 0 && sshPort == 22) ? vm.getPrivateIpAddress() : publicIpAddress;
@ -80,6 +94,37 @@ public class KubernetesClusterUpgradeWorker extends KubernetesClusterActionWorke
}
private void upgradeKubernetesClusterNodes() {
// kubeadm upgrade apply schedules a post-upgrade health check pod on a non-cordoned node.
// In air-gapped clusters, workers lack the new pause image until their own upgrade runs,
// causing the pod pull to fail and the upgrade to stall. Pre-seed all workers first so
// the image is present before the control-plane upgrade starts.
if (clusterVMs.size() > 1) {
for (int i = 1; i < clusterVMs.size(); ++i) {
UserVm vm = clusterVMs.get(i);
String errorMessage = String.format("Failed to upgrade Kubernetes cluster : %s, unable to pre-seed images on VM : %s",
kubernetesCluster.getName(), vm.getDisplayName());
for (int retry = KubernetesClusterService.KubernetesClusterUpgradeRetries.value(); retry >= 0; retry--) {
try {
Pair<Boolean, String> result = preseedImagesOnVM(vm, i);
if (result.first()) {
break;
}
if (retry > 0) {
logger.warn("{}, retries left: {}", errorMessage, retry);
} else {
logTransitStateDetachIsoAndThrow(Level.ERROR, errorMessage, kubernetesCluster, clusterVMs, KubernetesCluster.Event.OperationFailed, null);
}
} catch (Exception e) {
if (retry > 0) {
logger.warn("{} due to {}, retries left: {}", errorMessage, e, retry);
} else {
logTransitStateDetachIsoAndThrow(Level.ERROR, errorMessage, kubernetesCluster, clusterVMs, KubernetesCluster.Event.OperationFailed, e);
}
}
}
}
}
for (int i = 0; i < clusterVMs.size(); ++i) {
UserVm vm = clusterVMs.get(i);
String hostName = vm.getHostName();

10
plugins/integrations/kubernetes-service/src/main/resources/script/upgrade-kubernetes.sh
View File

@ -39,6 +39,10 @@ EXTERNAL_CNI=false
if [ $# -gt 4 ]; then
EXTERNAL_CNI="${5}"
fi
PRESEED_ONLY=false
if [ $# -gt 5 ]; then
PRESEED_ONLY="${6}"
fi
export PATH=$PATH:/opt/bin
if [[ "$PATH" != *:/usr/sbin && "$PATH" != *:/usr/sbin:* ]]; then
@ -120,6 +124,12 @@ if [ -d "$BINARIES_DIR" ]; then
sed -i "s|sandbox_image = .*|sandbox_image = \"$PAUSE_IMAGE\"|g" /etc/containerd/config.toml
fi
if [ "${PRESEED_ONLY}" == 'true' ]; then
systemctl restart containerd
umount "${ISO_MOUNT_DIR}" && rmdir "${ISO_MOUNT_DIR}"
exit 0
fi
tar -f "${BINARIES_DIR}/cni/cni-plugins-"*64.tgz -C /opt/cni/bin -xz
tar -f "${BINARIES_DIR}/cri-tools/crictl-linux-"*64.tar.gz -C /opt/bin -xz