etan39
/
cloudstack
mirror of https://github.com/apache/cloudstack.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Commit comprises of: 

- remove docker from systemvm template - use containerd as container runtime
- update create-k8s-binaries script to use ctr for all docker operations
- Update userdata sent to the k8s nodes
- update cksnode script, run during patching of the cks/k8s nodes
This commit is contained in:
Pearl Dsilva 2021-12-21 14:24:32 +05:30
parent 239ee80a88
commit ca4750d537
13 changed files with 77 additions and 66 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java
View File

@ -1169,20 +1169,6 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv
_storagePoolMgr = new KVMStoragePoolManager(_storage, _monitor);
_sysvmISOPath = (String)params.get("systemvm.iso.path");
if (_sysvmISOPath == null) {
final String[] isoPaths = {"/usr/share/cloudstack-common/vms/systemvm.iso"};
for (final String isoPath : isoPaths) {
if (_storage.exists(isoPath)) {
_sysvmISOPath = isoPath;
break;
}
}
if (_sysvmISOPath == null) {
s_logger.debug("Can't find system vm ISO");
}
}
final Map<String, String> bridges = new HashMap<String, String>();
params.put("libvirt.host.bridges", bridges);
@ -2903,14 +2889,12 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv
}
if (vmSpec.getType() != VirtualMachine.Type.User) {
if (_sysvmISOPath != null) {
final DiskDef iso = new DiskDef();
// iso.defISODisk(_sysvmISOPath);
if (_guestCpuArch != null && _guestCpuArch.equals("aarch64")) {
iso.setBusType(DiskDef.DiskBus.SCSI);
}
vm.getDevices().addDevice(iso);
final DiskDef iso = new DiskDef();
iso.defISODisk(_sysvmISOPath);
if (_guestCpuArch != null && _guestCpuArch.equals("aarch64")) {
iso.setBusType(DiskDef.DiskBus.SCSI);
}
vm.getDevices().addDevice(iso);
}
// For LXC, find and add the root filesystem, rbd data disks

10
plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node-add.yml
View File

@ -118,7 +118,7 @@ write_files:
fi
retval=0
set +e
docker load < "${BINARIES_DIR}/docker/$line"
ctr image import "${BINARIES_DIR}/docker/$line"
retval=$?
set -e
if [ $retval -eq 0 ]; then
@ -165,7 +165,7 @@ write_files:
fi
systemctl enable kubelet && systemctl start kubelet
modprobe br_netfilter && sysctl net.bridge.bridge-nf-call-iptables=1
modprobe overlay && modprobe br_netfilter && sysctl net.bridge.bridge-nf-call-iptables=1
if [ -d "$BINARIES_DIR" ] && [ "$ATTEMPT_ONLINE_INSTALL" = true ]; then
crucial_cmd_attempts=1
@ -176,7 +176,7 @@ write_files:
fi
retval=0
set +e
kubeadm config images pull
kubeadm config images pull --cri-socket /run/containerd/containerd.sock
retval=$?
set -e
if [ $retval -eq 0 ]; then
@ -218,8 +218,8 @@ write_files:
owner: root:root
content: |
[Unit]
Requires=docker.service
After=docker.service
Requires=containerd.service
After=containerd.service
[Service]
Type=simple

12
plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node.yml
View File

@ -138,7 +138,7 @@ write_files:
fi
retval=0
set +e
docker load < "${BINARIES_DIR}/docker/$line"
ctr image import "${BINARIES_DIR}/docker/$line"
retval=$?
set -e
if [ $retval -eq 0 ]; then
@ -187,7 +187,7 @@ write_files:
fi
systemctl enable kubelet && systemctl start kubelet
modprobe br_netfilter && sysctl net.bridge.bridge-nf-call-iptables=1
modprobe overlay && modprobe br_netfilter && sysctl net.bridge.bridge-nf-call-iptables=1
if [ -d "$BINARIES_DIR" ] && [ "$ATTEMPT_ONLINE_INSTALL" = true ]; then
crucial_cmd_attempts=1
@ -198,7 +198,7 @@ write_files:
fi
retval=0
set +e
kubeadm config images pull
kubeadm config images pull --cri-socket /run/containerd/containerd.sock
retval=$?
set -e
if [ $retval -eq 0 ]; then
@ -216,7 +216,7 @@ write_files:
fi
retval=0
set +e
kubeadm init --token {{ k8s_control_node.cluster.token }} --token-ttl 0 {{ k8s_control_node.cluster.initargs }}
kubeadm init --token {{ k8s_control_node.cluster.token }} --token-ttl 0 {{ k8s_control_node.cluster.initargs }} --cri-socket /run/containerd/containerd.sock
retval=$?
set -e
if [ $retval -eq 0 ]; then
@ -275,8 +275,8 @@ write_files:
owner: root:root
content: |
[Unit]
Requires=docker.service
After=docker.service
Requires=containerd.service
After=containerd.service
[Service]
Type=simple

10
plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-node.yml
View File

@ -118,7 +118,7 @@ write_files:
fi
retval=0
set +e
docker load < "${BINARIES_DIR}/docker/$line"
ctr image import "${BINARIES_DIR}/docker/$line"
retval=$?
set -e
if [ $retval -eq 0 ]; then
@ -165,7 +165,7 @@ write_files:
fi
systemctl enable kubelet && systemctl start kubelet
modprobe br_netfilter && sysctl net.bridge.bridge-nf-call-iptables=1
modprobe overlay && modprobe br_netfilter && sysctl net.bridge.bridge-nf-call-iptables=1
if [ -d "$BINARIES_DIR" ] && [ "$ATTEMPT_ONLINE_INSTALL" = true ]; then
crucial_cmd_attempts=1
@ -176,7 +176,7 @@ write_files:
fi
retval=0
set +e
kubeadm config images pull
kubeadm config images pull --cri-socket /run/containerd/containerd.sock
retval=$?
set -e
if [ $retval -eq 0 ]; then
@ -218,8 +218,8 @@ write_files:
owner: root:root
content: |
[Unit]
Requires=docker.service
After=docker.service
Requires=containerd.service
After=containerd.service
[Service]
Type=simple

2
plugins/integrations/kubernetes-service/src/main/resources/script/upgrade-kubernetes.sh
View File

@ -93,7 +93,7 @@ if [ -d "$BINARIES_DIR" ]; then
output=`ls ${BINARIES_DIR}/docker/`
if [ "$output" != "" ]; then
while read -r line; do
docker load < "${BINARIES_DIR}/docker/$line"
ctr image import "${BINARIES_DIR}/docker/$line"
done <<< "$output"
fi
if [ -e "${BINARIES_DIR}/provider.yaml" ]; then

22
scripts/util/create-kubernetes-binaries-iso.sh
View File

@ -98,19 +98,18 @@ provider_conf_file="${working_dir}/provider.yaml"
curl -sSL ${PROVIDER_URL} -o ${provider_conf_file}
echo "Fetching k8s docker images..."
docker -v
ctr -v
if [ $? -ne 0 ]; then
echo "Installing docker..."
echo "Installing containerd..."
if [ -f /etc/redhat-release ]; then
sudo yum -y remove docker-common docker container-selinux docker-selinux docker-engine
sudo yum -y install lvm2 device-mapper device-mapper-persistent-data device-mapper-event device-mapper-libs device-mapper-event-libs
sudo yum install -y http://mirror.centos.org/centos/7/extras/x86_64/Packages/container-selinux-2.107-3.el7.noarch.rpm
sudo wget https://download.docker.com/linux/centos/docker-ce.repo -O /etc/yum.repos.d/docker-ce.repo && sudo yum -y install docker-ce
sudo systemctl enable docker && sudo systemctl start docker
sudo yum install -y containerd.io
elif [ -f /etc/lsb-release ]; then
sudo apt update && sudo apt install docker.io -y
sudo systemctl enable docker && sudo systemctl start docker
sudo apt update && sudo apt install containerd.io -y
fi
sudo systemctl enable containerd && sudo systemctl start containerd
fi
mkdir -p "${working_dir}/docker"
output=`${k8s_dir}/kubeadm config images list --kubernetes-version=${RELEASE}`
@ -130,11 +129,14 @@ provider_image=`grep "image:" ${provider_conf_file} | cut -d ':' -f2- | tr -d '
output=`printf "%s\n" ${output} ${provider_image}`
while read -r line; do
echo "Downloading docker image $line ---"
sudo docker pull "$line"
echo "Downloading image $line ---"
if [[ $line == kubernetesui* ]] || [[ $line == apache* ]]; then
line="docker.io/${line}"
fi
sudo ctr image pull "$line"
image_name=`echo "$line" | grep -oE "[^/]+$"`
sudo docker save "$line" > "${working_dir}/docker/$image_name.tar"
sudo docker image rm "$line"
sudo ctr image export "${working_dir}/docker/$image_name.tar" "$line"
sudo ctr image rm "$line"
done <<< "$output"
echo "Restore kubeadm permissions..."

8
server/src/main/java/com/cloud/consoleproxy/ConsoleProxyManagerImpl.java
View File

@ -29,7 +29,9 @@ import java.util.Map;
import javax.inject.Inject;
import javax.naming.ConfigurationException;
import com.cloud.utils.PasswordGenerator;
import org.apache.cloudstack.agent.lb.IndirectAgentLB;
import org.apache.cloudstack.ca.CAManager;
import org.apache.cloudstack.context.CallContext;
import org.apache.cloudstack.engine.orchestration.service.NetworkOrchestrationService;
import org.apache.cloudstack.framework.config.ConfigKey;
@ -221,6 +223,10 @@ public class ConsoleProxyManagerImpl extends ManagerBase implements ConsoleProxy
private VirtualMachineManager virtualMachineManager;
@Inject
private IndirectAgentLB indirectAgentLB;
@Inject
private CAManager caManager;
@Inject
private NetworkOrchestrationService networkMgr;
private ConsoleProxyListener consoleProxyListener;
@ -1274,6 +1280,8 @@ public class ConsoleProxyManagerImpl extends ManagerBase implements ConsoleProxy
buf.append(" dns2=").append(dc.getDns2());
}
buf.append(" keystore_password=").append(PasswordGenerator.generateRandomPassword(16));
buf.append(" validity=").append(CAManager.CertValidityPeriod.value());
String bootArgs = buf.toString();
if (s_logger.isDebugEnabled()) {
s_logger.debug("Boot Args for " + profile + ": " + bootArgs);

9
systemvm/debian/opt/cloud/bin/setup/cksnode.sh
View File

@ -28,18 +28,23 @@ setup_k8s_node() {
# set default ssh port and restart sshd service
sed -i 's/3922/22/g' /etc/ssh/sshd_config
systemctl restart ssh
# Prevent root login
> /root/.ssh/authorized_keys
passwd -l root
#sed -i 's#root:x:0:0:root:/root:/bin/bash#root:x:0:0:root:/root:/sbin/nologin#' /etc/passwd
# Update containerd configuration
mkdir -p /etc/containerd
containerd config default>/etc/containerd/config.toml
systemctl restart containerd
swapoff -a
sudo sed -i '/ swap / s/^/#/' /etc/fstab
log_it "Swap disabled"
log_it "Setting up interfaces"
# setup_common eth0
setup_system_rfc1918_internal
log_it "Setting up entry in hosts"
@ -61,8 +66,6 @@ setup_k8s_node() {
log_it "Starting cloud-init services"
systemctl enable --now --no-block containerd
systemctl enable --now --no-block docker.socket
systemctl enable --now --no-block docker.service
if [ -f /home/core/success ]; then
systemctl stop cloud-init cloud-config cloud-final
systemctl disable cloud-init cloud-config cloud-final

29
systemvm/debian/opt/cloud/bin/setup/cloud-early-config
View File

@ -31,29 +31,40 @@ log_it() {
log_action_msg "$@"
}
validate_checksums() {
local oldmd5=
[ -f ${1} ] && oldmd5=$(cat ${1})
local newmd5=
[ -f ${2} ] && newmd5=$(md5sum ${2} | awk '{print $1}')
log_it "Scripts checksum detected: oldmd5=$oldmd5 newmd5=$newmd5" >> /dev/null 2>&1
echo "oldmd5='${oldmd5}'; newmd5='${newmd5}'"
}
patch() {
local PATCH_MOUNT=/home/cloud
local patchfile=$PATCH_MOUNT/cloud-scripts.tgz
local PATCH_SCRIPTS=cloud-scripts.tgz
local oldpatchfile=/usr/share/cloud/$PATCH_SCRIPTS
local patchfile=$PATCH_MOUNT/$PATCH_SCRIPTS
local privkey=$PATCH_MOUNT/authorized_keys
local md5file=/var/cache/cloud/cloud-scripts-signature
local cdrom_dev=
mkdir -p $PATCH_MOUNT
if [ -f /var/cache/cloud/authorized_keys ]; then
privkey=/var/cache/cloud/authorized_keys
fi
eval $(validate_checksums $md5file $oldpatchfile)
if [ "$oldmd5" == "$newmd5" ] && [ ! -f ${patchfile} ]; then
log_it "Checksum matches, do need to patch"
return 0
fi
retry=60
local patched=false
while [ $retry -gt 0 ]
do
if [ -f $patchfile ]; then
local oldmd5=
[ -f ${md5file} ] && oldmd5=$(cat ${md5file})
local newmd5=
[ -f ${patchfile} ] && newmd5=$(md5sum ${patchfile} | awk '{print $1}')
log_it "Scripts checksum detected: oldmd5=$oldmd5 newmd5=$newmd5"
log_it ls -lrt $PATCH_MOUNT
eval $(validate_checksums $md5file $patchfile)
if [ "$oldmd5" != "$newmd5" ] && [ -f ${patchfile} ] && [ "$newmd5" != "" ]
then
tar xzf $patchfile -C /
@ -84,7 +95,7 @@ patch() {
cleanup() {
rm -rf /home/cloud/agent.zip
rm -rf /home/cloud/cloud-scripts.tgz
mv /home/cloud/cloud-scripts.tgz /usr/share/cloud/cloud-scripts.tgz
}
start() {

6
systemvm/debian/opt/cloud/bin/setup/common.sh
View File

@ -762,6 +762,12 @@ parse_cmd_line() {
authorized_key)
export AUTHORIZED_KEYS=$VALUE
;;
keystore_password)
export KEYSTORE_PSSWD=$VALUE
;;
validity)
export VALIDITY=$VALUE
;;
esac
done
echo -e "\n\t}\n}" >> ${CHEF_TMP_FILE}

3
systemvm/patch-sysvms.sh
View File

@ -76,7 +76,8 @@ restart_services() {
cleanup_systemVM() {
rm -rf $backupfolder
rm -rf "$newpath""cloud-scripts.tgz" "$newpath""agent.zip" "$newpath""patch-sysvms.sh"
mv "$newpath"cloud-scripts.tgz /usr/share/cloud/cloud-scripts.tgz
rm -rf "$newpath""agent.zip" "$newpath""patch-sysvms.sh"
}
patch_systemvm() {

4
tools/appliance/systemvmtemplate/scripts/configure_systemvm_services.sh
View File

@ -127,10 +127,6 @@ function configure_services() {
# Disable container services
systemctl disable containerd
systemctl disable docker.service
systemctl stop docker.service
systemctl disable docker.socket
systemctl stop docker.socket
# Disable cloud init by default
cat <<EOF > /etc/cloud/cloud.cfg.d/cloudstack.cfg

2
tools/appliance/systemvmtemplate/scripts/install_systemvm_packages.sh
View File

@ -98,7 +98,7 @@ function install_packages() {
apt-key fingerprint 0EBFCD88
add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/debian $(lsb_release -cs) stable"
apt-get update
${apt_get} install docker-ce docker-ce-cli containerd.io
${apt_get} install containerd.io
apt_clean