mirror of https://github.com/apache/cloudstack.git
340 lines
13 KiB
YAML
340 lines
13 KiB
YAML
#cloud-config
|
|
# Licensed to the Apache Software Foundation (ASF) under one
|
|
# or more contributor license agreements. See the NOTICE file
|
|
# distributed with this work for additional information
|
|
# regarding copyright ownership. The ASF licenses this file
|
|
# to you under the Apache License, Version 2.0 (the
|
|
# "License"); you may not use this file except in compliance
|
|
# with the License. You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing,
|
|
# software distributed under the License is distributed on an
|
|
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
|
# KIND, either express or implied. See the License for the
|
|
# specific language governing permissions and limitations
|
|
# under the License.
|
|
|
|
---
|
|
users:
|
|
- name: cloud
|
|
sudo: ALL=(ALL) NOPASSWD:ALL
|
|
shell: /bin/bash
|
|
ssh_authorized_keys:
|
|
{{ k8s.ssh.pub.key }}
|
|
|
|
write_files:
|
|
- path: /etc/conf.d/nfs
|
|
permissions: '0644'
|
|
content: |
|
|
OPTS_RPC_MOUNTD=""
|
|
|
|
- path: /etc/kubernetes/pki/cloudstack/ca.crt
|
|
permissions: '0644'
|
|
content: |
|
|
{{ k8s_control_node.ca.crt }}
|
|
|
|
- path: /etc/kubernetes/pki/cloudstack/apiserver.crt
|
|
permissions: '0644'
|
|
content: |
|
|
{{ k8s_control_node.apiserver.crt }}
|
|
|
|
- path: /etc/kubernetes/pki/cloudstack/apiserver.key
|
|
permissions: '0600'
|
|
content: |
|
|
{{ k8s_control_node.apiserver.key }}
|
|
|
|
- path: /opt/bin/setup-kube-system
|
|
permissions: '0700'
|
|
owner: root:root
|
|
content: |
|
|
#!/bin/bash -e
|
|
|
|
if [[ -f "/home/cloud/success" ]]; then
|
|
echo "Already provisioned!"
|
|
exit 0
|
|
fi
|
|
|
|
sysctl net.ipv4.conf.default.arp_announce=0
|
|
sysctl net.ipv4.conf.default.arp_ignore=0
|
|
sysctl net.ipv4.conf.all.arp_announce=0
|
|
sysctl net.ipv4.conf.all.arp_ignore=0
|
|
sysctl net.ipv4.conf.eth0.arp_announce=0
|
|
sysctl net.ipv4.conf.eth0.arp_ignore=0
|
|
sed -i "s/net.ipv4.conf.default.arp_announce =.*$/net.ipv4.conf.default.arp_announce = 0/" /etc/sysctl.conf
|
|
sed -i "s/net.ipv4.conf.default.arp_ignore =.*$/net.ipv4.conf.default.arp_ignore = 0/" /etc/sysctl.conf
|
|
sed -i "s/net.ipv4.conf.all.arp_announce =.*$/net.ipv4.conf.all.arp_announce = 0/" /etc/sysctl.conf
|
|
sed -i "s/net.ipv4.conf.all.arp_ignore =.*$/net.ipv4.conf.all.arp_ignore = 0/" /etc/sysctl.conf
|
|
|
|
ISO_MOUNT_DIR=/mnt/k8sdisk
|
|
BINARIES_DIR=${ISO_MOUNT_DIR}/
|
|
K8S_CONFIG_SCRIPTS_COPY_DIR=/tmp/k8sconfigscripts/
|
|
ATTEMPT_ONLINE_INSTALL=false
|
|
setup_complete=false
|
|
|
|
OFFLINE_INSTALL_ATTEMPT_SLEEP=15
|
|
MAX_OFFLINE_INSTALL_ATTEMPTS=100
|
|
offline_attempts=1
|
|
MAX_SETUP_CRUCIAL_CMD_ATTEMPTS=3
|
|
EJECT_ISO_FROM_OS={{ k8s.eject.iso }}
|
|
crucial_cmd_attempts=1
|
|
iso_drive_path=""
|
|
while true; do
|
|
if (( "$offline_attempts" > "$MAX_OFFLINE_INSTALL_ATTEMPTS" )); then
|
|
echo "Warning: Offline install timed out!"
|
|
break
|
|
fi
|
|
set +e
|
|
output=`blkid -o device -t LABEL=CDROM`
|
|
set -e
|
|
if [ "$output" != "" ]; then
|
|
while read -r line; do
|
|
if [ ! -d "${ISO_MOUNT_DIR}" ]; then
|
|
mkdir "${ISO_MOUNT_DIR}"
|
|
fi
|
|
retval=0
|
|
set +e
|
|
mount -o ro "${line}" "${ISO_MOUNT_DIR}"
|
|
retval=$?
|
|
set -e
|
|
if [ $retval -eq 0 ]; then
|
|
if [ -d "$BINARIES_DIR" ]; then
|
|
iso_drive_path="${line}"
|
|
break
|
|
else
|
|
umount "${line}" && rmdir "${ISO_MOUNT_DIR}"
|
|
fi
|
|
fi
|
|
done <<< "$output"
|
|
fi
|
|
if [ -d "$BINARIES_DIR" ]; then
|
|
break
|
|
fi
|
|
echo "Waiting for Binaries directory $BINARIES_DIR to be available, sleeping for $OFFLINE_INSTALL_ATTEMPT_SLEEP seconds, attempt: $offline_attempts"
|
|
sleep $OFFLINE_INSTALL_ATTEMPT_SLEEP
|
|
offline_attempts=$[$offline_attempts + 1]
|
|
done
|
|
|
|
if [[ "$PATH" != *:/opt/bin && "$PATH" != *:/opt/bin:* ]]; then
|
|
export PATH=$PATH:/opt/bin
|
|
fi
|
|
|
|
if [ -d "$BINARIES_DIR" ]; then
|
|
### Binaries available offline ###
|
|
echo "Installing binaries from ${BINARIES_DIR}"
|
|
mkdir -p /opt/cni/bin
|
|
tar -f "${BINARIES_DIR}/cni/cni-plugins-"*64.tgz -C /opt/cni/bin -xz
|
|
|
|
mkdir -p /opt/bin
|
|
tar -f "${BINARIES_DIR}/cri-tools/crictl-linux-"*64.tar.gz -C /opt/bin -xz
|
|
|
|
mkdir -p /opt/bin
|
|
cd /opt/bin
|
|
cp -a ${BINARIES_DIR}/k8s/{kubeadm,kubelet,kubectl} .
|
|
chmod +x {kubeadm,kubelet,kubectl}
|
|
|
|
sed "s:/usr/bin:/opt/bin:g" ${BINARIES_DIR}/kubelet.service > /etc/systemd/system/kubelet.service
|
|
mkdir -p /etc/systemd/system/kubelet.service.d
|
|
sed "s:/usr/bin:/opt/bin:g" ${BINARIES_DIR}/10-kubeadm.conf > /etc/systemd/system/kubelet.service.d/10-kubeadm.conf
|
|
|
|
echo "KUBELET_EXTRA_ARGS=--cgroup-driver=systemd" > /etc/default/kubelet
|
|
|
|
output=`ls ${BINARIES_DIR}/docker/`
|
|
if [ "$output" != "" ]; then
|
|
while read -r line; do
|
|
crucial_cmd_attempts=1
|
|
while true; do
|
|
if (( "$crucial_cmd_attempts" > "$MAX_SETUP_CRUCIAL_CMD_ATTEMPTS" )); then
|
|
echo "Loading docker image ${BINARIES_DIR}/docker/$line failed!"
|
|
break;
|
|
fi
|
|
retval=0
|
|
set +e
|
|
ctr -n k8s.io image import "${BINARIES_DIR}/docker/$line"
|
|
retval=$?
|
|
set -e
|
|
if [ $retval -eq 0 ]; then
|
|
break;
|
|
fi
|
|
crucial_cmd_attempts=$[$crucial_cmd_attempts + 1]
|
|
done
|
|
done <<< "$output"
|
|
setup_complete=true
|
|
fi
|
|
mkdir -p "${K8S_CONFIG_SCRIPTS_COPY_DIR}"
|
|
cp ${BINARIES_DIR}/*.yaml "${K8S_CONFIG_SCRIPTS_COPY_DIR}"
|
|
if [ -e "${BINARIES_DIR}/autoscaler.yaml" ]; then
|
|
mkdir -p /opt/autoscaler
|
|
cp "${BINARIES_DIR}/autoscaler.yaml" /opt/autoscaler/autoscaler_tmpl.yaml
|
|
fi
|
|
if [ -e "${BINARIES_DIR}/provider.yaml" ]; then
|
|
mkdir -p /opt/provider
|
|
cp "${BINARIES_DIR}/provider.yaml" /opt/provider/provider.yaml
|
|
fi
|
|
|
|
PAUSE_IMAGE=`ctr -n k8s.io images ls -q | grep "pause" | sort | tail -n 1`
|
|
echo $PAUSE_IMAGE
|
|
if [ -n "$PAUSE_IMAGE" ]; then
|
|
sed -i "s|sandbox_image = .*|sandbox_image = \"$PAUSE_IMAGE\"|g" /etc/containerd/config.toml
|
|
fi
|
|
systemctl daemon-reload
|
|
systemctl restart containerd
|
|
|
|
umount "${ISO_MOUNT_DIR}" && rmdir "${ISO_MOUNT_DIR}"
|
|
if [ "$EJECT_ISO_FROM_OS" = true ] && [ "$iso_drive_path" != "" ]; then
|
|
eject "${iso_drive_path}"
|
|
fi
|
|
fi
|
|
if [ "$setup_complete" = false ] && [ "$ATTEMPT_ONLINE_INSTALL" = true ]; then
|
|
### Binaries not available offline ###
|
|
RELEASE="v1.16.3"
|
|
CNI_VERSION="v0.7.5"
|
|
CRICTL_VERSION="v1.16.0"
|
|
echo "Warning: ${BINARIES_DIR} not found. Will get binaries and docker images from Internet."
|
|
mkdir -p /opt/cni/bin
|
|
curl -L "https://github.com/containernetworking/plugins/releases/download/${CNI_VERSION}/cni-plugins-amd64-${CNI_VERSION}.tgz" | tar -C /opt/cni/bin -xz
|
|
|
|
mkdir -p /opt/bin
|
|
curl -L "https://github.com/kubernetes-incubator/cri-tools/releases/download/${CRICTL_VERSION}/crictl-${CRICTL_VERSION}-linux-amd64.tar.gz" | tar -C /opt/bin -xz
|
|
|
|
mkdir -p /opt/bin
|
|
cd /opt/bin
|
|
curl -L --remote-name-all https://storage.googleapis.com/kubernetes-release/release/${RELEASE}/bin/linux/amd64/{kubeadm,kubelet,kubectl}
|
|
chmod +x {kubeadm,kubelet,kubectl}
|
|
|
|
curl -sSL "https://raw.githubusercontent.com/kubernetes/kubernetes/${RELEASE}/build/debs/kubelet.service" | sed "s:/usr/bin:/opt/bin:g" > /etc/systemd/system/kubelet.service
|
|
mkdir -p /etc/systemd/system/kubelet.service.d
|
|
curl -sSL "https://raw.githubusercontent.com/kubernetes/kubernetes/${RELEASE}/build/debs/10-kubeadm.conf" | sed "s:/usr/bin:/opt/bin:g" > /etc/systemd/system/kubelet.service.d/10-kubeadm.conf
|
|
fi
|
|
|
|
systemctl enable kubelet && systemctl start kubelet
|
|
modprobe overlay && modprobe br_netfilter && sysctl net.bridge.bridge-nf-call-iptables=1
|
|
|
|
if [ -d "$BINARIES_DIR" ] && [ "$ATTEMPT_ONLINE_INSTALL" = true ]; then
|
|
crucial_cmd_attempts=1
|
|
while true; do
|
|
if (( "$crucial_cmd_attempts" > "$MAX_SETUP_CRUCIAL_CMD_ATTEMPTS" )); then
|
|
echo "Warning: kubeadm pull images failed after multiple tries!"
|
|
break;
|
|
fi
|
|
retval=0
|
|
set +e
|
|
kubeadm config images pull --cri-socket /run/containerd/containerd.sock
|
|
retval=$?
|
|
set -e
|
|
if [ $retval -eq 0 ]; then
|
|
break;
|
|
fi
|
|
crucial_cmd_attempts=$[$crucial_cmd_attempts + 1]
|
|
done
|
|
fi
|
|
|
|
- path: /opt/bin/deploy-kube-system
|
|
permissions: '0700'
|
|
owner: root:root
|
|
content: |
|
|
#!/bin/bash -e
|
|
|
|
if [[ -f "/home/cloud/success" ]]; then
|
|
echo "Already provisioned!"
|
|
exit 0
|
|
fi
|
|
|
|
if [[ "$PATH" != *:/opt/bin && "$PATH" != *:/opt/bin:* ]]; then
|
|
export PATH=$PATH:/opt/bin
|
|
fi
|
|
|
|
MAX_SETUP_CRUCIAL_CMD_ATTEMPTS=3
|
|
crucial_cmd_attempts=1
|
|
while true; do
|
|
if (( "$crucial_cmd_attempts" > "$MAX_SETUP_CRUCIAL_CMD_ATTEMPTS" )); then
|
|
echo "Error: kubeadm init failed!"
|
|
exit 1
|
|
fi
|
|
retval=0
|
|
set +e
|
|
kubeadm init --token {{ k8s_control_node.cluster.token }} --token-ttl 0 {{ k8s_control_node.cluster.initargs }} --cri-socket /run/containerd/containerd.sock
|
|
retval=$?
|
|
set -e
|
|
if [ $retval -eq 0 ]; then
|
|
break;
|
|
fi
|
|
crucial_cmd_attempts=$[$crucial_cmd_attempts + 1]
|
|
done
|
|
|
|
K8S_CONFIG_SCRIPTS_COPY_DIR=/tmp/k8sconfigscripts/
|
|
|
|
if [[ $(systemctl is-active setup-kube-system) != "inactive" ]]; then
|
|
echo "setup-kube-system is running!"
|
|
exit 1
|
|
fi
|
|
if [[ "$PATH" != *:/opt/bin && "$PATH" != *:/opt/bin:* ]]; then
|
|
export PATH=$PATH:/opt/bin
|
|
fi
|
|
export KUBECONFIG=/etc/kubernetes/admin.conf
|
|
|
|
mkdir -p /root/.kube
|
|
cp -i /etc/kubernetes/admin.conf /root/.kube/config
|
|
chown $(id -u):$(id -g) /root/.kube/config
|
|
echo export PATH=\$PATH:/opt/bin >> /root/.bashrc
|
|
|
|
if [ -d "$K8S_CONFIG_SCRIPTS_COPY_DIR" ]; then
|
|
### Network, dashboard configs available offline ###
|
|
echo "Offline configs are available!"
|
|
/opt/bin/kubectl apply -f ${K8S_CONFIG_SCRIPTS_COPY_DIR}/network.yaml
|
|
/opt/bin/kubectl apply -f ${K8S_CONFIG_SCRIPTS_COPY_DIR}/dashboard.yaml
|
|
rm -rf "${K8S_CONFIG_SCRIPTS_COPY_DIR}"
|
|
else
|
|
/opt/bin/kubectl apply -f "https://cloud.weave.works/k8s/net?k8s-version=$(/opt/bin/kubectl version | base64 | tr -d '\n')"
|
|
/opt/bin/kubectl apply -f https://raw.githubusercontent.com/kubernetes/dashboard/v2.0.0-beta6/aio/deploy/recommended.yaml
|
|
fi
|
|
|
|
/opt/bin/kubectl create rolebinding admin-binding --role=admin --user=admin || true
|
|
/opt/bin/kubectl create clusterrolebinding cluster-admin-binding --clusterrole=cluster-admin --user=admin || true
|
|
/opt/bin/kubectl create clusterrolebinding kubernetes-dashboard-ui --clusterrole=cluster-admin --serviceaccount=kubernetes-dashboard:kubernetes-dashboard || true
|
|
|
|
sudo touch /home/cloud/success
|
|
echo "true" > /home/cloud/success
|
|
|
|
- path: /opt/bin/setup-containerd
|
|
permissions: '0755'
|
|
owner: root:root
|
|
content: |
|
|
#!/bin/bash -e
|
|
|
|
export registryConfig="\\ [plugins.\"io.containerd.grpc.v1.cri\".registry.mirrors.\"{{registry.url.endpoint}}\"]\n \\ endpoint = [\"{{registry.url}}\"]"
|
|
export registryCredentials="\\ [plugins.\"io.containerd.grpc.v1.cri\".registry.configs.\"{{registry.url.endpoint}}\".auth]\n\tusername = \"{{registry.username}}\" \n\tpassword = \"{{registry.password}}\" \n\tidentitytoken = \"{{registry.token}}\""
|
|
|
|
echo "creating config file for containerd"
|
|
containerd config default > /etc/containerd/config.toml
|
|
sed -i '/\[plugins."io.containerd.grpc.v1.cri".registry\]/a '"${registryCredentials}"'' /etc/containerd/config.toml
|
|
sed -i '/\[plugins."io.containerd.grpc.v1.cri".registry.mirrors\]/a '"${registryConfig}"'' /etc/containerd/config.toml
|
|
|
|
echo "Restarting containerd service"
|
|
systemctl daemon-reload
|
|
systemctl restart containerd
|
|
|
|
- path: /etc/systemd/system/deploy-kube-system.service
|
|
permissions: '0755'
|
|
owner: root:root
|
|
content: |
|
|
[Unit]
|
|
Requires=containerd.service
|
|
After=containerd.service
|
|
|
|
[Service]
|
|
Type=simple
|
|
StartLimitInterval=0
|
|
Restart=on-failure
|
|
ExecStart=/opt/bin/deploy-kube-system
|
|
|
|
runcmd:
|
|
- chown -R cloud:cloud /home/cloud/.ssh
|
|
- containerd config default > /etc/containerd/config.toml
|
|
- sed -i 's/SystemdCgroup = false/SystemdCgroup = true/g' /etc/containerd/config.toml
|
|
- until [ -f /opt/bin/setup-kube-system ]; do sleep 5; done
|
|
- /opt/bin/setup-kube-system
|
|
- until [ -f /etc/systemd/system/deploy-kube-system.service ]; do sleep 5; done
|
|
- systemctl start deploy-kube-system
|