deploy/demo.sh
#!/bin/bash
set -o errexit
set -o nounset
set -o pipefail
set -o errtrace
KUBE_VERSION="${KUBE_VERSION:-1.14}"
CRIPROXY_DEB_URL="${CRIPROXY_DEB_URL:-https://github.com/Mirantis/criproxy/releases/download/v0.14.0/criproxy-nodeps_0.14.0_amd64.deb}"
NONINTERACTIVE="${NONINTERACTIVE:-}"
NO_VM_CONSOLE="${NO_VM_CONSOLE:-}"
INJECT_LOCAL_IMAGE="${INJECT_LOCAL_IMAGE:-}"
KDC_VERSION="${KDC_VERSION:-v0.2.0}"
KDC_BASE_LOCATION="${KDC_BASE_LOCATION:-https://github.com/kubernetes-sigs/kubeadm-dind-cluster/releases/download}"
kdc_script="dind-cluster-v${KUBE_VERSION}.sh"
kdc_url="${KDC_BASE_LOCATION}/${KDC_VERSION}/${kdc_script}"
kubectl="${HOME}/.kubeadm-dind-cluster/kubectl"
BASE_LOCATION="${BASE_LOCATION:-https://raw.githubusercontent.com/Mirantis/virtlet/master/}"
RELEASE_LOCATION="${RELEASE_LOCATION:-https://github.com/Mirantis/virtlet/releases/download/}"
VIRTLET_DEMO_RELEASE="${VIRTLET_DEMO_RELEASE:-}"
VIRTLET_DEMO_BRANCH="${VIRTLET_DEMO_BRANCH:-}"
VIRTLET_ON_MASTER="${VIRTLET_ON_MASTER:-}"
VIRTLET_MULTI_NODE="${VIRTLET_MULTI_NODE:-}"
IMAGE_REGEXP_TRANSLATION="${IMAGE_REGEXP_TRANSLATION:-1}"
MULTI_CNI="${MULTI_CNI:-}"
DEMO_LOG_LEVEL="${DEMO_LOG_LEVEL:-}"
DIND_CRI="${DIND_CRI:-containerd}"
# Convenience setting for local testing:
# BASE_LOCATION="${HOME}/work/kubernetes/src/github.com/Mirantis/virtlet"
cirros_key="demo-cirros-private-key"
# just initialize it
declare virtlet_release
declare virtlet_docker_tag
virtlet_nodes=()
if [[ ${VIRTLET_ON_MASTER} ]]; then
virtlet_nodes+=(kube-master)
fi
if [[ !${VIRTLET_ON_MASTER} || ${VIRTLET_MULTI_NODE} ]]; then
virtlet_nodes+=(kube-node-1)
fi
if [[ ${VIRTLET_MULTI_NODE} ]]; then
virtlet_nodes+=(kube-node-2)
fi
# In case of linuxkit / moby linux, -v will not work so we can't
# mount /lib/modules and /boot.
using_linuxkit=
if ! docker info|grep -s '^Operating System: .*Docker for Windows' > /dev/null 2>&1 ; then
if docker info|grep -s '^Kernel Version: .*-moby$' >/dev/null 2>&1 ||
docker info|grep -s '^Kernel Version: .*-linuxkit-' > /dev/null 2>&1 ; then
using_linuxkit=1
fi
fi
function demo::step {
local OPTS=""
if [ "$1" = "-n" ]; then
shift
OPTS+="-n"
fi
GREEN="$1"
shift
if [ -t 2 ] ; then
echo -e ${OPTS} "\x1B[97m* \x1B[92m${GREEN}\x1B[39m $*" >&2
else
echo ${OPTS} "* ${GREEN} $*" >&2
fi
}
function demo::ask-before-continuing {
if [[ ! ${NONINTERACTIVE} ]]; then
echo "Press Enter to continue or Ctrl-C to stop." >&2
read
fi
}
function demo::ask-user {
if [[ ${1:-} = "" ]]; then
echo "no prompt message provided" >&2
exit 1
fi
if [[ ${2:-} = "" ]]; then
echo "no return var name provided" >&2
exit 1
fi
local __resultvar=$2
local reply="false"
while true; do
read -p "$(tput bold)$(tput setaf 3) ${1} (yY/nN): $(tput sgr0)" reply
case $reply in
Y|y) reply="true"; break;;
N|n) echo "Abort"; reply="false"; break;;
*) echo "Please answer y[Y] or n[N].";;
esac
done
eval $__resultvar="'$reply'"
}
function demo::get-dind-cluster {
download="true"
if [[ -f ${kdc_script} ]]; then
demo::step "Will update ${kdc_script} script to the latest version"
if [[ ! ${NONINTERACTIVE} ]]; then
demo::ask-user "Do you want to redownload ${kdc_script} ?" download
if [[ ${download} = "true" ]]; then
rm "${kdc_script}"
fi
else
demo::step "Will now clear existing ${kdc_script}"
rm "${kdc_script}"
fi
fi
if [[ ${download} = "true" ]]; then
demo::step "Will download ${kdc_script} into current directory"
demo::ask-before-continuing
wget -O "${kdc_script}" "${kdc_url}"
chmod +x "${kdc_script}"
fi
}
function demo::get-cirros-ssh-keys {
if [[ -f ${cirros_key} ]]; then
return 0
fi
demo::step "Will download ${cirros_key} into current directory"
wget -O ${cirros_key} "https://raw.githubusercontent.com/Mirantis/virtlet/${virtlet_release}/examples/vmkey"
chmod 600 ${cirros_key}
}
function demo::start-dind-cluster {
demo::step "Will now clear any kubeadm-dind-cluster data on the current Docker"
if [[ ! ${NONINTERACTIVE} ]]; then
echo "Cirros ssh connection will be open after Virtlet setup is complete, press Ctrl-D to disconnect." >&2
fi
echo "To clean up the cluster, use './dind-cluster-v${KUBE_VERSION}.sh clean'" >&2
demo::ask-before-continuing
"./${kdc_script}" clean
"./${kdc_script}" up
}
function demo::jq-patch {
local node="${1}"
local expr="${2}"
local filename="${3}"
docker exec "${node}" \
bash -c "jq '${expr}' '${filename}' >/tmp/jqpatch.tmp && mv /tmp/jqpatch.tmp '${filename}'"
}
function demo::install-cni-genie {
"${kubectl}" apply -f https://docs.projectcalico.org/v2.6/getting-started/kubernetes/installation/hosted/kubeadm/1.6/calico.yaml
demo::wait-for "Calico etcd" demo::pods-ready k8s-app=calico-etcd
demo::wait-for "Calico node" demo::pods-ready k8s-app=calico-node
"${kubectl}" apply -f https://raw.githubusercontent.com/Huawei-PaaS/CNI-Genie/master/conf/1.8/genie-plugin.yaml
demo::wait-for "CNI Genie" demo::pods-ready k8s-app=genie
demo::jq-patch kube-node-1 '.cniVersion="0.3.0"|.default_plugin="calico,flannel"' /etc/cni/net.d/00-genie.conf
demo::jq-patch kube-node-1 '.cniVersion="0.3.0"' /etc/cni/net.d/10-calico.conf
demo::jq-patch kube-node-1 '.cniVersion="0.3.0"' /etc/cni/net.d/10-flannel.conflist
}
function demo::install-cri-proxy {
local virtlet_node="${1}"
demo::step "Installing CRI proxy package on ${virtlet_node} container"
if [[ ${DIND_CRI} = containerd ]]; then
docker exec "${virtlet_node}" /bin/bash -c 'echo criproxy-nodeps criproxy/primary_cri select containerd | debconf-set-selections'
fi
docker exec "${virtlet_node}" /bin/bash -c "curl -sSL '${CRIPROXY_DEB_URL}' >/criproxy.deb && DEBIAN_FRONTEND=noninteractive dpkg -i /criproxy.deb && rm /criproxy.deb"
}
function demo::fix-mounts {
local virtlet_node="${1}"
demo::step "Marking mounts used by virtlet as shared in ${virtlet_node} container"
docker exec "${virtlet_node}" mount --make-shared /dind
docker exec "${virtlet_node}" mount --make-shared /dev
if [[ ! ${using_linuxkit} ]]; then
docker exec "${virtlet_node}" mount --make-shared /boot
fi
docker exec "${virtlet_node}" mount --make-shared /sys/fs/cgroup
demo::step "Bind-mounting /var/lib/virtlet from a docker volume"
docker exec "${virtlet_node}" mkdir -p /dind/virtlet /var/lib/virtlet
docker exec "${virtlet_node}" mount --bind /dind/virtlet /var/lib/virtlet
}
function demo::inject-local-image {
local virtlet_node="${1}"
demo::step "Copying local mirantis/virtlet image into ${virtlet_node} container"
docker save mirantis/virtlet |
if [[ ${DIND_CRI} = containerd ]]; then
docker exec -i "${virtlet_node}" ctr -n k8s.io images import -
else
docker exec -i "${virtlet_node}" docker load
fi
}
function demo::label-and-untaint-node {
local virtlet_node="${1}"
demo::step "Applying label to ${virtlet_node}:" "extraRuntime=virtlet"
"${kubectl}" label node "${virtlet_node}" extraRuntime=virtlet
if [[ ${VIRTLET_ON_MASTER} ]]; then
demo::step "Checking/removing master taint from ${virtlet_node}"
if [[ $("${kubectl}" get node kube-master -o jsonpath='{.spec.taints[?(@.key=="node-role.kubernetes.io/master")]}') ]]; then
"${kubectl}" taint nodes kube-master node-role.kubernetes.io/master-
fi
fi
}
function demo::pods-ready {
local label="$1"
local out
if ! out="$("${kubectl}" get pod -l "${label}" -n kube-system \
-o jsonpath='{ .items[*].status.conditions[?(@.type == "Ready")].status }' 2>/dev/null)"; then
return 1
fi
if ! grep -v False <<<"${out}" | grep -q True; then
return 1
fi
return 0
}
function demo::service-ready {
local name="$1"
if ! "${kubectl}" describe service -n kube-system "${name}"|grep -q '^Endpoints:.*[0-9]\.'; then
return 1
fi
}
function demo::wait-for {
local title="$1"
local action="$2"
local what="$3"
shift 3
demo::step "Waiting for:" "${title}"
while ! "${action}" "${what}" "$@"; do
echo -n "." >&2
sleep 1
done
echo "[done]" >&2
}
virtlet_pod=
function demo::virsh {
local opts=
if [[ ${1:-} = "console" ]]; then
# using -it with `virsh list` causes it to use \r\n as line endings,
# which makes it less useful
local opts="-it"
fi
if [[ ! ${virtlet_pod} ]]; then
virtlet_pod=$("${kubectl}" get pods -n kube-system -l runtime=virtlet -o name|head -1|sed 's@.*/@@')
fi
"${kubectl}" exec ${opts} -n kube-system "${virtlet_pod}" -c virtlet -- virsh "$@"
}
function demo::ssh {
local cirros_ip=
demo::get-cirros-ssh-keys
if [[ ! ${virtlet_pod} ]]; then
virtlet_pod=$("${kubectl}" get pods -n kube-system -l runtime=virtlet -o name|head -1|sed 's@.*/@@')
fi
if [[ ! ${cirros_ip} ]]; then
while true; do
cirros_ip=$("${kubectl}" get pod cirros-vm -o jsonpath="{.status.podIP}")
if [[ ! ${cirros_ip} ]]; then
echo "Waiting for cirros IP..."
sleep 1
continue
fi
echo "Cirros IP is ${cirros_ip}."
break
done
fi
echo "Trying to establish ssh connection to cirros-vm..."
while ! internal::ssh ${virtlet_pod} ${cirros_ip} "echo Hello" | grep -q "Hello"; do
sleep 1
echo "Trying to establish ssh connection to cirros-vm..."
done
echo "Successfully established ssh connection. Press Ctrl-D to disconnect."
internal::ssh ${virtlet_pod} ${cirros_ip}
}
function internal::ssh {
virtlet_pod=${1}
cirros_ip=${2}
shift 2
ssh -oProxyCommand="${kubectl} exec -i -n kube-system ${virtlet_pod} -c virtlet -- nc -q0 ${cirros_ip} 22" \
-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -q \
-i ${cirros_key} cirros@cirros-vm "$@"
}
function demo::vm-ready {
local name="$1"
# note that the following is not a bulletproof check
if ! demo::virsh list --name | grep -q "${name}\$"; then
return 1
fi
}
function demo::kvm-ok {
demo::step "Checking for KVM support..."
# The check is done inside the node container because it has proper /lib/modules
# from the docker host. Also, it'll have to use mirantis/virtlet image
# later anyway.
if [[ ${using_linuxkit} ]]; then
return 1
fi
# use kube-master node as all of the DIND nodes in the cluster are similar
if ! docker exec kube-master docker run --privileged --rm -v /lib/modules:/lib/modules "mirantis/virtlet:${virtlet_docker_tag}" kvm-ok; then
return 1
fi
}
function demo::get-correct-virtlet-release {
# will use most recently published virtlet release
# (virtlet releases are pre-releases now so not returned in /latest)
local __resultvar=$1
local jq_filter=".[0].tag_name"
local last_release
last_release=$(curl --silent https://api.github.com/repos/Mirantis/virtlet/releases | docker exec -i kube-master jq "${jq_filter}" | sed 's/^\"\(.*\)\"$/\1/')
if [[ $__resultvar ]]; then
eval $__resultvar="'$last_release'"
else
echo "$last_release"
fi
}
function demo::start-virtlet {
local -a virtlet_config=(--from-literal=download_protocol=http --from-literal=image_regexp_translation="$IMAGE_REGEXP_TRANSLATION")
if [[ ${VIRTLET_DEMO_BRANCH} ]]; then
if [[ ${VIRTLET_DEMO_BRANCH} = "master" ]]; then
virtlet_release="master"
virtlet_docker_tag="latest"
else
virtlet_release="${VIRTLET_DEMO_BRANCH}"
virtlet_docker_tag=$(echo $VIRTLET_DEMO_BRANCH | sed -e "s/\//_/g")
BASE_LOCATION="https://raw.githubusercontent.com/Mirantis/virtlet/${virtlet_release}/"
fi
else
if [[ ${VIRTLET_DEMO_RELEASE} ]]; then
virtlet_release="${VIRTLET_DEMO_RELEASE}"
else
demo::get-correct-virtlet-release virtlet_release
fi
# set correct urls and names
virtlet_docker_tag="${virtlet_release}"
BASE_LOCATION="https://raw.githubusercontent.com/Mirantis/virtlet/${virtlet_release}/"
fi
echo "Will run demo using Virtlet:${virtlet_release} for demo and ${virtlet_docker_tag} as docker tag"
if demo::kvm-ok; then
demo::step "Setting up Virtlet configuration with KVM support"
else
demo::step "Setting up Virtlet configuration *without* KVM support"
virtlet_config+=(--from-literal=disable_kvm=y)
fi
"${kubectl}" create configmap -n kube-system virtlet-config "${virtlet_config[@]}"
# new functionality added post 0.8.2
# that logic could be removed later
if [[ ${BASE_LOCATION} == https://* ]]; then
# remote location so fetch file
rm -f demo_images.yaml
status_code=$(curl -w "%{http_code}" --silent -o demo_images.yaml "${BASE_LOCATION}"/deploy/images.yaml)
if [[ $status_code == "200" ]]; then
"${kubectl}" create configmap -n kube-system virtlet-image-translations --from-file demo_images.yaml
fi
else
if [[ -f ${BASE_LOCATION}/deploy/images.yaml ]]; then
"${kubectl}" create configmap -n kube-system virtlet-image-translations --from-file "${BASE_LOCATION}/deploy/images.yaml"
fi
fi
if [[ ${DEMO_LOG_LEVEL} ]]; then
demo::step "Deploying Virtlet CRDs"
docker run --rm "mirantis/virtlet:${virtlet_docker_tag}" virtletctl gen --crd |
"${kubectl}" apply -f -
"${kubectl}" apply -f - <<EOF
---
apiVersion: "virtlet.k8s/v1"
kind: VirtletConfigMapping
metadata:
name: demo-log-level
namespace: kube-system
spec:
config:
logLevel: ${DEMO_LOG_LEVEL}
EOF
fi
demo::step "Deploying Virtlet DaemonSet with docker tag ${virtlet_docker_tag}"
docker run --rm "mirantis/virtlet:${virtlet_docker_tag}" virtletctl gen --tag "${virtlet_docker_tag}" |
"${kubectl}" apply -f -
demo::wait-for "Virtlet DaemonSet" demo::pods-ready runtime=virtlet
}
function demo::start-nginx {
"${kubectl}" run nginx --image=nginx --expose --port 80
}
function demo::start-vm {
demo::step "Starting sample CirrOS VM"
"${kubectl}" create -f "${BASE_LOCATION}/examples/cirros-vm.yaml"
demo::wait-for "CirrOS VM" demo::vm-ready cirros-vm
if [[ ! "${NO_VM_CONSOLE:-}" ]]; then
demo::step "Establishing ssh connection to the VM. Use Ctrl-D to disconnect"
demo::ssh
fi
}
if [[ ${1:-} = "--help" || ${1:-} = "-h" ]]; then
cat <<EOF >&2
Usage: ./demo.sh
This script runs a simple demo of Virtlet[1] using kubeadm-dind-cluster[2]
ssh connection will be established after Virtlet setup is complete, Ctrl-D
can be used to disconnect from it.
Use 'curl http://nginx.default.svc.cluster.local' from VM console to test
cluster networking.
To clean up the cluster, use './dind-cluster-v${KUBE_VERSION}.sh clean'
[1] https://github.com/Mirantis/virtlet
[2] https://github.com/kubernetes-sigs/kubeadm-dind-cluster
EOF
exit 0
fi
demo::get-dind-cluster
if [[ ${MULTI_CNI} ]]; then
export NUM_NODES=1
export CNI_PLUGIN=flannel
fi
demo::start-dind-cluster
for virtlet_node in "${virtlet_nodes[@]}"; do
demo::fix-mounts "${virtlet_node}"
demo::install-cri-proxy "${virtlet_node}"
if [[ ${INJECT_LOCAL_IMAGE:-} ]]; then
demo::inject-local-image "${virtlet_node}"
fi
demo::label-and-untaint-node "${virtlet_node}"
done
if [[ ${MULTI_CNI} ]]; then
demo::install-cni-genie
fi
demo::start-virtlet
demo::start-nginx
demo::start-vm