diff --git a/docker-multinode/README.md b/docker-multinode/README.md index fbcb14f96..7769196e3 100644 --- a/docker-multinode/README.md +++ b/docker-multinode/README.md @@ -24,13 +24,19 @@ This pattern is necessary because the `flannel` daemon is responsible for settin all of the Docker containers created by Kubernetes. To achieve this, it must run outside of the _main_ Docker daemon. However, it is still useful to use containers for deployment and management, so we create a simpler _bootstrap_ daemon to achieve this. +### Versions supported + +v1.2.x and v1.3.x are supported versions for this deployment. +v1.3.0 alphas and betas might work, but be sure you know what you're doing if you're trying them out. ### Multi-arch solution Yeah, it's true. You may run this deployment setup seamlessly on `amd64`, `arm`, `arm64` and `ppc64le` hosts. See this tracking issue for more details: https://github.com/kubernetes/kubernetes/issues/17981 -(Small note: arm is available with `v1.3.0-alpha.2+`. arm64 and ppc64le are available with `v1.3.0-alpha.3+`. Also ppc64le encountered problems in `v1.3.0-alpha.5+`, so ppc64le has only `v1.3.0-alpha.(3|4)` images pushed) +v1.3.0 ships with support for amd64, arm and arm64. ppc64le isn't supported, due to a bug in the Go runtime, `hyperkube` (only!) isn't built for the stable v1.3.0 release, and therefore this guide can't run it. But you may still run Kubernetes on ppc64le via custom deployments. + +hyperkube was pushed for ppc64le at versions `v1.3.0-alpha.3` and `v1.3.0-alpha.4`, feel free to try them out, but there might be some unexpected bugs. ### Options/configuration @@ -43,8 +49,6 @@ The scripts will output something like this when starting: +++ [0611 12:50:12] FLANNEL_IPMASQ is set to: true +++ [0611 12:50:12] FLANNEL_NETWORK is set to: 10.1.0.0/16 +++ [0611 12:50:12] FLANNEL_BACKEND is set to: udp -+++ [0611 12:50:12] DNS_DOMAIN is set to: cluster.local -+++ [0611 12:50:12] DNS_SERVER_IP is set to: 10.0.0.10 +++ [0611 12:50:12] RESTART_POLICY is set to: unless-stopped +++ [0611 12:50:12] MASTER_IP is set to: 192.168.1.50 +++ [0611 12:50:12] ARCH is set to: amd64 @@ -88,4 +92,23 @@ Then, the main docker daemon is restarted and lastly `kubelet` is launched as a ## Addons -kube-dns and the dashboard are deployed automatically with `v1.3.0-alpha.5` and over +kube-dns and the dashboard are deployed automatically with v1.3.0 + +### Deploy DNS manually for v1.2.x + +Just specify the architecture, and deploy via these commands: + +```console +# Possible options: amd64, arm, arm64 and ppc64le +$ export ARCH=amd64 + +# If the kube-system namespace isn't already created, create it +$ kubectl get ns +$ kubectl create namespace kube-system + +$ sed -e "s/ARCH/${ARCH}/g;" skydns.yaml | kubectl create -f - +``` + +### Test if DNS works + +Follow [this link](https://releases.k8s.io/release-1.2/cluster/addons/dns#how-do-i-test-if-it-is-working) to check it out. diff --git a/docker-multinode/common.sh b/docker-multinode/common.sh index d11fcb918..c1d4e5591 100755 --- a/docker-multinode/common.sh +++ b/docker-multinode/common.sh @@ -27,9 +27,6 @@ kube::multinode::main(){ FLANNEL_BACKEND=${FLANNEL_BACKEND:-"udp"} FLANNEL_NETWORK=${FLANNEL_NETWORK:-"10.1.0.0/16"} - DNS_DOMAIN=${DNS_DOMAIN:-"cluster.local"} - DNS_SERVER_IP=${DNS_SERVER_IP:-"10.0.0.10"} - RESTART_POLICY=${RESTART_POLICY:-"unless-stopped"} CURRENT_PLATFORM=$(kube::helpers::host_platform) @@ -42,7 +39,7 @@ kube::multinode::main(){ TIMEOUT_FOR_SERVICES=20 BOOTSTRAP_DOCKER_SOCK="unix:///var/run/docker-bootstrap.sock" KUBELET_MOUNTS="\ - -v /sys:/sys:ro \ + -v /sys:/sys:rw \ -v /var/run:/var/run:rw \ -v /var/lib/docker:/var/lib/docker:rw \ -v /var/lib/kubelet:/var/lib/kubelet:shared \ @@ -86,8 +83,6 @@ kube::multinode::check_params() { kube::log::status "FLANNEL_IPMASQ is set to: ${FLANNEL_IPMASQ}" kube::log::status "FLANNEL_NETWORK is set to: ${FLANNEL_NETWORK}" kube::log::status "FLANNEL_BACKEND is set to: ${FLANNEL_BACKEND}" - kube::log::status "DNS_DOMAIN is set to: ${DNS_DOMAIN}" - kube::log::status "DNS_SERVER_IP is set to: ${DNS_SERVER_IP}" kube::log::status "RESTART_POLICY is set to: ${RESTART_POLICY}" kube::log::status "MASTER_IP is set to: ${MASTER_IP}" kube::log::status "ARCH is set to: ${ARCH}" @@ -116,11 +111,11 @@ kube::multinode::detect_lsb() { case "${lsb_dist}" in amzn|centos|debian|ubuntu|systemd) - ;; + ;; *) - kube::log::error "Error: We currently only support ubuntu|debian|amzn|centos|systemd." - exit 1 - ;; + kube::log::error "Error: We currently only support ubuntu|debian|amzn|centos|systemd." + exit 1 + ;; esac kube::log::status "Detected OS: ${lsb_dist}" @@ -169,7 +164,15 @@ kube::multinode::start_etcd() { --data-dir=/var/etcd/data # Wait for etcd to come up - sleep 5 + local SECONDS=0 + while [[ $(curl -fs http://localhost:4001/v2/machines 2>&1 1>/dev/null; echo $?) != 0 ]]; do + ((SECONDS++)) + if [[ ${SECONDS} == ${TIMEOUT_FOR_SERVICES} ]]; then + kube::log::error "etcd failed to start. Exiting..." + exit + fi + sleep 1 + done # Set flannel net config docker -H ${BOOTSTRAP_DOCKER_SOCK} run \ @@ -224,6 +227,7 @@ kube::multinode::restart_docker(){ case "${lsb_dist}" in amzn) DOCKER_CONF="/etc/sysconfig/docker" + kube::helpers::backup_file ${DOCKER_CONF} if ! kube::helpers::command_exists ifconfig; then yum -y -q install net-tools @@ -232,12 +236,14 @@ kube::multinode::restart_docker(){ yum -y -q install bridge-utils fi - kube::helpers::file_replace_line ${DOCKER_CONF} `# Replace content in this file` \ - "--bip" `# Find a line with this content...` \ - "OPTIONS=\"\$OPTIONS --mtu=${FLANNEL_MTU} --bip=${FLANNEL_SUBNET}\"" `# ...and replace the found line with this line` + # Is there an uncommented OPTIONS line at all? + if [[ -z $(grep "OPTIONS" ${DOCKER_CONF} | grep -v "#") ]]; then + echo "OPTIONS=\"--mtu=${FLANNEL_MTU} --bip=${FLANNEL_SUBNET}\"" >> ${DOCKER_CONF} + else + kube::helpers::replace_mtu_bip ${DOCKER_CONF} "OPTIONS" + fi ifconfig docker0 down - brctl delbr docker0 service docker restart ;; @@ -254,10 +260,15 @@ kube::multinode::restart_docker(){ kube::multinode::restart_docker_systemd else DOCKER_CONF="/etc/sysconfig/docker" + kube::helpers::backup_file ${DOCKER_CONF} + + # Is there an uncommented OPTIONS line at all? + if [[ -z $(grep "OPTIONS" ${DOCKER_CONF} | grep -v "#") ]]; then + echo "OPTIONS=\"--mtu=${FLANNEL_MTU} --bip=${FLANNEL_SUBNET}\"" >> ${DOCKER_CONF} + else + kube::helpers::replace_mtu_bip ${DOCKER_CONF} "OPTIONS" + fi - kube::helpers::file_replace_line ${DOCKER_CONF} `# Replace content in this file` \ - "--bip" `# Find a line with this content...` \ - "OPTIONS=\"\$OPTIONS --mtu=${FLANNEL_MTU} --bip=${FLANNEL_SUBNET}\"" `# ...and replace the found line with this line` ifconfig docker0 down brctl delbr docker0 systemctl restart docker @@ -273,15 +284,19 @@ kube::multinode::restart_docker(){ kube::multinode::restart_docker_systemd else DOCKER_CONF="/etc/default/docker" + kube::helpers::backup_file ${DOCKER_CONF} - kube::helpers::file_replace_line ${DOCKER_CONF} `# Replace content in this file` \ - "--bip" `# Find a line with this content...` \ - "OPTIONS=\"\$OPTIONS --mtu=${FLANNEL_MTU} --bip=${FLANNEL_SUBNET}\"" `# ...and replace the found line with this line` + # Is there an uncommented DOCKER_OPTS line at all? + if [[ -z $(grep "DOCKER_OPTS" $DOCKER_CONF | grep -v "#") ]]; then + echo "DOCKER_OPTS=\"--mtu=${FLANNEL_MTU} --bip=${FLANNEL_SUBNET}\"" >> ${DOCKER_CONF} + else + kube::helpers::replace_mtu_bip ${DOCKER_CONF} "DOCKER_OPTS" + fi ifconfig docker0 down brctl delbr docker0 service docker stop - while [[ $(ps aux | grep /usr/bin/docker | grep -v grep | wc -l) -gt 0 ]]; do + while [[ $(ps aux | grep $(which docker) | grep -v grep | wc -l) -gt 0 ]]; do kube::log::status "Waiting for docker to terminate" sleep 1 done @@ -300,23 +315,13 @@ kube::multinode::restart_docker(){ kube::multinode::restart_docker_systemd(){ DOCKER_CONF=$(systemctl cat docker | head -1 | awk '{print $2}') - - # This expression checks if the "--mtu" and "--bip" options are there - # If they aren't, they are inserted at the end of the docker command - if [[ -z $(grep -- "--mtu=" $DOCKER_CONF) ]]; then - sed -e "s@$(grep "/usr/bin/docker" $DOCKER_CONF)@$(grep "/usr/bin/docker" $DOCKER_CONF) --mtu=${FLANNEL_MTU}@g" -i $DOCKER_CONF - fi - if [[ -z $(grep -- "--bip=" $DOCKER_CONF) ]]; then - sed -e "s@$(grep "/usr/bin/docker" $DOCKER_CONF)@$(grep "/usr/bin/docker" $DOCKER_CONF) --bip=${FLANNEL_SUBNET}@g" -i $DOCKER_CONF - fi + kube::helpers::backup_file ${DOCKER_CONF} + kube::helpers::replace_mtu_bip ${DOCKER_CONF} $(which docker) ifconfig docker0 down brctl delbr docker0 - # Finds "--mtu=????" and replaces with "--mtu=${FLANNEL_MTU}" - # Also finds "--bip=??.??.??.??" and replaces with "--bip=${FLANNEL_SUBNET}" - sed -e "s@$(grep -o -- "--mtu=[[:graph:]]*" $DOCKER_CONF)@--mtu=${FLANNEL_MTU}@g;s@$(grep -o -- "--bip=[[:graph:]]*" $DOCKER_CONF)@--bip=${FLANNEL_SUBNET}@g" -i $DOCKER_CONF - sed -i.bak 's/^\(MountFlags=\).*/\1shared/' $DOCKER_CONF + sed -i.bak 's/^\(MountFlags=\).*/\1shared/' ${DOCKER_CONF} systemctl daemon-reload systemctl restart docker } @@ -340,8 +345,8 @@ kube::multinode::start_k8s_master() { --allow-privileged \ --api-servers=http://localhost:8080 \ --config=/etc/kubernetes/manifests-multi \ - --cluster-dns=${DNS_SERVER_IP} \ - --cluster-domain=${DNS_DOMAIN} \ + --cluster-dns=10.0.0.10 \ + --cluster-domain=cluster.local \ --hostname-override=$(ip -o -4 addr list ${NET_INTERFACE} | awk '{print $4}' | cut -d/ -f1) \ --v=2 } @@ -365,8 +370,8 @@ kube::multinode::start_k8s_worker() { /hyperkube kubelet \ --allow-privileged \ --api-servers=http://${MASTER_IP}:8080 \ - --cluster-dns=${DNS_SERVER_IP} \ - --cluster-domain=${DNS_DOMAIN} \ + --cluster-dns=10.0.0.10 \ + --cluster-domain=cluster.local \ --hostname-override=$(ip -o -4 addr list ${NET_INTERFACE} | awk '{print $4}' | cut -d/ -f1) \ --v=2 } @@ -377,22 +382,22 @@ kube::multinode::start_k8s_worker_proxy() { # Some quite complex version checking here... # If the version is under v1.3.0-alpha.5, kube-proxy is run manually in this script # In v1.3.0-alpha.5 and above, kube-proxy is run in a DaemonSet - if [[ $((VERSION_MINOR < 3)) == 1 || \ - $((VERSION_MINOR <= 3)) == 1 && \ - $(echo ${VERSION_EXTRA}) != "" && \ - ${VERSION_PRERELEASE} == "alpha" && \ - $((VERSION_PRERELEASE_REV < 5)) == 1 ]]; then - - kube::log::status "Launching kube-proxy..." - docker run -d \ - --net=host \ - --privileged \ - --restart=${RESTART_POLICY} \ - gcr.io/google_containers/hyperkube-${ARCH}:${K8S_VERSION} \ - /hyperkube proxy \ - --master=http://${MASTER_IP}:8080 \ - --v=2 - fi + # This has been uncommented for now, since the DaemonSet was inactivated in the stable v1.3 release + #if [[ $((VERSION_MINOR < 3)) == 1 || \ + # $((VERSION_MINOR <= 3)) == 1 && \ + # $(echo ${VERSION_EXTRA}) != "" && \ + # ${VERSION_PRERELEASE} == "alpha" && \ + # $((VERSION_PRERELEASE_REV < 5)) == 1 ]]; then + + kube::log::status "Launching kube-proxy..." + docker run -d \ + --net=host \ + --privileged \ + --restart=${RESTART_POLICY} \ + gcr.io/google_containers/hyperkube-${ARCH}:${K8S_VERSION} \ + /hyperkube proxy \ + --master=http://${MASTER_IP}:8080 \ + --v=2 } # Turndown the local cluster @@ -433,10 +438,11 @@ kube::multinode::turndown(){ read -p "Should we stop the other containers that are running too? [Y/n] " stop_containers case $stop_containers in - [n|N]*) + [nN]*) ;; # Do nothing *) docker kill $(docker ps -q) + ;; esac fi @@ -444,16 +450,22 @@ kube::multinode::turndown(){ read -p "Do you want to clean /var/lib/kubelet? [Y/n] " clean_kubelet_dir case $clean_kubelet_dir in - [n|N]*) + [nN]*) ;; # Do nothing *) # umount if there are mounts in /var/lib/kubelet if [[ ! -z $(mount | grep /var/lib/kubelet | awk '{print $3}') ]]; then - umount $(mount | grep /var/lib/kubelet | awk '{print $3}') + + # The umount command may be a little bit subborn sometimes, so run the commands twice to ensure the mounts are gone + mount | grep /var/lib/kubelet/* | awk '{print $3}' | xargs umount 1>/dev/null 2>/dev/null + mount | grep /var/lib/kubelet/* | awk '{print $3}' | xargs umount 1>/dev/null 2>/dev/null + umount /var/lib/kubelet 1>/dev/null 2>/dev/null + umount /var/lib/kubelet 1>/dev/null 2>/dev/null fi # Delete the directory rm -rf /var/lib/kubelet + ;; esac fi } @@ -476,6 +488,28 @@ kube::helpers::file_replace_line(){ fi } +kube::helpers::replace_mtu_bip(){ + local DOCKER_CONF=$1 + local SEARCH_FOR=$2 + + # Assuming is a $SEARCH_FOR statement already, and we should append the options if they do not exist + if [[ -z $(grep -- "--mtu=" $DOCKER_CONF) ]]; then + sed -e "s@$(grep "$SEARCH_FOR" $DOCKER_CONF)@$(grep "$SEARCH_FOR" $DOCKER_CONF) --mtu=${FLANNEL_MTU}@g" -i $DOCKER_CONF + fi + if [[ -z $(grep -- "--bip=" $DOCKER_CONF) ]]; then + sed -e "s@$(grep "$SEARCH_FOR" $DOCKER_CONF)@$(grep "$SEARCH_FOR" $DOCKER_CONF) --bip=${FLANNEL_SUBNET}@g" -i $DOCKER_CONF + fi + + # Finds "--mtu=????" and replaces with "--mtu=${FLANNEL_MTU}" + # Also finds "--bip=??.??.??.??" and replaces with "--bip=${FLANNEL_SUBNET}" + sed -e "s@$(grep -o -- "--mtu=[[:graph:]]*" $DOCKER_CONF)@--mtu=${FLANNEL_MTU}@g;s@$(grep -o -- "--bip=[[:graph:]]*" $DOCKER_CONF)@--bip=${FLANNEL_SUBNET}@g" -i $DOCKER_CONF +} + +kube::helpers::backup_file(){ + # Backup the current file + cp -f ${1} ${1}.backup +} + # Check if a process is running kube::helpers::is_running(){ if [[ ! -z $(ps aux | grep ${1} | grep -v grep) ]]; then diff --git a/docker-multinode/deployDNS.md b/docker-multinode/deployDNS.md deleted file mode 100644 index dac0e3163..000000000 --- a/docker-multinode/deployDNS.md +++ /dev/null @@ -1,31 +0,0 @@ -### Get the template file - -First of all, download the dns template - -[skydns template](skydns.yaml.in) - -### Set environment variables - -Then you need to set `ARCH` `DNS_REPLICAS`, `DNS_DOMAIN` and `DNS_SERVER_IP` envs - -```shell -# Supported: amd64, arm, arm64 and ppc64le -export ARCH=amd64 -export DNS_REPLICAS=1 -export DNS_DOMAIN=cluster.local -export DNS_SERVER_IP=10.0.0.10 -``` - -### Replace the corresponding value in the template and create the pod - -```shell -# If the kube-system namespace isn't already created, create it -$ kubectl get ns -$ kubectl create namespace kube-system - -$ sed -e "s/ARCH/${ARCH}/g;s/DNS_REPLICAS/${DNS_REPLICAS}/g;s/DNS_DOMAIN/${DNS_DOMAIN}/g;s/DNS_SERVER_IP/${DNS_SERVER_IP}/g" skydns.yaml.in | kubectl create -f - -``` - -### Test if DNS works - -Follow [this link](https://releases.k8s.io/release-1.2/cluster/addons/dns#how-do-i-test-if-it-is-working) to check it out. diff --git a/docker-multinode/skydns.yaml.in b/docker-multinode/skydns.yaml similarity index 89% rename from docker-multinode/skydns.yaml.in rename to docker-multinode/skydns.yaml index a4553527e..26fa8f72b 100644 --- a/docker-multinode/skydns.yaml.in +++ b/docker-multinode/skydns.yaml @@ -1,3 +1,5 @@ +#### IMPORTANT: This dns manifest is for v1.2.x only! With v1.3, dns is deployed automatically + apiVersion: v1 kind: ReplicationController metadata: @@ -8,7 +10,7 @@ metadata: version: v8 kubernetes.io/cluster-service: "true" spec: - replicas: DNS_REPLICAS + replicas: 1 selector: k8s-app: kube-dns version: v8 @@ -47,7 +49,7 @@ spec: memory: 50Mi args: # command = "/kube2sky" - - --domain=DNS_DOMAIN + - --domain=cluster.local - name: skydns image: gcr.io/google_containers/skydns-ARCH:1.0 resources: @@ -58,7 +60,7 @@ spec: # command = "/skydns" - -machines=http://localhost:4001 - -addr=0.0.0.0:53 - - -domain=DNS_DOMAIN. + - -domain=cluster.local. ports: - containerPort: 53 name: dns @@ -80,7 +82,7 @@ spec: cpu: 10m memory: 20Mi args: - - -cmd=nslookup kubernetes.default.svc.DNS_DOMAIN localhost >/dev/null + - -cmd=nslookup kubernetes.default.svc.cluster.local localhost >/dev/null - -port=8080 ports: - containerPort: 8080 @@ -102,7 +104,7 @@ metadata: spec: selector: k8s-app: kube-dns - clusterIP: DNS_SERVER_IP + clusterIP: 10.0.0.10 ports: - name: dns port: 53