Make docker multinode ready for the v1.3 release and update docs

kubernetes-retired · Jul 2, 2016 · eb1f29a · eb1f29a
1 parent 6eeedc9
commit eb1f29a
Show file tree

Hide file tree

Showing 4 changed files with 126 additions and 98 deletions.
diff --git a/docker-multinode/README.md b/docker-multinode/README.md
@@ -24,13 +24,19 @@ This pattern is necessary because the `flannel` daemon is responsible for settin
 all of the Docker containers created by Kubernetes. To achieve this, it must run outside of the _main_ Docker daemon. However,
 it is still useful to use containers for deployment and management, so we create a simpler _bootstrap_ daemon to achieve this.
 
+### Versions supported
+
+v1.2.x and v1.3.x are supported versions for this deployment.
+v1.3.0 alphas and betas might work, but be sure you know what you're doing if you're trying them out.
 
 ### Multi-arch solution
 
 Yeah, it's true. You may run this deployment setup seamlessly on `amd64`, `arm`, `arm64` and `ppc64le` hosts.
 See this tracking issue for more details: https://github.com/kubernetes/kubernetes/issues/17981
 
-(Small note: arm is available with `v1.3.0-alpha.2+`. arm64 and ppc64le are available with `v1.3.0-alpha.3+`. Also ppc64le encountered problems in `v1.3.0-alpha.5+`, so ppc64le has only `v1.3.0-alpha.(3|4)` images pushed)
+v1.3.0 ships with support for amd64, arm and arm64. ppc64le isn't supported, due to a bug in the Go runtime, `hyperkube` (only!) isn't built for the stable v1.3.0 release, and therefore this guide can't run it. But you may still run Kubernetes on ppc64le via custom deployments.
+
+hyperkube was pushed for ppc64le at versions `v1.3.0-alpha.3` and `v1.3.0-alpha.4`, feel free to try them out, but there might be some unexpected bugs.
 
 ### Options/configuration
 
@@ -43,8 +49,6 @@ The scripts will output something like this when starting:
 +++ [0611 12:50:12] FLANNEL_IPMASQ is set to: true
 +++ [0611 12:50:12] FLANNEL_NETWORK is set to: 10.1.0.0/16
 +++ [0611 12:50:12] FLANNEL_BACKEND is set to: udp
-+++ [0611 12:50:12] DNS_DOMAIN is set to: cluster.local
-+++ [0611 12:50:12] DNS_SERVER_IP is set to: 10.0.0.10
 +++ [0611 12:50:12] RESTART_POLICY is set to: unless-stopped
 +++ [0611 12:50:12] MASTER_IP is set to: 192.168.1.50
 +++ [0611 12:50:12] ARCH is set to: amd64
@@ -88,4 +92,23 @@ Then, the main docker daemon is restarted and lastly `kubelet` is launched as a
 
 ## Addons
 
-kube-dns and the dashboard are deployed automatically with `v1.3.0-alpha.5` and over
+kube-dns and the dashboard are deployed automatically with v1.3.0
+
+### Deploy DNS manually for v1.2.x
+
+Just specify the architecture, and deploy via these commands:
+
+```console
+# Possible options: amd64, arm, arm64 and ppc64le
+$ export ARCH=amd64
+
+# If the kube-system namespace isn't already created, create it
+$ kubectl get ns
+$ kubectl create namespace kube-system
+
+$ sed -e "s/ARCH/${ARCH}/g;" skydns.yaml | kubectl create -f -
+```
+
+### Test if DNS works
+
+Follow [this link](https://releases.k8s.io/release-1.2/cluster/addons/dns#how-do-i-test-if-it-is-working) to check it out.
diff --git a/docker-multinode/common.sh b/docker-multinode/common.sh
@@ -27,9 +27,6 @@ kube::multinode::main(){
   FLANNEL_BACKEND=${FLANNEL_BACKEND:-"udp"}
   FLANNEL_NETWORK=${FLANNEL_NETWORK:-"10.1.0.0/16"}
 
-  DNS_DOMAIN=${DNS_DOMAIN:-"cluster.local"}
-  DNS_SERVER_IP=${DNS_SERVER_IP:-"10.0.0.10"}
-
   RESTART_POLICY=${RESTART_POLICY:-"unless-stopped"}
 
   CURRENT_PLATFORM=$(kube::helpers::host_platform)
@@ -42,7 +39,7 @@ kube::multinode::main(){
   TIMEOUT_FOR_SERVICES=20
   BOOTSTRAP_DOCKER_SOCK="unix:///var/run/docker-bootstrap.sock"
   KUBELET_MOUNTS="\
-    -v /sys:/sys:ro \
+    -v /sys:/sys:rw \
     -v /var/run:/var/run:rw \
     -v /var/lib/docker:/var/lib/docker:rw \
     -v /var/lib/kubelet:/var/lib/kubelet:shared \
@@ -86,8 +83,6 @@ kube::multinode::check_params() {
   kube::log::status "FLANNEL_IPMASQ is set to: ${FLANNEL_IPMASQ}"
   kube::log::status "FLANNEL_NETWORK is set to: ${FLANNEL_NETWORK}"
   kube::log::status "FLANNEL_BACKEND is set to: ${FLANNEL_BACKEND}"
-  kube::log::status "DNS_DOMAIN is set to: ${DNS_DOMAIN}"
-  kube::log::status "DNS_SERVER_IP is set to: ${DNS_SERVER_IP}"
   kube::log::status "RESTART_POLICY is set to: ${RESTART_POLICY}"
   kube::log::status "MASTER_IP is set to: ${MASTER_IP}"
   kube::log::status "ARCH is set to: ${ARCH}"
@@ -116,11 +111,11 @@ kube::multinode::detect_lsb() {
 
   case "${lsb_dist}" in
       amzn|centos|debian|ubuntu|systemd)
-          ;;
+        ;;
       *)
-          kube::log::error "Error: We currently only support ubuntu|debian|amzn|centos|systemd."
-          exit 1
-          ;;
+        kube::log::error "Error: We currently only support ubuntu|debian|amzn|centos|systemd."
+        exit 1
+        ;;
   esac
 
   kube::log::status "Detected OS: ${lsb_dist}"
@@ -169,7 +164,15 @@ kube::multinode::start_etcd() {
       --data-dir=/var/etcd/data
 
   # Wait for etcd to come up
-  sleep 5
+  local SECONDS=0
+  while [[ $(curl -fs http://localhost:4001/v2/machines 2>&1 1>/dev/null; echo $?) != 0 ]]; do
+    ((SECONDS++))
+    if [[ ${SECONDS} == ${TIMEOUT_FOR_SERVICES} ]]; then
+      kube::log::error "etcd failed to start. Exiting..."
+      exit
+    fi
+    sleep 1
+  done
 
   # Set flannel net config
   docker -H ${BOOTSTRAP_DOCKER_SOCK} run \
@@ -224,6 +227,7 @@ kube::multinode::restart_docker(){
   case "${lsb_dist}" in
     amzn)
       DOCKER_CONF="/etc/sysconfig/docker"
+      kube::helpers::backup_file ${DOCKER_CONF}
 
       if ! kube::helpers::command_exists ifconfig; then
         yum -y -q install net-tools
@@ -232,12 +236,14 @@ kube::multinode::restart_docker(){
         yum -y -q install bridge-utils
       fi
 
-      kube::helpers::file_replace_line ${DOCKER_CONF} `# Replace content in this file` \
-        "--bip" `# Find a line with this content...` \
-        "OPTIONS=\"\$OPTIONS --mtu=${FLANNEL_MTU} --bip=${FLANNEL_SUBNET}\"" `# ...and replace the found line with this line`
+      # Is there an uncommented OPTIONS line at all?
+      if [[ -z $(grep "OPTIONS" ${DOCKER_CONF} | grep -v "#") ]]; then
+        echo "OPTIONS=\"--mtu=${FLANNEL_MTU} --bip=${FLANNEL_SUBNET}\"" >> ${DOCKER_CONF}
+      else
+        kube::helpers::replace_mtu_bip ${DOCKER_CONF} "OPTIONS"
+      fi
 
       ifconfig docker0 down
-
       brctl delbr docker0 
       service docker restart
       ;;
@@ -254,10 +260,15 @@ kube::multinode::restart_docker(){
         kube::multinode::restart_docker_systemd
       else
         DOCKER_CONF="/etc/sysconfig/docker"
+        kube::helpers::backup_file ${DOCKER_CONF}
+
+        # Is there an uncommented OPTIONS line at all?
+        if [[ -z $(grep "OPTIONS" ${DOCKER_CONF} | grep -v "#") ]]; then
+          echo "OPTIONS=\"--mtu=${FLANNEL_MTU} --bip=${FLANNEL_SUBNET}\"" >> ${DOCKER_CONF}
+        else
+          kube::helpers::replace_mtu_bip ${DOCKER_CONF} "OPTIONS"
+        fi
 
-        kube::helpers::file_replace_line ${DOCKER_CONF} `# Replace content in this file` \
-          "--bip" `# Find a line with this content...` \
-          "OPTIONS=\"\$OPTIONS --mtu=${FLANNEL_MTU} --bip=${FLANNEL_SUBNET}\"" `# ...and replace the found line with this line`
         ifconfig docker0 down
         brctl delbr docker0 
         systemctl restart docker
@@ -273,15 +284,19 @@ kube::multinode::restart_docker(){
         kube::multinode::restart_docker_systemd
       else
         DOCKER_CONF="/etc/default/docker"
+        kube::helpers::backup_file ${DOCKER_CONF}
 
-        kube::helpers::file_replace_line ${DOCKER_CONF} `# Replace content in this file` \
-          "--bip" `# Find a line with this content...` \
-          "OPTIONS=\"\$OPTIONS --mtu=${FLANNEL_MTU} --bip=${FLANNEL_SUBNET}\"" `# ...and replace the found line with this line`
+        # Is there an uncommented DOCKER_OPTS line at all?
+        if [[ -z $(grep "DOCKER_OPTS" $DOCKER_CONF | grep -v "#") ]]; then
+          echo "DOCKER_OPTS=\"--mtu=${FLANNEL_MTU} --bip=${FLANNEL_SUBNET}\"" >> ${DOCKER_CONF}
+        else
+          kube::helpers::replace_mtu_bip ${DOCKER_CONF} "DOCKER_OPTS"
+        fi
 
         ifconfig docker0 down
         brctl delbr docker0 
         service docker stop
-        while [[ $(ps aux | grep /usr/bin/docker | grep -v grep | wc -l) -gt 0 ]]; do
+        while [[ $(ps aux | grep $(which docker) | grep -v grep | wc -l) -gt 0 ]]; do
             kube::log::status "Waiting for docker to terminate"
             sleep 1
         done
@@ -300,23 +315,13 @@ kube::multinode::restart_docker(){
 kube::multinode::restart_docker_systemd(){
 
   DOCKER_CONF=$(systemctl cat docker | head -1 | awk '{print $2}')
-
-  # This expression checks if the "--mtu" and "--bip" options are there
-  # If they aren't, they are inserted at the end of the docker command
-  if [[ -z $(grep -- "--mtu=" $DOCKER_CONF) ]]; then
-    sed -e "s@$(grep "/usr/bin/docker" $DOCKER_CONF)@$(grep "/usr/bin/docker" $DOCKER_CONF) --mtu=${FLANNEL_MTU}@g" -i $DOCKER_CONF
-  fi
-  if [[ -z $(grep -- "--bip=" $DOCKER_CONF) ]]; then
-    sed -e "s@$(grep "/usr/bin/docker" $DOCKER_CONF)@$(grep "/usr/bin/docker" $DOCKER_CONF) --bip=${FLANNEL_SUBNET}@g" -i $DOCKER_CONF
-  fi
+  kube::helpers::backup_file ${DOCKER_CONF}
+  kube::helpers::replace_mtu_bip ${DOCKER_CONF} $(which docker)
 
   ifconfig docker0 down
   brctl delbr docker0
 
-  # Finds "--mtu=????" and replaces with "--mtu=${FLANNEL_MTU}"
-  # Also finds "--bip=??.??.??.??" and replaces with "--bip=${FLANNEL_SUBNET}"
-  sed -e "s@$(grep -o -- "--mtu=[[:graph:]]*" $DOCKER_CONF)@--mtu=${FLANNEL_MTU}@g;s@$(grep -o -- "--bip=[[:graph:]]*" $DOCKER_CONF)@--bip=${FLANNEL_SUBNET}@g" -i $DOCKER_CONF
-  sed -i.bak 's/^\(MountFlags=\).*/\1shared/' $DOCKER_CONF
+  sed -i.bak 's/^\(MountFlags=\).*/\1shared/' ${DOCKER_CONF}
   systemctl daemon-reload
   systemctl restart docker
 }
@@ -340,8 +345,8 @@ kube::multinode::start_k8s_master() {
       --allow-privileged \
       --api-servers=http://localhost:8080 \
       --config=/etc/kubernetes/manifests-multi \
-      --cluster-dns=${DNS_SERVER_IP} \
-      --cluster-domain=${DNS_DOMAIN} \
+      --cluster-dns=10.0.0.10 \
+      --cluster-domain=cluster.local \
       --hostname-override=$(ip -o -4 addr list ${NET_INTERFACE} | awk '{print $4}' | cut -d/ -f1) \
       --v=2
 }
@@ -365,8 +370,8 @@ kube::multinode::start_k8s_worker() {
     /hyperkube kubelet \
       --allow-privileged \
       --api-servers=http://${MASTER_IP}:8080 \
-      --cluster-dns=${DNS_SERVER_IP} \
-      --cluster-domain=${DNS_DOMAIN} \
+      --cluster-dns=10.0.0.10 \
+      --cluster-domain=cluster.local \
       --hostname-override=$(ip -o -4 addr list ${NET_INTERFACE} | awk '{print $4}' | cut -d/ -f1) \
       --v=2
 }
@@ -377,22 +382,22 @@ kube::multinode::start_k8s_worker_proxy() {
   # Some quite complex version checking here...
   # If the version is under v1.3.0-alpha.5, kube-proxy is run manually in this script
   # In v1.3.0-alpha.5 and above, kube-proxy is run in a DaemonSet 
-  if [[ $((VERSION_MINOR < 3)) == 1 || \
-        $((VERSION_MINOR <= 3)) == 1 && \
-        $(echo ${VERSION_EXTRA}) != "" && \
-        ${VERSION_PRERELEASE} == "alpha" && \
-        $((VERSION_PRERELEASE_REV < 5)) == 1 ]]; then
-
-    kube::log::status "Launching kube-proxy..."
-    docker run -d \
-      --net=host \
-      --privileged \
-      --restart=${RESTART_POLICY} \
-      gcr.io/google_containers/hyperkube-${ARCH}:${K8S_VERSION} \
-      /hyperkube proxy \
-          --master=http://${MASTER_IP}:8080 \
-          --v=2
-  fi
+  # This has been uncommented for now, since the DaemonSet was inactivated in the stable v1.3 release
+  #if [[ $((VERSION_MINOR < 3)) == 1 || \
+  #      $((VERSION_MINOR <= 3)) == 1 && \
+  #      $(echo ${VERSION_EXTRA}) != "" && \
+  #      ${VERSION_PRERELEASE} == "alpha" && \
+  #      $((VERSION_PRERELEASE_REV < 5)) == 1 ]]; then
+
+  kube::log::status "Launching kube-proxy..."
+  docker run -d \
+    --net=host \
+    --privileged \
+    --restart=${RESTART_POLICY} \
+    gcr.io/google_containers/hyperkube-${ARCH}:${K8S_VERSION} \
+    /hyperkube proxy \
+        --master=http://${MASTER_IP}:8080 \
+        --v=2
 }
 
 # Turndown the local cluster
@@ -433,27 +438,34 @@ kube::multinode::turndown(){
     read -p "Should we stop the other containers that are running too? [Y/n] " stop_containers
 
     case $stop_containers in
-      [n|N]*)
+      [nN]*)
         ;; # Do nothing
       *)
         docker kill $(docker ps -q)
+        ;;
     esac
   fi
 
   if [[ -d /var/lib/kubelet ]]; then
     read -p "Do you want to clean /var/lib/kubelet? [Y/n] " clean_kubelet_dir
 
     case $clean_kubelet_dir in
-      [n|N]*)
+      [nN]*)
         ;; # Do nothing
       *)
         # umount if there are mounts in /var/lib/kubelet
         if [[ ! -z $(mount | grep /var/lib/kubelet | awk '{print $3}') ]]; then
-          umount $(mount | grep /var/lib/kubelet | awk '{print $3}')
+
+          # The umount command may be a little bit subborn sometimes, so run the commands twice to ensure the mounts are gone
+          mount | grep /var/lib/kubelet/* | awk '{print $3}' | xargs umount 1>/dev/null 2>/dev/null
+          mount | grep /var/lib/kubelet/* | awk '{print $3}' | xargs umount 1>/dev/null 2>/dev/null
+          umount /var/lib/kubelet 1>/dev/null 2>/dev/null
+          umount /var/lib/kubelet 1>/dev/null 2>/dev/null
         fi
 
         # Delete the directory
         rm -rf /var/lib/kubelet
+        ;;
     esac
   fi
 }
@@ -476,6 +488,28 @@ kube::helpers::file_replace_line(){
   fi
 }
 
+kube::helpers::replace_mtu_bip(){
+  local DOCKER_CONF=$1
+  local SEARCH_FOR=$2
+
+  # Assuming is a $SEARCH_FOR statement already, and we should append the options if they do not exist
+  if [[ -z $(grep -- "--mtu=" $DOCKER_CONF) ]]; then
+    sed -e "s@$(grep "$SEARCH_FOR" $DOCKER_CONF)@$(grep "$SEARCH_FOR" $DOCKER_CONF) --mtu=${FLANNEL_MTU}@g" -i $DOCKER_CONF
+  fi
+  if [[ -z $(grep -- "--bip=" $DOCKER_CONF) ]]; then
+    sed -e "s@$(grep "$SEARCH_FOR" $DOCKER_CONF)@$(grep "$SEARCH_FOR" $DOCKER_CONF) --bip=${FLANNEL_SUBNET}@g" -i $DOCKER_CONF
+  fi
+
+  # Finds "--mtu=????" and replaces with "--mtu=${FLANNEL_MTU}"
+  # Also finds "--bip=??.??.??.??" and replaces with "--bip=${FLANNEL_SUBNET}"
+  sed -e "s@$(grep -o -- "--mtu=[[:graph:]]*" $DOCKER_CONF)@--mtu=${FLANNEL_MTU}@g;s@$(grep -o -- "--bip=[[:graph:]]*" $DOCKER_CONF)@--bip=${FLANNEL_SUBNET}@g" -i $DOCKER_CONF
+}
+
+kube::helpers::backup_file(){
+  # Backup the current file
+  cp -f ${1} ${1}.backup
+}
+
 # Check if a process is running
 kube::helpers::is_running(){
   if [[ ! -z $(ps aux | grep ${1} | grep -v grep) ]]; then

diff --git a/docker-multinode/deployDNS.md b/docker-multinode/deployDNS.md