update the new guide and some script modification

cluster/ubuntu/config-default.sh

@@ -22,7 +22,7 @@ export nodes="vcap@10.10.103.250 vcap@10.10.103.162 vcap@10.10.103.223"
 # Define all your nodes role: a(master) or i(minion) or ai(both master and minion), must be the order same 
 export roles=("ai" "i" "i")
 # Define minion numbers
-NUM_MINIONS=${NUM_MINIONS:-3}
+export NUM_MINIONS=${NUM_MINIONS:-3}
 # define the IP range used for service portal
 export PORTAL_NET=11.1.1.0/24
 

cluster/ubuntu/reconfDocker.sh

@@ -1,5 +1,19 @@
 #!/bin/bash
-# Run as root only
+
+# Copyright 2015 Google Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 # reconfigure docker network setting
 
 if [ "$(id -u)" != "0" ]; then

cluster/ubuntu/util.sh

@@ -15,7 +15,7 @@
 # limitations under the License.
 
 # A library of helper functions that each provider hosting Kubernetes must implement to use cluster/kube-*.sh scripts.
-set -ex
+set -e
 
 SSH_OPTS="-oStrictHostKeyChecking=no -oUserKnownHostsFile=/dev/null -oLogLevel=ERROR"
 
@@ -66,10 +66,12 @@ function setClusterInfo() {
 }
 
 
-# Verify prereqs, ensuring the binaries is downloaded.
+# Verify ssh prereqs
 function verify-prereqs {
-  if [ ! -f "ubuntu/binaries/kube-apiserver" ]; then
-    echo "warning: not enough binaries to build k8s, please run build.sh in cluster/ubuntu first"
+   # Expect at least one identity to be available.
+  if ! ssh-add -L 1> /dev/null 2> /dev/null; then
+    echo "Could not find or add an SSH identity."
+    echo "Please start ssh-agent, add your identity, and retry."
     exit 1
   fi
 }
@@ -304,6 +306,13 @@ function kube-up {
   KUBE_ROOT=$(dirname "${BASH_SOURCE}")/../..
   source "${KUBE_ROOT}/cluster/ubuntu/${KUBE_CONFIG_FILE-"config-default.sh"}"
 
+
+  # ensure the binaries are downloaded
+  if [ ! -f "ubuntu/binaries/kube-apiserver" ]; then
+    echo "warning: not enough binaries to build k8s, please run build.sh in cluster/ubuntu first"
+    exit 1
+  fi
+
   setClusterInfo
   ii=0
 
@@ -333,6 +342,7 @@ function kube-up {
 function provision-master() {
   # copy the binaries and scripts to the ~/kube directory on the master
   echo "Deploying master on machine ${MASTER_IP}"
+  echo 
   ssh $SSH_OPTS $MASTER "mkdir -p ~/kube/default"
   scp -r $SSH_OPTS ubuntu/config-default.sh ubuntu/util.sh ubuntu/master/* ubuntu/binaries/ "${MASTER}:~/kube"
 
@@ -371,6 +381,7 @@ function provision-minion() {
 function provision-masterandminion() {
   # copy the binaries and scripts to the ~/kube directory on the master
   echo "Deploying master and minion on machine ${MASTER_IP}"
+  echo 
   ssh $SSH_OPTS $MASTER "mkdir -p ~/kube/default"
   scp -r $SSH_OPTS ubuntu/config-default.sh ubuntu/util.sh ubuntu/master/* ubuntu/minion/* ubuntu/binaries/ "${MASTER}:~/kube"
 
@@ -398,8 +409,8 @@ function kube-down {
   for i in ${nodes}; do
   {
     echo "Cleaning on node ${i#*@}"
-    ssh $i 'pgrep etcd && echo password | sudo -S -p "[sudo] password for cleaning etcd data: " service etcd stop && sudo rm -rf /infra*'
-  } &
+    ssh -t $i 'pgrep etcd && sudo -p "[sudo] password for cleaning etcd data: " service etcd stop && sudo rm -rf /infra*'
+  } 
   done
   wait
 }

cluster/validate-cluster.sh

@@ -51,22 +51,6 @@ attempt=0
 while true; do
   kubectl_output=$("${KUBE_ROOT}/cluster/kubectl.sh" get cs)
 
-  # On vSphere, use minion IPs as their names
-  if [[ "${KUBERNETES_PROVIDER}" == "vsphere" ]] || [[ "${KUBERNETES_PROVIDER}" == "vagrant" ]] || [[ "${KUBERNETES_PROVIDER}" == "libvirt-coreos" ]] || [[ "${KUBERNETES_PROVIDER}" == "ubuntu" ]]; then
-    MINION_NAMES=("${KUBE_MINION_IP_ADDRESSES[@]}")
-  fi
-
-  # On AWS we can't really name the minions, so just trust that if the number is right, the right names are there.
-  if [[ "${KUBERNETES_PROVIDER}" == "aws" ]]; then
-    MINION_NAMES=("$(cat ${MINIONS_FILE})")
-    # /healthz validation isn't working for some reason on AWS.  So just hope for the best.
-    # TODO: figure out why and fix, it must be working in some form, or else clusters wouldn't work.
-    echo "Kubelet health checking on AWS isn't currently supported, assuming everything is good..."
-    echo -e "${color_green}Cluster validation succeeded${color_norm}"
-    exit 0
-  fi
-
-
   # The "kubectl componentstatuses" output is four columns like this:
   #
   #     COMPONENT            HEALTH    MSG       ERR
@@ -83,38 +67,6 @@ while true; do
   if ((non_success_count > 1)); then
     if ((attempt < 5)); then
       echo -e "${color_yellow}Cluster not working yet.${color_norm}"
-
-    name="${MINION_NAMES[$i]}"
-    if [ "$KUBERNETES_PROVIDER" != "vsphere" ] && [ "$KUBERNETES_PROVIDER" != "vagrant" ] && [ "$KUBERNETES_PROVIDER" != "libvirt-coreos" ] && [ "$KUBERNETES_PROVIDER" != "ubuntu" ]; then
-      # Grab fully qualified name
-      name=$(grep "${MINION_NAMES[$i]}\." "${MINIONS_FILE}")
-    fi
-
-    # Make sure the kubelet is healthy.
-    # Make several attempts to deal with slow cluster birth.
-    attempt=0
-    while true; do
-      echo -n "Attempt $((attempt+1)) at checking Kubelet installation on node ${MINION_NAMES[$i]} ..."
-      if [ "$KUBERNETES_PROVIDER" != "libvirt-coreos" ] && [ "$KUBERNETES_PROVIDER" != "ubuntu" ]; then
-        curl_output=$(curl -s --insecure --user "${KUBE_USER}:${KUBE_PASSWORD}" \
-          "https://${KUBE_MASTER_IP}/api/v1beta1/proxy/minions/${name}/healthz")
-      else
-        curl_output=$(curl -s \
-          "http://${KUBE_MASTER_IP}:8080/api/v1beta1/proxy/minions/${name}/healthz")
-      fi
-      if [[ "${curl_output}" != "ok" ]]; then
-          if (( attempt > 5 )); then
-            echo
-            echo -e "${color_red}Kubelet failed to install on node ${MINION_NAMES[$i]}. Your cluster is unlikely to work correctly."
-            echo -e "Please run ./cluster/kube-down.sh and re-create the cluster. (sorry!)${color_norm}"
-            exit 1
-          fi
-      else
-          echo -e " ${color_green}[working]${color_norm}"
-          break
-      fi
-      echo -e " ${color_yellow}[not working yet]${color_norm}"
-
       attempt=$((attempt+1))
       sleep 30
     else
@@ -130,4 +82,4 @@ done
 
 echo "Validate output:"
 echo "${kubectl_output}"
-echo -e "${color_green}Cluster validation succeeded${color_norm}"
+echo -e "${color_green}Cluster validation succeeded${color_norm}"

docs/getting-started-guides/README.md

@@ -23,8 +23,7 @@ Vmware         |              | Debian | OVS         | [docs](../../docs/getting
 AWS            | Saltstack    | Ubuntu | OVS         | [docs](../../docs/getting-started-guides/aws.md)       | Community (@justinsb)        | Uses K8s version 0.5.0
 Vmware         | CoreOS       | CoreOS | flannel     | [docs](../../docs/getting-started-guides/coreos.md)    | Community (@kelseyhightower) |
 Azure          | Saltstack    | Ubuntu | OpenVPN     | [docs](../../docs/getting-started-guides/azure.md)     | Community                    |
-Bare-metal     | custom       | Ubuntu | _none_      | [docs](../../docs/getting-started-guides/ubuntu_single_node.md) | Community (@jainvipin)       |
-Bare-metal     | custom       | Ubuntu Cluster | flannel | [docs](../../docs/getting-started-guides/ubuntu_multinodes_cluster.md) | Community (@resouer @WIZARD-CXY) | use k8s version 0.12.0
+Bare-metal     | custom       | Ubuntu | flannel     | [docs](../../docs/getting-started-guides/ubuntu.md)    | Community (@resouer @WIZARD-CXY)       | use k8s version 0.15.0
 Docker Single Node        | custom       | N/A    | local       | [docs](docker.md) | Project (@brendandburns) | Tested @ 0.14.1 |
 Docker Multi Node        | Flannel| N/A    | local       | [docs](docker-multinode.md) | Project (@brendandburns) | Tested @ 0.14.1 |
 Local          |              |        | _none_      | [docs](../../docs/getting-started-guides/locally.md)   | Community (@preillyme)                     |

docs/getting-started-guides/ubuntu.md

@@ -0,0 +1,131 @@
+# Kubernetes deployed on ubuntu nodes
+
+This document describes how to deploy kubernetes on ubuntu nodes, including 1 master node and 3 minion nodes, and people uses this approach can scale to **any number of minion nodes** by changing some settings with ease. Although there exists saltstack based ubuntu k8s installation ,  it may be tedious and hard for a guy that knows little about saltstack but want to build a really distributed k8s cluster. This new approach of kubernets deployment is much more easy and automatical than the previous one.
+
+[Cloud team from ZJU](https://github.com/ZJU-SEL) will keep updating this work.
+
+### **Prerequisites：**
+*1 The minion nodes have installed docker version 1.2+* 
+
+*2 All machines can communicate with each orther, no need to connect Internet (should use private docker registry in this case)*
+
+*3 These guide is tested OK on Ubuntu 14.04 LTS 64bit server, but it should also work on most Ubuntu versions*
+
+*4 Dependences of this guide: etcd-2.0.0, flannel-0.2.0, k8s-0.15.0, but it may work with higher versions*
+
+*5 All the remote servers can be ssh logged in without a password by using key authentication* 
+
+
+### **Main Steps**
+#### I. Make *kubernetes* , *etcd* and *flanneld* binaries
+
+On your laptop, copy `cluster/ubuntu` directory to your workspace.
+
+The `build.sh` will download and build all the needed binaries into `./binaries`.
+
+You can customize your etcd version or K8s version in the build.sh by changing  variable `ETCD_V` and `K8S_V` in build.sh, default etcd version is 2.0.0 and K8s version is 0.15.0.
+
+
+```
+$ cd cluster/ubuntu
+$ sudo ./build.sh
+```
+
+Please make sure that there are `kube-apiserver`, `kube-controller-manager`, `kube-scheduler`, `kubelet`, `kube-proxy`, `etcd`, `etcdctl` and `flannel` in the binaries directory. All the other files are not necessary for a build for now. 
+
+> We used flannel here because we want to use overlay network, but please remember it is not the only choice, and it is also not a k8s' necessary dependence. Actually you can just build up k8s cluster natively, or use flannel, Open vSwitch or any other SDN tool you like, we just choose flannel here as a example.
+
+#### II. Configue and install every components upstart script
+An example cluster is listed as below:
+
+| IP Address|Role |      
+|---------|------|
+|10.10.103.223|   minion   |
+|10.10.103.162|   minion   |
+|10.10.103.250| both master and minion|
+
+First configure the cluster information in cluster/ubuntu/config-default.sh, below is a simple sample.
+
+```
+export nodes="vcap@10.10.103.250 vcap@10.10.103.162 vcap@10.10.103.223"
+
+export roles=("ai" "i" "i")
+
+export NUM_MINIONS=${NUM_MINIONS:-3}
+
+export PORTAL_NET=11.1.1.0/24
+
+```
+
+The first variable `nodes` defines all your cluster nodes, MASTER node comes first and separated with blank space like `<user_1@ip_1> <user_2@ip_2> <user_3@ip_3> `
+
+Then the `roles ` variable defines the role of above machine in the same order, "ai" stands for machine acts as both master and minion, "a" stands for master, "i" stands for minion. So they are just defined the k8s cluster as the table above described.
+
+The `NUM_MINIONS` variable defines the total number of minions.
+
+The `PORTAL_NET` variable defines the kubernetes service portal ip range.
+
+After all the above variable being set correctly. We can use Below command in cluster/ directory to bring up the whole cluster.
+
+`$ KUBERNETES_PROVIDER=ubuntu ./kube-up.sh` 
+
+The scripts is automatically scp binaries and config files to all the machines and start the k8s service on them. The only thing you need to do is to type the sudo password when promoted. The current machine name is shown below like. So you will not type in the wrong password.
+
+```
+
+Deploying minion on machine 10.10.103.223
+
+...
+
+[sudo] password to copy files and start minion: 
+
+```
+
+If all things goes right, you will see the below message from console
+`Cluster validation succeeded` indicating the k8s is up.
+
+**All done !**
+
+You can also use kubectl command to see if the newly created k8s is working correctly. 
+
+For example , `$ kubectl get minions` to see if you get all your minion nodes comming up and ready. It may take some times for the minions be ready to use like below . 
+
+```
+NAME                 LABELS             STATUS
+
+10.10.103.162       <none>              Ready
+
+10.10.103.223       <none>              Ready
+
+10.10.103.250       <none>              Ready
+```
+
+Also you can run kubernetes [guest-example](https://github.com/GoogleCloudPlatform/kubernetes/tree/master/examples/guestbook) to build a redis backend cluster on the k8s．
+
+
+#### V. Trouble Shooting
+
+Generally, what of this approach did is quite simple: 
+
+1. Build and copy binaries and configuration files to proper dirctories on every node
+
+2. Configure `etcd` using IPs based on input from user 
+
+3. Create and start flannel network
+
+So, whenver you have problem, do not blame Kubernetes, **check etcd configuration first** 
+
+Please try:
+
+1. Check `/var/log/upstart/etcd.log` for suspicisous etcd log 
+
+2. Check `/etc/default/etcd`, as we do not have much input validation, a right config should be like:
+	```
+	ETCD_OPTS="-name infra1 -initial-advertise-peer-urls <http://ip_of_this_node:2380> -listen-peer-urls <http://ip_of_this_node:2380> -initial-cluster-token etcd-cluster-1 -initial-cluster infra1=<http://ip_of_this_node:2380>,infra2=<http://ip_of_another_node:2380>,infra3=<http://ip_of_another_node:2380> -initial-cluster-state new"
+	```
+
+3. You can use below command 
+   `$ KUBERNETES_PROVIDER=ubuntu ./kube-down.sh` to bring down the cluster and run
+   `$ KUBERNETES_PROVIDER=ubuntu ./kube-up.sh` again to start again.
+
+4. You can also customize your own settings in `/etc/default/{component_name}` after configured success.