[HA] Fully automated impl of containerized high available etcd+apiserver+controllermgr+scheduler

docs/high-availability.md

@@ -200,4 +200,10 @@ restarting the kubelets on each node.
 If you are turning up a fresh cluster, you will need to install the kubelet and kube-proxy on each worker node, and
 set the ```--apiserver``` flag to your replicated endpoint.
 
+##Vagrant up!
+
+We indeed have an initial proof of concept tester for this, which is available [here](../examples/high-availability/).
+
+It implements the major concepts (with a few minor reductions for simplicity), of the podmaster HA implementation alongside a quick smoke test using k8petstore.
+
 [![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/docs/high-availability.md?pixel)]()

examples/high-availability/Vagrantfile

@@ -0,0 +1,46 @@
+# -*- mode: ruby -*-
+# vi: set ft=ruby :
+
+# Vagrantfile API/syntax version. Don't touch unless you know what you're doing!
+VAGRANTFILE_API_VERSION = "2"
+
+Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
+
+  fedora = 'hansode/fedora-21-server-x86_64'
+  script = "provision.sh"
+  config.hostmanager.enabled = true
+
+  if Vagrant.has_plugin?("vagrant-cachier")
+    config.cache.scope = :box
+  end
+
+  config.vm.provider "virtualbox" do |v|
+     v.memory = 3000
+     v.cpus = 1
+  end
+
+  config.vm.define "kube0" do |kube0|
+    kube0.vm.box = fedora
+    kube0.vm.hostname = "kube0.ha"
+    kube0.vm.synced_folder ".", "/vagrant"
+    kube0.vm.network :private_network, ip: "192.168.4.100"
+    kube0.vm.provision "shell", path:script
+  end
+
+  config.vm.define "kube1" do |kube1|
+    kube1.vm.box = fedora
+    kube1.vm.hostname = "kube1.ha"
+    kube1.vm.synced_folder ".", "/vagrant"
+    kube1.vm.network :private_network, ip: "192.168.4.101"
+    kube1.vm.provision "shell", path:script
+  end
+
+  config.vm.define "kube2" do |kube2|
+    kube2.vm.box = fedora
+    kube2.vm.hostname = "kube2.ha"
+    kube2.vm.network :private_network, ip: "192.168.4.102"
+    kube2.vm.synced_folder ".", "/vagrant"
+    kube2.vm.provision "shell", path:script
+  end
+
+end

examples/high-availability/apiserver.crt

@@ -0,0 +1,18 @@
+-----BEGIN CERTIFICATE-----
+MIIC+zCCAeWgAwIBAgIBATALBgkqhkiG9w0BAQswHzEdMBsGA1UEAwwUMTAuMC4y
+LjE1QDE0MzQ1NTU1NTkwHhcNMTUwNjE3MTUzOTE5WhcNMTYwNjE2MTUzOTE5WjAf
+MR0wGwYDVQQDDBQxMC4wLjIuMTVAMTQzNDU1NTU1OTCCASIwDQYJKoZIhvcNAQEB
+BQADggEPADCCAQoCggEBAK3OFQlqz04iuOtmSIlbJkeTwecL+p8tdtmG9SRn4Fw6
+TeuuoLCiSqjCZGLV1pKiL6fcjPYWsHoUNIzTtb6E/gj9OfGgQuIqZWRjM3blBmik
+aZ7N7OwJ5SZy6e5wFtNJ08xRnDZjhOIhtSjPQHk0WsC3hKJav3rGNdh7C53LUiWB
+uL3ne8oWaiTI9vlgW0ZWx6LcSa0U4jXftwdzLPLbB5INYrz9chF1hpulYnPWY1UA
+GE6wJTEpQM0p88Ye1t8Ey5QRWp6tjxVfxDYScxSP6FS8Dcj36RF9+5zGYcQ1YbRC
+Hc1hq7k33H6k5uUp+iPofezG9v4xhWqPkNV6LPxB9k8CAwEAAaNGMEQwDgYDVR0P
+AQH/BAQDAgCgMBMGA1UdJQQMMAoGCCsGAQUFBwMBMAwGA1UdEwEB/wQCMAAwDwYD
+VR0RBAgwBocECgACDzALBgkqhkiG9w0BAQsDggEBAFAzOeP67fKtHH2t114lSvvD
+2wMj7YjSDLyp3xRAMqtAiQ2DXytXvJ0nG8HcI8rnYYx/0RhibpSM565KjMk3VKhV
+FMYBU5BgFmR84TmCtLeZe4szr817A1Bbr25djMLQgHtEhtA0NptmjrzSdJICXeXe
+ih29/5HCxELlbDl7Alb8C8ITQlWsVQpUyr2W5tPp2w1wUA5OD1jJAdQquOHG/lWn
+4JC/4Out213CNCRh9dZFQsIy0oVUIncqspfj7v+xxVmQMeMqu1H92e5NFIqzfKaV
+cL5lSqZj2tOKS4fKPqadZ6IBxOZVr28THCUlhbWwDrLEMk8Vu7W+iuhrl8Jthws=
+-----END CERTIFICATE-----

examples/high-availability/apiserver.key

@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEowIBAAKCAQEArc4VCWrPTiK462ZIiVsmR5PB5wv6ny122Yb1JGfgXDpN666g
+sKJKqMJkYtXWkqIvp9yM9hawehQ0jNO1voT+CP058aBC4iplZGMzduUGaKRpns3s
+7AnlJnLp7nAW00nTzFGcNmOE4iG1KM9AeTRawLeEolq/esY12HsLnctSJYG4ved7
+yhZqJMj2+WBbRlbHotxJrRTiNd+3B3Ms8tsHkg1ivP1yEXWGm6Vic9ZjVQAYTrAl
+MSlAzSnzxh7W3wTLlBFanq2PFV/ENhJzFI/oVLwNyPfpEX37nMZhxDVhtEIdzWGr
+uTfcfqTm5Sn6I+h97Mb2/jGFao+Q1Xos/EH2TwIDAQABAoIBAAN4BXt2okSQ9Nfd
+5ry3EQCEtm4CEzJyGdkllGbxm8y7bSqzBYoA8zSDDLXcRrgOfJ0BDd8rfMIy/QmC
+gDIZPWi4hRuJc0xIRFXIlRJeZ85W2bTNr1jWqbgmuvyDSDFXW97MEi4Ttmc8/Pyf
+hk3k+Zq3DvKelQvCQPjdG9I+aJlPAQ9jRpseiXADgxgGJRjMrNNvDmAXSy+vD/PA
+MzIPcmW48nQ6kvXs6UdRXmfZD8CySQnSMN+pOMzikN9bbyrPHnKxNzImsKSCpN78
+Uh8eNItDJmMLWv/SwnVS8/X5wMxRQ2t5vuGRnWCQEc+rLtw2mAkj36GnfFpZvSNd
+fVuVbCECgYEAygCErUVxFUgfBgrXDZa9VEiYKnuQBdrgKOH5JQ/Pifp9BuhNtLvH
+fHZ16cesZnQ8q+U4nUbnu0s4Gpl0RS96KcwJ3EjGPknclZoVZVPB6Ece/7JaaZcA
+OQuRRkYABJRPIcpPCtgeJO+OL6H3BFmvteT8GTrek6RY8wrw65nIPu0CgYEA3EP5
+guS3OoclBvFs5o2WyKokc+kq+L0zS9WX/Hv4nK2c2aS628TfhDqWeTeettuq0Jlf
+hGvNkNaehQfPpyrJzHwoATMWhi/cKM9sycC9oEFX1tuPAZit2gl+cjXZOX19msp6
+Sh1I5VKGM8pxGFrE3gDDq1woRr+Ke+lWOiDz5qsCgYBMhSm9YYLW5v0pks2oTiPm
+W6GY5jnGngUwN3njujuKLyNRjIpzHncRBObh6ku6e+nHzAtIOOXrozDbkqni03tZ
+fft2QPMoAV7YJQhZ3AKmdNqfTfbF7PeepG0hy76R/YSEbljG6NtybnTUQmyKb1cK
+dnWxMQXDtAwl9U0SUqzyeQKBgGANWGpHGMvyESiE8WtcxSs+XuUZAf80aoEgZMXa
+veB9KRAT8NRTdvEvp1y274RoKIYMzAVwCVWm8burW1dXpmaAoeVcBO+BQW2efp9A
+aLDQreBpIGSe0vlo+HYPm2mhmrt8nnVhbd9q7FD7b/Qh6QWyqaE5b+riLh648zwo
+EJQ/AoGBAKpDzePHNptD8zZCi/LEjPGeI2HPSiDs7/s6591l5gbSvfRtWyyRtDk3
+jRgbOAqjF3Eow+QOA1GNGaSYWoANBmhKuUwn3ETzsmQ8UFSj/Wmc3IohhYZtrh6h
+e0T8VGFcS6bg5OLbYfarzdaI+hL7zlOhjDAgc9E8rjYgBIvb8h9n
+-----END RSA PRIVATE KEY-----

examples/high-availability/etc_kubernetes_kubelet

@@ -0,0 +1,21 @@
+###
+# kubernetes kubelet (minion) config
+
+# The address for the info server to serve on (set to 0.0.0.0 or "" for all interfaces)
+KUBELET_ADDRESS=""
+#--address=0.0.0.0"
+
+# The port for the info server to serve on
+# KUBELET_PORT="--port=10250"
+
+# You may leave this blank to use the actual hostname
+# KUBELET_HOSTNAME="--hostname_override=0.0.0."
+
+# location of the api-server
+KUBELET_API_SERVER="--api_servers=http://0.0.0.0:8080,kube1.ha:8080,kube0.ha:8080 "
+# --cert-dir="/var/run/kubernetes": The directory where the TLS certs are located (by default /var/run/kubernetes). If --tls_cert_file and --tls_private_key_file are provided, this flag will be ignored.
+# --tls-cert-file="": File containing x509 Certificate for HTTPS.  (CA cert, if any, concatenated after server cert). If --tls_cert_file and --tls_private_key_file are not provided, a self-signed certificate and key are generated for the public address and saved to the directory passed to --cert_dir.
+#  --tls-private-key-file="": File containing x509 private key matching --tls_cert_file.
+
+# We modify kubelet args to do verbose logs + read from manifests dir.
+KUBELET_ARGS="--tls-cert-file=/vagrant/apiserver.crt --tls-private-key-file=/vagrant/apiserver.key --register-node=true --v=5 --config=/etc/kubernetes/manifests --kubeconfig=/vagrant/kubeconfig"

examples/high-availability/etcd.manifest

@@ -0,0 +1,97 @@
+{
+"apiVersion": "v1beta3",
+"kind": "Pod",
+"metadata": {"name":"etcd-server"},
+"spec":{
+    "hostNetwork": true,
+    "containers":[
+    {
+    "name": "etcd-container",
+    "image": "quay.io/coreos/etcd",
+    "command": [
+          "/etcd","--name","NODE_NAME",
+	      "--initial-advertise-peer-urls",  "http://NODE_IP:2380",
+	      "--listen-peer-urls",     	"http://NODE_IP:2380",
+	      "--advertise-client-urls",    	"http://NODE_IP:2379",
+	      "-initial-cluster", "kube0.ha=http://192.168.4.100:2380",
+	      "--listen-client-urls",   	"http://127.0.0.1:2379,http://NODE_IP:2379",
+          "--data-dir","/var/etcd/data"
+    ],
+    "ports":[
+        {
+            "name": "serverport", "containerPort": 2380, "hostPort": 2380
+        },
+        {
+            "name": "clientport","containerPort": 4001, "hostPort": 4001
+        }
+    ],
+    "volumeMounts": [
+        { "name": "varetcd",
+        "mountPath": "/var/etcd",
+        "readOnly": false},
+        { "name": "etcssl",
+        "mountPath": "/etc/ssl",
+        "readOnly": true},
+        { "name": "usrsharessl",
+        "mountPath": "/usr/share/ssl",
+        "readOnly": true},
+        { "name": "varssl",
+        "mountPath": "/var/ssl",
+        "readOnly": true},
+        { "name": "usrssl",
+        "mountPath": "/usr/ssl",
+        "readOnly": true},
+        { "name": "usrlibssl",
+        "mountPath": "/usr/lib/ssl",
+        "readOnly": true},
+        { "name": "usrlocalopenssl",
+        "mountPath": "/usr/local/openssl",
+        "readOnly": true},
+        { "name": "etcopenssl",
+        "mountPath": "/etc/openssl",
+        "readOnly": true},
+        { "name": "etcpkitls",
+        "mountPath": "/etc/pki/tls",
+        "readOnly": true}
+        ]
+       }
+],
+"volumes":[
+  { "name": "varetcd",
+    "hostPath": {
+        "path": "/var/etcd/data"}
+  },
+  { "name": "etcssl",
+    "hostPath": {
+        "path": "/etc/pki/tls/certs"}
+  },
+  { "name": "usrsharessl",
+    "hostPath": {
+        "path": "/usr/share/ssl"}
+  },
+  { "name": "varssl",
+    "hostPath": {
+        "path": "/var/ssl"}
+  },
+  { "name": "usrssl",
+    "hostPath": {
+        "path": "/usr/ssl"}
+  },
+  { "name": "usrlibssl",
+    "hostPath": {
+        "path": "/usr/lib/ssl"}
+  },
+  { "name": "usrlocalopenssl",
+    "hostPath": {
+        "path": "/usr/local/openssl"}
+  },
+  { "name": "etcopenssl",
+    "hostPath": {
+        "path": "/etc/openssl"}
+  },
+  { "name": "etcpkitls",
+    "hostPath": {
+        "path": "/etc/pki/tls"}
+  }
+]
+}}

examples/high-availability/kube-apiserver.manifest

@@ -0,0 +1,103 @@
+  {
+"apiVersion": "v1beta3",
+"kind": "Pod",
+"metadata": {"name":"kube-apiserver"},
+"spec":{
+"hostNetwork": true,
+"containers":[
+    {
+    "name": "kube-apiserver",
+    "image": "gcr.io/google_containers/kube-apiserver:9680e782e08a1a1c94c656190011bd02",
+    "command": [
+                 "/bin/sh",
+                 "-c",
+                 "/usr/local/bin/kube-apiserver --address=0.0.0.0 --etcd_servers=http://kube0.ha:2379 --service-cluster-ip-range=10.0.0.0/16 --v=4 --allow_privileged=True 1>>/var/log/kube-apiserver.log 2>&1"
+               ],
+    "ports":[
+      { "name": "https",
+        "containerPort": 443,
+        "hostPort": 443},{
+       "name": "http",
+        "containerPort": 7080,
+        "hostPort": 7080},{
+       "name": "local",
+        "containerPort": 8080,
+        "hostPort": 8080}
+        ],
+    "volumeMounts": [
+        { "name": "srvkube",
+        "mountPath": "/srv/kubernetes",
+        "readOnly": true},
+        { "name": "logfile",
+        "mountPath": "/var/log/kube-apiserver.log",
+        "readOnly": false},
+        { "name": "etcssl",
+        "mountPath": "/etc/ssl",
+        "readOnly": true},
+        { "name": "usrsharessl",
+        "mountPath": "/usr/share/ssl",
+        "readOnly": true},
+        { "name": "varssl",
+        "mountPath": "/var/ssl",
+        "readOnly": true},
+        { "name": "usrssl",
+        "mountPath": "/usr/ssl",
+        "readOnly": true},
+        { "name": "usrlibssl",
+        "mountPath": "/usr/lib/ssl",
+        "readOnly": true},
+        { "name": "usrlocalopenssl",
+        "mountPath": "/usr/local/openssl",
+        "readOnly": true},
+        { "name": "etcopenssl",
+        "mountPath": "/etc/openssl",
+        "readOnly": true},
+        { "name": "etcpkitls",
+        "mountPath": "/etc/pki/tls",
+        "readOnly": true}
+      ]
+    }
+],
+"volumes":[
+  { "name": "srvkube",
+    "hostPath": {
+        "path": "/srv/kubernetes"}
+  },
+  { "name": "logfile",
+    "hostPath": {
+        "path": "/var/log/kube-apiserver.log"}
+  },
+  { "name": "etcssl",
+    "hostPath": {
+        "path": "/etc/ssl"}
+  },
+  { "name": "usrsharessl",
+    "hostPath": {
+        "path": "/usr/share/ssl"}
+  },
+  { "name": "varssl",
+    "hostPath": {
+        "path": "/var/ssl"}
+  },
+  { "name": "usrssl",
+    "hostPath": {
+        "path": "/usr/ssl"}
+  },
+  { "name": "usrlibssl",
+    "hostPath": {
+        "path": "/usr/lib/ssl"}
+  },
+  { "name": "usrlocalopenssl",
+    "hostPath": {
+        "path": "/usr/local/openssl"}
+  },
+  { "name": "etcopenssl",
+    "hostPath": {
+        "path": "/etc/openssl"}
+  },
+  { "name": "etcpkitls",
+    "hostPath": {
+        "path": "/etc/pki/tls"}
+  }
+]
+}}