Implement kube-master HA for multiple masters

ansible/inventory.example.ha

@@ -1,3 +1,6 @@
+# TODO: Current multi-master configuration doesn't allow for
+# a single host to be both a master and node.
+# https://github.com/kubernetes/contrib/issues/838
 
 [masters]
 kube-master-test-[1:3].example.com

ansible/roles/kubernetes/defaults/main.yml

@@ -46,6 +46,11 @@ kube_token_dir: "{{ kube_config_dir }}/tokens"
 # pods on startup
 kube_manifest_dir: "{{ kube_config_dir }}/manifests"
 
+# This is where you can drop yaml/json files for podmaster to copy into kube_manifest_dir to be started by kubelet
+# Podmaster containers provide leader election for each pod in this directory, so that each cluster
+# only runs one pod of each type (e.g. controller-manager & scheduler, or other state-modifying services)
+kube_standby_manifest_dir: "{{ kube_config_dir }}/standby_manifests"
+
 # This is the group that the cert creation scripts chgrp the
 # cert files to. Not really changeable...
 kube_cert_group: kube-cert

ansible/roles/kubernetes/tasks/gen_certs.yml

@@ -38,6 +38,10 @@
     HTTP_PROXY: "{{ http_proxy|default('') }}"
     HTTPS_PROXY: "{{ https_proxy|default('') }}"
 
+- name: Create server.pem from crt and key
+  raw: cat {{ kube_cert_dir }}/server.crt {{ kube_cert_dir }}/server.key > {{ kube_cert_dir }}/server.pem
+  sudo: yes
+
 - name: Verify certificate permissions
   file:
     path={{ item }}
@@ -48,5 +52,7 @@
     - "{{ kube_cert_dir }}/ca.crt"
     - "{{ kube_cert_dir }}/server.crt"
     - "{{ kube_cert_dir }}/server.key"
+    - "{{ kube_cert_dir }}/kubelet.crt"
+    - "{{ kube_cert_dir }}/kubelet.key"
     - "{{ kube_cert_dir }}/kubecfg.crt"
     - "{{ kube_cert_dir }}/kubecfg.key"

ansible/roles/kubernetes/tasks/gen_tokens.yml

@@ -10,7 +10,7 @@
   environment:
     TOKEN_DIR: "{{ kube_token_dir }}"
   with_nested:
-    - [ "system:controller_manager", "system:scheduler", "system:kubectl" ]
+    - [ "system:controller_manager", "system:scheduler", "system:kubectl" , "system:kubelet" ]
     - "{{ groups['masters'] }}"
   register: gentoken
   changed_when: "'Added' in gentoken.stdout"

ansible/roles/kubernetes/tasks/main.yml

@@ -21,6 +21,12 @@
 - name: Create kubernetes config directory
   file: path={{ kube_config_dir }} state=directory
 
+- name: Create manifests directory
+  file: path={{ kube_manifest_dir }} state=directory mode=0755
+
+- name: Create standby-manifests directory
+  file: path={{ kube_standby_manifest_dir }} state=directory mode=0755
+
 - name: Create kubernetes script directory
   file: path={{ kube_script_dir }} state=directory
 

ansible/roles/kubernetes/tasks/secrets.yml

@@ -28,6 +28,20 @@
 - include: gen_certs.yml
   when: inventory_hostname == groups['masters'][0]
 
+- name: Archive certificates
+  shell: tar -czf /tmp/certs.tar.gz .
+  args:
+    chdir: "{{ kube_cert_dir }}"
+  delegate_to: "{{ groups['masters'][0] }}"
+
+- name: Fetch certificates
+  fetch: src=/tmp/certs.tar.gz dest=/tmp/certs.tar.gz flat=yes
+  delegate_to: "{{ groups['masters'][0] }}"
+
+- name: Place certs everywhere
+  unarchive: src=/tmp/certs.tar.gz dest="{{ kube_cert_dir }}"
+  when: inventory_hostname in groups['masters']
+
 - name: Read back the CA certificate
   slurp:
     src: "{{ kube_cert_dir }}/ca.crt"
@@ -72,5 +86,25 @@
   notify:
     - restart daemons
 
+- name: Copy the token gen script
+  copy:
+    src=kube-gen-token.sh
+    dest={{ kube_script_dir }}
+    mode=u+x
+
 - include: gen_tokens.yml
   when: inventory_hostname == groups['masters'][0]
+
+- name: Archive tokens
+  shell: tar -czf /tmp/known_tokens.tar.gz .
+  args:
+    chdir: "{{ kube_token_dir }}"
+  delegate_to: "{{ groups['masters'][0] }}"
+
+- name: Fetch tokens
+  fetch: src=/tmp/known_tokens.tar.gz dest=/tmp/known_tokens.tar.gz flat=yes
+  delegate_to: "{{ groups['masters'][0] }}"
+
+- name: Place tokens everywhere
+  unarchive: src=/tmp/known_tokens.tar.gz dest="{{ kube_token_dir }}"
+  when: inventory_hostname in groups['masters']

ansible/roles/kubernetes/templates/kube_internal_networkd.j2

@@ -0,0 +1,17 @@
+[Match]
+Name={{ kube_internal_interface }}
+
+[Network]
+Address={{ kube_internal_ip }}{{ kube_internal_cidr }}
+
+{% if kube_internal_routes %}
+{% for route in kube_internal_routes %}
+{% set route_dest=route.split(',')[0] %}
+{% set route_mask=route.split(',')[1] %}
+{% set route_next_hop=route.split(',')[2] %}
+[Route]
+Destination={{ route_dest }}{{ route_mask }}
+Gateway={{ route_next_hop }}
+
+{% endfor %}
+{% endif %}

ansible/roles/master/defaults/main.yml

@@ -1,7 +1,15 @@
 kube_master_insecure_port: 8080
 
+kube_master_api_port: 443
+
+kube_apiserver_interface: "{{ ansible_default_ipv4.interface }}"
+
 localBuildOutput: ../../_output/local/go/bin
 
 admission_controllers: NamespaceLifecycle,NamespaceExists,LimitRanger,ServiceAccount,ResourceQuota
 
 kube_apiserver_bind_address: "0.0.0.0"
+
+# hyperkube is an all-in-one kubernetes binary that is automatically pushed on every release
+# https://github.com/kubernetes/kubernetes/tree/master/cluster/images/hyperkube
+hyperkube_version: "v1.1.8"

ansible/roles/master/handlers/main.yml

@@ -14,12 +14,15 @@
 
 - name: restart apiserver
   service: name=kube-apiserver state=restarted
+  when: groups['masters']|length == 1
 
 - name: restart controller-manager
   service: name=kube-controller-manager state=restarted
+  when: groups['masters']|length == 1
 
 - name: restart scheduler
   service: name=kube-scheduler state=restarted
+  when: groups['masters']|length == 1
 
 - name: restart iptables
   service: name=iptables state=restarted

ansible/roles/master/tasks/ha_master.yml

@@ -0,0 +1,76 @@
+- name: Generic | Install kubernetes node
+  action: "{{ ansible_pkg_mgr }}"
+  args:
+    name: kubernetes-node
+    state: latest
+  notify:
+    - restart daemons
+  when: not is_coreos
+
+- name: CoreOS | Get Systemd Unit Files for kubelet
+  get_url:
+    url=https://raw.githubusercontent.com/kubernetes/contrib/master/init/systemd/{{ item }}.service
+    dest=/etc/systemd/system/{{ item }}.service
+    force=yes
+  register: "{{ item }}_service"
+  notify:
+    - reload systemd
+  with_items:
+    - kubelet
+  environment:
+    http_proxy: "{{ http_proxy|default('') }}"
+    https_proxy: "{{ https_proxy|default('') }}"
+    no_proxy: "{{ no_proxy|default('') }}"
+  when: is_coreos
+
+- name: CoreOS | Create dropin directories for kubelet
+  file: path=/etc/systemd/system/{{ item }}.service.d state=directory mode=0755
+  with_items:
+    - kubelet
+  when: is_coreos
+
+- name: CoreOS | Write kubelet drop-in file
+  template: src={{ item }}-dropin.j2 dest="/etc/systemd/system/{{ item }}.service.d/10-conf-file.conf"
+  register: "{{ item }}_dropin"
+  with_items:
+    - kubelet
+  notify:
+    - reload systemd
+  when: is_coreos
+
+- name: Set selinux permissive because tokens and selinux do not work together
+  selinux: state=permissive policy={{ ansible_selinux.type }}
+  when: ansible_selinux is defined and ansible_selinux.status == "enabled"
+
+- name: Create the kubelet working directory
+  file: path=/var/lib/kubelet state=directory
+
+- name: write the kubecfg (auth) file for kubelet
+  template: src=kubelet.kubeconfig.j2 dest={{ kube_config_dir }}/kubelet.kubeconfig
+
+- name: Write the pod manifest for the controller-manager
+  template: src=kube-controller-manager-hyperkube.yml.j2 dest={{ kube_standby_manifest_dir }}/kube-controller-manager-hyperkube.yml
+
+- name: Write the pod manifest for the api server
+  template: src=kube-api-hyperkube.yml.j2 dest={{ kube_manifest_dir }}/kube-api-hyperkube.yml
+
+- name: Write the pod manifest for the scheduler
+  template: src=kube-scheduler-hyperkube.yml.j2 dest={{ kube_standby_manifest_dir }}/kube-scheduler-hyperkube.yml
+
+- name: Write the pod manifest for podmaster
+  template: src=podmaster.yml.j2 dest={{ kube_manifest_dir }}/podmaster.yml
+
+- name: Make sure kube master log files exist
+  file: path=/var/log/{{ item }} state=touch mode=0755
+  with_items:
+    - kube-apiserver.log
+    - kube-controller-manager.log
+    - kube-scheduler.log
+
+- name: write the config files for kubelet
+  template: src=kubelet.j2 dest={{ kube_config_dir }}/kubelet
+  notify:
+    - restart kubelet
+
+- name: Enable kubelet
+  service: name=kubelet enabled=yes state=started

ansible/roles/master/tasks/main.yml

@@ -1,5 +1,7 @@
 ---
-- include: coreos.yml
+- name: CoreOS | Force source_type to github
+  set_fact:
+    source_type: "github-release"
   when: is_coreos
 
 - include: packageManagerInstall.yml
@@ -12,29 +14,21 @@
   tags:
     - binary-update
 
-- name: write the config file for the api server
-  template: src=apiserver.j2 dest={{ kube_config_dir }}/apiserver
-  notify:
-    - restart apiserver
+- name: Make sure manifests directory exists
+  file: path={{ kube_manifest_dir }} state=directory mode=0755 
 
 - name: Ensure that a token auth file exists (addons may populate it)
   file: path={{ kube_token_dir }}/known_tokens.csv state=touch
   changed_when: false
 
-- name: add cap_net_bind_service to kube-apiserver
-  capabilities: path=/usr/bin/kube-apiserver capability=cap_net_bind_service=ep state=present
-  when: not is_atomic and not is_coreos
-
-- name: Enable apiserver
-  service: name=kube-apiserver enabled=yes state=started
-
 - name: Get the master token values
   slurp:
     src: "{{ kube_token_dir }}/{{ item }}-{{ inventory_hostname }}.token"
   with_items:
     - "system:controller_manager"
     - "system:scheduler"
     - "system:kubectl"
+    - "system:kubelet"
   register: tokens
   delegate_to: "{{ groups['masters'][0] }}"
 
@@ -43,53 +37,26 @@
     controller_manager_token: "{{ tokens.results[0].content|b64decode }}"
     scheduler_token: "{{ tokens.results[1].content|b64decode }}"
     kubectl_token: "{{ tokens.results[2].content|b64decode }}"
-
-- name: write the config file for the controller-manager
-  template: src=controller-manager.j2 dest={{ kube_config_dir }}/controller-manager
-  notify:
-    - restart controller-manager
+    kubelet_token: "{{ tokens.results[3].content|b64decode }}"
 
 - name: write the kubecfg (auth) file for controller-manager
   template: src=controller-manager.kubeconfig.j2 dest={{ kube_config_dir }}/controller-manager.kubeconfig
   notify:
     - restart controller-manager
 
-- name: Enable controller-manager
-  service: name=kube-controller-manager enabled=yes state=started
-
-- name: write the config file for the scheduler
-  template: src=scheduler.j2 dest={{ kube_config_dir }}/scheduler
-  notify:
-    - restart scheduler
-
 - name: write the kubecfg (auth) file for scheduler
   template: src=scheduler.kubeconfig.j2 dest={{ kube_config_dir }}/scheduler.kubeconfig
   notify:
     - restart scheduler
 
-- name: Enable scheduler
-  service: name=kube-scheduler enabled=yes state=started
-
 - name: write the kubecfg (auth) file for kubectl
   template: src=kubectl.kubeconfig.j2 dest={{ kube_config_dir }}/kubectl.kubeconfig
 
-# Enable kubelet on master only when OpenContrail is in use; see
-# https://github.com/kubernetes/contrib/pull/183
-- name: write the config files for kubelet
-  template: src=kubelet.j2 dest={{ kube_config_dir }}/kubelet
-  notify:
-    - restart kubelet
-  when: networking == 'opencontrail'
-
-- name: Enable kubelet
-  service: name=kubelet enabled=yes state=started
-  when: networking == 'opencontrail'
-
-- name: write the delay-master-services target
-  copy: src=delay-master-services.target dest=/etc/systemd/system/ mode=0644
+- include: ha_master.yml
+  when: groups['masters']|length > 1
 
-- name: Enable delay-master-services
-  service: name=delay-master-services.target enabled=yes
+- include: single_master.yml
+  when: groups['masters']|length == 1
 
 - include: firewalld.yml
   when: has_firewalld

ansible/roles/master/tasks/single_master.yml

@@ -0,0 +1,51 @@
+- include: coreos.yml
+  when: is_coreos
+
+- name: write the config file for the api server
+  template: src=apiserver.j2 dest={{ kube_config_dir }}/apiserver
+  notify:
+    - restart apiserver
+
+- name: add cap_net_bind_service to kube-apiserver
+  capabilities: path=/usr/bin/kube-apiserver capability=cap_net_bind_service=ep state=present
+  when: not is_atomic and not is_coreos
+
+- name: Enable apiserver
+  service: name=kube-apiserver enabled=yes state=started
+
+- name: write the config file for the controller-manager
+  template: src=controller-manager.j2 dest={{ kube_config_dir }}/controller-manager
+  notify:
+    - restart controller-manager
+
+- name: Enable controller-manager
+  service: name=kube-controller-manager enabled=yes state=started
+
+- name: write the config file for the scheduler
+  template: src=scheduler.j2 dest={{ kube_config_dir }}/scheduler
+  notify:
+    - restart scheduler
+
+- name: Enable scheduler
+  service: name=kube-scheduler enabled=yes state=started
+
+- name: write the config files for kubelet
+  template: src=kubelet.j2 dest={{ kube_config_dir }}/kubelet
+  notify:
+    - restart kubelet
+  when: networking == 'opencontrail'
+
+- name: Enable kubelet
+  service: name=kubelet enabled=yes state=started
+  when: networking == 'opencontrail'
+
+# Enable kubelet on master only when OpenContrail is in use; see
+# https://github.com/kubernetes/contrib/pull/183
+- name: write the delay-master-services target
+  copy: src=delay-master-services.target dest=/etc/systemd/system/ mode=0644
+
+- name: Reload systemd configuration prior to master-services
+  command: systemctl daemon-reload
+
+- name: Enable delay-master-services
+  service: name=delay-master-services.target enabled=yes