corekube-cloudservers.yaml

 heat_template_version: 2013-05-23

 description: >
   Deploy Kubernetes on a CoreOS cluster that operates on
   Rackspace's Public Cloud (Virtual Machine/Server)
   Maintainer - Mike Metral <metral@gmail.com> | Github: @metral

 parameters:
   kubernetes_minion_count:
     description: Number of CoreOS machines to deploy as Kubernetes Minion
     type: number
     default: 3
     constraints:
     - range:
         min: 1
         max: 12
       description: Must be between 2 and 12 servers.
   keyname:
     type: string
     description: Name of keypair to be used for compute instance
   flavor:
     type: string
     default: 4 GB General Purpose v1
     constraints:
     - allowed_values:
       - 4 GB General Purpose v1
       - 8 GB General Purpose v1
       description: |
         Must be a valid Rackspace Cloud Server flavor for the region you have
         selected to deploy into.
   coreos_image:
     type: string
     description: Rackspace Cloud Servers CoreOS Stable (1010.5.0) UUID
     default: "3eba4fbb-51da-4233-b699-8a4030561add"
   git_command:
     type: string
     description: Git repo checkout command
     default: "/usr/bin/git clone https://github.com/metral/overlord ; /usr/bin/git -C overlord checkout -qf 54bc1daacc502c89a97ca68051ba336065022bd1"
   flannel_url:
     type: string
     description: Flannel (0.5.5) Binary URL
     default: "https://storage.googleapis.com/flanneld/flanneld"
   private_discovery_etcd_version:
     type: string
     description: Version of etcd used for Private Discovery node
     default: "2.3.1"
   discovery_net_interface:
     type: string
     description: Host network interface for which the Discovery node will operate over
     default: "eth1"
   kubernetes_net_interface:
     type: string
     description: Host network interface for which Kubernetes' overlay will operate over
     default: "eth2"

 resources:

   coreos_cluster_uuid:
     type: OS::Heat::RandomString

   priv_network:
     type: Rackspace::Cloud::Network
     properties:
       label: kubernetes
       cidr: 192.168.3.0/24

   discovery:
     type: OS::Nova::Server
     properties:
       key_name: { get_param: keyname }
       image: { get_param: coreos_image }
       flavor: { get_param: flavor }
       name: "discovery"
       user_data_format: RAW
       config_drive: "true"
       user_data:
         str_replace:
           template: |
             #cloud-config
             write_files:
               - path: /run/get_discovery_interface_ip.sh
                 permissions: '0755'
                 content: |
                   #!/bin/bash
                   # Get's the IP of the interface that discovery will be
                   # accessible over
                   DISCOVERY_IF=%discovery_net_interface%
                   /usr/bin/ip -4 addr show $DISCOVERY_IF | /usr/bin/awk '/inet/ {print $2}' | /usr/bin/cut -d/ -f1 > /run/IP
                   /usr/bin/sed -i 's/^/IP=/' /run/IP
             coreos:
               update:
                 group: stable
                 reboot-strategy: off
               units:
                 - name: private-discovery-setup.service
                   command: start
                   content: |
                     [Unit]
                     After=network-online.target
                     Requires=network-online.target

                     [Service]
                     ExecStart=/usr/bin/bash /run/get_discovery_interface_ip.sh
                 - name: private-discovery.service
                   command: start
                   content: |
                     [Unit]
                     After=network-online.target private-discovery-setup.service
                     Requires=network-online.target private-discovery-setup.service

                     [Service]
                     EnvironmentFile=/run/IP
                     Restart=on-failure
                     RestartSec=5s
                     ExecStartPre=/usr/bin/docker pull quay.io/coreos/etcd:v%private_discovery_etcd_version%
                     ExecStart=/usr/bin/docker run -d --name discovery \
                       -p 2379:2379 -p 2380:2380 \
                       -v /usr/share/ca-certificates/:/etc/ssl/certs \
                       --net host quay.io/coreos/etcd:v%private_discovery_etcd_version% -name discovery \
                       -initial-advertise-peer-urls http://${IP}:2380,http://${IP}:7001 \
                       -listen-peer-urls http://${IP}:2380,http://${IP}:7001 \
                       -initial-cluster discovery=http://${IP}:2380,discovery=http://${IP}:7001 \
                       -advertise-client-urls http://${IP}:2379,http://${IP}:4001 \
                       -listen-client-urls http://0.0.0.0:2379,http://0.0.0.0:4001
           params:
             "%discovery_net_interface%": { get_param: discovery_net_interface }
             "%private_discovery_etcd_version%": { get_param: private_discovery_etcd_version}

   overlord:
     type: OS::Nova::Server
     properties:
       key_name: { get_param: keyname }
       image: { get_param: coreos_image }
       flavor: { get_param: flavor }
       name: "overlord"
       user_data_format: RAW
       config_drive: "true"
       user_data:
         str_replace:
           template: |
             #cloud-config
             write_files:
               - path: /run/get_discovery_ip_port.sh
                 permissions: '0755'
                 content: |
                   #!/bin/bash
                   # Sets up environment file with the discovery node's IP &
                   # port so that redeploy.sh has it
                   /usr/bin/cat /run/systemd/system/etcd.service.d/20-cloudinit.conf | /usr/bin/grep -i discovery | /usr/bin/cut -f3 -d"=" | /usr/bin/awk -F '/v' '{print $1}' > /run/discovery_ip_port
                   /usr/bin/sed -i 's/^/DISCOVERY_IP_PORT=/' /run/discovery_ip_port
             coreos:
               etcd:
                 name: overlord
                 discovery: http://%discovery%:2379/v2/keys/discovery/%uuid%
                 addr: $private_ipv4:4001
                 peer-addr: $private_ipv4:7001
               update:
                 group: stable
                 reboot-strategy: off
               units:
                 - name: etcd.service
                   command: start
                 - name: fleet.socket
                   command: start
                   content: |
                     [Socket]
                     # Talk to the API over a Unix domain socket (default)
                     ListenStream=/var/run/fleet.sock

                     # Talk to the API over an exposed port
                     ListenStream=10001
                     Service=fleet-local.service

                     [Install]
                     WantedBy=sockets.target
                 - name: fleet-local.service
                   command: start
                   content: |
                     # fleet-local is kicked off by fleet.socket after API port
                     # is opened
                     [Unit]
                     Description=fleet-local
                     Wants=etcd.service
                     After=etcd.service

                     [Service]
                     Environment=FLEET_PUBLIC_IP=$private_ipv4
                     Environment=FLEET_METADATA=kubernetes_role=overlord
                     ExecStart=/usr/bin/fleet
                     Restart=always
                     RestartSec=10s
                 - name: get_discovery_ip_port.service
                   runtime: true
                   command: start
                   content: |
                     # Runs get_discovery_ip_port.sh to have discovery IP &
                     # port ready for consumption by reploy.sh when creating
                     [Unit]
                     After=network-online.target
                     Requires=network-online.target

                     [Service]
                     ExecStart=/usr/bin/bash /run/get_discovery_ip_port.sh
                 - name: overlord.service
                   command: start
                   content: |
                     # Overlord / logic layer service to deploy kubernetes to
                     # the cluster
                     [Unit]
                     After=network-online.target
                     Requires=network-online.target

                     [Service]
                     WorkingDirectory=/root
                     Environment="DIR=overlord"
                     ExecStartPre=/usr/bin/rm -rf $DIR
                     ExecStartPre=%git_command%
                     ExecStart=/usr/bin/bash ${DIR}/build_run.sh
           params:
             "%discovery%": { get_attr: [discovery, networks, private, 0] }
             "%uuid%": { get_attr: [coreos_cluster_uuid, value] }
             "%git_command%": { get_param: git_command }

   kubernetes_master:
     type: OS::Nova::Server
     properties:
       key_name: { get_param: keyname }
       image: { get_param: coreos_image }
       flavor: { get_param: flavor }
       name: "kubernetes_master"
       networks:
       - network: "00000000-0000-0000-0000-000000000000"
       - network: "11111111-1111-1111-1111-111111111111"
       - network: { get_resource: priv_network }
       user_data_format: RAW
       config_drive: "true"
       user_data:
         str_replace:
           template: |
             #cloud-config
             write_files:
               - path: /run/get_discovery_ip_port.sh
                 permissions: '0755'
                 content: |
                   #!/bin/bash
                   # Sets up environment file with the discovery node's IP &
                   # port so # that in Overlode's template 
                   # master-apiserver@.service it can be passed 
                   # in as an argument
                   /usr/bin/cat /run/systemd/system/etcd.service.d/20-cloudinit.conf | /usr/bin/grep -i discovery | /usr/bin/cut -f3 -d"=" | /usr/bin/awk -F '/v' '{print $1}' > /run/discovery_ip_port
                   /usr/bin/sed -i 's/^/DISCOVERY_IP_PORT=/' /run/discovery_ip_port
             coreos:
               etcd:
                 name: kubernetes_master
                 discovery: http://%discovery%:2379/v2/keys/discovery/%uuid%
                 addr: $private_ipv4:4001
                 peer-addr: $private_ipv4:7001
               fleet:
                 public-ip: $private_ipv4
                 metadata: kubernetes_role=master
               update:
                 group: stable
                 reboot-strategy: off
               units:
                 - name: etcd.service
                   command: start
                 - name: fleet.service
                   command: start
                 - name: get_discovery_ip_port.service
                   runtime: true
                   command: start
                   content: |
                     # Runs get_discovery_ip_port.sh to have discovery IP &
                     # port ready for consumption by overlord when creating
                     # the Kubernetes' master-api@.service template
                     [Unit]
                     After=network-online.target
                     Requires=network-online.target

                     [Service]
                     ExecStart=/usr/bin/bash /run/get_discovery_ip_port.sh
                 - name: flannel-install.service
                   command: start
                   content: |
                     # Installs flannel
                     [Unit]
                     After=network-online.target
                     Requires=network-online.target

                     [Service]
                     Type=oneshot
                     RemainAfterExit=yes
                     ExecStart=/usr/bin/wget -N -P /opt/bin %flannel_url%
                     ExecStart=/usr/bin/chmod +x /opt/bin/flanneld
                 - name: flannel.service
                   command: start
                   content: |
                     # Configures & starts flannel
                     [Unit]
                     After=network-online.target etcd.service flannel-install.service
                     Requires=network-online.target etcd.service flannel-install.service

                     [Service]
                     ExecStartPre=/usr/bin/etcdctl --endpoint=%discovery%:4001 mk /coreos.com/network/config '{"Network":"10.244.0.0/15", "Backend": {"Type": "vxlan"}}'
                     ExecStart=/opt/bin/flanneld -etcd-endpoints=http://%discovery%:4001 -iface=%kubernetes_net_interface%
                     Restart=always
                     RestartSec=5s
                 - name: generate-k8s-certs.service
                   command: start
                   content: |
                     [Unit]
                     Description=Generate Kubernetes API Server certificates
                     ConditionPathExists=!/srv/kubernetes/.certs.lock
                     Requires=network-online.target
                     After=network-online.target

                     [Service]
                     Type=oneshot
                     ExecStartPre=-/usr/sbin/groupadd -r kube-cert
                     ExecStartPre=/usr/bin/wget -q -N -P /opt/bin https://gist.githubusercontent.com/metral/0731466d7b35d4020dfd/raw/2fb0c5858e5eaefdf6b45b9492f1d6c75a7815b0/make-ca-cert.sh
                     ExecStartPre=/usr/bin/chmod u=rwx,go= /opt/bin/make-ca-cert.sh
                     ExecStart=/opt/bin/make-ca-cert.sh $private_ipv4 IP:10.1.0.1,IP:127.0.0.1,DNS:kubernetes,DNS:kubernetes.default,DNS:kubernetes.default.svc,DNS:kubernetes.default.svc.freeletics.local,DNS:localhost
                     RemainAfterExit=true
           params:
             "%discovery%": { get_attr: [discovery, networks, private, 0] }
             "%uuid%": { get_attr: [coreos_cluster_uuid, value] }
             "%flannel_url%": { get_param: flannel_url }
             "%kubernetes_net_interface%": { get_param: kubernetes_net_interface }

   kubernetes_minions:
     type: "OS::Heat::ResourceGroup"
     properties:
       count: { get_param: kubernetes_minion_count }
       resource_def:
         type: OS::Nova::Server
         properties:
           key_name: { get_param: keyname }
           image: { get_param: coreos_image }
           flavor: { get_param: flavor }
           name: kubernetes_minion_%index%
           networks:
           - network: "00000000-0000-0000-0000-000000000000"
           - network: "11111111-1111-1111-1111-111111111111"
           - network: { get_resource: priv_network }
           user_data_format: RAW
           config_drive: "true"
           user_data:
             str_replace:
               template: |
                 #cloud-config
                 coreos:
                   etcd:
                     name: kubernetes_minion_%index%
                     discovery: http://%discovery%:2379/v2/keys/discovery/%uuid%
                     addr: $private_ipv4:4001
                     peer-addr: $private_ipv4:7001
                   fleet:
                     public-ip: $private_ipv4
                     metadata: kubernetes_role=minion
                   update:
                     group: stable
                     reboot-strategy: off
                   units:
                     - name: etcd.service
                       command: start
                     - name: fleet.service
                       command: start
                     - name: flannel-install.service
                       command: start
                       content: |
                         # Installs flannel
                         [Unit]
                         After=network-online.target
                         Requires=network-online.target

                         [Service]
                         Type=oneshot
                         RemainAfterExit=yes
                         ExecStart=/usr/bin/wget -N -P /opt/bin %flannel_url%
                         ExecStart=/usr/bin/chmod +x /opt/bin/flanneld
                     - name: flannel.service
                       command: start
                       content: |
                         # Configures & starts flannel
                         [Unit]
                         After=etcd.service flannel-install.service
                         Requires=etcd.service flannel-install.service

                         [Service]
                         ExecStart=/opt/bin/flanneld -etcd-endpoints=http://%discovery%:4001 -iface=%kubernetes_net_interface%
                         Restart=always
                         RestartSec=5s
                     - name: flannel-env.path
                       command: start
                       content: |
                         # Ensures flannel env vars are set to use with Docker
                         [Path]
                         PathExists=/run/flannel/subnet.env
                         Unit=docker.service
                     - name: docker.service
                       command: start
                       content: |
                         # Starts new docker server that uses flannel 
                         [Unit]
                         After=flannel-env.path network-online.target flannel.service
                         Requires=flannel-env.path network-online.target flannel.service
                         Description=Docker Application Container Engine

                         [Service]
                         EnvironmentFile=/run/flannel/subnet.env
                         ExecStartPre=/bin/mount --make-rprivate /
                         ExecStartPre=/usr/bin/systemctl kill docker.service
                         ExecStart=/usr/bin/docker daemon --bip=${FLANNEL_SUBNET} --mtu=${FLANNEL_MTU}

                         [Install]
                         WantedBy=multi-user.target
               params:
                 "%discovery%": { get_attr: [discovery, networks, private, 0] }
                 "%uuid%": { get_attr: [coreos_cluster_uuid, value] }
                 "%flannel_url%": { get_param: flannel_url }
                 "%kubernetes_net_interface%": { get_param: kubernetes_net_interface }

 outputs:
   discovery_ip:
     value: { get_attr: [ discovery, accessIPv4 ] }
     description: The IP of the Discovery
   overlord_ip:
     value: { get_attr: [ overlord, accessIPv4 ] }
     description: The IP of the Overlord
   master_ip:
     value: { get_attr: [ kubernetes_master, accessIPv4 ] }
     description: The IP of the Kubernetes Master
   minion_ips:
     value: { get_attr: [ kubernetes_minions, accessIPv4 ] }
     description: The IP of the Kubernetes Minions