user-data

#cloud-config

hostname: k8solo-01

coreos:
  update:
    reboot-strategy: off
  etcd2:
    name: k8solo-01
    data-dir: /data/var/lib/etcd2
    initial-advertise-peer-urls: http://k8solo-01:2380
    initial-cluster-token: k8solo-01_etcd
    initial-cluster: k8solo-01=http://k8solo-01:2380
    initial-cluster-state: new
    listen-peer-urls: http://k8solo-01:2380,http://k8solo-01:7001
    listen-client-urls: http://0.0.0.0:2379,http://0.0.0.0:4001
    advertise-client-urls:  http://k8solo-01:2379,http://k8solo-01:4001
  units:
    - name: docker-tcp.socket
      command: start
      enable: true
      content: |
        [Unit]
        Description=Docker Socket for the API

        [Socket]
        ListenStream=2375
        BindIPv6Only=both
        Service=docker.service

        [Install]
        WantedBy=sockets.target
    - name: format-persistent-disk.service
      command: start
      content: |
        [Unit]
        Description=Formats the persistent drive (if not formated yet)
        [Service]
        Type=oneshot
        RemainAfterExit=yes
        ExecStart=/bin/sh -c '/usr/sbin/blkid -pi /dev/vda | grep TYPE= || /usr/sbin/mkfs.ext4 -i 8192 -L k8s_data /dev/vda'
    - name: data.mount
      command: start
      content: |
        [Unit]
        Description=Mounts ephemeral to /data
        Requires=format-persistent-disk.service etcd2.service docker.service rkt-metadata.socket
        After=format-persistent-disk.service
        Before=etcd2.service docker.service rkt-metadata.socket
        [Mount]
        What=/dev/vda
        Where=/data
        Type=ext4
    - name: persistent-data-checks.service
      command: start
      content: |
        [Unit]
        Description=prepare for etcd,docker,rkt,opt,kubernetes
        Requires=data.mount etcd2.service docker.service rkt-metadata.socket
        Before=etcd2.service docker.service rkt-metadata.socket
        After=data.mount
        [Service]
        Type=oneshot
        RemainAfterExit=yes
        ExecStart=/bin/sh -c '[[ -d /data/var/lib/etcd2 ]] || mkdir -p /data/var/lib/etcd2 && /usr/bin/chown etcd:etcd /data/var/lib/etcd2'
        ExecStart=/bin/sh -c '/usr/bin/chown etcd:etcd /data/var/lib/etcd2'
        ExecStart=/bin/sh -c '[[ -d /data/var/lib/docker ]] || mkdir -p /data/var/lib/docker'
        ExecStart=/bin/sh -c '[[ -d /data/var/lib/rkt ]] || mkdir -p /data/var/lib/rkt && /usr/bin/chown root:rkt /data/var/lib/rkt && /usr/bin/chmod g+s /data/var/lib/rkt'
        ExecStart=/bin/sh -c '/usr/bin/chown root:rkt /data/var/lib/rkt && /usr/bin/chmod g+s /data/var/lib/rkt'
        ExecStart=/bin/sh -c '[[ -d /data/opt/bin ]] || mkdir -p /data/opt/bin'
        ExecStart=/bin/sh -c '[[ -d /data/kubernetes/manifests ]] || mkdir -p /data/kubernetes/manifests && chmod 777 -R /data/kubernetes'
        ExecStart=/bin/sh -c '[[ -d /data/kubelet ]] || mkdir -p /data/kubelet'
        ExecStart=/bin/sh -c 'sudo ln -s /data/kubelet /var/lib/kubelet'
        ExecStart=/bin/sh -c 'sudo mkdir -p /opt/bin/ && sudo ln -s /data/opt/bin/socat /opt/bin/socat'
    - name: var-lib-docker.mount
      command: start
      content: |
        [Unit]
        Description=Binds /data/var/lib/docker to /var/lib/docker
        After=persistent-data-checks.service
        Requires=persistent-data-checks.service docker.service
        Before=docker.service
        [Mount]
        What=/data/var/lib/docker
        Where=/var/lib/docker
        Type=ext4
        Options=bind
        [Install]
        RequiredBy=docker.service
    - name: var-lib-rkt.mount
      command: start
      content: |
        [Unit]
        Description=Binds /data/var/lib/rkt to /var/lib/rkt
        Requires=persistent-data-checks.service rkt-metadata.socket
        After=persistent-data-checks.service
        Before=rkt-metadata.socket
        [Mount]
        What=/data/var/lib/rkt
        Where=/var/lib/rkt
        Type=ext4
        Options=bind
        [Install]
        RequiredBy=rkt-metadata.socket
    - name: etcd2.service
      command: start
    - name: fleet.service
      command: stop
    - name: flanneld.service
      command: start
      drop-ins:
      - name: 50-network-config.conf
        content: |
          [Unit]
          Requires=etcd2.service
          [Service]
          ExecStartPre=/usr/bin/etcdctl set /coreos.com/network/config '{"Network": "10.244.0.0/16", "SubnetLen": 24, "SubnetMin": "10.244.0.0", "Backend": {"Type": "vxlan"}}'
    - name: docker.service
      command: start
      drop-ins:
        - name: 50-insecure-registry.conf
          content: |
            [Unit]
            Requires=flanneld.service
            [Service]
            Environment="DOCKER_OPTS=--insecure-registry=0.0.0.0/0 --storage-driver=overlay2"
    - name: update-engine.service
      command: stop
    - name: kube-certs.service
      command: start
      content: |
        [Unit]
        Description=Generate Kubernetes API Server certificates
        ConditionPathExists=/opt/sbin/make-certs.sh
        Requires=network-online.target persistent-data-checks.service etcd2.service
        After=network-online.target persistent-data-checks.service etcd2.service
        ConditionPathExists=!/data/kubernetes/ca.crt
        [Service]
        EnvironmentFile=/etc/environment
        ExecStartPre=-/usr/sbin/groupadd -r kube-cert
        ExecStartPre=/usr/bin/chmod 755 /opt/sbin/make-certs.sh
        ExecStart=/opt/sbin/make-certs.sh ${COREOS_PUBLIC_IPV4} IP:${COREOS_PUBLIC_IPV4},IP:10.100.0.1,DNS:kubernetes,DNS:kubernetes.default,DNS:kubernetes.default.svc,DNS:kubernetes.default.svc.cluster.local
        ExecStartPost=/bin/bash -c 'sleep 20'
        Type=oneshot
        RemainAfterExit=true
    - name: kube-apiserver.service
      command: start
      content: |
        [Unit]
        Description=Kubernetes API Server
        Documentation=https://github.com/GoogleCloudPlatform/kubernetes
        ConditionPathExists=/data/opt/bin/kube-apiserver
        Requires=etcd2.service docker.service flanneld.service kube-certs.service
        After=etcd2.service docker.service flanneld.service kube-certs.service
        [Service]
        LimitNOFILE=491521
        # Check for certs
        ExecStartPre=/bin/sh -c 'while [ ! -f /data/kubernetes/ca.crt ]; do sleep 1; done'
        ExecStartPre=/bin/sh -c 'while [ ! -f /data/kubernetes/server.cert ]; do sleep 1; done'
        ExecStartPre=/bin/sh -c 'while [ ! -f /data/kubernetes/server.key ]; do sleep 1; done'
        # Check for binary
        ExecStartPre=/bin/sh -c 'while [ ! -f /data/opt/bin/kube-apiserver ]; do sleep 1; done'
        # Start API server
        ExecStart=/data/opt/bin/kube-apiserver \
          --client-ca-file=/data/kubernetes/ca.crt \
          --tls-cert-file=/data/kubernetes/server.cert \
          --tls-private-key-file=/data/kubernetes/server.key \
          --admission_control=NamespaceLifecycle,NamespaceAutoProvision,LimitRanger,ServiceAccount,ResourceQuota \
          --service_account_key_file=/data/kubernetes/server.key \
          --service_account_lookup=false \
          --allow_privileged=true \
          --insecure_bind_address=0.0.0.0 \
          --insecure_port=8080 \
          --kubelet_https=true \
          --secure_port=6443 \
          --service-cluster-ip-range=10.100.0.0/16 \
          --etcd_servers=http://127.0.0.1:2379 \
          --bind-address=0.0.0.0 \
          --logtostderr=true
        Restart=always
        RestartSec=5
    - name: kube-controller-manager.service
      command: start
      content: |
        [Unit]
        Description=Kubernetes Controller Manager
        Documentation=https://github.com/GoogleCloudPlatform/kubernetes
        Requires=kube-apiserver.service
        After=kube-apiserver.service
        [Service]
        # Check for binary
        ExecStartPre=/bin/sh -c 'while [ ! -f /data/opt/bin/kube-controller-manager ]; do sleep 1; done'
        # Wait for api server
        ExecStartPre=/opt/sbin/wupiao 127.0.0.1:8080
        ExecStart=/data/opt/bin/kube-controller-manager \
          --root-ca-file=/data/kubernetes/ca.crt \
          --service-account-private-key-file=/data/kubernetes/server.key \
          --master=127.0.0.1:8080 \
          --pod_eviction_timeout=30s \
          --logtostderr=true
        Restart=always
        RestartSec=10
    - name: kube-scheduler.service
      command: start
      content: |
        [Unit]
        Description=Kubernetes API Server
        Documentation=https://github.com/GoogleCloudPlatform/kubernetes
        Requires=kube-apiserver.service
        After=kube-apiserver.service
        [Service]
        # Check for binary
        ExecStartPre=/bin/sh -c 'while [ ! -f /data/opt/bin/kube-scheduler ]; do sleep 1; done'
        # Wait for api server
        ExecStartPre=/opt/sbin/wupiao 127.0.0.1:8080
        ExecStart=/data/opt/bin/kube-scheduler \
          --master=127.0.0.1:8080 \
          --logtostderr=true
        Restart=always
        RestartSec=10
    - name: kube-proxy.service
      command: start
      content: |
        [Unit]
        Description=Kubernetes Proxy
        Documentation=https://github.com/GoogleCloudPlatform/kubernetes
        [Service]
        # Check for binary
        ExecStartPre=/bin/sh -c 'while [ ! -f /data/opt/bin/kube-proxy ]; do sleep 1; done'
        # Wait for api server
        ExecStartPre=/opt/sbin/wupiao 127.0.0.1:8080
        ExecStart=/data/opt/bin/kube-proxy \
          --master=http://127.0.0.1:8080 \
          --logtostderr=true
        Restart=always
        RestartSec=10
    - name: kube-kubelet.service
      command: start
      content: |
        [Unit]
        Description=Kubernetes Kubelet
        Documentation=https://github.com/GoogleCloudPlatform/kubernetes
        [Service]
        EnvironmentFile=/etc/environment
        Environment=PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/opt/bin
        # small hacks around for Deis Workflow
        ExecStartPre=-/bin/sh -c 'sudo mkdir -p /data/deis/minio && sudo chown 106:107 /data/deis/minio'
        # Check for binary
        ExecStartPre=/bin/sh -c 'while [ ! -f /data/opt/bin/kubelet ]; do sleep 1; done'
        # Wait for api server
        ExecStartPre=/opt/sbin/wupiao 127.0.0.1:8080
        #
        ExecStart=/data/opt/bin/kubelet \
          --address=0.0.0.0 \
          --port=10250 \
          --hostname_override=${HOSTNAME} \
          --register-node=true \
          --container_runtime=docker \
          --api_servers=http://127.0.0.1:8080 \
          --allow_privileged=true \
          --cluster_dns=10.100.0.10 \
          --cluster_domain=cluster.local \
          --logtostderr=true \
          --config=/data/kubernetes/manifests \
          --cadvisor_port=4194 \
          --healthz_bind_address=0.0.0.0 \
          --healthz_port=10248
        Restart=always
        RestartSec=10
write-files:
 - path: /opt/sbin/wupiao
   permissions: '0755'
   content: |
      #!/bin/bash
      # [w]ait [u]ntil [p]ort [i]s [a]ctually [o]pen
      [ -n "$1" ] && \
        until curl -o /dev/null -sIf http://${1}; do \
          sleep 1 && echo .;
        done;
      exit $?
 - path: /opt/sbin/make-certs.sh
   permissions: '0755'
   content: |
      #!/bin/sh

      # Copyright 2014 The Kubernetes Authors All rights reserved.
      #

      #set -o errexit
      #set -o nounset
      #set -o pipefail

      cert_ip=$1
      extra_sans=${2:-}
      cert_dir=/data/kubernetes
      cert_group=kube-cert

      mkdir -p "$cert_dir"

      use_cn=false

      sans="IP:${cert_ip}"
      if [[ -n "${extra_sans}" ]]; then
        sans="${sans},${extra_sans}"
      fi

      tmpdir=$(mktemp -d --tmpdir kubernetes_cacert.XXXXXX)
      cd $tmpdir

      trap 'rm -rf "${tmpdir}"' EXIT
      #
      if [ ! -f /data/opt/tmp/easy-rsa.tar.gz ]
      then
         curl -L -o /data/opt/tmp/easy-rsa.tar.gz https://storage.googleapis.com/kubernetes-release/easy-rsa/easy-rsa.tar.gz > /dev/null 2>&1
         tar xzf /data/opt/tmp/easy-rsa.tar.gz > /dev/null 2>&1
      else
         tar xzf /data/opt/tmp/easy-rsa.tar.gz > /dev/null 2>&1
      fi

      cd easy-rsa-master/easyrsa3
      ./easyrsa init-pki > /dev/null 2>&1
      ./easyrsa --batch "--req-cn=$cert_ip@`date +%s`" build-ca nopass > /dev/null 2>&1
      if [ $use_cn = "true" ]; then
          ./easyrsa build-server-full $cert_ip nopass > /dev/null 2>&1
          cp -p pki/issued/$cert_ip.crt "${cert_dir}/server.cert" > /dev/null 2>&1
          cp -p pki/private/$cert_ip.key "${cert_dir}/server.key" > /dev/null 2>&1
      else
          ./easyrsa --subject-alt-name="${sans}" build-server-full kubernetes-master nopass > /dev/null 2>&1
          cp -p pki/issued/kubernetes-master.crt "${cert_dir}/server.cert" > /dev/null 2>&1
          cp -p pki/private/kubernetes-master.key "${cert_dir}/server.key" > /dev/null 2>&1
      fi
      ./easyrsa build-client-full kubecfg nopass > /dev/null 2>&1
      cp -p pki/ca.crt "${cert_dir}/ca.crt"
      cp -p pki/issued/kubecfg.crt "${cert_dir}/kubecfg.crt"
      cp -p pki/private/kubecfg.key "${cert_dir}/kubecfg.key"

      # Make server certs accessible to apiserver.
      chgrp $cert_group "${cert_dir}/server.key" "${cert_dir}/server.cert" "${cert_dir}/ca.crt"
      chmod 660 "${cert_dir}/server.key" "${cert_dir}/server.cert" "${cert_dir}/ca.crt"
 - path: /opt/sbin/restart-kube-units.sh
   permissions: '0755'
   content: |
      #!/bin/bash
      systemctl restart kube-apiserver
      systemctl restart kube-controller-manager
      systemctl restart kube-scheduler
      systemctl restart kube-kubelet
      systemctl restart kube-proxy