rke.go

package ranchhand

import (
	"bytes"
	"context"
	"encoding/base64"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"text/template"
	"time"

	"github.com/pkg/errors"
)

const (
	RKEKubeConfig = "kube_config_rancher-cluster.yml"

	RKEConfigFile = "rancher-cluster.yml"

	RKETemplate = `# DO NOT EDIT THIS FILE - GENERATED BY RANCHHAND
ssh_key_path: {{ .SSH.KeyPath }}
ignore_docker_version: false

nodes:
{{- range .Nodes }}
  - address: {{ .PublicIP }}
  {{- with .PrivateIP }}
    internal_address: {{ . }}
  {{- end }}
    user: {{ $.SSH.User }}
    port: {{ $.SSH.Port }}
    role: [controlplane,worker,etcd]
{{- end }}

services:
  etcd:
    snapshot: true
    creation: 6h
    retention: 24h
  kube-api:
    pod_security_policy: true
    extra_args:
      anonymous-auth: "false"
      profiling: "false"
      service-account-lookup: "true"
      enable-admission-plugins: "ServiceAccount,NamespaceLifecycle,LimitRanger,PersistentVolumeLabel,DefaultStorageClass,ResourceQuota,DefaultTolerationSeconds,AlwaysPullImages,DenyEscalatingExec,NodeRestriction,EventRateLimit,PodSecurityPolicy"
      admission-control-config-file: "{{ .AdmissionControlConfigFile }}"
      audit-log-path: "/var/log/kube-audit/audit-log.json"
      audit-log-maxage: "30"
      audit-log-maxbackup: "10"
      audit-log-maxsize: "100"
      audit-log-format: "json"
      audit-policy-file: "{{ .AuditPolicyFile }}"
    extra_binds:
    - "/var/log/kube-audit:/var/log/kube-audit"
  kube-controller:
    extra_args:
      profiling: "false"
      address: "127.0.0.1"
      terminated-pod-gc-threshold: "1000"
  kubelet:
    extra_args:
      streaming-connection-idle-timeout: "30m"
      protect-kernel-defaults: "false"
      make-iptables-util-chains: "true"
      event-qps: "0"
  scheduler:
    extra_args:
      profiling: "false"
      address: "127.0.0.1"

ingress:
  provider: nginx
  extra_args:
    default-ssl-certificate: ingress-nginx/ingress-default-cert

addons: |
  apiVersion: v1
  kind: Secret
  metadata:
    name: ingress-default-cert
    namespace: ingress-nginx
  type: kubernetes.io/tls
  data:
    tls.crt: {{ .CertPEM | base64Encode }}
    tls.key: {{ .KeyPEM | base64Encode }}
  ---
  apiVersion: rbac.authorization.k8s.io/v1
  kind: Role
  metadata:
    name: default-psp-role
    namespace: ingress-nginx
  rules:
  - apiGroups:
    - extensions
    resourceNames:
    - default-psp
    resources:
    - podsecuritypolicies
    verbs:
    - use
  ---
  apiVersion: rbac.authorization.k8s.io/v1
  kind: RoleBinding
  metadata:
    name: default-psp-rolebinding
    namespace: ingress-nginx
  roleRef:
    apiGroup: rbac.authorization.k8s.io
    kind: Role
    name: default-psp-role
  subjects:
  - apiGroup: rbac.authorization.k8s.io
    kind: Group
    name: system:serviceaccounts
  - apiGroup: rbac.authorization.k8s.io
    kind: Group
    name: system:authenticated
  ---
  apiVersion: v1
  kind: Namespace
  metadata:
    name: cattle-system
  ---
  apiVersion: rbac.authorization.k8s.io/v1
  kind: Role
  metadata:
    name: default-psp-role
    namespace: cattle-system
  rules:
  - apiGroups:
    - extensions
    resourceNames:
    - default-psp
    resources:
    - podsecuritypolicies
    verbs:
    - use
  ---
  apiVersion: rbac.authorization.k8s.io/v1
  kind: RoleBinding
  metadata:
    name: default-psp-rolebinding
    namespace: cattle-system
  roleRef:
    apiGroup: rbac.authorization.k8s.io
    kind: Role
    name: default-psp-role
  subjects:
  - apiGroup: rbac.authorization.k8s.io
    kind: Group
    name: system:serviceaccounts
  - apiGroup: rbac.authorization.k8s.io
    kind: Group
    name: system:authenticated
  ---
  apiVersion: extensions/v1beta1
  kind: PodSecurityPolicy
  metadata:
    name: restricted
  spec:
    requiredDropCapabilities:
    - NET_RAW
    privileged: false
    allowPrivilegeEscalation: false
    defaultAllowPrivilegeEscalation: false
    fsGroup:
      rule: RunAsAny
    runAsUser:
      rule: MustRunAsNonRoot
    seLinux:
      rule: RunAsAny
    supplementalGroups:
      rule: RunAsAny
    volumes:
    - emptyDir
    - secret
    - persistentVolumeClaim
    - downwardAPI
    - configMap
    - projected
  ---
  apiVersion: rbac.authorization.k8s.io/v1
  kind: ClusterRole
  metadata:
    name: psp:restricted
  rules:
  - apiGroups:
    - extensions
    resourceNames:
    - restricted
    resources:
    - podsecuritypolicies
    verbs:
    - use
  ---
  apiVersion: rbac.authorization.k8s.io/v1
  kind: ClusterRoleBinding
  metadata:
    name: psp:restricted
  roleRef:
    apiGroup: rbac.authorization.k8s.io
    kind: ClusterRole
    name: psp:restricted
  subjects:
  - apiGroup: rbac.authorization.k8s.io
    kind: Group
    name: system:serviceaccounts
  - apiGroup: rbac.authorization.k8s.io
    kind: Group
    name: system:authenticated
`
)

var tpl *template.Template

type tmplData struct {
	*Config
	CertPEM, KeyPEM                             []byte
	AdmissionControlConfigFile, AuditPolicyFile string
}

func launchRKE(cfg *Config, certPEM, keyPEM []byte) error {
	// render config template
	var buf bytes.Buffer
	tplData := tmplData{
		Config:                     cfg,
		CertPEM:                    certPEM,
		KeyPEM:                     keyPEM,
		AdmissionControlConfigFile: k8sConfigs["admission"].filename,
		AuditPolicyFile:            k8sConfigs["audit"].filename,
	}
	if err := tpl.Execute(&buf, tplData); err != nil {
		return errors.Wrap(err, "rke template render failed")
	}
	tplContents := buf.Bytes()

	// decided if file needs to be [over]written
	var writeConfig, saveState bool
	_, err := os.Stat(RKEConfigFile)
	switch {
	case os.IsNotExist(err):
		writeConfig = true
	case cfg.UpgradeKubernetes:
		configContents, err := ioutil.ReadFile(RKEConfigFile)
		if err != nil {
			return errors.Wrap(err, "rke config read failed")
		}
		writeConfig = !bytes.Equal(configContents, tplContents)
		saveState = writeConfig
	}

	// write config to file
	if writeConfig {
		err := ioutil.WriteFile(RKEConfigFile, tplContents, 0644)
		if err != nil {
			return errors.Wrap(err, "rke config write failed")
		}
	}

	// take a snapshot of etcd state before applying upgrade
	if saveState {
		snapshotName := fmt.Sprintf("snapshot-%s", time.Now().UTC().Format(time.RFC3339))
		cmd := exec.Command("rke", "etcd", "snapshot-save", "--name", snapshotName, "--config", RKEConfigFile)
		cmd.Stdout = os.Stdout
		cmd.Stderr = os.Stderr

		log.Info("taking snapshot of etcd state prior to upgrade")
		if err := cmd.Run(); err != nil {
			return errors.Wrap(cmd.Run(), "cannot snapshot etcd")
		}
	}

	// run `rke up` if file was just written or cluster has not converged
	if writeConfig || hasNotConverged() {
		cmd := exec.Command("rke", "up", "--config", RKEConfigFile)
		cmd.Stdout = os.Stdout
		cmd.Stderr = os.Stderr

		log.Info("launching rke up")
		return errors.Wrap(cmd.Run(), "cannot install kubernetes")
	}

	return nil
}

func hasNotConverged() bool {
	ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
	defer cancel()

	cmd := exec.CommandContext(ctx, "rke", "version", "--config", RKEConfigFile)
	return cmd.Run() != nil
}

func init() {
	tpl = template.New("rke-tmpl")
	tpl.Funcs(template.FuncMap{
		"base64Encode": func(bs []byte) string {
			return base64.StdEncoding.EncodeToString(bs)
		},
	})
	template.Must(tpl.Parse(RKETemplate))
}