kubermatic · mrIncompetent · Oct 13, 2018 · Oct 6, 2018
diff --git a/pkg/template/functions.go b/pkg/template/functions.go
diff --git a/pkg/test/helper.go b/pkg/test/helper.go
@@ -0,0 +1,41 @@
+package test
+
+import (
+	"io/ioutil"
+	"path/filepath"
+	"testing"
+
+	"github.com/pmezard/go-difflib/difflib"
+)
+
+func CompareOutput(t *testing.T, name, output string, update bool) {
+	golden, err := filepath.Abs(filepath.Join("testdata", name+".golden"))
+	if err != nil {
+		t.Fatalf("failed to get absolute path to goldan file: %v", err)
+	}
+	if update {
+		if err := ioutil.WriteFile(golden, []byte(output), 0644); err != nil {
+			t.Fatalf("failed to write updated fixture: %v", err)
+		}
+	}
+	expected, err := ioutil.ReadFile(golden)
+	if err != nil {
+		t.Fatalf("failed to read .golden file: %v", err)
+	}
+
+	diff := difflib.UnifiedDiff{
+		A:        difflib.SplitLines(string(expected)),
+		B:        difflib.SplitLines(output),
+		FromFile: "Fixture",
+		ToFile:   "Current",
+		Context:  3,
+	}
+	diffStr, err := difflib.GetUnifiedDiffString(diff)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if diffStr != "" {
+		t.Errorf("got diff between expected and actual result: \n%s\n", diffStr)
+	}
+}
diff --git a/pkg/userdata/centos/testdata/kubelet-v1.10-aws.golden b/pkg/userdata/centos/testdata/kubelet-v1.10-aws.golden
@@ -8,24 +8,26 @@ write_files:
   content: |
     [Journal]
     SystemMaxUse=5G
+
+
+- path: "/etc/modules-load.d/k8s.conf"
+  content: |
+    ip_vs
+    ip_vs_rr
+    ip_vs_wrr
+    ip_vs_sh
+    nf_conntrack_ipv4
+
 
 - path: "/etc/sysctl.d/k8s.conf"
   content: |
     net.bridge.bridge-nf-call-ip6tables = 1
     net.bridge.bridge-nf-call-iptables = 1
     kernel.panic_on_oops = 1
     kernel.panic = 10
+    net.ipv4.ip_forward = 1
     vm.overcommit_memory = 1
-
-- path: "/etc/yum.repos.d/kubernetes.repo"
-  content: |
-    [kubernetes]
-    name=Kubernetes
-    baseurl=https://packages.cloud.google.com/yum/repos/kubernetes-el7-$basearch
-    enabled=1
-    gpgcheck=1
-    repo_gpgcheck=1
-    gpgkey=https://packages.cloud.google.com/yum/doc/yum-key.gpg https://packages.cloud.google.com/yum/doc/rpm-package-key.gpg
+
 
 - path: /etc/sysconfig/selinux
   content: |
@@ -41,72 +43,58 @@ write_files:
     #     mls - Multi Level Security protection.
     SELINUXTYPE=targeted
 
-- path: "/etc/sysconfig/kubelet-overwrite"
-  content: |
-    KUBELET_DNS_ARGS=
-    KUBELET_EXTRA_ARGS=--authentication-token-webhook=true \
-      --cloud-provider=aws \
-      --cloud-config=/etc/kubernetes/cloud-config \
-      --hostname-override=node1 \
-      --read-only-port=0 \
-      --protect-kernel-defaults=true \
-      --cluster-dns= \
-      --cluster-domain=cluster.local
-- path: "/etc/systemd/system/kubelet.service.d/20-extra.conf"
-  content: |
-    [Service]
-    EnvironmentFile=/etc/sysconfig/kubelet
-
-- path: "/etc/kubernetes/cloud-config"
-  content: |
-    {aws-config:true}
-
-- path: "/usr/local/bin/setup"
-  permissions: "0755"
+- path: "/opt/bin/setup"
+  permissions: "0777"
   content: |
     #!/bin/bash
     set -xeuo pipefail
+
     setenforce 0 || true
+
+    # As we added some modules and don't want to reboot, restart the service 
+    systemctl restart systemd-modules-load.service
     sysctl --system
 
     yum install -y docker-1.13.1 \
-      kubelet-1.10.2 \
-      kubeadm-1.10.2 \
       ebtables \
       ethtool \
       nfs-utils \
       bash-completion \
-      sudo
-
-    cp /etc/sysconfig/kubelet-overwrite /etc/sysconfig/kubelet
-
-    systemctl enable --now docker
-    systemctl enable --now kubelet
-
-    if [[ ! -x /usr/local/bin/health-monitor.sh ]]; then
-      curl -Lfo /usr/local/bin/health-monitor.sh \
-        https://raw.githubusercontent.com/kubermatic/machine-controller/8b5b66e4910a6228dfaecccaa0a3b05ec4902f8e/pkg/userdata/scripts/health-monitor.sh
-      chmod +x /usr/local/bin/health-monitor.sh
+      sudo \
+      socat \
+      wget \
+      curl \
+      ipvsadm
+
+    #setup some common directories
+    mkdir -p /opt/bin/
+    mkdir -p /var/lib/calico
+    mkdir -p /etc/kubernetes/manifests
+    mkdir -p /etc/cni/net.d
+    mkdir -p /opt/cni/bin
+
+    # cni
+    if [ ! -f /opt/cni/bin/loopback ]; then
+        curl -L https://github.com/containernetworking/plugins/releases/download/v0.6.0/cni-plugins-amd64-v0.6.0.tgz | tar -xvzC /opt/cni/bin -f -
     fi
-
-    if ! [[ -e /etc/kubernetes/pki/ca.crt ]]; then
-      kubeadm join \
-        --token my-token \
-        --discovery-token-ca-cert-hash sha256:6caecce9fedcb55d4953d61a27dc6997361a2f226ad86d7e6004dde7526fc4b1 \
-        --ignore-preflight-errors=CRI \
-        server:443
+    # kubelet
+    if [ ! -f /opt/bin/kubelet ]; then
+        curl -Lfo /opt/bin/kubelet https://storage.googleapis.com/kubernetes-release/release/v1.10.2/bin/linux/amd64/kubelet
+        chmod +x /opt/bin/kubelet
     fi
-
-    if [[ ! -x /usr/local/bin/health-monitor.sh ]]; then
-      curl -Lfo /usr/local/bin/health-monitor.sh \
-        https://raw.githubusercontent.com/kubermatic/machine-controller/8b5b66e4910a6228dfaecccaa0a3b05ec4902f8e/pkg/userdata/scripts/health-monitor.sh
-      chmod +x /usr/local/bin/health-monitor.sh
+
+    if [[ ! -x /opt/bin/health-monitor.sh ]]; then
+        curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/8b5b66e4910a6228dfaecccaa0a3b05ec4902f8e/pkg/userdata/scripts/health-monitor.sh
+        chmod +x /opt/bin/health-monitor.sh
     fi
+
 
+    systemctl enable --now docker
+    systemctl enable --now kubelet
     systemctl enable --now --no-block kubelet-healthcheck.service
     systemctl enable --now --no-block docker-healthcheck.service
 
-- path: "/usr/local/bin/supervise.sh"
+- path: "/opt/bin/supervise.sh"
   permissions: "0755"
   content: |
     #!/bin/bash
@@ -115,7 +103,108 @@ write_files:
       sleep 1
     done
 
+- path: "/etc/systemd/system/kubelet.service"
+  content: |
+    [Unit]
+    After=docker.service
+    Requires=docker.service
+
+    Description=kubelet: The Kubernetes Node Agent
+    Documentation=https://kubernetes.io/docs/home/
+
+    [Service]
+    Restart=always
+    StartLimitInterval=0
+    RestartSec=10
+
+    Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/"
+
+    ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \
+      --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \
+      --kubeconfig=/etc/kubernetes/kubelet.conf \
+      --pod-manifest-path=/etc/kubernetes/manifests \
+      --allow-privileged=true \
+      --network-plugin=cni \
+      --cni-conf-dir=/etc/cni/net.d \
+      --cni-bin-dir=/opt/cni/bin \
+      --authorization-mode=Webhook \
+      --client-ca-file=/etc/kubernetes/pki/ca.crt \
+      --cadvisor-port=0 \
+      --rotate-certificates=true \
+      --cert-dir=/etc/kubernetes/pki \
+      --authentication-token-webhook=true \
+      --cloud-provider=aws \
+      --cloud-config=/etc/kubernetes/cloud-config \
+      --hostname-override=node1 \
+      --read-only-port=0 \
+      --exit-on-lock-contention \
+      --lock-file=/tmp/kubelet.lock \
+      --anonymous-auth=false \
+      --protect-kernel-defaults=true \
+      --cluster-dns= \
+      --cluster-domain=cluster.local
+
+    [Install]
+    WantedBy=multi-user.target
+
+- path: "/etc/systemd/system/kubelet.service.d/extras.conf"
+  content: |
+    [Service]
+    Environment="KUBELET_EXTRA_ARGS=--cgroup-driver=systemd"
+
+- path: "/etc/kubernetes/cloud-config"
+  content: |
+    {aws-config:true}
+
+- path: "/etc/kubernetes/bootstrap-kubelet.conf"
+  content: |
+    apiVersion: v1
+    clusters:
+    - cluster:
+        certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t
+        server: https://server:443
+      name: ""
+    contexts: []
+    current-context: ""
+    kind: Config
+    preferences: {}
+    users:
+    - name: ""
+      user:
+        token: my-token
+
+
+- path: "/etc/kubernetes/pki/ca.crt"
+  content: |
+    -----BEGIN CERTIFICATE-----
+    MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV
+    BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG
+    A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3
+    DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0
+    NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG
+    cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv
+    c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B
+    AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS
+    R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT
+    ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk
+    JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3
+    mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW
+    caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G
+    A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt
+    hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB
+    MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES
+    MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv
+    bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h
+    U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao
+    eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4
+    UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD
+    58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n
+    sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF
+    kPe6XoSbiLm/kxk32T0=
+    -----END CERTIFICATE-----
+
 - path: "/etc/systemd/system/setup.service"
+  permissions: "0644"
   content: |
     [Install]
     WantedBy=multi-user.target
@@ -127,33 +216,40 @@ write_files:
     [Service]
     Type=oneshot
     RemainAfterExit=true
-    ExecStart=/usr/local/bin/supervise.sh /usr/local/bin/setup
+    ExecStart=/opt/bin/supervise.sh /opt/bin/setup
+
+- path: "/etc/profile.d/opt-bin-path.sh"
+  permissions: "0644"
+  content: |
+    export PATH="/opt/bin:$PATH"
 
 - path: /etc/systemd/system/kubelet-healthcheck.service
   permissions: "0644"
   content: |
     [Unit]
     Requires=kubelet.service
     After=kubelet.service
-
+    
     [Service]
-    ExecStart=/usr/local/bin/health-monitor.sh kubelet
-
+    ExecStart=/opt/bin/health-monitor.sh kubelet
+    
     [Install]
     WantedBy=multi-user.target
+
 
 - path: /etc/systemd/system/docker-healthcheck.service
   permissions: "0644"
   content: |
     [Unit]
     Requires=docker.service
     After=docker.service
-
+    
     [Service]
-    ExecStart=/usr/local/bin/health-monitor.sh container-runtime
-
+    ExecStart=/opt/bin/health-monitor.sh container-runtime
+    
     [Install]
     WantedBy=multi-user.target
+
 
 runcmd:
 - systemctl enable --now setup.service