From 321070791bcf457b37404a0aa07b62964e397358 Mon Sep 17 00:00:00 2001 From: mrIncompetent Date: Sat, 6 Oct 2018 13:19:11 +0200 Subject: [PATCH] consolidate kubelet flags and remove kubeadm --- pkg/template/functions.go | 28 -- pkg/test/helper.go | 41 +++ .../centos/testdata/kubelet-v1.10-aws.golden | 226 ++++++++---- .../centos/testdata/kubelet-v1.11-aws.golden | 222 +++++++---- .../centos/testdata/kubelet-v1.12-aws.golden | 221 +++++++---- .../centos/testdata/kubelet-v1.9-aws.golden | 226 ++++++++---- pkg/userdata/centos/userdata.go | 195 ++++------ pkg/userdata/centos/userdata_test.go | 30 +- ...-openstack-kubelet-v-version-prefix.golden | 47 ++- ...-auto-update-openstack-multiple-dns.golden | 47 ++- .../v1.11.2-vsphere-static-ipconfig.golden | 47 ++- ....12.0-vsphere-overwrite-cloudconfig.golden | 47 ++- .../v1.9.2-disable-auto-update-aws.golden | 47 ++- pkg/userdata/coreos/userdata.go | 105 ++---- pkg/userdata/helper/common_test.go | 19 + .../helper/download_binaries_script.go | 59 +++ .../helper/download_binaries_script_test.go | 22 ++ pkg/userdata/helper/helper.go | 70 ++-- pkg/userdata/helper/kubelet.go | 140 +++++++ pkg/userdata/helper/kubelet_test.go | 62 ++++ pkg/userdata/helper/template_functions.go | 23 ++ .../testdata/download_binaries_v1.10.0.golden | 21 ++ .../download_binaries_v1.11.0-rc.2.golden | 21 ++ .../testdata/download_binaries_v1.11.0.golden | 21 ++ .../testdata/download_binaries_v1.11.3.golden | 21 ++ .../testdata/download_binaries_v1.12.0.golden | 21 ++ ...let_systemd_unit_cloud-provider-set.golden | 41 +++ ...t_systemd_unit_multiple-dns-servers.golden | 39 ++ ...kublet_systemd_unit_version-v1.10.0.golden | 39 ++ ...t_systemd_unit_version-v1.11.0-rc.2.golden | 39 ++ ...kublet_systemd_unit_version-v1.11.0.golden | 39 ++ ...kublet_systemd_unit_version-v1.11.3.golden | 39 ++ ...kublet_systemd_unit_version-v1.12.0.golden | 38 ++ pkg/userdata/test-machines/v1.10.8.yaml | 108 ++++++ pkg/userdata/test-machines/v1.11.3.yaml | 108 ++++++ pkg/userdata/test-machines/v1.12.1.yaml | 108 ++++++ pkg/userdata/test-machines/v1.9.11.yaml | 108 ++++++ pkg/userdata/ubuntu/testdata/1.11-aws.golden | 277 -------------- .../1.9.2-dist-upgrade-on-boot-aws.golden | 285 --------------- .../1.9.2-openstack-multiple-dns.golden | 285 --------------- .../testdata/dist-upgrade-on-boot.golden | 343 +++++++++++++++++ .../kubelet-version-without-v-prefix.golden | 342 +++++++++++++++++ .../testdata/multiple-dns-servers.golden | 342 +++++++++++++++++ .../ubuntu/testdata/multiple-ssh-keys.golden | 344 ++++++++++++++++++ .../openstack-kubelet-v-version-prefix.golden | 285 --------------- .../openstack-overwrite-cloud-config.golden | 321 +++++++++------- pkg/userdata/ubuntu/testdata/openstack.golden | 344 ++++++++++++++++++ .../ubuntu/testdata/version-1.10.10.golden | 342 +++++++++++++++++ .../ubuntu/testdata/version-1.11.3.golden | 342 +++++++++++++++++ .../ubuntu/testdata/version-1.12.1.golden | 341 +++++++++++++++++ .../ubuntu/testdata/version-1.9.10.golden | 342 +++++++++++++++++ pkg/userdata/ubuntu/userdata.go | 329 +++++++---------- pkg/userdata/ubuntu/userdata_test.go | 158 ++++---- test/e2e/provisioning/helper.go | 8 +- 54 files changed, 5616 insertions(+), 2149 deletions(-) delete mode 100644 pkg/template/functions.go create mode 100644 pkg/test/helper.go create mode 100644 pkg/userdata/helper/common_test.go create mode 100644 pkg/userdata/helper/download_binaries_script.go create mode 100644 pkg/userdata/helper/download_binaries_script_test.go create mode 100644 pkg/userdata/helper/kubelet.go create mode 100644 pkg/userdata/helper/kubelet_test.go create mode 100644 pkg/userdata/helper/template_functions.go create mode 100644 pkg/userdata/helper/testdata/download_binaries_v1.10.0.golden create mode 100644 pkg/userdata/helper/testdata/download_binaries_v1.11.0-rc.2.golden create mode 100644 pkg/userdata/helper/testdata/download_binaries_v1.11.0.golden create mode 100644 pkg/userdata/helper/testdata/download_binaries_v1.11.3.golden create mode 100644 pkg/userdata/helper/testdata/download_binaries_v1.12.0.golden create mode 100644 pkg/userdata/helper/testdata/kublet_systemd_unit_cloud-provider-set.golden create mode 100644 pkg/userdata/helper/testdata/kublet_systemd_unit_multiple-dns-servers.golden create mode 100644 pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.10.0.golden create mode 100644 pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.11.0-rc.2.golden create mode 100644 pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.11.0.golden create mode 100644 pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.11.3.golden create mode 100644 pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.12.0.golden create mode 100644 pkg/userdata/test-machines/v1.10.8.yaml create mode 100644 pkg/userdata/test-machines/v1.11.3.yaml create mode 100644 pkg/userdata/test-machines/v1.12.1.yaml create mode 100644 pkg/userdata/test-machines/v1.9.11.yaml delete mode 100644 pkg/userdata/ubuntu/testdata/1.11-aws.golden delete mode 100644 pkg/userdata/ubuntu/testdata/1.9.2-dist-upgrade-on-boot-aws.golden delete mode 100644 pkg/userdata/ubuntu/testdata/1.9.2-openstack-multiple-dns.golden create mode 100644 pkg/userdata/ubuntu/testdata/dist-upgrade-on-boot.golden create mode 100644 pkg/userdata/ubuntu/testdata/kubelet-version-without-v-prefix.golden create mode 100644 pkg/userdata/ubuntu/testdata/multiple-dns-servers.golden create mode 100644 pkg/userdata/ubuntu/testdata/multiple-ssh-keys.golden delete mode 100644 pkg/userdata/ubuntu/testdata/openstack-kubelet-v-version-prefix.golden create mode 100644 pkg/userdata/ubuntu/testdata/openstack.golden create mode 100644 pkg/userdata/ubuntu/testdata/version-1.10.10.golden create mode 100644 pkg/userdata/ubuntu/testdata/version-1.11.3.golden create mode 100644 pkg/userdata/ubuntu/testdata/version-1.12.1.golden create mode 100644 pkg/userdata/ubuntu/testdata/version-1.9.10.golden diff --git a/pkg/template/functions.go b/pkg/template/functions.go deleted file mode 100644 index 3f149b2b6..000000000 --- a/pkg/template/functions.go +++ /dev/null @@ -1,28 +0,0 @@ -package template - -import ( - "fmt" - "net" - "strings" - "text/template" - - "github.com/Masterminds/sprig" -) - -func ipSliceToCommaSeparatedString(ips []net.IP) string { - var s string - for _, ip := range ips { - s = s + fmt.Sprintf("%s,", ip.String()) - } - - return strings.TrimSuffix(s, ",") -} - -// TxtFuncMap returns an aggregated template function map. Currently (custom functions + sprig) -func TxtFuncMap() template.FuncMap { - funcMap := sprig.TxtFuncMap() - - funcMap["ipSliceToCommaSeparatedString"] = ipSliceToCommaSeparatedString - - return funcMap -} diff --git a/pkg/test/helper.go b/pkg/test/helper.go new file mode 100644 index 000000000..abb56335b --- /dev/null +++ b/pkg/test/helper.go @@ -0,0 +1,41 @@ +package test + +import ( + "io/ioutil" + "path/filepath" + "testing" + + "github.com/pmezard/go-difflib/difflib" +) + +func CompareOutput(t *testing.T, name, output string, update bool) { + golden, err := filepath.Abs(filepath.Join("testdata", name+".golden")) + if err != nil { + t.Fatalf("failed to get absolute path to goldan file: %v", err) + } + if update { + if err := ioutil.WriteFile(golden, []byte(output), 0644); err != nil { + t.Fatalf("failed to write updated fixture: %v", err) + } + } + expected, err := ioutil.ReadFile(golden) + if err != nil { + t.Fatalf("failed to read .golden file: %v", err) + } + + diff := difflib.UnifiedDiff{ + A: difflib.SplitLines(string(expected)), + B: difflib.SplitLines(output), + FromFile: "Fixture", + ToFile: "Current", + Context: 3, + } + diffStr, err := difflib.GetUnifiedDiffString(diff) + if err != nil { + t.Fatal(err) + } + + if diffStr != "" { + t.Errorf("got diff between expected and actual result: \n%s\n", diffStr) + } +} diff --git a/pkg/userdata/centos/testdata/kubelet-v1.10-aws.golden b/pkg/userdata/centos/testdata/kubelet-v1.10-aws.golden index 752fe597b..a5bfa5566 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.10-aws.golden +++ b/pkg/userdata/centos/testdata/kubelet-v1.10-aws.golden @@ -8,6 +8,16 @@ write_files: content: | [Journal] SystemMaxUse=5G + + +- path: "/etc/modules-load.d/k8s.conf" + content: | + ip_vs + ip_vs_rr + ip_vs_wrr + ip_vs_sh + nf_conntrack_ipv4 + - path: "/etc/sysctl.d/k8s.conf" content: | @@ -15,17 +25,9 @@ write_files: net.bridge.bridge-nf-call-iptables = 1 kernel.panic_on_oops = 1 kernel.panic = 10 + net.ipv4.ip_forward = 1 vm.overcommit_memory = 1 - -- path: "/etc/yum.repos.d/kubernetes.repo" - content: | - [kubernetes] - name=Kubernetes - baseurl=https://packages.cloud.google.com/yum/repos/kubernetes-el7-$basearch - enabled=1 - gpgcheck=1 - repo_gpgcheck=1 - gpgkey=https://packages.cloud.google.com/yum/doc/yum-key.gpg https://packages.cloud.google.com/yum/doc/rpm-package-key.gpg + - path: /etc/sysconfig/selinux content: | @@ -41,72 +43,58 @@ write_files: # mls - Multi Level Security protection. SELINUXTYPE=targeted -- path: "/etc/sysconfig/kubelet-overwrite" - content: | - KUBELET_DNS_ARGS= - KUBELET_EXTRA_ARGS=--authentication-token-webhook=true \ - --cloud-provider=aws \ - --cloud-config=/etc/kubernetes/cloud-config \ - --hostname-override=node1 \ - --read-only-port=0 \ - --protect-kernel-defaults=true \ - --cluster-dns= \ - --cluster-domain=cluster.local -- path: "/etc/systemd/system/kubelet.service.d/20-extra.conf" - content: | - [Service] - EnvironmentFile=/etc/sysconfig/kubelet - -- path: "/etc/kubernetes/cloud-config" - content: | - {aws-config:true} - -- path: "/usr/local/bin/setup" - permissions: "0755" +- path: "/opt/bin/setup" + permissions: "0777" content: | #!/bin/bash set -xeuo pipefail + setenforce 0 || true + + # As we added some modules and don't want to reboot, restart the service + systemctl restart systemd-modules-load.service sysctl --system yum install -y docker-1.13.1 \ - kubelet-1.10.2 \ - kubeadm-1.10.2 \ ebtables \ ethtool \ nfs-utils \ bash-completion \ - sudo - - cp /etc/sysconfig/kubelet-overwrite /etc/sysconfig/kubelet - - systemctl enable --now docker - systemctl enable --now kubelet - - if [[ ! -x /usr/local/bin/health-monitor.sh ]]; then - curl -Lfo /usr/local/bin/health-monitor.sh \ - https://raw.githubusercontent.com/kubermatic/machine-controller/8b5b66e4910a6228dfaecccaa0a3b05ec4902f8e/pkg/userdata/scripts/health-monitor.sh - chmod +x /usr/local/bin/health-monitor.sh + sudo \ + socat \ + wget \ + curl \ + ipvsadm + + #setup some common directories + mkdir -p /opt/bin/ + mkdir -p /var/lib/calico + mkdir -p /etc/kubernetes/manifests + mkdir -p /etc/cni/net.d + mkdir -p /opt/cni/bin + + # cni + if [ ! -f /opt/cni/bin/loopback ]; then + curl -L https://github.com/containernetworking/plugins/releases/download/v0.6.0/cni-plugins-amd64-v0.6.0.tgz | tar -xvzC /opt/cni/bin -f - fi - - if ! [[ -e /etc/kubernetes/pki/ca.crt ]]; then - kubeadm join \ - --token my-token \ - --discovery-token-ca-cert-hash sha256:6caecce9fedcb55d4953d61a27dc6997361a2f226ad86d7e6004dde7526fc4b1 \ - --ignore-preflight-errors=CRI \ - server:443 + # kubelet + if [ ! -f /opt/bin/kubelet ]; then + curl -Lfo /opt/bin/kubelet https://storage.googleapis.com/kubernetes-release/release/v1.10.2/bin/linux/amd64/kubelet + chmod +x /opt/bin/kubelet fi - - if [[ ! -x /usr/local/bin/health-monitor.sh ]]; then - curl -Lfo /usr/local/bin/health-monitor.sh \ - https://raw.githubusercontent.com/kubermatic/machine-controller/8b5b66e4910a6228dfaecccaa0a3b05ec4902f8e/pkg/userdata/scripts/health-monitor.sh - chmod +x /usr/local/bin/health-monitor.sh + + if [[ ! -x /opt/bin/health-monitor.sh ]]; then + curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/8b5b66e4910a6228dfaecccaa0a3b05ec4902f8e/pkg/userdata/scripts/health-monitor.sh + chmod +x /opt/bin/health-monitor.sh fi + + systemctl enable --now docker + systemctl enable --now kubelet systemctl enable --now --no-block kubelet-healthcheck.service systemctl enable --now --no-block docker-healthcheck.service -- path: "/usr/local/bin/supervise.sh" +- path: "/opt/bin/supervise.sh" permissions: "0755" content: | #!/bin/bash @@ -115,7 +103,108 @@ write_files: sleep 1 done +- path: "/etc/systemd/system/kubelet.service" + content: | + [Unit] + After=docker.service + Requires=docker.service + + Description=kubelet: The Kubernetes Node Agent + Documentation=https://kubernetes.io/docs/home/ + + [Service] + Restart=always + StartLimitInterval=0 + RestartSec=10 + + Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" + + ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ + --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ + --kubeconfig=/etc/kubernetes/kubelet.conf \ + --pod-manifest-path=/etc/kubernetes/manifests \ + --allow-privileged=true \ + --network-plugin=cni \ + --cni-conf-dir=/etc/cni/net.d \ + --cni-bin-dir=/opt/cni/bin \ + --authorization-mode=Webhook \ + --client-ca-file=/etc/kubernetes/pki/ca.crt \ + --cadvisor-port=0 \ + --rotate-certificates=true \ + --cert-dir=/etc/kubernetes/pki \ + --authentication-token-webhook=true \ + --cloud-provider=aws \ + --cloud-config=/etc/kubernetes/cloud-config \ + --hostname-override=node1 \ + --read-only-port=0 \ + --exit-on-lock-contention \ + --lock-file=/tmp/kubelet.lock \ + --anonymous-auth=false \ + --protect-kernel-defaults=true \ + --cluster-dns= \ + --cluster-domain=cluster.local + + [Install] + WantedBy=multi-user.target + +- path: "/etc/systemd/system/kubelet.service.d/extras.conf" + content: | + [Service] + Environment="KUBELET_EXTRA_ARGS=--cgroup-driver=systemd" + +- path: "/etc/kubernetes/cloud-config" + content: | + {aws-config:true} + +- path: "/etc/kubernetes/bootstrap-kubelet.conf" + content: | + apiVersion: v1 + clusters: + - cluster: + certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t + server: https://server:443 + name: "" + contexts: [] + current-context: "" + kind: Config + preferences: {} + users: + - name: "" + user: + token: my-token + + +- path: "/etc/kubernetes/pki/ca.crt" + content: | + -----BEGIN CERTIFICATE----- + MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV + BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG + A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 + DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 + NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG + cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv + c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B + AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS + R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT + ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk + JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 + mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW + caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G + A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt + hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB + MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES + MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv + bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h + U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao + eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 + UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD + 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n + sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF + kPe6XoSbiLm/kxk32T0= + -----END CERTIFICATE----- + - path: "/etc/systemd/system/setup.service" + permissions: "0644" content: | [Install] WantedBy=multi-user.target @@ -127,7 +216,12 @@ write_files: [Service] Type=oneshot RemainAfterExit=true - ExecStart=/usr/local/bin/supervise.sh /usr/local/bin/setup + ExecStart=/opt/bin/supervise.sh /opt/bin/setup + +- path: "/etc/profile.d/opt-bin-path.sh" + permissions: "0644" + content: | + export PATH="/opt/bin:$PATH" - path: /etc/systemd/system/kubelet-healthcheck.service permissions: "0644" @@ -135,12 +229,13 @@ write_files: [Unit] Requires=kubelet.service After=kubelet.service - + [Service] - ExecStart=/usr/local/bin/health-monitor.sh kubelet - + ExecStart=/opt/bin/health-monitor.sh kubelet + [Install] WantedBy=multi-user.target + - path: /etc/systemd/system/docker-healthcheck.service permissions: "0644" @@ -148,12 +243,13 @@ write_files: [Unit] Requires=docker.service After=docker.service - + [Service] - ExecStart=/usr/local/bin/health-monitor.sh container-runtime - + ExecStart=/opt/bin/health-monitor.sh container-runtime + [Install] WantedBy=multi-user.target + runcmd: - systemctl enable --now setup.service diff --git a/pkg/userdata/centos/testdata/kubelet-v1.11-aws.golden b/pkg/userdata/centos/testdata/kubelet-v1.11-aws.golden index 15b16ac3f..47f0f564a 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.11-aws.golden +++ b/pkg/userdata/centos/testdata/kubelet-v1.11-aws.golden @@ -8,6 +8,16 @@ write_files: content: | [Journal] SystemMaxUse=5G + + +- path: "/etc/modules-load.d/k8s.conf" + content: | + ip_vs + ip_vs_rr + ip_vs_wrr + ip_vs_sh + nf_conntrack_ipv4 + - path: "/etc/sysctl.d/k8s.conf" content: | @@ -15,17 +25,9 @@ write_files: net.bridge.bridge-nf-call-iptables = 1 kernel.panic_on_oops = 1 kernel.panic = 10 + net.ipv4.ip_forward = 1 vm.overcommit_memory = 1 - -- path: "/etc/yum.repos.d/kubernetes.repo" - content: | - [kubernetes] - name=Kubernetes - baseurl=https://packages.cloud.google.com/yum/repos/kubernetes-el7-$basearch - enabled=1 - gpgcheck=1 - repo_gpgcheck=1 - gpgkey=https://packages.cloud.google.com/yum/doc/yum-key.gpg https://packages.cloud.google.com/yum/doc/rpm-package-key.gpg + - path: /etc/sysconfig/selinux content: | @@ -41,68 +43,58 @@ write_files: # mls - Multi Level Security protection. SELINUXTYPE=targeted -- path: "/etc/sysconfig/kubelet-overwrite" - content: | - KUBELET_DNS_ARGS= - KUBELET_EXTRA_ARGS=--authentication-token-webhook=true \ - --cloud-provider=aws \ - --cloud-config=/etc/kubernetes/cloud-config \ - --hostname-override=node1 \ - --read-only-port=0 \ - --protect-kernel-defaults=true \ - --cluster-dns= \ - --cluster-domain=cluster.local - -- path: "/etc/kubernetes/cloud-config" - content: | - {aws-config:true} - -- path: "/usr/local/bin/setup" - permissions: "0755" +- path: "/opt/bin/setup" + permissions: "0777" content: | #!/bin/bash set -xeuo pipefail + setenforce 0 || true + + # As we added some modules and don't want to reboot, restart the service + systemctl restart systemd-modules-load.service sysctl --system yum install -y docker-1.13.1 \ - kubelet-1.11.3 \ - kubeadm-1.11.3 \ ebtables \ ethtool \ nfs-utils \ bash-completion \ - sudo - - cp /etc/sysconfig/kubelet-overwrite /etc/sysconfig/kubelet - - systemctl enable --now docker - systemctl enable --now kubelet - - if [[ ! -x /usr/local/bin/health-monitor.sh ]]; then - curl -Lfo /usr/local/bin/health-monitor.sh \ - https://raw.githubusercontent.com/kubermatic/machine-controller/8b5b66e4910a6228dfaecccaa0a3b05ec4902f8e/pkg/userdata/scripts/health-monitor.sh - chmod +x /usr/local/bin/health-monitor.sh + sudo \ + socat \ + wget \ + curl \ + ipvsadm + + #setup some common directories + mkdir -p /opt/bin/ + mkdir -p /var/lib/calico + mkdir -p /etc/kubernetes/manifests + mkdir -p /etc/cni/net.d + mkdir -p /opt/cni/bin + + # cni + if [ ! -f /opt/cni/bin/loopback ]; then + curl -L https://github.com/containernetworking/plugins/releases/download/v0.6.0/cni-plugins-amd64-v0.6.0.tgz | tar -xvzC /opt/cni/bin -f - fi - - if ! [[ -e /etc/kubernetes/pki/ca.crt ]]; then - kubeadm join \ - --token my-token \ - --discovery-token-ca-cert-hash sha256:6caecce9fedcb55d4953d61a27dc6997361a2f226ad86d7e6004dde7526fc4b1 \ - --ignore-preflight-errors=CRI \ - server:443 + # kubelet + if [ ! -f /opt/bin/kubelet ]; then + curl -Lfo /opt/bin/kubelet https://storage.googleapis.com/kubernetes-release/release/v1.11.3/bin/linux/amd64/kubelet + chmod +x /opt/bin/kubelet fi - - if [[ ! -x /usr/local/bin/health-monitor.sh ]]; then - curl -Lfo /usr/local/bin/health-monitor.sh \ - https://raw.githubusercontent.com/kubermatic/machine-controller/8b5b66e4910a6228dfaecccaa0a3b05ec4902f8e/pkg/userdata/scripts/health-monitor.sh - chmod +x /usr/local/bin/health-monitor.sh + + if [[ ! -x /opt/bin/health-monitor.sh ]]; then + curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/8b5b66e4910a6228dfaecccaa0a3b05ec4902f8e/pkg/userdata/scripts/health-monitor.sh + chmod +x /opt/bin/health-monitor.sh fi + + systemctl enable --now docker + systemctl enable --now kubelet systemctl enable --now --no-block kubelet-healthcheck.service systemctl enable --now --no-block docker-healthcheck.service -- path: "/usr/local/bin/supervise.sh" +- path: "/opt/bin/supervise.sh" permissions: "0755" content: | #!/bin/bash @@ -111,7 +103,108 @@ write_files: sleep 1 done +- path: "/etc/systemd/system/kubelet.service" + content: | + [Unit] + After=docker.service + Requires=docker.service + + Description=kubelet: The Kubernetes Node Agent + Documentation=https://kubernetes.io/docs/home/ + + [Service] + Restart=always + StartLimitInterval=0 + RestartSec=10 + + Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" + + ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ + --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ + --kubeconfig=/etc/kubernetes/kubelet.conf \ + --pod-manifest-path=/etc/kubernetes/manifests \ + --allow-privileged=true \ + --network-plugin=cni \ + --cni-conf-dir=/etc/cni/net.d \ + --cni-bin-dir=/opt/cni/bin \ + --authorization-mode=Webhook \ + --client-ca-file=/etc/kubernetes/pki/ca.crt \ + --cadvisor-port=0 \ + --rotate-certificates=true \ + --cert-dir=/etc/kubernetes/pki \ + --authentication-token-webhook=true \ + --cloud-provider=aws \ + --cloud-config=/etc/kubernetes/cloud-config \ + --hostname-override=node1 \ + --read-only-port=0 \ + --exit-on-lock-contention \ + --lock-file=/tmp/kubelet.lock \ + --anonymous-auth=false \ + --protect-kernel-defaults=true \ + --cluster-dns= \ + --cluster-domain=cluster.local + + [Install] + WantedBy=multi-user.target + +- path: "/etc/systemd/system/kubelet.service.d/extras.conf" + content: | + [Service] + Environment="KUBELET_EXTRA_ARGS=--cgroup-driver=systemd" + +- path: "/etc/kubernetes/cloud-config" + content: | + {aws-config:true} + +- path: "/etc/kubernetes/bootstrap-kubelet.conf" + content: | + apiVersion: v1 + clusters: + - cluster: + certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t + server: https://server:443 + name: "" + contexts: [] + current-context: "" + kind: Config + preferences: {} + users: + - name: "" + user: + token: my-token + + +- path: "/etc/kubernetes/pki/ca.crt" + content: | + -----BEGIN CERTIFICATE----- + MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV + BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG + A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 + DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 + NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG + cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv + c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B + AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS + R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT + ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk + JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 + mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW + caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G + A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt + hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB + MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES + MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv + bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h + U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao + eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 + UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD + 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n + sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF + kPe6XoSbiLm/kxk32T0= + -----END CERTIFICATE----- + - path: "/etc/systemd/system/setup.service" + permissions: "0644" content: | [Install] WantedBy=multi-user.target @@ -123,7 +216,12 @@ write_files: [Service] Type=oneshot RemainAfterExit=true - ExecStart=/usr/local/bin/supervise.sh /usr/local/bin/setup + ExecStart=/opt/bin/supervise.sh /opt/bin/setup + +- path: "/etc/profile.d/opt-bin-path.sh" + permissions: "0644" + content: | + export PATH="/opt/bin:$PATH" - path: /etc/systemd/system/kubelet-healthcheck.service permissions: "0644" @@ -131,12 +229,13 @@ write_files: [Unit] Requires=kubelet.service After=kubelet.service - + [Service] - ExecStart=/usr/local/bin/health-monitor.sh kubelet - + ExecStart=/opt/bin/health-monitor.sh kubelet + [Install] WantedBy=multi-user.target + - path: /etc/systemd/system/docker-healthcheck.service permissions: "0644" @@ -144,12 +243,13 @@ write_files: [Unit] Requires=docker.service After=docker.service - + [Service] - ExecStart=/usr/local/bin/health-monitor.sh container-runtime - + ExecStart=/opt/bin/health-monitor.sh container-runtime + [Install] WantedBy=multi-user.target + runcmd: - systemctl enable --now setup.service diff --git a/pkg/userdata/centos/testdata/kubelet-v1.12-aws.golden b/pkg/userdata/centos/testdata/kubelet-v1.12-aws.golden index 137e1db24..b739f1d85 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.12-aws.golden +++ b/pkg/userdata/centos/testdata/kubelet-v1.12-aws.golden @@ -8,6 +8,16 @@ write_files: content: | [Journal] SystemMaxUse=5G + + +- path: "/etc/modules-load.d/k8s.conf" + content: | + ip_vs + ip_vs_rr + ip_vs_wrr + ip_vs_sh + nf_conntrack_ipv4 + - path: "/etc/sysctl.d/k8s.conf" content: | @@ -15,17 +25,9 @@ write_files: net.bridge.bridge-nf-call-iptables = 1 kernel.panic_on_oops = 1 kernel.panic = 10 + net.ipv4.ip_forward = 1 vm.overcommit_memory = 1 - -- path: "/etc/yum.repos.d/kubernetes.repo" - content: | - [kubernetes] - name=Kubernetes - baseurl=https://packages.cloud.google.com/yum/repos/kubernetes-el7-$basearch - enabled=1 - gpgcheck=1 - repo_gpgcheck=1 - gpgkey=https://packages.cloud.google.com/yum/doc/yum-key.gpg https://packages.cloud.google.com/yum/doc/rpm-package-key.gpg + - path: /etc/sysconfig/selinux content: | @@ -41,68 +43,58 @@ write_files: # mls - Multi Level Security protection. SELINUXTYPE=targeted -- path: "/etc/sysconfig/kubelet-overwrite" - content: | - KUBELET_DNS_ARGS= - KUBELET_EXTRA_ARGS=--authentication-token-webhook=true \ - --cloud-provider=aws \ - --cloud-config=/etc/kubernetes/cloud-config \ - --hostname-override=node1 \ - --read-only-port=0 \ - --protect-kernel-defaults=true \ - --cluster-dns= \ - --cluster-domain=cluster.local - -- path: "/etc/kubernetes/cloud-config" - content: | - {aws-config:true} - -- path: "/usr/local/bin/setup" - permissions: "0755" +- path: "/opt/bin/setup" + permissions: "0777" content: | #!/bin/bash set -xeuo pipefail + setenforce 0 || true + + # As we added some modules and don't want to reboot, restart the service + systemctl restart systemd-modules-load.service sysctl --system yum install -y docker-1.13.1 \ - kubelet-1.12.0 \ - kubeadm-1.12.0 \ ebtables \ ethtool \ nfs-utils \ bash-completion \ - sudo - - cp /etc/sysconfig/kubelet-overwrite /etc/sysconfig/kubelet - - systemctl enable --now docker - systemctl enable --now kubelet - - if [[ ! -x /usr/local/bin/health-monitor.sh ]]; then - curl -Lfo /usr/local/bin/health-monitor.sh \ - https://raw.githubusercontent.com/kubermatic/machine-controller/8b5b66e4910a6228dfaecccaa0a3b05ec4902f8e/pkg/userdata/scripts/health-monitor.sh - chmod +x /usr/local/bin/health-monitor.sh + sudo \ + socat \ + wget \ + curl \ + ipvsadm + + #setup some common directories + mkdir -p /opt/bin/ + mkdir -p /var/lib/calico + mkdir -p /etc/kubernetes/manifests + mkdir -p /etc/cni/net.d + mkdir -p /opt/cni/bin + + # cni + if [ ! -f /opt/cni/bin/loopback ]; then + curl -L https://github.com/containernetworking/plugins/releases/download/v0.6.0/cni-plugins-amd64-v0.6.0.tgz | tar -xvzC /opt/cni/bin -f - fi - - if ! [[ -e /etc/kubernetes/pki/ca.crt ]]; then - kubeadm join \ - --token my-token \ - --discovery-token-ca-cert-hash sha256:6caecce9fedcb55d4953d61a27dc6997361a2f226ad86d7e6004dde7526fc4b1 \ - --ignore-preflight-errors=CRI \ - server:443 + # kubelet + if [ ! -f /opt/bin/kubelet ]; then + curl -Lfo /opt/bin/kubelet https://storage.googleapis.com/kubernetes-release/release/v1.12.0/bin/linux/amd64/kubelet + chmod +x /opt/bin/kubelet fi - - if [[ ! -x /usr/local/bin/health-monitor.sh ]]; then - curl -Lfo /usr/local/bin/health-monitor.sh \ - https://raw.githubusercontent.com/kubermatic/machine-controller/8b5b66e4910a6228dfaecccaa0a3b05ec4902f8e/pkg/userdata/scripts/health-monitor.sh - chmod +x /usr/local/bin/health-monitor.sh + + if [[ ! -x /opt/bin/health-monitor.sh ]]; then + curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/8b5b66e4910a6228dfaecccaa0a3b05ec4902f8e/pkg/userdata/scripts/health-monitor.sh + chmod +x /opt/bin/health-monitor.sh fi + + systemctl enable --now docker + systemctl enable --now kubelet systemctl enable --now --no-block kubelet-healthcheck.service systemctl enable --now --no-block docker-healthcheck.service -- path: "/usr/local/bin/supervise.sh" +- path: "/opt/bin/supervise.sh" permissions: "0755" content: | #!/bin/bash @@ -111,7 +103,107 @@ write_files: sleep 1 done +- path: "/etc/systemd/system/kubelet.service" + content: | + [Unit] + After=docker.service + Requires=docker.service + + Description=kubelet: The Kubernetes Node Agent + Documentation=https://kubernetes.io/docs/home/ + + [Service] + Restart=always + StartLimitInterval=0 + RestartSec=10 + + Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" + + ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ + --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ + --kubeconfig=/etc/kubernetes/kubelet.conf \ + --pod-manifest-path=/etc/kubernetes/manifests \ + --allow-privileged=true \ + --network-plugin=cni \ + --cni-conf-dir=/etc/cni/net.d \ + --cni-bin-dir=/opt/cni/bin \ + --authorization-mode=Webhook \ + --client-ca-file=/etc/kubernetes/pki/ca.crt \ + --rotate-certificates=true \ + --cert-dir=/etc/kubernetes/pki \ + --authentication-token-webhook=true \ + --cloud-provider=aws \ + --cloud-config=/etc/kubernetes/cloud-config \ + --hostname-override=node1 \ + --read-only-port=0 \ + --exit-on-lock-contention \ + --lock-file=/tmp/kubelet.lock \ + --anonymous-auth=false \ + --protect-kernel-defaults=true \ + --cluster-dns= \ + --cluster-domain=cluster.local + + [Install] + WantedBy=multi-user.target + +- path: "/etc/systemd/system/kubelet.service.d/extras.conf" + content: | + [Service] + Environment="KUBELET_EXTRA_ARGS=--cgroup-driver=systemd" + +- path: "/etc/kubernetes/cloud-config" + content: | + {aws-config:true} + +- path: "/etc/kubernetes/bootstrap-kubelet.conf" + content: | + apiVersion: v1 + clusters: + - cluster: + certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t + server: https://server:443 + name: "" + contexts: [] + current-context: "" + kind: Config + preferences: {} + users: + - name: "" + user: + token: my-token + + +- path: "/etc/kubernetes/pki/ca.crt" + content: | + -----BEGIN CERTIFICATE----- + MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV + BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG + A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 + DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 + NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG + cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv + c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B + AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS + R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT + ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk + JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 + mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW + caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G + A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt + hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB + MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES + MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv + bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h + U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao + eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 + UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD + 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n + sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF + kPe6XoSbiLm/kxk32T0= + -----END CERTIFICATE----- + - path: "/etc/systemd/system/setup.service" + permissions: "0644" content: | [Install] WantedBy=multi-user.target @@ -123,7 +215,12 @@ write_files: [Service] Type=oneshot RemainAfterExit=true - ExecStart=/usr/local/bin/supervise.sh /usr/local/bin/setup + ExecStart=/opt/bin/supervise.sh /opt/bin/setup + +- path: "/etc/profile.d/opt-bin-path.sh" + permissions: "0644" + content: | + export PATH="/opt/bin:$PATH" - path: /etc/systemd/system/kubelet-healthcheck.service permissions: "0644" @@ -131,12 +228,13 @@ write_files: [Unit] Requires=kubelet.service After=kubelet.service - + [Service] - ExecStart=/usr/local/bin/health-monitor.sh kubelet - + ExecStart=/opt/bin/health-monitor.sh kubelet + [Install] WantedBy=multi-user.target + - path: /etc/systemd/system/docker-healthcheck.service permissions: "0644" @@ -144,12 +242,13 @@ write_files: [Unit] Requires=docker.service After=docker.service - + [Service] - ExecStart=/usr/local/bin/health-monitor.sh container-runtime - + ExecStart=/opt/bin/health-monitor.sh container-runtime + [Install] WantedBy=multi-user.target + runcmd: - systemctl enable --now setup.service diff --git a/pkg/userdata/centos/testdata/kubelet-v1.9-aws.golden b/pkg/userdata/centos/testdata/kubelet-v1.9-aws.golden index 71c578729..118b5435c 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.9-aws.golden +++ b/pkg/userdata/centos/testdata/kubelet-v1.9-aws.golden @@ -8,6 +8,16 @@ write_files: content: | [Journal] SystemMaxUse=5G + + +- path: "/etc/modules-load.d/k8s.conf" + content: | + ip_vs + ip_vs_rr + ip_vs_wrr + ip_vs_sh + nf_conntrack_ipv4 + - path: "/etc/sysctl.d/k8s.conf" content: | @@ -15,17 +25,9 @@ write_files: net.bridge.bridge-nf-call-iptables = 1 kernel.panic_on_oops = 1 kernel.panic = 10 + net.ipv4.ip_forward = 1 vm.overcommit_memory = 1 - -- path: "/etc/yum.repos.d/kubernetes.repo" - content: | - [kubernetes] - name=Kubernetes - baseurl=https://packages.cloud.google.com/yum/repos/kubernetes-el7-$basearch - enabled=1 - gpgcheck=1 - repo_gpgcheck=1 - gpgkey=https://packages.cloud.google.com/yum/doc/yum-key.gpg https://packages.cloud.google.com/yum/doc/rpm-package-key.gpg + - path: /etc/sysconfig/selinux content: | @@ -41,72 +43,58 @@ write_files: # mls - Multi Level Security protection. SELINUXTYPE=targeted -- path: "/etc/sysconfig/kubelet-overwrite" - content: | - KUBELET_DNS_ARGS= - KUBELET_EXTRA_ARGS=--authentication-token-webhook=true \ - --cloud-provider=aws \ - --cloud-config=/etc/kubernetes/cloud-config \ - --hostname-override=node1 \ - --read-only-port=0 \ - --protect-kernel-defaults=true \ - --cluster-dns= \ - --cluster-domain=cluster.local -- path: "/etc/systemd/system/kubelet.service.d/20-extra.conf" - content: | - [Service] - EnvironmentFile=/etc/sysconfig/kubelet - -- path: "/etc/kubernetes/cloud-config" - content: | - {aws-config:true} - -- path: "/usr/local/bin/setup" - permissions: "0755" +- path: "/opt/bin/setup" + permissions: "0777" content: | #!/bin/bash set -xeuo pipefail + setenforce 0 || true + + # As we added some modules and don't want to reboot, restart the service + systemctl restart systemd-modules-load.service sysctl --system yum install -y docker-1.13.1 \ - kubelet-1.9.6 \ - kubeadm-1.9.6 \ ebtables \ ethtool \ nfs-utils \ bash-completion \ - sudo - - cp /etc/sysconfig/kubelet-overwrite /etc/sysconfig/kubelet - - systemctl enable --now docker - systemctl enable --now kubelet - - if [[ ! -x /usr/local/bin/health-monitor.sh ]]; then - curl -Lfo /usr/local/bin/health-monitor.sh \ - https://raw.githubusercontent.com/kubermatic/machine-controller/8b5b66e4910a6228dfaecccaa0a3b05ec4902f8e/pkg/userdata/scripts/health-monitor.sh - chmod +x /usr/local/bin/health-monitor.sh + sudo \ + socat \ + wget \ + curl \ + ipvsadm + + #setup some common directories + mkdir -p /opt/bin/ + mkdir -p /var/lib/calico + mkdir -p /etc/kubernetes/manifests + mkdir -p /etc/cni/net.d + mkdir -p /opt/cni/bin + + # cni + if [ ! -f /opt/cni/bin/loopback ]; then + curl -L https://github.com/containernetworking/plugins/releases/download/v0.6.0/cni-plugins-amd64-v0.6.0.tgz | tar -xvzC /opt/cni/bin -f - fi - - if ! [[ -e /etc/kubernetes/pki/ca.crt ]]; then - kubeadm join \ - --token my-token \ - --discovery-token-ca-cert-hash sha256:6caecce9fedcb55d4953d61a27dc6997361a2f226ad86d7e6004dde7526fc4b1 \ - --ignore-preflight-errors=CRI \ - server:443 + # kubelet + if [ ! -f /opt/bin/kubelet ]; then + curl -Lfo /opt/bin/kubelet https://storage.googleapis.com/kubernetes-release/release/v1.9.2/bin/linux/amd64/kubelet + chmod +x /opt/bin/kubelet fi - - if [[ ! -x /usr/local/bin/health-monitor.sh ]]; then - curl -Lfo /usr/local/bin/health-monitor.sh \ - https://raw.githubusercontent.com/kubermatic/machine-controller/8b5b66e4910a6228dfaecccaa0a3b05ec4902f8e/pkg/userdata/scripts/health-monitor.sh - chmod +x /usr/local/bin/health-monitor.sh + + if [[ ! -x /opt/bin/health-monitor.sh ]]; then + curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/8b5b66e4910a6228dfaecccaa0a3b05ec4902f8e/pkg/userdata/scripts/health-monitor.sh + chmod +x /opt/bin/health-monitor.sh fi + + systemctl enable --now docker + systemctl enable --now kubelet systemctl enable --now --no-block kubelet-healthcheck.service systemctl enable --now --no-block docker-healthcheck.service -- path: "/usr/local/bin/supervise.sh" +- path: "/opt/bin/supervise.sh" permissions: "0755" content: | #!/bin/bash @@ -115,7 +103,108 @@ write_files: sleep 1 done +- path: "/etc/systemd/system/kubelet.service" + content: | + [Unit] + After=docker.service + Requires=docker.service + + Description=kubelet: The Kubernetes Node Agent + Documentation=https://kubernetes.io/docs/home/ + + [Service] + Restart=always + StartLimitInterval=0 + RestartSec=10 + + Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" + + ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ + --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ + --kubeconfig=/etc/kubernetes/kubelet.conf \ + --pod-manifest-path=/etc/kubernetes/manifests \ + --allow-privileged=true \ + --network-plugin=cni \ + --cni-conf-dir=/etc/cni/net.d \ + --cni-bin-dir=/opt/cni/bin \ + --authorization-mode=Webhook \ + --client-ca-file=/etc/kubernetes/pki/ca.crt \ + --cadvisor-port=0 \ + --rotate-certificates=true \ + --cert-dir=/etc/kubernetes/pki \ + --authentication-token-webhook=true \ + --cloud-provider=aws \ + --cloud-config=/etc/kubernetes/cloud-config \ + --hostname-override=node1 \ + --read-only-port=0 \ + --exit-on-lock-contention \ + --lock-file=/tmp/kubelet.lock \ + --anonymous-auth=false \ + --protect-kernel-defaults=true \ + --cluster-dns= \ + --cluster-domain=cluster.local + + [Install] + WantedBy=multi-user.target + +- path: "/etc/systemd/system/kubelet.service.d/extras.conf" + content: | + [Service] + Environment="KUBELET_EXTRA_ARGS=--cgroup-driver=systemd" + +- path: "/etc/kubernetes/cloud-config" + content: | + {aws-config:true} + +- path: "/etc/kubernetes/bootstrap-kubelet.conf" + content: | + apiVersion: v1 + clusters: + - cluster: + certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t + server: https://server:443 + name: "" + contexts: [] + current-context: "" + kind: Config + preferences: {} + users: + - name: "" + user: + token: my-token + + +- path: "/etc/kubernetes/pki/ca.crt" + content: | + -----BEGIN CERTIFICATE----- + MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV + BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG + A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 + DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 + NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG + cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv + c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B + AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS + R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT + ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk + JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 + mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW + caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G + A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt + hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB + MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES + MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv + bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h + U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao + eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 + UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD + 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n + sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF + kPe6XoSbiLm/kxk32T0= + -----END CERTIFICATE----- + - path: "/etc/systemd/system/setup.service" + permissions: "0644" content: | [Install] WantedBy=multi-user.target @@ -127,7 +216,12 @@ write_files: [Service] Type=oneshot RemainAfterExit=true - ExecStart=/usr/local/bin/supervise.sh /usr/local/bin/setup + ExecStart=/opt/bin/supervise.sh /opt/bin/setup + +- path: "/etc/profile.d/opt-bin-path.sh" + permissions: "0644" + content: | + export PATH="/opt/bin:$PATH" - path: /etc/systemd/system/kubelet-healthcheck.service permissions: "0644" @@ -135,12 +229,13 @@ write_files: [Unit] Requires=kubelet.service After=kubelet.service - + [Service] - ExecStart=/usr/local/bin/health-monitor.sh kubelet - + ExecStart=/opt/bin/health-monitor.sh kubelet + [Install] WantedBy=multi-user.target + - path: /etc/systemd/system/docker-healthcheck.service permissions: "0644" @@ -148,12 +243,13 @@ write_files: [Unit] Requires=docker.service After=docker.service - + [Service] - ExecStart=/usr/local/bin/health-monitor.sh container-runtime - + ExecStart=/opt/bin/health-monitor.sh container-runtime + [Install] WantedBy=multi-user.target + runcmd: - systemctl enable --now setup.service diff --git a/pkg/userdata/centos/userdata.go b/pkg/userdata/centos/userdata.go index f6e3ca2b1..20ffef00d 100644 --- a/pkg/userdata/centos/userdata.go +++ b/pkg/userdata/centos/userdata.go @@ -13,7 +13,6 @@ import ( clientcmdapi "k8s.io/client-go/tools/clientcmd/api" "github.com/kubermatic/machine-controller/pkg/providerconfig" - machinetemplate "github.com/kubermatic/machine-controller/pkg/template" "github.com/kubermatic/machine-controller/pkg/userdata/cloud" userdatahelper "github.com/kubermatic/machine-controller/pkg/userdata/helper" @@ -47,7 +46,7 @@ func (p Provider) UserData( clusterDNSIPs []net.IP, ) (string, error) { - tmpl, err := template.New("user-data").Funcs(machinetemplate.TxtFuncMap()).Parse(ctTemplate) + tmpl, err := template.New("user-data").Funcs(userdatahelper.TxtFuncMap()).Parse(ctTemplate) if err != nil { return "", fmt.Errorf("failed to parse user-data template: %v", err) } @@ -80,45 +79,43 @@ func (p Provider) UserData( return "", fmt.Errorf("failed to parse OperatingSystemSpec: '%v'", err) } - bootstrapToken, err := userdatahelper.GetTokenFromKubeconfig(kubeconfig) + serverAddr, err := userdatahelper.GetServerAddressFromKubeconfig(kubeconfig) if err != nil { - return "", fmt.Errorf("error extracting token: %v", err) + return "", fmt.Errorf("error extracting server address from kubeconfig: %v", err) } - kubeadmCACertHash, err := userdatahelper.GetKubeadmCACertHash(kubeconfig) + kubeconfigString, err := userdatahelper.StringifyKubeconfig(kubeconfig) if err != nil { - return "", fmt.Errorf("error extracting kubeadm cacert hash: %v", err) + return "", err } - serverAddr, err := userdatahelper.GetServerAddressFromKubeconfig(kubeconfig) + kubernetesCACert, err := userdatahelper.GetCACert(kubeconfig) if err != nil { - return "", fmt.Errorf("error extracting server address from kubeconfig: %v", err) + return "", fmt.Errorf("error extracting cacert: %v", err) } data := struct { - MachineSpec clusterv1alpha1.MachineSpec - ProviderConfig *providerconfig.Config - OSConfig *Config - BoostrapToken string - CloudProvider string - CloudConfig string - KubeletVersion string - ClusterDNSIPs []net.IP - KubeadmCACertHash string - ServerAddr string - JournaldMaxSize string + MachineSpec clusterv1alpha1.MachineSpec + ProviderConfig *providerconfig.Config + OSConfig *Config + CloudProvider string + CloudConfig string + KubeletVersion string + ClusterDNSIPs []net.IP + ServerAddr string + Kubeconfig string + KubernetesCACert string }{ - MachineSpec: spec, - ProviderConfig: pconfig, - OSConfig: osConfig, - BoostrapToken: bootstrapToken, - CloudProvider: cpName, - CloudConfig: cpConfig, - KubeletVersion: kubeletVersion.String(), - ClusterDNSIPs: clusterDNSIPs, - KubeadmCACertHash: kubeadmCACertHash, - ServerAddr: serverAddr, - JournaldMaxSize: userdatahelper.JournaldMaxUse, + MachineSpec: spec, + ProviderConfig: pconfig, + OSConfig: osConfig, + CloudProvider: cpName, + CloudConfig: cpConfig, + KubeletVersion: kubeletVersion.String(), + ClusterDNSIPs: clusterDNSIPs, + ServerAddr: serverAddr, + Kubeconfig: kubeconfigString, + KubernetesCACert: kubernetesCACert, } b := &bytes.Buffer{} err = tmpl.Execute(b, data) @@ -148,26 +145,15 @@ ssh_authorized_keys: write_files: - path: "/etc/systemd/journald.conf.d/max_disk_use.conf" content: | - [Journal] - SystemMaxUse={{ .JournaldMaxSize }} +{{ journalDConfig | indent 4 }} -- path: "/etc/sysctl.d/k8s.conf" +- path: "/etc/modules-load.d/k8s.conf" content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - vm.overcommit_memory = 1 +{{ kernelModules | indent 4 }} -- path: "/etc/yum.repos.d/kubernetes.repo" +- path: "/etc/sysctl.d/k8s.conf" content: | - [kubernetes] - name=Kubernetes - baseurl=https://packages.cloud.google.com/yum/repos/kubernetes-el7-$basearch - enabled=1 - gpgcheck=1 - repo_gpgcheck=1 - gpgkey=https://packages.cloud.google.com/yum/doc/yum-key.gpg https://packages.cloud.google.com/yum/doc/rpm-package-key.gpg +{{ kernelSettings | indent 4 }} - path: /etc/sysconfig/selinux content: | @@ -183,79 +169,37 @@ write_files: # mls - Multi Level Security protection. SELINUXTYPE=targeted -- path: "/etc/sysconfig/kubelet-overwrite" - content: | - KUBELET_DNS_ARGS= - KUBELET_EXTRA_ARGS=--authentication-token-webhook=true \ - {{- if .CloudProvider }} - --cloud-provider={{ .CloudProvider }} \ - --cloud-config=/etc/kubernetes/cloud-config \ - {{- end}} - --hostname-override={{ .MachineSpec.Name }} \ - --read-only-port=0 \ - --protect-kernel-defaults=true \ - --cluster-dns={{ ipSliceToCommaSeparatedString .ClusterDNSIPs }} \ - --cluster-domain=cluster.local - -{{- if semverCompare "<1.11.0" .KubeletVersion }} -- path: "/etc/systemd/system/kubelet.service.d/20-extra.conf" - content: | - [Service] - EnvironmentFile=/etc/sysconfig/kubelet -{{- end }} - -- path: "/etc/kubernetes/cloud-config" - content: | -{{ if ne .CloudConfig "" }}{{ .CloudConfig | indent 4 }}{{ end }} - -- path: "/usr/local/bin/setup" - permissions: "0755" +- path: "/opt/bin/setup" + permissions: "0777" content: | #!/bin/bash set -xeuo pipefail + setenforce 0 || true + + # As we added some modules and don't want to reboot, restart the service + systemctl restart systemd-modules-load.service sysctl --system yum install -y docker-1.13.1 \ - kubelet-{{ .KubeletVersion }} \ - kubeadm-{{ .KubeletVersion }} \ ebtables \ ethtool \ nfs-utils \ bash-completion \ - sudo + sudo \ + socat \ + wget \ + curl \ + ipvsadm - cp /etc/sysconfig/kubelet-overwrite /etc/sysconfig/kubelet +{{ downloadBinariesScript .KubeletVersion true | indent 4 }} systemctl enable --now docker systemctl enable --now kubelet - - if [[ ! -x /usr/local/bin/health-monitor.sh ]]; then - curl -Lfo /usr/local/bin/health-monitor.sh \ - https://raw.githubusercontent.com/kubermatic/machine-controller/8b5b66e4910a6228dfaecccaa0a3b05ec4902f8e/pkg/userdata/scripts/health-monitor.sh - chmod +x /usr/local/bin/health-monitor.sh - fi - - if ! [[ -e /etc/kubernetes/pki/ca.crt ]]; then - kubeadm join \ - --token {{ .BoostrapToken }} \ - --discovery-token-ca-cert-hash sha256:{{ .KubeadmCACertHash }} \ - {{- if semverCompare ">=1.9.X" .KubeletVersion }} - --ignore-preflight-errors=CRI \ - {{- end }} - {{ .ServerAddr }} - fi - - if [[ ! -x /usr/local/bin/health-monitor.sh ]]; then - curl -Lfo /usr/local/bin/health-monitor.sh \ - https://raw.githubusercontent.com/kubermatic/machine-controller/8b5b66e4910a6228dfaecccaa0a3b05ec4902f8e/pkg/userdata/scripts/health-monitor.sh - chmod +x /usr/local/bin/health-monitor.sh - fi - systemctl enable --now --no-block kubelet-healthcheck.service systemctl enable --now --no-block docker-healthcheck.service -- path: "/usr/local/bin/supervise.sh" +- path: "/opt/bin/supervise.sh" permissions: "0755" content: | #!/bin/bash @@ -264,7 +208,29 @@ write_files: sleep 1 done +- path: "/etc/systemd/system/kubelet.service" + content: | +{{ kubeletSystemdUnit .KubeletVersion .CloudProvider .MachineSpec.Name .ClusterDNSIPs | indent 4 }} + +- path: "/etc/systemd/system/kubelet.service.d/extras.conf" + content: | + [Service] + Environment="KUBELET_EXTRA_ARGS=--cgroup-driver=systemd" + +- path: "/etc/kubernetes/cloud-config" + content: | +{{ .CloudConfig | indent 4 }} + +- path: "/etc/kubernetes/bootstrap-kubelet.conf" + content: | +{{ .Kubeconfig | indent 4 }} + +- path: "/etc/kubernetes/pki/ca.crt" + content: | +{{ .KubernetesCACert | indent 4 }} + - path: "/etc/systemd/system/setup.service" + permissions: "0644" content: | [Install] WantedBy=multi-user.target @@ -276,33 +242,22 @@ write_files: [Service] Type=oneshot RemainAfterExit=true - ExecStart=/usr/local/bin/supervise.sh /usr/local/bin/setup + ExecStart=/opt/bin/supervise.sh /opt/bin/setup -- path: /etc/systemd/system/kubelet-healthcheck.service +- path: "/etc/profile.d/opt-bin-path.sh" permissions: "0644" content: | - [Unit] - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/usr/local/bin/health-monitor.sh kubelet + export PATH="/opt/bin:$PATH" - [Install] - WantedBy=multi-user.target +- path: /etc/systemd/system/kubelet-healthcheck.service + permissions: "0644" + content: | +{{ kubeletHealthCheckSystemdUnit | indent 4 }} - path: /etc/systemd/system/docker-healthcheck.service permissions: "0644" content: | - [Unit] - Requires=docker.service - After=docker.service - - [Service] - ExecStart=/usr/local/bin/health-monitor.sh container-runtime - - [Install] - WantedBy=multi-user.target +{{ containerRuntimeHealthCheckSystemdUnit | indent 4 }} runcmd: - systemctl enable --now setup.service diff --git a/pkg/userdata/centos/userdata_test.go b/pkg/userdata/centos/userdata_test.go index 5766ef2fd..a2e79e5b7 100644 --- a/pkg/userdata/centos/userdata_test.go +++ b/pkg/userdata/centos/userdata_test.go @@ -2,14 +2,13 @@ package centos import ( "flag" - "io/ioutil" "net" - "path/filepath" "testing" + testhelper "github.com/kubermatic/machine-controller/pkg/test" + "k8s.io/apimachinery/pkg/runtime" - "github.com/pmezard/go-difflib/difflib" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" clientcmdapi "k8s.io/client-go/tools/clientcmd/api" @@ -68,7 +67,7 @@ func TestUserDataGeneration(t *testing.T) { spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.9.6", + Kubelet: "1.9.2", }, }, }, @@ -116,28 +115,7 @@ func TestUserDataGeneration(t *testing.T) { if err != nil { t.Errorf("error getting userdata: '%v'", err) } - golden := filepath.Join("testdata", test.name+".golden") - if *update { - ioutil.WriteFile(golden, []byte(userdata), 0644) - } - expected, err := ioutil.ReadFile(golden) - if err != nil { - t.Errorf("failed to read .golden file: %v", err) - } - if string(expected) != userdata { - diff := difflib.UnifiedDiff{ - A: difflib.SplitLines(string(expected)), - B: difflib.SplitLines(userdata), - FromFile: "Fixture", - ToFile: "Current", - Context: 3, - } - diffStr, err := difflib.GetUnifiedDiffString(diff) - if err != nil { - t.Fatal(err) - } - t.Errorf("got diff between expected and actual result: \n%s\n", diffStr) - } + testhelper.CompareOutput(t, test.name, userdata, *update) } } diff --git a/pkg/userdata/coreos/testdata/auto-update-openstack-kubelet-v-version-prefix.golden b/pkg/userdata/coreos/testdata/auto-update-openstack-kubelet-v-version-prefix.golden index 3c638dd59..c35eb7b37 100644 --- a/pkg/userdata/coreos/testdata/auto-update-openstack-kubelet-v-version-prefix.golden +++ b/pkg/userdata/coreos/testdata/auto-update-openstack-kubelet-v-version-prefix.golden @@ -29,13 +29,24 @@ }, "mode": 420 }, + { + "filesystem": "root", + "group": {}, + "path": "/etc/modules-load.d/k8s.conf", + "user": {}, + "contents": { + "source": "data:,ip_vs%0Aip_vs_rr%0Aip_vs_wrr%0Aip_vs_sh%0Anf_conntrack_ipv4%0A", + "verification": {} + }, + "mode": 420 + }, { "filesystem": "root", "group": {}, "path": "/etc/sysctl.d/k8s.conf", "user": {}, "contents": { - "source": "data:,kernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Avm.overcommit_memory%20%3D%201%0A", + "source": "data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0A", "verification": {} }, "mode": 420 @@ -76,7 +87,7 @@ { "filesystem": "root", "group": {}, - "path": "/etc/kubernetes/bootstrap.kubeconfig", + "path": "/etc/kubernetes/bootstrap-kubelet.conf", "user": {}, "contents": { "source": "data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20%5B%5D%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A", @@ -98,7 +109,7 @@ { "filesystem": "root", "group": {}, - "path": "/etc/kubernetes/ca.crt", + "path": "/etc/kubernetes/pki/ca.crt", "user": {}, "contents": { "source": "data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A", @@ -157,13 +168,13 @@ { "filesystem": "root", "group": {}, - "path": "/opt/bin/download-healthcheck-script.sh", + "path": "/opt/bin/download.sh", "user": {}, "contents": { - "source": "data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-xeuo%20pipefail%0Auntil%20%5B%5B%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20do%0A%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5C%0A%20%20%20%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F8b5b66e4910a6228dfaecccaa0a3b05ec4902f8e%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Adone%0A", + "source": "data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%23setup%20some%20common%20directories%0Amkdir%20-p%20%2Fopt%2Fbin%2F%0Amkdir%20-p%20%2Fvar%2Flib%2Fcalico%0Amkdir%20-p%20%2Fetc%2Fkubernetes%2Fmanifests%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%0Amkdir%20-p%20%2Fopt%2Fcni%2Fbin%0A%0A%23%20cni%0Aif%20%5B%20!%20-f%20%2Fopt%2Fcni%2Fbin%2Floopback%20%5D%3B%20then%0A%20%20%20%20curl%20-L%20https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2Fv0.6.0%2Fcni-plugins-amd64-v0.6.0.tgz%20%7C%20tar%20-xvzC%20%2Fopt%2Fcni%2Fbin%20-f%20-%0Afi%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F8b5b66e4910a6228dfaecccaa0a3b05ec4902f8e%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A", "verification": {} }, - "mode": 755 + "mode": 493 } ] }, @@ -174,29 +185,35 @@ "name": "docker.service" }, { - "contents": "[Unit]\nRequires=network-online.target\nAfter=network-online.target\n[Service]\nType=oneshot\nExecStart=/opt/bin/download-healthcheck-script.sh\n[Install]\nWantedBy=multi-user.target\n", + "contents": "[Unit]\nRequires=network-online.target\nAfter=network-online.target\n[Service]\nType=oneshot\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n", "enabled": true, "name": "download-healthcheck-script.service" }, { - "contents": "[Unit]\nRequires=download-healthcheck-script.service kubelet.service\nAfter=download-healthcheck-script.service kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n", - "enabled": true, - "name": "kubelet-healthcheck.service" - }, - { - "contents": "[Unit]\nRequires=download-healthcheck-script.service docker.service\nAfter=download-healthcheck-script.service docker.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh container-runtime\n\n[Install]\nWantedBy=multi-user.target\n", + "contents": "[Unit]\nRequires=docker.service\nAfter=docker.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh container-runtime\n\n[Install]\nWantedBy=multi-user.target\n", + "dropins": [ + { + "contents": "[Unit]\nRequires=download-healthcheck-script.service\nAfter=download-healthcheck-script.service\n", + "name": "40-docker.conf" + } + ], "enabled": true, "name": "docker-healthcheck.service" }, { - "contents": "[Unit]\nDescription=Kubernetes Kubelet\nRequires=docker.service\nAfter=docker.service\n[Service]\nTimeoutStartSec=5min\nEnvironment=KUBELET_IMAGE=docker://k8s.gcr.io/hyperkube-amd64:v1.9.2\nEnvironment=\"RKT_RUN_ARGS=--uuid-file-save=/var/cache/kubelet-pod.uuid \\\n --insecure-options=image \\\n --volume=resolv,kind=host,source=/etc/resolv.conf \\\n --mount volume=resolv,target=/etc/resolv.conf \\\n --volume cni-bin,kind=host,source=/opt/cni/bin \\\n --mount volume=cni-bin,target=/opt/cni/bin \\\n --volume cni-conf,kind=host,source=/etc/cni/net.d \\\n --mount volume=cni-conf,target=/etc/cni/net.d \\\n --volume etc-kubernetes,kind=host,source=/etc/kubernetes \\\n --mount volume=etc-kubernetes,target=/etc/kubernetes \\\n --volume var-log,kind=host,source=/var/log \\\n --mount volume=var-log,target=/var/log \\\n --volume var-lib-calico,kind=host,source=/var/lib/calico \\\n --mount volume=var-lib-calico,target=/var/lib/calico\"\nExecStartPre=/bin/mkdir -p /var/lib/calico\nExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests\nExecStartPre=/bin/mkdir -p /etc/cni/net.d\nExecStartPre=/bin/mkdir -p /opt/cni/bin\nExecStartPre=-/usr/bin/rkt rm --uuid-file=/var/cache/kubelet-pod.uuid\nExecStart=/usr/lib/coreos/kubelet-wrapper \\\n --container-runtime=docker \\\n --allow-privileged=true \\\n --cni-bin-dir=/opt/cni/bin \\\n --cni-conf-dir=/etc/cni/net.d \\\n --cluster-dns=10.10.10.10 \\\n --cluster-domain=cluster.local \\\n --authentication-token-webhook=true \\\n --hostname-override=node1 \\\n --network-plugin=cni \\\n --cloud-provider=openstack \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --cert-dir=/etc/kubernetes/ \\\n --pod-manifest-path=/etc/kubernetes/manifests \\\n --resolv-conf=/etc/resolv.conf \\\n --rotate-certificates=true \\\n --kubeconfig=/etc/kubernetes/kubeconfig \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap.kubeconfig \\\n --lock-file=/var/run/lock/kubelet.lock \\\n --exit-on-lock-contention \\\n --read-only-port=0 \\\n --protect-kernel-defaults=true \\\n --authorization-mode=Webhook \\\n --anonymous-auth=false \\\n --client-ca-file=/etc/kubernetes/ca.crt\nExecStop=-/usr/bin/rkt stop --uuid-file=/var/cache/kubelet-pod.uuid\nRestart=always\nRestartSec=10\n[Install]\nWantedBy=multi-user.target\n", + "contents": "[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n", "dropins": [ { - "contents": "[Unit]\nRequires=docker.service\nAfter=docker.service\n", + "contents": "[Unit]\nRequires=download-healthcheck-script.service\nAfter=download-healthcheck-script.service\n", "name": "40-docker.conf" } ], "enabled": true, + "name": "kubelet-healthcheck.service" + }, + { + "contents": "[Unit]\nDescription=Kubernetes Kubelet\nRequires=docker.service\nAfter=docker.service\n[Service]\nTimeoutStartSec=5min\nEnvironment=KUBELET_IMAGE=docker://k8s.gcr.io/hyperkube-amd64:v1.9.2\nEnvironment=\"RKT_RUN_ARGS=--uuid-file-save=/var/cache/kubelet-pod.uuid \\\n --insecure-options=image \\\n --volume=resolv,kind=host,source=/etc/resolv.conf \\\n --mount volume=resolv,target=/etc/resolv.conf \\\n --volume cni-bin,kind=host,source=/opt/cni/bin \\\n --mount volume=cni-bin,target=/opt/cni/bin \\\n --volume cni-conf,kind=host,source=/etc/cni/net.d \\\n --mount volume=cni-conf,target=/etc/cni/net.d \\\n --volume etc-kubernetes,kind=host,source=/etc/kubernetes \\\n --mount volume=etc-kubernetes,target=/etc/kubernetes \\\n --volume var-log,kind=host,source=/var/log \\\n --mount volume=var-log,target=/var/log \\\n --volume var-lib-calico,kind=host,source=/var/lib/calico \\\n --mount volume=var-lib-calico,target=/var/lib/calico\"\nExecStartPre=/bin/mkdir -p /var/lib/calico\nExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests\nExecStartPre=/bin/mkdir -p /etc/cni/net.d\nExecStartPre=/bin/mkdir -p /opt/cni/bin\nExecStartPre=-/usr/bin/rkt rm --uuid-file=/var/cache/kubelet-pod.uuid\nExecStart=/usr/lib/coreos/kubelet-wrapper \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/etc/kubernetes/kubelet.conf \\\n --pod-manifest-path=/etc/kubernetes/manifests \\\n --allow-privileged=true \\\n --network-plugin=cni \\\n --cni-conf-dir=/etc/cni/net.d \\\n --cni-bin-dir=/opt/cni/bin \\\n --authorization-mode=Webhook \\\n --client-ca-file=/etc/kubernetes/pki/ca.crt \\\n --cadvisor-port=0 \\\n --rotate-certificates=true \\\n --cert-dir=/etc/kubernetes/pki \\\n --authentication-token-webhook=true \\\n --cloud-provider=openstack \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --read-only-port=0 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --anonymous-auth=false \\\n --protect-kernel-defaults=true \\\n --cluster-dns=10.10.10.10 \\\n --cluster-domain=cluster.local\nExecStop=-/usr/bin/rkt stop --uuid-file=/var/cache/kubelet-pod.uuid\nRestart=always\nRestartSec=10\n[Install]\nWantedBy=multi-user.target\n", + "enabled": true, "name": "kubelet.service" } ] diff --git a/pkg/userdata/coreos/testdata/v1.10.3-auto-update-openstack-multiple-dns.golden b/pkg/userdata/coreos/testdata/v1.10.3-auto-update-openstack-multiple-dns.golden index 08bf7e004..8a27e766f 100644 --- a/pkg/userdata/coreos/testdata/v1.10.3-auto-update-openstack-multiple-dns.golden +++ b/pkg/userdata/coreos/testdata/v1.10.3-auto-update-openstack-multiple-dns.golden @@ -29,13 +29,24 @@ }, "mode": 420 }, + { + "filesystem": "root", + "group": {}, + "path": "/etc/modules-load.d/k8s.conf", + "user": {}, + "contents": { + "source": "data:,ip_vs%0Aip_vs_rr%0Aip_vs_wrr%0Aip_vs_sh%0Anf_conntrack_ipv4%0A", + "verification": {} + }, + "mode": 420 + }, { "filesystem": "root", "group": {}, "path": "/etc/sysctl.d/k8s.conf", "user": {}, "contents": { - "source": "data:,kernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Avm.overcommit_memory%20%3D%201%0A", + "source": "data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0A", "verification": {} }, "mode": 420 @@ -76,7 +87,7 @@ { "filesystem": "root", "group": {}, - "path": "/etc/kubernetes/bootstrap.kubeconfig", + "path": "/etc/kubernetes/bootstrap-kubelet.conf", "user": {}, "contents": { "source": "data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20%5B%5D%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A", @@ -98,7 +109,7 @@ { "filesystem": "root", "group": {}, - "path": "/etc/kubernetes/ca.crt", + "path": "/etc/kubernetes/pki/ca.crt", "user": {}, "contents": { "source": "data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A", @@ -157,13 +168,13 @@ { "filesystem": "root", "group": {}, - "path": "/opt/bin/download-healthcheck-script.sh", + "path": "/opt/bin/download.sh", "user": {}, "contents": { - "source": "data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-xeuo%20pipefail%0Auntil%20%5B%5B%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20do%0A%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5C%0A%20%20%20%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F8b5b66e4910a6228dfaecccaa0a3b05ec4902f8e%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Adone%0A", + "source": "data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%23setup%20some%20common%20directories%0Amkdir%20-p%20%2Fopt%2Fbin%2F%0Amkdir%20-p%20%2Fvar%2Flib%2Fcalico%0Amkdir%20-p%20%2Fetc%2Fkubernetes%2Fmanifests%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%0Amkdir%20-p%20%2Fopt%2Fcni%2Fbin%0A%0A%23%20cni%0Aif%20%5B%20!%20-f%20%2Fopt%2Fcni%2Fbin%2Floopback%20%5D%3B%20then%0A%20%20%20%20curl%20-L%20https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2Fv0.6.0%2Fcni-plugins-amd64-v0.6.0.tgz%20%7C%20tar%20-xvzC%20%2Fopt%2Fcni%2Fbin%20-f%20-%0Afi%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F8b5b66e4910a6228dfaecccaa0a3b05ec4902f8e%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A", "verification": {} }, - "mode": 755 + "mode": 493 } ] }, @@ -174,29 +185,35 @@ "name": "docker.service" }, { - "contents": "[Unit]\nRequires=network-online.target\nAfter=network-online.target\n[Service]\nType=oneshot\nExecStart=/opt/bin/download-healthcheck-script.sh\n[Install]\nWantedBy=multi-user.target\n", + "contents": "[Unit]\nRequires=network-online.target\nAfter=network-online.target\n[Service]\nType=oneshot\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n", "enabled": true, "name": "download-healthcheck-script.service" }, { - "contents": "[Unit]\nRequires=download-healthcheck-script.service kubelet.service\nAfter=download-healthcheck-script.service kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n", - "enabled": true, - "name": "kubelet-healthcheck.service" - }, - { - "contents": "[Unit]\nRequires=download-healthcheck-script.service docker.service\nAfter=download-healthcheck-script.service docker.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh container-runtime\n\n[Install]\nWantedBy=multi-user.target\n", + "contents": "[Unit]\nRequires=docker.service\nAfter=docker.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh container-runtime\n\n[Install]\nWantedBy=multi-user.target\n", + "dropins": [ + { + "contents": "[Unit]\nRequires=download-healthcheck-script.service\nAfter=download-healthcheck-script.service\n", + "name": "40-docker.conf" + } + ], "enabled": true, "name": "docker-healthcheck.service" }, { - "contents": "[Unit]\nDescription=Kubernetes Kubelet\nRequires=docker.service\nAfter=docker.service\n[Service]\nTimeoutStartSec=5min\nEnvironment=KUBELET_IMAGE=docker://k8s.gcr.io/hyperkube-amd64:v1.10.3\nEnvironment=\"RKT_RUN_ARGS=--uuid-file-save=/var/cache/kubelet-pod.uuid \\\n --insecure-options=image \\\n --volume=resolv,kind=host,source=/etc/resolv.conf \\\n --mount volume=resolv,target=/etc/resolv.conf \\\n --volume cni-bin,kind=host,source=/opt/cni/bin \\\n --mount volume=cni-bin,target=/opt/cni/bin \\\n --volume cni-conf,kind=host,source=/etc/cni/net.d \\\n --mount volume=cni-conf,target=/etc/cni/net.d \\\n --volume etc-kubernetes,kind=host,source=/etc/kubernetes \\\n --mount volume=etc-kubernetes,target=/etc/kubernetes \\\n --volume var-log,kind=host,source=/var/log \\\n --mount volume=var-log,target=/var/log \\\n --volume var-lib-calico,kind=host,source=/var/lib/calico \\\n --mount volume=var-lib-calico,target=/var/lib/calico\"\nExecStartPre=/bin/mkdir -p /var/lib/calico\nExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests\nExecStartPre=/bin/mkdir -p /etc/cni/net.d\nExecStartPre=/bin/mkdir -p /opt/cni/bin\nExecStartPre=-/usr/bin/rkt rm --uuid-file=/var/cache/kubelet-pod.uuid\nExecStart=/usr/lib/coreos/kubelet-wrapper \\\n --container-runtime=docker \\\n --allow-privileged=true \\\n --cni-bin-dir=/opt/cni/bin \\\n --cni-conf-dir=/etc/cni/net.d \\\n --cluster-dns=10.10.10.10,10.10.10.11,10.10.10.12 \\\n --cluster-domain=cluster.local \\\n --authentication-token-webhook=true \\\n --hostname-override=node1 \\\n --network-plugin=cni \\\n --cloud-provider=openstack \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --cert-dir=/etc/kubernetes/ \\\n --pod-manifest-path=/etc/kubernetes/manifests \\\n --resolv-conf=/etc/resolv.conf \\\n --rotate-certificates=true \\\n --kubeconfig=/etc/kubernetes/kubeconfig \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap.kubeconfig \\\n --lock-file=/var/run/lock/kubelet.lock \\\n --exit-on-lock-contention \\\n --read-only-port=0 \\\n --protect-kernel-defaults=true \\\n --authorization-mode=Webhook \\\n --anonymous-auth=false \\\n --client-ca-file=/etc/kubernetes/ca.crt\nExecStop=-/usr/bin/rkt stop --uuid-file=/var/cache/kubelet-pod.uuid\nRestart=always\nRestartSec=10\n[Install]\nWantedBy=multi-user.target\n", + "contents": "[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n", "dropins": [ { - "contents": "[Unit]\nRequires=docker.service\nAfter=docker.service\n", + "contents": "[Unit]\nRequires=download-healthcheck-script.service\nAfter=download-healthcheck-script.service\n", "name": "40-docker.conf" } ], "enabled": true, + "name": "kubelet-healthcheck.service" + }, + { + "contents": "[Unit]\nDescription=Kubernetes Kubelet\nRequires=docker.service\nAfter=docker.service\n[Service]\nTimeoutStartSec=5min\nEnvironment=KUBELET_IMAGE=docker://k8s.gcr.io/hyperkube-amd64:v1.10.3\nEnvironment=\"RKT_RUN_ARGS=--uuid-file-save=/var/cache/kubelet-pod.uuid \\\n --insecure-options=image \\\n --volume=resolv,kind=host,source=/etc/resolv.conf \\\n --mount volume=resolv,target=/etc/resolv.conf \\\n --volume cni-bin,kind=host,source=/opt/cni/bin \\\n --mount volume=cni-bin,target=/opt/cni/bin \\\n --volume cni-conf,kind=host,source=/etc/cni/net.d \\\n --mount volume=cni-conf,target=/etc/cni/net.d \\\n --volume etc-kubernetes,kind=host,source=/etc/kubernetes \\\n --mount volume=etc-kubernetes,target=/etc/kubernetes \\\n --volume var-log,kind=host,source=/var/log \\\n --mount volume=var-log,target=/var/log \\\n --volume var-lib-calico,kind=host,source=/var/lib/calico \\\n --mount volume=var-lib-calico,target=/var/lib/calico\"\nExecStartPre=/bin/mkdir -p /var/lib/calico\nExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests\nExecStartPre=/bin/mkdir -p /etc/cni/net.d\nExecStartPre=/bin/mkdir -p /opt/cni/bin\nExecStartPre=-/usr/bin/rkt rm --uuid-file=/var/cache/kubelet-pod.uuid\nExecStart=/usr/lib/coreos/kubelet-wrapper \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/etc/kubernetes/kubelet.conf \\\n --pod-manifest-path=/etc/kubernetes/manifests \\\n --allow-privileged=true \\\n --network-plugin=cni \\\n --cni-conf-dir=/etc/cni/net.d \\\n --cni-bin-dir=/opt/cni/bin \\\n --authorization-mode=Webhook \\\n --client-ca-file=/etc/kubernetes/pki/ca.crt \\\n --cadvisor-port=0 \\\n --rotate-certificates=true \\\n --cert-dir=/etc/kubernetes/pki \\\n --authentication-token-webhook=true \\\n --cloud-provider=openstack \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --read-only-port=0 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --anonymous-auth=false \\\n --protect-kernel-defaults=true \\\n --cluster-dns=10.10.10.10,10.10.10.11,10.10.10.12 \\\n --cluster-domain=cluster.local\nExecStop=-/usr/bin/rkt stop --uuid-file=/var/cache/kubelet-pod.uuid\nRestart=always\nRestartSec=10\n[Install]\nWantedBy=multi-user.target\n", + "enabled": true, "name": "kubelet.service" } ] diff --git a/pkg/userdata/coreos/testdata/v1.11.2-vsphere-static-ipconfig.golden b/pkg/userdata/coreos/testdata/v1.11.2-vsphere-static-ipconfig.golden index dd2e2ae85..705489a8a 100644 --- a/pkg/userdata/coreos/testdata/v1.11.2-vsphere-static-ipconfig.golden +++ b/pkg/userdata/coreos/testdata/v1.11.2-vsphere-static-ipconfig.golden @@ -36,13 +36,24 @@ }, "mode": 420 }, + { + "filesystem": "root", + "group": {}, + "path": "/etc/modules-load.d/k8s.conf", + "user": {}, + "contents": { + "source": "data:,ip_vs%0Aip_vs_rr%0Aip_vs_wrr%0Aip_vs_sh%0Anf_conntrack_ipv4%0A", + "verification": {} + }, + "mode": 420 + }, { "filesystem": "root", "group": {}, "path": "/etc/sysctl.d/k8s.conf", "user": {}, "contents": { - "source": "data:,kernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Avm.overcommit_memory%20%3D%201%0A", + "source": "data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0A", "verification": {} }, "mode": 420 @@ -83,7 +94,7 @@ { "filesystem": "root", "group": {}, - "path": "/etc/kubernetes/bootstrap.kubeconfig", + "path": "/etc/kubernetes/bootstrap-kubelet.conf", "user": {}, "contents": { "source": "data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20%5B%5D%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A", @@ -105,7 +116,7 @@ { "filesystem": "root", "group": {}, - "path": "/etc/kubernetes/ca.crt", + "path": "/etc/kubernetes/pki/ca.crt", "user": {}, "contents": { "source": "data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A", @@ -164,13 +175,13 @@ { "filesystem": "root", "group": {}, - "path": "/opt/bin/download-healthcheck-script.sh", + "path": "/opt/bin/download.sh", "user": {}, "contents": { - "source": "data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-xeuo%20pipefail%0Auntil%20%5B%5B%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20do%0A%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5C%0A%20%20%20%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F8b5b66e4910a6228dfaecccaa0a3b05ec4902f8e%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Adone%0A", + "source": "data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%23setup%20some%20common%20directories%0Amkdir%20-p%20%2Fopt%2Fbin%2F%0Amkdir%20-p%20%2Fvar%2Flib%2Fcalico%0Amkdir%20-p%20%2Fetc%2Fkubernetes%2Fmanifests%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%0Amkdir%20-p%20%2Fopt%2Fcni%2Fbin%0A%0A%23%20cni%0Aif%20%5B%20!%20-f%20%2Fopt%2Fcni%2Fbin%2Floopback%20%5D%3B%20then%0A%20%20%20%20curl%20-L%20https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2Fv0.6.0%2Fcni-plugins-amd64-v0.6.0.tgz%20%7C%20tar%20-xvzC%20%2Fopt%2Fcni%2Fbin%20-f%20-%0Afi%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F8b5b66e4910a6228dfaecccaa0a3b05ec4902f8e%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A", "verification": {} }, - "mode": 755 + "mode": 493 } ] }, @@ -189,29 +200,35 @@ "name": "docker.service" }, { - "contents": "[Unit]\nRequires=network-online.target\nAfter=network-online.target\n[Service]\nType=oneshot\nExecStart=/opt/bin/download-healthcheck-script.sh\n[Install]\nWantedBy=multi-user.target\n", + "contents": "[Unit]\nRequires=network-online.target\nAfter=network-online.target\n[Service]\nType=oneshot\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n", "enabled": true, "name": "download-healthcheck-script.service" }, { - "contents": "[Unit]\nRequires=download-healthcheck-script.service kubelet.service\nAfter=download-healthcheck-script.service kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n", - "enabled": true, - "name": "kubelet-healthcheck.service" - }, - { - "contents": "[Unit]\nRequires=download-healthcheck-script.service docker.service\nAfter=download-healthcheck-script.service docker.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh container-runtime\n\n[Install]\nWantedBy=multi-user.target\n", + "contents": "[Unit]\nRequires=docker.service\nAfter=docker.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh container-runtime\n\n[Install]\nWantedBy=multi-user.target\n", + "dropins": [ + { + "contents": "[Unit]\nRequires=download-healthcheck-script.service\nAfter=download-healthcheck-script.service\n", + "name": "40-docker.conf" + } + ], "enabled": true, "name": "docker-healthcheck.service" }, { - "contents": "[Unit]\nDescription=Kubernetes Kubelet\nRequires=docker.service\nAfter=docker.service\n[Service]\nTimeoutStartSec=5min\nEnvironment=KUBELET_IMAGE=docker://k8s.gcr.io/hyperkube-amd64:v1.11.2\nEnvironment=\"RKT_RUN_ARGS=--uuid-file-save=/var/cache/kubelet-pod.uuid \\\n --insecure-options=image \\\n --volume=resolv,kind=host,source=/etc/resolv.conf \\\n --mount volume=resolv,target=/etc/resolv.conf \\\n --volume cni-bin,kind=host,source=/opt/cni/bin \\\n --mount volume=cni-bin,target=/opt/cni/bin \\\n --volume cni-conf,kind=host,source=/etc/cni/net.d \\\n --mount volume=cni-conf,target=/etc/cni/net.d \\\n --volume etc-kubernetes,kind=host,source=/etc/kubernetes \\\n --mount volume=etc-kubernetes,target=/etc/kubernetes \\\n --volume var-log,kind=host,source=/var/log \\\n --mount volume=var-log,target=/var/log \\\n --volume var-lib-calico,kind=host,source=/var/lib/calico \\\n --mount volume=var-lib-calico,target=/var/lib/calico\"\nExecStartPre=/bin/mkdir -p /var/lib/calico\nExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests\nExecStartPre=/bin/mkdir -p /etc/cni/net.d\nExecStartPre=/bin/mkdir -p /opt/cni/bin\nExecStartPre=-/usr/bin/rkt rm --uuid-file=/var/cache/kubelet-pod.uuid\nExecStart=/usr/lib/coreos/kubelet-wrapper \\\n --container-runtime=docker \\\n --allow-privileged=true \\\n --cni-bin-dir=/opt/cni/bin \\\n --cni-conf-dir=/etc/cni/net.d \\\n --cluster-dns=10.10.10.10 \\\n --cluster-domain=cluster.local \\\n --authentication-token-webhook=true \\\n --hostname-override=node1 \\\n --network-plugin=cni \\\n --cloud-provider=vsphere \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --cert-dir=/etc/kubernetes/ \\\n --pod-manifest-path=/etc/kubernetes/manifests \\\n --resolv-conf=/etc/resolv.conf \\\n --rotate-certificates=true \\\n --kubeconfig=/etc/kubernetes/kubeconfig \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap.kubeconfig \\\n --lock-file=/var/run/lock/kubelet.lock \\\n --exit-on-lock-contention \\\n --read-only-port=0 \\\n --protect-kernel-defaults=true \\\n --authorization-mode=Webhook \\\n --anonymous-auth=false \\\n --client-ca-file=/etc/kubernetes/ca.crt\nExecStop=-/usr/bin/rkt stop --uuid-file=/var/cache/kubelet-pod.uuid\nRestart=always\nRestartSec=10\n[Install]\nWantedBy=multi-user.target\n", + "contents": "[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n", "dropins": [ { - "contents": "[Unit]\nRequires=docker.service\nAfter=docker.service\n", + "contents": "[Unit]\nRequires=download-healthcheck-script.service\nAfter=download-healthcheck-script.service\n", "name": "40-docker.conf" } ], "enabled": true, + "name": "kubelet-healthcheck.service" + }, + { + "contents": "[Unit]\nDescription=Kubernetes Kubelet\nRequires=docker.service\nAfter=docker.service\n[Service]\nTimeoutStartSec=5min\nEnvironment=KUBELET_IMAGE=docker://k8s.gcr.io/hyperkube-amd64:v1.11.2\nEnvironment=\"RKT_RUN_ARGS=--uuid-file-save=/var/cache/kubelet-pod.uuid \\\n --insecure-options=image \\\n --volume=resolv,kind=host,source=/etc/resolv.conf \\\n --mount volume=resolv,target=/etc/resolv.conf \\\n --volume cni-bin,kind=host,source=/opt/cni/bin \\\n --mount volume=cni-bin,target=/opt/cni/bin \\\n --volume cni-conf,kind=host,source=/etc/cni/net.d \\\n --mount volume=cni-conf,target=/etc/cni/net.d \\\n --volume etc-kubernetes,kind=host,source=/etc/kubernetes \\\n --mount volume=etc-kubernetes,target=/etc/kubernetes \\\n --volume var-log,kind=host,source=/var/log \\\n --mount volume=var-log,target=/var/log \\\n --volume var-lib-calico,kind=host,source=/var/lib/calico \\\n --mount volume=var-lib-calico,target=/var/lib/calico\"\nExecStartPre=/bin/mkdir -p /var/lib/calico\nExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests\nExecStartPre=/bin/mkdir -p /etc/cni/net.d\nExecStartPre=/bin/mkdir -p /opt/cni/bin\nExecStartPre=-/usr/bin/rkt rm --uuid-file=/var/cache/kubelet-pod.uuid\nExecStart=/usr/lib/coreos/kubelet-wrapper \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/etc/kubernetes/kubelet.conf \\\n --pod-manifest-path=/etc/kubernetes/manifests \\\n --allow-privileged=true \\\n --network-plugin=cni \\\n --cni-conf-dir=/etc/cni/net.d \\\n --cni-bin-dir=/opt/cni/bin \\\n --authorization-mode=Webhook \\\n --client-ca-file=/etc/kubernetes/pki/ca.crt \\\n --cadvisor-port=0 \\\n --rotate-certificates=true \\\n --cert-dir=/etc/kubernetes/pki \\\n --authentication-token-webhook=true \\\n --cloud-provider=vsphere \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --read-only-port=0 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --anonymous-auth=false \\\n --protect-kernel-defaults=true \\\n --cluster-dns=10.10.10.10 \\\n --cluster-domain=cluster.local\nExecStop=-/usr/bin/rkt stop --uuid-file=/var/cache/kubelet-pod.uuid\nRestart=always\nRestartSec=10\n[Install]\nWantedBy=multi-user.target\n", + "enabled": true, "name": "kubelet.service" } ] diff --git a/pkg/userdata/coreos/testdata/v1.12.0-vsphere-overwrite-cloudconfig.golden b/pkg/userdata/coreos/testdata/v1.12.0-vsphere-overwrite-cloudconfig.golden index 3c36cf346..b472fee5a 100644 --- a/pkg/userdata/coreos/testdata/v1.12.0-vsphere-overwrite-cloudconfig.golden +++ b/pkg/userdata/coreos/testdata/v1.12.0-vsphere-overwrite-cloudconfig.golden @@ -36,13 +36,24 @@ }, "mode": 420 }, + { + "filesystem": "root", + "group": {}, + "path": "/etc/modules-load.d/k8s.conf", + "user": {}, + "contents": { + "source": "data:,ip_vs%0Aip_vs_rr%0Aip_vs_wrr%0Aip_vs_sh%0Anf_conntrack_ipv4%0A", + "verification": {} + }, + "mode": 420 + }, { "filesystem": "root", "group": {}, "path": "/etc/sysctl.d/k8s.conf", "user": {}, "contents": { - "source": "data:,kernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Avm.overcommit_memory%20%3D%201%0A", + "source": "data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0A", "verification": {} }, "mode": 420 @@ -83,7 +94,7 @@ { "filesystem": "root", "group": {}, - "path": "/etc/kubernetes/bootstrap.kubeconfig", + "path": "/etc/kubernetes/bootstrap-kubelet.conf", "user": {}, "contents": { "source": "data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20%5B%5D%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A", @@ -105,7 +116,7 @@ { "filesystem": "root", "group": {}, - "path": "/etc/kubernetes/ca.crt", + "path": "/etc/kubernetes/pki/ca.crt", "user": {}, "contents": { "source": "data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A", @@ -153,13 +164,13 @@ { "filesystem": "root", "group": {}, - "path": "/opt/bin/download-healthcheck-script.sh", + "path": "/opt/bin/download.sh", "user": {}, "contents": { - "source": "data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-xeuo%20pipefail%0Auntil%20%5B%5B%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20do%0A%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5C%0A%20%20%20%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F8b5b66e4910a6228dfaecccaa0a3b05ec4902f8e%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Adone%0A", + "source": "data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%23setup%20some%20common%20directories%0Amkdir%20-p%20%2Fopt%2Fbin%2F%0Amkdir%20-p%20%2Fvar%2Flib%2Fcalico%0Amkdir%20-p%20%2Fetc%2Fkubernetes%2Fmanifests%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%0Amkdir%20-p%20%2Fopt%2Fcni%2Fbin%0A%0A%23%20cni%0Aif%20%5B%20!%20-f%20%2Fopt%2Fcni%2Fbin%2Floopback%20%5D%3B%20then%0A%20%20%20%20curl%20-L%20https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2Fv0.6.0%2Fcni-plugins-amd64-v0.6.0.tgz%20%7C%20tar%20-xvzC%20%2Fopt%2Fcni%2Fbin%20-f%20-%0Afi%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F8b5b66e4910a6228dfaecccaa0a3b05ec4902f8e%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A", "verification": {} }, - "mode": 755 + "mode": 493 } ] }, @@ -178,29 +189,35 @@ "name": "docker.service" }, { - "contents": "[Unit]\nRequires=network-online.target\nAfter=network-online.target\n[Service]\nType=oneshot\nExecStart=/opt/bin/download-healthcheck-script.sh\n[Install]\nWantedBy=multi-user.target\n", + "contents": "[Unit]\nRequires=network-online.target\nAfter=network-online.target\n[Service]\nType=oneshot\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n", "enabled": true, "name": "download-healthcheck-script.service" }, { - "contents": "[Unit]\nRequires=download-healthcheck-script.service kubelet.service\nAfter=download-healthcheck-script.service kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n", - "enabled": true, - "name": "kubelet-healthcheck.service" - }, - { - "contents": "[Unit]\nRequires=download-healthcheck-script.service docker.service\nAfter=download-healthcheck-script.service docker.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh container-runtime\n\n[Install]\nWantedBy=multi-user.target\n", + "contents": "[Unit]\nRequires=docker.service\nAfter=docker.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh container-runtime\n\n[Install]\nWantedBy=multi-user.target\n", + "dropins": [ + { + "contents": "[Unit]\nRequires=download-healthcheck-script.service\nAfter=download-healthcheck-script.service\n", + "name": "40-docker.conf" + } + ], "enabled": true, "name": "docker-healthcheck.service" }, { - "contents": "[Unit]\nDescription=Kubernetes Kubelet\nRequires=docker.service\nAfter=docker.service\n[Service]\nTimeoutStartSec=5min\nEnvironment=KUBELET_IMAGE=docker://k8s.gcr.io/hyperkube-amd64:v1.12.0\nEnvironment=\"RKT_RUN_ARGS=--uuid-file-save=/var/cache/kubelet-pod.uuid \\\n --insecure-options=image \\\n --volume=resolv,kind=host,source=/etc/resolv.conf \\\n --mount volume=resolv,target=/etc/resolv.conf \\\n --volume cni-bin,kind=host,source=/opt/cni/bin \\\n --mount volume=cni-bin,target=/opt/cni/bin \\\n --volume cni-conf,kind=host,source=/etc/cni/net.d \\\n --mount volume=cni-conf,target=/etc/cni/net.d \\\n --volume etc-kubernetes,kind=host,source=/etc/kubernetes \\\n --mount volume=etc-kubernetes,target=/etc/kubernetes \\\n --volume var-log,kind=host,source=/var/log \\\n --mount volume=var-log,target=/var/log \\\n --volume var-lib-calico,kind=host,source=/var/lib/calico \\\n --mount volume=var-lib-calico,target=/var/lib/calico\"\nExecStartPre=/bin/mkdir -p /var/lib/calico\nExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests\nExecStartPre=/bin/mkdir -p /etc/cni/net.d\nExecStartPre=/bin/mkdir -p /opt/cni/bin\nExecStartPre=-/usr/bin/rkt rm --uuid-file=/var/cache/kubelet-pod.uuid\nExecStart=/usr/lib/coreos/kubelet-wrapper \\\n --container-runtime=docker \\\n --allow-privileged=true \\\n --cni-bin-dir=/opt/cni/bin \\\n --cni-conf-dir=/etc/cni/net.d \\\n --cluster-dns=10.10.10.10 \\\n --cluster-domain=cluster.local \\\n --authentication-token-webhook=true \\\n --hostname-override=node1 \\\n --network-plugin=cni \\\n --cloud-provider=vsphere \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --cert-dir=/etc/kubernetes/ \\\n --pod-manifest-path=/etc/kubernetes/manifests \\\n --resolv-conf=/etc/resolv.conf \\\n --rotate-certificates=true \\\n --kubeconfig=/etc/kubernetes/kubeconfig \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap.kubeconfig \\\n --lock-file=/var/run/lock/kubelet.lock \\\n --exit-on-lock-contention \\\n --read-only-port=0 \\\n --protect-kernel-defaults=true \\\n --authorization-mode=Webhook \\\n --anonymous-auth=false \\\n --client-ca-file=/etc/kubernetes/ca.crt\nExecStop=-/usr/bin/rkt stop --uuid-file=/var/cache/kubelet-pod.uuid\nRestart=always\nRestartSec=10\n[Install]\nWantedBy=multi-user.target\n", + "contents": "[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n", "dropins": [ { - "contents": "[Unit]\nRequires=docker.service\nAfter=docker.service\n", + "contents": "[Unit]\nRequires=download-healthcheck-script.service\nAfter=download-healthcheck-script.service\n", "name": "40-docker.conf" } ], "enabled": true, + "name": "kubelet-healthcheck.service" + }, + { + "contents": "[Unit]\nDescription=Kubernetes Kubelet\nRequires=docker.service\nAfter=docker.service\n[Service]\nTimeoutStartSec=5min\nEnvironment=KUBELET_IMAGE=docker://k8s.gcr.io/hyperkube-amd64:v1.12.0\nEnvironment=\"RKT_RUN_ARGS=--uuid-file-save=/var/cache/kubelet-pod.uuid \\\n --insecure-options=image \\\n --volume=resolv,kind=host,source=/etc/resolv.conf \\\n --mount volume=resolv,target=/etc/resolv.conf \\\n --volume cni-bin,kind=host,source=/opt/cni/bin \\\n --mount volume=cni-bin,target=/opt/cni/bin \\\n --volume cni-conf,kind=host,source=/etc/cni/net.d \\\n --mount volume=cni-conf,target=/etc/cni/net.d \\\n --volume etc-kubernetes,kind=host,source=/etc/kubernetes \\\n --mount volume=etc-kubernetes,target=/etc/kubernetes \\\n --volume var-log,kind=host,source=/var/log \\\n --mount volume=var-log,target=/var/log \\\n --volume var-lib-calico,kind=host,source=/var/lib/calico \\\n --mount volume=var-lib-calico,target=/var/lib/calico\"\nExecStartPre=/bin/mkdir -p /var/lib/calico\nExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests\nExecStartPre=/bin/mkdir -p /etc/cni/net.d\nExecStartPre=/bin/mkdir -p /opt/cni/bin\nExecStartPre=-/usr/bin/rkt rm --uuid-file=/var/cache/kubelet-pod.uuid\nExecStart=/usr/lib/coreos/kubelet-wrapper \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/etc/kubernetes/kubelet.conf \\\n --pod-manifest-path=/etc/kubernetes/manifests \\\n --allow-privileged=true \\\n --network-plugin=cni \\\n --cni-conf-dir=/etc/cni/net.d \\\n --cni-bin-dir=/opt/cni/bin \\\n --authorization-mode=Webhook \\\n --client-ca-file=/etc/kubernetes/pki/ca.crt \\\n --rotate-certificates=true \\\n --cert-dir=/etc/kubernetes/pki \\\n --authentication-token-webhook=true \\\n --cloud-provider=vsphere \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --read-only-port=0 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --anonymous-auth=false \\\n --protect-kernel-defaults=true \\\n --cluster-dns=10.10.10.10 \\\n --cluster-domain=cluster.local\nExecStop=-/usr/bin/rkt stop --uuid-file=/var/cache/kubelet-pod.uuid\nRestart=always\nRestartSec=10\n[Install]\nWantedBy=multi-user.target\n", + "enabled": true, "name": "kubelet.service" } ] diff --git a/pkg/userdata/coreos/testdata/v1.9.2-disable-auto-update-aws.golden b/pkg/userdata/coreos/testdata/v1.9.2-disable-auto-update-aws.golden index af2cea382..4bcea2819 100644 --- a/pkg/userdata/coreos/testdata/v1.9.2-disable-auto-update-aws.golden +++ b/pkg/userdata/coreos/testdata/v1.9.2-disable-auto-update-aws.golden @@ -29,13 +29,24 @@ }, "mode": 420 }, + { + "filesystem": "root", + "group": {}, + "path": "/etc/modules-load.d/k8s.conf", + "user": {}, + "contents": { + "source": "data:,ip_vs%0Aip_vs_rr%0Aip_vs_wrr%0Aip_vs_sh%0Anf_conntrack_ipv4%0A", + "verification": {} + }, + "mode": 420 + }, { "filesystem": "root", "group": {}, "path": "/etc/sysctl.d/k8s.conf", "user": {}, "contents": { - "source": "data:,kernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Avm.overcommit_memory%20%3D%201%0A", + "source": "data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0A", "verification": {} }, "mode": 420 @@ -76,7 +87,7 @@ { "filesystem": "root", "group": {}, - "path": "/etc/kubernetes/bootstrap.kubeconfig", + "path": "/etc/kubernetes/bootstrap-kubelet.conf", "user": {}, "contents": { "source": "data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20%5B%5D%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A", @@ -98,7 +109,7 @@ { "filesystem": "root", "group": {}, - "path": "/etc/kubernetes/ca.crt", + "path": "/etc/kubernetes/pki/ca.crt", "user": {}, "contents": { "source": "data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A", @@ -157,13 +168,13 @@ { "filesystem": "root", "group": {}, - "path": "/opt/bin/download-healthcheck-script.sh", + "path": "/opt/bin/download.sh", "user": {}, "contents": { - "source": "data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-xeuo%20pipefail%0Auntil%20%5B%5B%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20do%0A%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5C%0A%20%20%20%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F8b5b66e4910a6228dfaecccaa0a3b05ec4902f8e%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Adone%0A", + "source": "data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%23setup%20some%20common%20directories%0Amkdir%20-p%20%2Fopt%2Fbin%2F%0Amkdir%20-p%20%2Fvar%2Flib%2Fcalico%0Amkdir%20-p%20%2Fetc%2Fkubernetes%2Fmanifests%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%0Amkdir%20-p%20%2Fopt%2Fcni%2Fbin%0A%0A%23%20cni%0Aif%20%5B%20!%20-f%20%2Fopt%2Fcni%2Fbin%2Floopback%20%5D%3B%20then%0A%20%20%20%20curl%20-L%20https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2Fv0.6.0%2Fcni-plugins-amd64-v0.6.0.tgz%20%7C%20tar%20-xvzC%20%2Fopt%2Fcni%2Fbin%20-f%20-%0Afi%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F8b5b66e4910a6228dfaecccaa0a3b05ec4902f8e%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A", "verification": {} }, - "mode": 755 + "mode": 493 } ] }, @@ -182,29 +193,35 @@ "name": "docker.service" }, { - "contents": "[Unit]\nRequires=network-online.target\nAfter=network-online.target\n[Service]\nType=oneshot\nExecStart=/opt/bin/download-healthcheck-script.sh\n[Install]\nWantedBy=multi-user.target\n", + "contents": "[Unit]\nRequires=network-online.target\nAfter=network-online.target\n[Service]\nType=oneshot\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n", "enabled": true, "name": "download-healthcheck-script.service" }, { - "contents": "[Unit]\nRequires=download-healthcheck-script.service kubelet.service\nAfter=download-healthcheck-script.service kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n", - "enabled": true, - "name": "kubelet-healthcheck.service" - }, - { - "contents": "[Unit]\nRequires=download-healthcheck-script.service docker.service\nAfter=download-healthcheck-script.service docker.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh container-runtime\n\n[Install]\nWantedBy=multi-user.target\n", + "contents": "[Unit]\nRequires=docker.service\nAfter=docker.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh container-runtime\n\n[Install]\nWantedBy=multi-user.target\n", + "dropins": [ + { + "contents": "[Unit]\nRequires=download-healthcheck-script.service\nAfter=download-healthcheck-script.service\n", + "name": "40-docker.conf" + } + ], "enabled": true, "name": "docker-healthcheck.service" }, { - "contents": "[Unit]\nDescription=Kubernetes Kubelet\nRequires=docker.service\nAfter=docker.service\n[Service]\nTimeoutStartSec=5min\nEnvironment=KUBELET_IMAGE=docker://k8s.gcr.io/hyperkube-amd64:v1.9.2\nEnvironment=\"RKT_RUN_ARGS=--uuid-file-save=/var/cache/kubelet-pod.uuid \\\n --insecure-options=image \\\n --volume=resolv,kind=host,source=/etc/resolv.conf \\\n --mount volume=resolv,target=/etc/resolv.conf \\\n --volume cni-bin,kind=host,source=/opt/cni/bin \\\n --mount volume=cni-bin,target=/opt/cni/bin \\\n --volume cni-conf,kind=host,source=/etc/cni/net.d \\\n --mount volume=cni-conf,target=/etc/cni/net.d \\\n --volume etc-kubernetes,kind=host,source=/etc/kubernetes \\\n --mount volume=etc-kubernetes,target=/etc/kubernetes \\\n --volume var-log,kind=host,source=/var/log \\\n --mount volume=var-log,target=/var/log \\\n --volume var-lib-calico,kind=host,source=/var/lib/calico \\\n --mount volume=var-lib-calico,target=/var/lib/calico\"\nExecStartPre=/bin/mkdir -p /var/lib/calico\nExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests\nExecStartPre=/bin/mkdir -p /etc/cni/net.d\nExecStartPre=/bin/mkdir -p /opt/cni/bin\nExecStartPre=-/usr/bin/rkt rm --uuid-file=/var/cache/kubelet-pod.uuid\nExecStart=/usr/lib/coreos/kubelet-wrapper \\\n --container-runtime=docker \\\n --allow-privileged=true \\\n --cni-bin-dir=/opt/cni/bin \\\n --cni-conf-dir=/etc/cni/net.d \\\n --cluster-dns=10.10.10.10 \\\n --cluster-domain=cluster.local \\\n --authentication-token-webhook=true \\\n --hostname-override=node1 \\\n --network-plugin=cni \\\n --cloud-provider=aws \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --cert-dir=/etc/kubernetes/ \\\n --pod-manifest-path=/etc/kubernetes/manifests \\\n --resolv-conf=/etc/resolv.conf \\\n --rotate-certificates=true \\\n --kubeconfig=/etc/kubernetes/kubeconfig \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap.kubeconfig \\\n --lock-file=/var/run/lock/kubelet.lock \\\n --exit-on-lock-contention \\\n --read-only-port=0 \\\n --protect-kernel-defaults=true \\\n --authorization-mode=Webhook \\\n --anonymous-auth=false \\\n --client-ca-file=/etc/kubernetes/ca.crt\nExecStop=-/usr/bin/rkt stop --uuid-file=/var/cache/kubelet-pod.uuid\nRestart=always\nRestartSec=10\n[Install]\nWantedBy=multi-user.target\n", + "contents": "[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n", "dropins": [ { - "contents": "[Unit]\nRequires=docker.service\nAfter=docker.service\n", + "contents": "[Unit]\nRequires=download-healthcheck-script.service\nAfter=download-healthcheck-script.service\n", "name": "40-docker.conf" } ], "enabled": true, + "name": "kubelet-healthcheck.service" + }, + { + "contents": "[Unit]\nDescription=Kubernetes Kubelet\nRequires=docker.service\nAfter=docker.service\n[Service]\nTimeoutStartSec=5min\nEnvironment=KUBELET_IMAGE=docker://k8s.gcr.io/hyperkube-amd64:v1.9.2\nEnvironment=\"RKT_RUN_ARGS=--uuid-file-save=/var/cache/kubelet-pod.uuid \\\n --insecure-options=image \\\n --volume=resolv,kind=host,source=/etc/resolv.conf \\\n --mount volume=resolv,target=/etc/resolv.conf \\\n --volume cni-bin,kind=host,source=/opt/cni/bin \\\n --mount volume=cni-bin,target=/opt/cni/bin \\\n --volume cni-conf,kind=host,source=/etc/cni/net.d \\\n --mount volume=cni-conf,target=/etc/cni/net.d \\\n --volume etc-kubernetes,kind=host,source=/etc/kubernetes \\\n --mount volume=etc-kubernetes,target=/etc/kubernetes \\\n --volume var-log,kind=host,source=/var/log \\\n --mount volume=var-log,target=/var/log \\\n --volume var-lib-calico,kind=host,source=/var/lib/calico \\\n --mount volume=var-lib-calico,target=/var/lib/calico\"\nExecStartPre=/bin/mkdir -p /var/lib/calico\nExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests\nExecStartPre=/bin/mkdir -p /etc/cni/net.d\nExecStartPre=/bin/mkdir -p /opt/cni/bin\nExecStartPre=-/usr/bin/rkt rm --uuid-file=/var/cache/kubelet-pod.uuid\nExecStart=/usr/lib/coreos/kubelet-wrapper \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/etc/kubernetes/kubelet.conf \\\n --pod-manifest-path=/etc/kubernetes/manifests \\\n --allow-privileged=true \\\n --network-plugin=cni \\\n --cni-conf-dir=/etc/cni/net.d \\\n --cni-bin-dir=/opt/cni/bin \\\n --authorization-mode=Webhook \\\n --client-ca-file=/etc/kubernetes/pki/ca.crt \\\n --cadvisor-port=0 \\\n --rotate-certificates=true \\\n --cert-dir=/etc/kubernetes/pki \\\n --authentication-token-webhook=true \\\n --cloud-provider=aws \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --read-only-port=0 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --anonymous-auth=false \\\n --protect-kernel-defaults=true \\\n --cluster-dns=10.10.10.10 \\\n --cluster-domain=cluster.local\nExecStop=-/usr/bin/rkt stop --uuid-file=/var/cache/kubelet-pod.uuid\nRestart=always\nRestartSec=10\n[Install]\nWantedBy=multi-user.target\n", + "enabled": true, "name": "kubelet.service" } ] diff --git a/pkg/userdata/coreos/userdata.go b/pkg/userdata/coreos/userdata.go index 305afb277..842ed6af4 100644 --- a/pkg/userdata/coreos/userdata.go +++ b/pkg/userdata/coreos/userdata.go @@ -13,7 +13,6 @@ import ( clientcmdapi "k8s.io/client-go/tools/clientcmd/api" "github.com/kubermatic/machine-controller/pkg/providerconfig" - machinetemplate "github.com/kubermatic/machine-controller/pkg/template" "github.com/kubermatic/machine-controller/pkg/userdata/cloud" userdatahelper "github.com/kubermatic/machine-controller/pkg/userdata/helper" @@ -48,7 +47,7 @@ func (p Provider) UserData( clusterDNSIPs []net.IP, ) (string, error) { - tmpl, err := template.New("user-data").Funcs(machinetemplate.TxtFuncMap()).Parse(ctTemplate) + tmpl, err := template.New("user-data").Funcs(userdatahelper.TxtFuncMap()).Parse(ctTemplate) if err != nil { return "", fmt.Errorf("failed to parse user-data template: %v", err) } @@ -97,7 +96,6 @@ func (p Provider) UserData( HyperkubeImageTag string ClusterDNSIPs []net.IP KubernetesCACert string - JournaldMaxSize string KubeletVersion string }{ MachineSpec: spec, @@ -109,7 +107,6 @@ func (p Provider) UserData( HyperkubeImageTag: fmt.Sprintf("v%s", kubeletVersion.String()), ClusterDNSIPs: clusterDNSIPs, KubernetesCACert: kubernetesCACert, - JournaldMaxSize: userdatahelper.JournaldMaxUse, KubeletVersion: kubeletVersion.String(), } b := &bytes.Buffer{} @@ -183,45 +180,34 @@ systemd: After=network-online.target [Service] Type=oneshot - ExecStart=/opt/bin/download-healthcheck-script.sh - [Install] - WantedBy=multi-user.target - - - - name: kubelet-healthcheck.service - enabled: true - contents: | - [Unit] - Requires=download-healthcheck-script.service kubelet.service - After=download-healthcheck-script.service kubelet.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh kubelet - + ExecStart=/opt/bin/download.sh [Install] WantedBy=multi-user.target - name: docker-healthcheck.service enabled: true + dropins: + - name: 40-docker.conf + contents: | + [Unit] + Requires=download-healthcheck-script.service + After=download-healthcheck-script.service contents: | - [Unit] - Requires=download-healthcheck-script.service docker.service - After=download-healthcheck-script.service docker.service +{{ containerRuntimeHealthCheckSystemdUnit | indent 10 }} - [Service] - ExecStart=/opt/bin/health-monitor.sh container-runtime - - [Install] - WantedBy=multi-user.target - - - name: kubelet.service + - name: kubelet-healthcheck.service enabled: true dropins: - name: 40-docker.conf contents: | [Unit] - Requires=docker.service - After=docker.service + Requires=download-healthcheck-script.service + After=download-healthcheck-script.service + contents: | +{{ kubeletHealthCheckSystemdUnit | indent 10 }} + + - name: kubelet.service + enabled: true contents: | [Unit] Description=Kubernetes Kubelet @@ -250,32 +236,7 @@ systemd: ExecStartPre=/bin/mkdir -p /opt/cni/bin ExecStartPre=-/usr/bin/rkt rm --uuid-file=/var/cache/kubelet-pod.uuid ExecStart=/usr/lib/coreos/kubelet-wrapper \ - --container-runtime=docker \ - --allow-privileged=true \ - --cni-bin-dir=/opt/cni/bin \ - --cni-conf-dir=/etc/cni/net.d \ - --cluster-dns={{ ipSliceToCommaSeparatedString .ClusterDNSIPs }} \ - --cluster-domain=cluster.local \ - --authentication-token-webhook=true \ - --hostname-override={{ .MachineSpec.Name }} \ - --network-plugin=cni \ - {{- if .CloudProvider }} - --cloud-provider={{ .CloudProvider }} \ - --cloud-config=/etc/kubernetes/cloud-config \ - {{- end }} - --cert-dir=/etc/kubernetes/ \ - --pod-manifest-path=/etc/kubernetes/manifests \ - --resolv-conf=/etc/resolv.conf \ - --rotate-certificates=true \ - --kubeconfig=/etc/kubernetes/kubeconfig \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap.kubeconfig \ - --lock-file=/var/run/lock/kubelet.lock \ - --exit-on-lock-contention \ - --read-only-port=0 \ - --protect-kernel-defaults=true \ - --authorization-mode=Webhook \ - --anonymous-auth=false \ - --client-ca-file=/etc/kubernetes/ca.crt +{{ kubeletFlags .KubeletVersion .CloudProvider .MachineSpec.Name .ClusterDNSIPs | indent 10 }} ExecStop=-/usr/bin/rkt stop --uuid-file=/var/cache/kubelet-pod.uuid Restart=always RestartSec=10 @@ -289,17 +250,21 @@ storage: mode: 0644 contents: inline: | - [Journal] - SystemMaxUse={{ .JournaldMaxSize }} +{{ journalDConfig | indent 10 }} + + - path: /etc/modules-load.d/k8s.conf + filesystem: root + mode: 0644 + contents: + inline: | +{{ kernelModules | indent 10 }} - path: /etc/sysctl.d/k8s.conf filesystem: root mode: 0644 contents: inline: | - kernel.panic_on_oops = 1 - kernel.panic = 10 - vm.overcommit_memory = 1 +{{ kernelSettings | indent 10 }} - path: /proc/sys/kernel/panic_on_oops filesystem: root @@ -322,7 +287,7 @@ storage: inline: | 1 - - path: /etc/kubernetes/bootstrap.kubeconfig + - path: /etc/kubernetes/bootstrap-kubelet.conf filesystem: root mode: 0400 contents: @@ -336,7 +301,7 @@ storage: inline: | {{ .CloudConfig | indent 10 }} - - path: /etc/kubernetes/ca.crt + - path: /etc/kubernetes/pki/ca.crt filesystem: root mode: 0644 contents: @@ -385,16 +350,12 @@ storage: [Service] Environment=DOCKER_OPTS=--storage-driver=overlay2 - - path: /opt/bin/download-healthcheck-script.sh + - path: /opt/bin/download.sh filesystem: root - mode: 755 + mode: 0755 contents: inline: | - #!/usr/bin/env bash + #!/bin/bash set -xeuo pipefail - until [[ -x /opt/bin/health-monitor.sh ]]; do - curl -Lfo /opt/bin/health-monitor.sh \ - https://raw.githubusercontent.com/kubermatic/machine-controller/8b5b66e4910a6228dfaecccaa0a3b05ec4902f8e/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh - done +{{ downloadBinariesScript .KubeletVersion false | indent 10 }} ` diff --git a/pkg/userdata/helper/common_test.go b/pkg/userdata/helper/common_test.go new file mode 100644 index 000000000..ac3c493aa --- /dev/null +++ b/pkg/userdata/helper/common_test.go @@ -0,0 +1,19 @@ +package helper + +import ( + "flag" + + "github.com/Masterminds/semver" +) + +var update = flag.Bool("update", false, "update .golden files") + +var ( + versions = []*semver.Version{ + semver.MustParse("v1.10.0"), + semver.MustParse("v1.11.0"), + semver.MustParse("v1.11.0-rc.2"), + semver.MustParse("v1.11.3"), + semver.MustParse("v1.12.0"), + } +) diff --git a/pkg/userdata/helper/download_binaries_script.go b/pkg/userdata/helper/download_binaries_script.go new file mode 100644 index 000000000..33ef6d69d --- /dev/null +++ b/pkg/userdata/helper/download_binaries_script.go @@ -0,0 +1,59 @@ +package helper + +import ( + "bytes" + "fmt" + "text/template" +) + +const ( + downloadBinariesTpl = `#setup some common directories +mkdir -p /opt/bin/ +mkdir -p /var/lib/calico +mkdir -p /etc/kubernetes/manifests +mkdir -p /etc/cni/net.d +mkdir -p /opt/cni/bin + +# cni +if [ ! -f /opt/cni/bin/loopback ]; then + curl -L https://github.com/containernetworking/plugins/releases/download/v0.6.0/cni-plugins-amd64-v0.6.0.tgz | tar -xvzC /opt/cni/bin -f - +fi + +{{- if .DownloadKubelet }} +# kubelet +if [ ! -f /opt/bin/kubelet ]; then + curl -Lfo /opt/bin/kubelet https://storage.googleapis.com/kubernetes-release/release/v{{ .KubeletVersion }}/bin/linux/amd64/kubelet + chmod +x /opt/bin/kubelet +fi +{{- end }} + +if [[ ! -x /opt/bin/health-monitor.sh ]]; then + curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/8b5b66e4910a6228dfaecccaa0a3b05ec4902f8e/pkg/userdata/scripts/health-monitor.sh + chmod +x /opt/bin/health-monitor.sh +fi +` +) + +// DownloadBinariesScript returns the script which is responsible to download all required binaries. +// Extracted into a dedicated function so we can use it to prepare custom images: TODO: Use it to prepare custom images... +func DownloadBinariesScript(kubeletVersion string, downloadKubelet bool) (string, error) { + tmpl, err := template.New("download-binaries").Funcs(TxtFuncMap()).Parse(downloadBinariesTpl) + if err != nil { + return "", fmt.Errorf("failed to parse download-binaries template: %v", err) + } + + data := struct { + KubeletVersion string + DownloadKubelet bool + }{ + KubeletVersion: kubeletVersion, + DownloadKubelet: downloadKubelet, + } + b := &bytes.Buffer{} + err = tmpl.Execute(b, data) + if err != nil { + return "", fmt.Errorf("failed to execute download-binaries template: %v", err) + } + + return string(b.String()), nil +} diff --git a/pkg/userdata/helper/download_binaries_script_test.go b/pkg/userdata/helper/download_binaries_script_test.go new file mode 100644 index 000000000..19a20a054 --- /dev/null +++ b/pkg/userdata/helper/download_binaries_script_test.go @@ -0,0 +1,22 @@ +package helper + +import ( + "fmt" + "testing" + + "github.com/kubermatic/machine-controller/pkg/test" +) + +func TestDownloadBinariesScript(t *testing.T) { + for _, version := range versions { + name := fmt.Sprintf("download_binaries_%s", version.Original()) + t.Run(name, func(t *testing.T) { + script, err := DownloadBinariesScript(version.String(), true) + if err != nil { + t.Error(err) + } + + test.CompareOutput(t, name, script, *update) + }) + } +} diff --git a/pkg/userdata/helper/helper.go b/pkg/userdata/helper/helper.go index 029de2d90..382f1f5b9 100644 --- a/pkg/userdata/helper/helper.go +++ b/pkg/userdata/helper/helper.go @@ -1,9 +1,6 @@ package helper import ( - "crypto/sha256" - "crypto/x509" - "encoding/pem" "fmt" "strings" @@ -11,12 +8,6 @@ import ( clientcmdapi "k8s.io/client-go/tools/clientcmd/api" ) -const ( - // JournaldMaxUse defines the maximum space that journalD logs can occupy. - // https://www.freedesktop.org/software/systemd/man/journald.conf.html#SystemMaxUse= - JournaldMaxUse = "5G" -) - func GetServerAddressFromKubeconfig(kubeconfig *clientcmdapi.Config) (string, error) { if len(kubeconfig.Clusters) != 1 { return "", fmt.Errorf("kubeconfig does not contain exactly one cluster, can not extract server address") @@ -42,37 +33,6 @@ func GetCACert(kubeconfig *clientcmdapi.Config) (string, error) { return "", fmt.Errorf("no CACert found") } -// GetKubeadmCACertHash returns a sha256sum of the Certificates RawSubjectPublicKeyInfo -func GetKubeadmCACertHash(kubeconfig *clientcmdapi.Config) (string, error) { - cacert, err := GetCACert(kubeconfig) - if err != nil { - return "", err - } - // _ is not an error but the remaining bytes in case the - // input to pem.Decode() contains more than one cert - certBlock, _ := pem.Decode([]byte(cacert)) - if certBlock == nil { - return "", fmt.Errorf("pem certificate is empty") - } - cert, err := x509.ParseCertificate(certBlock.Bytes) - if err != nil { - return "", fmt.Errorf("error parsing certificate: %v", err) - } - return fmt.Sprintf("%x", sha256.Sum256(cert.RawSubjectPublicKeyInfo)), nil -} - -func GetTokenFromKubeconfig(kubeconfig *clientcmdapi.Config) (string, error) { - if len(kubeconfig.AuthInfos) != 1 { - return "", fmt.Errorf("kubeconfig does not contain exactly one token, can not extract token") - } - - for _, authInfo := range kubeconfig.AuthInfos { - return string(authInfo.Token), nil - } - - return "", fmt.Errorf("no token found in kubeconfig") -} - // StringifyKubeconfig marshals a kubeconfig to its text form func StringifyKubeconfig(kubeconfig *clientcmdapi.Config) (string, error) { kubeconfigBytes, err := clientcmd.Write(*kubeconfig) @@ -82,3 +42,33 @@ func StringifyKubeconfig(kubeconfig *clientcmdapi.Config) (string, error) { return string(kubeconfigBytes), nil } + +// KernelModules returns the list of kernel modules required for a kubernetes worker node +func KernelModules() string { + return `ip_vs +ip_vs_rr +ip_vs_wrr +ip_vs_sh +nf_conntrack_ipv4 +` +} + +// KernelSettings returns the list of kernel settings required for a kubernetes worker node +func KernelSettings() string { + return `net.bridge.bridge-nf-call-ip6tables = 1 +net.bridge.bridge-nf-call-iptables = 1 +kernel.panic_on_oops = 1 +kernel.panic = 10 +net.ipv4.ip_forward = 1 +vm.overcommit_memory = 1 +` +} + +// JournalDConfig returns the journal config preferable on every node +func JournalDConfig() string { + // JournaldMaxUse defines the maximum space that journalD logs can occupy. + // https://www.freedesktop.org/software/systemd/man/journald.conf.html#SystemMaxUse= + return `[Journal] +SystemMaxUse=5G +` +} diff --git a/pkg/userdata/helper/kubelet.go b/pkg/userdata/helper/kubelet.go new file mode 100644 index 000000000..a9756c0a4 --- /dev/null +++ b/pkg/userdata/helper/kubelet.go @@ -0,0 +1,140 @@ +package helper + +import ( + "bytes" + "fmt" + "net" + "text/template" +) + +const ( + kubeletFlagsTpl = `--bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ +--kubeconfig=/etc/kubernetes/kubelet.conf \ +--pod-manifest-path=/etc/kubernetes/manifests \ +--allow-privileged=true \ +--network-plugin=cni \ +--cni-conf-dir=/etc/cni/net.d \ +--cni-bin-dir=/opt/cni/bin \ +--authorization-mode=Webhook \ +--client-ca-file=/etc/kubernetes/pki/ca.crt \ +{{- if semverCompare "<1.12.0-0" .KubeletVersion }} +--cadvisor-port=0 \ +{{- end }} +--rotate-certificates=true \ +--cert-dir=/etc/kubernetes/pki \ +--authentication-token-webhook=true \ +{{- if .CloudProvider }} +--cloud-provider={{ .CloudProvider }} \ +--cloud-config=/etc/kubernetes/cloud-config \ +{{- end }} +{{- if .Hostname }} +--hostname-override={{ .Hostname }} \ +{{- end }} +--read-only-port=0 \ +--exit-on-lock-contention \ +--lock-file=/tmp/kubelet.lock \ +--anonymous-auth=false \ +--protect-kernel-defaults=true \ +--cluster-dns={{ .ClusterDNSIPs | join "," }} \ +--cluster-domain=cluster.local` + + kubeletSystemdUnitTpl = `[Unit] +After=docker.service +Requires=docker.service + +Description=kubelet: The Kubernetes Node Agent +Documentation=https://kubernetes.io/docs/home/ + +[Service] +Restart=always +StartLimitInterval=0 +RestartSec=10 + +Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" + +ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ +{{ kubeletFlags .KubeletVersion .CloudProvider .Hostname .ClusterDNSIPs | indent 2 }} + +[Install] +WantedBy=multi-user.target` +) + +// KubeletSystemdUnit returns the systemd unit for the kubelet +func KubeletSystemdUnit(kubeletVersion, cloudProvider, hostname string, dnsIPs []net.IP) (string, error) { + tmpl, err := template.New("kubelet-systemd-unit").Funcs(TxtFuncMap()).Parse(kubeletSystemdUnitTpl) + if err != nil { + return "", fmt.Errorf("failed to parse kubelet-systemd-unit template: %v", err) + } + + data := struct { + KubeletVersion string + CloudProvider string + Hostname string + ClusterDNSIPs []net.IP + }{ + KubeletVersion: kubeletVersion, + CloudProvider: cloudProvider, + Hostname: hostname, + ClusterDNSIPs: dnsIPs, + } + b := &bytes.Buffer{} + err = tmpl.Execute(b, data) + if err != nil { + return "", fmt.Errorf("failed to execute kubelet-systemd-unit template: %v", err) + } + + return string(b.String()), nil +} + +// KubeletFlags returns the kubelet flags +func KubeletFlags(version, cloudProvider, hostname string, dnsIPs []net.IP) (string, error) { + tmpl, err := template.New("kubelet-flags").Funcs(TxtFuncMap()).Parse(kubeletFlagsTpl) + if err != nil { + return "", fmt.Errorf("failed to parse kubelet-flags template: %v", err) + } + + data := struct { + CloudProvider string + Hostname string + ClusterDNSIPs []net.IP + KubeletVersion string + }{ + CloudProvider: cloudProvider, + Hostname: hostname, + ClusterDNSIPs: dnsIPs, + KubeletVersion: version, + } + b := &bytes.Buffer{} + err = tmpl.Execute(b, data) + if err != nil { + return "", fmt.Errorf("failed to execute kubelet-flags template: %v", err) + } + + return string(b.String()), nil +} + +func KubeletHealthCheckSystemdUnit() string { + return `[Unit] +Requires=kubelet.service +After=kubelet.service + +[Service] +ExecStart=/opt/bin/health-monitor.sh kubelet + +[Install] +WantedBy=multi-user.target +` +} + +func ContainerRuntimeHealthCheckSystemdUnit() string { + return `[Unit] +Requires=docker.service +After=docker.service + +[Service] +ExecStart=/opt/bin/health-monitor.sh container-runtime + +[Install] +WantedBy=multi-user.target +` +} diff --git a/pkg/userdata/helper/kubelet_test.go b/pkg/userdata/helper/kubelet_test.go new file mode 100644 index 000000000..275686298 --- /dev/null +++ b/pkg/userdata/helper/kubelet_test.go @@ -0,0 +1,62 @@ +package helper + +import ( + "fmt" + "net" + "testing" + + testhelper "github.com/kubermatic/machine-controller/pkg/test" + + "github.com/Masterminds/semver" +) + +type kubeletFlagTestCase struct { + name string + version *semver.Version + dnsIPs []net.IP + hostname string + cloudProvider string +} + +func TestKubeletSystemdUnit(t *testing.T) { + var tests []kubeletFlagTestCase + for _, version := range versions { + tests = append(tests, kubeletFlagTestCase{ + name: fmt.Sprintf("version-%s", version.Original()), + version: version, + dnsIPs: []net.IP{net.ParseIP("10.10.10.10")}, + hostname: "some-test-node", + }) + } + tests = append(tests, []kubeletFlagTestCase{ + { + name: "multiple-dns-servers", + version: semver.MustParse("v1.10.1"), + dnsIPs: []net.IP{ + net.ParseIP("10.10.10.10"), + net.ParseIP("10.10.10.11"), + net.ParseIP("10.10.10.12"), + }, + hostname: "some-test-node", + }, + { + name: "cloud-provider-set", + version: semver.MustParse("v1.10.1"), + dnsIPs: []net.IP{net.ParseIP("10.10.10.10")}, + hostname: "some-test-node", + cloudProvider: "aws", + }, + }...) + + for _, test := range tests { + name := fmt.Sprintf("kublet_systemd_unit_%s", test.name) + t.Run(name, func(t *testing.T) { + out, err := KubeletSystemdUnit(test.version.String(), test.cloudProvider, test.hostname, test.dnsIPs) + if err != nil { + t.Error(err) + } + + testhelper.CompareOutput(t, name, out, *update) + }) + } +} diff --git a/pkg/userdata/helper/template_functions.go b/pkg/userdata/helper/template_functions.go new file mode 100644 index 000000000..5a08622fd --- /dev/null +++ b/pkg/userdata/helper/template_functions.go @@ -0,0 +1,23 @@ +package helper + +import ( + "text/template" + + "github.com/Masterminds/sprig" +) + +// TxtFuncMap returns an aggregated template function map. Currently (custom functions + sprig) +func TxtFuncMap() template.FuncMap { + funcMap := sprig.TxtFuncMap() + + funcMap["downloadBinariesScript"] = DownloadBinariesScript + funcMap["kubeletSystemdUnit"] = KubeletSystemdUnit + funcMap["kubeletFlags"] = KubeletFlags + funcMap["kernelModules"] = KernelModules + funcMap["kernelSettings"] = KernelSettings + funcMap["journalDConfig"] = JournalDConfig + funcMap["kubeletHealthCheckSystemdUnit"] = KubeletHealthCheckSystemdUnit + funcMap["containerRuntimeHealthCheckSystemdUnit"] = ContainerRuntimeHealthCheckSystemdUnit + + return funcMap +} diff --git a/pkg/userdata/helper/testdata/download_binaries_v1.10.0.golden b/pkg/userdata/helper/testdata/download_binaries_v1.10.0.golden new file mode 100644 index 000000000..07a89bcfe --- /dev/null +++ b/pkg/userdata/helper/testdata/download_binaries_v1.10.0.golden @@ -0,0 +1,21 @@ +#setup some common directories +mkdir -p /opt/bin/ +mkdir -p /var/lib/calico +mkdir -p /etc/kubernetes/manifests +mkdir -p /etc/cni/net.d +mkdir -p /opt/cni/bin + +# cni +if [ ! -f /opt/cni/bin/loopback ]; then + curl -L https://github.com/containernetworking/plugins/releases/download/v0.6.0/cni-plugins-amd64-v0.6.0.tgz | tar -xvzC /opt/cni/bin -f - +fi +# kubelet +if [ ! -f /opt/bin/kubelet ]; then + curl -Lfo /opt/bin/kubelet https://storage.googleapis.com/kubernetes-release/release/v1.10.0/bin/linux/amd64/kubelet + chmod +x /opt/bin/kubelet +fi + +if [[ ! -x /opt/bin/health-monitor.sh ]]; then + curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/8b5b66e4910a6228dfaecccaa0a3b05ec4902f8e/pkg/userdata/scripts/health-monitor.sh + chmod +x /opt/bin/health-monitor.sh +fi diff --git a/pkg/userdata/helper/testdata/download_binaries_v1.11.0-rc.2.golden b/pkg/userdata/helper/testdata/download_binaries_v1.11.0-rc.2.golden new file mode 100644 index 000000000..9fd374fa2 --- /dev/null +++ b/pkg/userdata/helper/testdata/download_binaries_v1.11.0-rc.2.golden @@ -0,0 +1,21 @@ +#setup some common directories +mkdir -p /opt/bin/ +mkdir -p /var/lib/calico +mkdir -p /etc/kubernetes/manifests +mkdir -p /etc/cni/net.d +mkdir -p /opt/cni/bin + +# cni +if [ ! -f /opt/cni/bin/loopback ]; then + curl -L https://github.com/containernetworking/plugins/releases/download/v0.6.0/cni-plugins-amd64-v0.6.0.tgz | tar -xvzC /opt/cni/bin -f - +fi +# kubelet +if [ ! -f /opt/bin/kubelet ]; then + curl -Lfo /opt/bin/kubelet https://storage.googleapis.com/kubernetes-release/release/v1.11.0-rc.2/bin/linux/amd64/kubelet + chmod +x /opt/bin/kubelet +fi + +if [[ ! -x /opt/bin/health-monitor.sh ]]; then + curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/8b5b66e4910a6228dfaecccaa0a3b05ec4902f8e/pkg/userdata/scripts/health-monitor.sh + chmod +x /opt/bin/health-monitor.sh +fi diff --git a/pkg/userdata/helper/testdata/download_binaries_v1.11.0.golden b/pkg/userdata/helper/testdata/download_binaries_v1.11.0.golden new file mode 100644 index 000000000..2f7dc7aee --- /dev/null +++ b/pkg/userdata/helper/testdata/download_binaries_v1.11.0.golden @@ -0,0 +1,21 @@ +#setup some common directories +mkdir -p /opt/bin/ +mkdir -p /var/lib/calico +mkdir -p /etc/kubernetes/manifests +mkdir -p /etc/cni/net.d +mkdir -p /opt/cni/bin + +# cni +if [ ! -f /opt/cni/bin/loopback ]; then + curl -L https://github.com/containernetworking/plugins/releases/download/v0.6.0/cni-plugins-amd64-v0.6.0.tgz | tar -xvzC /opt/cni/bin -f - +fi +# kubelet +if [ ! -f /opt/bin/kubelet ]; then + curl -Lfo /opt/bin/kubelet https://storage.googleapis.com/kubernetes-release/release/v1.11.0/bin/linux/amd64/kubelet + chmod +x /opt/bin/kubelet +fi + +if [[ ! -x /opt/bin/health-monitor.sh ]]; then + curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/8b5b66e4910a6228dfaecccaa0a3b05ec4902f8e/pkg/userdata/scripts/health-monitor.sh + chmod +x /opt/bin/health-monitor.sh +fi diff --git a/pkg/userdata/helper/testdata/download_binaries_v1.11.3.golden b/pkg/userdata/helper/testdata/download_binaries_v1.11.3.golden new file mode 100644 index 000000000..a03d31d55 --- /dev/null +++ b/pkg/userdata/helper/testdata/download_binaries_v1.11.3.golden @@ -0,0 +1,21 @@ +#setup some common directories +mkdir -p /opt/bin/ +mkdir -p /var/lib/calico +mkdir -p /etc/kubernetes/manifests +mkdir -p /etc/cni/net.d +mkdir -p /opt/cni/bin + +# cni +if [ ! -f /opt/cni/bin/loopback ]; then + curl -L https://github.com/containernetworking/plugins/releases/download/v0.6.0/cni-plugins-amd64-v0.6.0.tgz | tar -xvzC /opt/cni/bin -f - +fi +# kubelet +if [ ! -f /opt/bin/kubelet ]; then + curl -Lfo /opt/bin/kubelet https://storage.googleapis.com/kubernetes-release/release/v1.11.3/bin/linux/amd64/kubelet + chmod +x /opt/bin/kubelet +fi + +if [[ ! -x /opt/bin/health-monitor.sh ]]; then + curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/8b5b66e4910a6228dfaecccaa0a3b05ec4902f8e/pkg/userdata/scripts/health-monitor.sh + chmod +x /opt/bin/health-monitor.sh +fi diff --git a/pkg/userdata/helper/testdata/download_binaries_v1.12.0.golden b/pkg/userdata/helper/testdata/download_binaries_v1.12.0.golden new file mode 100644 index 000000000..354912f33 --- /dev/null +++ b/pkg/userdata/helper/testdata/download_binaries_v1.12.0.golden @@ -0,0 +1,21 @@ +#setup some common directories +mkdir -p /opt/bin/ +mkdir -p /var/lib/calico +mkdir -p /etc/kubernetes/manifests +mkdir -p /etc/cni/net.d +mkdir -p /opt/cni/bin + +# cni +if [ ! -f /opt/cni/bin/loopback ]; then + curl -L https://github.com/containernetworking/plugins/releases/download/v0.6.0/cni-plugins-amd64-v0.6.0.tgz | tar -xvzC /opt/cni/bin -f - +fi +# kubelet +if [ ! -f /opt/bin/kubelet ]; then + curl -Lfo /opt/bin/kubelet https://storage.googleapis.com/kubernetes-release/release/v1.12.0/bin/linux/amd64/kubelet + chmod +x /opt/bin/kubelet +fi + +if [[ ! -x /opt/bin/health-monitor.sh ]]; then + curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/8b5b66e4910a6228dfaecccaa0a3b05ec4902f8e/pkg/userdata/scripts/health-monitor.sh + chmod +x /opt/bin/health-monitor.sh +fi diff --git a/pkg/userdata/helper/testdata/kublet_systemd_unit_cloud-provider-set.golden b/pkg/userdata/helper/testdata/kublet_systemd_unit_cloud-provider-set.golden new file mode 100644 index 000000000..35de803cd --- /dev/null +++ b/pkg/userdata/helper/testdata/kublet_systemd_unit_cloud-provider-set.golden @@ -0,0 +1,41 @@ +[Unit] +After=docker.service +Requires=docker.service + +Description=kubelet: The Kubernetes Node Agent +Documentation=https://kubernetes.io/docs/home/ + +[Service] +Restart=always +StartLimitInterval=0 +RestartSec=10 + +Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" + +ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ + --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ + --kubeconfig=/etc/kubernetes/kubelet.conf \ + --pod-manifest-path=/etc/kubernetes/manifests \ + --allow-privileged=true \ + --network-plugin=cni \ + --cni-conf-dir=/etc/cni/net.d \ + --cni-bin-dir=/opt/cni/bin \ + --authorization-mode=Webhook \ + --client-ca-file=/etc/kubernetes/pki/ca.crt \ + --cadvisor-port=0 \ + --rotate-certificates=true \ + --cert-dir=/etc/kubernetes/pki \ + --authentication-token-webhook=true \ + --cloud-provider=aws \ + --cloud-config=/etc/kubernetes/cloud-config \ + --hostname-override=some-test-node \ + --read-only-port=0 \ + --exit-on-lock-contention \ + --lock-file=/tmp/kubelet.lock \ + --anonymous-auth=false \ + --protect-kernel-defaults=true \ + --cluster-dns=10.10.10.10 \ + --cluster-domain=cluster.local + +[Install] +WantedBy=multi-user.target \ No newline at end of file diff --git a/pkg/userdata/helper/testdata/kublet_systemd_unit_multiple-dns-servers.golden b/pkg/userdata/helper/testdata/kublet_systemd_unit_multiple-dns-servers.golden new file mode 100644 index 000000000..057beb7f4 --- /dev/null +++ b/pkg/userdata/helper/testdata/kublet_systemd_unit_multiple-dns-servers.golden @@ -0,0 +1,39 @@ +[Unit] +After=docker.service +Requires=docker.service + +Description=kubelet: The Kubernetes Node Agent +Documentation=https://kubernetes.io/docs/home/ + +[Service] +Restart=always +StartLimitInterval=0 +RestartSec=10 + +Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" + +ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ + --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ + --kubeconfig=/etc/kubernetes/kubelet.conf \ + --pod-manifest-path=/etc/kubernetes/manifests \ + --allow-privileged=true \ + --network-plugin=cni \ + --cni-conf-dir=/etc/cni/net.d \ + --cni-bin-dir=/opt/cni/bin \ + --authorization-mode=Webhook \ + --client-ca-file=/etc/kubernetes/pki/ca.crt \ + --cadvisor-port=0 \ + --rotate-certificates=true \ + --cert-dir=/etc/kubernetes/pki \ + --authentication-token-webhook=true \ + --hostname-override=some-test-node \ + --read-only-port=0 \ + --exit-on-lock-contention \ + --lock-file=/tmp/kubelet.lock \ + --anonymous-auth=false \ + --protect-kernel-defaults=true \ + --cluster-dns=10.10.10.10,10.10.10.11,10.10.10.12 \ + --cluster-domain=cluster.local + +[Install] +WantedBy=multi-user.target \ No newline at end of file diff --git a/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.10.0.golden b/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.10.0.golden new file mode 100644 index 000000000..e194f0b81 --- /dev/null +++ b/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.10.0.golden @@ -0,0 +1,39 @@ +[Unit] +After=docker.service +Requires=docker.service + +Description=kubelet: The Kubernetes Node Agent +Documentation=https://kubernetes.io/docs/home/ + +[Service] +Restart=always +StartLimitInterval=0 +RestartSec=10 + +Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" + +ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ + --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ + --kubeconfig=/etc/kubernetes/kubelet.conf \ + --pod-manifest-path=/etc/kubernetes/manifests \ + --allow-privileged=true \ + --network-plugin=cni \ + --cni-conf-dir=/etc/cni/net.d \ + --cni-bin-dir=/opt/cni/bin \ + --authorization-mode=Webhook \ + --client-ca-file=/etc/kubernetes/pki/ca.crt \ + --cadvisor-port=0 \ + --rotate-certificates=true \ + --cert-dir=/etc/kubernetes/pki \ + --authentication-token-webhook=true \ + --hostname-override=some-test-node \ + --read-only-port=0 \ + --exit-on-lock-contention \ + --lock-file=/tmp/kubelet.lock \ + --anonymous-auth=false \ + --protect-kernel-defaults=true \ + --cluster-dns=10.10.10.10 \ + --cluster-domain=cluster.local + +[Install] +WantedBy=multi-user.target \ No newline at end of file diff --git a/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.11.0-rc.2.golden b/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.11.0-rc.2.golden new file mode 100644 index 000000000..e194f0b81 --- /dev/null +++ b/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.11.0-rc.2.golden @@ -0,0 +1,39 @@ +[Unit] +After=docker.service +Requires=docker.service + +Description=kubelet: The Kubernetes Node Agent +Documentation=https://kubernetes.io/docs/home/ + +[Service] +Restart=always +StartLimitInterval=0 +RestartSec=10 + +Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" + +ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ + --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ + --kubeconfig=/etc/kubernetes/kubelet.conf \ + --pod-manifest-path=/etc/kubernetes/manifests \ + --allow-privileged=true \ + --network-plugin=cni \ + --cni-conf-dir=/etc/cni/net.d \ + --cni-bin-dir=/opt/cni/bin \ + --authorization-mode=Webhook \ + --client-ca-file=/etc/kubernetes/pki/ca.crt \ + --cadvisor-port=0 \ + --rotate-certificates=true \ + --cert-dir=/etc/kubernetes/pki \ + --authentication-token-webhook=true \ + --hostname-override=some-test-node \ + --read-only-port=0 \ + --exit-on-lock-contention \ + --lock-file=/tmp/kubelet.lock \ + --anonymous-auth=false \ + --protect-kernel-defaults=true \ + --cluster-dns=10.10.10.10 \ + --cluster-domain=cluster.local + +[Install] +WantedBy=multi-user.target \ No newline at end of file diff --git a/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.11.0.golden b/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.11.0.golden new file mode 100644 index 000000000..e194f0b81 --- /dev/null +++ b/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.11.0.golden @@ -0,0 +1,39 @@ +[Unit] +After=docker.service +Requires=docker.service + +Description=kubelet: The Kubernetes Node Agent +Documentation=https://kubernetes.io/docs/home/ + +[Service] +Restart=always +StartLimitInterval=0 +RestartSec=10 + +Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" + +ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ + --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ + --kubeconfig=/etc/kubernetes/kubelet.conf \ + --pod-manifest-path=/etc/kubernetes/manifests \ + --allow-privileged=true \ + --network-plugin=cni \ + --cni-conf-dir=/etc/cni/net.d \ + --cni-bin-dir=/opt/cni/bin \ + --authorization-mode=Webhook \ + --client-ca-file=/etc/kubernetes/pki/ca.crt \ + --cadvisor-port=0 \ + --rotate-certificates=true \ + --cert-dir=/etc/kubernetes/pki \ + --authentication-token-webhook=true \ + --hostname-override=some-test-node \ + --read-only-port=0 \ + --exit-on-lock-contention \ + --lock-file=/tmp/kubelet.lock \ + --anonymous-auth=false \ + --protect-kernel-defaults=true \ + --cluster-dns=10.10.10.10 \ + --cluster-domain=cluster.local + +[Install] +WantedBy=multi-user.target \ No newline at end of file diff --git a/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.11.3.golden b/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.11.3.golden new file mode 100644 index 000000000..e194f0b81 --- /dev/null +++ b/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.11.3.golden @@ -0,0 +1,39 @@ +[Unit] +After=docker.service +Requires=docker.service + +Description=kubelet: The Kubernetes Node Agent +Documentation=https://kubernetes.io/docs/home/ + +[Service] +Restart=always +StartLimitInterval=0 +RestartSec=10 + +Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" + +ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ + --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ + --kubeconfig=/etc/kubernetes/kubelet.conf \ + --pod-manifest-path=/etc/kubernetes/manifests \ + --allow-privileged=true \ + --network-plugin=cni \ + --cni-conf-dir=/etc/cni/net.d \ + --cni-bin-dir=/opt/cni/bin \ + --authorization-mode=Webhook \ + --client-ca-file=/etc/kubernetes/pki/ca.crt \ + --cadvisor-port=0 \ + --rotate-certificates=true \ + --cert-dir=/etc/kubernetes/pki \ + --authentication-token-webhook=true \ + --hostname-override=some-test-node \ + --read-only-port=0 \ + --exit-on-lock-contention \ + --lock-file=/tmp/kubelet.lock \ + --anonymous-auth=false \ + --protect-kernel-defaults=true \ + --cluster-dns=10.10.10.10 \ + --cluster-domain=cluster.local + +[Install] +WantedBy=multi-user.target \ No newline at end of file diff --git a/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.12.0.golden b/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.12.0.golden new file mode 100644 index 000000000..1af387455 --- /dev/null +++ b/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.12.0.golden @@ -0,0 +1,38 @@ +[Unit] +After=docker.service +Requires=docker.service + +Description=kubelet: The Kubernetes Node Agent +Documentation=https://kubernetes.io/docs/home/ + +[Service] +Restart=always +StartLimitInterval=0 +RestartSec=10 + +Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" + +ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ + --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ + --kubeconfig=/etc/kubernetes/kubelet.conf \ + --pod-manifest-path=/etc/kubernetes/manifests \ + --allow-privileged=true \ + --network-plugin=cni \ + --cni-conf-dir=/etc/cni/net.d \ + --cni-bin-dir=/opt/cni/bin \ + --authorization-mode=Webhook \ + --client-ca-file=/etc/kubernetes/pki/ca.crt \ + --rotate-certificates=true \ + --cert-dir=/etc/kubernetes/pki \ + --authentication-token-webhook=true \ + --hostname-override=some-test-node \ + --read-only-port=0 \ + --exit-on-lock-contention \ + --lock-file=/tmp/kubelet.lock \ + --anonymous-auth=false \ + --protect-kernel-defaults=true \ + --cluster-dns=10.10.10.10 \ + --cluster-domain=cluster.local + +[Install] +WantedBy=multi-user.target \ No newline at end of file diff --git a/pkg/userdata/test-machines/v1.10.8.yaml b/pkg/userdata/test-machines/v1.10.8.yaml new file mode 100644 index 000000000..8ce40abdd --- /dev/null +++ b/pkg/userdata/test-machines/v1.10.8.yaml @@ -0,0 +1,108 @@ +apiVersion: cluster.k8s.io/v1alpha1 +kind: Machine +metadata: + name: v1.10.8-ubuntu + namespace: kube-system +spec: + metadata: + name: v1.10.8-ubuntu + providerConfig: + value: + cloudProvider: aws + cloudProviderSpec: + accessKeyId: "" + ami: "" + availabilityZone: eu-central-1a + diskSize: 100 + diskType: gp2 + instanceProfile: kubermatic-instance-profile-5f4rjps8mv + instanceType: t2.medium + region: eu-central-1 + secretAccessKey: "" + securityGroupIDs: + - sg-0e1c9bcdb6fb5b057 + subnetId: subnet-2bff4f43 + tags: + kubernetes.io/cluster/5f4rjps8mv: "" + vpcId: vpc-819f62e9 + operatingSystem: ubuntu + operatingSystemSpec: + distUpgradeOnBoot: false + sshPublicKeys: + - ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQC496PqMk7zZuQKXpzcgl3g92LNZMcgEgkfnj4BNoa2CRs5f4wrHMt1Ii78CgY3Ziqw4y191v6IJ3Fby2gT26YWIaK6qSF35sXmjyG/MH+Iu96wIn7JcjlnTkizofGE7zOkRmEtMYcyOVbgnwEH4UNj1W1aalton5JCAqz8kFpKE7sp/2NpxZMqgO68NaqYSd0nDXdwM9HX1grWVF88lBXEJ5YSF03sEpnsOyw0QHhXHyNwHOpfL/wCG0/8OWel1dcXOeQKVVp5V045er0AyIFJVwHSTQCnfyKYCf5i0cLci4+61iSS1hBSl2qBqz6z6mdLy8O7XSP9kPoyQarnQguFO+DHOG8IyTP/Kk3PC/+yR8xGfhFxqM8fGdSdrRwpIgjGH0/7or1vbbUvp52evRMZIdT1YfndahRxafo0iZ36o+VgX1W5oWiZ8Bws/zrV2PCqrmuEbFYSpLati3XvxRQG+Om5p7dzdXx1mb86wqyNe4xpZ2gVInaUYINnwcUmLiaOtCHh4tE/RBMqjcRY7i75gl29H8nbDFwWHGLTULix1W+1FH0XyPiL831XdECYSlw8h+kJW/fYtlynTgeYSLSrIA0atEblf1ui0BPnd8imFX7h8mlUhPgxpfxasI+EQj+m4xjeJUalHTvFuE1wqls8A7A6+J8HP/Ol9cOlDFxexw== + henrik@loodse.com + versions: + kubelet: 1.10.8 +--- +apiVersion: cluster.k8s.io/v1alpha1 +kind: Machine +metadata: + name: v1.10.8-coreos + namespace: kube-system +spec: + metadata: + creationTimestamp: null + name: v1.10.8-coreos + providerConfig: + value: + cloudProvider: aws + cloudProviderSpec: + accessKeyId: "" + ami: "" + availabilityZone: eu-central-1a + diskSize: 100 + diskType: gp2 + instanceProfile: kubermatic-instance-profile-5f4rjps8mv + instanceType: t2.medium + region: eu-central-1 + secretAccessKey: "" + securityGroupIDs: + - sg-0e1c9bcdb6fb5b057 + subnetId: subnet-2bff4f43 + tags: + kubernetes.io/cluster/5f4rjps8mv: "" + vpcId: vpc-819f62e9 + operatingSystem: coreos + operatingSystemSpec: + disableAutoUpdate: false + sshPublicKeys: + - ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQC496PqMk7zZuQKXpzcgl3g92LNZMcgEgkfnj4BNoa2CRs5f4wrHMt1Ii78CgY3Ziqw4y191v6IJ3Fby2gT26YWIaK6qSF35sXmjyG/MH+Iu96wIn7JcjlnTkizofGE7zOkRmEtMYcyOVbgnwEH4UNj1W1aalton5JCAqz8kFpKE7sp/2NpxZMqgO68NaqYSd0nDXdwM9HX1grWVF88lBXEJ5YSF03sEpnsOyw0QHhXHyNwHOpfL/wCG0/8OWel1dcXOeQKVVp5V045er0AyIFJVwHSTQCnfyKYCf5i0cLci4+61iSS1hBSl2qBqz6z6mdLy8O7XSP9kPoyQarnQguFO+DHOG8IyTP/Kk3PC/+yR8xGfhFxqM8fGdSdrRwpIgjGH0/7or1vbbUvp52evRMZIdT1YfndahRxafo0iZ36o+VgX1W5oWiZ8Bws/zrV2PCqrmuEbFYSpLati3XvxRQG+Om5p7dzdXx1mb86wqyNe4xpZ2gVInaUYINnwcUmLiaOtCHh4tE/RBMqjcRY7i75gl29H8nbDFwWHGLTULix1W+1FH0XyPiL831XdECYSlw8h+kJW/fYtlynTgeYSLSrIA0atEblf1ui0BPnd8imFX7h8mlUhPgxpfxasI+EQj+m4xjeJUalHTvFuE1wqls8A7A6+J8HP/Ol9cOlDFxexw== + henrik@loodse.com + versions: + kubelet: 1.10.8 +--- +apiVersion: cluster.k8s.io/v1alpha1 +kind: Machine +metadata: + name: v1.10.8-centos + namespace: kube-system +spec: + metadata: + name: v1.10.8-centos + providerConfig: + value: + cloudProvider: aws + cloudProviderSpec: + accessKeyId: "" + ami: "" + availabilityZone: eu-central-1a + diskSize: 100 + diskType: gp2 + instanceProfile: kubermatic-instance-profile-5f4rjps8mv + instanceType: t2.medium + region: eu-central-1 + secretAccessKey: "" + securityGroupIDs: + - sg-0e1c9bcdb6fb5b057 + subnetId: subnet-2bff4f43 + tags: + kubernetes.io/cluster/5f4rjps8mv: "" + vpcId: vpc-819f62e9 + operatingSystem: centos + operatingSystemSpec: + distUpgradeOnBoot: false + sshPublicKeys: + - ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQC496PqMk7zZuQKXpzcgl3g92LNZMcgEgkfnj4BNoa2CRs5f4wrHMt1Ii78CgY3Ziqw4y191v6IJ3Fby2gT26YWIaK6qSF35sXmjyG/MH+Iu96wIn7JcjlnTkizofGE7zOkRmEtMYcyOVbgnwEH4UNj1W1aalton5JCAqz8kFpKE7sp/2NpxZMqgO68NaqYSd0nDXdwM9HX1grWVF88lBXEJ5YSF03sEpnsOyw0QHhXHyNwHOpfL/wCG0/8OWel1dcXOeQKVVp5V045er0AyIFJVwHSTQCnfyKYCf5i0cLci4+61iSS1hBSl2qBqz6z6mdLy8O7XSP9kPoyQarnQguFO+DHOG8IyTP/Kk3PC/+yR8xGfhFxqM8fGdSdrRwpIgjGH0/7or1vbbUvp52evRMZIdT1YfndahRxafo0iZ36o+VgX1W5oWiZ8Bws/zrV2PCqrmuEbFYSpLati3XvxRQG+Om5p7dzdXx1mb86wqyNe4xpZ2gVInaUYINnwcUmLiaOtCHh4tE/RBMqjcRY7i75gl29H8nbDFwWHGLTULix1W+1FH0XyPiL831XdECYSlw8h+kJW/fYtlynTgeYSLSrIA0atEblf1ui0BPnd8imFX7h8mlUhPgxpfxasI+EQj+m4xjeJUalHTvFuE1wqls8A7A6+J8HP/Ol9cOlDFxexw== + henrik@loodse.com + versions: + kubelet: 1.10.8 diff --git a/pkg/userdata/test-machines/v1.11.3.yaml b/pkg/userdata/test-machines/v1.11.3.yaml new file mode 100644 index 000000000..0861617b8 --- /dev/null +++ b/pkg/userdata/test-machines/v1.11.3.yaml @@ -0,0 +1,108 @@ +apiVersion: cluster.k8s.io/v1alpha1 +kind: Machine +metadata: + name: v1.11.3-ubuntu + namespace: kube-system +spec: + metadata: + name: v1.11.3-ubuntu + providerConfig: + value: + cloudProvider: aws + cloudProviderSpec: + accessKeyId: "" + ami: "" + availabilityZone: eu-central-1a + diskSize: 100 + diskType: gp2 + instanceProfile: kubermatic-instance-profile-5f4rjps8mv + instanceType: t2.medium + region: eu-central-1 + secretAccessKey: "" + securityGroupIDs: + - sg-0e1c9bcdb6fb5b057 + subnetId: subnet-2bff4f43 + tags: + kubernetes.io/cluster/5f4rjps8mv: "" + vpcId: vpc-819f62e9 + operatingSystem: ubuntu + operatingSystemSpec: + distUpgradeOnBoot: false + sshPublicKeys: + - ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQC496PqMk7zZuQKXpzcgl3g92LNZMcgEgkfnj4BNoa2CRs5f4wrHMt1Ii78CgY3Ziqw4y191v6IJ3Fby2gT26YWIaK6qSF35sXmjyG/MH+Iu96wIn7JcjlnTkizofGE7zOkRmEtMYcyOVbgnwEH4UNj1W1aalton5JCAqz8kFpKE7sp/2NpxZMqgO68NaqYSd0nDXdwM9HX1grWVF88lBXEJ5YSF03sEpnsOyw0QHhXHyNwHOpfL/wCG0/8OWel1dcXOeQKVVp5V045er0AyIFJVwHSTQCnfyKYCf5i0cLci4+61iSS1hBSl2qBqz6z6mdLy8O7XSP9kPoyQarnQguFO+DHOG8IyTP/Kk3PC/+yR8xGfhFxqM8fGdSdrRwpIgjGH0/7or1vbbUvp52evRMZIdT1YfndahRxafo0iZ36o+VgX1W5oWiZ8Bws/zrV2PCqrmuEbFYSpLati3XvxRQG+Om5p7dzdXx1mb86wqyNe4xpZ2gVInaUYINnwcUmLiaOtCHh4tE/RBMqjcRY7i75gl29H8nbDFwWHGLTULix1W+1FH0XyPiL831XdECYSlw8h+kJW/fYtlynTgeYSLSrIA0atEblf1ui0BPnd8imFX7h8mlUhPgxpfxasI+EQj+m4xjeJUalHTvFuE1wqls8A7A6+J8HP/Ol9cOlDFxexw== + henrik@loodse.com + versions: + kubelet: 1.11.3 +--- +apiVersion: cluster.k8s.io/v1alpha1 +kind: Machine +metadata: + name: v1.11.3-coreos + namespace: kube-system +spec: + metadata: + creationTimestamp: null + name: v1.11.3-coreos + providerConfig: + value: + cloudProvider: aws + cloudProviderSpec: + accessKeyId: "" + ami: "" + availabilityZone: eu-central-1a + diskSize: 100 + diskType: gp2 + instanceProfile: kubermatic-instance-profile-5f4rjps8mv + instanceType: t2.medium + region: eu-central-1 + secretAccessKey: "" + securityGroupIDs: + - sg-0e1c9bcdb6fb5b057 + subnetId: subnet-2bff4f43 + tags: + kubernetes.io/cluster/5f4rjps8mv: "" + vpcId: vpc-819f62e9 + operatingSystem: coreos + operatingSystemSpec: + disableAutoUpdate: false + sshPublicKeys: + - ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQC496PqMk7zZuQKXpzcgl3g92LNZMcgEgkfnj4BNoa2CRs5f4wrHMt1Ii78CgY3Ziqw4y191v6IJ3Fby2gT26YWIaK6qSF35sXmjyG/MH+Iu96wIn7JcjlnTkizofGE7zOkRmEtMYcyOVbgnwEH4UNj1W1aalton5JCAqz8kFpKE7sp/2NpxZMqgO68NaqYSd0nDXdwM9HX1grWVF88lBXEJ5YSF03sEpnsOyw0QHhXHyNwHOpfL/wCG0/8OWel1dcXOeQKVVp5V045er0AyIFJVwHSTQCnfyKYCf5i0cLci4+61iSS1hBSl2qBqz6z6mdLy8O7XSP9kPoyQarnQguFO+DHOG8IyTP/Kk3PC/+yR8xGfhFxqM8fGdSdrRwpIgjGH0/7or1vbbUvp52evRMZIdT1YfndahRxafo0iZ36o+VgX1W5oWiZ8Bws/zrV2PCqrmuEbFYSpLati3XvxRQG+Om5p7dzdXx1mb86wqyNe4xpZ2gVInaUYINnwcUmLiaOtCHh4tE/RBMqjcRY7i75gl29H8nbDFwWHGLTULix1W+1FH0XyPiL831XdECYSlw8h+kJW/fYtlynTgeYSLSrIA0atEblf1ui0BPnd8imFX7h8mlUhPgxpfxasI+EQj+m4xjeJUalHTvFuE1wqls8A7A6+J8HP/Ol9cOlDFxexw== + henrik@loodse.com + versions: + kubelet: 1.11.3 +--- +apiVersion: cluster.k8s.io/v1alpha1 +kind: Machine +metadata: + name: v1.11.3-centos + namespace: kube-system +spec: + metadata: + name: v1.11.3-centos + providerConfig: + value: + cloudProvider: aws + cloudProviderSpec: + accessKeyId: "" + ami: "" + availabilityZone: eu-central-1a + diskSize: 100 + diskType: gp2 + instanceProfile: kubermatic-instance-profile-5f4rjps8mv + instanceType: t2.medium + region: eu-central-1 + secretAccessKey: "" + securityGroupIDs: + - sg-0e1c9bcdb6fb5b057 + subnetId: subnet-2bff4f43 + tags: + kubernetes.io/cluster/5f4rjps8mv: "" + vpcId: vpc-819f62e9 + operatingSystem: centos + operatingSystemSpec: + distUpgradeOnBoot: false + sshPublicKeys: + - ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQC496PqMk7zZuQKXpzcgl3g92LNZMcgEgkfnj4BNoa2CRs5f4wrHMt1Ii78CgY3Ziqw4y191v6IJ3Fby2gT26YWIaK6qSF35sXmjyG/MH+Iu96wIn7JcjlnTkizofGE7zOkRmEtMYcyOVbgnwEH4UNj1W1aalton5JCAqz8kFpKE7sp/2NpxZMqgO68NaqYSd0nDXdwM9HX1grWVF88lBXEJ5YSF03sEpnsOyw0QHhXHyNwHOpfL/wCG0/8OWel1dcXOeQKVVp5V045er0AyIFJVwHSTQCnfyKYCf5i0cLci4+61iSS1hBSl2qBqz6z6mdLy8O7XSP9kPoyQarnQguFO+DHOG8IyTP/Kk3PC/+yR8xGfhFxqM8fGdSdrRwpIgjGH0/7or1vbbUvp52evRMZIdT1YfndahRxafo0iZ36o+VgX1W5oWiZ8Bws/zrV2PCqrmuEbFYSpLati3XvxRQG+Om5p7dzdXx1mb86wqyNe4xpZ2gVInaUYINnwcUmLiaOtCHh4tE/RBMqjcRY7i75gl29H8nbDFwWHGLTULix1W+1FH0XyPiL831XdECYSlw8h+kJW/fYtlynTgeYSLSrIA0atEblf1ui0BPnd8imFX7h8mlUhPgxpfxasI+EQj+m4xjeJUalHTvFuE1wqls8A7A6+J8HP/Ol9cOlDFxexw== + henrik@loodse.com + versions: + kubelet: 1.11.3 diff --git a/pkg/userdata/test-machines/v1.12.1.yaml b/pkg/userdata/test-machines/v1.12.1.yaml new file mode 100644 index 000000000..9f090b74a --- /dev/null +++ b/pkg/userdata/test-machines/v1.12.1.yaml @@ -0,0 +1,108 @@ +apiVersion: cluster.k8s.io/v1alpha1 +kind: Machine +metadata: + name: v1.12.1-ubuntu + namespace: kube-system +spec: + metadata: + name: v1.12.1-ubuntu + providerConfig: + value: + cloudProvider: aws + cloudProviderSpec: + accessKeyId: "" + ami: "" + availabilityZone: eu-central-1a + diskSize: 100 + diskType: gp2 + instanceProfile: kubermatic-instance-profile-5f4rjps8mv + instanceType: t2.medium + region: eu-central-1 + secretAccessKey: "" + securityGroupIDs: + - sg-0e1c9bcdb6fb5b057 + subnetId: subnet-2bff4f43 + tags: + kubernetes.io/cluster/5f4rjps8mv: "" + vpcId: vpc-819f62e9 + operatingSystem: ubuntu + operatingSystemSpec: + distUpgradeOnBoot: false + sshPublicKeys: + - ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQC496PqMk7zZuQKXpzcgl3g92LNZMcgEgkfnj4BNoa2CRs5f4wrHMt1Ii78CgY3Ziqw4y191v6IJ3Fby2gT26YWIaK6qSF35sXmjyG/MH+Iu96wIn7JcjlnTkizofGE7zOkRmEtMYcyOVbgnwEH4UNj1W1aalton5JCAqz8kFpKE7sp/2NpxZMqgO68NaqYSd0nDXdwM9HX1grWVF88lBXEJ5YSF03sEpnsOyw0QHhXHyNwHOpfL/wCG0/8OWel1dcXOeQKVVp5V045er0AyIFJVwHSTQCnfyKYCf5i0cLci4+61iSS1hBSl2qBqz6z6mdLy8O7XSP9kPoyQarnQguFO+DHOG8IyTP/Kk3PC/+yR8xGfhFxqM8fGdSdrRwpIgjGH0/7or1vbbUvp52evRMZIdT1YfndahRxafo0iZ36o+VgX1W5oWiZ8Bws/zrV2PCqrmuEbFYSpLati3XvxRQG+Om5p7dzdXx1mb86wqyNe4xpZ2gVInaUYINnwcUmLiaOtCHh4tE/RBMqjcRY7i75gl29H8nbDFwWHGLTULix1W+1FH0XyPiL831XdECYSlw8h+kJW/fYtlynTgeYSLSrIA0atEblf1ui0BPnd8imFX7h8mlUhPgxpfxasI+EQj+m4xjeJUalHTvFuE1wqls8A7A6+J8HP/Ol9cOlDFxexw== + henrik@loodse.com + versions: + kubelet: 1.12.1 +--- +apiVersion: cluster.k8s.io/v1alpha1 +kind: Machine +metadata: + name: v1.12.1-coreos + namespace: kube-system +spec: + metadata: + creationTimestamp: null + name: v1.12.1-coreos + providerConfig: + value: + cloudProvider: aws + cloudProviderSpec: + accessKeyId: "" + ami: "" + availabilityZone: eu-central-1a + diskSize: 100 + diskType: gp2 + instanceProfile: kubermatic-instance-profile-5f4rjps8mv + instanceType: t2.medium + region: eu-central-1 + secretAccessKey: "" + securityGroupIDs: + - sg-0e1c9bcdb6fb5b057 + subnetId: subnet-2bff4f43 + tags: + kubernetes.io/cluster/5f4rjps8mv: "" + vpcId: vpc-819f62e9 + operatingSystem: coreos + operatingSystemSpec: + disableAutoUpdate: false + sshPublicKeys: + - ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQC496PqMk7zZuQKXpzcgl3g92LNZMcgEgkfnj4BNoa2CRs5f4wrHMt1Ii78CgY3Ziqw4y191v6IJ3Fby2gT26YWIaK6qSF35sXmjyG/MH+Iu96wIn7JcjlnTkizofGE7zOkRmEtMYcyOVbgnwEH4UNj1W1aalton5JCAqz8kFpKE7sp/2NpxZMqgO68NaqYSd0nDXdwM9HX1grWVF88lBXEJ5YSF03sEpnsOyw0QHhXHyNwHOpfL/wCG0/8OWel1dcXOeQKVVp5V045er0AyIFJVwHSTQCnfyKYCf5i0cLci4+61iSS1hBSl2qBqz6z6mdLy8O7XSP9kPoyQarnQguFO+DHOG8IyTP/Kk3PC/+yR8xGfhFxqM8fGdSdrRwpIgjGH0/7or1vbbUvp52evRMZIdT1YfndahRxafo0iZ36o+VgX1W5oWiZ8Bws/zrV2PCqrmuEbFYSpLati3XvxRQG+Om5p7dzdXx1mb86wqyNe4xpZ2gVInaUYINnwcUmLiaOtCHh4tE/RBMqjcRY7i75gl29H8nbDFwWHGLTULix1W+1FH0XyPiL831XdECYSlw8h+kJW/fYtlynTgeYSLSrIA0atEblf1ui0BPnd8imFX7h8mlUhPgxpfxasI+EQj+m4xjeJUalHTvFuE1wqls8A7A6+J8HP/Ol9cOlDFxexw== + henrik@loodse.com + versions: + kubelet: 1.12.1 +--- +apiVersion: cluster.k8s.io/v1alpha1 +kind: Machine +metadata: + name: v1.12.1-centos + namespace: kube-system +spec: + metadata: + name: v1.12.1-centos + providerConfig: + value: + cloudProvider: aws + cloudProviderSpec: + accessKeyId: "" + ami: "" + availabilityZone: eu-central-1a + diskSize: 100 + diskType: gp2 + instanceProfile: kubermatic-instance-profile-5f4rjps8mv + instanceType: t2.medium + region: eu-central-1 + secretAccessKey: "" + securityGroupIDs: + - sg-0e1c9bcdb6fb5b057 + subnetId: subnet-2bff4f43 + tags: + kubernetes.io/cluster/5f4rjps8mv: "" + vpcId: vpc-819f62e9 + operatingSystem: centos + operatingSystemSpec: + distUpgradeOnBoot: false + sshPublicKeys: + - ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQC496PqMk7zZuQKXpzcgl3g92LNZMcgEgkfnj4BNoa2CRs5f4wrHMt1Ii78CgY3Ziqw4y191v6IJ3Fby2gT26YWIaK6qSF35sXmjyG/MH+Iu96wIn7JcjlnTkizofGE7zOkRmEtMYcyOVbgnwEH4UNj1W1aalton5JCAqz8kFpKE7sp/2NpxZMqgO68NaqYSd0nDXdwM9HX1grWVF88lBXEJ5YSF03sEpnsOyw0QHhXHyNwHOpfL/wCG0/8OWel1dcXOeQKVVp5V045er0AyIFJVwHSTQCnfyKYCf5i0cLci4+61iSS1hBSl2qBqz6z6mdLy8O7XSP9kPoyQarnQguFO+DHOG8IyTP/Kk3PC/+yR8xGfhFxqM8fGdSdrRwpIgjGH0/7or1vbbUvp52evRMZIdT1YfndahRxafo0iZ36o+VgX1W5oWiZ8Bws/zrV2PCqrmuEbFYSpLati3XvxRQG+Om5p7dzdXx1mb86wqyNe4xpZ2gVInaUYINnwcUmLiaOtCHh4tE/RBMqjcRY7i75gl29H8nbDFwWHGLTULix1W+1FH0XyPiL831XdECYSlw8h+kJW/fYtlynTgeYSLSrIA0atEblf1ui0BPnd8imFX7h8mlUhPgxpfxasI+EQj+m4xjeJUalHTvFuE1wqls8A7A6+J8HP/Ol9cOlDFxexw== + henrik@loodse.com + versions: + kubelet: 1.12.1 diff --git a/pkg/userdata/test-machines/v1.9.11.yaml b/pkg/userdata/test-machines/v1.9.11.yaml new file mode 100644 index 000000000..5c8700cad --- /dev/null +++ b/pkg/userdata/test-machines/v1.9.11.yaml @@ -0,0 +1,108 @@ +apiVersion: cluster.k8s.io/v1alpha1 +kind: Machine +metadata: + name: v1.9.11-ubuntu + namespace: kube-system +spec: + metadata: + name: v1.9.11-ubuntu + providerConfig: + value: + cloudProvider: aws + cloudProviderSpec: + accessKeyId: "" + ami: "" + availabilityZone: eu-central-1a + diskSize: 100 + diskType: gp2 + instanceProfile: kubermatic-instance-profile-5f4rjps8mv + instanceType: t2.medium + region: eu-central-1 + secretAccessKey: "" + securityGroupIDs: + - sg-0e1c9bcdb6fb5b057 + subnetId: subnet-2bff4f43 + tags: + kubernetes.io/cluster/5f4rjps8mv: "" + vpcId: vpc-819f62e9 + operatingSystem: ubuntu + operatingSystemSpec: + distUpgradeOnBoot: false + sshPublicKeys: + - ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQC496PqMk7zZuQKXpzcgl3g92LNZMcgEgkfnj4BNoa2CRs5f4wrHMt1Ii78CgY3Ziqw4y191v6IJ3Fby2gT26YWIaK6qSF35sXmjyG/MH+Iu96wIn7JcjlnTkizofGE7zOkRmEtMYcyOVbgnwEH4UNj1W1aalton5JCAqz8kFpKE7sp/2NpxZMqgO68NaqYSd0nDXdwM9HX1grWVF88lBXEJ5YSF03sEpnsOyw0QHhXHyNwHOpfL/wCG0/8OWel1dcXOeQKVVp5V045er0AyIFJVwHSTQCnfyKYCf5i0cLci4+61iSS1hBSl2qBqz6z6mdLy8O7XSP9kPoyQarnQguFO+DHOG8IyTP/Kk3PC/+yR8xGfhFxqM8fGdSdrRwpIgjGH0/7or1vbbUvp52evRMZIdT1YfndahRxafo0iZ36o+VgX1W5oWiZ8Bws/zrV2PCqrmuEbFYSpLati3XvxRQG+Om5p7dzdXx1mb86wqyNe4xpZ2gVInaUYINnwcUmLiaOtCHh4tE/RBMqjcRY7i75gl29H8nbDFwWHGLTULix1W+1FH0XyPiL831XdECYSlw8h+kJW/fYtlynTgeYSLSrIA0atEblf1ui0BPnd8imFX7h8mlUhPgxpfxasI+EQj+m4xjeJUalHTvFuE1wqls8A7A6+J8HP/Ol9cOlDFxexw== + henrik@loodse.com + versions: + kubelet: 1.9.11 +--- +apiVersion: cluster.k8s.io/v1alpha1 +kind: Machine +metadata: + name: v1.9.11-coreos + namespace: kube-system +spec: + metadata: + creationTimestamp: null + name: v1.9.11-coreos + providerConfig: + value: + cloudProvider: aws + cloudProviderSpec: + accessKeyId: "" + ami: "" + availabilityZone: eu-central-1a + diskSize: 100 + diskType: gp2 + instanceProfile: kubermatic-instance-profile-5f4rjps8mv + instanceType: t2.medium + region: eu-central-1 + secretAccessKey: "" + securityGroupIDs: + - sg-0e1c9bcdb6fb5b057 + subnetId: subnet-2bff4f43 + tags: + kubernetes.io/cluster/5f4rjps8mv: "" + vpcId: vpc-819f62e9 + operatingSystem: coreos + operatingSystemSpec: + disableAutoUpdate: false + sshPublicKeys: + - ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQC496PqMk7zZuQKXpzcgl3g92LNZMcgEgkfnj4BNoa2CRs5f4wrHMt1Ii78CgY3Ziqw4y191v6IJ3Fby2gT26YWIaK6qSF35sXmjyG/MH+Iu96wIn7JcjlnTkizofGE7zOkRmEtMYcyOVbgnwEH4UNj1W1aalton5JCAqz8kFpKE7sp/2NpxZMqgO68NaqYSd0nDXdwM9HX1grWVF88lBXEJ5YSF03sEpnsOyw0QHhXHyNwHOpfL/wCG0/8OWel1dcXOeQKVVp5V045er0AyIFJVwHSTQCnfyKYCf5i0cLci4+61iSS1hBSl2qBqz6z6mdLy8O7XSP9kPoyQarnQguFO+DHOG8IyTP/Kk3PC/+yR8xGfhFxqM8fGdSdrRwpIgjGH0/7or1vbbUvp52evRMZIdT1YfndahRxafo0iZ36o+VgX1W5oWiZ8Bws/zrV2PCqrmuEbFYSpLati3XvxRQG+Om5p7dzdXx1mb86wqyNe4xpZ2gVInaUYINnwcUmLiaOtCHh4tE/RBMqjcRY7i75gl29H8nbDFwWHGLTULix1W+1FH0XyPiL831XdECYSlw8h+kJW/fYtlynTgeYSLSrIA0atEblf1ui0BPnd8imFX7h8mlUhPgxpfxasI+EQj+m4xjeJUalHTvFuE1wqls8A7A6+J8HP/Ol9cOlDFxexw== + henrik@loodse.com + versions: + kubelet: 1.9.11 +--- +apiVersion: cluster.k8s.io/v1alpha1 +kind: Machine +metadata: + name: v1.9.11-centos + namespace: kube-system +spec: + metadata: + name: v1.9.11-centos + providerConfig: + value: + cloudProvider: aws + cloudProviderSpec: + accessKeyId: "" + ami: "" + availabilityZone: eu-central-1a + diskSize: 100 + diskType: gp2 + instanceProfile: kubermatic-instance-profile-5f4rjps8mv + instanceType: t2.medium + region: eu-central-1 + secretAccessKey: "" + securityGroupIDs: + - sg-0e1c9bcdb6fb5b057 + subnetId: subnet-2bff4f43 + tags: + kubernetes.io/cluster/5f4rjps8mv: "" + vpcId: vpc-819f62e9 + operatingSystem: centos + operatingSystemSpec: + distUpgradeOnBoot: false + sshPublicKeys: + - ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQC496PqMk7zZuQKXpzcgl3g92LNZMcgEgkfnj4BNoa2CRs5f4wrHMt1Ii78CgY3Ziqw4y191v6IJ3Fby2gT26YWIaK6qSF35sXmjyG/MH+Iu96wIn7JcjlnTkizofGE7zOkRmEtMYcyOVbgnwEH4UNj1W1aalton5JCAqz8kFpKE7sp/2NpxZMqgO68NaqYSd0nDXdwM9HX1grWVF88lBXEJ5YSF03sEpnsOyw0QHhXHyNwHOpfL/wCG0/8OWel1dcXOeQKVVp5V045er0AyIFJVwHSTQCnfyKYCf5i0cLci4+61iSS1hBSl2qBqz6z6mdLy8O7XSP9kPoyQarnQguFO+DHOG8IyTP/Kk3PC/+yR8xGfhFxqM8fGdSdrRwpIgjGH0/7or1vbbUvp52evRMZIdT1YfndahRxafo0iZ36o+VgX1W5oWiZ8Bws/zrV2PCqrmuEbFYSpLati3XvxRQG+Om5p7dzdXx1mb86wqyNe4xpZ2gVInaUYINnwcUmLiaOtCHh4tE/RBMqjcRY7i75gl29H8nbDFwWHGLTULix1W+1FH0XyPiL831XdECYSlw8h+kJW/fYtlynTgeYSLSrIA0atEblf1ui0BPnd8imFX7h8mlUhPgxpfxasI+EQj+m4xjeJUalHTvFuE1wqls8A7A6+J8HP/Ol9cOlDFxexw== + henrik@loodse.com + versions: + kubelet: 1.9.11 diff --git a/pkg/userdata/ubuntu/testdata/1.11-aws.golden b/pkg/userdata/ubuntu/testdata/1.11-aws.golden deleted file mode 100644 index 9ddfebc9c..000000000 --- a/pkg/userdata/ubuntu/testdata/1.11-aws.golden +++ /dev/null @@ -1,277 +0,0 @@ -#cloud-config -hostname: node1 - -ssh_pwauth: no - -ssh_authorized_keys: -- "ssh-rsa AAABBB" -- "ssh-rsa CCCDDD" - -write_files: -- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" - content: | - [Journal] - SystemMaxUse=5G - -- path: "/etc/sysctl.d/k8s.conf" - content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - vm.overcommit_memory = 1 - -- path: "/etc/kubernetes/cloud-config" - content: | - {aws-config:true} - -- path: "/etc/apt/sources.list.d/docker.list" - permissions: "0644" - content: deb [arch=amd64] https://download.docker.com/linux/ubuntu bionic stable - -- path: "/etc/apt/sources.list.d/kubernetes.list" - permissions: "0644" - content: deb http://apt.kubernetes.io/ kubernetes-xenial main - -- path: "/usr/local/bin/setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - - sysctl --system - mkdir -p /opt/bin - apt-key add /opt/docker.asc - apt-key add /opt/kubernetes.asc - apt-get update - - # Hetzner's Ubuntu Bionic comes with swap pre-configured, so we force it off. - systemctl mask swap.target - swapoff -a - - # If something failed during package installation but one of docker/kubeadm/kubelet was already installed - # an apt-mark hold after the install won't do it, which is why we test here if the binaries exist and if - # yes put them on hold - set +e - which docker && apt-mark hold docker.io docker-ce - which kubelet && apt-mark hold kubelet - which kubeadm && apt-mark hold kubeadm - - # When docker is started from within the apt installation it fails with a - # 'no sockets found via socket activation: make sure the service was started by systemd' - # Apparently the package is broken in a way that it gets started without its dependencies, manually starting - # it works fine thought - which docker && systemctl start docker - set -e - if [[ -e /var/run/reboot-required ]]; then - reboot - fi - - export CR_PKG='docker-ce=18.06.0~ce~3-0~ubuntu' - export CR_PKG='docker.io=17.12.1-0ubuntu1' - - - DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install -y \ - curl \ - ca-certificates \ - ceph-common \ - cifs-utils \ - conntrack \ - e2fsprogs \ - ebtables \ - ethtool \ - glusterfs-client \ - iptables \ - jq \ - kmod \ - openssh-client \ - nfs-common \ - socat \ - util-linux \ - ${CR_PKG} \ - open-vm-tools \ - kubelet=1.11.0-00 \ - kubeadm=1.11.0-00 \ - - cp /etc/default/kubelet-overwrite /etc/default/kubelet - - systemctl enable --now docker - systemctl enable kubelet - - if [[ ! -x /usr/local/bin/health-monitor.sh ]]; then - curl -Lfo /usr/local/bin/health-monitor.sh \ - https://raw.githubusercontent.com/kubermatic/machine-controller/8b5b66e4910a6228dfaecccaa0a3b05ec4902f8e/pkg/userdata/scripts/health-monitor.sh - chmod +x /usr/local/bin/health-monitor.sh - fi - - if ! [[ -e /etc/kubernetes/pki/ca.crt ]]; then - kubeadm join \ - --token my-token \ - --discovery-token-ca-cert-hash sha256:6caecce9fedcb55d4953d61a27dc6997361a2f226ad86d7e6004dde7526fc4b1 \ - --ignore-preflight-errors=CRI \ - server:443 - fi - - systemctl enable --now --no-block kubelet-healthcheck.service - systemctl enable --now --no-block docker-healthcheck.service - -- path: "/opt/kubernetes.asc" - permissions: "0400" - content: | - -----BEGIN PGP PUBLIC KEY BLOCK----- - - mQENBFrBaNsBCADrF18KCbsZlo4NjAvVecTBCnp6WcBQJ5oSh7+E98jX9YznUCrN - rgmeCcCMUvTDRDxfTaDJybaHugfba43nqhkbNpJ47YXsIa+YL6eEE9emSmQtjrSW - IiY+2YJYwsDgsgckF3duqkb02OdBQlh6IbHPoXB6H//b1PgZYsomB+841XW1LSJP - YlYbIrWfwDfQvtkFQI90r6NknVTQlpqQh5GLNWNYqRNrGQPmsB+NrUYrkl1nUt1L - RGu+rCe4bSaSmNbwKMQKkROE4kTiB72DPk7zH4Lm0uo0YFFWG4qsMIuqEihJ/9KN - X8GYBr+tWgyLooLlsdK3l+4dVqd8cjkJM1ExABEBAAG0QEdvb2dsZSBDbG91ZCBQ - YWNrYWdlcyBBdXRvbWF0aWMgU2lnbmluZyBLZXkgPGdjLXRlYW1AZ29vZ2xlLmNv - bT6JAT4EEwECACgFAlrBaNsCGy8FCQWjmoAGCwkIBwMCBhUIAgkKCwQWAgMBAh4B - AheAAAoJEGoDCyG6B/T78e8H/1WH2LN/nVNhm5TS1VYJG8B+IW8zS4BqyozxC9iJ - AJqZIVHXl8g8a/Hus8RfXR7cnYHcg8sjSaJfQhqO9RbKnffiuQgGrqwQxuC2jBa6 - M/QKzejTeP0Mgi67pyrLJNWrFI71RhritQZmzTZ2PoWxfv6b+Tv5v0rPaG+ut1J4 - 7pn+kYgtUaKdsJz1umi6HzK6AacDf0C0CksJdKG7MOWsZcB4xeOxJYuy6NuO6Kcd - Ez8/XyEUjIuIOlhYTd0hH8E/SEBbXXft7/VBQC5wNq40izPi+6WFK/e1O42DIpzQ - 749ogYQ1eodexPNhLzekKR3XhGrNXJ95r5KO10VrsLFNd8I= - =TKuP - -----END PGP PUBLIC KEY BLOCK----- - -- path: "/opt/docker.asc" - permissions: "0400" - content: | - -----BEGIN PGP PUBLIC KEY BLOCK----- - - mQINBFit2ioBEADhWpZ8/wvZ6hUTiXOwQHXMAlaFHcPH9hAtr4F1y2+OYdbtMuth - lqqwp028AqyY+PRfVMtSYMbjuQuu5byyKR01BbqYhuS3jtqQmljZ/bJvXqnmiVXh - 38UuLa+z077PxyxQhu5BbqntTPQMfiyqEiU+BKbq2WmANUKQf+1AmZY/IruOXbnq - L4C1+gJ8vfmXQt99npCaxEjaNRVYfOS8QcixNzHUYnb6emjlANyEVlZzeqo7XKl7 - UrwV5inawTSzWNvtjEjj4nJL8NsLwscpLPQUhTQ+7BbQXAwAmeHCUTQIvvWXqw0N - cmhh4HgeQscQHYgOJjjDVfoY5MucvglbIgCqfzAHW9jxmRL4qbMZj+b1XoePEtht - ku4bIQN1X5P07fNWzlgaRL5Z4POXDDZTlIQ/El58j9kp4bnWRCJW0lya+f8ocodo - vZZ+Doi+fy4D5ZGrL4XEcIQP/Lv5uFyf+kQtl/94VFYVJOleAv8W92KdgDkhTcTD - G7c0tIkVEKNUq48b3aQ64NOZQW7fVjfoKwEZdOqPE72Pa45jrZzvUFxSpdiNk2tZ - XYukHjlxxEgBdC/J3cMMNRE1F4NCA3ApfV1Y7/hTeOnmDuDYwr9/obA8t016Yljj - q5rdkywPf4JF8mXUW5eCN1vAFHxeg9ZWemhBtQmGxXnw9M+z6hWwc6ahmwARAQAB - tCtEb2NrZXIgUmVsZWFzZSAoQ0UgZGViKSA8ZG9ja2VyQGRvY2tlci5jb20+iQI3 - BBMBCgAhBQJYrefAAhsvBQsJCAcDBRUKCQgLBRYCAwEAAh4BAheAAAoJEI2BgDwO - v82IsskP/iQZo68flDQmNvn8X5XTd6RRaUH33kXYXquT6NkHJciS7E2gTJmqvMqd - tI4mNYHCSEYxI5qrcYV5YqX9P6+Ko+vozo4nseUQLPH/ATQ4qL0Zok+1jkag3Lgk - jonyUf9bwtWxFp05HC3GMHPhhcUSexCxQLQvnFWXD2sWLKivHp2fT8QbRGeZ+d3m - 6fqcd5Fu7pxsqm0EUDK5NL+nPIgYhN+auTrhgzhK1CShfGccM/wfRlei9Utz6p9P - XRKIlWnXtT4qNGZNTN0tR+NLG/6Bqd8OYBaFAUcue/w1VW6JQ2VGYZHnZu9S8LMc - FYBa5Ig9PxwGQOgq6RDKDbV+PqTQT5EFMeR1mrjckk4DQJjbxeMZbiNMG5kGECA8 - g383P3elhn03WGbEEa4MNc3Z4+7c236QI3xWJfNPdUbXRaAwhy/6rTSFbzwKB0Jm - ebwzQfwjQY6f55MiI/RqDCyuPj3r3jyVRkK86pQKBAJwFHyqj9KaKXMZjfVnowLh - 9svIGfNbGHpucATqREvUHuQbNnqkCx8VVhtYkhDb9fEP2xBu5VvHbR+3nfVhMut5 - G34Ct5RS7Jt6LIfFdtcn8CaSas/l1HbiGeRgc70X/9aYx/V/CEJv0lIe8gP6uDoW - FPIZ7d6vH+Vro6xuWEGiuMaiznap2KhZmpkgfupyFmplh0s6knymuQINBFit2ioB - EADneL9S9m4vhU3blaRjVUUyJ7b/qTjcSylvCH5XUE6R2k+ckEZjfAMZPLpO+/tF - M2JIJMD4SifKuS3xck9KtZGCufGmcwiLQRzeHF7vJUKrLD5RTkNi23ydvWZgPjtx - Q+DTT1Zcn7BrQFY6FgnRoUVIxwtdw1bMY/89rsFgS5wwuMESd3Q2RYgb7EOFOpnu - w6da7WakWf4IhnF5nsNYGDVaIHzpiqCl+uTbf1epCjrOlIzkZ3Z3Yk5CM/TiFzPk - z2lLz89cpD8U+NtCsfagWWfjd2U3jDapgH+7nQnCEWpROtzaKHG6lA3pXdix5zG8 - eRc6/0IbUSWvfjKxLLPfNeCS2pCL3IeEI5nothEEYdQH6szpLog79xB9dVnJyKJb - VfxXnseoYqVrRz2VVbUI5Blwm6B40E3eGVfUQWiux54DspyVMMk41Mx7QJ3iynIa - 1N4ZAqVMAEruyXTRTxc9XW0tYhDMA/1GYvz0EmFpm8LzTHA6sFVtPm/ZlNCX6P1X - zJwrv7DSQKD6GGlBQUX+OeEJ8tTkkf8QTJSPUdh8P8YxDFS5EOGAvhhpMBYD42kQ - pqXjEC+XcycTvGI7impgv9PDY1RCC1zkBjKPa120rNhv/hkVk/YhuGoajoHyy4h7 - ZQopdcMtpN2dgmhEegny9JCSwxfQmQ0zK0g7m6SHiKMwjwARAQABiQQ+BBgBCAAJ - BQJYrdoqAhsCAikJEI2BgDwOv82IwV0gBBkBCAAGBQJYrdoqAAoJEH6gqcPyc/zY - 1WAP/2wJ+R0gE6qsce3rjaIz58PJmc8goKrir5hnElWhPgbq7cYIsW5qiFyLhkdp - YcMmhD9mRiPpQn6Ya2w3e3B8zfIVKipbMBnke/ytZ9M7qHmDCcjoiSmwEXN3wKYI - mD9VHONsl/CG1rU9Isw1jtB5g1YxuBA7M/m36XN6x2u+NtNMDB9P56yc4gfsZVES - KA9v+yY2/l45L8d/WUkUi0YXomn6hyBGI7JrBLq0CX37GEYP6O9rrKipfz73XfO7 - JIGzOKZlljb/D9RX/g7nRbCn+3EtH7xnk+TK/50euEKw8SMUg147sJTcpQmv6UzZ - cM4JgL0HbHVCojV4C/plELwMddALOFeYQzTif6sMRPf+3DSj8frbInjChC3yOLy0 - 6br92KFom17EIj2CAcoeq7UPhi2oouYBwPxh5ytdehJkoo+sN7RIWua6P2WSmon5 - U888cSylXC0+ADFdgLX9K2zrDVYUG1vo8CX0vzxFBaHwN6Px26fhIT1/hYUHQR1z - VfNDcyQmXqkOnZvvoMfz/Q0s9BhFJ/zU6AgQbIZE/hm1spsfgvtsD1frZfygXJ9f - irP+MSAI80xHSf91qSRZOj4Pl3ZJNbq4yYxv0b1pkMqeGdjdCYhLU+LZ4wbQmpCk - SVe2prlLureigXtmZfkqevRz7FrIZiu9ky8wnCAPwC7/zmS18rgP/17bOtL4/iIz - QhxAAoAMWVrGyJivSkjhSGx1uCojsWfsTAm11P7jsruIL61ZzMUVE2aM3Pmj5G+W - 9AcZ58Em+1WsVnAXdUR//bMmhyr8wL/G1YO1V3JEJTRdxsSxdYa4deGBBY/Adpsw - 24jxhOJR+lsJpqIUeb999+R8euDhRHG9eFO7DRu6weatUJ6suupoDTRWtr/4yGqe - dKxV3qQhNLSnaAzqW/1nA3iUB4k7kCaKZxhdhDbClf9P37qaRW467BLCVO/coL3y - Vm50dwdrNtKpMBh3ZpbB1uJvgi9mXtyBOMJ3v8RZeDzFiG8HdCtg9RvIt/AIFoHR - H3S+U79NT6i0KPzLImDfs8T7RlpyuMc4Ufs8ggyg9v3Ae6cN3eQyxcK3w0cbBwsh - /nQNfsA6uu+9H7NhbehBMhYnpNZyrHzCmzyXkauwRAqoCbGCNykTRwsur9gS41TQ - M8ssD1jFheOJf3hODnkKU+HKjvMROl1DK7zdmLdNzA1cvtZH/nCC9KPj1z8QC47S - xx+dTZSx4ONAhwbS/LN3PoKtn8LPjY9NP9uDWI+TWYquS2U+KHDrBDlsgozDbs/O - jCxcpDzNmXpWQHEtHU7649OXHP7UeNST1mCUCH5qdank0V1iejF6/CfTFU4MfcrG - YT90qFF93M3v01BbxP+EIY2/9tiIPbrd - =0YYh - -----END PGP PUBLIC KEY BLOCK----- - -- path: "/usr/local/bin/supervise.sh" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - while ! "$@"; do - sleep 1 - done - -- path: "/etc/default/kubelet-overwrite" - content: | - KUBELET_DNS_ARGS= - KUBELET_EXTRA_ARGS=--authentication-token-webhook=true \ - --cloud-provider=aws \ - --cloud-config=/etc/kubernetes/cloud-config \ - --hostname-override=node1 \ - --read-only-port=0 \ - --protect-kernel-defaults=true \ - --cluster-dns=10.10.10.10 \ - --cluster-domain=cluster.local - - -- path: "/etc/systemd/system/setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - ExecStart=/usr/local/bin/supervise.sh /usr/local/bin/setup - -- path: /etc/systemd/system/docker.service.d/10-storage.conf - permissions: "0644" - content: | - [Service] - ExecStart= - ExecStart=/usr/bin/dockerd -H fd:// --storage-driver=overlay2 - -- path: /etc/systemd/system/kubelet-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/usr/local/bin/health-monitor.sh kubelet - - [Install] - WantedBy=multi-user.target - -- path: /etc/systemd/system/docker-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=docker.service - After=docker.service - - [Service] - ExecStart=/usr/local/bin/health-monitor.sh container-runtime - - [Install] - WantedBy=multi-user.target - -runcmd: -- systemctl enable --now setup.service diff --git a/pkg/userdata/ubuntu/testdata/1.9.2-dist-upgrade-on-boot-aws.golden b/pkg/userdata/ubuntu/testdata/1.9.2-dist-upgrade-on-boot-aws.golden deleted file mode 100644 index ff0984858..000000000 --- a/pkg/userdata/ubuntu/testdata/1.9.2-dist-upgrade-on-boot-aws.golden +++ /dev/null @@ -1,285 +0,0 @@ -#cloud-config -hostname: node1 - -ssh_pwauth: no - -ssh_authorized_keys: -- "ssh-rsa AAABBB" -- "ssh-rsa CCCDDD" - -write_files: -- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" - content: | - [Journal] - SystemMaxUse=5G - -- path: "/etc/sysctl.d/k8s.conf" - content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - vm.overcommit_memory = 1 - -- path: "/etc/kubernetes/cloud-config" - content: | - {aws-config:true} - -- path: "/etc/apt/sources.list.d/docker.list" - permissions: "0644" - content: deb [arch=amd64] https://download.docker.com/linux/ubuntu bionic stable - -- path: "/etc/apt/sources.list.d/kubernetes.list" - permissions: "0644" - content: deb http://apt.kubernetes.io/ kubernetes-xenial main - -- path: "/usr/local/bin/setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - - sysctl --system - mkdir -p /opt/bin - apt-key add /opt/docker.asc - apt-key add /opt/kubernetes.asc - apt-get update - - # Hetzner's Ubuntu Bionic comes with swap pre-configured, so we force it off. - systemctl mask swap.target - swapoff -a - - # If something failed during package installation but one of docker/kubeadm/kubelet was already installed - # an apt-mark hold after the install won't do it, which is why we test here if the binaries exist and if - # yes put them on hold - set +e - which docker && apt-mark hold docker.io docker-ce - which kubelet && apt-mark hold kubelet - which kubeadm && apt-mark hold kubeadm - - # When docker is started from within the apt installation it fails with a - # 'no sockets found via socket activation: make sure the service was started by systemd' - # Apparently the package is broken in a way that it gets started without its dependencies, manually starting - # it works fine thought - which docker && systemctl start docker - set -e - DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" dist-upgrade -y - if [[ -e /var/run/reboot-required ]]; then - reboot - fi - - export CR_PKG='docker-ce=18.06.0~ce~3-0~ubuntu' - export CR_PKG='docker.io=17.12.1-0ubuntu1' - - - DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install -y \ - curl \ - ca-certificates \ - ceph-common \ - cifs-utils \ - conntrack \ - e2fsprogs \ - ebtables \ - ethtool \ - glusterfs-client \ - iptables \ - jq \ - kmod \ - openssh-client \ - nfs-common \ - socat \ - util-linux \ - ${CR_PKG} \ - open-vm-tools \ - kubelet=1.9.2-00 \ - kubeadm=1.9.2-00 \ - - cp /etc/default/kubelet-overwrite /etc/default/kubelet - - systemctl enable --now docker - systemctl enable kubelet - - if [[ ! -x /usr/local/bin/health-monitor.sh ]]; then - curl -Lfo /usr/local/bin/health-monitor.sh \ - https://raw.githubusercontent.com/kubermatic/machine-controller/8b5b66e4910a6228dfaecccaa0a3b05ec4902f8e/pkg/userdata/scripts/health-monitor.sh - chmod +x /usr/local/bin/health-monitor.sh - fi - - if ! [[ -e /etc/kubernetes/pki/ca.crt ]]; then - kubeadm join \ - --token my-token \ - --discovery-token-ca-cert-hash sha256:6caecce9fedcb55d4953d61a27dc6997361a2f226ad86d7e6004dde7526fc4b1 \ - --ignore-preflight-errors=CRI \ - server:443 - fi - - systemctl enable --now --no-block kubelet-healthcheck.service - systemctl enable --now --no-block docker-healthcheck.service - -- path: "/opt/kubernetes.asc" - permissions: "0400" - content: | - -----BEGIN PGP PUBLIC KEY BLOCK----- - - mQENBFrBaNsBCADrF18KCbsZlo4NjAvVecTBCnp6WcBQJ5oSh7+E98jX9YznUCrN - rgmeCcCMUvTDRDxfTaDJybaHugfba43nqhkbNpJ47YXsIa+YL6eEE9emSmQtjrSW - IiY+2YJYwsDgsgckF3duqkb02OdBQlh6IbHPoXB6H//b1PgZYsomB+841XW1LSJP - YlYbIrWfwDfQvtkFQI90r6NknVTQlpqQh5GLNWNYqRNrGQPmsB+NrUYrkl1nUt1L - RGu+rCe4bSaSmNbwKMQKkROE4kTiB72DPk7zH4Lm0uo0YFFWG4qsMIuqEihJ/9KN - X8GYBr+tWgyLooLlsdK3l+4dVqd8cjkJM1ExABEBAAG0QEdvb2dsZSBDbG91ZCBQ - YWNrYWdlcyBBdXRvbWF0aWMgU2lnbmluZyBLZXkgPGdjLXRlYW1AZ29vZ2xlLmNv - bT6JAT4EEwECACgFAlrBaNsCGy8FCQWjmoAGCwkIBwMCBhUIAgkKCwQWAgMBAh4B - AheAAAoJEGoDCyG6B/T78e8H/1WH2LN/nVNhm5TS1VYJG8B+IW8zS4BqyozxC9iJ - AJqZIVHXl8g8a/Hus8RfXR7cnYHcg8sjSaJfQhqO9RbKnffiuQgGrqwQxuC2jBa6 - M/QKzejTeP0Mgi67pyrLJNWrFI71RhritQZmzTZ2PoWxfv6b+Tv5v0rPaG+ut1J4 - 7pn+kYgtUaKdsJz1umi6HzK6AacDf0C0CksJdKG7MOWsZcB4xeOxJYuy6NuO6Kcd - Ez8/XyEUjIuIOlhYTd0hH8E/SEBbXXft7/VBQC5wNq40izPi+6WFK/e1O42DIpzQ - 749ogYQ1eodexPNhLzekKR3XhGrNXJ95r5KO10VrsLFNd8I= - =TKuP - -----END PGP PUBLIC KEY BLOCK----- - -- path: "/opt/docker.asc" - permissions: "0400" - content: | - -----BEGIN PGP PUBLIC KEY BLOCK----- - - mQINBFit2ioBEADhWpZ8/wvZ6hUTiXOwQHXMAlaFHcPH9hAtr4F1y2+OYdbtMuth - lqqwp028AqyY+PRfVMtSYMbjuQuu5byyKR01BbqYhuS3jtqQmljZ/bJvXqnmiVXh - 38UuLa+z077PxyxQhu5BbqntTPQMfiyqEiU+BKbq2WmANUKQf+1AmZY/IruOXbnq - L4C1+gJ8vfmXQt99npCaxEjaNRVYfOS8QcixNzHUYnb6emjlANyEVlZzeqo7XKl7 - UrwV5inawTSzWNvtjEjj4nJL8NsLwscpLPQUhTQ+7BbQXAwAmeHCUTQIvvWXqw0N - cmhh4HgeQscQHYgOJjjDVfoY5MucvglbIgCqfzAHW9jxmRL4qbMZj+b1XoePEtht - ku4bIQN1X5P07fNWzlgaRL5Z4POXDDZTlIQ/El58j9kp4bnWRCJW0lya+f8ocodo - vZZ+Doi+fy4D5ZGrL4XEcIQP/Lv5uFyf+kQtl/94VFYVJOleAv8W92KdgDkhTcTD - G7c0tIkVEKNUq48b3aQ64NOZQW7fVjfoKwEZdOqPE72Pa45jrZzvUFxSpdiNk2tZ - XYukHjlxxEgBdC/J3cMMNRE1F4NCA3ApfV1Y7/hTeOnmDuDYwr9/obA8t016Yljj - q5rdkywPf4JF8mXUW5eCN1vAFHxeg9ZWemhBtQmGxXnw9M+z6hWwc6ahmwARAQAB - tCtEb2NrZXIgUmVsZWFzZSAoQ0UgZGViKSA8ZG9ja2VyQGRvY2tlci5jb20+iQI3 - BBMBCgAhBQJYrefAAhsvBQsJCAcDBRUKCQgLBRYCAwEAAh4BAheAAAoJEI2BgDwO - v82IsskP/iQZo68flDQmNvn8X5XTd6RRaUH33kXYXquT6NkHJciS7E2gTJmqvMqd - tI4mNYHCSEYxI5qrcYV5YqX9P6+Ko+vozo4nseUQLPH/ATQ4qL0Zok+1jkag3Lgk - jonyUf9bwtWxFp05HC3GMHPhhcUSexCxQLQvnFWXD2sWLKivHp2fT8QbRGeZ+d3m - 6fqcd5Fu7pxsqm0EUDK5NL+nPIgYhN+auTrhgzhK1CShfGccM/wfRlei9Utz6p9P - XRKIlWnXtT4qNGZNTN0tR+NLG/6Bqd8OYBaFAUcue/w1VW6JQ2VGYZHnZu9S8LMc - FYBa5Ig9PxwGQOgq6RDKDbV+PqTQT5EFMeR1mrjckk4DQJjbxeMZbiNMG5kGECA8 - g383P3elhn03WGbEEa4MNc3Z4+7c236QI3xWJfNPdUbXRaAwhy/6rTSFbzwKB0Jm - ebwzQfwjQY6f55MiI/RqDCyuPj3r3jyVRkK86pQKBAJwFHyqj9KaKXMZjfVnowLh - 9svIGfNbGHpucATqREvUHuQbNnqkCx8VVhtYkhDb9fEP2xBu5VvHbR+3nfVhMut5 - G34Ct5RS7Jt6LIfFdtcn8CaSas/l1HbiGeRgc70X/9aYx/V/CEJv0lIe8gP6uDoW - FPIZ7d6vH+Vro6xuWEGiuMaiznap2KhZmpkgfupyFmplh0s6knymuQINBFit2ioB - EADneL9S9m4vhU3blaRjVUUyJ7b/qTjcSylvCH5XUE6R2k+ckEZjfAMZPLpO+/tF - M2JIJMD4SifKuS3xck9KtZGCufGmcwiLQRzeHF7vJUKrLD5RTkNi23ydvWZgPjtx - Q+DTT1Zcn7BrQFY6FgnRoUVIxwtdw1bMY/89rsFgS5wwuMESd3Q2RYgb7EOFOpnu - w6da7WakWf4IhnF5nsNYGDVaIHzpiqCl+uTbf1epCjrOlIzkZ3Z3Yk5CM/TiFzPk - z2lLz89cpD8U+NtCsfagWWfjd2U3jDapgH+7nQnCEWpROtzaKHG6lA3pXdix5zG8 - eRc6/0IbUSWvfjKxLLPfNeCS2pCL3IeEI5nothEEYdQH6szpLog79xB9dVnJyKJb - VfxXnseoYqVrRz2VVbUI5Blwm6B40E3eGVfUQWiux54DspyVMMk41Mx7QJ3iynIa - 1N4ZAqVMAEruyXTRTxc9XW0tYhDMA/1GYvz0EmFpm8LzTHA6sFVtPm/ZlNCX6P1X - zJwrv7DSQKD6GGlBQUX+OeEJ8tTkkf8QTJSPUdh8P8YxDFS5EOGAvhhpMBYD42kQ - pqXjEC+XcycTvGI7impgv9PDY1RCC1zkBjKPa120rNhv/hkVk/YhuGoajoHyy4h7 - ZQopdcMtpN2dgmhEegny9JCSwxfQmQ0zK0g7m6SHiKMwjwARAQABiQQ+BBgBCAAJ - BQJYrdoqAhsCAikJEI2BgDwOv82IwV0gBBkBCAAGBQJYrdoqAAoJEH6gqcPyc/zY - 1WAP/2wJ+R0gE6qsce3rjaIz58PJmc8goKrir5hnElWhPgbq7cYIsW5qiFyLhkdp - YcMmhD9mRiPpQn6Ya2w3e3B8zfIVKipbMBnke/ytZ9M7qHmDCcjoiSmwEXN3wKYI - mD9VHONsl/CG1rU9Isw1jtB5g1YxuBA7M/m36XN6x2u+NtNMDB9P56yc4gfsZVES - KA9v+yY2/l45L8d/WUkUi0YXomn6hyBGI7JrBLq0CX37GEYP6O9rrKipfz73XfO7 - JIGzOKZlljb/D9RX/g7nRbCn+3EtH7xnk+TK/50euEKw8SMUg147sJTcpQmv6UzZ - cM4JgL0HbHVCojV4C/plELwMddALOFeYQzTif6sMRPf+3DSj8frbInjChC3yOLy0 - 6br92KFom17EIj2CAcoeq7UPhi2oouYBwPxh5ytdehJkoo+sN7RIWua6P2WSmon5 - U888cSylXC0+ADFdgLX9K2zrDVYUG1vo8CX0vzxFBaHwN6Px26fhIT1/hYUHQR1z - VfNDcyQmXqkOnZvvoMfz/Q0s9BhFJ/zU6AgQbIZE/hm1spsfgvtsD1frZfygXJ9f - irP+MSAI80xHSf91qSRZOj4Pl3ZJNbq4yYxv0b1pkMqeGdjdCYhLU+LZ4wbQmpCk - SVe2prlLureigXtmZfkqevRz7FrIZiu9ky8wnCAPwC7/zmS18rgP/17bOtL4/iIz - QhxAAoAMWVrGyJivSkjhSGx1uCojsWfsTAm11P7jsruIL61ZzMUVE2aM3Pmj5G+W - 9AcZ58Em+1WsVnAXdUR//bMmhyr8wL/G1YO1V3JEJTRdxsSxdYa4deGBBY/Adpsw - 24jxhOJR+lsJpqIUeb999+R8euDhRHG9eFO7DRu6weatUJ6suupoDTRWtr/4yGqe - dKxV3qQhNLSnaAzqW/1nA3iUB4k7kCaKZxhdhDbClf9P37qaRW467BLCVO/coL3y - Vm50dwdrNtKpMBh3ZpbB1uJvgi9mXtyBOMJ3v8RZeDzFiG8HdCtg9RvIt/AIFoHR - H3S+U79NT6i0KPzLImDfs8T7RlpyuMc4Ufs8ggyg9v3Ae6cN3eQyxcK3w0cbBwsh - /nQNfsA6uu+9H7NhbehBMhYnpNZyrHzCmzyXkauwRAqoCbGCNykTRwsur9gS41TQ - M8ssD1jFheOJf3hODnkKU+HKjvMROl1DK7zdmLdNzA1cvtZH/nCC9KPj1z8QC47S - xx+dTZSx4ONAhwbS/LN3PoKtn8LPjY9NP9uDWI+TWYquS2U+KHDrBDlsgozDbs/O - jCxcpDzNmXpWQHEtHU7649OXHP7UeNST1mCUCH5qdank0V1iejF6/CfTFU4MfcrG - YT90qFF93M3v01BbxP+EIY2/9tiIPbrd - =0YYh - -----END PGP PUBLIC KEY BLOCK----- - -- path: "/usr/local/bin/supervise.sh" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - while ! "$@"; do - sleep 1 - done - -- path: "/etc/default/kubelet-overwrite" - content: | - KUBELET_DNS_ARGS= - KUBELET_EXTRA_ARGS=--authentication-token-webhook=true \ - --cloud-provider=aws \ - --cloud-config=/etc/kubernetes/cloud-config \ - --hostname-override=node1 \ - --read-only-port=0 \ - --protect-kernel-defaults=true \ - --resolv-conf=/run/systemd/resolve/resolv.conf \ - --cluster-dns=10.10.10.10 \ - --cluster-domain=cluster.local - -- path: "/etc/systemd/system/kubelet.service.d/20-extra.conf" - permissions: "0644" - content: | - [Service] - EnvironmentFile=/etc/default/kubelet - - -- path: "/etc/systemd/system/setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - ExecStart=/usr/local/bin/supervise.sh /usr/local/bin/setup - -- path: /etc/systemd/system/docker.service.d/10-storage.conf - permissions: "0644" - content: | - [Service] - ExecStart= - ExecStart=/usr/bin/dockerd -H fd:// --storage-driver=overlay2 - -- path: /etc/systemd/system/kubelet-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/usr/local/bin/health-monitor.sh kubelet - - [Install] - WantedBy=multi-user.target - -- path: /etc/systemd/system/docker-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=docker.service - After=docker.service - - [Service] - ExecStart=/usr/local/bin/health-monitor.sh container-runtime - - [Install] - WantedBy=multi-user.target - -runcmd: -- systemctl enable --now setup.service diff --git a/pkg/userdata/ubuntu/testdata/1.9.2-openstack-multiple-dns.golden b/pkg/userdata/ubuntu/testdata/1.9.2-openstack-multiple-dns.golden deleted file mode 100644 index b5b2948e5..000000000 --- a/pkg/userdata/ubuntu/testdata/1.9.2-openstack-multiple-dns.golden +++ /dev/null @@ -1,285 +0,0 @@ -#cloud-config -hostname: node1 - -ssh_pwauth: no - -ssh_authorized_keys: -- "ssh-rsa AAABBB" -- "ssh-rsa CCCDDD" - -write_files: -- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" - content: | - [Journal] - SystemMaxUse=5G - -- path: "/etc/sysctl.d/k8s.conf" - content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - vm.overcommit_memory = 1 - -- path: "/etc/kubernetes/cloud-config" - content: | - {openstack-config:true} - -- path: "/etc/apt/sources.list.d/docker.list" - permissions: "0644" - content: deb [arch=amd64] https://download.docker.com/linux/ubuntu bionic stable - -- path: "/etc/apt/sources.list.d/kubernetes.list" - permissions: "0644" - content: deb http://apt.kubernetes.io/ kubernetes-xenial main - -- path: "/usr/local/bin/setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - - sysctl --system - mkdir -p /opt/bin - apt-key add /opt/docker.asc - apt-key add /opt/kubernetes.asc - apt-get update - - # Hetzner's Ubuntu Bionic comes with swap pre-configured, so we force it off. - systemctl mask swap.target - swapoff -a - - # If something failed during package installation but one of docker/kubeadm/kubelet was already installed - # an apt-mark hold after the install won't do it, which is why we test here if the binaries exist and if - # yes put them on hold - set +e - which docker && apt-mark hold docker.io docker-ce - which kubelet && apt-mark hold kubelet - which kubeadm && apt-mark hold kubeadm - - # When docker is started from within the apt installation it fails with a - # 'no sockets found via socket activation: make sure the service was started by systemd' - # Apparently the package is broken in a way that it gets started without its dependencies, manually starting - # it works fine thought - which docker && systemctl start docker - set -e - DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" dist-upgrade -y - if [[ -e /var/run/reboot-required ]]; then - reboot - fi - - export CR_PKG='docker-ce=18.06.0~ce~3-0~ubuntu' - export CR_PKG='docker.io=17.12.1-0ubuntu1' - - - DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install -y \ - curl \ - ca-certificates \ - ceph-common \ - cifs-utils \ - conntrack \ - e2fsprogs \ - ebtables \ - ethtool \ - glusterfs-client \ - iptables \ - jq \ - kmod \ - openssh-client \ - nfs-common \ - socat \ - util-linux \ - ${CR_PKG} \ - open-vm-tools \ - kubelet=1.9.2-00 \ - kubeadm=1.9.2-00 \ - - cp /etc/default/kubelet-overwrite /etc/default/kubelet - - systemctl enable --now docker - systemctl enable kubelet - - if [[ ! -x /usr/local/bin/health-monitor.sh ]]; then - curl -Lfo /usr/local/bin/health-monitor.sh \ - https://raw.githubusercontent.com/kubermatic/machine-controller/8b5b66e4910a6228dfaecccaa0a3b05ec4902f8e/pkg/userdata/scripts/health-monitor.sh - chmod +x /usr/local/bin/health-monitor.sh - fi - - if ! [[ -e /etc/kubernetes/pki/ca.crt ]]; then - kubeadm join \ - --token my-token \ - --discovery-token-ca-cert-hash sha256:6caecce9fedcb55d4953d61a27dc6997361a2f226ad86d7e6004dde7526fc4b1 \ - --ignore-preflight-errors=CRI \ - server:443 - fi - - systemctl enable --now --no-block kubelet-healthcheck.service - systemctl enable --now --no-block docker-healthcheck.service - -- path: "/opt/kubernetes.asc" - permissions: "0400" - content: | - -----BEGIN PGP PUBLIC KEY BLOCK----- - - mQENBFrBaNsBCADrF18KCbsZlo4NjAvVecTBCnp6WcBQJ5oSh7+E98jX9YznUCrN - rgmeCcCMUvTDRDxfTaDJybaHugfba43nqhkbNpJ47YXsIa+YL6eEE9emSmQtjrSW - IiY+2YJYwsDgsgckF3duqkb02OdBQlh6IbHPoXB6H//b1PgZYsomB+841XW1LSJP - YlYbIrWfwDfQvtkFQI90r6NknVTQlpqQh5GLNWNYqRNrGQPmsB+NrUYrkl1nUt1L - RGu+rCe4bSaSmNbwKMQKkROE4kTiB72DPk7zH4Lm0uo0YFFWG4qsMIuqEihJ/9KN - X8GYBr+tWgyLooLlsdK3l+4dVqd8cjkJM1ExABEBAAG0QEdvb2dsZSBDbG91ZCBQ - YWNrYWdlcyBBdXRvbWF0aWMgU2lnbmluZyBLZXkgPGdjLXRlYW1AZ29vZ2xlLmNv - bT6JAT4EEwECACgFAlrBaNsCGy8FCQWjmoAGCwkIBwMCBhUIAgkKCwQWAgMBAh4B - AheAAAoJEGoDCyG6B/T78e8H/1WH2LN/nVNhm5TS1VYJG8B+IW8zS4BqyozxC9iJ - AJqZIVHXl8g8a/Hus8RfXR7cnYHcg8sjSaJfQhqO9RbKnffiuQgGrqwQxuC2jBa6 - M/QKzejTeP0Mgi67pyrLJNWrFI71RhritQZmzTZ2PoWxfv6b+Tv5v0rPaG+ut1J4 - 7pn+kYgtUaKdsJz1umi6HzK6AacDf0C0CksJdKG7MOWsZcB4xeOxJYuy6NuO6Kcd - Ez8/XyEUjIuIOlhYTd0hH8E/SEBbXXft7/VBQC5wNq40izPi+6WFK/e1O42DIpzQ - 749ogYQ1eodexPNhLzekKR3XhGrNXJ95r5KO10VrsLFNd8I= - =TKuP - -----END PGP PUBLIC KEY BLOCK----- - -- path: "/opt/docker.asc" - permissions: "0400" - content: | - -----BEGIN PGP PUBLIC KEY BLOCK----- - - mQINBFit2ioBEADhWpZ8/wvZ6hUTiXOwQHXMAlaFHcPH9hAtr4F1y2+OYdbtMuth - lqqwp028AqyY+PRfVMtSYMbjuQuu5byyKR01BbqYhuS3jtqQmljZ/bJvXqnmiVXh - 38UuLa+z077PxyxQhu5BbqntTPQMfiyqEiU+BKbq2WmANUKQf+1AmZY/IruOXbnq - L4C1+gJ8vfmXQt99npCaxEjaNRVYfOS8QcixNzHUYnb6emjlANyEVlZzeqo7XKl7 - UrwV5inawTSzWNvtjEjj4nJL8NsLwscpLPQUhTQ+7BbQXAwAmeHCUTQIvvWXqw0N - cmhh4HgeQscQHYgOJjjDVfoY5MucvglbIgCqfzAHW9jxmRL4qbMZj+b1XoePEtht - ku4bIQN1X5P07fNWzlgaRL5Z4POXDDZTlIQ/El58j9kp4bnWRCJW0lya+f8ocodo - vZZ+Doi+fy4D5ZGrL4XEcIQP/Lv5uFyf+kQtl/94VFYVJOleAv8W92KdgDkhTcTD - G7c0tIkVEKNUq48b3aQ64NOZQW7fVjfoKwEZdOqPE72Pa45jrZzvUFxSpdiNk2tZ - XYukHjlxxEgBdC/J3cMMNRE1F4NCA3ApfV1Y7/hTeOnmDuDYwr9/obA8t016Yljj - q5rdkywPf4JF8mXUW5eCN1vAFHxeg9ZWemhBtQmGxXnw9M+z6hWwc6ahmwARAQAB - tCtEb2NrZXIgUmVsZWFzZSAoQ0UgZGViKSA8ZG9ja2VyQGRvY2tlci5jb20+iQI3 - BBMBCgAhBQJYrefAAhsvBQsJCAcDBRUKCQgLBRYCAwEAAh4BAheAAAoJEI2BgDwO - v82IsskP/iQZo68flDQmNvn8X5XTd6RRaUH33kXYXquT6NkHJciS7E2gTJmqvMqd - tI4mNYHCSEYxI5qrcYV5YqX9P6+Ko+vozo4nseUQLPH/ATQ4qL0Zok+1jkag3Lgk - jonyUf9bwtWxFp05HC3GMHPhhcUSexCxQLQvnFWXD2sWLKivHp2fT8QbRGeZ+d3m - 6fqcd5Fu7pxsqm0EUDK5NL+nPIgYhN+auTrhgzhK1CShfGccM/wfRlei9Utz6p9P - XRKIlWnXtT4qNGZNTN0tR+NLG/6Bqd8OYBaFAUcue/w1VW6JQ2VGYZHnZu9S8LMc - FYBa5Ig9PxwGQOgq6RDKDbV+PqTQT5EFMeR1mrjckk4DQJjbxeMZbiNMG5kGECA8 - g383P3elhn03WGbEEa4MNc3Z4+7c236QI3xWJfNPdUbXRaAwhy/6rTSFbzwKB0Jm - ebwzQfwjQY6f55MiI/RqDCyuPj3r3jyVRkK86pQKBAJwFHyqj9KaKXMZjfVnowLh - 9svIGfNbGHpucATqREvUHuQbNnqkCx8VVhtYkhDb9fEP2xBu5VvHbR+3nfVhMut5 - G34Ct5RS7Jt6LIfFdtcn8CaSas/l1HbiGeRgc70X/9aYx/V/CEJv0lIe8gP6uDoW - FPIZ7d6vH+Vro6xuWEGiuMaiznap2KhZmpkgfupyFmplh0s6knymuQINBFit2ioB - EADneL9S9m4vhU3blaRjVUUyJ7b/qTjcSylvCH5XUE6R2k+ckEZjfAMZPLpO+/tF - M2JIJMD4SifKuS3xck9KtZGCufGmcwiLQRzeHF7vJUKrLD5RTkNi23ydvWZgPjtx - Q+DTT1Zcn7BrQFY6FgnRoUVIxwtdw1bMY/89rsFgS5wwuMESd3Q2RYgb7EOFOpnu - w6da7WakWf4IhnF5nsNYGDVaIHzpiqCl+uTbf1epCjrOlIzkZ3Z3Yk5CM/TiFzPk - z2lLz89cpD8U+NtCsfagWWfjd2U3jDapgH+7nQnCEWpROtzaKHG6lA3pXdix5zG8 - eRc6/0IbUSWvfjKxLLPfNeCS2pCL3IeEI5nothEEYdQH6szpLog79xB9dVnJyKJb - VfxXnseoYqVrRz2VVbUI5Blwm6B40E3eGVfUQWiux54DspyVMMk41Mx7QJ3iynIa - 1N4ZAqVMAEruyXTRTxc9XW0tYhDMA/1GYvz0EmFpm8LzTHA6sFVtPm/ZlNCX6P1X - zJwrv7DSQKD6GGlBQUX+OeEJ8tTkkf8QTJSPUdh8P8YxDFS5EOGAvhhpMBYD42kQ - pqXjEC+XcycTvGI7impgv9PDY1RCC1zkBjKPa120rNhv/hkVk/YhuGoajoHyy4h7 - ZQopdcMtpN2dgmhEegny9JCSwxfQmQ0zK0g7m6SHiKMwjwARAQABiQQ+BBgBCAAJ - BQJYrdoqAhsCAikJEI2BgDwOv82IwV0gBBkBCAAGBQJYrdoqAAoJEH6gqcPyc/zY - 1WAP/2wJ+R0gE6qsce3rjaIz58PJmc8goKrir5hnElWhPgbq7cYIsW5qiFyLhkdp - YcMmhD9mRiPpQn6Ya2w3e3B8zfIVKipbMBnke/ytZ9M7qHmDCcjoiSmwEXN3wKYI - mD9VHONsl/CG1rU9Isw1jtB5g1YxuBA7M/m36XN6x2u+NtNMDB9P56yc4gfsZVES - KA9v+yY2/l45L8d/WUkUi0YXomn6hyBGI7JrBLq0CX37GEYP6O9rrKipfz73XfO7 - JIGzOKZlljb/D9RX/g7nRbCn+3EtH7xnk+TK/50euEKw8SMUg147sJTcpQmv6UzZ - cM4JgL0HbHVCojV4C/plELwMddALOFeYQzTif6sMRPf+3DSj8frbInjChC3yOLy0 - 6br92KFom17EIj2CAcoeq7UPhi2oouYBwPxh5ytdehJkoo+sN7RIWua6P2WSmon5 - U888cSylXC0+ADFdgLX9K2zrDVYUG1vo8CX0vzxFBaHwN6Px26fhIT1/hYUHQR1z - VfNDcyQmXqkOnZvvoMfz/Q0s9BhFJ/zU6AgQbIZE/hm1spsfgvtsD1frZfygXJ9f - irP+MSAI80xHSf91qSRZOj4Pl3ZJNbq4yYxv0b1pkMqeGdjdCYhLU+LZ4wbQmpCk - SVe2prlLureigXtmZfkqevRz7FrIZiu9ky8wnCAPwC7/zmS18rgP/17bOtL4/iIz - QhxAAoAMWVrGyJivSkjhSGx1uCojsWfsTAm11P7jsruIL61ZzMUVE2aM3Pmj5G+W - 9AcZ58Em+1WsVnAXdUR//bMmhyr8wL/G1YO1V3JEJTRdxsSxdYa4deGBBY/Adpsw - 24jxhOJR+lsJpqIUeb999+R8euDhRHG9eFO7DRu6weatUJ6suupoDTRWtr/4yGqe - dKxV3qQhNLSnaAzqW/1nA3iUB4k7kCaKZxhdhDbClf9P37qaRW467BLCVO/coL3y - Vm50dwdrNtKpMBh3ZpbB1uJvgi9mXtyBOMJ3v8RZeDzFiG8HdCtg9RvIt/AIFoHR - H3S+U79NT6i0KPzLImDfs8T7RlpyuMc4Ufs8ggyg9v3Ae6cN3eQyxcK3w0cbBwsh - /nQNfsA6uu+9H7NhbehBMhYnpNZyrHzCmzyXkauwRAqoCbGCNykTRwsur9gS41TQ - M8ssD1jFheOJf3hODnkKU+HKjvMROl1DK7zdmLdNzA1cvtZH/nCC9KPj1z8QC47S - xx+dTZSx4ONAhwbS/LN3PoKtn8LPjY9NP9uDWI+TWYquS2U+KHDrBDlsgozDbs/O - jCxcpDzNmXpWQHEtHU7649OXHP7UeNST1mCUCH5qdank0V1iejF6/CfTFU4MfcrG - YT90qFF93M3v01BbxP+EIY2/9tiIPbrd - =0YYh - -----END PGP PUBLIC KEY BLOCK----- - -- path: "/usr/local/bin/supervise.sh" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - while ! "$@"; do - sleep 1 - done - -- path: "/etc/default/kubelet-overwrite" - content: | - KUBELET_DNS_ARGS= - KUBELET_EXTRA_ARGS=--authentication-token-webhook=true \ - --cloud-provider=openstack \ - --cloud-config=/etc/kubernetes/cloud-config \ - --hostname-override=node1 \ - --read-only-port=0 \ - --protect-kernel-defaults=true \ - --resolv-conf=/run/systemd/resolve/resolv.conf \ - --cluster-dns=10.10.10.10,10.10.10.11,10.10.10.12 \ - --cluster-domain=cluster.local - -- path: "/etc/systemd/system/kubelet.service.d/20-extra.conf" - permissions: "0644" - content: | - [Service] - EnvironmentFile=/etc/default/kubelet - - -- path: "/etc/systemd/system/setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - ExecStart=/usr/local/bin/supervise.sh /usr/local/bin/setup - -- path: /etc/systemd/system/docker.service.d/10-storage.conf - permissions: "0644" - content: | - [Service] - ExecStart= - ExecStart=/usr/bin/dockerd -H fd:// --storage-driver=overlay2 - -- path: /etc/systemd/system/kubelet-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/usr/local/bin/health-monitor.sh kubelet - - [Install] - WantedBy=multi-user.target - -- path: /etc/systemd/system/docker-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=docker.service - After=docker.service - - [Service] - ExecStart=/usr/local/bin/health-monitor.sh container-runtime - - [Install] - WantedBy=multi-user.target - -runcmd: -- systemctl enable --now setup.service diff --git a/pkg/userdata/ubuntu/testdata/dist-upgrade-on-boot.golden b/pkg/userdata/ubuntu/testdata/dist-upgrade-on-boot.golden new file mode 100644 index 000000000..b424ef78f --- /dev/null +++ b/pkg/userdata/ubuntu/testdata/dist-upgrade-on-boot.golden @@ -0,0 +1,343 @@ +#cloud-config +hostname: node1 + +ssh_pwauth: no + +ssh_authorized_keys: +- "ssh-rsa AAABBB" + +write_files: +- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" + content: | + [Journal] + SystemMaxUse=5G + + +- path: "/etc/modules-load.d/k8s.conf" + content: | + ip_vs + ip_vs_rr + ip_vs_wrr + ip_vs_sh + nf_conntrack_ipv4 + + +- path: "/etc/sysctl.d/k8s.conf" + content: | + net.bridge.bridge-nf-call-ip6tables = 1 + net.bridge.bridge-nf-call-iptables = 1 + kernel.panic_on_oops = 1 + kernel.panic = 10 + net.ipv4.ip_forward = 1 + vm.overcommit_memory = 1 + + +- path: "/etc/apt/sources.list.d/docker.list" + permissions: "0644" + content: deb [arch=amd64] https://download.docker.com/linux/ubuntu bionic stable + +- path: "/opt/docker.asc" + permissions: "0400" + content: | + -----BEGIN PGP PUBLIC KEY BLOCK----- + + mQINBFit2ioBEADhWpZ8/wvZ6hUTiXOwQHXMAlaFHcPH9hAtr4F1y2+OYdbtMuth + lqqwp028AqyY+PRfVMtSYMbjuQuu5byyKR01BbqYhuS3jtqQmljZ/bJvXqnmiVXh + 38UuLa+z077PxyxQhu5BbqntTPQMfiyqEiU+BKbq2WmANUKQf+1AmZY/IruOXbnq + L4C1+gJ8vfmXQt99npCaxEjaNRVYfOS8QcixNzHUYnb6emjlANyEVlZzeqo7XKl7 + UrwV5inawTSzWNvtjEjj4nJL8NsLwscpLPQUhTQ+7BbQXAwAmeHCUTQIvvWXqw0N + cmhh4HgeQscQHYgOJjjDVfoY5MucvglbIgCqfzAHW9jxmRL4qbMZj+b1XoePEtht + ku4bIQN1X5P07fNWzlgaRL5Z4POXDDZTlIQ/El58j9kp4bnWRCJW0lya+f8ocodo + vZZ+Doi+fy4D5ZGrL4XEcIQP/Lv5uFyf+kQtl/94VFYVJOleAv8W92KdgDkhTcTD + G7c0tIkVEKNUq48b3aQ64NOZQW7fVjfoKwEZdOqPE72Pa45jrZzvUFxSpdiNk2tZ + XYukHjlxxEgBdC/J3cMMNRE1F4NCA3ApfV1Y7/hTeOnmDuDYwr9/obA8t016Yljj + q5rdkywPf4JF8mXUW5eCN1vAFHxeg9ZWemhBtQmGxXnw9M+z6hWwc6ahmwARAQAB + tCtEb2NrZXIgUmVsZWFzZSAoQ0UgZGViKSA8ZG9ja2VyQGRvY2tlci5jb20+iQI3 + BBMBCgAhBQJYrefAAhsvBQsJCAcDBRUKCQgLBRYCAwEAAh4BAheAAAoJEI2BgDwO + v82IsskP/iQZo68flDQmNvn8X5XTd6RRaUH33kXYXquT6NkHJciS7E2gTJmqvMqd + tI4mNYHCSEYxI5qrcYV5YqX9P6+Ko+vozo4nseUQLPH/ATQ4qL0Zok+1jkag3Lgk + jonyUf9bwtWxFp05HC3GMHPhhcUSexCxQLQvnFWXD2sWLKivHp2fT8QbRGeZ+d3m + 6fqcd5Fu7pxsqm0EUDK5NL+nPIgYhN+auTrhgzhK1CShfGccM/wfRlei9Utz6p9P + XRKIlWnXtT4qNGZNTN0tR+NLG/6Bqd8OYBaFAUcue/w1VW6JQ2VGYZHnZu9S8LMc + FYBa5Ig9PxwGQOgq6RDKDbV+PqTQT5EFMeR1mrjckk4DQJjbxeMZbiNMG5kGECA8 + g383P3elhn03WGbEEa4MNc3Z4+7c236QI3xWJfNPdUbXRaAwhy/6rTSFbzwKB0Jm + ebwzQfwjQY6f55MiI/RqDCyuPj3r3jyVRkK86pQKBAJwFHyqj9KaKXMZjfVnowLh + 9svIGfNbGHpucATqREvUHuQbNnqkCx8VVhtYkhDb9fEP2xBu5VvHbR+3nfVhMut5 + G34Ct5RS7Jt6LIfFdtcn8CaSas/l1HbiGeRgc70X/9aYx/V/CEJv0lIe8gP6uDoW + FPIZ7d6vH+Vro6xuWEGiuMaiznap2KhZmpkgfupyFmplh0s6knymuQINBFit2ioB + EADneL9S9m4vhU3blaRjVUUyJ7b/qTjcSylvCH5XUE6R2k+ckEZjfAMZPLpO+/tF + M2JIJMD4SifKuS3xck9KtZGCufGmcwiLQRzeHF7vJUKrLD5RTkNi23ydvWZgPjtx + Q+DTT1Zcn7BrQFY6FgnRoUVIxwtdw1bMY/89rsFgS5wwuMESd3Q2RYgb7EOFOpnu + w6da7WakWf4IhnF5nsNYGDVaIHzpiqCl+uTbf1epCjrOlIzkZ3Z3Yk5CM/TiFzPk + z2lLz89cpD8U+NtCsfagWWfjd2U3jDapgH+7nQnCEWpROtzaKHG6lA3pXdix5zG8 + eRc6/0IbUSWvfjKxLLPfNeCS2pCL3IeEI5nothEEYdQH6szpLog79xB9dVnJyKJb + VfxXnseoYqVrRz2VVbUI5Blwm6B40E3eGVfUQWiux54DspyVMMk41Mx7QJ3iynIa + 1N4ZAqVMAEruyXTRTxc9XW0tYhDMA/1GYvz0EmFpm8LzTHA6sFVtPm/ZlNCX6P1X + zJwrv7DSQKD6GGlBQUX+OeEJ8tTkkf8QTJSPUdh8P8YxDFS5EOGAvhhpMBYD42kQ + pqXjEC+XcycTvGI7impgv9PDY1RCC1zkBjKPa120rNhv/hkVk/YhuGoajoHyy4h7 + ZQopdcMtpN2dgmhEegny9JCSwxfQmQ0zK0g7m6SHiKMwjwARAQABiQQ+BBgBCAAJ + BQJYrdoqAhsCAikJEI2BgDwOv82IwV0gBBkBCAAGBQJYrdoqAAoJEH6gqcPyc/zY + 1WAP/2wJ+R0gE6qsce3rjaIz58PJmc8goKrir5hnElWhPgbq7cYIsW5qiFyLhkdp + YcMmhD9mRiPpQn6Ya2w3e3B8zfIVKipbMBnke/ytZ9M7qHmDCcjoiSmwEXN3wKYI + mD9VHONsl/CG1rU9Isw1jtB5g1YxuBA7M/m36XN6x2u+NtNMDB9P56yc4gfsZVES + KA9v+yY2/l45L8d/WUkUi0YXomn6hyBGI7JrBLq0CX37GEYP6O9rrKipfz73XfO7 + JIGzOKZlljb/D9RX/g7nRbCn+3EtH7xnk+TK/50euEKw8SMUg147sJTcpQmv6UzZ + cM4JgL0HbHVCojV4C/plELwMddALOFeYQzTif6sMRPf+3DSj8frbInjChC3yOLy0 + 6br92KFom17EIj2CAcoeq7UPhi2oouYBwPxh5ytdehJkoo+sN7RIWua6P2WSmon5 + U888cSylXC0+ADFdgLX9K2zrDVYUG1vo8CX0vzxFBaHwN6Px26fhIT1/hYUHQR1z + VfNDcyQmXqkOnZvvoMfz/Q0s9BhFJ/zU6AgQbIZE/hm1spsfgvtsD1frZfygXJ9f + irP+MSAI80xHSf91qSRZOj4Pl3ZJNbq4yYxv0b1pkMqeGdjdCYhLU+LZ4wbQmpCk + SVe2prlLureigXtmZfkqevRz7FrIZiu9ky8wnCAPwC7/zmS18rgP/17bOtL4/iIz + QhxAAoAMWVrGyJivSkjhSGx1uCojsWfsTAm11P7jsruIL61ZzMUVE2aM3Pmj5G+W + 9AcZ58Em+1WsVnAXdUR//bMmhyr8wL/G1YO1V3JEJTRdxsSxdYa4deGBBY/Adpsw + 24jxhOJR+lsJpqIUeb999+R8euDhRHG9eFO7DRu6weatUJ6suupoDTRWtr/4yGqe + dKxV3qQhNLSnaAzqW/1nA3iUB4k7kCaKZxhdhDbClf9P37qaRW467BLCVO/coL3y + Vm50dwdrNtKpMBh3ZpbB1uJvgi9mXtyBOMJ3v8RZeDzFiG8HdCtg9RvIt/AIFoHR + H3S+U79NT6i0KPzLImDfs8T7RlpyuMc4Ufs8ggyg9v3Ae6cN3eQyxcK3w0cbBwsh + /nQNfsA6uu+9H7NhbehBMhYnpNZyrHzCmzyXkauwRAqoCbGCNykTRwsur9gS41TQ + M8ssD1jFheOJf3hODnkKU+HKjvMROl1DK7zdmLdNzA1cvtZH/nCC9KPj1z8QC47S + xx+dTZSx4ONAhwbS/LN3PoKtn8LPjY9NP9uDWI+TWYquS2U+KHDrBDlsgozDbs/O + jCxcpDzNmXpWQHEtHU7649OXHP7UeNST1mCUCH5qdank0V1iejF6/CfTFU4MfcrG + YT90qFF93M3v01BbxP+EIY2/9tiIPbrd + =0YYh + -----END PGP PUBLIC KEY BLOCK----- + +- path: "/opt/bin/setup" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + + # As we added some modules and don't want to reboot, restart the service + systemctl restart systemd-modules-load.service + sysctl --system + + apt-key add /opt/docker.asc + apt-get update + + # Make sure we always disable swap - Otherwise the kubelet won't start'. + systemctl mask swap.target + swapoff -a + export CR_PKG='docker.io=17.12.1-0ubuntu1' + + DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install -y \ + curl \ + ca-certificates \ + ceph-common \ + cifs-utils \ + conntrack \ + e2fsprogs \ + ebtables \ + ethtool \ + glusterfs-client \ + iptables \ + jq \ + kmod \ + openssh-client \ + nfs-common \ + socat \ + util-linux \ + ${CR_PKG} \ + open-vm-tools \ + ipvsadm + + # If something failed during package installation but docker got installed, we need to put it on hold + apt-mark hold docker.io || true + apt-mark hold docker-ce || true + DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" dist-upgrade -y + if [[ -e /var/run/reboot-required ]]; then + reboot + fi + + #setup some common directories + mkdir -p /opt/bin/ + mkdir -p /var/lib/calico + mkdir -p /etc/kubernetes/manifests + mkdir -p /etc/cni/net.d + mkdir -p /opt/cni/bin + + # cni + if [ ! -f /opt/cni/bin/loopback ]; then + curl -L https://github.com/containernetworking/plugins/releases/download/v0.6.0/cni-plugins-amd64-v0.6.0.tgz | tar -xvzC /opt/cni/bin -f - + fi + # kubelet + if [ ! -f /opt/bin/kubelet ]; then + curl -Lfo /opt/bin/kubelet https://storage.googleapis.com/kubernetes-release/release/v1.11.3/bin/linux/amd64/kubelet + chmod +x /opt/bin/kubelet + fi + + if [[ ! -x /opt/bin/health-monitor.sh ]]; then + curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/8b5b66e4910a6228dfaecccaa0a3b05ec4902f8e/pkg/userdata/scripts/health-monitor.sh + chmod +x /opt/bin/health-monitor.sh + fi + + + systemctl enable --now docker + systemctl enable --now kubelet + systemctl enable --now --no-block kubelet-healthcheck.service + systemctl enable --now --no-block docker-healthcheck.service + +- path: "/opt/bin/supervise.sh" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + while ! "$@"; do + sleep 1 + done + +- path: "/etc/systemd/system/kubelet.service" + content: | + [Unit] + After=docker.service + Requires=docker.service + + Description=kubelet: The Kubernetes Node Agent + Documentation=https://kubernetes.io/docs/home/ + + [Service] + Restart=always + StartLimitInterval=0 + RestartSec=10 + + Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" + + ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ + --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ + --kubeconfig=/etc/kubernetes/kubelet.conf \ + --pod-manifest-path=/etc/kubernetes/manifests \ + --allow-privileged=true \ + --network-plugin=cni \ + --cni-conf-dir=/etc/cni/net.d \ + --cni-bin-dir=/opt/cni/bin \ + --authorization-mode=Webhook \ + --client-ca-file=/etc/kubernetes/pki/ca.crt \ + --cadvisor-port=0 \ + --rotate-certificates=true \ + --cert-dir=/etc/kubernetes/pki \ + --authentication-token-webhook=true \ + --hostname-override=node1 \ + --read-only-port=0 \ + --exit-on-lock-contention \ + --lock-file=/tmp/kubelet.lock \ + --anonymous-auth=false \ + --protect-kernel-defaults=true \ + --cluster-dns=10.10.10.10 \ + --cluster-domain=cluster.local + + [Install] + WantedBy=multi-user.target + +- path: "/etc/systemd/system/kubelet.service.d/extras.conf" + content: | + [Service] + Environment="KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf" + +- path: "/etc/kubernetes/cloud-config" + content: | + + +- path: "/etc/kubernetes/bootstrap-kubelet.conf" + content: | + apiVersion: v1 + clusters: + - cluster: + certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t + server: https://server:443 + name: "" + contexts: [] + current-context: "" + kind: Config + preferences: {} + users: + - name: "" + user: + token: my-token + + +- path: "/etc/kubernetes/pki/ca.crt" + content: | + -----BEGIN CERTIFICATE----- + MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV + BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG + A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 + DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 + NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG + cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv + c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B + AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS + R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT + ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk + JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 + mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW + caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G + A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt + hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB + MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES + MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv + bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h + U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao + eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 + UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD + 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n + sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF + kPe6XoSbiLm/kxk32T0= + -----END CERTIFICATE----- + +- path: "/etc/systemd/system/setup.service" + permissions: "0644" + content: | + [Install] + WantedBy=multi-user.target + + [Unit] + Requires=network-online.target + After=network-online.target + + [Service] + Type=oneshot + RemainAfterExit=true + ExecStart=/opt/bin/supervise.sh /opt/bin/setup + +- path: "/etc/profile.d/opt-bin-path.sh" + permissions: "0644" + content: | + export PATH="/opt/bin:$PATH" + +- path: /etc/systemd/system/docker.service.d/10-storage.conf + permissions: "0644" + content: | + [Service] + ExecStart= + ExecStart=/usr/bin/dockerd -H fd:// --storage-driver=overlay2 + +- path: /etc/systemd/system/kubelet-healthcheck.service + permissions: "0644" + content: | + [Unit] + Requires=kubelet.service + After=kubelet.service + + [Service] + ExecStart=/opt/bin/health-monitor.sh kubelet + + [Install] + WantedBy=multi-user.target + + +- path: /etc/systemd/system/docker-healthcheck.service + permissions: "0644" + content: | + [Unit] + Requires=docker.service + After=docker.service + + [Service] + ExecStart=/opt/bin/health-monitor.sh container-runtime + + [Install] + WantedBy=multi-user.target + + +runcmd: +- systemctl enable --now setup.service diff --git a/pkg/userdata/ubuntu/testdata/kubelet-version-without-v-prefix.golden b/pkg/userdata/ubuntu/testdata/kubelet-version-without-v-prefix.golden new file mode 100644 index 000000000..69439fc7b --- /dev/null +++ b/pkg/userdata/ubuntu/testdata/kubelet-version-without-v-prefix.golden @@ -0,0 +1,342 @@ +#cloud-config +hostname: node1 + +ssh_pwauth: no + +ssh_authorized_keys: +- "ssh-rsa AAABBB" + +write_files: +- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" + content: | + [Journal] + SystemMaxUse=5G + + +- path: "/etc/modules-load.d/k8s.conf" + content: | + ip_vs + ip_vs_rr + ip_vs_wrr + ip_vs_sh + nf_conntrack_ipv4 + + +- path: "/etc/sysctl.d/k8s.conf" + content: | + net.bridge.bridge-nf-call-ip6tables = 1 + net.bridge.bridge-nf-call-iptables = 1 + kernel.panic_on_oops = 1 + kernel.panic = 10 + net.ipv4.ip_forward = 1 + vm.overcommit_memory = 1 + + +- path: "/etc/apt/sources.list.d/docker.list" + permissions: "0644" + content: deb [arch=amd64] https://download.docker.com/linux/ubuntu bionic stable + +- path: "/opt/docker.asc" + permissions: "0400" + content: | + -----BEGIN PGP PUBLIC KEY BLOCK----- + + mQINBFit2ioBEADhWpZ8/wvZ6hUTiXOwQHXMAlaFHcPH9hAtr4F1y2+OYdbtMuth + lqqwp028AqyY+PRfVMtSYMbjuQuu5byyKR01BbqYhuS3jtqQmljZ/bJvXqnmiVXh + 38UuLa+z077PxyxQhu5BbqntTPQMfiyqEiU+BKbq2WmANUKQf+1AmZY/IruOXbnq + L4C1+gJ8vfmXQt99npCaxEjaNRVYfOS8QcixNzHUYnb6emjlANyEVlZzeqo7XKl7 + UrwV5inawTSzWNvtjEjj4nJL8NsLwscpLPQUhTQ+7BbQXAwAmeHCUTQIvvWXqw0N + cmhh4HgeQscQHYgOJjjDVfoY5MucvglbIgCqfzAHW9jxmRL4qbMZj+b1XoePEtht + ku4bIQN1X5P07fNWzlgaRL5Z4POXDDZTlIQ/El58j9kp4bnWRCJW0lya+f8ocodo + vZZ+Doi+fy4D5ZGrL4XEcIQP/Lv5uFyf+kQtl/94VFYVJOleAv8W92KdgDkhTcTD + G7c0tIkVEKNUq48b3aQ64NOZQW7fVjfoKwEZdOqPE72Pa45jrZzvUFxSpdiNk2tZ + XYukHjlxxEgBdC/J3cMMNRE1F4NCA3ApfV1Y7/hTeOnmDuDYwr9/obA8t016Yljj + q5rdkywPf4JF8mXUW5eCN1vAFHxeg9ZWemhBtQmGxXnw9M+z6hWwc6ahmwARAQAB + tCtEb2NrZXIgUmVsZWFzZSAoQ0UgZGViKSA8ZG9ja2VyQGRvY2tlci5jb20+iQI3 + BBMBCgAhBQJYrefAAhsvBQsJCAcDBRUKCQgLBRYCAwEAAh4BAheAAAoJEI2BgDwO + v82IsskP/iQZo68flDQmNvn8X5XTd6RRaUH33kXYXquT6NkHJciS7E2gTJmqvMqd + tI4mNYHCSEYxI5qrcYV5YqX9P6+Ko+vozo4nseUQLPH/ATQ4qL0Zok+1jkag3Lgk + jonyUf9bwtWxFp05HC3GMHPhhcUSexCxQLQvnFWXD2sWLKivHp2fT8QbRGeZ+d3m + 6fqcd5Fu7pxsqm0EUDK5NL+nPIgYhN+auTrhgzhK1CShfGccM/wfRlei9Utz6p9P + XRKIlWnXtT4qNGZNTN0tR+NLG/6Bqd8OYBaFAUcue/w1VW6JQ2VGYZHnZu9S8LMc + FYBa5Ig9PxwGQOgq6RDKDbV+PqTQT5EFMeR1mrjckk4DQJjbxeMZbiNMG5kGECA8 + g383P3elhn03WGbEEa4MNc3Z4+7c236QI3xWJfNPdUbXRaAwhy/6rTSFbzwKB0Jm + ebwzQfwjQY6f55MiI/RqDCyuPj3r3jyVRkK86pQKBAJwFHyqj9KaKXMZjfVnowLh + 9svIGfNbGHpucATqREvUHuQbNnqkCx8VVhtYkhDb9fEP2xBu5VvHbR+3nfVhMut5 + G34Ct5RS7Jt6LIfFdtcn8CaSas/l1HbiGeRgc70X/9aYx/V/CEJv0lIe8gP6uDoW + FPIZ7d6vH+Vro6xuWEGiuMaiznap2KhZmpkgfupyFmplh0s6knymuQINBFit2ioB + EADneL9S9m4vhU3blaRjVUUyJ7b/qTjcSylvCH5XUE6R2k+ckEZjfAMZPLpO+/tF + M2JIJMD4SifKuS3xck9KtZGCufGmcwiLQRzeHF7vJUKrLD5RTkNi23ydvWZgPjtx + Q+DTT1Zcn7BrQFY6FgnRoUVIxwtdw1bMY/89rsFgS5wwuMESd3Q2RYgb7EOFOpnu + w6da7WakWf4IhnF5nsNYGDVaIHzpiqCl+uTbf1epCjrOlIzkZ3Z3Yk5CM/TiFzPk + z2lLz89cpD8U+NtCsfagWWfjd2U3jDapgH+7nQnCEWpROtzaKHG6lA3pXdix5zG8 + eRc6/0IbUSWvfjKxLLPfNeCS2pCL3IeEI5nothEEYdQH6szpLog79xB9dVnJyKJb + VfxXnseoYqVrRz2VVbUI5Blwm6B40E3eGVfUQWiux54DspyVMMk41Mx7QJ3iynIa + 1N4ZAqVMAEruyXTRTxc9XW0tYhDMA/1GYvz0EmFpm8LzTHA6sFVtPm/ZlNCX6P1X + zJwrv7DSQKD6GGlBQUX+OeEJ8tTkkf8QTJSPUdh8P8YxDFS5EOGAvhhpMBYD42kQ + pqXjEC+XcycTvGI7impgv9PDY1RCC1zkBjKPa120rNhv/hkVk/YhuGoajoHyy4h7 + ZQopdcMtpN2dgmhEegny9JCSwxfQmQ0zK0g7m6SHiKMwjwARAQABiQQ+BBgBCAAJ + BQJYrdoqAhsCAikJEI2BgDwOv82IwV0gBBkBCAAGBQJYrdoqAAoJEH6gqcPyc/zY + 1WAP/2wJ+R0gE6qsce3rjaIz58PJmc8goKrir5hnElWhPgbq7cYIsW5qiFyLhkdp + YcMmhD9mRiPpQn6Ya2w3e3B8zfIVKipbMBnke/ytZ9M7qHmDCcjoiSmwEXN3wKYI + mD9VHONsl/CG1rU9Isw1jtB5g1YxuBA7M/m36XN6x2u+NtNMDB9P56yc4gfsZVES + KA9v+yY2/l45L8d/WUkUi0YXomn6hyBGI7JrBLq0CX37GEYP6O9rrKipfz73XfO7 + JIGzOKZlljb/D9RX/g7nRbCn+3EtH7xnk+TK/50euEKw8SMUg147sJTcpQmv6UzZ + cM4JgL0HbHVCojV4C/plELwMddALOFeYQzTif6sMRPf+3DSj8frbInjChC3yOLy0 + 6br92KFom17EIj2CAcoeq7UPhi2oouYBwPxh5ytdehJkoo+sN7RIWua6P2WSmon5 + U888cSylXC0+ADFdgLX9K2zrDVYUG1vo8CX0vzxFBaHwN6Px26fhIT1/hYUHQR1z + VfNDcyQmXqkOnZvvoMfz/Q0s9BhFJ/zU6AgQbIZE/hm1spsfgvtsD1frZfygXJ9f + irP+MSAI80xHSf91qSRZOj4Pl3ZJNbq4yYxv0b1pkMqeGdjdCYhLU+LZ4wbQmpCk + SVe2prlLureigXtmZfkqevRz7FrIZiu9ky8wnCAPwC7/zmS18rgP/17bOtL4/iIz + QhxAAoAMWVrGyJivSkjhSGx1uCojsWfsTAm11P7jsruIL61ZzMUVE2aM3Pmj5G+W + 9AcZ58Em+1WsVnAXdUR//bMmhyr8wL/G1YO1V3JEJTRdxsSxdYa4deGBBY/Adpsw + 24jxhOJR+lsJpqIUeb999+R8euDhRHG9eFO7DRu6weatUJ6suupoDTRWtr/4yGqe + dKxV3qQhNLSnaAzqW/1nA3iUB4k7kCaKZxhdhDbClf9P37qaRW467BLCVO/coL3y + Vm50dwdrNtKpMBh3ZpbB1uJvgi9mXtyBOMJ3v8RZeDzFiG8HdCtg9RvIt/AIFoHR + H3S+U79NT6i0KPzLImDfs8T7RlpyuMc4Ufs8ggyg9v3Ae6cN3eQyxcK3w0cbBwsh + /nQNfsA6uu+9H7NhbehBMhYnpNZyrHzCmzyXkauwRAqoCbGCNykTRwsur9gS41TQ + M8ssD1jFheOJf3hODnkKU+HKjvMROl1DK7zdmLdNzA1cvtZH/nCC9KPj1z8QC47S + xx+dTZSx4ONAhwbS/LN3PoKtn8LPjY9NP9uDWI+TWYquS2U+KHDrBDlsgozDbs/O + jCxcpDzNmXpWQHEtHU7649OXHP7UeNST1mCUCH5qdank0V1iejF6/CfTFU4MfcrG + YT90qFF93M3v01BbxP+EIY2/9tiIPbrd + =0YYh + -----END PGP PUBLIC KEY BLOCK----- + +- path: "/opt/bin/setup" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + + # As we added some modules and don't want to reboot, restart the service + systemctl restart systemd-modules-load.service + sysctl --system + + apt-key add /opt/docker.asc + apt-get update + + # Make sure we always disable swap - Otherwise the kubelet won't start'. + systemctl mask swap.target + swapoff -a + export CR_PKG='docker.io=17.12.1-0ubuntu1' + + DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install -y \ + curl \ + ca-certificates \ + ceph-common \ + cifs-utils \ + conntrack \ + e2fsprogs \ + ebtables \ + ethtool \ + glusterfs-client \ + iptables \ + jq \ + kmod \ + openssh-client \ + nfs-common \ + socat \ + util-linux \ + ${CR_PKG} \ + open-vm-tools \ + ipvsadm + + # If something failed during package installation but docker got installed, we need to put it on hold + apt-mark hold docker.io || true + apt-mark hold docker-ce || true + if [[ -e /var/run/reboot-required ]]; then + reboot + fi + + #setup some common directories + mkdir -p /opt/bin/ + mkdir -p /var/lib/calico + mkdir -p /etc/kubernetes/manifests + mkdir -p /etc/cni/net.d + mkdir -p /opt/cni/bin + + # cni + if [ ! -f /opt/cni/bin/loopback ]; then + curl -L https://github.com/containernetworking/plugins/releases/download/v0.6.0/cni-plugins-amd64-v0.6.0.tgz | tar -xvzC /opt/cni/bin -f - + fi + # kubelet + if [ ! -f /opt/bin/kubelet ]; then + curl -Lfo /opt/bin/kubelet https://storage.googleapis.com/kubernetes-release/release/v1.11.3/bin/linux/amd64/kubelet + chmod +x /opt/bin/kubelet + fi + + if [[ ! -x /opt/bin/health-monitor.sh ]]; then + curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/8b5b66e4910a6228dfaecccaa0a3b05ec4902f8e/pkg/userdata/scripts/health-monitor.sh + chmod +x /opt/bin/health-monitor.sh + fi + + + systemctl enable --now docker + systemctl enable --now kubelet + systemctl enable --now --no-block kubelet-healthcheck.service + systemctl enable --now --no-block docker-healthcheck.service + +- path: "/opt/bin/supervise.sh" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + while ! "$@"; do + sleep 1 + done + +- path: "/etc/systemd/system/kubelet.service" + content: | + [Unit] + After=docker.service + Requires=docker.service + + Description=kubelet: The Kubernetes Node Agent + Documentation=https://kubernetes.io/docs/home/ + + [Service] + Restart=always + StartLimitInterval=0 + RestartSec=10 + + Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" + + ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ + --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ + --kubeconfig=/etc/kubernetes/kubelet.conf \ + --pod-manifest-path=/etc/kubernetes/manifests \ + --allow-privileged=true \ + --network-plugin=cni \ + --cni-conf-dir=/etc/cni/net.d \ + --cni-bin-dir=/opt/cni/bin \ + --authorization-mode=Webhook \ + --client-ca-file=/etc/kubernetes/pki/ca.crt \ + --cadvisor-port=0 \ + --rotate-certificates=true \ + --cert-dir=/etc/kubernetes/pki \ + --authentication-token-webhook=true \ + --hostname-override=node1 \ + --read-only-port=0 \ + --exit-on-lock-contention \ + --lock-file=/tmp/kubelet.lock \ + --anonymous-auth=false \ + --protect-kernel-defaults=true \ + --cluster-dns=10.10.10.10 \ + --cluster-domain=cluster.local + + [Install] + WantedBy=multi-user.target + +- path: "/etc/systemd/system/kubelet.service.d/extras.conf" + content: | + [Service] + Environment="KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf" + +- path: "/etc/kubernetes/cloud-config" + content: | + + +- path: "/etc/kubernetes/bootstrap-kubelet.conf" + content: | + apiVersion: v1 + clusters: + - cluster: + certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t + server: https://server:443 + name: "" + contexts: [] + current-context: "" + kind: Config + preferences: {} + users: + - name: "" + user: + token: my-token + + +- path: "/etc/kubernetes/pki/ca.crt" + content: | + -----BEGIN CERTIFICATE----- + MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV + BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG + A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 + DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 + NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG + cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv + c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B + AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS + R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT + ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk + JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 + mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW + caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G + A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt + hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB + MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES + MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv + bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h + U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao + eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 + UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD + 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n + sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF + kPe6XoSbiLm/kxk32T0= + -----END CERTIFICATE----- + +- path: "/etc/systemd/system/setup.service" + permissions: "0644" + content: | + [Install] + WantedBy=multi-user.target + + [Unit] + Requires=network-online.target + After=network-online.target + + [Service] + Type=oneshot + RemainAfterExit=true + ExecStart=/opt/bin/supervise.sh /opt/bin/setup + +- path: "/etc/profile.d/opt-bin-path.sh" + permissions: "0644" + content: | + export PATH="/opt/bin:$PATH" + +- path: /etc/systemd/system/docker.service.d/10-storage.conf + permissions: "0644" + content: | + [Service] + ExecStart= + ExecStart=/usr/bin/dockerd -H fd:// --storage-driver=overlay2 + +- path: /etc/systemd/system/kubelet-healthcheck.service + permissions: "0644" + content: | + [Unit] + Requires=kubelet.service + After=kubelet.service + + [Service] + ExecStart=/opt/bin/health-monitor.sh kubelet + + [Install] + WantedBy=multi-user.target + + +- path: /etc/systemd/system/docker-healthcheck.service + permissions: "0644" + content: | + [Unit] + Requires=docker.service + After=docker.service + + [Service] + ExecStart=/opt/bin/health-monitor.sh container-runtime + + [Install] + WantedBy=multi-user.target + + +runcmd: +- systemctl enable --now setup.service diff --git a/pkg/userdata/ubuntu/testdata/multiple-dns-servers.golden b/pkg/userdata/ubuntu/testdata/multiple-dns-servers.golden new file mode 100644 index 000000000..fa4ce22b0 --- /dev/null +++ b/pkg/userdata/ubuntu/testdata/multiple-dns-servers.golden @@ -0,0 +1,342 @@ +#cloud-config +hostname: node1 + +ssh_pwauth: no + +ssh_authorized_keys: +- "ssh-rsa AAABBB" + +write_files: +- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" + content: | + [Journal] + SystemMaxUse=5G + + +- path: "/etc/modules-load.d/k8s.conf" + content: | + ip_vs + ip_vs_rr + ip_vs_wrr + ip_vs_sh + nf_conntrack_ipv4 + + +- path: "/etc/sysctl.d/k8s.conf" + content: | + net.bridge.bridge-nf-call-ip6tables = 1 + net.bridge.bridge-nf-call-iptables = 1 + kernel.panic_on_oops = 1 + kernel.panic = 10 + net.ipv4.ip_forward = 1 + vm.overcommit_memory = 1 + + +- path: "/etc/apt/sources.list.d/docker.list" + permissions: "0644" + content: deb [arch=amd64] https://download.docker.com/linux/ubuntu bionic stable + +- path: "/opt/docker.asc" + permissions: "0400" + content: | + -----BEGIN PGP PUBLIC KEY BLOCK----- + + mQINBFit2ioBEADhWpZ8/wvZ6hUTiXOwQHXMAlaFHcPH9hAtr4F1y2+OYdbtMuth + lqqwp028AqyY+PRfVMtSYMbjuQuu5byyKR01BbqYhuS3jtqQmljZ/bJvXqnmiVXh + 38UuLa+z077PxyxQhu5BbqntTPQMfiyqEiU+BKbq2WmANUKQf+1AmZY/IruOXbnq + L4C1+gJ8vfmXQt99npCaxEjaNRVYfOS8QcixNzHUYnb6emjlANyEVlZzeqo7XKl7 + UrwV5inawTSzWNvtjEjj4nJL8NsLwscpLPQUhTQ+7BbQXAwAmeHCUTQIvvWXqw0N + cmhh4HgeQscQHYgOJjjDVfoY5MucvglbIgCqfzAHW9jxmRL4qbMZj+b1XoePEtht + ku4bIQN1X5P07fNWzlgaRL5Z4POXDDZTlIQ/El58j9kp4bnWRCJW0lya+f8ocodo + vZZ+Doi+fy4D5ZGrL4XEcIQP/Lv5uFyf+kQtl/94VFYVJOleAv8W92KdgDkhTcTD + G7c0tIkVEKNUq48b3aQ64NOZQW7fVjfoKwEZdOqPE72Pa45jrZzvUFxSpdiNk2tZ + XYukHjlxxEgBdC/J3cMMNRE1F4NCA3ApfV1Y7/hTeOnmDuDYwr9/obA8t016Yljj + q5rdkywPf4JF8mXUW5eCN1vAFHxeg9ZWemhBtQmGxXnw9M+z6hWwc6ahmwARAQAB + tCtEb2NrZXIgUmVsZWFzZSAoQ0UgZGViKSA8ZG9ja2VyQGRvY2tlci5jb20+iQI3 + BBMBCgAhBQJYrefAAhsvBQsJCAcDBRUKCQgLBRYCAwEAAh4BAheAAAoJEI2BgDwO + v82IsskP/iQZo68flDQmNvn8X5XTd6RRaUH33kXYXquT6NkHJciS7E2gTJmqvMqd + tI4mNYHCSEYxI5qrcYV5YqX9P6+Ko+vozo4nseUQLPH/ATQ4qL0Zok+1jkag3Lgk + jonyUf9bwtWxFp05HC3GMHPhhcUSexCxQLQvnFWXD2sWLKivHp2fT8QbRGeZ+d3m + 6fqcd5Fu7pxsqm0EUDK5NL+nPIgYhN+auTrhgzhK1CShfGccM/wfRlei9Utz6p9P + XRKIlWnXtT4qNGZNTN0tR+NLG/6Bqd8OYBaFAUcue/w1VW6JQ2VGYZHnZu9S8LMc + FYBa5Ig9PxwGQOgq6RDKDbV+PqTQT5EFMeR1mrjckk4DQJjbxeMZbiNMG5kGECA8 + g383P3elhn03WGbEEa4MNc3Z4+7c236QI3xWJfNPdUbXRaAwhy/6rTSFbzwKB0Jm + ebwzQfwjQY6f55MiI/RqDCyuPj3r3jyVRkK86pQKBAJwFHyqj9KaKXMZjfVnowLh + 9svIGfNbGHpucATqREvUHuQbNnqkCx8VVhtYkhDb9fEP2xBu5VvHbR+3nfVhMut5 + G34Ct5RS7Jt6LIfFdtcn8CaSas/l1HbiGeRgc70X/9aYx/V/CEJv0lIe8gP6uDoW + FPIZ7d6vH+Vro6xuWEGiuMaiznap2KhZmpkgfupyFmplh0s6knymuQINBFit2ioB + EADneL9S9m4vhU3blaRjVUUyJ7b/qTjcSylvCH5XUE6R2k+ckEZjfAMZPLpO+/tF + M2JIJMD4SifKuS3xck9KtZGCufGmcwiLQRzeHF7vJUKrLD5RTkNi23ydvWZgPjtx + Q+DTT1Zcn7BrQFY6FgnRoUVIxwtdw1bMY/89rsFgS5wwuMESd3Q2RYgb7EOFOpnu + w6da7WakWf4IhnF5nsNYGDVaIHzpiqCl+uTbf1epCjrOlIzkZ3Z3Yk5CM/TiFzPk + z2lLz89cpD8U+NtCsfagWWfjd2U3jDapgH+7nQnCEWpROtzaKHG6lA3pXdix5zG8 + eRc6/0IbUSWvfjKxLLPfNeCS2pCL3IeEI5nothEEYdQH6szpLog79xB9dVnJyKJb + VfxXnseoYqVrRz2VVbUI5Blwm6B40E3eGVfUQWiux54DspyVMMk41Mx7QJ3iynIa + 1N4ZAqVMAEruyXTRTxc9XW0tYhDMA/1GYvz0EmFpm8LzTHA6sFVtPm/ZlNCX6P1X + zJwrv7DSQKD6GGlBQUX+OeEJ8tTkkf8QTJSPUdh8P8YxDFS5EOGAvhhpMBYD42kQ + pqXjEC+XcycTvGI7impgv9PDY1RCC1zkBjKPa120rNhv/hkVk/YhuGoajoHyy4h7 + ZQopdcMtpN2dgmhEegny9JCSwxfQmQ0zK0g7m6SHiKMwjwARAQABiQQ+BBgBCAAJ + BQJYrdoqAhsCAikJEI2BgDwOv82IwV0gBBkBCAAGBQJYrdoqAAoJEH6gqcPyc/zY + 1WAP/2wJ+R0gE6qsce3rjaIz58PJmc8goKrir5hnElWhPgbq7cYIsW5qiFyLhkdp + YcMmhD9mRiPpQn6Ya2w3e3B8zfIVKipbMBnke/ytZ9M7qHmDCcjoiSmwEXN3wKYI + mD9VHONsl/CG1rU9Isw1jtB5g1YxuBA7M/m36XN6x2u+NtNMDB9P56yc4gfsZVES + KA9v+yY2/l45L8d/WUkUi0YXomn6hyBGI7JrBLq0CX37GEYP6O9rrKipfz73XfO7 + JIGzOKZlljb/D9RX/g7nRbCn+3EtH7xnk+TK/50euEKw8SMUg147sJTcpQmv6UzZ + cM4JgL0HbHVCojV4C/plELwMddALOFeYQzTif6sMRPf+3DSj8frbInjChC3yOLy0 + 6br92KFom17EIj2CAcoeq7UPhi2oouYBwPxh5ytdehJkoo+sN7RIWua6P2WSmon5 + U888cSylXC0+ADFdgLX9K2zrDVYUG1vo8CX0vzxFBaHwN6Px26fhIT1/hYUHQR1z + VfNDcyQmXqkOnZvvoMfz/Q0s9BhFJ/zU6AgQbIZE/hm1spsfgvtsD1frZfygXJ9f + irP+MSAI80xHSf91qSRZOj4Pl3ZJNbq4yYxv0b1pkMqeGdjdCYhLU+LZ4wbQmpCk + SVe2prlLureigXtmZfkqevRz7FrIZiu9ky8wnCAPwC7/zmS18rgP/17bOtL4/iIz + QhxAAoAMWVrGyJivSkjhSGx1uCojsWfsTAm11P7jsruIL61ZzMUVE2aM3Pmj5G+W + 9AcZ58Em+1WsVnAXdUR//bMmhyr8wL/G1YO1V3JEJTRdxsSxdYa4deGBBY/Adpsw + 24jxhOJR+lsJpqIUeb999+R8euDhRHG9eFO7DRu6weatUJ6suupoDTRWtr/4yGqe + dKxV3qQhNLSnaAzqW/1nA3iUB4k7kCaKZxhdhDbClf9P37qaRW467BLCVO/coL3y + Vm50dwdrNtKpMBh3ZpbB1uJvgi9mXtyBOMJ3v8RZeDzFiG8HdCtg9RvIt/AIFoHR + H3S+U79NT6i0KPzLImDfs8T7RlpyuMc4Ufs8ggyg9v3Ae6cN3eQyxcK3w0cbBwsh + /nQNfsA6uu+9H7NhbehBMhYnpNZyrHzCmzyXkauwRAqoCbGCNykTRwsur9gS41TQ + M8ssD1jFheOJf3hODnkKU+HKjvMROl1DK7zdmLdNzA1cvtZH/nCC9KPj1z8QC47S + xx+dTZSx4ONAhwbS/LN3PoKtn8LPjY9NP9uDWI+TWYquS2U+KHDrBDlsgozDbs/O + jCxcpDzNmXpWQHEtHU7649OXHP7UeNST1mCUCH5qdank0V1iejF6/CfTFU4MfcrG + YT90qFF93M3v01BbxP+EIY2/9tiIPbrd + =0YYh + -----END PGP PUBLIC KEY BLOCK----- + +- path: "/opt/bin/setup" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + + # As we added some modules and don't want to reboot, restart the service + systemctl restart systemd-modules-load.service + sysctl --system + + apt-key add /opt/docker.asc + apt-get update + + # Make sure we always disable swap - Otherwise the kubelet won't start'. + systemctl mask swap.target + swapoff -a + export CR_PKG='docker.io=17.12.1-0ubuntu1' + + DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install -y \ + curl \ + ca-certificates \ + ceph-common \ + cifs-utils \ + conntrack \ + e2fsprogs \ + ebtables \ + ethtool \ + glusterfs-client \ + iptables \ + jq \ + kmod \ + openssh-client \ + nfs-common \ + socat \ + util-linux \ + ${CR_PKG} \ + open-vm-tools \ + ipvsadm + + # If something failed during package installation but docker got installed, we need to put it on hold + apt-mark hold docker.io || true + apt-mark hold docker-ce || true + if [[ -e /var/run/reboot-required ]]; then + reboot + fi + + #setup some common directories + mkdir -p /opt/bin/ + mkdir -p /var/lib/calico + mkdir -p /etc/kubernetes/manifests + mkdir -p /etc/cni/net.d + mkdir -p /opt/cni/bin + + # cni + if [ ! -f /opt/cni/bin/loopback ]; then + curl -L https://github.com/containernetworking/plugins/releases/download/v0.6.0/cni-plugins-amd64-v0.6.0.tgz | tar -xvzC /opt/cni/bin -f - + fi + # kubelet + if [ ! -f /opt/bin/kubelet ]; then + curl -Lfo /opt/bin/kubelet https://storage.googleapis.com/kubernetes-release/release/v1.11.3/bin/linux/amd64/kubelet + chmod +x /opt/bin/kubelet + fi + + if [[ ! -x /opt/bin/health-monitor.sh ]]; then + curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/8b5b66e4910a6228dfaecccaa0a3b05ec4902f8e/pkg/userdata/scripts/health-monitor.sh + chmod +x /opt/bin/health-monitor.sh + fi + + + systemctl enable --now docker + systemctl enable --now kubelet + systemctl enable --now --no-block kubelet-healthcheck.service + systemctl enable --now --no-block docker-healthcheck.service + +- path: "/opt/bin/supervise.sh" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + while ! "$@"; do + sleep 1 + done + +- path: "/etc/systemd/system/kubelet.service" + content: | + [Unit] + After=docker.service + Requires=docker.service + + Description=kubelet: The Kubernetes Node Agent + Documentation=https://kubernetes.io/docs/home/ + + [Service] + Restart=always + StartLimitInterval=0 + RestartSec=10 + + Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" + + ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ + --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ + --kubeconfig=/etc/kubernetes/kubelet.conf \ + --pod-manifest-path=/etc/kubernetes/manifests \ + --allow-privileged=true \ + --network-plugin=cni \ + --cni-conf-dir=/etc/cni/net.d \ + --cni-bin-dir=/opt/cni/bin \ + --authorization-mode=Webhook \ + --client-ca-file=/etc/kubernetes/pki/ca.crt \ + --cadvisor-port=0 \ + --rotate-certificates=true \ + --cert-dir=/etc/kubernetes/pki \ + --authentication-token-webhook=true \ + --hostname-override=node1 \ + --read-only-port=0 \ + --exit-on-lock-contention \ + --lock-file=/tmp/kubelet.lock \ + --anonymous-auth=false \ + --protect-kernel-defaults=true \ + --cluster-dns=10.10.10.10,10.10.10.11,10.10.10.12 \ + --cluster-domain=cluster.local + + [Install] + WantedBy=multi-user.target + +- path: "/etc/systemd/system/kubelet.service.d/extras.conf" + content: | + [Service] + Environment="KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf" + +- path: "/etc/kubernetes/cloud-config" + content: | + + +- path: "/etc/kubernetes/bootstrap-kubelet.conf" + content: | + apiVersion: v1 + clusters: + - cluster: + certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t + server: https://server:443 + name: "" + contexts: [] + current-context: "" + kind: Config + preferences: {} + users: + - name: "" + user: + token: my-token + + +- path: "/etc/kubernetes/pki/ca.crt" + content: | + -----BEGIN CERTIFICATE----- + MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV + BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG + A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 + DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 + NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG + cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv + c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B + AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS + R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT + ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk + JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 + mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW + caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G + A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt + hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB + MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES + MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv + bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h + U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao + eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 + UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD + 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n + sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF + kPe6XoSbiLm/kxk32T0= + -----END CERTIFICATE----- + +- path: "/etc/systemd/system/setup.service" + permissions: "0644" + content: | + [Install] + WantedBy=multi-user.target + + [Unit] + Requires=network-online.target + After=network-online.target + + [Service] + Type=oneshot + RemainAfterExit=true + ExecStart=/opt/bin/supervise.sh /opt/bin/setup + +- path: "/etc/profile.d/opt-bin-path.sh" + permissions: "0644" + content: | + export PATH="/opt/bin:$PATH" + +- path: /etc/systemd/system/docker.service.d/10-storage.conf + permissions: "0644" + content: | + [Service] + ExecStart= + ExecStart=/usr/bin/dockerd -H fd:// --storage-driver=overlay2 + +- path: /etc/systemd/system/kubelet-healthcheck.service + permissions: "0644" + content: | + [Unit] + Requires=kubelet.service + After=kubelet.service + + [Service] + ExecStart=/opt/bin/health-monitor.sh kubelet + + [Install] + WantedBy=multi-user.target + + +- path: /etc/systemd/system/docker-healthcheck.service + permissions: "0644" + content: | + [Unit] + Requires=docker.service + After=docker.service + + [Service] + ExecStart=/opt/bin/health-monitor.sh container-runtime + + [Install] + WantedBy=multi-user.target + + +runcmd: +- systemctl enable --now setup.service diff --git a/pkg/userdata/ubuntu/testdata/multiple-ssh-keys.golden b/pkg/userdata/ubuntu/testdata/multiple-ssh-keys.golden new file mode 100644 index 000000000..41578f254 --- /dev/null +++ b/pkg/userdata/ubuntu/testdata/multiple-ssh-keys.golden @@ -0,0 +1,344 @@ +#cloud-config +hostname: node1 + +ssh_pwauth: no + +ssh_authorized_keys: +- "ssh-rsa AAABBB" +- "ssh-rsa CCCDDD" +- "ssh-rsa EEEFFF" + +write_files: +- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" + content: | + [Journal] + SystemMaxUse=5G + + +- path: "/etc/modules-load.d/k8s.conf" + content: | + ip_vs + ip_vs_rr + ip_vs_wrr + ip_vs_sh + nf_conntrack_ipv4 + + +- path: "/etc/sysctl.d/k8s.conf" + content: | + net.bridge.bridge-nf-call-ip6tables = 1 + net.bridge.bridge-nf-call-iptables = 1 + kernel.panic_on_oops = 1 + kernel.panic = 10 + net.ipv4.ip_forward = 1 + vm.overcommit_memory = 1 + + +- path: "/etc/apt/sources.list.d/docker.list" + permissions: "0644" + content: deb [arch=amd64] https://download.docker.com/linux/ubuntu bionic stable + +- path: "/opt/docker.asc" + permissions: "0400" + content: | + -----BEGIN PGP PUBLIC KEY BLOCK----- + + mQINBFit2ioBEADhWpZ8/wvZ6hUTiXOwQHXMAlaFHcPH9hAtr4F1y2+OYdbtMuth + lqqwp028AqyY+PRfVMtSYMbjuQuu5byyKR01BbqYhuS3jtqQmljZ/bJvXqnmiVXh + 38UuLa+z077PxyxQhu5BbqntTPQMfiyqEiU+BKbq2WmANUKQf+1AmZY/IruOXbnq + L4C1+gJ8vfmXQt99npCaxEjaNRVYfOS8QcixNzHUYnb6emjlANyEVlZzeqo7XKl7 + UrwV5inawTSzWNvtjEjj4nJL8NsLwscpLPQUhTQ+7BbQXAwAmeHCUTQIvvWXqw0N + cmhh4HgeQscQHYgOJjjDVfoY5MucvglbIgCqfzAHW9jxmRL4qbMZj+b1XoePEtht + ku4bIQN1X5P07fNWzlgaRL5Z4POXDDZTlIQ/El58j9kp4bnWRCJW0lya+f8ocodo + vZZ+Doi+fy4D5ZGrL4XEcIQP/Lv5uFyf+kQtl/94VFYVJOleAv8W92KdgDkhTcTD + G7c0tIkVEKNUq48b3aQ64NOZQW7fVjfoKwEZdOqPE72Pa45jrZzvUFxSpdiNk2tZ + XYukHjlxxEgBdC/J3cMMNRE1F4NCA3ApfV1Y7/hTeOnmDuDYwr9/obA8t016Yljj + q5rdkywPf4JF8mXUW5eCN1vAFHxeg9ZWemhBtQmGxXnw9M+z6hWwc6ahmwARAQAB + tCtEb2NrZXIgUmVsZWFzZSAoQ0UgZGViKSA8ZG9ja2VyQGRvY2tlci5jb20+iQI3 + BBMBCgAhBQJYrefAAhsvBQsJCAcDBRUKCQgLBRYCAwEAAh4BAheAAAoJEI2BgDwO + v82IsskP/iQZo68flDQmNvn8X5XTd6RRaUH33kXYXquT6NkHJciS7E2gTJmqvMqd + tI4mNYHCSEYxI5qrcYV5YqX9P6+Ko+vozo4nseUQLPH/ATQ4qL0Zok+1jkag3Lgk + jonyUf9bwtWxFp05HC3GMHPhhcUSexCxQLQvnFWXD2sWLKivHp2fT8QbRGeZ+d3m + 6fqcd5Fu7pxsqm0EUDK5NL+nPIgYhN+auTrhgzhK1CShfGccM/wfRlei9Utz6p9P + XRKIlWnXtT4qNGZNTN0tR+NLG/6Bqd8OYBaFAUcue/w1VW6JQ2VGYZHnZu9S8LMc + FYBa5Ig9PxwGQOgq6RDKDbV+PqTQT5EFMeR1mrjckk4DQJjbxeMZbiNMG5kGECA8 + g383P3elhn03WGbEEa4MNc3Z4+7c236QI3xWJfNPdUbXRaAwhy/6rTSFbzwKB0Jm + ebwzQfwjQY6f55MiI/RqDCyuPj3r3jyVRkK86pQKBAJwFHyqj9KaKXMZjfVnowLh + 9svIGfNbGHpucATqREvUHuQbNnqkCx8VVhtYkhDb9fEP2xBu5VvHbR+3nfVhMut5 + G34Ct5RS7Jt6LIfFdtcn8CaSas/l1HbiGeRgc70X/9aYx/V/CEJv0lIe8gP6uDoW + FPIZ7d6vH+Vro6xuWEGiuMaiznap2KhZmpkgfupyFmplh0s6knymuQINBFit2ioB + EADneL9S9m4vhU3blaRjVUUyJ7b/qTjcSylvCH5XUE6R2k+ckEZjfAMZPLpO+/tF + M2JIJMD4SifKuS3xck9KtZGCufGmcwiLQRzeHF7vJUKrLD5RTkNi23ydvWZgPjtx + Q+DTT1Zcn7BrQFY6FgnRoUVIxwtdw1bMY/89rsFgS5wwuMESd3Q2RYgb7EOFOpnu + w6da7WakWf4IhnF5nsNYGDVaIHzpiqCl+uTbf1epCjrOlIzkZ3Z3Yk5CM/TiFzPk + z2lLz89cpD8U+NtCsfagWWfjd2U3jDapgH+7nQnCEWpROtzaKHG6lA3pXdix5zG8 + eRc6/0IbUSWvfjKxLLPfNeCS2pCL3IeEI5nothEEYdQH6szpLog79xB9dVnJyKJb + VfxXnseoYqVrRz2VVbUI5Blwm6B40E3eGVfUQWiux54DspyVMMk41Mx7QJ3iynIa + 1N4ZAqVMAEruyXTRTxc9XW0tYhDMA/1GYvz0EmFpm8LzTHA6sFVtPm/ZlNCX6P1X + zJwrv7DSQKD6GGlBQUX+OeEJ8tTkkf8QTJSPUdh8P8YxDFS5EOGAvhhpMBYD42kQ + pqXjEC+XcycTvGI7impgv9PDY1RCC1zkBjKPa120rNhv/hkVk/YhuGoajoHyy4h7 + ZQopdcMtpN2dgmhEegny9JCSwxfQmQ0zK0g7m6SHiKMwjwARAQABiQQ+BBgBCAAJ + BQJYrdoqAhsCAikJEI2BgDwOv82IwV0gBBkBCAAGBQJYrdoqAAoJEH6gqcPyc/zY + 1WAP/2wJ+R0gE6qsce3rjaIz58PJmc8goKrir5hnElWhPgbq7cYIsW5qiFyLhkdp + YcMmhD9mRiPpQn6Ya2w3e3B8zfIVKipbMBnke/ytZ9M7qHmDCcjoiSmwEXN3wKYI + mD9VHONsl/CG1rU9Isw1jtB5g1YxuBA7M/m36XN6x2u+NtNMDB9P56yc4gfsZVES + KA9v+yY2/l45L8d/WUkUi0YXomn6hyBGI7JrBLq0CX37GEYP6O9rrKipfz73XfO7 + JIGzOKZlljb/D9RX/g7nRbCn+3EtH7xnk+TK/50euEKw8SMUg147sJTcpQmv6UzZ + cM4JgL0HbHVCojV4C/plELwMddALOFeYQzTif6sMRPf+3DSj8frbInjChC3yOLy0 + 6br92KFom17EIj2CAcoeq7UPhi2oouYBwPxh5ytdehJkoo+sN7RIWua6P2WSmon5 + U888cSylXC0+ADFdgLX9K2zrDVYUG1vo8CX0vzxFBaHwN6Px26fhIT1/hYUHQR1z + VfNDcyQmXqkOnZvvoMfz/Q0s9BhFJ/zU6AgQbIZE/hm1spsfgvtsD1frZfygXJ9f + irP+MSAI80xHSf91qSRZOj4Pl3ZJNbq4yYxv0b1pkMqeGdjdCYhLU+LZ4wbQmpCk + SVe2prlLureigXtmZfkqevRz7FrIZiu9ky8wnCAPwC7/zmS18rgP/17bOtL4/iIz + QhxAAoAMWVrGyJivSkjhSGx1uCojsWfsTAm11P7jsruIL61ZzMUVE2aM3Pmj5G+W + 9AcZ58Em+1WsVnAXdUR//bMmhyr8wL/G1YO1V3JEJTRdxsSxdYa4deGBBY/Adpsw + 24jxhOJR+lsJpqIUeb999+R8euDhRHG9eFO7DRu6weatUJ6suupoDTRWtr/4yGqe + dKxV3qQhNLSnaAzqW/1nA3iUB4k7kCaKZxhdhDbClf9P37qaRW467BLCVO/coL3y + Vm50dwdrNtKpMBh3ZpbB1uJvgi9mXtyBOMJ3v8RZeDzFiG8HdCtg9RvIt/AIFoHR + H3S+U79NT6i0KPzLImDfs8T7RlpyuMc4Ufs8ggyg9v3Ae6cN3eQyxcK3w0cbBwsh + /nQNfsA6uu+9H7NhbehBMhYnpNZyrHzCmzyXkauwRAqoCbGCNykTRwsur9gS41TQ + M8ssD1jFheOJf3hODnkKU+HKjvMROl1DK7zdmLdNzA1cvtZH/nCC9KPj1z8QC47S + xx+dTZSx4ONAhwbS/LN3PoKtn8LPjY9NP9uDWI+TWYquS2U+KHDrBDlsgozDbs/O + jCxcpDzNmXpWQHEtHU7649OXHP7UeNST1mCUCH5qdank0V1iejF6/CfTFU4MfcrG + YT90qFF93M3v01BbxP+EIY2/9tiIPbrd + =0YYh + -----END PGP PUBLIC KEY BLOCK----- + +- path: "/opt/bin/setup" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + + # As we added some modules and don't want to reboot, restart the service + systemctl restart systemd-modules-load.service + sysctl --system + + apt-key add /opt/docker.asc + apt-get update + + # Make sure we always disable swap - Otherwise the kubelet won't start'. + systemctl mask swap.target + swapoff -a + export CR_PKG='docker.io=17.12.1-0ubuntu1' + + DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install -y \ + curl \ + ca-certificates \ + ceph-common \ + cifs-utils \ + conntrack \ + e2fsprogs \ + ebtables \ + ethtool \ + glusterfs-client \ + iptables \ + jq \ + kmod \ + openssh-client \ + nfs-common \ + socat \ + util-linux \ + ${CR_PKG} \ + open-vm-tools \ + ipvsadm + + # If something failed during package installation but docker got installed, we need to put it on hold + apt-mark hold docker.io || true + apt-mark hold docker-ce || true + if [[ -e /var/run/reboot-required ]]; then + reboot + fi + + #setup some common directories + mkdir -p /opt/bin/ + mkdir -p /var/lib/calico + mkdir -p /etc/kubernetes/manifests + mkdir -p /etc/cni/net.d + mkdir -p /opt/cni/bin + + # cni + if [ ! -f /opt/cni/bin/loopback ]; then + curl -L https://github.com/containernetworking/plugins/releases/download/v0.6.0/cni-plugins-amd64-v0.6.0.tgz | tar -xvzC /opt/cni/bin -f - + fi + # kubelet + if [ ! -f /opt/bin/kubelet ]; then + curl -Lfo /opt/bin/kubelet https://storage.googleapis.com/kubernetes-release/release/v1.11.3/bin/linux/amd64/kubelet + chmod +x /opt/bin/kubelet + fi + + if [[ ! -x /opt/bin/health-monitor.sh ]]; then + curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/8b5b66e4910a6228dfaecccaa0a3b05ec4902f8e/pkg/userdata/scripts/health-monitor.sh + chmod +x /opt/bin/health-monitor.sh + fi + + + systemctl enable --now docker + systemctl enable --now kubelet + systemctl enable --now --no-block kubelet-healthcheck.service + systemctl enable --now --no-block docker-healthcheck.service + +- path: "/opt/bin/supervise.sh" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + while ! "$@"; do + sleep 1 + done + +- path: "/etc/systemd/system/kubelet.service" + content: | + [Unit] + After=docker.service + Requires=docker.service + + Description=kubelet: The Kubernetes Node Agent + Documentation=https://kubernetes.io/docs/home/ + + [Service] + Restart=always + StartLimitInterval=0 + RestartSec=10 + + Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" + + ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ + --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ + --kubeconfig=/etc/kubernetes/kubelet.conf \ + --pod-manifest-path=/etc/kubernetes/manifests \ + --allow-privileged=true \ + --network-plugin=cni \ + --cni-conf-dir=/etc/cni/net.d \ + --cni-bin-dir=/opt/cni/bin \ + --authorization-mode=Webhook \ + --client-ca-file=/etc/kubernetes/pki/ca.crt \ + --cadvisor-port=0 \ + --rotate-certificates=true \ + --cert-dir=/etc/kubernetes/pki \ + --authentication-token-webhook=true \ + --hostname-override=node1 \ + --read-only-port=0 \ + --exit-on-lock-contention \ + --lock-file=/tmp/kubelet.lock \ + --anonymous-auth=false \ + --protect-kernel-defaults=true \ + --cluster-dns=10.10.10.10 \ + --cluster-domain=cluster.local + + [Install] + WantedBy=multi-user.target + +- path: "/etc/systemd/system/kubelet.service.d/extras.conf" + content: | + [Service] + Environment="KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf" + +- path: "/etc/kubernetes/cloud-config" + content: | + + +- path: "/etc/kubernetes/bootstrap-kubelet.conf" + content: | + apiVersion: v1 + clusters: + - cluster: + certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t + server: https://server:443 + name: "" + contexts: [] + current-context: "" + kind: Config + preferences: {} + users: + - name: "" + user: + token: my-token + + +- path: "/etc/kubernetes/pki/ca.crt" + content: | + -----BEGIN CERTIFICATE----- + MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV + BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG + A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 + DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 + NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG + cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv + c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B + AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS + R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT + ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk + JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 + mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW + caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G + A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt + hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB + MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES + MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv + bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h + U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao + eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 + UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD + 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n + sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF + kPe6XoSbiLm/kxk32T0= + -----END CERTIFICATE----- + +- path: "/etc/systemd/system/setup.service" + permissions: "0644" + content: | + [Install] + WantedBy=multi-user.target + + [Unit] + Requires=network-online.target + After=network-online.target + + [Service] + Type=oneshot + RemainAfterExit=true + ExecStart=/opt/bin/supervise.sh /opt/bin/setup + +- path: "/etc/profile.d/opt-bin-path.sh" + permissions: "0644" + content: | + export PATH="/opt/bin:$PATH" + +- path: /etc/systemd/system/docker.service.d/10-storage.conf + permissions: "0644" + content: | + [Service] + ExecStart= + ExecStart=/usr/bin/dockerd -H fd:// --storage-driver=overlay2 + +- path: /etc/systemd/system/kubelet-healthcheck.service + permissions: "0644" + content: | + [Unit] + Requires=kubelet.service + After=kubelet.service + + [Service] + ExecStart=/opt/bin/health-monitor.sh kubelet + + [Install] + WantedBy=multi-user.target + + +- path: /etc/systemd/system/docker-healthcheck.service + permissions: "0644" + content: | + [Unit] + Requires=docker.service + After=docker.service + + [Service] + ExecStart=/opt/bin/health-monitor.sh container-runtime + + [Install] + WantedBy=multi-user.target + + +runcmd: +- systemctl enable --now setup.service diff --git a/pkg/userdata/ubuntu/testdata/openstack-kubelet-v-version-prefix.golden b/pkg/userdata/ubuntu/testdata/openstack-kubelet-v-version-prefix.golden deleted file mode 100644 index 7e98523a2..000000000 --- a/pkg/userdata/ubuntu/testdata/openstack-kubelet-v-version-prefix.golden +++ /dev/null @@ -1,285 +0,0 @@ -#cloud-config -hostname: node1 - -ssh_pwauth: no - -ssh_authorized_keys: -- "ssh-rsa AAABBB" -- "ssh-rsa CCCDDD" - -write_files: -- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" - content: | - [Journal] - SystemMaxUse=5G - -- path: "/etc/sysctl.d/k8s.conf" - content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - vm.overcommit_memory = 1 - -- path: "/etc/kubernetes/cloud-config" - content: | - {openstack-config:true} - -- path: "/etc/apt/sources.list.d/docker.list" - permissions: "0644" - content: deb [arch=amd64] https://download.docker.com/linux/ubuntu bionic stable - -- path: "/etc/apt/sources.list.d/kubernetes.list" - permissions: "0644" - content: deb http://apt.kubernetes.io/ kubernetes-xenial main - -- path: "/usr/local/bin/setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - - sysctl --system - mkdir -p /opt/bin - apt-key add /opt/docker.asc - apt-key add /opt/kubernetes.asc - apt-get update - - # Hetzner's Ubuntu Bionic comes with swap pre-configured, so we force it off. - systemctl mask swap.target - swapoff -a - - # If something failed during package installation but one of docker/kubeadm/kubelet was already installed - # an apt-mark hold after the install won't do it, which is why we test here if the binaries exist and if - # yes put them on hold - set +e - which docker && apt-mark hold docker.io docker-ce - which kubelet && apt-mark hold kubelet - which kubeadm && apt-mark hold kubeadm - - # When docker is started from within the apt installation it fails with a - # 'no sockets found via socket activation: make sure the service was started by systemd' - # Apparently the package is broken in a way that it gets started without its dependencies, manually starting - # it works fine thought - which docker && systemctl start docker - set -e - DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" dist-upgrade -y - if [[ -e /var/run/reboot-required ]]; then - reboot - fi - - export CR_PKG='docker-ce=18.06.0~ce~3-0~ubuntu' - export CR_PKG='docker.io=17.12.1-0ubuntu1' - - - DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install -y \ - curl \ - ca-certificates \ - ceph-common \ - cifs-utils \ - conntrack \ - e2fsprogs \ - ebtables \ - ethtool \ - glusterfs-client \ - iptables \ - jq \ - kmod \ - openssh-client \ - nfs-common \ - socat \ - util-linux \ - ${CR_PKG} \ - open-vm-tools \ - kubelet=1.9.2-00 \ - kubeadm=1.9.2-00 \ - - cp /etc/default/kubelet-overwrite /etc/default/kubelet - - systemctl enable --now docker - systemctl enable kubelet - - if [[ ! -x /usr/local/bin/health-monitor.sh ]]; then - curl -Lfo /usr/local/bin/health-monitor.sh \ - https://raw.githubusercontent.com/kubermatic/machine-controller/8b5b66e4910a6228dfaecccaa0a3b05ec4902f8e/pkg/userdata/scripts/health-monitor.sh - chmod +x /usr/local/bin/health-monitor.sh - fi - - if ! [[ -e /etc/kubernetes/pki/ca.crt ]]; then - kubeadm join \ - --token my-token \ - --discovery-token-ca-cert-hash sha256:6caecce9fedcb55d4953d61a27dc6997361a2f226ad86d7e6004dde7526fc4b1 \ - --ignore-preflight-errors=CRI \ - server:443 - fi - - systemctl enable --now --no-block kubelet-healthcheck.service - systemctl enable --now --no-block docker-healthcheck.service - -- path: "/opt/kubernetes.asc" - permissions: "0400" - content: | - -----BEGIN PGP PUBLIC KEY BLOCK----- - - mQENBFrBaNsBCADrF18KCbsZlo4NjAvVecTBCnp6WcBQJ5oSh7+E98jX9YznUCrN - rgmeCcCMUvTDRDxfTaDJybaHugfba43nqhkbNpJ47YXsIa+YL6eEE9emSmQtjrSW - IiY+2YJYwsDgsgckF3duqkb02OdBQlh6IbHPoXB6H//b1PgZYsomB+841XW1LSJP - YlYbIrWfwDfQvtkFQI90r6NknVTQlpqQh5GLNWNYqRNrGQPmsB+NrUYrkl1nUt1L - RGu+rCe4bSaSmNbwKMQKkROE4kTiB72DPk7zH4Lm0uo0YFFWG4qsMIuqEihJ/9KN - X8GYBr+tWgyLooLlsdK3l+4dVqd8cjkJM1ExABEBAAG0QEdvb2dsZSBDbG91ZCBQ - YWNrYWdlcyBBdXRvbWF0aWMgU2lnbmluZyBLZXkgPGdjLXRlYW1AZ29vZ2xlLmNv - bT6JAT4EEwECACgFAlrBaNsCGy8FCQWjmoAGCwkIBwMCBhUIAgkKCwQWAgMBAh4B - AheAAAoJEGoDCyG6B/T78e8H/1WH2LN/nVNhm5TS1VYJG8B+IW8zS4BqyozxC9iJ - AJqZIVHXl8g8a/Hus8RfXR7cnYHcg8sjSaJfQhqO9RbKnffiuQgGrqwQxuC2jBa6 - M/QKzejTeP0Mgi67pyrLJNWrFI71RhritQZmzTZ2PoWxfv6b+Tv5v0rPaG+ut1J4 - 7pn+kYgtUaKdsJz1umi6HzK6AacDf0C0CksJdKG7MOWsZcB4xeOxJYuy6NuO6Kcd - Ez8/XyEUjIuIOlhYTd0hH8E/SEBbXXft7/VBQC5wNq40izPi+6WFK/e1O42DIpzQ - 749ogYQ1eodexPNhLzekKR3XhGrNXJ95r5KO10VrsLFNd8I= - =TKuP - -----END PGP PUBLIC KEY BLOCK----- - -- path: "/opt/docker.asc" - permissions: "0400" - content: | - -----BEGIN PGP PUBLIC KEY BLOCK----- - - mQINBFit2ioBEADhWpZ8/wvZ6hUTiXOwQHXMAlaFHcPH9hAtr4F1y2+OYdbtMuth - lqqwp028AqyY+PRfVMtSYMbjuQuu5byyKR01BbqYhuS3jtqQmljZ/bJvXqnmiVXh - 38UuLa+z077PxyxQhu5BbqntTPQMfiyqEiU+BKbq2WmANUKQf+1AmZY/IruOXbnq - L4C1+gJ8vfmXQt99npCaxEjaNRVYfOS8QcixNzHUYnb6emjlANyEVlZzeqo7XKl7 - UrwV5inawTSzWNvtjEjj4nJL8NsLwscpLPQUhTQ+7BbQXAwAmeHCUTQIvvWXqw0N - cmhh4HgeQscQHYgOJjjDVfoY5MucvglbIgCqfzAHW9jxmRL4qbMZj+b1XoePEtht - ku4bIQN1X5P07fNWzlgaRL5Z4POXDDZTlIQ/El58j9kp4bnWRCJW0lya+f8ocodo - vZZ+Doi+fy4D5ZGrL4XEcIQP/Lv5uFyf+kQtl/94VFYVJOleAv8W92KdgDkhTcTD - G7c0tIkVEKNUq48b3aQ64NOZQW7fVjfoKwEZdOqPE72Pa45jrZzvUFxSpdiNk2tZ - XYukHjlxxEgBdC/J3cMMNRE1F4NCA3ApfV1Y7/hTeOnmDuDYwr9/obA8t016Yljj - q5rdkywPf4JF8mXUW5eCN1vAFHxeg9ZWemhBtQmGxXnw9M+z6hWwc6ahmwARAQAB - tCtEb2NrZXIgUmVsZWFzZSAoQ0UgZGViKSA8ZG9ja2VyQGRvY2tlci5jb20+iQI3 - BBMBCgAhBQJYrefAAhsvBQsJCAcDBRUKCQgLBRYCAwEAAh4BAheAAAoJEI2BgDwO - v82IsskP/iQZo68flDQmNvn8X5XTd6RRaUH33kXYXquT6NkHJciS7E2gTJmqvMqd - tI4mNYHCSEYxI5qrcYV5YqX9P6+Ko+vozo4nseUQLPH/ATQ4qL0Zok+1jkag3Lgk - jonyUf9bwtWxFp05HC3GMHPhhcUSexCxQLQvnFWXD2sWLKivHp2fT8QbRGeZ+d3m - 6fqcd5Fu7pxsqm0EUDK5NL+nPIgYhN+auTrhgzhK1CShfGccM/wfRlei9Utz6p9P - XRKIlWnXtT4qNGZNTN0tR+NLG/6Bqd8OYBaFAUcue/w1VW6JQ2VGYZHnZu9S8LMc - FYBa5Ig9PxwGQOgq6RDKDbV+PqTQT5EFMeR1mrjckk4DQJjbxeMZbiNMG5kGECA8 - g383P3elhn03WGbEEa4MNc3Z4+7c236QI3xWJfNPdUbXRaAwhy/6rTSFbzwKB0Jm - ebwzQfwjQY6f55MiI/RqDCyuPj3r3jyVRkK86pQKBAJwFHyqj9KaKXMZjfVnowLh - 9svIGfNbGHpucATqREvUHuQbNnqkCx8VVhtYkhDb9fEP2xBu5VvHbR+3nfVhMut5 - G34Ct5RS7Jt6LIfFdtcn8CaSas/l1HbiGeRgc70X/9aYx/V/CEJv0lIe8gP6uDoW - FPIZ7d6vH+Vro6xuWEGiuMaiznap2KhZmpkgfupyFmplh0s6knymuQINBFit2ioB - EADneL9S9m4vhU3blaRjVUUyJ7b/qTjcSylvCH5XUE6R2k+ckEZjfAMZPLpO+/tF - M2JIJMD4SifKuS3xck9KtZGCufGmcwiLQRzeHF7vJUKrLD5RTkNi23ydvWZgPjtx - Q+DTT1Zcn7BrQFY6FgnRoUVIxwtdw1bMY/89rsFgS5wwuMESd3Q2RYgb7EOFOpnu - w6da7WakWf4IhnF5nsNYGDVaIHzpiqCl+uTbf1epCjrOlIzkZ3Z3Yk5CM/TiFzPk - z2lLz89cpD8U+NtCsfagWWfjd2U3jDapgH+7nQnCEWpROtzaKHG6lA3pXdix5zG8 - eRc6/0IbUSWvfjKxLLPfNeCS2pCL3IeEI5nothEEYdQH6szpLog79xB9dVnJyKJb - VfxXnseoYqVrRz2VVbUI5Blwm6B40E3eGVfUQWiux54DspyVMMk41Mx7QJ3iynIa - 1N4ZAqVMAEruyXTRTxc9XW0tYhDMA/1GYvz0EmFpm8LzTHA6sFVtPm/ZlNCX6P1X - zJwrv7DSQKD6GGlBQUX+OeEJ8tTkkf8QTJSPUdh8P8YxDFS5EOGAvhhpMBYD42kQ - pqXjEC+XcycTvGI7impgv9PDY1RCC1zkBjKPa120rNhv/hkVk/YhuGoajoHyy4h7 - ZQopdcMtpN2dgmhEegny9JCSwxfQmQ0zK0g7m6SHiKMwjwARAQABiQQ+BBgBCAAJ - BQJYrdoqAhsCAikJEI2BgDwOv82IwV0gBBkBCAAGBQJYrdoqAAoJEH6gqcPyc/zY - 1WAP/2wJ+R0gE6qsce3rjaIz58PJmc8goKrir5hnElWhPgbq7cYIsW5qiFyLhkdp - YcMmhD9mRiPpQn6Ya2w3e3B8zfIVKipbMBnke/ytZ9M7qHmDCcjoiSmwEXN3wKYI - mD9VHONsl/CG1rU9Isw1jtB5g1YxuBA7M/m36XN6x2u+NtNMDB9P56yc4gfsZVES - KA9v+yY2/l45L8d/WUkUi0YXomn6hyBGI7JrBLq0CX37GEYP6O9rrKipfz73XfO7 - JIGzOKZlljb/D9RX/g7nRbCn+3EtH7xnk+TK/50euEKw8SMUg147sJTcpQmv6UzZ - cM4JgL0HbHVCojV4C/plELwMddALOFeYQzTif6sMRPf+3DSj8frbInjChC3yOLy0 - 6br92KFom17EIj2CAcoeq7UPhi2oouYBwPxh5ytdehJkoo+sN7RIWua6P2WSmon5 - U888cSylXC0+ADFdgLX9K2zrDVYUG1vo8CX0vzxFBaHwN6Px26fhIT1/hYUHQR1z - VfNDcyQmXqkOnZvvoMfz/Q0s9BhFJ/zU6AgQbIZE/hm1spsfgvtsD1frZfygXJ9f - irP+MSAI80xHSf91qSRZOj4Pl3ZJNbq4yYxv0b1pkMqeGdjdCYhLU+LZ4wbQmpCk - SVe2prlLureigXtmZfkqevRz7FrIZiu9ky8wnCAPwC7/zmS18rgP/17bOtL4/iIz - QhxAAoAMWVrGyJivSkjhSGx1uCojsWfsTAm11P7jsruIL61ZzMUVE2aM3Pmj5G+W - 9AcZ58Em+1WsVnAXdUR//bMmhyr8wL/G1YO1V3JEJTRdxsSxdYa4deGBBY/Adpsw - 24jxhOJR+lsJpqIUeb999+R8euDhRHG9eFO7DRu6weatUJ6suupoDTRWtr/4yGqe - dKxV3qQhNLSnaAzqW/1nA3iUB4k7kCaKZxhdhDbClf9P37qaRW467BLCVO/coL3y - Vm50dwdrNtKpMBh3ZpbB1uJvgi9mXtyBOMJ3v8RZeDzFiG8HdCtg9RvIt/AIFoHR - H3S+U79NT6i0KPzLImDfs8T7RlpyuMc4Ufs8ggyg9v3Ae6cN3eQyxcK3w0cbBwsh - /nQNfsA6uu+9H7NhbehBMhYnpNZyrHzCmzyXkauwRAqoCbGCNykTRwsur9gS41TQ - M8ssD1jFheOJf3hODnkKU+HKjvMROl1DK7zdmLdNzA1cvtZH/nCC9KPj1z8QC47S - xx+dTZSx4ONAhwbS/LN3PoKtn8LPjY9NP9uDWI+TWYquS2U+KHDrBDlsgozDbs/O - jCxcpDzNmXpWQHEtHU7649OXHP7UeNST1mCUCH5qdank0V1iejF6/CfTFU4MfcrG - YT90qFF93M3v01BbxP+EIY2/9tiIPbrd - =0YYh - -----END PGP PUBLIC KEY BLOCK----- - -- path: "/usr/local/bin/supervise.sh" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - while ! "$@"; do - sleep 1 - done - -- path: "/etc/default/kubelet-overwrite" - content: | - KUBELET_DNS_ARGS= - KUBELET_EXTRA_ARGS=--authentication-token-webhook=true \ - --cloud-provider=openstack \ - --cloud-config=/etc/kubernetes/cloud-config \ - --hostname-override=node1 \ - --read-only-port=0 \ - --protect-kernel-defaults=true \ - --resolv-conf=/run/systemd/resolve/resolv.conf \ - --cluster-dns=10.10.10.10 \ - --cluster-domain=cluster.local - -- path: "/etc/systemd/system/kubelet.service.d/20-extra.conf" - permissions: "0644" - content: | - [Service] - EnvironmentFile=/etc/default/kubelet - - -- path: "/etc/systemd/system/setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - ExecStart=/usr/local/bin/supervise.sh /usr/local/bin/setup - -- path: /etc/systemd/system/docker.service.d/10-storage.conf - permissions: "0644" - content: | - [Service] - ExecStart= - ExecStart=/usr/bin/dockerd -H fd:// --storage-driver=overlay2 - -- path: /etc/systemd/system/kubelet-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/usr/local/bin/health-monitor.sh kubelet - - [Install] - WantedBy=multi-user.target - -- path: /etc/systemd/system/docker-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=docker.service - After=docker.service - - [Service] - ExecStart=/usr/local/bin/health-monitor.sh container-runtime - - [Install] - WantedBy=multi-user.target - -runcmd: -- systemctl enable --now setup.service diff --git a/pkg/userdata/ubuntu/testdata/openstack-overwrite-cloud-config.golden b/pkg/userdata/ubuntu/testdata/openstack-overwrite-cloud-config.golden index 48d607674..33ed8819d 100644 --- a/pkg/userdata/ubuntu/testdata/openstack-overwrite-cloud-config.golden +++ b/pkg/userdata/ubuntu/testdata/openstack-overwrite-cloud-config.golden @@ -5,13 +5,22 @@ ssh_pwauth: no ssh_authorized_keys: - "ssh-rsa AAABBB" -- "ssh-rsa CCCDDD" write_files: - path: "/etc/systemd/journald.conf.d/max_disk_use.conf" content: | [Journal] SystemMaxUse=5G + + +- path: "/etc/modules-load.d/k8s.conf" + content: | + ip_vs + ip_vs_rr + ip_vs_wrr + ip_vs_sh + nf_conntrack_ipv4 + - path: "/etc/sysctl.d/k8s.conf" content: | @@ -19,127 +28,14 @@ write_files: net.bridge.bridge-nf-call-iptables = 1 kernel.panic_on_oops = 1 kernel.panic = 10 + net.ipv4.ip_forward = 1 vm.overcommit_memory = 1 - -- path: "/etc/kubernetes/cloud-config" - content: | - custom - cloud - config + - path: "/etc/apt/sources.list.d/docker.list" permissions: "0644" content: deb [arch=amd64] https://download.docker.com/linux/ubuntu bionic stable -- path: "/etc/apt/sources.list.d/kubernetes.list" - permissions: "0644" - content: deb http://apt.kubernetes.io/ kubernetes-xenial main - -- path: "/usr/local/bin/setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - - sysctl --system - mkdir -p /opt/bin - apt-key add /opt/docker.asc - apt-key add /opt/kubernetes.asc - apt-get update - - # Hetzner's Ubuntu Bionic comes with swap pre-configured, so we force it off. - systemctl mask swap.target - swapoff -a - - # If something failed during package installation but one of docker/kubeadm/kubelet was already installed - # an apt-mark hold after the install won't do it, which is why we test here if the binaries exist and if - # yes put them on hold - set +e - which docker && apt-mark hold docker.io docker-ce - which kubelet && apt-mark hold kubelet - which kubeadm && apt-mark hold kubeadm - - # When docker is started from within the apt installation it fails with a - # 'no sockets found via socket activation: make sure the service was started by systemd' - # Apparently the package is broken in a way that it gets started without its dependencies, manually starting - # it works fine thought - which docker && systemctl start docker - set -e - DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" dist-upgrade -y - if [[ -e /var/run/reboot-required ]]; then - reboot - fi - - export CR_PKG='docker-ce=18.06.0~ce~3-0~ubuntu' - export CR_PKG='docker.io=17.12.1-0ubuntu1' - - - DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install -y \ - curl \ - ca-certificates \ - ceph-common \ - cifs-utils \ - conntrack \ - e2fsprogs \ - ebtables \ - ethtool \ - glusterfs-client \ - iptables \ - jq \ - kmod \ - openssh-client \ - nfs-common \ - socat \ - util-linux \ - ${CR_PKG} \ - open-vm-tools \ - kubelet=1.9.2-00 \ - kubeadm=1.9.2-00 \ - - cp /etc/default/kubelet-overwrite /etc/default/kubelet - - systemctl enable --now docker - systemctl enable kubelet - - if [[ ! -x /usr/local/bin/health-monitor.sh ]]; then - curl -Lfo /usr/local/bin/health-monitor.sh \ - https://raw.githubusercontent.com/kubermatic/machine-controller/8b5b66e4910a6228dfaecccaa0a3b05ec4902f8e/pkg/userdata/scripts/health-monitor.sh - chmod +x /usr/local/bin/health-monitor.sh - fi - - if ! [[ -e /etc/kubernetes/pki/ca.crt ]]; then - kubeadm join \ - --token my-token \ - --discovery-token-ca-cert-hash sha256:6caecce9fedcb55d4953d61a27dc6997361a2f226ad86d7e6004dde7526fc4b1 \ - --ignore-preflight-errors=CRI \ - server:443 - fi - - systemctl enable --now --no-block kubelet-healthcheck.service - systemctl enable --now --no-block docker-healthcheck.service - -- path: "/opt/kubernetes.asc" - permissions: "0400" - content: | - -----BEGIN PGP PUBLIC KEY BLOCK----- - - mQENBFrBaNsBCADrF18KCbsZlo4NjAvVecTBCnp6WcBQJ5oSh7+E98jX9YznUCrN - rgmeCcCMUvTDRDxfTaDJybaHugfba43nqhkbNpJ47YXsIa+YL6eEE9emSmQtjrSW - IiY+2YJYwsDgsgckF3duqkb02OdBQlh6IbHPoXB6H//b1PgZYsomB+841XW1LSJP - YlYbIrWfwDfQvtkFQI90r6NknVTQlpqQh5GLNWNYqRNrGQPmsB+NrUYrkl1nUt1L - RGu+rCe4bSaSmNbwKMQKkROE4kTiB72DPk7zH4Lm0uo0YFFWG4qsMIuqEihJ/9KN - X8GYBr+tWgyLooLlsdK3l+4dVqd8cjkJM1ExABEBAAG0QEdvb2dsZSBDbG91ZCBQ - YWNrYWdlcyBBdXRvbWF0aWMgU2lnbmluZyBLZXkgPGdjLXRlYW1AZ29vZ2xlLmNv - bT6JAT4EEwECACgFAlrBaNsCGy8FCQWjmoAGCwkIBwMCBhUIAgkKCwQWAgMBAh4B - AheAAAoJEGoDCyG6B/T78e8H/1WH2LN/nVNhm5TS1VYJG8B+IW8zS4BqyozxC9iJ - AJqZIVHXl8g8a/Hus8RfXR7cnYHcg8sjSaJfQhqO9RbKnffiuQgGrqwQxuC2jBa6 - M/QKzejTeP0Mgi67pyrLJNWrFI71RhritQZmzTZ2PoWxfv6b+Tv5v0rPaG+ut1J4 - 7pn+kYgtUaKdsJz1umi6HzK6AacDf0C0CksJdKG7MOWsZcB4xeOxJYuy6NuO6Kcd - Ez8/XyEUjIuIOlhYTd0hH8E/SEBbXXft7/VBQC5wNq40izPi+6WFK/e1O42DIpzQ - 749ogYQ1eodexPNhLzekKR3XhGrNXJ95r5KO10VrsLFNd8I= - =TKuP - -----END PGP PUBLIC KEY BLOCK----- - - path: "/opt/docker.asc" permissions: "0400" content: | @@ -206,7 +102,81 @@ write_files: =0YYh -----END PGP PUBLIC KEY BLOCK----- -- path: "/usr/local/bin/supervise.sh" +- path: "/opt/bin/setup" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + + # As we added some modules and don't want to reboot, restart the service + systemctl restart systemd-modules-load.service + sysctl --system + + apt-key add /opt/docker.asc + apt-get update + + # Make sure we always disable swap - Otherwise the kubelet won't start'. + systemctl mask swap.target + swapoff -a + export CR_PKG='docker.io=17.12.1-0ubuntu1' + + DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install -y \ + curl \ + ca-certificates \ + ceph-common \ + cifs-utils \ + conntrack \ + e2fsprogs \ + ebtables \ + ethtool \ + glusterfs-client \ + iptables \ + jq \ + kmod \ + openssh-client \ + nfs-common \ + socat \ + util-linux \ + ${CR_PKG} \ + open-vm-tools \ + ipvsadm + + # If something failed during package installation but docker got installed, we need to put it on hold + apt-mark hold docker.io || true + apt-mark hold docker-ce || true + if [[ -e /var/run/reboot-required ]]; then + reboot + fi + + #setup some common directories + mkdir -p /opt/bin/ + mkdir -p /var/lib/calico + mkdir -p /etc/kubernetes/manifests + mkdir -p /etc/cni/net.d + mkdir -p /opt/cni/bin + + # cni + if [ ! -f /opt/cni/bin/loopback ]; then + curl -L https://github.com/containernetworking/plugins/releases/download/v0.6.0/cni-plugins-amd64-v0.6.0.tgz | tar -xvzC /opt/cni/bin -f - + fi + # kubelet + if [ ! -f /opt/bin/kubelet ]; then + curl -Lfo /opt/bin/kubelet https://storage.googleapis.com/kubernetes-release/release/v1.11.3/bin/linux/amd64/kubelet + chmod +x /opt/bin/kubelet + fi + + if [[ ! -x /opt/bin/health-monitor.sh ]]; then + curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/8b5b66e4910a6228dfaecccaa0a3b05ec4902f8e/pkg/userdata/scripts/health-monitor.sh + chmod +x /opt/bin/health-monitor.sh + fi + + + systemctl enable --now docker + systemctl enable --now kubelet + systemctl enable --now --no-block kubelet-healthcheck.service + systemctl enable --now --no-block docker-healthcheck.service + +- path: "/opt/bin/supervise.sh" permissions: "0755" content: | #!/bin/bash @@ -215,25 +185,107 @@ write_files: sleep 1 done -- path: "/etc/default/kubelet-overwrite" +- path: "/etc/systemd/system/kubelet.service" content: | - KUBELET_DNS_ARGS= - KUBELET_EXTRA_ARGS=--authentication-token-webhook=true \ + [Unit] + After=docker.service + Requires=docker.service + + Description=kubelet: The Kubernetes Node Agent + Documentation=https://kubernetes.io/docs/home/ + + [Service] + Restart=always + StartLimitInterval=0 + RestartSec=10 + + Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" + + ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ + --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ + --kubeconfig=/etc/kubernetes/kubelet.conf \ + --pod-manifest-path=/etc/kubernetes/manifests \ + --allow-privileged=true \ + --network-plugin=cni \ + --cni-conf-dir=/etc/cni/net.d \ + --cni-bin-dir=/opt/cni/bin \ + --authorization-mode=Webhook \ + --client-ca-file=/etc/kubernetes/pki/ca.crt \ + --cadvisor-port=0 \ + --rotate-certificates=true \ + --cert-dir=/etc/kubernetes/pki \ + --authentication-token-webhook=true \ --cloud-provider=openstack \ --cloud-config=/etc/kubernetes/cloud-config \ --hostname-override=node1 \ --read-only-port=0 \ + --exit-on-lock-contention \ + --lock-file=/tmp/kubelet.lock \ + --anonymous-auth=false \ --protect-kernel-defaults=true \ - --resolv-conf=/run/systemd/resolve/resolv.conf \ --cluster-dns=10.10.10.10 \ --cluster-domain=cluster.local + + [Install] + WantedBy=multi-user.target -- path: "/etc/systemd/system/kubelet.service.d/20-extra.conf" - permissions: "0644" +- path: "/etc/systemd/system/kubelet.service.d/extras.conf" content: | [Service] - EnvironmentFile=/etc/default/kubelet + Environment="KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf" + +- path: "/etc/kubernetes/cloud-config" + content: | + custom + cloud + config +- path: "/etc/kubernetes/bootstrap-kubelet.conf" + content: | + apiVersion: v1 + clusters: + - cluster: + certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t + server: https://server:443 + name: "" + contexts: [] + current-context: "" + kind: Config + preferences: {} + users: + - name: "" + user: + token: my-token + + +- path: "/etc/kubernetes/pki/ca.crt" + content: | + -----BEGIN CERTIFICATE----- + MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV + BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG + A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 + DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 + NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG + cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv + c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B + AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS + R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT + ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk + JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 + mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW + caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G + A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt + hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB + MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES + MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv + bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h + U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao + eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 + UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD + 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n + sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF + kPe6XoSbiLm/kxk32T0= + -----END CERTIFICATE----- - path: "/etc/systemd/system/setup.service" permissions: "0644" @@ -248,7 +300,12 @@ write_files: [Service] Type=oneshot RemainAfterExit=true - ExecStart=/usr/local/bin/supervise.sh /usr/local/bin/setup + ExecStart=/opt/bin/supervise.sh /opt/bin/setup + +- path: "/etc/profile.d/opt-bin-path.sh" + permissions: "0644" + content: | + export PATH="/opt/bin:$PATH" - path: /etc/systemd/system/docker.service.d/10-storage.conf permissions: "0644" @@ -263,12 +320,13 @@ write_files: [Unit] Requires=kubelet.service After=kubelet.service - + [Service] - ExecStart=/usr/local/bin/health-monitor.sh kubelet - + ExecStart=/opt/bin/health-monitor.sh kubelet + [Install] WantedBy=multi-user.target + - path: /etc/systemd/system/docker-healthcheck.service permissions: "0644" @@ -276,12 +334,13 @@ write_files: [Unit] Requires=docker.service After=docker.service - + [Service] - ExecStart=/usr/local/bin/health-monitor.sh container-runtime - + ExecStart=/opt/bin/health-monitor.sh container-runtime + [Install] WantedBy=multi-user.target + runcmd: - systemctl enable --now setup.service diff --git a/pkg/userdata/ubuntu/testdata/openstack.golden b/pkg/userdata/ubuntu/testdata/openstack.golden new file mode 100644 index 000000000..747dc26eb --- /dev/null +++ b/pkg/userdata/ubuntu/testdata/openstack.golden @@ -0,0 +1,344 @@ +#cloud-config +hostname: node1 + +ssh_pwauth: no + +ssh_authorized_keys: +- "ssh-rsa AAABBB" + +write_files: +- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" + content: | + [Journal] + SystemMaxUse=5G + + +- path: "/etc/modules-load.d/k8s.conf" + content: | + ip_vs + ip_vs_rr + ip_vs_wrr + ip_vs_sh + nf_conntrack_ipv4 + + +- path: "/etc/sysctl.d/k8s.conf" + content: | + net.bridge.bridge-nf-call-ip6tables = 1 + net.bridge.bridge-nf-call-iptables = 1 + kernel.panic_on_oops = 1 + kernel.panic = 10 + net.ipv4.ip_forward = 1 + vm.overcommit_memory = 1 + + +- path: "/etc/apt/sources.list.d/docker.list" + permissions: "0644" + content: deb [arch=amd64] https://download.docker.com/linux/ubuntu bionic stable + +- path: "/opt/docker.asc" + permissions: "0400" + content: | + -----BEGIN PGP PUBLIC KEY BLOCK----- + + mQINBFit2ioBEADhWpZ8/wvZ6hUTiXOwQHXMAlaFHcPH9hAtr4F1y2+OYdbtMuth + lqqwp028AqyY+PRfVMtSYMbjuQuu5byyKR01BbqYhuS3jtqQmljZ/bJvXqnmiVXh + 38UuLa+z077PxyxQhu5BbqntTPQMfiyqEiU+BKbq2WmANUKQf+1AmZY/IruOXbnq + L4C1+gJ8vfmXQt99npCaxEjaNRVYfOS8QcixNzHUYnb6emjlANyEVlZzeqo7XKl7 + UrwV5inawTSzWNvtjEjj4nJL8NsLwscpLPQUhTQ+7BbQXAwAmeHCUTQIvvWXqw0N + cmhh4HgeQscQHYgOJjjDVfoY5MucvglbIgCqfzAHW9jxmRL4qbMZj+b1XoePEtht + ku4bIQN1X5P07fNWzlgaRL5Z4POXDDZTlIQ/El58j9kp4bnWRCJW0lya+f8ocodo + vZZ+Doi+fy4D5ZGrL4XEcIQP/Lv5uFyf+kQtl/94VFYVJOleAv8W92KdgDkhTcTD + G7c0tIkVEKNUq48b3aQ64NOZQW7fVjfoKwEZdOqPE72Pa45jrZzvUFxSpdiNk2tZ + XYukHjlxxEgBdC/J3cMMNRE1F4NCA3ApfV1Y7/hTeOnmDuDYwr9/obA8t016Yljj + q5rdkywPf4JF8mXUW5eCN1vAFHxeg9ZWemhBtQmGxXnw9M+z6hWwc6ahmwARAQAB + tCtEb2NrZXIgUmVsZWFzZSAoQ0UgZGViKSA8ZG9ja2VyQGRvY2tlci5jb20+iQI3 + BBMBCgAhBQJYrefAAhsvBQsJCAcDBRUKCQgLBRYCAwEAAh4BAheAAAoJEI2BgDwO + v82IsskP/iQZo68flDQmNvn8X5XTd6RRaUH33kXYXquT6NkHJciS7E2gTJmqvMqd + tI4mNYHCSEYxI5qrcYV5YqX9P6+Ko+vozo4nseUQLPH/ATQ4qL0Zok+1jkag3Lgk + jonyUf9bwtWxFp05HC3GMHPhhcUSexCxQLQvnFWXD2sWLKivHp2fT8QbRGeZ+d3m + 6fqcd5Fu7pxsqm0EUDK5NL+nPIgYhN+auTrhgzhK1CShfGccM/wfRlei9Utz6p9P + XRKIlWnXtT4qNGZNTN0tR+NLG/6Bqd8OYBaFAUcue/w1VW6JQ2VGYZHnZu9S8LMc + FYBa5Ig9PxwGQOgq6RDKDbV+PqTQT5EFMeR1mrjckk4DQJjbxeMZbiNMG5kGECA8 + g383P3elhn03WGbEEa4MNc3Z4+7c236QI3xWJfNPdUbXRaAwhy/6rTSFbzwKB0Jm + ebwzQfwjQY6f55MiI/RqDCyuPj3r3jyVRkK86pQKBAJwFHyqj9KaKXMZjfVnowLh + 9svIGfNbGHpucATqREvUHuQbNnqkCx8VVhtYkhDb9fEP2xBu5VvHbR+3nfVhMut5 + G34Ct5RS7Jt6LIfFdtcn8CaSas/l1HbiGeRgc70X/9aYx/V/CEJv0lIe8gP6uDoW + FPIZ7d6vH+Vro6xuWEGiuMaiznap2KhZmpkgfupyFmplh0s6knymuQINBFit2ioB + EADneL9S9m4vhU3blaRjVUUyJ7b/qTjcSylvCH5XUE6R2k+ckEZjfAMZPLpO+/tF + M2JIJMD4SifKuS3xck9KtZGCufGmcwiLQRzeHF7vJUKrLD5RTkNi23ydvWZgPjtx + Q+DTT1Zcn7BrQFY6FgnRoUVIxwtdw1bMY/89rsFgS5wwuMESd3Q2RYgb7EOFOpnu + w6da7WakWf4IhnF5nsNYGDVaIHzpiqCl+uTbf1epCjrOlIzkZ3Z3Yk5CM/TiFzPk + z2lLz89cpD8U+NtCsfagWWfjd2U3jDapgH+7nQnCEWpROtzaKHG6lA3pXdix5zG8 + eRc6/0IbUSWvfjKxLLPfNeCS2pCL3IeEI5nothEEYdQH6szpLog79xB9dVnJyKJb + VfxXnseoYqVrRz2VVbUI5Blwm6B40E3eGVfUQWiux54DspyVMMk41Mx7QJ3iynIa + 1N4ZAqVMAEruyXTRTxc9XW0tYhDMA/1GYvz0EmFpm8LzTHA6sFVtPm/ZlNCX6P1X + zJwrv7DSQKD6GGlBQUX+OeEJ8tTkkf8QTJSPUdh8P8YxDFS5EOGAvhhpMBYD42kQ + pqXjEC+XcycTvGI7impgv9PDY1RCC1zkBjKPa120rNhv/hkVk/YhuGoajoHyy4h7 + ZQopdcMtpN2dgmhEegny9JCSwxfQmQ0zK0g7m6SHiKMwjwARAQABiQQ+BBgBCAAJ + BQJYrdoqAhsCAikJEI2BgDwOv82IwV0gBBkBCAAGBQJYrdoqAAoJEH6gqcPyc/zY + 1WAP/2wJ+R0gE6qsce3rjaIz58PJmc8goKrir5hnElWhPgbq7cYIsW5qiFyLhkdp + YcMmhD9mRiPpQn6Ya2w3e3B8zfIVKipbMBnke/ytZ9M7qHmDCcjoiSmwEXN3wKYI + mD9VHONsl/CG1rU9Isw1jtB5g1YxuBA7M/m36XN6x2u+NtNMDB9P56yc4gfsZVES + KA9v+yY2/l45L8d/WUkUi0YXomn6hyBGI7JrBLq0CX37GEYP6O9rrKipfz73XfO7 + JIGzOKZlljb/D9RX/g7nRbCn+3EtH7xnk+TK/50euEKw8SMUg147sJTcpQmv6UzZ + cM4JgL0HbHVCojV4C/plELwMddALOFeYQzTif6sMRPf+3DSj8frbInjChC3yOLy0 + 6br92KFom17EIj2CAcoeq7UPhi2oouYBwPxh5ytdehJkoo+sN7RIWua6P2WSmon5 + U888cSylXC0+ADFdgLX9K2zrDVYUG1vo8CX0vzxFBaHwN6Px26fhIT1/hYUHQR1z + VfNDcyQmXqkOnZvvoMfz/Q0s9BhFJ/zU6AgQbIZE/hm1spsfgvtsD1frZfygXJ9f + irP+MSAI80xHSf91qSRZOj4Pl3ZJNbq4yYxv0b1pkMqeGdjdCYhLU+LZ4wbQmpCk + SVe2prlLureigXtmZfkqevRz7FrIZiu9ky8wnCAPwC7/zmS18rgP/17bOtL4/iIz + QhxAAoAMWVrGyJivSkjhSGx1uCojsWfsTAm11P7jsruIL61ZzMUVE2aM3Pmj5G+W + 9AcZ58Em+1WsVnAXdUR//bMmhyr8wL/G1YO1V3JEJTRdxsSxdYa4deGBBY/Adpsw + 24jxhOJR+lsJpqIUeb999+R8euDhRHG9eFO7DRu6weatUJ6suupoDTRWtr/4yGqe + dKxV3qQhNLSnaAzqW/1nA3iUB4k7kCaKZxhdhDbClf9P37qaRW467BLCVO/coL3y + Vm50dwdrNtKpMBh3ZpbB1uJvgi9mXtyBOMJ3v8RZeDzFiG8HdCtg9RvIt/AIFoHR + H3S+U79NT6i0KPzLImDfs8T7RlpyuMc4Ufs8ggyg9v3Ae6cN3eQyxcK3w0cbBwsh + /nQNfsA6uu+9H7NhbehBMhYnpNZyrHzCmzyXkauwRAqoCbGCNykTRwsur9gS41TQ + M8ssD1jFheOJf3hODnkKU+HKjvMROl1DK7zdmLdNzA1cvtZH/nCC9KPj1z8QC47S + xx+dTZSx4ONAhwbS/LN3PoKtn8LPjY9NP9uDWI+TWYquS2U+KHDrBDlsgozDbs/O + jCxcpDzNmXpWQHEtHU7649OXHP7UeNST1mCUCH5qdank0V1iejF6/CfTFU4MfcrG + YT90qFF93M3v01BbxP+EIY2/9tiIPbrd + =0YYh + -----END PGP PUBLIC KEY BLOCK----- + +- path: "/opt/bin/setup" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + + # As we added some modules and don't want to reboot, restart the service + systemctl restart systemd-modules-load.service + sysctl --system + + apt-key add /opt/docker.asc + apt-get update + + # Make sure we always disable swap - Otherwise the kubelet won't start'. + systemctl mask swap.target + swapoff -a + export CR_PKG='docker.io=17.12.1-0ubuntu1' + + DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install -y \ + curl \ + ca-certificates \ + ceph-common \ + cifs-utils \ + conntrack \ + e2fsprogs \ + ebtables \ + ethtool \ + glusterfs-client \ + iptables \ + jq \ + kmod \ + openssh-client \ + nfs-common \ + socat \ + util-linux \ + ${CR_PKG} \ + open-vm-tools \ + ipvsadm + + # If something failed during package installation but docker got installed, we need to put it on hold + apt-mark hold docker.io || true + apt-mark hold docker-ce || true + if [[ -e /var/run/reboot-required ]]; then + reboot + fi + + #setup some common directories + mkdir -p /opt/bin/ + mkdir -p /var/lib/calico + mkdir -p /etc/kubernetes/manifests + mkdir -p /etc/cni/net.d + mkdir -p /opt/cni/bin + + # cni + if [ ! -f /opt/cni/bin/loopback ]; then + curl -L https://github.com/containernetworking/plugins/releases/download/v0.6.0/cni-plugins-amd64-v0.6.0.tgz | tar -xvzC /opt/cni/bin -f - + fi + # kubelet + if [ ! -f /opt/bin/kubelet ]; then + curl -Lfo /opt/bin/kubelet https://storage.googleapis.com/kubernetes-release/release/v1.11.3/bin/linux/amd64/kubelet + chmod +x /opt/bin/kubelet + fi + + if [[ ! -x /opt/bin/health-monitor.sh ]]; then + curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/8b5b66e4910a6228dfaecccaa0a3b05ec4902f8e/pkg/userdata/scripts/health-monitor.sh + chmod +x /opt/bin/health-monitor.sh + fi + + + systemctl enable --now docker + systemctl enable --now kubelet + systemctl enable --now --no-block kubelet-healthcheck.service + systemctl enable --now --no-block docker-healthcheck.service + +- path: "/opt/bin/supervise.sh" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + while ! "$@"; do + sleep 1 + done + +- path: "/etc/systemd/system/kubelet.service" + content: | + [Unit] + After=docker.service + Requires=docker.service + + Description=kubelet: The Kubernetes Node Agent + Documentation=https://kubernetes.io/docs/home/ + + [Service] + Restart=always + StartLimitInterval=0 + RestartSec=10 + + Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" + + ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ + --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ + --kubeconfig=/etc/kubernetes/kubelet.conf \ + --pod-manifest-path=/etc/kubernetes/manifests \ + --allow-privileged=true \ + --network-plugin=cni \ + --cni-conf-dir=/etc/cni/net.d \ + --cni-bin-dir=/opt/cni/bin \ + --authorization-mode=Webhook \ + --client-ca-file=/etc/kubernetes/pki/ca.crt \ + --cadvisor-port=0 \ + --rotate-certificates=true \ + --cert-dir=/etc/kubernetes/pki \ + --authentication-token-webhook=true \ + --cloud-provider=openstack \ + --cloud-config=/etc/kubernetes/cloud-config \ + --hostname-override=node1 \ + --read-only-port=0 \ + --exit-on-lock-contention \ + --lock-file=/tmp/kubelet.lock \ + --anonymous-auth=false \ + --protect-kernel-defaults=true \ + --cluster-dns=10.10.10.10,10.10.10.11,10.10.10.12 \ + --cluster-domain=cluster.local + + [Install] + WantedBy=multi-user.target + +- path: "/etc/systemd/system/kubelet.service.d/extras.conf" + content: | + [Service] + Environment="KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf" + +- path: "/etc/kubernetes/cloud-config" + content: | + {openstack-config:true} + +- path: "/etc/kubernetes/bootstrap-kubelet.conf" + content: | + apiVersion: v1 + clusters: + - cluster: + certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t + server: https://server:443 + name: "" + contexts: [] + current-context: "" + kind: Config + preferences: {} + users: + - name: "" + user: + token: my-token + + +- path: "/etc/kubernetes/pki/ca.crt" + content: | + -----BEGIN CERTIFICATE----- + MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV + BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG + A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 + DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 + NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG + cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv + c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B + AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS + R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT + ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk + JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 + mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW + caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G + A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt + hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB + MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES + MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv + bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h + U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao + eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 + UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD + 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n + sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF + kPe6XoSbiLm/kxk32T0= + -----END CERTIFICATE----- + +- path: "/etc/systemd/system/setup.service" + permissions: "0644" + content: | + [Install] + WantedBy=multi-user.target + + [Unit] + Requires=network-online.target + After=network-online.target + + [Service] + Type=oneshot + RemainAfterExit=true + ExecStart=/opt/bin/supervise.sh /opt/bin/setup + +- path: "/etc/profile.d/opt-bin-path.sh" + permissions: "0644" + content: | + export PATH="/opt/bin:$PATH" + +- path: /etc/systemd/system/docker.service.d/10-storage.conf + permissions: "0644" + content: | + [Service] + ExecStart= + ExecStart=/usr/bin/dockerd -H fd:// --storage-driver=overlay2 + +- path: /etc/systemd/system/kubelet-healthcheck.service + permissions: "0644" + content: | + [Unit] + Requires=kubelet.service + After=kubelet.service + + [Service] + ExecStart=/opt/bin/health-monitor.sh kubelet + + [Install] + WantedBy=multi-user.target + + +- path: /etc/systemd/system/docker-healthcheck.service + permissions: "0644" + content: | + [Unit] + Requires=docker.service + After=docker.service + + [Service] + ExecStart=/opt/bin/health-monitor.sh container-runtime + + [Install] + WantedBy=multi-user.target + + +runcmd: +- systemctl enable --now setup.service diff --git a/pkg/userdata/ubuntu/testdata/version-1.10.10.golden b/pkg/userdata/ubuntu/testdata/version-1.10.10.golden new file mode 100644 index 000000000..fdfe9827e --- /dev/null +++ b/pkg/userdata/ubuntu/testdata/version-1.10.10.golden @@ -0,0 +1,342 @@ +#cloud-config +hostname: node1 + +ssh_pwauth: no + +ssh_authorized_keys: +- "ssh-rsa AAABBB" + +write_files: +- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" + content: | + [Journal] + SystemMaxUse=5G + + +- path: "/etc/modules-load.d/k8s.conf" + content: | + ip_vs + ip_vs_rr + ip_vs_wrr + ip_vs_sh + nf_conntrack_ipv4 + + +- path: "/etc/sysctl.d/k8s.conf" + content: | + net.bridge.bridge-nf-call-ip6tables = 1 + net.bridge.bridge-nf-call-iptables = 1 + kernel.panic_on_oops = 1 + kernel.panic = 10 + net.ipv4.ip_forward = 1 + vm.overcommit_memory = 1 + + +- path: "/etc/apt/sources.list.d/docker.list" + permissions: "0644" + content: deb [arch=amd64] https://download.docker.com/linux/ubuntu bionic stable + +- path: "/opt/docker.asc" + permissions: "0400" + content: | + -----BEGIN PGP PUBLIC KEY BLOCK----- + + mQINBFit2ioBEADhWpZ8/wvZ6hUTiXOwQHXMAlaFHcPH9hAtr4F1y2+OYdbtMuth + lqqwp028AqyY+PRfVMtSYMbjuQuu5byyKR01BbqYhuS3jtqQmljZ/bJvXqnmiVXh + 38UuLa+z077PxyxQhu5BbqntTPQMfiyqEiU+BKbq2WmANUKQf+1AmZY/IruOXbnq + L4C1+gJ8vfmXQt99npCaxEjaNRVYfOS8QcixNzHUYnb6emjlANyEVlZzeqo7XKl7 + UrwV5inawTSzWNvtjEjj4nJL8NsLwscpLPQUhTQ+7BbQXAwAmeHCUTQIvvWXqw0N + cmhh4HgeQscQHYgOJjjDVfoY5MucvglbIgCqfzAHW9jxmRL4qbMZj+b1XoePEtht + ku4bIQN1X5P07fNWzlgaRL5Z4POXDDZTlIQ/El58j9kp4bnWRCJW0lya+f8ocodo + vZZ+Doi+fy4D5ZGrL4XEcIQP/Lv5uFyf+kQtl/94VFYVJOleAv8W92KdgDkhTcTD + G7c0tIkVEKNUq48b3aQ64NOZQW7fVjfoKwEZdOqPE72Pa45jrZzvUFxSpdiNk2tZ + XYukHjlxxEgBdC/J3cMMNRE1F4NCA3ApfV1Y7/hTeOnmDuDYwr9/obA8t016Yljj + q5rdkywPf4JF8mXUW5eCN1vAFHxeg9ZWemhBtQmGxXnw9M+z6hWwc6ahmwARAQAB + tCtEb2NrZXIgUmVsZWFzZSAoQ0UgZGViKSA8ZG9ja2VyQGRvY2tlci5jb20+iQI3 + BBMBCgAhBQJYrefAAhsvBQsJCAcDBRUKCQgLBRYCAwEAAh4BAheAAAoJEI2BgDwO + v82IsskP/iQZo68flDQmNvn8X5XTd6RRaUH33kXYXquT6NkHJciS7E2gTJmqvMqd + tI4mNYHCSEYxI5qrcYV5YqX9P6+Ko+vozo4nseUQLPH/ATQ4qL0Zok+1jkag3Lgk + jonyUf9bwtWxFp05HC3GMHPhhcUSexCxQLQvnFWXD2sWLKivHp2fT8QbRGeZ+d3m + 6fqcd5Fu7pxsqm0EUDK5NL+nPIgYhN+auTrhgzhK1CShfGccM/wfRlei9Utz6p9P + XRKIlWnXtT4qNGZNTN0tR+NLG/6Bqd8OYBaFAUcue/w1VW6JQ2VGYZHnZu9S8LMc + FYBa5Ig9PxwGQOgq6RDKDbV+PqTQT5EFMeR1mrjckk4DQJjbxeMZbiNMG5kGECA8 + g383P3elhn03WGbEEa4MNc3Z4+7c236QI3xWJfNPdUbXRaAwhy/6rTSFbzwKB0Jm + ebwzQfwjQY6f55MiI/RqDCyuPj3r3jyVRkK86pQKBAJwFHyqj9KaKXMZjfVnowLh + 9svIGfNbGHpucATqREvUHuQbNnqkCx8VVhtYkhDb9fEP2xBu5VvHbR+3nfVhMut5 + G34Ct5RS7Jt6LIfFdtcn8CaSas/l1HbiGeRgc70X/9aYx/V/CEJv0lIe8gP6uDoW + FPIZ7d6vH+Vro6xuWEGiuMaiznap2KhZmpkgfupyFmplh0s6knymuQINBFit2ioB + EADneL9S9m4vhU3blaRjVUUyJ7b/qTjcSylvCH5XUE6R2k+ckEZjfAMZPLpO+/tF + M2JIJMD4SifKuS3xck9KtZGCufGmcwiLQRzeHF7vJUKrLD5RTkNi23ydvWZgPjtx + Q+DTT1Zcn7BrQFY6FgnRoUVIxwtdw1bMY/89rsFgS5wwuMESd3Q2RYgb7EOFOpnu + w6da7WakWf4IhnF5nsNYGDVaIHzpiqCl+uTbf1epCjrOlIzkZ3Z3Yk5CM/TiFzPk + z2lLz89cpD8U+NtCsfagWWfjd2U3jDapgH+7nQnCEWpROtzaKHG6lA3pXdix5zG8 + eRc6/0IbUSWvfjKxLLPfNeCS2pCL3IeEI5nothEEYdQH6szpLog79xB9dVnJyKJb + VfxXnseoYqVrRz2VVbUI5Blwm6B40E3eGVfUQWiux54DspyVMMk41Mx7QJ3iynIa + 1N4ZAqVMAEruyXTRTxc9XW0tYhDMA/1GYvz0EmFpm8LzTHA6sFVtPm/ZlNCX6P1X + zJwrv7DSQKD6GGlBQUX+OeEJ8tTkkf8QTJSPUdh8P8YxDFS5EOGAvhhpMBYD42kQ + pqXjEC+XcycTvGI7impgv9PDY1RCC1zkBjKPa120rNhv/hkVk/YhuGoajoHyy4h7 + ZQopdcMtpN2dgmhEegny9JCSwxfQmQ0zK0g7m6SHiKMwjwARAQABiQQ+BBgBCAAJ + BQJYrdoqAhsCAikJEI2BgDwOv82IwV0gBBkBCAAGBQJYrdoqAAoJEH6gqcPyc/zY + 1WAP/2wJ+R0gE6qsce3rjaIz58PJmc8goKrir5hnElWhPgbq7cYIsW5qiFyLhkdp + YcMmhD9mRiPpQn6Ya2w3e3B8zfIVKipbMBnke/ytZ9M7qHmDCcjoiSmwEXN3wKYI + mD9VHONsl/CG1rU9Isw1jtB5g1YxuBA7M/m36XN6x2u+NtNMDB9P56yc4gfsZVES + KA9v+yY2/l45L8d/WUkUi0YXomn6hyBGI7JrBLq0CX37GEYP6O9rrKipfz73XfO7 + JIGzOKZlljb/D9RX/g7nRbCn+3EtH7xnk+TK/50euEKw8SMUg147sJTcpQmv6UzZ + cM4JgL0HbHVCojV4C/plELwMddALOFeYQzTif6sMRPf+3DSj8frbInjChC3yOLy0 + 6br92KFom17EIj2CAcoeq7UPhi2oouYBwPxh5ytdehJkoo+sN7RIWua6P2WSmon5 + U888cSylXC0+ADFdgLX9K2zrDVYUG1vo8CX0vzxFBaHwN6Px26fhIT1/hYUHQR1z + VfNDcyQmXqkOnZvvoMfz/Q0s9BhFJ/zU6AgQbIZE/hm1spsfgvtsD1frZfygXJ9f + irP+MSAI80xHSf91qSRZOj4Pl3ZJNbq4yYxv0b1pkMqeGdjdCYhLU+LZ4wbQmpCk + SVe2prlLureigXtmZfkqevRz7FrIZiu9ky8wnCAPwC7/zmS18rgP/17bOtL4/iIz + QhxAAoAMWVrGyJivSkjhSGx1uCojsWfsTAm11P7jsruIL61ZzMUVE2aM3Pmj5G+W + 9AcZ58Em+1WsVnAXdUR//bMmhyr8wL/G1YO1V3JEJTRdxsSxdYa4deGBBY/Adpsw + 24jxhOJR+lsJpqIUeb999+R8euDhRHG9eFO7DRu6weatUJ6suupoDTRWtr/4yGqe + dKxV3qQhNLSnaAzqW/1nA3iUB4k7kCaKZxhdhDbClf9P37qaRW467BLCVO/coL3y + Vm50dwdrNtKpMBh3ZpbB1uJvgi9mXtyBOMJ3v8RZeDzFiG8HdCtg9RvIt/AIFoHR + H3S+U79NT6i0KPzLImDfs8T7RlpyuMc4Ufs8ggyg9v3Ae6cN3eQyxcK3w0cbBwsh + /nQNfsA6uu+9H7NhbehBMhYnpNZyrHzCmzyXkauwRAqoCbGCNykTRwsur9gS41TQ + M8ssD1jFheOJf3hODnkKU+HKjvMROl1DK7zdmLdNzA1cvtZH/nCC9KPj1z8QC47S + xx+dTZSx4ONAhwbS/LN3PoKtn8LPjY9NP9uDWI+TWYquS2U+KHDrBDlsgozDbs/O + jCxcpDzNmXpWQHEtHU7649OXHP7UeNST1mCUCH5qdank0V1iejF6/CfTFU4MfcrG + YT90qFF93M3v01BbxP+EIY2/9tiIPbrd + =0YYh + -----END PGP PUBLIC KEY BLOCK----- + +- path: "/opt/bin/setup" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + + # As we added some modules and don't want to reboot, restart the service + systemctl restart systemd-modules-load.service + sysctl --system + + apt-key add /opt/docker.asc + apt-get update + + # Make sure we always disable swap - Otherwise the kubelet won't start'. + systemctl mask swap.target + swapoff -a + export CR_PKG='docker.io=17.12.1-0ubuntu1' + + DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install -y \ + curl \ + ca-certificates \ + ceph-common \ + cifs-utils \ + conntrack \ + e2fsprogs \ + ebtables \ + ethtool \ + glusterfs-client \ + iptables \ + jq \ + kmod \ + openssh-client \ + nfs-common \ + socat \ + util-linux \ + ${CR_PKG} \ + open-vm-tools \ + ipvsadm + + # If something failed during package installation but docker got installed, we need to put it on hold + apt-mark hold docker.io || true + apt-mark hold docker-ce || true + if [[ -e /var/run/reboot-required ]]; then + reboot + fi + + #setup some common directories + mkdir -p /opt/bin/ + mkdir -p /var/lib/calico + mkdir -p /etc/kubernetes/manifests + mkdir -p /etc/cni/net.d + mkdir -p /opt/cni/bin + + # cni + if [ ! -f /opt/cni/bin/loopback ]; then + curl -L https://github.com/containernetworking/plugins/releases/download/v0.6.0/cni-plugins-amd64-v0.6.0.tgz | tar -xvzC /opt/cni/bin -f - + fi + # kubelet + if [ ! -f /opt/bin/kubelet ]; then + curl -Lfo /opt/bin/kubelet https://storage.googleapis.com/kubernetes-release/release/v1.10.10/bin/linux/amd64/kubelet + chmod +x /opt/bin/kubelet + fi + + if [[ ! -x /opt/bin/health-monitor.sh ]]; then + curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/8b5b66e4910a6228dfaecccaa0a3b05ec4902f8e/pkg/userdata/scripts/health-monitor.sh + chmod +x /opt/bin/health-monitor.sh + fi + + + systemctl enable --now docker + systemctl enable --now kubelet + systemctl enable --now --no-block kubelet-healthcheck.service + systemctl enable --now --no-block docker-healthcheck.service + +- path: "/opt/bin/supervise.sh" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + while ! "$@"; do + sleep 1 + done + +- path: "/etc/systemd/system/kubelet.service" + content: | + [Unit] + After=docker.service + Requires=docker.service + + Description=kubelet: The Kubernetes Node Agent + Documentation=https://kubernetes.io/docs/home/ + + [Service] + Restart=always + StartLimitInterval=0 + RestartSec=10 + + Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" + + ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ + --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ + --kubeconfig=/etc/kubernetes/kubelet.conf \ + --pod-manifest-path=/etc/kubernetes/manifests \ + --allow-privileged=true \ + --network-plugin=cni \ + --cni-conf-dir=/etc/cni/net.d \ + --cni-bin-dir=/opt/cni/bin \ + --authorization-mode=Webhook \ + --client-ca-file=/etc/kubernetes/pki/ca.crt \ + --cadvisor-port=0 \ + --rotate-certificates=true \ + --cert-dir=/etc/kubernetes/pki \ + --authentication-token-webhook=true \ + --hostname-override=node1 \ + --read-only-port=0 \ + --exit-on-lock-contention \ + --lock-file=/tmp/kubelet.lock \ + --anonymous-auth=false \ + --protect-kernel-defaults=true \ + --cluster-dns=10.10.10.10 \ + --cluster-domain=cluster.local + + [Install] + WantedBy=multi-user.target + +- path: "/etc/systemd/system/kubelet.service.d/extras.conf" + content: | + [Service] + Environment="KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf" + +- path: "/etc/kubernetes/cloud-config" + content: | + + +- path: "/etc/kubernetes/bootstrap-kubelet.conf" + content: | + apiVersion: v1 + clusters: + - cluster: + certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t + server: https://server:443 + name: "" + contexts: [] + current-context: "" + kind: Config + preferences: {} + users: + - name: "" + user: + token: my-token + + +- path: "/etc/kubernetes/pki/ca.crt" + content: | + -----BEGIN CERTIFICATE----- + MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV + BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG + A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 + DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 + NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG + cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv + c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B + AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS + R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT + ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk + JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 + mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW + caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G + A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt + hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB + MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES + MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv + bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h + U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao + eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 + UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD + 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n + sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF + kPe6XoSbiLm/kxk32T0= + -----END CERTIFICATE----- + +- path: "/etc/systemd/system/setup.service" + permissions: "0644" + content: | + [Install] + WantedBy=multi-user.target + + [Unit] + Requires=network-online.target + After=network-online.target + + [Service] + Type=oneshot + RemainAfterExit=true + ExecStart=/opt/bin/supervise.sh /opt/bin/setup + +- path: "/etc/profile.d/opt-bin-path.sh" + permissions: "0644" + content: | + export PATH="/opt/bin:$PATH" + +- path: /etc/systemd/system/docker.service.d/10-storage.conf + permissions: "0644" + content: | + [Service] + ExecStart= + ExecStart=/usr/bin/dockerd -H fd:// --storage-driver=overlay2 + +- path: /etc/systemd/system/kubelet-healthcheck.service + permissions: "0644" + content: | + [Unit] + Requires=kubelet.service + After=kubelet.service + + [Service] + ExecStart=/opt/bin/health-monitor.sh kubelet + + [Install] + WantedBy=multi-user.target + + +- path: /etc/systemd/system/docker-healthcheck.service + permissions: "0644" + content: | + [Unit] + Requires=docker.service + After=docker.service + + [Service] + ExecStart=/opt/bin/health-monitor.sh container-runtime + + [Install] + WantedBy=multi-user.target + + +runcmd: +- systemctl enable --now setup.service diff --git a/pkg/userdata/ubuntu/testdata/version-1.11.3.golden b/pkg/userdata/ubuntu/testdata/version-1.11.3.golden new file mode 100644 index 000000000..69439fc7b --- /dev/null +++ b/pkg/userdata/ubuntu/testdata/version-1.11.3.golden @@ -0,0 +1,342 @@ +#cloud-config +hostname: node1 + +ssh_pwauth: no + +ssh_authorized_keys: +- "ssh-rsa AAABBB" + +write_files: +- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" + content: | + [Journal] + SystemMaxUse=5G + + +- path: "/etc/modules-load.d/k8s.conf" + content: | + ip_vs + ip_vs_rr + ip_vs_wrr + ip_vs_sh + nf_conntrack_ipv4 + + +- path: "/etc/sysctl.d/k8s.conf" + content: | + net.bridge.bridge-nf-call-ip6tables = 1 + net.bridge.bridge-nf-call-iptables = 1 + kernel.panic_on_oops = 1 + kernel.panic = 10 + net.ipv4.ip_forward = 1 + vm.overcommit_memory = 1 + + +- path: "/etc/apt/sources.list.d/docker.list" + permissions: "0644" + content: deb [arch=amd64] https://download.docker.com/linux/ubuntu bionic stable + +- path: "/opt/docker.asc" + permissions: "0400" + content: | + -----BEGIN PGP PUBLIC KEY BLOCK----- + + mQINBFit2ioBEADhWpZ8/wvZ6hUTiXOwQHXMAlaFHcPH9hAtr4F1y2+OYdbtMuth + lqqwp028AqyY+PRfVMtSYMbjuQuu5byyKR01BbqYhuS3jtqQmljZ/bJvXqnmiVXh + 38UuLa+z077PxyxQhu5BbqntTPQMfiyqEiU+BKbq2WmANUKQf+1AmZY/IruOXbnq + L4C1+gJ8vfmXQt99npCaxEjaNRVYfOS8QcixNzHUYnb6emjlANyEVlZzeqo7XKl7 + UrwV5inawTSzWNvtjEjj4nJL8NsLwscpLPQUhTQ+7BbQXAwAmeHCUTQIvvWXqw0N + cmhh4HgeQscQHYgOJjjDVfoY5MucvglbIgCqfzAHW9jxmRL4qbMZj+b1XoePEtht + ku4bIQN1X5P07fNWzlgaRL5Z4POXDDZTlIQ/El58j9kp4bnWRCJW0lya+f8ocodo + vZZ+Doi+fy4D5ZGrL4XEcIQP/Lv5uFyf+kQtl/94VFYVJOleAv8W92KdgDkhTcTD + G7c0tIkVEKNUq48b3aQ64NOZQW7fVjfoKwEZdOqPE72Pa45jrZzvUFxSpdiNk2tZ + XYukHjlxxEgBdC/J3cMMNRE1F4NCA3ApfV1Y7/hTeOnmDuDYwr9/obA8t016Yljj + q5rdkywPf4JF8mXUW5eCN1vAFHxeg9ZWemhBtQmGxXnw9M+z6hWwc6ahmwARAQAB + tCtEb2NrZXIgUmVsZWFzZSAoQ0UgZGViKSA8ZG9ja2VyQGRvY2tlci5jb20+iQI3 + BBMBCgAhBQJYrefAAhsvBQsJCAcDBRUKCQgLBRYCAwEAAh4BAheAAAoJEI2BgDwO + v82IsskP/iQZo68flDQmNvn8X5XTd6RRaUH33kXYXquT6NkHJciS7E2gTJmqvMqd + tI4mNYHCSEYxI5qrcYV5YqX9P6+Ko+vozo4nseUQLPH/ATQ4qL0Zok+1jkag3Lgk + jonyUf9bwtWxFp05HC3GMHPhhcUSexCxQLQvnFWXD2sWLKivHp2fT8QbRGeZ+d3m + 6fqcd5Fu7pxsqm0EUDK5NL+nPIgYhN+auTrhgzhK1CShfGccM/wfRlei9Utz6p9P + XRKIlWnXtT4qNGZNTN0tR+NLG/6Bqd8OYBaFAUcue/w1VW6JQ2VGYZHnZu9S8LMc + FYBa5Ig9PxwGQOgq6RDKDbV+PqTQT5EFMeR1mrjckk4DQJjbxeMZbiNMG5kGECA8 + g383P3elhn03WGbEEa4MNc3Z4+7c236QI3xWJfNPdUbXRaAwhy/6rTSFbzwKB0Jm + ebwzQfwjQY6f55MiI/RqDCyuPj3r3jyVRkK86pQKBAJwFHyqj9KaKXMZjfVnowLh + 9svIGfNbGHpucATqREvUHuQbNnqkCx8VVhtYkhDb9fEP2xBu5VvHbR+3nfVhMut5 + G34Ct5RS7Jt6LIfFdtcn8CaSas/l1HbiGeRgc70X/9aYx/V/CEJv0lIe8gP6uDoW + FPIZ7d6vH+Vro6xuWEGiuMaiznap2KhZmpkgfupyFmplh0s6knymuQINBFit2ioB + EADneL9S9m4vhU3blaRjVUUyJ7b/qTjcSylvCH5XUE6R2k+ckEZjfAMZPLpO+/tF + M2JIJMD4SifKuS3xck9KtZGCufGmcwiLQRzeHF7vJUKrLD5RTkNi23ydvWZgPjtx + Q+DTT1Zcn7BrQFY6FgnRoUVIxwtdw1bMY/89rsFgS5wwuMESd3Q2RYgb7EOFOpnu + w6da7WakWf4IhnF5nsNYGDVaIHzpiqCl+uTbf1epCjrOlIzkZ3Z3Yk5CM/TiFzPk + z2lLz89cpD8U+NtCsfagWWfjd2U3jDapgH+7nQnCEWpROtzaKHG6lA3pXdix5zG8 + eRc6/0IbUSWvfjKxLLPfNeCS2pCL3IeEI5nothEEYdQH6szpLog79xB9dVnJyKJb + VfxXnseoYqVrRz2VVbUI5Blwm6B40E3eGVfUQWiux54DspyVMMk41Mx7QJ3iynIa + 1N4ZAqVMAEruyXTRTxc9XW0tYhDMA/1GYvz0EmFpm8LzTHA6sFVtPm/ZlNCX6P1X + zJwrv7DSQKD6GGlBQUX+OeEJ8tTkkf8QTJSPUdh8P8YxDFS5EOGAvhhpMBYD42kQ + pqXjEC+XcycTvGI7impgv9PDY1RCC1zkBjKPa120rNhv/hkVk/YhuGoajoHyy4h7 + ZQopdcMtpN2dgmhEegny9JCSwxfQmQ0zK0g7m6SHiKMwjwARAQABiQQ+BBgBCAAJ + BQJYrdoqAhsCAikJEI2BgDwOv82IwV0gBBkBCAAGBQJYrdoqAAoJEH6gqcPyc/zY + 1WAP/2wJ+R0gE6qsce3rjaIz58PJmc8goKrir5hnElWhPgbq7cYIsW5qiFyLhkdp + YcMmhD9mRiPpQn6Ya2w3e3B8zfIVKipbMBnke/ytZ9M7qHmDCcjoiSmwEXN3wKYI + mD9VHONsl/CG1rU9Isw1jtB5g1YxuBA7M/m36XN6x2u+NtNMDB9P56yc4gfsZVES + KA9v+yY2/l45L8d/WUkUi0YXomn6hyBGI7JrBLq0CX37GEYP6O9rrKipfz73XfO7 + JIGzOKZlljb/D9RX/g7nRbCn+3EtH7xnk+TK/50euEKw8SMUg147sJTcpQmv6UzZ + cM4JgL0HbHVCojV4C/plELwMddALOFeYQzTif6sMRPf+3DSj8frbInjChC3yOLy0 + 6br92KFom17EIj2CAcoeq7UPhi2oouYBwPxh5ytdehJkoo+sN7RIWua6P2WSmon5 + U888cSylXC0+ADFdgLX9K2zrDVYUG1vo8CX0vzxFBaHwN6Px26fhIT1/hYUHQR1z + VfNDcyQmXqkOnZvvoMfz/Q0s9BhFJ/zU6AgQbIZE/hm1spsfgvtsD1frZfygXJ9f + irP+MSAI80xHSf91qSRZOj4Pl3ZJNbq4yYxv0b1pkMqeGdjdCYhLU+LZ4wbQmpCk + SVe2prlLureigXtmZfkqevRz7FrIZiu9ky8wnCAPwC7/zmS18rgP/17bOtL4/iIz + QhxAAoAMWVrGyJivSkjhSGx1uCojsWfsTAm11P7jsruIL61ZzMUVE2aM3Pmj5G+W + 9AcZ58Em+1WsVnAXdUR//bMmhyr8wL/G1YO1V3JEJTRdxsSxdYa4deGBBY/Adpsw + 24jxhOJR+lsJpqIUeb999+R8euDhRHG9eFO7DRu6weatUJ6suupoDTRWtr/4yGqe + dKxV3qQhNLSnaAzqW/1nA3iUB4k7kCaKZxhdhDbClf9P37qaRW467BLCVO/coL3y + Vm50dwdrNtKpMBh3ZpbB1uJvgi9mXtyBOMJ3v8RZeDzFiG8HdCtg9RvIt/AIFoHR + H3S+U79NT6i0KPzLImDfs8T7RlpyuMc4Ufs8ggyg9v3Ae6cN3eQyxcK3w0cbBwsh + /nQNfsA6uu+9H7NhbehBMhYnpNZyrHzCmzyXkauwRAqoCbGCNykTRwsur9gS41TQ + M8ssD1jFheOJf3hODnkKU+HKjvMROl1DK7zdmLdNzA1cvtZH/nCC9KPj1z8QC47S + xx+dTZSx4ONAhwbS/LN3PoKtn8LPjY9NP9uDWI+TWYquS2U+KHDrBDlsgozDbs/O + jCxcpDzNmXpWQHEtHU7649OXHP7UeNST1mCUCH5qdank0V1iejF6/CfTFU4MfcrG + YT90qFF93M3v01BbxP+EIY2/9tiIPbrd + =0YYh + -----END PGP PUBLIC KEY BLOCK----- + +- path: "/opt/bin/setup" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + + # As we added some modules and don't want to reboot, restart the service + systemctl restart systemd-modules-load.service + sysctl --system + + apt-key add /opt/docker.asc + apt-get update + + # Make sure we always disable swap - Otherwise the kubelet won't start'. + systemctl mask swap.target + swapoff -a + export CR_PKG='docker.io=17.12.1-0ubuntu1' + + DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install -y \ + curl \ + ca-certificates \ + ceph-common \ + cifs-utils \ + conntrack \ + e2fsprogs \ + ebtables \ + ethtool \ + glusterfs-client \ + iptables \ + jq \ + kmod \ + openssh-client \ + nfs-common \ + socat \ + util-linux \ + ${CR_PKG} \ + open-vm-tools \ + ipvsadm + + # If something failed during package installation but docker got installed, we need to put it on hold + apt-mark hold docker.io || true + apt-mark hold docker-ce || true + if [[ -e /var/run/reboot-required ]]; then + reboot + fi + + #setup some common directories + mkdir -p /opt/bin/ + mkdir -p /var/lib/calico + mkdir -p /etc/kubernetes/manifests + mkdir -p /etc/cni/net.d + mkdir -p /opt/cni/bin + + # cni + if [ ! -f /opt/cni/bin/loopback ]; then + curl -L https://github.com/containernetworking/plugins/releases/download/v0.6.0/cni-plugins-amd64-v0.6.0.tgz | tar -xvzC /opt/cni/bin -f - + fi + # kubelet + if [ ! -f /opt/bin/kubelet ]; then + curl -Lfo /opt/bin/kubelet https://storage.googleapis.com/kubernetes-release/release/v1.11.3/bin/linux/amd64/kubelet + chmod +x /opt/bin/kubelet + fi + + if [[ ! -x /opt/bin/health-monitor.sh ]]; then + curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/8b5b66e4910a6228dfaecccaa0a3b05ec4902f8e/pkg/userdata/scripts/health-monitor.sh + chmod +x /opt/bin/health-monitor.sh + fi + + + systemctl enable --now docker + systemctl enable --now kubelet + systemctl enable --now --no-block kubelet-healthcheck.service + systemctl enable --now --no-block docker-healthcheck.service + +- path: "/opt/bin/supervise.sh" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + while ! "$@"; do + sleep 1 + done + +- path: "/etc/systemd/system/kubelet.service" + content: | + [Unit] + After=docker.service + Requires=docker.service + + Description=kubelet: The Kubernetes Node Agent + Documentation=https://kubernetes.io/docs/home/ + + [Service] + Restart=always + StartLimitInterval=0 + RestartSec=10 + + Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" + + ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ + --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ + --kubeconfig=/etc/kubernetes/kubelet.conf \ + --pod-manifest-path=/etc/kubernetes/manifests \ + --allow-privileged=true \ + --network-plugin=cni \ + --cni-conf-dir=/etc/cni/net.d \ + --cni-bin-dir=/opt/cni/bin \ + --authorization-mode=Webhook \ + --client-ca-file=/etc/kubernetes/pki/ca.crt \ + --cadvisor-port=0 \ + --rotate-certificates=true \ + --cert-dir=/etc/kubernetes/pki \ + --authentication-token-webhook=true \ + --hostname-override=node1 \ + --read-only-port=0 \ + --exit-on-lock-contention \ + --lock-file=/tmp/kubelet.lock \ + --anonymous-auth=false \ + --protect-kernel-defaults=true \ + --cluster-dns=10.10.10.10 \ + --cluster-domain=cluster.local + + [Install] + WantedBy=multi-user.target + +- path: "/etc/systemd/system/kubelet.service.d/extras.conf" + content: | + [Service] + Environment="KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf" + +- path: "/etc/kubernetes/cloud-config" + content: | + + +- path: "/etc/kubernetes/bootstrap-kubelet.conf" + content: | + apiVersion: v1 + clusters: + - cluster: + certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t + server: https://server:443 + name: "" + contexts: [] + current-context: "" + kind: Config + preferences: {} + users: + - name: "" + user: + token: my-token + + +- path: "/etc/kubernetes/pki/ca.crt" + content: | + -----BEGIN CERTIFICATE----- + MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV + BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG + A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 + DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 + NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG + cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv + c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B + AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS + R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT + ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk + JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 + mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW + caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G + A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt + hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB + MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES + MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv + bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h + U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao + eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 + UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD + 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n + sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF + kPe6XoSbiLm/kxk32T0= + -----END CERTIFICATE----- + +- path: "/etc/systemd/system/setup.service" + permissions: "0644" + content: | + [Install] + WantedBy=multi-user.target + + [Unit] + Requires=network-online.target + After=network-online.target + + [Service] + Type=oneshot + RemainAfterExit=true + ExecStart=/opt/bin/supervise.sh /opt/bin/setup + +- path: "/etc/profile.d/opt-bin-path.sh" + permissions: "0644" + content: | + export PATH="/opt/bin:$PATH" + +- path: /etc/systemd/system/docker.service.d/10-storage.conf + permissions: "0644" + content: | + [Service] + ExecStart= + ExecStart=/usr/bin/dockerd -H fd:// --storage-driver=overlay2 + +- path: /etc/systemd/system/kubelet-healthcheck.service + permissions: "0644" + content: | + [Unit] + Requires=kubelet.service + After=kubelet.service + + [Service] + ExecStart=/opt/bin/health-monitor.sh kubelet + + [Install] + WantedBy=multi-user.target + + +- path: /etc/systemd/system/docker-healthcheck.service + permissions: "0644" + content: | + [Unit] + Requires=docker.service + After=docker.service + + [Service] + ExecStart=/opt/bin/health-monitor.sh container-runtime + + [Install] + WantedBy=multi-user.target + + +runcmd: +- systemctl enable --now setup.service diff --git a/pkg/userdata/ubuntu/testdata/version-1.12.1.golden b/pkg/userdata/ubuntu/testdata/version-1.12.1.golden new file mode 100644 index 000000000..e8395348e --- /dev/null +++ b/pkg/userdata/ubuntu/testdata/version-1.12.1.golden @@ -0,0 +1,341 @@ +#cloud-config +hostname: node1 + +ssh_pwauth: no + +ssh_authorized_keys: +- "ssh-rsa AAABBB" + +write_files: +- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" + content: | + [Journal] + SystemMaxUse=5G + + +- path: "/etc/modules-load.d/k8s.conf" + content: | + ip_vs + ip_vs_rr + ip_vs_wrr + ip_vs_sh + nf_conntrack_ipv4 + + +- path: "/etc/sysctl.d/k8s.conf" + content: | + net.bridge.bridge-nf-call-ip6tables = 1 + net.bridge.bridge-nf-call-iptables = 1 + kernel.panic_on_oops = 1 + kernel.panic = 10 + net.ipv4.ip_forward = 1 + vm.overcommit_memory = 1 + + +- path: "/etc/apt/sources.list.d/docker.list" + permissions: "0644" + content: deb [arch=amd64] https://download.docker.com/linux/ubuntu bionic stable + +- path: "/opt/docker.asc" + permissions: "0400" + content: | + -----BEGIN PGP PUBLIC KEY BLOCK----- + + mQINBFit2ioBEADhWpZ8/wvZ6hUTiXOwQHXMAlaFHcPH9hAtr4F1y2+OYdbtMuth + lqqwp028AqyY+PRfVMtSYMbjuQuu5byyKR01BbqYhuS3jtqQmljZ/bJvXqnmiVXh + 38UuLa+z077PxyxQhu5BbqntTPQMfiyqEiU+BKbq2WmANUKQf+1AmZY/IruOXbnq + L4C1+gJ8vfmXQt99npCaxEjaNRVYfOS8QcixNzHUYnb6emjlANyEVlZzeqo7XKl7 + UrwV5inawTSzWNvtjEjj4nJL8NsLwscpLPQUhTQ+7BbQXAwAmeHCUTQIvvWXqw0N + cmhh4HgeQscQHYgOJjjDVfoY5MucvglbIgCqfzAHW9jxmRL4qbMZj+b1XoePEtht + ku4bIQN1X5P07fNWzlgaRL5Z4POXDDZTlIQ/El58j9kp4bnWRCJW0lya+f8ocodo + vZZ+Doi+fy4D5ZGrL4XEcIQP/Lv5uFyf+kQtl/94VFYVJOleAv8W92KdgDkhTcTD + G7c0tIkVEKNUq48b3aQ64NOZQW7fVjfoKwEZdOqPE72Pa45jrZzvUFxSpdiNk2tZ + XYukHjlxxEgBdC/J3cMMNRE1F4NCA3ApfV1Y7/hTeOnmDuDYwr9/obA8t016Yljj + q5rdkywPf4JF8mXUW5eCN1vAFHxeg9ZWemhBtQmGxXnw9M+z6hWwc6ahmwARAQAB + tCtEb2NrZXIgUmVsZWFzZSAoQ0UgZGViKSA8ZG9ja2VyQGRvY2tlci5jb20+iQI3 + BBMBCgAhBQJYrefAAhsvBQsJCAcDBRUKCQgLBRYCAwEAAh4BAheAAAoJEI2BgDwO + v82IsskP/iQZo68flDQmNvn8X5XTd6RRaUH33kXYXquT6NkHJciS7E2gTJmqvMqd + tI4mNYHCSEYxI5qrcYV5YqX9P6+Ko+vozo4nseUQLPH/ATQ4qL0Zok+1jkag3Lgk + jonyUf9bwtWxFp05HC3GMHPhhcUSexCxQLQvnFWXD2sWLKivHp2fT8QbRGeZ+d3m + 6fqcd5Fu7pxsqm0EUDK5NL+nPIgYhN+auTrhgzhK1CShfGccM/wfRlei9Utz6p9P + XRKIlWnXtT4qNGZNTN0tR+NLG/6Bqd8OYBaFAUcue/w1VW6JQ2VGYZHnZu9S8LMc + FYBa5Ig9PxwGQOgq6RDKDbV+PqTQT5EFMeR1mrjckk4DQJjbxeMZbiNMG5kGECA8 + g383P3elhn03WGbEEa4MNc3Z4+7c236QI3xWJfNPdUbXRaAwhy/6rTSFbzwKB0Jm + ebwzQfwjQY6f55MiI/RqDCyuPj3r3jyVRkK86pQKBAJwFHyqj9KaKXMZjfVnowLh + 9svIGfNbGHpucATqREvUHuQbNnqkCx8VVhtYkhDb9fEP2xBu5VvHbR+3nfVhMut5 + G34Ct5RS7Jt6LIfFdtcn8CaSas/l1HbiGeRgc70X/9aYx/V/CEJv0lIe8gP6uDoW + FPIZ7d6vH+Vro6xuWEGiuMaiznap2KhZmpkgfupyFmplh0s6knymuQINBFit2ioB + EADneL9S9m4vhU3blaRjVUUyJ7b/qTjcSylvCH5XUE6R2k+ckEZjfAMZPLpO+/tF + M2JIJMD4SifKuS3xck9KtZGCufGmcwiLQRzeHF7vJUKrLD5RTkNi23ydvWZgPjtx + Q+DTT1Zcn7BrQFY6FgnRoUVIxwtdw1bMY/89rsFgS5wwuMESd3Q2RYgb7EOFOpnu + w6da7WakWf4IhnF5nsNYGDVaIHzpiqCl+uTbf1epCjrOlIzkZ3Z3Yk5CM/TiFzPk + z2lLz89cpD8U+NtCsfagWWfjd2U3jDapgH+7nQnCEWpROtzaKHG6lA3pXdix5zG8 + eRc6/0IbUSWvfjKxLLPfNeCS2pCL3IeEI5nothEEYdQH6szpLog79xB9dVnJyKJb + VfxXnseoYqVrRz2VVbUI5Blwm6B40E3eGVfUQWiux54DspyVMMk41Mx7QJ3iynIa + 1N4ZAqVMAEruyXTRTxc9XW0tYhDMA/1GYvz0EmFpm8LzTHA6sFVtPm/ZlNCX6P1X + zJwrv7DSQKD6GGlBQUX+OeEJ8tTkkf8QTJSPUdh8P8YxDFS5EOGAvhhpMBYD42kQ + pqXjEC+XcycTvGI7impgv9PDY1RCC1zkBjKPa120rNhv/hkVk/YhuGoajoHyy4h7 + ZQopdcMtpN2dgmhEegny9JCSwxfQmQ0zK0g7m6SHiKMwjwARAQABiQQ+BBgBCAAJ + BQJYrdoqAhsCAikJEI2BgDwOv82IwV0gBBkBCAAGBQJYrdoqAAoJEH6gqcPyc/zY + 1WAP/2wJ+R0gE6qsce3rjaIz58PJmc8goKrir5hnElWhPgbq7cYIsW5qiFyLhkdp + YcMmhD9mRiPpQn6Ya2w3e3B8zfIVKipbMBnke/ytZ9M7qHmDCcjoiSmwEXN3wKYI + mD9VHONsl/CG1rU9Isw1jtB5g1YxuBA7M/m36XN6x2u+NtNMDB9P56yc4gfsZVES + KA9v+yY2/l45L8d/WUkUi0YXomn6hyBGI7JrBLq0CX37GEYP6O9rrKipfz73XfO7 + JIGzOKZlljb/D9RX/g7nRbCn+3EtH7xnk+TK/50euEKw8SMUg147sJTcpQmv6UzZ + cM4JgL0HbHVCojV4C/plELwMddALOFeYQzTif6sMRPf+3DSj8frbInjChC3yOLy0 + 6br92KFom17EIj2CAcoeq7UPhi2oouYBwPxh5ytdehJkoo+sN7RIWua6P2WSmon5 + U888cSylXC0+ADFdgLX9K2zrDVYUG1vo8CX0vzxFBaHwN6Px26fhIT1/hYUHQR1z + VfNDcyQmXqkOnZvvoMfz/Q0s9BhFJ/zU6AgQbIZE/hm1spsfgvtsD1frZfygXJ9f + irP+MSAI80xHSf91qSRZOj4Pl3ZJNbq4yYxv0b1pkMqeGdjdCYhLU+LZ4wbQmpCk + SVe2prlLureigXtmZfkqevRz7FrIZiu9ky8wnCAPwC7/zmS18rgP/17bOtL4/iIz + QhxAAoAMWVrGyJivSkjhSGx1uCojsWfsTAm11P7jsruIL61ZzMUVE2aM3Pmj5G+W + 9AcZ58Em+1WsVnAXdUR//bMmhyr8wL/G1YO1V3JEJTRdxsSxdYa4deGBBY/Adpsw + 24jxhOJR+lsJpqIUeb999+R8euDhRHG9eFO7DRu6weatUJ6suupoDTRWtr/4yGqe + dKxV3qQhNLSnaAzqW/1nA3iUB4k7kCaKZxhdhDbClf9P37qaRW467BLCVO/coL3y + Vm50dwdrNtKpMBh3ZpbB1uJvgi9mXtyBOMJ3v8RZeDzFiG8HdCtg9RvIt/AIFoHR + H3S+U79NT6i0KPzLImDfs8T7RlpyuMc4Ufs8ggyg9v3Ae6cN3eQyxcK3w0cbBwsh + /nQNfsA6uu+9H7NhbehBMhYnpNZyrHzCmzyXkauwRAqoCbGCNykTRwsur9gS41TQ + M8ssD1jFheOJf3hODnkKU+HKjvMROl1DK7zdmLdNzA1cvtZH/nCC9KPj1z8QC47S + xx+dTZSx4ONAhwbS/LN3PoKtn8LPjY9NP9uDWI+TWYquS2U+KHDrBDlsgozDbs/O + jCxcpDzNmXpWQHEtHU7649OXHP7UeNST1mCUCH5qdank0V1iejF6/CfTFU4MfcrG + YT90qFF93M3v01BbxP+EIY2/9tiIPbrd + =0YYh + -----END PGP PUBLIC KEY BLOCK----- + +- path: "/opt/bin/setup" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + + # As we added some modules and don't want to reboot, restart the service + systemctl restart systemd-modules-load.service + sysctl --system + + apt-key add /opt/docker.asc + apt-get update + + # Make sure we always disable swap - Otherwise the kubelet won't start'. + systemctl mask swap.target + swapoff -a + export CR_PKG='docker-ce=18.06.0~ce~3-0~ubuntu' + + DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install -y \ + curl \ + ca-certificates \ + ceph-common \ + cifs-utils \ + conntrack \ + e2fsprogs \ + ebtables \ + ethtool \ + glusterfs-client \ + iptables \ + jq \ + kmod \ + openssh-client \ + nfs-common \ + socat \ + util-linux \ + ${CR_PKG} \ + open-vm-tools \ + ipvsadm + + # If something failed during package installation but docker got installed, we need to put it on hold + apt-mark hold docker.io || true + apt-mark hold docker-ce || true + if [[ -e /var/run/reboot-required ]]; then + reboot + fi + + #setup some common directories + mkdir -p /opt/bin/ + mkdir -p /var/lib/calico + mkdir -p /etc/kubernetes/manifests + mkdir -p /etc/cni/net.d + mkdir -p /opt/cni/bin + + # cni + if [ ! -f /opt/cni/bin/loopback ]; then + curl -L https://github.com/containernetworking/plugins/releases/download/v0.6.0/cni-plugins-amd64-v0.6.0.tgz | tar -xvzC /opt/cni/bin -f - + fi + # kubelet + if [ ! -f /opt/bin/kubelet ]; then + curl -Lfo /opt/bin/kubelet https://storage.googleapis.com/kubernetes-release/release/v1.12.1/bin/linux/amd64/kubelet + chmod +x /opt/bin/kubelet + fi + + if [[ ! -x /opt/bin/health-monitor.sh ]]; then + curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/8b5b66e4910a6228dfaecccaa0a3b05ec4902f8e/pkg/userdata/scripts/health-monitor.sh + chmod +x /opt/bin/health-monitor.sh + fi + + + systemctl enable --now docker + systemctl enable --now kubelet + systemctl enable --now --no-block kubelet-healthcheck.service + systemctl enable --now --no-block docker-healthcheck.service + +- path: "/opt/bin/supervise.sh" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + while ! "$@"; do + sleep 1 + done + +- path: "/etc/systemd/system/kubelet.service" + content: | + [Unit] + After=docker.service + Requires=docker.service + + Description=kubelet: The Kubernetes Node Agent + Documentation=https://kubernetes.io/docs/home/ + + [Service] + Restart=always + StartLimitInterval=0 + RestartSec=10 + + Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" + + ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ + --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ + --kubeconfig=/etc/kubernetes/kubelet.conf \ + --pod-manifest-path=/etc/kubernetes/manifests \ + --allow-privileged=true \ + --network-plugin=cni \ + --cni-conf-dir=/etc/cni/net.d \ + --cni-bin-dir=/opt/cni/bin \ + --authorization-mode=Webhook \ + --client-ca-file=/etc/kubernetes/pki/ca.crt \ + --rotate-certificates=true \ + --cert-dir=/etc/kubernetes/pki \ + --authentication-token-webhook=true \ + --hostname-override=node1 \ + --read-only-port=0 \ + --exit-on-lock-contention \ + --lock-file=/tmp/kubelet.lock \ + --anonymous-auth=false \ + --protect-kernel-defaults=true \ + --cluster-dns=10.10.10.10 \ + --cluster-domain=cluster.local + + [Install] + WantedBy=multi-user.target + +- path: "/etc/systemd/system/kubelet.service.d/extras.conf" + content: | + [Service] + Environment="KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf" + +- path: "/etc/kubernetes/cloud-config" + content: | + + +- path: "/etc/kubernetes/bootstrap-kubelet.conf" + content: | + apiVersion: v1 + clusters: + - cluster: + certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t + server: https://server:443 + name: "" + contexts: [] + current-context: "" + kind: Config + preferences: {} + users: + - name: "" + user: + token: my-token + + +- path: "/etc/kubernetes/pki/ca.crt" + content: | + -----BEGIN CERTIFICATE----- + MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV + BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG + A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 + DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 + NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG + cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv + c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B + AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS + R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT + ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk + JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 + mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW + caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G + A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt + hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB + MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES + MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv + bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h + U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao + eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 + UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD + 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n + sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF + kPe6XoSbiLm/kxk32T0= + -----END CERTIFICATE----- + +- path: "/etc/systemd/system/setup.service" + permissions: "0644" + content: | + [Install] + WantedBy=multi-user.target + + [Unit] + Requires=network-online.target + After=network-online.target + + [Service] + Type=oneshot + RemainAfterExit=true + ExecStart=/opt/bin/supervise.sh /opt/bin/setup + +- path: "/etc/profile.d/opt-bin-path.sh" + permissions: "0644" + content: | + export PATH="/opt/bin:$PATH" + +- path: /etc/systemd/system/docker.service.d/10-storage.conf + permissions: "0644" + content: | + [Service] + ExecStart= + ExecStart=/usr/bin/dockerd -H fd:// --storage-driver=overlay2 + +- path: /etc/systemd/system/kubelet-healthcheck.service + permissions: "0644" + content: | + [Unit] + Requires=kubelet.service + After=kubelet.service + + [Service] + ExecStart=/opt/bin/health-monitor.sh kubelet + + [Install] + WantedBy=multi-user.target + + +- path: /etc/systemd/system/docker-healthcheck.service + permissions: "0644" + content: | + [Unit] + Requires=docker.service + After=docker.service + + [Service] + ExecStart=/opt/bin/health-monitor.sh container-runtime + + [Install] + WantedBy=multi-user.target + + +runcmd: +- systemctl enable --now setup.service diff --git a/pkg/userdata/ubuntu/testdata/version-1.9.10.golden b/pkg/userdata/ubuntu/testdata/version-1.9.10.golden new file mode 100644 index 000000000..319dcf436 --- /dev/null +++ b/pkg/userdata/ubuntu/testdata/version-1.9.10.golden @@ -0,0 +1,342 @@ +#cloud-config +hostname: node1 + +ssh_pwauth: no + +ssh_authorized_keys: +- "ssh-rsa AAABBB" + +write_files: +- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" + content: | + [Journal] + SystemMaxUse=5G + + +- path: "/etc/modules-load.d/k8s.conf" + content: | + ip_vs + ip_vs_rr + ip_vs_wrr + ip_vs_sh + nf_conntrack_ipv4 + + +- path: "/etc/sysctl.d/k8s.conf" + content: | + net.bridge.bridge-nf-call-ip6tables = 1 + net.bridge.bridge-nf-call-iptables = 1 + kernel.panic_on_oops = 1 + kernel.panic = 10 + net.ipv4.ip_forward = 1 + vm.overcommit_memory = 1 + + +- path: "/etc/apt/sources.list.d/docker.list" + permissions: "0644" + content: deb [arch=amd64] https://download.docker.com/linux/ubuntu bionic stable + +- path: "/opt/docker.asc" + permissions: "0400" + content: | + -----BEGIN PGP PUBLIC KEY BLOCK----- + + mQINBFit2ioBEADhWpZ8/wvZ6hUTiXOwQHXMAlaFHcPH9hAtr4F1y2+OYdbtMuth + lqqwp028AqyY+PRfVMtSYMbjuQuu5byyKR01BbqYhuS3jtqQmljZ/bJvXqnmiVXh + 38UuLa+z077PxyxQhu5BbqntTPQMfiyqEiU+BKbq2WmANUKQf+1AmZY/IruOXbnq + L4C1+gJ8vfmXQt99npCaxEjaNRVYfOS8QcixNzHUYnb6emjlANyEVlZzeqo7XKl7 + UrwV5inawTSzWNvtjEjj4nJL8NsLwscpLPQUhTQ+7BbQXAwAmeHCUTQIvvWXqw0N + cmhh4HgeQscQHYgOJjjDVfoY5MucvglbIgCqfzAHW9jxmRL4qbMZj+b1XoePEtht + ku4bIQN1X5P07fNWzlgaRL5Z4POXDDZTlIQ/El58j9kp4bnWRCJW0lya+f8ocodo + vZZ+Doi+fy4D5ZGrL4XEcIQP/Lv5uFyf+kQtl/94VFYVJOleAv8W92KdgDkhTcTD + G7c0tIkVEKNUq48b3aQ64NOZQW7fVjfoKwEZdOqPE72Pa45jrZzvUFxSpdiNk2tZ + XYukHjlxxEgBdC/J3cMMNRE1F4NCA3ApfV1Y7/hTeOnmDuDYwr9/obA8t016Yljj + q5rdkywPf4JF8mXUW5eCN1vAFHxeg9ZWemhBtQmGxXnw9M+z6hWwc6ahmwARAQAB + tCtEb2NrZXIgUmVsZWFzZSAoQ0UgZGViKSA8ZG9ja2VyQGRvY2tlci5jb20+iQI3 + BBMBCgAhBQJYrefAAhsvBQsJCAcDBRUKCQgLBRYCAwEAAh4BAheAAAoJEI2BgDwO + v82IsskP/iQZo68flDQmNvn8X5XTd6RRaUH33kXYXquT6NkHJciS7E2gTJmqvMqd + tI4mNYHCSEYxI5qrcYV5YqX9P6+Ko+vozo4nseUQLPH/ATQ4qL0Zok+1jkag3Lgk + jonyUf9bwtWxFp05HC3GMHPhhcUSexCxQLQvnFWXD2sWLKivHp2fT8QbRGeZ+d3m + 6fqcd5Fu7pxsqm0EUDK5NL+nPIgYhN+auTrhgzhK1CShfGccM/wfRlei9Utz6p9P + XRKIlWnXtT4qNGZNTN0tR+NLG/6Bqd8OYBaFAUcue/w1VW6JQ2VGYZHnZu9S8LMc + FYBa5Ig9PxwGQOgq6RDKDbV+PqTQT5EFMeR1mrjckk4DQJjbxeMZbiNMG5kGECA8 + g383P3elhn03WGbEEa4MNc3Z4+7c236QI3xWJfNPdUbXRaAwhy/6rTSFbzwKB0Jm + ebwzQfwjQY6f55MiI/RqDCyuPj3r3jyVRkK86pQKBAJwFHyqj9KaKXMZjfVnowLh + 9svIGfNbGHpucATqREvUHuQbNnqkCx8VVhtYkhDb9fEP2xBu5VvHbR+3nfVhMut5 + G34Ct5RS7Jt6LIfFdtcn8CaSas/l1HbiGeRgc70X/9aYx/V/CEJv0lIe8gP6uDoW + FPIZ7d6vH+Vro6xuWEGiuMaiznap2KhZmpkgfupyFmplh0s6knymuQINBFit2ioB + EADneL9S9m4vhU3blaRjVUUyJ7b/qTjcSylvCH5XUE6R2k+ckEZjfAMZPLpO+/tF + M2JIJMD4SifKuS3xck9KtZGCufGmcwiLQRzeHF7vJUKrLD5RTkNi23ydvWZgPjtx + Q+DTT1Zcn7BrQFY6FgnRoUVIxwtdw1bMY/89rsFgS5wwuMESd3Q2RYgb7EOFOpnu + w6da7WakWf4IhnF5nsNYGDVaIHzpiqCl+uTbf1epCjrOlIzkZ3Z3Yk5CM/TiFzPk + z2lLz89cpD8U+NtCsfagWWfjd2U3jDapgH+7nQnCEWpROtzaKHG6lA3pXdix5zG8 + eRc6/0IbUSWvfjKxLLPfNeCS2pCL3IeEI5nothEEYdQH6szpLog79xB9dVnJyKJb + VfxXnseoYqVrRz2VVbUI5Blwm6B40E3eGVfUQWiux54DspyVMMk41Mx7QJ3iynIa + 1N4ZAqVMAEruyXTRTxc9XW0tYhDMA/1GYvz0EmFpm8LzTHA6sFVtPm/ZlNCX6P1X + zJwrv7DSQKD6GGlBQUX+OeEJ8tTkkf8QTJSPUdh8P8YxDFS5EOGAvhhpMBYD42kQ + pqXjEC+XcycTvGI7impgv9PDY1RCC1zkBjKPa120rNhv/hkVk/YhuGoajoHyy4h7 + ZQopdcMtpN2dgmhEegny9JCSwxfQmQ0zK0g7m6SHiKMwjwARAQABiQQ+BBgBCAAJ + BQJYrdoqAhsCAikJEI2BgDwOv82IwV0gBBkBCAAGBQJYrdoqAAoJEH6gqcPyc/zY + 1WAP/2wJ+R0gE6qsce3rjaIz58PJmc8goKrir5hnElWhPgbq7cYIsW5qiFyLhkdp + YcMmhD9mRiPpQn6Ya2w3e3B8zfIVKipbMBnke/ytZ9M7qHmDCcjoiSmwEXN3wKYI + mD9VHONsl/CG1rU9Isw1jtB5g1YxuBA7M/m36XN6x2u+NtNMDB9P56yc4gfsZVES + KA9v+yY2/l45L8d/WUkUi0YXomn6hyBGI7JrBLq0CX37GEYP6O9rrKipfz73XfO7 + JIGzOKZlljb/D9RX/g7nRbCn+3EtH7xnk+TK/50euEKw8SMUg147sJTcpQmv6UzZ + cM4JgL0HbHVCojV4C/plELwMddALOFeYQzTif6sMRPf+3DSj8frbInjChC3yOLy0 + 6br92KFom17EIj2CAcoeq7UPhi2oouYBwPxh5ytdehJkoo+sN7RIWua6P2WSmon5 + U888cSylXC0+ADFdgLX9K2zrDVYUG1vo8CX0vzxFBaHwN6Px26fhIT1/hYUHQR1z + VfNDcyQmXqkOnZvvoMfz/Q0s9BhFJ/zU6AgQbIZE/hm1spsfgvtsD1frZfygXJ9f + irP+MSAI80xHSf91qSRZOj4Pl3ZJNbq4yYxv0b1pkMqeGdjdCYhLU+LZ4wbQmpCk + SVe2prlLureigXtmZfkqevRz7FrIZiu9ky8wnCAPwC7/zmS18rgP/17bOtL4/iIz + QhxAAoAMWVrGyJivSkjhSGx1uCojsWfsTAm11P7jsruIL61ZzMUVE2aM3Pmj5G+W + 9AcZ58Em+1WsVnAXdUR//bMmhyr8wL/G1YO1V3JEJTRdxsSxdYa4deGBBY/Adpsw + 24jxhOJR+lsJpqIUeb999+R8euDhRHG9eFO7DRu6weatUJ6suupoDTRWtr/4yGqe + dKxV3qQhNLSnaAzqW/1nA3iUB4k7kCaKZxhdhDbClf9P37qaRW467BLCVO/coL3y + Vm50dwdrNtKpMBh3ZpbB1uJvgi9mXtyBOMJ3v8RZeDzFiG8HdCtg9RvIt/AIFoHR + H3S+U79NT6i0KPzLImDfs8T7RlpyuMc4Ufs8ggyg9v3Ae6cN3eQyxcK3w0cbBwsh + /nQNfsA6uu+9H7NhbehBMhYnpNZyrHzCmzyXkauwRAqoCbGCNykTRwsur9gS41TQ + M8ssD1jFheOJf3hODnkKU+HKjvMROl1DK7zdmLdNzA1cvtZH/nCC9KPj1z8QC47S + xx+dTZSx4ONAhwbS/LN3PoKtn8LPjY9NP9uDWI+TWYquS2U+KHDrBDlsgozDbs/O + jCxcpDzNmXpWQHEtHU7649OXHP7UeNST1mCUCH5qdank0V1iejF6/CfTFU4MfcrG + YT90qFF93M3v01BbxP+EIY2/9tiIPbrd + =0YYh + -----END PGP PUBLIC KEY BLOCK----- + +- path: "/opt/bin/setup" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + + # As we added some modules and don't want to reboot, restart the service + systemctl restart systemd-modules-load.service + sysctl --system + + apt-key add /opt/docker.asc + apt-get update + + # Make sure we always disable swap - Otherwise the kubelet won't start'. + systemctl mask swap.target + swapoff -a + export CR_PKG='docker.io=17.12.1-0ubuntu1' + + DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install -y \ + curl \ + ca-certificates \ + ceph-common \ + cifs-utils \ + conntrack \ + e2fsprogs \ + ebtables \ + ethtool \ + glusterfs-client \ + iptables \ + jq \ + kmod \ + openssh-client \ + nfs-common \ + socat \ + util-linux \ + ${CR_PKG} \ + open-vm-tools \ + ipvsadm + + # If something failed during package installation but docker got installed, we need to put it on hold + apt-mark hold docker.io || true + apt-mark hold docker-ce || true + if [[ -e /var/run/reboot-required ]]; then + reboot + fi + + #setup some common directories + mkdir -p /opt/bin/ + mkdir -p /var/lib/calico + mkdir -p /etc/kubernetes/manifests + mkdir -p /etc/cni/net.d + mkdir -p /opt/cni/bin + + # cni + if [ ! -f /opt/cni/bin/loopback ]; then + curl -L https://github.com/containernetworking/plugins/releases/download/v0.6.0/cni-plugins-amd64-v0.6.0.tgz | tar -xvzC /opt/cni/bin -f - + fi + # kubelet + if [ ! -f /opt/bin/kubelet ]; then + curl -Lfo /opt/bin/kubelet https://storage.googleapis.com/kubernetes-release/release/v1.9.10/bin/linux/amd64/kubelet + chmod +x /opt/bin/kubelet + fi + + if [[ ! -x /opt/bin/health-monitor.sh ]]; then + curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/8b5b66e4910a6228dfaecccaa0a3b05ec4902f8e/pkg/userdata/scripts/health-monitor.sh + chmod +x /opt/bin/health-monitor.sh + fi + + + systemctl enable --now docker + systemctl enable --now kubelet + systemctl enable --now --no-block kubelet-healthcheck.service + systemctl enable --now --no-block docker-healthcheck.service + +- path: "/opt/bin/supervise.sh" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + while ! "$@"; do + sleep 1 + done + +- path: "/etc/systemd/system/kubelet.service" + content: | + [Unit] + After=docker.service + Requires=docker.service + + Description=kubelet: The Kubernetes Node Agent + Documentation=https://kubernetes.io/docs/home/ + + [Service] + Restart=always + StartLimitInterval=0 + RestartSec=10 + + Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" + + ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ + --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ + --kubeconfig=/etc/kubernetes/kubelet.conf \ + --pod-manifest-path=/etc/kubernetes/manifests \ + --allow-privileged=true \ + --network-plugin=cni \ + --cni-conf-dir=/etc/cni/net.d \ + --cni-bin-dir=/opt/cni/bin \ + --authorization-mode=Webhook \ + --client-ca-file=/etc/kubernetes/pki/ca.crt \ + --cadvisor-port=0 \ + --rotate-certificates=true \ + --cert-dir=/etc/kubernetes/pki \ + --authentication-token-webhook=true \ + --hostname-override=node1 \ + --read-only-port=0 \ + --exit-on-lock-contention \ + --lock-file=/tmp/kubelet.lock \ + --anonymous-auth=false \ + --protect-kernel-defaults=true \ + --cluster-dns=10.10.10.10 \ + --cluster-domain=cluster.local + + [Install] + WantedBy=multi-user.target + +- path: "/etc/systemd/system/kubelet.service.d/extras.conf" + content: | + [Service] + Environment="KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf" + +- path: "/etc/kubernetes/cloud-config" + content: | + + +- path: "/etc/kubernetes/bootstrap-kubelet.conf" + content: | + apiVersion: v1 + clusters: + - cluster: + certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t + server: https://server:443 + name: "" + contexts: [] + current-context: "" + kind: Config + preferences: {} + users: + - name: "" + user: + token: my-token + + +- path: "/etc/kubernetes/pki/ca.crt" + content: | + -----BEGIN CERTIFICATE----- + MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV + BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG + A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 + DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 + NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG + cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv + c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B + AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS + R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT + ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk + JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 + mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW + caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G + A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt + hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB + MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES + MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv + bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h + U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao + eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 + UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD + 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n + sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF + kPe6XoSbiLm/kxk32T0= + -----END CERTIFICATE----- + +- path: "/etc/systemd/system/setup.service" + permissions: "0644" + content: | + [Install] + WantedBy=multi-user.target + + [Unit] + Requires=network-online.target + After=network-online.target + + [Service] + Type=oneshot + RemainAfterExit=true + ExecStart=/opt/bin/supervise.sh /opt/bin/setup + +- path: "/etc/profile.d/opt-bin-path.sh" + permissions: "0644" + content: | + export PATH="/opt/bin:$PATH" + +- path: /etc/systemd/system/docker.service.d/10-storage.conf + permissions: "0644" + content: | + [Service] + ExecStart= + ExecStart=/usr/bin/dockerd -H fd:// --storage-driver=overlay2 + +- path: /etc/systemd/system/kubelet-healthcheck.service + permissions: "0644" + content: | + [Unit] + Requires=kubelet.service + After=kubelet.service + + [Service] + ExecStart=/opt/bin/health-monitor.sh kubelet + + [Install] + WantedBy=multi-user.target + + +- path: /etc/systemd/system/docker-healthcheck.service + permissions: "0644" + content: | + [Unit] + Requires=docker.service + After=docker.service + + [Service] + ExecStart=/opt/bin/health-monitor.sh container-runtime + + [Install] + WantedBy=multi-user.target + + +runcmd: +- systemctl enable --now setup.service diff --git a/pkg/userdata/ubuntu/userdata.go b/pkg/userdata/ubuntu/userdata.go index 9be9418de..ed2e95956 100644 --- a/pkg/userdata/ubuntu/userdata.go +++ b/pkg/userdata/ubuntu/userdata.go @@ -10,13 +10,13 @@ import ( "github.com/Masterminds/semver" "k8s.io/apimachinery/pkg/runtime" - clientcmdapi "k8s.io/client-go/tools/clientcmd/api" "github.com/kubermatic/machine-controller/pkg/providerconfig" - machinetemplate "github.com/kubermatic/machine-controller/pkg/template" "github.com/kubermatic/machine-controller/pkg/userdata/cloud" userdatahelper "github.com/kubermatic/machine-controller/pkg/userdata/helper" + clientcmdapi "k8s.io/client-go/tools/clientcmd/api" + clusterv1alpha1 "sigs.k8s.io/cluster-api/pkg/apis/cluster/v1alpha1" ) @@ -47,7 +47,7 @@ func (p Provider) UserData( clusterDNSIPs []net.IP, ) (string, error) { - tmpl, err := template.New("user-data").Funcs(machinetemplate.TxtFuncMap()).Parse(ctTemplate) + tmpl, err := template.New("user-data").Funcs(userdatahelper.TxtFuncMap()).Parse(ctTemplate) if err != nil { return "", fmt.Errorf("failed to parse user-data template: %v", err) } @@ -57,13 +57,6 @@ func (p Provider) UserData( return "", fmt.Errorf("invalid kubelet version: %v", err) } - var kubeadmDropInFilename string - if kubeletVersion.Minor() > 8 { - kubeadmDropInFilename = "10-kubeadm.conf" - } else { - kubeadmDropInFilename = "kubeadm-10.conf" - } - cpConfig, cpName, err := ccProvider.GetCloudConfig(spec) if err != nil { return "", fmt.Errorf("failed to get cloud config: %v", err) @@ -87,49 +80,43 @@ func (p Provider) UserData( return "", fmt.Errorf("failed to get ubuntu config from provider config: %v", err) } - bootstrapToken, err := userdatahelper.GetTokenFromKubeconfig(kubeconfig) + serverAddr, err := userdatahelper.GetServerAddressFromKubeconfig(kubeconfig) if err != nil { - return "", fmt.Errorf("error extracting token: %v", err) + return "", fmt.Errorf("error extracting server address from kubeconfig: %v", err) } - kubeadmCACertHash, err := userdatahelper.GetKubeadmCACertHash(kubeconfig) + kubeconfigString, err := userdatahelper.StringifyKubeconfig(kubeconfig) if err != nil { - return "", fmt.Errorf("error extracting kubeadm cacert hash: %v", err) + return "", err } - serverAddr, err := userdatahelper.GetServerAddressFromKubeconfig(kubeconfig) + kubernetesCACert, err := userdatahelper.GetCACert(kubeconfig) if err != nil { - return "", fmt.Errorf("error extracting server address from kubeconfig: %v", err) + return "", fmt.Errorf("error extracting cacert: %v", err) } data := struct { - MachineSpec clusterv1alpha1.MachineSpec - ProviderConfig *providerconfig.Config - OSConfig *Config - BoostrapToken string - CloudProvider string - CloudConfig string - CRAptPackage string - CRAptPackageVersion string - KubeadmDropInFilename string - ClusterDNSIPs []net.IP - KubeadmCACertHash string - ServerAddr string - JournaldMaxSize string - KubeletVersion string + MachineSpec clusterv1alpha1.MachineSpec + ProviderConfig *providerconfig.Config + OSConfig *Config + CloudProvider string + CloudConfig string + ClusterDNSIPs []net.IP + ServerAddr string + KubeletVersion string + Kubeconfig string + KubernetesCACert string }{ - MachineSpec: spec, - ProviderConfig: pconfig, - OSConfig: osConfig, - BoostrapToken: bootstrapToken, - CloudProvider: cpName, - CloudConfig: cpConfig, - KubeadmDropInFilename: kubeadmDropInFilename, - ClusterDNSIPs: clusterDNSIPs, - KubeadmCACertHash: kubeadmCACertHash, - ServerAddr: serverAddr, - JournaldMaxSize: userdatahelper.JournaldMaxUse, - KubeletVersion: kubeletVersion.String(), + MachineSpec: spec, + ProviderConfig: pconfig, + OSConfig: osConfig, + CloudProvider: cpName, + CloudConfig: cpConfig, + ClusterDNSIPs: clusterDNSIPs, + ServerAddr: serverAddr, + KubeletVersion: kubeletVersion.String(), + Kubeconfig: kubeconfigString, + KubernetesCACert: kubernetesCACert, } b := &bytes.Buffer{} err = tmpl.Execute(b, data) @@ -153,140 +140,20 @@ ssh_authorized_keys: write_files: - path: "/etc/systemd/journald.conf.d/max_disk_use.conf" content: | - [Journal] - SystemMaxUse={{ .JournaldMaxSize }} +{{ journalDConfig | indent 4 }} -- path: "/etc/sysctl.d/k8s.conf" +- path: "/etc/modules-load.d/k8s.conf" content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - vm.overcommit_memory = 1 +{{ kernelModules | indent 4 }} -- path: "/etc/kubernetes/cloud-config" +- path: "/etc/sysctl.d/k8s.conf" content: | -{{ if ne .CloudConfig "" }}{{ .CloudConfig | indent 4 }}{{ end }} +{{ kernelSettings | indent 4 }} - path: "/etc/apt/sources.list.d/docker.list" permissions: "0644" content: deb [arch=amd64] https://download.docker.com/linux/ubuntu bionic stable -- path: "/etc/apt/sources.list.d/kubernetes.list" - permissions: "0644" - content: deb http://apt.kubernetes.io/ kubernetes-xenial main - -- path: "/usr/local/bin/setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - - sysctl --system - mkdir -p /opt/bin - apt-key add /opt/docker.asc - apt-key add /opt/kubernetes.asc - apt-get update - - # Hetzner's Ubuntu Bionic comes with swap pre-configured, so we force it off. - systemctl mask swap.target - swapoff -a - - # If something failed during package installation but one of docker/kubeadm/kubelet was already installed - # an apt-mark hold after the install won't do it, which is why we test here if the binaries exist and if - # yes put them on hold - set +e - which docker && apt-mark hold docker.io docker-ce - which kubelet && apt-mark hold kubelet - which kubeadm && apt-mark hold kubeadm - - # When docker is started from within the apt installation it fails with a - # 'no sockets found via socket activation: make sure the service was started by systemd' - # Apparently the package is broken in a way that it gets started without its dependencies, manually starting - # it works fine thought - which docker && systemctl start docker - set -e - - {{- if .OSConfig.DistUpgradeOnBoot }} - DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" dist-upgrade -y - {{- end }} - if [[ -e /var/run/reboot-required ]]; then - reboot - fi - - export CR_PKG='docker-ce=18.06.0~ce~3-0~ubuntu' -{{- if semverCompare "<1.12.0" .KubeletVersion }} - export CR_PKG='docker.io=17.12.1-0ubuntu1' -{{ end }} - - DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install -y \ - curl \ - ca-certificates \ - ceph-common \ - cifs-utils \ - conntrack \ - e2fsprogs \ - ebtables \ - ethtool \ - glusterfs-client \ - iptables \ - jq \ - kmod \ - openssh-client \ - nfs-common \ - socat \ - util-linux \ - ${CR_PKG} \ - open-vm-tools \ - kubelet={{ .KubeletVersion }}-00 \ - kubeadm={{ .KubeletVersion }}-00 \ - - cp /etc/default/kubelet-overwrite /etc/default/kubelet - - systemctl enable --now docker - systemctl enable kubelet - - if [[ ! -x /usr/local/bin/health-monitor.sh ]]; then - curl -Lfo /usr/local/bin/health-monitor.sh \ - https://raw.githubusercontent.com/kubermatic/machine-controller/8b5b66e4910a6228dfaecccaa0a3b05ec4902f8e/pkg/userdata/scripts/health-monitor.sh - chmod +x /usr/local/bin/health-monitor.sh - fi - - if ! [[ -e /etc/kubernetes/pki/ca.crt ]]; then - kubeadm join \ - --token {{ .BoostrapToken }} \ - --discovery-token-ca-cert-hash sha256:{{ .KubeadmCACertHash }} \ - {{- if semverCompare ">=1.9.X" .KubeletVersion }} - --ignore-preflight-errors=CRI \ - {{- end }} - {{ .ServerAddr }} - fi - - systemctl enable --now --no-block kubelet-healthcheck.service - systemctl enable --now --no-block docker-healthcheck.service - -- path: "/opt/kubernetes.asc" - permissions: "0400" - content: | - -----BEGIN PGP PUBLIC KEY BLOCK----- - - mQENBFrBaNsBCADrF18KCbsZlo4NjAvVecTBCnp6WcBQJ5oSh7+E98jX9YznUCrN - rgmeCcCMUvTDRDxfTaDJybaHugfba43nqhkbNpJ47YXsIa+YL6eEE9emSmQtjrSW - IiY+2YJYwsDgsgckF3duqkb02OdBQlh6IbHPoXB6H//b1PgZYsomB+841XW1LSJP - YlYbIrWfwDfQvtkFQI90r6NknVTQlpqQh5GLNWNYqRNrGQPmsB+NrUYrkl1nUt1L - RGu+rCe4bSaSmNbwKMQKkROE4kTiB72DPk7zH4Lm0uo0YFFWG4qsMIuqEihJ/9KN - X8GYBr+tWgyLooLlsdK3l+4dVqd8cjkJM1ExABEBAAG0QEdvb2dsZSBDbG91ZCBQ - YWNrYWdlcyBBdXRvbWF0aWMgU2lnbmluZyBLZXkgPGdjLXRlYW1AZ29vZ2xlLmNv - bT6JAT4EEwECACgFAlrBaNsCGy8FCQWjmoAGCwkIBwMCBhUIAgkKCwQWAgMBAh4B - AheAAAoJEGoDCyG6B/T78e8H/1WH2LN/nVNhm5TS1VYJG8B+IW8zS4BqyozxC9iJ - AJqZIVHXl8g8a/Hus8RfXR7cnYHcg8sjSaJfQhqO9RbKnffiuQgGrqwQxuC2jBa6 - M/QKzejTeP0Mgi67pyrLJNWrFI71RhritQZmzTZ2PoWxfv6b+Tv5v0rPaG+ut1J4 - 7pn+kYgtUaKdsJz1umi6HzK6AacDf0C0CksJdKG7MOWsZcB4xeOxJYuy6NuO6Kcd - Ez8/XyEUjIuIOlhYTd0hH8E/SEBbXXft7/VBQC5wNq40izPi+6WFK/e1O42DIpzQ - 749ogYQ1eodexPNhLzekKR3XhGrNXJ95r5KO10VrsLFNd8I= - =TKuP - -----END PGP PUBLIC KEY BLOCK----- - - path: "/opt/docker.asc" permissions: "0400" content: | @@ -353,7 +220,69 @@ write_files: =0YYh -----END PGP PUBLIC KEY BLOCK----- -- path: "/usr/local/bin/supervise.sh" +- path: "/opt/bin/setup" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + + # As we added some modules and don't want to reboot, restart the service + systemctl restart systemd-modules-load.service + sysctl --system + + apt-key add /opt/docker.asc + apt-get update + + # Make sure we always disable swap - Otherwise the kubelet won't start'. + systemctl mask swap.target + swapoff -a + +{{- if semverCompare "<1.12.0" .KubeletVersion }} + export CR_PKG='docker.io=17.12.1-0ubuntu1' +{{- else }} + export CR_PKG='docker-ce=18.06.0~ce~3-0~ubuntu' +{{- end }} + + DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install -y \ + curl \ + ca-certificates \ + ceph-common \ + cifs-utils \ + conntrack \ + e2fsprogs \ + ebtables \ + ethtool \ + glusterfs-client \ + iptables \ + jq \ + kmod \ + openssh-client \ + nfs-common \ + socat \ + util-linux \ + ${CR_PKG} \ + open-vm-tools \ + ipvsadm + + # If something failed during package installation but docker got installed, we need to put it on hold + apt-mark hold docker.io || true + apt-mark hold docker-ce || true + + {{- if .OSConfig.DistUpgradeOnBoot }} + DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" dist-upgrade -y + {{- end }} + if [[ -e /var/run/reboot-required ]]; then + reboot + fi + +{{ downloadBinariesScript .KubeletVersion true | indent 4 }} + + systemctl enable --now docker + systemctl enable --now kubelet + systemctl enable --now --no-block kubelet-healthcheck.service + systemctl enable --now --no-block docker-healthcheck.service + +- path: "/opt/bin/supervise.sh" permissions: "0755" content: | #!/bin/bash @@ -362,29 +291,26 @@ write_files: sleep 1 done -- path: "/etc/default/kubelet-overwrite" +- path: "/etc/systemd/system/kubelet.service" content: | - KUBELET_DNS_ARGS= - KUBELET_EXTRA_ARGS=--authentication-token-webhook=true \ - {{- if .CloudProvider }} - --cloud-provider={{ .CloudProvider }} \ - --cloud-config=/etc/kubernetes/cloud-config \ - {{- end}} - --hostname-override={{ .MachineSpec.Name }} \ - --read-only-port=0 \ - --protect-kernel-defaults=true \ - {{- if semverCompare "<1.11.0" .KubeletVersion }} - --resolv-conf=/run/systemd/resolve/resolv.conf \ - {{- end }} - --cluster-dns={{ ipSliceToCommaSeparatedString .ClusterDNSIPs }} \ - --cluster-domain=cluster.local -{{ if semverCompare "<1.11.0" .KubeletVersion }} -- path: "/etc/systemd/system/kubelet.service.d/20-extra.conf" - permissions: "0644" +{{ kubeletSystemdUnit .KubeletVersion .CloudProvider .MachineSpec.Name .ClusterDNSIPs | indent 4 }} + +- path: "/etc/systemd/system/kubelet.service.d/extras.conf" content: | [Service] - EnvironmentFile=/etc/default/kubelet -{{ end }} + Environment="KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf" + +- path: "/etc/kubernetes/cloud-config" + content: | +{{ .CloudConfig | indent 4 }} + +- path: "/etc/kubernetes/bootstrap-kubelet.conf" + content: | +{{ .Kubeconfig | indent 4 }} + +- path: "/etc/kubernetes/pki/ca.crt" + content: | +{{ .KubernetesCACert | indent 4 }} - path: "/etc/systemd/system/setup.service" permissions: "0644" @@ -399,7 +325,12 @@ write_files: [Service] Type=oneshot RemainAfterExit=true - ExecStart=/usr/local/bin/supervise.sh /usr/local/bin/setup + ExecStart=/opt/bin/supervise.sh /opt/bin/setup + +- path: "/etc/profile.d/opt-bin-path.sh" + permissions: "0644" + content: | + export PATH="/opt/bin:$PATH" - path: /etc/systemd/system/docker.service.d/10-storage.conf permissions: "0644" @@ -411,28 +342,12 @@ write_files: - path: /etc/systemd/system/kubelet-healthcheck.service permissions: "0644" content: | - [Unit] - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/usr/local/bin/health-monitor.sh kubelet - - [Install] - WantedBy=multi-user.target +{{ kubeletHealthCheckSystemdUnit | indent 4 }} - path: /etc/systemd/system/docker-healthcheck.service permissions: "0644" content: | - [Unit] - Requires=docker.service - After=docker.service - - [Service] - ExecStart=/usr/local/bin/health-monitor.sh container-runtime - - [Install] - WantedBy=multi-user.target +{{ containerRuntimeHealthCheckSystemdUnit | indent 4 }} runcmd: - systemctl enable --now setup.service diff --git a/pkg/userdata/ubuntu/userdata_test.go b/pkg/userdata/ubuntu/userdata_test.go index af94c32a4..a98abc6b6 100644 --- a/pkg/userdata/ubuntu/userdata_test.go +++ b/pkg/userdata/ubuntu/userdata_test.go @@ -3,13 +3,13 @@ package ubuntu import ( "encoding/json" "flag" - "io/ioutil" + "fmt" "net" - "path/filepath" "testing" - "github.com/pmezard/go-difflib/difflib" + testhelper "github.com/kubermatic/machine-controller/pkg/test" + "github.com/Masterminds/semver" "github.com/kubermatic/machine-controller/pkg/providerconfig" "github.com/kubermatic/machine-controller/pkg/userdata/cloud" @@ -52,6 +52,10 @@ kPe6XoSbiLm/kxk32T0= AuthInfos: map[string]*clientcmdapi.AuthInfo{"": &clientcmdapi.AuthInfo{Token: "my-token"}}} ) +const ( + defaultVersion = "1.11.3" +) + type fakeCloudConfigProvider struct { config string name string @@ -64,104 +68,157 @@ func (p *fakeCloudConfigProvider) GetCloudConfig(spec clusterv1alpha1.MachineSpe var update = flag.Bool("update", false, "update .golden files") +func getSimpleVersionTests() []userDataTestCase { + versions := []*semver.Version{ + semver.MustParse("v1.9.10"), + semver.MustParse("v1.10.10"), + semver.MustParse("v1.11.3"), + semver.MustParse("v1.12.1"), + } + + var tests []userDataTestCase + for _, v := range versions { + tests = append(tests, userDataTestCase{ + name: fmt.Sprintf("version-%s", v.String()), + providerConfig: &providerconfig.Config{ + CloudProvider: "", + SSHPublicKeys: []string{"ssh-rsa AAABBB"}, + }, + spec: clusterv1alpha1.MachineSpec{ + ObjectMeta: metav1.ObjectMeta{Name: "node1"}, + Versions: clusterv1alpha1.MachineVersionInfo{ + Kubelet: v.String(), + }, + }, + ccProvider: &fakeCloudConfigProvider{name: "", config: "", err: nil}, + DNSIPs: []net.IP{net.ParseIP("10.10.10.10")}, + kubernetesCACert: "CACert", + osConfig: &Config{DistUpgradeOnBoot: false}, + }) + } + + return tests +} + +type userDataTestCase struct { + name string + spec clusterv1alpha1.MachineSpec + ccProvider cloud.ConfigProvider + osConfig *Config + providerConfig *providerconfig.Config + DNSIPs []net.IP + kubernetesCACert string +} + func TestProvider_UserData(t *testing.T) { t.Parallel() - tests := []struct { - name string - spec clusterv1alpha1.MachineSpec - ccProvider cloud.ConfigProvider - osConfig *Config - providerConfig *providerconfig.Config - DNSIPs []net.IP - kubernetesCACert string - }{ + + tests := getSimpleVersionTests() + tests = append(tests, []userDataTestCase{ { - name: "1.9.2-dist-upgrade-on-boot-aws", + name: "dist-upgrade-on-boot", providerConfig: &providerconfig.Config{ - CloudProvider: "aws", - SSHPublicKeys: []string{"ssh-rsa AAABBB", "ssh-rsa CCCDDD"}, + CloudProvider: "", + SSHPublicKeys: []string{"ssh-rsa AAABBB"}, }, spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.9.2", + Kubelet: defaultVersion, }, }, - ccProvider: &fakeCloudConfigProvider{name: "aws", config: "{aws-config:true}", err: nil}, + ccProvider: &fakeCloudConfigProvider{name: "", config: "", err: nil}, DNSIPs: []net.IP{net.ParseIP("10.10.10.10")}, kubernetesCACert: "CACert", osConfig: &Config{DistUpgradeOnBoot: true}, }, { - name: "1.11-aws", + name: "multiple-dns-servers", providerConfig: &providerconfig.Config{ - CloudProvider: "aws", - SSHPublicKeys: []string{"ssh-rsa AAABBB", "ssh-rsa CCCDDD"}, + CloudProvider: "", + SSHPublicKeys: []string{"ssh-rsa AAABBB"}, }, spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.11.0", + Kubelet: defaultVersion, }, }, - ccProvider: &fakeCloudConfigProvider{name: "aws", config: "{aws-config:true}", err: nil}, + ccProvider: &fakeCloudConfigProvider{name: "", config: "", err: nil}, + DNSIPs: []net.IP{net.ParseIP("10.10.10.10"), net.ParseIP("10.10.10.11"), net.ParseIP("10.10.10.12")}, + kubernetesCACert: "CACert", + osConfig: &Config{DistUpgradeOnBoot: false}, + }, + { + name: "kubelet-version-without-v-prefix", + providerConfig: &providerconfig.Config{ + CloudProvider: "", + SSHPublicKeys: []string{"ssh-rsa AAABBB"}, + }, + spec: clusterv1alpha1.MachineSpec{ + ObjectMeta: metav1.ObjectMeta{Name: "node1"}, + Versions: clusterv1alpha1.MachineVersionInfo{ + Kubelet: "1.11.3", + }, + }, + ccProvider: &fakeCloudConfigProvider{name: "", config: "", err: nil}, DNSIPs: []net.IP{net.ParseIP("10.10.10.10")}, kubernetesCACert: "CACert", osConfig: &Config{DistUpgradeOnBoot: false}, }, { - name: "1.9.2-openstack-multiple-dns", + name: "multiple-ssh-keys", providerConfig: &providerconfig.Config{ - CloudProvider: "openstack", - SSHPublicKeys: []string{"ssh-rsa AAABBB", "ssh-rsa CCCDDD"}, + CloudProvider: "", + SSHPublicKeys: []string{"ssh-rsa AAABBB", "ssh-rsa CCCDDD", "ssh-rsa EEEFFF"}, }, spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.9.2", + Kubelet: "1.11.3", }, }, - ccProvider: &fakeCloudConfigProvider{name: "openstack", config: "{openstack-config:true}", err: nil}, - DNSIPs: []net.IP{net.ParseIP("10.10.10.10"), net.ParseIP("10.10.10.11"), net.ParseIP("10.10.10.12")}, + ccProvider: &fakeCloudConfigProvider{name: "", config: "", err: nil}, + DNSIPs: []net.IP{net.ParseIP("10.10.10.10")}, kubernetesCACert: "CACert", - osConfig: &Config{DistUpgradeOnBoot: true}, + osConfig: &Config{DistUpgradeOnBoot: false}, }, { - name: "openstack-kubelet-v-version-prefix", + name: "openstack", providerConfig: &providerconfig.Config{ CloudProvider: "openstack", - SSHPublicKeys: []string{"ssh-rsa AAABBB", "ssh-rsa CCCDDD"}, + SSHPublicKeys: []string{"ssh-rsa AAABBB"}, }, spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "v1.9.2", + Kubelet: defaultVersion, }, }, ccProvider: &fakeCloudConfigProvider{name: "openstack", config: "{openstack-config:true}", err: nil}, - DNSIPs: []net.IP{net.ParseIP("10.10.10.10")}, + DNSIPs: []net.IP{net.ParseIP("10.10.10.10"), net.ParseIP("10.10.10.11"), net.ParseIP("10.10.10.12")}, kubernetesCACert: "CACert", - osConfig: &Config{DistUpgradeOnBoot: true}, + osConfig: &Config{DistUpgradeOnBoot: false}, }, { name: "openstack-overwrite-cloud-config", providerConfig: &providerconfig.Config{ CloudProvider: "openstack", - SSHPublicKeys: []string{"ssh-rsa AAABBB", "ssh-rsa CCCDDD"}, + SSHPublicKeys: []string{"ssh-rsa AAABBB"}, OverwriteCloudConfig: stringPtr("custom\ncloud\nconfig"), }, spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "v1.9.2", + Kubelet: "v1.11.3", }, }, ccProvider: &fakeCloudConfigProvider{name: "openstack", config: "{openstack-config:true}", err: nil}, DNSIPs: []net.IP{net.ParseIP("10.10.10.10")}, kubernetesCACert: "CACert", - osConfig: &Config{DistUpgradeOnBoot: true}, + osConfig: &Config{DistUpgradeOnBoot: false}, }, - } + }...) for _, test := range tests { t.Run(test.name, func(t *testing.T) { @@ -186,30 +243,7 @@ func TestProvider_UserData(t *testing.T) { t.Fatal(err) } - golden := filepath.Join("testdata", test.name+".golden") - if *update { - ioutil.WriteFile(golden, []byte(userdata), 0644) - } - expected, err := ioutil.ReadFile(golden) - if err != nil { - t.Errorf("failed to read .golden file: %v", err) - } - - diff := difflib.UnifiedDiff{ - A: difflib.SplitLines(string(expected)), - B: difflib.SplitLines(userdata), - FromFile: "Fixture", - ToFile: "Current", - Context: 3, - } - diffStr, err := difflib.GetUnifiedDiffString(diff) - if err != nil { - t.Fatal(err) - } - - if diffStr != "" { - t.Errorf("got diff between expected and actual result: \n%s\n", diffStr) - } + testhelper.CompareOutput(t, test.name, userdata, *update) }) } } diff --git a/test/e2e/provisioning/helper.go b/test/e2e/provisioning/helper.go index 7ab57f3ee..cdb7fe3a9 100644 --- a/test/e2e/provisioning/helper.go +++ b/test/e2e/provisioning/helper.go @@ -17,10 +17,10 @@ var ( scenarios = buildScenarios() versions = []*semver.Version{ - semver.MustParse("v1.9.10"), - semver.MustParse("v1.10.7"), - semver.MustParse("v1.11.2"), - semver.MustParse("v1.12.0"), + semver.MustParse("v1.9.11"), + semver.MustParse("v1.10.8"), + semver.MustParse("v1.11.3"), + semver.MustParse("v1.12.1"), } operatingSystems = []providerconfig.OperatingSystem{