Add CentOS support to VSphere

docs/vsphere.md

@@ -1,7 +1,9 @@
-# VMWware vSphere
+# VMware vSphere
 
 To use the machine-controller to create machines on VMWare vsphere, you must first
-create a template. Currently Ubuntu and Container Linux are supported.
+create a template.
+
+Ubuntu & CoreOS:
 
 1. Go into the VSphere WebUI, select your datacenter, right click onto it and choose "Deploy OVF Template"
 2. Fill in the "URL" field with the appropriate url:
@@ -12,3 +14,10 @@ create a template. Currently Ubuntu and Container Linux are supported.
 5. Select the same network you want to use for your machines
 6. Leave everyhting in the "Customize Template" and "Ready to complete" dialog as it is
 7. Wait until the VM got fully imported and the "Snapshots" => "Create Snapshot" button is not grayed out anymore
+
+CentOS:
+
+1. Download the CentOS cloud image to your local workstation from here: `https://cloud.centos.org/centos/7/images/CentOS-7-x86_64-GenericCloud.qcow2`
+1. Convert it to vmdk: `qemu-img convert -f qcow2 -O vmdk CentOS-7-x86_64-GenericCloud.qcow2 CentOS-7-x86_64-GenericCloud.vmdk`
+1. Upload it to a Datastore of your Vsphere installation
+1. Create a new virtual machine that uses the uploaded vmdk as rootdisk

pkg/userdata/centos/testdata/kubelet-v1.10-aws.golden

@@ -51,10 +51,14 @@ write_files:
 
     setenforce 0 || true
 
-    # As we added some modules and don't want to reboot, restart the service 
+    # As we added some modules and don't want to reboot, restart the service
     systemctl restart systemd-modules-load.service
     sysctl --system
 
+    # The normal way of setting it via cloud-init is broken:
+    # https://bugs.launchpad.net/cloud-init/+bug/1662542
+    hostnamectl set-hostname node1
+
     yum install -y docker-1.13.1 \
       ebtables \
       ethtool \
@@ -87,8 +91,6 @@ write_files:
         curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/8b5b66e4910a6228dfaecccaa0a3b05ec4902f8e/pkg/userdata/scripts/health-monitor.sh
         chmod +x /opt/bin/health-monitor.sh
     fi
-
-
     systemctl enable --now docker
     systemctl enable --now kubelet
     systemctl enable --now --no-block kubelet-healthcheck.service

pkg/userdata/centos/testdata/kubelet-v1.11-aws.golden

@@ -51,10 +51,14 @@ write_files:
 
     setenforce 0 || true
 
-    # As we added some modules and don't want to reboot, restart the service 
+    # As we added some modules and don't want to reboot, restart the service
     systemctl restart systemd-modules-load.service
     sysctl --system
 
+    # The normal way of setting it via cloud-init is broken:
+    # https://bugs.launchpad.net/cloud-init/+bug/1662542
+    hostnamectl set-hostname node1
+
     yum install -y docker-1.13.1 \
       ebtables \
       ethtool \
@@ -87,8 +91,6 @@ write_files:
         curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/8b5b66e4910a6228dfaecccaa0a3b05ec4902f8e/pkg/userdata/scripts/health-monitor.sh
         chmod +x /opt/bin/health-monitor.sh
     fi
-
-
     systemctl enable --now docker
     systemctl enable --now kubelet
     systemctl enable --now --no-block kubelet-healthcheck.service

pkg/userdata/centos/testdata/kubelet-v1.12-aws.golden

@@ -51,10 +51,14 @@ write_files:
 
     setenforce 0 || true
 
-    # As we added some modules and don't want to reboot, restart the service 
+    # As we added some modules and don't want to reboot, restart the service
     systemctl restart systemd-modules-load.service
     sysctl --system
 
+    # The normal way of setting it via cloud-init is broken:
+    # https://bugs.launchpad.net/cloud-init/+bug/1662542
+    hostnamectl set-hostname node1
+
     yum install -y docker-1.13.1 \
       ebtables \
       ethtool \
@@ -87,8 +91,6 @@ write_files:
         curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/8b5b66e4910a6228dfaecccaa0a3b05ec4902f8e/pkg/userdata/scripts/health-monitor.sh
         chmod +x /opt/bin/health-monitor.sh
     fi
-
-
     systemctl enable --now docker
     systemctl enable --now kubelet
     systemctl enable --now --no-block kubelet-healthcheck.service

pkg/userdata/centos/testdata/kubelet-v1.12-vsphere.golden

@@ -0,0 +1,259 @@
+#cloud-config
+hostname: node1
+
+ssh_pwauth: no
+
+write_files:
+- path: "/etc/systemd/journald.conf.d/max_disk_use.conf"
+  content: |
+    [Journal]
+    SystemMaxUse=5G
+
+
+- path: "/etc/modules-load.d/k8s.conf"
+  content: |
+    ip_vs
+    ip_vs_rr
+    ip_vs_wrr
+    ip_vs_sh
+    nf_conntrack_ipv4
+
+
+- path: "/etc/sysctl.d/k8s.conf"
+  content: |
+    net.bridge.bridge-nf-call-ip6tables = 1
+    net.bridge.bridge-nf-call-iptables = 1
+    kernel.panic_on_oops = 1
+    kernel.panic = 10
+    net.ipv4.ip_forward = 1
+    vm.overcommit_memory = 1
+
+
+- path: /etc/sysconfig/selinux
+  content: |
+    # This file controls the state of SELinux on the system.
+    # SELINUX= can take one of these three values:
+    #     enforcing - SELinux security policy is enforced.
+    #     permissive - SELinux prints warnings instead of enforcing.
+    #     disabled - No SELinux policy is loaded.
+    SELINUX=permissive
+    # SELINUXTYPE= can take one of three two values:
+    #     targeted - Targeted processes are protected,
+    #     minimum - Modification of targeted policy. Only selected processes are protected.
+    #     mls - Multi Level Security protection.
+    SELINUXTYPE=targeted
+
+- path: "/opt/bin/setup"
+  permissions: "0777"
+  content: |
+    #!/bin/bash
+    set -xeuo pipefail
+
+    setenforce 0 || true
+
+    # As we added some modules and don't want to reboot, restart the service
+    systemctl restart systemd-modules-load.service
+    sysctl --system
+
+    # The normal way of setting it via cloud-init is broken:
+    # https://bugs.launchpad.net/cloud-init/+bug/1662542
+    hostnamectl set-hostname node1
+
+    yum install -y docker-1.13.1 \
+      ebtables \
+      ethtool \
+      nfs-utils \
+      bash-completion \
+      sudo \
+      socat \
+      wget \
+      curl \
+      ipvsadm \
+      open-vm-tools
+
+    #setup some common directories
+    mkdir -p /opt/bin/
+    mkdir -p /var/lib/calico
+    mkdir -p /etc/kubernetes/manifests
+    mkdir -p /etc/cni/net.d
+    mkdir -p /opt/cni/bin
+
+    # cni
+    if [ ! -f /opt/cni/bin/loopback ]; then
+        curl -L https://github.com/containernetworking/plugins/releases/download/v0.6.0/cni-plugins-amd64-v0.6.0.tgz | tar -xvzC /opt/cni/bin -f -
+    fi
+    # kubelet
+    if [ ! -f /opt/bin/kubelet ]; then
+        curl -Lfo /opt/bin/kubelet https://storage.googleapis.com/kubernetes-release/release/v1.12.0/bin/linux/amd64/kubelet
+        chmod +x /opt/bin/kubelet
+    fi
+
+    if [[ ! -x /opt/bin/health-monitor.sh ]]; then
+        curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/8b5b66e4910a6228dfaecccaa0a3b05ec4902f8e/pkg/userdata/scripts/health-monitor.sh
+        chmod +x /opt/bin/health-monitor.sh
+    fi
+
+    systemctl enable --now vmtoolsd.service
+    systemctl enable --now docker
+    systemctl enable --now kubelet
+    systemctl enable --now --no-block kubelet-healthcheck.service
+    systemctl enable --now --no-block docker-healthcheck.service
+
+- path: "/opt/bin/supervise.sh"
+  permissions: "0755"
+  content: |
+    #!/bin/bash
+    set -xeuo pipefail
+    while ! "$@"; do
+      sleep 1
+    done
+
+- path: "/etc/systemd/system/kubelet.service"
+  content: |
+    [Unit]
+    After=docker.service
+    Requires=docker.service
+
+    Description=kubelet: The Kubernetes Node Agent
+    Documentation=https://kubernetes.io/docs/home/
+
+    [Service]
+    Restart=always
+    StartLimitInterval=0
+    RestartSec=10
+
+    Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/"
+
+    ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \
+      --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \
+      --kubeconfig=/etc/kubernetes/kubelet.conf \
+      --pod-manifest-path=/etc/kubernetes/manifests \
+      --allow-privileged=true \
+      --network-plugin=cni \
+      --cni-conf-dir=/etc/cni/net.d \
+      --cni-bin-dir=/opt/cni/bin \
+      --authorization-mode=Webhook \
+      --client-ca-file=/etc/kubernetes/pki/ca.crt \
+      --rotate-certificates=true \
+      --cert-dir=/etc/kubernetes/pki \
+      --authentication-token-webhook=true \
+      --cloud-provider=vsphere \
+      --cloud-config=/etc/kubernetes/cloud-config \
+      --hostname-override=node1 \
+      --read-only-port=0 \
+      --exit-on-lock-contention \
+      --lock-file=/tmp/kubelet.lock \
+      --anonymous-auth=false \
+      --protect-kernel-defaults=true \
+      --cluster-dns= \
+      --cluster-domain=cluster.local
+
+    [Install]
+    WantedBy=multi-user.target
+
+- path: "/etc/systemd/system/kubelet.service.d/extras.conf"
+  content: |
+    [Service]
+    Environment="KUBELET_EXTRA_ARGS=--cgroup-driver=systemd"
+
+- path: "/etc/kubernetes/cloud-config"
+  content: |
+    {config:true}
+
+- path: "/etc/kubernetes/bootstrap-kubelet.conf"
+  content: |
+    apiVersion: v1
+    clusters:
+    - cluster:
+        certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t
+        server: https://server:443
+      name: ""
+    contexts: []
+    current-context: ""
+    kind: Config
+    preferences: {}
+    users:
+    - name: ""
+      user:
+        token: my-token
+
+
+- path: "/etc/kubernetes/pki/ca.crt"
+  content: |
+    -----BEGIN CERTIFICATE-----
+    MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV
+    BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG
+    A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3
+    DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0
+    NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG
+    cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv
+    c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B
+    AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS
+    R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT
+    ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk
+    JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3
+    mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW
+    caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G
+    A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt
+    hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB
+    MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES
+    MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv
+    bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h
+    U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao
+    eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4
+    UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD
+    58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n
+    sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF
+    kPe6XoSbiLm/kxk32T0=
+    -----END CERTIFICATE-----
+
+- path: "/etc/systemd/system/setup.service"
+  permissions: "0644"
+  content: |
+    [Install]
+    WantedBy=multi-user.target
+
+    [Unit]
+    Requires=network-online.target
+    After=network-online.target
+
+    [Service]
+    Type=oneshot
+    RemainAfterExit=true
+    ExecStart=/opt/bin/supervise.sh /opt/bin/setup
+
+- path: "/etc/profile.d/opt-bin-path.sh"
+  permissions: "0644"
+  content: |
+    export PATH="/opt/bin:$PATH"
+
+- path: /etc/systemd/system/kubelet-healthcheck.service
+  permissions: "0644"
+  content: |
+    [Unit]
+    Requires=kubelet.service
+    After=kubelet.service
+
+    [Service]
+    ExecStart=/opt/bin/health-monitor.sh kubelet
+
+    [Install]
+    WantedBy=multi-user.target
+
+
+- path: /etc/systemd/system/docker-healthcheck.service
+  permissions: "0644"
+  content: |
+    [Unit]
+    Requires=docker.service
+    After=docker.service
+
+    [Service]
+    ExecStart=/opt/bin/health-monitor.sh container-runtime
+
+    [Install]
+    WantedBy=multi-user.target
+
+
+runcmd:
+- systemctl enable --now setup.service

pkg/userdata/centos/testdata/kubelet-v1.9-aws.golden

@@ -51,10 +51,14 @@ write_files:
 
     setenforce 0 || true
 
-    # As we added some modules and don't want to reboot, restart the service 
+    # As we added some modules and don't want to reboot, restart the service
     systemctl restart systemd-modules-load.service
     sysctl --system
 
+    # The normal way of setting it via cloud-init is broken:
+    # https://bugs.launchpad.net/cloud-init/+bug/1662542
+    hostnamectl set-hostname node1
+
     yum install -y docker-1.13.1 \
       ebtables \
       ethtool \
@@ -87,8 +91,6 @@ write_files:
         curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/8b5b66e4910a6228dfaecccaa0a3b05ec4902f8e/pkg/userdata/scripts/health-monitor.sh
         chmod +x /opt/bin/health-monitor.sh
     fi
-
-
     systemctl enable --now docker
     systemctl enable --now kubelet
     systemctl enable --now --no-block kubelet-healthcheck.service