osp-centos.yaml

# Copyright 2021 The Operating System Manager contributors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

apiVersion: operatingsystemmanager.k8c.io/v1alpha1
kind: OperatingSystemProfile
metadata:
  name: osp-centos
  namespace: kube-system
spec:
  osName: "centos"
  osVersion: "7.7"
  version: "v1.5.0"
  provisioningUtility: "cloud-init"
  supportedCloudProviders:
    - name: "alibaba"
    - name: "aws"
    - name: "azure"
    - name: "baremetal"
    - name: "digitalocean"
    - name: "equinixmetal"
    - name: "kubevirt"
    - name: "nutanix"
    - name: "openstack"
    - name: "vsphere"

  bootstrapConfig:
    templates:
      configureProxyScript: |-
        {{- if .HTTPProxy }}
        cat <<EOF | tee -a /etc/environment
        HTTP_PROXY={{ .HTTPProxy }}
        http_proxy={{ .HTTPProxy }}
        HTTPS_PROXY={{ .HTTPProxy }}
        https_proxy={{ .HTTPProxy }}
        EOF
        {{- end }}

        {{- if .NoProxy }}
        cat <<EOF | tee -a /etc/environment
        NO_PROXY={{ .NoProxy }}
        no_proxy={{ .NoProxy }}
        EOF
        {{- end }}

        source /etc/environment

      configureHostCABundle: |-
        {{- if .HostCACert }}
        cat <<EOF | tee /etc/pki/ca-trust/source/anchors/custom.crt
        {{ .HostCACert }}
        EOF
        update-ca-trust
        {{- end }}

    files:
      - path: /opt/bin/supervise.sh
        permissions: 755
        content:
          inline:
            encoding: b64
            data: |
              #!/bin/bash
              set -xeuo pipefail
              while ! "$@"; do
                sleep 1
              done

      - path: /opt/bin/bootstrap
        permissions: 755
        content:
          inline:
            encoding: b64
            data: |
              #!/bin/bash
              set -xeuo pipefail

              # Check if bootstrap phase has already completed. This is required when we run `cloud-init init` again since it tries to re-run
              # the bootstrap cloud-config as well, from the userdata.
              if [ -f /etc/bootstrap-complete ]; then
                exit 0
              fi

              {{- /* Configure proxy as the first step to ensure that all the phases of provisioning respect the proxy environment. */}}
              {{- template "configureProxyScript" }}
              {{- template "configureHostCABundle" }}

              source /etc/os-release
              if [ "$ID" == "centos" ] && [ "$VERSION_ID" == "8" ]; then
                sudo sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-*
                sudo sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-*
              fi
              yum install epel-release -y
              yum install -y curl jq

              curl -s -k -v --header 'Authorization: Bearer {{ .Token }}' {{ .ServerURL }}/api/v1/namespaces/cloud-init-settings/secrets/{{ .SecretName }} | jq '.data["cloud-config"]' -r| base64 -d > /etc/cloud/cloud.cfg.d/{{ .SecretName }}.cfg
              cloud-init clean
              CLOUD_INIT_VERSION=$(cloud-init --version | awk '{print $2}')
              # Compare the semver values of cloud-init versions to determine the correct command to run.
              # This is required because the command line arguments for cloud-init changed in version 24.1, for details: https://github.com/canonical/cloud-init/releases/tag/24.1.
              if [[ $(echo -e "24.0.0\n$CLOUD_INIT_VERSION" | sort -V | head -n1) = "24.0.0" ]]; then
                  cloud-init init --file /etc/cloud/cloud.cfg.d/{{ .SecretName }}.cfg
              else
                  cloud-init --file /etc/cloud/cloud.cfg.d/{{ .SecretName }}.cfg init
              fi

              systemctl daemon-reload

              # cloud-init should only run on the first boot. From this point forward we don't need cloud-init anymore.
              systemctl disable cloud-init
              touch /etc/cloud/cloud-init.disabled

              # Bootstrap phase for the machine is complete.
              touch /etc/bootstrap-complete
              systemctl disable bootstrap.service

              # Start provisioning phase for the machine.
              systemctl restart setup.service

      - path: /etc/systemd/system/bootstrap.service
        permissions: 644
        content:
          inline:
            encoding: b64
            data: |
              [Install]
              WantedBy=multi-user.target

              [Unit]
              Requires=network-online.target
              After=network-online.target
              [Service]
              Type=oneshot
              RemainAfterExit=true
              EnvironmentFile=-/etc/environment
              ExecStart=/opt/bin/supervise.sh /opt/bin/bootstrap

    modules:
      runcmd:
        - systemctl restart bootstrap.service
        - systemctl daemon-reload

  provisioningConfig:
    supportedContainerRuntimes:
      - name: containerd
        files:
          - path: /etc/systemd/system/containerd.service.d/environment.conf
            content:
              inline:
                data: |
                  [Service]
                  Restart=always
                  EnvironmentFile=-/etc/environment

          - path: /etc/crictl.yaml
            content:
              inline:
                data: |
                  runtime-endpoint: unix:///run/containerd/containerd.sock

          - path: /etc/containerd/config.toml
            permissions: 600
            content:
              inline:
                encoding: b64
                data: |
                  {{ .ContainerRuntimeConfig }}
        templates:
          containerRuntimeInstallation: |-
            yum install -y yum-utils
            yum-config-manager --add-repo=https://download.docker.com/linux/centos/docker-ce.repo
            {{- /*
                Due to DNF modules we have to do this on docker-ce repo
                More info at: https://bugzilla.redhat.com/show_bug.cgi?id=1756473
            */}}
            yum-config-manager --save --setopt=docker-ce-stable.module_hotfixes=true

            yum install -y containerd.io-1.6* yum-plugin-versionlock
            yum versionlock add containerd.io

            systemctl daemon-reload
            systemctl enable --now containerd

    templates:
      safeDownloadBinariesScript: |-
        {{- /* setup some common directories */}}
        opt_bin=/opt/bin
        usr_local_bin=/usr/local/bin
        cni_bin_dir=/opt/cni/bin

        {{- /* create all the necessary dirs */}}
        mkdir -p /etc/cni/net.d /etc/kubernetes/manifests "$opt_bin" "$cni_bin_dir"
        {{- /* HOST_ARCH can be defined outside of machine-controller (in kubeone for example) */}}
        arch=${HOST_ARCH-}
        if [ -z "$arch" ]
        then
        case $(uname -m) in
        x86_64)
            arch="amd64"
            ;;
        aarch64)
            arch="arm64"
            ;;
        *)
            echo "unsupported CPU architecture, exiting"
            exit 1
            ;;
        esac
        fi

        {{- /* # CNI variables */}}
        CNI_VERSION="${CNI_VERSION:-v1.4.1}"
        cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION"
        cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz"

        {{- /* download CNI */}}
        curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename"

        {{- /* download CNI checksum */}}
        cni_sum=$(curl -Lf "$cni_base_url/$cni_filename.sha256")
        cd "$cni_bin_dir"

        {{- /* verify CNI checksum */}}
        sha256sum -c <<<"$cni_sum"

        {{- /* unpack CNI */}}
        tar xvf "$cni_filename"


        {{- /* set ownership of the CNI binaries to root:root. Required due to https://github.com/cilium/cilium/issues/23838 */}}
        chown -R root:root "$cni_bin_dir"

        {{- /* # cri-tools variables */}}
        {{- if semverCompare "~1.27.0" .KubeVersion }}
        CRI_TOOLS_RELEASE="v1.27.1"
        {{- else if semverCompare "~1.28.0" .KubeVersion }}
        CRI_TOOLS_RELEASE="v1.28.0"
        {{- else if semverCompare "~1.29.0" .KubeVersion }}
        CRI_TOOLS_RELEASE="v1.29.0"
        {{- else if semverCompare "~1.30.0" .KubeVersion }}
        CRI_TOOLS_RELEASE="v1.30.0"
        {{- else }}
        {{- /* Fallback to the latest version */}}
        CRI_TOOLS_RELEASE="v1.30.0"
        {{- end }}

        CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.29.0}"
        cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/${CRI_TOOLS_RELEASE}"
        cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz"

        {{- /* download cri-tools */}}
        curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename"

        {{- /* download cri-tools checksum */}}
        {{- /* the cri-tools checksum file provides only the checksum without the file name, so we need to handle it specially */}}
        cri_tools_sum_value=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256")
        cri_tools_sum="$cri_tools_sum_value $cri_tools_filename"
        cd "$opt_bin"

        {{- /* verify cri-tools checksum */}}
        sha256sum -c <<<"$cri_tools_sum"

        {{- /* unpack cri-tools and symlink to path so it's available to all users */}}
        tar xvf "$cri_tools_filename"
        rm -f "$cri_tools_filename"
        ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped"
        cd -

        {{- /* kubelet */}}
        KUBE_VERSION="${KUBE_VERSION:-{{ .KubeVersion }}}"
        kube_dir="$opt_bin/kubernetes-$KUBE_VERSION"
        kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch"
        kube_sum_file="$kube_dir/sha256"

        {{- /* create versioned kube dir */}}
        mkdir -p "$kube_dir"
        : >"$kube_sum_file"

        for bin in kubelet kubeadm kubectl; do
            {{- /* download kube binary */}}
            curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin"
            chmod +x "$kube_dir/$bin"

            {{- /* download kube binary checksum */}}
            sum=$(curl -Lf "$kube_base_url/$bin.sha256")

            {{- /* save kube binary checksum */}}
            echo "$sum  $kube_dir/$bin" >>"$kube_sum_file"
        done

        {{- /* check kube binaries checksum */}}
        sha256sum -c "$kube_sum_file"

        for bin in kubelet kubeadm kubectl; do
            {{- /* link kube binaries from verioned dir to $opt_bin */}}
            ln -sf "$kube_dir/$bin" "$opt_bin"/$bin
        done

    files:
      - path: /opt/bin/health-monitor.sh
        permissions: 755
        content:
          inline:
            encoding: b64
            data: |
              #!/usr/bin/env bash

              # Copyright 2016 The Kubernetes Authors.
              #
              # Licensed under the Apache License, Version 2.0 (the "License");
              # you may not use this file except in compliance with the License.
              # You may obtain a copy of the License at
              #
              #     http://www.apache.org/licenses/LICENSE-2.0
              #
              # Unless required by applicable law or agreed to in writing, software
              # distributed under the License is distributed on an "AS IS" BASIS,
              # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
              # See the License for the specific language governing permissions and
              # limitations under the License.

              # This script is for master and node instance health monitoring, which is
              # packed in kube-manifest tarball. It is executed through a systemd service
              # in cluster/gce/gci/<master/node>.yaml. The env variables come from an env
              # file provided by the systemd service.

              # This script is a slightly adjusted version of
              # https://github.com/kubernetes/kubernetes/blob/e1a1aa211224fcd9b213420b80b2ae680669683d/cluster/gce/gci/health-monitor.sh
              # Adjustments are:
              # * Kubelet health port is 10248 not 10255
              # * Removal of all all references to the KUBE_ENV file

              set -o nounset
              set -o pipefail

              # We simply kill the process when there is a failure. Another systemd service will
              # automatically restart the process.
              function container_runtime_monitoring() {
                local -r max_attempts=5
                local attempt=1
                local -r container_runtime_name="${CONTAINER_RUNTIME_NAME:-docker}"
                # We still need to use 'docker ps' when container runtime is "docker". This is because
                # dockershim is still part of kubelet today. When kubelet is down, crictl pods
                # will also fail, and docker will be killed. This is undesirable especially when
                # docker live restore is disabled.
                local healthcheck_command="docker ps"
                if [[ "${CONTAINER_RUNTIME:-docker}" != "docker" ]]; then
                  healthcheck_command="crictl pods"
                fi
                # Container runtime startup takes time. Make initial attempts before starting
                # killing the container runtime.
                until timeout 60 ${healthcheck_command} > /dev/null; do
                  if ((attempt == max_attempts)); then
                    echo "Max attempt ${max_attempts} reached! Proceeding to monitor container runtime healthiness."
                    break
                  fi
                  echo "$attempt initial attempt \"${healthcheck_command}\"! Trying again in $attempt seconds..."
                  sleep "$((2 ** attempt++))"
                done
                while true; do
                  if ! timeout 60 ${healthcheck_command} > /dev/null; then
                    echo "Container runtime ${container_runtime_name} failed!"
                    if [[ "$container_runtime_name" == "docker" ]]; then
                      # Dump stack of docker daemon for investigation.
                      # Log file name looks like goroutine-stacks-TIMESTAMP and will be saved to
                      # the exec root directory, which is /var/run/docker/ on Ubuntu and COS.
                      pkill -SIGUSR1 dockerd
                    fi
                    systemctl kill --kill-who=main "${container_runtime_name}"
                    # Wait for a while, as we don't want to kill it again before it is really up.
                    sleep 120
                  else
                    sleep "${SLEEP_SECONDS}"
                  fi
                done
              }

              function kubelet_monitoring() {
                echo "Wait for 2 minutes for kubelet to be functional"
                sleep 120
                local -r max_seconds=10
                local output=""
                while true; do
                  local failed=false

                  if journalctl -u kubelet -n 1 | grep -q "use of closed network connection"; then
                    failed=true
                    echo "Kubelet stopped posting node status. Restarting"
                  elif ! output=$(curl -m "${max_seconds}" -f -s -S http://127.0.0.1:10248/healthz 2>&1); then
                    failed=true
                    # Print the response and/or errors.
                    echo "$output"
                  fi

                  if [[ "$failed" == "true" ]]; then
                    echo "Kubelet is unhealthy!"
                    systemctl kill kubelet
                    # Wait for a while, as we don't want to kill it again before it is really up.
                    sleep 60
                  else
                    sleep "${SLEEP_SECONDS}"
                  fi
                done
              }

              ############## Main Function ################
              if [[ "$#" -ne 1 ]]; then
                echo "Usage: health-monitor.sh <container-runtime/kubelet>"
                exit 1
              fi

              SLEEP_SECONDS=10
              component=$1
              echo "Start kubernetes health monitoring for ${component}"
              if [[ "${component}" == "container-runtime" ]]; then
                container_runtime_monitoring
              elif [[ "${component}" == "kubelet" ]]; then
                kubelet_monitoring
              else
                echo "Health monitoring for component ${component} is not supported!"
              fi

      - path: /etc/systemd/journald.conf.d/max_disk_use.conf
        content:
          inline:
            encoding: b64
            data: |
              [Journal]
              SystemMaxUse=5G

      - path: /opt/load-kernel-modules.sh
        permissions: 755
        content:
          inline:
            encoding: b64
            data: |
              #!/usr/bin/env bash
              set -euo pipefail

              modprobe ip_vs
              modprobe ip_vs_rr
              modprobe ip_vs_wrr
              modprobe ip_vs_sh

              if modinfo nf_conntrack_ipv4 &> /dev/null; then
                modprobe nf_conntrack_ipv4
              else
                modprobe nf_conntrack
              fi

      - path: /etc/sysctl.d/k8s.conf
        content:
          inline:
            encoding: b64
            data: |
              net.bridge.bridge-nf-call-ip6tables = 1
              net.bridge.bridge-nf-call-iptables = 1
              kernel.panic_on_oops = 1
              kernel.panic = 10
              net.ipv4.ip_forward = 1
              {{- if or (eq .NetworkIPFamily "IPv4+IPv6") (eq .NetworkIPFamily "IPv6+IPv4") (eq .NetworkIPFamily "IPv6") }}
              net.ipv6.conf.all.forwarding = 1
              # Configure Linux to accept router advertisements to ensure the default
              # IPv6 route is not removed from the routing table when the Docker service starts.
              # For more information: https://github.com/docker/for-linux/issues/844
              net.ipv6.conf.all.accept_ra		= 2
              {{- end }}
              vm.overcommit_memory = 1
              fs.inotify.max_user_watches = 1048576
              fs.inotify.max_user_instances = 8192

      - path: /etc/selinux/config
        content:
          inline:
            encoding: b64
            data: |
              # This file controls the state of SELinux on the system.
              # SELINUX= can take one of these three values:
              #     enforcing - SELinux security policy is enforced.
              #     permissive - SELinux prints warnings instead of enforcing.
              #     disabled - No SELinux policy is loaded.
              SELINUX=permissive
              # SELINUXTYPE= can take one of three two values:
              #     targeted - Targeted processes are protected,
              #     minimum - Modification of targeted policy. Only selected processes are protected.
              #     mls - Multi Level Security protection.
              SELINUXTYPE=targeted

      - path: /opt/bin/setup
        permissions: 755
        content:
          inline:
            encoding: b64
            data: |
              #!/bin/bash
              set -xeuo pipefail

              setenforce 0 || true

              {{- /* As we added some modules and don't want to reboot, restart the service */}}
              systemctl restart systemd-modules-load.service
              sysctl --system

              # Override hostname if /etc/machine-name exists
              if [ -x "$(command -v hostnamectl)" ] && [ -s /etc/machine-name ]; then
                machine_name=$(cat /etc/machine-name)
                hostnamectl set-hostname ${machine_name}
              fi

              yum install -y \
                device-mapper-persistent-data \
                lvm2 \
                ebtables \
                ethtool \
                nfs-utils \
                bash-completion \
                sudo \
                socat \
                wget \
                curl \
                {{- if or (eq .CloudProviderName "vsphere") (eq .CloudProviderName "vmware-cloud-director") }}
                open-vm-tools \
                {{- end }}
                {{- if eq .CloudProviderName "nutanix" }}
                iscsi-initiator-utils \
                {{- end }}
                ipvsadm

              systemctl disable --now firewalld || true

              {{- /* iscsid service is required on Nutanix machines for CSI driver to attach volumes. */}}
              {{- if eq .CloudProviderName "nutanix" }}
              systemctl enable --now iscsid
              {{ end }}

              {{- template "containerRuntimeInstallation" }}

              {{- template "safeDownloadBinariesScript" }}

              mkdir -p /etc/systemd/system/kubelet.service.d/
              # set kubelet nodeip environment variable
              /opt/bin/setup_net_env.sh

              {{ if eq .CloudProviderName "vsphere" }}
              systemctl enable --now vmtoolsd.service
              {{ end -}}

              {{- /* fetch kubelet bootstrapping kubeconfig */}}
              curl -s -k -v --header 'Authorization: Bearer {{ .Token }}' {{ .ServerURL }}/api/v1/namespaces/cloud-init-settings/secrets/{{ .BootstrapKubeconfigSecretName }} | jq '.data["kubeconfig"]' -r| base64 -d > /etc/kubernetes/bootstrap-kubelet.conf

              systemctl enable --now kubelet
              systemctl enable --now --no-block kubelet-healthcheck.service
              systemctl disable setup.service

      - path: /etc/systemd/system/kubelet.service
        content:
          inline:
            encoding: b64
            data: |
              [Unit]
              After={{ .ContainerRuntime }}.service
              Requires={{ .ContainerRuntime }}.service

              Description=kubelet: The Kubernetes Node Agent
              Documentation=https://kubernetes.io/docs/home/

              [Service]
              User=root
              Restart=always
              StartLimitInterval=0
              RestartSec=10
              CPUAccounting=true
              MemoryAccounting=true

              Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/"
              EnvironmentFile=-/etc/environment

              ExecStartPre=/bin/bash /opt/disable-swap.sh
              ExecStartPre=/bin/bash /opt/load-kernel-modules.sh
              ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh
              ExecStart=/opt/bin/kubelet \
                --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \
                --kubeconfig=/var/lib/kubelet/kubeconfig \
                --config=/etc/kubernetes/kubelet.conf \
                --cert-dir=/etc/kubernetes/pki \
                {{- if .ExternalCloudProvider }}
                --cloud-provider=external \
                {{- /* In-tree cloud providers have been disabled starting from k8s 1.29. For more information: https://github.com/kubernetes/kubernetes/pull/117503 */}}
                {{- else if and (.InTreeCCMAvailable) (semverCompare "<1.29" .KubeVersion) }}
                --cloud-provider={{- .CloudProviderName }} \
                --cloud-config=/etc/kubernetes/cloud-config \
                {{- end }}
                {{- if ne .CloudProviderName "aws" }}
                --hostname-override=${KUBELET_HOSTNAME} \
                {{- else if and (eq .CloudProviderName "aws") (.ExternalCloudProvider) }}
                --hostname-override=${KUBELET_HOSTNAME} \
                {{- end }}
                --exit-on-lock-contention \
                --lock-file=/tmp/kubelet.lock \
                {{- if .PauseImage }}
                --pod-infra-container-image={{ .PauseImage }} \
                {{- end }}
                {{- if .InitialTaints }}
                --register-with-taints={{- .InitialTaints }} \
                {{- end }}
                {{- if eq .ContainerRuntime "containerd" }}
                --container-runtime-endpoint=unix:///run/containerd/containerd.sock \
                {{- end }}
                {{- /* If external or in-tree CCM is in use we don't need to set --node-ip as the cloud provider will know what IPs to return.  */}}
                {{- if not (and (or (eq .NetworkIPFamily "IPv4+IPv6") (eq .NetworkIPFamily "IPv6+IPv4")) (or (.InTreeCCMAvailable) (.ExternalCloudProvider))) }}
                --node-ip ${KUBELET_NODE_IP}
                {{- end }}

              [Install]
              WantedBy=multi-user.target

      - path: /etc/kubernetes/cloud-config
        permissions: 600
        content:
          inline:
            encoding: b64
            data: |
              {{ .CloudConfig }}

      - path: /opt/bin/setup_net_env.sh
        permissions: 755
        content:
          inline:
            encoding: b64
            data: |
              #!/usr/bin/env bash
              echodate() {
                echo "[$(date -Is)]" "$@"
              }

              # get the default interface IP address
              {{- if eq .NetworkIPFamily "IPv6" }}
              DEFAULT_IFC_IP=$(ip -o -6 route get  1:: | grep -oP "src \K\S+")
              {{- else if eq .NetworkIPFamily "IPv4+IPv6" }}
              DEFAULT_IFC_IPv4=$(ip -o route get  1 | grep -oP "src \K\S+")
              DEFAULT_IFC_IPv6=$(ip -o -6 route get  1:: | grep -oP "src \K\S+")

              if [ -z "${DEFAULT_IFC_IPv6}" ]
              then
                echodate "Failed to get IPv6 address for the default route interface"
                exit 1
              fi
              DEFAULT_IFC_IP=$DEFAULT_IFC_IPv4,$DEFAULT_IFC_IPv6
              {{- else if eq .NetworkIPFamily "IPv6+IPv4" }}
              DEFAULT_IFC_IPv4=$(ip -o route get  1 | grep -oP "src \K\S+")
              DEFAULT_IFC_IPv6=$(ip -o -6 route get  1:: | grep -oP "src \K\S+")

              if [ -z "${DEFAULT_IFC_IPv6}" ]
              then
                echodate "Failed to get IPv6 address for the default route interface"
                exit 1
              fi

              DEFAULT_IFC_IP=$DEFAULT_IFC_IPv6,$DEFAULT_IFC_IPv4
              {{- else }}
              DEFAULT_IFC_IP=$(ip -o  route get 1 | grep -oP "src \K\S+")
              {{- end }}

              if [ -z "${DEFAULT_IFC_IP}" ]
              then
                echodate "Failed to get IP address for the default route interface"
                exit 1
              fi

              # get the full hostname
              FULL_HOSTNAME=$(hostname -f)
              # if /etc/machine-name is not empty then use the hostname from there
              if [ -s /etc/machine-name ]; then
                  FULL_HOSTNAME=$(cat /etc/machine-name)
              fi

              # write the nodeip_env file
              # we need the line below because flatcar has the same string "coreos" in that file
              if grep -q coreos /etc/os-release
              then
                echo "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf
              else
                mkdir -p /etc/systemd/system/kubelet.service.d
                echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf
              fi

      - path: /etc/kubernetes/pki/ca.crt
        content:
          inline:
            encoding: b64
            data: |
              {{ .KubernetesCACert }}

      - path: /etc/systemd/system/setup.service
        permissions: 644
        content:
          inline:
            encoding: b64
            data: |
              [Install]
              WantedBy=multi-user.target

              [Unit]
              Requires=network-online.target
              After=network-online.target

              [Service]
              Type=oneshot
              RemainAfterExit=true
              EnvironmentFile=-/etc/environment
              ExecStart=/opt/bin/supervise.sh /opt/bin/setup

      - path: /etc/profile.d/opt-bin-path.sh
        permissions: 644
        content:
          inline:
            encoding: b64
            data: |
              export PATH="/opt/bin:$PATH"

      - path: /etc/kubernetes/kubelet.conf
        content:
          inline:
            encoding: b64
            data: |
              apiVersion: kubelet.config.k8s.io/v1beta1
              kind: KubeletConfiguration
              authentication:
                anonymous:
                  enabled: false
                webhook:
                  cacheTTL: 0s
                  enabled: true
                x509:
                  clientCAFile: /etc/kubernetes/pki/ca.crt
              authorization:
                mode: Webhook
                webhook:
                  cacheAuthorizedTTL: 0s
                  cacheUnauthorizedTTL: 0s
              cgroupDriver: systemd
              clusterDNS:
              {{- range .ClusterDNSIPs }}
              - "{{ . }}"
              {{- end }}
              clusterDomain: cluster.local
              {{- if .ContainerLogMaxSize }}
              containerLogMaxSize: {{ .ContainerLogMaxSize }}
              {{- else }}
              containerLogMaxSize: 100Mi
              {{- end }}
              {{- if .ContainerLogMaxFiles }}
              containerLogMaxFiles: {{ .ContainerLogMaxFiles }}
              {{- else }}
              containerLogMaxFiles: 5
              {{- end }}
              cpuManagerReconcilePeriod: 0s
              evictionPressureTransitionPeriod: 0s
              featureGates:
              {{- if .KubeletFeatureGates -}}
                {{ range $key, $val := .KubeletFeatureGates }}
                {{ $key }}: {{ $val }}
                {{- end -}}
              {{- end }}
              fileCheckFrequency: 0s
              httpCheckFrequency: 0s
              imageMinimumGCAge: 0s
              logging:
                flushFrequency: 0
                options:
                  json:
                    infoBufferSize: "0"
                verbosity: 0
              memorySwap: {}
              nodeStatusReportFrequency: 0s
              nodeStatusUpdateFrequency: 0s
              protectKernelDefaults: true
              readOnlyPort: 0
              rotateCertificates: true
              runtimeRequestTimeout: 0s
              serverTLSBootstrap: true
              shutdownGracePeriod: 0s
              shutdownGracePeriodCriticalPods: 0s
              staticPodPath: /etc/kubernetes/manifests
              streamingConnectionIdleTimeout: 0s
              syncFrequency: 0s
              # Enable parallel image pulling.
              serializeImagePulls: false
              # Set max parallel image pulls to 10.
              maxParallelImagePulls: 10
              kubeReserved:
              {{- if .KubeReserved -}}
                {{ range $key, $val := .KubeReserved }}
                {{ $key }}: {{ $val }}
                {{- end -}}
              {{- else }}
                cpu: 200m
                ephemeral-storage: 1Gi
                memory: 200Mi
              {{- end }}
              systemReserved:
              {{- if .SystemReserved -}}
                {{ range $key, $val := .SystemReserved }}
                {{ $key }}: {{ $val }}
                {{- end -}}
              {{- else }}
                cpu: 200m
                ephemeral-storage: 1Gi
                memory: 200Mi
              {{- end }}
              evictionHard:
              {{- if .EvictionHard -}}
                {{ range $key, $val := .EvictionHard }}
                {{ $key }}: {{ $val }}
                {{- end -}}
              {{- else }}
                imagefs.available: 15%
                memory.available: 100Mi
                nodefs.available: 10%
                nodefs.inodesFree: 5%
              {{- end }}
              {{- if .MaxPods }}
              maxPods: {{ .MaxPods }}
              {{- end }}
              tlsCipherSuites:
              - TLS_AES_128_GCM_SHA256
              - TLS_AES_256_GCM_SHA384
              - TLS_CHACHA20_POLY1305_SHA256
              - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256
              - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384
              - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305
              - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256
              - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384
              - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305
              volumePluginDir: /var/lib/kubelet/volumeplugins
              volumeStatsAggPeriod: 0s

      - path: /etc/systemd/system/kubelet-healthcheck.service
        permissions: 644
        content:
          inline:
            encoding: b64
            data: |
              [Unit]
              Requires=kubelet.service
              After=kubelet.service

              [Service]
              EnvironmentFile=-/etc/environment
              ExecStart=/opt/bin/health-monitor.sh kubelet

              [Install]
              WantedBy=multi-user.target

      - path: /opt/disable-swap.sh
        permissions: 755
        content:
          inline:
            encoding: b64
            data: |
              #!/usr/bin/env bash
              set -euo pipefail

              # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud
              # providers swap gets enabled on reboot or after the setup script has finished executing.
              sed -i.orig '/.*swap.*/d' /etc/fstab
              swapoff -a