Add cluster directory and health-monitor.sh.

Signed-off-by: Lantao Liu <lantaol@google.com>
containerd · Oct 1, 2020 · 2ad761d · 2ad761d
1 parent 02d93ad
commit 2ad761d
Show file tree

Hide file tree

Showing 5 changed files with 594 additions and 0 deletions.
diff --git a/contrib/env b/contrib/env
@@ -0,0 +1,18 @@
+#!/bin/bash
+CLUSTER_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+# TODO(random-liu): Upload release tarball to user's own GCS, and use it. We should
+# not let all nodes of all users download tarball from cri-containerd-release.
+export KUBE_MASTER_EXTRA_METADATA="user-data=${CLUSTER_DIR}/gce/cloud-init/master.yaml,cri-containerd-configure-sh=${CLUSTER_DIR}/gce/configure.sh"
+export KUBE_NODE_EXTRA_METADATA="user-data=${CLUSTER_DIR}/gce/cloud-init/node.yaml,cri-containerd-configure-sh=${CLUSTER_DIR}/gce/configure.sh"
+if [ -n "${VERSION}" ]; then
+  version=$(mktemp /tmp/version.XXXX)
+  echo "${VERSION}" > "$version"
+  export KUBE_MASTER_EXTRA_METADATA="${KUBE_MASTER_EXTRA_METADATA},version=${version}"
+  export KUBE_NODE_EXTRA_METADATA="${KUBE_NODE_EXTRA_METADATA},version=${version}"
+fi
+export KUBE_CONTAINER_RUNTIME="remote"
+export KUBE_CONTAINER_RUNTIME_ENDPOINT="/var/run/cri-containerd.sock"
+export KUBE_LOAD_IMAGE_COMMAND="/home/cri-containerd/usr/local/bin/cri-containerd load"
+export NETWORK_POLICY_PROVIDER="calico"
+export NON_MASQUERADE_CIDR="0.0.0.0/0"
diff --git a/contrib/gce/cloud-init/master.yaml b/contrib/gce/cloud-init/master.yaml
@@ -0,0 +1,232 @@
+#cloud-config
+
+write_files:
+# Setup cri-containerd.
+  - path: /etc/systemd/system/cri-containerd-installation.service
+    permissions: 0644
+    owner: root
+    content: |
+      # installed by cloud-init
+      [Unit]
+      Description=Download and install cri-containerd binaries and configurations.
+      After=network-online.target
+
+      [Service]
+      Type=oneshot
+      RemainAfterExit=yes
+      ExecStartPre=/bin/mkdir -p /home/cri-containerd
+      ExecStartPre=/bin/mount --bind /home/cri-containerd /home/cri-containerd
+      ExecStartPre=/bin/mount -o remount,exec /home/cri-containerd
+      ExecStartPre=/usr/bin/curl --fail --retry 5 --retry-delay 3 --silent --show-error -H "X-Google-Metadata-Request: True" -o /home/cri-containerd/configure.sh http://metadata.google.internal/computeMetadata/v1/instance/attributes/cri-containerd-configure-sh
+      ExecStartPre=/bin/chmod 544 /home/cri-containerd/configure.sh
+      ExecStart=/home/cri-containerd/configure.sh
+
+      [Install]
+      WantedBy=cri-containerd.target
+
+  - path: /etc/containerd/config.toml
+    permissions: 0644
+    owner: root
+    content: |
+      # installed by cloud-init
+      oom_score = -999
+
+      [plugins.linux]
+        shim = "/home/cri-containerd/usr/local/bin/containerd-shim"
+        runtime = "/home/cri-containerd/usr/local/sbin/runc"
+
+  - path: /etc/systemd/system/containerd.service
+    permissions: 0644
+    owner: root
+    content: |
+      # installed by cloud-init
+      [Unit]
+      Description=containerd container runtime
+      Documentation=https://containerd.io
+      After=cri-containerd-installation.service
+
+      [Service]
+      Restart=always
+      RestartSec=5
+      Delegate=yes
+      KillMode=process
+      LimitNOFILE=1048576
+      # Having non-zero Limit*s causes performance problems due to accounting overhead
+      # in the kernel. We recommend using cgroups to do container-local accounting.
+      LimitNPROC=infinity
+      LimitCORE=infinity
+      ExecStartPre=/sbin/modprobe overlay
+      ExecStart=/home/cri-containerd/usr/local/bin/containerd --log-level debug
+
+      [Install]
+      WantedBy=cri-containerd.target
+
+  - path: /etc/systemd/system/cri-containerd.service
+    permissions: 0644
+    owner: root
+    content: |
+      # installed by cloud-init
+      [Unit]
+      Description=Kubernetes containerd CRI shim
+      Requires=network-online.target
+      After=cri-containerd-installation.service
+
+      [Service]
+      Restart=always
+      RestartSec=5
+      LimitNOFILE=1048576
+      # Having non-zero Limit*s causes performance problems due to accounting overhead
+      # in the kernel. We recommend using cgroups to do container-local accounting.
+      LimitNPROC=infinity
+      LimitCORE=infinity
+      # cri-containerd on master uses the cni binary and config in the
+      # release tarball.
+      ExecStart=/home/cri-containerd/usr/local/bin/cri-containerd \
+        --logtostderr --v=4 \
+        --network-bin-dir=/home/cri-containerd/opt/cni/bin \
+        --network-conf-dir=/home/cri-containerd/etc/cni/net.d
+
+      [Install]
+      WantedBy=cri-containerd.target
+
+  - path: /etc/systemd/system/cri-containerd-monitor.service
+    permissions: 0644
+    owner: root
+    content: |
+      [Unit]
+      Description=Kubernetes health monitoring for cri-containerd and containerd
+      After=containerd.service cri-containerd.service
+
+      [Service]
+      Restart=always
+      RestartSec=10
+      RemainAfterExit=yes
+      RemainAfterExit=yes
+      ExecStartPre=/bin/chmod 544 /home/cri-containerd/opt/cri-containerd/cluster/health-monitor.sh
+      ExecStart=/bin/bash -c 'CRICTL=/home/cri-containerd/usr/local/bin/crictl \
+      /home/cri-containerd/opt/cri-containerd/cluster/health-monitor.sh'
+
+      [Install]
+      WantedBy=cri-containerd.target
+
+  # TODO(random-liu): Guarantee order.
+  - path: /etc/systemd/system/cri-containerd.target
+    permissions: 0644
+    owner: root
+    content: |
+      [Unit]
+      Description=CRI Containerd
+
+      [Install]
+      WantedBy=kubernetes.target
+
+# Setup kubernetes.
+  - path: /etc/systemd/system/kube-master-installation.service
+    permissions: 0644
+    owner: root
+    content: |
+      [Unit]
+      Description=Download and install k8s binaries and configurations
+      After=network-online.target
+
+      [Service]
+      Type=oneshot
+      RemainAfterExit=yes
+      ExecStartPre=/bin/mkdir -p /home/kubernetes/bin
+      ExecStartPre=/bin/mount --bind /home/kubernetes/bin /home/kubernetes/bin
+      ExecStartPre=/bin/mount -o remount,exec /home/kubernetes/bin
+      ExecStartPre=/usr/bin/curl --fail --retry 5 --retry-delay 3 --silent --show-error	-H "X-Google-Metadata-Request: True" -o /home/kubernetes/bin/configure.sh http://metadata.google.internal/computeMetadata/v1/instance/attributes/configure-sh
+      ExecStartPre=/bin/chmod 544 /home/kubernetes/bin/configure.sh
+      ExecStart=/home/kubernetes/bin/configure.sh
+
+      [Install]
+      WantedBy=kubernetes.target
+
+  - path: /etc/systemd/system/kube-master-configuration.service
+    permissions: 0644
+    owner: root
+    content: |
+      [Unit]
+      Description=Configure kubernetes master
+      After=kube-master-installation.service
+
+      [Service]
+      Type=oneshot
+      RemainAfterExit=yes
+      ExecStartPre=/bin/chmod 544 /home/kubernetes/bin/configure-helper.sh
+      ExecStart=/home/kubernetes/bin/configure-helper.sh
+
+      [Install]
+      WantedBy=kubernetes.target
+
+  - path: /etc/systemd/system/kubelet-monitor.service
+    permissions: 0644
+    owner: root
+    content: |
+      [Unit]
+      Description=Kubernetes health monitoring for kubelet
+      After=kube-master-configuration.service
+
+      [Service]
+      Restart=always
+      RestartSec=10
+      RemainAfterExit=yes
+      RemainAfterExit=yes
+      ExecStartPre=/bin/chmod 544 /home/kubernetes/bin/health-monitor.sh
+      ExecStart=/home/kubernetes/bin/health-monitor.sh kubelet
+
+      [Install]
+      WantedBy=kubernetes.target
+
+  - path: /etc/systemd/system/kube-logrotate.timer
+    permissions: 0644
+    owner: root
+    content: |
+      [Unit]
+      Description=Hourly kube-logrotate invocation
+
+      [Timer]
+      OnCalendar=hourly
+
+      [Install]
+      WantedBy=kubernetes.target
+
+  - path: /etc/systemd/system/kube-logrotate.service
+    permissions: 0644
+    owner: root
+    content: |
+      [Unit]
+      Description=Kubernetes log rotation
+      After=kube-master-configuration.service
+
+      [Service]
+      Type=oneshot
+      ExecStart=-/usr/sbin/logrotate /etc/logrotate.conf
+
+      [Install]
+      WantedBy=kubernetes.target
+
+  - path: /etc/systemd/system/kubernetes.target
+    permissions: 0644
+    owner: root
+    content: |
+      [Unit]
+      Description=Kubernetes
+
+      [Install]
+      WantedBy=multi-user.target
+
+runcmd:
+  - systemctl daemon-reload
+  - systemctl enable containerd.service
+  - systemctl enable cri-containerd-installation.service
+  - systemctl enable cri-containerd.service
+  - systemctl enable cri-containerd-monitor.service
+  - systemctl enable cri-containerd.target
+  - systemctl enable kube-master-installation.service
+  - systemctl enable kube-master-configuration.service
+  - systemctl enable kubelet-monitor.service
+  - systemctl enable kube-logrotate.timer
+  - systemctl enable kube-logrotate.service
+  - systemctl enable kubernetes.target
+  - systemctl start kubernetes.target