Migrate Poseidon k8s cluster nodes to Debian 12

A local containerd deb package will be built using a Docker image because the respective version of the containerd deb package distributed by Debian's snapshot repositories does not work with my Poseidon Kubernetes cluster. The go tool now used as part of my Makefile will not need to be tied to a specific version of Golang. The only requirement is that the Golang version installed supports go modules. The sed commands are required to allow for building containerd with Golang versions '1.18'+. This fix follows how upstream patched this, see containerd/containerd#6716. pyyaml will now be added to the k8s_controllers client Python virtual environment because the debian/bookworm64 Vagrant box does not ship with a 'yaml' library installed in its system Python environment, causing the 'Add the Calico Helm repository' task to fail. Hence also why ansible_python_interpreter is now set for the 'Setup Kubernetes control planes (first control-planes) (poseidon)' play. The entire /etc/containerd directory is being removed as part of the 'Remove the default Debian packaged containerd configuration file' task because the containerd deb package installs a single configuration file within that directory. I could have just removed the file but I did not want to leave an empty directory on the filesystem. The 'Register the current grub configuration file inode' task now requires elevated permissions because the grub configuration file is not readable by the ansible_user. The 'Remove preset Vagrant box configurations' play was removed because the play was only applicable when using the generic/ubuntu2204 Vagrant box for the k8s_workers and k8s_controllers. gawk was changed to awk because gawk is not installed on the debian/bookworm64 Vagrant box, and I am fine with using mawk implementation. gnupg was added as a package to install as part of the Kubernetes apt repository dependencies because the gpg tool is not installed by default on the debian/bookworm64 Vagrant box. This should have been added as part of 966ad96 because that's when the roles started using said tool.
cavcrosby · Oct 12, 2023 · 00f942c · 00f942c
1 parent 6fbc99b
commit 00f942c
Show file tree

Hide file tree

Showing 10 changed files with 140 additions and 78 deletions.
diff --git a/Dockerfile b/Dockerfile
@@ -0,0 +1,41 @@
+FROM debian:bookworm@sha256:b91baba9c2cae5edbe3b0ff50ae8f05157e3ae6f018372dcfc3aba224acb392b
+
+ARG HOST_UID=1000
+ARG HOST_GID=1000
+ARG CONTAINERD_UPSTREAM_PREFIX
+
+ENV USERNAME="builder"
+WORKDIR "/build"
+
+# required to install build deps for a pkg
+RUN <<_EOF_
+sed \
+    --in-place \
+    's/Types: deb/Types: deb deb-src/g' \
+    "/etc/apt/sources.list.d/debian.sources"
+_EOF_
+
+RUN <<_EOF_
+apt-get update
+apt-get install --assume-yes "build-essential" "protobuf-compiler"
+apt-get build-dep --assume-yes "containerd"
+_EOF_
+
+RUN <<_EOF_
+groupadd --gid "${HOST_GID}" "${USERNAME}"
+useradd \
+    --uid "${HOST_UID}" \
+    --gid "${HOST_GID}" \
+    "${USERNAME}"
+_EOF_
+
+WORKDIR "/build/${CONTAINERD_UPSTREAM_PREFIX}"
+USER "${USERNAME}"
+
+# DEB_BUILD_OPTIONS=nocheck to ignore running tests after building the pkg, see
+# https://manpages.debian.org/testing/debhelper/dh_auto_test.1.en.html
+ENTRYPOINT [ \
+    "/bin/bash", \
+    "-c", \
+    "DEB_BUILD_OPTIONS=\"nocheck\" dpkg-buildpackage --build=\"binary\"" \
+]
diff --git a/Makefile b/Makefile
@@ -4,6 +4,8 @@
 
 # recursively expanded variables
 SHELL = /usr/bin/sh
+CONTAINERD_UPSTREAM_PREFIX = containerd-1.5.9~ds1
+BUILD_DIR_PATH = ./playbooks/build
 TRUTHY_VALUES = \
     true\
     1
@@ -40,6 +42,7 @@ PRODUCTION = production
 STAGING = staging
 ANSIBLE_SECRETS = ansible-secrets
 K8S_NODE_IMAGES = k8s-node-images
+CONTAINERD_DEB = containerd-deb
 LINT = lint
 DEVELOPMENT_SHELL = development-shell
 CLEAN = clean
@@ -83,6 +86,9 @@ PIP = pip
 NPM = npm
 NPX = npx
 PRE_COMMIT = pre-commit
+DGET = dget
+DOCKER = docker
+GO = go
 
 # simply expanded variables
 executables := \
@@ -100,13 +106,17 @@ executables := \
 	${BASH}\
 	${PYTHON}\
 	${NPM}\
-	${PACKER}
+	${PACKER}\
+	${DGET}\
+	${DOCKER}\
+	${GO}
 
 _check_executables := $(foreach exec,${executables},$(if $(shell command -v ${exec}),pass,$(error "No ${exec} in PATH")))
 src_yml := $(shell find . \( -type f \) \
 	-and \( -name '*.yml' \) \
 	-and ! \( -path './vendor/*' \) \
 	-and ! \( -path './node_modules/*' \) \
+	-and ! \( -path './playbooks/build/*' \) \
 )
 
 # provider VM identifiers
@@ -311,6 +321,29 @@ ${K8S_NODE_IMAGES}:
 		-var encrypted_ssh_password='$(value K8S_NODE_IMAGES_ENCRYPTED_SSH_PASSWORD)' \
 		"./k8s-nodes.pkr.hcl"
 
+.PHONY: ${CONTAINERD_DEB}
+${CONTAINERD_DEB}:
+	# group cmds right of || because otherwise dget always runs, see
+	# https://unix.stackexchange.com/questions/88850/precedence-of-the-shell-logical-operators#answer-88865
+>	@[ -d "${BUILD_DIR_PATH}/${CONTAINERD_UPSTREAM_PREFIX}" ] \
+		|| ( mkdir --parents "${BUILD_DIR_PATH}" \
+		&& cd "${BUILD_DIR_PATH}" \
+		&& ${DGET} \
+				--allow-unauthenticated \
+				"https://snapshot.debian.org/archive/debian/20220303T093013Z/pool/main/c/containerd/$(subst -,_,${CONTAINERD_UPSTREAM_PREFIX})-1.dsc" )
+
+>	cd "${BUILD_DIR_PATH}/${CONTAINERD_UPSTREAM_PREFIX}" \
+		&& ${GO} mod vendor \
+		&& sed --in-place 's_github.com/containerd/containerd => ./.empty-mod/__' "go.mod" \
+		&& sed --in-place 's_# github.com/containerd/containerd => ./.empty-mod/__' "./vendor/modules.txt"
+
+>	@[ -n "$$(${DOCKER} image ls --quiet "${CONTAINERD_DEB}")" ] \
+		|| ${DOCKER} build \
+				--build-arg CONTAINERD_UPSTREAM_PREFIX="${CONTAINERD_UPSTREAM_PREFIX}" \
+				--tag "${CONTAINERD_DEB}" "${CURDIR}"
+
+>	${DOCKER} run --rm --volume "${CURDIR}/playbooks/build:/build" "${CONTAINERD_DEB}"
+
 .PHONY: ${CLEAN}
 ${CLEAN}:
 >	rm --force *.log
@@ -320,6 +353,8 @@ ${CLEAN}:
 		"./playbooks/packer/qemu-poseidon_k8s_controller" \
 		"./playbooks/packer/qemu-poseidon_k8s_worker"
 
+>	rm --recursive --force "${BUILD_DIR_PATH}"
+>	${DOCKER} rmi --force "${CONTAINERD_DEB}"
 ifeq (${VAGRANT_PROVIDER}, ${LIBVIRT})
 	# There are times where vagrant may get into defunct state and will be unable to
 	# remove a domain known to libvirt (through 'vagrant destroy'). Hence the calls

diff --git a/playbooks/k8s_controllers.yml b/playbooks/k8s_controllers.yml
@@ -15,6 +15,7 @@
         python_virtualenvs_path: "{{ ansible_user_python_virtualenvs_path }}"
         python_virtualenv_pkgs:
           - kubernetes
+          - pyyaml
 
     - name: Add sourcing of the Python virtual environment
       ansible.builtin.import_role:
@@ -30,6 +31,8 @@
 - name: Perform initial setup for k8s_controller (poseidon)
   hosts: k8s_controllers:&poseidon
   tags: do_initial_poseidon_k8s_controller_setup
+  vars:
+    containerd_deb: containerd_{{ containerd_version }}_amd64.deb
   vars_files:
     - "{{ network_configs_path }}"
     - ./vars/{{ k8s_software_versions_file }}
@@ -49,14 +52,28 @@
       become: true
       when: ansible_os_family | lower == "debian"
 
+    - name: Copy the locally built containerd package over (debian-like)
+      ansible.builtin.copy:
+        dest: /tmp/{{ containerd_deb }}
+        src: ./build/{{ containerd_deb }}
+        mode: "755"
+      when: ansible_os_family | lower == "debian"
+
     - name: Install containerd (debian-like)
       ansible.builtin.apt:
-        name:
-          - containerd={{ containerd_version }}
+        deb: /tmp/{{ containerd_deb }}
         state: present
       become: true
       when: ansible_os_family | lower == "debian"
 
+    - name: Remove the default Debian packaged containerd configuration file
+      ansible.builtin.file:
+        path: /etc/containerd
+        state: absent
+      become: true
+      notify:
+        - Restart containerd
+
     - name: Prevent containerd from being upgraded
       ansible.builtin.dpkg_selections:
         name: containerd
@@ -151,6 +168,12 @@
         state: saved
       become: true
 
+    - name: Restart containerd
+      ansible.builtin.service:
+        name: containerd
+        state: restarted
+      become: true
+
 - name: Setup Kubernetes control planes (first control-planes)
   hosts: k8s_first_controllers
   tags: configure_as_k8s_first_controller
@@ -172,6 +195,8 @@
   vars_files:
     - ./vars/{{ k8s_software_versions_file }}
     - ./vars/ansible_secrets.yml
+  vars:
+    ansible_python_interpreter: "{{ ansible_user_python_virtualenvs_path }}/{{ ansible_user_python_virtualenv_name }}/bin/python"
 
   pre_tasks:
     - name: Setup the Calico CNI

diff --git a/playbooks/k8s_workers.yml b/playbooks/k8s_workers.yml
@@ -30,6 +30,8 @@
 - name: Perform initial setup for k8s_worker (poseidon)
   hosts: k8s_workers:&poseidon
   tags: do_initial_poseidon_k8s_worker_setup
+  vars:
+    containerd_deb: containerd_{{ containerd_version }}_amd64.deb
   vars_files:
     - ./vars/{{ k8s_software_versions_file }}
 
@@ -40,14 +42,28 @@
       become: true
       when: ansible_os_family | lower == "debian"
 
+    - name: Copy the locally built containerd package over (debian-like)
+      ansible.builtin.copy:
+        dest: /tmp/{{ containerd_deb }}
+        src: ./build/{{ containerd_deb }}
+        mode: "755"
+      when: ansible_os_family | lower == "debian"
+
     - name: Install containerd (debian-like)
       ansible.builtin.apt:
-        name:
-          - containerd={{ containerd_version }}
+        deb: /tmp/{{ containerd_deb }}
         state: present
       become: true
       when: ansible_os_family | lower == "debian"
 
+    - name: Remove the default Debian packaged containerd configuration file
+      ansible.builtin.file:
+        path: /etc/containerd
+        state: absent
+      become: true
+      notify:
+        - Restart containerd
+
     - name: Prevent containerd from being upgraded
       ansible.builtin.dpkg_selections:
         name: containerd
@@ -69,6 +85,12 @@
         state: saved
       become: true
 
+    - name: Restart containerd
+      ansible.builtin.service:
+        name: containerd
+        state: restarted
+      become: true
+
 - name: Configure k8s_worker to be a Kubernetes worker node
   hosts: k8s_workers
   tags: configure_host_as_k8s_worker_node

diff --git a/playbooks/vagrant_customizations.yml b/playbooks/vagrant_customizations.yml
@@ -42,6 +42,7 @@
           ansible.builtin.stat:
             path: "{{ grub_config_path }}"
             checksum_algorithm: sha256
+          become: true
           register: current_grub_config_inode
 
         - name: Update grub.cfg (debian-like)
@@ -50,7 +51,7 @@
             cmd: |
               set -o pipefail
               grub-mkconfig --output "{{ grub_config_path }}"
-              sha256sum "{{ grub_config_path }}" | gawk --field-separator " " '{ print $1 }'
+              sha256sum "{{ grub_config_path }}" | awk -F " " '{ print $1 }'
           changed_when: current_grub_config_inode.stat.checksum != grub_config_checksum.stdout
           become: true
           register: grub_config_checksum
@@ -64,66 +65,3 @@
       when:
         - default_grub_config_inode.stat.exists
         - line_check.found == 0
-
-- name: Remove preset Vagrant box configurations
-  hosts:
-    - k8s_workers
-    - k8s_controllers
-  tags: remove_preset_vagrant_box_configs
-
-  tasks:
-    - name: Get the current merged netplan configuration
-      ansible.builtin.command: netplan get all
-      changed_when: false
-      register: current_netplan_config
-
-    # The generic/ubuntu2204 vagrant box has DNS configs that causes
-    # /run/systemd/resolve/resolv.conf to have too many DNS nameservers added to it.
-    # Hence the need to undo some of the box's configs, for reference on how the
-    # box is configured in regards to DNS:
-    # https://github.com/lavabit/robox/blob/master/scripts/ubuntu2204/network.sh
-    #
-    # This above issue collides with Kubernetes when it tries to use the mentioned
-    # 'resolv.conf'. For reference on the issue I saw:
-    # https://github.com/kubernetes/kubernetes/issues/82756
-    - name: Remove created netplan yaml
-      ansible.builtin.file:
-        path: /etc/netplan/01-netcfg.yaml
-        state: absent
-      become: true
-      notify:
-        - Apply netplan configurations
-        - Restart systemd-networkd
-        - Restart systemd-resolved
-
-    # defaults can be restored when deleting th resolved.conf and all drop-ins
-    - name: Set the systemd resolved.conf configuration back to defaults
-      ansible.builtin.file:
-        path: /etc/systemd/resolved.conf
-        state: absent
-      become: true
-      notify:
-        - Restart systemd-networkd
-        - Restart systemd-resolved
-
-  handlers:
-    - name: Apply netplan configurations
-      ansible.builtin.shell: |
-        netplan generate
-        netplan get all
-      changed_when: current_netplan_config.stdout != netplan_config.stdout
-      become: true
-      register: netplan_config
-
-    - name: Restart systemd-networkd
-      ansible.builtin.systemd:
-        name: systemd-networkd
-        enabled: true
-        state: restarted
-      become: true
-
-    - name: Restart systemd-resolved
-      ansible.builtin.systemd:
-        name: systemd-resolved
-        state: restarted
-      become: true
diff --git a/playbooks/vars/poseidon_k8s_software_versions.yml b/playbooks/vars/poseidon_k8s_software_versions.yml
@@ -5,7 +5,7 @@ kubectl_version: 1.25.5-1.1
 kubelet_version: 1.25.5-1.1
 cri_tools_version: 1.25.0-1.1
 etcdctl_version: v3.5.6
-containerd_version: 1.5.9-0ubuntu3
+containerd_version: 1.5.9~ds1-1
 calico_chart_version: v3.24.3  # chart version points to app version v3.24.3
 metrics_server_chart_version: 3.8.3  # chart version points to app version 0.6.2
 ingress_nginx_chart_version: 4.2.1  # chart version points to app version 1.3.0
diff --git a/roles/helm/handlers/main.yml b/roles/helm/handlers/main.yml
@@ -8,7 +8,7 @@
     cmd: |
       set -o pipefail
       gpg --batch --output "{{ _helm_keyring_install_path }}" --dearmor "{{ _helm_keyring_download_path }}"
-      sha256sum "{{ _helm_keyring_install_path }}" | gawk --field-separator " " '{ print $1 }'
+      sha256sum "{{ _helm_keyring_install_path }}" | awk -F " " '{ print $1 }'
   changed_when: not current_installed_gpg_key_inode.stat.exists
                 or current_installed_gpg_key_inode.stat.checksum != installed_gpg_key_inode.stdout
   become: true

diff --git a/roles/k8s_kubeadm/handlers/main.yml b/roles/k8s_kubeadm/handlers/main.yml
@@ -5,7 +5,7 @@
     cmd: |
       set -o pipefail
       gpg --batch --output "{{ _k8s_keyring_install_path }}" --dearmor "{{ _k8s_keyring_download_path }}"
-      sha256sum "{{ _k8s_keyring_install_path }}" | gawk --field-separator " " '{ print $1 }'
+      sha256sum "{{ _k8s_keyring_install_path }}" | awk -F " " '{ print $1 }'
   changed_when: not current_installed_gpg_key_inode.stat.exists
                 or current_installed_gpg_key_inode.stat.checksum != installed_gpg_key_inode.stdout
   become: true

diff --git a/roles/k8s_kubeadm/tasks/main.yml b/roles/k8s_kubeadm/tasks/main.yml
@@ -26,6 +26,7 @@
       - apt-transport-https
       - ca-certificates
       - curl
+      - gnupg
     state: present
   become: true
   when: ansible_os_family | lower == "debian"