From 3949ed1e6dda7ad2d436dc53a173aacc4db431d6 Mon Sep 17 00:00:00 2001 From: Jeff Geerling Date: Wed, 14 Sep 2022 13:18:35 -0500 Subject: [PATCH 01/18] Update Kubernetes version to 1.25, see what happens. --- .github/workflows/ci.yml | 2 +- README.md | 4 ++-- defaults/main.yml | 4 ++-- meta/main.yml | 4 +++- 4 files changed, 8 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9d9d3db..b8bf14f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -43,7 +43,7 @@ jobs: include: - distro: rockylinux8 playbook: converge.yml - - distro: ubuntu2004 + - distro: ubuntu2204 playbook: converge.yml - distro: debian11 playbook: converge.yml diff --git a/README.md b/README.md index af41a8f..173f518 100644 --- a/README.md +++ b/README.md @@ -27,8 +27,8 @@ kubernetes_packages: Kubernetes packages to be installed on the server. You can either provide a list of package names, or set `name` and `state` to have more control over whether the package is `present`, `absent`, `latest`, etc. ```yaml -kubernetes_version: '1.20' -kubernetes_version_rhel_package: '1.20.4' +kubernetes_version: '1.25' +kubernetes_version_rhel_package: '1.25.0' ``` The minor version of Kubernetes to install. The plain `kubernetes_version` is used to pin an apt package version on Debian, and as the Kubernetes version passed into the `kubeadm init` command (see `kubernetes_version_kubeadm`). The `kubernetes_version_rhel_package` variable must be a specific Kubernetes release, and is used to pin the version on Red Hat / CentOS servers. diff --git a/defaults/main.yml b/defaults/main.yml index 9434768..172e1de 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -9,8 +9,8 @@ kubernetes_packages: - name: kubernetes-cni state: present -kubernetes_version: '1.20' -kubernetes_version_rhel_package: '1.20.4' +kubernetes_version: '1.25' +kubernetes_version_rhel_package: '1.25.0' kubernetes_role: master diff --git a/meta/main.yml b/meta/main.yml index 82729f8..4ecdc5b 100644 --- a/meta/main.yml +++ b/meta/main.yml @@ -7,12 +7,13 @@ galaxy_info: description: Kubernetes for Linux. company: "Midwestern Mac, LLC" license: "license (BSD, MIT)" - min_ansible_version: 2.4 + min_ansible_version: 2.10 platforms: - name: EL versions: - 7 - 8 + - 9 - name: Debian versions: - stretch @@ -23,6 +24,7 @@ galaxy_info: - xenial - bionic - focal + - jammy galaxy_tags: - system - containers From 1881f6161a12f3fee8bb22238e1ece828529f433 Mon Sep 17 00:00:00 2001 From: Jeff Geerling Date: Wed, 14 Sep 2022 14:26:10 -0500 Subject: [PATCH 02/18] Fix kubelet and kubeadm configs. --- .github/workflows/ci.yml | 2 +- templates/kubeadm-kubelet-config.j2 | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b8bf14f..9d9d3db 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -43,7 +43,7 @@ jobs: include: - distro: rockylinux8 playbook: converge.yml - - distro: ubuntu2204 + - distro: ubuntu2004 playbook: converge.yml - distro: debian11 playbook: converge.yml diff --git a/templates/kubeadm-kubelet-config.j2 b/templates/kubeadm-kubelet-config.j2 index 7c4318b..3c49c2e 100644 --- a/templates/kubeadm-kubelet-config.j2 +++ b/templates/kubeadm-kubelet-config.j2 @@ -1,10 +1,10 @@ --- -apiVersion: kubeadm.k8s.io/v1beta2 +apiVersion: kubeadm.k8s.io/v1beta3 kind: InitConfiguration {{ kubernetes_config_init_configuration | to_nice_yaml }} --- +apiVersion: kubeadm.k8s.io/v1beta3 kind: ClusterConfiguration -apiVersion: kubeadm.k8s.io/v1beta2 {{ kubernetes_config_cluster_configuration | to_nice_yaml }} {% if kubernetes_config_kubelet_configuration|length > 0 %} --- From 3a236239a727056965c7329305a2fbdf43b08c37 Mon Sep 17 00:00:00 2001 From: Jeff Geerling Date: Wed, 14 Sep 2022 14:42:00 -0500 Subject: [PATCH 03/18] Fixes #88: Switch tests and defaults from Docker to containerd. --- .travis.yml | 36 ------------------------------- README.md | 6 +++--- defaults/main.yml | 2 +- molecule/default/calico.yml | 6 +++--- molecule/default/converge.yml | 6 +++--- molecule/default/requirements.yml | 2 +- 6 files changed, 11 insertions(+), 47 deletions(-) delete mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 61c6b8e..0000000 --- a/.travis.yml +++ /dev/null @@ -1,36 +0,0 @@ ---- -language: python -services: docker - -env: - global: - - ROLE_NAME: kubernetes - matrix: - - MOLECULE_DISTRO: centos8 - - MOLECULE_DISTRO: centos7 - - MOLECULE_DISTRO: ubuntu1804 - - MOLECULE_DISTRO: debian10 - - - MOLECULE_DISTRO: debian10 - MOLECULE_PLAYBOOK: playbook-calico.yml - -before_install: - # Upgrade Docker to work with docker-py. - - curl https://gist.githubusercontent.com/geerlingguy/ce883ad4aec6a5f1187ef93bd338511e/raw/36612d28981d92863f839c5aefe5b7dd7193d6c6/travis-ci-docker-upgrade.sh | sudo bash - -install: - # Install test dependencies. - - pip install molecule[docker] yamllint ansible-lint docker - -before_script: - # Use actual Ansible Galaxy role name for the project directory. - - cd ../ - - mv ansible-role-$ROLE_NAME geerlingguy.$ROLE_NAME - - cd geerlingguy.$ROLE_NAME - -script: - # Run tests. - - molecule test - -notifications: - webhooks: https://galaxy.ansible.com/api/v1/notifications/ diff --git a/README.md b/README.md index 173f518..26a43f0 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ An Ansible Role that installs [Kubernetes](https://kubernetes.io) on Linux. ## Requirements -Requires Docker or another [Container Runtime](https://kubernetes.io/docs/setup/production-environment/container-runtimes) ; recommended role for Docker installation: `geerlingguy.docker`. +Requires a compatible [Container Runtime](https://kubernetes.io/docs/setup/production-environment/container-runtimes); recommended role for CRI installation: `geerlingguy.containerd`. ## Role Variables @@ -70,14 +70,14 @@ Options under `kind: ClusterConfiguration`. Including `kubernetes_pod_network.ci ```yaml kubernetes_config_kubelet_configuration: - cgroupDriver: cgroupfs + cgroupDriver: systemd ``` Options to configure kubelet on any nodes in your cluster through the `kubeadm init` process. To get the syntax of this options see https://kubernetes.io/docs/tasks/administer-cluster/kubelet-config-file and https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/kubelet-integration. NOTE: This is the recommended way to do the kubelet-configuration. Most command-line-options are deprecated. -NOTE: The recommended cgroupDriver depends on your [Container Runtime](https://kubernetes.io/docs/setup/production-environment/container-runtimes). When using this role with containerd instead of docker, this value should be changed to `systemd`. +NOTE: The recommended cgroupDriver depends on your [Container Runtime](https://kubernetes.io/docs/setup/production-environment/container-runtimes). When using this role with Docker instead of containerd, this value should be changed to `cgroupfs`. ```yaml kubernetes_config_kube_proxy_configuration: {} diff --git a/defaults/main.yml b/defaults/main.yml index 172e1de..2fc763e 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -30,7 +30,7 @@ kubernetes_pod_network: kubernetes_kubeadm_kubelet_config_file_path: '/etc/kubernetes/kubeadm-kubelet-config.yaml' kubernetes_config_kubelet_configuration: - cgroupDriver: "cgroupfs" + cgroupDriver: "systemd" kubernetes_config_init_configuration: localAPIEndpoint: diff --git a/molecule/default/calico.yml b/molecule/default/calico.yml index 2e2258e..1fd035c 100644 --- a/molecule/default/calico.yml +++ b/molecule/default/calico.yml @@ -8,8 +8,8 @@ cni: 'calico' cidr: '192.168.0.0/16' - # Allow swap in test environments (hard to control in some Docker envs). - kubernetes_kubelet_extra_args: "--fail-swap-on=false --cgroup-driver=cgroupfs" + # Allow swap in test environments (hard to control in some envs). + kubernetes_kubelet_extra_args: "--fail-swap-on=false --cgroup-driver=systemd" docker_install_compose: false pre_tasks: @@ -29,7 +29,7 @@ action: setup roles: - - role: geerlingguy.docker + - role: geerlingguy.containerd - role: geerlingguy.kubernetes post_tasks: diff --git a/molecule/default/converge.yml b/molecule/default/converge.yml index f93c7d7..46c5610 100644 --- a/molecule/default/converge.yml +++ b/molecule/default/converge.yml @@ -4,8 +4,8 @@ become: true vars: - # Allow swap in test environments (hard to control in some Docker envs). - kubernetes_kubelet_extra_args: "--fail-swap-on=false --cgroup-driver=cgroupfs" + # Allow swap in test environments (hard to control in some envs). + kubernetes_kubelet_extra_args: "--fail-swap-on=false --cgroup-driver=systemd" docker_install_compose: false pre_tasks: @@ -25,7 +25,7 @@ action: setup roles: - - role: geerlingguy.docker + - role: geerlingguy.containerd - role: geerlingguy.kubernetes post_tasks: diff --git a/molecule/default/requirements.yml b/molecule/default/requirements.yml index 3a013f3..8f7dff5 100644 --- a/molecule/default/requirements.yml +++ b/molecule/default/requirements.yml @@ -1,2 +1,2 @@ --- -- src: geerlingguy.docker +- src: geerlingguy.containerd From 926a8c909e71ab71c8581c28828ac010929bc6e3 Mon Sep 17 00:00:00 2001 From: Jeff Geerling Date: Wed, 14 Sep 2022 15:46:44 -0500 Subject: [PATCH 04/18] PR #107: Co-authored commit to change master naming to control plane. Co-authored-by: ra-rau --- README.md | 24 +++++++++---------- defaults/main.yml | 4 ++-- molecule/default/calico.yml | 6 ++++- molecule/default/converge.yml | 6 ++++- molecule/default/molecule.yml | 2 +- ...ster-setup.yml => control-plane-setup.yml} | 10 ++++---- tasks/main.yml | 10 ++++---- tasks/node-setup.yml | 2 +- 8 files changed, 36 insertions(+), 28 deletions(-) rename tasks/{master-setup.yml => control-plane-setup.yml} (88%) diff --git a/README.md b/README.md index 26a43f0..6e79a3d 100644 --- a/README.md +++ b/README.md @@ -34,10 +34,10 @@ kubernetes_version_rhel_package: '1.25.0' The minor version of Kubernetes to install. The plain `kubernetes_version` is used to pin an apt package version on Debian, and as the Kubernetes version passed into the `kubeadm init` command (see `kubernetes_version_kubeadm`). The `kubernetes_version_rhel_package` variable must be a specific Kubernetes release, and is used to pin the version on Red Hat / CentOS servers. ```yaml -kubernetes_role: master +kubernetes_role: control_plane ``` -Whether the particular server will serve as a Kubernetes `master` (default) or `node`. The master will have `kubeadm init` run on it to intialize the entire K8s control plane, while `node`s will have `kubeadm join` run on them to join them to the `master`. +Whether the particular server will serve as a Kubernetes `control_plane` (default) or `node`. The control plane will have `kubeadm init` run on it to intialize the entire K8s control plane, while `node`s will have `kubeadm join` run on them to join them to the `control_plane`. ### Variables to configure kubeadm and kubelet with `kubeadm init` through a config file (recommended) @@ -109,10 +109,10 @@ Extra args to pass to the generated `kubeadm join` command during K8s node initi ### Additional variables ```yaml -kubernetes_allow_pods_on_master: true +kubernetes_allow_pods_on_control_plane: true ``` -Whether to remove the taint that denies pods from being deployed to the Kubernetes master. If you have a single-node cluster, this should definitely be `True`. Otherwise, set to `False` if you want a dedicated Kubernetes master which doesn't run any other pods. +Whether to remove the taint that denies pods from being deployed to the Kubernetes control plane. If you have a single-node cluster, this should definitely be `True`. Otherwise, set to `False` if you want a dedicated Kubernetes control plane which doesn't run any other pods. ```yaml kubernetes_pod_network: @@ -137,7 +137,7 @@ kubernetes_version_kubeadm: 'stable-{{ kubernetes_version }}'` kubernetes_ignore_preflight_errors: 'all' ``` -Options passed to `kubeadm init` when initializing the Kubernetes master. The `kubernetes_apiserver_advertise_address` defaults to `ansible_default_ipv4.address` if it's left empty. +Options passed to `kubeadm init` when initializing the Kubernetes control plane. The `kubernetes_apiserver_advertise_address` defaults to `ansible_default_ipv4.address` if it's left empty. ```yaml kubernetes_apt_release_channel: main @@ -177,25 +177,25 @@ None. ## Example Playbooks -### Single node (master-only) cluster +### Single node (control-plane-only) cluster ```yaml - hosts: all vars: - kubernetes_allow_pods_on_master: true + kubernetes_allow_pods_on_control_plane: true roles: - geerlingguy.docker - geerlingguy.kubernetes ``` -### Two or more nodes (single master) cluster +### Two or more nodes (single control-plane) cluster -Master inventory vars: +Control plane inventory vars: ```yaml -kubernetes_role: "master" +kubernetes_role: "control_plane" ``` Node(s) inventory vars: @@ -210,14 +210,14 @@ Playbook: - hosts: all vars: - kubernetes_allow_pods_on_master: true + kubernetes_allow_pods_on_control_plane: true roles: - geerlingguy.docker - geerlingguy.kubernetes ``` -Then, log into the Kubernetes master, and run `kubectl get nodes` as root, and you should see a list of all the servers. +Then, log into the Kubernetes control plane, and run `kubectl get nodes` as root, and you should see a list of all the servers. ## License diff --git a/defaults/main.yml b/defaults/main.yml index 2fc763e..a1ca4b1 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -12,14 +12,14 @@ kubernetes_packages: kubernetes_version: '1.25' kubernetes_version_rhel_package: '1.25.0' -kubernetes_role: master +kubernetes_role: control_plane # This is deprecated. Please use kubernetes_config_kubelet_configuration instead. kubernetes_kubelet_extra_args: "" kubernetes_kubeadm_init_extra_opts: "" kubernetes_join_command_extra_opts: "" -kubernetes_allow_pods_on_master: true +kubernetes_allow_pods_on_control_plane: true kubernetes_pod_network: # Flannel CNI. cni: 'flannel' diff --git a/molecule/default/calico.yml b/molecule/default/calico.yml index 1fd035c..b8c9a70 100644 --- a/molecule/default/calico.yml +++ b/molecule/default/calico.yml @@ -9,7 +9,11 @@ cidr: '192.168.0.0/16' # Allow swap in test environments (hard to control in some envs). - kubernetes_kubelet_extra_args: "--fail-swap-on=false --cgroup-driver=systemd" + kubernetes_kubelet_extra_args: >- + --fail-swap-on=false + --cgroup-driver=systemd + --cgroups-per-qos=false + --enforce-node-allocatable="" docker_install_compose: false pre_tasks: diff --git a/molecule/default/converge.yml b/molecule/default/converge.yml index 46c5610..f89c518 100644 --- a/molecule/default/converge.yml +++ b/molecule/default/converge.yml @@ -5,7 +5,11 @@ vars: # Allow swap in test environments (hard to control in some envs). - kubernetes_kubelet_extra_args: "--fail-swap-on=false --cgroup-driver=systemd" + kubernetes_kubelet_extra_args: >- + --fail-swap-on=false + --cgroup-driver=systemd + --cgroups-per-qos=false + --enforce-node-allocatable="" docker_install_compose: false pre_tasks: diff --git a/molecule/default/molecule.yml b/molecule/default/molecule.yml index 3f1b2b5..cc47e45 100644 --- a/molecule/default/molecule.yml +++ b/molecule/default/molecule.yml @@ -10,7 +10,7 @@ platforms: command: ${MOLECULE_DOCKER_COMMAND:-""} volumes: - /sys/fs/cgroup:/sys/fs/cgroup:ro - - /var/lib/docker + - /var/lib/containerd privileged: true pre_build_image: true provisioner: diff --git a/tasks/master-setup.yml b/tasks/control-plane-setup.yml similarity index 88% rename from tasks/master-setup.yml rename to tasks/control-plane-setup.yml index 49feb78..347e5cc 100644 --- a/tasks/master-setup.yml +++ b/tasks/control-plane-setup.yml @@ -9,7 +9,7 @@ src: "kubeadm-kubelet-config.j2" dest: "{{ kubernetes_kubeadm_kubelet_config_file_path }}" -- name: Initialize Kubernetes master with kubeadm init +- name: Initialize Kubernetes control plane with kubeadm init command: > kubeadm init --config {{ kubernetes_kubeadm_kubelet_config_file_path }} @@ -17,7 +17,7 @@ register: kubeadmin_init when: (not kubernetes_init_stat.stat.exists) and (kubernetes_ignore_preflight_errors is not defined) -- name: Initialize Kubernetes master with kubeadm init and ignore_preflight_errors +- name: Initialize Kubernetes control plane with kubeadm init and ignore_preflight_errors command: > kubeadm init --config {{ kubernetes_kubeadm_kubelet_config_file_path }} @@ -73,8 +73,8 @@ # TODO: Check if taint exists with something like `kubectl describe nodes` # instead of using kubernetes_init_stat.stat.exists check. -- name: Allow pods on master node (if configured). - command: "kubectl taint nodes --all node-role.kubernetes.io/master-" +- name: Allow pods on control plane (if configured). + command: "kubectl taint nodes --all node-role.kubernetes.io/control-plane-" when: - - kubernetes_allow_pods_on_master | bool + - kubernetes_allow_pods_on_control_plane | bool - not kubernetes_init_stat.stat.exists diff --git a/tasks/main.yml b/tasks/main.yml index 1b57179..3fa409a 100644 --- a/tasks/main.yml +++ b/tasks/main.yml @@ -34,15 +34,15 @@ path: /etc/kubernetes/admin.conf register: kubernetes_init_stat -# Set up master. -- include_tasks: master-setup.yml - when: kubernetes_role == 'master' +# Set up control plane. +- include_tasks: control-plane-setup.yml + when: kubernetes_role == 'control_plane' # Set up nodes. -- name: Get the kubeadm join command from the Kubernetes master. +- name: Get the kubeadm join command from the Kubernetes control plane. command: kubeadm token create --print-join-command changed_when: false - when: kubernetes_role == 'master' + when: kubernetes_role == 'control_plane' register: kubernetes_join_command_result - name: Set the kubeadm join command globally. diff --git a/tasks/node-setup.yml b/tasks/node-setup.yml index 304cbf1..be684ee 100644 --- a/tasks/node-setup.yml +++ b/tasks/node-setup.yml @@ -1,5 +1,5 @@ --- -- name: Join node to Kubernetes master +- name: Join node to Kubernetes control plane. shell: > {{ kubernetes_join_command }} creates=/etc/kubernetes/kubelet.conf From 2fb5b1e993a7783edaab3d7f4dff89cc33e47ebd Mon Sep 17 00:00:00 2001 From: Jeff Geerling Date: Wed, 14 Sep 2022 17:11:18 -0500 Subject: [PATCH 05/18] Attempt to get systemd overlay stuff working correctly. --- molecule/default/calico.yml | 2 +- molecule/default/converge.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/molecule/default/calico.yml b/molecule/default/calico.yml index b8c9a70..cd86eca 100644 --- a/molecule/default/calico.yml +++ b/molecule/default/calico.yml @@ -14,7 +14,7 @@ --cgroup-driver=systemd --cgroups-per-qos=false --enforce-node-allocatable="" - docker_install_compose: false + containerd_config_cgroup_driver_systemd: true pre_tasks: - name: Update apt cache. diff --git a/molecule/default/converge.yml b/molecule/default/converge.yml index f89c518..45a38fc 100644 --- a/molecule/default/converge.yml +++ b/molecule/default/converge.yml @@ -10,7 +10,7 @@ --cgroup-driver=systemd --cgroups-per-qos=false --enforce-node-allocatable="" - docker_install_compose: false + containerd_config_cgroup_driver_systemd: true pre_tasks: - name: Update apt cache. From 724151c5fc584c5293a5001640828b9b83661941 Mon Sep 17 00:00:00 2001 From: Jeff Geerling Date: Thu, 15 Sep 2022 10:30:49 -0500 Subject: [PATCH 06/18] Debug. --- .github/workflows/ci.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9d9d3db..f97189e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -65,6 +65,9 @@ jobs: - name: Install test dependencies. run: pip3 install ansible molecule[docker] docker + - name: Tests + run: modprobe configs + - name: Run Molecule tests. run: molecule test env: From 004f583d96e1a3088c9d69de668e8e80cbde4064 Mon Sep 17 00:00:00 2001 From: Jeff Geerling Date: Thu, 15 Sep 2022 10:37:47 -0500 Subject: [PATCH 07/18] Remove that debug stuff. --- .github/workflows/ci.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f97189e..9d9d3db 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -65,9 +65,6 @@ jobs: - name: Install test dependencies. run: pip3 install ansible molecule[docker] docker - - name: Tests - run: modprobe configs - - name: Run Molecule tests. run: molecule test env: From 29daceb481134ae5ec6312af950b9dc34dbea475 Mon Sep 17 00:00:00 2001 From: Jeff Geerling Date: Thu, 15 Sep 2022 12:22:35 -0500 Subject: [PATCH 08/18] Debugging. --- tasks/control-plane-setup.yml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/tasks/control-plane-setup.yml b/tasks/control-plane-setup.yml index 347e5cc..a198ef8 100644 --- a/tasks/control-plane-setup.yml +++ b/tasks/control-plane-setup.yml @@ -25,6 +25,21 @@ {{ kubernetes_kubeadm_init_extra_opts }} register: kubeadmin_init when: (not kubernetes_init_stat.stat.exists) and (kubernetes_ignore_preflight_errors is defined) + failed_when: false # DELETEME START --- +- name: Test + command: systemctl status kubelet + register: systemctl_result + +- debug: var=systemctl_result + +- name: Test 2 + command: journalctl --no-pager -xeu kubelet + register: journalctl_result + +- debug: var=journalctl_result + +- fail: +# DELETEME END --- - name: Print the init output to screen. debug: From 641beb88bfa7d450a30b289ca4cdf79d22aead74 Mon Sep 17 00:00:00 2001 From: Jeff Geerling Date: Thu, 15 Sep 2022 13:11:19 -0500 Subject: [PATCH 09/18] Disable Calico test right now. --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9d9d3db..b53e578 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -48,8 +48,8 @@ jobs: - distro: debian11 playbook: converge.yml - - distro: debian11 - playbook: calico.yml + # - distro: debian11 + # playbook: calico.yml steps: - name: Check out the codebase. From 70761e474e89b7134cd7e5ad24b03ef510f824d6 Mon Sep 17 00:00:00 2001 From: Jeff Geerling Date: Thu, 15 Sep 2022 14:06:00 -0500 Subject: [PATCH 10/18] Turn off Debian 11 for now. --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b53e578..07ac33e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -45,8 +45,8 @@ jobs: playbook: converge.yml - distro: ubuntu2004 playbook: converge.yml - - distro: debian11 - playbook: converge.yml + # - distro: debian11 + # playbook: converge.yml # - distro: debian11 # playbook: calico.yml From 39c2a7c4bc281033fa8beefea5edeac36e9a2f10 Mon Sep 17 00:00:00 2001 From: Jeff Geerling Date: Thu, 15 Sep 2022 14:22:23 -0500 Subject: [PATCH 11/18] See if non-Debian OSes work. --- tasks/control-plane-setup.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tasks/control-plane-setup.yml b/tasks/control-plane-setup.yml index a198ef8..f275e4b 100644 --- a/tasks/control-plane-setup.yml +++ b/tasks/control-plane-setup.yml @@ -28,17 +28,17 @@ failed_when: false # DELETEME START --- - name: Test command: systemctl status kubelet + changed_when: false register: systemctl_result - debug: var=systemctl_result - name: Test 2 command: journalctl --no-pager -xeu kubelet + changed_when: false register: journalctl_result - debug: var=journalctl_result - -- fail: # DELETEME END --- - name: Print the init output to screen. From d752819b3f08efd62c7d265bac532039bc4a745c Mon Sep 17 00:00:00 2001 From: Jeff Geerling Date: Thu, 15 Sep 2022 14:33:19 -0500 Subject: [PATCH 12/18] See if non-Debian OSes work x2. --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 07ac33e..1454abd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -43,8 +43,8 @@ jobs: include: - distro: rockylinux8 playbook: converge.yml - - distro: ubuntu2004 - playbook: converge.yml + # - distro: ubuntu2004 + # playbook: converge.yml # - distro: debian11 # playbook: converge.yml From 57292bdfc82a95fea1d9b2bea9c2f07f7b27bd43 Mon Sep 17 00:00:00 2001 From: Jeff Geerling Date: Thu, 15 Sep 2022 14:54:53 -0500 Subject: [PATCH 13/18] Does it work at all on centos? --- tasks/control-plane-setup.yml | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/tasks/control-plane-setup.yml b/tasks/control-plane-setup.yml index f275e4b..609fd47 100644 --- a/tasks/control-plane-setup.yml +++ b/tasks/control-plane-setup.yml @@ -25,21 +25,21 @@ {{ kubernetes_kubeadm_init_extra_opts }} register: kubeadmin_init when: (not kubernetes_init_stat.stat.exists) and (kubernetes_ignore_preflight_errors is defined) - failed_when: false # DELETEME START --- -- name: Test - command: systemctl status kubelet - changed_when: false - register: systemctl_result +# failed_when: false # DELETEME START --- +# - name: Test +# command: systemctl status kubelet +# changed_when: false +# register: systemctl_result -- debug: var=systemctl_result +# - debug: var=systemctl_result -- name: Test 2 - command: journalctl --no-pager -xeu kubelet - changed_when: false - register: journalctl_result +# - name: Test 2 +# command: journalctl --no-pager -xeu kubelet +# changed_when: false +# register: journalctl_result -- debug: var=journalctl_result -# DELETEME END --- +# - debug: var=journalctl_result +# # DELETEME END --- - name: Print the init output to screen. debug: From 70e203de3f0f1d8a9c6058f1c3d9d8cad097ebb8 Mon Sep 17 00:00:00 2001 From: Jeff Geerling Date: Thu, 15 Sep 2022 17:41:21 -0500 Subject: [PATCH 14/18] Try Kubernetes 1.24.4. --- .github/workflows/ci.yml | 12 ++++++------ README.md | 4 ++-- defaults/main.yml | 4 ++-- molecule/default/molecule.yml | 2 +- tasks/control-plane-setup.yml | 15 --------------- 5 files changed, 11 insertions(+), 26 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1454abd..9d9d3db 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -43,13 +43,13 @@ jobs: include: - distro: rockylinux8 playbook: converge.yml - # - distro: ubuntu2004 - # playbook: converge.yml - # - distro: debian11 - # playbook: converge.yml + - distro: ubuntu2004 + playbook: converge.yml + - distro: debian11 + playbook: converge.yml - # - distro: debian11 - # playbook: calico.yml + - distro: debian11 + playbook: calico.yml steps: - name: Check out the codebase. diff --git a/README.md b/README.md index 6e79a3d..c0ff006 100644 --- a/README.md +++ b/README.md @@ -27,8 +27,8 @@ kubernetes_packages: Kubernetes packages to be installed on the server. You can either provide a list of package names, or set `name` and `state` to have more control over whether the package is `present`, `absent`, `latest`, etc. ```yaml -kubernetes_version: '1.25' -kubernetes_version_rhel_package: '1.25.0' +kubernetes_version: '1.24' +kubernetes_version_rhel_package: '1.24.4' ``` The minor version of Kubernetes to install. The plain `kubernetes_version` is used to pin an apt package version on Debian, and as the Kubernetes version passed into the `kubeadm init` command (see `kubernetes_version_kubeadm`). The `kubernetes_version_rhel_package` variable must be a specific Kubernetes release, and is used to pin the version on Red Hat / CentOS servers. diff --git a/defaults/main.yml b/defaults/main.yml index a1ca4b1..8aeaceb 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -9,8 +9,8 @@ kubernetes_packages: - name: kubernetes-cni state: present -kubernetes_version: '1.25' -kubernetes_version_rhel_package: '1.25.0' +kubernetes_version: '1.24' +kubernetes_version_rhel_package: '1.24.4' kubernetes_role: control_plane diff --git a/molecule/default/molecule.yml b/molecule/default/molecule.yml index cc47e45..514da27 100644 --- a/molecule/default/molecule.yml +++ b/molecule/default/molecule.yml @@ -9,7 +9,7 @@ platforms: image: "geerlingguy/docker-${MOLECULE_DISTRO:-centos7}-ansible:latest" command: ${MOLECULE_DOCKER_COMMAND:-""} volumes: - - /sys/fs/cgroup:/sys/fs/cgroup:ro + - /sys/fs/cgroup:/sys/fs/cgroup:rw - /var/lib/containerd privileged: true pre_build_image: true diff --git a/tasks/control-plane-setup.yml b/tasks/control-plane-setup.yml index 609fd47..347e5cc 100644 --- a/tasks/control-plane-setup.yml +++ b/tasks/control-plane-setup.yml @@ -25,21 +25,6 @@ {{ kubernetes_kubeadm_init_extra_opts }} register: kubeadmin_init when: (not kubernetes_init_stat.stat.exists) and (kubernetes_ignore_preflight_errors is defined) -# failed_when: false # DELETEME START --- -# - name: Test -# command: systemctl status kubelet -# changed_when: false -# register: systemctl_result - -# - debug: var=systemctl_result - -# - name: Test 2 -# command: journalctl --no-pager -xeu kubelet -# changed_when: false -# register: journalctl_result - -# - debug: var=journalctl_result -# # DELETEME END --- - name: Print the init output to screen. debug: From ad1afeebb600afb7e1e2660cdec2f8ffb88e38c9 Mon Sep 17 00:00:00 2001 From: Jeff Geerling Date: Fri, 16 Sep 2022 10:29:58 -0500 Subject: [PATCH 15/18] Back to 1.25. --- README.md | 4 ++-- defaults/main.yml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index c0ff006..a4018e5 100644 --- a/README.md +++ b/README.md @@ -27,8 +27,8 @@ kubernetes_packages: Kubernetes packages to be installed on the server. You can either provide a list of package names, or set `name` and `state` to have more control over whether the package is `present`, `absent`, `latest`, etc. ```yaml -kubernetes_version: '1.24' -kubernetes_version_rhel_package: '1.24.4' +kubernetes_version: '1.25' +kubernetes_version_rhel_package: '1.25.1' ``` The minor version of Kubernetes to install. The plain `kubernetes_version` is used to pin an apt package version on Debian, and as the Kubernetes version passed into the `kubeadm init` command (see `kubernetes_version_kubeadm`). The `kubernetes_version_rhel_package` variable must be a specific Kubernetes release, and is used to pin the version on Red Hat / CentOS servers. diff --git a/defaults/main.yml b/defaults/main.yml index 8aeaceb..9827631 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -9,8 +9,8 @@ kubernetes_packages: - name: kubernetes-cni state: present -kubernetes_version: '1.24' -kubernetes_version_rhel_package: '1.24.4' +kubernetes_version: '1.25' +kubernetes_version_rhel_package: '1.25.1' kubernetes_role: control_plane From 219c089243035c8a2507b3f2133a6bb9f80eb176 Mon Sep 17 00:00:00 2001 From: Jeff Geerling Date: Wed, 26 Oct 2022 10:08:27 -0500 Subject: [PATCH 16/18] Debugging with journalctl. --- tasks/control-plane-setup.yml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/tasks/control-plane-setup.yml b/tasks/control-plane-setup.yml index 347e5cc..f092cff 100644 --- a/tasks/control-plane-setup.yml +++ b/tasks/control-plane-setup.yml @@ -24,8 +24,23 @@ --ignore-preflight-errors={{ kubernetes_ignore_preflight_errors }} {{ kubernetes_kubeadm_init_extra_opts }} register: kubeadmin_init + failed_when: false # TODO REMOVE when: (not kubernetes_init_stat.stat.exists) and (kubernetes_ignore_preflight_errors is defined) +# TODO REMOVE +- name: Print kubeadm init output + debug: var=kubeadmin_init.stdout + +# TODO REMOVE +- name: Get kubelet status + command: journalctl --no-pager -xeu kubelet + register: journalctl + changed_when: false + +# TODO REMOVE +- name: Print kubelet status + debug: var=journalctl.stdout + - name: Print the init output to screen. debug: var: kubeadmin_init.stdout From 27837c56711e53e8c7c3302b688219e2714e230c Mon Sep 17 00:00:00 2001 From: Jeff Geerling Date: Wed, 26 Oct 2022 10:34:05 -0500 Subject: [PATCH 17/18] Use kubelet configuration for swap control and other vars. --- README.md | 4 ++-- molecule/default/calico.yml | 10 +++++----- molecule/default/converge.yml | 10 +++++----- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index a4018e5..8126133 100644 --- a/README.md +++ b/README.md @@ -73,7 +73,7 @@ kubernetes_config_kubelet_configuration: cgroupDriver: systemd ``` -Options to configure kubelet on any nodes in your cluster through the `kubeadm init` process. To get the syntax of this options see https://kubernetes.io/docs/tasks/administer-cluster/kubelet-config-file and https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/kubelet-integration. +Options to configure kubelet on any nodes in your cluster through the `kubeadm init` process. For syntax options read the [kubelet config file](https://kubernetes.io/docs/tasks/administer-cluster/kubelet-config-file) and [kubelet integration](https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/kubelet-integration) documentation. NOTE: This is the recommended way to do the kubelet-configuration. Most command-line-options are deprecated. @@ -92,7 +92,7 @@ kubernetes_kubelet_extra_args: "" kubernetes_kubelet_extra_args_config_file: /etc/default/kubelet ``` -Extra args to pass to `kubelet` during startup. E.g. to allow `kubelet` to start up even if there is swap is enabled on your server, set this to: `"--fail-swap-on=false"`. Or to specify the node-ip advertised by `kubelet`, set this to `"--node-ip={{ ansible_host }}"`. *This is deprecated. Please use `kubernetes_config_kubelet_configuration` instead.* +Extra args to pass to `kubelet` during startup. E.g. to allow `kubelet` to start up even if there is swap is enabled on your server, set this to: `"--fail-swap-on=false"`. Or to specify the node-ip advertised by `kubelet`, set this to `"--node-ip={{ ansible_host }}"`. **This option is deprecated. Please use `kubernetes_config_kubelet_configuration` instead.** ```yaml kubernetes_kubeadm_init_extra_opts: "" diff --git a/molecule/default/calico.yml b/molecule/default/calico.yml index cd86eca..b2060ba 100644 --- a/molecule/default/calico.yml +++ b/molecule/default/calico.yml @@ -9,11 +9,11 @@ cidr: '192.168.0.0/16' # Allow swap in test environments (hard to control in some envs). - kubernetes_kubelet_extra_args: >- - --fail-swap-on=false - --cgroup-driver=systemd - --cgroups-per-qos=false - --enforce-node-allocatable="" + kubernetes_config_kubelet_configuration: + cgroupDriver: "systemd" + failSwapOn: false + cgroupsPerQOS: false + enforceNodeAllocatable: "" containerd_config_cgroup_driver_systemd: true pre_tasks: diff --git a/molecule/default/converge.yml b/molecule/default/converge.yml index 45a38fc..248ed37 100644 --- a/molecule/default/converge.yml +++ b/molecule/default/converge.yml @@ -5,11 +5,11 @@ vars: # Allow swap in test environments (hard to control in some envs). - kubernetes_kubelet_extra_args: >- - --fail-swap-on=false - --cgroup-driver=systemd - --cgroups-per-qos=false - --enforce-node-allocatable="" + kubernetes_config_kubelet_configuration: + cgroupDriver: "systemd" + failSwapOn: false + cgroupsPerQOS: false + enforceNodeAllocatable: "" containerd_config_cgroup_driver_systemd: true pre_tasks: From 040a10d252a3003bc1397b51afd71e141ca64767 Mon Sep 17 00:00:00 2001 From: Jeff Geerling Date: Wed, 26 Oct 2022 12:38:20 -0500 Subject: [PATCH 18/18] Attempt to get tests working with debug. --- README.md | 2 +- molecule/default/calico.yml | 4 ++-- molecule/default/converge.yml | 4 ++-- molecule/default/molecule.yml | 1 + tasks/control-plane-setup.yml | 4 ++-- 5 files changed, 8 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 8126133..6fb94f4 100644 --- a/README.md +++ b/README.md @@ -49,7 +49,7 @@ kubernetes_kubeadm_kubelet_config_file_path: '/etc/kubernetes/kubeadm-kubelet-co Path for ``. If the directory does not exist, this role will create it. -The following variables are parsed as options to . To understand its syntax, see https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/kubelet-integration and https://kubernetes.io/docs/reference/setup-tools/kubeadm/kubeadm-init/#config-file . The skeleton (`apiVersion`, `kind`) of the config file will be created by this role, so do not define them within the variables. (See `templates/kubeadm-kubelet-config.j2`). +The following variables are parsed as options to . To understand its syntax, see [kubelet-integration](https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/kubelet-integration) and [kubeadm-config-file](https://kubernetes.io/docs/reference/setup-tools/kubeadm/kubeadm-init/#config-file) . The skeleton (`apiVersion`, `kind`) of the config file will be created by this role, so do not define them within the variables. (See `templates/kubeadm-kubelet-config.j2`). ```yaml kubernetes_config_init_configuration: diff --git a/molecule/default/calico.yml b/molecule/default/calico.yml index b2060ba..74998b2 100644 --- a/molecule/default/calico.yml +++ b/molecule/default/calico.yml @@ -12,8 +12,8 @@ kubernetes_config_kubelet_configuration: cgroupDriver: "systemd" failSwapOn: false - cgroupsPerQOS: false - enforceNodeAllocatable: "" + cgroupsPerQOS: true + enforceNodeAllocatable: ['pods'] containerd_config_cgroup_driver_systemd: true pre_tasks: diff --git a/molecule/default/converge.yml b/molecule/default/converge.yml index 248ed37..1e953b6 100644 --- a/molecule/default/converge.yml +++ b/molecule/default/converge.yml @@ -8,8 +8,8 @@ kubernetes_config_kubelet_configuration: cgroupDriver: "systemd" failSwapOn: false - cgroupsPerQOS: false - enforceNodeAllocatable: "" + cgroupsPerQOS: true + enforceNodeAllocatable: ['pods'] containerd_config_cgroup_driver_systemd: true pre_tasks: diff --git a/molecule/default/molecule.yml b/molecule/default/molecule.yml index 514da27..518aa05 100644 --- a/molecule/default/molecule.yml +++ b/molecule/default/molecule.yml @@ -11,6 +11,7 @@ platforms: volumes: - /sys/fs/cgroup:/sys/fs/cgroup:rw - /var/lib/containerd + cgroupns_mode: host privileged: true pre_build_image: true provisioner: diff --git a/tasks/control-plane-setup.yml b/tasks/control-plane-setup.yml index f092cff..96c20da 100644 --- a/tasks/control-plane-setup.yml +++ b/tasks/control-plane-setup.yml @@ -29,7 +29,7 @@ # TODO REMOVE - name: Print kubeadm init output - debug: var=kubeadmin_init.stdout + debug: var=kubeadmin_init # TODO REMOVE - name: Get kubelet status @@ -39,7 +39,7 @@ # TODO REMOVE - name: Print kubelet status - debug: var=journalctl.stdout + debug: var=journalctl - name: Print the init output to screen. debug: