add k0s support (#1123)

* ignore idea * Initial work on support for k0s * * Changed config param to bootstrap_distribution * Replaced democratic-csi with openebs * Added notes about system-upgrade-controller being ignored with k0s * Removed password for nodes * * Switched to single storage path for openebs * Removed democratic-csi helm repo * Added missing openebs helm repo * Fixed template formatting * Made bootstrap_private_github_repo hidden/undocumented * Updated flux cluster config to accommodate bootstrap_private_github_repo * * added the storageclass name to be declarative * * added newline * set chart wait false * * fixed spacing in K0sTasks * fixed openebs path * properly added openebs helm repo * * updated configure task to use venv * renamed bootstrap_kube_vip_addr to bootstrap_kubeapi_addr * renamed bootstrap_kube_vip_addr derived variables accordingly * * tweaked configure task cmd * started work on k0s-config.yaml template * * tweaked k0s task cmd * moved coredns under addons rather than optional * fixed typo in coredns task * normalized formatting in k0s-config * disabled telemetry in k0s-config * added renovate to k0s-config * removed tunnel and added routing mode in k0s-config cilium chart values * updated docs in config example * fixed issue with k0s-config controller role so it didn't default to worker+controller if more than one master * * fixed whitespace * add k0sctl to brew tasks * Update bootstrap/vars/config.sample.yaml * Update bootstrap/vars/config.sample.yaml * Update k0s-config.yaml.j2 * Update config.sample.yaml * controllerManager and scheduler bind to `0.0.0.0` for metrics * Update custom-cilium-helmchart.yaml.j2.j2 * Update helmvalues.yaml.j2 * Update bootstrap/tasks/addons/main.yaml * first pass at readme updates * update placement of k0sconfig options * disable telemetry on k0sctl too * disable k0s telemetry in env too * update: jinja spacing in k0sconfig Signed-off-by: Devin Buhl <devin@buhl.casa> * fix: ensure bootstrap_local_storage_path is delete on nuke Signed-off-by: Devin Buhl <devin@buhl.casa> * fix: update renovate regex for k3s/k0s Signed-off-by: Devin Buhl <devin@buhl.casa> * fix: update nuke playbook and only include k3s task when k3s Signed-off-by: Devin Buhl <devin@buhl.casa> * fix: update nuke playbook and only include k3s task when k3s and include jinja comments Signed-off-by: Devin Buhl <devin@buhl.casa> --------- Signed-off-by: Devin Buhl <devin@buhl.casa> Co-authored-by: Devin Buhl <onedr0p@users.noreply.github.com> Co-authored-by: Devin Buhl <devin@buhl.casa>
onedr0p · Jan 11, 2024 · d093d0a · d093d0a
1 parent 1b73543
commit d093d0a
Show file tree

Hide file tree

Showing 43 changed files with 355 additions and 153 deletions.
diff --git a/.envrc b/.envrc
@@ -8,3 +8,5 @@ export ANSIBLE_COLLECTIONS_PATH=$(expand_path ./.venv/galaxy)
 export ANSIBLE_ROLES_PATH=$(expand_path ./.venv/galaxy/ansible_roles)
 export ANSIBLE_VARS_ENABLED="host_group_vars,community.sops.sops"
 export K8S_AUTH_KUBECONFIG="$(expand_path ./kubeconfig)"
+# k0s
+export DISABLE_TELEMETRY="true"
diff --git a/.github/renovate.json5 b/.github/renovate.json5
@@ -78,10 +78,10 @@
     },
     // custom versioning
     {
-      "description": "Use custom versioning for k3s",
+      "description": "Use custom versioning for k0s/k3s",
       "matchDatasources": ["github-releases"],
-      "versioning": "regex:^v(?<major>\\d+)\\.(?<minor>\\d+)\\.(?<patch>\\d+)(?<compatibility>\\+k3s)(?<build>\\d+)$",
-      "matchPackagePatterns": ["k3s"]
+      "versioning": "regex:^v(?<major>\\d+)\\.(?<minor>\\d+)\\.(?<patch>\\d+)(?<compatibility>\\+k.s)\\.?(?<build>\\d+)$",
+      "matchPackagePatterns": ["k0s", "k3s"]
     },
     // commit message topics
     {

diff --git a/.gitignore b/.gitignore
@@ -12,3 +12,6 @@ kubeconfig
 .venv*
 # Taskfile
 .tasks
+# intellij
+.idea
+
diff --git a/.taskfiles/BrewTasks.yaml b/.taskfiles/BrewTasks.yaml
@@ -20,6 +20,7 @@ tasks:
         fluxcd/tap/flux
         helm
         jq
+        k0sproject/tap/k0sctl
         k9s
         kubernetes-cli
         kustomize

diff --git a/.taskfiles/FluxTasks.yaml b/.taskfiles/FluxTasks.yaml
@@ -26,9 +26,15 @@ tasks:
       - kubectl apply --server-side --filename {{.KUBERNETES_DIR}}/flux/vars/cluster-settings-user.yaml
       - kubectl apply --server-side --kustomize {{.KUBERNETES_DIR}}/flux/config
     preconditions:
-      - { msg: "Flux already appears installed", sh: "exit $(( ! $(kubectl get namespace flux-system) ))" }
+      - { msg: "Flux already appears installed", sh: "kubectl get namespace flux-system &>/dev/null && exit 1 || exit 0" }
       - { msg: "Age private key not found",      sh: "test -f {{.ROOT_DIR}}/age.key" }
 
+  github-deploy-key:
+    cmds:
+      - sops --decrypt {{.KUBERNETES_DIR}}/bootstrap/github-deploy-key.sops.yaml | kubectl apply -f -
+    preconditions:
+      - { msg: "Flux is not installed", sh: "kubectl get namespace flux-system &>/dev/null && exit 0 || exit 1" }
+
   apply:
     desc: Apply a Flux Kustomization resource for a cluster
     summary: |

diff --git a/.taskfiles/K0sTasks.yaml b/.taskfiles/K0sTasks.yaml
@@ -0,0 +1,28 @@
+---
+version: "3"
+
+env:
+  DISABLE_TELEMETRY: "true"
+
+tasks:
+
+  kubeconfig:
+    desc: Gets k0s cluster kubeconfig
+    cmds:
+      - k0sctl kubeconfig --config k0s-config.yaml > kubeconfig
+    preconditions:
+      - { msg: "k0s-config.yaml not found", sh: "test -f {{.ROOT_DIR}}/k0s-config.yaml" }
+
+  apply:
+    desc: Apply k0s cluster k0s-config.yaml
+    cmds:
+      - k0sctl apply --config k0s-config.yaml
+      - task: kubeconfig
+    preconditions:
+      - { msg: "k0s-config.yaml not found", sh: "test -f {{.ROOT_DIR}}/k0s-config.yaml" }
+
+  reset:
+    desc: Resets the k0s cluster
+    cmd: k0sctl reset --config k0s-config.yaml
+    preconditions:
+      - { msg: "k0s-config.yaml not found", sh: "test -f {{.ROOT_DIR}}/k0s-config.yaml" }
diff --git a/README.md b/README.md
@@ -1,6 +1,6 @@
 # Deploy a Kubernetes cluster backed by Flux
 
-Welcome to my highly opinionated template for deploying a single Kubernetes ([k3s](https://k3s.io)) cluster with [Ansible](https://www.ansible.com) and using [Flux](https://toolkit.fluxcd.io) to manage its state.
+Welcome to my highly opinionated template for deploying a single Kubernetes ([k3s](https://k3s.io) or [k0s](https://github.com/k0sproject/k0s)) cluster with [Ansible](https://www.ansible.com) and using [Flux](https://toolkit.fluxcd.io) to manage its state.
 
 ## 👋 Introduction
 
@@ -258,7 +258,7 @@ Once you have installed Debian on your nodes, there are six stages to getting a
 > └─📁 apps          # Apps deployed into the cluster grouped by namespace
 > ```
 
-### ⚡ Stage 4: Prepare your nodes for k3s
+### ⚡ Stage 4: Prepare your nodes for Kubernetes
 
 📍 _Here we will be running an Ansible playbook to prepare your nodes for running a Kubernetes cluster._
 
@@ -282,7 +282,7 @@ Once you have installed Debian on your nodes, there are six stages to getting a
     task ansible:run playbook=cluster-prepare
     ```
 
-### ⛵ Stage 5: Use Ansible to install k3s
+### ⛵ Stage 5: Install Kubernetes
 
 📍 _Here we will be running a Ansible Playbook to install [k3s](https://k3s.io/) with [this](https://galaxy.ansible.com/xanmanning/k3s) Ansible galaxy role. If you run into problems, you can run `task ansible:run playbook=cluster-nuke` to destroy the k3s cluster and start over from this point._
 
@@ -298,13 +298,16 @@ Once you have installed Debian on your nodes, there are six stages to getting a
     task ansible:ping
     ```
 
-3. Install k3s with Ansible
+3. Install Kubernetes
 
     ```sh
+    # k3s
     task ansible:run playbook=cluster-installation
+    # k0s
+    task k0s:apply
     ```
 
-4. Verify the nodes are online
+5. Verify the nodes are online
 
     📍 _If this command **fails** you likely haven't configured `direnv` as mentioned previously in the guide._
 
@@ -315,7 +318,7 @@ Once you have installed Debian on your nodes, there are six stages to getting a
     # k8s-1          Ready    worker                      1h      v1.27.3+k3s1
     ```
 
-5. The `kubeconfig` for interacting with your cluster should have been created in the root of your repository.
+6. The `kubeconfig` for interacting with your cluster should have been created in the root of your repository.
 
 ### 🔹 Stage 6: Install Flux in your cluster
 

diff --git a/Taskfile.yaml b/Taskfile.yaml
@@ -17,6 +17,7 @@ includes:
     aliases: ["k8s"]
     taskfile: .taskfiles/KubernetesTasks.yaml
   flux: .taskfiles/FluxTasks.yaml
+  k0s: .taskfiles/K0sTasks.yaml
 
 tasks:
 
@@ -45,7 +46,6 @@ tasks:
   configure:
     desc: Configure repository from Ansible vars
     prompt: Any conflicting config in the root kubernetes and ansible directories will be overwritten... continue?
-    dir: "{{.BOOTSTRAP_DIR}}"
-    cmd: ansible-playbook configure.yaml
+    cmd: ./.venv/bin/ansible-playbook {{.BOOTSTRAP_DIR}}/configure.yaml
     env:
       ANSIBLE_DISPLAY_SKIPPED_HOSTS: "false"
diff --git a/bootstrap/configure.yaml b/bootstrap/configure.yaml
@@ -17,18 +17,22 @@
       ansible.builtin.set_fact:
         repository_path: "{{ repository.stdout }}"
 
-    - name: Override kube-vip address when there is a single master node and no address is defined
-      when: bootstrap_nodes.master | length == 1 and not bootstrap_kube_vip_addr
+    - name: Override kubeapi address when there is a single master node and no address is defined
+      when: bootstrap_nodes.master | length == 1 and not bootstrap_kubeapi_addr
       ansible.builtin.set_fact:
         bootstrap_kube_vip_enabled: false
-        bootstrap_kube_vip_addr: "{{ bootstrap_nodes.master[0].address }}"
+        bootstrap_kubeapi_addr: "{{ bootstrap_nodes.master[0].address }}"
 
     - name: Verify configuration
       ansible.builtin.include_tasks: tasks/validation/main.yaml
 
     - name: Template Sops configuration
       ansible.builtin.include_tasks: tasks/sops/main.yaml
 
+    - name: Template k0s configuration
+      when: bootstrap_distribution == "k0s"
+      ansible.builtin.include_tasks: tasks/k0s/main.yaml
+
     - name: Template Ansible configuration
       ansible.builtin.include_tasks: tasks/ansible/main.yaml
 

diff --git a/bootstrap/tasks/addons/coredns.yaml b/bootstrap/tasks/addons/coredns.yaml
@@ -0,0 +1,34 @@
+---
+- name: Set addon facts
+  ansible.builtin.set_fact:
+    addon_name: coredns
+    addon_namespace: kube-system
+
+- name: Ensure directories exist for {{ addon_namespace }}/{{ addon_name }}
+  when: item.state == 'directory'
+  ansible.builtin.file:
+    path: "{{ repository_path }}/kubernetes/apps/{{ addon_namespace }}/{{ addon_name }}/{{ item.path }}"
+    state: directory
+    mode: "0755"
+  with_community.general.filetree: ["../templates/addons/{{ addon_name }}/"]
+
+- name: Template unencrypted files for {{ addon_namespace }}/{{ addon_name }}
+  when: item.state == 'file' and 'sops' not in item.path
+  ansible.builtin.template:
+    src: "{{ item.src }}"
+    dest: "{{ repository_path }}/kubernetes/apps/{{ addon_namespace }}/{{ addon_name }}/{{ item.path | replace('.j2', '') }}"
+    mode: "0644"
+  with_community.general.filetree: ["../templates/addons/{{ addon_name }}/"]
+
+- name: Template encrypted files for {{ addon_namespace }}/{{ addon_name }}
+  block:
+    - name: Template encrypted files
+      when: item.state == 'file' and 'sops' in item.path
+      community.sops.sops_encrypt:
+        path: "{{ repository_path }}/kubernetes/apps/{{ addon_namespace }}/{{ addon_name }}/{{ item.path | replace('.j2', '') }}"
+        encrypted_regex: ^(data|stringData)$
+        age: ["{{ bootstrap_age_public_key }}"]
+        content_yaml: "{{ lookup('ansible.builtin.template', item.src) | from_yaml }}"
+        mode: "0644"
+        force: true
+      with_community.general.filetree: ["../templates/addons/{{ addon_name }}/"]
diff --git a/bootstrap/tasks/addons/main.yaml b/bootstrap/tasks/addons/main.yaml
@@ -1,5 +1,9 @@
 ---
 
+- name: Process optional coredns
+  when: bootstrap_distribution == "k3s"
+  ansible.builtin.include_tasks: coredns.yaml
+
 - name: Process addon csi-driver-nfs
   when: csi_driver_nfs.enabled | default(false)
   ansible.builtin.include_tasks: csi_driver_nfs.yaml
@@ -21,7 +25,9 @@
   ansible.builtin.include_tasks: kube_prometheus_stack.yaml
 
 - name: Process addon system-upgrade-controller
-  when: system_upgrade_controller.enabled | default(false)
+  when:
+    - bootstrap_distribution == "k3s"
+    - system_upgrade_controller.enabled | default(false)
   ansible.builtin.include_tasks: system_upgrade_controller.yaml
 
 - name: Process addon weave-gitops

diff --git a/bootstrap/tasks/ansible/main.yaml b/bootstrap/tasks/ansible/main.yaml
@@ -27,13 +27,4 @@
         mode: "0644"
         force: true
       with_community.general.filetree: ["../templates/ansible/"]
-    - name: Template encrypted node secrets
-      community.sops.sops_encrypt:
-        path: "{{ repository_path }}/ansible/inventory/host_vars/{{ item.name }}.sops.yaml"
-        age: ["{{ bootstrap_age_public_key }}"]
-        content_yaml: "{{ lookup('ansible.builtin.template', 'templates/node.sops.yaml.j2', template_vars=dict(password=item.password)) | from_yaml }}"
-        mode: "0644"
-        force: true
-      loop: "{{ bootstrap_nodes.master + bootstrap_nodes.worker | default([]) }}"
-      loop_control:
-        label: "{{ item.address }}"
+
diff --git a/bootstrap/tasks/k0s/main.yaml b/bootstrap/tasks/k0s/main.yaml
@@ -0,0 +1,6 @@
+---
+- name: Template k0s configuration file
+  ansible.builtin.template:
+    src: "templates/k0s-config.yaml.j2"
+    dest: "{{ repository_path }}/k0s-config.yaml"
+    mode: "0644"
diff --git a/bootstrap/tasks/validation/github.yaml b/bootstrap/tasks/validation/github.yaml
@@ -14,6 +14,7 @@
     fail_msg: Github user {{ bootstrap_github_username }} does not exist
 
 - name: Query Github repo
+  when: not bootstrap_private_github_repo | default(false)
   ansible.builtin.uri:
     url: https://api.github.com/repos/{{ bootstrap_github_username }}/{{ bootstrap_github_repository_name }}
     timeout: 5
@@ -22,12 +23,14 @@
   register: result
 
 - name: Check if repo exists
+  when: not bootstrap_private_github_repo | default(false)
   ansible.builtin.assert:
     that: result.json.full_name == bootstrap_github_username + '/' + bootstrap_github_repository_name
     success_msg: Github repo {{ bootstrap_github_username }}/{{ bootstrap_github_repository_name }} exists
     fail_msg: Github repo {{ bootstrap_github_username }}/{{ bootstrap_github_repository_name }} does not exist
 
 - name: Query Github repo branch
+  when: not bootstrap_private_github_repo | default(false)
   ansible.builtin.uri:
     url: https://api.github.com/repos/{{ bootstrap_github_username }}/{{ bootstrap_github_repository_name }}/branches/{{ bootstrap_github_repository_branch | default('main', true) }}
     timeout: 5
@@ -36,6 +39,7 @@
   register: result
 
 - name: Check if repo branch exists
+  when: not bootstrap_private_github_repo | default(false)
   ansible.builtin.assert:
     that: result.json.name == bootstrap_github_repository_branch | default('main', true)
     success_msg: Github repo branch {{ bootstrap_github_repository_branch | default('main', true) }} exists

diff --git a/bootstrap/tasks/validation/net.yaml b/bootstrap/tasks/validation/net.yaml
@@ -98,17 +98,17 @@
     success_msg: external ingress address {{ bootstrap_external_ingress_addr }} is within {{ bootstrap_node_cidr }}.
     fail_msg: external ingress address {{ bootstrap_external_ingress_addr }} is not within {{ bootstrap_node_cidr }}.
 
-- name: Verify kube-vip
+- name: Verify kubeapi address
   ansible.builtin.assert:
-    that: bootstrap_kube_vip_addr is ansible.utils.ipv4
-    success_msg: kube-vip address {{ bootstrap_kube_vip_addr }} is valid.
-    fail_msg: kube-vip address {{ bootstrap_kube_vip_addr }} is invalid.
+    that: bootstrap_kubeapi_addr is ansible.utils.ipv4
+    success_msg: kubeapi address {{ bootstrap_kubeapi_addr }} is valid.
+    fail_msg: kubeapi address {{ bootstrap_kubeapi_addr }} is invalid.
 
-- name: Verify kube-vip in node CIDR
+- name: Verify kubeapi address in node CIDR
   ansible.builtin.assert:
-    that: bootstrap_node_cidr | ansible.utils.network_in_usable(bootstrap_kube_vip_addr)
-    success_msg: kube-vip address {{ bootstrap_kube_vip_addr }} is within {{ bootstrap_node_cidr }}.
-    fail_msg: kube-vip address {{ bootstrap_kube_vip_addr }} is not within {{ bootstrap_node_cidr }}.
+    that: bootstrap_node_cidr | ansible.utils.network_in_usable(bootstrap_kubeapi_addr)
+    success_msg: kubeapi address {{ bootstrap_kubeapi_addr }} is within {{ bootstrap_node_cidr }}.
+    fail_msg: kubeapi address {{ bootstrap_kubeapi_addr }} is not within {{ bootstrap_node_cidr }}.
 
 - name: Verify all IP addresses are unique
   ansible.builtin.assert:
@@ -117,7 +117,7 @@
         bootstrap_k8s_gateway_addr,
         bootstrap_external_ingress_addr,
         bootstrap_internal_ingress_addr,
-        bootstrap_kube_vip_addr
+        bootstrap_kubeapi_addr
       ] | unique | length == 4
     success_msg: All IP addresses are unique.
     fail_msg: All IP addresses are not unique.
@@ -133,12 +133,12 @@
   loop_control:
     label: "{{ item.address }}"
 
-- name: Verify nodes are not the same IPs as k8s_gateway, ingress external/internal or kube-vip
-  when: bootstrap_kube_vip_enabled | default(true)
+- name: Verify nodes are not the same IPs as k8s_gateway, ingress external/internal or kubeapi address
+  when: (bootstrap_distribution == "k3s") and (bootstrap_kube_vip_enabled | default(true))
   ansible.builtin.assert:
-    that: item.address not in (bootstrap_k8s_gateway_addr, bootstrap_external_ingress_addr, bootstrap_internal_ingress_addr, bootstrap_kube_vip_addr)
-    success_msg: Node address {{ item.address }} is different than k8s_gateway, ingress-nginx or kube-vip.
-    fail_msg: Node address {{ item.address }} is not different than k8s_gateway, ingress-nginx or kube-vip.
+    that: item.address not in (bootstrap_k8s_gateway_addr, bootstrap_external_ingress_addr, bootstrap_internal_ingress_addr, bootstrap_kubeapi_addr)
+    success_msg: Node address {{ item.address }} is different than k8s_gateway, ingress-nginx or kubeapi.
+    fail_msg: Node address {{ item.address }} is not different than k8s_gateway, ingress-nginx or kubeapi.
     quiet: true
   loop: "{{ bootstrap_nodes.master + bootstrap_nodes.worker | default([]) }}"
   loop_control:

diff --git a/bootstrap/tasks/validation/vars.yaml b/bootstrap/tasks/validation/vars.yaml
@@ -16,23 +16,24 @@
     - bootstrap_cloudflare_tunnel_id
     - bootstrap_cloudflare_tunnel_secret
     - bootstrap_cluster_cidr
+    - bootstrap_distribution
+    - bootstrap_external_ingress_addr
     - bootstrap_flux_github_webhook_token
-    - bootstrap_github_repository_name
     - bootstrap_github_repository_branch
+    - bootstrap_github_repository_name
     - bootstrap_github_username
-    - bootstrap_external_ingress_addr
     - bootstrap_internal_ingress_addr
     - bootstrap_ipv6_enabled
     - bootstrap_k8s_gateway_addr
-    - bootstrap_kube_vip_addr
-    - bootstrap_local_path_provisioner_path
+    - bootstrap_kubeapi_addr
+    - bootstrap_local_storage_path
     - bootstrap_node_cidr
     - bootstrap_service_cidr
     - bootstrap_timezone
 
 - name: Verify bootstrap node names are valid
   ansible.builtin.assert:
-    that: item.name is match('^[a-z0-9-]+$')
+    that: item.name is match('^[a-z0-9-\.]+$')
     success_msg: Node name {{ item.name }} is valid
     fail_msg: Node name {{ item.name }} is not valid
   loop: "{{ bootstrap_nodes.master + bootstrap_nodes.worker | default([]) }}"

diff --git a/...be-system/coredns/app/helmrelease.yaml.j2 → ...es/addons/coredns/app/helmrelease.yaml.j2 b/...be-system/coredns/app/helmrelease.yaml.j2 → ...es/addons/coredns/app/helmrelease.yaml.j2
diff --git a/...-system/coredns/app/kustomization.yaml.j2 → .../addons/coredns/app/kustomization.yaml.j2 b/...-system/coredns/app/kustomization.yaml.j2 → .../addons/coredns/app/kustomization.yaml.j2
diff --git a/...netes/apps/kube-system/coredns/ks.yaml.j2 → ...strap/templates/addons/coredns/ks.yaml.j2 b/...netes/apps/kube-system/coredns/ks.yaml.j2 → ...strap/templates/addons/coredns/ks.yaml.j2
diff --git a/bootstrap/templates/addons/grafana/app/helmrelease.yaml.j2 b/bootstrap/templates/addons/grafana/app/helmrelease.yaml.j2
@@ -24,7 +24,7 @@ spec:
   uninstall:
     keepHistory: false
   dependsOn:
-    - name: local-path-provisioner
+    - name: openebs
       namespace: storage
   values:
     deploymentStrategy:
@@ -168,6 +168,6 @@ spec:
             - *host
     persistence:
       enabled: true
-      storageClassName: local-hostpath
+      storageClassName: openebs-hostpath
     testFramework:
       enabled: false
diff --git a/bootstrap/templates/addons/kube-prometheus-stack/app/helmrelease.yaml.j2 b/bootstrap/templates/addons/kube-prometheus-stack/app/helmrelease.yaml.j2
@@ -27,7 +27,7 @@ spec:
   uninstall:
     keepHistory: false
   dependsOn:
-    - name: local-path-provisioner
+    - name: openebs
       namespace: storage
   valuesFrom:
     - name: kube-prometheus-stack-values