feat: add spegel support for k0s (#1178)

* feat: add spegel support for k0s

Signed-off-by: Devin Buhl <devin@buhl.casa>

* chore: add spegel to e2e tests

Signed-off-by: Devin Buhl <devin@buhl.casa>

* fix: update containerd file paths and run apply from k0s-config dir

Signed-off-by: Devin Buhl <devin@buhl.casa>

* fix: move containerd files after upload

Signed-off-by: Devin Buhl <devin@buhl.casa>

---------

Signed-off-by: Devin Buhl <devin@buhl.casa>

.github/renovate.json5

@@ -223,8 +223,7 @@
       "fileMatch": [
         "(^|/).taskfiles/.+\\.ya?ml$",
         "(^|/)ansible/.+\\.ya?ml(\\.j2)?(\\.j2)?$",
-        "(^|/)kubernetes/.+\\.ya?ml(\\.j2)?(\\.j2)?$",
-        "(^|/)k0s-config.ya?ml(\\.j2)?(\\.j2)?$"
+        "(^|/)kubernetes/.+\\.ya?ml(\\.j2)?(\\.j2)?$"
       ],
       "matchStrings": [
         // Example:

.github/tests/addons.yaml

@@ -32,3 +32,6 @@ discord_template_notifier:
 
 volsync:
   enabled: true
+
+spegel:
+  enabled: true

.taskfiles/K0s/Taskfile.yaml

@@ -6,12 +6,13 @@ env:
   DISABLE_UPGRADE_CHECK: "true"
 
 vars:
-  K0S_CONFIG_FILE: "{{.ROOT_DIR}}/k0s-config.yaml"
+  K0S_CONFIG_FILE: "{{.KUBERNETES_DIR}}/k0s/k0s-config.yaml"
 
 tasks:
 
   apply:
     desc: Apply k0s cluster config
+    dir: "{{.KUBERNETES_DIR}}/k0s" # Needed for uploading files in k0s-config.yaml
     cmds:
       - k0sctl apply --config {{.K0S_CONFIG_FILE}}
       - task: kubeconfig

bootstrap/templates/kubernetes/apps/kube-system/kustomization.yaml.j2

@@ -8,3 +8,6 @@ resources:
   - ./coredns/ks.yaml
   #% endif %#
   - ./metrics-server/ks.yaml
+  #% if bootstrap_distribution == "k0s" and spegel.enabled | default(false) %#
+  - ./spegel/ks.yaml
+  #% endif %#

bootstrap/templates/kubernetes/apps/kube-system/spegel/app/helmrelease.yaml.j2

@@ -0,0 +1,37 @@
+#% if bootstrap_distribution == "k0s" and spegel.enabled | default(false) %#
+---
+apiVersion: helm.toolkit.fluxcd.io/v2beta2
+kind: HelmRelease
+metadata:
+  name: spegel
+spec:
+  interval: 30m
+  chart:
+    spec:
+      chart: spegel
+      version: v0.0.17
+      sourceRef:
+        kind: HelmRepository
+        name: xenitab
+        namespace: flux-system
+  install:
+    remediation:
+      retries: 3
+  upgrade:
+    cleanupOnFail: true
+    remediation:
+      retries: 3
+  uninstall:
+    keepHistory: false
+  values:
+    spegel:
+      #% if bootstrap_distribution == "k0s" %#
+      containerdSock: /run/k0s/containerd.sock
+      containerdRegistryConfigPath: /var/lib/k0s/containerd/certs.d
+      #% endif %#
+    service:
+      registry:
+        hostPort: 29999
+    serviceMonitor:
+      enabled: true
+#% endif %#

bootstrap/templates/kubernetes/apps/kube-system/spegel/app/kustomization.yaml.j2

@@ -0,0 +1,7 @@
+#% if bootstrap_distribution == "k0s" and spegel.enabled | default(false) %#
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - ./helmrelease.yaml
+#% endif %#

bootstrap/templates/kubernetes/apps/kube-system/spegel/ks.yaml.j2

@@ -0,0 +1,22 @@
+#% if bootstrap_distribution == "k0s" and spegel.enabled | default(false) %#
+---
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: &app spegel
+  namespace: flux-system
+spec:
+  targetNamespace: kube-system
+  commonMetadata:
+    labels:
+      app.kubernetes.io/name: *app
+  path: ./kubernetes/apps/kube-system/spegel/app
+  prune: true
+  sourceRef:
+    kind: GitRepository
+    name: home-kubernetes
+  wait: false
+  interval: 30m
+  retryInterval: 1m
+  timeout: 5m
+#% endif %#

bootstrap/templates/kubernetes/flux/repositories/helm/kustomization.yaml.j2

@@ -21,3 +21,4 @@ resources:
   - ./prometheus-community.yaml
   - ./stakater.yaml
   - ./weave-gitops.yaml
+  - ./xenitab.yaml

bootstrap/templates/kubernetes/flux/repositories/helm/xenitab.yaml.j2

@@ -0,0 +1,10 @@
+---
+apiVersion: source.toolkit.fluxcd.io/v1beta2
+kind: HelmRepository
+metadata:
+  name: xenitab
+  namespace: flux-system
+spec:
+  type: oci
+  interval: 5m
+  url: oci://ghcr.io/xenitab/helm-charts

bootstrap/templates/k0s-config.yaml.j2 → bootstrap/templates/kubernetes/k0s/k0s-config.yaml.j2

@@ -16,13 +16,39 @@ spec:
         #% if item.role | default('') == 'controller+worker' %#
         - --no-taints
         #% endif %#
+      files:
+        - name: containerd-config
+          src: resources/containerd/
+          perm: 0644
+          user: root
+          group: root
+      hooks:
+        apply:
+          before:
+            #% if spegel.enabled | default(false) %#
+            - sudo mv ~/spegel.toml /etc/k0s/containerd.d/spegel.toml
+            #% endif %#
+            - sudo mv ~/unprivileged-ports.toml /etc/k0s/containerd.d/unprivileged-ports.toml
     #% endfor %#
     #% if bootstrap_nodes.worker | default([]) | length > 0 %#
     #% for item in bootstrap_nodes.worker %#
     - role: worker
       ssh:
         address: "#{ item.address }#"
         user: "#{ item.username }#"
+      files:
+        - name: containerd-config
+          src: resources/containerd/
+          perm: 0644
+          user: root
+          group: root
+      hooks:
+        apply:
+          before:
+            #% if spegel.enabled | default(false) %#
+            - sudo mv ~/spegel.toml /etc/k0s/containerd.d/spegel.toml
+            #% endif %#
+            - sudo mv ~/unprivileged-ports.toml /etc/k0s/containerd.d/unprivileged-ports.toml
     #% endfor %#
     #% endif %#
   k0s:
@@ -55,6 +81,15 @@ spec:
             - "#{ item.name }#"
             #% endif %#
             #% endfor %#
+        network:
+          kubeProxy:
+            disabled: true
+          #%  if bootstrap_nodes.master | length > 1 %#
+          nodeLocalLoadBalancing:
+            enabled: true
+            type: EnvoyProxy
+          #% endif %#
+          provider: custom
         extensions:
           helm:
             repositories:
@@ -119,13 +154,4 @@ spec:
                   routingMode: native
                   securityContext:
                     privileged: true
-        network:
-          kubeProxy:
-            disabled: true
-          #%  if bootstrap_nodes.master | length > 1 %#
-          nodeLocalLoadBalancing:
-            enabled: true
-            type: EnvoyProxy
-          #% endif %#
-          provider: custom
 #% endif %#

bootstrap/templates/kubernetes/k0s/resources/containerd/spegel.toml.j2

@@ -0,0 +1,6 @@
+#% if bootstrap_distribution == 'k0s' and spegel.enabled | default(false) %#
+[plugins."io.containerd.grpc.v1.cri".containerd]
+  discard_unpacked_layers = false
+[plugins."io.containerd.grpc.v1.cri".registry]
+  config_path = "/var/lib/k0s/containerd/certs.d"
+#% endif %#

bootstrap/templates/kubernetes/k0s/resources/containerd/unprivileged-ports.toml.j2

@@ -0,0 +1,5 @@
+#% if bootstrap_distribution == 'k0s' and spegel.enabled | default(false) %#
+[plugins."io.containerd.grpc.v1.cri"]
+  enable_unprivileged_ports = true
+  enable_unprivileged_icmp = true
+#% endif %#

bootstrap/vars/addons.sample.yaml

@@ -41,7 +41,7 @@ system_upgrade_controller:
   # WARNING: Only enable this if you also track the version of k3s in the
   # ansible configuration files. Running ansible against an already provisioned
   # cluster with this enabled might cause your cluster to be downgraded.
-  # Note that if bootstrap_distribution is set to k0s enable: true will be ignored.
+  # Note: If bootstrap_distribution is set to k0s this will be ignored.
   enabled: false
 
 # https://github.com/morphy2k/rss-forwarder
@@ -54,3 +54,8 @@ discord_template_notifier:
 # https://github.com/backube/volsync
 volsync:
   enabled: false
+
+# https://github.com/XenitAB/spegel
+spegel:
+  # Note: This only applies to k0s at the moment
+  enabled: false