diff --git a/.github/renovate.json5 b/.github/renovate.json5 index f87710b0220..31e40bdcf04 100644 --- a/.github/renovate.json5 +++ b/.github/renovate.json5 @@ -223,8 +223,7 @@ "fileMatch": [ "(^|/).taskfiles/.+\\.ya?ml$", "(^|/)ansible/.+\\.ya?ml(\\.j2)?(\\.j2)?$", - "(^|/)kubernetes/.+\\.ya?ml(\\.j2)?(\\.j2)?$", - "(^|/)k0s-config.ya?ml(\\.j2)?(\\.j2)?$" + "(^|/)kubernetes/.+\\.ya?ml(\\.j2)?(\\.j2)?$" ], "matchStrings": [ // Example: diff --git a/.github/tests/addons.yaml b/.github/tests/addons.yaml index c2248a942c1..ad7ac3d426a 100644 --- a/.github/tests/addons.yaml +++ b/.github/tests/addons.yaml @@ -32,3 +32,6 @@ discord_template_notifier: volsync: enabled: true + +spegel: + enabled: true diff --git a/.taskfiles/K0s/Taskfile.yaml b/.taskfiles/K0s/Taskfile.yaml index 94ec0923e67..4d40ec7df4a 100644 --- a/.taskfiles/K0s/Taskfile.yaml +++ b/.taskfiles/K0s/Taskfile.yaml @@ -6,12 +6,13 @@ env: DISABLE_UPGRADE_CHECK: "true" vars: - K0S_CONFIG_FILE: "{{.ROOT_DIR}}/k0s-config.yaml" + K0S_CONFIG_FILE: "{{.KUBERNETES_DIR}}/k0s/k0s-config.yaml" tasks: apply: desc: Apply k0s cluster config + dir: "{{.KUBERNETES_DIR}}/k0s" # Needed for uploading files in k0s-config.yaml cmds: - k0sctl apply --config {{.K0S_CONFIG_FILE}} - task: kubeconfig diff --git a/bootstrap/templates/kubernetes/apps/kube-system/kustomization.yaml.j2 b/bootstrap/templates/kubernetes/apps/kube-system/kustomization.yaml.j2 index aab3f9d66dd..d0da8c604e2 100644 --- a/bootstrap/templates/kubernetes/apps/kube-system/kustomization.yaml.j2 +++ b/bootstrap/templates/kubernetes/apps/kube-system/kustomization.yaml.j2 @@ -8,3 +8,6 @@ resources: - ./coredns/ks.yaml #% endif %# - ./metrics-server/ks.yaml + #% if bootstrap_distribution == "k0s" and spegel.enabled | default(false) %# + - ./spegel/ks.yaml + #% endif %# diff --git a/bootstrap/templates/kubernetes/apps/kube-system/spegel/app/helmrelease.yaml.j2 b/bootstrap/templates/kubernetes/apps/kube-system/spegel/app/helmrelease.yaml.j2 new file mode 100644 index 00000000000..f8cbf7d8111 --- /dev/null +++ b/bootstrap/templates/kubernetes/apps/kube-system/spegel/app/helmrelease.yaml.j2 @@ -0,0 +1,37 @@ +#% if bootstrap_distribution == "k0s" and spegel.enabled | default(false) %# +--- +apiVersion: helm.toolkit.fluxcd.io/v2beta2 +kind: HelmRelease +metadata: + name: spegel +spec: + interval: 30m + chart: + spec: + chart: spegel + version: v0.0.17 + sourceRef: + kind: HelmRepository + name: xenitab + namespace: flux-system + install: + remediation: + retries: 3 + upgrade: + cleanupOnFail: true + remediation: + retries: 3 + uninstall: + keepHistory: false + values: + spegel: + #% if bootstrap_distribution == "k0s" %# + containerdSock: /run/k0s/containerd.sock + containerdRegistryConfigPath: /var/lib/k0s/containerd/certs.d + #% endif %# + service: + registry: + hostPort: 29999 + serviceMonitor: + enabled: true +#% endif %# diff --git a/bootstrap/templates/kubernetes/apps/kube-system/spegel/app/kustomization.yaml.j2 b/bootstrap/templates/kubernetes/apps/kube-system/spegel/app/kustomization.yaml.j2 new file mode 100644 index 00000000000..96b4eb8e3be --- /dev/null +++ b/bootstrap/templates/kubernetes/apps/kube-system/spegel/app/kustomization.yaml.j2 @@ -0,0 +1,7 @@ +#% if bootstrap_distribution == "k0s" and spegel.enabled | default(false) %# +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - ./helmrelease.yaml +#% endif %# diff --git a/bootstrap/templates/kubernetes/apps/kube-system/spegel/ks.yaml.j2 b/bootstrap/templates/kubernetes/apps/kube-system/spegel/ks.yaml.j2 new file mode 100644 index 00000000000..cdb0566bede --- /dev/null +++ b/bootstrap/templates/kubernetes/apps/kube-system/spegel/ks.yaml.j2 @@ -0,0 +1,22 @@ +#% if bootstrap_distribution == "k0s" and spegel.enabled | default(false) %# +--- +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: &app spegel + namespace: flux-system +spec: + targetNamespace: kube-system + commonMetadata: + labels: + app.kubernetes.io/name: *app + path: ./kubernetes/apps/kube-system/spegel/app + prune: true + sourceRef: + kind: GitRepository + name: home-kubernetes + wait: false + interval: 30m + retryInterval: 1m + timeout: 5m +#% endif %# diff --git a/bootstrap/templates/kubernetes/flux/repositories/helm/kustomization.yaml.j2 b/bootstrap/templates/kubernetes/flux/repositories/helm/kustomization.yaml.j2 index e5744c789a2..07fc265e3e1 100644 --- a/bootstrap/templates/kubernetes/flux/repositories/helm/kustomization.yaml.j2 +++ b/bootstrap/templates/kubernetes/flux/repositories/helm/kustomization.yaml.j2 @@ -21,3 +21,4 @@ resources: - ./prometheus-community.yaml - ./stakater.yaml - ./weave-gitops.yaml + - ./xenitab.yaml diff --git a/bootstrap/templates/kubernetes/flux/repositories/helm/xenitab.yaml.j2 b/bootstrap/templates/kubernetes/flux/repositories/helm/xenitab.yaml.j2 new file mode 100644 index 00000000000..31cb1257bc9 --- /dev/null +++ b/bootstrap/templates/kubernetes/flux/repositories/helm/xenitab.yaml.j2 @@ -0,0 +1,10 @@ +--- +apiVersion: source.toolkit.fluxcd.io/v1beta2 +kind: HelmRepository +metadata: + name: xenitab + namespace: flux-system +spec: + type: oci + interval: 5m + url: oci://ghcr.io/xenitab/helm-charts diff --git a/bootstrap/templates/k0s-config.yaml.j2 b/bootstrap/templates/kubernetes/k0s/k0s-config.yaml.j2 similarity index 84% rename from bootstrap/templates/k0s-config.yaml.j2 rename to bootstrap/templates/kubernetes/k0s/k0s-config.yaml.j2 index f2bd6d4d4ff..e00c30de131 100644 --- a/bootstrap/templates/k0s-config.yaml.j2 +++ b/bootstrap/templates/kubernetes/k0s/k0s-config.yaml.j2 @@ -16,6 +16,19 @@ spec: #% if item.role | default('') == 'controller+worker' %# - --no-taints #% endif %# + files: + - name: containerd-config + src: resources/containerd/ + perm: 0644 + user: root + group: root + hooks: + apply: + before: + #% if spegel.enabled | default(false) %# + - sudo mv ~/spegel.toml /etc/k0s/containerd.d/spegel.toml + #% endif %# + - sudo mv ~/unprivileged-ports.toml /etc/k0s/containerd.d/unprivileged-ports.toml #% endfor %# #% if bootstrap_nodes.worker | default([]) | length > 0 %# #% for item in bootstrap_nodes.worker %# @@ -23,6 +36,19 @@ spec: ssh: address: "#{ item.address }#" user: "#{ item.username }#" + files: + - name: containerd-config + src: resources/containerd/ + perm: 0644 + user: root + group: root + hooks: + apply: + before: + #% if spegel.enabled | default(false) %# + - sudo mv ~/spegel.toml /etc/k0s/containerd.d/spegel.toml + #% endif %# + - sudo mv ~/unprivileged-ports.toml /etc/k0s/containerd.d/unprivileged-ports.toml #% endfor %# #% endif %# k0s: @@ -55,6 +81,15 @@ spec: - "#{ item.name }#" #% endif %# #% endfor %# + network: + kubeProxy: + disabled: true + #% if bootstrap_nodes.master | length > 1 %# + nodeLocalLoadBalancing: + enabled: true + type: EnvoyProxy + #% endif %# + provider: custom extensions: helm: repositories: @@ -119,13 +154,4 @@ spec: routingMode: native securityContext: privileged: true - network: - kubeProxy: - disabled: true - #% if bootstrap_nodes.master | length > 1 %# - nodeLocalLoadBalancing: - enabled: true - type: EnvoyProxy - #% endif %# - provider: custom #% endif %# diff --git a/bootstrap/templates/kubernetes/k0s/resources/containerd/spegel.toml.j2 b/bootstrap/templates/kubernetes/k0s/resources/containerd/spegel.toml.j2 new file mode 100644 index 00000000000..887ff9a7cdd --- /dev/null +++ b/bootstrap/templates/kubernetes/k0s/resources/containerd/spegel.toml.j2 @@ -0,0 +1,6 @@ +#% if bootstrap_distribution == 'k0s' and spegel.enabled | default(false) %# +[plugins."io.containerd.grpc.v1.cri".containerd] + discard_unpacked_layers = false +[plugins."io.containerd.grpc.v1.cri".registry] + config_path = "/var/lib/k0s/containerd/certs.d" +#% endif %# diff --git a/bootstrap/templates/kubernetes/k0s/resources/containerd/unprivileged-ports.toml.j2 b/bootstrap/templates/kubernetes/k0s/resources/containerd/unprivileged-ports.toml.j2 new file mode 100644 index 00000000000..66583b86e0e --- /dev/null +++ b/bootstrap/templates/kubernetes/k0s/resources/containerd/unprivileged-ports.toml.j2 @@ -0,0 +1,5 @@ +#% if bootstrap_distribution == 'k0s' and spegel.enabled | default(false) %# +[plugins."io.containerd.grpc.v1.cri"] + enable_unprivileged_ports = true + enable_unprivileged_icmp = true +#% endif %# diff --git a/bootstrap/vars/addons.sample.yaml b/bootstrap/vars/addons.sample.yaml index c6a9a0cce27..c91bcec1a13 100644 --- a/bootstrap/vars/addons.sample.yaml +++ b/bootstrap/vars/addons.sample.yaml @@ -41,7 +41,7 @@ system_upgrade_controller: # WARNING: Only enable this if you also track the version of k3s in the # ansible configuration files. Running ansible against an already provisioned # cluster with this enabled might cause your cluster to be downgraded. - # Note that if bootstrap_distribution is set to k0s enable: true will be ignored. + # Note: If bootstrap_distribution is set to k0s this will be ignored. enabled: false # https://github.com/morphy2k/rss-forwarder @@ -54,3 +54,8 @@ discord_template_notifier: # https://github.com/backube/volsync volsync: enabled: false + +# https://github.com/XenitAB/spegel +spegel: + # Note: This only applies to k0s at the moment + enabled: false