feat: Do not rely on k0s file uploads - instead use curl (#1229)

* feat: Do not rely on k0s file uploads - instead use curl Signed-off-by: Devin Buhl <devin@buhl.casa> * fix: add bootstrap_advanced_flags to validation Signed-off-by: Devin Buhl <devin@buhl.casa> * fix: only skip tests for non-static config Signed-off-by: Devin Buhl <devin@buhl.casa> * fix: only skip tests for non-static config Signed-off-by: Devin Buhl <devin@buhl.casa> * fix: issues in k0s scripts Signed-off-by: Devin Buhl <devin@buhl.casa> * update: readme Signed-off-by: Devin Buhl <devin@buhl.casa> * update: taskfile Signed-off-by: Devin Buhl <devin@buhl.casa> * update: taskfile Signed-off-by: Devin Buhl <devin@buhl.casa> * update: k0s config and remove ansible hook Signed-off-by: Devin Buhl <devin@buhl.casa> --------- Signed-off-by: Devin Buhl <devin@buhl.casa>
onedr0p · Jan 22, 2024 · d9c7fd4 · d9c7fd4
1 parent 5ef0438
commit d9c7fd4
Show file tree

Hide file tree

Showing 17 changed files with 120 additions and 109 deletions.
diff --git a/.github/tests/config-k0s.yaml b/.github/tests/config-k0s.yaml
@@ -4,10 +4,9 @@ skip_tests: true
 bootstrap_distribution: k0s
 bootstrap_github_username: onedr0p
 bootstrap_github_repository_name: flux-cluster-template
-bootstrap_github_repository_branch: main
 bootstrap_age_public_key: $BOOTSTRAP_AGE_PUBLIC_KEY
 bootstrap_timezone: Etc/UTC
-bootstrap_acme_email: fake
+bootstrap_acme_email: fake@example.com
 bootstrap_acme_production_enabled: false
 bootstrap_flux_github_webhook_token: fake
 bootstrap_cloudflare_domain: fake

diff --git a/.github/tests/config-k3s-ipv4.yaml b/.github/tests/config-k3s-ipv4.yaml
@@ -4,10 +4,9 @@ skip_tests: true
 bootstrap_distribution: k3s
 bootstrap_github_username: onedr0p
 bootstrap_github_repository_name: flux-cluster-template
-bootstrap_github_repository_branch: main
 bootstrap_age_public_key: $BOOTSTRAP_AGE_PUBLIC_KEY
 bootstrap_timezone: Etc/UTC
-bootstrap_acme_email: fake
+bootstrap_acme_email: fake@example.com
 bootstrap_acme_production_enabled: false
 bootstrap_flux_github_webhook_token: fake
 bootstrap_cloudflare_domain: fake

diff --git a/.github/tests/config-k3s-ipv6.yaml b/.github/tests/config-k3s-ipv6.yaml
@@ -4,10 +4,9 @@ skip_tests: true
 bootstrap_distribution: k3s
 bootstrap_github_username: onedr0p
 bootstrap_github_repository_name: flux-cluster-template
-bootstrap_github_repository_branch: main
 bootstrap_age_public_key: $BOOTSTRAP_AGE_PUBLIC_KEY
 bootstrap_timezone: Etc/UTC
-bootstrap_acme_email: fake
+bootstrap_acme_email: fake@example.com
 bootstrap_acme_production_enabled: false
 bootstrap_flux_github_webhook_token: fake
 bootstrap_cloudflare_domain: fake

diff --git a/.github/tests/config-talos.yaml b/.github/tests/config-talos.yaml
@@ -4,10 +4,9 @@ skip_tests: true
 bootstrap_distribution: talos
 bootstrap_github_username: onedr0p
 bootstrap_github_repository_name: flux-cluster-template
-bootstrap_github_repository_branch: main
 bootstrap_age_public_key: $BOOTSTRAP_AGE_PUBLIC_KEY
 bootstrap_timezone: Etc/UTC
-bootstrap_acme_email: fake
+bootstrap_acme_email: fake@example.com
 bootstrap_acme_production_enabled: false
 bootstrap_flux_github_webhook_token: fake
 bootstrap_cloudflare_domain: fake

diff --git a/.taskfiles/Ansible/Taskfile.yaml b/.taskfiles/Ansible/Taskfile.yaml
@@ -21,8 +21,8 @@ tasks:
 
   deps:
     desc: Set up Ansible dependencies
+    deps: [":setup-virtual-env"]
     cmds:
-      - task: :setup-virtual-env
       - .venv/bin/python3 -m pip install --upgrade --requirement "{{.ANSIBLE_PIP_REQUIREMENTS_FILE}}"
       - .venv/bin/ansible-galaxy install --role-file "{{.ANSIBLE_REQUIREMENTS_FILE}}" {{if eq .force "true"}}--force{{end}}
     preconditions:

diff --git a/README.md b/README.md
@@ -284,8 +284,17 @@ Once you have installed Talos or Debian on your nodes, there are six stages to g
     task configure
     ```
 
-8. Continue on to ⚡ [**Stage 4**](#-stage-4-prepare-your-nodes-for-kubernetes)
+8. Push you changes to git
 
+   📍 **Verify** all the `*.sops.yaml` and `*.sops.yaml` files under the `./ansible`, and `./kubernetes` directories are **encrypted** with SOPS
+
+    ```sh
+    git add -A
+    git commit -m "Initial commit :rocket:"
+    git push
+    ```
+
+9. Continue on to ⚡ [**Stage 4**](#-stage-4-prepare-your-nodes-for-kubernetes)
 
 ### ⚡ Stage 4: Prepare your nodes for Kubernetes
 
@@ -396,17 +405,7 @@ Once you have installed Talos or Debian on your nodes, there are six stages to g
     # ✔ prerequisites checks passed
     ```
 
-2. Push you changes to git
-
-   📍 **Verify** all the `*.sops.yaml` and `*.sops.yaml` files under the `./ansible`, and `./kubernetes` directories are **encrypted** with SOPS
-
-    ```sh
-    git add -A
-    git commit -m "Initial commit :rocket:"
-    git push
-    ```
-
-3. Install Flux and sync the cluster to the Git repository
+2. Install Flux and sync the cluster to the Git repository
 
     ```sh
     task flux:bootstrap
@@ -415,7 +414,7 @@ Once you have installed Talos or Debian on your nodes, there are six stages to g
     # ...
     ```
 
-4. Verify Flux components are running in the cluster
+3. Verify Flux components are running in the cluster
 
     ```sh
     kubectl -n flux-system get pods -o wide

diff --git a/Taskfile.yaml b/Taskfile.yaml
@@ -38,8 +38,8 @@ tasks:
 
   init:
     desc: Initialize virtual env and configuration files
+    deps: ["setup-virtual-env"]
     cmds:
-      - task: setup-virtual-env
       - mkdir -p {{.PRIVATE_DIR}}
       - cp -n {{.BOOTSTRAP_ADDONS_FILE | replace ".yaml" ".sample.yaml"}} {{.BOOTSTRAP_ADDONS_FILE}}
       - cp -n {{.BOOTSTRAP_CONFIG_FILE | replace ".yaml" ".sample.yaml"}} {{.BOOTSTRAP_CONFIG_FILE}}
@@ -53,38 +53,39 @@ tasks:
         silent: true
 
   configure:
-    desc: Configure repository from Ansible vars
+    desc: Configure repository from bootstrap vars
     prompt: Any conflicting config in the root kubernetes and ansible directories will be overwritten... continue?
+    deps: ["setup-virtual-env"]
     cmds:
       - task: .template
-      - task: .post-validate
+      - task: sops:encrypt
+      - task: .validate
+
+  setup-virtual-env:
+    desc: Set up virtual environment
+    cmds:
+      - "{{.PYTHON_BIN}} -m venv {{.ROOT_DIR}}/.venv"
+      - .venv/bin/python3 -m pip install --upgrade pip setuptools wheel
+      - .venv/bin/python3 -m pip install --upgrade --requirement "{{.PIP_REQUIREMENTS_FILE}}"
+    sources:
+      - "{{.PIP_REQUIREMENTS_FILE}}"
+    generates:
+      - "{{.ROOT_DIR}}/.venv/pyvenv.cfg"
+    preconditions:
+      - { msg: "Missing Pip requirements file", sh: "test -f {{.PIP_REQUIREMENTS_FILE}}" }
 
   .template:
     internal: true
     cmds:
       - ./.venv/bin/makejinja
-      - task: sops:encrypt
     preconditions:
       - { msg: "Missing virtual environment", sh: "test -d {{.ROOT_DIR}}/.venv" }
       - { msg: "Missing Makejinja config file", sh: "test -f {{.MAKEJINJA_CONFIG_FILE}}" }
       - { msg: "Missing Makejinja plugin file", sh: "test -f {{.BOOTSTRAP_DIR}}/scripts/plugin.py" }
       - { msg: "Missing bootstrap addons file", sh: "test -f {{.BOOTSTRAP_ADDONS_FILE}}" }
       - { msg: "Missing bootstrap config file", sh: "test -f {{.BOOTSTRAP_CONFIG_FILE}}" }
 
-  .post-validate:
+  .validate:
     internal: true
     cmds:
       - task: kubernetes:kubeconform
-
-  setup-virtual-env:
-    desc: Set up virtual environment
-    cmds:
-      - "{{.PYTHON_BIN}} -m venv {{.ROOT_DIR}}/.venv"
-      - .venv/bin/python3 -m pip install --upgrade pip setuptools wheel
-      - .venv/bin/python3 -m pip install --upgrade --requirement "{{.PIP_REQUIREMENTS_FILE}}"
-    sources:
-      - "{{.PIP_REQUIREMENTS_FILE}}"
-    generates:
-      - "{{.ROOT_DIR}}/.venv/pyvenv.cfg"
-    preconditions:
-      - { msg: "Missing Pip requirements file", sh: "test -f {{.PIP_REQUIREMENTS_FILE}}" }
diff --git a/bootstrap/scripts/plugin.py b/bootstrap/scripts/plugin.py
@@ -20,9 +20,6 @@ def encrypt(value: str) -> str:
 
 class Plugin(makejinja.plugin.Plugin):
     def __init__(self, data: dict[str, Any]):
-        if data.get("skip_tests", False):
-            return
-
         validation.validate(data)
 
     def filters(self) -> makejinja.plugin.Filters:

diff --git a/bootstrap/scripts/validation.py b/bootstrap/scripts/validation.py
@@ -100,14 +100,17 @@ def validate_distribution(distribution: str, **_) -> None:
     _validate_distribution(distribution)
 
 
-@required("bootstrap_github_username", "bootstrap_github_repository_name", "bootstrap_github_repository_branch")
-def validate_github(username: str, repository: str, branch: str, **_) -> None:
+@required("bootstrap_github_username", "bootstrap_github_repository_name", "bootstrap_advanced_flags")
+def validate_github(username: str, repository: str, advanced_flags: dict, **_) -> None:
     try:
-        request = requests.get(f"https://api.github.com/repos/{username}/{repository}/branches/{branch}")
+        request = requests.get(
+            f"https://api.github.com/repos/{username}/{repository}/branches/{advanced_flags.get('github_repository_branch', 'main')}")
         if request.status_code != 200:
-            raise ValueError(f"GitHub repository {username}/{repository} branch {branch} not found")
+            raise ValueError(
+                f"GitHub repository {username}/{repository} branch {advanced_flags.get('github_repository_branch', 'main')} not found")
     except requests.exceptions.RequestException as e:
-        raise ValueError(f"GitHub repository {username}/{repository} branch {branch} not found") from e
+        raise ValueError(
+            f"GitHub repository {username}/{repository} branch {advanced_flags.get('github_repository_branch', 'main')} not found") from e
 
 
 @required("bootstrap_age_public_key")
@@ -240,18 +243,27 @@ def validate_nodes(node_cidr: str, nodes: dict[list], distribution: str, **_) ->
         _validate_node(node, node_cidr, distribution)
 
 
+def massage(data: dict) -> dict:
+    data["bootstrap_advanced_flags"] = data.get("bootstrap_advanced_flags", {})
+    return data
+
 def validate(data: dict) -> None:
+    user_data = massage(data)
+
     validate_python_version()
-    validate_cli_tools(data)
-    validate_distribution(data)
-    if not data.get("bootstrap_private_github_repo"):
-        validate_github(data)
-    validate_age(data)
-    validate_timezone(data)
-    validate_acme_email(data)
-    validate_flux_github_webhook_token(data)
-    validate_cloudflare(data)
-    validate_host_network(data)
-    validate_bootstrap_dns_server(data)
-    validate_cluster_cidrs(data)
-    validate_nodes(data)
+    validate_cli_tools(user_data)
+    validate_distribution(user_data)
+    validate_age(user_data)
+    validate_timezone(user_data)
+    validate_bootstrap_dns_server(user_data)
+    validate_cluster_cidrs(user_data)
+    validate_flux_github_webhook_token(user_data)
+    validate_host_network(user_data)
+    validate_acme_email(user_data)
+
+    if not user_data.get("bootstrap_private_github_repo"):
+        validate_github(user_data)
+
+    if not user_data.get("skip_tests", False):
+        validate_cloudflare(user_data)
+        validate_nodes(user_data)
diff --git a/bootstrap/templates/ansible/playbooks/cluster-prepare.yaml.j2 b/bootstrap/templates/ansible/playbooks/cluster-prepare.yaml.j2
@@ -62,7 +62,7 @@
               ff02::2 ip6-allrouters
             dest: /etc/hosts
             mode: preserve
-        #% if bootstrap_advanced_flags.update_resolv_conf %#
+        #% if bootstrap_advanced_flags.update_resolv_conf|default(true) %#
         # https://github.com/onedr0p/flux-cluster-template/discussions/635
         - name: Network Configuration | Remove immutable flag from /etc/resolv.conf
           ansible.builtin.file:

diff --git a/bootstrap/templates/kubernetes/flux/config/cluster.yaml.j2 b/bootstrap/templates/kubernetes/flux/config/cluster.yaml.j2
@@ -7,7 +7,7 @@ metadata:
 spec:
   interval: 30m
   ref:
-    branch: "#{ bootstrap_github_repository_branch }#"
+    branch: "#{ bootstrap_advanced_flags.github_repository_branch|default('main', true) }#"
   #% if bootstrap_private_github_repo %#
   secretRef:
     name: github-deploy-key

diff --git a/bootstrap/templates/kubernetes/k0s/k0sctl.yaml.j2 b/bootstrap/templates/kubernetes/k0s/k0sctl.yaml.j2
@@ -14,46 +14,31 @@ spec:
       installFlags:
         - --disable-components=metrics-server
         - --no-taints
-      #% if loop.first %#
-      files: &files
-        - name: system-config
-          src: resources/hooks/
-          dstDir: /home/#{ item.username }#/k0s/hooks/
-          perm: 0644
-          user: root
-          group: root
-        - name: containerd-config
-          src: resources/containerd/
-          dstDir: /home/#{ item.username }#/k0s/containerd/
-          perm: 0644
-          user: root
-          group: root
-      #% else %#
-      files: *files
-      #% endif %#
-      #% if loop.first %#
-      hooks: &hooks
+      hooks:
         apply:
           before:
-            - sudo bash /home/#{ item.username }#/k0s/hooks/apply-system.sh
-            - sudo mv /home/#{ item.username }#/k0s/containerd/default.toml /etc/k0s/containerd.d/default.toml
+            - curl -sfL https://raw.githubusercontent.com/#{ bootstrap_github_username }#/#{ bootstrap_github_repository_name }#/#{ bootstrap_advanced_flags.github_repository_branch|default('main', true) }#/kubernetes/k0s/resources/apply-system.sh | INSTALL_K0S_HOSTNAME="#{ item.name }#" bash -s -
         reset:
           before:
-            - sudo bash /home/#{ item.username }#/k0s/hooks/reset-cilium.sh
+            - curl -sfL https://raw.githubusercontent.com/#{ bootstrap_github_username }#/#{ bootstrap_github_repository_name }#/#{ bootstrap_advanced_flags.github_repository_branch|default('main', true) }#/kubernetes/k0s/resources/reset-cilium.sh | bash -s -
           after:
-            - sudo bash /home/#{ item.username }#/k0s/hooks/reset-system.sh
-      #% else %#
-      hooks: *hooks
-      #% endif %#
+            - curl -sfL https://raw.githubusercontent.com/#{ bootstrap_github_username }#/#{ bootstrap_github_repository_name }#/#{ bootstrap_advanced_flags.github_repository_branch|default('main', true) }#/kubernetes/k0s/resources/reset-system.sh | bash -s -
     #% endfor %#
     #% if bootstrap_nodes.worker | length > 0 %#
     #% for item in bootstrap_nodes.worker %#
     - role: worker
       ssh:
         address: "#{ item.address }#"
         user: "#{ item.username }#"
-      files: *files
-      hooks: *hooks
+      hooks:
+        apply:
+          before:
+            - curl -sfL https://raw.githubusercontent.com/#{ bootstrap_github_username }#/#{ bootstrap_github_repository_name }#/#{ bootstrap_advanced_flags.github_repository_branch|default('main', true) }#/kubernetes/k0s/resources/apply-system.sh | INSTALL_K0S_HOSTNAME="#{ item.name }#" bash -s -
+        reset:
+          before:
+            - curl -sfL https://raw.githubusercontent.com/#{ bootstrap_github_username }#/#{ bootstrap_github_repository_name }#/#{ bootstrap_advanced_flags.github_repository_branch|default('main', true) }#/kubernetes/k0s/resources/reset-cilium.sh | bash -s -
+          after:
+            - curl -sfL https://raw.githubusercontent.com/#{ bootstrap_github_username }#/#{ bootstrap_github_repository_name }#/#{ bootstrap_advanced_flags.github_repository_branch|default('main', true) }#/kubernetes/k0s/resources/reset-system.sh | bash -s -
     #% endfor %#
     #% endif %#
   k0s:

diff --git a/...es/k0s/resources/hooks/apply-system.sh.j2 → ...bernetes/k0s/resources/apply-system.sh.j2 b/...es/k0s/resources/hooks/apply-system.sh.j2 → ...bernetes/k0s/resources/apply-system.sh.j2
@@ -1,11 +1,25 @@
 #% if bootstrap_distribution == 'k0s' %#
 #!/usr/bin/env bash
+set -e
+set -o noglob
 
 [ $(id -u) -eq 0 ] || exec sudo $0 $@
 
 # Prepare
 sudo apt-get update -y
 
+# Hostname
+if [ -n "${INSTALL_K0S_HOSTNAME}" ] && [ "$(hostnamectl hostname)" != "${INSTALL_K0S_HOSTNAME}" ]; then
+    hostnamectl set-hostname "${INSTALL_K0S_HOSTNAME}"
+tee /etc/hosts > /dev/null <<EOF
+127.0.0.1 localhost
+127.0.1.1 $INSTALL_K0S_HOSTNAME
+::1       localhost ip6-localhost ip6-loopback
+ff02::1   ip6-allnodes
+ff02::2   ip6-allrouters
+EOF
+fi
+
 # Timezone
 timedatectl set-timezone "#{ bootstrap_timezone }#"
 
@@ -16,7 +30,7 @@ apt-get install -y --no-install-recommends \
     nvme-cli open-iscsi parted psmisc python3 python3-apt python3-kubernetes python3-yaml \
     smartmontools socat software-properties-common unzip util-linux
 
-#% if bootstrap_advanced_flags.update_resolv_conf %#
+#% if bootstrap_advanced_flags.update_resolv_conf|default(true) %#
 # Update DNS
 chattr -i /etc/resolv.conf
 rm -f /etc/resolv.conf
@@ -37,9 +51,9 @@ systemctl restart systemd-modules-load.service
 
 # Sysctls
 cat <<EOF > /etc/sysctl.d/99-kubernetes.conf
-fs.inotify.max_queued_events: 65536
-fs.inotify.max_user_watches: 524288
-fs.inotify.max_user_instances: 8192
+fs.inotify.max_queued_events = 65536
+fs.inotify.max_user_watches = 524288
+fs.inotify.max_user_instances = 8192
 EOF
 sysctl -p /etc/sysctl.d/99-kubernetes.conf
 
@@ -57,6 +71,17 @@ neofetch --config none
 EOF
 chmod 755 /etc/profile.d/neofetch.sh
 
-# Ensure k0s containerd directory exists
+# Create containerd config
 mkdir -p /etc/k0s/containerd.d
+cat <<EOF > /etc/k0s/containerd.d/default.toml
+[plugins."io.containerd.grpc.v1.cri"]
+  enable_unprivileged_ports = true
+  enable_unprivileged_icmp = true
+[plugins."io.containerd.grpc.v1.cri".registry]
+  config_path = "/var/lib/k0s/containerd/certs.d"
+[plugins."io.containerd.grpc.v1.cri".containerd]
+  discard_unpacked_layers = false
+[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
+  discard_unpacked_layers = false
+EOF
 #% endif %#