fix: Updates to taskfiles and e2e workflow (#1144)

.envrc

@@ -7,6 +7,8 @@ export VIRTUAL_ENV="$(expand_path ./.venv)"
 export ANSIBLE_COLLECTIONS_PATH=$(expand_path ./.venv/galaxy)
 export ANSIBLE_ROLES_PATH=$(expand_path ./.venv/galaxy/ansible_roles)
 export ANSIBLE_VARS_ENABLED="host_group_vars,community.sops.sops"
+export ANSIBLE_LOCALHOST_WARNING="False"
+export ANSIBLE_INVENTORY_UNPARSED_WARNING="False"
 export K8S_AUTH_KUBECONFIG="$(expand_path ./kubeconfig)"
 # k0s
 export DISABLE_TELEMETRY="true"

.github/workflows/e2e.yaml

@@ -2,6 +2,7 @@
 name: "e2e"
 
 on:
+  workflow_dispatch:
   pull_request:
     branches: ["main"]
 
@@ -10,49 +11,80 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  e2e:
-    name: e2e
+  e2e-yay:
+    name: e2e-yay
     runs-on: ubuntu-latest
+    container:
+      image: greyltc/archlinux-aur:yay
+      options: --user root
     steps:
       - name: Checkout
         uses: actions/checkout@v4
 
+      - name: Test Arch dependencies
+        shell: bash
+        run: >-
+          sudo -E -u ab -D~ bash -c '
+          cd $GITHUB_WORKSPACE;
+          yay -Syu --needed --noconfirm --noprogressbar go-task;
+          go-task workstation:yay
+          '
+
+  e2e-configure:
+    name: e2e-configure
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Setup Homebrew
+        id: setup-homebrew
+        uses: Homebrew/actions/setup-homebrew@master
+
       - name: Setup Python
         uses: actions/setup-python@v5
         id: setup-python
         with:
           python-version: "3.12"
 
+      - name: Cache homebrew packages
+        if: ${{ github.event_name == 'pull_request' }}
+        uses: actions/cache@v3
+        id: cache-homebrew-packages
+        with:
+          key: homebrew-${{ runner.os }}-${{ steps.setup-homebrew.outputs.gems-hash }}-${{ hashFiles('.taskfiles/Workstation/Brewfile') }}
+          path: /home/linuxbrew/.linuxbrew
+
       - name: Cache venv
+        if: ${{ github.event_name == 'pull_request' }}
         uses: actions/cache@v3
         with:
           key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('requirements.txt', 'requirements.yaml') }}
           path: .venv
 
-      - name: Setup Homebrew
-        uses: Homebrew/actions/setup-homebrew@master
-
       - name: Setup Workflow Tools
+        if: ${{ github.event_name == 'pull_request' && steps.cache-homebrew-packages.outputs.cache-hit != 'true' }}
         shell: bash
         run: brew install go-task
 
       - name: Install Brew dependencies
+        if: ${{ github.event_name == 'pull_request' && steps.cache-homebrew-packages.outputs.cache-hit != 'true' }}
         shell: bash
-        run: task brew:deps
+        run: task workstation:brew
 
       - name: Initialize Direnv
         shell: bash
         run: direnv allow .
 
       - name: Initialize Sops Age key
         shell: bash
-        run: task sops:init
+        run: task sops:age-keygen
 
       - name: Install Ansible dependencies
         shell: bash
-        run: task ansible:deps
+        run: task ansible:deps force=false
 
-      - name: Initialize config
+      - name: Generate bootstrap config file
         shell: bash
         run: |
           task init
@@ -65,7 +97,11 @@ jobs:
         shell: bash
         run: task --yes configure
 
-      - name: List Hosts
+      - name: Run kubeconform
+        shell: bash
+        run: ./scripts/kubeconform.sh ./kubernetes
+
+      - name: List Hosts with Ansible
         shell: bash
         run: task ansible:list
 

.gitignore

@@ -14,6 +14,8 @@ kubeconfig
 .venv*
 # Taskfile
 .task
+# Brew
+Brewfile.lock.json
 # intellij
 .idea
 # wiki

.taskfiles/AnsibleTasks.yaml → .taskfiles/Ansible/Taskfile.yaml

@@ -4,20 +4,25 @@ version: "3"
 
 vars:
   PYTHON_BIN: python3
+  ANSIBLE_INVENTORY_FILE: "{{.ANSIBLE_DIR}}/inventory/hosts.yaml"
 
 env:
   PATH: "{{.ROOT_DIR}}/.venv/bin:$PATH"
   VIRTUAL_ENV: "{{.ROOT_DIR}}/.venv"
   ANSIBLE_COLLECTIONS_PATH: "{{.ROOT_DIR}}/.venv/galaxy"
   ANSIBLE_ROLES_PATH: "{{.ROOT_DIR}}/.venv/galaxy/ansible_roles"
   ANSIBLE_VARS_ENABLED: "host_group_vars,community.sops.sops"
+  ANSIBLE_LOCALHOST_WARNING: "False"
+  ANSIBLE_INVENTORY_UNPARSED_WARNING: "False"
 
 tasks:
 
   deps:
     desc: Set up Ansible dependencies for the environment
     cmds:
       - task: .venv
+        vars:
+          force: '{{.force | default "true"}}'
 
   run:
     desc: Run an Ansible playbook for configuring a cluster
@@ -27,54 +32,63 @@ tasks:
     prompt: Run Ansible playbook '{{.playbook}}'... continue?
     deps: ["deps"]
     cmd: |
-      .venv/bin/ansible-playbook \
-        --inventory {{.ANSIBLE_DIR}}/inventory/hosts.yaml \
-        {{.ANSIBLE_DIR}}/playbooks/{{.playbook}}.yaml {{.CLI_ARGS}}
+      .venv/bin/ansible-playbook --inventory {{.ANSIBLE_INVENTORY_FILE}} \
+          {{.ANSIBLE_DIR}}/playbooks/{{.playbook}}.yaml {{.CLI_ARGS}}
     preconditions:
       - { msg: "Argument (playbook) is required", sh: "test -n {{.playbook}}" }
       - { msg: "Venv not found",                  sh: "test -d {{.ROOT_DIR}}/.venv" }
-      - { msg: "Inventory not found",             sh: "test -f {{.ANSIBLE_DIR}}/inventory/hosts.yaml" }
+      - { msg: "Inventory not found",             sh: "test -f {{.ANSIBLE_INVENTORY_FILE}}" }
       - { msg: "Playbook not found",              sh: "test -f {{.ANSIBLE_DIR}}/playbooks/{{.playbook}}.yaml" }
 
   poweroff:
     desc: Shutdown all the k8s nodes
     deps: ["deps"]
-    cmd: .venv/bin/ansible kubernetes -i {{.ANSIBLE_DIR}}/inventory/hosts.yaml -a '/usr/bin/systemctl poweroff' --become
+    cmd: |
+      .venv/bin/ansible kubernetes --inventory {{.ANSIBLE_INVENTORY_FILE}} \
+          -a '/usr/bin/systemctl poweroff' --become
     preconditions:
       - { msg: "Venv not found",      sh: "test -d {{.ROOT_DIR}}/.venv" }
-      - { msg: "Inventory not found", sh: "test -f {{.ANSIBLE_DIR}}/inventory/hosts.yaml" }
+      - { msg: "Inventory not found", sh: "test -f {{.ANSIBLE_INVENTORY_FILE}}" }
 
   list:
     desc: List all the hosts
     deps: ["deps"]
-    cmd: .venv/bin/ansible kubernetes -i {{.ANSIBLE_DIR}}/inventory/hosts.yaml --list-hosts
+    cmd: |
+      .venv/bin/ansible kubernetes --inventory {{.ANSIBLE_INVENTORY_FILE}} \
+          --list-hosts
     preconditions:
       - { msg: "Venv not found",      sh: "test -d {{.ROOT_DIR}}/.venv" }
-      - { msg: "Inventory not found", sh: "test -f {{.ANSIBLE_DIR}}/inventory/hosts.yaml" }
+      - { msg: "Inventory not found", sh: "test -f {{.ANSIBLE_INVENTORY_FILE}}" }
 
   ping:
     desc: Ping all the hosts
     deps: ["deps"]
-    cmd: .venv/bin/ansible kubernetes -i {{.ANSIBLE_DIR}}/inventory/hosts.yaml --one-line -m 'ping'
+    cmd: |
+      .venv/bin/ansible kubernetes --inventory {{.ANSIBLE_INVENTORY_FILE}} \
+          --one-line -m 'ping'
     preconditions:
       - { msg: "Venv not found",      sh: "test -d {{.ROOT_DIR}}/.venv" }
-      - { msg: "Inventory not found", sh: "test -f {{.ANSIBLE_DIR}}/inventory/hosts.yaml" }
+      - { msg: "Inventory not found", sh: "test -f {{.ANSIBLE_INVENTORY_FILE}}" }
 
   uptime:
     desc: Uptime of all the hosts
     deps: ["deps"]
-    cmd: .venv/bin/ansible kubernetes -i {{.ANSIBLE_DIR}}/inventory/hosts.yaml --one-line -a 'uptime'
+    cmd: |
+      .venv/bin/ansible kubernetes --inventory {{.ANSIBLE_INVENTORY_FILE}} \
+          --one-line -a 'uptime'
     preconditions:
       - { msg: "Venv not found",      sh: "test -d {{.ROOT_DIR}}/.venv" }
-      - { msg: "Inventory not found", sh: "test -f {{.ANSIBLE_DIR}}/inventory/hosts.yaml" }
+      - { msg: "Inventory not found", sh: "test -f {{.ANSIBLE_INVENTORY_FILE}}" }
 
   .venv:
     internal: true
     cmds:
       - true && {{.PYTHON_BIN}} -m venv {{.ROOT_DIR}}/.venv
       - .venv/bin/python3 -m pip install --upgrade pip setuptools wheel
       - .venv/bin/python3 -m pip install --upgrade --requirement "{{.ROOT_DIR}}/requirements.txt"
-      - .venv/bin/ansible-galaxy install --role-file "{{.ROOT_DIR}}/requirements.yaml" --force
+      - .venv/bin/ansible-galaxy install --role-file "{{.ROOT_DIR}}/requirements.yaml" {{if eq .force "true"}}--force{{end}}
+    vars:
+      force: '{{.force | default "true"}}'
     sources:
       - "{{.ANSIBLE_DIR}}/requirements.txt"
       - "{{.ANSIBLE_DIR}}/requirements.yaml"

.taskfiles/FluxTasks.yaml → .taskfiles/Flux/Taskfile.yaml

@@ -57,7 +57,6 @@ tasks:
     cmd: flux reconcile --namespace flux-system kustomization cluster --with-source
 
   github-deploy-key:
-    cmds:
-      - sops --decrypt {{.KUBERNETES_DIR}}/bootstrap/github-deploy-key.sops.yaml | kubectl apply --server-side --filename -
+    cmd: sops --decrypt {{.KUBERNETES_DIR}}/bootstrap/github-deploy-key.sops.yaml | kubectl apply --server-side --filename -
     preconditions:
       - { msg: "Flux is not installed", sh: "kubectl get namespace flux-system &>/dev/null && exit 0 || exit 1" }

.taskfiles/RepoTasks.yaml → .taskfiles/Repo/Taskfile.yaml

@@ -12,7 +12,7 @@ tasks:
       - rm -rf {{.ROOT_DIR}}/.github/workflows/e2e.yaml
       # Move bootstrap directory to gitignored directory
       - mkdir -p {{.ROOT_DIR}}/.private
-      - mv {{.ROOT_DIR}}/bootstrap {{.ROOT_DIR}}/.private
+      - mv {{.ROOT_DIR}}/bootstrap {{.ROOT_DIR}}/.private/bootstrap-{{now | date "150405"}}
       # Update renovate.json5
       - sed -i {{if eq OS "darwin"}}''{{end}} 's/(..\.j2)\?(..\.j2)\?//g' {{.ROOT_DIR}}/.github/renovate.json5
       - sed -i {{if eq OS "darwin"}}''{{end}} '/addons/d' {{.ROOT_DIR}}/.github/renovate.json5

.taskfiles/SopsTasks.yaml → .taskfiles/Sops/Taskfile.yaml

@@ -4,14 +4,15 @@ version: "3"
 
 tasks:
 
-  init:
-    desc: Initialize Sops Age Key
+  age-keygen:
+    desc: Initialize Age Key for Sops
     cmd: age-keygen --output {{.ROOT_DIR}}/age.key
     status:
       - test -f "{{.ROOT_DIR}}/age.key"
 
   encrypt:
     desc: Encrypt a secret
+    aliases: ["e"]
     summary: |
       Args:
         file: Path to file to encrypt (required)
@@ -24,6 +25,7 @@ tasks:
 
   decrypt:
     desc: Decrypt a secret
+    aliases: ["d"]
     summary: |
       Args:
         file: Path to file to decrypt (required)

.taskfiles/Workstation/ArchPackages

@@ -0,0 +1,17 @@
+age
+cilium-cli
+cloudflared-bin
+direnv
+flux-bin
+helm
+jq
+k0sctl-bin
+k9s
+kubeconform
+kubectl-bin
+kustomize
+moreutils
+sops
+stern-bin
+go-task
+go-yq

.taskfiles/Workstation/Brewfile

@@ -0,0 +1,21 @@
+tap "fluxcd/tap"
+tap "go-task/tap"
+tap "k0sproject/tap"
+tap "kubecolor/tap"
+brew "age"
+brew "cilium-cli"
+brew "cloudflared"
+brew "direnv"
+brew "fluxcd/tap/flux"
+brew "helm"
+brew "jq"
+brew "k0sproject/tap/k0sctl"
+brew "k9s"
+brew "kubeconform"
+brew "kubecolor/tap/kubecolor"
+brew "kubernetes-cli"
+brew "kustomize"
+brew "moreutils"
+brew "sops"
+brew "stern"
+brew "yq"

.taskfiles/Workstation/Taskfile.yaml

@@ -0,0 +1,25 @@
+---
+# yaml-language-server: $schema=https://taskfile.dev/schema.json
+version: "3"
+
+tasks:
+
+  brew:
+    desc: Install workstation dependencies with Brew
+    cmd: brew bundle --file {{.ROOT_DIR}}/.taskfiles/Workstation/Brewfile
+    preconditions:
+      - sh: command -v brew
+        msg: |
+          Homebrew is not installed. Using MacOS, Linux or WSL?
+          Head over to https://brew.sh to get up and running.
+      - { msg: "Brewfile not found", sh: "test -f {{.ROOT_DIR}}/.taskfiles/Workstation/Brewfile" }
+
+  yay:
+    desc: Install workstation dependencies with Paru
+    cmd: yay -Syu --needed --noconfirm --noprogressbar $(cat {{.ROOT_DIR}}/.taskfiles/Workstation/ArchPackages | xargs)
+    preconditions:
+      - sh: command -v yay
+        msg: |
+          Yay is not installed. Using ArchLinux?
+          Head over to https://github.com/Jguer/yay to get up and running.
+      - { msg: "ArchPackages not found", sh: "test -f {{.ROOT_DIR}}/.taskfiles/Workstation/ArchPackages" }