WIP: tf-lock-url

.github/actions/basic-setup/action.yml

@@ -0,0 +1,65 @@
+name: Basic-Setup
+# This is derived from the Setup github action.
+# It omits several steps, and is mainly used by tacos_unlock.
+
+inputs:
+  ssh-private-key:
+    description: "Private SSH key to use for git clone"
+    type: string
+    default: ""
+  user:
+    description: the username that will be used for following steps
+    required: false
+    default: ${{github.triggering_actor}}
+  shell:
+    description: "private -- do not use"
+    default: env ./tacos-gha/lib/ci/default-shell {0}
+
+runs:
+  using: composite
+
+  steps:
+    - uses: ./tacos-gha/.github/actions/just-the-basics
+
+    - name: tell TF username and PR
+      uses: ./tacos-gha/.github/actions/set-username-and-hostname
+      with:
+        user: ${{inputs.user}}
+
+    - name: Set up SSH agent
+      if: inputs.ssh-private-key != ''
+      uses: webfactory/ssh-agent@v0.8.0
+      with:
+        ssh-private-key: ${{ inputs.ssh-private-key }}
+
+        # These fix most ownership, permission issues, but the .ssh config files
+        # still get the wrong ownership, fixed in the next step.
+        ssh-agent-cmd: |-
+          ./tacos-gha/lib/ci/bin/sudo-ssh-agent
+        ssh-add-cmd: |-
+          ./tacos-gha/lib/ci/bin/sudo-ssh-add
+    - name: Fix .ssh permissions
+      shell: ${{inputs.shell}}
+      if: inputs.ssh-private-key != ''
+      run: |
+        : fix ssh config ownership
+        sudo chown -v -R "$(id -un):$(id -gn)" ~/.ssh
+        : Show SSH agent pubkeys
+        ssh-add -L
+        : ... hashes too
+        ssh-add -l
+    # this should really be default behavior:
+    - shell: ${{inputs.shell}}
+      run: |
+        gha-set-env 'TF_VERSION' < "$(nearest-config-file .terraform-version)"
+        gha-set-env 'TERRAGRUNT_VERSION' < "$(nearest-config-file .terragrunt-version)"
+    - name: Setup Terragrunt
+      uses: autero1/action-terragrunt@v1.3.2
+      with:
+        terragrunt_version: ${{env.TERRAGRUNT_VERSION}}
+
+    - name: Setup Terraform
+      uses: hashicorp/setup-terraform@v3
+      with:
+        terraform_wrapper: false
+        terraform_version: ${{ env.TF_VERSION }}

.github/actions/matrix-fan-in/action.yml

@@ -31,8 +31,7 @@ runs:
       uses: actions/download-artifact@v4
       with:
         pattern: ${{ env.artifact_name }} *${{ inputs.pattern }}*
-        path: matrix-fan-in.tmp
-
+        path: ${{ inputs.path }}
     - name: fix up archive files
       shell: bash
       env:
@@ -41,3 +40,11 @@ runs:
       # note: "$GITHUB_ACTION_PATH" contains this action directory's path
       run: |
         "$GITHUB_ACTION_PATH/"rename-tmp-dirs.sh "$MATRIX_FAN_OUT_PATH"
+
+    ## DEBUG:
+    # - name: Start SSH
+    #   if: always()
+    #   uses: lhotari/action-upterm@v1
+    #   with:
+    #     ## limits ssh access and adds the ssh public keys of the listed GitHub users
+    #     limit-access-to-users: bukzor,kneeyo1

.github/actions/matrix-fan-in/rename-tmp-dirs.sh

@@ -14,7 +14,7 @@ path="$1"
 mkdir -p "$path"
 
 : directory name fixup
-find ./matrix-fan-in.tmp \
+find "$path"\
     -mindepth 1 \
     -maxdepth 1 \
     -print0 \

.github/actions/matrix-fan-out/action.yml

@@ -11,6 +11,9 @@ inputs:
   shell:
     description: "private -- do not use"
     default: bash -euxo pipefail {0}
+  matrix:
+    description: defaults to toJSON(matrix)
+    default: ${{ toJSON(matrix) }}
 
 runs:
   using: "composite"
@@ -19,7 +22,7 @@ runs:
     - shell: ${{ inputs.shell }}
       env:
         MATRIX_FAN_OUT_PATH: ${{ inputs.path }}
-        GHA_MATRIX_CONTEXT: ${{ toJSON(matrix) }}
+        GHA_MATRIX_CONTEXT: ${{ inputs.matrix }}
       run: |
         "$GITHUB_ACTION_PATH/"prepare.sh |
           tee -a "$GITHUB_ENV"

.github/workflows/selftest-matrix-io.yml

@@ -32,7 +32,7 @@ jobs:
           exec >&2  # our only output is logging
           printf "keys=[10, 27]" >> "$GITHUB_OUTPUT"
           # for scale testing:
-          ###seq 30 | shuf | jq -R | jq -cs | tee -a "$GITHUB_OUTPUT"
+          #seq 30 | shuf | jq -R | jq -cs | tee -a "$GITHUB_OUTPUT"
 
   fan-out:
     name: Compute Squares

.github/workflows/tacos_apply.yml

@@ -75,7 +75,7 @@ jobs:
 
     runs-on: ubuntu-latest
     permissions:
-      contents: read
+      contents: write
       pull-requests: write
       id-token: write
 

.github/workflows/tacos_detect_drift.yml

@@ -66,7 +66,7 @@ jobs:
 
     runs-on: ubuntu-latest
     permissions:
-      contents: read
+      contents: write
       pull-requests: write
       id-token: write
 

.github/workflows/tacos_plan.yml

@@ -15,7 +15,7 @@ on:
         default: refs/heads/stable
       debug:
         type: string
-        default: 0
+        default: "0"
     secrets:
       ssh-private-key:
         description: "Private SSH key to use for git clone"
@@ -78,7 +78,7 @@ jobs:
 
     runs-on: ubuntu-latest
     permissions:
-      contents: read
+      contents: write
       pull-requests: write
       id-token: write
 
@@ -128,12 +128,10 @@ jobs:
         id: main
         run: |
           "$TACOS_GHA_HOME/"lib/ci/tacos-plan
-
       - name: Save matrix result
         # we need to show any errors to end-users
         if: always()
         uses: ./tacos-gha/.github/actions/matrix-fan-out
-
   summary:
     needs: tacos_plan
     # we need to report failures, too

.github/workflows/tacos_unlock.yml

@@ -32,20 +32,18 @@ env:
   GETSENTRY_SAC_VERB: state-admin
 
 jobs:
-  determine-tf-root-modules:
-    name: List Slices
+  determine-terraformers:
+    name: list terraformers
     if: |
       false
       || github.event.action != 'labeled'
       || github.event.label.name == ':taco::unlock'
     outputs:
-      slices: ${{ steps.list-slices.outputs.slices }}
-
+      terraformers: ${{ steps.list-terraformers.outputs.terraformers }}
     runs-on: ubuntu-latest
     permissions:
       contents: read
       pull-requests: write
-
     steps:
       - name: Checkout IAC
         uses: actions/checkout@v4
@@ -55,30 +53,32 @@ jobs:
           repository: ${{inputs.tacos_gha_repo}}
           ref: ${{inputs.tacos_gha_ref}}
           path: tacos-gha
-
-      - name: List Slices
-        id: list-slices
-        uses: ./tacos-gha/.github/actions/list-slices
+      - name: basic-setup
+        uses: ./tacos-gha/.github/actions/basic-setup
+      - name: List Terraformers
+        id: list-terraformers
+        run: |
+          gha-log-as-step-summary \
+            "$TACOS_GHA_HOME/"lib/ci/list-terraformers
 
   tacos_unlock:
     name: TACOS Unlock
-    needs: [determine-tf-root-modules]
-    if: |
-      needs.determine-tf-root-modules.outputs.slices != '[]'
+    needs: [determine-terraformers]
     strategy:
       fail-fast: false
       matrix:
-        tf-root-module:
-          ${{ fromJSON(needs.determine-tf-root-modules.outputs.slices) }}
-
+        terraformer:
+          ${{ fromJSON(needs.determine-terraformers.outputs.terraformers) }}
     runs-on: ubuntu-latest
     permissions:
       contents: read
       pull-requests: write
       id-token: write
 
     env:
-      TF_ROOT_MODULE: ${{matrix.tf-root-module}}
+      SUDO_GCP_SERVICE_ACCOUNT: ${{fromJSON(matrix.terraformer).SUDO_GCP_SERVICE_ACCOUNT}}
+      GETSENTRY_SAC_OIDC: ${{fromJSON(matrix.terraformer).GETSENTRY_SAC_OIDC}}
+      SLICES: ${{toJSON(fromJSON(matrix.terraformer).slices)}}
     steps:
       - name: Checkout IAC
         uses: actions/checkout@v4
@@ -88,8 +88,8 @@ jobs:
           repository: ${{inputs.tacos_gha_repo}}
           ref: ${{inputs.tacos_gha_ref}}
           path: tacos-gha
-      - name: Setup
-        uses: ./tacos-gha/.github/actions/setup
+      - name: basic-setup
+        uses: ./tacos-gha/.github/actions/basic-setup
         with:
           ssh-private-key: ${{ secrets.ssh-private-key }}
           # We explicitly list the low-concern actions, during which users will
@@ -113,15 +113,28 @@ jobs:
               || github.triggering_actor
             }}
 
+      - name: gcp auth
+        id: auth
+        uses: google-github-actions/auth@v2.1.1
+        with:
+          workload_identity_provider: ${{env.GETSENTRY_SAC_OIDC}}
+          service_account: ${{env.SUDO_GCP_SERVICE_ACCOUNT}}
+
       - name: Unlock
         id: main
         run: |
-          tf-step-summary "TACOS Unlock" "$TACOS_GHA_HOME/"lib/tacos/unlock
+          # release all tfstate locks currently held 
+          jq <<< "$SLICES" -r '.[]' | ./tacos-gha/lib/ci/unlock
 
       - name: Save matrix result
         # we need to show any errors to end-users
         if: always()
         uses: ./tacos-gha/.github/actions/matrix-fan-out
+        with:
+          path: |
+            **/matrix-fan-out
+          matrix: |
+            { "terraformer": "${{env.SUDO_GCP_SERVICE_ACCOUNT}}" }
 
   summary:
     needs: tacos_unlock
@@ -142,6 +155,9 @@ jobs:
         uses: ./tacos-gha/.github/actions/just-the-basics
       - name: Run matrix-fan-in
         uses: ./tacos-gha/.github/actions/matrix-fan-in
+        with:
+          path: |
+            **/matrix-fan-out
       - name: Summarize
         id: summary
         run: |

activate.sh

@@ -2,6 +2,8 @@
 _here="$(readlink -f "$(dirname "${BASH_SOURCE:-$0}")")"
 
 export TACOS_GHA_HOME="$_here"
+
+export PYTHONPATH="$TACOS_GHA_HOME${PYTHONPATH:+:$PYTHONPATH}"
 export PATH="$TACOS_GHA_HOME/bin${PATH:+:$PATH}}"
 export DEBUG="${DEBUG:-}"
 

bin/tf-lock-url

@@ -0,0 +1 @@
+../lib/tf_lock/tf-lock-url

lib/ci/bin/default-shell-post-sudo

@@ -5,6 +5,7 @@ set -euo pipefail
 HERE="$(dirname "$(readlink -f "$0")")"
 umask 002  # stuff is group-writable by default
 
+export DEBUG="${DEBUG:-}"
 if (( DEBUG > 0 )); then
   gha-printenv post-sudo
 fi

lib/ci/bin/terragrunt-noninteractive

@@ -32,10 +32,12 @@ export DEBUG="${DEBUG:-}"
 if (( DEBUG >= 1 )); then
   export TF_LOG=debug
   export TERRAGRUNT_LOG_LEVEL=info
-  set -x
 fi
 if (( DEBUG >= 3 )); then
   export TERRAGRUNT_LOG_LEVEL=debug
+  if (( DEBUG >= 4 )); then
+    set -x
+  fi
 elif (( DEBUG <= 0 )); then
   export TERRAGRUNT_LOG_LEVEL=error
 fi

lib/ci/bin/unlock-one

@@ -0,0 +1,6 @@
+#!/bin/bash
+set -euo pipefail
+
+export TF_ROOT_MODULE=$1
+
+tf-step-summary "TACOS Unlock" tf-lock-release

lib/ci/list-terraformers

@@ -0,0 +1,10 @@
+#!/bin/bash
+set -euo pipefail
+
+python3 -m lib.tacos.terraformers |
+jq -R |
+jq -cs |
+gha-set-output terraformers |
+# prettify
+jq . \
+;

lib/ci/tacos_summary.py

@@ -25,6 +25,9 @@
 FILE_NOT_FOUND = "(file not found: {!r}"
 
 SectionFunction = Callable[[Sequence["SliceSummary"], int], Lines]
+TacosSummary = Callable[
+    [Collection["SliceSummary"], ByteBudget, str, int], Lines
+]
 
 
 def ensmallen(lines: Lines, size_limit: int) -> Lines:
@@ -318,11 +321,20 @@ def error_section(
     return mksection(budget, slices, title="Errors", first=True)
 
 
-def main_helper(
-    tacos_summary: Callable[
-        [Collection[SliceSummary], ByteBudget, str, int], Lines
-    ],
-) -> ExitCode:
+def process_slices(
+    tacos_summary: TacosSummary, slices: Collection[SliceSummary]
+) -> Iterable[Line]:
+    budget = ByteBudget(COMMENT_SIZE_LIMIT - 1000)
+
+    from os import environ
+
+    run_id = int(environ["GITHUB_RUN_ID"])
+    repository = environ["GITHUB_REPOSITORY"]
+
+    return tacos_summary(slices, budget, repository, run_id)
+
+
+def main_helper(tacos_summary: TacosSummary) -> ExitCode:
     from sys import argv
 
     try:
@@ -332,14 +344,8 @@ def main_helper(
 
     path = OSPath(arg)
     slices = tuple(SliceSummary.from_matrix_fan_in(path))
-    budget = ByteBudget(COMMENT_SIZE_LIMIT - 1000)
-
-    from os import environ
-
-    run_id = int(environ["GITHUB_RUN_ID"])
-    repository = environ["GITHUB_REPOSITORY"]
 
-    for line in tacos_summary(slices, budget, repository, run_id):
+    for line in process_slices(tacos_summary, slices):
         print(line)
 
     return 0