diff --git a/.github/actions/basic-setup/action.yml b/.github/actions/basic-setup/action.yml new file mode 100644 index 00000000..668f3de9 --- /dev/null +++ b/.github/actions/basic-setup/action.yml @@ -0,0 +1,65 @@ +name: Basic-Setup +# This is derived from the Setup github action. +# It omits several steps, and is mainly used by tacos_unlock. + +inputs: + ssh-private-key: + description: "Private SSH key to use for git clone" + type: string + default: "" + user: + description: the username that will be used for following steps + required: false + default: ${{github.triggering_actor}} + shell: + description: "private -- do not use" + default: env ./tacos-gha/lib/ci/default-shell {0} + +runs: + using: composite + + steps: + - uses: ./tacos-gha/.github/actions/just-the-basics + + - name: tell TF username and PR + uses: ./tacos-gha/.github/actions/set-username-and-hostname + with: + user: ${{inputs.user}} + + - name: Set up SSH agent + if: inputs.ssh-private-key != '' + uses: webfactory/ssh-agent@v0.8.0 + with: + ssh-private-key: ${{ inputs.ssh-private-key }} + + # These fix most ownership, permission issues, but the .ssh config files + # still get the wrong ownership, fixed in the next step. + ssh-agent-cmd: |- + ./tacos-gha/lib/ci/bin/sudo-ssh-agent + ssh-add-cmd: |- + ./tacos-gha/lib/ci/bin/sudo-ssh-add + - name: Fix .ssh permissions + shell: ${{inputs.shell}} + if: inputs.ssh-private-key != '' + run: | + : fix ssh config ownership + sudo chown -v -R "$(id -un):$(id -gn)" ~/.ssh + : Show SSH agent pubkeys + ssh-add -L + : ... hashes too + ssh-add -l + # this should really be default behavior: + - shell: ${{inputs.shell}} + run: | + gha-set-env 'TF_VERSION' < "$(nearest-config-file .terraform-version)" + gha-set-env 'TERRAGRUNT_VERSION' < "$(nearest-config-file .terragrunt-version)" + - name: Setup Terragrunt + uses: autero1/action-terragrunt@v1.3.2 + with: + terragrunt_version: ${{env.TERRAGRUNT_VERSION}} + + - name: Setup Terraform + uses: hashicorp/setup-terraform@v3 + with: + terraform_wrapper: false + terraform_version: ${{ env.TF_VERSION }} diff --git a/.github/actions/matrix-fan-in/action.yml b/.github/actions/matrix-fan-in/action.yml index 638acc82..b502abc6 100644 --- a/.github/actions/matrix-fan-in/action.yml +++ b/.github/actions/matrix-fan-in/action.yml @@ -31,8 +31,7 @@ runs: uses: actions/download-artifact@v4 with: pattern: ${{ env.artifact_name }} *${{ inputs.pattern }}* - path: matrix-fan-in.tmp - + path: ${{ inputs.path }} - name: fix up archive files shell: bash env: @@ -41,3 +40,11 @@ runs: # note: "$GITHUB_ACTION_PATH" contains this action directory's path run: | "$GITHUB_ACTION_PATH/"rename-tmp-dirs.sh "$MATRIX_FAN_OUT_PATH" + + ## DEBUG: + # - name: Start SSH + # if: always() + # uses: lhotari/action-upterm@v1 + # with: + # ## limits ssh access and adds the ssh public keys of the listed GitHub users + # limit-access-to-users: bukzor,kneeyo1 diff --git a/.github/actions/matrix-fan-in/rename-tmp-dirs.sh b/.github/actions/matrix-fan-in/rename-tmp-dirs.sh index 3ce91532..e241a2c8 100755 --- a/.github/actions/matrix-fan-in/rename-tmp-dirs.sh +++ b/.github/actions/matrix-fan-in/rename-tmp-dirs.sh @@ -14,7 +14,7 @@ path="$1" mkdir -p "$path" : directory name fixup -find ./matrix-fan-in.tmp \ +find "$path"\ -mindepth 1 \ -maxdepth 1 \ -print0 \ diff --git a/.github/actions/matrix-fan-out/action.yml b/.github/actions/matrix-fan-out/action.yml index 3a651a6b..65d998fa 100644 --- a/.github/actions/matrix-fan-out/action.yml +++ b/.github/actions/matrix-fan-out/action.yml @@ -11,6 +11,9 @@ inputs: shell: description: "private -- do not use" default: bash -euxo pipefail {0} + matrix: + description: defaults to toJSON(matrix) + default: ${{ toJSON(matrix) }} runs: using: "composite" @@ -19,7 +22,7 @@ runs: - shell: ${{ inputs.shell }} env: MATRIX_FAN_OUT_PATH: ${{ inputs.path }} - GHA_MATRIX_CONTEXT: ${{ toJSON(matrix) }} + GHA_MATRIX_CONTEXT: ${{ inputs.matrix }} run: | "$GITHUB_ACTION_PATH/"prepare.sh | tee -a "$GITHUB_ENV" diff --git a/.github/workflows/selftest-matrix-io.yml b/.github/workflows/selftest-matrix-io.yml index ba65cfa6..2c87cc54 100644 --- a/.github/workflows/selftest-matrix-io.yml +++ b/.github/workflows/selftest-matrix-io.yml @@ -32,7 +32,7 @@ jobs: exec >&2 # our only output is logging printf "keys=[10, 27]" >> "$GITHUB_OUTPUT" # for scale testing: - ###seq 30 | shuf | jq -R | jq -cs | tee -a "$GITHUB_OUTPUT" + #seq 30 | shuf | jq -R | jq -cs | tee -a "$GITHUB_OUTPUT" fan-out: name: Compute Squares diff --git a/.github/workflows/tacos_apply.yml b/.github/workflows/tacos_apply.yml index f164cc34..fbfaa2cd 100644 --- a/.github/workflows/tacos_apply.yml +++ b/.github/workflows/tacos_apply.yml @@ -75,7 +75,7 @@ jobs: runs-on: ubuntu-latest permissions: - contents: read + contents: write pull-requests: write id-token: write diff --git a/.github/workflows/tacos_detect_drift.yml b/.github/workflows/tacos_detect_drift.yml index ac52b725..9d45d0e4 100644 --- a/.github/workflows/tacos_detect_drift.yml +++ b/.github/workflows/tacos_detect_drift.yml @@ -66,7 +66,7 @@ jobs: runs-on: ubuntu-latest permissions: - contents: read + contents: write pull-requests: write id-token: write diff --git a/.github/workflows/tacos_plan.yml b/.github/workflows/tacos_plan.yml index fb0fb17e..d2fa63ec 100644 --- a/.github/workflows/tacos_plan.yml +++ b/.github/workflows/tacos_plan.yml @@ -15,7 +15,7 @@ on: default: refs/heads/stable debug: type: string - default: 0 + default: "0" secrets: ssh-private-key: description: "Private SSH key to use for git clone" @@ -78,7 +78,7 @@ jobs: runs-on: ubuntu-latest permissions: - contents: read + contents: write pull-requests: write id-token: write @@ -128,12 +128,10 @@ jobs: id: main run: | "$TACOS_GHA_HOME/"lib/ci/tacos-plan - - name: Save matrix result # we need to show any errors to end-users if: always() uses: ./tacos-gha/.github/actions/matrix-fan-out - summary: needs: tacos_plan # we need to report failures, too diff --git a/.github/workflows/tacos_unlock.yml b/.github/workflows/tacos_unlock.yml index b9076984..6357e880 100644 --- a/.github/workflows/tacos_unlock.yml +++ b/.github/workflows/tacos_unlock.yml @@ -32,20 +32,18 @@ env: GETSENTRY_SAC_VERB: state-admin jobs: - determine-tf-root-modules: - name: List Slices + determine-terraformers: + name: list terraformers if: | false || github.event.action != 'labeled' || github.event.label.name == ':taco::unlock' outputs: - slices: ${{ steps.list-slices.outputs.slices }} - + terraformers: ${{ steps.list-terraformers.outputs.terraformers }} runs-on: ubuntu-latest permissions: contents: read pull-requests: write - steps: - name: Checkout IAC uses: actions/checkout@v4 @@ -55,22 +53,22 @@ jobs: repository: ${{inputs.tacos_gha_repo}} ref: ${{inputs.tacos_gha_ref}} path: tacos-gha - - - name: List Slices - id: list-slices - uses: ./tacos-gha/.github/actions/list-slices + - name: basic-setup + uses: ./tacos-gha/.github/actions/basic-setup + - name: List Terraformers + id: list-terraformers + run: | + gha-log-as-step-summary \ + "$TACOS_GHA_HOME/"lib/ci/list-terraformers tacos_unlock: name: TACOS Unlock - needs: [determine-tf-root-modules] - if: | - needs.determine-tf-root-modules.outputs.slices != '[]' + needs: [determine-terraformers] strategy: fail-fast: false matrix: - tf-root-module: - ${{ fromJSON(needs.determine-tf-root-modules.outputs.slices) }} - + terraformer: + ${{ fromJSON(needs.determine-terraformers.outputs.terraformers) }} runs-on: ubuntu-latest permissions: contents: read @@ -78,7 +76,9 @@ jobs: id-token: write env: - TF_ROOT_MODULE: ${{matrix.tf-root-module}} + SUDO_GCP_SERVICE_ACCOUNT: ${{fromJSON(matrix.terraformer).SUDO_GCP_SERVICE_ACCOUNT}} + GETSENTRY_SAC_OIDC: ${{fromJSON(matrix.terraformer).GETSENTRY_SAC_OIDC}} + SLICES: ${{toJSON(fromJSON(matrix.terraformer).slices)}} steps: - name: Checkout IAC uses: actions/checkout@v4 @@ -88,8 +88,8 @@ jobs: repository: ${{inputs.tacos_gha_repo}} ref: ${{inputs.tacos_gha_ref}} path: tacos-gha - - name: Setup - uses: ./tacos-gha/.github/actions/setup + - name: basic-setup + uses: ./tacos-gha/.github/actions/basic-setup with: ssh-private-key: ${{ secrets.ssh-private-key }} # We explicitly list the low-concern actions, during which users will @@ -113,15 +113,28 @@ jobs: || github.triggering_actor }} + - name: gcp auth + id: auth + uses: google-github-actions/auth@v2.1.1 + with: + workload_identity_provider: ${{env.GETSENTRY_SAC_OIDC}} + service_account: ${{env.SUDO_GCP_SERVICE_ACCOUNT}} + - name: Unlock id: main run: | - tf-step-summary "TACOS Unlock" "$TACOS_GHA_HOME/"lib/tacos/unlock + # release all tfstate locks currently held + jq <<< "$SLICES" -r '.[]' | ./tacos-gha/lib/ci/unlock - name: Save matrix result # we need to show any errors to end-users if: always() uses: ./tacos-gha/.github/actions/matrix-fan-out + with: + path: | + **/matrix-fan-out + matrix: | + { "terraformer": "${{env.SUDO_GCP_SERVICE_ACCOUNT}}" } summary: needs: tacos_unlock @@ -142,6 +155,9 @@ jobs: uses: ./tacos-gha/.github/actions/just-the-basics - name: Run matrix-fan-in uses: ./tacos-gha/.github/actions/matrix-fan-in + with: + path: | + **/matrix-fan-out - name: Summarize id: summary run: | diff --git a/activate.sh b/activate.sh index a5f8db1b..f7e7aa94 100644 --- a/activate.sh +++ b/activate.sh @@ -2,6 +2,8 @@ _here="$(readlink -f "$(dirname "${BASH_SOURCE:-$0}")")" export TACOS_GHA_HOME="$_here" + +export PYTHONPATH="$TACOS_GHA_HOME${PYTHONPATH:+:$PYTHONPATH}" export PATH="$TACOS_GHA_HOME/bin${PATH:+:$PATH}}" export DEBUG="${DEBUG:-}" diff --git a/bin/tf-lock-url b/bin/tf-lock-url new file mode 120000 index 00000000..94a2c6f1 --- /dev/null +++ b/bin/tf-lock-url @@ -0,0 +1 @@ +../lib/tf_lock/tf-lock-url \ No newline at end of file diff --git a/lib/ci/bin/default-shell-post-sudo b/lib/ci/bin/default-shell-post-sudo index 5cb34696..0260e2ef 100755 --- a/lib/ci/bin/default-shell-post-sudo +++ b/lib/ci/bin/default-shell-post-sudo @@ -5,6 +5,7 @@ set -euo pipefail HERE="$(dirname "$(readlink -f "$0")")" umask 002 # stuff is group-writable by default +export DEBUG="${DEBUG:-}" if (( DEBUG > 0 )); then gha-printenv post-sudo fi diff --git a/lib/ci/bin/terragrunt-noninteractive b/lib/ci/bin/terragrunt-noninteractive index 29745096..4527b436 100755 --- a/lib/ci/bin/terragrunt-noninteractive +++ b/lib/ci/bin/terragrunt-noninteractive @@ -32,10 +32,12 @@ export DEBUG="${DEBUG:-}" if (( DEBUG >= 1 )); then export TF_LOG=debug export TERRAGRUNT_LOG_LEVEL=info - set -x fi if (( DEBUG >= 3 )); then export TERRAGRUNT_LOG_LEVEL=debug + if (( DEBUG >= 4 )); then + set -x + fi elif (( DEBUG <= 0 )); then export TERRAGRUNT_LOG_LEVEL=error fi diff --git a/lib/ci/bin/unlock-one b/lib/ci/bin/unlock-one new file mode 100755 index 00000000..522ccc56 --- /dev/null +++ b/lib/ci/bin/unlock-one @@ -0,0 +1,6 @@ +#!/bin/bash +set -euo pipefail + +export TF_ROOT_MODULE=$1 + +tf-step-summary "TACOS Unlock" tf-lock-release diff --git a/lib/ci/list-terraformers b/lib/ci/list-terraformers new file mode 100755 index 00000000..24e78bf9 --- /dev/null +++ b/lib/ci/list-terraformers @@ -0,0 +1,10 @@ +#!/bin/bash +set -euo pipefail + +python3 -m lib.tacos.terraformers | +jq -R | +jq -cs | +gha-set-output terraformers | +# prettify +jq . \ +; \ No newline at end of file diff --git a/lib/ci/tacos_summary.py b/lib/ci/tacos_summary.py index 1bad1f09..4f172fce 100644 --- a/lib/ci/tacos_summary.py +++ b/lib/ci/tacos_summary.py @@ -25,6 +25,9 @@ FILE_NOT_FOUND = "(file not found: {!r}" SectionFunction = Callable[[Sequence["SliceSummary"], int], Lines] +TacosSummary = Callable[ + [Collection["SliceSummary"], ByteBudget, str, int], Lines +] def ensmallen(lines: Lines, size_limit: int) -> Lines: @@ -318,11 +321,20 @@ def error_section( return mksection(budget, slices, title="Errors", first=True) -def main_helper( - tacos_summary: Callable[ - [Collection[SliceSummary], ByteBudget, str, int], Lines - ], -) -> ExitCode: +def process_slices( + tacos_summary: TacosSummary, slices: Collection[SliceSummary] +) -> Iterable[Line]: + budget = ByteBudget(COMMENT_SIZE_LIMIT - 1000) + + from os import environ + + run_id = int(environ["GITHUB_RUN_ID"]) + repository = environ["GITHUB_REPOSITORY"] + + return tacos_summary(slices, budget, repository, run_id) + + +def main_helper(tacos_summary: TacosSummary) -> ExitCode: from sys import argv try: @@ -332,14 +344,8 @@ def main_helper( path = OSPath(arg) slices = tuple(SliceSummary.from_matrix_fan_in(path)) - budget = ByteBudget(COMMENT_SIZE_LIMIT - 1000) - - from os import environ - - run_id = int(environ["GITHUB_RUN_ID"]) - repository = environ["GITHUB_REPOSITORY"] - for line in tacos_summary(slices, budget, repository, run_id): + for line in process_slices(tacos_summary, slices): print(line) return 0 diff --git a/lib/ci/tacos_unlock_summary.py b/lib/ci/tacos_unlock_summary.py index 893b8239..2635c0f8 100755 --- a/lib/ci/tacos_unlock_summary.py +++ b/lib/ci/tacos_unlock_summary.py @@ -7,13 +7,15 @@ from lib.byte_budget import ByteBudget from lib.byte_budget import Lines from lib.byte_budget import Log +from lib.sh import sh from lib.types import ExitCode +from lib.types import OSPath from .tacos_summary import GHA_RUN_URL from .tacos_summary import SKIPPED_MESSAGE from .tacos_summary import SliceSummary from .tacos_summary import error_section -from .tacos_summary import main_helper +from .tacos_summary import process_slices def header( @@ -77,7 +79,14 @@ def tacos_unlock_summary( def main() -> ExitCode: - return main_helper(tacos_unlock_summary) + slices: list[SliceSummary] = [] + for matrix_fan_out in sh.lines(("find", ".", "-name", "matrix-fan-out")): + slices.append(SliceSummary.from_matrix_fan_out(OSPath(matrix_fan_out))) + + for line in process_slices(tacos_unlock_summary, slices): + print(line) + + return 0 if __name__ == "__main__": diff --git a/lib/ci/unlock b/lib/ci/unlock new file mode 100755 index 00000000..cc3cf377 --- /dev/null +++ b/lib/ci/unlock @@ -0,0 +1,31 @@ +#!/bin/bash +set -euo pipefail +set -ex +exec 1>&2 # stdout is reserved for tf plan/apply results +( + echo "### TACOS Unlock" + echo + if results=$(xargs -r -P1 -n1 unlock-one 2>&1); then + cat < + +Success! all slices have been unlocked. + +\`\`\`console +$results +\`\`\` + +EOF + else + cat < + +Some slices failed to unlock. + +\`\`\`console +$results +\`\`\` + +EOF + fi +) | gha-step-summary; diff --git a/lib/functions.py b/lib/functions.py index 4ea5411b..0fc491a9 100644 --- a/lib/functions.py +++ b/lib/functions.py @@ -4,6 +4,8 @@ from typing import Iterable from typing import TypeVar +from .types import Lines + T = TypeVar("T") @@ -34,3 +36,13 @@ def one(xs: Iterable[T]) -> T: def noop() -> None: pass + + +def config_lines(lines: Lines) -> Lines: + """Strip commented and empty lines from configuration.""" + for line in lines: + line = line.strip() + if not line or line.startswith("#"): + continue + + yield line diff --git a/lib/gcloud/gcloud-auth-export-access-token b/lib/gcloud/gcloud-auth-export-access-token index 6d651a23..76335e8b 100755 --- a/lib/gcloud/gcloud-auth-export-access-token +++ b/lib/gcloud/gcloud-auth-export-access-token @@ -35,7 +35,7 @@ info() { echo >&2 "$@"; } error() { info "$@"; exit 1; } export DEBUG="${DEBUG:-}" -if (( DEBUG >= 1 )); then +if (( DEBUG >= 5 )); then set -x fi diff --git a/lib/gcloud/sudo-gcp b/lib/gcloud/sudo-gcp index 3a6cc1f2..6896d475 100755 --- a/lib/gcloud/sudo-gcp +++ b/lib/gcloud/sudo-gcp @@ -4,7 +4,7 @@ HERE="$(dirname "$(readlink -f "$0")")" export DEBUG="${DEBUG:-}" -if (( DEBUG >= 1 )); then +if (( DEBUG >= 5 )); then set -x fi diff --git a/lib/gcloud/sudo-gcp-service-account b/lib/gcloud/sudo-gcp-service-account index 519ece40..fa92f85b 100755 --- a/lib/gcloud/sudo-gcp-service-account +++ b/lib/gcloud/sudo-gcp-service-account @@ -23,7 +23,7 @@ interpolate_line() { export DEBUG="${DEBUG:-}" -if (( DEBUG >= 1 )); then +if (( DEBUG >= 5 )); then set -x fi diff --git a/lib/sh/cd.py b/lib/sh/cd.py index 734461b7..95581545 100644 --- a/lib/sh/cd.py +++ b/lib/sh/cd.py @@ -19,14 +19,14 @@ @contextmanager def cd( - dirname: Path, env: Environ = environ, *, direnv: bool = True + dirname: OSPath, env: Environ = environ, *, direnv: bool = True ) -> Generator[Path]: oldpwd = Path.cwd(env) newpwd = oldpwd / dirname cwd = OSPath.cwd() if newpwd == oldpwd and cwd.samefile(newpwd): # we're already there - yield oldpwd + yield newpwd return xtrace(("cd", dirname)) diff --git a/lib/sh/core.py b/lib/sh/core.py index 27a35a19..6e993d21 100755 --- a/lib/sh/core.py +++ b/lib/sh/core.py @@ -6,11 +6,14 @@ from lib.functions import LessThanOneError as LessThanOneError from lib.functions import MoreThanOneError as MoreThanOneError +from lib.functions import config_lines from lib.functions import one from lib.types import Environ from lib.types import OSPath from .constant import UTF8 +from .errors import ShError +from .io import quote from .io import xtrace from .types import Command from .types import Generator @@ -66,12 +69,7 @@ def lines(cmd: Command, *, encoding: str = UTF8) -> Generator[Line]: """ process = _popen(cmd, encoding=encoding, capture_output=True) assert process.stdout, process.stdout - for line in process.stdout: - line = line.strip() - if not line or line.startswith("#"): - continue - - yield line + yield from config_lines(process.stdout) # handle termination and error codes _wait(process) @@ -185,8 +183,10 @@ def _wait( raise retcode = process.poll() if check and retcode: - raise subprocess.CalledProcessError( - retcode, process.args, output=stdout, stderr=stderr + assert isinstance(process.args, tuple) + raise ShError( + f"Command failed: (code {retcode})\n\n {quote(process.args)}\n", + code=retcode, ) assert retcode is not None, retcode return subprocess.CompletedProcess(process.args, retcode, stdout, stderr) diff --git a/lib/sh/errors.py b/lib/sh/errors.py new file mode 100644 index 00000000..a2e8c026 --- /dev/null +++ b/lib/sh/errors.py @@ -0,0 +1,9 @@ +from __future__ import annotations + +from subprocess import CalledProcessError + +from lib.user_error import UserError + + +class ShError(UserError, CalledProcessError): + pass diff --git a/lib/sh/io.py b/lib/sh/io.py index 3111100d..d81ee2bd 100644 --- a/lib/sh/io.py +++ b/lib/sh/io.py @@ -4,6 +4,7 @@ from os import getenv from typing import ContextManager from typing import Iterable +from typing import TypeVar from lib import ansi @@ -21,8 +22,8 @@ # 2 - debug # 3 - trace # note: empty-string vars should be treated as unset -DEBUG: int = int(getenv("DEBUG") or "1") - +DEBUG: int = int(getenv("DEBUG", "1") or "0") +T = TypeVar("T") Uniq = set[tuple[object, ...]] UNIQ: Uniq | None = None @@ -110,12 +111,23 @@ def verbosity(newvalue: int) -> Generator[int]: DEBUG = orig +@contextlib.contextmanager +def noop_context(x: T) -> Generator[T]: + yield x + + def quiet() -> ContextManager[int]: - return verbosity(0) + if getenv("DEBUG") is None: + return verbosity(0) + else: + return noop_context(DEBUG) def loud() -> ContextManager[int]: - return verbosity(2) + if getenv("DEBUG") is None: + return verbosity(2) + else: + return noop_context(DEBUG) @contextlib.contextmanager diff --git a/lib/sh/sh.py b/lib/sh/sh.py index 6587cd05..90921aeb 100644 --- a/lib/sh/sh.py +++ b/lib/sh/sh.py @@ -13,6 +13,7 @@ from .core import run as run from .core import stdout as stdout from .core import success as success +from .errors import ShError as ShError from .io import * # info, debug, banner, comment, and 10+ more from .json import jq as jq from .json import json as json diff --git a/lib/tacos/lib/handle-tflock-cache b/lib/tacos/lib/handle-tflock-cache new file mode 100755 index 00000000..b2dfd21a --- /dev/null +++ b/lib/tacos/lib/handle-tflock-cache @@ -0,0 +1,40 @@ +#!/bin/bash +set -euo pipefail + +cache=.config/tf-lock-info + +#todo: make this an artifact instead +# we expect at most 6 slices to be fighting +git_push_aggressively() { + sleep=3 + limit=20 # 3 min + while ! (set -ex; git push -q); do + if (( sleep > limit )); then exit 1; fi + echo "failed! trying again after $sleep seconds..." + sleep $sleep + (( sleep += 3 )) + (set -ex + git pull --rebase origin "$GITHUB_HEAD_REF" + ) + done +} + +if [[ -d $cache ]]; then + git add -f $cache + if ! git diff --cached --exit-code $cache; then + git config --global user.email "$USER@$HOSTNAME" + git config --global user.name "$USER" + git config --global push.default current + #git checkout -b "$GITHUB_HEAD_REF" + ( set -ex + git fetch origin --depth=1 "$GITHUB_HEAD_REF" + git checkout "$GITHUB_HEAD_REF" + # graft="$(git rev-parse HEAD)" + # add .terraform-lock changes + git add -u . + git status + git commit -m "updating tf-lock cache: $TF_ROOT_MODULE" + ) + git_push_aggressively + fi + fi \ No newline at end of file diff --git a/lib/tacos/plan b/lib/tacos/plan index ebfc80da..2305ead6 100755 --- a/lib/tacos/plan +++ b/lib/tacos/plan @@ -12,8 +12,8 @@ if "$TACOS_LOCK"; then ( set -ex env GETSENTRY_SAC_VERB=state-admin sudo-gcp tf-lock-acquire ) + "$TACOS_GHA_HOME"/lib/tacos/lib/handle-tflock-cache fi - quietly sudo-gcp terragrunt run-all init if "$TACOS_LOCK"; then diff --git a/lib/tacos/terraformers.py b/lib/tacos/terraformers.py new file mode 100755 index 00000000..6c99b8d9 --- /dev/null +++ b/lib/tacos/terraformers.py @@ -0,0 +1,91 @@ +#!/usr/bin/env python3.12 +from __future__ import annotations + +from dataclasses import dataclass +from typing import Iterable + +from lib import json +from lib.constants import TACOS_GHA_HOME +from lib.sh import sh +from lib.types import Generator +from lib.types import OSPath + +from .dependent_slices import TFCategorized +from .dependent_slices import TopLevelTFModule + + +@dataclass(frozen=True) +class TerraformerResult: + GETSENTRY_SAC_OIDC: str + SUDO_GCP_SERVICE_ACCOUNT: str + slices: set[TopLevelTFModule] + + def to_json(self) -> json.Value: + return { + "GETSENTRY_SAC_OIDC": self.GETSENTRY_SAC_OIDC, + "SUDO_GCP_SERVICE_ACCOUNT": self.SUDO_GCP_SERVICE_ACCOUNT, + # Convert each TopLevelTFModule in the set to a string, then convert the set to a list + "slices": [str(path) for path in self.slices], + } + + +def get_cached_slices() -> Iterable[TopLevelTFModule]: + """List terraform/grunt slices that have a cached tflock path. + + Slices without such a file have never been locked. + """ + for slice in sorted(TFCategorized.from_git().slices): + if OSPath(slice / ".config/tf-lock-info/Path").exists(): + yield slice + + +def list_terraformers( + slices: Iterable[TopLevelTFModule], +) -> Iterable[TerraformerResult]: + """List unlockable slices and the oidc provider and terraformer of that slice""" + for slice in slices: + with sh.cd(OSPath(slice)): + oidc_provider = sh.stdout( + (TACOS_GHA_HOME / "lib/getsentry-sac/oidc-provider",) + ) + terraformer = sh.stdout(("sudo-gcp-service-account",)) + + yield TerraformerResult(oidc_provider, terraformer, set([slice])) + + +def terraformers() -> Generator[TerraformerResult]: + """Which slices need to be unlocked?""" + from collections import defaultdict + + by_terraformer: defaultdict[tuple[str, str], set[TopLevelTFModule]] = ( + defaultdict(set) + ) + + slices = get_cached_slices() + for tf_result in list_terraformers(slices): + key = ( + tf_result.GETSENTRY_SAC_OIDC, + tf_result.SUDO_GCP_SERVICE_ACCOUNT, + ) + for slice in tf_result.slices: + by_terraformer[key].add(slice) + + for key in by_terraformer: + oidc_provider, terraformer = key + yield TerraformerResult( + oidc_provider, terraformer, by_terraformer[key] + ) + + +def main() -> int: + import json + + for result in terraformers(): + # use custom conversion here, because json doesn't like sets or OSPaths + print(json.dumps(result.to_json())) + + return 0 + + +if __name__ == "__main__": + exit(main()) diff --git a/lib/tf_lock/TESTING.md b/lib/tf_lock/TESTING.md index 4f25c0b7..fb8fde09 100644 --- a/lib/tf_lock/TESTING.md +++ b/lib/tf_lock/TESTING.md @@ -1,3 +1,5 @@ +# TESTING + FIXME: automated testing for lib/tf_lock ```console diff --git a/lib/tf_lock/lib/env.py b/lib/tf_lock/lib/env.py index 9cba80bb..706c9b71 100755 --- a/lib/tf_lock/lib/env.py +++ b/lib/tf_lock/lib/env.py @@ -6,14 +6,35 @@ from os import environ +from lib import json from lib.sh import sh +from lib.types import Environ from lib.types import OSPath from lib.types import Path -from ..release import get_current_host -from ..release import get_current_user -HERE = sh.get_HERE(__file__) +def get_current_host(env: Environ) -> str: + for var in ("HOST", "HOSTNAME"): + if var in env: + return env[var] + else: + import socket + + return socket.gethostname() + + +def get_current_user(env: Environ) -> str: + for var in ("USER", "LOGNAME"): + if var in env: + return env[var] + else: + import getpass + + return getpass.getuser() + + +here: OSPath = sh.get_HERE(__file__) +LIB = here USER = environ["USER"] = get_current_user(environ) HOST = environ["HOST"] = get_current_host(environ) HOSTNAME = environ["HOSTNAME"] = environ["HOST"] @@ -21,15 +42,16 @@ TF_LOCK_ENONE = 2 TF_LOCK_EHELD = 3 - -TACOS_GHA_HOME = environ.setdefault("TACOS_GHA_HOME", str(HERE / "../../..")) +TACOS_GHA_HOME = Path( + environ.setdefault("TACOS_GHA_HOME", str(here / "../../..")) +) -def path_prepend(env_name: str, env_val: str) -> None: +def path_prepend(env_name: str, env_val: object) -> None: r"""Preprend to a colon delimited environment variable. >>> path_prepend('name', 'val') - >>> path_prepend('name', 'val2') + >>> path_prepend('name', Path('val2')) >>> environ['name'] 'val2:val' """ @@ -38,33 +60,35 @@ def path_prepend(env_name: str, env_val: str) -> None: pythonpath_list = pythonpath.split(":") else: pythonpath_list = [] - pythonpath_list.insert(0, env_val) + pythonpath_list.insert(0, str(env_val)) environ[env_name] = ":".join(pythonpath_list) path_prepend("PYTHONPATH", TACOS_GHA_HOME) -path_prepend("PATH", TACOS_GHA_HOME + "/bin") -path_prepend("PATH", TACOS_GHA_HOME + "/lib/tf_lock/bin") +path_prepend("PATH", TACOS_GHA_HOME / "/bin") +path_prepend("PATH", here / "bin") -def tf_working_dir(root_module: OSPath) -> Path: +def tf_working_dir(root_module: OSPath) -> OSPath: if (root_module / "terragrunt.hcl").exists(): with sh.cd(root_module): # validate-inputs makes terragrunt generate its templates - sh.run(( - "terragrunt", - "--terragrunt-no-auto-init=false", - "validate-inputs", - )) - terragrunt_info = sh.json(( - "terragrunt", - "--terragrunt-no-auto-init=false", - "terragrunt-info", - )) - assert isinstance(terragrunt_info, dict), terragrunt_info - working_dir = terragrunt_info.get("WorkingDir") - assert isinstance(working_dir, str) - return Path(working_dir) + sh.run( + ( + "terragrunt", + "--terragrunt-no-auto-init=false", + "validate-inputs", + ) + ) + terragrunt_info = sh.json( + ( + "terragrunt", + "--terragrunt-no-auto-init=false", + "terragrunt-info", + ) + ) + terragrunt_info = json.assert_dict_of_strings(terragrunt_info) + return OSPath(terragrunt_info["WorkingDir"]) else: return root_module diff --git a/lib/tf_lock/lib/env.sh b/lib/tf_lock/lib/env.sh index 9b74a9d9..aa980bb7 100644 --- a/lib/tf_lock/lib/env.sh +++ b/lib/tf_lock/lib/env.sh @@ -30,6 +30,6 @@ tf_working_dir() { } export DEBUG="${DEBUG:-}" -if (( DEBUG >= 0 )); then +if (( DEBUG >= 3 )); then set -x fi diff --git a/lib/tf_lock/lib/tf-lock-info-uncached b/lib/tf_lock/lib/tf-lock-info-uncached new file mode 100755 index 00000000..16910a6b --- /dev/null +++ b/lib/tf_lock/lib/tf-lock-info-uncached @@ -0,0 +1,58 @@ +#!/bin/bash +set -eEuo pipefail +HERE="$(dirname "$(readlink -f "$0")")" +. "$HERE/"env.sh + +root_module="${1:-"$PWD"}" +working_dir="$(tf_working_dir "$root_module")" + +# at most 1 reiteration should be necessary +i=0 +limit=3 +while (( i < limit )); do + (( i += 1 )) + + + fancy_error="$( + cd "$working_dir" || exit 1 + # swap stdout and stderr: + terraform force-unlock -force -- -1 3>&2 2>&1 1>&3 <<< "" + )" && status=$? || status=$? + # strip ansi fanciness from error messages, for automated consumption + error="$( + uncolor <<< "$fancy_error" | + sed -r 's/^│ //; /^[╷╵]$/d' + )" + + if grep -Eq <<< "$error" $'ID:'; then + "$HERE/"error2json <<< "$error" + exit 0 + elif grep -Eq <<< "$error" '^Failed to unlock state: LocalState not locked$' + then + echo >&2 "No remote tfstate configured, path: '$root_module'" + exit 1 + elif grep -Eq <<< "$error" $'^\t\* storage: object doesn'\''t exist$'; then + echo '{"lock": false}' + exit 0 + elif grep -Eq <<< "$error" \ + 'Error: .*(Backend initialization required, please run "terraform init"|Required plugins are not installed)' + then + ( # NB: need to undo the cd to keep relative paths valid + cd "$working_dir" + if ! noise="$(terraform init 2>&1)" || (( i == limit )); then + echo >&2 Terraform init failed! + echo >&2 "$noise" + exit 1 + fi + ) + continue + else # an unexpected error: show it + set +x + echo -n >&2 "$fancy_error" # avoid the newline appended by <<< + exit "$status" + fi + +done + +echo >&2 AssertionError: this should be impossible: "$i" +exit 99 diff --git a/lib/tf_lock/release.py b/lib/tf_lock/release.py index be03a746..4b6ba40e 100644 --- a/lib/tf_lock/release.py +++ b/lib/tf_lock/release.py @@ -6,14 +6,18 @@ from typing import Tuple from lib import ansi -from lib import json from lib.parse import Parse from lib.sh import sh +from lib.tf_lock.lib.env import tf_working_dir from lib.types import Environ from lib.types import OSPath -from lib.types import Path from lib.user_error import UserError +from .lib.env import get_current_host +from .lib.env import get_current_user +from .tf_lock_info import cache_get +from .tf_lock_info import tf_lock_info + HERE = sh.get_HERE(__file__) TF_LOCK_EHELD = 3 @@ -73,26 +77,6 @@ def info(msg: object) -> None: print(msg, file=stderr, flush=True) -def get_current_user(env: Environ) -> str: - for var in ("USER", "LOGNAME"): - if var in env: - return env[var] - else: - import getpass - - return getpass.getuser() - - -def get_current_host(env: Environ) -> str: - for var in ("HOST", "HOSTNAME"): - if var in env: - return env[var] - else: - import socket - - return socket.gethostname() - - def assert_dict_of_strings(json: object) -> dict[str, str]: assert isinstance(json, dict), json @@ -106,14 +90,9 @@ def assert_dict_of_strings(json: object) -> dict[str, str]: return cast(dict[str, str], json) -def get_lock_info(root_module: Path) -> Tuple[bool, dict[str, str]]: - try: - result = sh.json((HERE / "tf-lock-info", str(root_module))) - except sh.CalledProcessError as error: - # error message was already printed by subcommand - raise UserError(code=error.returncode) +def get_lock_info(root_module: OSPath) -> Tuple[bool, dict[str, str]]: - assert isinstance(result, dict), result + result = dict(tf_lock_info(root_module)) lock = result.pop("lock") assert isinstance(lock, bool), lock @@ -121,7 +100,7 @@ def get_lock_info(root_module: Path) -> Tuple[bool, dict[str, str]]: return lock, assert_dict_of_strings(result) -def tf_lock_release(root_module: Path, env: Environ) -> None: +def tf_lock_release(root_module: OSPath, env: Environ) -> None: lock, lock_info = get_lock_info(root_module) if not lock: info(f"tf-lock-release: success: {root_module}") @@ -130,8 +109,14 @@ def tf_lock_release(root_module: Path, env: Environ) -> None: tf_user = f"{get_current_user(env)}@{get_current_host(env)}" lock_user = lock_info["Who"] if tf_user == lock_user: - try: - with sh.cd(tf_working_dir(root_module)): + cache = cache_get(root_module) + with sh.cd(tf_working_dir(root_module)): + if cache: + try: + sh.json(("gcloud", "storage", "rm", cache)) + except sh.ShError: + pass # already unlocked + else: sh.run(( "terraform", "force-unlock", @@ -139,10 +124,6 @@ def tf_lock_release(root_module: Path, env: Environ) -> None: "--", lock_info["ID"], )) - except sh.CalledProcessError as error: - # error message was already printed by subcommand - raise UserError(code=error.returncode) - info(f"tf-lock-release: success: {root_module}({lock_user})") else: @@ -157,19 +138,6 @@ def tf_lock_release(root_module: Path, env: Environ) -> None: ) -def tf_working_dir(root_module: Path) -> Path: - """dereference terragrunt indirection, if any""" - - if OSPath(root_module / "terragrunt.hcl").exists(): - with sh.cd(root_module): - sh.run(("terragrunt", "validate-inputs")) - info = sh.json(("terragrunt", "terragrunt-info")) - info = json.assert_dict_of_strings(info) - return Path(info["WorkingDir"]) - else: - return root_module - - @UserError.handler def main() -> None: from os import environ @@ -177,9 +145,9 @@ def main() -> None: args = argv[1:] if args: - paths = [Path(arg) for arg in args] + paths = [OSPath(arg) for arg in args] else: - paths = [Path(".")] + paths = [OSPath(".")] from os import environ diff --git a/lib/tf_lock/tf-lock-info b/lib/tf_lock/tf-lock-info index 50dd9ddb..f931d550 100755 --- a/lib/tf_lock/tf-lock-info +++ b/lib/tf_lock/tf-lock-info @@ -1,58 +1,7 @@ #!/bin/bash -set -eEuo pipefail +set -euo pipefail HERE="$(dirname "$(readlink -f "$0")")" . "$HERE/"lib/env.sh -root_module="${1:-"$PWD"}" -working_dir="$(tf_working_dir "$root_module")" - -# at most 1 reiteration should be necessary -i=0 -limit=3 -while (( i < limit )); do - (( i += 1 )) - - - fancy_error="$( - cd "$working_dir" || exit 1 - # swap stdout and stderr: - terraform force-unlock -force -- -1 3>&2 2>&1 1>&3 <<< "" - )" && status=$? || status=$? - # strip ansi fanciness from error messages, for automated consumption - error="$( - uncolor <<< "$fancy_error" | - sed -r 's/^│ //; /^[╷╵]$/d' - )" - - if grep -Eq <<< "$error" $'ID:'; then - "$HERE/"lib/error2json <<< "$error" - exit 0 - elif grep -Eq <<< "$error" '^Failed to unlock state: LocalState not locked$' - then - echo >&2 "No remote tfstate configured, path: '$root_module'" - exit 1 - elif grep -Eq <<< "$error" $'^\t\* storage: object doesn'\''t exist$'; then - echo '{"lock": false}' - exit 0 - elif grep -Eq <<< "$error" \ - 'Error: .*(Backend initialization required, please run "terraform init"|Required plugins are not installed)' - then - ( # NB: need to undo the cd to keep relative paths valid - cd "$working_dir" - if ! noise="$(terraform init 2>&1)" || (( i == limit )); then - echo >&2 Terraform init failed! - echo >&2 "$noise" - exit 1 - fi - ) - continue - else # an unexpected error: show it - set +x - echo -n >&2 "$fancy_error" # avoid the newline appended by <<< - exit "$status" - fi - -done - -echo >&2 AssertionError: this should be impossible: "$i" -exit 99 +# FIXME: we need pip packaging +python3.12 -m lib.tf_lock.tf_lock_info "$@" diff --git a/lib/tf_lock/tf-lock-url b/lib/tf_lock/tf-lock-url new file mode 100755 index 00000000..e6f2b598 --- /dev/null +++ b/lib/tf_lock/tf-lock-url @@ -0,0 +1,7 @@ +#!/bin/bash +set -euo pipefail +HERE="$(dirname "$(readlink -f "$0")")" +. "$HERE/"lib/env.sh + +# FIXME: we need pip packaging +python3.12 -m lib.tf_lock.tf_lock_url "$@" diff --git a/lib/tf_lock/tf_lock_info.py b/lib/tf_lock/tf_lock_info.py new file mode 100755 index 00000000..95608e29 --- /dev/null +++ b/lib/tf_lock/tf_lock_info.py @@ -0,0 +1,106 @@ +#!/usr/bin/env python3 +"""Print the metadata of the terraform state lock. + +No arguments: like terraform/terragrunt this operates on $PWD. + + +$ sudo-gcp tf-lock-info | jq . +{ + "ID": "1710368348617077", + "Path": "gs://sac-dev-tf--team-sre/regions/multi-tenant/tacos-gha/de/terraform.tfstate/default.tflock", + "Operation": "OperationTypeInvalid", + "Who": "bukzor@9685.ops.getsentry.github.invalid", + "Version": "1.5.3", + "Created": "2024-03-13 22:19:08.45158468 +0000 UTC", + "Info": "", + "lock": true +} +""" + +from __future__ import annotations + +from lib import json +from lib.functions import config_lines +from lib.functions import one +from lib.sh import sh +from lib.types import ExitCode +from lib.types import OSPath +from lib.types import Path +from lib.user_error import UserError + +from .lib.env import LIB + +CACHE_PATH = Path(".config/tf-lock-info/Path") + + +def cache_get(tg_root_module: OSPath) -> str | None: + if (tg_root_module / CACHE_PATH).exists(): + with (tg_root_module / CACHE_PATH).open() as cache: + return one(config_lines(cache)) + return None + + +def cache_put(tg_root_module: OSPath, path: str) -> None: + cache = tg_root_module / CACHE_PATH + cache.parent.mkdir(parents=True, exist_ok=True) + cache.write_text(path) + + +def tf_lock_info(tg_root_module: OSPath) -> json.Object: + with sh.cd(tg_root_module): + path = cache_get(tg_root_module) + lock_info: json.Value + if path is None: + lock_info = sh.json((LIB / "tf-lock-info-uncached",)) + else: + try: + lock_info = sh.json(("gcloud", "storage", "cat", path)) + except sh.ShError: + lock_info = {} + lock_info["lock"] = False + else: + assert isinstance(lock_info, dict) + lock_info["lock"] = True + + # the ID from the lockfile is the UUID, not the actual lock ID. + # https://github.com/hashicorp/terraform/blob/main/internal/backend/remote-state/gcs/client.go#L117 + # We can pull out the lock id from the generation value. + metadata = sh.json(("gcloud", "storage", "ls", path, "--json")) + assert isinstance(metadata, list) + for metadata in metadata: + assert isinstance(metadata, dict) + metadata = metadata["metadata"] + assert isinstance(metadata, dict) + lock_info["ID"] = metadata["generation"] + + assert isinstance(lock_info, dict) + + if lock_info["lock"]: + path = lock_info["Path"] + assert isinstance(path, str) + cache_put(tg_root_module, path) + else: + if path is not None: + lock_info.setdefault("Path", path) + + return lock_info + + +@UserError.handler +def main() -> ExitCode: + from sys import argv + + with sh.quiet(): + import json + + if len(argv) >= 2 and argv[1]: + # root_module="${1:-"$PWD"}" + root_module = OSPath(argv[1]) + else: + root_module = OSPath.cwd() + print(json.dumps(tf_lock_info(root_module))) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/lib/tf_lock/tf_lock_url.py b/lib/tf_lock/tf_lock_url.py new file mode 100755 index 00000000..7f9954c8 --- /dev/null +++ b/lib/tf_lock/tf_lock_url.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python3 +"""Print the URL to the terraform state lock for the IAC. + +This will require lock-acquire permission if the lock url is not already cached. +No arguments: like terraform/terragrunt this operates on $PWD. +""" + +from __future__ import annotations + +from lib.sh import sh +from lib.types import ExitCode +from lib.types import OSPath +from lib.user_error import UserError + +from .tf_lock_info import cache_get +from .tf_lock_info import tf_lock_info + + +def tf_lock_url(tg_root_module: OSPath) -> str: + path = cache_get(tg_root_module) + if path is not None: + return path + + # cache miss! go figure out the lock url (slowly) + with sh.cd(tg_root_module): + sh.run(("tf-lock-acquire",)) + tf_lock_info(tg_root_module) + sh.run(("tf-lock-release",)) + + path = cache_get(tg_root_module) + assert isinstance(path, str) + return path + + +@UserError.handler +def main() -> ExitCode: + # with sh.quiet(): + print(tf_lock_url(OSPath.cwd())) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/lib/unix/quietly b/lib/unix/quietly index 80209134..4cac66ab 100755 --- a/lib/unix/quietly +++ b/lib/unix/quietly @@ -6,7 +6,8 @@ tmp="$(mktemp)" trap 'rm "$tmp"' EXIT exec >&2 # only logging output here -if (( "${DEBUG:-0}" >= 1 )); then +export DEBUG="${DEBUG:-}" +if (( DEBUG >= 1 )); then "$@" elif ( set -x; "$@" >"$tmp" 2>&1 ); then length=$(wc -l <"$tmp") diff --git a/pyproject.toml b/pyproject.toml index 649d6689..536b3ee9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,6 +3,8 @@ line-length = 79 target-version = ['py312'] skip-magic-trailing-comma = true preview = true +enable-unstable-feature = ['hug_parens_with_braces_and_square_brackets'] + [tool.isort] profile = "black" @@ -12,9 +14,7 @@ multi_line_output = 3 include_trailing_comma = true sections = ['FUTURE', 'STDLIB', 'THIRDPARTY', 'FIRSTPARTY', 'LOCALFOLDER'] lines_between_sections = 1 -add_imports=[ - "from __future__ import annotations", -] +add_imports = ["from __future__ import annotations"] [tool.coverage.run] plugins = ["covdefaults"] @@ -32,13 +32,13 @@ xfail_strict = true addopts = "-p lib.pytest.plugin.cap1fd -v --durations=0 --durations-min=10 --doctest-modules --last-failed" testpaths = ["."] norecursedirs = [ - "**/__pycache__", - "**/.*", - "venv", - "tmp*", - "*tmp", - "**/bak", - "**/scratch", + "**/__pycache__", + "**/.*", + "venv", + "tmp*", + "*tmp", + "**/bak", + "**/scratch", ] python_files = ["*.py"] python_classes = ["Test", "Describe"] @@ -48,22 +48,20 @@ python_functions = ["test", "it"] [tool.pyright] include = ["."] exclude = [ - "**/__pycache__", - "**/.*", - "venv", - "tmp*", - "*tmp", - "**/bak", - "**/scratch", + "**/__pycache__", + "**/.*", + "venv", + "tmp*", + "*tmp", + "**/bak", + "**/scratch", ] follow_imports_for_stubs = true pythonPlatform = "Linux" pythonVersion = "3.12" -extraPaths = [ - "venv/lib/python3.12/site-packages/" -] +extraPaths = ["venv/lib/python3.12/site-packages/"] typeCheckingMode = "strict" @@ -78,7 +76,7 @@ reportMissingSuperCall = "error" reportPropertyTypeMismatch = "error" reportUninitializedInstanceVariable = "error" reportUnnecessaryTypeIgnoreComment = "error" -reportUnusedCallResult = "none" # too noisy +reportUnusedCallResult = "none" # too noisy # maintainer has an strong anti-idomatic stance on what "constant" means # https://github.com/microsoft/pyright/issues/5265 @@ -89,11 +87,7 @@ reportConstantRedefinition = false python_version = "3.12" files = ["."] -exclude = [ - "(^|/)venv/$", - "(^|/)bak/$", - "(^|/)scratch/$", -] +exclude = ["(^|/)venv/$", "(^|/)bak/$", "(^|/)scratch/$"] scripts_are_modules = true # Strict mode; enables the following flags: (in mypy 1.7.1) diff --git a/requirements-dev.txt b/requirements-dev.txt index 4ac245a5..76fdbf90 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -4,11 +4,11 @@ # # pip-compile --strip-extras requirements-dev.in # -astroid==3.1.0 +astroid==3.2.2 # via pylint -black==24.1.1 +black==24.4.2 # via -r requirements-dev.in -build==1.1.1 +build==1.2.1 # via pip-tools cffi==1.16.0 # via cryptography @@ -20,24 +20,24 @@ click==8.1.7 # pip-tools covdefaults==2.3.0 # via -r requirements-dev.in -coverage==7.4.3 +coverage==7.5.1 # via # -r requirements-dev.in # covdefaults # coverage-enable-subprocess coverage-enable-subprocess==1.0 # via -r requirements-dev.in -cryptography==42.0.5 +cryptography==42.0.7 # via pyjwt dill==0.3.8 # via pylint distlib==0.3.8 # via virtualenv -execnet==2.0.2 +execnet==2.1.1 # via pytest-xdist -filelock==3.13.1 +filelock==3.14.0 # via virtualenv -identify==2.5.35 +identify==2.5.36 # via pre-commit iniconfig==2.0.0 # via pytest @@ -47,7 +47,7 @@ isort==5.13.2 # pylint mccabe==0.7.0 # via pylint -mypy==1.8.0 +mypy==1.10.0 # via -r requirements-dev.in mypy-extensions==1.0.0 # via @@ -57,7 +57,7 @@ nodeenv==1.8.0 # via # pre-commit # pyright -packaging==23.2 +packaging==24.0 # via # black # build @@ -66,42 +66,42 @@ pathspec==0.12.1 # via black pip-tools==7.4.1 # via -r requirements-dev.in -platformdirs==4.2.0 +platformdirs==4.2.2 # via # black # pylint # virtualenv -pluggy==1.4.0 +pluggy==1.5.0 # via pytest -pre-commit==3.6.2 +pre-commit==3.7.1 # via -r requirements-dev.in -pycparser==2.21 +pycparser==2.22 # via cffi pyjwt==2.8.0 # via -r requirements-dev.in -pylint==3.1.0 +pylint==3.2.2 # via -r requirements-dev.in -pyproject-hooks==1.0.0 +pyproject-hooks==1.1.0 # via # build # pip-tools -pyright==1.1.352 +pyright==1.1.364 # via -r requirements-dev.in -pytest==8.0.2 +pytest==8.2.1 # via # -r requirements-dev.in # pytest-xdist -pytest-xdist==3.5.0 +pytest-xdist==3.6.1 # via -r requirements-dev.in pyyaml==6.0.1 # via pre-commit -tomlkit==0.12.4 +tomlkit==0.12.5 # via pylint -typing-extensions==4.10.0 +typing-extensions==4.11.0 # via mypy -virtualenv==20.25.1 +virtualenv==20.26.2 # via pre-commit -wheel==0.42.0 +wheel==0.43.0 # via pip-tools # The following packages are considered to be unsafe in a requirements file: diff --git a/spec/lib/gh/check_run.py b/spec/lib/gh/check_run.py index c10f611f..0d9101b3 100644 --- a/spec/lib/gh/check_run.py +++ b/spec/lib/gh/check_run.py @@ -99,9 +99,14 @@ def relevance(self) -> tuple[object, ...]: try: result.append( - ("NEUTRAL", "SUCCESS", "CANCELLED", "", "FAILURE").index( - self.conclusion - ) + ( + "SKIPPED", + "NEUTRAL", + "SUCCESS", + "CANCELLED", + "", + "FAILURE", + ).index(self.conclusion) ) except ValueError as error: raise AssertionError( @@ -132,13 +137,15 @@ def job(self) -> str: def get_runs_json(pr_url: URL) -> Generator[json.Value]: """Get the json of all runs, for the named check.""" # https://docs.github.com/en/graphql/reference/objects#statuscheckrollup - return sh.jq(( - "gh", - "pr", - "view", - pr_url, - "--json", - "statusCheckRollup", - "--jq", - ".statusCheckRollup[]", - )) + return sh.jq( + ( + "gh", + "pr", + "view", + pr_url, + "--json", + "statusCheckRollup", + "--jq", + ".statusCheckRollup[]", + ) + ) diff --git a/spec/lib/gh/pr.py b/spec/lib/gh/pr.py index 2a0594d6..9885a496 100644 --- a/spec/lib/gh/pr.py +++ b/spec/lib/gh/pr.py @@ -23,11 +23,6 @@ from .types import WorkflowName from .up_to_date import up_to_date -APP_INSTALLATION_REVIEWER = ( - "op://Team Tacos gha dev/tacos-gha-reviewer/installation.json" -) - - Comment = str # a PR comment if TYPE_CHECKING: