diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 878a93c..1267aeb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -29,12 +29,6 @@ jobs: fetch-depth: 0 persist-credentials: false - - name: Guard src-py-lib subtree - if: github.event_name == 'pull_request' - env: - ALLOW_SRC_PY_LIB_SUBTREE_CHANGE: ${{ contains(github.event.pull_request.labels.*.name, 'src-py-lib subtree') && '1' || '' }} - run: ./dev/check-src-py-lib-subtree.sh --branch '${{ github.event.pull_request.base.sha }}' - - name: Set up Python uses: actions/setup-python@v6 with: @@ -50,10 +44,10 @@ jobs: run: uv lock --check - name: Lint - run: uv run --frozen ruff check src_auth_perms_sync/ tests/ + run: uv run --frozen ruff check . - name: Check formatting - run: uv run --frozen ruff format --check src_auth_perms_sync/ tests/ + run: uv run --frozen ruff format --check . - name: Type check run: uv run --frozen pyright @@ -65,16 +59,14 @@ jobs: run: uv run --frozen src-auth-perms-sync --help >/tmp/src-auth-perms-sync-help.txt - name: Build wheel - run: | - uv build --wheel git-subtree/src-py-lib --out-dir dist --no-create-gitignore - uv build --wheel --out-dir dist --no-create-gitignore + run: uv build --wheel --out-dir dist --no-create-gitignore - name: Smoke test installed wheel run: | python -m venv build/ci-venv . build/ci-venv/bin/activate python -m pip install --upgrade pip - python -m pip install dist/src_py_lib-*.whl dist/src_auth_perms_sync-*.whl + python -m pip install dist/src_auth_perms_sync-*.whl src-auth-perms-sync --help >/tmp/src-auth-perms-sync-installed-help.txt python -m src_auth_perms_sync --help >/tmp/src-auth-perms-sync-module-help.txt diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index be7584e..9bac6c4 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -80,6 +80,14 @@ jobs: echo "::error title=Missing tag::Tag '${release_tag}' was not fetched. Create and push it before running this workflow." exit 1 fi + tag_revision="$(git rev-list -n 1 "${release_tag}")" + git fetch --no-tags origin main + main_revision="$(git rev-parse origin/main)" + if ! git merge-base --is-ancestor "${tag_revision}" "${main_revision}"; then + echo "::error title=Tag is not on main::Tag '${release_tag}' points at ${tag_revision}, which is not reachable from origin/main." + echo "::error::Merge the release PR first, then tag the main commit." + exit 1 + fi project_version=$(uv run --frozen python - <<'PY' import tomllib @@ -109,8 +117,8 @@ jobs: fi uv lock --check - uv run --frozen ruff check src_auth_perms_sync/ - uv run --frozen ruff format --check src_auth_perms_sync/ + uv run --frozen ruff check src/src_auth_perms_sync/ + uv run --frozen ruff format --check src/src_auth_perms_sync/ uv run --frozen pyright uv run --frozen src-auth-perms-sync --help >/tmp/src-auth-perms-sync-help.txt @@ -129,7 +137,6 @@ jobs: rm -rf build/release mkdir -p "${wheelhouse_dir}" "${dist_dir}" - uv build --wheel git-subtree/src-py-lib --out-dir "${wheelhouse_dir}" --no-create-gitignore uv build --wheel --out-dir "${dist_dir}" --no-create-gitignore project_wheels=("${dist_dir}"/*.whl) if [[ "${#project_wheels[@]}" -ne 1 ]]; then @@ -152,7 +159,11 @@ jobs: --frozen \ --output-file "${requirements_file}" - grep -v '^\./git-subtree/src-py-lib$' "${requirements_file}" > "${runtime_requirements_file}" + cp "${requirements_file}" "${runtime_requirements_file}" + if grep -q '^\./' "${runtime_requirements_file}"; then + echo "::error title=Unexpected local dependency::Runtime requirements must resolve from PyPI." + exit 1 + fi python -m pip wheel \ --only-binary=:all: \ @@ -165,8 +176,6 @@ jobs: echo "::error title=Unexpected src-py-lib wheel count::Expected one src-py-lib wheel, found ${#src_py_lib_wheels[@]}." exit 1 fi - src_py_lib_wheel_path="${src_py_lib_wheels[0]}" - src_py_lib_wheel_name="$(basename "${src_py_lib_wheel_path}")" cat > "${wheelhouse_dir}/INSTALL.txt" < WHEELS.sha256) @@ -210,8 +221,6 @@ jobs: echo "checksum_path=${checksum_path}" >> "${GITHUB_OUTPUT}" echo "project_wheel_path=${project_wheel_path}" >> "${GITHUB_OUTPUT}" echo "project_wheel_name=${project_wheel_name}" >> "${GITHUB_OUTPUT}" - echo "src_py_lib_wheel_path=${src_py_lib_wheel_path}" >> "${GITHUB_OUTPUT}" - echo "src_py_lib_wheel_name=${src_py_lib_wheel_name}" >> "${GITHUB_OUTPUT}" - name: Validate offline install from tarball run: | @@ -235,7 +244,6 @@ jobs: run: | release_tag="${{ steps.release.outputs.tag }}" project_wheel_name="${{ steps.build.outputs.project_wheel_name }}" - src_py_lib_wheel_name="${{ steps.build.outputs.src_py_lib_wheel_name }}" notes_path="build/release/release-notes.md" cat > "${notes_path}" <> "${GITHUB_OUTPUT}" @@ -285,9 +297,15 @@ jobs: ${{ steps.build.outputs.asset_path }} ${{ steps.build.outputs.checksum_path }} ${{ steps.build.outputs.project_wheel_path }} - ${{ steps.build.outputs.src_py_lib_wheel_path }} ${{ steps.notes.outputs.path }} + - name: Upload PyPI artifact + if: matrix.platform == 'linux-x86_64' + uses: actions/upload-artifact@v7 + with: + name: pypi-distributions + path: ${{ steps.build.outputs.project_wheel_path }} + - name: Publish GitHub release assets env: GH_TOKEN: ${{ github.token }} @@ -296,19 +314,43 @@ jobs: asset_path="${{ steps.build.outputs.asset_path }}" checksum_path="${{ steps.build.outputs.checksum_path }}" project_wheel_path="${{ steps.build.outputs.project_wheel_path }}" - src_py_lib_wheel_path="${{ steps.build.outputs.src_py_lib_wheel_path }}" notes_path="${{ steps.notes.outputs.path }}" + release_assets=("${asset_path}" "${checksum_path}") + + if [[ "${{ matrix.platform }}" == "linux-x86_64" ]]; then + release_assets+=("${project_wheel_path}") + fi if gh release view "${release_tag}" >/dev/null 2>&1; then gh release edit "${release_tag}" --title "${release_tag}" --notes-file "${notes_path}" - gh release upload "${release_tag}" "${asset_path}" "${checksum_path}" "${project_wheel_path}" "${src_py_lib_wheel_path}" --clobber + gh release upload "${release_tag}" "${release_assets[@]}" --clobber else gh release create "${release_tag}" \ - "${asset_path}" \ - "${checksum_path}" \ - "${project_wheel_path}" \ - "${src_py_lib_wheel_path}" \ + "${release_assets[@]}" \ --title "${release_tag}" \ --notes-file "${notes_path}" \ --verify-tag fi + + pypi: + name: Publish PyPI package + needs: wheelhouse + runs-on: ubuntu-24.04 + permissions: + contents: read + id-token: write + environment: + name: pypi + url: https://pypi.org/project/src-auth-perms-sync/ + + steps: + - name: Download built distribution + uses: actions/download-artifact@v7 + with: + name: pypi-distributions + path: dist + + - name: Publish PyPI package + uses: pypa/gh-action-pypi-publish@release/v1 + with: + packages-dir: dist diff --git a/.gitignore b/.gitignore index e0c9f06..60d5349 100644 --- a/.gitignore +++ b/.gitignore @@ -1,18 +1,23 @@ # Ignore -__pycache__/ +__pycache__ +.DS_Store +.playwright-mcp +.pypirc +.pyright +.pytest_cache .ruff_cache .venv -*.egg-info -*.env +*.env* *.gql +*.py[cod] *.py[oc] *.yaml -src-auth-perms-sync-runs/ build/ dist/ +src-auth-perms-sync-runs/ wheels/ # Allow !.env.example -!git-subtree/** +!.markdownlint-cli2.yaml !maps-example.yaml diff --git a/git-subtree/src-py-lib/.markdownlint-cli2.yaml b/.markdownlint-cli2.yaml similarity index 83% rename from git-subtree/src-py-lib/.markdownlint-cli2.yaml rename to .markdownlint-cli2.yaml index 578c97d..92ff342 100644 --- a/git-subtree/src-py-lib/.markdownlint-cli2.yaml +++ b/.markdownlint-cli2.yaml @@ -4,7 +4,6 @@ ignores: - ".venv/**" - "build/**" - "dist/**" - - "node_modules/**" config: MD013: line_length: 100 diff --git a/.markdownlint.jsonc b/.markdownlint.jsonc deleted file mode 100644 index a99bd1d..0000000 --- a/.markdownlint.jsonc +++ /dev/null @@ -1,11 +0,0 @@ -{ - // Relax line length inside code blocks and tables, which routinely contain - // long bash commands, GraphQL queries, or column-aligned examples that hurt - // when wrapped. Prose stays at 100 to match the Python line-length in - // pyproject.toml. - "MD013": { - "line_length": 100, - "code_blocks": false, - "tables": false - } -} diff --git a/AGENTS.md b/AGENTS.md index f64b8c4..5e41ee9 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,24 +1,18 @@ # AGENTS.md -## git subtrees - -- Do not edit files under git-subtree/, within this repo -- Find their original repo, make the change there, then update the subtree ref - in this repo - ## Linting -```sh +```bash ### Markdown files -npx --no-install markdownlint-cli2 *.md +npx --yes markdownlint-cli2 ### Python files # Lint + auto-fix safe issues -uv run ruff check src_auth_perms_sync/ --fix +uv run ruff check src/src_auth_perms_sync/ --fix # Format -uv run ruff format src_auth_perms_sync/ +uv run ruff format src/src_auth_perms_sync/ # Type check uv run pyright @@ -31,7 +25,7 @@ uv run src-auth-perms-sync --help - First run a dry-run (default behaviour, without `--apply` flag) against a Sourcegraph instance -```sh +```bash uv run src-auth-perms-sync [--get] uv run src-auth-perms-sync --set maps.yaml --full uv run src-auth-perms-sync --restore backups///before.json @@ -45,6 +39,152 @@ uv run src-auth-perms-sync --restore backups///before.json - Always inspect the before / after snapshots in `src-auth-perms-sync-runs//backups/` afterward to confirm the diff matches what you expected +## Release process + +- The tagged source commit must already contain the package version it + releases. Do not make the customer release workflow edit `pyproject.toml`. +- Prepare the version bump on a branch. Set `VERSION`, then copy / paste: + +```bash +set -euo pipefail + +VERSION=0.2.1 +BRANCH="release-v${VERSION}" + +git fetch origin --tags --prune +git switch main +git pull --ff-only +git switch -c "${BRANCH}" + +uv run python - "${VERSION}" <<'PY' +from pathlib import Path +import re +import sys + +version = sys.argv[1] +path = Path("pyproject.toml") +text = path.read_text() +new_text = re.sub( + r'(?m)^version = "[^"]+"$', + f'version = "{version}"', + text, + count=1, +) +if new_text == text: + raise SystemExit("pyproject.toml version was not updated") +path.write_text(new_text) +PY + +uv lock +``` + +- Validate the release candidate before opening / merging the PR: + +```bash +set -euo pipefail + +uv lock --check +uv run ruff check src/src_auth_perms_sync/ tests/ +uv run ruff format --check src/src_auth_perms_sync/ tests/ +uv run pyright +uv run python -m unittest discover -s tests +uv run src-auth-perms-sync --help +npx --yes markdownlint-cli2 +uv build --wheel --out-dir /tmp/src-auth-perms-sync-release-check --no-create-gitignore +rm -rf /tmp/src-auth-perms-sync-release-check +``` + +- Commit, push, open the PR, wait for checks, then merge it. If review is + required, stop after `gh pr checks` and ask for review before merging. + +```bash +set -euo pipefail + +VERSION=0.2.1 +BRANCH="release-v${VERSION}" +GH_REPO="sourcegraph/src-auth-perms-sync" + +git add pyproject.toml uv.lock +git commit -m "Release v${VERSION}" +git push -u origin "${BRANCH}" + +gh pr create \ + --repo "${GH_REPO}" \ + --base main \ + --head "${BRANCH}" \ + --title "Release v${VERSION}" \ + --body "Bump src-auth-perms-sync package metadata to ${VERSION}." + +gh pr checks "${BRANCH}" --repo "${GH_REPO}" --watch --fail-fast +gh pr merge "${BRANCH}" --repo "${GH_REPO}" --squash --delete-branch +``` + +- Tag the merged `main` commit. Do not tag a feature branch commit. + +```bash +set -euo pipefail + +VERSION=0.2.1 + +git fetch origin --tags --prune +git switch main +git pull --ff-only +git tag "v${VERSION}" +git push origin "v${VERSION}" +``` + +- Watch the customer release workflow and confirm the GitHub release assets + are uploaded: + +```bash +set -euo pipefail + +VERSION=0.2.1 +GH_REPO="sourcegraph/src-auth-perms-sync" + +RUN_ID="$( + gh run list \ + --repo "${GH_REPO}" \ + --workflow release.yml \ + --branch "v${VERSION}" \ + --limit 1 \ + --json databaseId \ + --jq '.[0].databaseId // empty' +)" +test -n "${RUN_ID}" +gh run watch "${RUN_ID}" --repo "${GH_REPO}" --exit-status +gh release view "v${VERSION}" --repo "${GH_REPO}" +``` + +- If a pushed tag points at the wrong commit, move it only after explicit + human approval: + +```bash +set -euo pipefail + +VERSION=0.2.1 +GH_REPO="sourcegraph/src-auth-perms-sync" + +git fetch origin --tags --prune +git switch main +git pull --ff-only +git tag -f "v${VERSION}" origin/main +git push origin "refs/tags/v${VERSION}" --force + +RUN_ID="$( + gh run list \ + --repo "${GH_REPO}" \ + --workflow release.yml \ + --branch "v${VERSION}" \ + --limit 1 \ + --json databaseId \ + --jq '.[0].databaseId // empty' +)" +test -n "${RUN_ID}" +gh run watch "${RUN_ID}" --repo "${GH_REPO}" --exit-status +gh release view "v${VERSION}" --repo "${GH_REPO}" +``` + ## Hard invariants — do not break Violating these can silently grant the wrong users access to the wrong @@ -83,7 +223,7 @@ organization sync maps SAML groups to Sourcegraph org membership. Read ## Layout -CLI lives in `src_auth_perms_sync/`; invoke with `uv run src-auth-perms-sync`. +CLI lives in `src/src_auth_perms_sync/`; invoke with `uv run src-auth-perms-sync`. Strict pyright covers the package. Root modules are entrypoints only: - `cli.py` — `main()`, arg parsing, owns the CLI description. diff --git a/README.md b/README.md index 6d9d0fc..4b2c697 100644 --- a/README.md +++ b/README.md @@ -230,24 +230,4 @@ src-auth-perms-sync-runs/endpoint/ - An `after.json` file, capturing the new state - A `diff.json` file, a shorter, reviewable file containing the diffs between before and after -## Git subtree: `src-py-lib` - -- This repo includes [src-py-lib](https://github.com/sourcegraph/src-py-lib) as a Git subtree - under `git-subtree/src-py-lib` -- Do not edit files under `git-subtree/` directly -- Make changes in the upstream repo first, merge them there, then update this repo's subtree: - - ```bash - git remote add src-py-lib https://github.com/sourcegraph/src-py-lib.git 2>/dev/null || true - git subtree pull \ - --prefix git-subtree/src-py-lib \ - --squash \ - -S \ - src-py-lib \ - main - uv lock - ``` - -- The `-S` flag signs the subtree update commits with your configured Git signing key - diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..9108946 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,3 @@ +# Security Policy + +Please find our security policy at diff --git a/dev/TODO.md b/dev/TODO.md index 5d77321..a6ec344 100644 --- a/dev/TODO.md +++ b/dev/TODO.md @@ -69,8 +69,13 @@ If/when we revisit: `allowGroups`-style enforcement exists on more than just SAML, but only SAML actually persists the group list. Recovery options for each: -| Provider | `allowGroups`-equivalent config | Stored in `account_data`? | Recovery path | -| --- | --- | --- | --- | -| OIDC | None — no `allowGroups` field on `OpenIDConnectAuthProvider` | No — `UserClaims` only stores name/email fields; `groups` claim is never parsed | Upstream change to persist the claim | -| GitHub OAuth | `allowOrgs`, `allowOrgsMap` (org→teams), `requiredSsoOrgs` | No — orgs/teams checked live in `verifyUserOrgs`/`verifyUserTeams` and discarded | Upstream change to persist the claim | -| GitLab OAuth | `allowGroups` | No — `verifyUserGroups` calls `glClient.IsGroupMember` live and discards | Upstream change to persist the claim | +- OIDC has no `allowGroups` field on `OpenIDConnectAuthProvider`. + `UserClaims` stores only name/email fields; the `groups` claim is never + parsed. Recovery needs an upstream change to persist the claim. +- GitHub OAuth has `allowOrgs`, `allowOrgsMap` (org→teams), and + `requiredSsoOrgs`. Org/team checks happen live in `verifyUserOrgs` / + `verifyUserTeams` and are discarded. Recovery needs an upstream change to + persist the claim. +- GitLab OAuth has `allowGroups`, but `verifyUserGroups` calls + `glClient.IsGroupMember` live and discards the result. Recovery needs an + upstream change to persist the claim. diff --git a/dev/check-src-py-lib-subtree.sh b/dev/check-src-py-lib-subtree.sh deleted file mode 100755 index 1ea2511..0000000 --- a/dev/check-src-py-lib-subtree.sh +++ /dev/null @@ -1,81 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -subtree_path="git-subtree/src-py-lib" -allow_environment_variable="ALLOW_SRC_PY_LIB_SUBTREE_CHANGE" - -usage() { - cat <&2 -Do not edit ${subtree_path} directly in src-auth-perms-sync. - -Make changes in sourcegraph/src-py-lib first, merge them upstream, then update this repo with: - - git subtree pull --prefix ${subtree_path} src-py-lib main --squash - -For an intentional subtree import/update, rerun with ${allow_environment_variable}=1. - -Changed files: -EOF - printf '%s\n' "$@" >&2 - exit 1 -} - -case "$mode" in - --staged) - if git diff --cached --quiet -- "$subtree_path"; then - exit 0 - fi - - changed_files="$(git diff --cached --name-only -- "$subtree_path")" - reject_change "$changed_files" - ;; - - --branch) - base_revision="${2:-origin/main}" - if ! git rev-parse --verify --quiet "${base_revision}^{commit}" >/dev/null; then - cat <&2 -Could not find base revision '${base_revision}' for subtree guard. -Fetch the base branch or pass an explicit base revision. -EOF - exit 1 - fi - - merge_base_revision="$(git merge-base "$base_revision" HEAD)" - if git diff --quiet "${merge_base_revision}...HEAD" -- "$subtree_path"; then - exit 0 - fi - - if git log --format=%B "${merge_base_revision}..HEAD" \ - | grep -Fx "git-subtree-dir: ${subtree_path}" >/dev/null; then - exit 0 - fi - - changed_files="$(git diff --name-only "${merge_base_revision}...HEAD" -- "$subtree_path")" - reject_change "$changed_files" - ;; - - *) - usage >&2 - exit 2 - ;; -esac diff --git a/dev/dead-code-audit.md b/dev/dead-code-audit.md index 3f91bdc..031b417 100644 --- a/dev/dead-code-audit.md +++ b/dev/dead-code-audit.md @@ -50,7 +50,7 @@ possibly test-only". or only self/test references, is a signal to inspect manually. 4. Use the function call graph, if exposed by the active Sourcegraph tools: - Traverse callees from the root set. - - Compare reachable functions to all functions under `src_auth_perms_sync/`. + - Compare reachable functions to all functions under `src/src_auth_perms_sync/`. - Treat unresolved dynamic calls as manual-review barriers, not as proof of dead code. 5. If call-graph traversal is not exposed through MCP, use Deep Search plus @@ -64,21 +64,21 @@ dependency unless we decide to maintain a repeatable CI/local audit. Run the strict pass first: ```sh -uv run --with vulture vulture --min-confidence 80 --sort-by-size src_auth_perms_sync tests +uv run --with vulture vulture --min-confidence 80 --sort-by-size src/src_auth_perms_sync tests ``` Then run the full advisory passes: ```sh -uv run --with vulture vulture --sort-by-size src_auth_perms_sync tests -uv run --with vulture vulture --sort-by-size src_auth_perms_sync +uv run --with vulture vulture --sort-by-size src/src_auth_perms_sync tests +uv run --with vulture vulture --sort-by-size src/src_auth_perms_sync uv run --with vulture vulture --sort-by-size tests ``` Interpret the results this way: -- Findings in `src_auth_perms_sync tests` are stronger "unused anywhere" candidates. -- Findings only in `src_auth_perms_sync` may be test-only helpers or symmetric APIs. +- Findings in `src/src_auth_perms_sync tests` are stronger "unused anywhere" candidates. +- Findings only in `src/src_auth_perms_sync` may be test-only helpers or symmetric APIs. - Findings only in `tests` may be stale fixtures or test helpers. - `TypedDict` fields and GraphQL/JSON wire keys are usually false positives. - Low-confidence findings are still useful, but require exact reference checks. @@ -87,7 +87,7 @@ For each real-looking Vulture finding, run an exact local search and, when available, a Sourcegraph reference search before deleting: ```sh -rg -n "symbol_name" src_auth_perms_sync tests +rg -n "symbol_name" src/src_auth_perms_sync tests ``` ## Complexity workflow @@ -99,23 +99,25 @@ Run them transiently with `uv run --with` instead of adding project dependencies Start with the file under suspicion: ```sh -uv run --with radon radon cc -s -a src_auth_perms_sync/cli.py -uv run --with radon radon mi -s src_auth_perms_sync/cli.py -uv run --with radon radon raw -s src_auth_perms_sync/cli.py -uv run --with lizard lizard src_auth_perms_sync/cli.py +uv run --with radon radon cc -s -a src/src_auth_perms_sync/cli.py +uv run --with radon radon mi -s src/src_auth_perms_sync/cli.py +uv run --with radon radon raw -s src/src_auth_perms_sync/cli.py +uv run --with lizard lizard src/src_auth_perms_sync/cli.py ``` Then scan the package for bigger hotspots: ```sh -uv run --with radon radon cc -s --min C src_auth_perms_sync -uv run --with lizard lizard -C 15 -L 80 src_auth_perms_sync +uv run --with radon radon cc -s --min C src/src_auth_perms_sync +uv run --with lizard lizard -C 15 -L 80 src/src_auth_perms_sync ``` Optional Ruff check for functions over a chosen cyclomatic-complexity threshold: ```sh -uv run ruff check src_auth_perms_sync/cli.py --select C901 --config 'lint.mccabe.max-complexity = 10' +uv run ruff check src/src_auth_perms_sync/cli.py \ + --select C901 \ + --config 'lint.mccabe.max-complexity = 10' ``` Interpret the results this way: @@ -169,8 +171,8 @@ Run the narrowest checks that cover the deleted code. For normal Python dead-cod deletions, run: ```sh -uv run ruff check src_auth_perms_sync/ --fix -uv run ruff format src_auth_perms_sync/ +uv run ruff check src/src_auth_perms_sync/ --fix +uv run ruff format src/src_auth_perms_sync/ uv run pyright uv run python -m unittest discover -s tests uv run src-auth-perms-sync --help diff --git a/dev/git-worktrees.md b/dev/git-worktrees.md index 5e4f387..6f83ed5 100644 --- a/dev/git-worktrees.md +++ b/dev/git-worktrees.md @@ -64,7 +64,7 @@ Good parallelism: Risky parallelism: -- Two agents refactor `src_auth_perms_sync/cli.py` at the same time. +- Two agents refactor `src/src_auth_perms_sync/cli.py` at the same time. - One branch renames functions while another branch edits their call sites. - Two branches change the same GraphQL mutation flow. diff --git a/dev/hooks/pre-commit b/dev/hooks/pre-commit index 8e4887a..0518138 100755 --- a/dev/hooks/pre-commit +++ b/dev/hooks/pre-commit @@ -1,6 +1,8 @@ #!/usr/bin/env bash set -euo pipefail +export PATH="$HOME/.local/bin:/opt/homebrew/bin:/usr/local/bin:$PATH" + repo_root="$(git rev-parse --show-toplevel)" cd "$repo_root" @@ -12,14 +14,13 @@ run() { printf 'Checking repository before commit in %s\n' "$repo_root" run git status --short -run git diff --check -- . ":(exclude)git-subtree/src-py-lib" -run git diff --cached --check -- . ":(exclude)git-subtree/src-py-lib" +run git diff --check +run git diff --cached --check run git diff --cached --stat run git diff --stat -run dev/check-src-py-lib-subtree.sh --staged -run uv run ruff check src_auth_perms_sync/ -run uv run ruff format src_auth_perms_sync/ --check +run uv run ruff check . +run uv run ruff format . --check run uv run pyright run uv run python -m unittest discover -s tests run uv run src-auth-perms-sync --help diff --git a/dev/python-versions.md b/dev/python-versions.md index 61a4435..fe5f2ed 100644 --- a/dev/python-versions.md +++ b/dev/python-versions.md @@ -12,10 +12,10 @@ Keep these in sync when changing the baseline: - GitHub Actions CI / release `PYTHON_VERSION` - Customer release wheelhouse labels and install instructions - `README.md` install commands -- `src-py-lib`'s matching Python support and pinned commit in `pyproject.toml` +- `src-py-lib`'s matching Python support and published version in `pyproject.toml` -Do not change `requires-python` until both this repo and `src-py-lib` are tested -with the target version and matching customer wheelhouses can be built. +Do not change `requires-python` until this repo and its PyPI dependencies are +tested with the target version and matching customer wheelhouses can be built. ## Renovate notes diff --git a/dev/test-command-permutations.py b/dev/test-command-permutations.py index 8f07140..b731891 100755 --- a/dev/test-command-permutations.py +++ b/dev/test-command-permutations.py @@ -5,9 +5,10 @@ uses the same CLI entrypoint an operator uses (`uv run src-auth-perms-sync`) and checks both process exit codes and structured `run` log records. -The script runs every case: read-only, dry-run, invalid-argument, no-op apply, -mutating apply, and full overwrite/restore. Mutating cases are refused outside -test-looking endpoints unless explicitly allowed. +The script covers every major command path: read-only, dry-run, +invalid-argument, no-op apply, mutating apply, and overwrite/restore. It avoids +running the same expensive full-snapshot path more than once when another case +already covers that behavior. """ from __future__ import annotations @@ -204,6 +205,7 @@ def __init__( *, iteration: int, keep_going: bool, + trace: bool, sample_interval: float, external_sample_interval: float, ) -> None: @@ -211,6 +213,7 @@ def __init__( self.environment = environment self.iteration = iteration self.keep_going = keep_going + self.trace = trace self.sample_interval = sample_interval self.external_sample_interval = external_sample_interval self.results: list[CommandResult] = [] @@ -235,6 +238,7 @@ def _run_process(self, case: CommandCase) -> CommandResult: full_command = [ *self.variant.executable, *case.arguments, + *(("--trace",) if self.trace else ()), "--sample-interval", str(self.sample_interval), ] @@ -385,6 +389,7 @@ def main() -> None: environment, iteration=iteration, keep_going=arguments.keep_going, + trace=arguments.trace, sample_interval=arguments.sample_interval, external_sample_interval=arguments.external_sample_interval, ) @@ -494,6 +499,11 @@ def parse_arguments() -> argparse.Namespace: action="store_true", help="Continue after assertion failures where it is safe to do so", ) + parser.add_argument( + "--trace", + action="store_true", + help="Pass --trace to each child src-auth-perms-sync command", + ) parser.add_argument( "--sample-interval", type=float, @@ -666,7 +676,6 @@ def run_matrix( ) runner.run(users_without_explicit_permissions_no_op_case(arguments)) - runner.run(sync_saml_dry_run_case()) runner.run(sync_saml_apply_case()) return baseline_repositories @@ -674,7 +683,7 @@ def run_matrix( def invalid_configuration_cases(arguments: argparse.Namespace) -> list[CommandCase]: restore_placeholder = "definitely-missing-before.json" missing_maps = "definitely-missing-command-permutation-maps.yaml" - command_pairs = [ + command_pairs: list[tuple[str, tuple[str, ...]]] = [ ("get-set", ("--get", "--set", "maps.yaml")), ("get-restore", ("--get", "--restore", restore_placeholder)), ("set-restore", ("--set", "maps.yaml", "--restore", restore_placeholder)), @@ -816,12 +825,6 @@ def read_only_cases(arguments: argparse.Namespace) -> list[CommandCase]: expected_log_command="get", must_contain=("Selected 0 user(s) for get output",), ), - CommandCase( - name="explicit-get-all-users", - arguments=("--get",), - expected_log_command="get", - must_contain=("Wrote before-snapshot",), - ), CommandCase( name="get-sync-saml-orgs-dry-run", arguments=("--get", "--sync-saml-orgs"), @@ -833,22 +836,6 @@ def read_only_cases(arguments: argparse.Namespace) -> list[CommandCase]: def run_safe_set_cases(arguments: argparse.Namespace, runner: CommandPermutationRunner) -> None: - runner.run( - CommandCase( - name="set-default-full-no-op-apply", - arguments=( - "--set", - "--created-after", - arguments.future_date, - "--apply", - "--no-backup", - "--parallelism", - str(arguments.parallelism), - ), - expected_log_command="set_full", - must_contain=("No repos resolved across any mapping",), - ) - ) runner.run( CommandCase( name="set-explicit-full-no-op-apply", @@ -949,15 +936,6 @@ def restore_scoped_apply_case(snapshot: Path, arguments: argparse.Namespace) -> ) -def sync_saml_dry_run_case() -> CommandCase: - return CommandCase( - name="sync-saml-orgs-dry-run", - arguments=("--sync-saml-orgs",), - expected_log_command="sync_saml_orgs", - must_contain=("Dry run complete",), - ) - - def sync_saml_apply_case() -> CommandCase: return CommandCase( name="sync-saml-orgs-apply", @@ -1001,7 +979,6 @@ def run_full_apply_cases(arguments: argparse.Namespace, runner: CommandPermutati must_contain=("VALIDATION OK",), ) ) - runner.run(restore_full_dry_run_case("restore-full-dry-run", baseline_snapshot, arguments)) finally: runner.run( restore_full_apply_case( @@ -1037,6 +1014,9 @@ def run_full_apply_cases(arguments: argparse.Namespace, runner: CommandPermutati ) ) + # Covers the combined set+SAML dispatch and SAML dry-run path without + # repeating the full set apply and full restore cleanup paths, which are + # already covered above. runner.run( CommandCase( name="set-full-sync-saml-orgs-dry-run", @@ -1045,46 +1025,6 @@ def run_full_apply_cases(arguments: argparse.Namespace, runner: CommandPermutati must_contain=("Dry run complete",), ) ) - try: - runner.run( - CommandCase( - name="set-full-sync-saml-orgs-apply", - arguments=( - "--set", - "--sync-saml-orgs", - "--apply", - "--parallelism", - str(arguments.parallelism), - ), - expected_log_command="set_full_sync_saml_orgs", - must_contain=("VALIDATION OK",), - ) - ) - finally: - runner.run( - restore_full_apply_case( - "restore-full-after-sync-cleanup", - baseline_snapshot, - arguments, - no_backup=False, - ) - ) - - -def restore_full_dry_run_case( - name: str, snapshot: Path, arguments: argparse.Namespace -) -> CommandCase: - return CommandCase( - name=name, - arguments=( - "--restore", - str(snapshot), - "--parallelism", - str(arguments.full_restore_parallelism), - ), - expected_log_command="restore", - must_contain_one_of=("Dry run complete", "Nothing to restore"), - ) def restore_full_apply_case( @@ -1416,7 +1356,7 @@ def print_memory_summary(results: list[CommandResult], limit: int) -> None: if not rows: print("\nMemory summary: no structured peak_rss_mb records found.") return - rows.sort(key=lambda result: result.memory.peak_rss_mb if result.memory else 0.0, reverse=True) + rows.sort(key=lambda result: result_peak_rss_mb(result) or 0.0, reverse=True) print("\nMemory summary (highest peak RSS first):") print( "variant,iteration,case,peak_rss_mib,sampled_peak_rss_mib," diff --git a/git-subtree/src-py-lib/.github/workflows/ci.yml b/git-subtree/src-py-lib/.github/workflows/ci.yml deleted file mode 100644 index 98eb145..0000000 --- a/git-subtree/src-py-lib/.github/workflows/ci.yml +++ /dev/null @@ -1,93 +0,0 @@ -name: CI - -on: - pull_request: - push: - branches: - - main - -permissions: - contents: read - -defaults: - run: - shell: bash - -jobs: - test: - name: Build and test - runs-on: ubuntu-24.04 - env: - IMPORT_NAME: src_py_lib - PYTHON_VERSION: "3.11" - UV_VERSION: "0.11.7" - - steps: - - name: Check out code - uses: actions/checkout@v6 - with: - persist-credentials: false - - - name: Set up Python - uses: actions/setup-python@v6 - with: - python-version: ${{ env.PYTHON_VERSION }} - cache: pip - - - name: Install uv - run: | - python -m pip install --upgrade pip - python -m pip install "uv==${UV_VERSION}" - - - name: Validate lockfile - run: uv lock --check - - - name: Lint Markdown - run: npx --yes markdownlint-cli2 - - - name: Lint Python - run: uv run --frozen ruff check . - - - name: Check Python formatting - run: uv run --frozen ruff format --check . - - - name: Type check - run: uv run --frozen pyright - - - name: Run tests - run: uv run --frozen python -m unittest discover -s tests - - - name: Smoke test source checkout import - run: | - uv run --frozen python - <<'PY' - import os - - import src_py_lib - - if src_py_lib.__name__ != os.environ["IMPORT_NAME"]: - raise SystemExit(f"unexpected import name: {src_py_lib.__name__}") - PY - - - name: Build wheel - run: uv build --wheel --out-dir dist --no-create-gitignore - - - name: Smoke test installed wheel - run: | - python -m venv build/ci-venv - . build/ci-venv/bin/activate - python -m pip install --upgrade pip - python -m pip install dist/*.whl - python - <<'PY' - import os - - import src_py_lib - - if src_py_lib.__name__ != os.environ["IMPORT_NAME"]: - raise SystemExit(f"unexpected import name: {src_py_lib.__name__}") - PY - - - name: Upload wheel artifact - uses: actions/upload-artifact@v7 - with: - name: src-py-lib-wheel - path: dist/*.whl diff --git a/git-subtree/src-py-lib/.github/workflows/release.yml b/git-subtree/src-py-lib/.github/workflows/release.yml deleted file mode 100644 index ce4ee24..0000000 --- a/git-subtree/src-py-lib/.github/workflows/release.yml +++ /dev/null @@ -1,190 +0,0 @@ -name: Build release - -on: - push: - tags: - - "v*" - workflow_dispatch: - inputs: - tag: - description: "Existing release tag to publish, for example v0.1.0" - required: true - type: string - -permissions: - contents: write - -concurrency: - group: release-${{ github.event.inputs.tag || github.ref_name }} - cancel-in-progress: false - -defaults: - run: - shell: bash - -jobs: - wheel: - name: Build wheel - runs-on: ubuntu-24.04 - env: - IMPORT_NAME: src_py_lib - PYTHON_VERSION: "3.11" - UV_VERSION: "0.11.7" - - steps: - - name: Check out release ref - uses: actions/checkout@v6 - with: - fetch-depth: 0 - persist-credentials: false - ref: ${{ github.event.inputs.tag || github.ref }} - - - name: Set up Python - uses: actions/setup-python@v6 - with: - python-version: ${{ env.PYTHON_VERSION }} - cache: pip - - - name: Install build tools - run: | - python -m pip install --upgrade pip - python -m pip install "uv==${UV_VERSION}" - - - name: Validate release inputs - id: release - run: | - release_tag="${{ github.event.inputs.tag || github.ref_name }}" - if [[ ! "${release_tag}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then - echo "::error title=Invalid release tag::Use a vMAJOR.MINOR.PATCH tag, got '${release_tag}'." - exit 1 - fi - if ! git rev-parse --verify --quiet "refs/tags/${release_tag}" >/dev/null; then - echo "::error title=Missing tag::Tag '${release_tag}' was not fetched. Create and push it before running this workflow." - exit 1 - fi - - project_version=$(uv run --frozen python - <<'PY' - import tomllib - - with open("pyproject.toml", "rb") as pyproject_file: - print(tomllib.load(pyproject_file)["project"]["version"]) - PY - ) - if [[ "v${project_version}" != "${release_tag}" ]]; then - echo "::error title=Version mismatch::pyproject.toml version '${project_version}' does not match tag '${release_tag}'." - exit 1 - fi - - echo "tag=${release_tag}" >> "${GITHUB_OUTPUT}" - - - name: Validate package - run: | - uv lock --check - uv run --frozen ruff check . - uv run --frozen ruff format --check . - uv run --frozen pyright - uv run --frozen python -m unittest discover -s tests - uv run --frozen python - <<'PY' - import os - - import src_py_lib - - if src_py_lib.__name__ != os.environ["IMPORT_NAME"]: - raise SystemExit(f"unexpected import name: {src_py_lib.__name__}") - PY - - - name: Build wheel - id: build - run: | - dist_dir="build/release/dist" - rm -rf build/release - mkdir -p "${dist_dir}" - - uv build --wheel --out-dir "${dist_dir}" --no-create-gitignore - project_wheels=("${dist_dir}"/*.whl) - if [[ "${#project_wheels[@]}" -ne 1 ]]; then - echo "::error title=Unexpected wheel count::Expected one project wheel, found ${#project_wheels[@]}." - exit 1 - fi - wheel_path="${project_wheels[0]}" - wheel_name="$(basename "${wheel_path}")" - checksum_path="${wheel_path}.sha256" - - ( - cd "$(dirname "${wheel_path}")" - shasum -a 256 "${wheel_name}" > "$(basename "${checksum_path}")" - ) - - echo "wheel_path=${wheel_path}" >> "${GITHUB_OUTPUT}" - echo "wheel_name=${wheel_name}" >> "${GITHUB_OUTPUT}" - echo "checksum_path=${checksum_path}" >> "${GITHUB_OUTPUT}" - - - name: Smoke test installed wheel - run: | - python -m venv build/release/install-venv - . build/release/install-venv/bin/activate - python -m pip install --upgrade pip - python -m pip install "${{ steps.build.outputs.wheel_path }}" - python - <<'PY' - import os - - import src_py_lib - - if src_py_lib.__name__ != os.environ["IMPORT_NAME"]: - raise SystemExit(f"unexpected import name: {src_py_lib.__name__}") - PY - - - name: Write release notes - id: notes - run: | - release_tag="${{ steps.release.outputs.tag }}" - wheel_name="${{ steps.build.outputs.wheel_name }}" - notes_path="build/release/release-notes.md" - cat > "${notes_path}" <> "${GITHUB_OUTPUT}" - - - name: Upload workflow artifact - uses: actions/upload-artifact@v7 - with: - name: src-py-lib-release - path: | - ${{ steps.build.outputs.wheel_path }} - ${{ steps.build.outputs.checksum_path }} - ${{ steps.notes.outputs.path }} - - - name: Publish GitHub release assets - env: - GH_TOKEN: ${{ github.token }} - run: | - release_tag="${{ steps.release.outputs.tag }}" - wheel_path="${{ steps.build.outputs.wheel_path }}" - checksum_path="${{ steps.build.outputs.checksum_path }}" - notes_path="${{ steps.notes.outputs.path }}" - - if gh release view "${release_tag}" >/dev/null 2>&1; then - gh release edit "${release_tag}" --title "${release_tag}" --notes-file "${notes_path}" - gh release upload "${release_tag}" "${wheel_path}" "${checksum_path}" --clobber - else - gh release create "${release_tag}" \ - "${wheel_path}" \ - "${checksum_path}" \ - --title "${release_tag}" \ - --notes-file "${notes_path}" \ - --verify-tag - fi diff --git a/git-subtree/src-py-lib/.gitignore b/git-subtree/src-py-lib/.gitignore deleted file mode 100644 index 25c45a3..0000000 --- a/git-subtree/src-py-lib/.gitignore +++ /dev/null @@ -1,225 +0,0 @@ -# Byte-compiled / optimized / DLL files -__pycache__/ -*.py[codz] -*$py.class - -# C extensions -*.so - -# Distribution / packaging -.Python -build/ -develop-eggs/ -dist/ -downloads/ -eggs/ -.eggs/ -lib/ -lib64/ -parts/ -sdist/ -var/ -wheels/ -share/python-wheels/ -*.egg-info/ -.installed.cfg -*.egg -MANIFEST - -# PyInstaller -# Usually these files are written by a python script from a template -# before PyInstaller builds the exe, so as to inject date/other infos into it. -*.manifest -*.spec - -# Installer logs -pip-log.txt -pip-delete-this-directory.txt - -# Unit test / coverage reports -htmlcov/ -.tox/ -.nox/ -.coverage -.coverage.* -.cache -nosetests.xml -coverage.xml -*.cover -*.py.cover -.hypothesis/ -.pytest_cache/ -cover/ - -# Translations -*.mo -*.pot - -# Django stuff: -*.log -local_settings.py -db.sqlite3 -db.sqlite3-journal - -# Flask stuff: -instance/ -.webassets-cache - -# Scrapy stuff: -.scrapy - -# Sphinx documentation -docs/_build/ - -# PyBuilder -.pybuilder/ -target/ - -# Jupyter Notebook -.ipynb_checkpoints - -# IPython -profile_default/ -ipython_config.py - -# pyenv -# For a library or package, you might want to ignore these files since the code is -# intended to run in multiple environments; otherwise, check them in: -# .python-version - -# pipenv -# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. -# However, in case of collaboration, if having platform-specific dependencies or dependencies -# having no cross-platform support, pipenv may install dependencies that don't work, or not -# install all needed dependencies. -# Pipfile.lock - -# UV -# Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control. -# This is especially recommended for binary packages to ensure reproducibility, and is more -# commonly ignored for libraries. -# uv.lock - -# poetry -# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. -# This is especially recommended for binary packages to ensure reproducibility, and is more -# commonly ignored for libraries. -# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control -# poetry.lock -# poetry.toml - -# pdm -# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. -# pdm recommends including project-wide configuration in pdm.toml, but excluding .pdm-python. -# https://pdm-project.org/en/latest/usage/project/#working-with-version-control -# pdm.lock -# pdm.toml -.pdm-python -.pdm-build/ - -# pixi -# Similar to Pipfile.lock, it is generally recommended to include pixi.lock in version control. -# pixi.lock -# Pixi creates a virtual environment in the .pixi directory, just like venv module creates one -# in the .venv directory. It is recommended not to include this directory in version control. -.pixi - -# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm -__pypackages__/ - -# Celery stuff -celerybeat-schedule -celerybeat.pid - -# Redis -*.rdb -*.aof -*.pid - -# RabbitMQ -mnesia/ -rabbitmq/ -rabbitmq-data/ - -# ActiveMQ -activemq-data/ - -# SageMath parsed files -*.sage.py - -# Environments -.env -.envrc -.venv -env/ -venv/ -ENV/ -env.bak/ -venv.bak/ - -# Spyder project settings -.spyderproject -.spyproject - -# Rope project settings -.ropeproject - -# mkdocs documentation -/site - -# mypy -.mypy_cache/ -.dmypy.json -dmypy.json - -# Pyre type checker -.pyre/ - -# pytype static type analyzer -.pytype/ - -# Cython debug symbols -cython_debug/ - -# PyCharm -# JetBrains specific template is maintained in a separate JetBrains.gitignore that can -# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore -# and can be added to the global gitignore or merged into this file. For a more nuclear -# option (not recommended) you can uncomment the following to ignore the entire idea folder. -# .idea/ - -# Abstra -# Abstra is an AI-powered process automation framework. -# Ignore directories containing user credentials, local state, and settings. -# Learn more at https://abstra.io/docs -.abstra/ - -# Visual Studio Code -# Visual Studio Code specific template is maintained in a separate VisualStudioCode.gitignore -# that can be found at https://github.com/github/gitignore/blob/main/Global/VisualStudioCode.gitignore -# and can be added to the global gitignore or merged into this file. However, if you prefer, -# you could uncomment the following to ignore the entire vscode folder -# .vscode/ -# Temporary file for partial code execution -tempCodeRunnerFile.py - -# Ruff stuff: -.ruff_cache/ - -# PyPI configuration file -.pypirc - -# Marimo -marimo/_static/ -marimo/_lsp/ -__marimo__/ - -# Streamlit -.streamlit/secrets.toml - -.DS_Store -.env.* -.pyright/ -.venv/ -*.py[cod] -!.env.example diff --git a/git-subtree/src-py-lib/.python-version b/git-subtree/src-py-lib/.python-version deleted file mode 100644 index 2c07333..0000000 --- a/git-subtree/src-py-lib/.python-version +++ /dev/null @@ -1 +0,0 @@ -3.11 diff --git a/git-subtree/src-py-lib/AGENTS.md b/git-subtree/src-py-lib/AGENTS.md deleted file mode 100644 index c874569..0000000 --- a/git-subtree/src-py-lib/AGENTS.md +++ /dev/null @@ -1,57 +0,0 @@ -# Agents - - - -## Project principles - -- This repo is public, never write non-public information in this repo -- Keep code and docs brief, for humans to read / understand / audit quickly -- Reuse and improve existing solutions / approaches / designs / helpers / tools / patterns, - before adding new / similar ones -- Keep runtime dependencies minimal; justify new dependencies in code review -- Preserve unrelated user or agent edits in the worktree - -## Standard commands - -```sh -npx --yes markdownlint-cli2 -uv sync -uv run ruff format . -uv run ruff check . -uv run pyright -uv run python -m unittest discover -s tests -``` - - - - - -## Toolchain - -- Use `uv` for dependency management, virtualenv creation, and command running -- Use pyright in strict mode; fix linting / typing issues instead of suppressing them -- Use ruff for formatting, import sorting, and linting - -## Runtime standards - -- Configure the root logger by default (`logger_name=""`) so project modules - and shared `src_py_lib` modules are captured by the same handlers -- Startup logs should include command, sanitized runtime config, commit when - available, and log file path when applicable -- Use shared HTTP/client helpers for timeout policy, API error wrapping, and - rate-limit handling - -## Code organization - -- Put importable package code under `src/` -- Put tests under `tests/` -- Keep module-level constants near the top of each module, after imports -- Prefer specific package/module names over broad `helpers` or `utils` modules - -## Before finishing changes - -- Re-read edited files for organization and stale comments -- Update `README.md` when setup or user-facing behavior changes -- Update this `AGENTS.md` only with durable project-specific discoveries - - diff --git a/git-subtree/src-py-lib/README.md b/git-subtree/src-py-lib/README.md deleted file mode 100644 index 944002c..0000000 --- a/git-subtree/src-py-lib/README.md +++ /dev/null @@ -1,140 +0,0 @@ -# src-py-lib - -Reusable libraries for Sourcegraph-adjacent Python projects - -This repo is the shared implementation layer for patterns which get -rebuilt in separate scripts: API clients, HTTP retries/timeouts, structured logging, -etc. - -## Experimental - This is not a supported Sourcegraph product - -This repo was created for Sourcegraph Implementation Engineering deployments, -and is not intended, designed, built, or supported for use in any other scenario. -Feel free to open issues or PRs, but responses are best effort. - -## Install from another project - -```sh -uv add git+https://github.com/sourcegraph/src-py-lib.git -``` - -## What is included - -- `src_py_lib.utils.logging` — centralized human stderr logs plus optional structured - JSONL events, run IDs, git commit metadata, context fields, event timing, - retention, startup metadata, and sanitized config snapshots. -- `src_py_lib.utils.config` — Pydantic-backed `Config` models loaded from code - defaults, `python-dotenv` `.env` parsing, shell environment, and CLI - overrides, with typed values, required checks, safe snapshots, and `op://...` - reference resolution. -- `src_py_lib.utils.http` — pooled `httpx` JSON HTTP client with a shared - 30-second timeout, retry policy, `Retry-After` support, and contextual errors. -- `src_py_lib.utils.tsv` — padded TSV writer for human-readable tabular exports, - with newline/tab cleanup, URL preservation, and Unicode-aware column widths. -- `src_py_lib.clients.graphql` — shared GraphQL execution with automatic cursor - pagination, batched alias lookups, and schema introspection export. -- `src_py_lib.clients.sourcegraph` — Sourcegraph GraphQL client with token - validation, endpoint normalization, connection streaming, and shared config - fields for `SRC_ENDPOINT` (default: `https://sourcegraph.com`) and - `SRC_ACCESS_TOKEN`. -- `src_py_lib.clients.linear` — Linear GraphQL client with automatic cursor - handling, token validation, shared config fields, and injectable HTTP policy. -- `src_py_lib.clients.slack` — Slack Web API client with token validation, - cursor pagination, and method pacing. Consider `slack_sdk` if usage grows - beyond simple GET, pagination, and rate-limit handling. -- `src_py_lib.clients.github` — GitHub GraphQL client, PR URL parsing, and - batched PR lookups, with token validation. Defaults to `https://github.com`; - pass `github_url` for GitHub Enterprise Server. Keep lightweight for GraphQL; - GitHub SDKs help more for REST. -- `src_py_lib.clients.one_password` — tiny 1Password CLI wrapper for signing in, - validating authenticated `op` access, and resolving `op://...` references after config loading. -- `src_py_lib.clients.google_sheets` — Google Sheets API primitives with - spreadsheet access validation using gcloud Application Default Credentials or - a provided access token. Prefer Google's official libraries if Sheets usage - grows beyond small primitives, because auth, quota project, token refresh, - batching, and error shapes are subtle. - -Prefer this library for shared logging, HTTP policy, and thin API wrappers. -Prefer vendor SDKs when they replace tricky auth, token refresh, retries, -pagination, quota behavior, or complex request models. - -## Example - -Define one project-specific `Config` model, then load it once at CLI startup. -For common CLI and client usage, import the curated root API: - -```python -from pathlib import Path - -import src_py_lib as src - - -class LinearExportConfig(src.LinearClientConfig): - output_dir: Path = src.config_field( - default=Path("."), - env_var="LINEAR_EXPORT_OUTPUT_DIR", - cli_flag="--output-dir", - metavar="PATH", - help="Directory for generated files.", - ) - -config = src.parse_args(LinearExportConfig, description="Export Linear data.") -client = src.linear_client_from_config(config) -print(f"Writing files under {config.output_dir}") -``` - -Config precedence is: code defaults, `.env`, shell environment, then CLI -overrides. API client modules can provide shared Config base classes such as -`LinearClientConfig`, and `parse_args` resolves `op://...` references by -default. `config_field(default=...)` supports aliases, store-true / -store-false command flags, optional values, numeric bounds, and string patterns -for simple CLIs. Pass a custom `argparse.ArgumentParser` to `parse_args` only when you -need parsing beyond Config fields. Help text preserves description and -argument-help newlines, and reserves enough option-column width for long config -flags. Mark sensitive fields with `secret=True` so snapshots do not expose -resolved values. - -## Logging example - -Configure logging once at process startup. Prefer configuring the root logger -(`logger_name=""`, the default) so project modules and shared `src_py_lib` modules -such as `src_py_lib.utils.http` are captured by the same terminal and JSONL handlers. -Use `logging()` in CLIs to configure logging, add the command field to all -structured events, and emit standard run/startup/run-end metadata. -Use `debug()`, `info()`, `warning()`, `error()`, and `critical()` for one-off -structured events. Use `event()` blocks around timed work; they emit `trace`, -`span`, and nested `parent_span` fields. Use `start_level="debug"` to hide -noisy start events while keeping end timing visible, and -`omit_success_status=True` for very high-volume success events. Use `stage()` -for workflow context such as `stage="apply"`. -When the root logger is configured, noisy `httpx`/`httpcore` records are suppressed; -`HTTPClient` emits structured `http_request` events instead. -Run-end events include HTTP attempt/byte/status/retry counters. Set -`LoggingSettings.resource_sample_interval_seconds` to emit DEBUG -`resource_sample` events and include process resource totals on run end. Set -`SRC_LOG_LEVEL=INFO` for a run to omit DEBUG events from the log file. -`LoggingConfig` includes `--verbose/-v`, `--quiet/-q`, and `--silent/-s` -shortcuts (also available as `SRC_LOG_VERBOSE`, `SRC_LOG_QUIET`, and -`SRC_LOG_SILENT`). Use `logging_settings_from_config()` to build -`LoggingSettings` from those conventions. - -```python -import src_py_lib as src - -with src.logging({"src_token": "provided"}): - src.info("sync_started", repository_count=3) - - client = src.SourcegraphClient("https://sourcegraph.example.com", "token") - data = client.graphql("query Viewer { currentUser { username } }") -``` - -## Development - -```sh -uv sync -uv run ruff format . -uv run ruff check . -uv run pyright -uv run python -m unittest discover -s tests -npx --yes markdownlint-cli2 -``` diff --git a/git-subtree/src-py-lib/pyproject.toml b/git-subtree/src-py-lib/pyproject.toml deleted file mode 100644 index 545474c..0000000 --- a/git-subtree/src-py-lib/pyproject.toml +++ /dev/null @@ -1,40 +0,0 @@ -[project] -name = "src-py-lib" -version = "0.1.0" -description = "Reusable Python helpers for Python projects." -readme = "README.md" -requires-python = ">=3.11" -dependencies = [ - "httpx>=0.28,<1", - "pydantic>=2,<3", - "python-dotenv>=1.2,<2", -] - -[build-system] -requires = ["hatchling"] -build-backend = "hatchling.build" - -[tool.hatch.build.targets.wheel] -packages = ["src/src_py_lib"] - -[tool.uv] -package = true - -[tool.pyright] -include = ["src/src_py_lib"] -typeCheckingMode = "strict" -extraPaths = ["src"] -pythonVersion = "3.11" - -[tool.ruff] -target-version = "py311" -line-length = 100 - -[tool.ruff.lint] -select = ["E", "F", "I", "B", "UP", "SIM"] - -[dependency-groups] -dev = [ - "pyright>=1.1.409", - "ruff>=0.7.0", -] diff --git a/git-subtree/src-py-lib/renovate.json b/git-subtree/src-py-lib/renovate.json deleted file mode 100644 index 63ffc65..0000000 --- a/git-subtree/src-py-lib/renovate.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "$schema": "https://docs.renovatebot.com/renovate-schema.json", - "extends": [ - "local>sourcegraph/renovate-config" - ], - "packageRules": [ - { - "matchPackageNames": [ - "python" - ], - "allowedVersions": "<3.12" - } - ] -} diff --git a/git-subtree/src-py-lib/src/src_py_lib/__init__.py b/git-subtree/src-py-lib/src/src_py_lib/__init__.py deleted file mode 100644 index 66e2710..0000000 --- a/git-subtree/src-py-lib/src/src_py_lib/__init__.py +++ /dev/null @@ -1,170 +0,0 @@ -"""Public interface for src-py-lib consumers.""" - -from __future__ import annotations - -import sys -from collections.abc import Callable, Mapping -from contextlib import AbstractContextManager -from pathlib import Path -from typing import Any - -from src_py_lib.clients.github import GitHubClient, PullRequest, gh_cli_token, pr_ref_from_url -from src_py_lib.clients.google_sheets import ( - GoogleSheetsClient, - GoogleSheetsError, - gcloud_adc_access_token, - quota_project_from_adc, -) -from src_py_lib.clients.graphql import ( - GraphQLClient, - GraphQLError, - aliased_batched_query, - introspect_schema, - stream_connection_nodes, -) -from src_py_lib.clients.linear import ( - LinearClient, - LinearClientConfig, - linear_client_from_config, -) -from src_py_lib.clients.slack import ( - SlackClient, - SlackClientConfig, - SlackError, - SlackPacer, - slack_client_from_config, -) -from src_py_lib.clients.sourcegraph import ( - SourcegraphClient, - SourcegraphClientConfig, - normalize_sourcegraph_endpoint, - sourcegraph_client_from_config, -) -from src_py_lib.utils.config import ( - Config, - ConfigError, - config_field, - config_snapshot, -) -from src_py_lib.utils.config import ( - config_parse_args as parse_args, -) -from src_py_lib.utils.http import HTTPClient, HTTPClientError -from src_py_lib.utils.json_cache import load_json_cache, load_json_subset, save_json_cache -from src_py_lib.utils.json_types import ( - JSONDict, - json_dict, - json_dicts, - json_int, - json_list, - json_str, - json_strs, -) -from src_py_lib.utils.logging import ( - LoggingConfig, - LoggingSettings, - configure_logging, - critical, - debug, - error, - event, - info, - log, - log_context, - logging_context, - logging_settings_from_config, - resolve_log_level_name, - stage, - startup_event, - submit_with_log_context, - warning, -) -from src_py_lib.utils.tsv import write_tsv - - -def logging( - config: object | None = None, - *, - command: str | None = None, - git_cwd: Path | str | None = None, - logging_config: LoggingSettings | None = None, - run_fields: Mapping[str, Any] | None = None, - run_summary: Callable[[], Mapping[str, Any]] | None = None, -) -> AbstractContextManager[Path | None]: - """Configure standard CLI logging and emit startup metadata.""" - return logging_context( - command or _script_name(), - config, - git_cwd=git_cwd, - logging_config=logging_config, - run_fields=run_fields, - run_summary=run_summary, - ) - - -def _script_name() -> str: - return Path(sys.argv[0]).stem or "python" - - -__all__ = [ - "Config", - "ConfigError", - "GraphQLError", - "GraphQLClient", - "GitHubClient", - "GoogleSheetsClient", - "GoogleSheetsError", - "HTTPClient", - "HTTPClientError", - "JSONDict", - "LinearClient", - "LinearClientConfig", - "LoggingConfig", - "LoggingSettings", - "PullRequest", - "SlackClient", - "SlackClientConfig", - "SlackError", - "SlackPacer", - "SourcegraphClient", - "SourcegraphClientConfig", - "aliased_batched_query", - "config_field", - "config_snapshot", - "configure_logging", - "critical", - "debug", - "error", - "event", - "gh_cli_token", - "gcloud_adc_access_token", - "info", - "introspect_schema", - "json_dict", - "json_dicts", - "json_int", - "json_list", - "json_str", - "json_strs", - "linear_client_from_config", - "load_json_cache", - "load_json_subset", - "logging", - "logging_settings_from_config", - "log", - "log_context", - "normalize_sourcegraph_endpoint", - "parse_args", - "pr_ref_from_url", - "quota_project_from_adc", - "resolve_log_level_name", - "save_json_cache", - "slack_client_from_config", - "sourcegraph_client_from_config", - "stage", - "startup_event", - "stream_connection_nodes", - "submit_with_log_context", - "warning", - "write_tsv", -] diff --git a/git-subtree/src-py-lib/src/src_py_lib/clients/__init__.py b/git-subtree/src-py-lib/src/src_py_lib/clients/__init__.py deleted file mode 100644 index f489d06..0000000 --- a/git-subtree/src-py-lib/src/src_py_lib/clients/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -"""API clients built on src_py_lib HTTP and logging primitives.""" - -from __future__ import annotations diff --git a/git-subtree/src-py-lib/src/src_py_lib/clients/github.py b/git-subtree/src-py-lib/src/src_py_lib/clients/github.py deleted file mode 100644 index ebff03e..0000000 --- a/git-subtree/src-py-lib/src/src_py_lib/clients/github.py +++ /dev/null @@ -1,157 +0,0 @@ -"""GitHub GraphQL API client.""" - -from __future__ import annotations - -import json -import re -import subprocess -from dataclasses import dataclass, field -from typing import TypedDict, cast -from urllib.parse import urlsplit - -from src_py_lib.clients.graphql import GraphQLClient, aliased_batched_query -from src_py_lib.utils.http import HTTPClient -from src_py_lib.utils.json_types import JSONDict, json_dict, json_str - -DEFAULT_GITHUB_URL = "https://github.com" -DEFAULT_PR_BATCH_SIZE = 50 -GITHUB_VALIDATE_QUERY = """ -query GitHubClientValidate { - viewer { - login - } -} -""" -PR_REF_RE = re.compile(r"^(?P[^/]+)/(?P[^/#]+)#(?P\d+)$") -PR_URL_RE = re.compile( - r"https?://[^/\s)>|]+/(?P[^/\s)>|]+)/(?P[^/\s)>|]+)/pull/(?P\d+)" -) - - -class PullRequest(TypedDict): - title: str - url: str - state: str - createdAt: str - mergedAt: str - closedAt: str - author: str - - -@dataclass -class GitHubClient: - token: str - github_url: str = DEFAULT_GITHUB_URL - http: HTTPClient = field(default_factory=HTTPClient) - - @classmethod - def from_gh_cli( - cls, *, github_url: str = DEFAULT_GITHUB_URL, http: HTTPClient | None = None - ) -> GitHubClient: - token = gh_cli_token(github_url=github_url) - if not token: - raise RuntimeError("No GitHub token from `gh auth token`.") - return cls(token=token, github_url=github_url, http=http or HTTPClient()) - - def graphql(self, query: str, variables: JSONDict | None = None) -> JSONDict: - return GraphQLClient( - url=graphql_api_url(self.github_url), - headers={"Authorization": f"bearer {self.token}"}, - label="GitHub", - http=self.http, - tolerate_partial_errors=True, - ).execute(query, variables) - - def validate(self) -> JSONDict: - """Validate the token with a cheap viewer query and return the viewer.""" - viewer = json_dict(self.graphql(GITHUB_VALIDATE_QUERY).get("viewer")) - if not viewer.get("login"): - raise RuntimeError("GitHub viewer response did not include viewer.login.") - return viewer - - def get_pull_requests( - self, refs: list[str], *, batch_size: int = DEFAULT_PR_BATCH_SIZE - ) -> dict[str, PullRequest]: - return cast( - dict[str, PullRequest], - aliased_batched_query( - refs, - batch_size=batch_size, - build_alias=_build_pr_alias, - parse_node=_project_pull_request, - post=self.graphql, - ), - ) - - -def graphql_api_url(github_url: str = DEFAULT_GITHUB_URL) -> str: - """Return the GraphQL API URL for github.com or a GitHub Enterprise host.""" - normalized = _normalize_github_url(github_url) - split = urlsplit(normalized) - if split.hostname == "github.com": - return f"{split.scheme}://api.github.com/graphql" - return f"{normalized}/api/graphql" - - -def gh_cli_token(*, github_url: str = DEFAULT_GITHUB_URL) -> str | None: - """Return `gh auth token`, or None when gh is unavailable/not logged in.""" - split = urlsplit(_normalize_github_url(github_url)) - command = ["gh", "auth", "token"] - if split.hostname and split.hostname != "github.com": - command.extend(["--hostname", split.netloc]) - try: - result = subprocess.run(command, capture_output=True, text=True, timeout=5, check=False) - except OSError: - return None - except subprocess.SubprocessError: - return None - token = result.stdout.strip() - return token if result.returncode == 0 and token else None - - -def _normalize_github_url(github_url: str) -> str: - stripped = github_url.strip().rstrip("/") - if "://" not in stripped: - stripped = f"https://{stripped}" - return stripped - - -def parse_pr_ref(ref: str) -> tuple[str, str, int]: - match = PR_REF_RE.match(ref) - if not match: - raise ValueError(f"invalid GitHub PR ref: {ref!r}") - return match.group("owner"), match.group("repo"), int(match.group("number")) - - -def pr_ref_from_url(url: str) -> str | None: - match = PR_URL_RE.search(url) - if not match: - return None - return f"{match.group('owner')}/{match.group('repo')}#{match.group('number')}" - - -def _build_pr_alias(_index: int, ref: str) -> str | None: - try: - owner, repo, number = parse_pr_ref(ref) - except ValueError: - return None - return ( - f"repository(owner: {json.dumps(owner)}, name: {json.dumps(repo)}) " - f"{{ pullRequest(number: {number}) " - "{ title url state createdAt mergedAt closedAt author { login } } }" - ) - - -def _project_pull_request(node: JSONDict) -> PullRequest | None: - pull_request = json_dict(node.get("pullRequest")) - if not pull_request: - return None - return { - "title": json_str(pull_request, "title"), - "url": json_str(pull_request, "url"), - "state": json_str(pull_request, "state"), - "createdAt": json_str(pull_request, "createdAt"), - "mergedAt": json_str(pull_request, "mergedAt"), - "closedAt": json_str(pull_request, "closedAt"), - "author": json_str(json_dict(pull_request.get("author")), "login"), - } diff --git a/git-subtree/src-py-lib/src/src_py_lib/clients/google_sheets.py b/git-subtree/src-py-lib/src/src_py_lib/clients/google_sheets.py deleted file mode 100644 index 7f4aeb1..0000000 --- a/git-subtree/src-py-lib/src/src_py_lib/clients/google_sheets.py +++ /dev/null @@ -1,131 +0,0 @@ -"""Google Sheets API client using an already-available OAuth access token.""" - -from __future__ import annotations - -import json -import os -import subprocess -from dataclasses import dataclass, field -from pathlib import Path -from typing import cast - -from src_py_lib.utils.http import HTTPClient, HTTPClientError -from src_py_lib.utils.json_types import JSONDict, json_dict, json_int, json_list, json_str - -SHEETS_API_URL = "https://sheets.googleapis.com/v4/spreadsheets" -DEFAULT_ADC_FILE = Path("~/.config/gcloud/application_default_credentials.json").expanduser() - - -class GoogleSheetsError(RuntimeError): - """Raised for Google Sheets client errors.""" - - -@dataclass(frozen=True) -class LinkRun: - start: int - end: int - uri: str - - -@dataclass(frozen=True) -class Cell: - text: str - links: tuple[LinkRun, ...] = () - - -CellValue = str | Cell - - -@dataclass -class GoogleSheetsClient: - spreadsheet_id: str - access_token: str - quota_project: str | None = None - http: HTTPClient = field(default_factory=HTTPClient) - - @classmethod - def from_gcloud_adc( - cls, - spreadsheet_id: str, - *, - credentials_file: Path = DEFAULT_ADC_FILE, - http: HTTPClient | None = None, - ) -> GoogleSheetsClient: - return cls( - spreadsheet_id=spreadsheet_id, - access_token=gcloud_adc_access_token(credentials_file), - quota_project=quota_project_from_adc(credentials_file), - http=http or HTTPClient(), - ) - - def request(self, method: str, path: str, body: JSONDict | None = None) -> JSONDict: - headers = {"Authorization": f"Bearer {self.access_token}"} - if self.quota_project: - headers["X-Goog-User-Project"] = self.quota_project - try: - return self.http.json( - method, - f"{SHEETS_API_URL}/{self.spreadsheet_id}{path}", - headers=headers, - json_body=body, - ) - except HTTPClientError as exception: - raise GoogleSheetsError( - f"Google Sheets {method} {path} failed: {exception}" - ) from exception - - def metadata(self) -> JSONDict: - return self.request("GET", "?fields=sheets.properties(sheetId,title,gridProperties)") - - def validate(self) -> JSONDict: - """Validate spreadsheet access and return spreadsheet metadata.""" - metadata = self.metadata() - if not isinstance(metadata.get("sheets"), list): - raise GoogleSheetsError("Google Sheets metadata response did not include sheets.") - return metadata - - def tab_ids_by_title(self) -> dict[str, int]: - return { - json_str(properties, "title"): json_int(properties, "sheetId") - for sheet in json_list(self.metadata().get("sheets")) - if (properties := json_dict(json_dict(sheet).get("properties"))) - } - - def batch_update(self, requests: list[JSONDict]) -> JSONDict: - return self.request("POST", ":batchUpdate", cast(JSONDict, {"requests": requests})) - - -def hyperlink_cell(url: str, text: str) -> CellValue: - if not url: - return "" - return Cell(text=text, links=(LinkRun(0, len(text), url),)) - - -def quota_project_from_adc(credentials_file: Path = DEFAULT_ADC_FILE) -> str: - if not credentials_file.exists(): - raise GoogleSheetsError(f"Application Default Credentials not found at {credentials_file}.") - data = json_dict(json.loads(credentials_file.read_text(encoding="utf-8"))) - quota_project = json_str(data, "quota_project_id") - if not quota_project: - raise GoogleSheetsError(f"{credentials_file} does not contain quota_project_id.") - return quota_project - - -def gcloud_adc_access_token(credentials_file: Path = DEFAULT_ADC_FILE) -> str: - env = os.environ.copy() - env["GOOGLE_APPLICATION_CREDENTIALS"] = str(credentials_file) - try: - result = subprocess.run( - ["gcloud", "auth", "application-default", "print-access-token"], - env=env, - capture_output=True, - text=True, - timeout=10, - check=False, - ) - except (OSError, subprocess.SubprocessError) as exception: - raise GoogleSheetsError("Could not run gcloud to fetch an ADC access token.") from exception - token = result.stdout.strip() - if result.returncode != 0 or not token: - raise GoogleSheetsError(result.stderr.strip() or "gcloud did not return an access token.") - return token diff --git a/git-subtree/src-py-lib/src/src_py_lib/clients/graphql.py b/git-subtree/src-py-lib/src/src_py_lib/clients/graphql.py deleted file mode 100644 index fe4e226..0000000 --- a/git-subtree/src-py-lib/src/src_py_lib/clients/graphql.py +++ /dev/null @@ -1,476 +0,0 @@ -"""Shared GraphQL client primitives.""" - -from __future__ import annotations - -import json -import re -from collections.abc import Callable, Iterator, Mapping, Sequence -from dataclasses import dataclass, field -from pathlib import Path -from typing import cast - -from src_py_lib.utils.http import HTTPClient, HTTPClientError -from src_py_lib.utils.json_types import JSONDict, JSONValue, json_dict, json_list, json_str -from src_py_lib.utils.logging import event - -_OPERATION_NAME_RE = re.compile(r"\b(?:query|mutation|subscription)\s+(\w+)") - -GRAPHQL_INTROSPECTION_QUERY = """ -query IntrospectionQuery { - __schema { - queryType { name } - mutationType { name } - subscriptionType { name } - types { - ...FullType - } - directives { - name - description - locations - args { - ...InputValue - } - } - } -} - -fragment FullType on __Type { - kind - name - description - fields(includeDeprecated: true) { - name - description - args { - ...InputValue - } - type { - ...TypeRef - } - isDeprecated - deprecationReason - } - inputFields { - ...InputValue - } - interfaces { - ...TypeRef - } - enumValues(includeDeprecated: true) { - name - description - isDeprecated - deprecationReason - } - possibleTypes { - ...TypeRef - } -} - -fragment InputValue on __InputValue { - name - description - type { ...TypeRef } - defaultValue -} - -fragment TypeRef on __Type { - kind - name - ofType { - kind - name - ofType { - kind - name - ofType { - kind - name - ofType { - kind - name - ofType { - kind - name - ofType { - kind - name - ofType { - kind - name - } - } - } - } - } - } - } -} -""".strip() - - -class GraphQLError(RuntimeError): - """Raised for GraphQL transport or application errors.""" - - def __init__( - self, - message: str, - *, - status_code: int | None = None, - is_application_error: bool = False, - ) -> None: - super().__init__(message) - self.status_code = status_code - self.is_application_error = is_application_error - - -@dataclass -class GraphQLClient: - """POST JSON GraphQL operations and return the `data` object.""" - - url: str - headers: dict[str, str] - label: str - http: HTTPClient = field(default_factory=HTTPClient) - tolerate_partial_errors: bool = False - - def execute( - self, - query: str, - variables: Mapping[str, JSONValue] | None = None, - *, - follow_pages: bool = True, - page_size: int | None = None, - first_variable: str = "first", - after_variable: str = "after", - ) -> JSONDict: - page_variables: JSONDict = dict(variables) if variables is not None else {} - if page_size is not None: - page_variables[first_variable] = page_size - if ( - follow_pages - and after_variable not in page_variables - and _query_uses_variable(query, after_variable) - ): - page_variables[after_variable] = None - - page_number = 1 - data = self._execute_once( - query, - page_variables, - page_number=page_number, - first_variable=first_variable, - after_variable=after_variable, - ) - if follow_pages: - - def execute_next_page(next_variables: JSONDict) -> JSONDict: - nonlocal page_number - page_number += 1 - return self._execute_once( - query, - next_variables, - page_number=page_number, - first_variable=first_variable, - after_variable=after_variable, - ) - - _fetch_remaining_pages( - execute_next_page, - data, - page_variables, - after_variable=after_variable, - query_uses_after_variable=_query_uses_variable(query, after_variable), - ) - return data - - def stream_connection_nodes( - self, - query: str, - variables: Mapping[str, JSONValue] | None = None, - *, - connection_path: Sequence[str], - page_size: int | None = None, - first_variable: str = "first", - after_variable: str = "after", - ) -> Iterator[JSONDict]: - """Stream one GraphQL connection's nodes page by page. - - `connection_path` is the response path to the connection object that - contains `nodes` and `pageInfo`, for example `("viewer", "items")`. - Unlike `execute(..., follow_pages=True)`, this does not accumulate all - nodes in memory before returning. - """ - page_number = 1 - - def execute_page( - operation: str, page_variables: Mapping[str, JSONValue] | None - ) -> JSONDict: - nonlocal page_number - data = self._execute_once( - operation, - dict(page_variables or {}), - page_number=page_number, - first_variable=first_variable, - after_variable=after_variable, - ) - page_number += 1 - return data - - yield from stream_connection_nodes( - execute_page, - query, - variables, - connection_path=connection_path, - page_size=page_size, - first_variable=first_variable, - after_variable=after_variable, - ) - - def _execute_once( - self, - query: str, - variables: JSONDict, - *, - page_number: int = 1, - first_variable: str = "first", - after_variable: str = "after", - ) -> JSONDict: - body = {"query": query, "variables": variables or {}} - with event( - "graphql_query", - level="debug", - graphql_client=self.label, - query_name=operation_name(query), - page_number=page_number, - page_size=_int_variable(variables, first_variable), - cursor_present=variables.get(after_variable) is not None, - url=self.url, - variable_names=sorted(variables), - query_bytes=len(query.encode("utf-8")), - ) as fields: - try: - payload = self.http.json("POST", self.url, headers=self.headers, json_body=body) - except HTTPClientError as exception: - raise GraphQLError( - f"{self.label} GraphQL request failed: {exception}", - status_code=exception.status_code, - ) from exception - errors = payload.get("errors") - data = json_dict(payload.get("data")) - fields["response_fields"] = sorted(data) - if errors: - fields["graphql_errors"] = len(errors) if isinstance(errors, list) else 1 - if errors and not (self.tolerate_partial_errors and data): - raise GraphQLError( - f"{self.label} GraphQL errors: {errors}", - is_application_error=True, - ) - return data - - -def operation_name(query: str) -> str: - """Extract the operation name from a GraphQL document.""" - match = _OPERATION_NAME_RE.search(query) - return match.group(1) if match else "anonymous" - - -def stream_connection_nodes( - execute: Callable[[str, Mapping[str, JSONValue] | None], JSONDict], - query: str, - variables: Mapping[str, JSONValue] | None = None, - *, - connection_path: Sequence[str], - page_size: int | None = None, - first_variable: str = "first", - after_variable: str = "after", -) -> Iterator[JSONDict]: - """Stream one GraphQL connection's nodes through any execute callable.""" - page_variables: JSONDict = dict(variables) if variables is not None else {} - if page_size is not None: - page_variables[first_variable] = page_size - query_uses_after_variable = _query_uses_variable(query, after_variable) - if query_uses_after_variable and after_variable not in page_variables: - page_variables[after_variable] = None - - path = tuple(connection_path) - current_cursor = page_variables.get(after_variable) - while True: - data = execute(query, dict(page_variables)) - page = _node_page_at_path(data, path) - for node in json_list(page.get("nodes")): - yield json_dict(node) - - page_info = json_dict(page.get("pageInfo")) - has_next_page = page_info.get("hasNextPage") - if not isinstance(has_next_page, bool): - raise GraphQLError( - f"GraphQL pagination path {_path_label(path)} missing pageInfo.hasNextPage" - ) - if not has_next_page: - return - if not query_uses_after_variable: - raise GraphQLError( - f"GraphQL query returned more pages but does not use ${after_variable}" - ) - next_cursor = _next_page_cursor(page_info, path, current_cursor) - page_variables[after_variable] = next_cursor - current_cursor = next_cursor - - -def _int_variable(variables: JSONDict, name: str) -> int | None: - value = variables.get(name) - return value if isinstance(value, int) else None - - -def introspect_schema( - client_or_execute: GraphQLClient | Callable[[str], JSONDict], - *, - output_file: Path | str | None = None, -) -> JSONDict | None: - """Fetch a GraphQL introspection schema or write it to `output_file`. - - Pass either a `GraphQLClient` or a callable such as `SourcegraphClient.graphql`. - When `output_file` is supplied, the schema JSON is written there and `None` is - returned. Otherwise, the introspection `__schema` object is returned. - """ - if isinstance(client_or_execute, GraphQLClient): - data = client_or_execute.execute(GRAPHQL_INTROSPECTION_QUERY, follow_pages=False) - else: - data = client_or_execute(GRAPHQL_INTROSPECTION_QUERY) - schema = json_dict(data.get("__schema")) - if not schema: - raise GraphQLError("GraphQL introspection response did not include __schema.") - if output_file is None: - return schema - - path = Path(output_file) - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(json.dumps(schema, indent=2) + "\n", encoding="utf-8") - return None - - -def aliased_batched_query( - keys: list[str], - *, - batch_size: int, - build_alias: Callable[[int, str], str | None], - parse_node: Callable[[JSONDict], object | None], - post: Callable[[str], JSONDict], -) -> dict[str, object]: - """Look up many keys with GraphQL aliases in fixed-size batches.""" - results: dict[str, object] = {} - for chunk_start in range(0, len(keys), batch_size): - chunk = keys[chunk_start : chunk_start + batch_size] - parts: list[str] = [] - for index, key in enumerate(chunk): - alias = build_alias(index, key) - if alias is not None: - parts.append(f"q{index}: {alias}") - if not parts: - continue - data = post("query { " + " ".join(parts) + " }") - for index, key in enumerate(chunk): - node = json_dict(data.get(f"q{index}")) - if not node: - continue - value = parse_node(node) - if value is not None: - results[key] = value - return results - - -def _fetch_remaining_pages( - execute: Callable[[JSONDict], JSONDict], - data: JSONDict, - variables: JSONDict, - *, - after_variable: str, - query_uses_after_variable: bool, -) -> None: - paths = _next_page_paths(data) - if not paths: - return - if len(paths) > 1: - joined = ", ".join(".".join(path) for path in paths) - raise GraphQLError(f"GraphQL query returned multiple paginated node lists: {joined}") - if not query_uses_after_variable: - raise GraphQLError(f"GraphQL query returned more pages but does not use ${after_variable}") - - path = paths[0] - target_page = _node_page_at_path(data, path) - target_nodes = json_list(target_page.get("nodes")) - page_info = json_dict(target_page.get("pageInfo")) - after = _next_page_cursor(page_info, path, variables.get(after_variable)) - - while after: - page_variables = dict(variables) - page_variables[after_variable] = after - next_data = execute(page_variables) - next_page = _node_page_at_path(next_data, path) - target_nodes.extend(json_list(next_page.get("nodes"))) - target_page["nodes"] = target_nodes - target_page["pageInfo"] = next_page.get("pageInfo") - - next_page_info = json_dict(next_page.get("pageInfo")) - has_next_page = next_page_info.get("hasNextPage") - if not isinstance(has_next_page, bool): - raise GraphQLError( - f"GraphQL pagination path {'.'.join(path)} missing pageInfo.hasNextPage" - ) - if not has_next_page: - return - after = _next_page_cursor(next_page_info, path, after) - - -def _next_page_paths(data: JSONDict) -> list[tuple[str, ...]]: - paths: list[tuple[str, ...]] = [] - - def visit(value: object, path: tuple[str, ...]) -> None: - if isinstance(value, dict): - mapping = cast(JSONDict, value) - page_info = json_dict(mapping.get("pageInfo")) - if isinstance(mapping.get("nodes"), list) and page_info.get("hasNextPage") is True: - paths.append(path) - return - for key, child in mapping.items(): - visit(child, (*path, key)) - elif isinstance(value, list): - for child in cast(list[object], value): - visit(child, path) - - visit(data, ()) - return paths - - -def _node_page_at_path(data: JSONDict, path: tuple[str, ...]) -> JSONDict: - current: object = data - for key in path: - current = json_dict(current).get(key) - page = json_dict(current) - if not page: - raise GraphQLError(f"GraphQL response did not include pagination path {_path_label(path)}") - return page - - -def _next_page_cursor(page_info: JSONDict, path: tuple[str, ...], current_cursor: object) -> str: - next_cursor = json_str(page_info, "endCursor") - if not next_cursor: - raise GraphQLError( - f"GraphQL pagination path {_path_label(path)} missing pageInfo.endCursor" - ) - if isinstance(current_cursor, str) and next_cursor == current_cursor: - raise GraphQLError( - f"GraphQL pagination path {_path_label(path)} stalled: " - f"pageInfo.endCursor did not advance from {current_cursor!r}" - ) - return next_cursor - - -def _path_label(path: tuple[str, ...]) -> str: - return ".".join(path) or "" - - -def _query_uses_variable(query: str, variable: str) -> bool: - return re.search(rf"\${re.escape(variable)}\b", query) is not None diff --git a/git-subtree/src-py-lib/src/src_py_lib/clients/linear.py b/git-subtree/src-py-lib/src/src_py_lib/clients/linear.py deleted file mode 100644 index fe81257..0000000 --- a/git-subtree/src-py-lib/src/src_py_lib/clients/linear.py +++ /dev/null @@ -1,101 +0,0 @@ -"""Linear GraphQL API client.""" - -from __future__ import annotations - -from collections.abc import Mapping -from dataclasses import dataclass, field - -from src_py_lib.clients.graphql import GraphQLClient -from src_py_lib.utils.config import Config, config_field -from src_py_lib.utils.http import HTTPClient -from src_py_lib.utils.json_types import JSONDict, JSONValue, json_dict, json_dicts - -LINEAR_API_URL = "https://api.linear.app/graphql" -LINEAR_VALIDATE_QUERY = """ -query LinearClientValidate { - viewer { - email - } -} -""" -LINEAR_USERS_QUERY = """ -query LinearUsers($first: Int!, $after: String) { - users(first: $first, after: $after, includeArchived: true) { - nodes { - id - name - displayName - email - teamMemberships(first: 25) { - nodes { - team { - id - key - name - } - } - } - } - pageInfo { - hasNextPage - endCursor - } - } -} -""" - - -class LinearClientConfig(Config): - """Config fields needed to build a Linear API client.""" - - linear_api_token: str = config_field( - default="", - env_var="LINEAR_API_TOKEN", - cli_flag="--linear-api-token", - metavar="TOKEN", - help="Linear API token or op:// secret reference", - secret=True, - required=True, - ) - - -@dataclass -class LinearClient: - token: str - http: HTTPClient = field(default_factory=HTTPClient) - - def graphql( - self, - query: str, - variables: Mapping[str, JSONValue] | None = None, - *, - page_size: int | None = None, - ) -> JSONDict: - - return GraphQLClient( - url=LINEAR_API_URL, - headers={"Authorization": self.token}, - label="Linear", - http=self.http, - ).execute(query, variables=variables, page_size=page_size) - - def validate(self) -> JSONDict: - """Validate the token with a cheap viewer query and return the viewer.""" - viewer = json_dict(self.graphql(LINEAR_VALIDATE_QUERY).get("viewer")) - if not viewer.get("email"): - raise RuntimeError("Linear viewer response did not include viewer.email.") - return viewer - - def list_users(self, *, page_size: int = 100) -> list[JSONDict]: - """Return every Linear user with common people-directory fields.""" - data = self.graphql(LINEAR_USERS_QUERY, page_size=page_size) - return json_dicts(json_dict(data.get("users")).get("nodes")) - - -def linear_client_from_config( - config: LinearClientConfig, *, http: HTTPClient | None = None -) -> LinearClient: - """Return a Linear API client from shared Linear Config fields.""" - if http is None: - return LinearClient(config.linear_api_token) - return LinearClient(config.linear_api_token, http=http) diff --git a/git-subtree/src-py-lib/src/src_py_lib/clients/one_password.py b/git-subtree/src-py-lib/src/src_py_lib/clients/one_password.py deleted file mode 100644 index de7edb5..0000000 --- a/git-subtree/src-py-lib/src/src_py_lib/clients/one_password.py +++ /dev/null @@ -1,95 +0,0 @@ -"""Small 1Password CLI client.""" - -from __future__ import annotations - -import json -import subprocess -from dataclasses import dataclass - -from src_py_lib.utils.json_types import JSONDict, json_dict - - -class OnePasswordError(RuntimeError): - """Raised when resolving a 1Password reference fails.""" - - -@dataclass(frozen=True) -class OnePasswordClient: - """Resolve `op://...` references through the `op` CLI.""" - - op_binary: str = "op" - - def signin(self) -> JSONDict: - """Run an interactive 1Password CLI sign-in, then return account info.""" - try: - subprocess.run([self.op_binary, "signin"], check=True) - except FileNotFoundError as exception: - raise OnePasswordError("1Password CLI (`op`) was not found on PATH.") from exception - except subprocess.CalledProcessError as exception: - stderr = exception.stderr.strip() if isinstance(exception.stderr, str) else "" - raise OnePasswordError( - f"Failed to sign in to 1Password CLI (`op`): {stderr or exception}" - ) from exception - - return self.validate() - - def validate(self) -> JSONDict: - """Validate that the 1Password CLI is authenticated and return account info.""" - try: - result = subprocess.run( - [self.op_binary, "whoami", "--format", "json"], - check=True, - text=True, - capture_output=True, - ) - except FileNotFoundError as exception: - raise OnePasswordError("1Password CLI (`op`) was not found on PATH.") from exception - except subprocess.CalledProcessError as exception: - stderr = exception.stderr.strip() - raise OnePasswordError( - f"1Password CLI (`op`) is not authenticated: {stderr or exception}" - ) from exception - - try: - account = json_dict(json.loads(result.stdout)) - except json.JSONDecodeError as exception: - raise OnePasswordError("`op whoami --format json` returned invalid JSON") from exception - if not account: - raise OnePasswordError("`op whoami --format json` returned an empty account") - return account - - def read(self, secret_ref: str) -> str: - """Return the resolved value for one `op://...` reference.""" - try: - result = subprocess.run( - [self.op_binary, "read", secret_ref], - check=True, - text=True, - capture_output=True, - ) - except FileNotFoundError as exception: - raise OnePasswordError("1Password CLI (`op`) was not found on PATH.") from exception - except subprocess.CalledProcessError as exception: - stderr = exception.stderr.strip() - raise OnePasswordError( - f"Failed to resolve 1Password reference: {stderr or exception}" - ) from exception - - secret = result.stdout.strip() - if not secret: - raise OnePasswordError(f"`op read {secret_ref}` returned an empty value") - return secret - - -def resolve_op_secret_ref(value: str, *, client: OnePasswordClient | None = None) -> str: - """Resolve `value` if it is an `op://...` reference; otherwise return it. - - This is useful for config values that may be either a raw value or a - 1Password reference. The resolved value is returned, not logged. - """ - stripped = value.strip() - if not stripped: - raise OnePasswordError("1Password reference value is empty") - if not stripped.startswith("op://"): - return stripped - return (client or OnePasswordClient()).read(stripped) diff --git a/git-subtree/src-py-lib/src/src_py_lib/clients/slack.py b/git-subtree/src-py-lib/src/src_py_lib/clients/slack.py deleted file mode 100644 index 9ad660f..0000000 --- a/git-subtree/src-py-lib/src/src_py_lib/clients/slack.py +++ /dev/null @@ -1,146 +0,0 @@ -"""Slack Web API client with cursor pagination and rate-limit handling.""" - -from __future__ import annotations - -import logging -import threading -import time -from dataclasses import dataclass, field -from typing import Any, Final, cast - -from src_py_lib.utils.config import Config, config_field -from src_py_lib.utils.http import HTTPClient, HTTPClientError, retry_after_seconds -from src_py_lib.utils.json_types import JSONDict, json_dict, json_list, json_str - -SLACK_API_URL: Final[str] = "https://slack.com/api" -DEFAULT_PAGE_LIMIT: Final[int] = 200 -DEFAULT_METHOD_INTERVAL_SECONDS: Final[float] = 1.3 - -logger = logging.getLogger(__name__) - - -class SlackError(RuntimeError): - """Raised for Slack API errors.""" - - -class SlackClientConfig(Config): - """Config fields needed to build a Slack API client.""" - - slack_bot_token: str = config_field( - default="", - env_var="SLACK_BOT_TOKEN", - cli_flag="--slack-bot-token", - metavar="TOKEN", - help="Slack bot token or op:// secret reference", - secret=True, - required=True, - ) - - -@dataclass -class SlackPacer: - """Reserve spaced request slots per Slack method to avoid 429 bursts.""" - - default_interval_seconds: float = DEFAULT_METHOD_INTERVAL_SECONDS - method_intervals: dict[str, float] = field(default_factory=lambda: cast(dict[str, float], {})) - _lock: threading.Lock = field(default_factory=threading.Lock, init=False) - _next_slot: dict[str, float] = field( - default_factory=lambda: cast(dict[str, float], {}), init=False - ) - - def wait_for_slot(self, method: str) -> None: - interval = self.method_intervals.get(method, self.default_interval_seconds) - with self._lock: - now = time.time() - slot = max(self._next_slot.get(method, 0.0), now) - self._next_slot[method] = slot + interval - delay = slot - time.time() - if delay > 0: - time.sleep(delay) - - def bump_after_rate_limit(self, method: str, wait_seconds: float) -> None: - with self._lock: - self._next_slot[method] = max( - self._next_slot.get(method, 0.0), time.time() + wait_seconds - ) - - -@dataclass -class SlackClient: - token: str - http: HTTPClient = field(default_factory=lambda: HTTPClient(max_attempts=1)) - pacer: SlackPacer = field(default_factory=SlackPacer) - - def get(self, method: str, params: dict[str, Any] | None = None) -> JSONDict: - while True: - self.pacer.wait_for_slot(method) - try: - data = self.http.json( - "GET", - f"{SLACK_API_URL}/{method}", - headers={"Authorization": f"Bearer {self.token}"}, - query=params or {}, - ) - except HTTPClientError as exception: - if exception.status_code == 429: - wait_seconds = retry_after_seconds(exception.headers.get("retry-after")) or 5.0 - logger.warning("Slack %s rate-limited; sleeping %.0fs.", method, wait_seconds) - self.pacer.bump_after_rate_limit(method, wait_seconds) - continue - raise SlackError(f"Slack request to {method} failed: {exception}") from exception - if data.get("ok") is True: - return data - if data.get("error") == "ratelimited": - self.pacer.bump_after_rate_limit(method, 5) - continue - raise SlackError(f"Slack API error on {method}: {data.get('error')}") - - def paginate( - self, - method: str, - *, - collection_key: str, - params: dict[str, Any] | None = None, - limit: int = DEFAULT_PAGE_LIMIT, - ) -> list[JSONDict]: - out: list[JSONDict] = [] - cursor = "" - while True: - page_params = {**(params or {}), "limit": limit} - if cursor: - page_params["cursor"] = cursor - data = self.get(method, page_params) - out.extend(json_dict(item) for item in json_list(data.get(collection_key))) - cursor = json_str(json_dict(data.get("response_metadata")), "next_cursor") - if not cursor: - return out - - def list_users(self) -> list[JSONDict]: - return self.paginate("users.list", collection_key="members") - - def validate(self) -> JSONDict: - """Validate the token with Slack auth.test and return the response.""" - data = self.get("auth.test") - if not json_str(data, "user_id"): - raise SlackError("Slack auth.test response did not include user_id.") - return data - - def workspace_url(self) -> str: - url = json_str(self.get("auth.test"), "url").strip().rstrip("/") - if not url: - raise SlackError("Slack auth.test response did not include workspace URL.") - return url - - -def slack_client_from_config( - config: SlackClientConfig, - *, - http: HTTPClient | None = None, - pacer: SlackPacer | None = None, -) -> SlackClient: - """Return a Slack API client from shared Slack Config fields.""" - return SlackClient( - config.slack_bot_token, - http=http or HTTPClient(max_attempts=1), - pacer=pacer or SlackPacer(), - ) diff --git a/git-subtree/src-py-lib/src/src_py_lib/clients/sourcegraph.py b/git-subtree/src-py-lib/src/src_py_lib/clients/sourcegraph.py deleted file mode 100644 index ec9b158..0000000 --- a/git-subtree/src-py-lib/src/src_py_lib/clients/sourcegraph.py +++ /dev/null @@ -1,127 +0,0 @@ -"""Sourcegraph GraphQL API client.""" - -from __future__ import annotations - -from collections.abc import Iterator, Mapping, Sequence -from dataclasses import dataclass, field -from urllib.parse import urlsplit - -from src_py_lib.clients.graphql import GraphQLClient, stream_connection_nodes -from src_py_lib.utils.config import Config, config_field -from src_py_lib.utils.http import HTTPClient -from src_py_lib.utils.json_types import JSONDict, JSONValue, json_dict - -DEFAULT_SOURCEGRAPH_ENDPOINT = "https://sourcegraph.com" -SOURCEGRAPH_VALIDATE_QUERY = """ -query SourcegraphClientValidate { - currentUser { - username - } -} -""" - - -def normalize_sourcegraph_endpoint(endpoint: str, *, require_https: bool = False) -> str: - """Return a stable Sourcegraph base URL, or raise ValueError.""" - normalized_endpoint = endpoint.strip().rstrip("/") - endpoint_parts = urlsplit(normalized_endpoint) - if require_https and endpoint_parts.scheme != "https": - raise ValueError( - f"Sourcegraph endpoint must be an https:// URL (got {endpoint_parts.scheme!r})" - ) - if endpoint_parts.scheme not in {"http", "https"}: - raise ValueError( - "Sourcegraph endpoint must be an http:// or https:// URL " - f"(got {endpoint_parts.scheme!r})" - ) - if not endpoint_parts.hostname: - raise ValueError( - f"could not parse hostname from Sourcegraph endpoint {normalized_endpoint!r}" - ) - return normalized_endpoint - - -class SourcegraphClientConfig(Config): - """Config fields needed to build a Sourcegraph API client.""" - - src_endpoint: str = config_field( - default=DEFAULT_SOURCEGRAPH_ENDPOINT, - env_var="SRC_ENDPOINT", - cli_flag="--src-endpoint", - metavar="URL", - help=f"Sourcegraph instance URL (default: {DEFAULT_SOURCEGRAPH_ENDPOINT})", - ) - src_access_token: str = config_field( - default="", - env_var="SRC_ACCESS_TOKEN", - cli_flag="--src-access-token", - metavar="TOKEN", - help="Sourcegraph access token, or op:// secret reference", - secret=True, - required=True, - ) - - -@dataclass -class SourcegraphClient: - """Small Sourcegraph GraphQL client. - - `endpoint` should be the instance base URL, for example - `https://sourcegraph.example.com`. - """ - - endpoint: str - token: str - http: HTTPClient = field(default_factory=HTTPClient) - - def __post_init__(self) -> None: - self.endpoint = normalize_sourcegraph_endpoint(self.endpoint) - - def graphql(self, query: str, variables: Mapping[str, JSONValue] | None = None) -> JSONDict: - return self._client().execute(query, variables) - - def stream_connection_nodes( - self, - query: str, - variables: Mapping[str, JSONValue] | None = None, - *, - connection_path: Sequence[str], - page_size: int | None = None, - first_variable: str = "first", - after_variable: str = "after", - ) -> Iterator[JSONDict]: - """Stream one Sourcegraph GraphQL connection's nodes.""" - return stream_connection_nodes( - self.graphql, - query, - variables, - connection_path=connection_path, - page_size=page_size, - first_variable=first_variable, - after_variable=after_variable, - ) - - def validate(self) -> JSONDict: - """Validate the token with a cheap current user query and return the user.""" - current_user = json_dict(self.graphql(SOURCEGRAPH_VALIDATE_QUERY).get("currentUser")) - if not current_user.get("username"): - raise RuntimeError( - "Sourcegraph current user response did not include currentUser.username." - ) - return current_user - - def _client(self) -> GraphQLClient: - return GraphQLClient( - url=f"{self.endpoint}/.api/graphql", - headers={"Authorization": f"token {self.token}"}, - label="Sourcegraph", - http=self.http, - ) - - -def sourcegraph_client_from_config(config: SourcegraphClientConfig) -> SourcegraphClient: - """Return a Sourcegraph API client from shared Sourcegraph Config fields.""" - return SourcegraphClient( - endpoint=config.src_endpoint, - token=config.src_access_token, - ) diff --git a/git-subtree/src-py-lib/src/src_py_lib/py.typed b/git-subtree/src-py-lib/src/src_py_lib/py.typed deleted file mode 100644 index e69de29..0000000 diff --git a/git-subtree/src-py-lib/src/src_py_lib/utils/__init__.py b/git-subtree/src-py-lib/src/src_py_lib/utils/__init__.py deleted file mode 100644 index 4f3dc23..0000000 --- a/git-subtree/src-py-lib/src/src_py_lib/utils/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -"""Foundational utilities used by src_py_lib clients and consumers.""" - -from __future__ import annotations diff --git a/git-subtree/src-py-lib/src/src_py_lib/utils/config.py b/git-subtree/src-py-lib/src/src_py_lib/utils/config.py deleted file mode 100644 index f186a3c..0000000 --- a/git-subtree/src-py-lib/src/src_py_lib/utils/config.py +++ /dev/null @@ -1,603 +0,0 @@ -"""Pydantic-backed Config loading for small CLIs and scripts. - -Config values use this precedence: - - code defaults < .env file < shell environment < CLI flags - -Any field may hold a raw value or an `op://...` reference. Mark truly sensitive -fields with `secret=True` so snapshots redact them after references are resolved. -""" - -from __future__ import annotations - -import argparse -import os -from collections.abc import Iterable, Mapping, Sequence -from dataclasses import dataclass, replace -from pathlib import Path -from types import UnionType -from typing import Any, Final, Literal, TypeVar, Union, cast, get_args, get_origin - -from dotenv import dotenv_values -from pydantic import BaseModel, ConfigDict, Field, ValidationError -from pydantic.config import JsonDict -from pydantic.fields import FieldInfo - -from src_py_lib.clients.one_password import ( - OnePasswordClient, - OnePasswordError, - resolve_op_secret_ref, -) - -DEFAULT_CONFIG_ENV_FILE: Final[Path] = Path(".env") -CONFIG_HELP_MIN_POSITION: Final[int] = 24 -CONFIG_HELP_MAX_POSITION_LIMIT: Final[int] = 48 -CONFIG_HELP_PADDING: Final[int] = 4 -_CONFIG_OPTION_KEY: Final[str] = "src_py_lib_config_option" -_MISSING: Final[object] = object() - - -class ConfigHelpFormatter(argparse.RawTextHelpFormatter): - """Help formatter for Config-backed CLIs.""" - - def __init__( - self, - prog: str, - indent_increment: int = 2, - max_help_position: int = CONFIG_HELP_MIN_POSITION, - width: int | None = None, - ) -> None: - super().__init__(prog, indent_increment, max_help_position, width) - - -class ConfigError(RuntimeError): - """Raised when Config loading, validation, or reference resolution fails.""" - - -@dataclass(frozen=True) -class ConfigOption: - """Environment and CLI metadata for one Config field.""" - - field_name: str - env_var: str - cli_flag: str = "" - cli_aliases: tuple[str, ...] = () - cli_action: Literal["auto", "store_true", "store_false"] = "auto" - cli_nargs: str | int | None = None - cli_const: object | None = None - metavar: str | None = None - help: str = "" - secret: bool = False - required: bool = False - - -class Config(BaseModel): - """Base class for project-specific Pydantic Config models.""" - - model_config = ConfigDict(extra="forbid") - - -ConfigType = TypeVar("ConfigType", bound=Config) - - -def config_field( - *, - default: Any = ..., - env_var: str, - cli_flag: str | None = None, - cli_aliases: Sequence[str] = (), - cli_action: Literal["auto", "store_true", "store_false"] = "auto", - cli_nargs: str | int | None = None, - cli_const: object | None = None, - metavar: str | None = None, - help: str = "", - secret: bool = False, - required: bool = False, - gt: int | float | None = None, - ge: int | float | None = None, - lt: int | float | None = None, - le: int | float | None = None, - pattern: str | None = None, -) -> Any: - """Return a Pydantic field with Config environment and CLI metadata.""" - option = ConfigOption( - field_name="", - env_var=env_var, - cli_flag=cli_flag or "", - cli_aliases=tuple(cli_aliases), - cli_action=cli_action, - cli_nargs=cli_nargs, - cli_const=cli_const, - metavar=metavar, - help=help, - secret=secret, - required=required, - ) - field_kwargs: dict[str, Any] = { - "description": help or None, - "json_schema_extra": cast(JsonDict, {_CONFIG_OPTION_KEY: _config_option_payload(option)}), - } - if gt is not None: - field_kwargs["gt"] = gt - if ge is not None: - field_kwargs["ge"] = ge - if lt is not None: - field_kwargs["lt"] = lt - if le is not None: - field_kwargs["le"] = le - if pattern is not None: - field_kwargs["pattern"] = pattern - return Field(default, **field_kwargs) - - -def config_options(config_cls: type[Config]) -> tuple[ConfigOption, ...]: - """Return all Config-backed fields declared on `config_cls`.""" - options: list[ConfigOption] = [] - for field_name, field_info in config_cls.model_fields.items(): - option = _config_option_from_field(field_name, field_info) - if option is not None: - options.append(option) - return tuple(options) - - -def load_config_env_file(path: Path | None) -> dict[str, str]: - """Load key/value pairs from a `.env` file. - - Missing files are ignored. Bare keys without `=` are ignored. - """ - if path is None or not path.exists(): - return {} - return {key: value for key, value in dotenv_values(path).items() if value is not None} - - -def load_config( - config_cls: type[ConfigType], - *, - env_file: Path | None = DEFAULT_CONFIG_ENV_FILE, - cli_overrides: Mapping[str, object] | None = None, - env: Mapping[str, str] | None = None, - base_dir: Path | None = None, - resolve_op_refs: bool = False, - op_client: OnePasswordClient | None = None, - require: Iterable[str] = (), -) -> ConfigType: - """Load, merge, and validate a Config model.""" - base = Path.cwd() if base_dir is None else base_dir - resolved_env_file = _path_for_source(env_file, base) if env_file is not None else None - env_file_values = load_config_env_file(resolved_env_file) - shell_values = os.environ if env is None else env - override_values = cli_overrides or {} - - values: dict[str, object] = {} - for option in config_options(config_cls): - raw = _selected_raw_value(option, env_file_values, shell_values, override_values) - if raw is not _MISSING: - if resolve_op_refs: - raw = _resolve_source_ref(option, raw, client=op_client) - field_info = config_cls.model_fields[option.field_name] - values[option.field_name] = _prepare_source_value(raw, field_info.annotation, base) - - try: - config = config_cls.model_validate(values) - except ValidationError as exception: - raise ConfigError(f"Invalid Config: {exception}") from exception - - required = tuple(option.field_name for option in config_options(config_cls) if option.required) - require_config_values(config, (*required, *tuple(require))) - return config - - -def add_config_arguments( - parser: argparse.ArgumentParser, - config_cls: type[Config], - *, - include_env_file: bool = True, -) -> None: - """Add Config CLI flags to an argparse parser.""" - group = parser.add_argument_group( - "Config", - "These options override matching environment variables and .env values", - ) - if include_env_file: - group.add_argument( - "--env-file", - dest="env_file", - default=None, - metavar="PATH", - help="Read Config .env values from PATH (default: .env)", - ) - - for option in config_options(config_cls): - field_info = config_cls.model_fields[option.field_name] - argument_kwargs: dict[str, Any] = { - "dest": option.field_name, - "default": None, - "help": option.help, - } - if option.metavar is not None: - argument_kwargs["metavar"] = option.metavar - if option.cli_nargs is not None: - argument_kwargs["nargs"] = option.cli_nargs - if option.cli_const is not None: - argument_kwargs["const"] = option.cli_const - if _is_bool_annotation(field_info.annotation): - if option.cli_action == "auto": - argument_kwargs["action"] = argparse.BooleanOptionalAction - else: - argument_kwargs["action"] = option.cli_action - group.add_argument(option.cli_flag, *option.cli_aliases, **argument_kwargs) - - -def config_parse_args( - config_cls: type[ConfigType], - *, - parser: argparse.ArgumentParser | None = None, - argv: Sequence[str] | None = None, - description: str | None = None, - include_env_file: bool = True, - env: Mapping[str, str] | None = None, - base_dir: Path | None = None, - resolve_op_refs: bool = True, - op_client: OnePasswordClient | None = None, - require: Iterable[str] = (), -) -> ConfigType: - """Parse Config CLI flags and return a validated Config model.""" - max_help_position = _config_help_max_position(config_cls, include_env_file=include_env_file) - argument_parser = parser or argparse.ArgumentParser( - description=description, - formatter_class=_config_help_formatter(max_help_position), - ) - add_config_arguments(argument_parser, config_cls, include_env_file=include_env_file) - args = argument_parser.parse_args(argv) - try: - return load_config_from_args( - config_cls, - args, - env=env, - base_dir=base_dir, - resolve_op_refs=resolve_op_refs, - op_client=op_client, - require=require, - ) - except ConfigError as exception: - argument_parser.error(str(exception)) - - -def _config_help_formatter(max_help_position: int) -> type[argparse.HelpFormatter]: - """Return a formatter class with this parser's computed help position.""" - - class DynamicConfigHelpFormatter(ConfigHelpFormatter): - def __init__(self, prog: str) -> None: - super().__init__(prog, max_help_position=max_help_position) - - return DynamicConfigHelpFormatter - - -def _config_help_max_position( - config_cls: type[Config], - *, - include_env_file: bool, -) -> int: - """Return help-column width based on this Config's CLI arguments.""" - invocation_lengths = [len("--env-file PATH")] if include_env_file else [] - invocation_lengths.extend( - _config_option_invocation_length(config_cls, option) - for option in config_options(config_cls) - ) - longest_invocation = max(invocation_lengths, default=0) - return min( - max(CONFIG_HELP_MIN_POSITION, longest_invocation + CONFIG_HELP_PADDING), - CONFIG_HELP_MAX_POSITION_LIMIT, - ) - - -def _config_option_invocation_length(config_cls: type[Config], option: ConfigOption) -> int: - """Return argparse-style option invocation length for help alignment.""" - field_info = config_cls.model_fields[option.field_name] - option_strings = _config_option_strings(option, field_info) - if _config_option_takes_value(option, field_info): - arguments = _config_option_arguments(option) - return len(", ".join(f"{option_string} {arguments}" for option_string in option_strings)) - return len(", ".join(option_strings)) - - -def _config_option_strings(option: ConfigOption, field_info: FieldInfo) -> tuple[str, ...]: - """Return option strings as argparse will display them.""" - if _is_bool_annotation(field_info.annotation) and option.cli_action == "auto": - long_options = tuple( - f"--no-{option_string.removeprefix('--')}" - for option_string in (option.cli_flag, *option.cli_aliases) - if option_string.startswith("--") - ) - return (option.cli_flag, *long_options, *option.cli_aliases) - return (option.cli_flag, *option.cli_aliases) - - -def _config_option_takes_value(option: ConfigOption, field_info: FieldInfo) -> bool: - """Return whether argparse displays a value placeholder for this option.""" - if not _is_bool_annotation(field_info.annotation): - return True - return option.cli_action == "auto" and option.cli_nargs is not None - - -def _config_option_arguments(option: ConfigOption) -> str: - """Return the argparse-style value placeholder for this option.""" - metavar = option.metavar or option.field_name.upper() - if option.cli_nargs == "?": - return f"[{metavar}]" - if option.cli_nargs == "*": - return f"[{metavar} ...]" - if option.cli_nargs == "+": - return f"{metavar} [{metavar} ...]" - if isinstance(option.cli_nargs, int): - return " ".join(metavar for _ in range(option.cli_nargs)) - return metavar - - -def config_overrides_from_args( - config_cls: type[Config], args: argparse.Namespace -) -> dict[str, object]: - """Return Config CLI overrides from parsed argparse args.""" - overrides: dict[str, object] = {} - for option in config_options(config_cls): - value = getattr(args, option.field_name, None) - if value is not None: - overrides[option.field_name] = value - return overrides - - -def config_env_file_from_args(args: argparse.Namespace, *, attr: str = "env_file") -> Path | None: - """Return the Config `.env` path from parsed argparse args, when supplied.""" - value = getattr(args, attr, None) - return Path(cast(str, value)).expanduser() if value else None - - -def load_config_from_args( - config_cls: type[ConfigType], - args: argparse.Namespace, - *, - env: Mapping[str, str] | None = None, - base_dir: Path | None = None, - resolve_op_refs: bool = True, - op_client: OnePasswordClient | None = None, - require: Iterable[str] = (), -) -> ConfigType: - """Load Config using argparse values produced by `add_config_arguments`. - - Secret references are resolved by default because CLI entrypoints usually need - ready-to-use Config values. - """ - return load_config( - config_cls, - env_file=config_env_file_from_args(args) or DEFAULT_CONFIG_ENV_FILE, - cli_overrides=config_overrides_from_args(config_cls, args), - env=env, - base_dir=base_dir, - resolve_op_refs=resolve_op_refs, - op_client=op_client, - require=require, - ) - - -def require_config_values(config: Config, fields: Iterable[str]) -> None: - """Raise when any named Config field or environment variable is missing.""" - missing: list[str] = [] - options = config_options(type(config)) - for name in dict.fromkeys(fields): - option = _option_by_name(options, name) - value = getattr(config, option.field_name) - if _value_is_missing(value): - missing.append(option.env_var) - if missing: - raise ConfigError("Missing required Config value(s): " + ", ".join(missing)) - - -def resolve_config_refs( - config: ConfigType, - *, - fields: Iterable[str] | None = None, - client: OnePasswordClient | None = None, -) -> ConfigType: - """Resolve `op://...` string fields and return an updated Config.""" - selected = set(fields) if fields is not None else None - updates: dict[str, str] = {} - for option in config_options(type(config)): - if not _option_is_selected(option, selected): - continue - value = getattr(config, option.field_name) - if not isinstance(value, str) or not value.strip().startswith("op://"): - continue - try: - updates[option.field_name] = resolve_op_secret_ref(value, client=client) - except OnePasswordError as exception: - raise ConfigError(f"Failed to resolve {option.env_var}: {exception}") from exception - if not updates: - return config - return config.model_copy(update=updates) - - -def config_snapshot(config: Config) -> dict[str, object]: - """Return a Config snapshot with secret values reduced to safe states.""" - snapshot: dict[str, object] = {} - for option in sorted(config_options(type(config)), key=lambda option: option.env_var): - value = getattr(config, option.field_name) - snapshot[option.env_var] = _secret_state(value) if option.secret else _snapshot_value(value) - return snapshot - - -def _config_option_from_field(field_name: str, field_info: FieldInfo) -> ConfigOption | None: - extra = field_info.json_schema_extra - if not isinstance(extra, Mapping): - return None - option_payload = cast(Mapping[str, object], extra).get(_CONFIG_OPTION_KEY) - if not isinstance(option_payload, Mapping): - return None - option = _config_option_from_payload(cast(Mapping[str, object], option_payload)) - if option is None: - return None - return replace( - option, - field_name=field_name, - cli_flag=option.cli_flag or f"--{field_name.replace('_', '-')}", - ) - - -def _config_option_payload(option: ConfigOption) -> dict[str, object]: - return { - "env_var": option.env_var, - "cli_flag": option.cli_flag, - "cli_aliases": list(option.cli_aliases), - "cli_action": option.cli_action, - "cli_nargs": option.cli_nargs, - "cli_const": option.cli_const, - "metavar": option.metavar, - "help": option.help, - "secret": option.secret, - "required": option.required, - } - - -def _config_option_from_payload(payload: Mapping[str, object]) -> ConfigOption | None: - env_var = payload.get("env_var") - if not isinstance(env_var, str) or not env_var: - return None - cli_flag = payload.get("cli_flag") - cli_aliases = payload.get("cli_aliases") - cli_action = payload.get("cli_action") - cli_nargs = payload.get("cli_nargs") - metavar = payload.get("metavar") - help_text = payload.get("help") - return ConfigOption( - field_name="", - env_var=env_var, - cli_flag=cli_flag if isinstance(cli_flag, str) else "", - cli_aliases=_string_tuple(cli_aliases), - cli_action=_cli_action(cli_action), - cli_nargs=cli_nargs if isinstance(cli_nargs, str | int) else None, - cli_const=payload.get("cli_const"), - metavar=metavar if isinstance(metavar, str) else None, - help=help_text if isinstance(help_text, str) else "", - secret=payload.get("secret") is True, - required=payload.get("required") is True, - ) - - -def _string_tuple(value: object) -> tuple[str, ...]: - if not isinstance(value, Sequence) or isinstance(value, str | bytes): - return () - return tuple(item for item in cast(Sequence[object], value) if isinstance(item, str)) - - -def _cli_action(value: object) -> Literal["auto", "store_true", "store_false"]: - if value in {"store_true", "store_false"}: - return cast(Literal["store_true", "store_false"], value) - return "auto" - - -def _selected_raw_value( - option: ConfigOption, - env_file_values: Mapping[str, str], - shell_values: Mapping[str, str], - override_values: Mapping[str, object], -) -> object: - raw: object = _MISSING - if option.env_var in env_file_values: - raw = env_file_values[option.env_var] - if option.env_var in shell_values: - raw = shell_values[option.env_var] - if option.env_var in override_values: - raw = override_values[option.env_var] - if option.field_name in override_values: - raw = override_values[option.field_name] - return raw - - -def _resolve_source_ref( - option: ConfigOption, raw: object, *, client: OnePasswordClient | None -) -> object: - if not isinstance(raw, str) or not raw.strip().startswith("op://"): - return raw - try: - return resolve_op_secret_ref(raw, client=client) - except OnePasswordError as exception: - raise ConfigError(f"Failed to resolve {option.env_var}: {exception}") from exception - - -def _prepare_source_value(raw: object, annotation: object, base_dir: Path) -> object: - if not isinstance(raw, str): - return raw - if _is_collection_annotation(annotation): - return tuple(part.strip() for part in raw.split(",") if part.strip()) - if _is_path_annotation(annotation): - return _path_for_source(Path(raw), base_dir) - return raw - - -def _path_for_source(path: Path | str, base_dir: Path) -> Path: - expanded = Path(path).expanduser() - return expanded if expanded.is_absolute() else base_dir / expanded - - -def _without_none(annotation: object) -> object: - origin = get_origin(annotation) - if origin not in (Union, UnionType): - return annotation - args = tuple(arg for arg in get_args(annotation) if arg is not type(None)) - return args[0] if len(args) == 1 else annotation - - -def _is_bool_annotation(annotation: object) -> bool: - return _without_none(annotation) is bool - - -def _is_collection_annotation(annotation: object) -> bool: - target = _without_none(annotation) - return get_origin(target) in (list, tuple, set, frozenset) - - -def _is_path_annotation(annotation: object) -> bool: - target = _without_none(annotation) - try: - return isinstance(target, type) and issubclass(target, Path) - except TypeError: - return False - - -def _option_by_name(options: Iterable[ConfigOption], name: str) -> ConfigOption: - for option in options: - if name in {option.field_name, option.env_var}: - return option - raise ConfigError(f"Unknown Config field or environment variable: {name}") - - -def _option_is_selected(option: ConfigOption, selected: set[str] | None) -> bool: - return selected is None or option.field_name in selected or option.env_var in selected - - -def _value_is_missing(value: object) -> bool: - if value is None: - return True - if isinstance(value, str): - return not value.strip() - if isinstance(value, list | tuple | set | frozenset | dict): - return not value - return False - - -def _secret_state(value: object) -> str: - if _value_is_missing(value): - return "missing" - if isinstance(value, str) and value.strip().startswith("op://"): - return "reference" - return "provided" - - -def _snapshot_value(value: object) -> object: - if isinstance(value, Path): - return str(value) - if isinstance(value, list | tuple | set | frozenset): - items = cast(Iterable[object], value) - return [str(item) for item in items] - if isinstance(value, str | int | float | bool) or value is None: - return value - return str(value) diff --git a/git-subtree/src-py-lib/src/src_py_lib/utils/http.py b/git-subtree/src-py-lib/src/src_py_lib/utils/http.py deleted file mode 100644 index 4b2ffe9..0000000 --- a/git-subtree/src-py-lib/src/src_py_lib/utils/http.py +++ /dev/null @@ -1,279 +0,0 @@ -"""Shared HTTP transport with timeouts, retries, and useful errors.""" - -from __future__ import annotations - -import json -import logging -import random -import time -import urllib.parse -from collections.abc import Iterable, Mapping -from dataclasses import dataclass, field -from typing import Final, cast - -import httpx - -from src_py_lib.utils.json_types import JSONDict, json_dict -from src_py_lib.utils.logging import event, record_http_attempt, record_http_retry - -DEFAULT_TIMEOUT_SECONDS: Final[float] = 30.0 -DEFAULT_MAX_CONNECTIONS: Final[int] = 20 -DEFAULT_MAX_ATTEMPTS: Final[int] = 3 -DEFAULT_RETRY_BASE_DELAY_SECONDS: Final[float] = 0.5 -DEFAULT_RETRY_MAX_DELAY_SECONDS: Final[float] = 30.0 -RETRYABLE_STATUS_CODES: Final[frozenset[int]] = frozenset({408, 429, 500, 502, 503, 504}) -ERROR_BODY_PREVIEW_CHARS: Final[int] = 500 -REDACTED_HEADER_VALUE: Final[str] = "[redacted]" -SENSITIVE_HEADER_FRAGMENTS: Final[tuple[str, ...]] = ( - "api-key", - "api_key", - "authorization", - "cookie", - "password", - "secret", - "token", -) - -logger = logging.getLogger(__name__) - - -class HTTPClientError(RuntimeError): - """Raised when an HTTP request fails after retries.""" - - def __init__( - self, - message: str, - *, - status_code: int | None = None, - body: str = "", - headers: Mapping[str, str] | None = None, - ) -> None: - super().__init__(message) - self.status_code = status_code - self.body = body - self.headers = {key.lower(): value for key, value in dict(headers or {}).items()} - - -@dataclass -class HTTPClient: - """HTTPX-backed HTTP client for JSON APIs with pooled connections.""" - - timeout: float | httpx.Timeout = DEFAULT_TIMEOUT_SECONDS - user_agent: str = "src-py-lib" - max_connections: int = DEFAULT_MAX_CONNECTIONS - max_attempts: int = DEFAULT_MAX_ATTEMPTS - retry_base_delay_seconds: float = DEFAULT_RETRY_BASE_DELAY_SECONDS - retry_max_delay_seconds: float = DEFAULT_RETRY_MAX_DELAY_SECONDS - retryable_status_codes: frozenset[int] = RETRYABLE_STATUS_CODES - transport: httpx.BaseTransport | None = field(default=None, repr=False) - _client: httpx.Client = field(init=False, repr=False) - - def __post_init__(self) -> None: - if self.max_connections < 1: - raise ValueError("max_connections must be at least 1") - if self.max_attempts < 1: - raise ValueError("max_attempts must be at least 1") - self._client = httpx.Client( - timeout=self.timeout, - limits=httpx.Limits( - max_connections=self.max_connections, - max_keepalive_connections=self.max_connections, - ), - transport=self.transport, - ) - - def __enter__(self) -> HTTPClient: - return self - - def __exit__(self, *_args: object) -> None: - self.close() - - def close(self) -> None: - """Close the underlying pooled HTTP transport.""" - self._client.close() - - def request( - self, - method: str, - url: str, - *, - headers: Mapping[str, str] | None = None, - query: Mapping[str, str | int | float | bool | None] | None = None, - json_body: object | None = None, - data: bytes | None = None, - ) -> bytes: - """Make an HTTP request and return raw response bytes.""" - request_url = _with_query(url, query) - body = data - request_headers = {"User-Agent": self.user_agent, **dict(headers or {})} - if json_body is not None: - body = json.dumps(json_body).encode("utf-8") - request_headers.setdefault("Content-Type", "application/json") - for attempt in range(1, self.max_attempts + 1): - try: - with event( - "http_request", - level="debug", - method=method, - url=_safe_url(request_url), - attempt=attempt, - request_headers=_headers_for_log(request_headers), - request_bytes=len(body or b""), - ) as fields: - response = self._client.request( - method, - request_url, - headers=request_headers, - content=body, - ) - payload = response.content - fields["status_code"] = response.status_code - fields["reason_phrase"] = response.reason_phrase - fields["response_headers"] = _headers_for_log(response.headers) - fields["response_bytes"] = len(payload) - http_version = _response_http_version(response) - if http_version is not None: - fields["http_version"] = http_version - record_http_attempt( - request_bytes=len(body or b""), - response_bytes=len(payload), - status_code=response.status_code, - ) - if response.status_code >= 400: - body_text = _body_preview(payload) - if not self._should_retry(response.status_code, attempt): - raise HTTPClientError( - f"HTTP {response.status_code} for {method} " - f"{_safe_url(request_url)}: {body_text}", - status_code=response.status_code, - body=body_text, - headers=dict(response.headers), - ) - record_http_retry() - self._sleep_before_retry(attempt, response.headers.get("Retry-After")) - else: - return payload - except HTTPClientError: - raise - except httpx.TransportError as exception: - record_http_attempt(request_bytes=len(body or b""), transport_error=True) - if not self._should_retry(None, attempt): - failure = ( - "timed out" if isinstance(exception, httpx.TimeoutException) else "failed" - ) - raise HTTPClientError( - f"HTTP request {failure} for {method} {_safe_url(request_url)}: " - f"{_exception_message(exception)}" - ) from exception - record_http_retry() - self._sleep_before_retry(attempt, None) - raise AssertionError("HTTP retry loop exited without returning or raising") - - def json( - self, - method: str, - url: str, - *, - headers: Mapping[str, str] | None = None, - query: Mapping[str, str | int | float | bool | None] | None = None, - json_body: object | None = None, - ) -> JSONDict: - """Make an HTTP request and decode a JSON object response.""" - raw = self.request(method, url, headers=headers, query=query, json_body=json_body) - try: - return json_dict(json.loads(raw.decode("utf-8")) if raw else {}) - except json.JSONDecodeError as exception: - raise HTTPClientError( - f"Invalid JSON response from {method} {_safe_url(url)}" - ) from exception - - def _should_retry(self, status_code: int | None, attempt: int) -> bool: - if attempt >= self.max_attempts: - return False - return status_code is None or status_code in self.retryable_status_codes - - def _sleep_before_retry(self, attempt: int, retry_after: str | None) -> None: - delay = retry_after_seconds(retry_after) - if delay is None: - delay = min( - self.retry_base_delay_seconds * (2 ** (attempt - 1)), - self.retry_max_delay_seconds, - ) * random.uniform(0.5, 1.5) - logger.warning("HTTP request failed; retrying in %.2fs (attempt %d).", delay, attempt + 1) - time.sleep(delay) - - -def _with_query( - url: str, - query: Mapping[str, str | int | float | bool | None] | None, -) -> str: - if not query: - return url - filtered = {key: value for key, value in query.items() if value is not None} - separator = "&" if urllib.parse.urlsplit(url).query else "?" - return f"{url}{separator}{urllib.parse.urlencode(filtered)}" - - -def _safe_url(url: str) -> str: - split = urllib.parse.urlsplit(url) - return urllib.parse.urlunsplit((split.scheme, split.netloc, split.path, split.query, "")) - - -def _headers_for_log(headers: Mapping[str, str] | httpx.Headers) -> dict[str, str | list[str]]: - values: dict[str, str | list[str]] = {} - for name, value in _header_items(headers): - key = name.lower() - logged_value = REDACTED_HEADER_VALUE if _is_sensitive_header(key) else value - existing = values.get(key) - if existing is None: - values[key] = logged_value - elif isinstance(existing, list): - existing.append(logged_value) - else: - values[key] = [existing, logged_value] - return {key: values[key] for key in sorted(values)} - - -def _header_items(headers: Mapping[str, str] | httpx.Headers) -> Iterable[tuple[str, str]]: - if isinstance(headers, httpx.Headers): - return headers.multi_items() - return headers.items() - - -def _is_sensitive_header(name: str) -> bool: - lowered = name.lower() - return any(fragment in lowered for fragment in SENSITIVE_HEADER_FRAGMENTS) - - -def _response_http_version(response: httpx.Response) -> str | None: - version = response.extensions.get("http_version") - if isinstance(version, bytes): - return version.decode("latin-1", errors="replace") - if isinstance(version, str): - return version - return None - - -def _body_preview(raw: bytes) -> str: - text = raw.decode("utf-8", errors="replace").strip() - if len(text) <= ERROR_BODY_PREVIEW_CHARS: - return text - return f"{text[:ERROR_BODY_PREVIEW_CHARS]}... (+{len(text) - ERROR_BODY_PREVIEW_CHARS} chars)" - - -def _exception_message(exception: Exception) -> str: - return str(exception) or type(exception).__name__ - - -def retry_after_seconds(value: str | None) -> float | None: - if not value: - return None - try: - return max(float(value), 0.0) - except ValueError: - return None - - -def cast_json_dict(value: object) -> JSONDict: - """Compatibility wrapper for call sites that want an explicit boundary cast.""" - return cast(JSONDict, value) if isinstance(value, dict) else {} diff --git a/git-subtree/src-py-lib/src/src_py_lib/utils/json_cache.py b/git-subtree/src-py-lib/src/src_py_lib/utils/json_cache.py deleted file mode 100644 index 8847a37..0000000 --- a/git-subtree/src-py-lib/src/src_py_lib/utils/json_cache.py +++ /dev/null @@ -1,42 +0,0 @@ -"""Small on-disk JSON cache helpers.""" - -from __future__ import annotations - -import json -from collections.abc import Callable, Mapping -from pathlib import Path -from typing import Any, TypeVar, cast - -Entry = TypeVar("Entry") - - -def load_json_cache( - path: Path, - parse: Callable[[Any], Entry] | None = None, -) -> dict[str, Entry]: - """Load `path` as a string-keyed cache. Missing files return `{}`.""" - if not path.exists(): - return {} - raw = cast(dict[str, Any], json.loads(path.read_text(encoding="utf-8"))) - if parse is None: - return cast(dict[str, Entry], raw) - return {key: parse(value) for key, value in raw.items()} - - -def save_json_cache(path: Path, cache: Mapping[str, object]) -> None: - """Write a string-keyed JSON cache with stable formatting.""" - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(json.dumps(dict(cache), indent=2, sort_keys=True) + "\n", encoding="utf-8") - - -def load_json_subset( - path: Path, - keys: list[str], - parse: Callable[[Any], Entry] | None = None, -) -> dict[str, Entry]: - """Load only `keys` that are present in a string-keyed JSON cache.""" - cache = load_json_cache(path, parse=parse) - return {key: cache[key] for key in keys if key in cache} - - -__all__ = ["load_json_cache", "load_json_subset", "save_json_cache"] diff --git a/git-subtree/src-py-lib/src/src_py_lib/utils/json_types.py b/git-subtree/src-py-lib/src/src_py_lib/utils/json_types.py deleted file mode 100644 index 61b56bb..0000000 --- a/git-subtree/src-py-lib/src/src_py_lib/utils/json_types.py +++ /dev/null @@ -1,54 +0,0 @@ -"""Small JSON type aliases and projection helpers.""" - -from __future__ import annotations - -from typing import Any, TypeAlias, cast - -JSONValue: TypeAlias = None | bool | int | float | str | list["JSONValue"] | dict[str, "JSONValue"] -JSONDict: TypeAlias = dict[str, JSONValue] -JSONArray: TypeAlias = list[JSONValue] - - -def json_dict(value: object) -> JSONDict: - """Return `value` as a JSON object, or an empty object when it is not one.""" - return cast(JSONDict, value) if isinstance(value, dict) else {} - - -def json_list(value: object) -> JSONArray: - """Return `value` as a JSON array, or an empty array when it is not one.""" - return cast(JSONArray, value) if isinstance(value, list) else [] - - -def json_dicts(value: object) -> list[JSONDict]: - """Return `value` as a list of JSON objects, filtering non-objects out.""" - if not isinstance(value, list): - return [] - items = cast(list[object], value) - return [cast(JSONDict, item) for item in items if isinstance(item, dict)] - - -def json_strs(value: object) -> list[str]: - """Return `value` as a list of strings, filtering non-strings out.""" - if not isinstance(value, list): - return [] - items = cast(list[object], value) - return [item for item in items if isinstance(item, str)] - - -def json_str(mapping: JSONDict, key: str, default: str = "") -> str: - """Read a string value from a JSON object.""" - value = mapping.get(key) - return value if isinstance(value, str) else default - - -def json_int(mapping: JSONDict, key: str, default: int = 0) -> int: - """Read an integer value from a JSON object, excluding booleans.""" - value = mapping.get(key) - return value if isinstance(value, int) and not isinstance(value, bool) else default - - -def require_json_dict(value: Any, *, where: str) -> JSONDict: - """Return `value` as a JSON object, or raise a clear error.""" - if isinstance(value, dict): - return cast(JSONDict, value) - raise TypeError(f"{where} must be a JSON object") diff --git a/git-subtree/src-py-lib/src/src_py_lib/utils/logging.py b/git-subtree/src-py-lib/src/src_py_lib/utils/logging.py deleted file mode 100644 index 569f70c..0000000 --- a/git-subtree/src-py-lib/src/src_py_lib/utils/logging.py +++ /dev/null @@ -1,950 +0,0 @@ -"""Central structured logging for small CLIs and scripts. - -Use `configure_logging()` once near process startup. Other modules should use -`logging.getLogger(__name__)` for human-readable operator messages and -`event()` / `log()` for structured JSONL events. -""" - -from __future__ import annotations - -import ast -import contextlib -import contextvars -import datetime as _datetime -import json -import logging -import os -import secrets -import subprocess -import sys -import threading -import time -from collections.abc import Callable, Generator, Iterable, Mapping -from concurrent.futures import Executor, Future -from dataclasses import dataclass, field -from pathlib import Path -from typing import Any, Final, Self, cast - -if sys.platform != "win32": - import resource - -from pydantic import model_validator - -from src_py_lib.utils.config import Config, config_field, config_snapshot - -RUN: Final[str] = secrets.token_hex(4) -DEFAULT_LOGS_DIR: Final[Path] = Path("logs") -DEFAULT_RETAIN_FILES: Final[int] = 50 -DEFAULT_LOG_FILE_LEVEL: Final[str] = "debug" -SRC_LOG_LEVEL: Final[str] = "SRC_LOG_LEVEL" -SRC_LOG_VERBOSE: Final[str] = "SRC_LOG_VERBOSE" -SRC_LOG_QUIET: Final[str] = "SRC_LOG_QUIET" -SRC_LOG_SILENT: Final[str] = "SRC_LOG_SILENT" -TRACE_SPAN_BYTES: Final[int] = 4 -MEBIBYTE: Final[int] = 1024 * 1024 -SECRET_FIELD_FRAGMENTS: Final[tuple[str, ...]] = ( - "api_key", - "authorization", - "cookie", - "password", - "secret", - "token", -) -LOG_FIELD_ORDER: Final[tuple[str, ...]] = ( - "ts", - "command", - "level", - "run", - "trace", - "span", - "parent_span", - "logger", - "event", - "phase", - "stage", - "message", -) - -_STRUCTURED_EVENT_ATTR: Final[str] = "_src_py_lib_structured_event" -_STRUCTURED_FIELDS_ATTR: Final[str] = "_src_py_lib_structured_fields" -_HTTPCORE_RESPONSE_HEADERS_PREFIX: Final[str] = "receive_response_headers.complete return_value=" -_HTTPX_REQUEST_PREFIX: Final[str] = "HTTP Request: " -_HTTP_DEPENDENCY_LOGGER_PREFIXES: Final[tuple[str, ...]] = ("httpx", "httpcore") -_CONTEXT: contextvars.ContextVar[dict[str, Any]] = contextvars.ContextVar("src_py_lib_log_context") - - -@dataclass(frozen=True) -class LoggingSettings: - """Logging destinations and levels.""" - - logger_name: str = "" - terminal_level: str = "info" - log_file_level: str | None = None - log_file: Path | None = None - logs_dir: Path | None = DEFAULT_LOGS_DIR - run: str = RUN - retain_log_files: int = DEFAULT_RETAIN_FILES - suppress_http_dependency_logs: bool = True - resource_sample_interval_seconds: float | None = None - - -class LoggingConfig(Config): - """Config fields for logging-related CLI and environment options.""" - - src_log_level: str | None = config_field( - default="INFO", - env_var=SRC_LOG_LEVEL, - cli_flag="--src-log-level", - metavar="LEVEL", - help="Log level (default: INFO)", - ) - verbose: bool = config_field( - default=False, - env_var=SRC_LOG_VERBOSE, - cli_flag="--verbose", - cli_aliases=("-v",), - cli_action="store_true", - help="Alias for --src-log-level DEBUG", - ) - quiet: bool = config_field( - default=False, - env_var=SRC_LOG_QUIET, - cli_flag="--quiet", - cli_aliases=("-q",), - cli_action="store_true", - help="Alias for --src-log-level WARNING", - ) - silent: bool = config_field( - default=False, - env_var=SRC_LOG_SILENT, - cli_flag="--silent", - cli_aliases=("-s",), - cli_action="store_true", - help="Alias for --src-log-level ERROR", - ) - - @model_validator(mode="after") - def validate_log_level_alias(self) -> Self: - """Require at most one alias for the terminal/log-file level.""" - if sum((self.verbose, self.quiet, self.silent)) > 1: - raise ValueError("choose only one of --verbose/-v, --quiet/-q, or --silent/-s") - return self - - -def resolve_log_level_name( - config: object | None = None, - *, - log_level: str | None = None, - verbose: bool | None = None, - quiet: bool | None = None, - silent: bool | None = None, -) -> str | None: - """Resolve common CLI log-level alias to a level name. - - Alias flags intentionally only map to strings. Explicit log-level - values are returned unchanged so `configure_logging()` owns parsing - and fallback behavior. - """ - resolved_verbose = verbose if verbose is not None else bool(getattr(config, "verbose", False)) - resolved_quiet = quiet if quiet is not None else bool(getattr(config, "quiet", False)) - resolved_silent = silent if silent is not None else bool(getattr(config, "silent", False)) - if resolved_verbose: - return "DEBUG" - if resolved_quiet: - return "WARNING" - if resolved_silent: - return "ERROR" - if log_level is not None: - return log_level - return _src_log_level_from_config(config) - - -def logging_settings_from_config( - config: object | None = None, - *, - terminal_default: str = "INFO", - log_file_default: str | None = DEFAULT_LOG_FILE_LEVEL, - logger_name: str = "", - log_file: Path | None = None, - logs_dir: Path | None = DEFAULT_LOGS_DIR, - run: str = RUN, - retain_log_files: int = DEFAULT_RETAIN_FILES, - suppress_http_dependency_logs: bool = True, - resource_sample_interval_seconds: float | None = None, -) -> LoggingSettings: - """Return `LoggingSettings` using common CLI log-level alias.""" - explicit_level = resolve_log_level_name(config) - return LoggingSettings( - logger_name=logger_name, - terminal_level=explicit_level or terminal_default, - log_file_level=explicit_level or log_file_default, - log_file=log_file, - logs_dir=logs_dir, - run=run, - retain_log_files=retain_log_files, - suppress_http_dependency_logs=suppress_http_dependency_logs, - resource_sample_interval_seconds=resource_sample_interval_seconds, - ) - - -@dataclass(frozen=True) -class _SpanContext: - trace: str - span: str - parent_span: str | None = None - - -_SPAN_CONTEXT: contextvars.ContextVar[_SpanContext | None] = contextvars.ContextVar( - "src_py_lib_span_context", default=None -) - -_HTTP_METRICS_LOCK: Final[threading.Lock] = threading.Lock() -_HTTP_METRICS: dict[str, int] = { - "http_request_attempt_count": 0, - "http_request_bytes_total": 0, - "http_response_bytes_total": 0, - "http_retry_count": 0, - "http_2xx_count": 0, - "http_3xx_count": 0, - "http_4xx_count": 0, - "http_429_count": 0, - "http_5xx_count": 0, - "http_transport_error_count": 0, -} - - -@dataclass -class ResourceSampler: - """Emit optional process resource samples and summarize usage at run end.""" - - interval_seconds: float - _stop: threading.Event = field(init=False, default_factory=threading.Event) - _thread: threading.Thread | None = field(init=False, default=None) - _started_at: float = field(init=False, default_factory=time.perf_counter) - _last_sample_at: float = field(init=False, default_factory=time.perf_counter) - _last_cpu_seconds: float = field(init=False, default=0.0) - _start_usage: Any = field(init=False, default=None) - _peak_rss_bytes: int = field(init=False, default=0) - - def __post_init__(self) -> None: - if self.interval_seconds < 0: - raise ValueError("resource_sample_interval_seconds must be >= 0") - self._start_usage = _resource_usage() - if self._start_usage is not None: - self._last_cpu_seconds = _cpu_seconds(self._start_usage) - - def start(self) -> None: - """Start periodic sampling, if enabled by a positive interval.""" - if self.interval_seconds <= 0: - return - context = contextvars.copy_context() - self._thread = threading.Thread( - target=context.run, - args=(self._loop,), - name="ResourceSampler", - daemon=True, - ) - self._thread.start() - self.emit_sample() - - def emit_sample(self) -> None: - """Emit one DEBUG `resource_sample` event.""" - log("debug", "resource_sample", **self._sample_fields()) - - def stop_and_summary(self) -> dict[str, Any]: - """Stop periodic sampling and return run-end resource fields.""" - if self.interval_seconds > 0: - self.emit_sample() - self._stop.set() - if self._thread is not None: - self._thread.join(timeout=2.0) - usage = _resource_usage() - summary: dict[str, Any] = { - "cpu_count_logical": os.cpu_count() or 0, - "num_threads": threading.active_count(), - } - file_descriptors = _num_file_descriptors() - if file_descriptors is not None: - summary["num_fds"] = file_descriptors - rss_bytes = _rss_bytes(usage) - if rss_bytes is not None: - self._peak_rss_bytes = max(self._peak_rss_bytes, rss_bytes) - if self._peak_rss_bytes: - summary["peak_rss_mb"] = _bytes_to_mib(self._peak_rss_bytes) - if usage is not None and self._start_usage is not None: - summary["cpu_user_seconds"] = round( - float(usage.ru_utime) - float(self._start_usage.ru_utime), 3 - ) - summary["cpu_system_seconds"] = round( - float(usage.ru_stime) - float(self._start_usage.ru_stime), 3 - ) - summary["io_read_count"] = int(usage.ru_inblock) - int(self._start_usage.ru_inblock) - summary["io_write_count"] = int(usage.ru_oublock) - int(self._start_usage.ru_oublock) - return summary - - def _loop(self) -> None: - while not self._stop.wait(self.interval_seconds): - self.emit_sample() - - def _sample_fields(self) -> dict[str, Any]: - now = time.perf_counter() - usage = _resource_usage() - fields: dict[str, Any] = { - "num_threads": threading.active_count(), - } - rss_bytes = _rss_bytes(usage) - if rss_bytes is not None: - self._peak_rss_bytes = max(self._peak_rss_bytes, rss_bytes) - fields["rss_mb"] = _bytes_to_mib(rss_bytes) - file_descriptors = _num_file_descriptors() - if file_descriptors is not None: - fields["num_fds"] = file_descriptors - if usage is not None: - cpu_seconds = _cpu_seconds(usage) - elapsed = max(now - self._last_sample_at, 0.001) - fields["process_cpu_percent"] = round( - max(cpu_seconds - self._last_cpu_seconds, 0.0) / elapsed * 100.0, - 1, - ) - self._last_cpu_seconds = cpu_seconds - self._last_sample_at = now - return fields - - -class _DropStructuredEvents(logging.Filter): - def filter(self, record: logging.LogRecord) -> bool: - return not hasattr(record, _STRUCTURED_EVENT_ATTR) - - -class _DropHTTPDependencyLogs(logging.Filter): - def filter(self, record: logging.LogRecord) -> bool: - return not any( - record.name == prefix or record.name.startswith(f"{prefix}.") - for prefix in _HTTP_DEPENDENCY_LOGGER_PREFIXES - ) - - -class JSONLogFileHandler(logging.Handler): - """Write every log record as one JSON object line.""" - - def __init__(self, path: Path, *, run: str, level: int) -> None: - super().__init__(level=level) - self.path = path - self._run = run - self._lock = threading.Lock() - self._file = path.open("w", encoding="utf-8", buffering=1) - - def emit(self, record: logging.LogRecord) -> None: - try: - timestamp = _datetime.datetime.now(_datetime.UTC).isoformat(timespec="milliseconds") - structured_event = getattr(record, _STRUCTURED_EVENT_ATTR, None) - if isinstance(structured_event, str): - fields = getattr(record, _STRUCTURED_FIELDS_ATTR, {}) - structured_fields: dict[str, Any] = ( - dict(cast(Mapping[str, Any], fields)) if isinstance(fields, Mapping) else {} - ) - payload = { - "ts": timestamp, - "run": self._run, - "level": record.levelname, - "event": structured_event, - **structured_fields, - } - else: - message, log_fields = _structured_log_fields(record) - payload = { - "ts": timestamp, - "run": self._run, - "level": record.levelname, - "event": "log", - "logger": record.name, - "message": message, - } - payload.update(log_fields) - payload.update(_current_log_fields(payload)) - if record.exc_info: - payload["exc_info"] = self.format(record) - with self._lock: - self._file.write(json.dumps(_ordered_payload(payload), default=str) + "\n") - except Exception: - self.handleError(record) - - def close(self) -> None: - with contextlib.suppress(Exception), self._lock: - self._file.flush() - self._file.close() - super().close() - - -def configure_logging(config: LoggingSettings | None = None) -> Path | None: - """Configure terminal logging and optional JSON log-file logging. - - Returns the JSON log-file path when file logging is enabled. - """ - config = config or LoggingSettings() - reset_observability_metrics() - terminal_level = _log_level(config.terminal_level) - log_file_level = _log_file_level(config.log_file_level) - log_file = config.log_file - if log_file is None and config.logs_dir is not None: - log_file = default_log_file(config.logs_dir, run=config.run) - root_or_package_logger = logging.getLogger(config.logger_name) - root_or_package_logger.handlers.clear() - root_or_package_logger.setLevel( - min( - terminal_level, - log_file_level if log_file else terminal_level, - ) - ) - root_or_package_logger.propagate = False - - terminal_handler = logging.StreamHandler() - terminal_handler.setLevel(terminal_level) - terminal_handler.setFormatter(logging.Formatter("%(asctime)s [%(levelname)s] %(message)s")) - terminal_handler.addFilter(_DropStructuredEvents()) - if config.suppress_http_dependency_logs and config.logger_name == "": - terminal_handler.addFilter(_DropHTTPDependencyLogs()) - root_or_package_logger.addHandler(terminal_handler) - - if log_file is None: - return None - - log_file.parent.mkdir(parents=True, exist_ok=True) - _prune_old_log_files(log_file.parent, config.retain_log_files) - log_file_handler = JSONLogFileHandler( - log_file, - run=config.run, - level=log_file_level, - ) - if config.suppress_http_dependency_logs and config.logger_name == "": - log_file_handler.addFilter(_DropHTTPDependencyLogs()) - root_or_package_logger.addHandler(log_file_handler) - root_or_package_logger.info("Writing log events to %s.", log_file) - return log_file - - -def reset_observability_metrics() -> None: - """Reset process-wide HTTP counters used by `logging_context()` run summaries.""" - with _HTTP_METRICS_LOCK: - for metric_name in _HTTP_METRICS: - _HTTP_METRICS[metric_name] = 0 - - -def record_http_attempt( - *, - request_bytes: int, - response_bytes: int = 0, - status_code: int | None = None, - transport_error: bool = False, -) -> None: - """Record one HTTP attempt for the current run summary.""" - with _HTTP_METRICS_LOCK: - _HTTP_METRICS["http_request_attempt_count"] += 1 - _HTTP_METRICS["http_request_bytes_total"] += request_bytes - _HTTP_METRICS["http_response_bytes_total"] += response_bytes - if transport_error: - _HTTP_METRICS["http_transport_error_count"] += 1 - if status_code is None: - return - status_group = 5 if status_code >= 500 else status_code // 100 - metric_name = { - 2: "http_2xx_count", - 3: "http_3xx_count", - 4: "http_4xx_count", - 5: "http_5xx_count", - }.get(status_group) - if metric_name is not None: - _HTTP_METRICS[metric_name] += 1 - if status_code == 429: - _HTTP_METRICS["http_429_count"] += 1 - - -def record_http_retry() -> None: - """Record that an HTTP attempt will be retried.""" - with _HTTP_METRICS_LOCK: - _HTTP_METRICS["http_retry_count"] += 1 - - -def observability_summary() -> dict[str, Any]: - """Return process-wide counters accumulated since logging was configured.""" - with _HTTP_METRICS_LOCK: - return dict(_HTTP_METRICS) - - -@contextlib.contextmanager -def logging_context( - name: str, - config: object | None = None, - *, - git_cwd: Path | str | None = None, - logging_config: LoggingSettings | None = None, - run_fields: Mapping[str, Any] | None = None, - run_summary: Callable[[], Mapping[str, Any]] | None = None, -) -> Generator[Path | None]: - """Configure logging, install command context, and emit startup metadata.""" - resolved_logging_config = logging_config or LoggingSettings( - log_file_level=_src_log_level_from_config(config) - ) - log_file = configure_logging(resolved_logging_config) - sampler = _resource_sampler(resolved_logging_config) - started = time.perf_counter() - error: BaseException | None = None - with log_context(command=name): - if sampler is not None: - sampler.start() - start_fields = {"phase": "start", **dict(run_fields or {})} - info("run", logger_name=resolved_logging_config.logger_name, **start_fields) - try: - startup_event( - command=name, - config=config, - log_file=log_file, - git_cwd=_git_cwd_path(git_cwd), - logger_name=resolved_logging_config.logger_name, - ) - yield log_file - except BaseException as exception: - error = exception - raise - finally: - error_type = _run_error_type(error) - summary: dict[str, Any] = {} - if sampler is not None: - summary.update(sampler.stop_and_summary()) - summary.update(observability_summary()) - summary["exit_code"] = _run_exit_code(error) - if run_summary is not None: - summary.update(dict(run_summary())) - end_fields = { - "phase": "end", - "duration_ms": round((time.perf_counter() - started) * 1000.0), - "status": "error" if error_type else "ok", - "error_type": error_type, - **dict(run_fields or {}), - **summary, - } - log( - "error" if error_type else "info", - "run", - logger_name=resolved_logging_config.logger_name, - **end_fields, - ) - - -def default_log_file(logs_dir: Path = DEFAULT_LOGS_DIR, *, run: str = RUN) -> Path: - """Return a timestamped log-file path under `logs_dir`.""" - timestamp = _datetime.datetime.now(_datetime.UTC).strftime("%Y-%m-%d-%H-%M-%S-%z") - timestamp = timestamp.replace("+", "", 1) - return logs_dir / f"{timestamp}-{run}.json" - - -def log(level: str, key: str, *, logger_name: str = "", **fields: Any) -> None: - """Log one structured event through the configured logger.""" - numeric_level = _log_level(level) - logger = logging.getLogger(logger_name) - if not logger.isEnabledFor(numeric_level): - return - logger.log( - numeric_level, - "event=%s", - key, - extra={ - _STRUCTURED_EVENT_ATTR: key, - _STRUCTURED_FIELDS_ATTR: {**_current_log_fields(), **fields}, - }, - ) - - -def debug(key: str, *, logger_name: str = "", **fields: Any) -> None: - """Log a DEBUG structured event.""" - log("debug", key, logger_name=logger_name, **fields) - - -def info(key: str, *, logger_name: str = "", **fields: Any) -> None: - """Log an INFO structured event.""" - log("info", key, logger_name=logger_name, **fields) - - -def warning(key: str, *, logger_name: str = "", **fields: Any) -> None: - """Log a WARNING structured event.""" - log("warning", key, logger_name=logger_name, **fields) - - -def error(key: str, *, logger_name: str = "", **fields: Any) -> None: - """Log an ERROR structured event.""" - log("error", key, logger_name=logger_name, **fields) - - -def critical(key: str, *, logger_name: str = "", **fields: Any) -> None: - """Log a CRITICAL structured event.""" - log("critical", key, logger_name=logger_name, **fields) - - -@contextlib.contextmanager -def log_context(**fields: Any) -> Generator[None]: - """Add inherited structured fields for nested `log()` calls.""" - reset_token = _CONTEXT.set({**_CONTEXT.get({}), **fields}) - try: - yield - finally: - _CONTEXT.reset(reset_token) - - -@contextlib.contextmanager -def stage(name: str, **fields: Any) -> Generator[None]: - """Add a workflow stage field for nested logs and structured events.""" - with log_context(stage=name, **fields): - yield - - -@contextlib.contextmanager -def event( - key: str, - *, - level: str = "info", - start_level: str | None = None, - omit_success_status: bool = False, - logger_name: str = "", - **fields: Any, -) -> Generator[dict[str, Any]]: - """Emit start/end structured events around a block of work.""" - parent = _SPAN_CONTEXT.get() - span = _SpanContext( - trace=parent.trace if parent else secrets.token_hex(TRACE_SPAN_BYTES), - span=secrets.token_hex(TRACE_SPAN_BYTES), - parent_span=parent.span if parent else None, - ) - reset_token = _SPAN_CONTEXT.set(span) - try: - log(start_level or level, key, logger_name=logger_name, phase="start", **fields) - started = time.perf_counter() - extra: dict[str, Any] = {} - error: BaseException | None = None - try: - yield extra - except BaseException as exception: - error = exception - raise - finally: - end_fields = { - **fields, - **extra, - "phase": "end", - "duration_ms": round((time.perf_counter() - started) * 1000.0), - } - if error: - end_fields["status"] = "error" - end_fields["error_type"] = type(error).__name__ - elif not omit_success_status: - end_fields["status"] = "ok" - end_fields["error_type"] = None - log( - "error" if error else level, - key, - logger_name=logger_name, - **end_fields, - ) - finally: - _SPAN_CONTEXT.reset(reset_token) - - -def submit_with_log_context( - executor: Executor, - function: Callable[..., Any], - *args: Any, - **kwargs: Any, -) -> Future[Any]: - """Submit work to an executor with current logging ContextVars propagated.""" - context = contextvars.copy_context() - return executor.submit(context.run, function, *args, **kwargs) - - -def sanitized_config_snapshot(config: object) -> dict[str, Any]: - """Return a log-safe snapshot of dataclass/object/dict config values.""" - if isinstance(config, Mapping): - items: Iterable[tuple[object, object]] = cast(Mapping[object, object], config).items() - else: - object_items: list[tuple[object, object]] = [] - for name in dir(config): - if name.startswith("_"): - continue - object_items.append((name, getattr(config, name))) - items = object_items - snapshot: dict[str, Any] = {} - for key, value in items: - if callable(value): - continue - key_text = str(key) - if any(fragment in key_text.lower() for fragment in SECRET_FIELD_FRAGMENTS): - snapshot[key_text] = _secret_state(value) - elif isinstance(value, Path): - snapshot[key_text] = str(value) - elif isinstance(value, str | int | float | bool) or value is None: - snapshot[key_text] = value - else: - snapshot[key_text] = str(value) - return snapshot - - -def _current_log_fields(protected: Mapping[str, Any] | None = None) -> dict[str, Any]: - protected_keys = set(protected or {}) - fields = {key: value for key, value in _CONTEXT.get({}).items() if key not in protected_keys} - span = _SPAN_CONTEXT.get() - if span is None: - return fields - if "parent_span" not in protected_keys and span.parent_span is not None: - fields["parent_span"] = span.parent_span - if "span" not in protected_keys: - fields["span"] = span.span - if "trace" not in protected_keys: - fields["trace"] = span.trace - return fields - - -def startup_event( - *, - command: str, - config: object | None = None, - log_file: Path | None = None, - git_commit: str | None = None, - git_cwd: Path | None = None, - logger_name: str = "", -) -> None: - """Emit standard startup metadata after logging is configured.""" - fields: dict[str, Any] = { - "command": command, - "log_file": str(log_file) if log_file else None, - } - commit = git_commit or git_short_hash(git_cwd) - if commit: - fields["git_commit"] = commit - if config is not None: - config_value = config_snapshot(config) if isinstance(config, Config) else config - fields["config"] = sanitized_config_snapshot(config_value) - info("startup", logger_name=logger_name, **fields) - - -def git_short_hash(cwd: Path | None = None) -> str | None: - """Return the current git short hash, or None outside a git checkout.""" - try: - result = subprocess.run( - ["git", "rev-parse", "--short", "HEAD"], - cwd=cwd, - capture_output=True, - text=True, - timeout=2, - check=False, - ) - except OSError: - return None - except subprocess.SubprocessError: - return None - commit = result.stdout.strip() - return commit if result.returncode == 0 and commit else None - - -def _ordered_payload(payload: Mapping[str, Any]) -> dict[str, Any]: - ordered: dict[str, Any] = {} - for key in LOG_FIELD_ORDER: - if key in payload: - ordered[key] = payload[key] - for key in sorted(key for key in payload if key not in ordered): - ordered[key] = payload[key] - return ordered - - -def _log_file_level(configured_level: str | None) -> int: - if configured_level is not None: - return _log_level(configured_level) - env_level = os.environ.get(SRC_LOG_LEVEL) - if env_level: - return _log_level(env_level) - return _log_level(DEFAULT_LOG_FILE_LEVEL) - - -def _src_log_level_from_config(config: object | None) -> str | None: - value = getattr(config, "src_log_level", None) - return value if isinstance(value, str) else None - - -def _git_cwd_path(value: Path | str | None) -> Path | None: - if value is None: - return None - path = Path(value) - return path.parent if path.is_file() else path - - -def _log_level(value: int | str) -> int: - if isinstance(value, int): - return value - normalized = value.strip().upper() - if not normalized: - return logging.INFO - if normalized.isdecimal(): - return int(normalized) - levels = logging.getLevelNamesMapping() - level = levels.get(normalized) - if level is None: - return logging.INFO - return level - - -def _structured_log_fields(record: logging.LogRecord) -> tuple[str, dict[str, Any]]: - message = record.getMessage() - fields: dict[str, Any] = ( - {"level": "DEBUG"} - if record.name == "httpx" and message.startswith(_HTTPX_REQUEST_PREFIX) - else {} - ) - if not message.startswith(_HTTPCORE_RESPONSE_HEADERS_PREFIX): - return message, fields - try: - literal_value = cast( - object, - ast.literal_eval(message.removeprefix(_HTTPCORE_RESPONSE_HEADERS_PREFIX)), - ) - except (SyntaxError, ValueError): - return message, fields - if not isinstance(literal_value, tuple): - return message, fields - - return_value = cast(tuple[object, ...], literal_value) - if len(return_value) != 4: - return message, fields - http_version, status_code, reason_phrase, raw_headers = return_value - headers = _http_headers(raw_headers) - if not headers: - return message, fields - - fields["headers"] = headers - decoded_version = _decode_http_bytes(http_version) - if decoded_version is not None: - fields["http_version"] = decoded_version - if isinstance(status_code, int): - fields["status_code"] = status_code - decoded_reason = _decode_http_bytes(reason_phrase) - if decoded_reason is not None: - fields["reason_phrase"] = decoded_reason - return "receive_response_headers.complete", fields - - -def _http_headers(raw_headers: object) -> dict[str, str | list[str]]: - if not isinstance(raw_headers, list | tuple): - return {} - headers: dict[str, str | list[str]] = {} - for item in cast(Iterable[object], raw_headers): - if not isinstance(item, tuple): - continue - header = cast(tuple[object, ...], item) - if len(header) != 2: - continue - raw_name, raw_value = header - name = _decode_http_bytes(raw_name) - value = _decode_http_bytes(raw_value) - if name is None or value is None: - continue - key = name.lower() - existing = headers.get(key) - if existing is None: - headers[key] = value - elif isinstance(existing, list): - existing.append(value) - else: - headers[key] = [existing, value] - return {key: headers[key] for key in sorted(headers)} - - -def _decode_http_bytes(value: object) -> str | None: - if isinstance(value, bytes): - return value.decode("latin-1", errors="replace") - if isinstance(value, str): - return value - return None - - -def _secret_state(value: object) -> str: - if value is None or value == "": - return "missing" - return "reference" if isinstance(value, str) and value.startswith("op://") else "provided" - - -def _resource_sampler(config: LoggingSettings) -> ResourceSampler | None: - interval_seconds = config.resource_sample_interval_seconds - return ResourceSampler(interval_seconds) if interval_seconds is not None else None - - -def _run_error_type(exception: BaseException | None) -> str | None: - if exception is None: - return None - if isinstance(exception, SystemExit) and exception.code in (None, 0): - return None - return type(exception).__name__ - - -def _run_exit_code(exception: BaseException | None) -> int: - if exception is None: - return 0 - if isinstance(exception, SystemExit): - return exception.code if isinstance(exception.code, int) else 1 - return 1 - - -def _resource_usage() -> Any | None: - if sys.platform == "win32": - return None - return resource.getrusage(resource.RUSAGE_SELF) - - -def _cpu_seconds(usage: Any) -> float: - return float(usage.ru_utime) + float(usage.ru_stime) - - -def _rss_bytes(usage: Any | None) -> int | None: - current = _linux_current_rss_bytes() - if current is not None: - return current - if usage is None: - return None - # Linux reports ru_maxrss in KiB; macOS reports bytes. - max_rss = int(usage.ru_maxrss) - return max_rss if sys.platform == "darwin" else max_rss * 1024 - - -def _linux_current_rss_bytes() -> int | None: - statm = Path("/proc/self/statm") - if not statm.exists(): - return None - try: - fields = statm.read_text(encoding="utf-8").split() - if len(fields) < 2: - return None - return int(fields[1]) * os.sysconf("SC_PAGE_SIZE") - except (OSError, ValueError): - return None - - -def _num_file_descriptors() -> int | None: - for directory in (Path("/proc/self/fd"), Path("/dev/fd")): - if not directory.exists(): - continue - try: - return len(list(directory.iterdir())) - except OSError: - continue - return None - - -def _bytes_to_mib(byte_count: int) -> float: - return round(byte_count / MEBIBYTE, 2) - - -def _prune_old_log_files(logs_dir: Path, retain_files: int) -> None: - if retain_files <= 0 or not logs_dir.exists(): - return - log_files = sorted( - [*logs_dir.glob("????-??-??-??-??-??-*.json"), *logs_dir.glob("events-*.json")], - key=lambda path: path.stat().st_mtime, - ) - for old_file in log_files[:-retain_files]: - with contextlib.suppress(OSError): - old_file.unlink() diff --git a/git-subtree/src-py-lib/src/src_py_lib/utils/tsv.py b/git-subtree/src-py-lib/src/src_py_lib/utils/tsv.py deleted file mode 100644 index 87fda1d..0000000 --- a/git-subtree/src-py-lib/src/src_py_lib/utils/tsv.py +++ /dev/null @@ -1,95 +0,0 @@ -"""Aligned TSV file writing.""" - -from __future__ import annotations - -import unicodedata -from collections.abc import Iterable, Mapping -from pathlib import Path -from typing import Final - -DEFAULT_MAX_COLUMN_WIDTH: Final[int] = 100 -_ZERO_WIDTH_CATEGORIES: Final[frozenset[str]] = frozenset({"Cf", "Me", "Mn"}) - - -def write_tsv( - path: Path, - rows: Iterable[Mapping[str, object]], - *, - max_column_width: int = DEFAULT_MAX_COLUMN_WIDTH, -) -> None: - """Write rows as a padded TSV table, inferring the header from row keys.""" - data_rows = list(rows) - fieldnames = list(data_rows[0]) if data_rows else [] - table: list[Mapping[str, object]] = [] - if fieldnames: - table.append(dict(zip(fieldnames, fieldnames, strict=True))) - table.extend(data_rows) - - widths = { - field: max( - display_width( - format_tsv_value( - row.get(field, ""), - field, - max_column_width=max_column_width, - ) - ) - for row in table - ) - for field in fieldnames - } - - path.parent.mkdir(parents=True, exist_ok=True) - with path.open("w", encoding="utf-8") as file: - for row in table: - values = [ - pad_display( - format_tsv_value( - row.get(field, ""), - field, - max_column_width=max_column_width, - ), - widths[field], - ) - for field in fieldnames - ] - file.write("\t".join(values) + "\n") - - -def format_tsv_value( - value: object, - field: str, - *, - max_column_width: int = DEFAULT_MAX_COLUMN_WIDTH, -) -> str: - """Return a single-line TSV cell value, truncating non-URL fields.""" - text = str(value).replace("\r", " ").replace("\n", " ").replace("\t", " ") - if field == "url" or field.endswith("_url"): - return text - return text[:max_column_width] - - -def display_width(value: str) -> int: - """Return terminal display width for padding aligned text columns.""" - width = 0 - for character in value: - if unicodedata.combining(character): - continue - if unicodedata.category(character) in _ZERO_WIDTH_CATEGORIES: - continue - width += 2 if unicodedata.east_asian_width(character) in {"F", "W"} else 1 - return width - - -def pad_display(value: str, width: int) -> str: - """Pad text to a target display width.""" - return value + " " * max(width - display_width(value), 0) - - -__all__ = [ - "DEFAULT_MAX_COLUMN_WIDTH", - "display_width", - "format_tsv_value", - "pad_display", - "write_tsv", -] diff --git a/git-subtree/src-py-lib/tests/test_import.py b/git-subtree/src-py-lib/tests/test_import.py deleted file mode 100644 index 856d024..0000000 --- a/git-subtree/src-py-lib/tests/test_import.py +++ /dev/null @@ -1,51 +0,0 @@ -"""Basic package import smoke test.""" - -from __future__ import annotations - -import unittest - -import src_py_lib - - -class PackageImportTest(unittest.TestCase): - """Verify the package can be imported.""" - - def test_package_imports(self) -> None: - self.assertIsNotNone(src_py_lib) - - def test_root_public_api_exports_common_entrypoints(self) -> None: - self.assertIsNotNone(src_py_lib.GitHubClient) - self.assertIsNotNone(src_py_lib.GraphQLClient) - self.assertIsNotNone(src_py_lib.HTTPClient) - self.assertIsNotNone(src_py_lib.JSONDict) - self.assertIsNotNone(src_py_lib.LinearClientConfig) - self.assertIsNotNone(src_py_lib.LoggingConfig) - self.assertIsNotNone(src_py_lib.LoggingSettings) - self.assertIsNotNone(src_py_lib.resolve_log_level_name) - self.assertIsNotNone(src_py_lib.SlackClient) - self.assertIsNotNone(src_py_lib.SlackPacer) - self.assertIsNotNone(src_py_lib.SourcegraphClient) - self.assertIsNotNone(src_py_lib.SourcegraphClientConfig) - self.assertIsNotNone(src_py_lib.config_field) - self.assertIsNotNone(src_py_lib.gh_cli_token) - self.assertIsNotNone(src_py_lib.gcloud_adc_access_token) - self.assertIsNotNone(src_py_lib.info) - self.assertIsNotNone(src_py_lib.json_dicts) - self.assertIsNotNone(src_py_lib.json_str) - self.assertIsNotNone(src_py_lib.log) - self.assertIsNotNone(src_py_lib.logging) - self.assertIsNotNone(src_py_lib.logging_settings_from_config) - self.assertIsNotNone(src_py_lib.linear_client_from_config) - self.assertIsNotNone(src_py_lib.load_json_cache) - self.assertIsNotNone(src_py_lib.normalize_sourcegraph_endpoint) - self.assertIsNotNone(src_py_lib.parse_args) - self.assertIsNotNone(src_py_lib.quota_project_from_adc) - self.assertIsNotNone(src_py_lib.save_json_cache) - self.assertIsNotNone(src_py_lib.slack_client_from_config) - self.assertIsNotNone(src_py_lib.sourcegraph_client_from_config) - self.assertIsNotNone(src_py_lib.stream_connection_nodes) - self.assertIsNotNone(src_py_lib.write_tsv) - - -if __name__ == "__main__": - unittest.main() diff --git a/git-subtree/src-py-lib/tests/test_logging_http_clients.py b/git-subtree/src-py-lib/tests/test_logging_http_clients.py deleted file mode 100644 index 0566ca1..0000000 --- a/git-subtree/src-py-lib/tests/test_logging_http_clients.py +++ /dev/null @@ -1,1981 +0,0 @@ -"""Focused tests for logging, HTTP, and API-client primitives.""" - -from __future__ import annotations - -import argparse -import io -import json -import logging -import subprocess -import tempfile -import unittest -from collections.abc import Mapping -from contextlib import redirect_stderr, redirect_stdout -from pathlib import Path -from typing import Any -from unittest.mock import patch - -import httpx - -import src_py_lib as src -from src_py_lib.clients.github import GitHubClient, graphql_api_url, pr_ref_from_url -from src_py_lib.clients.google_sheets import GoogleSheetsClient -from src_py_lib.clients.graphql import ( - GraphQLClient, - GraphQLError, - introspect_schema, - stream_connection_nodes, -) -from src_py_lib.clients.linear import LinearClient, LinearClientConfig, linear_client_from_config -from src_py_lib.clients.one_password import ( - OnePasswordClient, - OnePasswordError, - resolve_op_secret_ref, -) -from src_py_lib.clients.slack import SlackClient -from src_py_lib.clients.sourcegraph import ( - SourcegraphClient, - SourcegraphClientConfig, - normalize_sourcegraph_endpoint, - sourcegraph_client_from_config, -) -from src_py_lib.utils.config import ( - Config, - ConfigError, - add_config_arguments, - config_env_file_from_args, - config_field, - config_overrides_from_args, - config_parse_args, - config_snapshot, - load_config, - load_config_env_file, - load_config_from_args, - resolve_config_refs, -) -from src_py_lib.utils.http import HTTPClient, HTTPClientError -from src_py_lib.utils.json_types import JSONDict, json_dict, json_list -from src_py_lib.utils.logging import ( - LoggingConfig, - LoggingSettings, - configure_logging, - critical, - debug, - default_log_file, - error, - event, - info, - log, - log_context, - logging_settings_from_config, - resolve_log_level_name, - startup_event, - warning, -) - - -class RecordingHTTP(HTTPClient): - """HTTPClient test double that records JSON request arguments.""" - - def __init__(self, responses: list[dict[str, Any]] | None = None) -> None: - super().__init__() - self.responses = list(responses or []) - self.calls: list[dict[str, Any]] = [] - - def json( - self, - method: str, - url: str, - *, - headers: Mapping[str, str] | None = None, - query: Mapping[str, str | int | float | bool | None] | None = None, - json_body: object | None = None, - ) -> dict[str, Any]: - self.calls.append( - { - "method": method, - "url": url, - "headers": headers, - "query": query, - "json_body": json_body, - } - ) - if self.responses: - return self.responses.pop(0) - return {"data": {"viewer": {"username": "alice"}}} - - -class FakeOnePasswordClient(OnePasswordClient): - """1Password test double that avoids shelling out.""" - - def read(self, secret_ref: str) -> str: - if secret_ref == "op://vault/item/field": - return "resolved-secret" - if secret_ref == "op://vault/page-size/value": - return "40" - if secret_ref == "op://vault/labels/value": - return "gamma, delta" - if secret_ref == "op://vault/name/value": - return "resolved-name" - raise OnePasswordError(f"unexpected secret ref: {secret_ref}") - - -class ExampleConfig(Config): - """Config model used by Config tests.""" - - token: str = config_field( - default="", - env_var="EXAMPLE_TOKEN", - cli_flag="--token", - metavar="TOKEN", - help="Example token", - secret=True, - ) - page_size: int = config_field( - default=25, - env_var="EXAMPLE_PAGE_SIZE", - cli_flag="--page-size", - metavar="N", - help="Example page size", - ) - include_archived: bool = config_field( - default=False, - env_var="EXAMPLE_INCLUDE_ARCHIVED", - cli_flag="--include-archived", - help="Include archived examples", - ) - output_dir: Path = config_field( - default=Path("out"), - env_var="EXAMPLE_OUTPUT_DIR", - cli_flag="--output-dir", - metavar="PATH", - help="Example output directory", - ) - labels: tuple[str, ...] = config_field( - default=(), - env_var="EXAMPLE_LABELS", - cli_flag="--labels", - metavar="CSV", - help="Example labels", - ) - - -class RequiredConfig(Config): - """Config model with a required secret field.""" - - token: str = config_field( - default="", - env_var="REQUIRED_TOKEN", - cli_flag="--token", - metavar="TOKEN", - help="Required token", - secret=True, - required=True, - ) - name: str = config_field( - default="", - env_var="REQUIRED_NAME", - cli_flag="--name", - metavar="NAME", - help="Non-secret required config name", - ) - - -class MultilineHelpConfig(Config): - """Config model with multiline CLI help text.""" - - notes: str = config_field( - default="", - env_var="MULTILINE_HELP_NOTES", - cli_flag="--notes", - metavar="TEXT", - help="First line.\nSecond line.\n Indented detail.", - ) - - -class SnapshotOrderConfig(Config): - """Config model whose field names and env-var names sort differently.""" - - alpha: str = config_field(default="a", env_var="ZZZ_ALPHA") - zulu: str = config_field(default="z", env_var="AAA_ZULU") - - -class BoundedConfig(Config): - """Config model with numeric bounds.""" - - page_size: int = config_field( - default=25, - env_var="BOUNDED_PAGE_SIZE", - cli_flag="--page-size", - metavar="N", - ge=1, - ) - sample_interval: float = config_field( - default=10.0, - env_var="BOUNDED_SAMPLE_INTERVAL", - cli_flag="--sample-interval", - metavar="SECS", - ge=0, - ) - - -class PatternConfig(Config): - """Config model with a string pattern constraint.""" - - date: str | None = config_field( - default=None, - env_var="PATTERN_DATE", - cli_flag="--date", - metavar="YYYY-MM-DD", - pattern=r"^\d{4}-\d{2}-\d{2}$", - ) - - -class CommandStyleConfig(Config): - """Config model with command-style flags.""" - - get: bool = config_field( - default=False, - env_var="COMMAND_STYLE_GET", - cli_flag="--get", - cli_action="store_true", - ) - verbose: bool = config_field( - default=False, - env_var="COMMAND_STYLE_VERBOSE", - cli_flag="--verbose", - cli_aliases=("-v",), - cli_action="store_true", - ) - schema_path: Path | None = config_field( - default=None, - env_var="COMMAND_STYLE_SCHEMA_PATH", - cli_flag="--get-schema", - cli_nargs="?", - cli_const="schema.gql", - metavar="FILE", - ) - - -class LinearExampleConfig(LinearClientConfig): - """Config model composed from Linear client fields and app fields.""" - - page_size: int = config_field( - default=25, - env_var="LINEAR_EXAMPLE_PAGE_SIZE", - cli_flag="--page-size", - metavar="N", - help="Example page size", - ) - - -class SourcegraphExampleConfig(SourcegraphClientConfig): - """Config model composed from Sourcegraph client fields and app fields.""" - - repo_query: str = config_field( - default="", - env_var="SOURCEGRAPH_EXAMPLE_REPO_QUERY", - cli_flag="--repo-query", - metavar="QUERY", - help="Example Sourcegraph repository query", - ) - - -class LoggingExampleConfig(LoggingConfig): - """Config model composed from shared logging fields.""" - - -class ConfigTest(unittest.TestCase): - def test_load_config_env_file_uses_dotenv_parser(self) -> None: - with tempfile.TemporaryDirectory() as directory: - env_file = Path(directory) / ".env" - env_file.write_text( - "\n".join( - ( - "# comment", - "export EXAMPLE_TOKEN='quoted token'", - "EXAMPLE_PAGE_SIZE=10 # inline comment", - "EXAMPLE_OUTPUT_DIR=${EXAMPLE_TOKEN}/out", - "BARE_KEY", - ) - ), - encoding="utf-8", - ) - - self.assertEqual( - load_config_env_file(env_file), - { - "EXAMPLE_TOKEN": "quoted token", - "EXAMPLE_PAGE_SIZE": "10", - "EXAMPLE_OUTPUT_DIR": "quoted token/out", - }, - ) - - def test_client_config_mixin_adds_linear_token_and_builds_client(self) -> None: - parser = argparse.ArgumentParser() - add_config_arguments(parser, LinearExampleConfig) - args = parser.parse_args(["--linear-api-token", "test-token", "--page-size", "50"]) - - config = load_config_from_args( - LinearExampleConfig, - args, - env={}, - resolve_op_refs=False, - ) - http = RecordingHTTP() - client = linear_client_from_config(config, http=http) - - self.assertEqual(config.linear_api_token, "test-token") - self.assertEqual(config.page_size, 50) - self.assertEqual(client.token, "test-token") - self.assertIs(client.http, http) - - def test_client_config_mixin_adds_sourcegraph_fields_and_builds_client(self) -> None: - parser = argparse.ArgumentParser() - add_config_arguments(parser, SourcegraphExampleConfig) - args = parser.parse_args( - [ - "--src-access-token", - "test-token", - "--repo-query", - "repo:example", - ] - ) - - config = load_config_from_args( - SourcegraphExampleConfig, - args, - env={}, - resolve_op_refs=False, - ) - client = sourcegraph_client_from_config(config) - - self.assertEqual(config.src_endpoint, "https://sourcegraph.com") - self.assertEqual(config.src_access_token, "test-token") - self.assertEqual(config.repo_query, "repo:example") - self.assertEqual(client.endpoint, "https://sourcegraph.com") - self.assertEqual(client.token, "test-token") - - def test_load_config_uses_precedence_and_pydantic_types(self) -> None: - with tempfile.TemporaryDirectory() as directory: - base_dir = Path(directory) - env_file = base_dir / ".env" - env_file.write_text( - "\n".join( - ( - "EXAMPLE_TOKEN=op://vault/item/field", - "EXAMPLE_PAGE_SIZE=10", - "EXAMPLE_INCLUDE_ARCHIVED=false", - "EXAMPLE_OUTPUT_DIR=from-env-file", - "EXAMPLE_LABELS=op://vault/labels/value", - ) - ), - encoding="utf-8", - ) - - config = load_config( - ExampleConfig, - env_file=env_file, - env={ - "EXAMPLE_PAGE_SIZE": "op://vault/page-size/value", - "EXAMPLE_OUTPUT_DIR": "from-shell", - }, - cli_overrides={ - "include_archived": True, - "output_dir": "from-cli", - }, - base_dir=base_dir, - resolve_op_refs=True, - op_client=FakeOnePasswordClient(), - ) - - self.assertEqual(config.token, "resolved-secret") - self.assertEqual(config.page_size, 40) - self.assertTrue(config.include_archived) - self.assertEqual(config.output_dir, base_dir / "from-cli") - self.assertEqual(config.labels, ("gamma", "delta")) - snapshot = config_snapshot(config) - self.assertEqual( - list(snapshot), - [ - "EXAMPLE_INCLUDE_ARCHIVED", - "EXAMPLE_LABELS", - "EXAMPLE_OUTPUT_DIR", - "EXAMPLE_PAGE_SIZE", - "EXAMPLE_TOKEN", - ], - ) - self.assertEqual( - snapshot, - { - "EXAMPLE_INCLUDE_ARCHIVED": True, - "EXAMPLE_LABELS": ["gamma", "delta"], - "EXAMPLE_OUTPUT_DIR": str(base_dir / "from-cli"), - "EXAMPLE_PAGE_SIZE": 40, - "EXAMPLE_TOKEN": "provided", - }, - ) - - def test_config_snapshot_sorts_emitted_keys(self) -> None: - snapshot = config_snapshot(SnapshotOrderConfig()) - - self.assertEqual(list(snapshot), ["AAA_ZULU", "ZZZ_ALPHA"]) - self.assertEqual(snapshot, {"AAA_ZULU": "z", "ZZZ_ALPHA": "a"}) - - def test_argparse_helpers_add_flags_and_collect_overrides(self) -> None: - parser = argparse.ArgumentParser() - add_config_arguments(parser, ExampleConfig) - - args = parser.parse_args( - [ - "--env-file", - "custom.env", - "--token", - "raw-token", - "--page-size", - "50", - "--no-include-archived", - "--labels", - "one,two", - ] - ) - - self.assertEqual(config_env_file_from_args(args), Path("custom.env")) - self.assertEqual( - config_overrides_from_args(ExampleConfig, args), - { - "token": "raw-token", - "page_size": "50", - "include_archived": False, - "labels": "one,two", - }, - ) - - def test_config_arguments_support_aliases_actions_and_optional_values(self) -> None: - parser = argparse.ArgumentParser() - add_config_arguments(parser, CommandStyleConfig) - - default_schema_args = parser.parse_args(["--get", "-v", "--get-schema"]) - named_schema_args = parser.parse_args(["--get-schema", "custom.gql"]) - - default_schema_config = load_config_from_args( - CommandStyleConfig, - default_schema_args, - env={}, - resolve_op_refs=False, - ) - named_schema_config = load_config_from_args( - CommandStyleConfig, - named_schema_args, - env={}, - resolve_op_refs=False, - ) - - self.assertTrue(default_schema_config.get) - self.assertTrue(default_schema_config.verbose) - self.assertEqual(default_schema_config.schema_path, Path.cwd() / "schema.gql") - self.assertEqual(named_schema_config.schema_path, Path.cwd() / "custom.gql") - - def test_config_field_supports_numeric_bounds(self) -> None: - config = load_config( - BoundedConfig, - env_file=None, - env={"BOUNDED_PAGE_SIZE": "1", "BOUNDED_SAMPLE_INTERVAL": "0"}, - resolve_op_refs=False, - ) - - self.assertEqual(config.page_size, 1) - self.assertEqual(config.sample_interval, 0) - with self.assertRaisesRegex(ConfigError, "greater than or equal to 1"): - load_config( - BoundedConfig, - env_file=None, - env={"BOUNDED_PAGE_SIZE": "0"}, - resolve_op_refs=False, - ) - with self.assertRaisesRegex(ConfigError, "greater than or equal to 0"): - load_config( - BoundedConfig, - env_file=None, - env={"BOUNDED_SAMPLE_INTERVAL": "-0.1"}, - resolve_op_refs=False, - ) - - def test_config_field_supports_string_pattern(self) -> None: - config = load_config( - PatternConfig, - env_file=None, - env={"PATTERN_DATE": "2026-01-31"}, - resolve_op_refs=False, - ) - - self.assertEqual(config.date, "2026-01-31") - with self.assertRaisesRegex(ConfigError, "String should match pattern"): - load_config( - PatternConfig, - env_file=None, - env={"PATTERN_DATE": "2026-1-31"}, - resolve_op_refs=False, - ) - with self.assertRaisesRegex(ConfigError, "String should match pattern"): - load_config( - PatternConfig, - env_file=None, - env={"PATTERN_DATE": "2026-01-31T00:00:00Z"}, - resolve_op_refs=False, - ) - - def test_logging_config_mixin_adds_log_level_from_cli_and_env(self) -> None: - parser = argparse.ArgumentParser() - add_config_arguments(parser, LoggingExampleConfig) - args = parser.parse_args(["--src-log-level", "INFO", "-v"]) - - cli_config = load_config_from_args( - LoggingExampleConfig, - args, - env={"SRC_LOG_LEVEL": "WARNING"}, - resolve_op_refs=False, - ) - env_config = load_config( - LoggingExampleConfig, - env_file=None, - env={"SRC_LOG_LEVEL": "ERROR"}, - resolve_op_refs=False, - ) - - self.assertEqual(cli_config.src_log_level, "INFO") - self.assertTrue(cli_config.verbose) - self.assertEqual(env_config.src_log_level, "ERROR") - - def test_logging_config_rejects_multiple_log_level_alias(self) -> None: - with self.assertRaisesRegex(ConfigError, "choose only one of --verbose"): - load_config( - LoggingExampleConfig, - env_file=None, - env={"SRC_LOG_VERBOSE": "true", "SRC_LOG_QUIET": "true"}, - resolve_op_refs=False, - ) - - def test_resolve_log_level_name_maps_cli_alias(self) -> None: - self.assertEqual(resolve_log_level_name(verbose=True), "DEBUG") - self.assertEqual(resolve_log_level_name(quiet=True), "WARNING") - self.assertEqual(resolve_log_level_name(silent=True), "ERROR") - self.assertEqual(resolve_log_level_name(log_level="trace"), "trace") - self.assertIsNone(resolve_log_level_name(object())) - - config = LoggingExampleConfig(src_log_level="INFO") - self.assertEqual(resolve_log_level_name(config), "INFO") - verbose_config = LoggingExampleConfig(src_log_level="INFO", verbose=True) - self.assertEqual(resolve_log_level_name(verbose_config), "DEBUG") - quiet_config = config_parse_args( - LoggingExampleConfig, - argv=["-q"], - env={}, - resolve_op_refs=False, - ) - self.assertEqual(resolve_log_level_name(quiet_config), "WARNING") - env_config = load_config( - LoggingExampleConfig, - env_file=None, - env={"SRC_LOG_SILENT": "true"}, - resolve_op_refs=False, - ) - self.assertTrue(env_config.silent) - self.assertEqual(resolve_log_level_name(env_config), "ERROR") - - def test_logging_settings_from_config_maps_common_cli_levels(self) -> None: - default_settings = logging_settings_from_config( - resource_sample_interval_seconds=2.5, - ) - self.assertEqual(default_settings.terminal_level, "INFO") - self.assertEqual(default_settings.log_file_level, "debug") - self.assertEqual(default_settings.resource_sample_interval_seconds, 2.5) - - quiet_config = LoggingExampleConfig(src_log_level="INFO", quiet=True) - quiet_settings = logging_settings_from_config(quiet_config) - self.assertEqual(quiet_settings.terminal_level, "WARNING") - self.assertEqual(quiet_settings.log_file_level, "WARNING") - - log_level_config = LoggingExampleConfig(src_log_level="ERROR") - log_level_settings = logging_settings_from_config(log_level_config) - self.assertEqual(log_level_settings.terminal_level, "ERROR") - self.assertEqual(log_level_settings.log_file_level, "ERROR") - - def test_config_parse_args_loads_config_and_reports_config_errors(self) -> None: - config = config_parse_args( - ExampleConfig, - argv=["--token", "raw-token", "--page-size", "50"], - env={}, - resolve_op_refs=False, - description="Example CLI.", - ) - - self.assertEqual(config.token, "raw-token") - self.assertEqual(config.page_size, 50) - - stderr = io.StringIO() - with redirect_stderr(stderr), self.assertRaises(SystemExit) as raised: - config_parse_args(RequiredConfig, argv=[], env={}, resolve_op_refs=False) - - self.assertEqual(raised.exception.code, 2) - self.assertIn("REQUIRED_TOKEN", stderr.getvalue()) - - def test_config_parse_args_preserves_description_newlines_in_help(self) -> None: - description = "Example CLI.\n\nSteps:\n 1. Collect data.\n 2. Export data." - stdout = io.StringIO() - - with redirect_stdout(stdout), self.assertRaises(SystemExit) as raised: - config_parse_args( - ExampleConfig, - argv=["--help"], - description=description, - env={}, - resolve_op_refs=False, - ) - - self.assertEqual(raised.exception.code, 0) - self.assertIn(description, stdout.getvalue()) - - def test_config_parse_args_keeps_long_options_on_help_line(self) -> None: - stdout = io.StringIO() - - with redirect_stdout(stdout), self.assertRaises(SystemExit) as raised: - config_parse_args( - SourcegraphExampleConfig, - argv=["--help"], - env={}, - resolve_op_refs=False, - ) - - self.assertEqual(raised.exception.code, 0) - help_text = stdout.getvalue() - self.assertNotIn("--src-access-token TOKEN\n", help_text) - self.assertRegex(help_text, r"--src-access-token TOKEN +Sourcegraph access token") - - def test_config_parse_args_preserves_argument_help_newlines(self) -> None: - stdout = io.StringIO() - - with redirect_stdout(stdout), self.assertRaises(SystemExit) as raised: - config_parse_args( - MultilineHelpConfig, - argv=["--help"], - env={}, - resolve_op_refs=False, - ) - - self.assertEqual(raised.exception.code, 0) - help_text = stdout.getvalue() - self.assertIn("First line.\n", help_text) - self.assertRegex(help_text, r"\n +Second line\.\n") - self.assertRegex(help_text, r"\n + Indented detail\.") - - def test_config_field_requires_named_default(self) -> None: - config_field_any: Any = config_field - - with self.assertRaises(TypeError): - config_field_any("", env_var="POSITIONAL_DEFAULT") - - def test_required_values_and_reference_resolution(self) -> None: - with self.assertRaisesRegex(ConfigError, "REQUIRED_TOKEN"): - load_config(RequiredConfig, env_file=None, env={}) - - config = load_config( - RequiredConfig, - env_file=None, - env={ - "REQUIRED_TOKEN": "op://vault/item/field", - "REQUIRED_NAME": "op://vault/name/value", - }, - ) - resolved = resolve_config_refs(config, client=FakeOnePasswordClient()) - - self.assertEqual(config.token, "op://vault/item/field") - self.assertEqual(config.name, "op://vault/name/value") - self.assertEqual(resolved.token, "resolved-secret") - self.assertEqual(resolved.name, "resolved-name") - - -class GraphQLTest(unittest.TestCase): - def test_introspect_schema_returns_schema_with_documentation_query(self) -> None: - schema: JSONDict = { - "description": "Example schema.", - "queryType": {"name": "Query"}, - "types": [{"kind": "OBJECT", "name": "Query", "description": "Root query."}], - } - http = RecordingHTTP([{"data": {"__schema": schema}}]) - client = GraphQLClient("https://example.com/graphql", {}, "Example", http=http) - - self.assertEqual(introspect_schema(client), schema) - body = json_dict(http.calls[0]["json_body"]) - query = str(body.get("query") or "") - self.assertIn("description", query) - self.assertIn("fields(includeDeprecated: true)", query) - self.assertIn("inputFields", query) - self.assertIn("enumValues(includeDeprecated: true)", query) - self.assertIn("deprecationReason", query) - self.assertNotIn("__schema {\n description", query) - self.assertNotIn("isRepeatable", query) - self.assertNotIn("args(includeDeprecated: true)", query) - - def test_introspect_schema_writes_schema_file(self) -> None: - schema: JSONDict = { - "description": "Example schema.", - "queryType": {"name": "Query"}, - "types": [{"kind": "OBJECT", "name": "Query"}], - } - seen: dict[str, str] = {} - - def execute(query: str) -> JSONDict: - seen["query"] = query - return {"__schema": schema} - - with tempfile.TemporaryDirectory() as directory: - output_file = Path(directory) / "schema" / "schema.json" - - result = introspect_schema(execute, output_file=output_file) - - self.assertIsNone(result) - self.assertIn("IntrospectionQuery", seen["query"]) - self.assertEqual(json.loads(output_file.read_text(encoding="utf-8")), schema) - - -class LoggingTest(unittest.TestCase): - def test_default_log_file_uses_dashed_timestamp_offset_and_run(self) -> None: - path = default_log_file(Path("logs"), run="1ea51330") - - self.assertEqual(path.parent, Path("logs")) - self.assertRegex( - path.name, - r"^\d{4}-\d{2}-\d{2}-\d{2}-\d{2}-\d{2}-\d{4}-1ea51330\.json$", - ) - - def test_configure_logging_defaults_log_file_under_logs_dir(self) -> None: - with tempfile.TemporaryDirectory() as directory: - logs_dir = Path(directory) / "logs" - logger_name = "src_py_lib_test_default_logs_dir" - log_file = configure_logging( - LoggingSettings( - logger_name=logger_name, - terminal_level="critical", - logs_dir=logs_dir, - run="test-run", - ) - ) - try: - info("default_log_path", logger_name=logger_name) - finally: - logger = logging.getLogger(logger_name) - for handler in list(logger.handlers): - logger.removeHandler(handler) - handler.close() - - if log_file is None: - self.fail("configure_logging did not return a default log file") - self.assertEqual(log_file.parent, logs_dir) - self.assertRegex( - log_file.name, - r"^\d{4}-\d{2}-\d{2}-\d{2}-\d{2}-\d{2}-\d{4}-test-run\.json$", - ) - rows = [json.loads(line) for line in log_file.read_text().splitlines()] - self.assertTrue(any(row.get("event") == "default_log_path" for row in rows)) - - def test_src_log_level_env_controls_log_file_level(self) -> None: - with tempfile.TemporaryDirectory() as directory: - log_file = Path(directory) / "events.json" - logger_name = "src_py_lib_test_log_level" - with patch.dict("os.environ", {"SRC_LOG_LEVEL": "INFO"}): - configure_logging( - LoggingSettings( - logger_name=logger_name, - terminal_level="critical", - log_file=log_file, - run="test-run", - ) - ) - try: - debug("debug_event", logger_name=logger_name) - info("info_event", logger_name=logger_name) - finally: - logger = logging.getLogger(logger_name) - for handler in list(logger.handlers): - logger.removeHandler(handler) - handler.close() - - rows = [json.loads(line) for line in log_file.read_text().splitlines()] - events = [row["event"] for row in rows] - self.assertNotIn("debug_event", events) - self.assertIn("info_event", events) - - def test_log_and_level_helpers_use_string_levels(self) -> None: - with tempfile.TemporaryDirectory() as directory: - log_file = Path(directory) / "events.json" - logger_name = "src_py_lib_test_string_levels" - configure_logging( - LoggingSettings( - logger_name=logger_name, - terminal_level="critical", - log_file_level="debug", - log_file=log_file, - run="test-run", - ) - ) - try: - log("bogus", "fallback_info", logger_name=logger_name) - warning("warning_event", logger_name=logger_name) - error("error_event", logger_name=logger_name) - critical("critical_event", logger_name=logger_name) - finally: - logger = logging.getLogger(logger_name) - for handler in list(logger.handlers): - logger.removeHandler(handler) - handler.close() - - rows = [json.loads(line) for line in log_file.read_text().splitlines()] - levels = {row["event"]: row["level"] for row in rows} - self.assertEqual(levels["fallback_info"], "INFO") - self.assertEqual(levels["warning_event"], "WARNING") - self.assertEqual(levels["error_event"], "ERROR") - self.assertEqual(levels["critical_event"], "CRITICAL") - - def test_logging_configures_logging_context_and_startup(self) -> None: - with tempfile.TemporaryDirectory() as directory: - log_file = Path(directory) / "events.json" - logger_name = "src_py_lib_test_logging_context" - config = ExampleConfig(token="secret-token") - try: - with src.logging( - config, - command="unit-test", - git_cwd=__file__, - logging_config=LoggingSettings( - logger_name=logger_name, - terminal_level="critical", - log_file=log_file, - run="test-run", - ), - ) as context_log_file: - self.assertEqual(context_log_file, log_file) - info("inside_command", logger_name=logger_name) - finally: - logger = logging.getLogger(logger_name) - for handler in list(logger.handlers): - logger.removeHandler(handler) - handler.close() - - rows = [json.loads(line) for line in log_file.read_text().splitlines()] - startup = next(row for row in rows if row["event"] == "startup") - inside = next(row for row in rows if row["event"] == "inside_command") - self.assertEqual(startup["command"], "unit-test") - self.assertEqual(startup["config"]["EXAMPLE_TOKEN"], "provided") - self.assertEqual(inside["command"], "unit-test") - - def test_structured_log_file_includes_context_and_sanitized_terminal_omits_event( - self, - ) -> None: - with tempfile.TemporaryDirectory() as directory: - log_file = Path(directory) / "events.json" - logger_name = "src_py_lib_test_logging" - configure_logging( - LoggingSettings( - logger_name=logger_name, - terminal_level="info", - log_file=log_file, - run="test-run", - ) - ) - try: - startup_event( - command="unit-test", - logger_name=logger_name, - git_commit="abc1234", - ) - with log_context(command="unit-test"): - info("example", logger_name=logger_name, answer=42) - finally: - logger = logging.getLogger(logger_name) - for handler in list(logger.handlers): - logger.removeHandler(handler) - handler.close() - - rows = [json.loads(line) for line in log_file.read_text().splitlines()] - startup = next(row for row in rows if row["event"] == "startup") - self.assertEqual(startup["git_commit"], "abc1234") - self.assertFalse(any("git_commit" in row for row in rows if row["event"] != "startup")) - self.assertEqual( - list(rows[0]), - ["ts", "level", "run", "logger", "event", "message"], - ) - self.assertEqual( - rows[0]["message"], - f"Writing log events to {log_file}.", - ) - self.assertEqual( - list(startup), - [ - "ts", - "command", - "level", - "run", - "event", - "git_commit", - "log_file", - ], - ) - self.assertEqual( - list(rows[-1]), - ["ts", "command", "level", "run", "event", "answer"], - ) - self.assertEqual(rows[-1]["event"], "example") - self.assertEqual(rows[-1]["run"], "test-run") - self.assertEqual(rows[-1]["command"], "unit-test") - self.assertEqual(rows[-1]["answer"], 42) - - def test_event_context_adds_trace_and_span_fields(self) -> None: - with tempfile.TemporaryDirectory() as directory: - log_file = Path(directory) / "events.json" - logger_name = "src_py_lib_test_traces" - configure_logging( - LoggingSettings( - logger_name=logger_name, - terminal_level="info", - log_file=log_file, - run="test-run", - ) - ) - try: - with event("outer", logger_name=logger_name): - info("inside", logger_name=logger_name, answer=42) - with event("inner", logger_name=logger_name): - logging.getLogger(logger_name).info("inside nested span") - finally: - logger = logging.getLogger(logger_name) - for handler in list(logger.handlers): - logger.removeHandler(handler) - handler.close() - - rows = [json.loads(line) for line in log_file.read_text().splitlines()] - outer_start = next( - row for row in rows if row["event"] == "outer" and row["phase"] == "start" - ) - outer_end = next( - row for row in rows if row["event"] == "outer" and row["phase"] == "end" - ) - inside = next(row for row in rows if row["event"] == "inside") - inner_start = next( - row for row in rows if row["event"] == "inner" and row["phase"] == "start" - ) - inner_end = next( - row for row in rows if row["event"] == "inner" and row["phase"] == "end" - ) - inner_log = next(row for row in rows if row.get("message") == "inside nested span") - - self.assertEqual( - list(outer_start), - ["ts", "level", "run", "trace", "span", "event", "phase"], - ) - self.assertEqual(outer_start["trace"], outer_end["trace"]) - self.assertEqual(outer_start["span"], outer_end["span"]) - self.assertEqual(len(outer_start["trace"]), 8) - self.assertEqual(len(outer_start["span"]), 8) - self.assertNotIn("parent_span", outer_start) - - self.assertEqual(inside["trace"], outer_start["trace"]) - self.assertEqual(inside["span"], outer_start["span"]) - - self.assertEqual( - list(inner_start), - [ - "ts", - "level", - "run", - "trace", - "span", - "parent_span", - "event", - "phase", - ], - ) - self.assertEqual(inner_start["trace"], outer_start["trace"]) - self.assertEqual(inner_start["span"], inner_end["span"]) - self.assertEqual(len(inner_start["span"]), 8) - self.assertEqual(inner_start["parent_span"], outer_start["span"]) - self.assertNotEqual(inner_start["span"], outer_start["span"]) - - self.assertEqual( - list(inner_log), - [ - "ts", - "level", - "run", - "trace", - "span", - "parent_span", - "logger", - "event", - "message", - ], - ) - self.assertEqual(inner_log["trace"], outer_start["trace"]) - self.assertEqual(inner_log["span"], inner_start["span"]) - self.assertEqual(inner_log["parent_span"], outer_start["span"]) - - def test_event_can_lower_start_level_and_omit_success_status(self) -> None: - with tempfile.TemporaryDirectory() as directory: - log_file = Path(directory) / "events.json" - logger_name = "src_py_lib_test_quiet_event" - configure_logging( - LoggingSettings( - logger_name=logger_name, - terminal_level="critical", - log_file_level="info", - log_file=log_file, - run="test-run", - ) - ) - try: - with event( - "quiet_start", - logger_name=logger_name, - level="info", - start_level="debug", - omit_success_status=True, - ): - pass - finally: - logger = logging.getLogger(logger_name) - for handler in list(logger.handlers): - logger.removeHandler(handler) - handler.close() - - rows = [json.loads(line) for line in log_file.read_text().splitlines()] - quiet_rows = [row for row in rows if row["event"] == "quiet_start"] - self.assertEqual(len(quiet_rows), 1) - self.assertEqual(quiet_rows[0]["phase"], "end") - self.assertNotIn("status", quiet_rows[0]) - self.assertNotIn("error_type", quiet_rows[0]) - - def test_logging_context_emits_run_summary_resource_and_http_metrics(self) -> None: - attempts = 0 - - def handler(_request: httpx.Request) -> httpx.Response: - nonlocal attempts - attempts += 1 - if attempts == 1: - return httpx.Response(429, json={"retry": True}, headers={"Retry-After": "0"}) - return httpx.Response(200, json={"ok": True}) - - with tempfile.TemporaryDirectory() as directory: - log_file = Path(directory) / "events.json" - try: - with src.logging( - command="unit-test", - logging_config=LoggingSettings( - terminal_level="critical", - log_file_level="debug", - log_file=log_file, - run="test-run", - resource_sample_interval_seconds=0, - ), - run_fields={"endpoint": "https://example.com"}, - run_summary=lambda: {"custom_count": 7}, - ): - client = HTTPClient( - max_attempts=2, - retry_base_delay_seconds=0, - retry_max_delay_seconds=0, - transport=httpx.MockTransport(handler), - ) - self.assertEqual( - client.json( - "POST", - "https://example.com/api", - json_body={"hello": "world"}, - ), - {"ok": True}, - ) - finally: - logger = logging.getLogger("") - for handler_ in list(logger.handlers): - logger.removeHandler(handler_) - handler_.close() - - rows = [json.loads(line) for line in log_file.read_text().splitlines()] - run_end = next(row for row in rows if row["event"] == "run" and row["phase"] == "end") - self.assertEqual(run_end["status"], "ok") - self.assertEqual(run_end["exit_code"], 0) - self.assertEqual(run_end["endpoint"], "https://example.com") - self.assertEqual(run_end["custom_count"], 7) - self.assertEqual(run_end["http_request_attempt_count"], 2) - self.assertEqual(run_end["http_retry_count"], 1) - self.assertEqual(run_end["http_2xx_count"], 1) - self.assertEqual(run_end["http_429_count"], 1) - self.assertGreater(run_end["http_request_bytes_total"], 0) - self.assertGreater(run_end["http_response_bytes_total"], 0) - self.assertIn("cpu_count_logical", run_end) - - def test_logging_context_records_system_exit_code(self) -> None: - with tempfile.TemporaryDirectory() as directory: - log_file = Path(directory) / "events.json" - try: - with ( - self.assertRaises(SystemExit), - src.logging( - command="unit-test", - logging_config=LoggingSettings( - terminal_level="critical", - log_file_level="debug", - log_file=log_file, - run="test-run", - ), - ), - ): - raise SystemExit(3) - finally: - logger = logging.getLogger("") - for handler_ in list(logger.handlers): - logger.removeHandler(handler_) - handler_.close() - - rows = [json.loads(line) for line in log_file.read_text().splitlines()] - run_end = next(row for row in rows if row["event"] == "run" and row["phase"] == "end") - self.assertEqual(run_end["status"], "error") - self.assertEqual(run_end["error_type"], "SystemExit") - self.assertEqual(run_end["exit_code"], 3) - - def test_httpx_request_logs_are_debug_events(self) -> None: - with tempfile.TemporaryDirectory() as directory: - log_file = Path(directory) / "events.json" - logger_name = "httpx" - configure_logging( - LoggingSettings( - logger_name=logger_name, - terminal_level="critical", - log_file_level="debug", - log_file=log_file, - run="test-run", - ) - ) - try: - logging.getLogger(logger_name).info( - 'HTTP Request: POST https://api.linear.app/graphql "HTTP/1.1 200 OK"' - ) - finally: - logger = logging.getLogger(logger_name) - for handler in list(logger.handlers): - logger.removeHandler(handler) - handler.close() - - rows = [json.loads(line) for line in log_file.read_text().splitlines()] - request_log = next( - row for row in rows if row.get("message", "").startswith("HTTP Request:") - ) - self.assertEqual(request_log["level"], "DEBUG") - - def test_httpcore_response_headers_are_structured(self) -> None: - with tempfile.TemporaryDirectory() as directory: - log_file = Path(directory) / "events.json" - logger_name = "httpcore" - configure_logging( - LoggingSettings( - logger_name=logger_name, - terminal_level="info", - log_file_level="debug", - log_file=log_file, - run="test-run", - ) - ) - try: - logging.getLogger("httpcore.http11").debug( - "receive_response_headers.complete " - "return_value=(b'HTTP/1.1', 200, b'OK', " - "[(b'Zed', b'last'), (b'Content-Type', b'application/json'), " - "(b'Alpha', b'first')])" - ) - finally: - logger = logging.getLogger(logger_name) - for handler in list(logger.handlers): - logger.removeHandler(handler) - handler.close() - - rows = [json.loads(line) for line in log_file.read_text().splitlines()] - response_headers = next( - row for row in rows if row.get("message") == "receive_response_headers.complete" - ) - - self.assertEqual(response_headers["logger"], "httpcore.http11") - self.assertEqual(response_headers["http_version"], "HTTP/1.1") - self.assertEqual(response_headers["status_code"], 200) - self.assertEqual(response_headers["reason_phrase"], "OK") - self.assertEqual(list(response_headers["headers"]), ["alpha", "content-type", "zed"]) - self.assertEqual( - response_headers["headers"], - { - "alpha": "first", - "content-type": "application/json", - "zed": "last", - }, - ) - - -class HTTPClientTest(unittest.TestCase): - def test_json_request_adds_query_headers_and_decodes_object(self) -> None: - seen: dict[str, Any] = {} - - def handler(request: httpx.Request) -> httpx.Response: - seen["url"] = str(request.url) - seen["authorization"] = request.headers["Authorization"] - seen["user_agent"] = request.headers["User-Agent"] - seen["body"] = request.content - return httpx.Response(200, json={"ok": True}) - - client = HTTPClient( - timeout=12, - max_attempts=1, - max_connections=7, - transport=httpx.MockTransport(handler), - ) - payload = client.json( - "POST", - "https://example.com/api", - headers={"Authorization": "Bearer token"}, - query={"limit": 10, "skip": None}, - json_body={"hello": "world"}, - ) - - self.assertEqual(payload, {"ok": True}) - self.assertEqual(seen["url"], "https://example.com/api?limit=10") - self.assertEqual(seen["authorization"], "Bearer token") - self.assertEqual(seen["user_agent"], "src-py-lib") - self.assertEqual(json.loads(seen["body"]), {"hello": "world"}) - self.assertEqual(client.max_connections, 7) - - def test_json_request_emits_structured_http_event(self) -> None: - def handler(_request: httpx.Request) -> httpx.Response: - return httpx.Response( - 200, - json={"ok": True}, - headers={ - "Zed": "last", - "Content-Type": "application/json", - "Set-Cookie": "session=secret", - }, - ) - - with tempfile.TemporaryDirectory() as directory: - log_file = Path(directory) / "events.json" - configure_logging( - LoggingSettings( - terminal_level="critical", - log_file_level="debug", - log_file=log_file, - run="test-run", - ) - ) - try: - client = HTTPClient(max_attempts=1, transport=httpx.MockTransport(handler)) - payload = client.json( - "POST", - "https://example.com/api", - headers={"Authorization": "Bearer token"}, - json_body={"hello": "world"}, - ) - finally: - logger = logging.getLogger("") - for handler_ in list(logger.handlers): - logger.removeHandler(handler_) - handler_.close() - - self.assertEqual(payload, {"ok": True}) - rows = [json.loads(line) for line in log_file.read_text().splitlines()] - http_request = next( - row - for row in rows - if row.get("event") == "http_request" and row.get("phase") == "end" - ) - - self.assertFalse(any(row.get("logger") in {"httpx", "httpcore"} for row in rows)) - self.assertEqual(http_request["status_code"], 200) - self.assertEqual(http_request["reason_phrase"], "OK") - self.assertEqual(http_request["request_bytes"], len(b'{"hello": "world"}')) - self.assertEqual(http_request["request_headers"]["authorization"], "[redacted]") - self.assertEqual( - list(http_request["response_headers"]), sorted(http_request["response_headers"]) - ) - self.assertEqual(http_request["response_headers"]["content-type"], "application/json") - self.assertEqual(http_request["response_headers"]["set-cookie"], "[redacted]") - self.assertEqual(http_request["response_headers"]["zed"], "last") - - def test_json_request_wraps_timeouts(self) -> None: - def handler(_request: httpx.Request) -> httpx.Response: - raise httpx.ReadTimeout("read timed out") - - client = HTTPClient( - timeout=12, - max_attempts=1, - transport=httpx.MockTransport(handler), - ) - - with self.assertRaisesRegex(HTTPClientError, "read timed out"): - client.json("POST", "https://example.com/api") - - def test_json_request_wraps_http_errors_with_body(self) -> None: - def handler(_request: httpx.Request) -> httpx.Response: - return httpx.Response(429, text="rate limited", headers={"Retry-After": "0"}) - - client = HTTPClient( - max_attempts=1, - transport=httpx.MockTransport(handler), - ) - - with self.assertRaisesRegex(HTTPClientError, "rate limited") as raised: - client.json("GET", "https://example.com/api") - - self.assertEqual(raised.exception.status_code, 429) - self.assertEqual(raised.exception.body, "rate limited") - - -class ClientTest(unittest.TestCase): - def test_normalize_sourcegraph_endpoint(self) -> None: - self.assertEqual( - normalize_sourcegraph_endpoint(" https://sourcegraph.example.com/ "), - "https://sourcegraph.example.com", - ) - self.assertEqual( - normalize_sourcegraph_endpoint("http://localhost:3080/"), - "http://localhost:3080", - ) - with self.assertRaisesRegex(ValueError, "https:// URL"): - normalize_sourcegraph_endpoint("http://localhost:3080", require_https=True) - with self.assertRaisesRegex(ValueError, "http:// or https:// URL"): - normalize_sourcegraph_endpoint("sourcegraph.example.com") - - def test_sourcegraph_client_builds_graphql_request(self) -> None: - http = RecordingHTTP([{"data": {"currentUser": {"username": "alice"}}}]) - client = SourcegraphClient(" https://sourcegraph.example.com/ ", "token", http=http) - data = client.graphql("query Viewer { currentUser { username } }") - - self.assertEqual(client.endpoint, "https://sourcegraph.example.com") - self.assertEqual(data, {"currentUser": {"username": "alice"}}) - self.assertEqual(http.calls[0]["method"], "POST") - self.assertEqual(http.calls[0]["url"], "https://sourcegraph.example.com/.api/graphql") - self.assertEqual(http.calls[0]["headers"], {"Authorization": "token token"}) - - def test_sourcegraph_client_streams_connection_nodes(self) -> None: - http = RecordingHTTP( - [ - { - "data": { - "users": { - "nodes": [{"username": "alice"}], - "pageInfo": {"hasNextPage": True, "endCursor": "cursor-1"}, - } - } - }, - { - "data": { - "users": { - "nodes": [{"username": "bob"}], - "pageInfo": {"hasNextPage": False, "endCursor": None}, - } - } - }, - ] - ) - client = SourcegraphClient("https://sourcegraph.example.com", "token", http=http) - nodes = list( - client.stream_connection_nodes( - """ - query Users($first: Int, $after: String) { - users(first: $first, after: $after) { - nodes { username } - pageInfo { hasNextPage endCursor } - } - } - """, - connection_path=("users",), - page_size=1, - ) - ) - - self.assertEqual(nodes, [{"username": "alice"}, {"username": "bob"}]) - first_body = json_dict(http.calls[0]["json_body"]) - second_body = json_dict(http.calls[1]["json_body"]) - self.assertEqual(first_body["variables"], {"first": 1, "after": None}) - self.assertEqual(second_body["variables"], {"first": 1, "after": "cursor-1"}) - - def test_sourcegraph_client_validate_queries_current_user(self) -> None: - http = RecordingHTTP([{"data": {"currentUser": {"username": "alice"}}}]) - client = SourcegraphClient("https://sourcegraph.example.com/", "token", http=http) - - self.assertEqual(client.validate(), {"username": "alice"}) - body = json_dict(http.calls[0]["json_body"]) - self.assertIn("SourcegraphClientValidate", str(body.get("query") or "")) - self.assertIn("currentUser", str(body.get("query") or "")) - - def test_graphql_client_paginates_cursor_results(self) -> None: - http = RecordingHTTP( - [ - { - "data": { - "viewer": { - "items": { - "nodes": [{"id": "1"}], - "pageInfo": { - "hasNextPage": True, - "endCursor": "cursor-1", - }, - } - } - } - }, - { - "data": { - "viewer": { - "items": { - "nodes": [{"id": "2"}], - "pageInfo": { - "hasNextPage": False, - "endCursor": None, - }, - } - } - } - }, - ] - ) - client = GraphQLClient("https://example.com/graphql", {}, "Example", http=http) - query = """ -query Items($first: Int!, $after: String, $userId: ID!) { - viewer { items { nodes { id } pageInfo { hasNextPage endCursor } } } -} -""" - - data = client.execute( - query, - variables={"userId": "u1"}, - page_size=2, - ) - nodes = json_list(json_dict(json_dict(data.get("viewer")).get("items")).get("nodes")) - - self.assertEqual(nodes, [{"id": "1"}, {"id": "2"}]) - self.assertEqual( - http.calls[0]["json_body"]["variables"], - {"userId": "u1", "first": 2, "after": None}, - ) - self.assertEqual( - http.calls[1]["json_body"]["variables"], - {"userId": "u1", "first": 2, "after": "cursor-1"}, - ) - - def test_graphql_client_streams_connection_nodes(self) -> None: - http = RecordingHTTP( - [ - { - "data": { - "viewer": { - "items": { - "nodes": [{"id": "1"}], - "pageInfo": { - "hasNextPage": True, - "endCursor": "cursor-1", - }, - } - } - } - }, - { - "data": { - "viewer": { - "items": { - "nodes": [{"id": "2"}], - "pageInfo": { - "hasNextPage": False, - "endCursor": None, - }, - } - } - } - }, - ] - ) - client = GraphQLClient("https://example.com/graphql", {}, "Example", http=http) - query = """ -query Items($first: Int!, $after: String, $userId: ID!) { - viewer { items { nodes { id } pageInfo { hasNextPage endCursor } } } -} -""" - - nodes = list( - client.stream_connection_nodes( - query, - variables={"userId": "u1"}, - connection_path=("viewer", "items"), - page_size=2, - ) - ) - - self.assertEqual(nodes, [{"id": "1"}, {"id": "2"}]) - self.assertEqual( - http.calls[0]["json_body"]["variables"], - {"userId": "u1", "first": 2, "after": None}, - ) - self.assertEqual( - http.calls[1]["json_body"]["variables"], - {"userId": "u1", "first": 2, "after": "cursor-1"}, - ) - - def test_stream_connection_nodes_accepts_execute_callback(self) -> None: - calls: list[dict[str, Any]] = [] - responses: list[JSONDict] = [ - { - "viewer": { - "items": { - "nodes": [{"id": "1"}], - "pageInfo": { - "hasNextPage": True, - "endCursor": "cursor-1", - }, - } - } - }, - { - "viewer": { - "items": { - "nodes": [{"id": "2"}], - "pageInfo": { - "hasNextPage": False, - "endCursor": None, - }, - } - } - }, - ] - - def execute(query: str, variables: Mapping[str, Any] | None) -> JSONDict: - calls.append({"query": query, "variables": dict(variables or {})}) - return responses.pop(0) - - query = """ -query Items($first: Int!, $after: String, $userId: ID!) { - viewer { items { nodes { id } pageInfo { hasNextPage endCursor } } } -} -""" - - nodes = list( - stream_connection_nodes( - execute, - query, - variables={"userId": "u1"}, - connection_path=("viewer", "items"), - page_size=2, - ) - ) - - self.assertEqual(nodes, [{"id": "1"}, {"id": "2"}]) - self.assertEqual( - [call["variables"] for call in calls], - [ - {"userId": "u1", "first": 2, "after": None}, - {"userId": "u1", "first": 2, "after": "cursor-1"}, - ], - ) - - def test_graphql_client_emits_query_debug_events(self) -> None: - http = RecordingHTTP( - [ - { - "data": { - "viewer": { - "items": { - "nodes": [{"id": "1"}], - "pageInfo": { - "hasNextPage": True, - "endCursor": "cursor-1", - }, - } - } - } - }, - { - "data": { - "viewer": { - "items": { - "nodes": [{"id": "2"}], - "pageInfo": { - "hasNextPage": False, - "endCursor": None, - }, - } - } - } - }, - ] - ) - client = GraphQLClient("https://example.com/graphql", {}, "Example", http=http) - query = """ -query Items($first: Int!, $after: String, $userId: ID!) { - viewer { items { nodes { id } pageInfo { hasNextPage endCursor } } } -} -""" - - with tempfile.TemporaryDirectory() as directory: - log_file = Path(directory) / "events.json" - configure_logging( - LoggingSettings( - terminal_level="critical", - log_file_level="debug", - log_file=log_file, - run="test-run", - ) - ) - try: - client.execute(query, variables={"userId": "u1"}, page_size=2) - finally: - logger = logging.getLogger("") - for handler in list(logger.handlers): - logger.removeHandler(handler) - handler.close() - - rows = [json.loads(line) for line in log_file.read_text().splitlines()] - starts = [ - row - for row in rows - if row.get("event") == "graphql_query" and row.get("phase") == "start" - ] - ends = [ - row - for row in rows - if row.get("event") == "graphql_query" and row.get("phase") == "end" - ] - - self.assertEqual([row["query_name"] for row in starts], ["Items", "Items"]) - self.assertEqual([row["page_number"] for row in starts], [1, 2]) - self.assertEqual([row["page_size"] for row in starts], [2, 2]) - self.assertEqual([row["cursor_present"] for row in starts], [False, True]) - self.assertEqual(starts[0]["graphql_client"], "Example") - self.assertEqual(starts[0]["variable_names"], ["after", "first", "userId"]) - self.assertEqual(ends[0]["response_fields"], ["viewer"]) - - def test_graphql_client_requires_end_cursor_for_next_page(self) -> None: - http = RecordingHTTP( - [ - { - "data": { - "items": { - "nodes": [], - "pageInfo": {"hasNextPage": True, "endCursor": None}, - } - } - } - ] - ) - client = GraphQLClient("https://example.com/graphql", {}, "Example", http=http) - query = """ -query Items($first: Int!, $after: String) { - items { nodes { id } pageInfo { hasNextPage endCursor } } -} -""" - - with self.assertRaisesRegex(GraphQLError, "endCursor"): - client.execute( - query, - page_size=100, - ) - - def test_graphql_client_rejects_stalled_cursor(self) -> None: - http = RecordingHTTP( - [ - { - "data": { - "items": { - "nodes": [], - "pageInfo": {"hasNextPage": True, "endCursor": "cursor-1"}, - } - } - }, - { - "data": { - "items": { - "nodes": [], - "pageInfo": {"hasNextPage": True, "endCursor": "cursor-1"}, - } - } - }, - ] - ) - client = GraphQLClient("https://example.com/graphql", {}, "Example", http=http) - query = """ -query Items($first: Int!, $after: String) { - items { nodes { id } pageInfo { hasNextPage endCursor } } -} -""" - - with self.assertRaisesRegex(GraphQLError, "stalled"): - client.execute( - query, - page_size=100, - ) - - def test_graphql_client_preserves_http_status_on_transport_errors(self) -> None: - class FailingHTTP(RecordingHTTP): - def json( - self, - method: str, - url: str, - *, - headers: Mapping[str, str] | None = None, - query: Mapping[str, str | int | float | bool | None] | None = None, - json_body: object | None = None, - ) -> dict[str, Any]: - raise HTTPClientError("unavailable", status_code=503) - - client = GraphQLClient("https://example.com/graphql", {}, "Example", http=FailingHTTP()) - - with self.assertRaises(GraphQLError) as raised: - client.execute("query Viewer { viewer { login } }", follow_pages=False) - - self.assertEqual(raised.exception.status_code, 503) - self.assertFalse(raised.exception.is_application_error) - - def test_graphql_client_marks_application_errors(self) -> None: - http = RecordingHTTP( - [ - { - "data": {}, - "errors": [{"message": "field does not exist"}], - } - ] - ) - client = GraphQLClient("https://example.com/graphql", {}, "Example", http=http) - - with self.assertRaises(GraphQLError) as raised: - client.execute("query Broken { missingField }", follow_pages=False) - - self.assertIsNone(raised.exception.status_code) - self.assertTrue(raised.exception.is_application_error) - - def test_github_pr_ref_from_url(self) -> None: - self.assertEqual( - pr_ref_from_url("https://github.com/sourcegraph/amp/pull/1234"), - "sourcegraph/amp#1234", - ) - - def test_github_client_defaults_to_github_dot_com(self) -> None: - http = RecordingHTTP() - client = GitHubClient("token", http=http) - client.graphql("query Viewer { viewer { login } }") - - self.assertEqual(http.calls[0]["url"], "https://api.github.com/graphql") - - def test_github_client_can_target_github_enterprise(self) -> None: - http = RecordingHTTP() - client = GitHubClient("token", github_url="https://github.example.com", http=http) - client.graphql("query Viewer { viewer { login } }") - - self.assertEqual(http.calls[0]["url"], "https://github.example.com/api/graphql") - self.assertEqual( - graphql_api_url("github.example.com"), "https://github.example.com/api/graphql" - ) - - def test_github_client_validate_queries_viewer(self) -> None: - http = RecordingHTTP([{"data": {"viewer": {"login": "alice"}}}]) - client = GitHubClient("token", http=http) - - self.assertEqual(client.validate(), {"login": "alice"}) - body = json_dict(http.calls[0]["json_body"]) - self.assertIn("GitHubClientValidate", str(body.get("query") or "")) - - def test_slack_client_validate_calls_auth_test(self) -> None: - response = {"ok": True, "url": "https://example.slack.com/", "user_id": "U1"} - http = RecordingHTTP([response]) - client = SlackClient("token", http=http) - - self.assertEqual(client.validate(), response) - self.assertEqual(http.calls[0]["url"], "https://slack.com/api/auth.test") - self.assertEqual(http.calls[0]["headers"], {"Authorization": "Bearer token"}) - - def test_google_sheets_client_validate_fetches_metadata(self) -> None: - metadata = {"sheets": [{"properties": {"sheetId": 1, "title": "Sheet1"}}]} - http = RecordingHTTP([metadata]) - client = GoogleSheetsClient("spreadsheet-id", "token", quota_project="quota", http=http) - - self.assertEqual(client.validate(), metadata) - self.assertEqual( - http.calls[0]["url"], - "https://sheets.googleapis.com/v4/spreadsheets/spreadsheet-id" - "?fields=sheets.properties(sheetId,title,gridProperties)", - ) - self.assertEqual( - http.calls[0]["headers"], - {"Authorization": "Bearer token", "X-Goog-User-Project": "quota"}, - ) - - def test_one_password_client_validate_returns_authenticated_account(self) -> None: - with patch("src_py_lib.clients.one_password.subprocess.run") as run: - run.return_value = subprocess.CompletedProcess( - ["op", "whoami", "--format", "json"], - 0, - stdout='{ "email": "alice@example.com", "account_uuid": "A1" }\n', - stderr="", - ) - - self.assertEqual( - OnePasswordClient().validate(), - {"email": "alice@example.com", "account_uuid": "A1"}, - ) - - run.assert_called_once_with( - ["op", "whoami", "--format", "json"], - check=True, - text=True, - capture_output=True, - ) - - def test_one_password_client_validate_requires_authentication(self) -> None: - with patch("src_py_lib.clients.one_password.subprocess.run") as run: - run.side_effect = subprocess.CalledProcessError( - 1, - ["op", "whoami", "--format", "json"], - stderr="not signed in", - ) - - with self.assertRaisesRegex(OnePasswordError, "not authenticated"): - OnePasswordClient().validate() - - def test_one_password_client_signin_runs_signin_then_validates(self) -> None: - with patch("src_py_lib.clients.one_password.subprocess.run") as run: - run.side_effect = [ - subprocess.CompletedProcess(["op", "signin"], 0), - subprocess.CompletedProcess( - ["op", "whoami", "--format", "json"], - 0, - stdout='{ "email": "alice@example.com" }\n', - stderr="", - ), - ] - - self.assertEqual( - OnePasswordClient().signin(), - {"email": "alice@example.com"}, - ) - - self.assertEqual(run.call_count, 2) - run.assert_any_call(["op", "signin"], check=True) - run.assert_any_call( - ["op", "whoami", "--format", "json"], - check=True, - text=True, - capture_output=True, - ) - - def test_linear_client_builds_graphql_request(self) -> None: - http = RecordingHTTP() - with patch("src_py_lib.clients.linear.GraphQLClient") as client_cls: - client_cls.return_value.execute.return_value = { - "viewer": {"email": "alice@example.com"} - } - data = LinearClient("token", http=http).graphql( - "query Viewer { viewer { email } }", - {"first": 1}, - page_size=10, - ) - - self.assertEqual(data, {"viewer": {"email": "alice@example.com"}}) - client_cls.assert_called_once_with( - url="https://api.linear.app/graphql", - headers={"Authorization": "token"}, - label="Linear", - http=http, - ) - client_cls.return_value.execute.assert_called_once_with( - "query Viewer { viewer { email } }", - variables={"first": 1}, - page_size=10, - ) - - def test_linear_client_validate_queries_viewer(self) -> None: - http = RecordingHTTP([{"data": {"viewer": {"email": "alice@example.com"}}}]) - client = LinearClient("token", http=http) - - self.assertEqual(client.validate(), {"email": "alice@example.com"}) - body = json_dict(http.calls[0]["json_body"]) - self.assertIn("LinearClientValidate", str(body.get("query") or "")) - self.assertNotIn("\n id\n", str(body.get("query") or "")) - self.assertEqual(http.calls[0]["headers"], {"Authorization": "token"}) - - def test_linear_client_validate_requires_viewer_email(self) -> None: - http = RecordingHTTP([{"data": {"viewer": {}}}]) - client = LinearClient("token", http=http) - - with self.assertRaisesRegex(RuntimeError, "viewer.email"): - client.validate() - - def test_linear_client_list_users_paginates(self) -> None: - http = RecordingHTTP( - [ - { - "data": { - "users": { - "nodes": [{"id": "U1", "name": "Alice"}], - "pageInfo": { - "hasNextPage": True, - "endCursor": "cursor-1", - }, - } - } - }, - { - "data": { - "users": { - "nodes": [{"id": "U2", "name": "Bob"}], - "pageInfo": { - "hasNextPage": False, - "endCursor": None, - }, - } - } - }, - ] - ) - - users = LinearClient("token", http=http).list_users(page_size=25) - - self.assertEqual([user["id"] for user in users], ["U1", "U2"]) - first_body = json_dict(http.calls[0]["json_body"]) - second_body = json_dict(http.calls[1]["json_body"]) - self.assertEqual( - json_dict(first_body.get("variables")), - {"first": 25, "after": None}, - ) - self.assertEqual( - json_dict(second_body.get("variables")), - {"first": 25, "after": "cursor-1"}, - ) - - def test_json_cache_helpers_round_trip_and_parse(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - path = Path(tmp) / "nested" / "cache.json" - - src.save_json_cache(path, {"b": {"name": "Bob"}, "a": {"name": "Alice"}}) - parsed = src.load_json_cache(path, parse=lambda value: str(value.get("name", ""))) - subset = src.load_json_subset(path, ["a", "missing"], parse=lambda value: value) - - self.assertEqual(parsed, {"a": "Alice", "b": "Bob"}) - self.assertEqual(subset, {"a": {"name": "Alice"}}) - - def test_resolve_op_secret_ref_leaves_raw_values_alone(self) -> None: - self.assertEqual(resolve_op_secret_ref(" raw-secret "), "raw-secret") - - def test_resolve_op_secret_ref_uses_one_password_client_for_refs(self) -> None: - self.assertEqual( - resolve_op_secret_ref("op://vault/item/field", client=FakeOnePasswordClient()), - "resolved-secret", - ) - - def test_resolve_op_secret_ref_rejects_empty_values(self) -> None: - with self.assertRaises(OnePasswordError): - resolve_op_secret_ref(" ") - - -if __name__ == "__main__": - unittest.main() diff --git a/git-subtree/src-py-lib/tests/test_tsv.py b/git-subtree/src-py-lib/tests/test_tsv.py deleted file mode 100644 index 1e77cd7..0000000 --- a/git-subtree/src-py-lib/tests/test_tsv.py +++ /dev/null @@ -1,65 +0,0 @@ -"""Tests for aligned TSV writing.""" - -from __future__ import annotations - -import tempfile -import unittest -from pathlib import Path - -import src_py_lib as src -from src_py_lib.utils.tsv import display_width, format_tsv_value, pad_display - - -class TSVTest(unittest.TestCase): - def test_format_tsv_value_sanitizes_and_truncates_non_url_fields(self) -> None: - self.assertEqual( - format_tsv_value("hello\rthere\nfriend\tnow", "note"), - "hello there friend now", - ) - self.assertEqual( - format_tsv_value("abcdef", "note", max_column_width=3), - "abc", - ) - self.assertEqual( - format_tsv_value("https://example.test/abcdef", "url", max_column_width=3), - "https://example.test/abcdef", - ) - self.assertEqual( - format_tsv_value("https://example.test/abcdef", "project_url", max_column_width=3), - "https://example.test/abcdef", - ) - - def test_display_width_handles_wide_and_combining_characters(self) -> None: - self.assertEqual(display_width("a"), 1) - self.assertEqual(display_width("測"), 2) - self.assertEqual(display_width("e\u0301"), 1) - self.assertEqual(pad_display("測", 4), "測 ") - - def test_write_tsv_creates_aligned_table(self) -> None: - with tempfile.TemporaryDirectory() as directory: - output_file = Path(directory) / "nested" / "table.tsv" - - src.write_tsv( - output_file, - [ - {"name": "al", "n": 1}, - {"name": "bob", "n": 2}, - ], - ) - - self.assertEqual( - output_file.read_text(encoding="utf-8"), - "name\tn\nal \t1\nbob \t2\n", - ) - - def test_write_tsv_writes_empty_file_for_empty_rows(self) -> None: - with tempfile.TemporaryDirectory() as directory: - output_file = Path(directory) / "table.tsv" - - src.write_tsv(output_file, []) - - self.assertEqual(output_file.read_text(encoding="utf-8"), "") - - -if __name__ == "__main__": - unittest.main() diff --git a/git-subtree/src-py-lib/uv.lock b/git-subtree/src-py-lib/uv.lock deleted file mode 100644 index 8325c53..0000000 --- a/git-subtree/src-py-lib/uv.lock +++ /dev/null @@ -1,303 +0,0 @@ -version = 1 -revision = 3 -requires-python = ">=3.11" - -[[package]] -name = "annotated-types" -version = "0.7.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/ee/67/531ea369ba64dcff5ec9c3402f9f51bf748cec26dde048a2f973a4eea7f5/annotated_types-0.7.0.tar.gz", hash = "sha256:aff07c09a53a08bc8cfccb9c85b05f1aa9a2a6f23728d790723543408344ce89", size = 16081, upload-time = "2024-05-20T21:33:25.928Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/78/b6/6307fbef88d9b5ee7421e68d78a9f162e0da4900bc5f5793f6d3d0e34fb8/annotated_types-0.7.0-py3-none-any.whl", hash = "sha256:1f02e8b43a8fbbc3f3e0d4f0f4bfc8131bcb4eebe8849b8e5c773f3a1c582a53", size = 13643, upload-time = "2024-05-20T21:33:24.1Z" }, -] - -[[package]] -name = "anyio" -version = "4.13.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "idna" }, - { name = "typing-extensions", marker = "python_full_version < '3.13'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/19/14/2c5dd9f512b66549ae92767a9c7b330ae88e1932ca57876909410251fe13/anyio-4.13.0.tar.gz", hash = "sha256:334b70e641fd2221c1505b3890c69882fe4a2df910cba14d97019b90b24439dc", size = 231622, upload-time = "2026-03-24T12:59:09.671Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/da/42/e921fccf5015463e32a3cf6ee7f980a6ed0f395ceeaa45060b61d86486c2/anyio-4.13.0-py3-none-any.whl", hash = "sha256:08b310f9e24a9594186fd75b4f73f4a4152069e3853f1ed8bfbf58369f4ad708", size = 114353, upload-time = "2026-03-24T12:59:08.246Z" }, -] - -[[package]] -name = "certifi" -version = "2026.5.20" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/f3/ce/ee2ecad540810a79593028e88299baeae54d346cc7a0d94b6199988b89b1/certifi-2026.5.20.tar.gz", hash = "sha256:69dea482ab64caa7b9f6aba1c6bf48bb6a5448d1c0f1b17ab42ad8c763a5344d", size = 135422, upload-time = "2026-05-20T11:46:50.073Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/59/8c/57e832b7af6d7c5abe66eb3fbe3a3a32f4d11ea23a1aa7131371035be991/certifi-2026.5.20-py3-none-any.whl", hash = "sha256:3c52e209ba0a4ad7aebe60436a4ab349c39e1e602e8c134221e546902ad25897", size = 134134, upload-time = "2026-05-20T11:46:48.578Z" }, -] - -[[package]] -name = "h11" -version = "0.16.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/01/ee/02a2c011bdab74c6fb3c75474d40b3052059d95df7e73351460c8588d963/h11-0.16.0.tar.gz", hash = "sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1", size = 101250, upload-time = "2025-04-24T03:35:25.427Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/04/4b/29cac41a4d98d144bf5f6d33995617b185d14b22401f75ca86f384e87ff1/h11-0.16.0-py3-none-any.whl", hash = "sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86", size = 37515, upload-time = "2025-04-24T03:35:24.344Z" }, -] - -[[package]] -name = "httpcore" -version = "1.0.9" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "certifi" }, - { name = "h11" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/06/94/82699a10bca87a5556c9c59b5963f2d039dbd239f25bc2a63907a05a14cb/httpcore-1.0.9.tar.gz", hash = "sha256:6e34463af53fd2ab5d807f399a9b45ea31c3dfa2276f15a2c3f00afff6e176e8", size = 85484, upload-time = "2025-04-24T22:06:22.219Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/7e/f5/f66802a942d491edb555dd61e3a9961140fd64c90bce1eafd741609d334d/httpcore-1.0.9-py3-none-any.whl", hash = "sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55", size = 78784, upload-time = "2025-04-24T22:06:20.566Z" }, -] - -[[package]] -name = "httpx" -version = "0.28.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "anyio" }, - { name = "certifi" }, - { name = "httpcore" }, - { name = "idna" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/b1/df/48c586a5fe32a0f01324ee087459e112ebb7224f646c0b5023f5e79e9956/httpx-0.28.1.tar.gz", hash = "sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc", size = 141406, upload-time = "2024-12-06T15:37:23.222Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/2a/39/e50c7c3a983047577ee07d2a9e53faf5a69493943ec3f6a384bdc792deb2/httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad", size = 73517, upload-time = "2024-12-06T15:37:21.509Z" }, -] - -[[package]] -name = "idna" -version = "3.15" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/82/77/7b3966d0b9d1d31a36ddf1746926a11dface89a83409bf1483f0237aa758/idna-3.15.tar.gz", hash = "sha256:ca962446ea538f7092a95e057da437618e886f4d349216d2b1e294abfdb65fdc", size = 199245, upload-time = "2026-05-12T22:45:57.011Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/d2/23/408243171aa9aaba178d3e2559159c24c1171a641aa83b67bdd3394ead8e/idna-3.15-py3-none-any.whl", hash = "sha256:048adeaf8c2d788c40fee287673ccaa74c24ffd8dcf09ffa555a2fbb59f10ac8", size = 72340, upload-time = "2026-05-12T22:45:55.733Z" }, -] - -[[package]] -name = "nodeenv" -version = "1.10.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/24/bf/d1bda4f6168e0b2e9e5958945e01910052158313224ada5ce1fb2e1113b8/nodeenv-1.10.0.tar.gz", hash = "sha256:996c191ad80897d076bdfba80a41994c2b47c68e224c542b48feba42ba00f8bb", size = 55611, upload-time = "2025-12-20T14:08:54.006Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/88/b2/d0896bdcdc8d28a7fc5717c305f1a861c26e18c05047949fb371034d98bd/nodeenv-1.10.0-py2.py3-none-any.whl", hash = "sha256:5bb13e3eed2923615535339b3c620e76779af4cb4c6a90deccc9e36b274d3827", size = 23438, upload-time = "2025-12-20T14:08:52.782Z" }, -] - -[[package]] -name = "pydantic" -version = "2.13.4" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "annotated-types" }, - { name = "pydantic-core" }, - { name = "typing-extensions" }, - { name = "typing-inspection" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/18/a5/b60d21ac674192f8ab0ba4e9fd860690f9b4a6e51ca5df118733b487d8d6/pydantic-2.13.4.tar.gz", hash = "sha256:c40756b57adaa8b1efeeced5c196f3f3b7c435f90e84ea7f443901bec8099ef6", size = 844775, upload-time = "2026-05-06T13:43:05.343Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/fd/7b/122376b1fd3c62c1ed9dc80c931ace4844b3c55407b6fb2d199377c9736f/pydantic-2.13.4-py3-none-any.whl", hash = "sha256:45a282cde31d808236fd7ea9d919b128653c8b38b393d1c4ab335c62924d9aba", size = 472262, upload-time = "2026-05-06T13:43:02.641Z" }, -] - -[[package]] -name = "pydantic-core" -version = "2.46.4" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/9d/56/921726b776ace8d8f5db44c4ef961006580d91dc52b803c489fafd1aa249/pydantic_core-2.46.4.tar.gz", hash = "sha256:62f875393d7f270851f20523dd2e29f082bcc82292d66db2b64ea71f64b6e1c1", size = 471464, upload-time = "2026-05-06T13:37:06.98Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/5c/fa/6d7708d2cfc1a832acb6aeb0cd16e801902df8a0f583bb3b4b527fde022e/pydantic_core-2.46.4-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:0e96592440881c74a213e5ad528e2b24d3d4f940de2766bed9010ab1d9e51594", size = 2111872, upload-time = "2026-05-06T13:40:27.596Z" }, - { url = "https://files.pythonhosted.org/packages/ae/6f/aa064a3e74b5745afbdf250594f38e7ead05e2d651bcb35994b9417a0d4d/pydantic_core-2.46.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:e0d65b8c354be7fb5f720c3caa8bc940bc2d20ce749c8e06135f07f8ed95dd7c", size = 1948255, upload-time = "2026-05-06T13:39:12.574Z" }, - { url = "https://files.pythonhosted.org/packages/43/3a/41114a9f7569b84b4d84e7a018c57c56347dac30c0d4a872946ec4e36c46/pydantic_core-2.46.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7bfb192b3f4b9e8a89b6277b6ce787564f62cfd272055f6e685726b111dc7826", size = 1972827, upload-time = "2026-05-06T13:38:19.841Z" }, - { url = "https://files.pythonhosted.org/packages/ef/25/1ab42e8048fe551934d9884e8d64daa7e990ad386f310a15981aeb6a5b08/pydantic_core-2.46.4-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:9037063db01f09b09e237c282b6792bd4da634b5402c4e7f0c61effed7701a04", size = 2041051, upload-time = "2026-05-06T13:38:10.447Z" }, - { url = "https://files.pythonhosted.org/packages/94/c2/1a934597ddf08da410385b3b7aae91956a5a76c635effef456074fad7e88/pydantic_core-2.46.4-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:fc010ab034c8c7452522748bf937df58020d256ccae0874463d1f4d01758af8e", size = 2221314, upload-time = "2026-05-06T13:40:13.089Z" }, - { url = "https://files.pythonhosted.org/packages/02/6d/9e8ad178c9c4df27ad3c8f25d1fe2a7ab0d2ba0559fad4aee5d3d1f16771/pydantic_core-2.46.4-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8c5dac79fa1614d1e06ca695109c6105923bd9c7d1d6c918d4e637b7e6b32fd3", size = 2285146, upload-time = "2026-05-06T13:38:59.224Z" }, - { url = "https://files.pythonhosted.org/packages/80/50/540cd3aeefc041beb111125c4bff779831a2111fc6b15a9138cda277d32c/pydantic_core-2.46.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f9fa868638bf362d3d138ea55829cefb3d5f4b0d7f142234382a15e2485dbec4", size = 2089685, upload-time = "2026-05-06T13:38:17.762Z" }, - { url = "https://files.pythonhosted.org/packages/6b/a4/b440ad35f05f6a38f89fa0f149accb3f0e02be94ca5e15f3c449a61b4bc9/pydantic_core-2.46.4-cp311-cp311-manylinux_2_31_riscv64.whl", hash = "sha256:17299feefe090f2caa5b8e37222bb5f663e4935a8bfa6931d4102e5df1a9f398", size = 2115420, upload-time = "2026-05-06T13:37:58.195Z" }, - { url = "https://files.pythonhosted.org/packages/99/61/de4f55db8dfd57bfdfa9a12ec90fe1b57c4f41062f7ca86f08586b3e0ac0/pydantic_core-2.46.4-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:4c63ebc82684aa89d9a3bcbd13d515b3be44250dc68dd3bd81526c1cb31286c3", size = 2165122, upload-time = "2026-05-06T13:37:01.167Z" }, - { url = "https://files.pythonhosted.org/packages/f7/52/7c529d7bdb2d1068bd52f51fe32572c8301f9a4febf1948f10639f1436f5/pydantic_core-2.46.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:aaa2a54443eff1950ba5ddc6b6ccda0d9c84a364276a62f969bdf2a390650848", size = 2182573, upload-time = "2026-05-06T13:38:45.04Z" }, - { url = "https://files.pythonhosted.org/packages/37/b3/7c40325848ba78247f2812dcf9c7274e38cd801820ca6dd9fe63bcfb0eb4/pydantic_core-2.46.4-cp311-cp311-musllinux_1_1_armv7l.whl", hash = "sha256:18e5ceec2ab67e6d5f1a9085e5a24c9c4e2ac4545730bfe668680bca05e555f3", size = 2317139, upload-time = "2026-05-06T13:37:15.539Z" }, - { url = "https://files.pythonhosted.org/packages/d9/37/f913f81a657c865b75da6c0dbed79876073c2a43b5bd9edbe8da785e4d49/pydantic_core-2.46.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:a0f62d0a58f4e7da165457e995725421e0064f2255d8eccebc49f41bbc23b109", size = 2360433, upload-time = "2026-05-06T13:37:30.099Z" }, - { url = "https://files.pythonhosted.org/packages/c4/67/6acaa1be2567f9256b056d8477158cac7240813956ce86e49deae8e173b4/pydantic_core-2.46.4-cp311-cp311-win32.whl", hash = "sha256:041bde0a48fd37cf71cab1c9d56d3e8625a3793fef1f7dd232b3ff37e978ecda", size = 1985513, upload-time = "2026-05-06T13:38:15.669Z" }, - { url = "https://files.pythonhosted.org/packages/aa/e6/c505f83dfeda9a2e5c995cfd872949e4d05e12f7feb3dca72f633daefa94/pydantic_core-2.46.4-cp311-cp311-win_amd64.whl", hash = "sha256:6f2eeda33a839975441c86a4119e1383c50b47faf0cbb5176985565c6bb02c33", size = 2071114, upload-time = "2026-05-06T13:40:35.416Z" }, - { url = "https://files.pythonhosted.org/packages/0f/da/7a263a96d965d9d0df5e8de8a475f33495451117035b09acb110288c381f/pydantic_core-2.46.4-cp311-cp311-win_arm64.whl", hash = "sha256:14f4c5d6db102bd796a627bbb3a17b4cf4574b9ae861d8b7c9a9661c6dd3362d", size = 2044298, upload-time = "2026-05-06T13:38:29.754Z" }, - { url = "https://files.pythonhosted.org/packages/ce/8c/af022f0af448d7747c5154288d46b5f2bc5f17366eaa0e23e9aa04d59f3b/pydantic_core-2.46.4-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:3245406455a5d98187ec35530fd772b1d799b26667980872c8d4614991e2c4a2", size = 2106158, upload-time = "2026-05-06T13:38:57.215Z" }, - { url = "https://files.pythonhosted.org/packages/19/95/6195171e385007300f0f5574592e467c568becce2d937a0b6804f218bc49/pydantic_core-2.46.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:962ccbab7b642487b1d8b7df90ef677e03134cf1fd8880bf698649b22a69371f", size = 1951724, upload-time = "2026-05-06T13:37:02.697Z" }, - { url = "https://files.pythonhosted.org/packages/8e/bc/f47d1ff9cbb1620e1b5b697eef06010035735f07820180e74178226b27b3/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8233f2947cf85404441fd7e0085f53b10c93e0ee78611099b5c7237e36aacbf7", size = 1975742, upload-time = "2026-05-06T13:37:09.448Z" }, - { url = "https://files.pythonhosted.org/packages/5b/11/9b9a5b0306345664a2da6410877af6e8082481b5884b3ddd78d47c6013ce/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:3a233125ac121aa3ffba9a2b59edfc4a985a76092dc8279586ab4b71390875e7", size = 2052418, upload-time = "2026-05-06T13:37:38.234Z" }, - { url = "https://files.pythonhosted.org/packages/f1/b7/a65fec226f5d78fc39f4a13c4cc0c768c22b113438f60c14adc9d2865038/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5b712b53160b79a5850310b912a5ef8e57e56947c8ad690c227f5c9d7e561712", size = 2232274, upload-time = "2026-05-06T13:38:27.753Z" }, - { url = "https://files.pythonhosted.org/packages/68/f0/92039db98b907ef49269a8271f67db9cb78ae2fc68062ef7e4e77adb5f61/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9401557acd873c3a7f3eb9383edef8ac4968f9510e340f4808d427e75667e7b4", size = 2309940, upload-time = "2026-05-06T13:38:05.353Z" }, - { url = "https://files.pythonhosted.org/packages/5f/97/2aab507d3d00ca626e8e57c1eac6a79e4e5fbcc63eb99733ff55d1717f65/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:926c9541b14b12b1681dca8a0b75feb510b06c6341b70a8e500c2fdcff837cce", size = 2094516, upload-time = "2026-05-06T13:39:10.577Z" }, - { url = "https://files.pythonhosted.org/packages/22/37/a8aca44d40d737dde2bc05b3c6c07dff0de07ce6f82e9f3167aeaf4d5dea/pydantic_core-2.46.4-cp312-cp312-manylinux_2_31_riscv64.whl", hash = "sha256:56cb4851bcaf3d117eddcef4fe66afd750a50274b0da8e22be256d10e5611987", size = 2136854, upload-time = "2026-05-06T13:40:22.59Z" }, - { url = "https://files.pythonhosted.org/packages/24/99/fcef1b79238c06a8cbec70819ac722ba76e02bc8ada9b0fd66eba40da01b/pydantic_core-2.46.4-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:c68fcd102d71ea85c5b2dfac3f4f8476eff42a9e078fd5faefff6d145063536b", size = 2180306, upload-time = "2026-05-06T13:40:10.666Z" }, - { url = "https://files.pythonhosted.org/packages/ae/6c/fc44000918855b42779d007ae63b0532794739027b2f417321cddbc44f6a/pydantic_core-2.46.4-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:b2f69dec1725e79a012d920df1707de5caf7ed5e08f3be4435e25803efc47458", size = 2190044, upload-time = "2026-05-06T13:40:43.231Z" }, - { url = "https://files.pythonhosted.org/packages/6b/65/d9cadc9f1920d7a127ad2edba16c1db7916e59719285cd6c94600b0080ba/pydantic_core-2.46.4-cp312-cp312-musllinux_1_1_armv7l.whl", hash = "sha256:8d0820e8192167f80d88d64038e609c31452eeca865b4e1d9950a27a4609b00b", size = 2329133, upload-time = "2026-05-06T13:39:57.365Z" }, - { url = "https://files.pythonhosted.org/packages/d0/cf/c873d91679f3a30bcf5e7ac280ce5573483e72295307685120d0d5ad3416/pydantic_core-2.46.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:fbdb89b3e1c94a30cc5edfce477c6e6a5dc4d8f84665b455c27582f211a1c72c", size = 2374464, upload-time = "2026-05-06T13:38:06.976Z" }, - { url = "https://files.pythonhosted.org/packages/47/bd/6f2fc8188f31bf10590f1e98e7b306336161fac930a8c514cd7bd828c7dc/pydantic_core-2.46.4-cp312-cp312-win32.whl", hash = "sha256:9aa768456404a8bf48a4406685ac2bec8e72b62c69313734fa3b73cf33b3a894", size = 1974823, upload-time = "2026-05-06T13:40:47.985Z" }, - { url = "https://files.pythonhosted.org/packages/40/8c/985c1d41ea1107c2534abd9870e4ed5c8e7669b5c308297835c001e7a1c4/pydantic_core-2.46.4-cp312-cp312-win_amd64.whl", hash = "sha256:e9c26f834c65f5752f3f06cb08cb86a913ceb7274d0db6e267808a708b46bc89", size = 2072919, upload-time = "2026-05-06T13:39:21.153Z" }, - { url = "https://files.pythonhosted.org/packages/c4/ba/f463d006e0c47373ca7ec5e1a261c59dc01ef4d62b2657af925fb0deee3a/pydantic_core-2.46.4-cp312-cp312-win_arm64.whl", hash = "sha256:4fc73cb559bdb54b1134a706a2802a4cddd27a0633f5abb7e53056268751ac6a", size = 2027604, upload-time = "2026-05-06T13:39:03.753Z" }, - { url = "https://files.pythonhosted.org/packages/51/a2/5d30b469c5267a17b39dec53208222f76a8d351dfac4af661888c5aee77d/pydantic_core-2.46.4-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:5d5902252db0d3cedf8d4a1bc68f70eeb430f7e4c7104c8c476753519b423008", size = 2106306, upload-time = "2026-05-06T13:37:48.029Z" }, - { url = "https://files.pythonhosted.org/packages/c1/81/4fa520eaffa8bd7d1525e644cd6d39e7d60b1592bc5b516693c7340b50f1/pydantic_core-2.46.4-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:c94f0688e7b8d0a67abf40e57a7eaaecd17cc9586706a31b76c031f63df052b4", size = 1951906, upload-time = "2026-05-06T13:37:17.012Z" }, - { url = "https://files.pythonhosted.org/packages/03/d5/fd02da45b659668b05923b17ba3a0100a0a3d5541e3bd8fcc4ecb711309e/pydantic_core-2.46.4-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f027324c56cd5406ca49c124b0db10e56c69064fec039acc571c29020cc87c76", size = 1976802, upload-time = "2026-05-06T13:37:35.113Z" }, - { url = "https://files.pythonhosted.org/packages/21/f2/95727e1368be3d3ed485eaab7adbd7dda408f33f7a36e8b48e0144002b91/pydantic_core-2.46.4-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:e739fee756ba1010f8bcccb534252e85a35fe45ae92c295a06059ce58b74ccd3", size = 2052446, upload-time = "2026-05-06T13:37:12.313Z" }, - { url = "https://files.pythonhosted.org/packages/9c/86/5d99feea3f77c7234b8718075b23db11532773c1a0dbd9b9490215dc2eeb/pydantic_core-2.46.4-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9d56801be94b86a9da183e5f3766e6310752b99ff647e38b09a9500d88e46e76", size = 2232757, upload-time = "2026-05-06T13:39:01.149Z" }, - { url = "https://files.pythonhosted.org/packages/d2/3a/508ac615935ef7588cf6d9e9b91309fdc2da751af865e02a9098de88258c/pydantic_core-2.46.4-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2412e734dcb48da14d4e4006b82b46b74f2518b8a26ee7e58c6844a6cd6d03c4", size = 2309275, upload-time = "2026-05-06T13:37:41.406Z" }, - { url = "https://files.pythonhosted.org/packages/07/f8/41db9de19d7987d6b04715a02b3b40aea467000275d9d758ffaa31af7d50/pydantic_core-2.46.4-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9551187363ffc0de2a00b2e47c25aeaeb1020b69b668762966df15fc5659dd5a", size = 2094467, upload-time = "2026-05-06T13:39:18.847Z" }, - { url = "https://files.pythonhosted.org/packages/2c/e2/f35033184cb11d0052daf4416e8e10a502ea2ac006fc4f459aee872727d1/pydantic_core-2.46.4-cp313-cp313-manylinux_2_31_riscv64.whl", hash = "sha256:0186750b482eefa11d7f435892b09c5c606193ef3375bcf94aa00ae6bfb66262", size = 2134417, upload-time = "2026-05-06T13:40:17.944Z" }, - { url = "https://files.pythonhosted.org/packages/7e/7b/6ceeb1cc90e193862f444ebe373d8fdf613f0a82572dde03fb10734c6c71/pydantic_core-2.46.4-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:5855698a4856556d86e8e6cd8434bc3ac0314ee8e12089ae0e143f64c6256e4e", size = 2179782, upload-time = "2026-05-06T13:40:32.618Z" }, - { url = "https://files.pythonhosted.org/packages/5a/f2/c8d7773ede6af08036423a00ae0ceffce266c3c52a096c435d68c896083f/pydantic_core-2.46.4-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:cbaf13819775b7f769bf4a1f066cb6df7a28d4480081a589828ef190226881cd", size = 2188782, upload-time = "2026-05-06T13:36:51.018Z" }, - { url = "https://files.pythonhosted.org/packages/59/31/0c864784e31f09f05cdd87606f08923b9c9e7f6e51dd27f20f62f975ce9f/pydantic_core-2.46.4-cp313-cp313-musllinux_1_1_armv7l.whl", hash = "sha256:633147d34cf4550417f12e2b1a0383973bdf5cdfde212cb09e9a581cf10820be", size = 2328334, upload-time = "2026-05-06T13:40:37.764Z" }, - { url = "https://files.pythonhosted.org/packages/c2/eb/4f6c8a41efa30baa755590f4141abf3a8c370fab610915733e74134a7270/pydantic_core-2.46.4-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:82cf5301172168103724d49a1444d3378cb20cdee30b116a1bd6031236298a5d", size = 2372986, upload-time = "2026-05-06T13:39:34.152Z" }, - { url = "https://files.pythonhosted.org/packages/5b/24/b375a480d53113860c299764bfe9f349a3dc9108b3adc0d7f0d786492ebf/pydantic_core-2.46.4-cp313-cp313-win32.whl", hash = "sha256:9fa8ae11da9e2b3126c6426f147e0fba88d96d65921799bb30c6abd1cb2c97fb", size = 1973693, upload-time = "2026-05-06T13:37:55.072Z" }, - { url = "https://files.pythonhosted.org/packages/7e/e8/cff247591966f2d22ec8c003cd7587e27b7ba7b81ab2fb888e3ab75dc285/pydantic_core-2.46.4-cp313-cp313-win_amd64.whl", hash = "sha256:6b3ace8194b0e5204818c92802dcdca7fc6d88aabbb799d7c795540d9cd6d292", size = 2071819, upload-time = "2026-05-06T13:38:49.139Z" }, - { url = "https://files.pythonhosted.org/packages/c6/1a/f4aee670d5670e9e148e0c82c7db98d780be566c6e6a97ee8035528ca0b3/pydantic_core-2.46.4-cp313-cp313-win_arm64.whl", hash = "sha256:184c081504d17f1c1066e430e117142b2c77d9448a97f7b65c6ac9fd9aee238d", size = 2027411, upload-time = "2026-05-06T13:40:45.796Z" }, - { url = "https://files.pythonhosted.org/packages/8d/74/228a26ddad29c6672b805d9fd78e8d251cd04004fa7eed0e622096cd0250/pydantic_core-2.46.4-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:428e04521a40150c85216fc8b85e8d39fece235a9cf5e383761238c7fa9b96fb", size = 2102079, upload-time = "2026-05-06T13:38:41.019Z" }, - { url = "https://files.pythonhosted.org/packages/ad/1f/8970b150a4b4365623ae00fc88603491f763c627311ae8031e3111356d6e/pydantic_core-2.46.4-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:23ace664830ee0bfe014a0c7bc248b1f7f25ed7ad103852c317624a1083af462", size = 1952179, upload-time = "2026-05-06T13:36:59.812Z" }, - { url = "https://files.pythonhosted.org/packages/95/30/5211a831ae054928054b2f79731661087a2bc5c01e825c672b3a4a8f1b3e/pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ce5c1d2a8b27468f433ca974829c44060b8097eedc39933e3c206a90ee49c4a9", size = 1978926, upload-time = "2026-05-06T13:37:39.933Z" }, - { url = "https://files.pythonhosted.org/packages/57/e9/689668733b1eb67adeef047db3c2e8788fcf65a7fd9c9e2b46b7744fe245/pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:7283d57845ecf5a163403eb0702dfc220cc4fbdd18919cb5ccea4f95ee1cdab4", size = 2046785, upload-time = "2026-05-06T13:38:01.995Z" }, - { url = "https://files.pythonhosted.org/packages/60/d9/6715260422ff50a2109878fd24d948a6c3446bb2664f34ee78cd972b3acd/pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8daafc69c93ee8a0204506a3b6b30f586ef54028f52aeeeb5c4cfc5184fd5914", size = 2228733, upload-time = "2026-05-06T13:40:50.371Z" }, - { url = "https://files.pythonhosted.org/packages/18/ae/fdb2f64316afca925640f8e70bb1a564b0ec2721c1389e25b8eb4bf9a299/pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cd2213145bcc2ba85884d0ac63d222fece9209678f77b9b4d76f054c561adb28", size = 2307534, upload-time = "2026-05-06T13:37:21.531Z" }, - { url = "https://files.pythonhosted.org/packages/89/1d/8eff589b45bb8190a9d12c49cfad0f176a5cbd1534908a6b5125e2886239/pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7a5f930472650a82629163023e630d160863fce524c616f4e5186e5de9d9a49b", size = 2099732, upload-time = "2026-05-06T13:39:31.942Z" }, - { url = "https://files.pythonhosted.org/packages/06/d5/ee5a3366637fee41dee51a1fc91562dcf12ddbc68fda34e6b253da2324bb/pydantic_core-2.46.4-cp314-cp314-manylinux_2_31_riscv64.whl", hash = "sha256:c1b3f518abeca3aa13c712fd202306e145abf59a18b094a6bafb2d2bbf59192c", size = 2129627, upload-time = "2026-05-06T13:37:25.033Z" }, - { url = "https://files.pythonhosted.org/packages/94/33/2414be571d2c6a6c4d08be21f9292b6d3fdb08949a97b6dfe985017821db/pydantic_core-2.46.4-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:1a7dd0b3ee80d90150e3495a3a13ac34dbcbfd4f012996a6a1d8900e91b5c0fb", size = 2179141, upload-time = "2026-05-06T13:37:14.046Z" }, - { url = "https://files.pythonhosted.org/packages/7b/79/7daa95be995be0eecc4cf75064cb33f9bbbfe3fe0158caf2f0d4a996a5c7/pydantic_core-2.46.4-cp314-cp314-musllinux_1_1_aarch64.whl", hash = "sha256:3fb702cd90b0446a3a1c5e470bfa0dd23c0233b676a9099ddcc964fa6ca13898", size = 2184325, upload-time = "2026-05-06T13:36:53.615Z" }, - { url = "https://files.pythonhosted.org/packages/9f/cb/d0a382f5c0de8a222dc61c65348e0ce831b1f68e0a018450d31c2cace3a5/pydantic_core-2.46.4-cp314-cp314-musllinux_1_1_armv7l.whl", hash = "sha256:b8458003118a712e66286df6a707db01c52c0f52f7db8e4a38f0da1d3b94fc4e", size = 2323990, upload-time = "2026-05-06T13:40:29.971Z" }, - { url = "https://files.pythonhosted.org/packages/05/db/d9ba624cc4a5aced1598e88c04fdbd8310c8a69b9d38b9a3d39ce3a61ed7/pydantic_core-2.46.4-cp314-cp314-musllinux_1_1_x86_64.whl", hash = "sha256:372429a130e469c9cd698925ce5fc50940b7a1336b0d82038e63d5bbc4edc519", size = 2369978, upload-time = "2026-05-06T13:37:23.027Z" }, - { url = "https://files.pythonhosted.org/packages/f2/20/d15df15ba918c423461905802bfd2981c3af0bfa0e40d05e13edbfa48bc3/pydantic_core-2.46.4-cp314-cp314-win32.whl", hash = "sha256:85bb3611ff1802f3ee7fdd7dbff26b56f343fb432d57a4728fdd49b6ef35e2f4", size = 1966354, upload-time = "2026-05-06T13:38:03.499Z" }, - { url = "https://files.pythonhosted.org/packages/fc/b6/6b8de4c0a7d7ab3004c439c80c5c1e0a3e8d78bbae19379b01960383d9e5/pydantic_core-2.46.4-cp314-cp314-win_amd64.whl", hash = "sha256:811ff8e9c313ab425368bcbb36e5c4ebd7108c2bbf4e4089cfbb0b01eff63fac", size = 2072238, upload-time = "2026-05-06T13:39:40.807Z" }, - { url = "https://files.pythonhosted.org/packages/32/36/51eb763beec1f4cf59b1db243a7dcc39cbb41230f050a09b9d69faaf0a48/pydantic_core-2.46.4-cp314-cp314-win_arm64.whl", hash = "sha256:bfec22eab3c8cc2ceec0248aec886624116dc079afa027ecc8ad4a7e62010f8a", size = 2018251, upload-time = "2026-05-06T13:37:26.72Z" }, - { url = "https://files.pythonhosted.org/packages/e8/91/855af51d625b23aa987116a19e231d2aaef9c4a415273ddc189b79a45fee/pydantic_core-2.46.4-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:af8244b2bef6aaad6d92cda81372de7f8c8d36c9f0c3ea36e827c60e7d9467a0", size = 2099593, upload-time = "2026-05-06T13:39:47.682Z" }, - { url = "https://files.pythonhosted.org/packages/fb/1b/8784a54c65edb5f49f0a14d6977cf1b209bba85a4c77445b255c2de58ab3/pydantic_core-2.46.4-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:5a4330cdbc57162e4b3aa303f588ba752257694c9c9be3e7ebb11b4aca659b5d", size = 1935226, upload-time = "2026-05-06T13:40:40.428Z" }, - { url = "https://files.pythonhosted.org/packages/e8/e7/1955d28d1afc56dd4b3ad7cc0cf39df1b9852964cf16e5d13912756d6d6b/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:29c61fc04a3d840155ff08e475a04809278972fe6aef51e2720554e96367e34b", size = 1974605, upload-time = "2026-05-06T13:37:32.029Z" }, - { url = "https://files.pythonhosted.org/packages/93/e2/3fedbf0ba7a22850e6e9fd78117f1c0f10f950182344d8a6c535d468fdd8/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:c50f2528cf200c5eed56faf3f4e22fcd5f38c157a8b78576e6ba3168ec35f000", size = 2030777, upload-time = "2026-05-06T13:38:55.239Z" }, - { url = "https://files.pythonhosted.org/packages/f8/61/46be275fcaaba0b4f5b9669dd852267ce1ff616592dccf7a7845588df091/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:0cbe8b01f948de4286c74cdd6c667aceb38f5c1e26f0693b3983d9d74887c65e", size = 2236641, upload-time = "2026-05-06T13:37:08.096Z" }, - { url = "https://files.pythonhosted.org/packages/60/db/12e93e46a8bac9988be3c016860f83293daea8c716c029c9ace279036f2f/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:617d7e2ca7dcb8c5cf6bcb8c59b8832c94b36196bbf1cbd1bfb56ed341905edd", size = 2286404, upload-time = "2026-05-06T13:40:20.221Z" }, - { url = "https://files.pythonhosted.org/packages/e2/4a/4d8b19008f38d31c53b8219cfedc2e3d5de5fe99d90076b7e767de29274f/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7027560ee92211647d0d34e3f7cd6f50da56399d26a9c8ad0da286d3869a53f3", size = 2109219, upload-time = "2026-05-06T13:38:12.153Z" }, - { url = "https://files.pythonhosted.org/packages/88/70/3cbc40978fefb7bb09c6708d40d4ad1a5d70fd7213c3d17f971de868ec1f/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_31_riscv64.whl", hash = "sha256:f99626688942fb746e545232e7726926f3be91b5975f8b55327665fafda991c7", size = 2110594, upload-time = "2026-05-06T13:40:02.971Z" }, - { url = "https://files.pythonhosted.org/packages/9d/20/b8d36736216e29491125531685b2f9e61aa5b4b2599893f8268551da3338/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:fc3e9034a63de20e15e8ade85358bc6efc614008cab72898b4b4952bea0509ff", size = 2159542, upload-time = "2026-05-06T13:39:27.506Z" }, - { url = "https://files.pythonhosted.org/packages/1d/a2/367df868eb584dacf6bf82a389272406d7178e301c4ac82545ab98bc2dd9/pydantic_core-2.46.4-cp314-cp314t-musllinux_1_1_aarch64.whl", hash = "sha256:97e7cf2be5c77b7d1a9713a05605d49460d02c6078d38d8bef3cbe323c548424", size = 2168146, upload-time = "2026-05-06T13:38:31.93Z" }, - { url = "https://files.pythonhosted.org/packages/c1/b8/4460f77f7e201893f649a29ab355dddd3beee8a97bcb1a320db414f9a06e/pydantic_core-2.46.4-cp314-cp314t-musllinux_1_1_armv7l.whl", hash = "sha256:3bf92c5d0e00fefaab325a4d27828fe6b6e2a21848686b5b60d2d9eeb09d76c6", size = 2306309, upload-time = "2026-05-06T13:37:44.717Z" }, - { url = "https://files.pythonhosted.org/packages/64/c4/be2639293acd87dc8ddbcec41a73cee9b2ebf996fe6d892a1a74e88ad3f7/pydantic_core-2.46.4-cp314-cp314t-musllinux_1_1_x86_64.whl", hash = "sha256:3ecbc122d18468d06ca279dc26a8c2e2d5acb10943bb35e36ae92096dc3b5565", size = 2369736, upload-time = "2026-05-06T13:37:05.645Z" }, - { url = "https://files.pythonhosted.org/packages/30/a6/9f9f380dbb301f67023bf8f707aaa75daadf84f7152d95c410fd7e81d994/pydantic_core-2.46.4-cp314-cp314t-win32.whl", hash = "sha256:e846ae7835bf0703ae43f534ab79a867146dadd59dc9ca5c8b53d5c8f7c9ef02", size = 1955575, upload-time = "2026-05-06T13:38:51.116Z" }, - { url = "https://files.pythonhosted.org/packages/40/1f/f1eb9eb350e795d1af8586289746f5c5677d16043040d63710e22abc43c9/pydantic_core-2.46.4-cp314-cp314t-win_amd64.whl", hash = "sha256:2108ba5c1c1eca18030634489dc544844144ee36357f2f9f780b93e7ddbb44b5", size = 2051624, upload-time = "2026-05-06T13:38:21.672Z" }, - { url = "https://files.pythonhosted.org/packages/f6/d2/42dd53d0a85c27606f316d3aa5d2869c4e8470a5ed6dec30e4a1abe19192/pydantic_core-2.46.4-cp314-cp314t-win_arm64.whl", hash = "sha256:4fcbe087dbc2068af7eda3aa87634eba216dbda64d1ae73c8684b621d33f6596", size = 2017325, upload-time = "2026-05-06T13:40:52.723Z" }, - { url = "https://files.pythonhosted.org/packages/ee/a4/73995fd4ebbb46ba0ee51e6fa049b8f02c40daebb762208feda8a6b7894d/pydantic_core-2.46.4-graalpy311-graalpy242_311_native-macosx_10_12_x86_64.whl", hash = "sha256:14d4edf427bdcf950a8a02d7cb44a08614388dd6e1bdcbf4f67504fa7887da9c", size = 2111589, upload-time = "2026-05-06T13:37:10.817Z" }, - { url = "https://files.pythonhosted.org/packages/fb/7f/f37d3a5e8bfcc2e403f5c57a730f2d815693fb42119e8ea48b3789335af1/pydantic_core-2.46.4-graalpy311-graalpy242_311_native-macosx_11_0_arm64.whl", hash = "sha256:0ce40cd7b21210e99342afafbd4d0f76d784eb5b1d60f3bdc566be4983c6c73b", size = 1944552, upload-time = "2026-05-06T13:36:56.717Z" }, - { url = "https://files.pythonhosted.org/packages/15/3c/d7eb777b3ff43e8433a4efb39a17aa8fd98a4ee8561a24a67ef5db07b2d6/pydantic_core-2.46.4-graalpy311-graalpy242_311_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:90884113d8b48f760e9587002789ddd741e76ab9f89518cd1e43b1f1a52ec44b", size = 1982984, upload-time = "2026-05-06T13:39:06.207Z" }, - { url = "https://files.pythonhosted.org/packages/63/87/70b9f40170a81afd55ca26c9b2acb25c20d64bcfbf888fafecb3ba077d4c/pydantic_core-2.46.4-graalpy311-graalpy242_311_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:66ce7632c22d837c95301830e111ad0128a32b8207533b60896a96c4915192ea", size = 2138417, upload-time = "2026-05-06T13:39:45.476Z" }, - { url = "https://files.pythonhosted.org/packages/9d/1d/8987ad40f65ae1432753072f214fb5c74fe47ffbd0698bb9cbbb585664f8/pydantic_core-2.46.4-graalpy312-graalpy250_312_native-macosx_10_12_x86_64.whl", hash = "sha256:1d8ba486450b14f3b1d63bc521d410ec7565e52f887b9fb671791886436a42f7", size = 2095527, upload-time = "2026-05-06T13:39:52.283Z" }, - { url = "https://files.pythonhosted.org/packages/64/d3/84c282a7eee1d3ac4c0377546ef5a1ea436ce26840d9ac3b7ed54a377507/pydantic_core-2.46.4-graalpy312-graalpy250_312_native-macosx_11_0_arm64.whl", hash = "sha256:3009f12e4e90b7f88b4f9adb1b0c4a3d58fe7820f3238c190047209d148026df", size = 1936024, upload-time = "2026-05-06T13:40:15.671Z" }, - { url = "https://files.pythonhosted.org/packages/d7/ca/eac61596cdeb4d7e174d3dc0bd8a6238f14f75f97a24e7b7db4c7e7340a0/pydantic_core-2.46.4-graalpy312-graalpy250_312_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ad785e92e6dc634c21555edc8bd6b64957ab844541bcb96a1366c202951ae526", size = 1990696, upload-time = "2026-05-06T13:38:34.717Z" }, - { url = "https://files.pythonhosted.org/packages/fa/c3/7c8b240552251faf6b3a957db200fcfbbcec36763c050428b601e0c9b83b/pydantic_core-2.46.4-graalpy312-graalpy250_312_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:00c603d540afdd6b80eb39f078f33ebd46211f02f33e34a32d9f053bba711de0", size = 2147590, upload-time = "2026-05-06T13:39:29.883Z" }, - { url = "https://files.pythonhosted.org/packages/11/cb/428de0385b6c8d44b716feba566abfacfbd23ee3c4439faa789a1456242f/pydantic_core-2.46.4-pp311-pypy311_pp73-macosx_10_12_x86_64.whl", hash = "sha256:0c563b08bca408dc7f65f700633d8442fffb2421fc47b8101377e9fd65051ff0", size = 2112782, upload-time = "2026-05-06T13:37:04.016Z" }, - { url = "https://files.pythonhosted.org/packages/0b/b5/6a17bdadd0fc1f170adfd05a20d37c832f52b117b4d9131da1f41bb097ce/pydantic_core-2.46.4-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:db06ffe51636ffe9ca531fe9023dd64bdd794be8754cb5df57c5498ae5b518a7", size = 1952146, upload-time = "2026-05-06T13:39:43.092Z" }, - { url = "https://files.pythonhosted.org/packages/2a/dc/03734d80e362cd43ef65428e9de77c730ce7f2f11c60d2b1e1b39f0fbf99/pydantic_core-2.46.4-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:133878133d271ade3d41d1bfb2a45ec38dbdbda40bc065921c6b04e4630127e2", size = 2134492, upload-time = "2026-05-06T13:36:58.124Z" }, - { url = "https://files.pythonhosted.org/packages/de/df/5e5ffc085ed07cc22d298134d3d911c63e91f6a0eb91fe646750a3209910/pydantic_core-2.46.4-pp311-pypy311_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:9bc519fbf2b7578398853d815009ae5e4d4603d12f4e3f91da8c06852d3da3e9", size = 2156604, upload-time = "2026-05-06T13:37:49.88Z" }, - { url = "https://files.pythonhosted.org/packages/81/44/6e112a4253e56f5705467cbab7ab5e91ee7398ba3d56d358635958893d3e/pydantic_core-2.46.4-pp311-pypy311_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:c7a7bd4e39e8e4c12c39cd480356842b6a8a06e41b23a55a5e3e191718838ddf", size = 2183828, upload-time = "2026-05-06T13:37:43.053Z" }, - { url = "https://files.pythonhosted.org/packages/ac/ad/5565071e937d8e752842ac241463944c9eb14c87e2d269f2658a5bd05e98/pydantic_core-2.46.4-pp311-pypy311_pp73-musllinux_1_1_armv7l.whl", hash = "sha256:d396ec2b979760aaf3218e76c24e65bd0aca24983298653b3a9d7a45f9e47b30", size = 2310000, upload-time = "2026-05-06T13:37:56.694Z" }, - { url = "https://files.pythonhosted.org/packages/4f/c3/66883a5cec183e7fba4d024b4cbbe61851a63750ef606b0afecc46d1f2bf/pydantic_core-2.46.4-pp311-pypy311_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:86e1a4418c6cd97d60c95c71164158eaf7324fae7b0923264016baa993eba6fc", size = 2361286, upload-time = "2026-05-06T13:40:05.667Z" }, - { url = "https://files.pythonhosted.org/packages/4b/2d/69abac8f838090bbecd5df894befb2c2619e7996a98ddb949db9f3b93225/pydantic_core-2.46.4-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:d51026d73fcfd93610abc7b27789c26b313920fcfb20e27462d74a7f8b06e983", size = 2193071, upload-time = "2026-05-06T13:38:08.682Z" }, -] - -[[package]] -name = "pyright" -version = "1.1.409" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "nodeenv" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/51/4e/3aa27f74211522dba7e9cbc3e74de779c6d4b654c54e50a4840623be8014/pyright-1.1.409.tar.gz", hash = "sha256:986ee05beca9e077c165758ad123667c679e050059a2546aa02473930394bc93", size = 4430434, upload-time = "2026-04-23T11:02:03.799Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/16/6b/330d8ebae582b30c2959a1ef4c3bc344ebde48c2ff0c3f113c4710735e11/pyright-1.1.409-py3-none-any.whl", hash = "sha256:aa3ea228cab90c845c7a60d28db7a844c04315356392aa09fafcee98c8c22fb3", size = 6438161, upload-time = "2026-04-23T11:02:01.309Z" }, -] - -[[package]] -name = "python-dotenv" -version = "1.2.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/82/ed/0301aeeac3e5353ef3d94b6ec08bbcabd04a72018415dcb29e588514bba8/python_dotenv-1.2.2.tar.gz", hash = "sha256:2c371a91fbd7ba082c2c1dc1f8bf89ca22564a087c2c287cd9b662adde799cf3", size = 50135, upload-time = "2026-03-01T16:00:26.196Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/0b/d7/1959b9648791274998a9c3526f6d0ec8fd2233e4d4acce81bbae76b44b2a/python_dotenv-1.2.2-py3-none-any.whl", hash = "sha256:1d8214789a24de455a8b8bd8ae6fe3c6b69a5e3d64aa8a8e5d68e694bbcb285a", size = 22101, upload-time = "2026-03-01T16:00:25.09Z" }, -] - -[[package]] -name = "ruff" -version = "0.15.13" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/24/21/a7d5c126d5b557715ef81098f3db2fe20f622a039ff2e626af28d674ab80/ruff-0.15.13.tar.gz", hash = "sha256:f9d89f17f7ba7fb2ed42921f0df75da797a9a5d71bc39049e2c687cf2baf44b7", size = 4678180, upload-time = "2026-05-14T13:44:37.869Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/c6/61/11d458dc6ac22504fd8e237b29dfd40504c7fbbcc8930402cfe51a8e63ed/ruff-0.15.13-py3-none-linux_armv6l.whl", hash = "sha256:444b580fc72fd6887e650acd3e575e18cdc79dbcf42fb4030b491057921f61f8", size = 10738279, upload-time = "2026-05-14T13:44:18.7Z" }, - { url = "https://files.pythonhosted.org/packages/86/ca/caa871ee7be718c45256fada4e16a218ee3e33f0c4a46b729a60a24912e6/ruff-0.15.13-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:6590d009e7cb7ebf36f83dbdd44a3fa48a0994ff6f1cdc1b08006abe58f98dc7", size = 11124798, upload-time = "2026-05-14T13:44:06.427Z" }, - { url = "https://files.pythonhosted.org/packages/d3/19/43f5f2e568dddde567fc41f8471f9432c09563e19d3e617a48cfa52f8f0a/ruff-0.15.13-py3-none-macosx_11_0_arm64.whl", hash = "sha256:1c26d2f66163deeb6e08d8b39fbbe983ce3c71cea06a6d7591cfd1421793c629", size = 10460761, upload-time = "2026-05-14T13:44:04.375Z" }, - { url = "https://files.pythonhosted.org/packages/99/df/cf938cd6de3003178f03ad7c1ea2a6c099468c03a35037985070b37e76be/ruff-0.15.13-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9dbd6f94b434f896308e4d57fb7bfde0d02b99f7a64b3bdab0fdfa6a864203a5", size = 10804451, upload-time = "2026-05-14T13:44:25.221Z" }, - { url = "https://files.pythonhosted.org/packages/c7/7d/5d0973129b154ded2225729169d7068f26b467760b146493fde138415f23/ruff-0.15.13-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:bf3259f3be4d181bda591da5db2571aed6853c6a048157756448020bc6c5cd22", size = 10534285, upload-time = "2026-05-14T13:44:08.888Z" }, - { url = "https://files.pythonhosted.org/packages/1f/e3/6b999bbc66cd51e5f073842bc2a3995e99c5e0e72e16b15e7261f7abf57a/ruff-0.15.13-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ae9c17e5eb4430c154e76abc25d79a318190f5a997f38fb6b114416c5319ffc9", size = 11312063, upload-time = "2026-05-14T13:44:11.274Z" }, - { url = "https://files.pythonhosted.org/packages/af/5a/642639e9f5db04f1e97fbd6e091c6fd20725bdf072fb114d00eefb9e6eb8/ruff-0.15.13-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2e2e39bff6c341f4b577a21b801326fab0b11847f48fcaa83f00a113c9b3cb55", size = 12183079, upload-time = "2026-05-14T13:44:01.634Z" }, - { url = "https://files.pythonhosted.org/packages/19/4c/7585735f6b53b0f12de13618b2f7d250a844f018822efc899df2e7b8295f/ruff-0.15.13-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e8d9a8e08013542e94d3220bc5b62cc3e5ef87c5f74bff367d3fac14fab013e6", size = 11440833, upload-time = "2026-05-14T13:43:59.043Z" }, - { url = "https://files.pythonhosted.org/packages/e8/31/bf1a0803d077e679cfeee5f2f67290a0fa79c7385b5d9a8c17b9db2c48f0/ruff-0.15.13-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cc411dfebe5eebe55ce041c6ae080eb7668955e866daa2fbb16692a784f1c4ca", size = 11434486, upload-time = "2026-05-14T13:44:27.761Z" }, - { url = "https://files.pythonhosted.org/packages/e1/4e/62c9b999875d4f14db80f277c030578f5e249c9852d65b7ac7ad0b43c041/ruff-0.15.13-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:768494eb08b9cee54e2fd27969966f74db5a57f6eaa7a90fcb3306af34dfc4bd", size = 11385189, upload-time = "2026-05-14T13:44:13.704Z" }, - { url = "https://files.pythonhosted.org/packages/fc/89/7e959047a104df3eb12863447c110140191fc5b6c4f379ea2e803fcdb0e4/ruff-0.15.13-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:fb75f9a3a7e42ffe117d734494e6c5e5cb3565d66e12612cb63d0e572a41a5b6", size = 10781380, upload-time = "2026-05-14T13:43:56.734Z" }, - { url = "https://files.pythonhosted.org/packages/ff/52/5fd18f3b88cab63e88aa11516b3b4e1e5f720e5c330f8dbe5c26210f41f8/ruff-0.15.13-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:8cb74dd33bb2f6613faf7fc03b660053b5ac4f80e706d5788c6335e2a8048d51", size = 10540605, upload-time = "2026-05-14T13:44:20.748Z" }, - { url = "https://files.pythonhosted.org/packages/e8/e0/9e35f338990d3e41a82875ff7053ffe97541dae81c9d02143177f381d572/ruff-0.15.13-py3-none-musllinux_1_2_i686.whl", hash = "sha256:7ef823f817fcd191dc934e984be9cf4094f808effa16f2542ad8e821ba02bbf2", size = 11036554, upload-time = "2026-05-14T13:44:16.256Z" }, - { url = "https://files.pythonhosted.org/packages/c2/13/070fb048c24080fba188f66371e2a92785be257ad02242066dc7255ac6e9/ruff-0.15.13-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:f345a13937bd7f09f6f5d19fa0721b0c103e00e7f62bc67089a8e5e037719e0b", size = 11528133, upload-time = "2026-05-14T13:44:22.808Z" }, - { url = "https://files.pythonhosted.org/packages/6b/8c/b1e1666aef7fc6555094d73ae6cd981701781ae85b97ceefc0eebd0b4668/ruff-0.15.13-py3-none-win32.whl", hash = "sha256:4044f94208b3b05ba0fc4a4abd0558cf4d6459bd18325eead7fd8cc66f909b41", size = 10721455, upload-time = "2026-05-14T13:44:35.697Z" }, - { url = "https://files.pythonhosted.org/packages/ab/a6/870a3e8a50590bb92be184ad928c2922f088b00d9dc5c5ec7b924ee08c22/ruff-0.15.13-py3-none-win_amd64.whl", hash = "sha256:7064884d442b7d477b4e7473d12da7f08851d2b1982763c5d3f388a19468a1a4", size = 11900409, upload-time = "2026-05-14T13:44:30.389Z" }, - { url = "https://files.pythonhosted.org/packages/9b/36/9c015cd052fca743dae8cb2aeb16b551444787467db42ceab0fc968865af/ruff-0.15.13-py3-none-win_arm64.whl", hash = "sha256:2471da9bd1068c8c064b5fd9c0c4b6dddffd6369cb1cd68b29993b1709ff1b21", size = 11179336, upload-time = "2026-05-14T13:44:33.026Z" }, -] - -[[package]] -name = "src-py-lib" -version = "0.1.0" -source = { editable = "." } -dependencies = [ - { name = "httpx" }, - { name = "pydantic" }, - { name = "python-dotenv" }, -] - -[package.dev-dependencies] -dev = [ - { name = "pyright" }, - { name = "ruff" }, -] - -[package.metadata] -requires-dist = [ - { name = "httpx", specifier = ">=0.28,<1" }, - { name = "pydantic", specifier = ">=2,<3" }, - { name = "python-dotenv", specifier = ">=1.2,<2" }, -] - -[package.metadata.requires-dev] -dev = [ - { name = "pyright", specifier = ">=1.1.409" }, - { name = "ruff", specifier = ">=0.7.0" }, -] - -[[package]] -name = "typing-extensions" -version = "4.15.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/72/94/1a15dd82efb362ac84269196e94cf00f187f7ed21c242792a923cdb1c61f/typing_extensions-4.15.0.tar.gz", hash = "sha256:0cea48d173cc12fa28ecabc3b837ea3cf6f38c6d1136f85cbaaf598984861466", size = 109391, upload-time = "2025-08-25T13:49:26.313Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/18/67/36e9267722cc04a6b9f15c7f3441c2363321a3ea07da7ae0c0707beb2a9c/typing_extensions-4.15.0-py3-none-any.whl", hash = "sha256:f0fa19c6845758ab08074a0cfa8b7aecb71c999ca73d62883bc25cc018c4e548", size = 44614, upload-time = "2025-08-25T13:49:24.86Z" }, -] - -[[package]] -name = "typing-inspection" -version = "0.4.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/55/e3/70399cb7dd41c10ac53367ae42139cf4b1ca5f36bb3dc6c9d33acdb43655/typing_inspection-0.4.2.tar.gz", hash = "sha256:ba561c48a67c5958007083d386c3295464928b01faa735ab8547c5692e87f464", size = 75949, upload-time = "2025-10-01T02:14:41.687Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/dc/9b/47798a6c91d8bdb567fe2698fe81e0c6b7cb7ef4d13da4114b41d239f65d/typing_inspection-0.4.2-py3-none-any.whl", hash = "sha256:4ed1cacbdc298c220f1bd249ed5287caa16f34d44ef4e9c3d0cbad5b521545e7", size = 14611, upload-time = "2025-10-01T02:14:40.154Z" }, -] diff --git a/pyproject.toml b/pyproject.toml index 92f3d0b..904ed9d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,52 +1,66 @@ +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[dependency-groups] +dev = [ + "pyright>=1.1.409", + "ruff>=0.15.14", + "types-pyyaml>=6.0.12.20260518", +] + [project] name = "src-auth-perms-sync" -version = "0.2.0" +version = "0.2.1" description = "Set Sourcegraph permissions from authentication provider data" readme = "README.md" requires-python = ">=3.11" +license = "MIT" +license-files = ["LICENSE"] +authors = [ + { name="Sourcegraph" }, +] +classifiers = [ + "Development Status :: 3 - Alpha", + "Environment :: Console", + "Intended Audience :: Developers", + "License :: OSI Approved :: MIT License", + "Operating System :: OS Independent", + "Programming Language :: Python :: 3", + "Typing :: Typed", +] dependencies = [ "json5>=0.14.0", "pyyaml>=6.0.3", - "src-py-lib", + "src-py-lib==0.1.1", +] +keywords = [ + "Sourcegraph" ] [project.scripts] src-auth-perms-sync = "src_auth_perms_sync.cli:main" -[build-system] -requires = ["hatchling"] -build-backend = "hatchling.build" +[project.urls] +Homepage = "https://github.com/sourcegraph/src-auth-perms-sync" +Issues = "https://github.com/sourcegraph/src-auth-perms-sync/issues" [tool.hatch.build.targets.wheel] -packages = ["src_auth_perms_sync"] - -[tool.uv] -package = true - -[tool.uv.sources] -src-py-lib = { path = "git-subtree/src-py-lib" } +packages = ["src/src_auth_perms_sync"] [tool.pyright] -include = ["src_auth_perms_sync"] +include = ["src", "tests", "dev"] +exclude = ["src-auth-perms-sync-runs", ".venv", "build", "dist"] typeCheckingMode = "strict" pythonVersion = "3.11" [tool.ruff] target-version = "py311" -# Slightly above the default 88 — accommodates the YAML-header literal in -# permissions/maps.py and a few log-message strings without sacrificing readability. line-length = 100 +extend-exclude = ["src-auth-perms-sync-runs"] [tool.ruff.lint] -# E = pycodestyle, F = pyflakes (unused imports, undefined names), -# I = import sorting, B = bugbear (likely-bug patterns), -# UP = pyupgrade (modernise syntax for py311), -# SIM = simplify (collapse trivial constructs). select = ["E", "F", "I", "B", "UP", "SIM"] -[dependency-groups] -dev = [ - "pyright>=1.1.409", - "ruff>=0.15.14", - "types-pyyaml>=6.0.12.20260518", -] +[tool.uv] +package = true diff --git a/src_auth_perms_sync/__init__.py b/src/src_auth_perms_sync/__init__.py similarity index 100% rename from src_auth_perms_sync/__init__.py rename to src/src_auth_perms_sync/__init__.py diff --git a/src_auth_perms_sync/__main__.py b/src/src_auth_perms_sync/__main__.py similarity index 100% rename from src_auth_perms_sync/__main__.py rename to src/src_auth_perms_sync/__main__.py diff --git a/src_auth_perms_sync/cli.py b/src/src_auth_perms_sync/cli.py similarity index 89% rename from src_auth_perms_sync/cli.py rename to src/src_auth_perms_sync/cli.py index 17c5319..58b6149 100644 --- a/src_auth_perms_sync/cli.py +++ b/src/src_auth_perms_sync/cli.py @@ -11,7 +11,9 @@ import logging import os +import secrets import sys +from collections.abc import Mapping from concurrent.futures import ThreadPoolExecutor from dataclasses import dataclass from pathlib import Path @@ -159,6 +161,13 @@ class SrcAuthPermissionsSyncConfig(src.SourcegraphClientConfig, src.LoggingConfi cli_action="store_true", help="With mutating commands: actually mutate state. Default is dry-run", ) + no_backup: bool = src.config_field( + default=False, + env_var="SRC_AUTH_PERMS_SYNC_NO_BACKUP", + cli_flag="--no-backup", + cli_action="store_true", + help="With mutating commands: skip before/after snapshots and validation", + ) parallelism: int = src.config_field( default=16, env_var="SRC_AUTH_PERMS_SYNC_PARALLELISM", @@ -167,6 +176,16 @@ class SrcAuthPermissionsSyncConfig(src.SourcegraphClientConfig, src.LoggingConfi ge=1, help="Concurrent Sourcegraph API worker threads (default: 16)", ) + explicit_permissions_batch_size: int = src.config_field( + default=25, + env_var="SRC_AUTH_PERMS_SYNC_EXPLICIT_PERMISSIONS_BATCH_SIZE", + cli_flag="--explicit-permissions-batch-size", + metavar="N", + ge=1, + help=( + "Users per GraphQL request when capturing explicit repository permissions (default: 25)" + ), + ) max_attempts: int = src.config_field( default=5, env_var="SRC_AUTH_PERMS_SYNC_MAX_ATTEMPTS", @@ -175,13 +194,6 @@ class SrcAuthPermissionsSyncConfig(src.SourcegraphClientConfig, src.LoggingConfi ge=1, help="Max attempts per HTTP request before giving up (default: 5)", ) - no_backup: bool = src.config_field( - default=False, - env_var="SRC_AUTH_PERMS_SYNC_NO_BACKUP", - cli_flag="--no-backup", - cli_action="store_true", - help="With mutating commands: skip before/after snapshots and validation", - ) sample_interval: float = src.config_field( default=10.0, env_var="SRC_AUTH_PERMS_SYNC_SAMPLE_INTERVAL", @@ -190,6 +202,16 @@ class SrcAuthPermissionsSyncConfig(src.SourcegraphClientConfig, src.LoggingConfi ge=0, help="Seconds between logging compute resource samples; set 0 to disable (default: 10)", ) + trace: bool = src.config_field( + default=False, + env_var="SRC_AUTH_PERMS_SYNC_TRACE", + cli_flag="--trace", + cli_action="store_true", + help=( + "Force Sourcegraph trace sampling by sending a sampled traceparent " + "header on each HTTP request" + ), + ) def config_error(message: str) -> NoReturn: @@ -367,6 +389,8 @@ def run_fields( "apply_flag": config.apply, "endpoint": endpoint, "parallelism": config.parallelism, + "explicit_permissions_batch_size": config.explicit_permissions_batch_size, + "trace": config.trace, "max_attempts": config.max_attempts, "no_backup": config.no_backup, "sample_interval": config.sample_interval, @@ -377,6 +401,45 @@ def run_fields( } +class TraceSamplingHTTPClient(src.HTTPClient): + """HTTP client that asks Sourcegraph to retain Jaeger traces for every request.""" + + def request( + self, + method: str, + url: str, + *, + headers: Mapping[str, str] | None = None, + query: Mapping[str, str | int | float | bool | None] | None = None, + json_body: object | None = None, + data: bytes | None = None, + ) -> bytes: + request_headers = dict(headers or {}) + if not any(name.lower() == "traceparent" for name in request_headers): + request_headers["traceparent"] = sampled_traceparent() + return super().request( + method, + url, + headers=request_headers, + query=query, + json_body=json_body, + data=data, + ) + + +def sampled_traceparent() -> str: + """Return a W3C traceparent header value with the sampled flag set.""" + return f"00-{nonzero_hex(16)}-{nonzero_hex(8)}-01" + + +def nonzero_hex(byte_count: int) -> str: + """Return a random hex string that is not all zeroes.""" + while True: + value = secrets.token_hex(byte_count) + if any(character != "0" for character in value): + return value + + def run_with_client( config: SrcAuthPermissionsSyncConfig, command: ResolvedCommand, @@ -384,7 +447,8 @@ def run_with_client( worker_pool: ThreadPoolExecutor, ) -> None: """Create a client, run the selected command, and always close HTTP resources.""" - http = src.HTTPClient( + http_class = TraceSamplingHTTPClient if config.trace else src.HTTPClient + http = http_class( user_agent="src-auth-perms-sync/0.1 (+python)", max_attempts=config.max_attempts, max_connections=config.parallelism, @@ -448,6 +512,7 @@ def run_set( command.set_options, dry_run=not config.apply, parallelism=config.parallelism, + explicit_permissions_batch_size=config.explicit_permissions_batch_size, bind_id_mode=sourcegraph_site_config.bind_id_mode, saml_groups_attribute_name_by_config_id=( sourcegraph_site_config.saml_groups_attribute_name_by_config_id @@ -471,6 +536,7 @@ def run_restore( config.restore_path, dry_run=not config.apply, parallelism=config.parallelism, + explicit_permissions_batch_size=config.explicit_permissions_batch_size, bind_id_mode=sourcegraph_site_config.bind_id_mode, do_backup=not config.no_backup, worker_pool=worker_pool, @@ -522,6 +588,7 @@ def run_get( users_without_explicit_perms=config.users_without_explicit_perms, user_created_after=config.created_after, parallelism=config.parallelism, + explicit_permissions_batch_size=config.explicit_permissions_batch_size, bind_id_mode=sourcegraph_site_config.bind_id_mode, saml_groups_attribute_name_by_config_id=( sourcegraph_site_config.saml_groups_attribute_name_by_config_id diff --git a/src_auth_perms_sync/orgs/__init__.py b/src/src_auth_perms_sync/orgs/__init__.py similarity index 100% rename from src_auth_perms_sync/orgs/__init__.py rename to src/src_auth_perms_sync/orgs/__init__.py diff --git a/src_auth_perms_sync/orgs/command.py b/src/src_auth_perms_sync/orgs/command.py similarity index 100% rename from src_auth_perms_sync/orgs/command.py rename to src/src_auth_perms_sync/orgs/command.py diff --git a/src_auth_perms_sync/orgs/queries.py b/src/src_auth_perms_sync/orgs/queries.py similarity index 100% rename from src_auth_perms_sync/orgs/queries.py rename to src/src_auth_perms_sync/orgs/queries.py diff --git a/src_auth_perms_sync/orgs/sync.py b/src/src_auth_perms_sync/orgs/sync.py similarity index 100% rename from src_auth_perms_sync/orgs/sync.py rename to src/src_auth_perms_sync/orgs/sync.py diff --git a/src_auth_perms_sync/orgs/types.py b/src/src_auth_perms_sync/orgs/types.py similarity index 100% rename from src_auth_perms_sync/orgs/types.py rename to src/src_auth_perms_sync/orgs/types.py diff --git a/src_auth_perms_sync/permissions/__init__.py b/src/src_auth_perms_sync/permissions/__init__.py similarity index 100% rename from src_auth_perms_sync/permissions/__init__.py rename to src/src_auth_perms_sync/permissions/__init__.py diff --git a/src_auth_perms_sync/permissions/apply.py b/src/src_auth_perms_sync/permissions/apply.py similarity index 100% rename from src_auth_perms_sync/permissions/apply.py rename to src/src_auth_perms_sync/permissions/apply.py diff --git a/src_auth_perms_sync/permissions/command.py b/src/src_auth_perms_sync/permissions/command.py similarity index 98% rename from src_auth_perms_sync/permissions/command.py rename to src/src_auth_perms_sync/permissions/command.py index cf46e43..5ee5e6b 100644 --- a/src_auth_perms_sync/permissions/command.py +++ b/src/src_auth_perms_sync/permissions/command.py @@ -89,6 +89,7 @@ def cmd_get( users_without_explicit_perms: bool, user_created_after: str | None, parallelism: int, + explicit_permissions_batch_size: int, bind_id_mode: str, saml_groups_attribute_name_by_config_id: dict[str, str], auth_providers_by_config_id: dict[str, dict[str, Any]], @@ -141,7 +142,8 @@ def cmd_get( # SAML-only: tally distinct users per (serviceID, clientID, group) # by parsing each user's SAML AssertionInfo `accountData`. Surfaced # in the YAML so operators can size groups before authoring a - # `authProvider.samlGroup` mapping rule. See `src_auth_perms_sync/shared/saml_groups.py`. + # `authProvider.samlGroup` mapping rule. See + # `src/src_auth_perms_sync/shared/saml_groups.py`. saml_group_counts = saml_groups.count_users_per_saml_group( users, attribute_names_by_provider ) @@ -183,6 +185,7 @@ def cmd_get( bind_id_mode, maps_path, total_users=len(users), + explicit_permissions_batch_size=explicit_permissions_batch_size, worker_pool=worker_pool, ) before_path = snapshot_path(maps_path, timestamp, client.endpoint, "get", "before") @@ -307,6 +310,7 @@ def cmd_set( options: permission_types.SetCommandOptions, dry_run: bool, parallelism: int, + explicit_permissions_batch_size: int, bind_id_mode: str, saml_groups_attribute_name_by_config_id: dict[str, str], do_backup: bool, @@ -321,6 +325,7 @@ def cmd_set( options.user_created_after, dry_run, parallelism, + explicit_permissions_batch_size, bind_id_mode, saml_groups_attribute_name_by_config_id, do_backup, @@ -527,10 +532,9 @@ def _plan_additions_for_user( desired_repos[repository["id"]] = repository if existing_repo_ids is None: - existing_repo_ids = { - repository["id"] - for repository in permissions_sourcegraph.list_user_explicit_repos(client, user["id"]) - } + existing_repo_ids = set( + permissions_sourcegraph.list_user_explicit_repo_ids(client, user["id"]) + ) additions = [ permissions_apply.PermissionAddition( user_id=user["id"], @@ -896,6 +900,7 @@ def cmd_restore( snapshot_path: Path, dry_run: bool, parallelism: int, + explicit_permissions_batch_size: int, bind_id_mode: str, do_backup: bool, worker_pool: ThreadPoolExecutor | None = None, @@ -906,6 +911,7 @@ def cmd_restore( snapshot_path, dry_run, parallelism, + explicit_permissions_batch_size, bind_id_mode, do_backup, worker_pool, diff --git a/src_auth_perms_sync/permissions/full_set.py b/src/src_auth_perms_sync/permissions/full_set.py similarity index 97% rename from src_auth_perms_sync/permissions/full_set.py rename to src/src_auth_perms_sync/permissions/full_set.py index 38a4d5b..d41ec90 100644 --- a/src_auth_perms_sync/permissions/full_set.py +++ b/src/src_auth_perms_sync/permissions/full_set.py @@ -95,6 +95,7 @@ def _capture_full_set_snapshot_state( client: src.SourcegraphClient, input_path: Path, parallelism: int, + explicit_permissions_batch_size: int, bind_id_mode: str, worker_pool: ThreadPoolExecutor | None = None, ) -> _FullSetUserState: @@ -114,6 +115,7 @@ def _capture_full_set_snapshot_state( bind_id_mode, input_path, total_users=total_users, + explicit_permissions_batch_size=explicit_permissions_batch_size, worker_pool=worker_pool, ) log.info( @@ -134,6 +136,7 @@ def _load_full_set_snapshot_state( client: src.SourcegraphClient, input_path: Path, parallelism: int, + explicit_permissions_batch_size: int, bind_id_mode: str, capture_before: bool, worker_pool: ThreadPoolExecutor | None = None, @@ -144,6 +147,7 @@ def _load_full_set_snapshot_state( client, input_path, parallelism, + explicit_permissions_batch_size, bind_id_mode, worker_pool, ) @@ -512,6 +516,7 @@ def _finish_full_set_apply_with_backup( plan: _FullSetPlan, apply_result: _FullSetApplyResult, parallelism: int, + explicit_permissions_batch_size: int, bind_id_mode: str, worker_pool: ThreadPoolExecutor | None = None, ) -> None: @@ -527,6 +532,7 @@ def _finish_full_set_apply_with_backup( bind_id_mode, input_path, total_users=len(snapshot_state.users), + explicit_permissions_batch_size=explicit_permissions_batch_size, worker_pool=worker_pool, ) @@ -607,6 +613,7 @@ def _finish_empty_full_set_mapping_rules( command_name: str, dry_run: bool, parallelism: int, + explicit_permissions_batch_size: int, bind_id_mode: str, do_backup: bool, command_event: dict[str, Any], @@ -620,6 +627,7 @@ def _finish_empty_full_set_mapping_rules( client, input_path, parallelism, + explicit_permissions_batch_size, bind_id_mode, worker_pool, ) @@ -639,6 +647,7 @@ def _load_full_set_plan( mapping_rules: list[permission_types.MappingRule], user_created_after: str | None, parallelism: int, + explicit_permissions_batch_size: int, bind_id_mode: str, saml_groups_attribute_name_by_config_id: dict[str, str], capture_before: bool, @@ -651,6 +660,7 @@ def _load_full_set_plan( client, input_path, parallelism, + explicit_permissions_batch_size, bind_id_mode, capture_before=capture_before, worker_pool=worker_pool, @@ -727,6 +737,7 @@ def _run_full_set_apply( plan: _FullSetPlan, mapping_count: int, parallelism: int, + explicit_permissions_batch_size: int, bind_id_mode: str, do_backup: bool, before_path: Path | None, @@ -766,6 +777,7 @@ def _run_full_set_apply( plan, apply_result, parallelism, + explicit_permissions_batch_size, bind_id_mode, worker_pool, ) @@ -779,6 +791,7 @@ def cmd_set_full( user_created_after: str | None, dry_run: bool, parallelism: int, + explicit_permissions_batch_size: int, bind_id_mode: str, saml_groups_attribute_name_by_config_id: dict[str, str], do_backup: bool, @@ -803,6 +816,7 @@ def cmd_set_full( command_name, dry_run, parallelism, + explicit_permissions_batch_size, bind_id_mode, do_backup, command_event, @@ -816,6 +830,7 @@ def cmd_set_full( mapping_rules, user_created_after, parallelism, + explicit_permissions_batch_size, bind_id_mode, saml_groups_attribute_name_by_config_id, capture_before=dry_run or do_backup, @@ -855,6 +870,7 @@ def cmd_set_full( plan, len(mapping_rules), parallelism, + explicit_permissions_batch_size, bind_id_mode, do_backup, loaded_plan.apply_before_path, diff --git a/src_auth_perms_sync/permissions/mapping.py b/src/src_auth_perms_sync/permissions/mapping.py similarity index 99% rename from src_auth_perms_sync/permissions/mapping.py rename to src/src_auth_perms_sync/permissions/mapping.py index f2ac05b..6d0bd5b 100644 --- a/src_auth_perms_sync/permissions/mapping.py +++ b/src/src_auth_perms_sync/permissions/mapping.py @@ -5,11 +5,11 @@ `codeHostConnection`, and `regex`). Within a matcher, the supplied keys AND together against the discovered auth-provider / external- service entries. Across mapping rules, `cmd_set` unions the per-repo -user sets at apply time — see `src_auth_perms_sync/permissions/types.py` for the rationale. +user sets at apply time — see `src/src_auth_perms_sync/permissions/types.py` for the rationale. Adding a new matcher type: - 1. Add the TypedDict in `src_auth_perms_sync/permissions/types.py`. + 1. Add the TypedDict in `src/src_auth_perms_sync/permissions/types.py`. 2. Add it as a sibling key on `UsersFilter` or `ReposFilter`. 3. Add a branch in `resolve_users` / `resolve_repos` below. 4. Add structural validation in `validate_mapping_rules`. @@ -218,7 +218,7 @@ def resolve_users( `saml_groups_attribute_names` overrides the default `"groups"` SAML assertion attribute name per (serviceID, clientID) — see - `src_auth_perms_sync/shared/saml_groups.py`. When + `src/src_auth_perms_sync/shared/saml_groups.py`. When `None`, every SAML provider falls back to the default. Only consulted by the `authProvider.samlGroup` sub-field. diff --git a/src_auth_perms_sync/permissions/maps.py b/src/src_auth_perms_sync/permissions/maps.py similarity index 99% rename from src_auth_perms_sync/permissions/maps.py rename to src/src_auth_perms_sync/permissions/maps.py index ee8604e..c04994e 100644 --- a/src_auth_perms_sync/permissions/maps.py +++ b/src/src_auth_perms_sync/permissions/maps.py @@ -53,7 +53,7 @@ def auth_provider_to_yaml( `site_config_entry`, when provided, is the matching `auth.providers[*]` JSONC entry (already stripped of redacted/secret fields by - `src_auth_perms_sync/shared/site_config.py`). Any + `src/src_auth_perms_sync/shared/site_config.py`). Any fields it carries that aren't already emitted from GraphQL are surfaced verbatim, so operators see the full provider config in the YAML — e.g. `identityProviderMetadataURL`, `serviceProviderIssuer`, diff --git a/src_auth_perms_sync/permissions/queries.py b/src/src_auth_perms_sync/permissions/queries.py similarity index 95% rename from src_auth_perms_sync/permissions/queries.py rename to src/src_auth_perms_sync/permissions/queries.py index 42fff3f..afa83b5 100644 --- a/src_auth_perms_sync/permissions/queries.py +++ b/src/src_auth_perms_sync/permissions/queries.py @@ -144,8 +144,7 @@ permissionsInfo { repositories(source: API, first: $first, after: $after) { nodes { - repository { id name } - updatedAt + id } pageInfo { hasNextPage endCursor } } @@ -155,13 +154,13 @@ } """ -QUERY_USER_EXPLICIT_REPO_COUNT = """ -query UserExplicitRepoCount($id: ID!) { +QUERY_USER_EXPLICIT_REPO_EXISTS = """ +query UserExplicitRepoExists($id: ID!) { node(id: $id) { ... on User { permissionsInfo { repositories(source: API, first: 1) { - totalCount + nodes { id } } } } diff --git a/src_auth_perms_sync/permissions/restore.py b/src/src_auth_perms_sync/permissions/restore.py similarity index 97% rename from src_auth_perms_sync/permissions/restore.py rename to src/src_auth_perms_sync/permissions/restore.py index 674a221..da7ab1a 100644 --- a/src_auth_perms_sync/permissions/restore.py +++ b/src/src_auth_perms_sync/permissions/restore.py @@ -29,7 +29,7 @@ @dataclass(frozen=True) -class _RestoreSnapshotState: +class RestoreSnapshotState: """Target and live snapshots needed for a full restore.""" target_snapshot: permission_snapshot.Snapshot @@ -38,7 +38,7 @@ class _RestoreSnapshotState: @dataclass(frozen=True) -class _RestorePlan: +class RestorePlan: """Per-repo overwrite plan for a full restore.""" overwrites: list[permission_types.RepositoryUsernameOverwrite] @@ -533,9 +533,10 @@ def _capture_restore_snapshot_state( snapshot_path: Path, target_snapshot: permission_snapshot.Snapshot, parallelism: int, + explicit_permissions_batch_size: int, bind_id_mode: str, worker_pool: ThreadPoolExecutor | None = None, -) -> _RestoreSnapshotState: +) -> RestoreSnapshotState: """Capture the live full-instance state needed to plan a restore.""" total_users = shared_sourcegraph.count_users(client) log.info( @@ -552,6 +553,7 @@ def _capture_restore_snapshot_state( bind_id_mode, snapshot_path, total_users=total_users, + explicit_permissions_batch_size=explicit_permissions_batch_size, worker_pool=worker_pool, ) log.info( @@ -561,14 +563,14 @@ def _capture_restore_snapshot_state( current_snapshot["stats"]["repos_with_explicit_grants"], current_snapshot["stats"]["total_grants"], ) - return _RestoreSnapshotState( + return RestoreSnapshotState( target_snapshot=target_snapshot, current_snapshot=current_snapshot, users=permission_snapshot.compact_snapshot_users(users), ) -def _plan_full_restore(snapshot_state: _RestoreSnapshotState) -> _RestorePlan: +def plan_full_restore(snapshot_state: RestoreSnapshotState) -> RestorePlan: """Build only the per-repo overwrite plans needed to match the snapshot.""" target_repos = snapshot_state.target_snapshot["repos"] current_repos = snapshot_state.current_snapshot["repos"] @@ -597,7 +599,7 @@ def _plan_full_restore(snapshot_state: _RestoreSnapshotState) -> _RestorePlan: usernames=(), ) ) - return _RestorePlan( + return RestorePlan( overwrites=overwrites, snapshot_repo_count=len(target_repos), extra_repo_count=len(extra_repo_ids), @@ -637,7 +639,7 @@ def _finish_empty_restore_plan( ) -def _log_full_restore_plan(snapshot_state: _RestoreSnapshotState, plan: _RestorePlan) -> None: +def _log_full_restore_plan(snapshot_state: RestoreSnapshotState, plan: RestorePlan) -> None: log.info( "Restore plan: %d mutation(s) (%d snapshot repo(s), %d unchanged skipped, " "%d extra repo(s) to wipe).", @@ -658,7 +660,7 @@ def _log_full_restore_plan(snapshot_state: _RestoreSnapshotState, plan: _Restore def _finish_restore_dry_run( client: src.SourcegraphClient, snapshot_path: Path, - snapshot_state: _RestoreSnapshotState, + snapshot_state: RestoreSnapshotState, ) -> None: """Write dry-run restore artifacts and stop before mutation.""" timestamp = backups.backup_timestamp() @@ -733,8 +735,8 @@ def _apply_restore_overwrites( def _record_restore_event_fields( command_event: dict[str, Any], - snapshot_state: _RestoreSnapshotState, - plan: _RestorePlan, + snapshot_state: RestoreSnapshotState, + plan: RestorePlan, mutations: shared_types.MutationCounts, ) -> None: command_event["plan_size"] = len(plan.overwrites) @@ -750,8 +752,9 @@ def _finish_restore_apply_with_backup( client: src.SourcegraphClient, snapshot_path: Path, timestamp: str, - snapshot_state: _RestoreSnapshotState, + snapshot_state: RestoreSnapshotState, parallelism: int, + explicit_permissions_batch_size: int, bind_id_mode: str, worker_pool: ThreadPoolExecutor | None = None, ) -> None: @@ -764,6 +767,7 @@ def _finish_restore_apply_with_backup( bind_id_mode, snapshot_path, total_users=len(snapshot_state.users), + explicit_permissions_batch_size=explicit_permissions_batch_size, worker_pool=worker_pool, ) after_restore_path = snapshot_artifact_path( @@ -823,6 +827,7 @@ def cmd_restore( snapshot_path: Path, dry_run: bool, parallelism: int, + explicit_permissions_batch_size: int, bind_id_mode: str, do_backup: bool, worker_pool: ThreadPoolExecutor | None = None, @@ -861,10 +866,11 @@ def cmd_restore( snapshot_path, target_full_snapshot, parallelism, + explicit_permissions_batch_size, bind_id_mode, worker_pool, ) - plan = _plan_full_restore(snapshot_state) + plan = plan_full_restore(snapshot_state) if not plan.overwrites: _finish_empty_restore_plan( client, @@ -899,6 +905,7 @@ def cmd_restore( timestamp, snapshot_state, parallelism, + explicit_permissions_batch_size, bind_id_mode, worker_pool, ) diff --git a/src_auth_perms_sync/permissions/snapshot.py b/src/src_auth_perms_sync/permissions/snapshot.py similarity index 93% rename from src_auth_perms_sync/permissions/snapshot.py rename to src/src_auth_perms_sync/permissions/snapshot.py index c05cebe..fd57375 100644 --- a/src_auth_perms_sync/permissions/snapshot.py +++ b/src/src_auth_perms_sync/permissions/snapshot.py @@ -161,6 +161,7 @@ def capture_explicit_grants( client: src.SourcegraphClient, users: Iterable[SnapshotUserInput], parallelism: int, + explicit_permissions_batch_size: int, total_users: int | None = None, worker_pool: ThreadPoolExecutor | None = None, ) -> tuple[dict[str, RepoSnapshot], int]: @@ -191,14 +192,13 @@ def capture_explicit_grants( that need the user-count statistic don't have to materialize the iterator twice or measure it themselves. """ - # Invert directly as each per-user fetch completes. This avoids - # retaining a second, per-user copy of every repository grant before - # building the repo-centric snapshot. - repos_out: dict[str, RepoSnapshot] = {} + # Invert directly as each per-user fetch completes. Store only repo IDs + # first, then hydrate each unique repo name once after all users complete. + usernames_by_repository_id: dict[str, list[str]] = {} def _fetch( batch_users: list[SnapshotUserInput], - ) -> tuple[dict[str, list[permission_types.Repository]], int]: + ) -> tuple[dict[str, list[str]], int]: # High-frequency (one per user-batch): # - log the whole event (start + end) at DEBUG; failures still # get bumped to ERROR by the event() helper @@ -212,9 +212,10 @@ def _fetch( user_count=len(batch_users), ) as fetch_event: try: - repos_by_user_id = permissions_sourcegraph.list_users_explicit_repos( + repository_ids_by_user_id = permissions_sourcegraph.list_users_explicit_repo_ids( client, [user["id"] for user in batch_users], + batch_size=explicit_permissions_batch_size, ) failures = 0 except Exception as exception: @@ -224,24 +225,29 @@ def _fetch( len(batch_users), exception, ) - repos_by_user_id, failures = _fetch_one_user_at_a_time(batch_users) - repos_by_username = { - user["username"]: repos_by_user_id.get(user["id"], []) for user in batch_users + repository_ids_by_user_id, failures = _fetch_one_user_at_a_time(batch_users) + repository_ids_by_username = { + user["username"]: repository_ids_by_user_id.get(user["id"], []) + for user in batch_users } - fetch_event["repo_count"] = sum(len(repos) for repos in repos_by_username.values()) + fetch_event["repo_count"] = sum( + len(repository_ids) for repository_ids in repository_ids_by_username.values() + ) fetch_event["per_user_failures"] = failures - return repos_by_username, failures + return repository_ids_by_username, failures def _fetch_one_user_at_a_time( batch_users: list[SnapshotUserInput], - ) -> tuple[dict[str, list[permission_types.Repository]], int]: - repos_by_user_id: dict[str, list[permission_types.Repository]] = {} + ) -> tuple[dict[str, list[str]], int]: + repository_ids_by_user_id: dict[str, list[str]] = {} failures = 0 for user in batch_users: try: - repos_by_user_id[user["id"]] = permissions_sourcegraph.list_user_explicit_repos( - client, - user["id"], + repository_ids_by_user_id[user["id"]] = ( + permissions_sourcegraph.list_user_explicit_repo_ids( + client, + user["id"], + ) ) except Exception as exception: failures += 1 @@ -250,10 +256,14 @@ def _fetch_one_user_at_a_time( user["username"], exception, ) - repos_by_user_id[user["id"]] = [] - return repos_by_user_id, failures + repository_ids_by_user_id[user["id"]] = [] + return repository_ids_by_user_id, failures - with src.event("capture_explicit_grants", total_users=total_users) as capture_event: + with src.event( + "capture_explicit_grants", + total_users=total_users, + explicit_permissions_batch_size=explicit_permissions_batch_size, + ) as capture_event: capture_failures = 0 futures: dict[Any, list[SnapshotUserInput]] = {} submitted_user_count = 0 @@ -292,15 +302,14 @@ def _record_completed_futures(done_futures: Iterable[Any]) -> None: submitted_batch = futures.pop(future) completed += len(submitted_batch) try: - repos_by_username, failures = future.result() + repository_ids_by_username, failures = future.result() capture_failures += failures - for username, repos in repos_by_username.items(): - for repo in repos: - entry = repos_out.setdefault( - repo["id"], - {"name": repo["name"], "explicit_permissions_users": []}, - ) - entry["explicit_permissions_users"].append(username) + for username, repository_ids in repository_ids_by_username.items(): + for repository_id in repository_ids: + usernames_by_repository_id.setdefault( + repository_id, + [], + ).append(username) except Exception as exception: # Don't blow up the whole capture; warn so the operator # can see the users whose grants were treated as empty. @@ -346,7 +355,7 @@ def _record_completed_futures(done_futures: Iterable[Any]) -> None: batch_users: list[SnapshotUserInput] = [] for user in users: batch_users.append(user) - if len(batch_users) >= permissions_sourcegraph.USER_EXPLICIT_REPOS_BATCH_SIZE: + if len(batch_users) >= explicit_permissions_batch_size: _submit_batch(executor, batch_users) batch_users = [] if len(futures) >= max_pending_batches: @@ -363,19 +372,52 @@ def _record_completed_futures(done_futures: Iterable[Any]) -> None: capture_event["max_pending_batches"] = max_pending_batches # Stable sort: users alphabetical within each repo. - for entry in repos_out.values(): - entry["explicit_permissions_users"].sort() + for usernames in usernames_by_repository_id.values(): + usernames.sort() + + with src.event( + "hydrate_explicit_repository_names", + repository_count=len(usernames_by_repository_id), + ) as hydrate_event: + repositories_by_id = permissions_sourcegraph.list_repositories_by_ids( + client, + usernames_by_repository_id.keys(), + ) + hydrate_event["hydrated_repository_count"] = len(repositories_by_id) + + repos_out: dict[str, RepoSnapshot] = {} + for repository_id, usernames in usernames_by_repository_id.items(): + repos_out[repository_id] = { + "name": _snapshot_repository_name(repositories_by_id, repository_id), + "explicit_permissions_users": usernames, + } return repos_out, submitted_user_count +def _snapshot_repository_name( + repositories_by_id: dict[str, permission_types.Repository], + repository_id: str, +) -> str: + repository = repositories_by_id.get(repository_id) + if repository is not None: + return repository["name"] + try: + decoded_repository_id = id_codec.decode_repository_id(repository_id) + return f"" + except ValueError: + return f"" + + def build_snapshot( client: src.SourcegraphClient, users: Iterable[SnapshotUserInput], parallelism: int, bind_id_mode: str, config_path: Path | None = None, + *, total_users: int | None = None, + explicit_permissions_batch_size: int, worker_pool: ThreadPoolExecutor | None = None, ) -> Snapshot: """Capture a full Snapshot: explicit grants + pending-bindIDs + metadata. @@ -393,6 +435,7 @@ def build_snapshot( client, users, parallelism, + explicit_permissions_batch_size, total_users=total_users, worker_pool=worker_pool, ) diff --git a/src_auth_perms_sync/permissions/sourcegraph.py b/src/src_auth_perms_sync/permissions/sourcegraph.py similarity index 59% rename from src_auth_perms_sync/permissions/sourcegraph.py rename to src/src_auth_perms_sync/permissions/sourcegraph.py index 2e7730f..08aaae4 100644 --- a/src_auth_perms_sync/permissions/sourcegraph.py +++ b/src/src_auth_perms_sync/permissions/sourcegraph.py @@ -2,17 +2,19 @@ from __future__ import annotations -from collections.abc import Sequence +import logging +from collections.abc import Iterable, Iterator, Sequence from typing import Any, cast import src_py_lib as src +from ..shared import id_codec from ..shared import sourcegraph as shared_sourcegraph from ..shared import types as shared_types from . import queries from . import types as permission_types -USER_EXPLICIT_REPOS_BATCH_SIZE = 25 +log = logging.getLogger(__name__) def list_external_services(client: src.SourcegraphClient) -> list[permission_types.ExternalService]: @@ -104,7 +106,7 @@ def user_has_explicit_repos(client: src.SourcegraphClient, user_id: str) -> bool data = cast( dict[str, Any], client.graphql( - queries.QUERY_USER_EXPLICIT_REPO_COUNT, + queries.QUERY_USER_EXPLICIT_REPO_EXISTS, cast(src.JSONDict, {"id": user_id}), ), ) @@ -115,7 +117,7 @@ def user_has_explicit_repos(client: src.SourcegraphClient, user_id: str) -> bool if permissions_info is None: return False repositories = cast(dict[str, Any], permissions_info["repositories"]) - return cast(int, repositories["totalCount"]) > 0 + return bool(src.json_list(repositories.get("nodes"))) def list_user_explicit_repos( @@ -127,32 +129,48 @@ def list_user_explicit_repos( Repository TypedDict shape). Empty list if the user has no explicit grants OR if `permissionsInfo` is null (e.g. soft-deleted user). """ - repos: list[permission_types.Repository] = [] + return _repositories_from_ids(client, list_user_explicit_repo_ids(client, user_id)) + + +def list_user_explicit_repo_ids(client: src.SourcegraphClient, user_id: str) -> list[str]: + """Return repository IDs with `source: API` grants for `user_id`.""" + repository_ids: list[str] = [] for node in client.stream_connection_nodes( queries.QUERY_USER_EXPLICIT_REPOS, {"id": user_id}, connection_path=("node", "permissionsInfo", "repositories"), page_size=shared_sourcegraph.DEFAULT_PAGE_SIZE, ): - repo = cast(permission_types.Repository | None, node.get("repository")) - if repo is not None: - repos.append(repo) - return repos + repository_id = _permission_node_repository_id(node) + if repository_id is not None: + repository_ids.append(repository_id) + return repository_ids def list_users_explicit_repos( client: src.SourcegraphClient, user_ids: Sequence[str], *, - batch_size: int = USER_EXPLICIT_REPOS_BATCH_SIZE, + batch_size: int, ) -> dict[str, list[permission_types.Repository]]: """Return explicit API repository grants for many users using GraphQL aliases.""" + return _repositories_by_user_id( + client, + list_users_explicit_repo_ids(client, user_ids, batch_size=batch_size), + ) + + +def list_users_explicit_repo_ids( + client: src.SourcegraphClient, + user_ids: Sequence[str], + *, + batch_size: int, +) -> dict[str, list[str]]: + """Return explicit API repository IDs for many users using GraphQL aliases.""" if batch_size < 1: raise ValueError("batch_size must be at least 1") - repos_by_user_id: dict[str, list[permission_types.Repository]] = { - user_id: [] for user_id in user_ids - } + repository_ids_by_user_id: dict[str, list[str]] = {user_id: [] for user_id in user_ids} pending_pages: list[tuple[str, str | None]] = [(user_id, None) for user_id in user_ids] graphql_client = _graphql_client_without_auto_pagination(client) while pending_pages: @@ -167,7 +185,7 @@ def list_users_explicit_repos( connection = _user_explicit_repos_connection(data, index) if connection is None: continue - repos_by_user_id[user_id].extend(_connection_repositories(connection)) + repository_ids_by_user_id[user_id].extend(_connection_repository_ids(connection)) page_info = src.json_dict(connection.get("pageInfo")) has_next_page = page_info.get("hasNextPage") if not isinstance(has_next_page, bool): @@ -185,7 +203,39 @@ def list_users_explicit_repos( f"UserExplicitReposBatch user{index} cursor stalled at {next_cursor!r}" ) pending_pages.append((user_id, next_cursor)) - return repos_by_user_id + return repository_ids_by_user_id + + +def list_repositories_by_ids( + client: src.SourcegraphClient, + repository_ids: Iterable[str], + *, + batch_size: int = shared_sourcegraph.DEFAULT_PAGE_SIZE, +) -> dict[str, permission_types.Repository]: + """Return repository `{id, name}` objects for unique GraphQL repository IDs.""" + if batch_size < 1: + raise ValueError("batch_size must be at least 1") + + unique_repository_ids = list(dict.fromkeys(repository_ids)) + repositories: dict[str, permission_types.Repository] = {} + for batch in _batches(unique_repository_ids, batch_size): + data = cast( + dict[str, Any], + client.graphql( + _repositories_by_id_query(len(batch)), + _repositories_by_id_variables(batch), + ), + ) + for index, requested_repository_id in enumerate(batch): + repository = src.json_dict(data.get(f"repo{index}")) + returned_repository_id = repository.get("id") + repository_name = repository.get("name") + if isinstance(returned_repository_id, str) and isinstance(repository_name, str): + repositories[requested_repository_id] = { + "id": returned_repository_id, + "name": repository_name, + } + return repositories def _graphql_client_without_auto_pagination(client: src.SourcegraphClient) -> src.GraphQLClient: @@ -197,6 +247,11 @@ def _graphql_client_without_auto_pagination(client: src.SourcegraphClient) -> sr ) +def _batches(values: Sequence[str], batch_size: int) -> Iterator[Sequence[str]]: + for start_index in range(0, len(values), batch_size): + yield values[start_index : start_index + batch_size] + + def _user_explicit_repos_batch_query(batch_size: int) -> str: variables = ["$first: Int!"] fields: list[str] = [] @@ -209,8 +264,7 @@ def _user_explicit_repos_batch_query(batch_size: int) -> str: permissionsInfo {{ repositories(source: API, first: $first, after: $after{index}) {{ nodes {{ - repository {{ id name }} - updatedAt + id }} pageInfo {{ hasNextPage endCursor }} }} @@ -238,16 +292,98 @@ def _user_explicit_repos_connection(data: src.JSONDict, index: int) -> src.JSOND return connection or None -def _connection_repositories(connection: src.JSONDict) -> list[permission_types.Repository]: - repos: list[permission_types.Repository] = [] +def _connection_repository_ids(connection: src.JSONDict) -> list[str]: + repository_ids: list[str] = [] for permission_node_value in src.json_list(connection.get("nodes")): permission_node = src.json_dict(permission_node_value) - repository = src.json_dict(permission_node.get("repository")) - repo_id = repository.get("id") - repo_name = repository.get("name") - if isinstance(repo_id, str) and isinstance(repo_name, str): - repos.append({"id": repo_id, "name": repo_name}) - return repos + repository_id = _permission_node_repository_id(permission_node) + if repository_id is not None: + repository_ids.append(repository_id) + return repository_ids + + +def _permission_node_repository_id(permission_node: src.JSONDict) -> str | None: + repository_id = permission_node.get("id") + return repository_id if isinstance(repository_id, str) else None + + +def _repositories_from_ids( + client: src.SourcegraphClient, + repository_ids: Sequence[str], +) -> list[permission_types.Repository]: + repositories_by_id = list_repositories_by_ids(client, repository_ids) + return [ + _repository_or_placeholder(repositories_by_id, repository_id) + for repository_id in repository_ids + ] + + +def _repositories_by_user_id( + client: src.SourcegraphClient, + repository_ids_by_user_id: dict[str, list[str]], +) -> dict[str, list[permission_types.Repository]]: + unique_repository_ids = list( + dict.fromkeys( + repository_id + for repository_ids in repository_ids_by_user_id.values() + for repository_id in repository_ids + ) + ) + repositories_by_id = list_repositories_by_ids(client, unique_repository_ids) + missing_repository_ids = set(unique_repository_ids) - set(repositories_by_id) + if missing_repository_ids: + log.warning( + "Could not hydrate names for %d repository ID(s); using ID placeholders.", + len(missing_repository_ids), + ) + return { + user_id: [ + _repository_or_placeholder(repositories_by_id, repository_id) + for repository_id in repository_ids + ] + for user_id, repository_ids in repository_ids_by_user_id.items() + } + + +def _repository_or_placeholder( + repositories_by_id: dict[str, permission_types.Repository], + repository_id: str, +) -> permission_types.Repository: + repository = repositories_by_id.get(repository_id) + if repository is not None: + return repository + return _missing_repository(repository_id) + + +def _missing_repository(repository_id: str) -> permission_types.Repository: + try: + decoded_repository_id = id_codec.decode_repository_id(repository_id) + repository_name = f"" + except ValueError: + repository_name = f"" + return {"id": repository_id, "name": repository_name} + + +def _repositories_by_id_query(batch_size: int) -> str: + variables = [f"$repo{index}: ID!" for index in range(batch_size)] + fields = [ + f""" + repo{index}: node(id: $repo{index}) {{ + ... on Repository {{ + id + name + }} + }}""" + for index in range(batch_size) + ] + return "query RepositoryNamesByID(" + ", ".join(variables) + ") {" + "".join(fields) + "\n}" + + +def _repositories_by_id_variables(repository_ids: Sequence[str]) -> src.JSONDict: + return cast( + src.JSONDict, + {f"repo{index}": repository_id for index, repository_id in enumerate(repository_ids)}, + ) def list_pending_bind_ids(client: src.SourcegraphClient) -> list[str]: diff --git a/src_auth_perms_sync/permissions/types.py b/src/src_auth_perms_sync/permissions/types.py similarity index 100% rename from src_auth_perms_sync/permissions/types.py rename to src/src_auth_perms_sync/permissions/types.py diff --git a/src_auth_perms_sync/permissions/workflow.py b/src/src_auth_perms_sync/permissions/workflow.py similarity index 100% rename from src_auth_perms_sync/permissions/workflow.py rename to src/src_auth_perms_sync/permissions/workflow.py diff --git a/src_auth_perms_sync/shared/__init__.py b/src/src_auth_perms_sync/shared/__init__.py similarity index 100% rename from src_auth_perms_sync/shared/__init__.py rename to src/src_auth_perms_sync/shared/__init__.py diff --git a/src_auth_perms_sync/shared/backups.py b/src/src_auth_perms_sync/shared/backups.py similarity index 100% rename from src_auth_perms_sync/shared/backups.py rename to src/src_auth_perms_sync/shared/backups.py diff --git a/src_auth_perms_sync/shared/id_codec.py b/src/src_auth_perms_sync/shared/id_codec.py similarity index 100% rename from src_auth_perms_sync/shared/id_codec.py rename to src/src_auth_perms_sync/shared/id_codec.py diff --git a/src_auth_perms_sync/shared/queries.py b/src/src_auth_perms_sync/shared/queries.py similarity index 100% rename from src_auth_perms_sync/shared/queries.py rename to src/src_auth_perms_sync/shared/queries.py diff --git a/src_auth_perms_sync/shared/run_context.py b/src/src_auth_perms_sync/shared/run_context.py similarity index 100% rename from src_auth_perms_sync/shared/run_context.py rename to src/src_auth_perms_sync/shared/run_context.py diff --git a/src_auth_perms_sync/shared/saml_groups.py b/src/src_auth_perms_sync/shared/saml_groups.py similarity index 100% rename from src_auth_perms_sync/shared/saml_groups.py rename to src/src_auth_perms_sync/shared/saml_groups.py diff --git a/src_auth_perms_sync/shared/site_config.py b/src/src_auth_perms_sync/shared/site_config.py similarity index 100% rename from src_auth_perms_sync/shared/site_config.py rename to src/src_auth_perms_sync/shared/site_config.py diff --git a/src_auth_perms_sync/shared/sourcegraph.py b/src/src_auth_perms_sync/shared/sourcegraph.py similarity index 100% rename from src_auth_perms_sync/shared/sourcegraph.py rename to src/src_auth_perms_sync/shared/sourcegraph.py diff --git a/src_auth_perms_sync/shared/types.py b/src/src_auth_perms_sync/shared/types.py similarity index 94% rename from src_auth_perms_sync/shared/types.py rename to src/src_auth_perms_sync/shared/types.py index 62268d0..9429a41 100644 --- a/src_auth_perms_sync/shared/types.py +++ b/src/src_auth_perms_sync/shared/types.py @@ -21,7 +21,7 @@ class ExternalAccount(TypedDict): clientID: str # Provider-specific JSON; for SAML this is the gosaml2 AssertionInfo # (Assertions[].AttributeStatement.Attributes[].{Name,Values[].Value}). - # See `src_auth_perms_sync/shared/saml_groups.py` for the parser. Site-admin only; + # See `src/src_auth_perms_sync/shared/saml_groups.py` for the parser. Site-admin only; # null for accounts where the server does not expose it. accountData: NotRequired[dict[str, Any] | None] diff --git a/tests/unit/test_cli_config.py b/tests/unit/test_cli_config.py index b91fb3a..431ebf6 100644 --- a/tests/unit/test_cli_config.py +++ b/tests/unit/test_cli_config.py @@ -2,12 +2,16 @@ import contextlib import io +import re import tempfile import unittest from concurrent.futures import ThreadPoolExecutor from pathlib import Path +from typing import cast from unittest import mock +import httpx +import src_py_lib as src from src_py_lib.utils import config as shared_config from src_auth_perms_sync import cli @@ -169,6 +173,44 @@ def test_created_after_config_rejects_values_outside_yyyy_mm_dd_shape(self) -> N ): load_config_from_env(SRC_AUTH_PERMS_SYNC_CREATED_AFTER=invalid_value) + def test_explicit_permissions_batch_size_config_is_loaded_from_env(self) -> None: + config = load_config_from_env(SRC_AUTH_PERMS_SYNC_EXPLICIT_PERMISSIONS_BATCH_SIZE="50") + + self.assertEqual(50, config.explicit_permissions_batch_size) + + def test_explicit_permissions_batch_size_rejects_values_below_one(self) -> None: + with self.assertRaisesRegex(shared_config.ConfigError, "greater than or equal to 1"): + load_config_from_env(SRC_AUTH_PERMS_SYNC_EXPLICIT_PERMISSIONS_BATCH_SIZE="0") + + def test_trace_config_is_loaded_from_env(self) -> None: + config = load_config_from_env(SRC_AUTH_PERMS_SYNC_TRACE="true") + + self.assertTrue(config.trace) + + def test_trace_sampling_http_client_adds_sampled_traceparent(self) -> None: + traceparents: list[str] = [] + + def handler(request: httpx.Request) -> httpx.Response: + traceparents.append(request.headers["traceparent"]) + return httpx.Response(200, json={"ok": True}) + + client = cli.TraceSamplingHTTPClient( + max_attempts=1, + transport=httpx.MockTransport(handler), + ) + try: + self.assertEqual( + {"ok": True}, + client.json("POST", "https://sourcegraph.example.com/.api/graphql"), + ) + finally: + client.close() + + self.assertEqual(1, len(traceparents)) + self.assertRegex(traceparents[0], r"^00-[0-9a-f]{32}-[0-9a-f]{16}-01$") + self.assertNotEqual("0" * 32, re.split("-", traceparents[0])[1]) + self.assertNotEqual("0" * 16, re.split("-", traceparents[0])[2]) + def test_validate_config_rejects_multiple_set_modes(self) -> None: self.assert_config_error( make_config(set_path=Path("maps.yaml"), full=True, user="alice"), @@ -204,11 +246,13 @@ def test_run_fields_include_concrete_command(self) -> None: self.assertEqual("set", fields["base_cmd"]) self.assertEqual("user", fields["set_mode"]) self.assertEqual(True, fields["apply_flag"]) + self.assertEqual(25, fields["explicit_permissions_batch_size"]) + self.assertEqual(False, fields["trace"]) def test_run_command_passes_primary_data_to_combined_sync(self) -> None: configuration = make_config(get=True, sync_saml_organizations=True) command = cli.resolve_command(configuration) - client = object() + client = cast(src.SourcegraphClient, object()) sourcegraph_site_config = object() command_data = cli.run_context.CommandData() diff --git a/tests/unit/test_maps.py b/tests/unit/test_maps.py index 9bd00c6..34bb483 100644 --- a/tests/unit/test_maps.py +++ b/tests/unit/test_maps.py @@ -7,6 +7,7 @@ import yaml from src_auth_perms_sync.permissions import maps +from src_auth_perms_sync.shared import types as shared_types class MapsTests(unittest.TestCase): @@ -31,7 +32,7 @@ def test_default_maps_yaml_is_valid_yaml(self) -> None: self.assertEqual({"maps": [{"name": "Map 1"}]}, yaml.safe_load(maps_path.read_text())) def test_count_users_per_provider_counts_each_user_once_per_provider(self) -> None: - users = [ + users: list[shared_types.User] = [ { "id": "user-1", "username": "alice", diff --git a/tests/unit/test_restore.py b/tests/unit/test_restore.py index 54803aa..ac7f1e4 100644 --- a/tests/unit/test_restore.py +++ b/tests/unit/test_restore.py @@ -40,13 +40,13 @@ def test_plan_full_restore_skips_repos_that_already_match(self) -> None: ), } ) - snapshot_state = permission_restore._RestoreSnapshotState( + snapshot_state = permission_restore.RestoreSnapshotState( target_snapshot=target_snapshot, current_snapshot=current_snapshot, users=[], ) - plan = permission_restore._plan_full_restore(snapshot_state) + plan = permission_restore.plan_full_restore(snapshot_state) self.assertEqual(2, len(plan.overwrites)) self.assertEqual(2, plan.snapshot_repo_count) diff --git a/tests/unit/test_saml_groups.py b/tests/unit/test_saml_groups.py index b6002b0..7640785 100644 --- a/tests/unit/test_saml_groups.py +++ b/tests/unit/test_saml_groups.py @@ -47,7 +47,7 @@ def test_extract_saml_groups_from_flattened_saml_values(self) -> None: ) def test_attribute_names_by_provider_key_uses_only_saml_providers_with_overrides(self) -> None: - providers = [ + providers: list[shared_types.AuthProvider] = [ { "serviceType": "saml", "serviceID": "https://idp.example.com", @@ -80,7 +80,7 @@ def test_attribute_names_by_provider_key_uses_only_saml_providers_with_overrides ) def test_count_users_per_saml_group_counts_missing_and_deduplicates_user_groups(self) -> None: - users = [ + users: list[shared_types.User] = [ { "id": "user-1", "username": "alice", @@ -159,7 +159,7 @@ def test_count_users_per_saml_group_counts_missing_and_deduplicates_user_groups( ) def test_compact_saml_group_users_keeps_only_org_sync_fields(self) -> None: - providers = [ + providers: list[shared_types.AuthProvider] = [ { "serviceType": "saml", "serviceID": "https://idp.example.com", @@ -177,7 +177,7 @@ def test_compact_saml_group_users_keeps_only_org_sync_fields(self) -> None: "configID": "github", }, ] - users = [ + users: list[shared_types.User] = [ { "id": "user-1", "username": "alice", diff --git a/tests/unit/test_snapshot.py b/tests/unit/test_snapshot.py index a25bd40..e23d5c4 100644 --- a/tests/unit/test_snapshot.py +++ b/tests/unit/test_snapshot.py @@ -3,15 +3,18 @@ import json import tempfile import unittest -from collections.abc import Iterable +from collections.abc import Iterable, Sequence from concurrent.futures import Future from pathlib import Path from types import SimpleNamespace -from typing import Any +from typing import Any, cast from unittest.mock import patch +import src_py_lib as src + from src_auth_perms_sync.permissions import snapshot as permission_snapshot from src_auth_perms_sync.permissions import sourcegraph as permissions_sourcegraph +from src_auth_perms_sync.permissions import types as permission_types from src_auth_perms_sync.permissions import workflow as permission_workflow from src_auth_perms_sync.shared import backups, id_codec @@ -20,47 +23,59 @@ class SnapshotTests(unittest.TestCase): def test_capture_explicit_grants_inverts_repos_without_per_user_buffer(self) -> None: repo_one_id = id_codec.encode_repository_id(1) repo_two_id = id_codec.encode_repository_id(2) - users = [ - { - "id": "user-1", - "username": "carol", - "builtinAuth": True, - "externalAccounts": {"nodes": []}, - }, - { - "id": "user-2", - "username": "alice", - "builtinAuth": True, - "externalAccounts": {"nodes": []}, - }, - { - "id": "user-3", - "username": "bob", - "builtinAuth": True, - "externalAccounts": {"nodes": []}, - }, + users: list[permission_snapshot.SnapshotUser] = [ + {"id": "user-1", "username": "carol"}, + {"id": "user-2", "username": "alice"}, + {"id": "user-3", "username": "bob"}, ] - repos_by_user_id = { - "user-1": [ - {"id": repo_one_id, "name": "github.com/sourcegraph/one"}, - {"id": repo_two_id, "name": "github.com/sourcegraph/two"}, - ], - "user-2": [{"id": repo_one_id, "name": "github.com/sourcegraph/one"}], + repository_ids_by_user_id: dict[str, list[str]] = { + "user-1": [repo_one_id, repo_two_id], + "user-2": [repo_one_id], "user-3": [], } + repositories_by_id: dict[str, permission_types.Repository] = { + repo_one_id: {"id": repo_one_id, "name": "github.com/sourcegraph/one"}, + repo_two_id: {"id": repo_two_id, "name": "github.com/sourcegraph/two"}, + } + hydrated_repository_ids: list[str] = [] + + def list_repo_ids( + _client: src.SourcegraphClient, + user_ids: Sequence[str], + *, + batch_size: int, + ) -> dict[str, list[str]]: + return {user_id: repository_ids_by_user_id[user_id] for user_id in user_ids} + + def list_repositories_by_ids( + _client: src.SourcegraphClient, + repository_ids: Iterable[str], + ) -> dict[str, permission_types.Repository]: + hydrated_repository_ids.extend(repository_ids) + return repositories_by_id - with patch.object( - permission_snapshot.permissions_sourcegraph, - "list_users_explicit_repos", - side_effect=lambda _client, user_ids: { - user_id: repos_by_user_id[user_id] for user_id in user_ids - }, + with ( + patch.object( + permission_snapshot.permissions_sourcegraph, + "list_users_explicit_repo_ids", + side_effect=list_repo_ids, + ), + patch.object( + permission_snapshot.permissions_sourcegraph, + "list_repositories_by_ids", + side_effect=list_repositories_by_ids, + ), ): repos, user_count = permission_snapshot.capture_explicit_grants( - object(), users, parallelism=1, total_users=len(users) + cast(src.SourcegraphClient, object()), + users, + parallelism=1, + explicit_permissions_batch_size=25, + total_users=len(users), ) self.assertEqual(3, user_count) + self.assertEqual([repo_one_id, repo_two_id], hydrated_repository_ids) self.assertEqual( { repo_one_id: { @@ -76,14 +91,8 @@ def test_capture_explicit_grants_inverts_repos_without_per_user_buffer(self) -> ) def test_capture_explicit_grants_bounds_pending_batches(self) -> None: - users = [ - { - "id": f"user-{index}", - "username": f"user-{index}", - "builtinAuth": True, - "externalAccounts": {"nodes": []}, - } - for index in range(9) + users: list[permission_snapshot.SnapshotUser] = [ + {"id": f"user-{index}", "username": f"user-{index}"} for index in range(9) ] pending_counts: list[int] = [] real_wait = permission_snapshot.wait @@ -93,17 +102,33 @@ def recording_wait(futures: Iterable[Future[Any]], **kwargs: Any) -> Any: pending_counts.append(len(futures_list)) return real_wait(futures_list, **kwargs) + def list_repo_ids( + _client: src.SourcegraphClient, + user_ids: Sequence[str], + *, + batch_size: int, + ) -> dict[str, list[str]]: + return {user_id: [] for user_id in user_ids} + with ( - patch.object(permissions_sourcegraph, "USER_EXPLICIT_REPOS_BATCH_SIZE", 1), patch.object( permission_snapshot.permissions_sourcegraph, - "list_users_explicit_repos", - side_effect=lambda _client, user_ids: {user_id: [] for user_id in user_ids}, + "list_users_explicit_repo_ids", + side_effect=list_repo_ids, + ), + patch.object( + permission_snapshot.permissions_sourcegraph, + "list_repositories_by_ids", + return_value={}, ), patch.object(permission_snapshot, "wait", side_effect=recording_wait), ): _, user_count = permission_snapshot.capture_explicit_grants( - object(), users, parallelism=2, total_users=len(users) + cast(src.SourcegraphClient, object()), + users, + parallelism=2, + explicit_permissions_batch_size=1, + total_users=len(users), ) self.assertEqual(9, user_count) @@ -111,25 +136,45 @@ def recording_wait(futures: Iterable[Future[Any]], **kwargs: Any) -> Any: self.assertLessEqual(max(pending_counts), 4) def test_list_users_explicit_repos_batches_aliases_and_follows_pages(self) -> None: - repo_one = {"id": id_codec.encode_repository_id(1), "name": "github.com/sourcegraph/one"} - repo_two = {"id": id_codec.encode_repository_id(2), "name": "github.com/sourcegraph/two"} - repo_three = { + repo_one: permission_types.Repository = { + "id": id_codec.encode_repository_id(1), + "name": "github.com/sourcegraph/one", + } + repo_two: permission_types.Repository = { + "id": id_codec.encode_repository_id(2), + "name": "github.com/sourcegraph/two", + } + repo_three: permission_types.Repository = { "id": id_codec.encode_repository_id(3), "name": "github.com/sourcegraph/three", } - calls: list[tuple[str, dict[str, object], bool]] = [] - responses = [ - { - "user0": self.user_explicit_repos_page([repo_one], has_next_page=False), - "user1": self.user_explicit_repos_page( - [repo_two], - has_next_page=True, - end_cursor="cursor-two", - ), - }, - { - "user0": self.user_explicit_repos_page([repo_three], has_next_page=False), - }, + calls: list[tuple[str, src.JSONDict, bool]] = [] + responses: list[src.JSONDict] = [ + cast( + src.JSONDict, + { + "user0": self.user_explicit_repos_page([repo_one], has_next_page=False), + "user1": self.user_explicit_repos_page( + [repo_two], + has_next_page=True, + end_cursor="cursor-two", + ), + }, + ), + cast( + src.JSONDict, + { + "user0": self.user_explicit_repos_page([repo_three], has_next_page=False), + }, + ), + cast( + src.JSONDict, + { + "repo0": repo_one, + "repo1": repo_two, + "repo2": repo_three, + }, + ), ] class FakeGraphQLClient: @@ -139,17 +184,24 @@ def __init__(self, **_kwargs: object) -> None: def execute( self, query: str, - variables: dict[str, object], + variables: src.JSONDict, *, follow_pages: bool = True, - ) -> dict[str, object]: + ) -> src.JSONDict: calls.append((query, dict(variables), follow_pages)) return responses.pop(0) - client = SimpleNamespace( - endpoint="https://sourcegraph.example.com", - token="secret", - http=object(), + def graphql(query: str, variables: object = None) -> src.JSONDict: + return FakeGraphQLClient().execute(query, cast(src.JSONDict, variables or {})) + + client = cast( + src.SourcegraphClient, + SimpleNamespace( + endpoint="https://sourcegraph.example.com", + token="secret", + http=object(), + graphql=graphql, + ), ) with patch.object(permissions_sourcegraph.src, "GraphQLClient", FakeGraphQLClient): repos_by_user_id = permissions_sourcegraph.list_users_explicit_repos( @@ -167,6 +219,8 @@ def execute( ) self.assertIn("user0: node(id: $user0)", calls[0][0]) self.assertIn("user1: node(id: $user1)", calls[0][0]) + self.assertNotIn("repository {", calls[0][0]) + self.assertNotIn("updatedAt", calls[0][0]) self.assertFalse(calls[0][2]) self.assertEqual("user-1", calls[0][1]["user0"]) self.assertEqual("user-2", calls[0][1]["user1"]) @@ -175,6 +229,10 @@ def execute( self.assertFalse(calls[1][2]) self.assertEqual("user-2", calls[1][1]["user0"]) self.assertEqual("cursor-two", calls[1][1]["after0"]) + self.assertIn("repo0: node(id: $repo0)", calls[2][0]) + self.assertEqual(repo_one["id"], calls[2][1]["repo0"]) + self.assertEqual(repo_two["id"], calls[2][1]["repo1"]) + self.assertEqual(repo_three["id"], calls[2][1]["repo2"]) def test_write_snapshot_uses_username_list_for_explicit_permissions(self) -> None: snapshot = self.make_snapshot() @@ -308,19 +366,19 @@ def make_snapshot(self) -> permission_snapshot.Snapshot: def user_explicit_repos_page( self, - repositories: list[dict[str, str]], + repositories: list[permission_types.Repository], *, has_next_page: bool, end_cursor: str | None = None, - ) -> dict[str, object]: - return { - "permissionsInfo": { - "repositories": { - "nodes": [ - {"repository": repository, "updatedAt": "2026-05-27T00:00:00Z"} - for repository in repositories - ], - "pageInfo": {"hasNextPage": has_next_page, "endCursor": end_cursor}, + ) -> src.JSONDict: + return cast( + src.JSONDict, + { + "permissionsInfo": { + "repositories": { + "nodes": [{"id": repository["id"]} for repository in repositories], + "pageInfo": {"hasNextPage": has_next_page, "endCursor": end_cursor}, + } } - } - } + }, + ) diff --git a/uv.lock b/uv.lock index fbf1781..9550354 100644 --- a/uv.lock +++ b/uv.lock @@ -318,7 +318,7 @@ wheels = [ [[package]] name = "src-auth-perms-sync" -version = "0.2.0" +version = "0.2.1" source = { editable = "." } dependencies = [ { name = "json5" }, @@ -337,7 +337,7 @@ dev = [ requires-dist = [ { name = "json5", specifier = ">=0.14.0" }, { name = "pyyaml", specifier = ">=6.0.3" }, - { name = "src-py-lib", directory = "git-subtree/src-py-lib" }, + { name = "src-py-lib", specifier = "==0.1.1" }, ] [package.metadata.requires-dev] @@ -349,25 +349,16 @@ dev = [ [[package]] name = "src-py-lib" -version = "0.1.0" -source = { directory = "git-subtree/src-py-lib" } +version = "0.1.1" +source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "httpx" }, { name = "pydantic" }, { name = "python-dotenv" }, ] - -[package.metadata] -requires-dist = [ - { name = "httpx", specifier = ">=0.28,<1" }, - { name = "pydantic", specifier = ">=2,<3" }, - { name = "python-dotenv", specifier = ">=1.2,<2" }, -] - -[package.metadata.requires-dev] -dev = [ - { name = "pyright", specifier = ">=1.1.409" }, - { name = "ruff", specifier = ">=0.7.0" }, +sdist = { url = "https://files.pythonhosted.org/packages/af/66/5f31e15424d5f4e1f0074e4e5ce30a3ea879693a71dfd9e016552b9d64fd/src_py_lib-0.1.1.tar.gz", hash = "sha256:b73dece6b9e1d24992df7a5272c2ad2a3f9b491bcf002891d3b2632ce4d39aa7", size = 62598, upload-time = "2026-05-29T00:37:40.513Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8f/64/a627e17fcf0c9e307d109453f3890d306ffc36c03a5ca21bc0759aa790ef/src_py_lib-0.1.1-py3-none-any.whl", hash = "sha256:d40d0cac7f3fadc7793aae96a98f6639d6a6f5a16fdd7456d99a50f30ea46301", size = 39082, upload-time = "2026-05-29T00:37:39.436Z" }, ] [[package]]