From 767bf69687d5ae0a78530a5a3e603b03ecbcdb27 Mon Sep 17 00:00:00 2001 From: Sebastian Mendel Date: Mon, 4 May 2026 15:46:36 +0200 Subject: [PATCH 1/4] fix(checkpoints): rewrite GH-06/23/30/31 to satisfy assessment runner allowlist MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The automated-assessment runner enforces a command allowlist that rejects command-chaining metacharacters (; && || backticks $()) and only accepts specific base commands. Four checkpoints failed allowlist validation and were never actually evaluated against target projects. GH-06 and GH-23 used `test -f X || test -f Y` chains to test for any of several files. Rewritten as `type: file_exists` with brace expansion, which is the runner's first-class idiom for "any of these files". GH-30 and GH-31 used multi-line YAML literal-block scalars (`target: |`) to invoke `gh api` for branch protection audits. The runner's simple line parser sees the literal-block indicator as the first token and rejects with "'|' not in allowed command whitelist". Even with the allowlist passing, these cannot be executed mechanically — they require GitHub API auth context. Converted to `type: gh_api`, which the runner recognises and skips with a clear evidence string. The semantically equivalent audit is preserved in GH-32 (llm_reviews). --- skills/github-project/checkpoints.yaml | 61 +++++++------------------- 1 file changed, 16 insertions(+), 45 deletions(-) diff --git a/skills/github-project/checkpoints.yaml b/skills/github-project/checkpoints.yaml index 7baf069..ff19798 100644 --- a/skills/github-project/checkpoints.yaml +++ b/skills/github-project/checkpoints.yaml @@ -37,8 +37,8 @@ mechanical: desc: "CODEOWNERS should exist for review assignments" - id: GH-06 - type: command - target: "test -f .github/dependabot.yml || test -f renovate.json || test -f .github/renovate.json || test -f .renovaterc.json || test -f .renovaterc" + type: file_exists + target: "{.github/dependabot.yml,renovate.json,.github/renovate.json,.renovaterc.json,.renovaterc}" severity: warning desc: "Dependency update automation should be configured (Dependabot or Renovate)" @@ -144,8 +144,8 @@ mechanical: # === AUTO-MERGE WORKFLOW CHECKS === - id: GH-23 - type: command - target: "test -f .github/workflows/auto-merge-deps.yml || test -f .github/workflows/auto-merge.yml" + type: file_exists + target: "{.github/workflows/auto-merge-deps.yml,.github/workflows/auto-merge.yml}" severity: warning desc: "Auto-merge workflow should exist for Dependabot/Renovate PRs" @@ -212,52 +212,23 @@ mechanical: reviewDecision) # === BRANCH PROTECTION AUDIT === + # GH-30 / GH-31 require GitHub API access (gh CLI + auth) which is outside + # the assessment runner's command allowlist. Declared as type: gh_api so the + # runner skips them with a clear evidence string instead of rejecting them. + # The semantically equivalent LLM-driven audit lives in GH-32 (llm_reviews). - id: GH-30 - type: command - target: | - # Require GitHub CLI and a valid GitHub remote slug - command -v gh >/dev/null 2>&1 || exit 1 - REPO_SLUG=$(git config --get remote.origin.url 2>/dev/null | sed -E 's|.*github\.com[:/](.+/[^.]+)(\.git)?$|\1|') - test -n "$REPO_SLUG" || exit 1 - DEFAULT_BRANCH=$(gh api "repos/$REPO_SLUG" --jq '.default_branch' 2>/dev/null || echo 'main') - RESULT=$(gh api "repos/$REPO_SLUG/branches/$DEFAULT_BRANCH/protection" --jq '.enforce_admins.enabled' 2>/dev/null || echo '') - if [ "$RESULT" = "true" ]; then - exit 0 - fi - # Fallback: check rulesets when classic branch protection is not configured - RULESET_OK=$(gh api "repos/$REPO_SLUG/rulesets" \ - --jq 'map(select(.enforcement == "active" and .target == "branch" and ((.bypass_actors // []) | length == 0))) | any' \ - 2>/dev/null || echo 'false') - test "$RULESET_OK" = "true" + type: gh_api + endpoint: "repos/{owner}/{repo}/branches/{default_branch}/protection" + json_path: ".enforce_admins.enabled" severity: error - desc: "enforce_admins must be true on default branch — without it admins can bypass required status checks and review requirements" + desc: "enforce_admins must be true on default branch — without it admins can bypass required status checks and review requirements (audited by GH-32 llm_review; runner skips because gh API access requires auth)" - id: GH-31 - type: command - target: | - # Require GitHub CLI and a valid GitHub remote slug - command -v gh >/dev/null 2>&1 || exit 1 - REPO_SLUG=$(git config --get remote.origin.url 2>/dev/null | sed -E 's|.*github\.com[:/](.+/[^.]+)(\.git)?$|\1|') - test -n "$REPO_SLUG" || exit 1 - DEFAULT_BRANCH=$(gh api "repos/$REPO_SLUG" --jq '.default_branch' 2>/dev/null || echo 'main') - RESULT=$(gh api "repos/$REPO_SLUG/branches/$DEFAULT_BRANCH/protection" --jq '.required_conversation_resolution.enabled' 2>/dev/null || echo '') - if [ "$RESULT" = "true" ]; then - exit 0 - fi - # Fallback: check rulesets for review thread resolution - JQ_FILTER='map(select( - .enforcement == "active" and .target == "branch" - and ((.bypass_actors // []) | length == 0) - and any(.rules[]?; - .type == "pull_request" and - (.parameters.required_review_thread_resolution // false) - ) - )) | any' - RULESET_OK=$(gh api "repos/$REPO_SLUG/rulesets" \ - --jq "$JQ_FILTER" 2>/dev/null || echo 'false') - test "$RULESET_OK" = "true" + type: gh_api + endpoint: "repos/{owner}/{repo}/branches/{default_branch}/protection" + json_path: ".required_conversation_resolution.enabled" severity: error - desc: "required_conversation_resolution must be enabled — combined with enforce_admins, ensures unresolved review threads block ALL merges including admins" + desc: "required_conversation_resolution must be enabled — combined with enforce_admins, ensures unresolved review threads block ALL merges including admins (audited by GH-32 llm_review; runner skips because gh API access requires auth)" llm_reviews: # === BRANCH PROTECTION + MERGE QUEUE COMPATIBILITY === From 9de1aedef9886c1ceae38b1d19d2abb3abc8995e Mon Sep 17 00:00:00 2001 From: Sebastian Mendel Date: Mon, 4 May 2026 15:46:54 +0200 Subject: [PATCH 2/4] fix(checkpoints): broaden GH-02 LICENSE check to recognise split-licence repos Skill repos commonly use split licences with the SPDX-style multi-file naming convention (LICENSE-MIT for code + LICENSE-CC-BY-SA-4.0 for prose, LICENSE-APACHE + LICENSE-MIT dual licensing, etc.). The previous check only looked for a literal `LICENSE` filename and failed against any repo following this convention even though they are properly licensed. Replace the literal target with a brace expansion covering the common single-file names (LICENSE, LICENSE.md, LICENSE.txt) plus the SPDX-style LICENSE-* variants. file_exists with brace expansion passes if any one of the alternatives is present. --- skills/github-project/checkpoints.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/skills/github-project/checkpoints.yaml b/skills/github-project/checkpoints.yaml index ff19798..803218b 100644 --- a/skills/github-project/checkpoints.yaml +++ b/skills/github-project/checkpoints.yaml @@ -14,9 +14,9 @@ mechanical: - id: GH-02 type: file_exists - target: LICENSE + target: "{LICENSE,LICENSE.md,LICENSE.txt,LICENSE-MIT,LICENSE-APACHE,LICENSE-APACHE-2.0,LICENSE-BSD,LICENSE-CC-BY-SA-4.0,LICENSE-CC0-1.0,LICENSE-GPL,LICENSE-AGPL,LICENSE-MPL}" severity: error - desc: "LICENSE file must exist" + desc: "LICENSE file must exist (split licenses with multi-file naming like LICENSE-MIT/LICENSE-CC-BY-SA-4.0 are accepted)" - id: GH-03 type: file_exists From 86421c8470caa185402b2777ae22c741ea2b9924 Mon Sep 17 00:00:00 2001 From: Sebastian Mendel Date: Mon, 4 May 2026 15:47:18 +0200 Subject: [PATCH 3/4] fix(checkpoints): demote GH-03/04/05 to info, add brace expansion for in-repo locations MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit GH-03 (SECURITY.md), GH-04 (CONTRIBUTING.md), and GH-05 (CODEOWNERS) check for project-local files. GitHub honours these files org-wide via the special `.github` repo (e.g. netresearch/.github), so many repos that appear "missing" these files actually have them satisfied at the org level. The assessment runner cannot cheaply verify org-wide files (it would have to call the GitHub API, which requires auth and is outside the command allowlist). Demoting these from `warning` to `info` better reflects the real situation: nice-to-have at the repo level, often satisfied elsewhere. The descriptions now mention the org-wide fallback explicitly. Also broaden each target with brace expansion for the documented in-repo locations: - SECURITY.md → {SECURITY.md, .github/SECURITY.md, docs/SECURITY.md} - CONTRIBUTING.md → {CONTRIBUTING.md, .github/CONTRIBUTING.md, docs/CONTRIBUTING.md} - CODEOWNERS → {.github/CODEOWNERS, CODEOWNERS, docs/CODEOWNERS} GitHub itself accepts CODEOWNERS in any of those locations. --- skills/github-project/checkpoints.yaml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/skills/github-project/checkpoints.yaml b/skills/github-project/checkpoints.yaml index 803218b..7e610c3 100644 --- a/skills/github-project/checkpoints.yaml +++ b/skills/github-project/checkpoints.yaml @@ -20,21 +20,21 @@ mechanical: - id: GH-03 type: file_exists - target: SECURITY.md - severity: warning - desc: "SECURITY.md should exist for vulnerability reporting" + target: "{SECURITY.md,.github/SECURITY.md,docs/SECURITY.md}" + severity: info + desc: "SECURITY.md should exist for vulnerability reporting (satisfied org-wide if the GitHub owner has a `.github` repo providing these files)" - id: GH-04 type: file_exists - target: CONTRIBUTING.md - severity: warning - desc: "CONTRIBUTING.md should exist" + target: "{CONTRIBUTING.md,.github/CONTRIBUTING.md,docs/CONTRIBUTING.md}" + severity: info + desc: "CONTRIBUTING.md should exist (satisfied org-wide if the GitHub owner has a `.github` repo providing these files)" - id: GH-05 type: file_exists - target: .github/CODEOWNERS - severity: warning - desc: "CODEOWNERS should exist for review assignments" + target: "{.github/CODEOWNERS,CODEOWNERS,docs/CODEOWNERS}" + severity: info + desc: "CODEOWNERS should exist for review assignments (satisfied org-wide if the GitHub owner has a `.github` repo providing these files)" - id: GH-06 type: file_exists From 932b1a919f34d0149199f380708ba2487ef4f527 Mon Sep 17 00:00:00 2001 From: Sebastian Mendel Date: Mon, 4 May 2026 19:03:22 +0200 Subject: [PATCH 4/4] fix(checkpoints): broaden license/dependabot/workflow targets and revert GH-05 demotion MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Addresses PR #69 review feedback: GH-02 (Copilot, Gemini): added missing license filenames the skill itself documents — COPYING, COPYING.md, COPYING.txt, LICENSE-BSD-2-Clause, LICENSE-BSD-3-Clause, LICENSE-GPL-2.0, LICENSE-GPL-3.0, LICENSE-LGPL, LICENSE-LGPL-3.0, LICENSE-AGPL-3.0, LICENSE-MPL-2.0. GH-05 (Copilot): reverted demotion to info — CODEOWNERS must exist in the repository itself on the default branch (.github/, root, or docs/), and the org-wide .github mechanism explicitly does NOT cover it (that mechanism only provides templates and community-health files, never review-routing rules). Severity restored to warning; description corrected to remove the misleading org-wide claim. GH-06 (Gemini): added .github/dependabot.yaml, renovate.json5, renovate config variants, and the .json5 form for Renovate. GH-13/14 (Gemini follow-on): brace-expanded targets to also accept .github/dependabot.yaml. GH-19/20/21 (Gemini): glob target now `.github/workflows/*.{yml,yaml}` to match either extension. GH-24..27 (Gemini): glob target now `.github/workflows/auto-merge*.{yml,yaml}`. Push-back on Copilot 'auto-merge.yml weakens GH-23' comment: GH-24..27 already use a glob (auto-merge*.yml — now expanded to *.{yml,yaml}) that matches both filenames, so adding auto-merge.yml to GH-23 does not produce false failures downstream. GH-30/31 desc: wrapped long lines as YAML folded scalars (no impact on runner — these are gh_api types, the desc field is human-readable). .yamllint.yml: line-length bumped 160 → 360 to accommodate single-line brace-expansion targets that the runner cannot read as folded scalars (the runner's parser is bash regex, not a YAML library). --- .yamllint.yml | 7 +++- skills/github-project/checkpoints.yaml | 55 ++++++++++++++++++-------- 2 files changed, 44 insertions(+), 18 deletions(-) diff --git a/.yamllint.yml b/.yamllint.yml index 51de3c0..d352ac5 100644 --- a/.yamllint.yml +++ b/.yamllint.yml @@ -1,5 +1,10 @@ extends: default rules: - line-length: {max: 160} + # Bumped to 280 to accommodate single-line brace-expansion targets in + # checkpoints.yaml (e.g. GH-02 enumerates ~24 LICENSE filenames). The + # checkpoint runner does not unfold YAML folded scalars (`>-`) for + # `target:` fields — its parser is bash regex, not a real YAML parser — + # so these long enumerations cannot be wrapped. + line-length: {max: 360} truthy: {allowed-values: ['true', 'false', 'on']} document-start: disable diff --git a/skills/github-project/checkpoints.yaml b/skills/github-project/checkpoints.yaml index 7e610c3..458ce4b 100644 --- a/skills/github-project/checkpoints.yaml +++ b/skills/github-project/checkpoints.yaml @@ -12,11 +12,17 @@ mechanical: severity: error desc: "README.md must exist" + # Brace expansion covers SPDX-style split-license filenames common in + # multi-licence skill repos plus the GNU COPYING convention. Single line + # because the runner's YAML parser doesn't unfold `>-` folded scalars + # for `target:` fields (it's bash regex, not a real YAML parser). - id: GH-02 type: file_exists - target: "{LICENSE,LICENSE.md,LICENSE.txt,LICENSE-MIT,LICENSE-APACHE,LICENSE-APACHE-2.0,LICENSE-BSD,LICENSE-CC-BY-SA-4.0,LICENSE-CC0-1.0,LICENSE-GPL,LICENSE-AGPL,LICENSE-MPL}" + target: "{LICENSE,LICENSE.md,LICENSE.txt,COPYING,COPYING.md,COPYING.txt,LICENSE-MIT,LICENSE-APACHE,LICENSE-APACHE-2.0,LICENSE-BSD,LICENSE-BSD-2-Clause,LICENSE-BSD-3-Clause,LICENSE-CC-BY-SA-4.0,LICENSE-CC0-1.0,LICENSE-GPL,LICENSE-GPL-2.0,LICENSE-GPL-3.0,LICENSE-LGPL,LICENSE-LGPL-3.0,LICENSE-AGPL,LICENSE-AGPL-3.0,LICENSE-MPL,LICENSE-MPL-2.0}" severity: error - desc: "LICENSE file must exist (split licenses with multi-file naming like LICENSE-MIT/LICENSE-CC-BY-SA-4.0 are accepted)" + desc: >- + LICENSE file must exist. Accepts split-license naming (LICENSE-MIT, + LICENSE-CC-BY-SA-4.0, etc.) and the GNU COPYING convention. - id: GH-03 type: file_exists @@ -33,14 +39,22 @@ mechanical: - id: GH-05 type: file_exists target: "{.github/CODEOWNERS,CODEOWNERS,docs/CODEOWNERS}" - severity: info - desc: "CODEOWNERS should exist for review assignments (satisfied org-wide if the GitHub owner has a `.github` repo providing these files)" + severity: warning + desc: >- + CODEOWNERS must exist in the repository itself for GitHub to assign + reviewers (only `.github/`, root, or `docs/` on the default branch are + recognised). Org-wide `.github` does NOT satisfy CODEOWNERS — that + mechanism only provides templates and community-health files, not + review-routing rules. - id: GH-06 type: file_exists - target: "{.github/dependabot.yml,renovate.json,.github/renovate.json,.renovaterc.json,.renovaterc}" + target: "{.github/dependabot.yml,.github/dependabot.yaml,renovate.json,renovate.json5,.github/renovate.json,.github/renovate.json5,.renovaterc.json,.renovaterc.json5,.renovaterc}" severity: warning - desc: "Dependency update automation should be configured (Dependabot or Renovate)" + desc: >- + Dependency update automation should be configured (Dependabot or + Renovate). Accepts both .yml/.yaml for Dependabot and .json/.json5 + for Renovate. - id: GH-07 type: file_exists @@ -88,14 +102,14 @@ mechanical: # === DEPENDABOT CHECKS === - id: GH-13 type: regex - target: .github/dependabot.yml + target: "{.github/dependabot.yml,.github/dependabot.yaml}" pattern: 'package-ecosystem:\s*"?(composer|gomod|npm|pip)"?' severity: warning desc: "Dependabot should monitor language-specific dependencies (composer, gomod, npm, pip)" - id: GH-14 type: regex - target: .github/dependabot.yml + target: "{.github/dependabot.yml,.github/dependabot.yaml}" pattern: 'package-ecosystem:\s*"?github-actions"?' severity: warning desc: "Dependabot should monitor GitHub Actions" @@ -120,14 +134,14 @@ mechanical: # pattern as ER-19/20 in enterprise-readiness-skill PR #55. - id: GH-19 type: regex - target: .github/workflows/*.yml + target: ".github/workflows/*.{yml,yaml}" pattern: 'uses:[[:space:]]*github/codeql-action|uses:[[:space:]]*netresearch/[.]github/[.]github/workflows/codeql[.]yml' severity: warning desc: "CodeQL must be wired up (dedicated workflow or reusable-workflow job calling github/codeql-action or netresearch/.github/...codeql.yml)" - id: GH-20 type: regex - target: .github/workflows/*.yml + target: ".github/workflows/*.{yml,yaml}" pattern: 'uses:[[:space:]]*ossf/scorecard-action|uses:[[:space:]]*netresearch/[.]github/[.]github/workflows/scorecard[.]yml' severity: info desc: >- @@ -137,7 +151,7 @@ mechanical: # === SLSA PROVENANCE === - id: GH-21 type: regex_not - target: .github/workflows/*.yml + target: ".github/workflows/*.{yml,yaml}" pattern: 'slsa-framework/slsa-github-generator' severity: warning desc: "Should migrate from slsa-github-generator to actions/attest-build-provenance (cannot be SHA-pinned)" @@ -167,28 +181,28 @@ mechanical: # contain the netresearch/... substring identically). - id: GH-24 type: regex - target: .github/workflows/auto-merge*.yml + target: ".github/workflows/auto-merge*.{yml,yaml}" pattern: 'netresearch/\.github/\.github/workflows/auto-merge-deps\.yml|on:[[:space:]]*\n[[:space:]]*pull_request_target:' severity: error desc: "Auto-merge workflow must delegate to netresearch reusable workflow OR use pull_request_target trigger for bot PR write permissions" - id: GH-25 type: regex - target: .github/workflows/auto-merge*.yml + target: ".github/workflows/auto-merge*.{yml,yaml}" pattern: 'netresearch/\.github/\.github/workflows/auto-merge-deps\.yml|github\.event\.pull_request\.user\.login' severity: warning desc: "Auto-merge should delegate to reusable workflow OR check github.event.pull_request.user.login (not github.actor which changes on reruns)" - id: GH-26 type: regex - target: .github/workflows/auto-merge*.yml + target: ".github/workflows/auto-merge*.{yml,yaml}" pattern: 'netresearch/\.github/\.github/workflows/auto-merge-deps\.yml|--auto' severity: warning desc: "Auto-merge should delegate to reusable workflow OR use gh pr merge --auto to respect branch protection and merge queues" - id: GH-27 type: regex - target: .github/workflows/auto-merge*.yml + target: ".github/workflows/auto-merge*.{yml,yaml}" pattern: 'netresearch/\.github/\.github/workflows/auto-merge-deps\.yml|gh api.*repos/\$' severity: info desc: "Auto-merge should delegate to reusable workflow OR dynamically detect merge strategy from repo settings" @@ -221,14 +235,21 @@ mechanical: endpoint: "repos/{owner}/{repo}/branches/{default_branch}/protection" json_path: ".enforce_admins.enabled" severity: error - desc: "enforce_admins must be true on default branch — without it admins can bypass required status checks and review requirements (audited by GH-32 llm_review; runner skips because gh API access requires auth)" + desc: >- + enforce_admins must be true on default branch — without it admins can + bypass required status checks and review requirements (audited by + GH-32 llm_review; runner skips because gh API access requires auth). - id: GH-31 type: gh_api endpoint: "repos/{owner}/{repo}/branches/{default_branch}/protection" json_path: ".required_conversation_resolution.enabled" severity: error - desc: "required_conversation_resolution must be enabled — combined with enforce_admins, ensures unresolved review threads block ALL merges including admins (audited by GH-32 llm_review; runner skips because gh API access requires auth)" + desc: >- + required_conversation_resolution must be enabled — combined with + enforce_admins, ensures unresolved review threads block ALL merges + including admins (audited by GH-32 llm_review; runner skips because + gh API access requires auth). llm_reviews: # === BRANCH PROTECTION + MERGE QUEUE COMPATIBILITY ===