From a4eb0f9155ce5fcac46dc5388370df7e18deb9a8 Mon Sep 17 00:00:00 2001 From: Tom Softreck Date: Sat, 1 Aug 2026 20:34:46 +0200 Subject: [PATCH 1/7] ci: adopt benchmark-qualified Koru judge --- .github/workflows/koru-code-review.yml | 4 +-- project/ticket-018/README.md | 16 ++++++++++++ project/ticket-018/ai-codex-logs.txt | 35 ++++++++++++++++++++++++++ project/ticket-018/ai-codex.md | 6 +++++ project/ticket-018/changelog.md | 11 ++++++++ 5 files changed, 70 insertions(+), 2 deletions(-) diff --git a/.github/workflows/koru-code-review.yml b/.github/workflows/koru-code-review.yml index cfe9b28..bd53308 100644 --- a/.github/workflows/koru-code-review.yml +++ b/.github/workflows/koru-code-review.yml @@ -32,7 +32,7 @@ jobs: env: KORU_VERSION: '0.1.444' VALLM_VERSION: '0.1.94' - REVIEW_MODEL: openrouter/deepseek/deepseek-v4-pro + REVIEW_MODEL: openrouter/google/gemini-3.1-pro-preview OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} BASE_SHA: ${{ inputs.base_sha || github.event.pull_request.base.sha }} HEAD_SHA: ${{ inputs.head_sha || github.event.pull_request.head.sha }} @@ -119,7 +119,7 @@ jobs: export VALLM_LLM_MODEL="$REVIEW_MODEL" export VALLM_LLM_BASE_URL=https://openrouter.ai/api/v1 vallm batch "${files[@]}" \ - --semantic --security --regression \ + --semantic --security \ --model "$REVIEW_MODEL" \ --format json --output .koru-review/vallm --show-issues BASH diff --git a/project/ticket-018/README.md b/project/ticket-018/README.md index 526a549..dc9884a 100644 --- a/project/ticket-018/README.md +++ b/project/ticket-018/README.md @@ -76,6 +76,14 @@ bounded Vallm review round. The review combines deterministic syntax, complexity and security checks with an OpenRouter semantic judge supplied by the existing organization-level `OPENROUTER_API_KEY` secret. +The semantic judge is `google/gemini-3.1-pro-preview`, selected from the current +live `llm-code-benchmark/v1` report because it is the only compared model that +qualified for both repair and validation (repair 1.000, validation 0.929, +security 1.000 and availability 100%). Vallm's Python-oriented `--regression` +mode is intentionally not used for TypeScript: the separate required `verify` +job owns the repository's 335-test regression suite, while Koru remains the +read-only semantic, syntax, complexity and security review boundary. + The workflow will never use `pull_request_target`, check out untrusted code with a write-capable token, modify source, auto-fix, commit, push or submit a GitHub `APPROVE` review. A missing secret or semantic-provider failure is an @@ -234,6 +242,14 @@ remain historical evidence, not evidence for AC-11..AC-17. artifact upload and attestation still succeeded. The attested report digest is `sha256:fa0f4d0c1f780bb8d21f56ca74d8ae901e184fb4996f9e84832a87846adfc1d8`. No credential value appears in the workflow output. +- Pull request #3 exposed that the original Koru configuration had drifted from + the current benchmark winner. Run `30712589077` still used + `openrouter/deepseek/deepseek-v4-pro`; Vallm also attempted `pytest` for the + TypeScript diff and the semantic request failed with OpenRouter 401 `User not + found`. The workflow now uses the qualified Gemini model and delegates + regression to the already passing required `verify` job. The 401 cannot be + repaired in repository code: a trusted repository or organization owner must + rotate the `OPENROUTER_API_KEY` Actions secret and rerun the exact commit. - Repository ruleset `20186914` is staged with no bypass actors and `current_user_can_bypass: never`. It targets the default branch, requires a pull request, dismisses stale review evidence, rejects deletion/force-push, diff --git a/project/ticket-018/ai-codex-logs.txt b/project/ticket-018/ai-codex-logs.txt index 7991c18..802d028 100644 --- a/project/ticket-018/ai-codex-logs.txt +++ b/project/ticket-018/ai-codex-logs.txt @@ -233,3 +233,38 @@ bypass actors: none current_user_can_bypass: never rules: pull request, dismiss stale reviews, block deletion/force-push, strict required checks governance / enforce and koru / code-review + +2026-08-01 KORU BENCHMARK MODEL ADOPTION +$ inspect llm-code-benchmark/reports/latest/REPORT.md +standard: llm-code-benchmark/v1 +winner: google/gemini-3.1-pro-preview +repair=1.000 validator=0.929 security=1.000 availability=100.0% +selection: only compared model qualified for both repair and validation + +$ inspect pull request #3 Koru run 30712589077 artifact +configured model: openrouter/deepseek/deepseek-v4-pro (stale) +selected files: src/config/env.ts, test/config-env.test.ts +Vallm regression error: pytest is not installed for the TypeScript repository +semantic provider error: OpenRouter 401 User not found +report, artifact upload and provenance attestation: PASS +enforced verdict: FAIL + +Decision: +- use openrouter/google/gemini-3.1-pro-preview for the Koru semantic judge; +- omit Vallm --regression because the required Node verify job owns regression; +- retain fail-closed behavior for provider errors; +- do not read, copy or overwrite the GitHub Actions secret. A trusted owner must + rotate OPENROUTER_API_KEY and rerun the exact commit. + +$ npm run verify:workflows +Workflow YAML verified: 2 file(s), no duplicate top-level keys. + +$ npm run verify +tests 335; pass 334; fail 0; skipped 1 (local JDK unavailable) +module boundaries: 114 modules, 521 internal imports, no cycles +workflow, schema, no-LLM and generated-analysis gates: PASS + +$ make governance +Four inherited ticket-019 findings remain: GOV-CONFLICT-001, +GOV-DEPENDENCY-002, GOV-WORKSTREAM-003 and GOV-WORKSTREAM-004. +No ticket-018 scope, ownership or secret finding was emitted. diff --git a/project/ticket-018/ai-codex.md b/project/ticket-018/ai-codex.md index 1cc7be3..b77d117 100644 --- a/project/ticket-018/ai-codex.md +++ b/project/ticket-018/ai-codex.md @@ -150,6 +150,12 @@ Current verified baseline: and Koru status checks, mandatory pull requests, stale-evidence dismissal and force-push/deletion prevention. It remains disabled solely for the final bootstrap evidence merge and will be activated afterward. +- Revalidated Koru after the model-adoption benchmark. Pull request #3 showed + that the workflow still used stale DeepSeek, invoked Python `pytest` for a + TypeScript diff and received OpenRouter 401. Updated the semantic judge to + benchmark-qualified Gemini 3.1 Pro Preview, left regression ownership with + the required Node `verify` job, and did not read or overwrite the external + Actions secret. ## Blockers diff --git a/project/ticket-018/changelog.md b/project/ticket-018/changelog.md index cf4007f..060cbeb 100644 --- a/project/ticket-018/changelog.md +++ b/project/ticket-018/changelog.md @@ -1,5 +1,16 @@ # Ticket Changelog (ticket-018) +## [0.2.1] - 2026-08-01 + +- Replaced the stale DeepSeek Koru judge with benchmark-qualified + `google/gemini-3.1-pro-preview`. +- Removed Vallm's Python-specific regression mode from the TypeScript review; + the independent required `verify` job remains the authoritative regression + gate. +- Recorded pull request #3 run `30712589077`: Koru evidence generation and + attestation worked, while the live semantic call failed with OpenRouter 401 + `User not found`. Secret rotation remains an external trusted-owner action. + ## [0.2.0] - 2026-08-01 - Evolved the plan for concurrent humans/agents: named workstreams, From 920fcb913725529fc4dee712b78234f130be2235 Mon Sep 17 00:00:00 2001 From: Tom Softreck Date: Sat, 1 Aug 2026 20:36:57 +0200 Subject: [PATCH 2/7] docs(governance): record Koru model evidence --- project/ticket-018/README.md | 6 ++++++ project/ticket-018/ai-codex-logs.txt | 17 +++++++++++++++++ project/ticket-018/changelog.md | 2 ++ 3 files changed, 25 insertions(+) diff --git a/project/ticket-018/README.md b/project/ticket-018/README.md index dc9884a..0bf8428 100644 --- a/project/ticket-018/README.md +++ b/project/ticket-018/README.md @@ -250,6 +250,12 @@ remain historical evidence, not evidence for AC-11..AC-17. regression to the already passing required `verify` job. The 401 cannot be repaired in repository code: a trusted repository or organization owner must rotate the `OPENROUTER_API_KEY` Actions secret and rerun the exact commit. +- Pull request #4 run `30712853708` passed the read-only Koru gate for commit + `a4eb0f9`. Its attested `t2c.koru-code-review/v1` report records + `openrouter/google/gemini-3.1-pro-preview` and an empty supported-source set, + so no provider request or cost occurred. This proves the deployed workflow + configuration and no-source path; it does not supersede the required live + rerun after secret rotation. - Repository ruleset `20186914` is staged with no bypass actors and `current_user_can_bypass: never`. It targets the default branch, requires a pull request, dismisses stale review evidence, rejects deletion/force-push, diff --git a/project/ticket-018/ai-codex-logs.txt b/project/ticket-018/ai-codex-logs.txt index 802d028..bb59ae0 100644 --- a/project/ticket-018/ai-codex-logs.txt +++ b/project/ticket-018/ai-codex-logs.txt @@ -268,3 +268,20 @@ $ make governance Four inherited ticket-019 findings remain: GOV-CONFLICT-001, GOV-DEPENDENCY-002, GOV-WORKSTREAM-003 and GOV-WORKSTREAM-004. No ticket-018 scope, ownership or secret finding was emitted. + +2026-08-01 KORU REMOTE MODEL CONFIGURATION EVIDENCE +$ GitHub pull request #4 / workflow run 30712853708 +reviewed base: 2e87205dcd0b06286529ca9fa8ab9518694e9068 +reviewed head: a4eb0f9155ce5fcac46dc5388370df7e18deb9a8 +report schema: t2c.koru-code-review/v1 +reported model: openrouter/google/gemini-3.1-pro-preview +selected supported source files: 0 +provider request/cost: none; credential step skipped by design +Koru report/artifact/attestation/enforced verdict: PASS +Node verify including Docker smoke: PASS +Java adapter with required JDK 17 fixture: PASS +governance: FAIL on inherited ticket-019 state only + +Interpretation: this is commit-bound evidence for the deployed model and the +no-source path. A live TypeScript semantic rerun is still required after a +trusted owner rotates the failing OPENROUTER_API_KEY Actions secret. diff --git a/project/ticket-018/changelog.md b/project/ticket-018/changelog.md index 060cbeb..2f5ce04 100644 --- a/project/ticket-018/changelog.md +++ b/project/ticket-018/changelog.md @@ -10,6 +10,8 @@ - Recorded pull request #3 run `30712589077`: Koru evidence generation and attestation worked, while the live semantic call failed with OpenRouter 401 `User not found`. Secret rotation remains an external trusted-owner action. +- Verified the new judge in pull request #4's attested no-source report; Koru, + Node/Docker verification and the required Java fixture passed remotely. ## [0.2.0] - 2026-08-01 From 4de71f9f2523a6e1f8851df069aa4acd51730628 Mon Sep 17 00:00:00 2001 From: Tom Softreck Date: Sat, 1 Aug 2026 20:42:19 +0200 Subject: [PATCH 3/7] docs(governance): record live Gemini Koru blocker --- project/ticket-018/README.md | 25 +++++++++++++++++++------ project/ticket-018/ai-codex-logs.txt | 23 +++++++++++++++++++++++ project/ticket-018/ai-codex.md | 5 +++++ project/ticket-018/changelog.md | 5 +++++ 4 files changed, 52 insertions(+), 6 deletions(-) diff --git a/project/ticket-018/README.md b/project/ticket-018/README.md index 0bf8428..0f35064 100644 --- a/project/ticket-018/README.md +++ b/project/ticket-018/README.md @@ -72,17 +72,22 @@ The user requested automated code review through Koru. The implementation will add a read-only GitHub check named `koru / code-review`, run for pull requests and explicit historical-review dispatches. It will pin Koru 0.1.444 and Vallm 0.1.94, select only changed supported source files, and let Koru execute one -bounded Vallm review round. The review combines deterministic syntax, -complexity and security checks with an OpenRouter semantic judge supplied by -the existing organization-level `OPENROUTER_API_KEY` secret. +bounded Vallm review round. The review runs deterministic complexity and +security checks, attempts Vallm syntax analysis, and uses an OpenRouter +semantic judge supplied by the existing organization-level +`OPENROUTER_API_KEY` secret. The semantic judge is `google/gemini-3.1-pro-preview`, selected from the current live `llm-code-benchmark/v1` report because it is the only compared model that qualified for both repair and validation (repair 1.000, validation 0.929, security 1.000 and availability 100%). Vallm's Python-oriented `--regression` mode is intentionally not used for TypeScript: the separate required `verify` -job owns the repository's 335-test regression suite, while Koru remains the -read-only semantic, syntax, complexity and security review boundary. +job owns TypeScript compilation and the repository's 335-test regression +suite. Koru remains the read-only semantic, complexity and security review +boundary. Vallm still attempts syntax analysis, but 0.1.94 passes the uppercase +language enum `TYPESCRIPT` to a parser that accepts lowercase `typescript`; its +`syntax.unsupported` warning is retained as an open upstream defect rather than +being presented as successful syntax evidence. The workflow will never use `pull_request_target`, check out untrusted code with a write-capable token, modify source, auto-fix, commit, push or submit a @@ -156,7 +161,7 @@ agent self-approved. - [x] AC-20: Koru 0.1.444 runs exactly one read-only Vallm 0.1.94 review round over changed supported source files; auto-fix, commit, push and mutable dependency versions are absent. -- [x] AC-21: Deterministic syntax/complexity/security checks and semantic +- [ ] AC-21: Deterministic syntax/complexity/security checks and semantic LLM-as-judge review fail closed on findings, missing credentials, malformed output or provider failure, with no secret value in logs. - [x] AC-22: The structured report records repository, base/head SHA, selected @@ -256,6 +261,14 @@ remain historical evidence, not evidence for AC-11..AC-17. so no provider request or cost occurred. This proves the deployed workflow configuration and no-source path; it does not supersede the required live rerun after secret rotation. +- Workflow dispatch `30713017811` then exercised that workflow against the + exact two-file TypeScript diff from pull request #3. The report records the + Gemini judge and no longer contains a regression/`pytest` error. It rejects + fail-closed because OpenRouter still returns 401 `User not found`; it also + retains Vallm 0.1.94's `TYPESCRIPT` parser warning. Report construction, + artifact upload and provenance attestation passed. AC-21 therefore remains + open until the secret is rotated and the upstream parser defect is fixed or + replaced with equivalent deterministic Koru-job evidence. - Repository ruleset `20186914` is staged with no bypass actors and `current_user_can_bypass: never`. It targets the default branch, requires a pull request, dismisses stale review evidence, rejects deletion/force-push, diff --git a/project/ticket-018/ai-codex-logs.txt b/project/ticket-018/ai-codex-logs.txt index bb59ae0..f8e52c6 100644 --- a/project/ticket-018/ai-codex-logs.txt +++ b/project/ticket-018/ai-codex-logs.txt @@ -285,3 +285,26 @@ governance: FAIL on inherited ticket-019 state only Interpretation: this is commit-bound evidence for the deployed model and the no-source path. A live TypeScript semantic rerun is still required after a trusted owner rotates the failing OPENROUTER_API_KEY Actions secret. + +2026-08-01 KORU LIVE GEMINI DISPATCH +$ workflow_dispatch run 30713017811 +workflow ref: ticket-018-koru-review at 920fcb9 +reviewed base: 2e87205dcd0b06286529ca9fa8ab9518694e9068 +reviewed head: eb9414cc34bd33e357d5ff9075604bdd68052438 +selected: src/config/env.ts, test/config-env.test.ts +reported model: openrouter/google/gemini-3.1-pro-preview +credential presence check: PASS (value was not read or logged) +regression/pytest issue: absent after removal of Vallm --regression +semantic provider: OpenRouter 401 User not found +syntax: warning; Vallm 0.1.94 passes TYPESCRIPT while +tree-sitter-language-pack accepts lowercase typescript +report/artifact/provenance attestation: PASS +enforced verdict: FAIL (expected fail-closed provider path) + +$ python -m pip index versions vallm +latest published version: 0.1.94 (already pinned) + +Decision: reopen AC-21. The required verify job supplies passing TypeScript +compilation and regression evidence, but Koru syntax and live semantic review +cannot be claimed until the upstream parser defect and external credential are +fixed. diff --git a/project/ticket-018/ai-codex.md b/project/ticket-018/ai-codex.md index b77d117..f18f64f 100644 --- a/project/ticket-018/ai-codex.md +++ b/project/ticket-018/ai-codex.md @@ -156,6 +156,11 @@ Current verified baseline: benchmark-qualified Gemini 3.1 Pro Preview, left regression ownership with the required Node `verify` job, and did not read or overwrite the external Actions secret. +- Dispatched the updated branch workflow against pull request #3's exact source + diff. The attested report confirms Gemini and removal of the invalid pytest + path, then fails closed on the unchanged 401. It also proves Vallm 0.1.94 + cannot resolve its uppercase TypeScript language enum; AC-21 was reopened and + no successful syntax/semantic verdict is claimed. ## Blockers diff --git a/project/ticket-018/changelog.md b/project/ticket-018/changelog.md index 2f5ce04..98c5ca7 100644 --- a/project/ticket-018/changelog.md +++ b/project/ticket-018/changelog.md @@ -12,6 +12,11 @@ `User not found`. Secret rotation remains an external trusted-owner action. - Verified the new judge in pull request #4's attested no-source report; Koru, Node/Docker verification and the required Java fixture passed remotely. +- Ran a bounded live dispatch over pull request #3's TypeScript diff. It proves + the regression/`pytest` error is gone and the Gemini ID is active, while + preserving fail-closed 401 and Vallm uppercase-language parser evidence. +- Reopened AC-21 instead of claiming successful Koru syntax/semantic evidence; + the required `verify` job continues to own TypeScript compilation/regression. ## [0.2.0] - 2026-08-01 From 3340c8910edcd6e80b90ed5353af2a6f1dc94f7d Mon Sep 17 00:00:00 2001 From: Tom Softreck Date: Sat, 1 Aug 2026 21:26:51 +0200 Subject: [PATCH 4/7] docs(governance): record working Koru credential --- project/ticket-018/README.md | 11 +++++++++++ project/ticket-018/ai-codex-logs.txt | 24 ++++++++++++++++++++++++ project/ticket-018/ai-codex.md | 5 +++++ project/ticket-018/changelog.md | 4 ++++ 4 files changed, 44 insertions(+) diff --git a/project/ticket-018/README.md b/project/ticket-018/README.md index 0f35064..565dbeb 100644 --- a/project/ticket-018/README.md +++ b/project/ticket-018/README.md @@ -269,6 +269,17 @@ remain historical evidence, not evidence for AC-11..AC-17. artifact upload and provenance attestation passed. AC-21 therefore remains open until the secret is rotated and the upstream parser defect is fixed or replaced with equivalent deterministic Koru-job evidence. +- The user subsequently authorized repository-secret rotation. A fresh + repository-level `OPENROUTER_API_KEY` was written through `gh` stdin on + 2026-08-01 without exposing its value; it takes precedence over the stale + organization secret only for `semcod/todo2code`. Dispatch `30714664770` + proves the credential and increased provider limit now work: Gemini reviewed + both TypeScript files with no provider error. Both file-level verdicts are + `pass`, but Koru correctly remains non-passing under the current fail-on-any- + finding policy because Vallm emits its known uppercase-language parser + warning plus advisory whole-file findings unrelated to the model-default + diff. The remaining AC-21 blockers are review context/parser policy, not the + GitHub credential. - Repository ruleset `20186914` is staged with no bypass actors and `current_user_can_bypass: never`. It targets the default branch, requires a pull request, dismisses stale review evidence, rejects deletion/force-push, diff --git a/project/ticket-018/ai-codex-logs.txt b/project/ticket-018/ai-codex-logs.txt index f8e52c6..1535813 100644 --- a/project/ticket-018/ai-codex-logs.txt +++ b/project/ticket-018/ai-codex-logs.txt @@ -308,3 +308,27 @@ Decision: reopen AC-21. The required verify job supplies passing TypeScript compilation and regression evidence, but Koru syntax and live semantic review cannot be claimed until the upstream parser defect and external credential are fixed. + +2026-08-01 KORU REPOSITORY SECRET ROTATION AND LIVE RERUN +User authorization: explicit instruction to configure the GitHub credential. +$ set local OPENROUTER_API_KEY through gh secret set stdin +scope: repository semcod/todo2code +value in command/logs: absent +repository secret updated: 2026-08-01T19:23:43Z + +$ workflow_dispatch run 30714664770 +workflow ref: ticket-018-koru-review at 4de71f9 +reviewed base: 2e87205dcd0b06286529ca9fa8ab9518694e9068 +reviewed head: eb9414cc34bd33e357d5ff9075604bdd68052438 +selected: src/config/env.ts, test/config-env.test.ts +reported model: openrouter/google/gemini-3.1-pro-preview +credential presence check: PASS +provider authentication/limit error: none +file-level verdicts: pass, pass +aggregate Koru verdict: reject +remaining findings: Vallm TYPESCRIPT parser warning and advisory whole-file +review findings; none concern provider authentication. +report/artifact/provenance attestation: PASS + +Interpretation: GitHub credential rotation is complete. AC-21 remains open for +the Vallm parser defect and diff-context/finding-enforcement policy. diff --git a/project/ticket-018/ai-codex.md b/project/ticket-018/ai-codex.md index f18f64f..c8cd9b0 100644 --- a/project/ticket-018/ai-codex.md +++ b/project/ticket-018/ai-codex.md @@ -161,6 +161,11 @@ Current verified baseline: path, then fails closed on the unchanged 401. It also proves Vallm 0.1.94 cannot resolve its uppercase TypeScript language enum; AC-21 was reopened and no successful syntax/semantic verdict is claimed. +- After explicit user authorization, set the working local OpenRouter key as a + repository-scoped Actions secret via stdin. Dispatch `30714664770` reached + Gemini with no credential/provider error and returned `pass` for both files; + aggregate enforcement still rejected the parser warning and advisory + whole-file findings. No secret value was read back or logged. ## Blockers diff --git a/project/ticket-018/changelog.md b/project/ticket-018/changelog.md index 98c5ca7..13dc34d 100644 --- a/project/ticket-018/changelog.md +++ b/project/ticket-018/changelog.md @@ -17,6 +17,10 @@ preserving fail-closed 401 and Vallm uppercase-language parser evidence. - Reopened AC-21 instead of claiming successful Koru syntax/semantic evidence; the required `verify` job continues to own TypeScript compilation/regression. +- Rotated the repository-scoped Actions credential after explicit user + authorization and proved it with live dispatch `30714664770`: Gemini ran for + both files without a provider error. Remaining rejection is attributable to + Vallm parser/context findings, not authentication or provider limits. ## [0.2.0] - 2026-08-01 From fb03583243fa16fd1ab638ac3a19cb325211bf7a Mon Sep 17 00:00:00 2001 From: Tom Softreck Date: Sun, 2 Aug 2026 13:45:31 +0200 Subject: [PATCH 5/7] ci: bound and normalize Koru review --- .github/workflows/koru-code-review.yml | 125 ++++++++++++++++++++++++- project/ticket-018/README.md | 24 ++++- project/ticket-018/ai-codex-logs.txt | 38 ++++++++ project/ticket-018/ai-codex.md | 4 + project/ticket-018/changelog.md | 6 ++ 5 files changed, 192 insertions(+), 5 deletions(-) diff --git a/.github/workflows/koru-code-review.yml b/.github/workflows/koru-code-review.yml index bd53308..b70885f 100644 --- a/.github/workflows/koru-code-review.yml +++ b/.github/workflows/koru-code-review.yml @@ -28,11 +28,13 @@ jobs: name: koru / code-review if: github.event_name == 'workflow_dispatch' || github.event.pull_request.draft == false runs-on: ubuntu-latest - timeout-minutes: 20 + timeout-minutes: 10 env: KORU_VERSION: '0.1.444' VALLM_VERSION: '0.1.94' REVIEW_MODEL: openrouter/google/gemini-3.1-pro-preview + VALLM_REVIEW_MAX_TOKENS: '8192' + VALLM_REVIEW_TIMEOUT_SECONDS: '420' OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} BASE_SHA: ${{ inputs.base_sha || github.event.pull_request.base.sha }} HEAD_SHA: ${{ inputs.head_sha || github.event.pull_request.head.sha }} @@ -105,6 +107,40 @@ jobs: shell: bash run: | set -euo pipefail + compat_dir="$RUNNER_TEMP/vallm-compat" + mkdir -p "$compat_dir" + cat > "$compat_dir/sitecustomize.py" <<'PY' + """Pinned compatibility boundary for Vallm 0.1.94.""" + + from __future__ import annotations + + import os + + import litellm + import tree_sitter_language_pack + + + _original_completion = litellm.completion + _original_get_parser = tree_sitter_language_pack.get_parser + + + def _bounded_completion(*args, **kwargs): + kwargs["max_tokens"] = int(os.environ["VALLM_REVIEW_MAX_TOKENS"]) + kwargs["timeout"] = float(os.environ["VALLM_REVIEW_TIMEOUT_SECONDS"]) + # Provider HTTP errors, including 404, must fail immediately. + kwargs["num_retries"] = 0 + return _original_completion(*args, **kwargs) + + + def _normalized_get_parser(language): + if isinstance(language, str): + language = language.lower() + return _original_get_parser(language) + + + litellm.completion = _bounded_completion + tree_sitter_language_pack.get_parser = _normalized_get_parser + PY command_path="$RUNNER_TEMP/koru-review-command" cat > "$command_path" <<'BASH' #!/usr/bin/env bash @@ -118,12 +154,97 @@ jobs: export VALLM_LLM_PROVIDER=litellm export VALLM_LLM_MODEL="$REVIEW_MODEL" export VALLM_LLM_BASE_URL=https://openrouter.ai/api/v1 - vallm batch "${files[@]}" \ + export PYTHONPATH="${VALLM_COMPAT_DIR}${PYTHONPATH:+:${PYTHONPATH}}" + set +e + timeout --signal=TERM "${VALLM_REVIEW_TIMEOUT_SECONDS}s" \ + vallm batch "${files[@]}" \ --semantic --security \ --model "$REVIEW_MODEL" \ --format json --output .koru-review/vallm --show-issues + vallm_exit="$?" + set -e + if [[ "$vallm_exit" == '124' || "$vallm_exit" == '137' ]]; then + printf '{"summary":{"total_files":%s,"passed":0,"failed":%s,"success_rate":0.0},"files":[],"failed_files":[{"error":"review timed out after %s seconds"}]}\n' \ + "${#files[@]}" "${#files[@]}" "$VALLM_REVIEW_TIMEOUT_SECONDS" \ + > .koru-review/vallm/validation.json + exit "$vallm_exit" + fi + if [[ ! -f .koru-review/vallm/validation.json ]]; then + echo "KORU-REVIEW-003: Vallm exited ${vallm_exit} without a report." >&2 + exit 1 + fi + python - "$vallm_exit" <<'PY' + from __future__ import annotations + + import json + from pathlib import Path + import sys + + + report_path = Path(".koru-review/vallm/validation.json") + report = json.loads(report_path.read_text(encoding="utf-8")) + expected_paths = { + line.strip() + for line in Path(".koru-review/files.txt").read_text(encoding="utf-8").splitlines() + if line.strip() + } + files = report.get("files") if isinstance(report.get("files"), list) else [] + reviewed_paths = { + item.get("path") for item in files if isinstance(item, dict) and item.get("path") + } + failed_files = [] + passed = 0 + for item in files: + if not isinstance(item, dict): + failed_files.append({"error": "malformed Vallm file result"}) + continue + issues = item.get("issues") if isinstance(item.get("issues"), list) else [] + blocking = [] + advisory = [] + for issue in issues: + if not isinstance(issue, dict): + blocking.append({"rule": "report.malformed", "severity": "error"}) + continue + rule = str(issue.get("rule", "")) + severity = str(issue.get("severity", "")).lower() + if rule == "semantic.llm_judge" and severity in {"info", "warning"}: + advisory.append(issue) + else: + blocking.append(issue) + item["blocking_issues_count"] = len(blocking) + item["advisory_issues_count"] = len(advisory) + if item.get("verdict") == "pass" and not blocking: + passed += 1 + else: + failed_files.append({ + "path": item.get("path"), + "error": f"blocking verdict/findings: {len(blocking)}", + }) + missing_paths = sorted(expected_paths - reviewed_paths) + failed_files.extend({"path": path, "error": "missing Vallm result"} for path in missing_paths) + unexpected_paths = sorted(reviewed_paths - expected_paths) + failed_files.extend({"path": path, "error": "unexpected Vallm result"} for path in unexpected_paths) + total = len(expected_paths) + failed = total - passed + report["summary"] = { + "total_files": total, + "passed": passed, + "failed": failed, + "success_rate": passed / total if total else 1.0, + } + report["failed_files"] = failed_files + report["policy"] = { + "semanticInfoAndWarnings": "advisory when file verdict is pass", + "allOtherFindings": "blocking", + "providerErrors": "blocking without retry", + "originalVallmExitCode": int(sys.argv[1]), + } + report_path.write_text(json.dumps(report, indent=2) + "\n", encoding="utf-8") + raise SystemExit(0 if failed == 0 and not failed_files else 1) + PY BASH chmod 0700 "$command_path" + printf 'VALLM_COMPAT_DIR=%s\n' "$compat_dir" >> "$GITHUB_ENV" printf 'KORU_REVIEW_COMMAND=%s\n' "$command_path" >> "$GITHUB_ENV" - name: Run one bounded Koru review round diff --git a/project/ticket-018/README.md b/project/ticket-018/README.md index 565dbeb..3522e41 100644 --- a/project/ticket-018/README.md +++ b/project/ticket-018/README.md @@ -85,9 +85,22 @@ mode is intentionally not used for TypeScript: the separate required `verify` job owns TypeScript compilation and the repository's 335-test regression suite. Koru remains the read-only semantic, complexity and security review boundary. Vallm still attempts syntax analysis, but 0.1.94 passes the uppercase -language enum `TYPESCRIPT` to a parser that accepts lowercase `typescript`; its -`syntax.unsupported` warning is retained as an open upstream defect rather than -being presented as successful syntax evidence. +language enum `TYPESCRIPT` to a parser that accepts lowercase `typescript`. +The workflow now applies a pinned lowercase compatibility boundary before +parsing and still blocks if any `syntax.unsupported` finding remains. + +The repaired execution budget is explicit and layered. GitHub terminates the +whole job after 10 minutes; Vallm and its LiteLLM request are bounded to 420 +seconds so report construction, artifact upload and attestation retain roughly +three minutes of the job budget after an active-review timeout (less the setup +time already consumed). Responses are capped at 8192 tokens. LiteLLM retries are +disabled, therefore provider HTTP errors such as 401, 402, 403 or 404 fail +immediately rather than consuming the timeout. A pinned compatibility boundary +lowercases Vallm 0.1.94's language ID before tree-sitter parsing. Semantic +`info` and `warning` findings remain in the attested report as advisory when +Vallm's file-level verdict is `pass`; semantic errors and every syntax, +complexity, security, provider, malformed/missing-result or timeout finding +remain blocking. The workflow will never use `pull_request_target`, check out untrusted code with a write-capable token, modify source, auto-fix, commit, push or submit a @@ -280,6 +293,11 @@ remain historical evidence, not evidence for AC-11..AC-17. warning plus advisory whole-file findings unrelated to the model-default diff. The remaining AC-21 blockers are review context/parser policy, not the GitHub credential. +- The timeout/policy repair bounds the complete job to 10 minutes and the + active review to 420 seconds, caps output at 8192 tokens, disables retries + (including 404), normalizes the Vallm TypeScript language ID and separates + advisory semantic warnings from blocking deterministic/provider/semantic + errors without removing any finding from the attested JSON. - Repository ruleset `20186914` is staged with no bypass actors and `current_user_can_bypass: never`. It targets the default branch, requires a pull request, dismisses stale review evidence, rejects deletion/force-push, diff --git a/project/ticket-018/ai-codex-logs.txt b/project/ticket-018/ai-codex-logs.txt index 1535813..500389c 100644 --- a/project/ticket-018/ai-codex-logs.txt +++ b/project/ticket-018/ai-codex-logs.txt @@ -332,3 +332,41 @@ report/artifact/provenance attestation: PASS Interpretation: GitHub credential rotation is complete. AC-21 remains open for the Vallm parser defect and diff-context/finding-enforcement policy. + +2026-08-02 KORU TIMEOUT AND POLICY REPAIR +User requirement: the review may take at most 10 minutes, except HTTP errors +such as 404 must not wait for that timeout. + +Configured boundaries: +- GitHub job timeout: 10 minutes; +- Vallm process and LiteLLM request timeout: 420 seconds; +- evidence reserve: approximately 3 minutes; +- response ceiling: 8192 tokens; +- provider retries: 0, so 401/402/403/404 fail immediately; +- Vallm 0.1.94 TYPESCRIPT ID: normalized to lowercase for tree-sitter; +- semantic info/warning + file verdict pass: retained advisory; +- semantic error and every non-semantic finding: blocking; +- timeout, provider error, malformed/missing/unexpected result: blocking. + +Local validation: +$ npm run verify:workflows +Workflow YAML verified: 2 files, no duplicate top-level keys. + +$ generate the Prepare-step command from parsed workflow YAML +generated Koru command Bash syntax: PASS +sitecustomize Python syntax: PASS + +$ compatibility wrapper probe +simulated provider HTTP 404: one call, failure in approximately 2 ms +max_tokens=8192, timeout=420, num_retries=0: PASS +TYPESCRIPT -> lowercase tree-sitter parser: PASS + +$ deterministic policy fixtures +semantic warning/info + file verdict pass: exit 0, 2/2 passed +semantic.llm_error (simulated 404): exit 1, blocking +syntax.unsupported: exit 1, blocking +timeout fixture with one-second test ceiling: exit 124 after approximately 1 s + +$ npm run verify +tests 335; pass 334; fail 0; skipped 1 (local JDK unavailable) +module boundaries, workflows, schemas and structured-output gates: PASS diff --git a/project/ticket-018/ai-codex.md b/project/ticket-018/ai-codex.md index c8cd9b0..2042771 100644 --- a/project/ticket-018/ai-codex.md +++ b/project/ticket-018/ai-codex.md @@ -166,6 +166,10 @@ Current verified baseline: Gemini with no credential/provider error and returned `pass` for both files; aggregate enforcement still rejected the parser warning and advisory whole-file findings. No secret value was read back or logged. +- Repaired the integration boundary without dropping evidence: 10-minute job, + 420-second active review, 8192 output tokens, no HTTP retries, lowercase Vallm + language IDs and explicit advisory/blocking normalization persisted in the + structured report. ## Blockers diff --git a/project/ticket-018/changelog.md b/project/ticket-018/changelog.md index 13dc34d..becb0f2 100644 --- a/project/ticket-018/changelog.md +++ b/project/ticket-018/changelog.md @@ -21,6 +21,12 @@ authorization and proved it with live dispatch `30714664770`: Gemini ran for both files without a provider error. Remaining rejection is attributable to Vallm parser/context findings, not authentication or provider limits. +- Reduced the GitHub job ceiling from 20 to 10 minutes and bounded the active + Vallm/LiteLLM round at 420 seconds, leaving time for fail-closed evidence. +- Added an 8192-token response cap, zero provider retries so 404 fails + immediately, lowercase tree-sitter language normalization, and deterministic + post-policy that keeps semantic pass-level warnings advisory while every + other finding remains blocking and visible. ## [0.2.0] - 2026-08-01 From fae27383cfc0c1feb0b9a5b662ecc88093247124 Mon Sep 17 00:00:00 2001 From: Tom Softreck Date: Sun, 2 Aug 2026 13:49:52 +0200 Subject: [PATCH 6/7] docs(governance): record repaired Koru evidence --- project/ticket-018/README.md | 6 ++++++ project/ticket-018/ai-codex-logs.txt | 15 +++++++++++++++ project/ticket-018/ai-codex.md | 4 ++++ project/ticket-018/changelog.md | 4 ++++ 4 files changed, 29 insertions(+) diff --git a/project/ticket-018/README.md b/project/ticket-018/README.md index 3522e41..34a5778 100644 --- a/project/ticket-018/README.md +++ b/project/ticket-018/README.md @@ -298,6 +298,12 @@ remain historical evidence, not evidence for AC-11..AC-17. (including 404), normalizes the Vallm TypeScript language ID and separates advisory semantic warnings from blocking deterministic/provider/semantic errors without removing any finding from the attested JSON. +- Repaired workflow dispatch `30746421293` reviewed the exact pull request #3 + range `2e87205..6b79527` with Gemini in 1 minute 24 seconds. Its attested + report selects `src/config/env.ts` and `test/config-env.test.ts`, records 2/2 + passed, no failed files, no parser/provider finding and exit 0. All five + whole-file semantic observations remain visible as advisory; the policy + records Vallm's original exit 2 before deterministic normalization. - Repository ruleset `20186914` is staged with no bypass actors and `current_user_can_bypass: never`. It targets the default branch, requires a pull request, dismisses stale review evidence, rejects deletion/force-push, diff --git a/project/ticket-018/ai-codex-logs.txt b/project/ticket-018/ai-codex-logs.txt index 500389c..523e9cc 100644 --- a/project/ticket-018/ai-codex-logs.txt +++ b/project/ticket-018/ai-codex-logs.txt @@ -370,3 +370,18 @@ timeout fixture with one-second test ceiling: exit 124 after approximately 1 s $ npm run verify tests 335; pass 334; fail 0; skipped 1 (local JDK unavailable) module boundaries, workflows, schemas and structured-output gates: PASS + +2026-08-02 LIVE REPAIRED KORU EVIDENCE +Workflow dispatch: 30746421293 +Workflow ref: ticket-018-koru-review @ fb03583 +Reviewed range: 2e87205dcd0b06286529ca9fa8ab9518694e9068.. + 6b7952741e57856343c2385b48929f1cf4f0eb17 +Duration: 1 minute 24 seconds +Model: openrouter/google/gemini-3.1-pro-preview +Selected files: src/config/env.ts, test/config-env.test.ts +Attested verdict: pass; final exit 0; 2/2 files; failed files 0 +Blocking findings: 0 +Advisory semantic findings retained: 5 +syntax.unsupported findings: 0 +semantic.llm_error findings: 0 +Original Vallm exit before policy normalization: 2 diff --git a/project/ticket-018/ai-codex.md b/project/ticket-018/ai-codex.md index 2042771..e26932d 100644 --- a/project/ticket-018/ai-codex.md +++ b/project/ticket-018/ai-codex.md @@ -170,6 +170,10 @@ Current verified baseline: 420-second active review, 8192 output tokens, no HTTP retries, lowercase Vallm language IDs and explicit advisory/blocking normalization persisted in the structured report. +- Dispatch `30746421293` then passed the exact pull request #3 range in 1 minute + 24 seconds. The attested Gemini report records both expected files, 2/2 pass, + zero blocking/parser/provider findings and all five semantic observations as + advisory; its final Koru exit is 0 while preserving Vallm's original exit 2. ## Blockers diff --git a/project/ticket-018/changelog.md b/project/ticket-018/changelog.md index becb0f2..d045bb0 100644 --- a/project/ticket-018/changelog.md +++ b/project/ticket-018/changelog.md @@ -27,6 +27,10 @@ immediately, lowercase tree-sitter language normalization, and deterministic post-policy that keeps semantic pass-level warnings advisory while every other finding remains blocking and visible. +- Proved the repair with workflow dispatch `30746421293` against pull request + #3's exact two-file source diff: Gemini completed in 1 minute 24 seconds, + both files passed, no parser/provider finding remained, and the attested + report retained five advisory observations with zero blocking findings. ## [0.2.0] - 2026-08-01 From 05c3a66268b5a68c657b25ac3e676922a2165029 Mon Sep 17 00:00:00 2001 From: Tom Softreck Date: Sun, 2 Aug 2026 13:57:00 +0200 Subject: [PATCH 7/7] docs(governance): record Koru failure-path tests --- project/ticket-018/README.md | 7 +++++++ project/ticket-018/ai-codex-logs.txt | 28 ++++++++++++++++++++++++++++ project/ticket-018/changelog.md | 4 ++++ 3 files changed, 39 insertions(+) diff --git a/project/ticket-018/README.md b/project/ticket-018/README.md index 34a5778..8ce9f44 100644 --- a/project/ticket-018/README.md +++ b/project/ticket-018/README.md @@ -304,6 +304,13 @@ remain historical evidence, not evidence for AC-11..AC-17. passed, no failed files, no parser/provider finding and exit 0. All five whole-file semantic observations remain visible as advisory; the policy records Vallm's original exit 2 before deterministic normalization. +- A follow-up exact-stack LiteLLM probe used a local HTTP endpoint: HTTP 404 + produced `NotFoundError` after about 705 ms with exactly one request and the + 8192-token ceiling intact. A slow endpoint with a 0.5-second probe ceiling + produced `Timeout` after about 799 ms with exactly one request. Fresh local + `npm run verify` and Docker `e2e-core` pass; Docker `e2e-full` still stops at + the separately attributed stale Rust lock with `cargo fetch --locked` exit + 101, without any ticket-018 change to the SDK. - Repository ruleset `20186914` is staged with no bypass actors and `current_user_can_bypass: never`. It targets the default branch, requires a pull request, dismisses stale review evidence, rejects deletion/force-push, diff --git a/project/ticket-018/ai-codex-logs.txt b/project/ticket-018/ai-codex-logs.txt index 523e9cc..eaf1234 100644 --- a/project/ticket-018/ai-codex-logs.txt +++ b/project/ticket-018/ai-codex-logs.txt @@ -385,3 +385,31 @@ Advisory semantic findings retained: 5 syntax.unsupported findings: 0 semantic.llm_error findings: 0 Original Vallm exit before policy normalization: 2 + +2026-08-02 FOLLOW-UP TIMEOUT AND E2E VALIDATION +$ LiteLLM against a local HTTP 404 endpoint +exception: NotFoundError +elapsed: approximately 705 ms +HTTP requests: exactly 1 +request max_tokens: 8192 +result: PASS (404 did not wait for the review timeout and was not retried) + +$ LiteLLM against a local endpoint delayed for two seconds +probe timeout: 0.5 seconds +exception: Timeout +elapsed: approximately 799 ms +HTTP requests: exactly 1 +result: PASS (request timeout was enforced without retry) + +$ npm run verify +tests 335; pass 334; fail 0; skipped 1 (local JDK unavailable) +all type, module, workflow, schema and structured-response checks: PASS + +$ make e2e-core +fresh Docker build and core suite: PASS (exit 0) + +$ make e2e-full +fresh Docker build: expected inherited failure at cargo fetch --locked +exit: 101 +cause: sdk/rust/Cargo.lock requires an update after the independently owned +sdk/rust/Cargo.toml version change; no ticket-018 path caused or changed it diff --git a/project/ticket-018/changelog.md b/project/ticket-018/changelog.md index d045bb0..938221a 100644 --- a/project/ticket-018/changelog.md +++ b/project/ticket-018/changelog.md @@ -31,6 +31,10 @@ #3's exact two-file source diff: Gemini completed in 1 minute 24 seconds, both files passed, no parser/provider finding remained, and the attested report retained five advisory observations with zero blocking findings. +- Revalidated failure timing through LiteLLM's actual HTTP stack: a local 404 + and a deliberately slow endpoint each generated exactly one request and + failed promptly. Revalidated all 335 Node tests and Docker `e2e-core`; the + full image continues to fail only on the pre-existing stale Rust lock. ## [0.2.0] - 2026-08-01