diff --git a/.github/workflows/test-pipeline.yml b/.github/workflows/test-pipeline.yml index fecf2dd8c5..593465c853 100644 --- a/.github/workflows/test-pipeline.yml +++ b/.github/workflows/test-pipeline.yml @@ -19,6 +19,7 @@ jobs: build: name: Build Tests uses: ./.github/workflows/test-suite.yml + secrets: inherit with: suite: build judge_mode: ${{ inputs.judge_mode || 'simple' }} @@ -27,6 +28,7 @@ jobs: name: Smoke Tests needs: build uses: ./.github/workflows/test-suite.yml + secrets: inherit with: suite: smoke judge_mode: ${{ inputs.judge_mode || 'simple' }} @@ -35,6 +37,7 @@ jobs: name: Auth Tests needs: smoke uses: ./.github/workflows/test-suite.yml + secrets: inherit with: suite: auth judge_mode: ${{ inputs.judge_mode || 'simple' }} @@ -43,6 +46,7 @@ jobs: name: CRUD Tests needs: auth uses: ./.github/workflows/test-suite.yml + secrets: inherit with: suite: crud judge_mode: ${{ inputs.judge_mode || 'simple' }} @@ -51,6 +55,7 @@ jobs: name: Workflow Tests needs: crud uses: ./.github/workflows/test-suite.yml + secrets: inherit with: suite: workflow judge_mode: ${{ inputs.judge_mode || 'simple' }} diff --git a/.github/workflows/test-suite.yml b/.github/workflows/test-suite.yml index d36a2eac93..b596f2391e 100644 --- a/.github/workflows/test-suite.yml +++ b/.github/workflows/test-suite.yml @@ -25,9 +25,9 @@ on: - "simple" - "dual" judge_model: - description: "LLM model for judging (if dual mode)" + description: "LLM model (leave blank to use vars.LLM_JUDGE_MODEL or framework default gemma3:4b)" required: false - default: "llama3:8b" + default: "" type: string workflow_call: inputs: @@ -41,9 +41,9 @@ on: default: "simple" type: string judge_model: - description: "LLM model for judging" + description: "LLM model override; empty means use vars.LLM_JUDGE_MODEL / framework default" required: false - default: "llama3:8b" + default: "" type: string outputs: result: @@ -71,13 +71,25 @@ jobs: - name: Run tests id: run-tests + env: + # Flow secrets/vars through to the test framework. When unset, + # the scripts' defaults kick in (see cicd/tests/src/config.ts and + # cicd/scripts/init-db.sh). Local dev keeps using cicd/tests/.env + # — workflows never materialize that file. See FR-005. + LLM_JUDGE_URL: ${{ secrets.LLM_JUDGE_URL }} + LLM_JUDGE_MODEL: ${{ vars.LLM_JUDGE_MODEL }} + TL_DEV_KEY: ${{ secrets.TL_DEV_KEY }} run: | - # Build judge flags based on input + # Build judge flags based on input. + # In dual mode, only append --judge-model when the operator typed + # one at dispatch time. Otherwise let LLM_JUDGE_MODEL (from the + # env: block above, fed by vars.LLM_JUDGE_MODEL) or the framework + # default in config.ts take effect — a CLI flag would shadow them. JUDGE_FLAGS="" if [ "${{ inputs.judge_mode }}" = "simple" ] || [ -z "${{ inputs.judge_mode }}" ]; then JUDGE_FLAGS="--no-llm" - else - JUDGE_FLAGS="--judge-model ${{ inputs.judge_model || 'llama3:8b' }}" + elif [ -n "${{ inputs.judge_model }}" ]; then + JUDGE_FLAGS="--judge-model ${{ inputs.judge_model }}" fi # Build suite flag diff --git a/cicd/CI_SETUP.md b/cicd/CI_SETUP.md new file mode 100644 index 0000000000..023ba6c2ee --- /dev/null +++ b/cicd/CI_SETUP.md @@ -0,0 +1,33 @@ +# CI setup — GitHub Actions + +How the `.github/workflows/*.yml` workflows consume configuration, and what a new collaborator needs to add to their fork to run them. + +Design record: [FR-005](../docs/feature-requests/FR-005-github-actions-env-vars-and-secrets.md). Issue: [#19](https://github.com/dogkeeper886/testlink-code/issues/19). + +## The basics + +Workflows are **manual only** (`workflow_dispatch`). Trigger them from the Actions tab against any ref. They do not fire on push or PR. That is intentional. + +The default `judge_mode` is `simple`, which invokes `run-tests.sh` with `--no-llm`. The simple regex judge carries CI on its own. No LLM endpoint is required for a green CI run. + +Switch `judge_mode` to `dual` at dispatch time if you want the LLM judge to run too — that requires an accessible Ollama endpoint (see below). + +## Secrets and variables + +Add these in your fork's **Settings → Secrets and variables → Actions**. All are optional — the framework falls back to defaults when they are unset. + +| Name | Kind | Purpose | Default if unset | +|---|---|---|---| +| `LLM_JUDGE_URL` | Secret | Ollama endpoint for the LLM judge | `http://localhost:11434` (unreachable on hosted runners) | +| `LLM_JUDGE_MODEL` | Variable | Model tag the judge asks Ollama to load | `gemma3:4b` | +| `TL_DEV_KEY` | Secret | Rotated admin API key (32 hex chars) | `a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4` (seeded by `init-db.sh`) | + +Leave them unset for simple-judge-only CI. Set `LLM_JUDGE_URL` + `LLM_JUDGE_MODEL` when you want `dual` mode to work against a real endpoint. + +## Why these aren't in a `.env` file on the runner + +Local developers put these in `cicd/tests/.env` (gitignored). The workflow does **not** materialize that file on the runner — values flow through the job's `env:` block straight into the test framework's process environment. See FR-005's *Alternatives considered* for the reasoning. + +## Local dev is unchanged + +Your `cicd/tests/.env` keeps working exactly as before. `run-tests.sh` sources it only when present, and the runner has no `.env` to source. diff --git a/cicd/tests/src/config.ts b/cicd/tests/src/config.ts index 3798b81896..246c11d387 100644 --- a/cicd/tests/src/config.ts +++ b/cicd/tests/src/config.ts @@ -40,7 +40,7 @@ export const CONFIG = { // --judge-model CLI flags. llm: { defaultUrl: process.env.LLM_JUDGE_URL || 'http://localhost:11434', - defaultModel: process.env.LLM_JUDGE_MODEL || 'llama3:8b', + defaultModel: process.env.LLM_JUDGE_MODEL || 'gemma3:4b', timeout: 300000, stdoutLimit: 1000, stderrLimit: 500, diff --git a/cicd/tests/src/types.ts b/cicd/tests/src/types.ts index d4378eb44a..dbde3e3f88 100644 --- a/cicd/tests/src/types.ts +++ b/cicd/tests/src/types.ts @@ -264,6 +264,6 @@ export const DEFAULT_CONFIG: Partial = { dryRun: false, noLlm: false, judgeUrl: 'http://localhost:11434', - judgeModel: 'llama3:8b', + judgeModel: 'gemma3:4b', outputFormat: 'console', }; diff --git a/docs/feature-requests/FR-005-github-actions-env-vars-and-secrets.md b/docs/feature-requests/FR-005-github-actions-env-vars-and-secrets.md index 38524f75d0..f2a83b9b54 100644 --- a/docs/feature-requests/FR-005-github-actions-env-vars-and-secrets.md +++ b/docs/feature-requests/FR-005-github-actions-env-vars-and-secrets.md @@ -1,7 +1,7 @@ --- fr: FR-005 title: Wire GitHub Actions runner with env vars + secrets for the CI suite -status: draft +status: in-progress issue: https://github.com/dogkeeper886/testlink-code/issues/19 authors: ["dogkeeper886"] created: 2026-04-20 @@ -24,7 +24,7 @@ The CI test framework reads configuration from several environment variables: | Var | Default (when unset) | Who uses it | |---|---|---| | `LLM_JUDGE_URL` | `http://localhost:11434` | `cicd/tests/src/config.ts` — Ollama endpoint for semantic judge | -| `LLM_JUDGE_MODEL` | `llama3:8b` | Model to load on that endpoint | +| `LLM_JUDGE_MODEL` | `gemma3:4b` | Model to load on that endpoint | | `TL_PORT` | `8091` | `docker-compose.ci.yml` host port | | `TL_URL` | `http://localhost:8091` | `ci-up.sh` health check, `xmlrpc-capture.sh` default | | `TL_DEV_KEY` | `a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4` | `executor.ts` → `{{devKey}}`, `init-db.sh` → seeded admin key | diff --git a/docs/feature-requests/README.md b/docs/feature-requests/README.md index e689085f10..fd0f8b3aaa 100644 --- a/docs/feature-requests/README.md +++ b/docs/feature-requests/README.md @@ -46,4 +46,4 @@ If you happen to use Claude Code, there's an optional personal skill (`~/.claude | [FR-002](./FR-002-delete-build-and-remove-test-case-from-test-plan.md) | Add `deleteBuild` and `removeTestCaseFromTestPlan` | done | [#8](https://github.com/dogkeeper886/testlink-code/issues/8) | | [FR-003](./FR-003-requirement-spec-and-requirement-crud.md) | Add Requirement Spec + Requirement CRUD | draft | [#9](https://github.com/dogkeeper886/testlink-code/issues/9) | | [FR-004](./FR-004-llm-judge-evidence-grounding.md) | LLM judge — enforce grounded evidence citations | done | [#14](https://github.com/dogkeeper886/testlink-code/issues/14) | -| [FR-005](./FR-005-github-actions-env-vars-and-secrets.md) | GitHub Actions runner env vars + secrets | draft | [#19](https://github.com/dogkeeper886/testlink-code/issues/19) | +| [FR-005](./FR-005-github-actions-env-vars-and-secrets.md) | GitHub Actions runner env vars + secrets | in-progress | [#19](https://github.com/dogkeeper886/testlink-code/issues/19) |