fix(reports): GitHub Actions annotations never reached the pull request

.env.example

@@ -45,6 +45,7 @@ BASHUNIT_RERUN_FAILED=              # Default: false (replay only last run's fai
 BASHUNIT_ORDER_BY=                  # Default: defined (or defects, random)
 BASHUNIT_FAIL_ON_FLAKY=             # Default: false (treat retry-passed tests as failed)
 BASHUNIT_REPEAT=                    # Default: 1 (run each test N times)
+BASHUNIT_GHA_ANNOTATIONS=           # Default: auto (or always, never)
 BASHUNIT_CHANGED=                   # Default: false (run only test files changed since a git ref)
 BASHUNIT_CHANGED_REF=               # Default: empty (--changed ref: origin/HEAD, then HEAD)
 BASHUNIT_EXCLUDE_FILTER=            # Default: empty (skip tests whose name matches)

CHANGELOG.md

@@ -3,6 +3,7 @@
 ## Unreleased
 
 ### Added
+- `--gha-annotations <auto|always|never>` controls GitHub Actions annotations on stdout; `auto` turns them on inside GitHub Actions and stays quiet everywhere else (#1014)
 - `--repeat <n>` runs each selected test n times so flakiness can be hunted before it reaches CI. The test is reported once with the aggregate outcome, a failure names the iteration it happened on, and repeat wraps `--retry` rather than the other way round (#1013)
 - Flaky is a first-class outcome: a test that only passed after a retry is counted separately, kept inside the pass total so the exit code is unchanged, and carried into JUnit (`<flakyFailure>`), TAP, JSON, HTML and GitHub Actions along with the first attempt's failure message. `--fail-on-flaky` turns such a run red (#1012)
 - `--order-by <mode>` picks the execution order: `defined` (default), `defects` (last run's failures first, whole suite still runs) or `random`. `--random-order` and `--seed` keep working unchanged (#1011)

action.yml

@@ -27,6 +27,10 @@ inputs:
     description: 'If set, run "bashunit <args>" after installing (e.g. "tests/ --strict"). Empty = install only.'
     required: false
     default: ''
+  annotations:
+    description: 'Annotate failing tests on the pull request. bashunit detects GitHub Actions on its own, so this only needs setting to turn them off ("never").'
+    required: false
+    default: 'auto'
 
 outputs:
   path:
@@ -47,6 +51,7 @@ runs:
         BASHUNIT_ADD_TO_PATH: ${{ inputs.add-to-path }}
         BASHUNIT_VERIFY_CHECKSUM: ${{ inputs.verify-checksum }}
         BASHUNIT_ARGS: ${{ inputs.args }}
+        BASHUNIT_GHA_ANNOTATIONS: ${{ inputs.annotations }}
         BASHUNIT_ACTION_PATH: ${{ github.action_path }}
       run: |
         set -euo pipefail

completions/_bashunit

@@ -69,6 +69,7 @@ _bashunit() {
     '--exclude-filter[Skip tests whose name matches]:name:' \
     '--exclude-tag[Skip tests with matching @tag]:tag:' \
     '--log-junit[Write JUnit XML report]:file:_files' \
+    '--gha-annotations[GitHub Actions annotations on stdout]:mode:(auto always never)' \
     '--report-junit[Write JUnit XML report]:file:_files' \
     '--log-gha[Write GitHub Actions workflow-commands log]:file:_files' \
     '(-j --jobs)'{-j,--jobs}'[Max N parallel jobs, or auto]:jobs:(auto)' \

completions/bashunit.bash

@@ -17,7 +17,7 @@ _BASHUNIT_COMPLETIONS_DOC_OPTS="--custom -e --env --boot -h --help"
 _BASHUNIT_COMPLETIONS_TEST_OPTS="--assert --boot --changed --coverage --coverage-exclude \
 --coverage-min --coverage-paths --coverage-report --coverage-report-html \
 --debug --detailed --dry-run --env --exclude-filter --exclude-tag --fail-on-flaky --fail-on-risky --failures-only \
---filter --help --jobs --list --list-format --log-gha --log-junit --login --no-color \
+--filter --gha-annotations --help --jobs --list --list-format --log-gha --log-junit --login --no-color \
 --no-coverage-report --no-output --no-output-on-failure --no-parallel \
 --no-progress --no-snapshot-create --order-by --output --parallel --profile \
 --random-order --repeat --report-html \
@@ -84,6 +84,10 @@ _bashunit_completions() {
     COMPREPLY=($(compgen -W "defined defects random" -- "$cur"))
     return 0
     ;;
+  --gha-annotations)
+    COMPREPLY=($(compgen -W "auto always never" -- "$cur"))
+    return 0
+    ;;
   -f | --filter | --exclude-filter | --tag | --exclude-tag | --repeat | --retry | --seed | --shard | --test-timeout)
     return 0
     ;;

docs/command-line.md

@@ -68,6 +68,7 @@ bashunit test tests/ --parallel --simple
 | `-w, --watch`                  | Watch files and re-run tests on change           |
 | `--log-junit, --report-junit <file>` | Write JUnit XML report                     |
 | `--log-gha <file>`             | Write GitHub Actions workflow-commands log       |
+| `--gha-annotations <mode>`     | Annotations on stdout: `auto` (default), `always` or `never` |
 | `-j, --jobs <N\|auto>`         | Run tests in parallel with max N concurrent jobs (`auto` = CPU cores) |
 | `-p, --parallel`               | Run tests in parallel                            |
 | `--no-parallel`                | Run tests sequentially                           |
@@ -448,15 +449,48 @@ bashunit test tests/ --log-junit results.xml
 bashunit test tests/ --report-html report.html
 ```
 ```bash [GitHub Actions]
-# Stream annotations straight to the runner log:
-bashunit test tests/ --log-gha /dev/stdout
+# Nothing to configure: annotations are automatic on a runner.
+bashunit test tests/
 ```
 ```bash [JSON]
 bashunit test tests/ --report-json report.json
 ```
 :::
 
-The `--log-gha` flag writes GitHub Actions workflow commands (`::error`, `::warning`, `::notice`) for failed, risky and incomplete tests, including the failing test's `file` and `line`. Point it at `/dev/stdout` (or stream a log file to stdout) on a runner and the failures appear as inline annotations in the "Files changed" tab of a pull request.
+### GitHub Actions annotations
+
+Inside GitHub Actions, bashunit annotates failing tests on the pull request by
+itself. No flag, no configuration:
+
+```
+::error file=tests/math_test.sh,line=42,title=Sums::Expected '4' but got '5'
+```
+
+GitHub parses workflow commands from the **job log**, so the annotations go to
+stdout. They carry the failing test's `file` and `line`, which is what puts them
+on the right line of the "Files changed" tab: `::error` for failures, `::warning`
+for risky and flaky tests, `::notice` for incomplete ones. Messages are
+percent-encoded, so a multi-line failure stays a single annotation.
+
+Detection is `GITHUB_ACTIONS=true`, and `--gha-annotations` overrides it:
+
+| Mode | Behaviour |
+|------|-----------|
+| `auto` | On inside GitHub Actions, silent everywhere else. The default. |
+| `always` | On everywhere, useful for another CI that understands the format |
+| `never` | Off, including inside GitHub Actions |
+
+```bash
+bashunit test tests/ --gha-annotations never
+```
+
+`auto` also stays quiet under `--output tap`, whose stdout is a machine format an
+annotation line would corrupt.
+
+The separate `--log-gha <file>` flag still writes the same workflow commands to a
+file. It is independent of the stdout annotations, so using both does not
+duplicate anything in the job log.
+
 
 The `--report-json` flag writes machine-readable results for scripts, dashboards and bots. Strings are escaped in pure Bash, so no `jq` is needed to produce it. Its schema is:
 

src/config/env.sh

@@ -255,6 +255,8 @@ _BASHUNIT_DEFAULT_SHARD_INDEX=""
 _BASHUNIT_DEFAULT_SHARD_TOTAL=""
 # Replay only the tests recorded as failing by the previous run
 _BASHUNIT_DEFAULT_RERUN_FAILED="false"
+# When to print GitHub Actions annotations to stdout: auto|always|never
+_BASHUNIT_DEFAULT_GHA_ANNOTATIONS="auto"
 # Run each selected test n times; the test fails if any iteration fails
 _BASHUNIT_DEFAULT_REPEAT="1"
 # Treat a test that only passed after a retry as a failure for the exit code
@@ -316,6 +318,22 @@ _BASHUNIT_DEFAULT_SNAPSHOT_REPORT_UNUSED="false"
 : "${BASHUNIT_ORDER_BY:=$_BASHUNIT_DEFAULT_ORDER_BY}"
 : "${BASHUNIT_FAIL_ON_FLAKY:=$_BASHUNIT_DEFAULT_FAIL_ON_FLAKY}"
 : "${BASHUNIT_REPEAT:=$_BASHUNIT_DEFAULT_REPEAT}"
+: "${BASHUNIT_GHA_ANNOTATIONS:=$_BASHUNIT_DEFAULT_GHA_ANNOTATIONS}"
+
+# GITHUB_ACTIONS is inherited by every child process, so a nested bashunit run
+# (bashunit's own acceptance suite, or a user's script under test that calls
+# bashunit) would annotate the parent's job log with its own fixtures'
+# failures. Only the outermost run owns that log.
+#
+# Deliberately exported, unlike the run-mode flags: the nested run is exactly
+# the consumer that has to see it. Reading it before claiming it is what makes
+# the outermost process the one that wins.
+if [ -n "${_BASHUNIT_GHA_ANNOTATIONS_CLAIMED:-}" ]; then
+  _BASHUNIT_IS_OUTERMOST_RUN=false
+else
+  _BASHUNIT_IS_OUTERMOST_RUN=true
+fi
+export _BASHUNIT_GHA_ANNOTATIONS_CLAIMED=1
 : "${BASHUNIT_SHARD_INDEX:=$_BASHUNIT_DEFAULT_SHARD_INDEX}"
 : "${BASHUNIT_SHARD_TOTAL:=$_BASHUNIT_DEFAULT_SHARD_TOTAL}"
 # No bare RERUN_FAILED alias, same reasoning as RETRY/SEED above. The default
@@ -632,6 +650,24 @@ function bashunit::env::is_fail_on_risky_enabled() {
   [ "$BASHUNIT_FAIL_ON_RISKY" = "true" ]
 }
 
+##
+# Whether workflow-command annotations go to stdout. GitHub parses them from the
+# job log, so stdout is the only sink that reaches a pull request.
+#
+# `auto` stays quiet outside GitHub Actions, and quiet under --output tap, whose
+# stdout is a machine format an annotation line would corrupt.
+##
+function bashunit::env::should_print_gha_annotations() {
+  case "${BASHUNIT_GHA_ANNOTATIONS:-auto}" in
+  never) return 1 ;;
+  always) return 0 ;;
+  esac
+
+  [ "${_BASHUNIT_IS_OUTERMOST_RUN:-true}" = true ] &&
+    [ "${GITHUB_ACTIONS:-}" = "true" ] &&
+    ! bashunit::env::is_tap_output_enabled
+}
+
 function bashunit::env::is_fail_on_flaky_enabled() {
   [ "${BASHUNIT_FAIL_ON_FLAKY:-false}" = "true" ]
 }

src/console/header.sh

@@ -125,6 +125,8 @@ Options:
                               Supports 'a&&b' (AND) and '!a' (NOT)
   --exclude-tag <name>        Skip tests with matching @tag (repeatable, exclude wins)
   --log-junit, --report-junit <file>  Write JUnit XML report
+  --log-gha <file>            Write GitHub Actions annotations to a file
+  --gha-annotations <mode>    Annotations on stdout: auto (in GitHub Actions), always or never
   -j, --jobs <N|auto>         Run tests in parallel with max N concurrent jobs ("auto" = CPU cores)
   -p, --parallel              Run tests in parallel (unlimited concurrency)
   --no-parallel               Run tests sequentially

src/main/run.sh

@@ -160,6 +160,13 @@ function bashunit::main::exec_tests() {
 
   bashunit::reports::load_spooled
 
+  # To stdout, not to a file: GitHub reads workflow commands from the job log.
+  # After load_spooled so a --parallel run annotates the rows its workers
+  # spooled, which the parent would otherwise never have seen (#1004).
+  if bashunit::env::should_print_gha_annotations; then
+    bashunit::reports::print_gha_annotations all
+  fi
+
   if [ -n "$BASHUNIT_LOG_JUNIT" ]; then
     bashunit::reports::generate_junit_xml "$BASHUNIT_LOG_JUNIT"
   fi

src/main/test.sh

@@ -240,6 +240,11 @@ function bashunit::main::cmd_test() {
       export -n BASHUNIT_LOG_GHA
       shift
       ;;
+    --gha-annotations)
+      BASHUNIT_GHA_ANNOTATIONS="$2"
+      export -n BASHUNIT_GHA_ANNOTATIONS
+      shift
+      ;;
     -r | --report-html)
       BASHUNIT_REPORT_HTML="$2"
       export -n BASHUNIT_REPORT_HTML

src/main/validate.sh

@@ -153,6 +153,17 @@ function bashunit::main::validate_config_or_exit() {
     ;;
   esac
 
+  # Same shape as --output above: an unrecognised mode would otherwise fall back
+  # to auto and look like it was honoured.
+  case "${BASHUNIT_GHA_ANNOTATIONS:-auto}" in
+  auto | always | never) ;;
+  *)
+    printf "%sError: unsupported mode '%s' for --gha-annotations. Supported: auto, always, never.%s\n" \
+      "${_BASHUNIT_COLOR_FAILED}" "${BASHUNIT_GHA_ANNOTATIONS}" "${_BASHUNIT_COLOR_DEFAULT}" >&2
+    exit 1
+    ;;
+  esac
+
   # Same shape as --output above: an unrecognised mode would otherwise leave the
   # suite in definition order and look like it was honoured.
   case "${BASHUNIT_ORDER_BY:-defined}" in

src/reports/collect.sh

@@ -60,7 +60,8 @@ function bashunit::reports::is_enabled() {
     [ -n "${BASHUNIT_REPORT_HTML:-}" ] ||
     [ -n "${BASHUNIT_LOG_GHA:-}" ] ||
     [ -n "${BASHUNIT_REPORT_TAP:-}" ] ||
-    [ -n "${BASHUNIT_REPORT_JSON:-}" ]
+    [ -n "${BASHUNIT_REPORT_JSON:-}" ] ||
+    bashunit::env::should_print_gha_annotations
 }
 
 function bashunit::reports::add_test() {

tests/acceptance/bashunit_gha_annotations_test.sh

@@ -0,0 +1,112 @@
+#!/usr/bin/env bash
+
+# GitHub parses workflow commands from the job log, so an annotation only lands
+# on a pull request if it reaches stdout. Writing it to a file nobody cats
+# produced exactly zero annotations.
+#
+# Clearing _BASHUNIT_GHA_ANNOTATIONS_CLAIMED below is how a nested run says
+# "pretend I am the top-level one": this suite is itself a bashunit run and has
+# already claimed the job log for its process tree, which is the very pollution
+# the marker exists to prevent.
+
+function set_up_before_script() {
+  TEST_ENV_FILE="tests/acceptance/fixtures/.env.default"
+  FIXTURE="./tests/acceptance/fixtures/test_bashunit_when_log_junit.sh"
+}
+
+function test_annotations_reach_stdout_inside_github_actions() {
+  local output
+  output="$(_BASHUNIT_GHA_ANNOTATIONS_CLAIMED='' GITHUB_ACTIONS=true ./bashunit --no-parallel --no-color \
+    --env "$TEST_ENV_FILE" "$FIXTURE")" || true
+
+  assert_contains "::error file=$FIXTURE" "$output"
+  assert_contains "title=Failure" "$output"
+}
+
+function test_the_annotation_carries_the_failing_line() {
+  local output
+  output="$(_BASHUNIT_GHA_ANNOTATIONS_CLAIMED='' GITHUB_ACTIONS=true ./bashunit --no-parallel --no-color \
+    --env "$TEST_ENV_FILE" "$FIXTURE")" || true
+
+  assert_matches "::error file=[^,]*,line=[0-9]+,title=" "$output"
+}
+
+function test_nothing_extra_is_printed_outside_github_actions() {
+  local output
+  output="$(GITHUB_ACTIONS='' ./bashunit --no-parallel --no-color \
+    --env "$TEST_ENV_FILE" "$FIXTURE")" || true
+
+  assert_not_contains "::error" "$output"
+}
+
+# The claim marker is left alone here, so this is a genuinely nested run. Under
+# CI it inherits GITHUB_ACTIONS=true and must still stay quiet, or every nested
+# run in a suite would annotate the parent's log with its own fixtures.
+function test_a_nested_run_never_annotates_the_parents_log() {
+  local output
+  output="$(GITHUB_ACTIONS=true ./bashunit --no-parallel --no-color \
+    --env "$TEST_ENV_FILE" "$FIXTURE")" || true
+
+  assert_not_contains "::error" "$output"
+}
+
+function test_never_suppresses_annotations_inside_github_actions() {
+  local output
+  output="$(_BASHUNIT_GHA_ANNOTATIONS_CLAIMED='' GITHUB_ACTIONS=true ./bashunit --no-parallel --no-color \
+    --env "$TEST_ENV_FILE" --gha-annotations never "$FIXTURE")" || true
+
+  assert_not_contains "::error" "$output"
+}
+
+function test_always_emits_annotations_outside_github_actions() {
+  local output
+  output="$(./bashunit --no-parallel --no-color --env "$TEST_ENV_FILE" \
+    --gha-annotations always "$FIXTURE")" || true
+
+  assert_contains "::error file=$FIXTURE" "$output"
+}
+
+function test_a_multi_line_message_stays_one_annotation() {
+  local output
+  output="$(_BASHUNIT_GHA_ANNOTATIONS_CLAIMED='' GITHUB_ACTIONS=true ./bashunit --no-parallel --no-color \
+    --env "$TEST_ENV_FILE" "$FIXTURE")" || true
+
+  # One failing test, so one ::error line, with the newlines percent-encoded.
+  assert_same "1" "$(printf '%s\n' "$output" | grep -c '^::error' | tr -d ' ')"
+  assert_contains "%0A" "$output"
+}
+
+function test_log_gha_still_writes_the_file_without_duplicating_stdout() {
+  local log_file
+  log_file="$(bashunit::temp_file)"
+
+  local output
+  output="$(_BASHUNIT_GHA_ANNOTATIONS_CLAIMED='' GITHUB_ACTIONS=true ./bashunit --no-parallel --no-color \
+    --env "$TEST_ENV_FILE" --log-gha "$log_file" "$FIXTURE")" || true
+
+  assert_contains "::error" "$(cat "$log_file")"
+  assert_same "1" "$(printf '%s\n' "$output" | grep -c '^::error' | tr -d ' ')"
+}
+
+function test_annotations_survive_parallel_aggregation() {
+  local output
+  output="$(_BASHUNIT_GHA_ANNOTATIONS_CLAIMED='' GITHUB_ACTIONS=true ./bashunit --parallel --no-color \
+    --env "$TEST_ENV_FILE" "$FIXTURE")" || true
+
+  assert_contains "::error file=$FIXTURE" "$output"
+}
+
+function test_log_gha_appears_in_the_help() {
+  assert_contains "--log-gha" "$(./bashunit test --help)"
+}
+
+function test_an_unknown_mode_is_a_usage_error() {
+  local ec=0
+  local output
+  output="$(./bashunit --no-parallel --env "$TEST_ENV_FILE" \
+    --gha-annotations sometimes "$FIXTURE" 2>&1)" || ec=$?
+
+  assert_general_error "" "" "$ec"
+  assert_contains "sometimes" "$output"
+  assert_contains "auto, always, never" "$output"
+}