feat(runner): report a test that only passed on retry as flaky

.env.example

@@ -43,6 +43,7 @@ BASHUNIT_PARALLEL_RUN=              # Default: false
 BASHUNIT_STOP_ON_FAILURE=           # Default: false (stop suite on first failure)
 BASHUNIT_RERUN_FAILED=              # Default: false (replay only last run's failing tests)
 BASHUNIT_ORDER_BY=                  # Default: defined (or defects, random)
+BASHUNIT_FAIL_ON_FLAKY=             # Default: false (treat retry-passed tests as failed)
 BASHUNIT_CHANGED=                   # Default: false (run only test files changed since a git ref)
 BASHUNIT_CHANGED_REF=               # Default: empty (--changed ref: origin/HEAD, then HEAD)
 BASHUNIT_EXCLUDE_FILTER=            # Default: empty (skip tests whose name matches)

CHANGELOG.md

@@ -3,6 +3,7 @@
 ## Unreleased
 
 ### Added
+- Flaky is a first-class outcome: a test that only passed after a retry is counted separately, kept inside the pass total so the exit code is unchanged, and carried into JUnit (`<flakyFailure>`), TAP, JSON, HTML and GitHub Actions along with the first attempt's failure message. `--fail-on-flaky` turns such a run red (#1012)
 - `--order-by <mode>` picks the execution order: `defined` (default), `defects` (last run's failures first, whole suite still runs) or `random`. `--random-order` and `--seed` keep working unchanged (#1011)
 - `--changed [<ref>]` runs only the test files git reports as touched since `<ref>` (default `origin/HEAD`, then `HEAD`), covering committed, staged, unstaged and untracked changes. Deletions are dropped, a rename selects its new path, and a missing work tree or unresolvable ref fails the run instead of selecting nothing (#1010)
 - `--list` (alias `--dry-run`) prints the tests a run would execute, without running them; `--list-format json` emits file, function, name, line and tags. Honours every selection flag, including `--shard` and `--random-order --seed` ordering (#1007)

completions/_bashunit

@@ -85,6 +85,7 @@ _bashunit() {
     '--test-timeout[Fail a test running longer than N seconds]:seconds:' \
     '--retry[Rerun a failed test up to N extra times]:count:' \
     '--random-order[Randomize test execution order]' \
+    '--fail-on-flaky[Treat tests that only passed after a retry as failures]' \
     '--order-by[Execution order]:mode:(defined defects random)' \
     '--seed[Seed for random order]:seed:' \
     '--shard[Run shard i of n]:shard:' \

completions/bashunit.bash

@@ -16,7 +16,7 @@ _BASHUNIT_COMPLETIONS_DOC_OPTS="--custom -e --env --boot -h --help"
 
 _BASHUNIT_COMPLETIONS_TEST_OPTS="--assert --boot --changed --coverage --coverage-exclude \
 --coverage-min --coverage-paths --coverage-report --coverage-report-html \
---debug --detailed --dry-run --env --exclude-filter --exclude-tag --fail-on-risky --failures-only \
+--debug --detailed --dry-run --env --exclude-filter --exclude-tag --fail-on-flaky --fail-on-risky --failures-only \
 --filter --help --jobs --list --list-format --log-gha --log-junit --login --no-color \
 --no-coverage-report --no-output --no-output-on-failure --no-parallel \
 --no-progress --no-snapshot-create --order-by --output --parallel --profile \

docs/command-line.md

@@ -98,6 +98,7 @@ bashunit test tests/ --parallel --simple
 | `--no-output`                  | Suppress all output                              |
 | `--failures-only`              | Only show failures                               |
 | `--fail-on-risky`              | Treat risky tests (no assertions) as failures    |
+| `--fail-on-flaky`              | Treat flaky tests (passed only after a retry) as failures |
 | `--profile`                    | Report the slowest tests after a run             |
 | `--no-progress`                | Suppress real-time progress, show only summary   |
 | `--show-output`                | Show test output on failure (default)            |
@@ -822,6 +823,56 @@ BASHUNIT_RERUN_FAILED=true bashunit test tests/
 ```
 :::
 
+### Flaky tests
+
+> `bashunit test --retry 2 --fail-on-flaky`
+
+A test that failed and then passed on a retry is **flaky**: it passed, so the
+run stays green, but the summary says so.
+
+```
+Tests:      12 passed, 1 flaky, 12 total
+```
+
+The flaky count is a facet of `passed`, not a seventh outcome, which is why it
+is not added to the total. Without it a retried failure is indistinguishable
+from a clean run, and flakiness never gets triaged.
+
+Every report carries the status, along with the **first attempt's** failure
+message (the diagnostic value, otherwise discarded when the retry overwrites
+it) and the retry count:
+
+| Format | Output |
+|--------|--------|
+| JUnit | `<flakyFailure>` inside the `<testcase>`, rendered natively by Jenkins and GitLab. Not counted in `failures` |
+| TAP | `ok N - name # TODO flaky (retried 1/2)` |
+| JSON | `"status": "flaky"`, `"retries": N`, plus a `flaky` key in the summary |
+| HTML | its own row styling |
+| GitHub Actions | a `::warning` annotation |
+
+Add `--fail-on-flaky` to turn a flaky run red, mirroring
+[`--fail-on-risky`](#test-options):
+
+```bash
+bashunit test tests/ --retry 2 --fail-on-flaky
+```
+
+Notes:
+
+- `--retry 0` (the default) can never produce a flaky result: nothing is retried.
+- Counters are correct under `--parallel`; the retry count crosses the fork in
+  the per-test payload.
+- Flaky never changes the exit code on its own.
+
+::: code-group
+```bash [Surface flakiness in CI]
+bashunit test --retry 2 --report-junit report.xml
+```
+```bash [Env variable]
+BASHUNIT_FAIL_ON_FLAKY=true bashunit test tests/ --retry 2
+```
+:::
+
 ### Order by
 
 > `bashunit test --order-by <mode>`

src/config/env.sh

@@ -255,6 +255,8 @@ _BASHUNIT_DEFAULT_SHARD_INDEX=""
 _BASHUNIT_DEFAULT_SHARD_TOTAL=""
 # Replay only the tests recorded as failing by the previous run
 _BASHUNIT_DEFAULT_RERUN_FAILED="false"
+# Treat a test that only passed after a retry as a failure for the exit code
+_BASHUNIT_DEFAULT_FAIL_ON_FLAKY="false"
 # Execution order: defined (definition order), defects (last run's failures
 # first) or random (equivalent to --random-order)
 _BASHUNIT_DEFAULT_ORDER_BY="defined"
@@ -308,8 +310,9 @@ _BASHUNIT_DEFAULT_SNAPSHOT_REPORT_UNUSED="false"
 # up unrelated environment values.
 : "${BASHUNIT_RANDOM_ORDER:=$_BASHUNIT_DEFAULT_RANDOM_ORDER}"
 : "${BASHUNIT_SEED:=$_BASHUNIT_DEFAULT_SEED}"
-# No bare ORDER_BY alias, same reasoning as RETRY/SEED above.
+# No bare ORDER_BY/FAIL_ON_FLAKY aliases, same reasoning as RETRY/SEED above.
 : "${BASHUNIT_ORDER_BY:=$_BASHUNIT_DEFAULT_ORDER_BY}"
+: "${BASHUNIT_FAIL_ON_FLAKY:=$_BASHUNIT_DEFAULT_FAIL_ON_FLAKY}"
 : "${BASHUNIT_SHARD_INDEX:=$_BASHUNIT_DEFAULT_SHARD_INDEX}"
 : "${BASHUNIT_SHARD_TOTAL:=$_BASHUNIT_DEFAULT_SHARD_TOTAL}"
 # No bare RERUN_FAILED alias, same reasoning as RETRY/SEED above. The default
@@ -613,6 +616,10 @@ function bashunit::env::is_fail_on_risky_enabled() {
   [ "$BASHUNIT_FAIL_ON_RISKY" = "true" ]
 }
 
+function bashunit::env::is_fail_on_flaky_enabled() {
+  [ "${BASHUNIT_FAIL_ON_FLAKY:-false}" = "true" ]
+}
+
 function bashunit::env::is_profile_enabled() {
   [ "$BASHUNIT_PROFILE" = "true" ]
 }

src/console/header.sh

@@ -154,6 +154,7 @@ Options:
   --no-output                 Suppress all output
   --failures-only             Only show failures (suppress passed/skipped/incomplete)
   --fail-on-risky             Treat risky tests (no assertions) as failures
+  --fail-on-flaky             Treat flaky tests (passed only after a retry) as failures
   --profile                   Report the slowest tests (count: BASHUNIT_PROFILE_COUNT, default 10)
   --no-progress               Suppress real-time progress, show only final results
   --show-output               Show test output on failure (default: enabled)

src/console/summary.sh

@@ -30,6 +30,7 @@ function bashunit::console_results::render_result() {
   local tests_snapshot=$_BASHUNIT_TESTS_SNAPSHOT
   local tests_failed=$_BASHUNIT_TESTS_FAILED
   local tests_risky=$_BASHUNIT_TESTS_RISKY
+  local tests_flaky=$_BASHUNIT_TESTS_FLAKY
   local assertions_passed=$_BASHUNIT_ASSERTIONS_PASSED
   local assertions_skipped=$_BASHUNIT_ASSERTIONS_SKIPPED
   local assertions_incomplete=$_BASHUNIT_ASSERTIONS_INCOMPLETE
@@ -70,6 +71,11 @@ function bashunit::console_results::render_result() {
   if [ "$tests_risky" -gt 0 ]; then
     printf " %s%s risky%s," "$_BASHUNIT_COLOR_RISKY" "$tests_risky" "$_BASHUNIT_COLOR_DEFAULT"
   fi
+  # Deliberately absent from total_tests: these tests are already inside the
+  # passed count, so adding them would make the total exceed the tests run.
+  if [ "$tests_flaky" -gt 0 ]; then
+    printf " %s%s flaky%s," "$_BASHUNIT_COLOR_INCOMPLETE" "$tests_flaky" "$_BASHUNIT_COLOR_DEFAULT"
+  fi
   printf " %s total\n" "$total_tests"
 
   printf "%sAssertions:%s" "$_BASHUNIT_COLOR_FAINT" "$_BASHUNIT_COLOR_DEFAULT"
@@ -96,6 +102,14 @@ function bashunit::console_results::render_result() {
     return 1
   fi
 
+  # Ranked above risky so a run that is both reports the outcome that turns it
+  # red. Without the flag flaky is a pass, so the ladder falls straight through.
+  if [ "$tests_flaky" -gt 0 ] && bashunit::env::is_fail_on_flaky_enabled; then
+    printf "\n%s%s%s\n" "$_BASHUNIT_COLOR_RETURN_ERROR" " Some tests flaky " "$_BASHUNIT_COLOR_DEFAULT"
+    bashunit::console_results::print_execution_time
+    return 1
+  fi
+
   if [ "$tests_risky" -gt 0 ]; then
     printf "\n%s%s%s\n" "$_BASHUNIT_COLOR_RETURN_RISKY" " Some tests risky (no assertions) " "$_BASHUNIT_COLOR_DEFAULT"
     bashunit::console_results::print_execution_time

src/main/test.sh

@@ -278,6 +278,10 @@ function bashunit::main::cmd_test() {
       BASHUNIT_FAIL_ON_RISKY=true
       export -n BASHUNIT_FAIL_ON_RISKY
       ;;
+    --fail-on-flaky)
+      BASHUNIT_FAIL_ON_FLAKY=true
+      export -n BASHUNIT_FAIL_ON_FLAKY
+      ;;
     --profile)
       BASHUNIT_PROFILE=true
       export -n BASHUNIT_PROFILE

src/reports/collect.sh

@@ -17,6 +17,7 @@ _BASHUNIT_REPORTS_TEST_DURATIONS=()
 _BASHUNIT_REPORTS_TEST_ASSERTIONS=()
 _BASHUNIT_REPORTS_TEST_FAILURES=()
 _BASHUNIT_REPORTS_TEST_LINES=()
+_BASHUNIT_REPORTS_TEST_RETRIES=()
 
 function bashunit::reports::add_test_snapshot() {
   bashunit::reports::add_test "$1" "$2" "$3" "$4" "snapshot"
@@ -42,6 +43,17 @@ function bashunit::reports::add_test_failed() {
   bashunit::reports::add_test "$1" "$2" "$3" "$4" "failed" "$5"
 }
 
+##
+# A test that passed, but not on the first attempt. Carries the retry count and
+# the first attempt's failure message, which is the whole diagnostic value and
+# is otherwise discarded when the retry loop overwrites the losing attempt.
+# Arguments: $1 file, $2 name, $3 duration, $4 assertions, $5 first failure,
+# $6 retries.
+##
+function bashunit::reports::add_test_flaky() {
+  bashunit::reports::add_test "$1" "$2" "$3" "$4" "flaky" "$5" "$6"
+}
+
 # Returns 0 when any report output is requested.
 function bashunit::reports::is_enabled() {
   [ -n "${BASHUNIT_LOG_JUNIT:-}" ] ||
@@ -61,6 +73,7 @@ function bashunit::reports::add_test() {
   local assertions="$4"
   local status="$5"
   local failure_message="${6:-}"
+  local retries="${7:-0}"
 
   # Capture the line number from the current test location ("file:line"),
   # but only when it belongs to this test's file, so a stale location from a
@@ -86,14 +99,15 @@ function bashunit::reports::add_test() {
   # Fields are base64-encoded because a failure message carries newlines and
   # arbitrary text, either of which would break a delimited line.
   if bashunit::parallel::is_enabled; then
-    printf '%s|%s|%s|%s|%s|%s|%s\n' \
+    printf '%s|%s|%s|%s|%s|%s|%s|%s\n' \
       "$(bashunit::helper::encode_base64 "$file")" \
       "$(bashunit::helper::encode_base64 "$test_name")" \
       "$(bashunit::helper::encode_base64 "$status")" \
       "$(bashunit::helper::encode_base64 "$duration")" \
       "$(bashunit::helper::encode_base64 "$assertions")" \
       "$(bashunit::helper::encode_base64 "$failure_message")" \
       "$(bashunit::helper::encode_base64 "$line")" \
+      "$(bashunit::helper::encode_base64 "$retries")" \
       >>"${REPORTS_OUTPUT_PATH:-/dev/null}" 2>/dev/null || true
   fi
 
@@ -104,6 +118,7 @@ function bashunit::reports::add_test() {
   _BASHUNIT_REPORTS_TEST_DURATIONS[${#_BASHUNIT_REPORTS_TEST_DURATIONS[@]}]="$duration"
   _BASHUNIT_REPORTS_TEST_FAILURES[${#_BASHUNIT_REPORTS_TEST_FAILURES[@]}]="$failure_message"
   _BASHUNIT_REPORTS_TEST_LINES[${#_BASHUNIT_REPORTS_TEST_LINES[@]}]="$line"
+  _BASHUNIT_REPORTS_TEST_RETRIES[${#_BASHUNIT_REPORTS_TEST_RETRIES[@]}]="$retries"
 }
 
 ##
@@ -115,8 +130,8 @@ function bashunit::reports::load_spooled() {
   bashunit::reports::is_enabled || return 0
   [ -f "${REPORTS_OUTPUT_PATH:-}" ] || return 0
 
-  local file test_name status duration assertions failure_message line n
-  while IFS='|' read -r file test_name status duration assertions failure_message line; do
+  local file test_name status duration assertions failure_message line retries n
+  while IFS='|' read -r file test_name status duration assertions failure_message line retries; do
     [ -n "$file" ] || continue
     local n=${#_BASHUNIT_REPORTS_TEST_FILES[@]}
     _BASHUNIT_REPORTS_TEST_FILES[n]=$(bashunit::helper::decode_base64 "$file")
@@ -126,5 +141,6 @@ function bashunit::reports::load_spooled() {
     _BASHUNIT_REPORTS_TEST_ASSERTIONS[n]=$(bashunit::helper::decode_base64 "$assertions")
     _BASHUNIT_REPORTS_TEST_FAILURES[n]=$(bashunit::helper::decode_base64 "$failure_message")
     _BASHUNIT_REPORTS_TEST_LINES[n]=$(bashunit::helper::decode_base64 "$line")
+    _BASHUNIT_REPORTS_TEST_RETRIES[n]=$(bashunit::helper::decode_base64 "$retries")
   done <"$REPORTS_OUTPUT_PATH"
 }

src/reports/gha.sh

@@ -37,6 +37,10 @@ function bashunit::reports::print_gha_annotations() {
         level="warning"
         message="Test has no assertions (risky)"
         ;;
+      flaky)
+        level="warning"
+        message="Test passed only after ${_BASHUNIT_REPORTS_TEST_RETRIES[$i]:-0} retries: $failure_message"
+        ;;
       incomplete)
         level="notice"
         message="Test incomplete"

src/reports/html.sh

@@ -48,6 +48,7 @@ function bashunit::reports::generate_report_html() {
     echo "    .incomplete { background-color: #d9edf7; }"
     echo "    .snapshot { background-color: #dfe6e9; }"
     echo "    .risky { background-color: #f5e6f5; }"
+    echo "    .flaky { background-color: #ffe8cc; }"
     echo "  </style>"
     echo "</head>"
     echo "<body>"

src/reports/json.sh

@@ -20,14 +20,20 @@ function bashunit::reports::generate_report_json() {
   local output_file="$1"
   local total="${#_BASHUNIT_REPORTS_TEST_NAMES[@]}"
 
-  local passed=0 failed=0 skipped=0 incomplete=0 duration_total=0
+  local passed=0 failed=0 skipped=0 incomplete=0 flaky=0 duration_total=0
   local i
   for i in "${!_BASHUNIT_REPORTS_TEST_NAMES[@]}"; do
     duration_total=$((duration_total + ${_BASHUNIT_REPORTS_TEST_DURATIONS[$i]:-0}))
     case "${_BASHUNIT_REPORTS_TEST_STATUSES[$i]:-}" in
     failed) failed=$((failed + 1)) ;;
     skipped) skipped=$((skipped + 1)) ;;
     incomplete) incomplete=$((incomplete + 1)) ;;
+    # Flaky is counted twice on purpose: it passed, so it belongs in passed, and
+    # the separate tally is what makes it triageable.
+    flaky)
+      flaky=$((flaky + 1))
+      passed=$((passed + 1))
+      ;;
     # snapshot and risky ran without failing, so they count as passed here; the
     # per-test "status" field below preserves the exact category.
     *) passed=$((passed + 1)) ;;
@@ -38,8 +44,8 @@ function bashunit::reports::generate_report_json() {
     printf '{\n'
     printf '  "summary": { "total": %d, "passed": %d, "failed": %d,' \
       "$total" "$passed" "$failed"
-    printf ' "skipped": %d, "incomplete": %d, "duration_ms": %d },\n' \
-      "$skipped" "$incomplete" "$duration_total"
+    printf ' "skipped": %d, "incomplete": %d, "flaky": %d, "duration_ms": %d },\n' \
+      "$skipped" "$incomplete" "$flaky" "$duration_total"
     printf '  "tests": [\n'
     local seq=0
     for i in "${!_BASHUNIT_REPORTS_TEST_NAMES[@]}"; do
@@ -51,8 +57,10 @@ function bashunit::reports::generate_report_json() {
       message=$(bashunit::reports::__json_escape "${_BASHUNIT_REPORTS_TEST_FAILURES[$i]:-}")
       sep=","
       [ "$seq" -eq "$((total - 1))" ] && sep=""
-      printf '    { "file": "%s", "name": "%s", "status": "%s", "duration_ms": %d, "message": "%s" }%s\n' \
-        "$file" "$name" "$status" "$duration" "$message" "$sep"
+      printf '    { "file": "%s", "name": "%s", "status": "%s", "duration_ms": %d,' \
+        "$file" "$name" "$status" "$duration"
+      printf ' "retries": %d, "message": "%s" }%s\n' \
+        "${_BASHUNIT_REPORTS_TEST_RETRIES[$i]:-0}" "$message" "$sep"
       seq=$((seq + 1))
     done
     printf '  ]\n'

src/reports/junit.sh

@@ -51,6 +51,13 @@ function bashunit::reports::generate_junit_xml() {
         local escaped_message
         escaped_message=$(bashunit::reports::__xml_escape "$failure_message")
         echo "      <failure message=\"Test failed\">$escaped_message</failure>"
+      elif [ "$status" = "flaky" ]; then
+        # Jenkins and GitLab render flakyFailure natively, and it does not count
+        # towards failures="" -- which is the point: the test passed.
+        local escaped_flaky
+        escaped_flaky=$(bashunit::reports::__xml_escape "$failure_message")
+        echo "      <flakyFailure message=\"Test passed after ${_BASHUNIT_REPORTS_TEST_RETRIES[$i]:-0} \
+retries\">$escaped_flaky</flakyFailure>"
       elif [ "$status" = "risky" ]; then
         echo "      <skipped message=\"Test has no assertions (risky)\"/>"
       elif [ "$status" = "skipped" ]; then

src/reports/tap.sh

@@ -50,6 +50,11 @@ function bashunit::reports::generate_report_tap() {
       incomplete)
         echo "ok $seq - $name # TODO"
         ;;
+      flaky)
+        # `ok` because it passed; the TODO directive is how TAP consumers mark a
+        # result that needs attention without failing the run.
+        echo "ok $seq - $name # TODO flaky (retried ${_BASHUNIT_REPORTS_TEST_RETRIES[$i]:-0}/${BASHUNIT_RETRY:-0})"
+        ;;
       *)
         echo "ok $seq - $name"
         ;;