ci: add cron stress harness for serve with fault injection + metrics

.github/workflows/serve-stress-real.yml

@@ -0,0 +1,87 @@
+name: Serve Stress (real repos)
+
+# Operational stress + performance-trend validation for `bazel-diff serve` against REAL repos of
+# three sizes, complementing the hermetic serve-stress.yml. Each matrix leg runs
+# tools/serve_stress.py in --real-repo-url mode against its target: it clones the repo, stands serve
+# up against it, and drives real commit SHAs (HEAD~1..HEAD) through concurrent cold/hot load plus the
+# index.lock self-heal / storm phases -- exercising real large-tree checkouts and a real `bazel
+# query` that the fabricated genrule workspace cannot. Checks are invariants (200s / cross-request
+# consistency / self-heal), not a hardcoded impacted set.
+#
+# The three legs (small / medium / bazel) each emit a per-size metrics JSON + markdown summary as a
+# 90-day artifact, so throughput / latency / time-to-ready can be compared *across sizes* and *across
+# runs over time*. Runs daily alongside the other serve crons; each leg gets a 90-min timeout and
+# fail-fast:false because the `bazel` leg's full `bazel query` on bazelbuild/bazel is minutes-scale
+# and network-heavy. If a daily run of the large leg is more runner cost than the trend data is
+# worth, dial the cron below back to weekly (e.g. "30 8 * * 1").
+#
+# The targets are wired here, not in code (the harness is preset-free). Swap a repo by editing its
+# matrix row; from/to default to HEAD~1..HEAD so every run tests each repo's latest commit.
+# Note: scheduled runs execute from the default branch, so this must land on master to fire.
+on:
+  workflow_dispatch:
+  schedule:
+    # Daily at 08:30 UTC, offset from serve-stress (30 6) and serve-harness (0 */12) so the jobs
+    # never contend for runners. Change to "30 8 * * 1" for a weekly cadence on the heavy legs.
+    - cron: "30 8 * * *"
+
+jobs:
+  ServeStressReal:
+    strategy:
+      fail-fast: false  # a slow/broken leg must not cancel the others' comparison data
+      matrix:
+        include:
+          - size: small
+            repo: "https://github.com/bazelbuild/bazel-skylib.git"
+            clone_args: "--depth=50"
+          - size: medium
+            repo: "https://github.com/abseil/abseil-cpp.git"
+            clone_args: "--depth=50"
+          - size: bazel
+            repo: "https://github.com/bazelbuild/bazel.git"
+            clone_args: "--depth=50"
+    name: ServeStressReal (${{ matrix.size }})
+    runs-on: ubuntu-latest
+    timeout-minutes: 90
+    steps:
+      - name: Setup Java JDK
+        uses: actions/setup-java@v4
+        with:
+          distribution: 'temurin'
+          java-version: '21'
+      - name: Setup Go environment
+        uses: actions/setup-go@v5
+        with:
+          go-version: ^1.17
+        id: go
+      - name: Setup Bazelisk
+        run: go install github.com/bazelbuild/bazelisk@latest
+      - uses: actions/checkout@v4
+      - name: Run serve stress harness (${{ matrix.size }})
+        # --bazel points the harness (and the serve subprocess) at bazelisk; the bazel version for
+        # the CLONED repo comes from that repo's own .bazelversion, which bazelisk honors.
+        run: >
+          python3 tools/serve_stress.py
+          --bazel "$HOME/go/bin/bazelisk"
+          --real-repo-url "${{ matrix.repo }}"
+          --real-from HEAD~1
+          --real-to HEAD
+          --real-clone-args "${{ matrix.clone_args }}"
+          --metrics-out "serve-stress-real-${{ matrix.size }}-metrics.json"
+          --summary-out "serve-stress-real-${{ matrix.size }}-summary.md"
+      - name: Publish step summary
+        if: always()
+        run: |
+          if [ -f "serve-stress-real-${{ matrix.size }}-summary.md" ]; then
+            cat "serve-stress-real-${{ matrix.size }}-summary.md" >> "$GITHUB_STEP_SUMMARY"
+          fi
+      - name: Upload metrics artifact
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: serve-stress-real-${{ matrix.size }}-${{ github.run_id }}
+          path: |
+            serve-stress-real-${{ matrix.size }}-metrics.json
+            serve-stress-real-${{ matrix.size }}-summary.md
+          if-no-files-found: warn
+          retention-days: 90

.github/workflows/serve-stress.yml

@@ -0,0 +1,66 @@
+name: Serve Stress
+
+# Operational stress validation for the `bazel-diff serve` query service. Runs
+# tools/serve_stress.py, which drives the real binary over HTTP against a live git:// remote
+# under concurrent load and injected faults (orphaned git index.lock checkouts, an index.lock
+# storm, a git-remote outage, lame-duck startup) and captures throughput / response-time
+# metrics for every phase. Complements the functional matrix in serve-harness.yml.
+#
+# Scheduled runs use the full profile; on-demand runs from the Actions tab can select the
+# reduced quick profile. Note: scheduled + dispatched workflows run from the default branch,
+# so this must land on master before the cron fires.
+on:
+  workflow_dispatch:
+    inputs:
+      quick:
+        description: "Run the reduced quick profile"
+        type: boolean
+        default: false
+  schedule:
+    # Daily, offset from the serve-harness cron (0 */12) so the two never contend for runners.
+    - cron: "30 6 * * *"
+
+jobs:
+  ServeStress:
+    runs-on: ubuntu-latest
+    timeout-minutes: 45
+    steps:
+      - name: Setup Java JDK
+        uses: actions/setup-java@v4
+        with:
+          distribution: 'temurin'
+          java-version: '21'
+      - name: Setup Go environment
+        uses: actions/setup-go@v5
+        with:
+          go-version: ^1.17
+        id: go
+      - name: Setup Bazelisk
+        run: go install github.com/bazelbuild/bazelisk@latest
+      - uses: actions/checkout@v4
+      - name: Run serve stress harness
+        # git daemon ships with git (preinstalled on the runner); python3 is preinstalled.
+        # --bazel points the harness (and the serve subprocesses it spawns) at bazelisk; the
+        # bazel version comes from .bazelversion, which the harness copies into its workspaces.
+        run: >
+          python3 tools/serve_stress.py
+          --bazel "$HOME/go/bin/bazelisk"
+          --metrics-out serve-stress-metrics.json
+          --summary-out serve-stress-summary.md
+          ${{ inputs.quick && '--quick' || '' }}
+      - name: Publish step summary
+        if: always()
+        run: |
+          if [ -f serve-stress-summary.md ]; then
+            cat serve-stress-summary.md >> "$GITHUB_STEP_SUMMARY"
+          fi
+      - name: Upload metrics artifact
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: serve-stress-metrics-${{ github.run_id }}
+          path: |
+            serve-stress-metrics.json
+            serve-stress-summary.md
+          if-no-files-found: warn
+          retention-days: 90

tools/BUILD

@@ -55,7 +55,8 @@ py_test(
     deps = [":coverage_check_lib"],
 )
 
-# Note: the serve integration harness (serve_harness.py) is intentionally NOT wrapped in a
-# py_binary. It shells out to `bazel build //cli:bazel-diff`, which would deadlock on the output-base
-# lock if it were itself launched via `bazel run` (nested bazel, same output base). Run it directly:
-# `python3 tools/serve_harness.py` (see its module docstring). It uses only the Python stdlib.
+# Note: the serve integration harnesses (serve_harness.py, serve_stress.py) are intentionally NOT
+# wrapped in py_binary targets. They shell out to `bazel build //cli:bazel-diff`, which would
+# deadlock on the output-base lock if they were themselves launched via `bazel run` (nested bazel,
+# same output base). Run them directly: `python3 tools/serve_harness.py` /
+# `python3 tools/serve_stress.py` (see their module docstrings). They use only the Python stdlib.

tools/serve_harness.py

@@ -364,15 +364,23 @@ def serve(
     initial_fetch: bool,
     track_deps: bool,
     ready_timeout: float,
+    verbose_server: bool = False,
 ):
     """Launches serve, waits for a health verdict, yields (Serve, health) then tears down.
 
     health is one of: "ready" (200), "lameduck" (up but stuck 503), "down" (never bound).
+
+    [verbose_server] launches the subprocess with `-v` so its StderrLogger emits the `[Info]` /
+    `[Warning]` git lines (checkouts, the "cleared stale git index.lock" self-heal) to the captured
+    stderr; without it only `[Error]` lines appear. The stress harness asserts on those lines.
     """
     port = free_port()
     stderr_path = workspace.parent / f"serve.{workspace.name}.stderr.log"
     args = [
         str(LAUNCHER),
+        # `-v` is a parent-command option with INHERIT scope, so it applies to the `serve`
+        # subcommand whether it precedes or follows it.
+        *(["-v"] if verbose_server else []),
         "serve",
         "-w",
         str(workspace),