Forbid ::warning:: advisory pattern; pip-audit + benchmark fail-fast

.github/workflows/_required.yml

@@ -82,6 +82,35 @@ jobs:
           fi
           echo 'OK: no "continue-on-error: true" found'
 
+      - name: "Reject advisory annotation pattern"
+        run: |
+          set -euo pipefail
+          mapfile -t files < <(git ls-files -- '.github/workflows/*.yml' '.github/workflows/*.yaml')
+          # The advisory-annotation pattern in workflow run blocks is morally
+          # equivalent to continue-on-error: true: it lets a tool fail without
+          # failing the CI step. The runbook (Bucket F) requires every tool
+          # to be fail-fast. The only legitimate annotation is informational
+          # text not tied to a tool's exit code — those should use
+          # echo "WARN:" to stdout instead. See docs/runbooks/no-silent-failures.md.
+          declare -a scan_files=()
+          for f in "${files[@]}"; do
+            # Exempt this workflow file (heredoc would otherwise self-match).
+            case "$f" in
+              .github/workflows/_required.yml) continue ;;
+            esac
+            scan_files+=("$f")
+          done
+          if [ "${#scan_files[@]}" -eq 0 ]; then
+            echo "No files to scan"
+            exit 0
+          fi
+          if grep -nF '::warning::' "${scan_files[@]}"; then
+            echo ""
+            echo '::error::Found advisory annotations above. Make the underlying tool fail-fast or use echo "WARN:" to stdout. See docs/runbooks/no-silent-failures.md Bucket F.'
+            exit 1
+          fi
+          echo 'OK: no advisory annotations found'
+
   # ============================================================
   # 1. lint
   #    clang-format, ruff, pre-commit, CMake cycle check,
@@ -686,13 +715,23 @@ jobs:
       - name: Python dependency audit (pip-audit)
         run: |
           set -euo pipefail
-          pip install -e ".[dev]" --quiet
-          # pip-audit --strict exits non-zero on any finding. We want to surface
-          # the report as a warning (Trivy below is the gating scan) without
-          # silently masking the rc.
-          if ! pip-audit --strict; then
-            echo "::warning::pip-audit found Python dependency advisories — see step log above"
-          fi
+          # pip-audit --strict exits non-zero on any finding — fail-fast per
+          # docs/runbooks/no-silent-failures.md (Bucket F). If a finding is
+          # genuinely unfixable, add it to a `--ignore-vuln` allowlist with an
+          # inline comment naming the tracking issue and planned fix date.
+          #
+          # Auditing the project path directly resolves pyproject.toml's
+          # production dependency closure without installing the project
+          # itself. This avoids the editable-distribution edge case in
+          # pip-audit 2.10+ where `--strict --skip-editable` errors with
+          # "distribution marked as editable" because projectkeystone is
+          # installed via `pip install -e` but is intentionally absent from
+          # PyPI (Keystone is a C++20 library; pyproject.toml exists only
+          # for dev/test tooling).
+          #
+          # Dev extras (mypy, conan, pytest, ...) are tools, not shipped
+          # code, and are intentionally out of scope here.
+          pip-audit --strict .
 
       - name: Run Trivy filesystem scan (SARIF)
         uses: aquasecurity/trivy-action@v0.36.0

.github/workflows/extras.yml

@@ -66,13 +66,7 @@ jobs:
         run: sccache --show-stats
 
       - name: Run benchmarks
-        run: |
-          set -euo pipefail
-          # Benchmarks can legitimately regress without blocking the workflow —
-          # report-only. Surface the rc as a warning instead of silently masking.
-          if ! make benchmark.native; then
-            echo "::warning::benchmark.native exited non-zero — see step log above"
-          fi
+        run: make benchmark.native
 
       - name: Upload benchmark results
         uses: actions/upload-artifact@v7

.pre-commit-config.yaml

@@ -138,6 +138,25 @@ repos:
         files: ^\.github/workflows/.*\.ya?ml$
         pass_filenames: true
 
+      - id: forbid-advisory-warnings
+        name: "forbid advisory ::warning:: pattern in workflow run blocks"
+        description: >
+          The advisory-warning workflow annotation is morally equivalent to
+          `continue-on-error: true` when it replaces a tool's failure exit
+          with a benign-looking warning. The CI step still passes, the
+          underlying problem still goes unfixed. Make the step fail-fast
+          instead. The only legitimate use is annotating a step that runs
+          BEFORE the failing tool (e.g., advising on a deprecated config),
+          not wrapping the tool's own exit. See docs/runbooks/no-silent-failures.md.
+        language: pygrep
+        # Match the GitHub Actions advisory-annotation prefix. We exempt
+        # this workflow file (the lint rule itself contains the literal
+        # for self-documentation) via the `exclude` regex below.
+        entry: '::warning::'
+        files: ^\.github/workflows/.*\.ya?ml$
+        exclude: ^\.github/workflows/_required\.yml$
+        pass_filenames: true
+
   # ============================================================================
   # Git Commit Message Linting (Optional)
   # ============================================================================

scripts/run_benchmarks.sh

@@ -26,7 +26,16 @@ NC='\033[0m' # No Color
 
 # Configuration
 PROJECT_ROOT="$(git rev-parse --show-toplevel 2>/dev/null || pwd)"
-BUILD_DIR="${BUILD_DIR:-$PROJECT_ROOT/build/release/bin}"
+# BUILD_DIR is the CMake build directory; binaries land in $BUILD_DIR/bin
+# because CMakeLists.txt sets CMAKE_RUNTIME_OUTPUT_DIRECTORY to
+# "${CMAKE_BINARY_DIR}/bin". The Makefile uses $(BUILD_DIR)/$(BUILD_SUBDIR)
+# as its CMake binary dir, which defaults to build/x86.release
+# (BUILD_DIR=build, BUILD_SUBDIR=x86, CMAKE_BUILD_TYPE=Release -> .release).
+# Callers may override BUILD_DIR (full path to CMake binary dir),
+# BUILD_SUBDIR (suffix only), or BENCH_BIN_DIR (full path to binaries).
+: "${BUILD_SUBDIR:=x86.release}"
+BUILD_DIR="${BUILD_DIR:-$PROJECT_ROOT/build/$BUILD_SUBDIR}"
+BENCH_BIN_DIR="${BENCH_BIN_DIR:-$BUILD_DIR/bin}"
 BENCHMARK_DIR="${BENCHMARK_OUTPUT_DIR:-$PROJECT_ROOT/build/reports/benchmarks}"
 RESULTS_DIR="$BENCHMARK_DIR/results"
 mkdir -p "$RESULTS_DIR"
@@ -89,7 +98,7 @@ BENCHMARKS=(
 # Check that benchmarks exist
 missing=0
 for bench in "${BENCHMARKS[@]}"; do
-    if [[ ! -f "$BUILD_DIR/$bench" ]]; then
+    if [[ ! -f "$BENCH_BIN_DIR/$bench" ]]; then
         echo -e "${YELLOW}Warning: $bench not found, skipping${NC}"
         missing=$((missing + 1))
     fi
@@ -101,23 +110,31 @@ if [[ $missing -eq ${#BENCHMARKS[@]} ]]; then
     exit 1
 fi
 
-# Timestamp for results
+# Timestamp for results.
+# Exported so the embedded `python3 << EOF` heredocs below can read them via
+# os.environ.get(). Heredocs run as a subprocess of bash and only inherit
+# *exported* variables; without `export` the python sees None and emits
+# "No result files found matching None/*_None.json".
+export TIMESTAMP
 TIMESTAMP=$(date +%Y%m%d_%H%M%S)
+export RESULTS_DIR
+export RESULTS_FILE
 RESULTS_FILE="$RESULTS_DIR/results_$TIMESTAMP.json"
+export COMPARE_BASELINE
 
 echo -e "${YELLOW}Running benchmarks...${NC}"
 echo ""
 
 # Run each benchmark suite
 for bench in "${BENCHMARKS[@]}"; do
-    if [[ ! -f "$BUILD_DIR/$bench" ]]; then
+    if [[ ! -f "$BENCH_BIN_DIR/$bench" ]]; then
         continue
     fi
 
     echo -e "${BLUE}Running $bench...${NC}"
 
     # Build benchmark command
-    BENCH_CMD="$BUILD_DIR/$bench"
+    BENCH_CMD="$BENCH_BIN_DIR/$bench"
     BENCH_ARGS="--benchmark_out_format=json --benchmark_out=$RESULTS_DIR/${bench}_$TIMESTAMP.json"
 
     # Add filter if specified