Add Earlgrey OpenTitan regression flow

.github/workflows/opentitan_regression.yaml

@@ -0,0 +1,60 @@
+#------------------------------------------------------------------------------
+# OpenTitan regression runner
+#
+# Copyright (c) 2025 lowRISC CIC
+# SPDX-License-Identifier: Apache License 2.0
+#------------------------------------------------------------------------------
+
+name: OpenTitan Regression
+on:
+  pull_request:
+  workflow_dispatch:
+    inputs:
+      opentitan_ref:
+        description: Branch, tag, or commit ref from OpenTitan to test
+        required: true
+        default: earlgrey_1.0.0
+        type: string
+
+jobs:
+  earlgrey:
+    name: Earlgrey
+    runs-on: ubuntu-22.04
+    steps:
+      - name: Checkout OpenTitan
+        uses: actions/checkout@v5
+        with:
+          repository: lowRISC/opentitan
+          ref: ${{ inputs.opentitan_ref || 'earlgrey_1.0.0' }}
+
+      - name: Prepare OpenTitan environment
+        uses: ./.github/actions/prepare-env
+
+      - name: Checkout QEMU
+        uses: actions/checkout@v5
+        with:
+          path: qemu
+
+      - name: Build QEMU
+        working-directory: qemu
+        run: |
+          ./configure \
+            --target-list=riscv32-softmmu \
+            --without-default-features \
+            --enable-tcg \
+            --disable-tools \
+            --enable-trace-backends=log
+
+          ninja -C build qemu-system-riscv32
+
+      - name: Run OpenTitan regressions
+        run: |
+          ./qemu/scripts/opentitan/run-bazel-tests.sh ./ qemu \
+            | tee test_results.txt
+
+      - name: Upload Bazel test results
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: bazel-test-results
+          path: test_results.txt

scripts/opentitan/run-bazel-tests.sh

@@ -0,0 +1,123 @@
+#!/usr/bin/env sh
+# Copyright (c) 2025 lowRISC contributors.
+
+set -e
+
+# This script will run all QEMU tests in the provided OpenTitan repository,
+# comparing the results with a list of expected passes.
+#
+# USAGE: run-bazel-tests.sh path/to/opentitan/repo path/to/qemu/repo
+#
+# There are two companion files in this directory read by this script:
+#
+# * `tests-passing.txt`
+# * `tests-flaky.txt`
+# 
+# The idea is to keep the list of passing tests up to date as QEMU changes.
+#
+# * If a test starts passing, add it to `tests-passing.txt` to catch
+#   regressions.
+# * When a test starts failing, investigate why and fix it, or remove it
+#   from the `tests-passing.txt` list if necessary.
+#
+# Tests which are "flaky" and may pass/fail randomly will still be run,
+# but can be recorded below to prevent them from being checked.
+
+# CLI arguments:
+opentitan_path="$1"
+qemu_path="$2"
+if [ ! -d "$opentitan_path" ] || [ ! -d "$qemu_path" ]; then
+  echo "USAGE: ${0} <OPENTITAN REPO> <QEMU REPO>"
+  exit 1
+fi
+opentitan_path="$(realpath "$opentitan_path")"
+qemu_path="$(realpath "$qemu_path")"
+
+# Lists of passing and flaky OpenTitan tests:
+passing_tests_path="${qemu_path}/scripts/opentitan/tests-passing.txt"
+flaky_tests_path="${qemu_path}/scripts/opentitan/tests-flaky.txt"
+
+# Check the test lists are sorted:
+if ! sort --check "$passing_tests_path" || ! sort --check "$flaky_tests_path"; then
+  echo >&2 "ERROR: test lists must be sorted!"
+  exit 1
+fi
+
+# Ensure QEMU has already been built in `./build`:
+if [ ! -x "${qemu_path}/build/qemu-system-riscv32" ]; then
+  echo >&2 "ERROR: expected QEMU binary at '${qemu_path}/build/qemu-system-riscv32'"
+  exit 1
+fi
+
+# Temporary files used by this script:
+results="$(mktemp)"
+flaky="$(mktemp)"
+expected="$(mktemp)"
+all_passed="$(mktemp)"
+passed="$(mktemp)"
+cleanup() {
+  rm -f "${qemu_path}/REPO.bazel" "${qemu_path}/BUILD"
+  rm -f "$results" "$flaky" "$expected" "$all_passed" "$passed" 
+}
+trap "cleanup" EXIT
+
+## Add temporary `REPO.bazel` and `BUILD` files from Bazel:
+touch "${qemu_path}/REPO.bazel"
+ln -s                                                             \
+  "${opentitan_path}/third_party/qemu/BUILD.qemu_opentitan.bazel" \
+  "${qemu_path}/BUILD"
+
+## RUN BAZEL TESTS
+cd "$opentitan_path" >/dev/null
+
+./bazelisk.sh test //...                                    \
+  --test_tag_filters="qemu"                                 \
+  --test_summary="short"                                    \
+  --test_output=all                                         \
+  --override_repository="+qemu+qemu_opentitan=${qemu_path}" \
+  --build_tests_only                                        \
+  | tee "$results"
+
+## COMPARE RESULTS
+
+# Write the flaky test list to a flat file:
+printf "%s\n" "$flaky_tests_path" | sort > "$flaky"
+
+# Load the list of passing tests and strip flaky tests:
+comm -23 "$passing_tests_path" "$flaky" > "$expected"
+
+# Find all the tests which passed in Bazel:
+grep "PASSED" "$results" | cut -d' ' -f1 | sort > "$all_passed"
+
+# Filter out the flaky tests:
+comm -23 "$all_passed" "$flaky" > "$passed"
+
+## REPORT MISMATCHES
+
+unexpected_failures="$(comm -13 "$passed" "$expected")"
+unexpected_passes="$(comm -23 "$passed" "$expected")"
+
+status=0
+
+if [ -n "$unexpected_failures" ]; then
+  echo >&2
+  echo >&2 Tests that we expected to pass which did NOT pass:
+  echo "$unexpected_failures" | awk '$0="  "$0' >&2
+  status=1
+fi
+
+if [ -n "$unexpected_passes" ]; then
+  echo >&2
+  echo >&2 Tests which passed but we did NOT expect them to pass:
+  echo "$unexpected_passes" | awk '$0="  "$0' >&2
+  status=1
+fi
+
+# Print helpful errors in CI.
+if [ $status -ne 0 ] && [ -n "$CI" ]; then
+  echo >&2
+  echo >&2 "::error::Bazel test results did not match the list of expected passing tests."
+  echo >&2 "::error::Consider fixing the tests or updating the test list in ${0}."
+fi
+
+exit $status

scripts/opentitan/tests-flaky.txt

@@ -0,0 +1,5 @@
+//sw/device/silicon_creator/lib/ownership:owner_verify_functest_sim_qemu_rom_with_fake_keys
+//sw/device/silicon_creator/lib/sigverify:mod_exp_ibex_functest_hardcoded_sim_qemu_rom_with_fake_keys
+//sw/device/tests/autogen:alert_test_sim_qemu_rom_with_fake_keys
+//sw/device/tests:edn_auto_mode_sim_qemu_rom_with_fake_keys
+//sw/device/tests:rv_core_ibex_rnd_test_sim_qemu_rom_with_fake_keys