ci: Update Earlgrey test lists & add new regression run

.github/workflows/ci_regression.yml

@@ -31,6 +31,16 @@ jobs:
       opentitan_repo: ${{ inputs.opentitan_repo || 'lowRISC/opentitan' }}
       opentitan_ref: ${{ inputs.opentitan_ref || 'earlgrey_1.0.0' }}
 
+  earlgrey_rom_ext:
+    name: ROM_EXT
+    uses: ./.github/workflows/eg_regression.yml
+    secrets: inherit
+    with:
+      exec_env: sim_qemu_rom_ext
+      test_timeout: 150
+      opentitan_repo: ${{ inputs.opentitan_repo || 'lowRISC/opentitan' }}
+      opentitan_ref: ${{ inputs.opentitan_ref || 'earlgrey_1.0.0' }}
+
   earlgrey_sival_rom_ext:
     name: SiVal ROM_EXT
     uses: ./.github/workflows/eg_regression.yml

.github/workflows/eg_regression.yml

@@ -14,6 +14,10 @@ on:
         required: true
         default: qemu  # Default tag, captures all QEMU exec envs in OpenTitan
         type: string
+      test_timeout:
+        description: The maximum timeout in seconds to use per executed test
+        required: false
+        type: string
       opentitan_repo:
         description: OpenTitan repository to be checked out
         required: true
@@ -62,6 +66,7 @@ jobs:
           set -o pipefail
           ./qemu/scripts/opentitan/run-bazel-tests.sh ./ qemu \
             ${{ inputs.exec_env }} \
+            ${{ inputs.test_timeout }} \
             | tee ${{ inputs.exec_env }}_test_results.txt
 
       - name: Upload Bazel test results

docs/opentitan/regressions.md

@@ -7,20 +7,32 @@ are supported.
 With a checkout of OpenTitan, the script can be run like this:
 
 ```sh
-./scripts/opentitan/run-bazel-tests.sh path/to/opentitan path/to/qemu
+./scripts/opentitan/run-bazel-tests.sh path/to/opentitan path/to/qemu \
+    [execution_environment] [timeout]
 ```
 
 The script will execute all QEMU-compatible tests using QEMU as it was built
-at the given path. The test results will be compared against two lists checked
-into this repository:
+at the given path. The test results will be compared against a list checked
+into this repository at `tests/opentitan/data/earlgrey-tests.txt`.
 
-* `scripts/opentitan/tests-passing.txt`
-* `scripts/opentitan/tests-flaky.txt`
-
-All tests in `tests-passing.txt` are expected to pass. Failures indicate a
+Each test occupies its own line, prefixed by the expected status.
+Tests with a `pass` status are expected to pass - failures indicate a
 regression in either QEMU or Earlgrey. The script will fail if there is a
 mismatch between the tests that we expect to pass and the actual results.
 
 Some tests may be flaky and pass or fail on different runs of the same QEMU
-and OpenTitan checkouts. These tests can be added to the `tests-flaky.txt` list
-to cause the script to ignore them.
+and OpenTitan checkouts. These tests can be prefixed with the `flaky` status
+to cause the script to ignore them. This will suppress these tests from being
+printed as unexpected passes when they do succeed, but will also stop their
+failures from causing the script to fail.
+
+After each test, a comment can be provided (starting with a `#`), which can
+help log the reasons that tests might have become flaky.
+
+Specifying an execution environment when running the script will restrict to
+only running and comparing against tests for that execution environment.
+This can be used for more granular testing to break down large test workloads.
+If you do not specify an execution environment, _all_ available QEMU tests
+will be executed. When specifying an execution environment, a maximum test
+timeout (in seconds) to use for that environment can also be provided as
+a subsequent argument to the script.

scripts/opentitan/run-bazel-tests.sh

@@ -6,22 +6,21 @@ set -e
 # This script will run all QEMU tests in the provided OpenTitan repository,
 # comparing the results with a list of expected passes.
 #
-# USAGE: run-bazel-tests.sh path/to/opentitan/repo path/to/qemu/repo
+# USAGE: run-bazel-tests.sh path/to/opentitan/repo path/to/qemu/repo \
+#            [execution environment] [test_timeout]
 #
-# There are two companion files in this directory read by this script:
-#
-# * `tests-passing.txt`
-# * `tests-flaky.txt`
+# There is a companion file `tests/opentitan/data/earlgrey-tests.txt` that
+# is read by this script.
 # 
 # The idea is to keep the list of passing tests up to date as QEMU changes.
 #
-# * If a test starts passing, add it to `tests-passing.txt` to catch
-#   regressions.
+# * If a test starts passing, add it with a `pass` status to the test
+#   list to catch regressions.
 # * When a test starts failing, investigate why and fix it, or remove it
-#   from the `tests-passing.txt` list if necessary.
+#   from the test list if necessary.
 #
-# Tests which are "flaky" and may pass/fail randomly will still be run,
-# but can be recorded below to prevent them from being checked.
+# Tests which are marked `flaky` and may pass/fail randomly will still be run,
+# but can be marked as such to prevent them from causing warnings or failures.
 
 # CI-only job summary feature - write to `/dev/null` when run locally.
 GITHUB_STEP_SUMMARY="${GITHUB_STEP_SUMMARY:-/dev/null}"
@@ -30,13 +29,14 @@ GITHUB_STEP_SUMMARY="${GITHUB_STEP_SUMMARY:-/dev/null}"
 opentitan_path="$1"
 qemu_path="$2"
 qemu_exec_env="$3"
+test_timeout="$4"
 if [ ! -d "$opentitan_path" ] || [ ! -d "$qemu_path" ]; then
-  echo "USAGE: ${0} <OPENTITAN REPO> <QEMU REPO> [<EXEC ENV>]"
+  echo "USAGE: ${0} <OPENTITAN REPO> <QEMU REPO> [<EXEC ENV>] [<TIMEOUT>]"
   exit 1
 fi
 opentitan_path="$(realpath "$opentitan_path")"
 qemu_path="$(realpath "$qemu_path")"
-if [ ! "$qemu_exec_env" ]; then
+if [ -z "$qemu_exec_env" ]; then
   # catch-all tag for all QEMU exec envs
   qemu_exec_env="qemu"
   echo "Using default 'qemu' tag to test all exec envs"
@@ -87,12 +87,18 @@ trap "cleanup" EXIT
 ## RUN BAZEL TESTS
 cd "$opentitan_path" >/dev/null
 
+test_args=""
+if [ -n "$test_timeout" ]; then
+  test_args="--test_timeout=$test_timeout"
+fi
+
 ./bazelisk.sh test //...                                    \
   --test_tag_filters="$qemu_exec_env"                       \
   --test_summary="short"                                    \
   --test_output=all                                         \
   --override_repository="+qemu+qemu_opentitan=${qemu_path}" \
   --build_tests_only                                        \
+  $test_args                                                \
   | tee "$results"
 
 ## COMPARE RESULTS