diff --git a/.github/workflows/opentitan_regression.yaml b/.github/workflows/opentitan_regression.yaml index 4092242009685..466155fe3ff71 100644 --- a/.github/workflows/opentitan_regression.yaml +++ b/.github/workflows/opentitan_regression.yaml @@ -48,7 +48,9 @@ jobs: ninja -C build qemu-system-riscv32 - name: Run OpenTitan regressions + shell: bash run: | + set -o pipefail ./qemu/scripts/opentitan/run-bazel-tests.sh ./ qemu \ | tee test_results.txt diff --git a/docs/opentitan/regressions.md b/docs/opentitan/regressions.md new file mode 100644 index 0000000000000..7f475502a694f --- /dev/null +++ b/docs/opentitan/regressions.md @@ -0,0 +1,26 @@ +# OpenTitan regressions + +The `run-bazel-tests.sh` script and accompanying CI workflow allow us to run +regression tests against the OpenTitan repository. Currently only Earlgrey tests +are supported. + +With a checkout of OpenTitan, the script can be run like this: + +```sh +./scripts/opentitan/run-bazel-tests.sh path/to/opentitan path/to/qemu +``` + +The script will execute all QEMU-compatible tests using QEMU as it was built +at the given path. The test results will be compared against two lists checked +into this repository: + +* `scripts/opentitan/tests-passing.txt` +* `scripts/opentitan/tests-flaky.txt` + +All tests in `tests-passing.txt` are expected to pass. Failures indicate a +regression in either QEMU or Earlgrey. The script will fail if there is a +mismatch between the tests that we expect to pass and the actual results. + +Some tests may be flaky and pass or fail on different runs of the same QEMU +and OpenTitan checkouts. These tests can be added to the `tests-flaky.txt` list +to cause the script to ignore them. diff --git a/docs/opentitan/tools.md b/docs/opentitan/tools.md index 279deca9fba7b..0e7a318604862 100644 --- a/docs/opentitan/tools.md +++ b/docs/opentitan/tools.md @@ -57,6 +57,8 @@ of options and the available features. * `treillis/` directory contains the test application to test the [GPIO](gpio.md) device. * [`uartmux.py`](uartmux.md) is a tiny stream wrapper to help dealing with multiple QEMU output streams, typically multiple virtual UARTs. +* [`run-bazel-tests.sh`](regressions.md) is a script for running the OpenTitan test suite against + a local build of QEMU. ## Python modules diff --git a/scripts/opentitan/run-bazel-tests.sh b/scripts/opentitan/run-bazel-tests.sh index db20b6815e500..e7420c141a9e4 100755 --- a/scripts/opentitan/run-bazel-tests.sh +++ b/scripts/opentitan/run-bazel-tests.sh @@ -37,12 +37,6 @@ qemu_path="$(realpath "$qemu_path")" passing_tests_path="${qemu_path}/scripts/opentitan/tests-passing.txt" flaky_tests_path="${qemu_path}/scripts/opentitan/tests-flaky.txt" -# Check the test lists are sorted: -if ! sort --check "$passing_tests_path" || ! sort --check "$flaky_tests_path"; then - echo >&2 "ERROR: test lists must be sorted!" - exit 1 -fi - # Ensure QEMU has already been built in `./build`: if [ ! -x "${qemu_path}/build/qemu-system-riscv32" ]; then echo >&2 "ERROR: expected QEMU binary at '${qemu_path}/build/qemu-system-riscv32'" @@ -80,14 +74,14 @@ cd "$opentitan_path" >/dev/null ## COMPARE RESULTS -# Write the flaky test list to a flat file: -printf "%s\n" "$flaky_tests_path" | sort > "$flaky" +# Ensure the flaky tests are sorted with the current locale. +sort "$flaky_tests_path" > "$flaky" # Load the list of passing tests and strip flaky tests: -comm -23 "$passing_tests_path" "$flaky" > "$expected" +sort "$passing_tests_path" | comm -23 - "$flaky" > "$expected" # Find all the tests which passed in Bazel: -grep "PASSED" "$results" | cut -d' ' -f1 | sort > "$all_passed" +grep "PASSED[^:]" "$results" | cut -d' ' -f1 | sort > "$all_passed" # Filter out the flaky tests: comm -23 "$all_passed" "$flaky" > "$passed" diff --git a/scripts/opentitan/tests-flaky.txt b/scripts/opentitan/tests-flaky.txt index 50bbbefb17a35..02b9b7546f5c9 100644 --- a/scripts/opentitan/tests-flaky.txt +++ b/scripts/opentitan/tests-flaky.txt @@ -1,5 +1,15 @@ //sw/device/silicon_creator/lib/ownership:owner_verify_functest_sim_qemu_rom_with_fake_keys //sw/device/silicon_creator/lib/sigverify:mod_exp_ibex_functest_hardcoded_sim_qemu_rom_with_fake_keys +//sw/device/silicon_creator/lib/sigverify/sphincsplus/test:verify_test_kat1_sim_qemu_rom_with_fake_keys +//sw/device/silicon_creator/lib/sigverify/sphincsplus/test:verify_test_kat2_sim_qemu_rom_with_fake_keys +//sw/device/silicon_creator/lib/sigverify/sphincsplus/test:verify_test_kat3_sim_qemu_rom_with_fake_keys +//sw/device/silicon_creator/lib/sigverify/sphincsplus/test:verify_test_kat4_sim_qemu_rom_with_fake_keys +//sw/device/silicon_creator/lib/sigverify/sphincsplus/test:verify_test_kat5_sim_qemu_rom_with_fake_keys +//sw/device/silicon_creator/lib/sigverify/sphincsplus/test:verify_test_kat6_sim_qemu_rom_with_fake_keys +//sw/device/silicon_creator/lib/sigverify/sphincsplus/test:verify_test_kat7_sim_qemu_rom_with_fake_keys +//sw/device/silicon_creator/lib/sigverify/sphincsplus/test:verify_test_kat9_sim_qemu_rom_with_fake_keys //sw/device/tests/autogen:alert_test_sim_qemu_rom_with_fake_keys +//sw/device/tests/crypto:aes_gcm_timing_test_sim_qemu_rom_with_fake_keys +//sw/device/tests/crypto:rsa_2048_keygen_functest_sim_qemu_rom_with_fake_keys //sw/device/tests:edn_auto_mode_sim_qemu_rom_with_fake_keys //sw/device/tests:rv_core_ibex_rnd_test_sim_qemu_rom_with_fake_keys diff --git a/scripts/opentitan/tests-passing.txt b/scripts/opentitan/tests-passing.txt index 6bcbb8df40cd7..48bc7266d6382 100644 --- a/scripts/opentitan/tests-passing.txt +++ b/scripts/opentitan/tests-passing.txt @@ -11,6 +11,10 @@ //sw/device/silicon_creator/lib/drivers:rstmgr_functest_sim_qemu_rom_with_fake_keys //sw/device/silicon_creator/lib/drivers:uart_functest_sim_qemu_rom_with_fake_keys //sw/device/silicon_creator/lib/ownership:owner_verify_functest_sim_qemu_rom_with_fake_keys +//sw/device/silicon_creator/lib/sigverify:ecdsa_p256_verify_functest_sim_qemu_rom_with_fake_keys +//sw/device/silicon_creator/lib/sigverify:rsa_verify_functest_sim_qemu_rom_with_fake_keys +//sw/device/silicon_creator/lib/sigverify:sigverify_dynamic_functest_hardcoded_sim_qemu_rom_with_fake_keys +//sw/device/silicon_creator/lib/sigverify:sigverify_dynamic_functest_wycheproof_sim_qemu_rom_with_fake_keys //sw/device/silicon_creator/lib/sigverify/sphincsplus/test:fors_test_sim_qemu_rom_with_fake_keys //sw/device/silicon_creator/lib/sigverify/sphincsplus/test:mgf1_test_sim_qemu_rom_with_fake_keys //sw/device/silicon_creator/lib/sigverify/sphincsplus/test:thash_test_sim_qemu_rom_with_fake_keys @@ -26,11 +30,23 @@ //sw/device/silicon_creator/lib/sigverify/sphincsplus/test:verify_test_kat8_sim_qemu_rom_with_fake_keys //sw/device/silicon_creator/lib/sigverify/sphincsplus/test:verify_test_kat9_sim_qemu_rom_with_fake_keys //sw/device/silicon_creator/lib/sigverify/sphincsplus/test:wots_test_sim_qemu_rom_with_fake_keys -//sw/device/silicon_creator/lib/sigverify:ecdsa_p256_verify_functest_sim_qemu_rom_with_fake_keys -//sw/device/silicon_creator/lib/sigverify:rsa_verify_functest_sim_qemu_rom_with_fake_keys -//sw/device/silicon_creator/lib/sigverify:sigverify_dynamic_functest_hardcoded_sim_qemu_rom_with_fake_keys -//sw/device/silicon_creator/lib/sigverify:sigverify_dynamic_functest_wycheproof_sim_qemu_rom_with_fake_keys //sw/device/silicon_creator/lib/sigverify:spx_verify_functest_sim_qemu_rom_with_fake_keys +//sw/device/tests:aes_entropy_test_sim_qemu_rom_with_fake_keys +//sw/device/tests:aes_idle_test_sim_qemu_rom_with_fake_keys +//sw/device/tests:aes_smoketest_sim_qemu_rom_with_fake_keys +//sw/device/tests:alert_handler_lpg_clkoff_test_sim_qemu_rom_with_fake_keys +//sw/device/tests:alert_handler_lpg_reset_toggle_test_sim_qemu_rom_with_fake_keys +//sw/device/tests:alert_handler_ping_ok_test_sim_qemu_rom_with_fake_keys +//sw/device/tests:aon_timer_irq_test_sim_qemu_rom_with_fake_keys +//sw/device/tests:clkmgr_jitter_frequency_test_sim_qemu_rom_with_fake_keys +//sw/device/tests:clkmgr_jitter_test_sim_qemu_rom_with_fake_keys +//sw/device/tests:clkmgr_off_aes_trans_test_sim_qemu_rom_with_fake_keys +//sw/device/tests:clkmgr_off_hmac_trans_test_sim_qemu_rom_with_fake_keys +//sw/device/tests:clkmgr_off_kmac_trans_test_sim_qemu_rom_with_fake_keys +//sw/device/tests:clkmgr_off_otbn_trans_test_sim_qemu_rom_with_fake_keys +//sw/device/tests:clkmgr_smoketest_sim_qemu_rom_with_fake_keys +//sw/device/tests:coverage_test_sim_qemu_rom_with_fake_keys +//sw/device/tests:crt_test_sim_qemu_rom_with_fake_keys //sw/device/tests/crypto:aes_functest_sim_qemu_rom_with_fake_keys //sw/device/tests/crypto:aes_gcm_functest_sim_qemu_rom_with_fake_keys //sw/device/tests/crypto:aes_kwp_functest_sim_qemu_rom_with_fake_keys @@ -60,22 +76,6 @@ //sw/device/tests/crypto:sha384_functest_sim_qemu_rom_with_fake_keys //sw/device/tests/crypto:sha512_functest_sim_qemu_rom_with_fake_keys //sw/device/tests/crypto:symmetric_keygen_functest_sim_qemu_rom_with_fake_keys -//sw/device/tests:aes_entropy_test_sim_qemu_rom_with_fake_keys -//sw/device/tests:aes_idle_test_sim_qemu_rom_with_fake_keys -//sw/device/tests:aes_smoketest_sim_qemu_rom_with_fake_keys -//sw/device/tests:alert_handler_lpg_clkoff_test_sim_qemu_rom_with_fake_keys -//sw/device/tests:alert_handler_lpg_reset_toggle_test_sim_qemu_rom_with_fake_keys -//sw/device/tests:alert_handler_ping_ok_test_sim_qemu_rom_with_fake_keys -//sw/device/tests:aon_timer_irq_test_sim_qemu_rom_with_fake_keys -//sw/device/tests:clkmgr_jitter_frequency_test_sim_qemu_rom_with_fake_keys -//sw/device/tests:clkmgr_jitter_test_sim_qemu_rom_with_fake_keys -//sw/device/tests:clkmgr_off_aes_trans_test_sim_qemu_rom_with_fake_keys -//sw/device/tests:clkmgr_off_hmac_trans_test_sim_qemu_rom_with_fake_keys -//sw/device/tests:clkmgr_off_kmac_trans_test_sim_qemu_rom_with_fake_keys -//sw/device/tests:clkmgr_off_otbn_trans_test_sim_qemu_rom_with_fake_keys -//sw/device/tests:clkmgr_smoketest_sim_qemu_rom_with_fake_keys -//sw/device/tests:coverage_test_sim_qemu_rom_with_fake_keys -//sw/device/tests:crt_test_sim_qemu_rom_with_fake_keys //sw/device/tests:csrng_kat_test_sim_qemu_rom_with_fake_keys //sw/device/tests:csrng_smoketest_sim_qemu_rom_with_fake_keys //sw/device/tests:edn_sw_mode_sim_qemu_rom_with_fake_keys @@ -131,19 +131,19 @@ //third_party/riscv-compliance:C-J_sim_qemu_rom_with_fake_keys //third_party/riscv-compliance:C-LI_sim_qemu_rom_with_fake_keys //third_party/riscv-compliance:C-LUI_sim_qemu_rom_with_fake_keys -//third_party/riscv-compliance:C-LWSP_sim_qemu_rom_with_fake_keys //third_party/riscv-compliance:C-LW_sim_qemu_rom_with_fake_keys +//third_party/riscv-compliance:C-LWSP_sim_qemu_rom_with_fake_keys //third_party/riscv-compliance:C-MV_sim_qemu_rom_with_fake_keys //third_party/riscv-compliance:C-OR_sim_qemu_rom_with_fake_keys //third_party/riscv-compliance:C-SLLI_sim_qemu_rom_with_fake_keys //third_party/riscv-compliance:C-SRAI_sim_qemu_rom_with_fake_keys //third_party/riscv-compliance:C-SRLI_sim_qemu_rom_with_fake_keys //third_party/riscv-compliance:C-SUB_sim_qemu_rom_with_fake_keys -//third_party/riscv-compliance:C-SWSP_sim_qemu_rom_with_fake_keys //third_party/riscv-compliance:C-SW_sim_qemu_rom_with_fake_keys +//third_party/riscv-compliance:C-SWSP_sim_qemu_rom_with_fake_keys //third_party/riscv-compliance:C-XOR_sim_qemu_rom_with_fake_keys -//third_party/riscv-compliance:DIVU_sim_qemu_rom_with_fake_keys //third_party/riscv-compliance:DIV_sim_qemu_rom_with_fake_keys +//third_party/riscv-compliance:DIVU_sim_qemu_rom_with_fake_keys //third_party/riscv-compliance:I-ADD-01_sim_qemu_rom_with_fake_keys //third_party/riscv-compliance:I-ADDI-01_sim_qemu_rom_with_fake_keys //third_party/riscv-compliance:I-AND-01_sim_qemu_rom_with_fake_keys @@ -192,9 +192,9 @@ //third_party/riscv-compliance:I-SW-01_sim_qemu_rom_with_fake_keys //third_party/riscv-compliance:I-XOR-01_sim_qemu_rom_with_fake_keys //third_party/riscv-compliance:I-XORI-01_sim_qemu_rom_with_fake_keys +//third_party/riscv-compliance:MULH_sim_qemu_rom_with_fake_keys //third_party/riscv-compliance:MULHSU_sim_qemu_rom_with_fake_keys //third_party/riscv-compliance:MULHU_sim_qemu_rom_with_fake_keys -//third_party/riscv-compliance:MULH_sim_qemu_rom_with_fake_keys //third_party/riscv-compliance:MUL_sim_qemu_rom_with_fake_keys -//third_party/riscv-compliance:REMU_sim_qemu_rom_with_fake_keys //third_party/riscv-compliance:REM_sim_qemu_rom_with_fake_keys +//third_party/riscv-compliance:REMU_sim_qemu_rom_with_fake_keys