From 09c10bbbd3aa4c5d9b8214a6afb96834b7918024 Mon Sep 17 00:00:00 2001 From: Alex Jones Date: Wed, 22 Oct 2025 13:57:16 +0100 Subject: [PATCH 1/8] [ot] tests/opentitan: earlgrey-tests: Remove RSA test The `rsa_3072_verify` test was removed from Upstream OpenTitan Earlgrey due to being obsolete, but was previously expected to be passing here. Remove it from the list of passing tests. Signed-off-by: Alex Jones --- tests/opentitan/data/earlgrey-tests.txt | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/opentitan/data/earlgrey-tests.txt b/tests/opentitan/data/earlgrey-tests.txt index d1e331509b448..be20efb1f4523 100644 --- a/tests/opentitan/data/earlgrey-tests.txt +++ b/tests/opentitan/data/earlgrey-tests.txt @@ -170,8 +170,6 @@ pass: //sw/device/tests/crypto:rsa_3072_encryption_functest_sim_qemu_rom_with_fa pass: //sw/device/tests/crypto:rsa_3072_encryption_functest_sim_qemu_sival_rom_ext pass: //sw/device/tests/crypto:rsa_3072_signature_functest_sim_qemu_rom_with_fake_keys pass: //sw/device/tests/crypto:rsa_3072_signature_functest_sim_qemu_sival_rom_ext -pass: //sw/device/tests/crypto:rsa_3072_verify_functest_hardcoded_sim_qemu_rom_with_fake_keys -pass: //sw/device/tests/crypto:rsa_3072_verify_functest_hardcoded_sim_qemu_sival_rom_ext flaky: //sw/device/tests/crypto:rsa_4096_encryption_functest_sim_qemu_rom_with_fake_keys # Fails timing (max cycle count) check ~25% of the time flaky: //sw/device/tests/crypto:rsa_4096_encryption_functest_sim_qemu_sival_rom_ext # Fails timing (max cycle count) check ~25% of the time flaky: //sw/device/tests/crypto:rsa_4096_signature_functest_sim_qemu_rom_with_fake_keys # Fails timing (max cycle count) check ~25% of the time From f843ee1a6c410afd950442f6424c8044289c636e Mon Sep 17 00:00:00 2001 From: Alex Jones Date: Wed, 22 Oct 2025 19:34:12 +0100 Subject: [PATCH 2/8] [ot] tests/opentitan: earlgrey-tests: Mark sigverify test flaky We expect this test to pass, but recent upstream changes increase the time to emulate this test by a lot and it will often take 2-5+ minutes to execute depending on host performance and load. Mark it flaky instead of removing it because it could theoretically pass if the runner is fast enough, but it is not expected to pass with the standard 60 second timeout we give. Signed-off-by: Alex Jones --- tests/opentitan/data/earlgrey-tests.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/opentitan/data/earlgrey-tests.txt b/tests/opentitan/data/earlgrey-tests.txt index be20efb1f4523..e6a1a0bc294a0 100644 --- a/tests/opentitan/data/earlgrey-tests.txt +++ b/tests/opentitan/data/earlgrey-tests.txt @@ -33,8 +33,8 @@ pass: //sw/device/silicon_creator/lib/sigverify:rsa_verify_functest_sim_qemu_rom pass: //sw/device/silicon_creator/lib/sigverify:rsa_verify_functest_sim_qemu_sival_rom_ext pass: //sw/device/silicon_creator/lib/sigverify:sigverify_dynamic_functest_hardcoded_sim_qemu_rom_with_fake_keys pass: //sw/device/silicon_creator/lib/sigverify:sigverify_dynamic_functest_hardcoded_sim_qemu_sival_rom_ext -pass: //sw/device/silicon_creator/lib/sigverify:sigverify_dynamic_functest_wycheproof_sim_qemu_rom_with_fake_keys -pass: //sw/device/silicon_creator/lib/sigverify:sigverify_dynamic_functest_wycheproof_sim_qemu_sival_rom_ext +flaky: //sw/device/silicon_creator/lib/sigverify:sigverify_dynamic_functest_wycheproof_sim_qemu_rom_with_fake_keys # We expect this to pass, but it takes takes >> 60 seconds to run +flaky: //sw/device/silicon_creator/lib/sigverify:sigverify_dynamic_functest_wycheproof_sim_qemu_sival_rom_ext # We expect this to pass, but it takes takes >> 60 seconds to run pass: //sw/device/silicon_creator/lib/sigverify/sphincsplus/test:fors_test_sim_qemu_rom_with_fake_keys pass: //sw/device/silicon_creator/lib/sigverify/sphincsplus/test:fors_test_sim_qemu_sival_rom_ext pass: //sw/device/silicon_creator/lib/sigverify/sphincsplus/test:mgf1_test_sim_qemu_rom_with_fake_keys From a7448dcc13f60af5cab72953fddd9724e627b958 Mon Sep 17 00:00:00 2001 From: Alex Jones Date: Wed, 22 Oct 2025 16:08:05 +0100 Subject: [PATCH 3/8] [ot] tests/opentitan: earlgrey-tests: Add common flaky tests Add a couple more OpenTitan tests that are commonly seen as being flaky in the logs (previously were not expected to pass). Signed-off-by: Alex Jones --- tests/opentitan/data/earlgrey-tests.txt | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/opentitan/data/earlgrey-tests.txt b/tests/opentitan/data/earlgrey-tests.txt index e6a1a0bc294a0..8dc411429d13e 100644 --- a/tests/opentitan/data/earlgrey-tests.txt +++ b/tests/opentitan/data/earlgrey-tests.txt @@ -270,6 +270,8 @@ pass: //sw/device/tests:pwrmgr_sleep_disabled_test_sim_qemu_rom_with_fake_keys pass: //sw/device/tests:pwrmgr_sleep_disabled_test_sim_qemu_sival_rom_ext pass: //sw/device/tests:pwrmgr_wdog_reset_reqs_test_sim_qemu_rom_with_fake_keys pass: //sw/device/tests:pwrmgr_wdog_reset_reqs_test_sim_qemu_sival_rom_ext +flaky: //sw/device/tests/qemu:i2c_qemu_target_device_test_sim_qemu_rom_with_fake_keys # Fails occassionally when running in parallel on a slow host, may be a timing issue. +flaky: //sw/device/tests/qemu:i2c_qemu_target_device_test_sim_qemu_sival_rom_ext # Fails occassionally when running in parallel on a slow host, may be a timing issue. flaky: //sw/device/tests/qemu:i2c_qemu_test_sim_qemu_rom_with_fake_keys # Fails ~10% of the time when running in parallel. Root cause unknown. pass: //sw/device/tests:rom_exit_immediately_sim_qemu_base pass: //sw/device/tests:rstmgr_smoketest_sim_qemu_rom_with_fake_keys @@ -278,6 +280,8 @@ pass: //sw/device/tests:rstmgr_sw_req_test_sim_qemu_rom_with_fake_keys pass: //sw/device/tests:rstmgr_sw_req_test_sim_qemu_sival_rom_ext pass: //sw/device/tests:rv_core_ibex_mem_test_prod_sim_qemu_rom_with_fake_keys pass: //sw/device/tests:rv_core_ibex_mem_test_prod_sim_qemu_sival_rom_ext +flaky: //sw/device/tests:rv_core_ibex_rnd_test_sim_qemu_rom_with_fake_keys # Passes and fails randomly, likely due to the test using randomness. +flaky: //sw/device/tests:rv_core_ibex_rnd_test_sim_qemu_sival_rom_ext # Passes and fails randomly, likely due to the test using randomness. pass: //sw/device/tests:rv_timer_smoketest_sim_qemu_rom_with_fake_keys pass: //sw/device/tests:rv_timer_smoketest_sim_qemu_sival_rom_ext pass: //sw/device/tests:spi_device_flash_smoketest_sim_qemu_rom_with_fake_keys From a340c12d6255948da956342c0af0f642a62c7152 Mon Sep 17 00:00:00 2001 From: Alex Jones Date: Wed, 22 Oct 2025 14:08:24 +0100 Subject: [PATCH 4/8] [ot] docs/opentitan: regression: Update regression documentation When the script was extended to support specifying different execution environments and to collect all the Earlgrey test statuses together in one place, I forgot to update the accompanying documentation. Make sure that is up to date with this commit. Signed-off-by: Alex Jones --- docs/opentitan/regressions.md | 28 +++++++++++++++++++--------- scripts/opentitan/run-bazel-tests.sh | 19 +++++++++---------- 2 files changed, 28 insertions(+), 19 deletions(-) diff --git a/docs/opentitan/regressions.md b/docs/opentitan/regressions.md index 7f475502a694f..467940a5ff270 100644 --- a/docs/opentitan/regressions.md +++ b/docs/opentitan/regressions.md @@ -7,20 +7,30 @@ are supported. With a checkout of OpenTitan, the script can be run like this: ```sh -./scripts/opentitan/run-bazel-tests.sh path/to/opentitan path/to/qemu +./scripts/opentitan/run-bazel-tests.sh path/to/opentitan path/to/qemu \ + [execution_environment] ``` The script will execute all QEMU-compatible tests using QEMU as it was built -at the given path. The test results will be compared against two lists checked -into this repository: +at the given path. The test results will be compared against a list checked +into this repository at `tests/opentitan/data/earlgrey-tests.txt`. -* `scripts/opentitan/tests-passing.txt` -* `scripts/opentitan/tests-flaky.txt` - -All tests in `tests-passing.txt` are expected to pass. Failures indicate a +Each test occupies its own line, prefixed by the expected status. +Tests with a `pass` status are expected to pass - failures indicate a regression in either QEMU or Earlgrey. The script will fail if there is a mismatch between the tests that we expect to pass and the actual results. Some tests may be flaky and pass or fail on different runs of the same QEMU -and OpenTitan checkouts. These tests can be added to the `tests-flaky.txt` list -to cause the script to ignore them. +and OpenTitan checkouts. These tests can be prefixed with the `flaky` status +to cause the script to ignore them. This will suppress these tests from being +printed as unexpected passes when they do succeed, but will also stop their +failures from causing the script to fail. + +After each test, a comment can be provided (starting with a `#`), which can +help log the reasons that tests might have become flaky. + +Specifying an execution environment when running the script will restrict to +only running and comparing against tests for that execution environment. +This can be used for more granular testing to break down large test workloads. +If you do not specify an execution environment, _all_ available QEMU tests +will be executed. diff --git a/scripts/opentitan/run-bazel-tests.sh b/scripts/opentitan/run-bazel-tests.sh index ae76072ba0d79..b343bbaad806d 100755 --- a/scripts/opentitan/run-bazel-tests.sh +++ b/scripts/opentitan/run-bazel-tests.sh @@ -6,22 +6,21 @@ set -e # This script will run all QEMU tests in the provided OpenTitan repository, # comparing the results with a list of expected passes. # -# USAGE: run-bazel-tests.sh path/to/opentitan/repo path/to/qemu/repo +# USAGE: run-bazel-tests.sh path/to/opentitan/repo path/to/qemu/repo \ +# [execution environment] # -# There are two companion files in this directory read by this script: -# -# * `tests-passing.txt` -# * `tests-flaky.txt` +# There is a companion file `tests/opentitan/data/earlgrey-tests.txt` that +# is read by this script. # # The idea is to keep the list of passing tests up to date as QEMU changes. # -# * If a test starts passing, add it to `tests-passing.txt` to catch -# regressions. +# * If a test starts passing, add it with a `pass` status to the test +# list to catch regressions. # * When a test starts failing, investigate why and fix it, or remove it -# from the `tests-passing.txt` list if necessary. +# from the test list if necessary. # -# Tests which are "flaky" and may pass/fail randomly will still be run, -# but can be recorded below to prevent them from being checked. +# Tests which are marked `flaky` and may pass/fail randomly will still be run, +# but can be marked as such to prevent them from causing warnings or failures. # CI-only job summary feature - write to `/dev/null` when run locally. GITHUB_STEP_SUMMARY="${GITHUB_STEP_SUMMARY:-/dev/null}" From 8f88abd930b6e160762491e388e62e05c708da68 Mon Sep 17 00:00:00 2001 From: Alex Jones Date: Wed, 22 Oct 2025 14:09:07 +0100 Subject: [PATCH 5/8] [ot] tests/opentitan: earlgrey-tests: Add newly passing E2E ROM tests With appropriate upstream OpenTitan support, we now expect a large amount of ROM end-to-end tests to pass in QEMU. Add all of these tests to the test list and mark them as expected passes to catch regressions. Signed-off-by: Alex Jones --- tests/opentitan/data/earlgrey-tests.txt | 153 ++++++++++++++++++++++++ 1 file changed, 153 insertions(+) diff --git a/tests/opentitan/data/earlgrey-tests.txt b/tests/opentitan/data/earlgrey-tests.txt index 8dc411429d13e..60cd39b81e661 100644 --- a/tests/opentitan/data/earlgrey-tests.txt +++ b/tests/opentitan/data/earlgrey-tests.txt @@ -73,6 +73,159 @@ flaky: //sw/device/silicon_creator/lib:irq_asm_functest_sim_qemu_sival_rom_ext pass: //sw/device/silicon_creator/rom/e2e/keymgr:rom_e2e_keymgr_init_otp_invalid_meas_sim_qemu_rom_with_fake_keys pass: //sw/device/silicon_creator/rom/e2e/keymgr:rom_e2e_keymgr_init_otp_meas_sim_qemu_rom_with_fake_keys pass: //sw/device/silicon_creator/rom/e2e/keymgr:rom_e2e_keymgr_init_otp_no_meas_sim_qemu_rom_with_fake_keys +pass: //sw/device/silicon_creator/rom/e2e/sigverify_always:sigverify_always_dev_a_bad_b_bad_sim_qemu_rom_with_fake_keys +pass: //sw/device/silicon_creator/rom/e2e/sigverify_always:sigverify_always_dev_a_bad_b_nothing_sim_qemu_rom_with_fake_keys +pass: //sw/device/silicon_creator/rom/e2e/sigverify_always:sigverify_always_dev_a_nothing_b_bad_sim_qemu_rom_with_fake_keys +pass: //sw/device/silicon_creator/rom/e2e/sigverify_always:sigverify_always_dev_a_nothing_b_nothing_sim_qemu_rom_with_fake_keys +pass: //sw/device/silicon_creator/rom/e2e/sigverify_always:sigverify_always_prod_a_bad_b_bad_sim_qemu_rom_with_fake_keys +pass: //sw/device/silicon_creator/rom/e2e/sigverify_always:sigverify_always_prod_a_bad_b_nothing_sim_qemu_rom_with_fake_keys +pass: //sw/device/silicon_creator/rom/e2e/sigverify_always:sigverify_always_prod_a_nothing_b_bad_sim_qemu_rom_with_fake_keys +pass: //sw/device/silicon_creator/rom/e2e/sigverify_always:sigverify_always_prod_a_nothing_b_nothing_sim_qemu_rom_with_fake_keys +pass: //sw/device/silicon_creator/rom/e2e/sigverify_always:sigverify_always_prod_end_a_bad_b_bad_sim_qemu_rom_with_fake_keys +pass: //sw/device/silicon_creator/rom/e2e/sigverify_always:sigverify_always_prod_end_a_bad_b_nothing_sim_qemu_rom_with_fake_keys +pass: //sw/device/silicon_creator/rom/e2e/sigverify_always:sigverify_always_prod_end_a_nothing_b_bad_sim_qemu_rom_with_fake_keys +pass: //sw/device/silicon_creator/rom/e2e/sigverify_always:sigverify_always_prod_end_a_nothing_b_nothing_sim_qemu_rom_with_fake_keys +pass: //sw/device/silicon_creator/rom/e2e/sigverify_always:sigverify_always_rma_a_bad_b_bad_sim_qemu_rom_with_fake_keys +pass: //sw/device/silicon_creator/rom/e2e/sigverify_always:sigverify_always_rma_a_bad_b_nothing_sim_qemu_rom_with_fake_keys +pass: //sw/device/silicon_creator/rom/e2e/sigverify_always:sigverify_always_rma_a_nothing_b_bad_sim_qemu_rom_with_fake_keys +pass: //sw/device/silicon_creator/rom/e2e/sigverify_always:sigverify_always_rma_a_nothing_b_nothing_sim_qemu_rom_with_fake_keys +pass: //sw/device/silicon_creator/rom/e2e/sigverify_always:sigverify_always_test_unlocked0_a_bad_b_bad_sim_qemu_rom_with_fake_keys +pass: //sw/device/silicon_creator/rom/e2e/sigverify_always:sigverify_always_test_unlocked0_a_bad_b_nothing_sim_qemu_rom_with_fake_keys +pass: //sw/device/silicon_creator/rom/e2e/sigverify_always:sigverify_always_test_unlocked0_a_nothing_b_bad_sim_qemu_rom_with_fake_keys +pass: //sw/device/silicon_creator/rom/e2e/sigverify_always:sigverify_always_test_unlocked0_a_nothing_b_nothing_sim_qemu_rom_with_fake_keys +pass: //sw/device/silicon_creator/rom/e2e/sigverify_key_type:sigverify_key_type_dev_ecdsa_dev_key_0_sim_qemu_rom_with_fake_keys +pass: //sw/device/silicon_creator/rom/e2e/sigverify_key_type:sigverify_key_type_dev_ecdsa_prod_key_0_sim_qemu_rom_with_fake_keys +pass: //sw/device/silicon_creator/rom/e2e/sigverify_key_type:sigverify_key_type_dev_ecdsa_test_key_0_sim_qemu_rom_with_fake_keys +pass: //sw/device/silicon_creator/rom/e2e/sigverify_key_type:sigverify_key_type_dev_spx_dev_key_0_sim_qemu_rom_with_fake_keys +pass: //sw/device/silicon_creator/rom/e2e/sigverify_key_type:sigverify_key_type_dev_spx_prod_key_0_sim_qemu_rom_with_fake_keys +pass: //sw/device/silicon_creator/rom/e2e/sigverify_key_type:sigverify_key_type_dev_spx_test_key_0_sim_qemu_rom_with_fake_keys +pass: //sw/device/silicon_creator/rom/e2e/sigverify_key_type:sigverify_key_type_prod_ecdsa_dev_key_0_sim_qemu_rom_with_fake_keys +pass: //sw/device/silicon_creator/rom/e2e/sigverify_key_type:sigverify_key_type_prod_ecdsa_prod_key_0_sim_qemu_rom_with_fake_keys +pass: //sw/device/silicon_creator/rom/e2e/sigverify_key_type:sigverify_key_type_prod_ecdsa_test_key_0_sim_qemu_rom_with_fake_keys +pass: //sw/device/silicon_creator/rom/e2e/sigverify_key_type:sigverify_key_type_prod_end_ecdsa_dev_key_0_sim_qemu_rom_with_fake_keys +pass: //sw/device/silicon_creator/rom/e2e/sigverify_key_type:sigverify_key_type_prod_end_ecdsa_prod_key_0_sim_qemu_rom_with_fake_keys +pass: //sw/device/silicon_creator/rom/e2e/sigverify_key_type:sigverify_key_type_prod_end_ecdsa_test_key_0_sim_qemu_rom_with_fake_keys +pass: //sw/device/silicon_creator/rom/e2e/sigverify_key_type:sigverify_key_type_prod_end_spx_dev_key_0_sim_qemu_rom_with_fake_keys +pass: //sw/device/silicon_creator/rom/e2e/sigverify_key_type:sigverify_key_type_prod_end_spx_prod_key_0_sim_qemu_rom_with_fake_keys +pass: //sw/device/silicon_creator/rom/e2e/sigverify_key_type:sigverify_key_type_prod_end_spx_test_key_0_sim_qemu_rom_with_fake_keys +pass: //sw/device/silicon_creator/rom/e2e/sigverify_key_type:sigverify_key_type_prod_spx_dev_key_0_sim_qemu_rom_with_fake_keys +pass: //sw/device/silicon_creator/rom/e2e/sigverify_key_type:sigverify_key_type_prod_spx_prod_key_0_sim_qemu_rom_with_fake_keys +pass: //sw/device/silicon_creator/rom/e2e/sigverify_key_type:sigverify_key_type_prod_spx_test_key_0_sim_qemu_rom_with_fake_keys +pass: //sw/device/silicon_creator/rom/e2e/sigverify_key_type:sigverify_key_type_rma_ecdsa_dev_key_0_sim_qemu_rom_with_fake_keys +pass: //sw/device/silicon_creator/rom/e2e/sigverify_key_type:sigverify_key_type_rma_ecdsa_prod_key_0_sim_qemu_rom_with_fake_keys +pass: //sw/device/silicon_creator/rom/e2e/sigverify_key_type:sigverify_key_type_rma_ecdsa_test_key_0_sim_qemu_rom_with_fake_keys +pass: //sw/device/silicon_creator/rom/e2e/sigverify_key_type:sigverify_key_type_rma_spx_dev_key_0_sim_qemu_rom_with_fake_keys +pass: //sw/device/silicon_creator/rom/e2e/sigverify_key_type:sigverify_key_type_rma_spx_prod_key_0_sim_qemu_rom_with_fake_keys +pass: //sw/device/silicon_creator/rom/e2e/sigverify_key_type:sigverify_key_type_rma_spx_test_key_0_sim_qemu_rom_with_fake_keys +pass: //sw/device/silicon_creator/rom/e2e/sigverify_key_type:sigverify_key_type_test_unlocked0_ecdsa_dev_key_0_sim_qemu_rom_with_fake_keys +pass: //sw/device/silicon_creator/rom/e2e/sigverify_key_type:sigverify_key_type_test_unlocked0_ecdsa_prod_key_0_sim_qemu_rom_with_fake_keys +pass: //sw/device/silicon_creator/rom/e2e/sigverify_key_type:sigverify_key_type_test_unlocked0_ecdsa_test_key_0_sim_qemu_rom_with_fake_keys +pass: //sw/device/silicon_creator/rom/e2e/sigverify_key_type:sigverify_key_type_test_unlocked0_spx_dev_key_0_sim_qemu_rom_with_fake_keys +pass: //sw/device/silicon_creator/rom/e2e/sigverify_key_type:sigverify_key_type_test_unlocked0_spx_prod_key_0_sim_qemu_rom_with_fake_keys +pass: //sw/device/silicon_creator/rom/e2e/sigverify_key_type:sigverify_key_type_test_unlocked0_spx_test_key_0_sim_qemu_rom_with_fake_keys +pass: //sw/device/silicon_creator/rom/e2e/sigverify_key_validity:sigverify_key_validity_ecdsa_blank_dev_dev_key_0_dev_key_0_sim_qemu_rom_with_fake_keys +pass: //sw/device/silicon_creator/rom/e2e/sigverify_key_validity:sigverify_key_validity_ecdsa_blank_dev_prod_key_0_prod_key_0_sim_qemu_rom_with_fake_keys +pass: //sw/device/silicon_creator/rom/e2e/sigverify_key_validity:sigverify_key_validity_ecdsa_blank_prod_end_prod_key_0_prod_key_0_sim_qemu_rom_with_fake_keys +pass: //sw/device/silicon_creator/rom/e2e/sigverify_key_validity:sigverify_key_validity_ecdsa_blank_prod_prod_key_0_prod_key_0_sim_qemu_rom_with_fake_keys +pass: //sw/device/silicon_creator/rom/e2e/sigverify_key_validity:sigverify_key_validity_ecdsa_blank_rma_dev_key_0_dev_key_0_sim_qemu_rom_with_fake_keys +pass: //sw/device/silicon_creator/rom/e2e/sigverify_key_validity:sigverify_key_validity_ecdsa_blank_rma_prod_key_0_prod_key_0_sim_qemu_rom_with_fake_keys +pass: //sw/device/silicon_creator/rom/e2e/sigverify_key_validity:sigverify_key_validity_ecdsa_blank_rma_test_key_0_test_key_0_sim_qemu_rom_with_fake_keys +pass: //sw/device/silicon_creator/rom/e2e/sigverify_key_validity:sigverify_key_validity_ecdsa_blank_test_unlocked0_dev_key_0_dev_key_0_sim_qemu_rom_with_fake_keys +pass: //sw/device/silicon_creator/rom/e2e/sigverify_key_validity:sigverify_key_validity_ecdsa_blank_test_unlocked0_prod_key_0_prod_key_0_sim_qemu_rom_with_fake_keys +pass: //sw/device/silicon_creator/rom/e2e/sigverify_key_validity:sigverify_key_validity_ecdsa_blank_test_unlocked0_test_key_0_test_key_0_sim_qemu_rom_with_fake_keys +pass: //sw/device/silicon_creator/rom/e2e/sigverify_key_validity:sigverify_key_validity_ecdsa_revoked_dev_dev_key_0_dev_key_0_sim_qemu_rom_with_fake_keys +pass: //sw/device/silicon_creator/rom/e2e/sigverify_key_validity:sigverify_key_validity_ecdsa_revoked_dev_prod_key_0_prod_key_0_sim_qemu_rom_with_fake_keys +pass: //sw/device/silicon_creator/rom/e2e/sigverify_key_validity:sigverify_key_validity_ecdsa_revoked_prod_end_prod_key_0_prod_key_0_sim_qemu_rom_with_fake_keys +pass: //sw/device/silicon_creator/rom/e2e/sigverify_key_validity:sigverify_key_validity_ecdsa_revoked_prod_prod_key_0_prod_key_0_sim_qemu_rom_with_fake_keys +pass: //sw/device/silicon_creator/rom/e2e/sigverify_key_validity:sigverify_key_validity_ecdsa_revoked_rma_dev_key_0_dev_key_0_sim_qemu_rom_with_fake_keys +pass: //sw/device/silicon_creator/rom/e2e/sigverify_key_validity:sigverify_key_validity_ecdsa_revoked_rma_prod_key_0_prod_key_0_sim_qemu_rom_with_fake_keys +pass: //sw/device/silicon_creator/rom/e2e/sigverify_key_validity:sigverify_key_validity_ecdsa_revoked_rma_test_key_0_test_key_0_sim_qemu_rom_with_fake_keys +pass: //sw/device/silicon_creator/rom/e2e/sigverify_key_validity:sigverify_key_validity_ecdsa_revoked_test_unlocked0_dev_key_0_dev_key_0_sim_qemu_rom_with_fake_keys +pass: //sw/device/silicon_creator/rom/e2e/sigverify_key_validity:sigverify_key_validity_ecdsa_revoked_test_unlocked0_prod_key_0_prod_key_0_sim_qemu_rom_with_fake_keys +pass: //sw/device/silicon_creator/rom/e2e/sigverify_key_validity:sigverify_key_validity_ecdsa_revoked_test_unlocked0_test_key_0_test_key_0_sim_qemu_rom_with_fake_keys +pass: //sw/device/silicon_creator/rom/e2e/sigverify_key_validity:sigverify_key_validity_spx_blank_dev_dev_key_0_dev_key_0_sim_qemu_rom_with_fake_keys +pass: //sw/device/silicon_creator/rom/e2e/sigverify_key_validity:sigverify_key_validity_spx_blank_dev_prod_key_0_prod_key_0_sim_qemu_rom_with_fake_keys +pass: //sw/device/silicon_creator/rom/e2e/sigverify_key_validity:sigverify_key_validity_spx_blank_prod_end_prod_key_0_prod_key_0_sim_qemu_rom_with_fake_keys +pass: //sw/device/silicon_creator/rom/e2e/sigverify_key_validity:sigverify_key_validity_spx_blank_prod_prod_key_0_prod_key_0_sim_qemu_rom_with_fake_keys +pass: //sw/device/silicon_creator/rom/e2e/sigverify_key_validity:sigverify_key_validity_spx_blank_rma_dev_key_0_dev_key_0_sim_qemu_rom_with_fake_keys +pass: //sw/device/silicon_creator/rom/e2e/sigverify_key_validity:sigverify_key_validity_spx_blank_rma_prod_key_0_prod_key_0_sim_qemu_rom_with_fake_keys +pass: //sw/device/silicon_creator/rom/e2e/sigverify_key_validity:sigverify_key_validity_spx_blank_rma_test_key_0_test_key_0_sim_qemu_rom_with_fake_keys +pass: //sw/device/silicon_creator/rom/e2e/sigverify_key_validity:sigverify_key_validity_spx_blank_test_unlocked0_dev_key_0_dev_key_0_sim_qemu_rom_with_fake_keys +pass: //sw/device/silicon_creator/rom/e2e/sigverify_key_validity:sigverify_key_validity_spx_blank_test_unlocked0_prod_key_0_prod_key_0_sim_qemu_rom_with_fake_keys +pass: //sw/device/silicon_creator/rom/e2e/sigverify_key_validity:sigverify_key_validity_spx_blank_test_unlocked0_test_key_0_test_key_0_sim_qemu_rom_with_fake_keys +pass: //sw/device/silicon_creator/rom/e2e/sigverify_key_validity:sigverify_key_validity_spx_revoked_dev_dev_key_0_dev_key_0_sim_qemu_rom_with_fake_keys +pass: //sw/device/silicon_creator/rom/e2e/sigverify_key_validity:sigverify_key_validity_spx_revoked_dev_prod_key_0_prod_key_0_sim_qemu_rom_with_fake_keys +pass: //sw/device/silicon_creator/rom/e2e/sigverify_key_validity:sigverify_key_validity_spx_revoked_prod_end_prod_key_0_prod_key_0_sim_qemu_rom_with_fake_keys +pass: //sw/device/silicon_creator/rom/e2e/sigverify_key_validity:sigverify_key_validity_spx_revoked_prod_prod_key_0_prod_key_0_sim_qemu_rom_with_fake_keys +pass: //sw/device/silicon_creator/rom/e2e/sigverify_key_validity:sigverify_key_validity_spx_revoked_rma_dev_key_0_dev_key_0_sim_qemu_rom_with_fake_keys +pass: //sw/device/silicon_creator/rom/e2e/sigverify_key_validity:sigverify_key_validity_spx_revoked_rma_prod_key_0_prod_key_0_sim_qemu_rom_with_fake_keys +pass: //sw/device/silicon_creator/rom/e2e/sigverify_key_validity:sigverify_key_validity_spx_revoked_rma_test_key_0_test_key_0_sim_qemu_rom_with_fake_keys +pass: //sw/device/silicon_creator/rom/e2e/sigverify_key_validity:sigverify_key_validity_spx_revoked_test_unlocked0_dev_key_0_dev_key_0_sim_qemu_rom_with_fake_keys +pass: //sw/device/silicon_creator/rom/e2e/sigverify_key_validity:sigverify_key_validity_spx_revoked_test_unlocked0_prod_key_0_prod_key_0_sim_qemu_rom_with_fake_keys +pass: //sw/device/silicon_creator/rom/e2e/sigverify_key_validity:sigverify_key_validity_spx_revoked_test_unlocked0_test_key_0_test_key_0_sim_qemu_rom_with_fake_keys +pass: //sw/device/silicon_creator/rom/e2e/sigverify_spx:sigverify_spx_dev_disabled_sim_qemu_rom_with_fake_keys +pass: //sw/device/silicon_creator/rom/e2e/sigverify_spx:sigverify_spx_dev_enabled_true_sim_qemu_rom_with_fake_keys +pass: //sw/device/silicon_creator/rom/e2e/sigverify_spx:sigverify_spx_dev_enabled_zero_sim_qemu_rom_with_fake_keys +pass: //sw/device/silicon_creator/rom/e2e/sigverify_spx:sigverify_spx_prehashed_test_sim_qemu_rom_with_fake_keys +pass: //sw/device/silicon_creator/rom/e2e/sigverify_spx:sigverify_spx_prod_disabled_sim_qemu_rom_with_fake_keys +pass: //sw/device/silicon_creator/rom/e2e/sigverify_spx:sigverify_spx_prod_enabled_true_sim_qemu_rom_with_fake_keys +pass: //sw/device/silicon_creator/rom/e2e/sigverify_spx:sigverify_spx_prod_enabled_zero_sim_qemu_rom_with_fake_keys +pass: //sw/device/silicon_creator/rom/e2e/sigverify_spx:sigverify_spx_prod_end_disabled_sim_qemu_rom_with_fake_keys +pass: //sw/device/silicon_creator/rom/e2e/sigverify_spx:sigverify_spx_prod_end_enabled_true_sim_qemu_rom_with_fake_keys +pass: //sw/device/silicon_creator/rom/e2e/sigverify_spx:sigverify_spx_prod_end_enabled_zero_sim_qemu_rom_with_fake_keys +pass: //sw/device/silicon_creator/rom/e2e/sigverify_spx:sigverify_spx_rma_disabled_sim_qemu_rom_with_fake_keys +pass: //sw/device/silicon_creator/rom/e2e/sigverify_spx:sigverify_spx_rma_enabled_true_sim_qemu_rom_with_fake_keys +pass: //sw/device/silicon_creator/rom/e2e/sigverify_spx:sigverify_spx_rma_enabled_zero_sim_qemu_rom_with_fake_keys +pass: //sw/device/silicon_creator/rom/e2e/sigverify_spx:sigverify_spx_test_unlocked0_disabled_sim_qemu_rom_with_fake_keys +pass: //sw/device/silicon_creator/rom/e2e/sigverify_spx:sigverify_spx_test_unlocked0_enabled_true_sim_qemu_rom_with_fake_keys +pass: //sw/device/silicon_creator/rom/e2e/sigverify_spx:sigverify_spx_test_unlocked0_enabled_zero_sim_qemu_rom_with_fake_keys +pass: //sw/device/silicon_creator/rom/e2e/sigverify_usage_constraints:sigverify_usage_constraint_all_constraints_mf_lc_dev_bs_lc_dev_sim_qemu_rom_with_fake_keys +pass: //sw/device/silicon_creator/rom/e2e/sigverify_usage_constraints:sigverify_usage_constraint_all_constraints_mf_lc_dev_bs_lc_prod_end_sim_qemu_rom_with_fake_keys +pass: //sw/device/silicon_creator/rom/e2e/sigverify_usage_constraints:sigverify_usage_constraint_all_constraints_mf_lc_dev_bs_lc_prod_sim_qemu_rom_with_fake_keys +pass: //sw/device/silicon_creator/rom/e2e/sigverify_usage_constraints:sigverify_usage_constraint_all_constraints_mf_lc_dev_bs_lc_rma_sim_qemu_rom_with_fake_keys +pass: //sw/device/silicon_creator/rom/e2e/sigverify_usage_constraints:sigverify_usage_constraint_all_constraints_mf_lc_dev_bs_lc_test_unlocked0_sim_qemu_rom_with_fake_keys +pass: //sw/device/silicon_creator/rom/e2e/sigverify_usage_constraints:sigverify_usage_constraint_all_constraints_mf_lc_prod_bs_lc_dev_sim_qemu_rom_with_fake_keys +pass: //sw/device/silicon_creator/rom/e2e/sigverify_usage_constraints:sigverify_usage_constraint_all_constraints_mf_lc_prod_bs_lc_prod_end_sim_qemu_rom_with_fake_keys +pass: //sw/device/silicon_creator/rom/e2e/sigverify_usage_constraints:sigverify_usage_constraint_all_constraints_mf_lc_prod_bs_lc_prod_sim_qemu_rom_with_fake_keys +pass: //sw/device/silicon_creator/rom/e2e/sigverify_usage_constraints:sigverify_usage_constraint_all_constraints_mf_lc_prod_bs_lc_rma_sim_qemu_rom_with_fake_keys +pass: //sw/device/silicon_creator/rom/e2e/sigverify_usage_constraints:sigverify_usage_constraint_all_constraints_mf_lc_prod_bs_lc_test_unlocked0_sim_qemu_rom_with_fake_keys +pass: //sw/device/silicon_creator/rom/e2e/sigverify_usage_constraints:sigverify_usage_constraint_all_constraints_mf_lc_prod_end_bs_lc_dev_sim_qemu_rom_with_fake_keys +pass: //sw/device/silicon_creator/rom/e2e/sigverify_usage_constraints:sigverify_usage_constraint_all_constraints_mf_lc_prod_end_bs_lc_prod_end_sim_qemu_rom_with_fake_keys +pass: //sw/device/silicon_creator/rom/e2e/sigverify_usage_constraints:sigverify_usage_constraint_all_constraints_mf_lc_prod_end_bs_lc_prod_sim_qemu_rom_with_fake_keys +pass: //sw/device/silicon_creator/rom/e2e/sigverify_usage_constraints:sigverify_usage_constraint_all_constraints_mf_lc_prod_end_bs_lc_rma_sim_qemu_rom_with_fake_keys +pass: //sw/device/silicon_creator/rom/e2e/sigverify_usage_constraints:sigverify_usage_constraint_all_constraints_mf_lc_prod_end_bs_lc_test_unlocked0_sim_qemu_rom_with_fake_keys +pass: //sw/device/silicon_creator/rom/e2e/sigverify_usage_constraints:sigverify_usage_constraint_all_constraints_mf_lc_rma_bs_lc_dev_sim_qemu_rom_with_fake_keys +pass: //sw/device/silicon_creator/rom/e2e/sigverify_usage_constraints:sigverify_usage_constraint_all_constraints_mf_lc_rma_bs_lc_prod_end_sim_qemu_rom_with_fake_keys +pass: //sw/device/silicon_creator/rom/e2e/sigverify_usage_constraints:sigverify_usage_constraint_all_constraints_mf_lc_rma_bs_lc_prod_sim_qemu_rom_with_fake_keys +pass: //sw/device/silicon_creator/rom/e2e/sigverify_usage_constraints:sigverify_usage_constraint_all_constraints_mf_lc_rma_bs_lc_rma_sim_qemu_rom_with_fake_keys +pass: //sw/device/silicon_creator/rom/e2e/sigverify_usage_constraints:sigverify_usage_constraint_all_constraints_mf_lc_rma_bs_lc_test_unlocked0_sim_qemu_rom_with_fake_keys +pass: //sw/device/silicon_creator/rom/e2e/sigverify_usage_constraints:sigverify_usage_constraint_all_constraints_mf_lc_test_bs_lc_dev_sim_qemu_rom_with_fake_keys +pass: //sw/device/silicon_creator/rom/e2e/sigverify_usage_constraints:sigverify_usage_constraint_all_constraints_mf_lc_test_bs_lc_prod_end_sim_qemu_rom_with_fake_keys +pass: //sw/device/silicon_creator/rom/e2e/sigverify_usage_constraints:sigverify_usage_constraint_all_constraints_mf_lc_test_bs_lc_prod_sim_qemu_rom_with_fake_keys +pass: //sw/device/silicon_creator/rom/e2e/sigverify_usage_constraints:sigverify_usage_constraint_all_constraints_mf_lc_test_bs_lc_rma_sim_qemu_rom_with_fake_keys +pass: //sw/device/silicon_creator/rom/e2e/sigverify_usage_constraints:sigverify_usage_constraint_all_constraints_mf_lc_test_bs_lc_test_unlocked0_sim_qemu_rom_with_fake_keys +pass: //sw/device/silicon_creator/rom/e2e/sigverify_usage_constraints:sigverify_usage_constraint_device_id_family_match_sim_qemu_rom_with_fake_keys +pass: //sw/device/silicon_creator/rom/e2e/sigverify_usage_constraints:sigverify_usage_constraint_device_id_family_no_match_sim_qemu_rom_with_fake_keys +pass: //sw/device/silicon_creator/rom/e2e/sigverify_usage_constraints:sigverify_usage_constraint_device_id_match_sim_qemu_rom_with_fake_keys +pass: //sw/device/silicon_creator/rom/e2e/sigverify_usage_constraints:sigverify_usage_constraint_device_id_no_match_sim_qemu_rom_with_fake_keys +pass: //sw/device/silicon_creator/rom/e2e/sigverify_usage_constraints:sigverify_usage_constraint_invalid_unselected_device_id_sim_qemu_rom_with_fake_keys +pass: //sw/device/silicon_creator/rom/e2e/sigverify_usage_constraints:sigverify_usage_constraint_invalid_unselected_life_cycle_state_sim_qemu_rom_with_fake_keys +pass: //sw/device/silicon_creator/rom/e2e/sigverify_usage_constraints:sigverify_usage_constraint_invalid_unselected_manuf_state_creator_sim_qemu_rom_with_fake_keys +pass: //sw/device/silicon_creator/rom/e2e/sigverify_usage_constraints:sigverify_usage_constraint_invalid_unselected_manuf_state_owner_sim_qemu_rom_with_fake_keys +pass: //sw/device/silicon_creator/rom/e2e/sigverify_usage_constraints:sigverify_usage_constraint_lc_state_dev_match_sim_qemu_rom_with_fake_keys +pass: //sw/device/silicon_creator/rom/e2e/sigverify_usage_constraints:sigverify_usage_constraint_lc_state_dev_no_match_sim_qemu_rom_with_fake_keys +pass: //sw/device/silicon_creator/rom/e2e/sigverify_usage_constraints:sigverify_usage_constraint_lc_state_prod_end_match_sim_qemu_rom_with_fake_keys +pass: //sw/device/silicon_creator/rom/e2e/sigverify_usage_constraints:sigverify_usage_constraint_lc_state_prod_end_no_match_sim_qemu_rom_with_fake_keys +pass: //sw/device/silicon_creator/rom/e2e/sigverify_usage_constraints:sigverify_usage_constraint_lc_state_prod_match_sim_qemu_rom_with_fake_keys +pass: //sw/device/silicon_creator/rom/e2e/sigverify_usage_constraints:sigverify_usage_constraint_lc_state_prod_no_match_sim_qemu_rom_with_fake_keys +pass: //sw/device/silicon_creator/rom/e2e/sigverify_usage_constraints:sigverify_usage_constraint_lc_state_rma_match_sim_qemu_rom_with_fake_keys +pass: //sw/device/silicon_creator/rom/e2e/sigverify_usage_constraints:sigverify_usage_constraint_lc_state_rma_no_match_sim_qemu_rom_with_fake_keys +pass: //sw/device/silicon_creator/rom/e2e/sigverify_usage_constraints:sigverify_usage_constraint_lc_state_test_unlocked0_match_sim_qemu_rom_with_fake_keys +pass: //sw/device/silicon_creator/rom/e2e/sigverify_usage_constraints:sigverify_usage_constraint_lc_state_test_unlocked0_no_match_sim_qemu_rom_with_fake_keys +pass: //sw/device/silicon_creator/rom/e2e/sigverify_usage_constraints:sigverify_usage_constraint_manuf_state_creator_match_sim_qemu_rom_with_fake_keys +pass: //sw/device/silicon_creator/rom/e2e/sigverify_usage_constraints:sigverify_usage_constraint_manuf_state_creator_no_match_sim_qemu_rom_with_fake_keys +pass: //sw/device/silicon_creator/rom/e2e/sigverify_usage_constraints:sigverify_usage_constraint_manuf_state_owner_match_sim_qemu_rom_with_fake_keys +pass: //sw/device/silicon_creator/rom/e2e/sigverify_usage_constraints:sigverify_usage_constraint_manuf_state_owner_no_match_sim_qemu_rom_with_fake_keys pass: //sw/device/silicon_creator/rom/e2e:rom_e2e_flash_ctrl_init_sim_qemu_rom_with_fake_keys pass: //sw/device/tests:aes_entropy_test_sim_qemu_rom_with_fake_keys pass: //sw/device/tests:aes_entropy_test_sim_qemu_sival_rom_ext From 6a5979e7707992931dc118f4bce772094fc14878 Mon Sep 17 00:00:00 2001 From: Alex Jones Date: Wed, 22 Oct 2025 15:22:12 +0100 Subject: [PATCH 6/8] [ot] scripts/opentitan: run-bazel-tests.sh: Add test timeout option When testing a specific execution environment, you can set a maximum test timeout to use for that environment. For environments where tests are expected to be quick this can be used to decrease CI times (tests using unimplemented features may timeout, so this provides a smaller upper bound on test execution). For environments where tests may take a while when executed in parallel on a standard GitHub runner (maybe longer than 60 seconds), this can be used to increase the timeout for just executions in that environment. Signed-off-by: Alex Jones --- .github/workflows/eg_regression.yml | 5 +++++ docs/opentitan/regressions.md | 6 ++++-- scripts/opentitan/run-bazel-tests.sh | 13 ++++++++++--- 3 files changed, 19 insertions(+), 5 deletions(-) diff --git a/.github/workflows/eg_regression.yml b/.github/workflows/eg_regression.yml index 723e16b2452c3..85e3927d001ea 100644 --- a/.github/workflows/eg_regression.yml +++ b/.github/workflows/eg_regression.yml @@ -14,6 +14,10 @@ on: required: true default: qemu # Default tag, captures all QEMU exec envs in OpenTitan type: string + test_timeout: + description: The maximum timeout in seconds to use per executed test + required: false + type: string opentitan_repo: description: OpenTitan repository to be checked out required: true @@ -62,6 +66,7 @@ jobs: set -o pipefail ./qemu/scripts/opentitan/run-bazel-tests.sh ./ qemu \ ${{ inputs.exec_env }} \ + ${{ inputs.test_timeout }} \ | tee ${{ inputs.exec_env }}_test_results.txt - name: Upload Bazel test results diff --git a/docs/opentitan/regressions.md b/docs/opentitan/regressions.md index 467940a5ff270..d2707d3845aa5 100644 --- a/docs/opentitan/regressions.md +++ b/docs/opentitan/regressions.md @@ -8,7 +8,7 @@ With a checkout of OpenTitan, the script can be run like this: ```sh ./scripts/opentitan/run-bazel-tests.sh path/to/opentitan path/to/qemu \ - [execution_environment] + [execution_environment] [timeout] ``` The script will execute all QEMU-compatible tests using QEMU as it was built @@ -33,4 +33,6 @@ Specifying an execution environment when running the script will restrict to only running and comparing against tests for that execution environment. This can be used for more granular testing to break down large test workloads. If you do not specify an execution environment, _all_ available QEMU tests -will be executed. +will be executed. When specifying an execution environment, a maximum test +timeout (in seconds) to use for that environment can also be provided as +a subsequent argument to the script. diff --git a/scripts/opentitan/run-bazel-tests.sh b/scripts/opentitan/run-bazel-tests.sh index b343bbaad806d..ad5d42b4c3158 100755 --- a/scripts/opentitan/run-bazel-tests.sh +++ b/scripts/opentitan/run-bazel-tests.sh @@ -7,7 +7,7 @@ set -e # comparing the results with a list of expected passes. # # USAGE: run-bazel-tests.sh path/to/opentitan/repo path/to/qemu/repo \ -# [execution environment] +# [execution environment] [test_timeout] # # There is a companion file `tests/opentitan/data/earlgrey-tests.txt` that # is read by this script. @@ -29,13 +29,14 @@ GITHUB_STEP_SUMMARY="${GITHUB_STEP_SUMMARY:-/dev/null}" opentitan_path="$1" qemu_path="$2" qemu_exec_env="$3" +test_timeout="$4" if [ ! -d "$opentitan_path" ] || [ ! -d "$qemu_path" ]; then - echo "USAGE: ${0} []" + echo "USAGE: ${0} [] []" exit 1 fi opentitan_path="$(realpath "$opentitan_path")" qemu_path="$(realpath "$qemu_path")" -if [ ! "$qemu_exec_env" ]; then +if [ -z "$qemu_exec_env" ]; then # catch-all tag for all QEMU exec envs qemu_exec_env="qemu" echo "Using default 'qemu' tag to test all exec envs" @@ -86,12 +87,18 @@ trap "cleanup" EXIT ## RUN BAZEL TESTS cd "$opentitan_path" >/dev/null +test_args="" +if [ -n "$test_timeout" ]; then + test_args="--test_timeout=$test_timeout" +fi + ./bazelisk.sh test //... \ --test_tag_filters="$qemu_exec_env" \ --test_summary="short" \ --test_output=all \ --override_repository="+qemu+qemu_opentitan=${qemu_path}" \ --build_tests_only \ + $test_args \ | tee "$results" ## COMPARE RESULTS From 0bf581c631ce895b0dd4abb810d2a96420906738 Mon Sep 17 00:00:00 2001 From: Alex Jones Date: Wed, 22 Oct 2025 14:11:03 +0100 Subject: [PATCH 7/8] [ot] .github/workflows: ci: Add Earlgrey ROM_EXT CI regression With upstream support, there are now a variety of ROM_EXT end-to-end tests that we might expect to pass or at least be flaky. To ensure that regressions on these more complicated tests are being caught, add a regression run for the non-SiVal ROM_EXT environment, which most of these tests use. Since this is a separate job in the regression workflow it will be run in parallel by a separate runner, and so should not increase existing CI times (unless the time to execute all ROM_EXT test overshadows the time for executing other existing execution environments). Signed-off-by: Alex Jones --- .github/workflows/ci_regression.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.github/workflows/ci_regression.yml b/.github/workflows/ci_regression.yml index 2a643268a75db..2a236b8318ddc 100644 --- a/.github/workflows/ci_regression.yml +++ b/.github/workflows/ci_regression.yml @@ -31,6 +31,16 @@ jobs: opentitan_repo: ${{ inputs.opentitan_repo || 'lowRISC/opentitan' }} opentitan_ref: ${{ inputs.opentitan_ref || 'earlgrey_1.0.0' }} + earlgrey_rom_ext: + name: ROM_EXT + uses: ./.github/workflows/eg_regression.yml + secrets: inherit + with: + exec_env: sim_qemu_rom_ext + test_timeout: 150 + opentitan_repo: ${{ inputs.opentitan_repo || 'lowRISC/opentitan' }} + opentitan_ref: ${{ inputs.opentitan_ref || 'earlgrey_1.0.0' }} + earlgrey_sival_rom_ext: name: SiVal ROM_EXT uses: ./.github/workflows/eg_regression.yml From 85a485737902e32815552c40827f68b955633d90 Mon Sep 17 00:00:00 2001 From: Alex Jones Date: Wed, 22 Oct 2025 15:23:44 +0100 Subject: [PATCH 8/8] [ot] tests/opentitan: earlgrey-tests: Add new passing ROM_EXT tests Add newly passing ROM_EXT tests to be checked in Earlgrey regressions now that these are properly supported in upstream OpenTitan. Uses the increased timeout to ensure that even when running in parallel on slower runners, these tests should pass most of the time. Mark the couple of tests that are most likely to be flaky due to strict timing checks in the ROM as such, to hopefully avoid CI flakiness. Signed-off-by: Alex Jones --- tests/opentitan/data/earlgrey-tests.txt | 86 +++++++++++++++++++++++++ 1 file changed, 86 insertions(+) diff --git a/tests/opentitan/data/earlgrey-tests.txt b/tests/opentitan/data/earlgrey-tests.txt index 60cd39b81e661..c39b5ae21576f 100644 --- a/tests/opentitan/data/earlgrey-tests.txt +++ b/tests/opentitan/data/earlgrey-tests.txt @@ -11,6 +11,7 @@ pass: //sw/device/lib/crypto/drivers:entropy_test_sim_qemu_sival_rom_ext pass: //sw/device/lib/crypto/drivers:keymgr_test_sim_qemu_rom_with_fake_keys pass: //sw/device/lib/crypto/drivers:rv_core_ibex_test_sim_qemu_rom_with_fake_keys pass: //sw/device/lib/crypto/drivers:rv_core_ibex_test_sim_qemu_sival_rom_ext +pass: //sw/device/silicon_creator/lib/cert:dice_cwt_functest_sim_qemu_rom_ext pass: //sw/device/silicon_creator/lib/drivers:hmac_functest_sim_qemu_rom_with_fake_keys pass: //sw/device/silicon_creator/lib/drivers:hmac_functest_sim_qemu_sival_rom_ext pass: //sw/device/silicon_creator/lib/drivers:kmac_functest_sim_qemu_rom_with_fake_keys @@ -70,6 +71,91 @@ pass: //sw/device/silicon_creator/lib/sigverify:spx_verify_functest_sim_qemu_siv pass: //sw/device/silicon_creator/lib:boot_data_functest_sim_qemu_rom_with_fake_keys flaky: //sw/device/silicon_creator/lib:irq_asm_functest_sim_qemu_rom_with_fake_keys # Fails ~80% of the time due to an unknown BFV on intentional reset via watchdog/rstmgr. Unknown cause, perhaps timing. flaky: //sw/device/silicon_creator/lib:irq_asm_functest_sim_qemu_sival_rom_ext # Fails ~80% of the time due to an unknown BFV on intentional reset via watchdog/rstmgr. Unknown cause, perhaps timing. +pass: //sw/device/silicon_creator/rom_ext/e2e/attestation:print_certs_test_sim_qemu_rom_ext +pass: //sw/device/silicon_creator/rom_ext/e2e/boot_svc:boot_svc_bad_next_test_sim_qemu_rom_ext +pass: //sw/device/silicon_creator/rom_ext/e2e/boot_svc:boot_svc_bad_primary_test_sim_qemu_rom_ext +pass: //sw/device/silicon_creator/rom_ext/e2e/boot_svc:boot_svc_empty_test_sim_qemu_rom_ext +pass: //sw/device/silicon_creator/rom_ext/e2e/boot_svc:boot_svc_enter_rescue_test_sim_qemu_rom_ext +pass: //sw/device/silicon_creator/rom_ext/e2e/boot_svc:boot_svc_min_sec_ver_a_test_sim_qemu_rom_ext +pass: //sw/device/silicon_creator/rom_ext/e2e/boot_svc:boot_svc_min_sec_ver_b_test_sim_qemu_rom_ext +pass: //sw/device/silicon_creator/rom_ext/e2e/boot_svc:boot_svc_next_aab_test_sim_qemu_rom_ext +pass: //sw/device/silicon_creator/rom_ext/e2e/boot_svc:boot_svc_next_aba_test_sim_qemu_rom_ext +pass: //sw/device/silicon_creator/rom_ext/e2e/boot_svc:boot_svc_next_abb_test_sim_qemu_rom_ext +pass: //sw/device/silicon_creator/rom_ext/e2e/boot_svc:boot_svc_primary_test_sim_qemu_rom_ext +pass: //sw/device/silicon_creator/rom_ext/e2e/dice_chain:corrupted_digest_test_sim_qemu_rom_ext +pass: //sw/device/silicon_creator/rom_ext/e2e/dice_chain:debug_mode_off_dice_cwt_test_sim_qemu_rom_ext +pass: //sw/device/silicon_creator/rom_ext/e2e/dice_chain:debug_mode_off_dice_x509_test_sim_qemu_rom_ext +pass: //sw/device/silicon_creator/rom_ext/e2e/dice_chain:debug_mode_on_dice_cwt_test_sim_qemu_rom_ext +pass: //sw/device/silicon_creator/rom_ext/e2e/dice_chain:debug_mode_on_dice_x509_test_sim_qemu_rom_ext +pass: //sw/device/silicon_creator/rom_ext/e2e/dice_chain:no_refresh_dice_cwt_test_sim_qemu_rom_ext +pass: //sw/device/silicon_creator/rom_ext/e2e/dice_chain:no_refresh_dice_x509_test_sim_qemu_rom_ext +pass: //sw/device/silicon_creator/rom_ext/e2e/ownership:bad_activate_test_sim_qemu_rom_ext +pass: //sw/device/silicon_creator/rom_ext/e2e/ownership:bad_appkey_constraint_test_sim_qemu_rom_ext +pass: //sw/device/silicon_creator/rom_ext/e2e/ownership:bad_app_key_test_sim_qemu_rom_ext +pass: //sw/device/silicon_creator/rom_ext/e2e/ownership:bad_endorsee_test_sim_qemu_rom_ext +pass: //sw/device/silicon_creator/rom_ext/e2e/ownership:bad_locked_update_no_exec_test_sim_qemu_rom_ext +pass: //sw/device/silicon_creator/rom_ext/e2e/ownership:bad_locked_update_test_sim_qemu_rom_ext +pass: //sw/device/silicon_creator/rom_ext/e2e/ownership:bad_owner_block_test_sim_qemu_rom_ext +pass: //sw/device/silicon_creator/rom_ext/e2e/ownership:bad_ownership_key_alg_test_sim_qemu_rom_ext +pass: //sw/device/silicon_creator/rom_ext/e2e/ownership:bad_ownership_key_alg_with_detached_sig_test_sim_qemu_rom_ext +pass: //sw/device/silicon_creator/rom_ext/e2e/ownership:bad_unlock_signature_test_sim_qemu_rom_ext +pass: //sw/device/silicon_creator/rom_ext/e2e/ownership:bad_unlock_test_sim_qemu_rom_ext +pass: //sw/device/silicon_creator/rom_ext/e2e/ownership:flash_error_test_sim_qemu_rom_ext +pass: //sw/device/silicon_creator/rom_ext/e2e/ownership:flash_permission_test_slot_aa_sim_qemu_rom_ext +pass: //sw/device/silicon_creator/rom_ext/e2e/ownership:flash_permission_test_slot_ab_sim_qemu_rom_ext +pass: //sw/device/silicon_creator/rom_ext/e2e/ownership:flash_permission_test_slot_ba_sim_qemu_rom_ext +pass: //sw/device/silicon_creator/rom_ext/e2e/ownership:flash_permission_test_slot_bb_sim_qemu_rom_ext +pass: //sw/device/silicon_creator/rom_ext/e2e/ownership:good_appkey_constraint_test_sim_qemu_rom_ext +pass: //sw/device/silicon_creator/rom_ext/e2e/ownership:install_owner_upgrade_test_sim_qemu_rom_ext +pass: //sw/device/silicon_creator/rom_ext/e2e/ownership:invalid_device_id_request_test_sim_qemu_rom_ext +pass: //sw/device/silicon_creator/rom_ext/e2e/ownership:invalid_key_alg_activate_request_test_sim_qemu_rom_ext +pass: //sw/device/silicon_creator/rom_ext/e2e/ownership:invalid_nonce_request_test_sim_qemu_rom_ext +pass: //sw/device/silicon_creator/rom_ext/e2e/ownership:locked_update_test_sim_qemu_rom_ext +pass: //sw/device/silicon_creator/rom_ext/e2e/ownership:newversion_badlock_test_sim_qemu_rom_ext +pass: //sw/device/silicon_creator/rom_ext/e2e/ownership:newversion_nodelock_test_sim_qemu_rom_ext +pass: //sw/device/silicon_creator/rom_ext/e2e/ownership:newversion_noupdate_test_sim_qemu_rom_ext +pass: //sw/device/silicon_creator/rom_ext/e2e/ownership:newversion_noupdate_with_bad_owner_block_test_sim_qemu_rom_ext +pass: //sw/device/silicon_creator/rom_ext/e2e/ownership:newversion_pq_downgrade_test_sim_qemu_rom_ext +pass: //sw/device/silicon_creator/rom_ext/e2e/ownership:newversion_pq_to_pq_update_test_sim_qemu_rom_ext +pass: //sw/device/silicon_creator/rom_ext/e2e/ownership:newversion_update_test_sim_qemu_rom_ext +flaky: //sw/device/silicon_creator/rom_ext/e2e/ownership:rescue_limit_test_sim_qemu_rom_ext # Xmodem flow has tight timing on UART transfer which may fail when run in parallel on slower hosts. +flaky: //sw/device/silicon_creator/rom_ext/e2e/ownership:rescue_permission_test_sim_qemu_rom_ext # Xmodem flow has tight timing on UART transfer which may fail when run in parallel on slower hosts. +pass: //sw/device/silicon_creator/rom_ext/e2e/ownership:transfer_any_test_sim_qemu_rom_ext +pass: //sw/device/silicon_creator/rom_ext/e2e/ownership:transfer_bl0_slot_b_test_sim_qemu_rom_ext +pass: //sw/device/silicon_creator/rom_ext/e2e/ownership:transfer_ecdsa_to_pq_test_sim_qemu_rom_ext +pass: //sw/device/silicon_creator/rom_ext/e2e/ownership:transfer_endorsed_test_sim_qemu_rom_ext +pass: //sw/device/silicon_creator/rom_ext/e2e/ownership:transfer_keymgr_test_sim_qemu_rom_ext +pass: //sw/device/silicon_creator/rom_ext/e2e/ownership:transfer_pq_to_pq_test_sim_qemu_rom_ext +pass: //sw/device/silicon_creator/rom_ext/e2e/ownership:transfer_spx_pure_to_spx_prehash_test_sim_qemu_rom_ext +pass: //sw/device/silicon_creator/rom_ext/e2e/ownership:unlock_detached_sig_test_sim_qemu_rom_ext +pass: //sw/device/silicon_creator/rom_ext/e2e/ownership:unlock_when_recovery_state_test_sim_qemu_rom_ext +pass: //sw/device/silicon_creator/rom_ext/e2e/ownership:unlock_with_owner_recovery_key_test_sim_qemu_rom_ext +pass: //sw/device/silicon_creator/rom_ext/e2e/verified_boot:bad_manifest_test_sim_qemu_rom_ext +pass: //sw/device/silicon_creator/rom_ext/e2e/verified_boot:bad_spx_manifest_test_sim_qemu_rom_ext +pass: //sw/device/silicon_creator/rom_ext/e2e/verified_boot:isfb_boot_test_sim_qemu_rom_ext +pass: //sw/device/silicon_creator/rom_ext/e2e/verified_boot:key_dev_hybrid_spx_prehashed_sim_qemu_rom_ext +pass: //sw/device/silicon_creator/rom_ext/e2e/verified_boot:key_dev_sim_qemu_rom_ext +pass: //sw/device/silicon_creator/rom_ext/e2e/verified_boot:key_prod_hybrid_spx_pure_sim_qemu_rom_ext +pass: //sw/device/silicon_creator/rom_ext/e2e/verified_boot:key_prod_sim_qemu_rom_ext +pass: //sw/device/silicon_creator/rom_ext/e2e/verified_boot:key_test_sim_qemu_rom_ext +pass: //sw/device/silicon_creator/rom_ext/e2e/verified_boot:key_unauthorized_sim_qemu_rom_ext +pass: //sw/device/silicon_creator/rom_ext/e2e/verified_boot:position_imm_section_virtual_a_sim_qemu_rom_ext +pass: //sw/device/silicon_creator/rom_ext/e2e/verified_boot:position_owner_slot_a_sim_qemu_rom_ext +pass: //sw/device/silicon_creator/rom_ext/e2e/verified_boot:position_owner_slot_b_sim_qemu_rom_ext +pass: //sw/device/silicon_creator/rom_ext/e2e/verified_boot:position_owner_virtual_a_sim_qemu_rom_ext +pass: //sw/device/silicon_creator/rom_ext/e2e/verified_boot:position_owner_virtual_b_sim_qemu_rom_ext +pass: //sw/device/silicon_creator/rom_ext/e2e/verified_boot:position_romext_slot_a_sim_qemu_rom_ext +pass: //sw/device/silicon_creator/rom_ext/e2e/verified_boot:position_romext_slot_b_sim_qemu_rom_ext +pass: //sw/device/silicon_creator/rom_ext/e2e/verified_boot:position_romext_virtual_a_sim_qemu_rom_ext +pass: //sw/device/silicon_creator/rom_ext/e2e/verified_boot:position_romext_virtual_a_with_imm_romext_enabled_sim_qemu_rom_ext +pass: //sw/device/silicon_creator/rom_ext/e2e/verified_boot:position_romext_virtual_b_sim_qemu_rom_ext +pass: //sw/device/silicon_creator/rom/e2e/address_translation:rom_ext_a_flash_a_bad_addr_trans_sim_qemu_rom_ext +pass: //sw/device/silicon_creator/rom/e2e/address_translation:rom_ext_a_flash_a_sim_qemu_rom_ext +pass: //sw/device/silicon_creator/rom/e2e/address_translation:rom_ext_a_flash_b_sim_qemu_rom_ext +pass: //sw/device/silicon_creator/rom/e2e/address_translation:rom_ext_b_flash_a_sim_qemu_rom_ext +pass: //sw/device/silicon_creator/rom/e2e/address_translation:rom_ext_b_flash_b_sim_qemu_rom_ext +pass: //sw/device/silicon_creator/rom/e2e/address_translation:rom_ext_v_flash_a_sim_qemu_rom_ext +pass: //sw/device/silicon_creator/rom/e2e/address_translation:rom_ext_v_flash_b_sim_qemu_rom_ext pass: //sw/device/silicon_creator/rom/e2e/keymgr:rom_e2e_keymgr_init_otp_invalid_meas_sim_qemu_rom_with_fake_keys pass: //sw/device/silicon_creator/rom/e2e/keymgr:rom_e2e_keymgr_init_otp_meas_sim_qemu_rom_with_fake_keys pass: //sw/device/silicon_creator/rom/e2e/keymgr:rom_e2e_keymgr_init_otp_no_meas_sim_qemu_rom_with_fake_keys