Skip to content

chore: set seed for flaky tests, use etherscan api key #12255

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 4 commits into from
Oct 23, 2025
Merged

chore: set seed for flaky tests, use etherscan api key #12255

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
258c719
chore: set seed for flaky tests
grandizzy Oct 22, 2025
6a74e84
env var
grandizzy Oct 23, 2025
d3ae9bb
fix
grandizzy Oct 23, 2025
02b5d26
Merge branch 'master' into grandizzy/flaky-fuzz-tests
grandizzy Oct 23, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .github/workflows/test.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -112,4 +112,5 @@ jobs:
SVM_TARGET_PLATFORM: ${{ matrix.svm_target_platform }}
HTTP_ARCHIVE_URLS: ${{ secrets.HTTP_ARCHIVE_URLS }}
WS_ARCHIVE_URLS: ${{ secrets.WS_ARCHIVE_URLS }}
ETHERSCAN_KEY: ${{ secrets.ETHERSCAN_API_KEY }}
Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

if set env var to ETHERSCAN_API_KEY then it's picked up by testdata test wich fails expected config (and other inline tests), hence used different ETHERSCAN_KEY key

run: cargo nextest run ${{ matrix.flags }}
4 changes: 2 additions & 2 deletions crates/cast/tests/cli/main.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2887,9 +2887,9 @@ contract CounterInExternalLibScript is Script {
.unwrap()
.tx_hash();

// Cache project selectors.
// Build and cache project selectors.
cmd.forge_fuse().args(["build"]).assert_success();
cmd.forge_fuse().args(["selectors", "cache"]).assert_success();

// Assert cast with local artifacts can decode external lib signature.
cmd.cast_fuse()
.args(["run", format!("{tx_hash}").as_str(), "--rpc-url", &handle.http_endpoint()])
Expand Down
3 changes: 2 additions & 1 deletion crates/cast/tests/cli/selectors.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -186,7 +186,8 @@ contract ContractWithCustomError {
}
"#,
);
// Store selectors in local cache.
// Build and cache project selectors.
cmd.forge_fuse().args(["build"]).assert_success();
cmd.forge_fuse().args(["selectors", "cache"]).assert_success();

// Assert cast can decode custom error with local cache.
Expand Down
1 change: 1 addition & 0 deletions crates/forge/tests/cli/failure_assertions.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -199,6 +199,7 @@ forgetest!(expect_emit_tests_should_fail, |prj, cmd| {
prj.add_source("ExpectEmitFailures.sol", expect_emit_failure_tests);

cmd.forge_fuse().arg("build").assert_success();
cmd.forge_fuse().args(["selectors", "cache"]).assert_success();

cmd.forge_fuse().args(["test", "--mc", "ExpectEmitFailureTest"]).assert_failure().stdout_eq(str![[r#"No files changed, compilation skipped
...
Expand Down
8 changes: 7 additions & 1 deletion crates/forge/tests/cli/test_cmd/fuzz.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -117,7 +117,10 @@ Ran 1 test suite [ELAPSED]: 1 tests passed, 0 failed, 0 skipped (1 total tests)

forgetest_init!(test_fuzz_fail_on_revert, |prj, cmd| {
prj.wipe_contracts();
prj.update_config(|config| config.fuzz.fail_on_revert = false);
prj.update_config(|config| {
config.fuzz.fail_on_revert = false;
config.fuzz.seed = Some(U256::from(100u32));
});
prj.add_source(
"Counter.sol",
r#"
Expand Down Expand Up @@ -672,6 +675,9 @@ Ran 1 test suite [ELAPSED]: 3 tests passed, 0 failed, 0 skipped (3 total tests)
// random values (instead edge cases) are generated if no fixtures defined
forgetest_init!(fuzz_uint, |prj, cmd| {
prj.wipe_contracts();
prj.update_config(|config| {
config.fuzz.seed = Some(U256::from(100u32));
});
prj.add_test(
"FuzzUint.t.sol",
r#"
Expand Down
44 changes: 5 additions & 39 deletions crates/forge/tests/cli/test_cmd/invariant/common.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -188,6 +188,7 @@ forgetest!(invariant_calldata_dictionary, |prj, cmd| {
prj.wipe_contracts();
prj.insert_utils();
prj.update_config(|config| {
config.fuzz.seed = Some(U256::from(1));
config.invariant.depth = 10;
});

Expand Down Expand Up @@ -293,7 +294,7 @@ Ran 1 test for test/InvariantCalldataDictionary.t.sol:InvariantCalldataDictionar
[SEQUENCE]
invariant_owner_never_changes() ([RUNS])

[STATS]
...

Suite result: FAILED. 0 passed; 1 failed; 0 skipped; [ELAPSED]

Expand Down Expand Up @@ -1056,35 +1057,17 @@ contract InvariantRollForkStateTest is Test {

assert_invariant(cmd.args(["test", "-j1"])).failure().stdout_eq(str![[r#"
...
Ran 1 test for test/InvariantRollFork.t.sol:InvariantRollForkStateTest
[FAIL: wrong supply]
[SEQUENCE]
invariant_fork_handler_state() ([RUNS])

[STATS]
Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

suspect some cases where [STATS] redact doesn't work as expected, I trimmed the output only to relevant


Suite result: FAILED. 0 passed; 1 failed; 0 skipped; [ELAPSED]

Ran 1 test for test/InvariantRollFork.t.sol:InvariantRollForkBlockTest
[FAIL: too many blocks mined]
[SEQUENCE]
invariant_fork_handler_block() ([RUNS])

[STATS]

Suite result: FAILED. 0 passed; 1 failed; 0 skipped; [ELAPSED]

Ran 2 test suites [ELAPSED]: 0 tests passed, 2 failed, 0 skipped (2 total tests)

Failing tests:
Encountered 1 failing test in test/InvariantRollFork.t.sol:InvariantRollForkBlockTest
[FAIL: too many blocks mined]
[SEQUENCE]
...
invariant_fork_handler_block() ([RUNS])

Encountered 1 failing test in test/InvariantRollFork.t.sol:InvariantRollForkStateTest
[FAIL: wrong supply]
[SEQUENCE]
...
invariant_fork_handler_state() ([RUNS])

Encountered a total of 2 failing tests, 0 tests succeeded
Expand All @@ -1098,6 +1081,7 @@ forgetest_init!(invariant_scrape_values, |prj, cmd| {
prj.wipe_contracts();
prj.update_config(|config| {
config.invariant.depth = 10;
config.fuzz.seed = Some(U256::from(100u32));
});

prj.add_test(
Expand Down Expand Up @@ -1173,24 +1157,6 @@ contract FindFromLogValueTest is Test {

assert_invariant(cmd.args(["test", "-j1"])).failure().stdout_eq(str![[r#"
...
Ran 1 test for test/InvariantScrapeValues.t.sol:FindFromReturnValueTest
[FAIL: value from return found]
[SEQUENCE]
Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

same as above, left only relevant

invariant_value_not_found() ([RUNS])

[STATS]

Suite result: FAILED. 0 passed; 1 failed; 0 skipped; [ELAPSED]

Ran 1 test for test/InvariantScrapeValues.t.sol:FindFromLogValueTest
[FAIL: value from logs found]
[SEQUENCE]
invariant_value_not_found() ([RUNS])

[STATS]

Suite result: FAILED. 0 passed; 1 failed; 0 skipped; [ELAPSED]

Ran 2 test suites [ELAPSED]: 0 tests passed, 2 failed, 0 skipped (2 total tests)

Failing tests:
Expand Down
25 changes: 2 additions & 23 deletions crates/forge/tests/cli/test_cmd/invariant/target.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -842,33 +842,12 @@ contract DynamicTargetContract is Test {

assert_invariant(cmd.args(["test", "-j1"])).failure().stdout_eq(str![[r#"
...
Ran 1 test for test/FuzzedTargetContracts.t.sol:ExplicitTargetContract
[PASS] invariant_explicit_target() ([RUNS])

[STATS]

Suite result: ok. 1 passed; 0 failed; 0 skipped; [ELAPSED]

Ran 1 test for test/FuzzedTargetContracts.t.sol:DynamicTargetContract
[FAIL: wrong target selector called]
[SEQUENCE]
invariant_dynamic_targets() ([RUNS])

[STATS]

Suite result: FAILED. 0 passed; 1 failed; 0 skipped; [ELAPSED]

Ran 2 test suites [ELAPSED]: 1 tests passed, 1 failed, 0 skipped (2 total tests)

Failing tests:
Encountered 1 failing test in test/FuzzedTargetContracts.t.sol:DynamicTargetContract
...
[FAIL: wrong target selector called]
[SEQUENCE]
invariant_dynamic_targets() ([RUNS])

Encountered a total of 1 failing tests, 1 tests succeeded

Tip: Run `forge test --rerun` to retry only the 1 failed test
...

"#]]);
});
2 changes: 1 addition & 1 deletion crates/forge/tests/cli/test_cmd/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4055,7 +4055,7 @@ contract ContractWithCustomError {
"#,
);
// Build and cache project selectors.
cmd.forge_fuse().args(["build"]).assert_success();
cmd.forge_fuse().args(["build", "--force"]).assert_success();

// Assert cast can decode custom error with local cache.
cmd.cast_fuse()
Expand Down
14 changes: 10 additions & 4 deletions crates/test-utils/src/rpc.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,12 @@ use foundry_config::{
RpcEndpointUrl, RpcEndpoints,
};
use rand::seq::SliceRandom;
use std::sync::{
LazyLock,
atomic::{AtomicUsize, Ordering},
use std::{
env,
sync::{
LazyLock,
atomic::{AtomicUsize, Ordering},
},
};

macro_rules! shuffled_list {
Expand Down Expand Up @@ -153,7 +156,10 @@ fn next_archive_url(is_ws: bool) -> String {

/// Returns the next etherscan api key.
pub fn next_etherscan_api_key() -> String {
let key = ETHERSCAN_KEYS.next().to_string();
let mut key = env::var("ETHERSCAN_KEY").unwrap_or_default();
if key.is_empty() {
key = ETHERSCAN_KEYS.next().to_string();
}
test_debug!("next_etherscan_api_key() = {}...", &key[..6]);
key
}
Expand Down
Loading