From fd04f2356ce9a72d7766e5db4e94a50a1a6f1c06 Mon Sep 17 00:00:00 2001 From: colinlyguo Date: Tue, 25 Mar 2025 15:51:42 +0800 Subject: [PATCH 01/10] add more logs in testWitness --- eth/api.go | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/eth/api.go b/eth/api.go index 8a0f094eef2..77aeb2b347a 100644 --- a/eth/api.go +++ b/eth/api.go @@ -373,31 +373,39 @@ func generateWitness(blockchain *core.BlockChain, block *types.Block) (*stateles func testWitness(blockchain *core.BlockChain, block *types.Block, witness *stateless.Witness) error { stateRoot := witness.Root() - if diskRoot, _ := rawdb.ReadDiskStateRoot(blockchain.Database(), stateRoot); diskRoot != (common.Hash{}) { + diskRoot, err := rawdb.ReadDiskStateRoot(blockchain.Database(), stateRoot) + if err != nil { + return fmt.Errorf("failed to read disk state root for stateRoot %s: %w", stateRoot.Hex(), err) + } + if diskRoot != (common.Hash{}) { stateRoot = diskRoot } // Create and populate the state database to serve as the stateless backend statedb, err := state.New(stateRoot, state.NewDatabase(witness.MakeHashDB()), nil) if err != nil { - return fmt.Errorf("failed to create state database: %w", err) + return fmt.Errorf("failed to create state database with stateRoot %s: %w", stateRoot.Hex(), err) } receipts, _, usedGas, err := blockchain.Processor().Process(block, statedb, *blockchain.GetVMConfig()) if err != nil { - return fmt.Errorf("failed to process block %d: %w", block.Number(), err) + return fmt.Errorf("failed to process block %d (hash: %s): %w", block.Number(), block.Hash().Hex(), err) } if err := blockchain.Validator().ValidateState(block, statedb, receipts, usedGas); err != nil { - return fmt.Errorf("failed to validate block %d: %w", block.Number(), err) + return fmt.Errorf("failed to validate block %d (hash: %s): %w", block.Number(), block.Hash().Hex(), err) } postStateRoot := block.Root() - if diskRoot, _ := rawdb.ReadDiskStateRoot(blockchain.Database(), postStateRoot); diskRoot != (common.Hash{}) { + diskRoot, err = rawdb.ReadDiskStateRoot(blockchain.Database(), postStateRoot) + if err != nil { + return fmt.Errorf("failed to read disk state root for postStateRoot %s: %w", postStateRoot.Hex(), err) + } + if diskRoot != (common.Hash{}) { postStateRoot = diskRoot } if statedb.GetRootHash() != postStateRoot { - return fmt.Errorf("failed to commit statelessly %d: %w", block.Number(), err) + return fmt.Errorf("state root mismatch after processing block %d (hash: %s): expected %s, got %s", block.Number(), block.Hash().Hex(), postStateRoot.Hex(), statedb.GetRootHash().Hex()) } return nil } From b2e53dfaf2152bfeb61505d27c8303abdad848a6 Mon Sep 17 00:00:00 2001 From: colinlyguo Date: Tue, 25 Mar 2025 15:56:06 +0800 Subject: [PATCH 02/10] bump version --- params/version.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/params/version.go b/params/version.go index 3b6ac95466d..47169196477 100644 --- a/params/version.go +++ b/params/version.go @@ -24,7 +24,7 @@ import ( const ( VersionMajor = 5 // Major version component of the current release VersionMinor = 8 // Minor version component of the current release - VersionPatch = 31 // Patch version component of the current release + VersionPatch = 32 // Patch version component of the current release VersionMeta = "mainnet" // Version metadata to append to the version string ) From ab49f0e48a94cc8bbc709bd4b6d23d0e09db4297 Mon Sep 17 00:00:00 2001 From: colinlyguo Date: Tue, 25 Mar 2025 17:28:43 +0800 Subject: [PATCH 03/10] add more logs --- eth/api.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/eth/api.go b/eth/api.go index 77aeb2b347a..7aec35a688a 100644 --- a/eth/api.go +++ b/eth/api.go @@ -405,7 +405,12 @@ func testWitness(blockchain *core.BlockChain, block *types.Block, witness *state postStateRoot = diskRoot } if statedb.GetRootHash() != postStateRoot { - return fmt.Errorf("state root mismatch after processing block %d (hash: %s): expected %s, got %s", block.Number(), block.Hash().Hex(), postStateRoot.Hex(), statedb.GetRootHash().Hex()) + executionWitness := ToExecutionWitness(witness) + jsonStr, err := json.Marshal(executionWitness) + if err != nil { + return fmt.Errorf("state root mismatch after processing block %d (hash: %s): expected %s, got %s, but failed to marshal witness: %w", block.Number(), block.Hash().Hex(), postStateRoot.Hex(), statedb.GetRootHash().Hex(), err) + } + return fmt.Errorf("state root mismatch after processing block %d (hash: %s): expected %s, got %s, witness: %s", block.Number(), block.Hash().Hex(), postStateRoot.Hex(), statedb.GetRootHash().Hex(), string(jsonStr)) } return nil } From 760177f4b14e79c6b2189a62d4d51a7a88ffe17b Mon Sep 17 00:00:00 2001 From: colinlyguo Date: Tue, 25 Mar 2025 19:34:52 +0800 Subject: [PATCH 04/10] use statedb.IntermediateRoot --- eth/api.go | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/eth/api.go b/eth/api.go index 7aec35a688a..a96758da876 100644 --- a/eth/api.go +++ b/eth/api.go @@ -378,6 +378,7 @@ func testWitness(blockchain *core.BlockChain, block *types.Block, witness *state return fmt.Errorf("failed to read disk state root for stateRoot %s: %w", stateRoot.Hex(), err) } if diskRoot != (common.Hash{}) { + log.Debug("Using disk root for state root", "stateRoot", stateRoot.Hex(), "diskRoot", diskRoot.Hex()) stateRoot = diskRoot } @@ -402,15 +403,17 @@ func testWitness(blockchain *core.BlockChain, block *types.Block, witness *state return fmt.Errorf("failed to read disk state root for postStateRoot %s: %w", postStateRoot.Hex(), err) } if diskRoot != (common.Hash{}) { + log.Debug("Using disk root for post state root", "postStateRoot", postStateRoot.Hex(), "diskRoot", diskRoot.Hex()) postStateRoot = diskRoot } - if statedb.GetRootHash() != postStateRoot { + computedRoot := statedb.IntermediateRoot(blockchain.Config().IsEIP158(block.Number())) + if computedRoot != postStateRoot { executionWitness := ToExecutionWitness(witness) jsonStr, err := json.Marshal(executionWitness) if err != nil { - return fmt.Errorf("state root mismatch after processing block %d (hash: %s): expected %s, got %s, but failed to marshal witness: %w", block.Number(), block.Hash().Hex(), postStateRoot.Hex(), statedb.GetRootHash().Hex(), err) + return fmt.Errorf("state root mismatch after processing block %d (hash: %s): expected %s, got %s, but failed to marshal witness: %w", block.Number(), block.Hash().Hex(), postStateRoot.Hex(), computedRoot, err) } - return fmt.Errorf("state root mismatch after processing block %d (hash: %s): expected %s, got %s, witness: %s", block.Number(), block.Hash().Hex(), postStateRoot.Hex(), statedb.GetRootHash().Hex(), string(jsonStr)) + return fmt.Errorf("state root mismatch after processing block %d (hash: %s): expected %s, got %s, witness: %s", block.Number(), block.Hash().Hex(), postStateRoot.Hex(), computedRoot, string(jsonStr)) } return nil } From ec42bfd803ff7f253e6d58c962e92fec1da37fe3 Mon Sep 17 00:00:00 2001 From: colinlyguo Date: Tue, 25 Mar 2025 20:17:34 +0800 Subject: [PATCH 05/10] add retries --- eth/api.go | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/eth/api.go b/eth/api.go index a96758da876..bc6a7d321a7 100644 --- a/eth/api.go +++ b/eth/api.go @@ -406,14 +406,21 @@ func testWitness(blockchain *core.BlockChain, block *types.Block, witness *state log.Debug("Using disk root for post state root", "postStateRoot", postStateRoot.Hex(), "diskRoot", diskRoot.Hex()) postStateRoot = diskRoot } - computedRoot := statedb.IntermediateRoot(blockchain.Config().IsEIP158(block.Number())) - if computedRoot != postStateRoot { - executionWitness := ToExecutionWitness(witness) - jsonStr, err := json.Marshal(executionWitness) - if err != nil { - return fmt.Errorf("state root mismatch after processing block %d (hash: %s): expected %s, got %s, but failed to marshal witness: %w", block.Number(), block.Hash().Hex(), postStateRoot.Hex(), computedRoot, err) + retryLimit := 5 + for retries := 0; retries <= retryLimit; retries++ { + computedRoot := statedb.IntermediateRoot(blockchain.Config().IsEIP158(block.Number())) + if computedRoot == postStateRoot { + break + } + log.Debug("State root mismatch", "block", block.Number(), "expected", postStateRoot.Hex(), "got", computedRoot) + if retries == retryLimit { + executionWitness := ToExecutionWitness(witness) + jsonStr, err := json.Marshal(executionWitness) + if err != nil { + return fmt.Errorf("state root mismatch after processing block %d (hash: %s): expected %s, got %s, but failed to marshal witness: %w", block.Number(), block.Hash().Hex(), postStateRoot.Hex(), computedRoot, err) + } + return fmt.Errorf("state root mismatch after processing block %d (hash: %s): expected %s, got %s, witness: %s", block.Number(), block.Hash().Hex(), postStateRoot.Hex(), computedRoot, string(jsonStr)) } - return fmt.Errorf("state root mismatch after processing block %d (hash: %s): expected %s, got %s, witness: %s", block.Number(), block.Hash().Hex(), postStateRoot.Hex(), computedRoot, string(jsonStr)) } return nil } From 6e50f8c870258e463edb87175dbecbb029f7f1bb Mon Sep 17 00:00:00 2001 From: colinlyguo Date: Tue, 25 Mar 2025 20:40:43 +0800 Subject: [PATCH 06/10] add more retries --- eth/api.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/eth/api.go b/eth/api.go index bc6a7d321a7..93b5189b27a 100644 --- a/eth/api.go +++ b/eth/api.go @@ -368,6 +368,13 @@ func generateWitness(blockchain *core.BlockChain, block *types.Block) (*stateles if err := blockchain.Validator().ValidateState(block, statedb, receipts, usedGas); err != nil { return nil, fmt.Errorf("failed to validate block %d: %w", block.Number(), err) } + + for retries := 0; retries <= 3; retries++ { + err = testWitness(blockchain, block, witness) + if err == nil { + return witness, nil + } + } return witness, testWitness(blockchain, block, witness) } @@ -408,6 +415,7 @@ func testWitness(blockchain *core.BlockChain, block *types.Block, witness *state } retryLimit := 5 for retries := 0; retries <= retryLimit; retries++ { + statedb.Commit(blockchain.Config().IsEIP158(block.Number())) computedRoot := statedb.IntermediateRoot(blockchain.Config().IsEIP158(block.Number())) if computedRoot == postStateRoot { break From c34fc0a16655951fd9116e774136b522d2d75c7e Mon Sep 17 00:00:00 2001 From: colinlyguo Date: Tue, 25 Mar 2025 21:05:09 +0800 Subject: [PATCH 07/10] remove root compute retries --- eth/api.go | 21 +++++++-------------- 1 file changed, 7 insertions(+), 14 deletions(-) diff --git a/eth/api.go b/eth/api.go index 93b5189b27a..10bff58ae9b 100644 --- a/eth/api.go +++ b/eth/api.go @@ -413,22 +413,15 @@ func testWitness(blockchain *core.BlockChain, block *types.Block, witness *state log.Debug("Using disk root for post state root", "postStateRoot", postStateRoot.Hex(), "diskRoot", diskRoot.Hex()) postStateRoot = diskRoot } - retryLimit := 5 - for retries := 0; retries <= retryLimit; retries++ { - statedb.Commit(blockchain.Config().IsEIP158(block.Number())) - computedRoot := statedb.IntermediateRoot(blockchain.Config().IsEIP158(block.Number())) - if computedRoot == postStateRoot { - break - } + computedRoot := statedb.GetRootHash() + if computedRoot != postStateRoot { log.Debug("State root mismatch", "block", block.Number(), "expected", postStateRoot.Hex(), "got", computedRoot) - if retries == retryLimit { - executionWitness := ToExecutionWitness(witness) - jsonStr, err := json.Marshal(executionWitness) - if err != nil { - return fmt.Errorf("state root mismatch after processing block %d (hash: %s): expected %s, got %s, but failed to marshal witness: %w", block.Number(), block.Hash().Hex(), postStateRoot.Hex(), computedRoot, err) - } - return fmt.Errorf("state root mismatch after processing block %d (hash: %s): expected %s, got %s, witness: %s", block.Number(), block.Hash().Hex(), postStateRoot.Hex(), computedRoot, string(jsonStr)) + executionWitness := ToExecutionWitness(witness) + jsonStr, err := json.Marshal(executionWitness) + if err != nil { + return fmt.Errorf("state root mismatch after processing block %d (hash: %s): expected %s, got %s, but failed to marshal witness: %w", block.Number(), block.Hash().Hex(), postStateRoot.Hex(), computedRoot, err) } + return fmt.Errorf("state root mismatch after processing block %d (hash: %s): expected %s, got %s, witness: %s", block.Number(), block.Hash().Hex(), postStateRoot.Hex(), computedRoot, string(jsonStr)) } return nil } From c63e349e272d5082c015723af657a5fa15d72fdc Mon Sep 17 00:00:00 2001 From: colinlyguo Date: Tue, 25 Mar 2025 21:08:03 +0800 Subject: [PATCH 08/10] tweak --- eth/api.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/eth/api.go b/eth/api.go index 10bff58ae9b..64b2c3ad07e 100644 --- a/eth/api.go +++ b/eth/api.go @@ -369,13 +369,12 @@ func generateWitness(blockchain *core.BlockChain, block *types.Block) (*stateles return nil, fmt.Errorf("failed to validate block %d: %w", block.Number(), err) } - for retries := 0; retries <= 3; retries++ { - err = testWitness(blockchain, block, witness) - if err == nil { + for retries := 0; retries < 5; retries++ { + if err = testWitness(blockchain, block, witness); err == nil { return witness, nil } } - return witness, testWitness(blockchain, block, witness) + return witness, err } func testWitness(blockchain *core.BlockChain, block *types.Block, witness *stateless.Witness) error { From 16757906147d2442092174ea585ac6cb6ba1b641 Mon Sep 17 00:00:00 2001 From: colinlyguo Date: Tue, 25 Mar 2025 21:20:56 +0800 Subject: [PATCH 09/10] add a fixme --- eth/api.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/eth/api.go b/eth/api.go index 64b2c3ad07e..7f79a000d01 100644 --- a/eth/api.go +++ b/eth/api.go @@ -369,6 +369,8 @@ func generateWitness(blockchain *core.BlockChain, block *types.Block) (*stateles return nil, fmt.Errorf("failed to validate block %d: %w", block.Number(), err) } + // FIXME: testWitness will fail from time to time, the problem is caused by occasional state root mismatch + // after processing the block based on witness. We need to investigate the root cause and fix it. for retries := 0; retries < 5; retries++ { if err = testWitness(blockchain, block, witness); err == nil { return witness, nil From 838c8f1dc1753b394aabd0ffae50cc75604cb82d Mon Sep 17 00:00:00 2001 From: colinlyguo Date: Wed, 26 Mar 2025 16:59:00 +0800 Subject: [PATCH 10/10] print error --- eth/api.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/eth/api.go b/eth/api.go index 7f79a000d01..0b380a57b2a 100644 --- a/eth/api.go +++ b/eth/api.go @@ -374,6 +374,8 @@ func generateWitness(blockchain *core.BlockChain, block *types.Block) (*stateles for retries := 0; retries < 5; retries++ { if err = testWitness(blockchain, block, witness); err == nil { return witness, nil + } else { + log.Warn("Failed to validate witness", "block", block.Number(), "error", err) } } return witness, err