diff --git a/graft/coreth/core/blockchain.go b/graft/coreth/core/blockchain.go index f39e82e05bdb..c993b611c222 100644 --- a/graft/coreth/core/blockchain.go +++ b/graft/coreth/core/blockchain.go @@ -33,6 +33,7 @@ import ( "errors" "fmt" "io" + "math" "math/big" "runtime" "strings" @@ -106,6 +107,7 @@ var ( blockTrieOpsTimer = metrics.GetOrRegisterCounter("chain/block/trie", nil) blockValidationTimer = metrics.GetOrRegisterCounter("chain/block/validations/state", nil) blockWriteTimer = metrics.GetOrRegisterCounter("chain/block/writes", nil) + blockAcceptTimer = metrics.GetOrRegisterCounter("chain/block/accepts", nil) acceptorQueueGauge = metrics.GetOrRegisterGauge("chain/acceptor/queue/size", nil) acceptorWorkTimer = metrics.GetOrRegisterCounter("chain/acceptor/work", nil) @@ -320,11 +322,13 @@ type BlockChain struct { currentBlock atomic.Pointer[types.Header] // Current head of the block chain - bodyCache *lru.Cache[common.Hash, *types.Body] // Cache for the most recent block bodies - receiptsCache *lru.Cache[common.Hash, []*types.Receipt] // Cache for the most recent receipts per block - blockCache *lru.Cache[common.Hash, *types.Block] // Cache for the most recent entire blocks - txLookupCache *lru.Cache[common.Hash, txLookup] // Cache for the most recent transaction lookup data. - badBlocks *lru.Cache[common.Hash, *badBlock] // Cache for bad blocks + bodyCache *lru.Cache[common.Hash, *types.Body] // Cache for the most recent block bodies + receiptsCache *lru.Cache[common.Hash, []*types.Receipt] // Cache for the most recent receipts per block + blockCache *lru.Cache[common.Hash, *types.Block] // Cache for the most recent entire blocks + txLookupCache *lru.Cache[common.Hash, txLookup] // Cache for the most recent transaction lookup data. + badBlocks *lru.Cache[common.Hash, *badBlock] // Cache for bad blocks + verifiedBlockCache *lru.Cache[common.Hash, *types.Block] // cache for verified but not accepted blocks + verifiedReceiptsCache *lru.Cache[common.Hash, types.Receipts] // cache for verified but not accepted receipts stopping atomic.Bool // false if chain is running, true when stopped @@ -413,21 +417,23 @@ func NewBlockChain( log.Info("") bc := &BlockChain{ - chainConfig: chainConfig, - cacheConfig: cacheConfig, - db: db, - triedb: triedb, - bodyCache: lru.NewCache[common.Hash, *types.Body](bodyCacheLimit), - receiptsCache: lru.NewCache[common.Hash, []*types.Receipt](receiptsCacheLimit), - blockCache: lru.NewCache[common.Hash, *types.Block](blockCacheLimit), - txLookupCache: lru.NewCache[common.Hash, txLookup](txLookupCacheLimit), - badBlocks: lru.NewCache[common.Hash, *badBlock](badBlockLimit), - engine: engine, - vmConfig: vmConfig, - senderCacher: NewTxSenderCacher(runtime.NumCPU()), - acceptorQueue: make(chan *types.Block, cacheConfig.AcceptorQueueLimit), - quit: make(chan struct{}), - acceptedLogsCache: NewFIFOCache[common.Hash, [][]*types.Log](cacheConfig.AcceptedCacheSize), + chainConfig: chainConfig, + cacheConfig: cacheConfig, + db: db, + triedb: triedb, + bodyCache: lru.NewCache[common.Hash, *types.Body](bodyCacheLimit), + receiptsCache: lru.NewCache[common.Hash, []*types.Receipt](receiptsCacheLimit), + blockCache: lru.NewCache[common.Hash, *types.Block](blockCacheLimit), + txLookupCache: lru.NewCache[common.Hash, txLookup](txLookupCacheLimit), + badBlocks: lru.NewCache[common.Hash, *badBlock](badBlockLimit), + verifiedBlockCache: lru.NewCache[common.Hash, *types.Block](math.MaxInt), + verifiedReceiptsCache: lru.NewCache[common.Hash, types.Receipts](math.MaxInt), + engine: engine, + vmConfig: vmConfig, + senderCacher: NewTxSenderCacher(runtime.NumCPU()), + acceptorQueue: make(chan *types.Block, cacheConfig.AcceptorQueueLimit), + quit: make(chan struct{}), + acceptedLogsCache: NewFIFOCache[common.Hash, [][]*types.Log](cacheConfig.AcceptedCacheSize), } bc.stateCache = extstate.NewDatabaseWithNodeDB(bc.db, bc.triedb) bc.validator = NewBlockValidator(chainConfig, bc, engine) @@ -592,6 +598,20 @@ func (bc *BlockChain) warmAcceptedCaches() { log.Info("Warmed accepted caches", "start", startIndex, "end", lastAccepted, "t", time.Since(startTime)) } +func (bc *BlockChain) writeAcceptedBlock(b *types.Block) { + batch := bc.db.NewBatch() + rawdb.WriteBlock(batch, b) + if receipts, ok := bc.verifiedReceiptsCache.Get(b.Hash()); ok { + rawdb.WriteReceipts(batch, b.Hash(), b.NumberU64(), receipts) + } + if err := batch.Write(); err != nil { + log.Crit("Failed to write accepted block and receipts", "err", err) + } + + bc.verifiedBlockCache.Remove(b.Hash()) + bc.verifiedReceiptsCache.Remove(b.Hash()) +} + // startAcceptor starts processing items on the [acceptorQueue]. If a [nil] // object is placed on the [acceptorQueue], the [startAcceptor] will exit. func (bc *BlockChain) startAcceptor() { @@ -723,6 +743,11 @@ func (bc *BlockChain) loadLastState(lastAcceptedHash common.Hash) error { return bc.loadGenesisState() } + lastAcceptedBlock := bc.GetBlockByHash(lastAcceptedHash) + if lastAcceptedBlock == nil { + return fmt.Errorf("could not load last accepted block %s", lastAcceptedHash.Hex()) + } + // Restore the last known head block head := rawdb.ReadHeadBlockHash(bc.db) if head == (common.Hash{}) { @@ -731,28 +756,28 @@ func (bc *BlockChain) loadLastState(lastAcceptedHash common.Hash) error { // Make sure the entire head block is available headBlock := bc.GetBlockByHash(head) if headBlock == nil { - return fmt.Errorf("could not load head block %s", head.Hex()) + log.Info( + "Head block is missing when loading last state, falling back to the last accepted block", + "hash", lastAcceptedBlock.Hash(), + "number", lastAcceptedBlock.Number(), + ) + + // ReadHeadBlockHash stores the hash of the last inserted/verified block. + // This means it can be missing if the blockchain crashed before the block + // was accepted. If this happens, we set the head block to the last accepted block. + headBlock = lastAcceptedBlock + bc.writeHeadBlock(headBlock) } - // Everything seems to be fine, set as the head block - bc.currentBlock.Store(headBlock.Header()) - // Restore the last known head header + bc.currentBlock.Store(headBlock.Header()) currentHeader := headBlock.Header() - if head := rawdb.ReadHeadHeaderHash(bc.db); head != (common.Hash{}) { - if header := bc.GetHeaderByHash(head); header != nil { - currentHeader = header - } - } bc.hc.SetCurrentHeader(currentHeader) log.Info("Loaded most recent local header", "number", currentHeader.Number, "hash", currentHeader.Hash(), "age", common.PrettyAge(time.Unix(int64(currentHeader.Time), 0))) log.Info("Loaded most recent local full block", "number", headBlock.Number(), "hash", headBlock.Hash(), "age", common.PrettyAge(time.Unix(int64(headBlock.Time()), 0))) // Otherwise, set the last accepted block and perform a re-org. - bc.lastAccepted = bc.GetBlockByHash(lastAcceptedHash) - if bc.lastAccepted == nil { - return fmt.Errorf("could not load last accepted block") - } + bc.lastAccepted = lastAcceptedBlock // This ensures that the head block is updated to the last accepted block on startup if err := bc.setPreference(bc.lastAccepted); err != nil { @@ -1090,6 +1115,8 @@ func (bc *BlockChain) Accept(block *types.Block) error { bc.chainmu.Lock() defer bc.chainmu.Unlock() + start := time.Now() + // The parent of [block] must be the last accepted block. if bc.lastAccepted.Hash() != block.ParentHash() { return fmt.Errorf( @@ -1110,6 +1137,7 @@ func (bc *BlockChain) Accept(block *types.Block) error { return fmt.Errorf("could not set new preferred block %d:%s as preferred: %w", block.Number(), block.Hash(), err) } } + bc.writeAcceptedBlock(block) // Enqueue block in the acceptor bc.lastAccepted = block @@ -1140,6 +1168,7 @@ func (bc *BlockChain) Accept(block *types.Block) error { latestMinDelayExcessGauge.Update(int64(delayExcess)) } } + blockAcceptTimer.Inc(time.Since(start).Milliseconds()) return nil } @@ -1167,6 +1196,8 @@ func (bc *BlockChain) Reject(block *types.Block) error { // Remove the block from the block cache (ignore return value of whether it was in the cache) _ = bc.blockCache.Remove(block.Hash()) + bc.verifiedBlockCache.Remove(block.Hash()) + bc.verifiedReceiptsCache.Remove(block.Hash()) return nil } @@ -1225,13 +1256,13 @@ func (bc *BlockChain) writeBlockAndSetHead(block *types.Block, parentRoot common // writeBlockWithState writes the block and all associated state to the database, // but it expects the chain mutex to be held. func (bc *BlockChain) writeBlockWithState(block *types.Block, parentRoot common.Hash, receipts []*types.Receipt, state *state.StateDB) error { - // Irrelevant of the canonical status, write the block itself to the database. - // - // Note all the components of block(hash->number map, header, body, receipts) - // should be written atomically. BlockBatch is used for containing all components. + // Cache block and receipts before they are written to disk in Accept + bc.verifiedBlockCache.Add(block.Hash(), block) + bc.verifiedReceiptsCache.Add(block.Hash(), receipts) + + // Write all other block data to disk blockBatch := bc.db.NewBatch() - rawdb.WriteBlock(blockBatch, block) - rawdb.WriteReceipts(blockBatch, block.Hash(), block.NumberU64(), receipts) + rawdb.WriteHeaderNumber(blockBatch, block.Hash(), block.NumberU64()) rawdb.WritePreimages(blockBatch, state.Preimages()) if err := blockBatch.Write(); err != nil { log.Crit("Failed to write block into disk", "err", err) @@ -1465,7 +1496,7 @@ func (bc *BlockChain) collectUnflattenedLogs(b *types.Block, removed bool) [][]* if excessBlobGas != nil { blobGasPrice = eip4844.CalcBlobFee(*excessBlobGas) } - receipts := rawdb.ReadRawReceipts(bc.db, b.Hash(), b.NumberU64()) + receipts := bc.GetReceiptsByHash(b.Hash()) if err := receipts.DeriveFields(bc.chainConfig, b.Hash(), b.NumberU64(), b.Time(), b.BaseFee(), blobGasPrice, b.Transactions()); err != nil { log.Error("Failed to derive block receipts fields", "hash", b.Hash(), "number", b.NumberU64(), "err", err) } diff --git a/graft/coreth/core/blockchain_ext_test.go b/graft/coreth/core/blockchain_ext_test.go index 1920a8ae60f1..7de7d51c3544 100644 --- a/graft/coreth/core/blockchain_ext_test.go +++ b/graft/coreth/core/blockchain_ext_test.go @@ -1701,6 +1701,10 @@ func ReexecCorruptedStateTest(t *testing.T, create ReexecTestFunc) { require.NoError(t, blockchain.writeBlockAcceptedIndices(chain[1])) blockchain.Stop() + // Write accepted block to disk as we are no longer writing it on verify + rawdb.WriteBlock(blockchain.db, chain[1]) + blockchain.Stop() + // Restart blockchain with existing state newDir := copyDir(t, tempDir) // avoid file lock restartedBlockchain, err := create(chainDB, gspec, chain[1].Hash(), newDir, 4096) diff --git a/graft/coreth/core/blockchain_reader.go b/graft/coreth/core/blockchain_reader.go index 8b3eb48f7096..9bf700146e91 100644 --- a/graft/coreth/core/blockchain_reader.go +++ b/graft/coreth/core/blockchain_reader.go @@ -61,18 +61,28 @@ func (bc *BlockChain) HasHeader(hash common.Hash, number uint64) bool { // GetHeader retrieves a block header from the database by hash and number, // caching it if found. func (bc *BlockChain) GetHeader(hash common.Hash, number uint64) *types.Header { + if block, ok := bc.verifiedBlockCache.Get(hash); ok { + return block.Header() + } return bc.hc.GetHeader(hash, number) } // GetHeaderByHash retrieves a block header from the database by hash, caching it if // found. func (bc *BlockChain) GetHeaderByHash(hash common.Hash) *types.Header { + if block, ok := bc.verifiedBlockCache.Get(hash); ok { + return block.Header() + } return bc.hc.GetHeaderByHash(hash) } // GetHeaderByNumber retrieves a block header from the database by number, // caching it (associated with its hash) if found. func (bc *BlockChain) GetHeaderByNumber(number uint64) *types.Header { + hash := rawdb.ReadCanonicalHash(bc.db, number) + if block, ok := bc.verifiedBlockCache.Get(hash); ok { + return block.Header() + } return bc.hc.GetHeaderByNumber(number) } @@ -83,6 +93,9 @@ func (bc *BlockChain) GetBody(hash common.Hash) *types.Body { if cached, ok := bc.bodyCache.Get(hash); ok { return cached } + if block, ok := bc.verifiedBlockCache.Get(hash); ok { + return block.Body() + } number := bc.hc.GetBlockNumber(hash) if number == nil { return nil @@ -101,6 +114,9 @@ func (bc *BlockChain) HasBlock(hash common.Hash, number uint64) bool { if bc.blockCache.Contains(hash) { return true } + if bc.verifiedBlockCache.Contains(hash) { + return true + } if !bc.HasHeader(hash, number) { return false } @@ -125,6 +141,9 @@ func (bc *BlockChain) GetBlock(hash common.Hash, number uint64) *types.Block { if block, ok := bc.blockCache.Get(hash); ok { return block } + if block, ok := bc.verifiedBlockCache.Get(hash); ok { + return block + } block := rawdb.ReadBlock(bc.db, hash, number) if block == nil { return nil @@ -177,6 +196,9 @@ func (bc *BlockChain) GetReceiptsByHash(hash common.Hash) types.Receipts { if receipts, ok := bc.receiptsCache.Get(hash); ok { return receipts } + if receipts, ok := bc.verifiedReceiptsCache.Get(hash); ok { + return receipts + } number := rawdb.ReadHeaderNumber(bc.db, hash) if number == nil { return nil diff --git a/graft/coreth/core/blockchain_repair_test.go b/graft/coreth/core/blockchain_repair_test.go index 0f72012b70d4..514dfb85ecd1 100644 --- a/graft/coreth/core/blockchain_repair_test.go +++ b/graft/coreth/core/blockchain_repair_test.go @@ -638,8 +638,11 @@ func testRepairWithScheme(t *testing.T, tt *rewindTest, snapshots bool, scheme s // Iterate over all the remaining blocks and ensure there are no gaps verifyNoGaps(t, newChain, true, canonblocks) verifyNoGaps(t, newChain, false, sideblocks) - verifyCutoff(t, newChain, true, canonblocks, tt.expCanonicalBlocks) - verifyCutoff(t, newChain, false, sideblocks, tt.expSidechainBlocks) + // Only accepted blocks are persisted after restart. + cutoffHead := int(newChain.LastAcceptedBlock().NumberU64()) + verifyCutoff(t, newChain, true, canonblocks, cutoffHead) + // Sidechain blocks are not persisted after restart; expect absence. + verifyCutoff(t, newChain, false, sideblocks, 0) if head := newChain.CurrentHeader(); head.Number.Uint64() != tt.expHeadBlock { t.Errorf("Head header mismatch: have %d, want %d", head.Number, tt.expHeadBlock) diff --git a/graft/coreth/core/blockchain_snapshot_test.go b/graft/coreth/core/blockchain_snapshot_test.go index bbfdc596d905..e28682d6b3a7 100644 --- a/graft/coreth/core/blockchain_snapshot_test.go +++ b/graft/coreth/core/blockchain_snapshot_test.go @@ -143,7 +143,9 @@ func (basic *snapshotTestBasic) prepare(t *testing.T) (*BlockChain, []*types.Blo func (basic *snapshotTestBasic) verify(t *testing.T, chain *BlockChain, blocks []*types.Block) { // Iterate over all the remaining blocks and ensure there are no gaps verifyNoGaps(t, chain, true, blocks) - verifyCutoff(t, chain, true, blocks, basic.expCanonicalBlocks) + // Only accepted blocks are persisted after restart. + acceptedHead := int(chain.LastAcceptedBlock().NumberU64()) + verifyCutoff(t, chain, true, blocks, acceptedHead) if head := chain.CurrentHeader(); head.Number.Uint64() != basic.expHeadBlock { t.Errorf("Head header mismatch: have %d, want %d", head.Number, basic.expHeadBlock) diff --git a/graft/coreth/core/headerchain_test.go b/graft/coreth/core/headerchain_test.go index 497a4c2fd306..5667bdf96a83 100644 --- a/graft/coreth/core/headerchain_test.go +++ b/graft/coreth/core/headerchain_test.go @@ -54,7 +54,7 @@ func verifyUnbrokenCanonchain(bc *BlockChain) error { if h.Number.Uint64() == 0 { break } - h = bc.hc.GetHeader(h.ParentHash, h.Number.Uint64()-1) + h = bc.GetHeader(h.ParentHash, h.Number.Uint64()-1) } return nil } diff --git a/graft/coreth/eth/filters/filter_test.go b/graft/coreth/eth/filters/filter_test.go index 95b2c9177d58..4c54d10bca43 100644 --- a/graft/coreth/eth/filters/filter_test.go +++ b/graft/coreth/eth/filters/filter_test.go @@ -273,6 +273,16 @@ func TestFilters(t *testing.T) { t.Fatal(err) } + // Persist headers/bodies/receipts for testBackend lookups. + // The filter testBackend resolves headers directly from the database via + // hash->number and header reads, and our insert path does not write headers + // for unaccepted blocks. + for _, block := range chain { + rawdb.WriteBlock(db, block) + receipts := bc.GetReceiptsByHash(block.Hash()) + rawdb.WriteReceipts(db, block.Hash(), block.NumberU64(), receipts) + } + // Set block 998 as Finalized (-3) // bc.SetFinalized(chain[998].Header()) err = customrawdb.WriteAcceptorTip(db, chain[998].Hash()) diff --git a/graft/coreth/plugin/evm/vm_warp_test.go b/graft/coreth/plugin/evm/vm_warp_test.go index 353f0e45f977..59f5eeee1379 100644 --- a/graft/coreth/plugin/evm/vm_warp_test.go +++ b/graft/coreth/plugin/evm/vm_warp_test.go @@ -12,7 +12,6 @@ import ( "time" "github.com/ava-labs/libevm/common" - "github.com/ava-labs/libevm/core/rawdb" "github.com/ava-labs/libevm/core/types" "github.com/ava-labs/libevm/crypto" "github.com/stretchr/testify/require" @@ -125,7 +124,7 @@ func testSendWarpMessage(t *testing.T, scheme string) { // Verify that the constructed block contains the expected log with an unsigned warp message in the log data ethBlock1 := blk.(*chain.BlockWrapper).Block.(*wrappedBlock).ethBlock require.Len(ethBlock1.Transactions(), 1) - receipts := rawdb.ReadReceipts(vm.chaindb, ethBlock1.Hash(), ethBlock1.NumberU64(), ethBlock1.Time(), vm.chainConfig) + receipts := vm.blockChain.GetReceiptsByHash(ethBlock1.Hash()) require.Len(receipts, 1) require.Len(receipts[0].Logs, 1) diff --git a/graft/coreth/plugin/evm/wrapped_block.go b/graft/coreth/plugin/evm/wrapped_block.go index 2db44fa8b263..61e50ab753b6 100644 --- a/graft/coreth/plugin/evm/wrapped_block.go +++ b/graft/coreth/plugin/evm/wrapped_block.go @@ -12,7 +12,6 @@ import ( "time" "github.com/ava-labs/libevm/common" - "github.com/ava-labs/libevm/core/rawdb" "github.com/ava-labs/libevm/core/types" "github.com/ava-labs/libevm/log" "github.com/ava-labs/libevm/rlp" @@ -146,11 +145,11 @@ func (b *wrappedBlock) handlePrecompileAccept(rules extras.Rules) error { } // Read receipts from disk - receipts := rawdb.ReadReceipts(b.vm.chaindb, b.ethBlock.Hash(), b.ethBlock.NumberU64(), b.ethBlock.Time(), b.vm.chainConfig) + receipts := b.vm.blockChain.GetReceiptsByHash(b.ethBlock.Hash()) // If there are no receipts, ReadReceipts may be nil, so we check the length and confirm the ReceiptHash // is empty to ensure that missing receipts results in an error on accept. if len(receipts) == 0 && b.ethBlock.ReceiptHash() != types.EmptyRootHash { - return fmt.Errorf("failed to fetch receipts for accepted block with non-empty root hash (%s) (Block: %s, Height: %d)", b.ethBlock.ReceiptHash(), b.ethBlock.Hash(), b.ethBlock.NumberU64()) + return fmt.Errorf("failed to fetch receipts for verified block with non-empty root hash (%s) (Block: %s, Height: %d)", b.ethBlock.ReceiptHash(), b.ethBlock.Hash(), b.ethBlock.NumberU64()) } acceptCtx := &precompileconfig.AcceptContext{ SnowCtx: b.vm.ctx,