(coreth) Write block header/body/receipts on Accept

graft/coreth/core/blockchain.go

@@ -33,6 +33,7 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"math"
 	"math/big"
 	"runtime"
 	"strings"
@@ -106,6 +107,7 @@ var (
 	blockTrieOpsTimer           = metrics.GetOrRegisterCounter("chain/block/trie", nil)
 	blockValidationTimer        = metrics.GetOrRegisterCounter("chain/block/validations/state", nil)
 	blockWriteTimer             = metrics.GetOrRegisterCounter("chain/block/writes", nil)
+	blockAcceptTimer            = metrics.GetOrRegisterCounter("chain/block/accepts", nil)
 
 	acceptorQueueGauge           = metrics.GetOrRegisterGauge("chain/acceptor/queue/size", nil)
 	acceptorWorkTimer            = metrics.GetOrRegisterCounter("chain/acceptor/work", nil)
@@ -320,11 +322,13 @@ type BlockChain struct {
 
 	currentBlock atomic.Pointer[types.Header] // Current head of the block chain
 
-	bodyCache     *lru.Cache[common.Hash, *types.Body]      // Cache for the most recent block bodies
-	receiptsCache *lru.Cache[common.Hash, []*types.Receipt] // Cache for the most recent receipts per block
-	blockCache    *lru.Cache[common.Hash, *types.Block]     // Cache for the most recent entire blocks
-	txLookupCache *lru.Cache[common.Hash, txLookup]         // Cache for the most recent transaction lookup data.
-	badBlocks     *lru.Cache[common.Hash, *badBlock]        // Cache for bad blocks
+	bodyCache             *lru.Cache[common.Hash, *types.Body]      // Cache for the most recent block bodies
+	receiptsCache         *lru.Cache[common.Hash, []*types.Receipt] // Cache for the most recent receipts per block
+	blockCache            *lru.Cache[common.Hash, *types.Block]     // Cache for the most recent entire blocks
+	txLookupCache         *lru.Cache[common.Hash, txLookup]         // Cache for the most recent transaction lookup data.
+	badBlocks             *lru.Cache[common.Hash, *badBlock]        // Cache for bad blocks
+	verifiedBlockCache    *lru.Cache[common.Hash, *types.Block]     // cache for verified but not accepted blocks
+	verifiedReceiptsCache *lru.Cache[common.Hash, types.Receipts]   // cache for verified but not accepted receipts
 
 	stopping atomic.Bool // false if chain is running, true when stopped
 
@@ -413,21 +417,23 @@ func NewBlockChain(
 	log.Info("")
 
 	bc := &BlockChain{
-		chainConfig:       chainConfig,
-		cacheConfig:       cacheConfig,
-		db:                db,
-		triedb:            triedb,
-		bodyCache:         lru.NewCache[common.Hash, *types.Body](bodyCacheLimit),
-		receiptsCache:     lru.NewCache[common.Hash, []*types.Receipt](receiptsCacheLimit),
-		blockCache:        lru.NewCache[common.Hash, *types.Block](blockCacheLimit),
-		txLookupCache:     lru.NewCache[common.Hash, txLookup](txLookupCacheLimit),
-		badBlocks:         lru.NewCache[common.Hash, *badBlock](badBlockLimit),
-		engine:            engine,
-		vmConfig:          vmConfig,
-		senderCacher:      NewTxSenderCacher(runtime.NumCPU()),
-		acceptorQueue:     make(chan *types.Block, cacheConfig.AcceptorQueueLimit),
-		quit:              make(chan struct{}),
-		acceptedLogsCache: NewFIFOCache[common.Hash, [][]*types.Log](cacheConfig.AcceptedCacheSize),
+		chainConfig:           chainConfig,
+		cacheConfig:           cacheConfig,
+		db:                    db,
+		triedb:                triedb,
+		bodyCache:             lru.NewCache[common.Hash, *types.Body](bodyCacheLimit),
+		receiptsCache:         lru.NewCache[common.Hash, []*types.Receipt](receiptsCacheLimit),
+		blockCache:            lru.NewCache[common.Hash, *types.Block](blockCacheLimit),
+		txLookupCache:         lru.NewCache[common.Hash, txLookup](txLookupCacheLimit),
+		badBlocks:             lru.NewCache[common.Hash, *badBlock](badBlockLimit),
+		verifiedBlockCache:    lru.NewCache[common.Hash, *types.Block](math.MaxInt),
+		verifiedReceiptsCache: lru.NewCache[common.Hash, types.Receipts](math.MaxInt),
+		engine:                engine,
+		vmConfig:              vmConfig,
+		senderCacher:          NewTxSenderCacher(runtime.NumCPU()),
+		acceptorQueue:         make(chan *types.Block, cacheConfig.AcceptorQueueLimit),
+		quit:                  make(chan struct{}),
+		acceptedLogsCache:     NewFIFOCache[common.Hash, [][]*types.Log](cacheConfig.AcceptedCacheSize),
 	}
 	bc.stateCache = extstate.NewDatabaseWithNodeDB(bc.db, bc.triedb)
 	bc.validator = NewBlockValidator(chainConfig, bc, engine)
@@ -592,6 +598,20 @@ func (bc *BlockChain) warmAcceptedCaches() {
 	log.Info("Warmed accepted caches", "start", startIndex, "end", lastAccepted, "t", time.Since(startTime))
 }
 
+func (bc *BlockChain) writeAcceptedBlock(b *types.Block) {
+	batch := bc.db.NewBatch()
+	rawdb.WriteBlock(batch, b)
+	if receipts, ok := bc.verifiedReceiptsCache.Get(b.Hash()); ok {
+		rawdb.WriteReceipts(batch, b.Hash(), b.NumberU64(), receipts)
+	}
+	if err := batch.Write(); err != nil {
+		log.Crit("Failed to write accepted block and receipts", "err", err)
+	}
+
+	bc.verifiedBlockCache.Remove(b.Hash())
+	bc.verifiedReceiptsCache.Remove(b.Hash())
+}
+
 // startAcceptor starts processing items on the [acceptorQueue]. If a [nil]
 // object is placed on the [acceptorQueue], the [startAcceptor] will exit.
 func (bc *BlockChain) startAcceptor() {
@@ -723,6 +743,11 @@ func (bc *BlockChain) loadLastState(lastAcceptedHash common.Hash) error {
 		return bc.loadGenesisState()
 	}
 
+	lastAcceptedBlock := bc.GetBlockByHash(lastAcceptedHash)
+	if lastAcceptedBlock == nil {
+		return fmt.Errorf("could not load last accepted block %s", lastAcceptedHash.Hex())
+	}
+
 	// Restore the last known head block
 	head := rawdb.ReadHeadBlockHash(bc.db)
 	if head == (common.Hash{}) {
@@ -731,28 +756,28 @@ func (bc *BlockChain) loadLastState(lastAcceptedHash common.Hash) error {
 	// Make sure the entire head block is available
 	headBlock := bc.GetBlockByHash(head)
 	if headBlock == nil {
-		return fmt.Errorf("could not load head block %s", head.Hex())
+		log.Info(
+			"Head block is missing when loading last state, falling back to the last accepted block",
+			"hash", lastAcceptedBlock.Hash(),
+			"number", lastAcceptedBlock.Number(),
+		)
+
+		// ReadHeadBlockHash stores the hash of the last inserted/verified block.
+		// This means it can be missing if the blockchain crashed before the block
+		// was accepted. If this happens, we set the head block to the last accepted block.
+		headBlock = lastAcceptedBlock
+		bc.writeHeadBlock(headBlock)
 	}
-	// Everything seems to be fine, set as the head block
-	bc.currentBlock.Store(headBlock.Header())
 
-	// Restore the last known head header
+	bc.currentBlock.Store(headBlock.Header())
 	currentHeader := headBlock.Header()
-	if head := rawdb.ReadHeadHeaderHash(bc.db); head != (common.Hash{}) {
-		if header := bc.GetHeaderByHash(head); header != nil {
-			currentHeader = header
-		}
-	}
 	bc.hc.SetCurrentHeader(currentHeader)
 
 	log.Info("Loaded most recent local header", "number", currentHeader.Number, "hash", currentHeader.Hash(), "age", common.PrettyAge(time.Unix(int64(currentHeader.Time), 0)))
 	log.Info("Loaded most recent local full block", "number", headBlock.Number(), "hash", headBlock.Hash(), "age", common.PrettyAge(time.Unix(int64(headBlock.Time()), 0)))
 
 	// Otherwise, set the last accepted block and perform a re-org.
-	bc.lastAccepted = bc.GetBlockByHash(lastAcceptedHash)
-	if bc.lastAccepted == nil {
-		return fmt.Errorf("could not load last accepted block")
-	}
+	bc.lastAccepted = lastAcceptedBlock
 
 	// This ensures that the head block is updated to the last accepted block on startup
 	if err := bc.setPreference(bc.lastAccepted); err != nil {
@@ -1090,6 +1115,8 @@ func (bc *BlockChain) Accept(block *types.Block) error {
 	bc.chainmu.Lock()
 	defer bc.chainmu.Unlock()
 
+	start := time.Now()
+
 	// The parent of [block] must be the last accepted block.
 	if bc.lastAccepted.Hash() != block.ParentHash() {
 		return fmt.Errorf(
@@ -1110,6 +1137,7 @@ func (bc *BlockChain) Accept(block *types.Block) error {
 			return fmt.Errorf("could not set new preferred block %d:%s as preferred: %w", block.Number(), block.Hash(), err)
 		}
 	}
+	bc.writeAcceptedBlock(block)
 
 	// Enqueue block in the acceptor
 	bc.lastAccepted = block
@@ -1140,6 +1168,7 @@ func (bc *BlockChain) Accept(block *types.Block) error {
 			latestMinDelayExcessGauge.Update(int64(delayExcess))
 		}
 	}
+	blockAcceptTimer.Inc(time.Since(start).Milliseconds())
 	return nil
 }
 
@@ -1167,6 +1196,8 @@ func (bc *BlockChain) Reject(block *types.Block) error {
 
 	// Remove the block from the block cache (ignore return value of whether it was in the cache)
 	_ = bc.blockCache.Remove(block.Hash())
+	bc.verifiedBlockCache.Remove(block.Hash())
+	bc.verifiedReceiptsCache.Remove(block.Hash())
 
 	return nil
 }
@@ -1225,13 +1256,13 @@ func (bc *BlockChain) writeBlockAndSetHead(block *types.Block, parentRoot common
 // writeBlockWithState writes the block and all associated state to the database,
 // but it expects the chain mutex to be held.
 func (bc *BlockChain) writeBlockWithState(block *types.Block, parentRoot common.Hash, receipts []*types.Receipt, state *state.StateDB) error {
-	// Irrelevant of the canonical status, write the block itself to the database.
-	//
-	// Note all the components of block(hash->number map, header, body, receipts)
-	// should be written atomically. BlockBatch is used for containing all components.
+	// Cache block and receipts before they are written to disk in Accept
+	bc.verifiedBlockCache.Add(block.Hash(), block)
+	bc.verifiedReceiptsCache.Add(block.Hash(), receipts)
+
+	// Write all other block data to disk
 	blockBatch := bc.db.NewBatch()
-	rawdb.WriteBlock(blockBatch, block)
-	rawdb.WriteReceipts(blockBatch, block.Hash(), block.NumberU64(), receipts)
+	rawdb.WriteHeaderNumber(blockBatch, block.Hash(), block.NumberU64())
 	rawdb.WritePreimages(blockBatch, state.Preimages())
 	if err := blockBatch.Write(); err != nil {
 		log.Crit("Failed to write block into disk", "err", err)
@@ -1465,7 +1496,7 @@ func (bc *BlockChain) collectUnflattenedLogs(b *types.Block, removed bool) [][]*
 	if excessBlobGas != nil {
 		blobGasPrice = eip4844.CalcBlobFee(*excessBlobGas)
 	}
-	receipts := rawdb.ReadRawReceipts(bc.db, b.Hash(), b.NumberU64())
+	receipts := bc.GetReceiptsByHash(b.Hash())
 	if err := receipts.DeriveFields(bc.chainConfig, b.Hash(), b.NumberU64(), b.Time(), b.BaseFee(), blobGasPrice, b.Transactions()); err != nil {
 		log.Error("Failed to derive block receipts fields", "hash", b.Hash(), "number", b.NumberU64(), "err", err)
 	}

graft/coreth/core/blockchain_ext_test.go

@@ -1701,6 +1701,10 @@ func ReexecCorruptedStateTest(t *testing.T, create ReexecTestFunc) {
 	require.NoError(t, blockchain.writeBlockAcceptedIndices(chain[1]))
 	blockchain.Stop()
 
+	// Write accepted block to disk as we are no longer writing it on verify
+	rawdb.WriteBlock(blockchain.db, chain[1])
+	blockchain.Stop()
+
 	// Restart blockchain with existing state
 	newDir := copyDir(t, tempDir) // avoid file lock
 	restartedBlockchain, err := create(chainDB, gspec, chain[1].Hash(), newDir, 4096)

graft/coreth/core/blockchain_reader.go

@@ -61,18 +61,28 @@ func (bc *BlockChain) HasHeader(hash common.Hash, number uint64) bool {
 // GetHeader retrieves a block header from the database by hash and number,
 // caching it if found.
 func (bc *BlockChain) GetHeader(hash common.Hash, number uint64) *types.Header {
+	if block, ok := bc.verifiedBlockCache.Get(hash); ok {
+		return block.Header()
+	}
 	return bc.hc.GetHeader(hash, number)
 }
 
 // GetHeaderByHash retrieves a block header from the database by hash, caching it if
 // found.
 func (bc *BlockChain) GetHeaderByHash(hash common.Hash) *types.Header {
+	if block, ok := bc.verifiedBlockCache.Get(hash); ok {
+		return block.Header()
+	}
 	return bc.hc.GetHeaderByHash(hash)
 }
 
 // GetHeaderByNumber retrieves a block header from the database by number,
 // caching it (associated with its hash) if found.
 func (bc *BlockChain) GetHeaderByNumber(number uint64) *types.Header {
+	hash := rawdb.ReadCanonicalHash(bc.db, number)
+	if block, ok := bc.verifiedBlockCache.Get(hash); ok {
+		return block.Header()
+	}
 	return bc.hc.GetHeaderByNumber(number)
 }
 
@@ -83,6 +93,9 @@ func (bc *BlockChain) GetBody(hash common.Hash) *types.Body {
 	if cached, ok := bc.bodyCache.Get(hash); ok {
 		return cached
 	}
+	if block, ok := bc.verifiedBlockCache.Get(hash); ok {
+		return block.Body()
+	}
 	number := bc.hc.GetBlockNumber(hash)
 	if number == nil {
 		return nil
@@ -101,6 +114,9 @@ func (bc *BlockChain) HasBlock(hash common.Hash, number uint64) bool {
 	if bc.blockCache.Contains(hash) {
 		return true
 	}
+	if bc.verifiedBlockCache.Contains(hash) {
+		return true
+	}
 	if !bc.HasHeader(hash, number) {
 		return false
 	}
@@ -125,6 +141,9 @@ func (bc *BlockChain) GetBlock(hash common.Hash, number uint64) *types.Block {
 	if block, ok := bc.blockCache.Get(hash); ok {
 		return block
 	}
+	if block, ok := bc.verifiedBlockCache.Get(hash); ok {
+		return block
+	}
 	block := rawdb.ReadBlock(bc.db, hash, number)
 	if block == nil {
 		return nil
@@ -177,6 +196,9 @@ func (bc *BlockChain) GetReceiptsByHash(hash common.Hash) types.Receipts {
 	if receipts, ok := bc.receiptsCache.Get(hash); ok {
 		return receipts
 	}
+	if receipts, ok := bc.verifiedReceiptsCache.Get(hash); ok {
+		return receipts
+	}
 	number := rawdb.ReadHeaderNumber(bc.db, hash)
 	if number == nil {
 		return nil

graft/coreth/core/blockchain_repair_test.go

@@ -638,8 +638,11 @@ func testRepairWithScheme(t *testing.T, tt *rewindTest, snapshots bool, scheme s
 	// Iterate over all the remaining blocks and ensure there are no gaps
 	verifyNoGaps(t, newChain, true, canonblocks)
 	verifyNoGaps(t, newChain, false, sideblocks)
-	verifyCutoff(t, newChain, true, canonblocks, tt.expCanonicalBlocks)
-	verifyCutoff(t, newChain, false, sideblocks, tt.expSidechainBlocks)
+	// Only accepted blocks are persisted after restart.
+	cutoffHead := int(newChain.LastAcceptedBlock().NumberU64())
+	verifyCutoff(t, newChain, true, canonblocks, cutoffHead)
+	// Sidechain blocks are not persisted after restart; expect absence.
+	verifyCutoff(t, newChain, false, sideblocks, 0)
 
 	if head := newChain.CurrentHeader(); head.Number.Uint64() != tt.expHeadBlock {
 		t.Errorf("Head header mismatch: have %d, want %d", head.Number, tt.expHeadBlock)

graft/coreth/core/blockchain_snapshot_test.go

@@ -143,7 +143,9 @@ func (basic *snapshotTestBasic) prepare(t *testing.T) (*BlockChain, []*types.Blo
 func (basic *snapshotTestBasic) verify(t *testing.T, chain *BlockChain, blocks []*types.Block) {
 	// Iterate over all the remaining blocks and ensure there are no gaps
 	verifyNoGaps(t, chain, true, blocks)
-	verifyCutoff(t, chain, true, blocks, basic.expCanonicalBlocks)
+	// Only accepted blocks are persisted after restart.
+	acceptedHead := int(chain.LastAcceptedBlock().NumberU64())
+	verifyCutoff(t, chain, true, blocks, acceptedHead)
 
 	if head := chain.CurrentHeader(); head.Number.Uint64() != basic.expHeadBlock {
 		t.Errorf("Head header mismatch: have %d, want %d", head.Number, basic.expHeadBlock)

graft/coreth/core/headerchain_test.go

@@ -54,7 +54,7 @@ func verifyUnbrokenCanonchain(bc *BlockChain) error {
 		if h.Number.Uint64() == 0 {
 			break
 		}
-		h = bc.hc.GetHeader(h.ParentHash, h.Number.Uint64()-1)
+		h = bc.GetHeader(h.ParentHash, h.Number.Uint64()-1)
 	}
 	return nil
 }

graft/coreth/eth/filters/filter_test.go

@@ -273,6 +273,16 @@ func TestFilters(t *testing.T) {
 		t.Fatal(err)
 	}
 
+	// Persist headers/bodies/receipts for testBackend lookups.
+	// The filter testBackend resolves headers directly from the database via
+	// hash->number and header reads, and our insert path does not write headers
+	// for unaccepted blocks.
+	for _, block := range chain {
+		rawdb.WriteBlock(db, block)
+		receipts := bc.GetReceiptsByHash(block.Hash())
+		rawdb.WriteReceipts(db, block.Hash(), block.NumberU64(), receipts)
+	}
+
 	// Set block 998 as Finalized (-3)
 	// bc.SetFinalized(chain[998].Header())
 	err = customrawdb.WriteAcceptorTip(db, chain[998].Hash())

graft/coreth/plugin/evm/vm_warp_test.go

@@ -12,7 +12,6 @@ import (
 	"time"
 
 	"github.com/ava-labs/libevm/common"
-	"github.com/ava-labs/libevm/core/rawdb"
 	"github.com/ava-labs/libevm/core/types"
 	"github.com/ava-labs/libevm/crypto"
 	"github.com/stretchr/testify/require"
@@ -125,7 +124,7 @@ func testSendWarpMessage(t *testing.T, scheme string) {
 	// Verify that the constructed block contains the expected log with an unsigned warp message in the log data
 	ethBlock1 := blk.(*chain.BlockWrapper).Block.(*wrappedBlock).ethBlock
 	require.Len(ethBlock1.Transactions(), 1)
-	receipts := rawdb.ReadReceipts(vm.chaindb, ethBlock1.Hash(), ethBlock1.NumberU64(), ethBlock1.Time(), vm.chainConfig)
+	receipts := vm.blockChain.GetReceiptsByHash(ethBlock1.Hash())
 	require.Len(receipts, 1)
 
 	require.Len(receipts[0].Logs, 1)

graft/coreth/plugin/evm/wrapped_block.go

@@ -12,7 +12,6 @@ import (
 	"time"
 
 	"github.com/ava-labs/libevm/common"
-	"github.com/ava-labs/libevm/core/rawdb"
 	"github.com/ava-labs/libevm/core/types"
 	"github.com/ava-labs/libevm/log"
 	"github.com/ava-labs/libevm/rlp"
@@ -146,11 +145,11 @@ func (b *wrappedBlock) handlePrecompileAccept(rules extras.Rules) error {
 	}
 
 	// Read receipts from disk
-	receipts := rawdb.ReadReceipts(b.vm.chaindb, b.ethBlock.Hash(), b.ethBlock.NumberU64(), b.ethBlock.Time(), b.vm.chainConfig)
+	receipts := b.vm.blockChain.GetReceiptsByHash(b.ethBlock.Hash())
 	// If there are no receipts, ReadReceipts may be nil, so we check the length and confirm the ReceiptHash
 	// is empty to ensure that missing receipts results in an error on accept.
 	if len(receipts) == 0 && b.ethBlock.ReceiptHash() != types.EmptyRootHash {
-		return fmt.Errorf("failed to fetch receipts for accepted block with non-empty root hash (%s) (Block: %s, Height: %d)", b.ethBlock.ReceiptHash(), b.ethBlock.Hash(), b.ethBlock.NumberU64())
+		return fmt.Errorf("failed to fetch receipts for verified block with non-empty root hash (%s) (Block: %s, Height: %d)", b.ethBlock.ReceiptHash(), b.ethBlock.Hash(), b.ethBlock.NumberU64())
 	}
 	acceptCtx := &precompileconfig.AcceptContext{
 		SnowCtx: b.vm.ctx,