Move EEI into EVM

packages/client/lib/execution/vmexecution.ts

@@ -34,6 +34,18 @@ export class VMExecution extends Execution {
   /** Maximally tolerated block time before giving a warning on console */
   private MAX_TOLERATED_BLOCK_TIME = 12
 
+  /**
+   * Delete cache items if not read or modfied by
+   * STATE_CACHE_THRESHOLD_NUM_BLOCKS number of blocks
+   */
+  private STATE_CACHE_THRESHOLD_NUM_BLOCKS = 25000
+
+  /**
+   * Display state cache stats every num blocks
+   */
+  private CACHE_STATS_NUM_BLOCKS = 500
+  private cacheStatsCount = 0
+
   /**
    * Create new VM execution module
    */
@@ -48,6 +60,7 @@ export class VMExecution extends Execution {
 
       const stateManager = new DefaultStateManager({
         trie,
+        deactivateCache: false,
       })
 
       this.vm = new (VM as any)({
@@ -280,6 +293,12 @@ export class VMExecution extends Execution {
           if (!headBlock || reorg) {
             const headBlock = await blockchain.getBlock(block.header.parentHash)
             parentState = headBlock.header.stateRoot
+
+            if (reorg) {
+              this.config.logger.info(
+                `Chain reorg happened, set new head to block number=${headBlock.header.number}, clearing state cache for VM execution.`
+              )
+            }
           }
           // run block, update head if valid
           try {
@@ -317,6 +336,7 @@ export class VMExecution extends Execution {
               const result = await this.vm.runBlock({
                 block,
                 root: parentState,
+                cacheClearingOptions: this.cacheStatsAndOptions(this.vm, number, reorg),
                 skipBlockValidation,
                 skipHeaderValidation: true,
               })
@@ -482,7 +502,7 @@ export class VMExecution extends Execution {
       const block = await vm.blockchain.getBlock(blockNumber)
       const parentBlock = await vm.blockchain.getBlock(block.header.parentHash)
       // Set the correct state root
-      await vm.stateManager.setStateRoot(parentBlock.header.stateRoot)
+      const root = parentBlock.header.stateRoot
       if (typeof vm.blockchain.getTotalDifficulty !== 'function') {
         throw new Error('cannot get iterator head: blockchain has no getTotalDifficulty function')
       }
@@ -493,7 +513,12 @@ export class VMExecution extends Execution {
         // we are skipping header validation because the block has been picked from the
         // blockchain and header should have already been validated while putBlock
         const beforeTS = Date.now()
-        const res = await vm.runBlock({ block, skipHeaderValidation: true })
+        const res = await vm.runBlock({
+          block,
+          root,
+          cacheClearingOptions: this.cacheStatsAndOptions(vm, BigInt(blockNumber), false),
+          skipHeaderValidation: true,
+        })
         const afterTS = Date.now()
         const diffSec = Math.round((afterTS - beforeTS) / 1000)
         const msg = `Executed block num=${blockNumber} hash=0x${block.hash().toString('hex')} txs=${
@@ -529,4 +554,28 @@ export class VMExecution extends Execution {
       }
     }
   }
+
+  cacheStatsAndOptions(vm: VM, blockNumber: bigint, reorg: boolean) {
+    this.cacheStatsCount += 1
+    if (this.cacheStatsCount === this.CACHE_STATS_NUM_BLOCKS) {
+      const stats = vm.stateManager.cache!.stats()
+      this.config.logger.info(
+        `State cache stats size=${stats.cache.size} reads=${stats.cache.reads} hits=${stats.cache.hits} writes=${stats.cache.writes} | trie reads=${stats.trie.reads} writes=${stats.trie.writes}`
+      )
+      this.cacheStatsCount = 0
+    }
+    // Only apply cache threshold in selected block intervals
+    // for performance reasons (whole cache iteration needed)
+    let useThreshold
+    if (blockNumber % BigInt(100) === BigInt(0)) {
+      useThreshold = blockNumber - BigInt(this.STATE_CACHE_THRESHOLD_NUM_BLOCKS)
+    }
+
+    const cacheClearingOptions = {
+      clear: reorg ? true : false,
+      useThreshold,
+      comparand: blockNumber,
+    }
+    return cacheClearingOptions
+  }
 }

packages/client/lib/rpc/modules/eth.ts

@@ -25,7 +25,6 @@ import type { Block, JsonRpcBlock } from '@ethereumjs/block'
 import type { Log } from '@ethereumjs/evm'
 import type { Proof } from '@ethereumjs/statemanager'
 import type { FeeMarketEIP1559Transaction, Transaction, TypedTransaction } from '@ethereumjs/tx'
-import type { Account } from '@ethereumjs/util'
 import type { PostByzantiumTxReceipt, PreByzantiumTxReceipt, TxReceipt, VM } from '@ethereumjs/vm'
 
 type GetLogsParams = {
@@ -514,6 +513,9 @@ export class Eth {
     const vm = await this._vm.copy()
     await vm.stateManager.setStateRoot(block.header.stateRoot)
     const account = await vm.stateManager.getAccount(address)
+    if (account === undefined) {
+      throw new Error(`could not read account`)
+    }
     return bigIntToHex(account.balance)
   }
 
@@ -684,7 +686,10 @@ export class Eth {
     await vm.stateManager.setStateRoot(block.header.stateRoot)
 
     const address = Address.fromString(addressHex)
-    const account: Account = await vm.stateManager.getAccount(address)
+    const account = await vm.stateManager.getAccount(address)
+    if (account === undefined) {
+      throw new Error(`could not read account`)
+    }
     return bigIntToHex(account.nonce)
   }
 

packages/client/lib/service/txpool.ts

@@ -1,5 +1,5 @@
 import { BlobEIP4844Transaction, Capability } from '@ethereumjs/tx'
-import { Address, bufferToHex } from '@ethereumjs/util'
+import { Account, Address, bufferToHex } from '@ethereumjs/util'
 import Heap = require('qheap')
 
 import type { Config } from '../config'
@@ -303,7 +303,10 @@ export class TxPool {
     const vmCopy = await this.vm.copy()
     // Set state root to latest block so that account balance is correct when doing balance check
     await vmCopy.stateManager.setStateRoot(block.stateRoot)
-    const account = await vmCopy.stateManager.getAccount(senderAddress)
+    let account = await vmCopy.stateManager.getAccount(senderAddress)
+    if (account === undefined) {
+      account = new Account()
+    }
     if (account.nonce > tx.nonce) {
       throw new Error(
         `0x${sender} tries to send a tx with nonce ${tx.nonce}, but account has nonce ${account.nonce} (tx nonce too low)`
@@ -694,7 +697,11 @@ export class TxPool {
         .map((obj) => obj.tx)
         .sort((a, b) => Number(a.nonce - b.nonce))
       // Check if the account nonce matches the lowest known tx nonce
-      const { nonce } = await vm.eei.getAccount(new Address(Buffer.from(address, 'hex')))
+      let account = await vm.eei.getAccount(new Address(Buffer.from(address, 'hex')))
+      if (account === undefined) {
+        account = new Account()
+      }
+      const { nonce } = account
       if (txsSortedByNonce[0].nonce !== nonce) {
         // Account nonce does not match the lowest known tx nonce,
         // therefore no txs from this address are currently executable

packages/client/test/rpc/engine/getBlobsBundleV1.spec.ts

@@ -1,7 +1,7 @@
 import { Hardfork } from '@ethereumjs/common'
 import { DefaultStateManager } from '@ethereumjs/statemanager'
 import { TransactionFactory, initKZG } from '@ethereumjs/tx'
-import { Address } from '@ethereumjs/util'
+import { Account, Address } from '@ethereumjs/util'
 import * as kzg from 'c-kzg'
 import * as tape from 'tape'
 
@@ -66,10 +66,11 @@ tape(`${method}: call with known payload`, async (t) => {
     'hex'
   )
   const address = Address.fromPrivateKey(pkey)
+  await service.execution.vm.stateManager.putAccount(address, new Account())
   const account = await service.execution.vm.stateManager.getAccount(address)
 
-  account.balance = 0xfffffffffffffffn
-  await service.execution.vm.stateManager.putAccount(address, account)
+  account!.balance = 0xfffffffffffffffn
+  await service.execution.vm.stateManager.putAccount(address, account!)
   let req = params('engine_forkchoiceUpdatedV2', validPayload)
   let payloadId
   let expectRes = (res: any) => {

packages/client/test/rpc/engine/getPayloadBodiesByHashV1.spec.ts

@@ -2,7 +2,7 @@ import { Block, BlockHeader } from '@ethereumjs/block'
 import { Hardfork } from '@ethereumjs/common'
 import { DefaultStateManager } from '@ethereumjs/statemanager'
 import { TransactionFactory } from '@ethereumjs/tx'
-import { Address } from '@ethereumjs/util'
+import { Account, Address } from '@ethereumjs/util'
 import { randomBytes } from 'crypto'
 import * as tape from 'tape'
 
@@ -47,10 +47,11 @@ tape(`${method}: call with valid parameters`, async (t) => {
     'hex'
   )
   const address = Address.fromPrivateKey(pkey)
+  await service.execution.vm.stateManager.putAccount(address, new Account())
   const account = await service.execution.vm.stateManager.getAccount(address)
 
-  account.balance = 0xfffffffffffffffn
-  await service.execution.vm.stateManager.putAccount(address, account)
+  account!.balance = 0xfffffffffffffffn
+  await service.execution.vm.stateManager.putAccount(address, account!)
   const tx = TransactionFactory.fromTxData(
     {
       type: 0x01,
@@ -139,10 +140,11 @@ tape(`${method}: call with valid parameters on pre-Shanghai block`, async (t) =>
     'hex'
   )
   const address = Address.fromPrivateKey(pkey)
+  await service.execution.vm.stateManager.putAccount(address, new Account())
   const account = await service.execution.vm.stateManager.getAccount(address)
 
-  account.balance = 0xfffffffffffffffn
-  await service.execution.vm.stateManager.putAccount(address, account)
+  account!.balance = 0xfffffffffffffffn
+  await service.execution.vm.stateManager.putAccount(address, account!)
   const tx = TransactionFactory.fromTxData(
     {
       type: 0x01,

packages/client/test/rpc/engine/getPayloadBodiesByRangeV1.spec.ts

@@ -2,7 +2,7 @@ import { Block, BlockHeader } from '@ethereumjs/block'
 import { Hardfork } from '@ethereumjs/common'
 import { DefaultStateManager } from '@ethereumjs/statemanager'
 import { TransactionFactory } from '@ethereumjs/tx'
-import { Address } from '@ethereumjs/util'
+import { Account, Address } from '@ethereumjs/util'
 import * as tape from 'tape'
 
 import { INVALID_PARAMS, TOO_LARGE_REQUEST } from '../../../lib/rpc/error-code'
@@ -55,10 +55,11 @@ tape(`${method}: call with valid parameters`, async (t) => {
     'hex'
   )
   const address = Address.fromPrivateKey(pkey)
+  await service.execution.vm.stateManager.putAccount(address, new Account())
   const account = await service.execution.vm.stateManager.getAccount(address)
 
-  account.balance = 0xfffffffffffffffn
-  await service.execution.vm.stateManager.putAccount(address, account)
+  account!.balance = 0xfffffffffffffffn
+  await service.execution.vm.stateManager.putAccount(address, account!)
   const tx = TransactionFactory.fromTxData(
     {
       type: 0x01,
@@ -150,10 +151,11 @@ tape(`${method}: call with valid parameters on pre-Shanghai hardfork`, async (t)
     'hex'
   )
   const address = Address.fromPrivateKey(pkey)
+  await service.execution.vm.stateManager.putAccount(address, new Account())
   const account = await service.execution.vm.stateManager.getAccount(address)
 
-  account.balance = 0xfffffffffffffffn
-  await service.execution.vm.stateManager.putAccount(address, account)
+  account!.balance = 0xfffffffffffffffn
+  await service.execution.vm.stateManager.putAccount(address, account!)
   const tx = TransactionFactory.fromTxData(
     {
       type: 0x01,

packages/client/test/rpc/eth/sendRawTransaction.spec.ts

@@ -12,7 +12,7 @@ import {
   commitmentsToVersionedHashes,
   getBlobs,
 } from '@ethereumjs/tx/dist/utils/blobHelpers'
-import { toBuffer } from '@ethereumjs/util'
+import { Account, toBuffer } from '@ethereumjs/util'
 import * as kzg from 'c-kzg'
 import { randomBytes } from 'crypto'
 import * as tape from 'tape'
@@ -45,9 +45,10 @@ tape(`${method}: call with valid arguments`, async (t) => {
   const address = transaction.getSenderAddress()
   const vm = (client.services.find((s) => s.name === 'eth') as FullEthereumService).execution.vm
 
+  await vm.stateManager.putAccount(address, new Account())
   const account = await vm.stateManager.getAccount(address)
-  account.balance = BigInt('40100000')
-  await vm.stateManager.putAccount(address, account)
+  account!.balance = BigInt('40100000')
+  await vm.stateManager.putAccount(address, account!)
 
   const req = params(method, [txData])
   const expectRes = (res: any) => {
@@ -189,9 +190,10 @@ tape(`${method}: call with no peers`, async (t) => {
   const address = transaction.getSenderAddress()
   const vm = (client.services.find((s) => s.name === 'eth') as FullEthereumService).execution.vm
 
+  await vm.stateManager.putAccount(address, new Account())
   const account = await vm.stateManager.getAccount(address)
-  account.balance = BigInt('40100000')
-  await vm.stateManager.putAccount(address, account)
+  account!.balance = BigInt('40100000')
+  await vm.stateManager.putAccount(address, account!)
 
   const req = params(method, [txData])
 
@@ -268,9 +270,10 @@ tape('blob EIP 4844 transaction', async (t) => {
     { common }
   ).sign(pk)
   const vm = (client.services.find((s) => s.name === 'eth') as FullEthereumService).execution.vm
+  await vm.stateManager.putAccount(tx.getSenderAddress(), new Account())
   const account = await vm.stateManager.getAccount(tx.getSenderAddress())
-  account.balance = BigInt(0xfffffffffffff)
-  await vm.stateManager.putAccount(tx.getSenderAddress(), account)
+  account!.balance = BigInt(0xfffffffffffff)
+  await vm.stateManager.putAccount(tx.getSenderAddress(), account!)
 
   const req = params(method, ['0x' + tx.serializeNetworkWrapper().toString('hex')])
   const req2 = params(method, ['0x' + replacementTx.serializeNetworkWrapper().toString('hex')])