Blockchain/Ethash/Trie: remove level dependency from blockchain

packages/blockchain/package.json

@@ -45,10 +45,8 @@
     "@ethereumjs/trie": "^5.0.4",
     "@ethereumjs/tx": "^4.1.1",
     "@ethereumjs/util": "^8.0.5",
-    "abstract-level": "^1.0.3",
     "debug": "^4.3.3",
     "ethereum-cryptography": "^1.1.2",
-    "level": "^8.0.0",
     "lru-cache": "^5.1.1",
     "memory-level": "^1.0.0"
   },

packages/blockchain/src/blockchain.ts

@@ -1,8 +1,7 @@
 import { Block, BlockHeader } from '@ethereumjs/block'
 import { Chain, Common, ConsensusAlgorithm, ConsensusType, Hardfork } from '@ethereumjs/common'
-import { KECCAK256_RLP, Lock, concatBytesNoTypeCheck } from '@ethereumjs/util'
+import { DB, KECCAK256_RLP, Lock, concatBytesNoTypeCheck } from '@ethereumjs/util'
 import { bytesToHex, equalsBytes, hexToBytes } from 'ethereum-cryptography/utils'
-import { MemoryLevel } from 'memory-level'
 
 import { CasperConsensus, CliqueConsensus, EthashConsensus } from './consensus'
 import { DBOp, DBSaveLookups, DBSetBlockOrHeader, DBSetHashToNumber, DBSetTD } from './db/helpers'
@@ -17,14 +16,14 @@ import type { BlockchainInterface, BlockchainOptions, OnBlock } from './types'
 import type { BlockData } from '@ethereumjs/block'
 import type { CliqueConfig } from '@ethereumjs/common'
 import type { BigIntLike } from '@ethereumjs/util'
-import type { AbstractLevel } from 'abstract-level'
+import { MapDB } from './db/map'
 
 /**
  * This class stores and interacts with blocks.
  */
 export class Blockchain implements BlockchainInterface {
   consensus: Consensus
-  db: AbstractLevel<string | Uint8Array | Uint8Array, string | Uint8Array, string | Uint8Array>
+  db: DB<Uint8Array | string, Uint8Array | string>
   dbManager: DBManager
 
   private _genesisBlock?: Block /** The genesis block of this blockchain */
@@ -115,7 +114,7 @@ export class Blockchain implements BlockchainInterface {
     this._validateBlocks = opts.validateBlocks ?? true
     this._customGenesisState = opts.genesisState
 
-    this.db = opts.db ? opts.db : new MemoryLevel()
+    this.db = opts.db ? opts.db : new MapDB()
     this.dbManager = new DBManager(this.db, this._common)
 
     if (opts.consensus) {

packages/blockchain/src/consensus/clique.ts

@@ -2,7 +2,7 @@ import { ConsensusAlgorithm } from '@ethereumjs/common'
 import { RLP } from '@ethereumjs/rlp'
 import { Address, TypeOutput, bigIntToBytes, bytesToBigInt, toType } from '@ethereumjs/util'
 import { debug as createDebugLogger } from 'debug'
-import { equalsBytes, hexToBytes } from 'ethereum-cryptography/utils'
+import { equalsBytes, hexToBytes, utf8ToBytes } from 'ethereum-cryptography/utils'
 
 import type { Blockchain } from '..'
 import type { Consensus, ConsensusOptions } from './interface'
@@ -253,7 +253,7 @@ export class CliqueConsensus implements Consensus {
       bigIntToBytes(state[0]),
       state[1].map((a) => a.toBytes()),
     ])
-    await this.blockchain!.db.put(CLIQUE_SIGNERS_KEY, RLP.encode(formatted), DB_OPTS)
+    await this.blockchain!.db.put(CLIQUE_SIGNERS_KEY, RLP.encode(formatted) /* , DB_OPTS */)
     // Output active signers for debugging purposes
     if (signerState !== undefined) {
       let i = 0
@@ -414,7 +414,7 @@ export class CliqueConsensus implements Consensus {
       bigIntToBytes(v[0]),
       [v[1][0].toBytes(), v[1][1].toBytes(), v[1][2]],
     ])
-    await this.blockchain!.db.put(CLIQUE_VOTES_KEY, RLP.encode(formatted), DB_OPTS)
+    await this.blockchain!.db.put(CLIQUE_VOTES_KEY, RLP.encode(formatted) /* , DB_OPTS */)
   }
 
   /**
@@ -523,7 +523,10 @@ export class CliqueConsensus implements Consensus {
       bigIntToBytes(b[0]),
       b[1].toBytes(),
     ])
-    await this.blockchain!.db.put(CLIQUE_BLOCK_SIGNERS_SNAPSHOT_KEY, RLP.encode(formatted), DB_OPTS)
+    await this.blockchain!.db.put(
+      CLIQUE_BLOCK_SIGNERS_SNAPSHOT_KEY,
+      RLP.encode(formatted) /* , DB_OPTS */
+    )
   }
 
   /**
@@ -532,9 +535,9 @@ export class CliqueConsensus implements Consensus {
    */
   private async getCliqueLatestSignerStates(): Promise<CliqueLatestSignerStates> {
     try {
-      const signerStates = await this.blockchain!.db.get<string, Uint8Array>(
-        CLIQUE_SIGNERS_KEY,
-        DB_OPTS
+      const signerStates = await this.blockchain!.db.get(
+        CLIQUE_SIGNERS_KEY
+        /* DB_OPTS */
       )
       const states = RLP.decode(signerStates) as [Uint8Array, Uint8Array[]]
       return states.map((state) => {
@@ -556,9 +559,9 @@ export class CliqueConsensus implements Consensus {
    */
   private async getCliqueLatestVotes(): Promise<CliqueLatestVotes> {
     try {
-      const signerVotes = await this.blockchain!.db.get<string, Uint8Array>(
-        CLIQUE_VOTES_KEY,
-        DB_OPTS
+      const signerVotes = await this.blockchain!.db.get(
+        CLIQUE_VOTES_KEY
+        /* DB_OPTS */
       )
       const votes = RLP.decode(signerVotes) as [Uint8Array, [Uint8Array, Uint8Array, Uint8Array]]
       return votes.map((vote) => {
@@ -582,9 +585,8 @@ export class CliqueConsensus implements Consensus {
    */
   private async getCliqueLatestBlockSigners(): Promise<CliqueLatestBlockSigners> {
     try {
-      const blockSigners = await this.blockchain!.db.get<string, Uint8Array>(
-        CLIQUE_BLOCK_SIGNERS_SNAPSHOT_KEY,
-        DB_OPTS
+      const blockSigners = await this.blockchain!.db.get(
+        CLIQUE_BLOCK_SIGNERS_SNAPSHOT_KEY /* DB_OPTS */
       )
       const signers = RLP.decode(blockSigners) as [Uint8Array, Uint8Array][]
       return signers.map((s) => {

packages/blockchain/src/db/manager.ts

@@ -1,6 +1,13 @@
 import { Block, BlockHeader, valuesArrayToHeaderData } from '@ethereumjs/block'
 import { RLP } from '@ethereumjs/rlp'
-import { KECCAK256_RLP, KECCAK256_RLP_ARRAY, bytesToBigInt, equalsBytes } from '@ethereumjs/util'
+import {
+  DB,
+  KECCAK256_RLP,
+  KECCAK256_RLP_ARRAY,
+  bytesToBigInt,
+  equalsBytes,
+  toBytes,
+} from '@ethereumjs/util'
 import { hexToBytes } from 'ethereum-cryptography/utils'
 
 import { Cache } from './cache'
@@ -9,7 +16,6 @@ import { DBOp, DBTarget } from './operation'
 import type { DBOpData, DatabaseKey } from './operation'
 import type { BlockBodyBytes, BlockBytes, BlockOptions } from '@ethereumjs/block'
 import type { Common } from '@ethereumjs/common'
-import type { AbstractLevel } from 'abstract-level'
 
 class NotFoundError extends Error {
   public code: string = 'LEVEL_NOT_FOUND'
@@ -43,12 +49,9 @@ export type CacheMap = { [key: string]: Cache<Uint8Array> }
 export class DBManager {
   private _cache: CacheMap
   private _common: Common
-  private _db: AbstractLevel<string | Uint8Array, string | Uint8Array, string | Uint8Array>
+  private _db: DB<Uint8Array | string, Uint8Array | string>
 
-  constructor(
-    db: AbstractLevel<string | Uint8Array, string | Uint8Array, string | Uint8Array>,
-    common: Common
-  ) {
+  constructor(db: DB<Uint8Array | string, Uint8Array | string>, common: Common) {
     this._db = db
     this._common = common
     this._cache = {
@@ -219,9 +222,10 @@ export class DBManager {
       }
       let value = this._cache[cacheString].get(dbKey)
       if (!value) {
-        value = await this._db.get(dbKey, dbOpts)
+        value = ((await this._db.get(dbKey)) as Uint8Array | null) ?? undefined
 
         if (value !== undefined) {
+          // TODO: Check if this comment is still valid
           // Always cast values to Uint8Array since db sometimes returns values as `Buffer`
           this._cache[cacheString].set(dbKey, Uint8Array.from(value))
         }
@@ -230,7 +234,7 @@ export class DBManager {
       return value
     }
 
-    return this._db.get(dbKey, dbOpts)
+    return this._db.get(dbKey /* , dbOpts */)
   }
 
   /**

packages/blockchain/src/db/map.ts

@@ -0,0 +1,42 @@
+import { DB, BatchDBOp } from '@ethereumjs/util'
+import { bytesToHex } from 'ethereum-cryptography/utils'
+
+export class MapDB<TKey extends Uint8Array | string, TValue extends Uint8Array | string>
+  implements DB<TKey, TValue>
+{
+  _database: Map<TKey, TValue>
+
+  constructor(database?: Map<TKey, TValue>) {
+    this._database = database ?? new Map<TKey, TValue>()
+  }
+
+  async get(key: TKey): Promise<TValue | null> {
+    const result = this._database.get(key)
+
+    return result ?? null
+  }
+
+  async put(key: TKey, val: TValue): Promise<void> {
+    this._database.set(key, val)
+  }
+
+  async del(key: TKey): Promise<void> {
+    this._database.delete(key)
+  }
+
+  async batch(opStack: BatchDBOp<TKey, TValue>[]): Promise<void> {
+    for (const op of opStack) {
+      if (op.type === 'del') {
+        await this.del(op.key)
+      }
+
+      if (op.type === 'put') {
+        await this.put(op.key, op.value)
+      }
+    }
+  }
+
+  copy(): DB<TKey, TValue> {
+    return new MapDB<TKey, TValue>(this._database)
+  }
+}

packages/blockchain/src/types.ts

@@ -1,8 +1,8 @@
+import { DB } from '@ethereumjs/util'
 import type { Consensus } from './consensus'
 import type { GenesisState } from './genesisStates'
 import type { Block, BlockHeader } from '@ethereumjs/block'
 import type { Common } from '@ethereumjs/common'
-import type { AbstractLevel } from 'abstract-level'
 
 export type OnBlock = (block: Block, reorg: boolean) => Promise<void> | void
 
@@ -105,14 +105,9 @@ export interface BlockchainOptions {
 
   /**
    * Database to store blocks and metadata.
-   * Should be an `abstract-leveldown` compliant store
-   * wrapped with `encoding-down`.
-   * For example:
-   *   `levelup(encode(leveldown('./db1')))`
-   * or use the `level` convenience package:
-   *   `new MemoryLevel('./db1')`
-   */
-  db?: AbstractLevel<string | Uint8Array, string | Uint8Array, string | Uint8Array>
+   * Can be any database implementation that adheres to the `DB` interface
+   */
+  db?: DB<Uint8Array | string, Uint8Array | string>
 
   /**
    * This flags indicates if a block should be validated along the consensus algorithm

packages/blockchain/test/index.spec.ts

@@ -11,6 +11,7 @@ import * as testDataPreLondon from './testdata/testdata_pre-london.json'
 import { createTestDB, generateBlockchain, generateBlocks, isConsecutive } from './util'
 
 import type { BlockOptions } from '@ethereumjs/block'
+import { MapDB } from '@ethereumjs/trie'
 
 tape('blockchain test', (t) => {
   t.test('should not crash on getting head of a blockchain without a genesis', async (st) => {
@@ -609,7 +610,7 @@ tape('blockchain test', (t) => {
   })
 
   t.test('should save headers', async (st) => {
-    const db = new MemoryLevel<string | Uint8Array, string | Uint8Array>()
+    const db = new MapDB()
     const gasLimit = 8000000
 
     const common = new Common({ chain: Chain.Mainnet, hardfork: Hardfork.Istanbul })

packages/blockchain/test/util.ts

@@ -1,15 +1,13 @@
 import { Block, BlockHeader } from '@ethereumjs/block'
 import { Chain, Common, Hardfork } from '@ethereumjs/common'
 import { RLP } from '@ethereumjs/rlp'
-import { toBytes } from '@ethereumjs/util'
+import { DB, toBytes } from '@ethereumjs/util'
 import { keccak256 } from 'ethereum-cryptography/keccak'
 import { bytesToHex, equalsBytes, hexToBytes, utf8ToBytes } from 'ethereum-cryptography/utils'
 import { MemoryLevel } from 'memory-level'
 
 import { Blockchain } from '../src'
 
-import type { Level } from 'level'
-
 export const generateBlocks = (numberOfBlocks: number, existingBlocks?: Block[]): Block[] => {
   const blocks = existingBlocks ? existingBlocks : []
 
@@ -115,7 +113,9 @@ export const isConsecutive = (blocks: Block[]) => {
   })
 }
 
-export const createTestDB = async (): Promise<[Level<any, any>, Block]> => {
+export const createTestDB = async (): Promise<
+  [DB<string | Uint8Array, string | Uint8Array>, Block]
+> => {
   const common = new Common({ chain: Chain.Mainnet, hardfork: Hardfork.Chainstart })
   const genesis = Block.fromBlockData({ header: { number: 0 } }, { common })
   const db = new MemoryLevel<any, any>()

packages/client/bin/cli.ts

@@ -6,6 +6,7 @@ import { Chain, Common, ConsensusAlgorithm, Hardfork } from '@ethereumjs/common'
 import { RLP } from '@ethereumjs/rlp'
 import {
   Address,
+  DB,
   bytesToHex,
   bytesToPrefixedHexString,
   hexStringToBytes,
@@ -424,7 +425,7 @@ async function startClient(config: Config, customGenesisState?: GenesisState) {
   if (customGenesisState !== undefined) {
     const validateConsensus = config.chainCommon.consensusAlgorithm() === ConsensusAlgorithm.Clique
     blockchain = await Blockchain.create({
-      db: dbs.chainDB,
+      db: dbs.chainDB as any,
       genesisState: customGenesisState,
       common: config.chainCommon,
       hardforkByHeadBlockNumber: true,

packages/client/lib/blockchain/chain.ts

@@ -155,7 +155,7 @@ export class Chain {
     this.config = options.config
     this.blockchain = options.blockchain!
 
-    this.chainDB = this.blockchain.db
+    this.chainDB = this.blockchain.db as any
     this.opened = false
   }
 
@@ -213,7 +213,7 @@ export class Chain {
    */
   async open(): Promise<boolean | void> {
     if (this.opened) return false
-    await this.blockchain.db.open()
+    await (this.blockchain.db as any)?.open()
     await (this.blockchain as any)._init()
     this.opened = true
     await this.update(false)
@@ -231,7 +231,7 @@ export class Chain {
   async close(): Promise<boolean | void> {
     if (!this.opened) return false
     this.reset()
-    await this.blockchain.db.close()
+    await (this.blockchain.db as any)?.close()
     this.opened = false
   }
 

packages/client/lib/execution/level.ts

@@ -1,6 +1,6 @@
 import { MemoryLevel } from 'memory-level'
 
-import type { BatchDBOp, DB } from '@ethereumjs/trie'
+import type { BatchDBOp, DB } from '@ethereumjs/util'
 import type { AbstractLevel } from 'abstract-level'
 
 export const ENCODING_OPTS = { keyEncoding: 'view', valueEncoding: 'view' }

packages/client/lib/sync/fetcher/bytecodefetcher.ts

@@ -1,6 +1,6 @@
 import { CODEHASH_PREFIX } from '@ethereumjs/statemanager'
 import { Trie } from '@ethereumjs/trie'
-import { bytesToHex, concatBytes, equalsBytes } from '@ethereumjs/util'
+import { BatchDBOp, bytesToHex, concatBytes, equalsBytes } from '@ethereumjs/util'
 import { debug as createDebugLogger } from 'debug'
 import { keccak256 } from 'ethereum-cryptography/keccak'
 
@@ -9,7 +9,6 @@ import { Fetcher } from './fetcher'
 import type { Peer } from '../../net/peer'
 import type { FetcherOptions } from './fetcher'
 import type { Job } from './types'
-import type { BatchDBOp } from '@ethereumjs/trie'
 import type { Debugger } from 'debug'
 
 type ByteCodeDataResponse = Uint8Array[] & { completed?: boolean }

packages/trie/src/db/checkpoint.ts

@@ -1,6 +1,6 @@
-import { bytesToHex, hexStringToBytes } from '@ethereumjs/util'
+import { BatchDBOp, DB, bytesToHex, hexStringToBytes } from '@ethereumjs/util'
 
-import type { BatchDBOp, Checkpoint, CheckpointDBOpts, DB } from '../types'
+import type { Checkpoint, CheckpointDBOpts } from '../types'
 import type LRUCache from 'lru-cache'
 
 const LRU = require('lru-cache')

packages/trie/src/db/map.ts

@@ -1,7 +1,6 @@
+import { DB, BatchDBOp } from '@ethereumjs/util'
 import { bytesToHex } from 'ethereum-cryptography/utils'
 
-import type { BatchDBOp, DB } from '../types'
-
 export class MapDB implements DB {
   _database: Map<string, Uint8Array>
 
@@ -12,11 +11,7 @@ export class MapDB implements DB {
   async get(key: Uint8Array): Promise<Uint8Array | null> {
     const result = this._database.get(bytesToHex(key))
 
-    if (result !== undefined) {
-      return result
-    }
-
-    return null
+    return result ?? null
   }
 
   async put(key: Uint8Array, val: Uint8Array): Promise<void> {