Skip to content

Commit

Permalink
WIP removal of level DB stuff (#2673)
Browse files Browse the repository at this point in the history
Co-authored-by: Gabriel Rocheleau <contact@rockwaterweb.com>
  • Loading branch information
acolytec3 and gabrocheleau committed May 4, 2023
1 parent 89f4741 commit 6c9d0b1
Show file tree
Hide file tree
Showing 43 changed files with 445 additions and 352 deletions.
4 changes: 0 additions & 4 deletions package-lock.json

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

149 changes: 65 additions & 84 deletions packages/blockchain/src/blockchain.ts
@@ -1,11 +1,17 @@
import { Block, BlockHeader } from '@ethereumjs/block'
import { Chain, Common, ConsensusAlgorithm, ConsensusType, Hardfork } from '@ethereumjs/common'
import { DB, KECCAK256_RLP, Lock, concatBytesNoTypeCheck } from '@ethereumjs/util'
import {
KECCAK256_RLP,
Lock,
bytesToPrefixedHexString,
concatBytesNoTypeCheck,
} from '@ethereumjs/util'
import { bytesToHex, equalsBytes, hexToBytes } from 'ethereum-cryptography/utils'

import { CasperConsensus, CliqueConsensus, EthashConsensus } from './consensus'
import { DBOp, DBSaveLookups, DBSetBlockOrHeader, DBSetHashToNumber, DBSetTD } from './db/helpers'
import { DBManager } from './db/manager'
import { MapDB } from './db/map'
import { DBTarget } from './db/operation'
import { genesisStateRoot } from './genesisStates'
import {} from './utils'
Expand All @@ -15,8 +21,7 @@ import type { GenesisState } from './genesisStates'
import type { BlockchainInterface, BlockchainOptions, OnBlock } from './types'
import type { BlockData } from '@ethereumjs/block'
import type { CliqueConfig } from '@ethereumjs/common'
import type { BigIntLike } from '@ethereumjs/util'
import { MapDB } from './db/map'
import type { BigIntLike, DB } from '@ethereumjs/util'

/**
* This class stores and interacts with blocks.
Expand Down Expand Up @@ -114,7 +119,8 @@ export class Blockchain implements BlockchainInterface {
this._validateBlocks = opts.validateBlocks ?? true
this._customGenesisState = opts.genesisState

this.db = opts.db ? opts.db : new MapDB()
this.db = opts.db !== undefined ? opts.db : new MapDB()

this.dbManager = new DBManager(this.db, this._common)

if (opts.consensus) {
Expand Down Expand Up @@ -190,17 +196,13 @@ export class Blockchain implements BlockchainInterface {
await this.consensus.setup({ blockchain: this })

if (this._isInitialized) return
let dbGenesisBlock
try {
const genesisHash = await this.dbManager.numberToHash(BigInt(0))
dbGenesisBlock = await this.dbManager.getBlock(genesisHash)
} catch (error: any) {
if (error.code !== 'LEVEL_NOT_FOUND') {
throw error
}
}

if (!genesisBlock) {
let genesisHash = await this.dbManager.numberToHash(BigInt(0))

const dbGenesisBlock =
genesisHash !== undefined ? await this.dbManager.getBlock(genesisHash) : undefined

if (genesisBlock === undefined) {
let stateRoot
if (this._common.chainId() === BigInt(1) && this._customGenesisState === undefined) {
// For mainnet use the known genesis stateRoot to quicken setup
Expand All @@ -213,13 +215,13 @@ export class Blockchain implements BlockchainInterface {

// If the DB has a genesis block, then verify that the genesis block in the
// DB is indeed the Genesis block generated or assigned.
if (dbGenesisBlock && !equalsBytes(genesisBlock.hash(), dbGenesisBlock.hash())) {
if (dbGenesisBlock !== undefined && !equalsBytes(genesisBlock.hash(), dbGenesisBlock.hash())) {
throw new Error(
'The genesis block in the DB has a different hash than the provided genesis block.'
)
}

const genesisHash = genesisBlock.hash()
genesisHash = genesisBlock.hash()

if (!dbGenesisBlock) {
// If there is no genesis block put the genesis block in the DB.
Expand All @@ -238,37 +240,16 @@ export class Blockchain implements BlockchainInterface {
this._genesisBlock = genesisBlock

// load verified iterator heads
try {
const heads = await this.dbManager.getHeads()
this._heads = heads
} catch (error: any) {
if (error.code !== 'LEVEL_NOT_FOUND') {
throw error
}
this._heads = {}
}
const heads = await this.dbManager.getHeads()
this._heads = heads !== undefined ? heads : {}

// load headerchain head
try {
const hash = await this.dbManager.getHeadHeader()
this._headHeaderHash = hash
} catch (error: any) {
if (error.code !== 'LEVEL_NOT_FOUND') {
throw error
}
this._headHeaderHash = genesisHash
}
let hash = await this.dbManager.getHeadHeader()
this._headHeaderHash = hash !== undefined ? hash : genesisHash

// load blockchain head
try {
const hash = await this.dbManager.getHeadBlock()
this._headBlockHash = hash
} catch (error: any) {
if (error.code !== 'LEVEL_NOT_FOUND') {
throw error
}
this._headBlockHash = genesisHash
}
hash = await this.dbManager.getHeadBlock()
this._headBlockHash = hash !== undefined ? hash : genesisHash

if (this._hardforkByHeadBlockNumber) {
const latestHeader = await this._getHeader(this._headHeaderHash)
Expand Down Expand Up @@ -422,6 +403,9 @@ export class Blockchain implements BlockchainInterface {
async resetCanonicalHead(canonicalHead: bigint) {
await this.runWithLock<void>(async () => {
const hash = await this.dbManager.numberToHash(canonicalHead)
if (hash === undefined) {
throw new Error(`no block for ${canonicalHead} found in DB`)
}
const header = await this._getHeader(hash, canonicalHead)
const td = await this.getParentTD(header)

Expand Down Expand Up @@ -737,18 +721,16 @@ export class Blockchain implements BlockchainInterface {
// in the `VM` if we encounter a `BLOCKHASH` opcode: then a bigint is used we
// need to then read the block from the canonical chain Q: is this safe? We
// know it is OK if we call it from the iterator... (runBlock)
try {
return await this.dbManager.getBlock(blockId)
} catch (error: any) {
if (error.code === 'LEVEL_NOT_FOUND') {
if (typeof blockId === 'object') {
error.message = `Block with hash ${bytesToHex(blockId)} not found in DB (NotFound)`
} else {
error.message = `Block number ${blockId} not found in DB (NotFound)`
}
const block = await this.dbManager.getBlock(blockId)

if (block === undefined) {
if (typeof blockId === 'object') {
throw new Error(`Block with hash ${bytesToHex(blockId)} not found in DB`)
} else {
throw new Error(`Block number ${blockId} not found in DB`)
}
throw error
}
return block
}

/**
Expand All @@ -757,6 +739,9 @@ export class Blockchain implements BlockchainInterface {
public async getTotalDifficulty(hash: Uint8Array, number?: bigint): Promise<bigint> {
if (number === undefined) {
number = await this.dbManager.hashToNumber(hash)
if (number === undefined) {
throw new Error(`Block with hash ${bytesToPrefixedHexString(hash)} not found in DB`)
}
}
return this.dbManager.getTotalDifficulty(hash, number)
}
Expand Down Expand Up @@ -793,12 +778,12 @@ export class Blockchain implements BlockchainInterface {
let block
try {
block = await this.getBlock(blockId)
} catch (error: any) {
if (error.code !== 'LEVEL_NOT_FOUND') {
throw error
}
return
} catch (err: any) {
if (err.message.includes('not found in DB') === true) {
return
} else throw err
}

i++
const nextBlockNumber = block.header.number + BigInt(reverse ? -1 : 1)
if (i !== 0 && skip && i % (skip + 1) !== 0) {
Expand Down Expand Up @@ -834,11 +819,12 @@ export class Blockchain implements BlockchainInterface {
let number
try {
number = await this.dbManager.hashToNumber(hashes[mid])
} catch (error: any) {
if (error.code !== 'LEVEL_NOT_FOUND') {
throw error
}
} catch (err: any) {
if (err.message.includes('not found in DB') === true) {
number = undefined
} else throw err
}

if (number !== undefined) {
min = mid + 1
} else {
Expand Down Expand Up @@ -944,9 +930,9 @@ export class Blockchain implements BlockchainInterface {
try {
const childHeader = await this.getCanonicalHeader(blockNumber + BigInt(1))
await this._delChild(childHeader.hash(), childHeader.number, headHash, ops)
} catch (error: any) {
if (error.code !== 'LEVEL_NOT_FOUND') {
throw error
} catch (err: any) {
if (err.message.includes('not found in canonical chain') !== true) {
throw err
}
}
}
Expand Down Expand Up @@ -976,7 +962,8 @@ export class Blockchain implements BlockchainInterface {
}

let headBlockNumber = await this.dbManager.hashToNumber(headHash)
let nextBlockNumber = headBlockNumber + BigInt(1)
// `headBlockNumber` should always exist since it defaults to the genesis block
let nextBlockNumber = headBlockNumber! + BigInt(1)
let blocksRanCounter = 0
let lastBlock: Block | undefined

Expand All @@ -990,7 +977,7 @@ export class Blockchain implements BlockchainInterface {
// If reorg has happened, the _heads must have been updated so lets reload the counters
headHash = this._heads[name] ?? this.genesisBlock.hash()
headBlockNumber = await this.dbManager.hashToNumber(headHash)
nextBlockNumber = headBlockNumber + BigInt(1)
nextBlockNumber = headBlockNumber! + BigInt(1)
nextBlock = await this.getBlock(nextBlockNumber)
}
this._heads[name] = nextBlock.hash()
Expand All @@ -1008,7 +995,7 @@ export class Blockchain implements BlockchainInterface {
nextBlockNumber++
blocksRanCounter++
} catch (error: any) {
if (error.code === 'LEVEL_NOT_FOUND') {
if ((error.message as string).includes('not found in DB')) {
break
} else {
throw error
Expand Down Expand Up @@ -1181,13 +1168,9 @@ export class Blockchain implements BlockchainInterface {
staleHeadBlock = true
}

try {
header = await this._getHeader(header.parentHash, --currentNumber)
} catch (error: any) {
header = await this._getHeader(header.parentHash, --currentNumber)
if (header === undefined) {
staleHeads = []
if (error.code !== 'LEVEL_NOT_FOUND') {
throw error
}
break
}
}
Expand Down Expand Up @@ -1241,6 +1224,8 @@ export class Blockchain implements BlockchainInterface {
private async _getHeader(hash: Uint8Array, number?: bigint) {
if (number === undefined) {
number = await this.dbManager.hashToNumber(hash)
if (number === undefined)
throw new Error(`no header for ${bytesToPrefixedHexString(hash)} found in DB`)
}
return this.dbManager.getHeader(hash, number)
}
Expand Down Expand Up @@ -1284,25 +1269,21 @@ export class Blockchain implements BlockchainInterface {
*/
async getCanonicalHeader(number: bigint) {
const hash = await this.dbManager.numberToHash(number)
if (hash === undefined) {
throw new Error(`header with number ${number} not found in canonical chain`)
}
return this._getHeader(hash, number)
}

/**
* This method either returns a Uint8Array if there exists one in the DB or if it
* does not exist (DB throws a `NotFoundError`) then return false If DB throws
* does not exist then return false If DB throws
* any other error, this function throws.
* @param number
*/
async safeNumberToHash(number: bigint): Promise<Uint8Array | false> {
try {
const hash = await this.dbManager.numberToHash(number)
return hash
} catch (error: any) {
if (error.code !== 'LEVEL_NOT_FOUND') {
throw error
}
return false
}
const hash = await this.dbManager.numberToHash(number)
return hash !== undefined ? hash : false
}

/**
Expand Down

1 comment on commit 6c9d0b1

@github-actions
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚠️ Performance Alert ⚠️

Possible performance regression was detected for benchmark.
Benchmark result of this commit is worse than the previous benchmark result exceeding threshold 2.

Benchmark suite Current: 6c9d0b1 Previous: c9b0a45 Ratio
Block 9422905 7040 ops/sec (±2.84%) 16231 ops/sec (±2.19%) 2.31
Block 9422906 7139 ops/sec (±3.11%) 15484 ops/sec (±4.00%) 2.17
Block 9422907 7174 ops/sec (±2.05%) 16302 ops/sec (±1.39%) 2.27
Block 9422908 6831 ops/sec (±4.90%) 15689 ops/sec (±1.70%) 2.30
Block 9422910 7130 ops/sec (±2.65%) 14940 ops/sec (±5.00%) 2.10

This comment was automatically generated by workflow using github-action-benchmark.

Please sign in to comment.