Client: RPC Block Retrieval By Portal Network (experimental)

package.json

@@ -17,7 +17,7 @@
     "prepare": "git config --local core.hooksPath .githooks",
     "prettier": "prettier --write \"./**/*.{js,json,md,ts,yml}\"",
     "sort-package-json": "sort-package-json \"package.json\" \"packages/*/package.json\"",
-    "install-browser-deps": "npm install webdriverio @vitest/browser web-streams-polyfill"
+    "install-browser-deps": "npm install webdriverio @vitest/browser@1.5.0 web-streams-polyfill"
   },
   "devDependencies": {
     "@types/estree": "^1.0.1",

packages/client/bin/cli.ts

@@ -35,6 +35,7 @@ import { loadKZG } from 'kzg-wasm'
 import { Level } from 'level'
 import { homedir } from 'os'
 import * as path from 'path'
+import { NetworkId, PortalNetwork } from 'portalnetwork'
 import * as promClient from 'prom-client'
 import * as readline from 'readline'
 import * as url from 'url'
@@ -60,6 +61,7 @@ import type { CustomCrypto } from '@ethereumjs/common'
 import type { GenesisState, PrefixedHexString } from '@ethereumjs/util'
 import type { AbstractLevel } from 'abstract-level'
 import type { Server as RPCServer } from 'jayson/promise/index.js'
+import type { NetworkConfig } from 'portalnetwork'
 
 type Account = [address: Address, privateKey: Uint8Array]
 
@@ -461,6 +463,36 @@ const args: ClientOpts = yargs
     boolean: true,
     default: false,
   })
+  .option('enablePortalHistory', {
+    describe: 'Use Portal Network for historical block retrieval',
+    boolean: true,
+    default: false,
+  })
+  .option('portalHistory', {
+    describe: 'node radius (exponent) for history portal network client',
+    number: true,
+    default: 0,
+  })
+  .option('enablePortalState', {
+    describe: 'Use Portal Network for historical state retrieval',
+    boolean: true,
+    default: false,
+  })
+  .option('portalState', {
+    describe: 'node radius (exponent) for state portal network client',
+    number: true,
+    default: 0,
+  })
+  .option('enablePortalBeacon', {
+    describe: 'Use Portal Network for beacon light client data retrieval',
+    boolean: true,
+    default: false,
+  })
+  .option('portalBeacon', {
+    describe: 'node radius (exponent) for beacon portal network client',
+    number: true,
+    default: 0,
+  })
   .completion()
   // strict() ensures that yargs throws when an invalid arg is provided
   .strict()
@@ -868,6 +900,7 @@ const stopClient = async (
 ) => {
   config.logger.info('Caught interrupt signal. Obtaining client handle for clean shutdown...')
   config.logger.info('(This might take a little longer if client not yet fully started)')
+  await config.portal?.stop()
   let timeoutHandle
   if (clientStartPromise?.toString().includes('Promise') === true)
     // Client hasn't finished starting up so setting timeout to terminate process if not already shutdown gracefully
@@ -1101,6 +1134,50 @@ async function run() {
     metricsServer.listen(args.prometheusPort)
   }
 
+  let portal
+  if (
+    args.enablePortalHistory === true ||
+    args.enablePortalState === true ||
+    args.enablePortalBeacon === true
+  ) {
+    const supportedNetworks: NetworkConfig[] = []
+    if (args.enablePortalHistory === true) {
+      supportedNetworks.push({
+        networkId: NetworkId.HistoryNetwork,
+        radius: BigInt(2) ** BigInt(args.portalHistory) - BigInt(1),
+      })
+    }
+    if (args.enablePortalState === true) {
+      supportedNetworks.push({
+        networkId: NetworkId.StateNetwork,
+        radius: BigInt(2) ** BigInt(args.portalState) - BigInt(1),
+      })
+    }
+    if (args.enablePortalBeacon === true) {
+      supportedNetworks.push({
+        networkId: NetworkId.BeaconLightClientNetwork,
+        radius: BigInt(2) ** BigInt(args.portalBeacon) - BigInt(1),
+      })
+    }
+    portal = await PortalNetwork.create({
+      bindAddress: '0.0.0.0',
+      supportedNetworks,
+      bootnodes: [
+        'enr:-Jy4QIs2pCyiKna9YWnAF0zgf7bT0GzlAGoF8MEKFJOExmtofBIqzm71zDvmzRiiLkxaEJcs_Amr7XIhLI74k1rtlXICY5Z0IDAuMS4xLWFscGhhLjEtMTEwZjUwgmlkgnY0gmlwhKEjVaWJc2VjcDI1NmsxoQLSC_nhF1iRwsCw0n3J4jRjqoaRxtKgsEe5a-Dz7y0JloN1ZHCCIyg',
+        'enr:-Jy4QKSLYMpku9F0Ebk84zhIhwTkmn80UnYvE4Z4sOcLukASIcofrGdXVLAUPVHh8oPCfnEOZm1W1gcAxB9kV2FJywkCY5Z0IDAuMS4xLWFscGhhLjEtMTEwZjUwgmlkgnY0gmlwhJO2oc6Jc2VjcDI1NmsxoQLMSGVlxXL62N3sPtaV-n_TbZFCEM5AR7RDyIwOadbQK4N1ZHCCIyg',
+        'enr:-Jy4QH4_H4cW--ejWDl_W7ngXw2m31MM2GT8_1ZgECnfWxMzZTiZKvHDgkmwUS_l2aqHHU54Q7hcFSPz6VGzkUjOqkcCY5Z0IDAuMS4xLWFscGhhLjEtMTEwZjUwgmlkgnY0gmlwhJ31OTWJc2VjcDI1NmsxoQPC0eRkjRajDiETr_DRa5N5VJRm-ttCWDoO1QAMMCg5pIN1ZHCCIyg',
+        'enr:-IS4QGUtAA29qeT3cWVr8lmJfySmkceR2wp6oFQtvO_uMe7KWaK_qd1UQvd93MJKXhMnubSsTQPJ6KkbIu0ywjvNdNEBgmlkgnY0gmlwhMIhKO6Jc2VjcDI1NmsxoQJ508pIqRqsjsvmUQfYGvaUFTxfsELPso_62FKDqlxI24N1ZHCCI40',
+        'enr:-IS4QNaaoQuHGReAMJKoDd6DbQKMbQ4Mked3Gi3GRatwgRVVPXynPlO_-gJKRF_ZSuJr3wyHfwMHyJDbd6q1xZQVZ2kBgmlkgnY0gmlwhMIhKO6Jc2VjcDI1NmsxoQM2kBHT5s_Uh4gsNiOclQDvLK4kPpoQucge3mtbuLuUGYN1ZHCCI44',
+        'enr:-IS4QBdIjs6S1ZkvlahSkuYNq5QW3DbD-UDcrm1l81f2PPjnNjb_NDa4B5x4olHCXtx0d2ZeZBHQyoHyNnuVZ-P1GVkBgmlkgnY0gmlwhMIhKO-Jc2VjcDI1NmsxoQOO3gFuaCAyQKscaiNLC9HfLbVzFdIerESFlOGcEuKWH4N1ZHCCI40',
+        'enr:-IS4QM731tV0CvQXLTDcZNvgFyhhpAjYDKU5XLbM7sZ1WEzIRq4zsakgrv3KO3qyOYZ8jFBK-VzENF8o-vnykuQ99iABgmlkgnY0gmlwhMIhKO-Jc2VjcDI1NmsxoQMTq6Cdx3HmL3Q9sitavcPHPbYKyEibKPKvyVyOlNF8J4N1ZHCCI44',
+        'enr:-IS4QFV_wTNknw7qiCGAbHf6LxB-xPQCktyrCEZX-b-7PikMOIKkBg-frHRBkfwhI3XaYo_T-HxBYmOOQGNwThkBBHYDgmlkgnY0gmlwhKRc9_OJc2VjcDI1NmsxoQKHPt5CQ0D66ueTtSUqwGjfhscU_LiwS28QvJ0GgJFd-YN1ZHCCE4k',
+        'enr:-IS4QDpUz2hQBNt0DECFm8Zy58Hi59PF_7sw780X3qA0vzJEB2IEd5RtVdPUYZUbeg4f0LMradgwpyIhYUeSxz2Tfa8DgmlkgnY0gmlwhKRc9_OJc2VjcDI1NmsxoQJd4NAVKOXfbdxyjSOUJzmA4rjtg43EDeEJu1f8YRhb_4N1ZHCCE4o',
+        'enr:-IS4QGG6moBhLW1oXz84NaKEHaRcim64qzFn1hAG80yQyVGNLoKqzJe887kEjthr7rJCNlt6vdVMKMNoUC9OCeNK-EMDgmlkgnY0gmlwhKRc9-KJc2VjcDI1NmsxoQLJhXByb3LmxHQaqgLDtIGUmpANXaBbFw3ybZWzGqb9-IN1ZHCCE4k',
+        'enr:-IS4QA5hpJikeDFf1DD1_Le6_ylgrLGpdwn3SRaneGu9hY2HUI7peHep0f28UUMzbC0PvlWjN8zSfnqMG07WVcCyBhADgmlkgnY0gmlwhKRc9-KJc2VjcDI1NmsxoQJMpHmGj1xSP1O-Mffk_jYIHVcg6tY5_CjmWVg1gJEsPIN1ZHCCE4o',
+      ],
+    })
+    void portal.start()
+  }
   const config = new Config({
     accounts,
     bootnodes,
@@ -1147,6 +1224,7 @@ async function run() {
         : args.engineNewpayloadMaxExecute,
     ignoreStatelessInvalidExecs: args.ignoreStatelessInvalidExecs,
     prometheusMetrics,
+    portal,
   })
   config.events.setMaxListeners(50)
   config.events.on(Event.SERVER_LISTENING, (details) => {

packages/client/package.json

@@ -86,6 +86,7 @@
     "kzg-wasm": "^0.4.0",
     "level": "^8.0.0",
     "memory-level": "^1.0.0",
+    "portalnetwork": "0.0.2-rc3",
     "prom-client": "^15.1.0",
     "winston": "^3.3.3",
     "winston-daily-rotate-file": "^4.5.5",

packages/client/src/client.ts

@@ -10,6 +10,7 @@ import type { MultiaddrLike } from './types.js'
 import type { Blockchain } from '@ethereumjs/blockchain'
 import type { GenesisState } from '@ethereumjs/util'
 import type { AbstractLevel } from 'abstract-level'
+import type { PortalNetwork } from 'portalnetwork'
 
 export interface EthereumClientOptions {
   /** Client configuration */
@@ -70,6 +71,7 @@ export class EthereumClient {
   public config: Config
   public chain: Chain
   public services: (FullEthereumService | LightEthereumService)[] = []
+  public portal: PortalNetwork | undefined
 
   public opened: boolean
   public started: boolean
@@ -133,8 +135,15 @@ export class EthereumClient {
       )
     )
     this.config.logger.info(
-      `Initializing Ethereumjs client version=v${packageJson.version} network=${name} chainId=${chainId}`
+      `Initializing Ethereumjs client version=v${packageJson.version} network=${name} chainId=${chainId}}}`
     )
+    if (this.config.portal !== undefined) {
+      this.config.logger.info(
+        `Starting Portal client. enr=${this.config.portal.discv5.enr.encodeTxt()} nodeId=0x${
+          this.config.portal.discv5.enr.nodeId
+        } networks=${[...this.config.portal.networks.values()].map((n) => n.constructor.name)}`
+      )
+    }
 
     this.config.events.on(Event.SERVER_ERROR, (error) => {
       this.config.logger.warn(`Server error: ${error.name} - ${error.message}`)

packages/client/src/config.ts

@@ -13,6 +13,7 @@ import type { EventBusType, MultiaddrLike, PrometheusMetrics } from './types.js'
 import type { BlockHeader } from '@ethereumjs/block'
 import type { VM, VMProfilerOpts } from '@ethereumjs/vm'
 import type { Multiaddr } from '@multiformats/multiaddr'
+import type { PortalNetwork } from 'portalnetwork'
 
 export enum DataDirectory {
   Chain = 'chain',
@@ -344,6 +345,11 @@ export interface ConfigOptions {
    * Enables Prometheus Metrics that can be collected for monitoring client health
    */
   prometheusMetrics?: PrometheusMetrics
+
+  /**
+   * Optional Portal Network client
+   */
+  portal?: PortalNetwork
 }
 
 export class Config {
@@ -470,6 +476,8 @@ export class Config {
 
   public readonly metrics: PrometheusMetrics | undefined
 
+  public readonly portal: PortalNetwork | undefined = undefined
+
   constructor(options: ConfigOptions = {}) {
     this.events = new EventBus() as EventBusType
 
@@ -578,6 +586,8 @@ export class Config {
     this.events.once(Event.CLIENT_SHUTDOWN, () => {
       this.shutdown = true
     })
+
+    this.portal = options.portal
   }
 
   /**

packages/client/src/rpc/modules/eth.ts

@@ -114,7 +114,6 @@ const jsonRpcBlock = async (
           withdrawals: json.withdrawals,
         }
       : {}
-  const td = await chain.getTd(block.hash(), block.header.number)
   return {
     number: header.number!,
     hash: bytesToHex(block.hash()),
@@ -128,7 +127,7 @@ const jsonRpcBlock = async (
     receiptsRoot: header.receiptTrie!,
     miner: header.coinbase!,
     difficulty: header.difficulty!,
-    totalDifficulty: bigIntToHex(td),
+    totalDifficulty: header.difficulty!,
     extraData: header.extraData!,
     size: intToHex(utf8ToBytes(JSON.stringify(json)).byteLength),
     gasLimit: header.gasLimit!,
@@ -648,7 +647,16 @@ export class Eth {
    */
   async getBlockByHash(params: [PrefixedHexString, boolean]) {
     const [blockHash, includeTransactions] = params
-
+    if (this.client.config.portal !== undefined && this.client.config.portal.discv5.isStarted()) {
+      const network = this.client.config.portal.network()['0x500b']!
+      const block = await network.ETH.getBlockByHash(blockHash, includeTransactions)
+      if (block === undefined)
+        throw {
+          code: INVALID_PARAMS,
+          message: 'Unknown block',
+        }
+      return jsonRpcBlock(block, this._chain, includeTransactions)
+    }
     try {
       const block = await this._chain.getBlock(hexToBytes(blockHash))
       return await jsonRpcBlock(block, this._chain, includeTransactions)
@@ -668,6 +676,19 @@ export class Eth {
    */
   async getBlockByNumber(params: [string, boolean]) {
     const [blockOpt, includeTransactions] = params
+    if (this.client.config.portal !== undefined && this.client.config.portal.discv5.isStarted()) {
+      const block = await this.client.config.portal.ETH.getBlockByNumber(
+        parseInt(blockOpt, 16),
+        includeTransactions
+      )
+      if (block === undefined)
+        throw {
+          code: INVALID_PARAMS,
+          message: 'Unknown block',
+        }
+      //@ts-ignore -- Ultralight isn't using the latest EthJS code (which includes Pectra stuff)
+      return jsonRpcBlock(block, this._chain, includeTransactions)
+    }
     const block = await getBlockByOption(blockOpt, this._chain)
     return jsonRpcBlock(block, this._chain, includeTransactions)
   }

packages/client/src/types.ts

@@ -175,6 +175,12 @@ export interface ClientOpts {
   skipEngineExec?: boolean
   ignoreStatelessInvalidExecs?: boolean
   useJsCrypto?: boolean
+  enablePortalHistory?: boolean
+  enablePortalState?: boolean
+  enablePortalBeacon?: boolean
+  portalHistory: number
+  portalState: number
+  portalBeacon: number
 }
 
 export type PrometheusMetrics = {

packages/client/tsconfig.prod.esm.json

@@ -4,7 +4,8 @@
   "compilerOptions": {
     "outDir": "dist/esm",
     "typeRoots": ["node_modules/@types", "src/@types"],
-    "composite": true
+    "composite": true,
+    "skipLibCheck": true
   },
   "references": [
     { "path": "../block/tsconfig.prod.cjs.json" },