feat: add call support to verified requests for prover (#5462)

* Update the readme with details

* Restructure utility methods to make consistent pattern

* Add call support for verified requests

* Update the evm logic

* Update the logger data type

* Add more structured tests

* Move e2e tests to mainnet to make them stable for now till setup CI

* Update readme

* Fix a typo

packages/prover/README.md

@@ -18,12 +18,14 @@ import Web3 from "web3";
 import {createVerifiedExecutionProvider, LCTransport} from "@lodestar/prover";
 
 const {provider, proofProvider} = createVerifiedExecutionProvider(
-  new Web3.providers.HttpProvider("https://lodestar-sepoliarpc.chainsafe.io"), {
-  transport: LCTransport.Rest,
-  urls: ["https://lodestar-sepolia.chainsafe.io"],
-  network: "sepolia",
-  wsCheckpoint: "trusted-checkpoint"
-});
+  new Web3.providers.HttpProvider("https://lodestar-sepoliarpc.chainsafe.io"),
+  {
+    transport: LCTransport.Rest,
+    urls: ["https://lodestar-sepolia.chainsafe.io"],
+    network: "sepolia",
+    wsCheckpoint: "trusted-checkpoint",
+  }
+);
 
 const web3 = new Web3(provider);
 
@@ -35,7 +37,7 @@ console.log({balance, address});
 You can also invoke the package as binary.
 
 ```bash
-npm -i g @lodestar/prover 
+npm -i g @lodestar/prover
 
 lodestar-prover start \
   --network sepolia \
@@ -45,6 +47,67 @@ lodestar-prover start \
   --port 8080
 ```
 
+## Supported Web3 Methods
+
+✅ - Completed
+
+⌛ - Todo
+
+➡️ - Request will be forward to EL without any intermediary manipulations.
+
+| Group             | Method                                  | Status | Version |
+| ----------------- | --------------------------------------- | ------ | ------- |
+| Block             | eth_getBlockByHash                      | ✅     | v0      |
+|                   | eth_getBlockByNumber                    | ✅     | v0      |
+|                   | eth_getBlockTransactionCountByHash      | ⌛     | v2      |
+|                   | eth_getBlockTransactionCountByNumber    | ⌛     | v2      |
+|                   | eth_getUncleCountByBlockHash            | ⌛     | v2      |
+|                   | eth_getUncleCountByBlockNumber          | ⌛     | v2      |
+| Chain/Network     | eth_chainId                             | ➡️     |
+|                   | eth_syncing                             | ⌛     | v1      |
+|                   | eth_coinbase                            | ⌛     | v2      |
+|                   | eth_accounts                            | ➡️     |
+|                   | eth_blockNumber                         | ➡️     |
+| Call and Estimate | eth_call                                | ✅     | v0      |
+|                   | eth_estimateGas                         | ⌛     | v0      |
+|                   | eth_createAccessList                    | ⌛     | v2      |
+|                   | eth_gasPrice                            | ⌛     | v1      |
+|                   | eth_maxPriorityFeePerGas                | ⌛     | v1      |
+|                   | eth_feeHistory                          | ⌛     | v2      |
+| Filters           | eth_newFilter                           | ⌛     | v2      |
+|                   | eth_newBlockFilter                      | ⌛     | v2      |
+|                   | eth_newPendingTransactionFilter         | ⌛     | v2      |
+|                   | eth_uninstallFilter                     | ⌛     | v2      |
+|                   | eth_getFilterChanges                    | ⌛     | v2      |
+|                   | eth_getFilterLogs                       | ⌛     | v2      |
+|                   | eth_getLogs                             | ⌛     | v1      |
+| Mining            | eth_mining                              | ➡️     |
+|                   | eth_hashrate                            | ➡️     |
+|                   | eth_getWork                             | ➡️     |
+|                   | eth_submitWork                          | ➡️     |
+|                   | eth_submitHashrate                      | ➡️     |
+| Signing           | eth_sign                                | ➡️     |
+|                   | eth_signTransaction                     | ➡️     |
+| State             | eth_getBalance                          | ✅     | v0      |
+|                   | eth_getStorageAt                        | ⌛     | v1      |
+|                   | eth_getTransactionCount                 | ⌛     | v2      |
+|                   | eth_getCode                             | ✅     | v0      |
+|                   | eth_getProof                            | ➡️     |
+| Transactions      | eth_sendTransaction                     | ➡️     |
+|                   | eth_sendRawTransaction                  | ➡️     |
+|                   | eth_getTransactionByHash                | ⌛     | v2      |
+|                   | eth_getTransactionByBlockHashAndIndex   | ⌛     | v2      |
+|                   | eth_getTransactionByBlockNumberAndIndex | ⌛     | v2      |
+|                   | eth_getTransactionReceipt               | ⌛     | v2      |
+
+## Non-supported features
+
+- Currently does not support batch requests.
+
+## Warnings
+
+- To use this prover the ehtereum provider must support the `eth_getProof` method. Unfortunately, Infura does not currently support this endpoint. As an alternative, we suggest using Alchemy.
+
 ## Prerequisites
 
 - [Lerna](https://github.com/lerna/lerna)

packages/prover/package.json

@@ -58,11 +58,13 @@
     "generate-fixtures": "npx ts-node --esm scripts/generate_fixtures.ts"
   },
   "dependencies": {
-    "@ethereumjs/block": "^4.2.1",
-    "@ethereumjs/common": "^3.1.1",
+    "@ethereumjs/block": "^4.2.2",
+    "@ethereumjs/common": "^3.1.2",  
     "@ethereumjs/rlp": "^4.0.1",
-    "@ethereumjs/trie": "^5.0.4",
-    "@ethereumjs/util": "^8.0.5",
+    "@ethereumjs/trie": "^5.0.5",
+    "@ethereumjs/util": "^8.0.6",
+    "@ethereumjs/vm": "^6.4.2",
+    "@ethereumjs/blockchain": "^6.2.2",
     "@lodestar/api": "^1.8.0",
     "@lodestar/config": "^1.8.0",
     "@lodestar/light-client": "^1.8.0",

packages/prover/scripts/generate_fixtures.ts

@@ -23,41 +23,104 @@ const networkURLs: Record<NETWORK, {beacon: string; rpc: string}> = {
 
 let idIndex = Math.floor(Math.random() * 1000000);
 
-async function rawEth(network: NETWORK, payload: Record<string, unknown>): Promise<Record<string, unknown>> {
-  return (await axios({url: networkURLs[network].rpc, method: "post", data: payload, responseType: "json"}))
-    .data as Record<string, unknown>;
+async function rawEth(
+  network: NETWORK,
+  method: string,
+  params: unknown[]
+): Promise<{payload: Record<string, unknown>; response: Record<string, unknown>}> {
+  const payload = {id: idIndex++, jsonrpc: "2.0", method, params};
+
+  const response = (
+    await axios({
+      url: networkURLs[network].rpc,
+      method: "post",
+      data: payload,
+      responseType: "json",
+    })
+  ).data as Record<string, unknown>;
+
+  return {payload, response};
 }
 
 async function rawBeacon(network: NETWORK, path: string): Promise<Record<string, unknown>> {
   return (await axios.get(`${networkURLs[network].beacon}/${path}`)).data as Record<string, unknown>;
 }
 
-async function generateFixture(
-  label: string,
-  {method, params}: {method: string; params: unknown[]},
-  {slot}: {slot: number | string},
-  network: NETWORK = "sepolia"
-): Promise<void> {
-  const request = {id: idIndex++, jsonrpc: "2.0", method, params};
-  const response = await rawEth(network, request);
-  // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-explicit-any
-  const executionPayload: unknown = ((await rawBeacon(network, `eth/v2/beacon/blocks/${slot}`)) as any).data.message
-    .body.execution_payload;
-
-  // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-explicit-any
-  const headers: unknown = ((await rawBeacon(network, `eth/v1/beacon/headers/${slot}`)) as any).data;
+type Generator = (opts: {latest: string; finalized: string}) => {
+  request: {method: string; params: unknown[]};
+  slot: number | string;
+  dependentRequests?: {method: string; params: unknown[]}[];
+};
+
+type Block = {hash: string; number: string};
+
+const getBlockByNumber = async (network: NETWORK, number: string | number, dehydrate = false): Promise<Block> => {
+  const {response} = await rawEth(network, "eth_getBlockByNumber", [number, dehydrate]);
+
+  if (response.error || !response.result) {
+    throw new Error("Invalid response" + JSON.stringify(response));
+  }
+
+  return response.result as Block;
+};
+
+const getBlockByHash = async (network: NETWORK, hash: string | number, dehydrate = false): Promise<Block> => {
+  const {response} = await rawEth(network, "eth_getBlockByHash", [hash, dehydrate]);
+
+  if (response.error || !response.result) {
+    throw new Error("Invalid response" + JSON.stringify(response));
+  }
+
+  return response.result as Block;
+};
+
+async function generateFixture(label: string, generator: Generator, network: NETWORK = "sepolia"): Promise<void> {
+  const latest = await getBlockByNumber(network, "latest");
+  const finalized = await getBlockByNumber(network, "finalized");
 
+  const opts = generator({latest: latest.hash, finalized: finalized.hash});
+  const slot = opts.slot;
+  const {payload: request, response} = await rawEth(network, opts.request.method, opts.request.params);
   if (response.error || !response.result) {
     throw new Error("Invalid response" + JSON.stringify(response));
   }
 
+  const beacon = {
+    // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-explicit-any
+    executionPayload: ((await rawBeacon(network, `eth/v2/beacon/blocks/${slot}`)) as any).data.message.body
+      .execution_payload,
+    // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-explicit-any
+    headers: ((await rawBeacon(network, `eth/v1/beacon/headers/${slot}`)) as any).data,
+  };
+
+  const payloadBlock = await getBlockByHash(
+    network,
+    // eslint-disable-next-line @typescript-eslint/naming-convention
+    (beacon.executionPayload as {block_hash: string}).block_hash,
+    true
+  );
+
+  const execution = {
+    block: payloadBlock,
+  };
+
+  const dependentRequests = [];
+  for (const {method, params} of opts.dependentRequests ?? []) {
+    const {payload, response} = await rawEth(network, method, params);
+    if (response.error || !response.result) {
+      throw new Error("Invalid response" + JSON.stringify(response));
+    }
+    dependentRequests.push({payload, response});
+  }
+
   const fixture = {
     label,
     network,
     request,
     response,
-    executionPayload,
-    headers,
+    beacon,
+    execution,
+    dependentRequests,
   };
 
   const dir = path.join(__dirname, "..", "test/fixtures", network);
@@ -67,56 +130,92 @@ async function generateFixture(
   console.info("Generated fixture:", label);
 }
 
-await generateFixture(
-  "eth_getBlock_with_no_accessList",
-  {
+await generateFixture("eth_getBlock_with_no_accessList", () => ({
+  request: {
     method: "eth_getBlockByHash",
     params: ["0x75b10426177f0f4bd8683999e2c7c597007c6e7c4551d6336c0f880b12c6f3bf", true],
   },
-  {slot: 2144468}
-);
+  slot: 2144468,
+}));
 
-await generateFixture(
-  "eth_getBlock_with_contractCreation",
-  {
+await generateFixture("eth_getBlock_with_contractCreation", () => ({
+  request: {
     method: "eth_getBlockByHash",
     params: ["0x3a0225b38d5927a37cc95fd48254e83c4e9b70115918a103d9fd7e36464030d4", true],
   },
-  {slot: 625024}
-);
-
-await generateFixture(
-  "eth_getBalance_eoa",
-  {method: "eth_getBalance", params: ["0xC4bFccB1668d6E464F33a76baDD8C8D7D341e04A", "latest"]},
-  {slot: "head"}
-);
-
-await generateFixture(
-  "eth_getBalance_eoa",
-  {method: "eth_getBalance", params: ["0xC4bFccB1668d6E464F33a76baDD8C8D7D341e04A", "latest"]},
-  {slot: "head"}
-);
-
-await generateFixture(
-  "eth_getBalance_eoa_proof",
-  {method: "eth_getProof", params: ["0xC4bFccB1668d6E464F33a76baDD8C8D7D341e04A", [], "latest"]},
-  {slot: "head"}
-);
-
-await generateFixture(
-  "eth_getBalance_contract",
-  {method: "eth_getBalance", params: ["0xa54aeF0dAB669e8e1A164BCcB323549a818a0497", "latest"]},
-  {slot: "head"}
-);
-
-await generateFixture(
-  "eth_getBalance_contract_proof",
-  {method: "eth_getProof", params: ["0xa54aeF0dAB669e8e1A164BCcB323549a818a0497", [], "latest"]},
-  {slot: "head"}
-);
+  slot: 625024,
+}));
+
+await generateFixture("eth_getBalance_eoa", ({latest}) => ({
+  request: {method: "eth_getBalance", params: ["0xC4bFccB1668d6E464F33a76baDD8C8D7D341e04A", latest]},
+  slot: "head",
+  dependentRequests: [{method: "eth_getProof", params: ["0xC4bFccB1668d6E464F33a76baDD8C8D7D341e04A", [], latest]}],
+}));
+
+await generateFixture("eth_getBalance_contract", ({latest}) => ({
+  request: {method: "eth_getBalance", params: ["0xa54aeF0dAB669e8e1A164BCcB323549a818a0497", latest]},
+  slot: "head",
+  dependentRequests: [{method: "eth_getProof", params: ["0xa54aeF0dAB669e8e1A164BCcB323549a818a0497", [], latest]}],
+}));
+
+await generateFixture("eth_getCode", ({latest}) => ({
+  request: {method: "eth_getCode", params: ["0xa54aeF0dAB669e8e1A164BCcB323549a818a0497", latest]},
+  slot: "head",
+  dependentRequests: [{method: "eth_getProof", params: ["0xa54aeF0dAB669e8e1A164BCcB323549a818a0497", [], latest]}],
+}));
+
+await generateFixture("eth_getTransactionCount", ({latest}) => ({
+  request: {method: "eth_getTransactionCount", params: ["0xC4bFccB1668d6E464F33a76baDD8C8D7D341e04A", latest]},
+  slot: "head",
+  dependentRequests: [{method: "eth_getProof", params: ["0xC4bFccB1668d6E464F33a76baDD8C8D7D341e04A", [], latest]}],
+}));
 
 await generateFixture(
-  "eth_getCode",
-  {method: "eth_getCode", params: ["0xa54aeF0dAB669e8e1A164BCcB323549a818a0497", "latest"]},
-  {slot: "head"}
+  "eth_call",
+  ({latest}) => ({
+    request: {
+      method: "eth_call",
+      params: [
+        {
+          data: "0xe6cb901300000000000000000000000000000000000000000000000000000000000000040000000000000000000000000000000000000000000000000000000000000005",
+          to: "0xade2a9c8b033d60ffcdb8cfc974dd87b2a9c1f27",
+        },
+        latest,
+      ],
+    },
+    slot: "head",
+    dependentRequests: [
+      {
+        method: "eth_createAccessList",
+        params: [
+          {
+            to: "0xade2a9c8b033d60ffcdb8cfc974dd87b2a9c1f27",
+            from: "0x0000000000000000000000000000000000000000",
+            data: "0xe6cb901300000000000000000000000000000000000000000000000000000000000000040000000000000000000000000000000000000000000000000000000000000005",
+            value: undefined,
+            gas: "0x1c9c380",
+            gasPrice: "0x0",
+          },
+          latest,
+        ],
+      },
+      {
+        method: "eth_getProof",
+        params: ["0x0000000000000000000000000000000000000000", [], latest],
+      },
+      {
+        method: "eth_getCode",
+        params: ["0x0000000000000000000000000000000000000000", latest],
+      },
+      {
+        method: "eth_getProof",
+        params: ["0xade2a9c8b033d60ffcdb8cfc974dd87b2a9c1f27", [], latest],
+      },
+      {
+        method: "eth_getCode",
+        params: ["0xade2a9c8b033d60ffcdb8cfc974dd87b2a9c1f27", latest],
+      },
+    ],
+  }),
+  "mainnet"
 );

packages/prover/src/constants.ts

@@ -2,3 +2,4 @@
 export const MAX_REQUEST_LIGHT_CLIENT_UPDATES = 128;
 export const MAX_PAYLOAD_HISTORY = 32;
 export const UNVERIFIED_RESPONSE_CODE = -33091;
+export const ZERO_ADDRESS = "0x0000000000000000000000000000000000000000";

packages/prover/src/interfaces.ts

@@ -22,6 +22,9 @@ export type ELRequestHandler<Params = unknown[], Response = unknown> = (
   payload: ELRequestPayload<Params>
 ) => Promise<ELResponse<Response> | undefined>;
 
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+export type ELRequestHandlerAny = ELRequestHandler<any, any>;
+
 // Modern providers uses this structure e.g. Web3 4.x
 export interface EIP1193Provider {
   request: (payload: ELRequestPayload) => Promise<ELResponse>;

packages/prover/src/proof_provider/payload_store.ts

@@ -100,6 +100,8 @@ export class PayloadStore {
       if (latestPayload && latestPayload.blockNumber < payload.blockNumber) {
         this.latestBlockRoot = blockRoot;
       }
+    } else {
+      this.latestBlockRoot = blockRoot;
     }
 
     if (finalized) {