EIP-4844: Force ExternalData arg in state transition fn (#4846)

* Add ExternalData to state transition function

* Explicitly state validation rules

* Fix tests

* Fix unit test types

packages/beacon-node/src/api/impl/beacon/state/utils.ts

@@ -7,6 +7,8 @@ import {
   createCachedBeaconState,
   createEmptyEpochContextImmutableData,
   PubkeyIndexMap,
+  ExecutionPayloadStatus,
+  DataAvailableStatus,
 } from "@lodestar/state-transition";
 import {BLSPubkey, phase0} from "@lodestar/types";
 import {stateTransition, processSlots} from "@lodestar/state-transition";
@@ -222,6 +224,9 @@ async function getFinalizedState(
   // process blocks up to the requested slot
   for await (const block of db.blockArchive.valuesStream({gt: state.slot, lte: slot})) {
     state = stateTransition(state, block, {
+      // Replaying finalized blocks, all data is considered valid
+      executionPayloadStatus: ExecutionPayloadStatus.valid,
+      dataAvailableStatus: DataAvailableStatus.available,
       verifyStateRoot: false,
       verifyProposer: false,
       verifySignatures: false,

packages/beacon-node/src/chain/blocks/verifyBlocksStateTransitionOnly.ts

@@ -1,5 +1,10 @@
-import {CachedBeaconStateAllForks, stateTransition} from "@lodestar/state-transition";
 import {ErrorAborted, ILogger, sleep} from "@lodestar/utils";
+import {
+  CachedBeaconStateAllForks,
+  stateTransition,
+  ExecutionPayloadStatus,
+  DataAvailableStatus,
+} from "@lodestar/state-transition";
 import {IMetrics} from "../../metrics/index.js";
 import {BlockError, BlockErrorCode} from "../errors/index.js";
 import {BlockProcessOpts} from "../options.js";
@@ -37,6 +42,11 @@ export async function verifyBlocksStateTransitionOnly(
       preState,
       block,
       {
+        // NOTE: Assume valid for now while sending payload to execution engine in parallel
+        // Latter verifyBlocksInEpoch() will make sure that payload is indeed valid
+        executionPayloadStatus: ExecutionPayloadStatus.valid,
+        // TODO EIP-4844: Conditionally validate blobs
+        dataAvailableStatus: DataAvailableStatus.preEIP4844,
         // false because it's verified below with better error typing
         verifyStateRoot: false,
         // if block is trusted don't verify proposer or op signature

packages/beacon-node/src/chain/produceBlock/computeNewStateRoot.ts

@@ -1,4 +1,9 @@
-import {CachedBeaconStateAllForks, stateTransition} from "@lodestar/state-transition";
+import {
+  CachedBeaconStateAllForks,
+  DataAvailableStatus,
+  ExecutionPayloadStatus,
+  stateTransition,
+} from "@lodestar/state-transition";
 import {allForks, Root} from "@lodestar/types";
 import {ZERO_HASH} from "../../constants/index.js";
 import {IMetrics} from "../../metrics/index.js";
@@ -22,10 +27,18 @@ export function computeNewStateRoot(
   const postState = stateTransition(
     state,
     blockEmptySig,
-    // verifyStateRoot: false  | the root in the block is zero-ed, it's being computed here
-    // verifyProposer: false   | as the block signature is zero-ed
-    // verifySignatures: false | since the data to assemble the block is trusted
-    {verifyStateRoot: false, verifyProposer: false, verifySignatures: false},
+    {
+      // ExecutionPayloadStatus.valid: Assume payload valid, it has been produced by a trusted EL
+      executionPayloadStatus: ExecutionPayloadStatus.valid,
+      // DataAvailableStatus.available: Assume the blobs to be available, have just been produced by trusted EL
+      dataAvailableStatus: DataAvailableStatus.available,
+      // verifyStateRoot: false  | the root in the block is zero-ed, it's being computed here
+      verifyStateRoot: false,
+      // verifyProposer: false   | as the block signature is zero-ed
+      verifyProposer: false,
+      // verifySignatures: false | since the data to assemble the block is trusted
+      verifySignatures: false,
+    },
     metrics
   );
 

packages/beacon-node/src/chain/regen/regen.ts

@@ -3,6 +3,8 @@ import {
   CachedBeaconStateAllForks,
   computeEpochAtSlot,
   computeStartSlotAtEpoch,
+  DataAvailableStatus,
+  ExecutionPayloadStatus,
   processSlots,
   stateTransition,
 } from "@lodestar/state-transition";
@@ -173,6 +175,9 @@ export class StateRegenerator implements IStateRegenerator {
           state,
           block,
           {
+            // Replay previously imported blocks, assume valid and available
+            executionPayloadStatus: ExecutionPayloadStatus.valid,
+            dataAvailableStatus: DataAvailableStatus.available,
             verifyStateRoot: false,
             verifyProposer: false,
             verifySignatures: false,

packages/beacon-node/test/spec/presets/finality.ts

@@ -1,4 +1,9 @@
-import {BeaconStateAllForks, stateTransition} from "@lodestar/state-transition";
+import {
+  BeaconStateAllForks,
+  DataAvailableStatus,
+  ExecutionPayloadStatus,
+  stateTransition,
+} from "@lodestar/state-transition";
 import {altair, bellatrix, ssz} from "@lodestar/types";
 import {ForkName} from "@lodestar/params";
 import {createCachedBeaconStateTest} from "../../utils/cachedBeaconState.js";
@@ -18,6 +23,9 @@ export const finality: TestRunnerFn<FinalityTestCase, BeaconStateAllForks> = (fo
         const signedBlock = testcase[`blocks_${i}`] as bellatrix.SignedBeaconBlock;
 
         state = stateTransition(state, signedBlock, {
+          // TODO EIP-4844: Should assume valid and available for this test?
+          executionPayloadStatus: ExecutionPayloadStatus.valid,
+          dataAvailableStatus: DataAvailableStatus.available,
           verifyStateRoot: false,
           verifyProposer: verify,
           verifySignatures: verify,

packages/beacon-node/test/spec/presets/operations.ts

@@ -2,6 +2,8 @@ import {
   BeaconStateAllForks,
   CachedBeaconStateAllForks,
   CachedBeaconStateBellatrix,
+  DataAvailableStatus,
+  ExecutionPayloadStatus,
   getBlockRootAtSlot,
 } from "@lodestar/state-transition";
 import * as blockFns from "@lodestar/state-transition/block";
@@ -72,7 +74,13 @@ const operationFns: Record<string, BlockProcessFn<CachedBeaconStateAllForks>> =
       fork,
       (state as CachedBeaconStateAllForks) as CachedBeaconStateBellatrix,
       testCase.execution_payload,
-      {notifyNewPayload: () => testCase.execution.execution_valid}
+      {
+        executionPayloadStatus: testCase.execution.execution_valid
+          ? ExecutionPayloadStatus.valid
+          : ExecutionPayloadStatus.invalid,
+        // TODO EIP-4844: Make this value dynamic on fork EIP4844
+        dataAvailableStatus: DataAvailableStatus.preEIP4844,
+      }
     );
   },
 };

packages/beacon-node/test/spec/presets/sanity.ts

@@ -1,5 +1,11 @@
 import {InputType} from "@lodestar/spec-test-util";
-import {BeaconStateAllForks, processSlots, stateTransition} from "@lodestar/state-transition";
+import {
+  BeaconStateAllForks,
+  DataAvailableStatus,
+  ExecutionPayloadStatus,
+  processSlots,
+  stateTransition,
+} from "@lodestar/state-transition";
 import {allForks, bellatrix, ssz} from "@lodestar/types";
 import {ForkName} from "@lodestar/params";
 import {bnToNum} from "@lodestar/utils";
@@ -57,6 +63,9 @@ export const sanityBlocks: TestRunnerFn<SanityBlocksTestCase, BeaconStateAllFork
       for (let i = 0; i < testcase.meta.blocks_count; i++) {
         const signedBlock = testcase[`blocks_${i}`] as bellatrix.SignedBeaconBlock;
         wrappedState = stateTransition(wrappedState, signedBlock, {
+          // TODO EIP-4844: Should assume valid and available for this test?
+          executionPayloadStatus: ExecutionPayloadStatus.valid,
+          dataAvailableStatus: DataAvailableStatus.available,
           verifyStateRoot: verify,
           verifyProposer: verify,
           verifySignatures: verify,

packages/beacon-node/test/spec/presets/transition.ts

@@ -1,4 +1,9 @@
-import {BeaconStateAllForks, stateTransition} from "@lodestar/state-transition";
+import {
+  BeaconStateAllForks,
+  DataAvailableStatus,
+  ExecutionPayloadStatus,
+  stateTransition,
+} from "@lodestar/state-transition";
 import {allForks, ssz} from "@lodestar/types";
 import {createIChainForkConfig, IChainConfig} from "@lodestar/config";
 import {ForkName} from "@lodestar/params";
@@ -45,6 +50,9 @@ export const transition: TestRunnerFn<TransitionTestCase, BeaconStateAllForks> =
       for (let i = 0; i < meta.blocks_count; i++) {
         const signedBlock = testcase[`blocks_${i}`] as allForks.SignedBeaconBlock;
         state = stateTransition(state, signedBlock, {
+          // TODO EIP-4844: Should assume valid and available for this test?
+          executionPayloadStatus: ExecutionPayloadStatus.valid,
+          dataAvailableStatus: DataAvailableStatus.available,
           verifyStateRoot: true,
           verifyProposer: false,
           verifySignatures: false,

packages/state-transition/src/block/externalData.ts

@@ -0,0 +1,25 @@
+/**
+ * Should emulate the return value of `ExecutionEngine.notifyNewPayload()`, such that:
+ *
+ * Returns ``True`` iff ``execution_payload`` is valid with respect to ``self.execution_state``.
+ *
+ * Note: `processExecutionPayload()` depends on process_randao function call as it retrieves the most recent randao
+ * mix from the state. Implementations that are considering parallel processing of execution payload with respect to
+ * beacon chain state transition function should work around this dependency.
+ */
+export enum ExecutionPayloadStatus {
+  preMerge = "preMerge",
+  invalid = "invalid",
+  valid = "valid",
+}
+
+export enum DataAvailableStatus {
+  preEIP4844 = "preEIP4844",
+  notAvailable = "notAvailable",
+  available = "available",
+}
+
+export interface BlockExternalData {
+  executionPayloadStatus: ExecutionPayloadStatus;
+  dataAvailableStatus: DataAvailableStatus;
+}

packages/state-transition/src/block/index.ts

@@ -1,6 +1,5 @@
 import {ForkSeq} from "@lodestar/params";
 import {allForks, altair, capella} from "@lodestar/types";
-import {ExecutionEngine} from "../util/executionEngine.js";
 import {getFullOrBlindedPayload, isExecutionEnabled} from "../util/execution.js";
 import {CachedBeaconStateAllForks, CachedBeaconStateCapella, CachedBeaconStateBellatrix} from "../types.js";
 import {processExecutionPayload} from "./processExecutionPayload.js";
@@ -9,6 +8,7 @@ import {processBlockHeader} from "./processBlockHeader.js";
 import {processEth1Data} from "./processEth1Data.js";
 import {processOperations} from "./processOperations.js";
 import {processRandao} from "./processRandao.js";
+import {BlockExternalData} from "./externalData.js";
 import {processWithdrawals} from "./processWithdrawals.js";
 
 // Spec tests
@@ -24,14 +24,22 @@ export * from "./processOperations.js";
 
 export * from "./initiateValidatorExit.js";
 export * from "./isValidIndexedAttestation.js";
+export * from "./externalData.js";
+
+export interface ProcessBlockOpts {
+  verifySignatures?: boolean;
+  disabledWithdrawals?: boolean;
+}
 
 export function processBlock(
   fork: ForkSeq,
   state: CachedBeaconStateAllForks,
   block: allForks.FullOrBlindedBeaconBlock,
-  verifySignatures = true,
-  executionEngine: ExecutionEngine | null
+  externalData: BlockExternalData & ProcessBlockOpts,
+  opts?: ProcessBlockOpts
 ): void {
+  const {verifySignatures = true} = opts ?? {};
+
   processBlockHeader(state, block);
 
   // The call to the process_execution_payload must happen before the call to the process_randao as the former depends
@@ -44,7 +52,7 @@ export function processBlock(
         fullOrBlindedPayload as capella.FullOrBlindedExecutionPayload
       );
     }
-    processExecutionPayload(fork, state as CachedBeaconStateBellatrix, fullOrBlindedPayload, executionEngine);
+    processExecutionPayload(fork, state as CachedBeaconStateBellatrix, fullOrBlindedPayload, externalData);
   }
 
   processRandao(state, block, verifySignatures);

packages/state-transition/src/block/processExecutionPayload.ts

@@ -1,22 +1,16 @@
-import {ssz, allForks} from "@lodestar/types";
+import {ssz, allForks, capella} from "@lodestar/types";
 import {toHexString, byteArrayEquals} from "@chainsafe/ssz";
 import {ForkSeq} from "@lodestar/params";
-
 import {CachedBeaconStateBellatrix, CachedBeaconStateCapella} from "../types.js";
 import {getRandaoMix} from "../util/index.js";
-import {ExecutionEngine} from "../util/executionEngine.js";
-import {
-  isExecutionPayload,
-  isMergeTransitionComplete,
-  isCapellaPayload,
-  isCapellaPayloadHeader,
-} from "../util/execution.js";
+import {isExecutionPayload, isMergeTransitionComplete} from "../util/execution.js";
+import {BlockExternalData, ExecutionPayloadStatus} from "./externalData.js";
 
 export function processExecutionPayload(
   fork: ForkSeq,
   state: CachedBeaconStateBellatrix | CachedBeaconStateCapella,
   payload: allForks.FullOrBlindedExecutionPayload,
-  executionEngine: ExecutionEngine | null
+  externalData: BlockExternalData
 ): void {
   // Verify consistency of the parent hash, block number, base fee per gas and gas limit
   // with respect to the previous execution payload header
@@ -54,14 +48,25 @@ export function processExecutionPayload(
   // if executionEngine is null, executionEngine.onPayload MUST be called after running processBlock to get the
   // correct randao mix. Since executionEngine will be an async call in most cases it is called afterwards to keep
   // the state transition sync
-  if (isExecutionPayload(payload) && executionEngine && !executionEngine.notifyNewPayload(fork, payload)) {
-    throw Error("Invalid execution payload");
+  //
+  // Equivalent to `assert executionEngine.notifyNewPayload(payload)`
+  if (isExecutionPayload(payload)) {
+    switch (externalData.executionPayloadStatus) {
+      case ExecutionPayloadStatus.preMerge:
+        throw Error("executionPayloadStatus preMerge");
+      case ExecutionPayloadStatus.invalid:
+        throw Error("Invalid execution payload");
+      case ExecutionPayloadStatus.valid:
+        break; // ok
+    }
   }
 
+  // For blinded or full payload -> return common header
   const transactionsRoot = isExecutionPayload(payload)
     ? ssz.bellatrix.Transactions.hashTreeRoot(payload.transactions)
     : payload.transactionsRoot;
-  const bellatrixPayloadFields = {
+
+  const bellatrixPayloadFields: allForks.ExecutionPayloadHeader = {
     parentHash: payload.parentHash,
     feeRecipient: payload.feeRecipient,
     stateRoot: payload.stateRoot,
@@ -78,21 +83,15 @@ export function processExecutionPayload(
     transactionsRoot,
   };
 
-  const withdrawalsRoot = isCapellaPayload(payload)
-    ? isCapellaPayloadHeader(payload)
-      ? payload.withdrawalsRoot
-      : ssz.capella.Withdrawals.hashTreeRoot(payload.withdrawals)
-    : undefined;
-
-  // Cache execution payload header
-  if (withdrawalsRoot !== undefined) {
-    (state as CachedBeaconStateCapella).latestExecutionPayloadHeader = ssz.capella.ExecutionPayloadHeader.toViewDU({
-      ...bellatrixPayloadFields,
-      withdrawalsRoot,
-    });
-  } else {
-    (state as CachedBeaconStateBellatrix).latestExecutionPayloadHeader = ssz.bellatrix.ExecutionPayloadHeader.toViewDU(
-      bellatrixPayloadFields
+  if (fork >= ForkSeq.capella) {
+    (bellatrixPayloadFields as capella.ExecutionPayloadHeader).withdrawalsRoot = ssz.capella.Withdrawals.hashTreeRoot(
+      (payload as capella.ExecutionPayload).withdrawals
     );
   }
+
+  // TODO EIP-4844: Types are not happy by default. Since it's a generic allForks type going through ViewDU
+  // transformation then into allForks, probably some weird intersection incompatibility happens
+  state.latestExecutionPayloadHeader = state.config
+    .getExecutionForkTypes(state.slot)
+    .ExecutionPayloadHeader.toViewDU(bellatrixPayloadFields) as typeof state.latestExecutionPayloadHeader;
 }

packages/state-transition/src/index.ts

@@ -41,6 +41,7 @@ export {isValidVoluntaryExit} from "./block/processVoluntaryExit.js";
 export {assertValidBlsToExecutionChange} from "./block/processBlsToExecutionChange.js";
 export {assertValidProposerSlashing} from "./block/processProposerSlashing.js";
 export {assertValidAttesterSlashing} from "./block/processAttesterSlashing.js";
+export {ExecutionPayloadStatus, DataAvailableStatus, BlockExternalData} from "./block/externalData.js";
 
 // BeaconChain, to prepare new blocks
 export {becomesNewEth1Data} from "./block/processEth1Data.js";