Implement Kintsugi specs 🍵 (the Merge November sprint PR) (#3418)

* completing ethereum/consensus-specs#2617

* add TBH_ACTIVATION_EPOCH ethereum/consensus-specs#2682

* remove prepare payload specs 2682, add payload id as return to notify_forkchoice... 2711

* remove uniion from transaction (2683), fix gossip and tx size (2686), remove gas validation(2699)

* remove extraneous p2p condition (2687)

* params e2e test fix

* update penalty params for Merge (2698)

* updating spec version

* spec runner merge sanity and operations fixes

* removing the beacon block gossip validations as per 1.1.4

* feedback cleanup

* spec v1.1.5, fixed blockhash (2710), payloadid (2711) already changed, tbh activation check(2712) already correct

* kintsugi geth interop

* ee test fixes

* assetterminalpowblock refac and root comparision fix

* runGethPreMerge test case

* runGethPreMerge scenario with ignoring geth side ttd not reached error

* assertvalidterminalpow block fix

* tracker in comments for geth preMerge to postMerge issue

* merge transition scenario with ttd > 0 check fix deployed on geth:kintsugi-spec

* cleanup as merge-interop test file scenaros updated and working

* handling prepare payload failure scenarios

* seperating optimistic sync, fixing and testing the transaction submission/execution

.github/workflows/test-sim-merge.yml

@@ -34,7 +34,7 @@ jobs:
       # Install Geth merge interop
       - uses: actions/setup-go@v2
       - name: Clone Geth merge interop branch
-        run: git clone -b merge-interop-spec https://github.com/MariusVanDerWijden/go-ethereum.git
+        run: git clone -b kintsugi-spec https://github.com/MariusVanDerWijden/go-ethereum.git
       - name: Build Geth
         run: cd go-ethereum && make
 

packages/beacon-state-transition/src/allForks/block/slashValidator.ts

@@ -4,6 +4,7 @@ import {
   ForkName,
   MIN_SLASHING_PENALTY_QUOTIENT,
   MIN_SLASHING_PENALTY_QUOTIENT_ALTAIR,
+  MIN_SLASHING_PENALTY_QUOTIENT_MERGE,
   PROPOSER_REWARD_QUOTIENT,
   PROPOSER_WEIGHT,
   WEIGHT_DENOMINATOR,
@@ -35,7 +36,11 @@ export function slashValidatorAllForks(
   state.slashings[epoch % EPOCHS_PER_SLASHINGS_VECTOR] += BigInt(effectiveBalance);
 
   const minSlashingPenaltyQuotient =
-    fork === ForkName.phase0 ? MIN_SLASHING_PENALTY_QUOTIENT : MIN_SLASHING_PENALTY_QUOTIENT_ALTAIR;
+    fork === ForkName.phase0
+      ? MIN_SLASHING_PENALTY_QUOTIENT
+      : fork === ForkName.altair
+      ? MIN_SLASHING_PENALTY_QUOTIENT_ALTAIR
+      : MIN_SLASHING_PENALTY_QUOTIENT_MERGE;
   decreaseBalance(state, slashedIndex, Math.floor(effectiveBalance / minSlashingPenaltyQuotient));
 
   // apply proposer and whistleblower rewards

packages/beacon-state-transition/src/allForks/epoch/processSlashings.ts

@@ -6,6 +6,7 @@ import {
   ForkName,
   PROPORTIONAL_SLASHING_MULTIPLIER,
   PROPORTIONAL_SLASHING_MULTIPLIER_ALTAIR,
+  PROPORTIONAL_SLASHING_MULTIPLIER_MERGE,
 } from "@chainsafe/lodestar-params";
 
 import {decreaseBalance} from "../../util";
@@ -35,7 +36,11 @@ export function processSlashingsAllForks(
   const totalSlashings = Array.from(readonlyValues(state.slashings)).reduce((a, b) => a + b, BigInt(0));
 
   const proportionalSlashingMultiplier =
-    fork === ForkName.phase0 ? PROPORTIONAL_SLASHING_MULTIPLIER : PROPORTIONAL_SLASHING_MULTIPLIER_ALTAIR;
+    fork === ForkName.phase0
+      ? PROPORTIONAL_SLASHING_MULTIPLIER
+      : fork === ForkName.altair
+      ? PROPORTIONAL_SLASHING_MULTIPLIER_ALTAIR
+      : PROPORTIONAL_SLASHING_MULTIPLIER_MERGE;
 
   const adjustedTotalSlashingBalance = bigIntMin(totalSlashings * BigInt(proportionalSlashingMultiplier), totalBalance);
   const increment = EFFECTIVE_BALANCE_INCREMENT;

packages/beacon-state-transition/src/altair/epoch/balance.ts

@@ -2,11 +2,13 @@ import {allForks, altair} from "@chainsafe/lodestar-types";
 import {
   EFFECTIVE_BALANCE_INCREMENT,
   INACTIVITY_PENALTY_QUOTIENT_ALTAIR,
+  INACTIVITY_PENALTY_QUOTIENT_MERGE,
   PARTICIPATION_FLAG_WEIGHTS,
   TIMELY_HEAD_FLAG_INDEX,
   TIMELY_SOURCE_FLAG_INDEX,
   TIMELY_TARGET_FLAG_INDEX,
   WEIGHT_DENOMINATOR,
+  ForkName,
 } from "@chainsafe/lodestar-params";
 import {
   CachedBeaconState,
@@ -56,7 +58,12 @@ export function getRewardsPenaltiesDeltas(
   // so there are limited values of them like 32000000000, 31000000000, 30000000000
   const rewardPenaltyItemCache = new Map<number, IRewardPenaltyItem>();
   const {config, epochCtx} = state;
-  const penaltyDenominator = config.INACTIVITY_SCORE_BIAS * INACTIVITY_PENALTY_QUOTIENT_ALTAIR;
+  const fork = config.getForkName(state.slot);
+
+  const inactivityPenalityMultiplier =
+    fork === ForkName.altair ? INACTIVITY_PENALTY_QUOTIENT_ALTAIR : INACTIVITY_PENALTY_QUOTIENT_MERGE;
+  const penaltyDenominator = config.INACTIVITY_SCORE_BIAS * inactivityPenalityMultiplier;
+
   const {statuses} = process;
   for (let i = 0; i < statuses.length; i++) {
     const status = statuses[i];

packages/beacon-state-transition/src/merge/block/index.ts

@@ -2,11 +2,15 @@ import {allForks, altair, merge} from "@chainsafe/lodestar-types";
 
 import {CachedBeaconState} from "../../allForks/util";
 import {processBlockHeader, processEth1Data, processRandao} from "../../allForks/block";
-import {processOperations} from "../../altair/block/processOperations";
+import {processOperations} from "./processOperations";
 import {processSyncAggregate} from "../../altair/block/processSyncCommittee";
 import {processExecutionPayload} from "./processExecutionPayload";
 import {ExecutionEngine} from "../executionEngine";
 import {isExecutionEnabled} from "../utils";
+import {processAttesterSlashing} from "./processAttesterSlashing";
+import {processProposerSlashing} from "./processProposerSlashing";
+
+export {processOperations, processAttesterSlashing, processProposerSlashing};
 
 export function processBlock(
   state: CachedBeaconState<merge.BeaconState>,
@@ -23,6 +27,6 @@ export function processBlock(
 
   processRandao(state as CachedBeaconState<allForks.BeaconState>, block, verifySignatures);
   processEth1Data(state as CachedBeaconState<allForks.BeaconState>, block.body);
-  processOperations((state as unknown) as CachedBeaconState<altair.BeaconState>, block.body, verifySignatures);
+  processOperations(state, block.body, verifySignatures);
   processSyncAggregate((state as unknown) as CachedBeaconState<altair.BeaconState>, block, verifySignatures);
 }

packages/beacon-state-transition/src/merge/block/processAttesterSlashing.ts

@@ -0,0 +1,17 @@
+import {allForks, merge, phase0} from "@chainsafe/lodestar-types";
+import {ForkName} from "@chainsafe/lodestar-params";
+import {CachedBeaconState} from "../../allForks/util";
+import {processAttesterSlashing as processAttesterSlashingAllForks} from "../../allForks/block";
+
+export function processAttesterSlashing(
+  state: CachedBeaconState<merge.BeaconState>,
+  attesterSlashing: phase0.AttesterSlashing,
+  verifySignatures = true
+): void {
+  processAttesterSlashingAllForks(
+    ForkName.merge,
+    state as CachedBeaconState<allForks.BeaconState>,
+    attesterSlashing,
+    verifySignatures
+  );
+}

packages/beacon-state-transition/src/merge/block/processExecutionPayload.ts

@@ -1,35 +1,10 @@
-import {GAS_LIMIT_DENOMINATOR, MIN_GAS_LIMIT} from "@chainsafe/lodestar-params";
 import {merge, ssz} from "@chainsafe/lodestar-types";
 import {byteArrayEquals, List, toHexString} from "@chainsafe/ssz";
 import {CachedBeaconState} from "../../allForks";
 import {getRandaoMix} from "../../util";
 import {ExecutionEngine} from "../executionEngine";
 import {isMergeComplete} from "../utils";
 
-function isValidGasLimit(payload: merge.ExecutionPayload, parent: merge.ExecutionPayloadHeader): boolean {
-  const parentGasLimit = parent.gasLimit;
-
-  // Check if the payload used too much gas
-  if (payload.gasUsed > payload.gasLimit) {
-    return false;
-  }
-
-  // Check if the payload changed the gas limit too much
-  if (payload.gasLimit >= parentGasLimit + Math.floor(parentGasLimit / GAS_LIMIT_DENOMINATOR)) {
-    return false;
-  }
-  if (payload.gasLimit <= parentGasLimit - Math.floor(parentGasLimit / GAS_LIMIT_DENOMINATOR)) {
-    return false;
-  }
-
-  // Check if the gas limit is at least the minimum gas limit
-  if (payload.gasLimit < MIN_GAS_LIMIT) {
-    return false;
-  }
-
-  return true;
-}
-
 export function processExecutionPayload(
   state: CachedBeaconState<merge.BeaconState>,
   payload: merge.ExecutionPayload,
@@ -46,16 +21,6 @@ export function processExecutionPayload(
         )}`
       );
     }
-    if (payload.blockNumber !== latestExecutionPayloadHeader.blockNumber + 1) {
-      throw Error(
-        `Invalid execution payload blockNumber ${payload.blockNumber} parent=${latestExecutionPayloadHeader.blockNumber}`
-      );
-    }
-    if (!isValidGasLimit(payload, latestExecutionPayloadHeader)) {
-      throw Error(
-        `Invalid gasLimit gasUsed=${payload.gasUsed} gasLimit=${payload.gasLimit} parentGasLimit=${latestExecutionPayloadHeader.gasLimit}`
-      );
-    }
   }
 
   // Verify random

packages/beacon-state-transition/src/merge/block/processOperations.ts

@@ -0,0 +1,44 @@
+import {readonlyValues} from "@chainsafe/ssz";
+import {altair, merge} from "@chainsafe/lodestar-types";
+
+import {CachedBeaconState} from "../../allForks/util";
+import {processProposerSlashing} from "./processProposerSlashing";
+import {processAttesterSlashing} from "./processAttesterSlashing";
+import {processAttestations} from "../../altair/block/processAttestation";
+import {processDeposit} from "../../altair/block/processDeposit";
+import {processVoluntaryExit} from "../../altair/block/processVoluntaryExit";
+import {MAX_DEPOSITS} from "@chainsafe/lodestar-params";
+
+export function processOperations(
+  state: CachedBeaconState<merge.BeaconState>,
+  body: merge.BeaconBlockBody,
+  verifySignatures = true
+): void {
+  // verify that outstanding deposits are processed up to the maximum number of deposits
+  const maxDeposits = Math.min(MAX_DEPOSITS, state.eth1Data.depositCount - state.eth1DepositIndex);
+  if (body.deposits.length !== maxDeposits) {
+    throw new Error(
+      `Block contains incorrect number of deposits: depositCount=${body.deposits.length} expected=${maxDeposits}`
+    );
+  }
+
+  for (const proposerSlashing of readonlyValues(body.proposerSlashings)) {
+    processProposerSlashing(state, proposerSlashing, verifySignatures);
+  }
+  for (const attesterSlashing of readonlyValues(body.attesterSlashings)) {
+    processAttesterSlashing(state, attesterSlashing, verifySignatures);
+  }
+
+  processAttestations(
+    (state as unknown) as CachedBeaconState<altair.BeaconState>,
+    Array.from(readonlyValues(body.attestations)),
+    verifySignatures
+  );
+
+  for (const deposit of readonlyValues(body.deposits)) {
+    processDeposit((state as unknown) as CachedBeaconState<altair.BeaconState>, deposit);
+  }
+  for (const voluntaryExit of readonlyValues(body.voluntaryExits)) {
+    processVoluntaryExit((state as unknown) as CachedBeaconState<altair.BeaconState>, voluntaryExit, verifySignatures);
+  }
+}

packages/beacon-state-transition/src/merge/block/processProposerSlashing.ts

@@ -0,0 +1,17 @@
+import {allForks, merge, phase0} from "@chainsafe/lodestar-types";
+import {ForkName} from "@chainsafe/lodestar-params";
+import {CachedBeaconState} from "../../allForks/util";
+import {processProposerSlashing as processProposerSlashingAllForks} from "../../allForks/block";
+
+export function processProposerSlashing(
+  state: CachedBeaconState<merge.BeaconState>,
+  proposerSlashing: phase0.ProposerSlashing,
+  verifySignatures = true
+): void {
+  processProposerSlashingAllForks(
+    ForkName.merge,
+    state as CachedBeaconState<allForks.BeaconState>,
+    proposerSlashing,
+    verifySignatures
+  );
+}

packages/beacon-state-transition/src/merge/epoch/index.ts

@@ -0,0 +1 @@
+export {processSlashings} from "./processSlashings";

packages/beacon-state-transition/src/merge/epoch/processSlashings.ts

@@ -0,0 +1,8 @@
+import {allForks, merge} from "@chainsafe/lodestar-types";
+import {ForkName} from "@chainsafe/lodestar-params";
+import {CachedBeaconState, IEpochProcess} from "../../allForks/util";
+import {processSlashingsAllForks} from "../../allForks/epoch/processSlashings";
+
+export function processSlashings(state: CachedBeaconState<merge.BeaconState>, epochProcess: IEpochProcess): void {
+  processSlashingsAllForks(ForkName.merge, state as CachedBeaconState<allForks.BeaconState>, epochProcess);
+}

packages/beacon-state-transition/src/merge/index.ts

@@ -1,5 +1,6 @@
-export {processBlock} from "./block";
-export {upgradeState} from "./upgradeState";
+export * from "./block";
+export * from "./epoch";
+export * from "./upgradeState";
 export * from "./utils";
 
 // re-export merge lodestar types for ergonomic usage downstream

packages/config/src/chainConfig/presets/mainnet.ts

@@ -10,6 +10,7 @@ export const chainConfig: IChainConfig = {
   // TBD, 2**256-1 is a placeholder
   TERMINAL_TOTAL_DIFFICULTY: BigInt(115792089237316195423570985008687907853269984665640564039457584007913129639935),
   TERMINAL_BLOCK_HASH: b("0x0000000000000000000000000000000000000000000000000000000000000000"),
+  TERMINAL_BLOCK_HASH_ACTIVATION_EPOCH: Infinity,
 
   // Genesis
   // ---------------------------------------------------------------

packages/config/src/chainConfig/presets/minimal.ts

@@ -11,6 +11,7 @@ export const chainConfig: IChainConfig = {
   // TBD, 2**256-1 is a placeholder
   TERMINAL_TOTAL_DIFFICULTY: BigInt(115792089237316195423570985008687907853269984665640564039457584007913129639935),
   TERMINAL_BLOCK_HASH: b("0x0000000000000000000000000000000000000000000000000000000000000000"),
+  TERMINAL_BLOCK_HASH_ACTIVATION_EPOCH: Infinity,
 
   // Genesis
   // ---------------------------------------------------------------

packages/config/src/chainConfig/sszTypes.ts

@@ -13,6 +13,8 @@ export const ChainConfig = new ContainerType<IChainConfig>({
 
     // Transition
     TERMINAL_TOTAL_DIFFICULTY: ssz.Uint256,
+    TERMINAL_BLOCK_HASH: ssz.Root,
+    TERMINAL_BLOCK_HASH_ACTIVATION_EPOCH: ssz.Number64,
 
     // Genesis
     MIN_GENESIS_ACTIVE_VALIDATOR_COUNT: ssz.Number64,

packages/config/src/chainConfig/types.ts

@@ -10,6 +10,7 @@ export interface IChainConfig {
   // Transition
   TERMINAL_TOTAL_DIFFICULTY: bigint;
   TERMINAL_BLOCK_HASH: Uint8Array;
+  TERMINAL_BLOCK_HASH_ACTIVATION_EPOCH: number;
 
   // Genesis
   MIN_GENESIS_ACTIVE_VALIDATOR_COUNT: number;

packages/fork-choice/src/forkChoice/forkChoice.ts

@@ -16,7 +16,7 @@ import {ProtoArray} from "../protoArray/protoArray";
 
 import {IForkChoiceMetrics} from "../metrics";
 import {ForkChoiceError, ForkChoiceErrorCode, InvalidBlockCode, InvalidAttestationCode} from "./errors";
-import {IForkChoice, ILatestMessage, IQueuedAttestation, OnBlockPrecachedData, PowBlockHex} from "./interface";
+import {IForkChoice, ILatestMessage, IQueuedAttestation, OnBlockPrecachedData} from "./interface";
 import {IForkChoiceStore, CheckpointWithHex, toCheckpointWithHex, equalCheckpointWithHex} from "./store";
 
 /* eslint-disable max-len */
@@ -299,14 +299,8 @@ export class ForkChoice implements IForkChoice {
       merge.isMergeStateType(state) &&
       merge.isMergeBlockBodyType(block.body) &&
       merge.isMergeBlock(state, block.body)
-    ) {
-      const {powBlock, powBlockParent} = preCachedData || {};
-      if (!powBlock) throw Error("onBlock preCachedData must include powBlock");
-      if (!powBlockParent) throw Error("onBlock preCachedData must include powBlock");
-      if (!isValidTerminalPowBlock(this.config, powBlock, powBlockParent)) {
-        throw Error("Not valid terminal POW block");
-      }
-    }
+    )
+      assertValidTerminalPowBlock(this.config, (block as unknown) as merge.BeaconBlock, preCachedData);
 
     let shouldUpdateJustified = false;
     const {finalizedCheckpoint} = state;
@@ -910,11 +904,33 @@ export class ForkChoice implements IForkChoice {
   }
 }
 
-function isValidTerminalPowBlock(config: IChainConfig, block: PowBlockHex, parent: PowBlockHex): boolean {
-  if (block.blockhash === toHexString(config.TERMINAL_BLOCK_HASH)) {
-    return true;
+function assertValidTerminalPowBlock(
+  config: IChainConfig,
+  block: merge.BeaconBlock,
+  preCachedData?: OnBlockPrecachedData
+): void {
+  if (!ssz.Root.equals(config.TERMINAL_BLOCK_HASH, ZERO_HASH)) {
+    if (computeEpochAtSlot(block.slot) < config.TERMINAL_BLOCK_HASH_ACTIVATION_EPOCH)
+      throw Error(`Terminal block activation epoch ${config.TERMINAL_BLOCK_HASH_ACTIVATION_EPOCH} not reached`);
+
+    // powBock.blockhash is hex, so we just pick the corresponding root
+    if (!ssz.Root.equals(block.body.executionPayload.parentHash, config.TERMINAL_BLOCK_HASH))
+      throw new Error(
+        `Invalid terminal block hash, expected: ${toHexString(config.TERMINAL_BLOCK_HASH)}, actual: ${toHexString(
+          block.body.executionPayload.parentHash
+        )}`
+      );
+  } else {
+    // If no TERMINAL_BLOCK_HASH override, check ttd
+    const {powBlock, powBlockParent} = preCachedData || {};
+    if (!powBlock) throw Error("onBlock preCachedData must include powBlock");
+    if (!powBlockParent) throw Error("onBlock preCachedData must include powBlock");
+
+    const isTotalDifficultyReached = powBlock.totalDifficulty >= config.TERMINAL_TOTAL_DIFFICULTY;
+    const isParentTotalDifficultyValid = powBlockParent.totalDifficulty < config.TERMINAL_TOTAL_DIFFICULTY;
+    if (!isTotalDifficultyReached || !isParentTotalDifficultyValid)
+      throw Error(
+        `Invalid terminal POW block: total difficulty not reached ${powBlockParent.totalDifficulty} < ${powBlock.totalDifficulty}`
+      );
   }
-  const isTotalDifficultyReached = block.totalDifficulty >= config.TERMINAL_TOTAL_DIFFICULTY;
-  const isParentTotalDifficultyValid = parent.totalDifficulty < config.TERMINAL_TOTAL_DIFFICULTY;
-  return isTotalDifficultyReached && isParentTotalDifficultyValid;
 }

packages/lodestar/src/api/impl/validator/index.ts

@@ -146,7 +146,7 @@ export function getValidatorApi({chain, config, logger, metrics, network, sync}:
 
       timer = metrics?.blockProductionTime.startTimer();
       const block = await assembleBlock(
-        {chain, metrics},
+        {chain, metrics, logger},
         {
           slot,
           randaoReveal,