Merge 4f19966 into 6793305

.github/workflows/test-sim-merge.yml

@@ -14,6 +14,7 @@ env:
   GETH_WITHDRAWALS_IMAGE: ethpandaops/geth:withdrawals-timestamp-e92fb91
   ETHEREUMJS_WITHDRAWALS_IMAGE: g11tech/ethereumjs:withdrawals-07b106
   NETHERMIND_WITHDRAWALS_IMAGE: nethermindeth/nethermind:withdrawals_yolo
+  GETH_4844_IMAGE: g11tech/geth:4844-c3121
 
 jobs:
   sim-merge-tests:
@@ -122,6 +123,16 @@ jobs:
       #     EL_BINARY_DIR: ${{ env.NETHERMIND_WITHDRAWALS_IMAGE }}
       #     EL_SCRIPT_DIR: netherminddocker
 
+      - name: Pull geth eip4844
+        run: docker pull $GETH_4844_IMAGE
+
+      - name: Test Lodestar <> geth 4844
+        run: yarn test:sim:eip4844
+        working-directory: packages/beacon-node
+        env:
+          EL_BINARY_DIR: ${{ env.GETH_4844_IMAGE }}
+          EL_SCRIPT_DIR: gethdocker
+
       - name: Upload debug log test files
         if: ${{ always() }}
         uses: actions/upload-artifact@v2

packages/beacon-node/package.json

@@ -83,6 +83,7 @@
     "test:sim:merge-interop": "mocha 'test/sim/merge-interop.test.ts'",
     "test:sim:mergemock": "mocha 'test/sim/mergemock.test.ts'",
     "test:sim:withdrawals": "mocha 'test/sim/withdrawal-interop.test.ts'",
+    "test:sim:eip4844": "mocha 'test/sim/4844-interop.test.ts'",
     "download-spec-tests": "node --loader=ts-node/esm test/spec/downloadTests.ts",
     "check-spec-tests": "mocha test/spec/checkCoverage.ts",
     "test:spec-bls-general": "mocha --config .mocharc.spec.cjs 'test/spec/bls/**/*.test.ts' 'test/spec/general/**/*.test.ts'",
@@ -98,6 +99,7 @@
     "@chainsafe/discv5": "^3.0.0",
     "@chainsafe/libp2p-gossipsub": "^6.1.0",
     "@chainsafe/libp2p-noise": "^11.0.0",
+    "@chainsafe/libp2p-yamux": "^3.0.3",
     "@chainsafe/persistent-merkle-tree": "^0.4.2",
     "@chainsafe/snappy-stream": "^5.1.2",
     "@chainsafe/ssz": "^0.9.2",
@@ -128,6 +130,7 @@
     "@multiformats/multiaddr": "^11.0.0",
     "@types/datastore-level": "^3.0.0",
     "buffer-xor": "^2.0.2",
+    "c-kzg": "^1.0.8",
     "cross-fetch": "^3.1.4",
     "datastore-core": "^8.0.1",
     "datastore-level": "^9.0.1",

packages/beacon-node/src/network/gossip/topic.ts

@@ -97,6 +97,8 @@ export function getGossipSSZType(topic: GossipTopic) {
       return ssz.altair.LightClientFinalityUpdate;
     case GossipType.bls_to_execution_change:
       return ssz.capella.SignedBLSToExecutionChange;
+    default:
+      throw new Error(`No ssz gossip type for ${(topic as GossipTopic).type}`);
   }
 }
 

packages/beacon-node/src/network/nodejs/bundle.ts

@@ -9,6 +9,7 @@ import type {PeerDiscovery} from "@libp2p/interface-peer-discovery";
 import type {Components} from "libp2p/components";
 import {prometheusMetrics} from "@libp2p/prometheus-metrics";
 import {Registry} from "prom-client";
+import {yamux} from "@chainsafe/libp2p-yamux";
 import {Libp2p} from "../interface.js";
 import {createNoise} from "./noise.js";
 
@@ -41,6 +42,10 @@ export async function createNodejsLibp2p(options: ILibp2pOptions): Promise<Libp2
       peerDiscovery.push(mdns());
     }
   }
+
+  const yamuxInstance = yamux();
+  const mplexInstance = mplex({maxInboundStreams: 256});
+
   return (await createLibp2p({
     peerId: options.peerId,
     addresses: {
@@ -58,7 +63,12 @@ export async function createNodejsLibp2p(options: ILibp2pOptions): Promise<Libp2
         },
       }),
     ],
-    streamMuxers: [mplex({maxInboundStreams: 256})],
+    streamMuxers: [
+      //
+      mplexInstance,
+      // Cast because yamux.StreamMuxerFactory is different than libp2p.StreamMuxerFactory
+      yamuxInstance as typeof mplexInstance,
+    ],
     peerDiscovery,
     metrics: options.metrics
       ? prometheusMetrics({

packages/beacon-node/src/network/reqresp/doBeaconBlocksMaybeBlobsByRange.ts

@@ -26,7 +26,7 @@ export async function doBeaconBlocksMaybeBlobsByRange(
   else if (computeEpochAtSlot(request.startSlot) >= currentEpoch - config.MIN_EPOCHS_FOR_BLOBS_SIDECARS_REQUESTS) {
     // TODO Deneb: Do two requests at once for blocks and blobs
     const blocks = await reqResp.beaconBlocksByRange(peerId, request);
-    const blobsSidecars = await reqResp.blobsSidecarsByRange(peerId, request);
+    const blobsSidecars = blocks.length > 0 ? await reqResp.blobsSidecarsByRange(peerId, request) : [];
 
     const blockInputs: BlockInput[] = [];
     let blobSideCarIndex = 0;

packages/beacon-node/src/sync/sync.ts

@@ -47,7 +47,8 @@ export class BeaconSync implements IBeaconSync {
     this.logger = logger;
     this.rangeSync = new RangeSync(modules, opts);
     this.unknownBlockSync = new UnknownBlockSync(config, network, chain, logger, metrics, opts);
-    this.slotImportTolerance = SLOTS_PER_EPOCH;
+    // TODO EIP-4844 TEMP, for fast tests
+    this.slotImportTolerance = 2 * SLOTS_PER_EPOCH;
 
     // Subscribe to RangeSync completing a SyncChain and recompute sync state
     if (!opts.disableRangeSync) {

packages/beacon-node/test/e2e/eip4844.ts

@@ -0,0 +1,96 @@
+import {IChainConfig} from "@lodestar/config";
+import {SLOTS_PER_EPOCH} from "@lodestar/params";
+import {fromHex, sleep, TimestampFormatCode} from "@lodestar/utils";
+import {LogLevel, testLogger, TestLoggerOpts} from "../utils/logger.js";
+import {getDevBeaconNode} from "../utils/node/beacon.js";
+import {getAndInitDevValidators} from "../utils/node/validator.js";
+
+// ```
+// lodestar/packages/beacon-node$
+// LODESTAR_PRESET=minimal SKIP_SYNC_COMMITTEE=true IGNORE_EIP4844_VALIDAtION=true IGNORE_TRUSTED_SETUP_ERRORS=true ../../node_modules/.bin/ts-node --esm test/e2e/eip4844.ts
+// ```
+
+const testParams: Pick<
+  IChainConfig,
+  "SECONDS_PER_SLOT" | "ALTAIR_FORK_EPOCH" | "BELLATRIX_FORK_EPOCH" | "CAPELLA_FORK_EPOCH" | "EIP4844_FORK_EPOCH"
+> = {
+  /* eslint-disable @typescript-eslint/naming-convention */
+  SECONDS_PER_SLOT: 2,
+  ALTAIR_FORK_EPOCH: 0,
+  BELLATRIX_FORK_EPOCH: 0,
+  CAPELLA_FORK_EPOCH: 0,
+  EIP4844_FORK_EPOCH: 0,
+};
+
+const restPort = 9000;
+const validatorCount = 8;
+const genesisEth1BlockHash = "0xbebebebebebebebebebebebebebebebebebebebebebebebebebebebebebebebe";
+
+// delay a bit so regular sync sees it's up to date and sync is completed from the beginning
+// also delay to allow bls workers to be transpiled/initialized
+const genesisTimeDelaySec = 6;
+const genesisTime = Math.floor(Date.now() / 1000) + genesisTimeDelaySec;
+
+const testLoggerOpts: TestLoggerOpts = {
+  logLevel: LogLevel.debug,
+  timestampFormat: {
+    format: TimestampFormatCode.EpochSlot,
+    genesisTime,
+    slotsPerEpoch: SLOTS_PER_EPOCH,
+    secondsPerSlot: testParams.SECONDS_PER_SLOT,
+  },
+};
+
+const loggerNodeA = testLogger("Node", testLoggerOpts);
+
+const bn = await getDevBeaconNode({
+  params: testParams,
+  options: {
+    sync: {isSingleNode: true},
+    network: {allowPublishToZeroPeers: true},
+    api: {rest: {enabled: false, api: ["node"], port: restPort, address: "localhost"}},
+    chain: {blsVerifyAllMainThread: true},
+    executionEngine: {mode: "mock", genesisBlockHash: genesisEth1BlockHash},
+  },
+  validatorCount,
+  genesisTime,
+  logger: loggerNodeA,
+  eth1BlockHash: fromHex(genesisEth1BlockHash),
+});
+
+await getAndInitDevValidators({
+  node: bn,
+  validatorsPerClient: validatorCount,
+  validatorClientCount: 1,
+  startIndex: 0,
+  useRestApi: false,
+  testLoggerOpts: {...testLoggerOpts, logLevel: LogLevel.error},
+});
+
+await waitForEpoch(genesisTime, bn.config, 3);
+
+const loggerNodeSync = testLogger("Sync", testLoggerOpts);
+const bn2 = await getDevBeaconNode({
+  params: testParams,
+  options: {
+    sync: {isSingleNode: false},
+    network: {allowPublishToZeroPeers: true},
+    api: {rest: {enabled: false, api: [], port: restPort + 1, address: "localhost"}},
+    chain: {blsVerifyAllMainThread: true},
+    executionEngine: {mode: "mock", genesisBlockHash: genesisEth1BlockHash},
+  },
+  validatorCount,
+  genesisTime,
+  logger: loggerNodeSync,
+  eth1BlockHash: fromHex(genesisEth1BlockHash),
+});
+
+const bnIdentity = await bn.api.node.getNetworkIdentity();
+await bn2.api.lodestar.connectPeer(bnIdentity.data.peerId, bnIdentity.data.p2pAddresses);
+
+await new Promise((r) => setTimeout(r, 300_000));
+
+function waitForEpoch(genesisTime: number, config: IChainConfig, epoch: number): Promise<void> {
+  const epochTimeSec = genesisTime + epoch * SLOTS_PER_EPOCH * config.SECONDS_PER_SLOT;
+  return sleep(1000 * epochTimeSec - Date.now());
+}

packages/beacon-node/test/e2e/network/gossipsub.test.ts

@@ -1,6 +1,7 @@
 import sinon from "sinon";
 import {expect} from "chai";
 import {createIBeaconConfig, createIChainForkConfig, defaultChainConfig} from "@lodestar/config";
+import {chainConfig} from "@lodestar/config/default";
 import {capella, altair, phase0, ssz} from "@lodestar/types";
 import {sleep} from "@lodestar/utils";
 
@@ -44,6 +45,11 @@ describe("gossipsub", function () {
 
   const logger = testLogger();
 
+  const ALTAIR_FORK_EPOCH = 128;
+  const genesisValidatorsRoot = Buffer.alloc(32, 0xdd);
+  // eslint-disable-next-line @typescript-eslint/naming-convention
+  const config = createIBeaconConfig({...chainConfig, ALTAIR_FORK_EPOCH}, genesisValidatorsRoot);
+
   const afterEachCallbacks: (() => Promise<void> | void)[] = [];
   afterEach(async () => {
     while (afterEachCallbacks.length > 0) {
@@ -64,13 +70,12 @@ describe("gossipsub", function () {
       },
     });
 
-    const beaconConfig = createIBeaconConfig(config, state.genesisValidatorsRoot);
     const chain = new MockBeaconChain({
       genesisTime: 0,
       chainId: 0,
       networkId: BigInt(0),
       state,
-      config: beaconConfig,
+      config,
     });
 
     chain.forkChoice.getHead = () => {
@@ -89,7 +94,7 @@ describe("gossipsub", function () {
     const loggerB = testLogger("B");
 
     const modules = {
-      config: beaconConfig,
+      config,
       chain,
       db,
       reqRespHandlers,