From bfe5da8b9e81b44bc043d7201fd81a5a44dd2663 Mon Sep 17 00:00:00 2001 From: Nazar Hussain Date: Wed, 16 Nov 2022 10:36:51 +0100 Subject: [PATCH] Merge e2e tests to sim tests (#4759) * Remove already covered tests * Move some instance functions to pure functions to be reuseable * Add uknonw block parent assertions * Update the wait time * Update block error message * Disable range sync for the uknown block sync node * Update the failing unit tests * Use getBlockV2 instead of getBlock * Fix linter warnings * Remove unknownBlockSync test covered in sim tests * Move few e2e tests to sim tests * Update the sim run command * Update docker runner * Fix child process stopping issue * Remove unnecessary e2e tests * Fix unit tests for new beacon options * Update a unit test * Remove an accidental committed file * Add sync check for the unkown sync node * Add simulation repoter interface * Update tracker event handling * Update repoter * Add logging for debugging * Update logic for head checking --- .../api/impl/beacon/state/endpoint.test.ts | 183 ------------ .../test/e2e/api/impl/node/endpoint.test.ts | 90 ------ .../test/e2e/sync/endpoint.test.ts | 61 ---- .../test/e2e/sync/finalizedSync.test.ts | 96 ------- .../test/e2e/sync/unknownBlockSync.test.ts | 113 -------- .../beacon-node/test/e2e/sync/wss.test.ts | 128 --------- packages/cli/package.json | 2 +- .../cli/src/options/beaconNodeOptions/sync.ts | 10 + .../test/simulation/beacon_endpoints.test.ts | 135 +++++++++ .../cli/test/simulation/multi_fork.test.ts | 89 +++++- .../unit/options/beaconNodeOptions.test.ts | 2 + .../utils/simulation/SimulationEnvironment.ts | 139 ++++------ .../utils/simulation/SimulationTracker.ts | 262 +++++++++--------- .../test/utils/simulation/TableReporter.ts | 115 ++++++++ .../simulation/assertions/defaults/index.ts | 19 ++ .../utils/simulation/assertions/matchers.ts | 3 + .../utils/simulation/cl_clients/lodestar.ts | 21 +- .../test/utils/simulation/el_clients/geth.ts | 8 +- .../utils/simulation/el_clients/nethermind.ts | 10 +- .../cli/test/utils/simulation/interfaces.ts | 51 +++- .../utils/simulation/runner/DockerRunner.ts | 23 +- .../utils/simulation/utils/child_process.ts | 2 +- .../test/utils/simulation/utils/el_genesis.ts | 9 +- .../cli/test/utils/simulation/utils/index.ts | 13 +- .../test/utils/simulation/utils/network.ts | 94 +++++++ 25 files changed, 710 insertions(+), 968 deletions(-) delete mode 100644 packages/beacon-node/test/e2e/api/impl/beacon/state/endpoint.test.ts delete mode 100644 packages/beacon-node/test/e2e/api/impl/node/endpoint.test.ts delete mode 100644 packages/beacon-node/test/e2e/sync/endpoint.test.ts delete mode 100644 packages/beacon-node/test/e2e/sync/finalizedSync.test.ts delete mode 100644 packages/beacon-node/test/e2e/sync/unknownBlockSync.test.ts delete mode 100644 packages/beacon-node/test/e2e/sync/wss.test.ts create mode 100644 packages/cli/test/simulation/beacon_endpoints.test.ts create mode 100644 packages/cli/test/utils/simulation/TableReporter.ts create mode 100644 packages/cli/test/utils/simulation/assertions/defaults/index.ts diff --git a/packages/beacon-node/test/e2e/api/impl/beacon/state/endpoint.test.ts b/packages/beacon-node/test/e2e/api/impl/beacon/state/endpoint.test.ts deleted file mode 100644 index d777bca5fa81..000000000000 --- a/packages/beacon-node/test/e2e/api/impl/beacon/state/endpoint.test.ts +++ /dev/null @@ -1,183 +0,0 @@ -import {expect} from "chai"; -import {createIBeaconConfig, IChainConfig} from "@lodestar/config"; -import {chainConfig as chainConfigDef} from "@lodestar/config/default"; -import {getClient} from "@lodestar/api"; -import {toHexString} from "@chainsafe/ssz"; -import {LogLevel, testLogger, TestLoggerOpts} from "../../../../../utils/logger.js"; -import {getDevBeaconNode} from "../../../../../utils/node/beacon.js"; -import {getAndInitDevValidators} from "../../../../../utils/node/validator.js"; - -/* eslint-disable @typescript-eslint/naming-convention */ -describe("lodestar / api / impl / state", function () { - const SECONDS_PER_SLOT = 2; - const ALTAIR_FORK_EPOCH = 0; - const restPort = 9596; - const chainConfig: IChainConfig = {...chainConfigDef, SECONDS_PER_SLOT, ALTAIR_FORK_EPOCH}; - const genesisValidatorsRoot = Buffer.alloc(32, 0xaa); - const config = createIBeaconConfig(chainConfig, genesisValidatorsRoot); - const testLoggerOpts: TestLoggerOpts = {logLevel: LogLevel.info}; - const loggerNodeA = testLogger("Node-A", testLoggerOpts); - - describe("eth/v1/beacon/states/{state_id}/validators", function () { - this.timeout("10 min"); - const testParams: Pick = { - SECONDS_PER_SLOT: 2, - }; - - const afterEachCallbacks: (() => Promise | void)[] = []; - afterEach(async () => { - while (afterEachCallbacks.length > 0) { - const callback = afterEachCallbacks.pop(); - if (callback) await callback(); - } - }); - - it("should return all validators when getStateValidators called without filters", async function () { - const validatorCount = 2; - const bn = await getDevBeaconNode({ - params: testParams, - options: { - sync: {isSingleNode: true}, - api: {rest: {enabled: true, port: restPort}}, - chain: {blsVerifyAllMainThread: true}, - }, - validatorCount, - logger: loggerNodeA, - }); - afterEachCallbacks.push(() => bn.close()); - - const {validators} = await getAndInitDevValidators({ - node: bn, - validatorsPerClient: validatorCount, - validatorClientCount: 1, - startIndex: 0, - useRestApi: false, - testLoggerOpts, - }); - afterEachCallbacks.push(() => Promise.all(validators.map((validator) => validator.close()))); - - const client = getClient({baseUrl: `http://127.0.0.1:${restPort}`}, {config}).beacon; - - const response = await client.getStateValidators("head"); - expect(response.data.length).to.be.equal(validatorCount); - expect(response.data[0].index).to.be.equal(0); - expect(response.data[1].index).to.be.equal(1); - }); - - it("should return filtered validators when getStateValidators called with filters", async function () { - const validatorCount = 2; - const filterPubKey = - "0xa99a76ed7796f7be22d5b7e85deeb7c5677e88e511e0b337618f8c4eb61349b4bf2d153f649f7b53359fe8b94a38e44c"; - - const bn = await getDevBeaconNode({ - params: testParams, - options: { - sync: {isSingleNode: true}, - api: {rest: {enabled: true, port: restPort}}, - chain: {blsVerifyAllMainThread: true}, - }, - validatorCount, - logger: loggerNodeA, - }); - afterEachCallbacks.push(() => bn.close()); - - const {validators} = await getAndInitDevValidators({ - node: bn, - validatorsPerClient: validatorCount, - validatorClientCount: 1, - startIndex: 0, - useRestApi: false, - testLoggerOpts, - }); - afterEachCallbacks.push(() => Promise.all(validators.map((validator) => validator.close()))); - - const client = getClient({baseUrl: `http://127.0.0.1:${restPort}`}, {config}).beacon; - - const response = await client.getStateValidators("head", { - id: [filterPubKey], - }); - - expect(response.data.length).to.be.equal(1); - expect(toHexString(response.data[0].validator.pubkey)).to.be.equal(filterPubKey); - }); - }); - - describe("/eth/v1/beacon/states/{state_id}/validators/{validator_id}", function () { - this.timeout("10 min"); - const testParams: Pick = { - SECONDS_PER_SLOT: 2, - }; - - const afterEachCallbacks: (() => Promise | void)[] = []; - afterEach(async () => { - while (afterEachCallbacks.length > 0) { - const callback = afterEachCallbacks.pop(); - if (callback) await callback(); - } - }); - - it("should return the validator when getStateValidator is called with the validator index", async function () { - const validatorIndex = "0"; - const validatorCount = 2; - const bn = await getDevBeaconNode({ - params: testParams, - options: { - sync: {isSingleNode: true}, - api: {rest: {enabled: true, port: restPort}}, - chain: {blsVerifyAllMainThread: true}, - }, - validatorCount, - logger: loggerNodeA, - }); - afterEachCallbacks.push(() => bn.close()); - - const {validators} = await getAndInitDevValidators({ - node: bn, - validatorsPerClient: validatorCount, - validatorClientCount: 1, - startIndex: 0, - useRestApi: false, - testLoggerOpts, - }); - afterEachCallbacks.push(() => Promise.all(validators.map((validator) => validator.close()))); - - const client = getClient({baseUrl: `http://127.0.0.1:${restPort}`}, {config}).beacon; - - const response = await client.getStateValidator("head", validatorIndex); - // TODO: the index in data should be a string instead of an integer - expect(response.data.index).to.be.equal(parseInt(validatorIndex)); - }); - - it("should return the validator when getStateValidator is called with the hex encoded public key", async function () { - const hexPubKey = - "0xa99a76ed7796f7be22d5b7e85deeb7c5677e88e511e0b337618f8c4eb61349b4bf2d153f649f7b53359fe8b94a38e44c"; - const validatorCount = 2; - const bn = await getDevBeaconNode({ - params: testParams, - options: { - sync: {isSingleNode: true}, - api: {rest: {enabled: true, port: restPort}}, - chain: {blsVerifyAllMainThread: true}, - }, - validatorCount, - logger: loggerNodeA, - }); - afterEachCallbacks.push(() => bn.close()); - - const {validators} = await getAndInitDevValidators({ - node: bn, - validatorsPerClient: validatorCount, - validatorClientCount: 1, - startIndex: 0, - useRestApi: false, - testLoggerOpts, - }); - afterEachCallbacks.push(() => Promise.all(validators.map((validator) => validator.close()))); - - const client = getClient({baseUrl: `http://127.0.0.1:${restPort}`}, {config}).beacon; - - const response = await client.getStateValidator("head", hexPubKey); - expect(toHexString(response.data.validator.pubkey)).to.be.equal(hexPubKey); - }); - }); -}); diff --git a/packages/beacon-node/test/e2e/api/impl/node/endpoint.test.ts b/packages/beacon-node/test/e2e/api/impl/node/endpoint.test.ts deleted file mode 100644 index fe6e925c6a48..000000000000 --- a/packages/beacon-node/test/e2e/api/impl/node/endpoint.test.ts +++ /dev/null @@ -1,90 +0,0 @@ -import {expect} from "chai"; -import {createIBeaconConfig, IChainConfig} from "@lodestar/config"; -import {chainConfig as chainConfigDef} from "@lodestar/config/default"; -import {getClient, routes} from "@lodestar/api"; -import {getDevBeaconNode} from "../../../../utils/node/beacon.js"; -import {LogLevel, testLogger, TestLoggerOpts} from "../../../../utils/logger.js"; - -/* eslint-disable @typescript-eslint/naming-convention */ -describe("lodestar / sync", function () { - const SECONDS_PER_SLOT = 2; - const ALTAIR_FORK_EPOCH = 0; - const validatorCount = 1; - const restPort = 9596; - const chainConfig: IChainConfig = {...chainConfigDef, SECONDS_PER_SLOT, ALTAIR_FORK_EPOCH}; - const genesisValidatorsRoot = Buffer.alloc(32, 0xaa); - const config = createIBeaconConfig(chainConfig, genesisValidatorsRoot); - const testLoggerOpts: TestLoggerOpts = {logLevel: LogLevel.info}; - const loggerNodeA = testLogger("Node-A", testLoggerOpts); - - describe("/eth/v1/node/health", function () { - const testParams: Pick = { - SECONDS_PER_SLOT: 2, - }; - - const genesisTime = Math.floor(Date.now() / 1000) + 16 * testParams.SECONDS_PER_SLOT; - - const afterEachCallbacks: (() => Promise | void)[] = []; - afterEach(async () => { - while (afterEachCallbacks.length > 0) { - const callback = afterEachCallbacks.pop(); - if (callback) await callback(); - } - }); - - it("Return READY pre genesis / BN Synced", async function () { - this.timeout("10 min"); - const bn = await getDevBeaconNode({ - params: testParams, - options: { - sync: {isSingleNode: true}, - api: {rest: {enabled: true, port: restPort}}, - chain: {blsVerifyAllMainThread: true}, - }, - validatorCount, - genesisTime: genesisTime, - logger: loggerNodeA, - }); - - afterEachCallbacks.push(() => bn.close()); - - const client = getClient({baseUrl: "http://127.0.0.1:9596"}, {config}).node; - - const expectedSyncStatus: routes.node.SyncingStatus = { - headSlot: "0", - syncDistance: "0", - isSyncing: false, - isOptimistic: false, - }; - await expect(client.getSyncingStatus()).to.eventually.be.deep.equal({data: expectedSyncStatus}); - await expect(client.getHealth()).to.eventually.be.equal(routes.node.NodeHealth.READY); - }); - - it("Return READY pre genesis / BN Not Synced", async function () { - this.timeout("10 min"); - const bn = await getDevBeaconNode({ - params: testParams, - options: { - sync: {isSingleNode: false}, - api: {rest: {enabled: true, port: restPort}}, - chain: {blsVerifyAllMainThread: true}, - }, - validatorCount, - genesisTime: genesisTime, - logger: loggerNodeA, - }); - - afterEachCallbacks.push(() => bn.close()); - - const client = getClient({baseUrl: "http://127.0.0.1:9596"}, {config}).node; - const expectedSyncStatus: routes.node.SyncingStatus = { - headSlot: "0", - syncDistance: "0", - isSyncing: false, - isOptimistic: false, - }; - await expect(client.getSyncingStatus()).to.eventually.be.deep.equal({data: expectedSyncStatus}); - await expect(client.getHealth()).to.eventually.be.equal(routes.node.NodeHealth.READY); - }); - }); -}); diff --git a/packages/beacon-node/test/e2e/sync/endpoint.test.ts b/packages/beacon-node/test/e2e/sync/endpoint.test.ts deleted file mode 100644 index 456a10105599..000000000000 --- a/packages/beacon-node/test/e2e/sync/endpoint.test.ts +++ /dev/null @@ -1,61 +0,0 @@ -import {expect} from "chai"; -import {createIBeaconConfig, IChainConfig} from "@lodestar/config"; -import {chainConfig as chainConfigDef} from "@lodestar/config/default"; -import {getClient, routes} from "@lodestar/api"; -import {getDevBeaconNode} from "../../utils/node/beacon.js"; -import {LogLevel, testLogger, TestLoggerOpts} from "../../utils/logger.js"; - -/* eslint-disable @typescript-eslint/naming-convention */ -describe("lodestar / sync", function () { - const SECONDS_PER_SLOT = 2; - const ALTAIR_FORK_EPOCH = 0; - const validatorCount = 1; - const restPort = 9596; - const chainConfig: IChainConfig = {...chainConfigDef, SECONDS_PER_SLOT, ALTAIR_FORK_EPOCH}; - const genesisValidatorsRoot = Buffer.alloc(32, 0xaa); - const config = createIBeaconConfig(chainConfig, genesisValidatorsRoot); - const testLoggerOpts: TestLoggerOpts = {logLevel: LogLevel.info}; - const loggerNodeA = testLogger("Node-A", testLoggerOpts); - - describe("/eth/v1/node/syncing", function () { - const testParams: Pick = { - SECONDS_PER_SLOT: 2, - }; - - const afterEachCallbacks: (() => Promise | void)[] = []; - afterEach(async () => { - while (afterEachCallbacks.length > 0) { - const callback = afterEachCallbacks.pop(); - if (callback) await callback(); - } - }); - - it("getSyncingStatus", async function () { - this.timeout("10 min"); - const bn = await getDevBeaconNode({ - params: testParams, - options: { - sync: {isSingleNode: true}, - api: {rest: {enabled: true, port: restPort}}, - chain: {blsVerifyAllMainThread: true}, - }, - validatorCount, - logger: loggerNodeA, - }); - - afterEachCallbacks.push(() => bn.close()); - - const client = getClient({baseUrl: "http://127.0.0.1:9596"}, {config}).node; - - const expectedSyncStatus: routes.node.SyncingStatus = { - headSlot: "0", - syncDistance: "0", - isSyncing: false, - isOptimistic: false, - }; - - // expect headSlot and syncDistance to be string - await expect(client.getSyncingStatus()).to.eventually.be.deep.equal({data: expectedSyncStatus}); - }); - }); -}); diff --git a/packages/beacon-node/test/e2e/sync/finalizedSync.test.ts b/packages/beacon-node/test/e2e/sync/finalizedSync.test.ts deleted file mode 100644 index f6f4f326a2b8..000000000000 --- a/packages/beacon-node/test/e2e/sync/finalizedSync.test.ts +++ /dev/null @@ -1,96 +0,0 @@ -import {assert} from "chai"; -import {IChainConfig} from "@lodestar/config"; -import {phase0, ssz} from "@lodestar/types"; -import {fromHexString} from "@chainsafe/ssz"; -import {getDevBeaconNode} from "../../utils/node/beacon.js"; -import {waitForEvent} from "../../utils/events/resolver.js"; -import {getAndInitDevValidators} from "../../utils/node/validator.js"; -import {ChainEvent} from "../../../src/chain/index.js"; -import {connect} from "../../utils/network.js"; -import {testLogger, LogLevel, TestLoggerOpts} from "../../utils/logger.js"; - -describe("sync / finalized sync", function () { - const validatorCount = 8; - const beaconParams: Partial = { - // eslint-disable-next-line @typescript-eslint/naming-convention - SECONDS_PER_SLOT: 2, - }; - - const afterEachCallbacks: (() => Promise | void)[] = []; - afterEach(async () => { - while (afterEachCallbacks.length > 0) { - const callback = afterEachCallbacks.pop(); - if (callback) await callback(); - } - }); - - it("should do a finalized sync from another BN", async function () { - this.timeout("10 min"); - - const testLoggerOpts: TestLoggerOpts = {logLevel: LogLevel.info}; - const loggerNodeA = testLogger("Node-A", testLoggerOpts); - const loggerNodeB = testLogger("Node-B", testLoggerOpts); - // delay a bit so regular sync sees it's up to date and sync is completed from the beginning - // the node needs time to transpile/initialize bls worker threads - const genesisSlotsDelay = 16; - // eslint-disable-next-line @typescript-eslint/no-non-null-assertion - const genesisTime = Math.floor(Date.now() / 1000) + genesisSlotsDelay * beaconParams.SECONDS_PER_SLOT!; - - const bn = await getDevBeaconNode({ - params: beaconParams, - options: { - sync: {isSingleNode: true}, - network: {allowPublishToZeroPeers: true}, - chain: {blsVerifyAllMainThread: true}, - }, - validatorCount, - genesisTime, - logger: loggerNodeA, - }); - - afterEachCallbacks.push(() => bn.close()); - - const {validators} = await getAndInitDevValidators({ - node: bn, - validatorsPerClient: validatorCount, - validatorClientCount: 1, - startIndex: 0, - useRestApi: false, - testLoggerOpts, - }); - - afterEachCallbacks.push(() => Promise.all(validators.map((validator) => validator.close()))); - - // stop beacon node after validators - afterEachCallbacks.push(() => bn.close()); - - await waitForEvent(bn.chain.emitter, ChainEvent.finalized, 240000); - loggerNodeA.info("Node A emitted finalized checkpoint event"); - - const bn2 = await getDevBeaconNode({ - params: beaconParams, - options: {api: {rest: {enabled: false}}}, - validatorCount, - genesisTime: bn.chain.getHeadState().genesisTime, - logger: loggerNodeB, - }); - afterEachCallbacks.push(() => bn2.close()); - - afterEachCallbacks.push(() => bn2.close()); - - const headSummary = bn.chain.forkChoice.getHead(); - const head = await bn.db.block.get(fromHexString(headSummary.blockRoot)); - if (!head) throw Error("First beacon node has no head block"); - const waitForSynced = waitForEvent(bn2.chain.emitter, ChainEvent.block, 100000, (block) => - ssz.phase0.SignedBeaconBlock.equals(block, head) - ); - - await connect(bn2.network, bn.network.peerId, bn.network.localMultiaddrs); - - try { - await waitForSynced; - } catch (e) { - assert.fail("Failed to sync to other node in time"); - } - }); -}); diff --git a/packages/beacon-node/test/e2e/sync/unknownBlockSync.test.ts b/packages/beacon-node/test/e2e/sync/unknownBlockSync.test.ts deleted file mode 100644 index 3c6f87f4ddf7..000000000000 --- a/packages/beacon-node/test/e2e/sync/unknownBlockSync.test.ts +++ /dev/null @@ -1,113 +0,0 @@ -import {IChainConfig} from "@lodestar/config"; -import {phase0, ssz} from "@lodestar/types"; -import {fromHexString} from "@chainsafe/ssz"; -import {TimestampFormatCode} from "@lodestar/utils"; -import {SLOTS_PER_EPOCH} from "@lodestar/params"; -import {getDevBeaconNode} from "../../utils/node/beacon.js"; -import {waitForEvent} from "../../utils/events/resolver.js"; -import {getAndInitDevValidators} from "../../utils/node/validator.js"; -import {ChainEvent} from "../../../src/chain/index.js"; -import {NetworkEvent} from "../../../src/network/index.js"; -import {connect} from "../../utils/network.js"; -import {testLogger, LogLevel, TestLoggerOpts} from "../../utils/logger.js"; -import {BlockError, BlockErrorCode} from "../../../src/chain/errors/index.js"; - -describe("sync / unknown block sync", function () { - const validatorCount = 8; - const testParams: Pick = { - // eslint-disable-next-line @typescript-eslint/naming-convention - SECONDS_PER_SLOT: 2, - }; - - const afterEachCallbacks: (() => Promise | void)[] = []; - afterEach(async () => { - while (afterEachCallbacks.length > 0) { - const callback = afterEachCallbacks.pop(); - if (callback) await callback(); - } - }); - - it("should do an unknown block sync from another BN", async function () { - this.timeout("10 min"); - - // the node needs time to transpile/initialize bls worker threads - const genesisSlotsDelay = 16; - const genesisTime = Math.floor(Date.now() / 1000) + genesisSlotsDelay * testParams.SECONDS_PER_SLOT; - const testLoggerOpts: TestLoggerOpts = { - logLevel: LogLevel.info, - timestampFormat: { - format: TimestampFormatCode.EpochSlot, - genesisTime, - slotsPerEpoch: SLOTS_PER_EPOCH, - secondsPerSlot: testParams.SECONDS_PER_SLOT, - }, - }; - - const loggerNodeA = testLogger("Node-A", testLoggerOpts); - const loggerNodeB = testLogger("Node-B", testLoggerOpts); - - const bn = await getDevBeaconNode({ - params: testParams, - options: { - sync: {isSingleNode: true}, - network: {allowPublishToZeroPeers: true}, - chain: {blsVerifyAllMainThread: true}, - }, - validatorCount, - logger: loggerNodeA, - }); - - afterEachCallbacks.push(() => bn.close()); - - const {validators} = await getAndInitDevValidators({ - node: bn, - validatorsPerClient: validatorCount, - validatorClientCount: 1, - startIndex: 0, - useRestApi: false, - testLoggerOpts, - }); - - afterEachCallbacks.push(() => Promise.all(validators.map((v) => v.close()))); - - // stop bn after validators - afterEachCallbacks.push(() => bn.close()); - - await waitForEvent(bn.chain.emitter, ChainEvent.checkpoint, 240000); - loggerNodeA.info("Node A emitted checkpoint event"); - - const bn2 = await getDevBeaconNode({ - params: testParams, - options: { - api: {rest: {enabled: false}}, - sync: {disableRangeSync: true}, - chain: {blsVerifyAllMainThread: true}, - }, - validatorCount, - genesisTime: bn.chain.getHeadState().genesisTime, - logger: loggerNodeB, - }); - - afterEachCallbacks.push(() => bn2.close()); - - const headSummary = bn.chain.forkChoice.getHead(); - const head = await bn.db.block.get(fromHexString(headSummary.blockRoot)); - if (!head) throw Error("First beacon node has no head block"); - const waitForSynced = waitForEvent(bn2.chain.emitter, ChainEvent.block, 100000, (block) => - ssz.phase0.SignedBeaconBlock.equals(block, head) - ); - - await connect(bn2.network, bn.network.peerId, bn.network.localMultiaddrs); - await bn2.chain.processBlock(head).catch((e) => { - if (e instanceof BlockError && e.type.code === BlockErrorCode.PARENT_UNKNOWN) { - // Expected - bn2.network.events.emit(NetworkEvent.unknownBlockParent, head, bn2.network.peerId.toString()); - } else { - throw e; - } - }); - - // Wait for NODE-A head to be processed in NODE-B without range sync - await waitForSynced; - }); -}); diff --git a/packages/beacon-node/test/e2e/sync/wss.test.ts b/packages/beacon-node/test/e2e/sync/wss.test.ts deleted file mode 100644 index 2efb9a146acc..000000000000 --- a/packages/beacon-node/test/e2e/sync/wss.test.ts +++ /dev/null @@ -1,128 +0,0 @@ -import {GENESIS_SLOT, SLOTS_PER_EPOCH} from "@lodestar/params"; -import {phase0, Slot} from "@lodestar/types"; -import {IChainConfig} from "@lodestar/config"; -import {config} from "@lodestar/config/default"; -import {TimestampFormatCode} from "@lodestar/utils"; -import {fetchWeakSubjectivityState} from "../../../../cli/src/networks/index.js"; -import {getDevBeaconNode} from "../../utils/node/beacon.js"; -import {waitForEvent} from "../../utils/events/resolver.js"; -import {getAndInitDevValidators} from "../../utils/node/validator.js"; -import {ChainEvent} from "../../../src/chain/index.js"; -import {BeaconRestApiServerOpts} from "../../../src/api/rest/index.js"; -import {testLogger, TestLoggerOpts} from "../../utils/logger.js"; -import {connect} from "../../utils/network.js"; -import {BackfillSyncEvent} from "../../../src/sync/backfill/index.js"; - -/* eslint-disable @typescript-eslint/naming-convention */ -describe("Start from WSS", function () { - const testParams: Pick = { - SECONDS_PER_SLOT: 2, - }; - - const afterEachCallbacks: (() => Promise | unknown)[] = []; - afterEach(async () => Promise.all(afterEachCallbacks.splice(0, afterEachCallbacks.length))); - - it("using another node", async function () { - // Should reach justification in 3 epochs max, and finalization in 4 epochs max - const expectedEpochsToFinish = 4; - // 1 epoch of margin of error - const epochsOfMargin = 1; - const timeoutSetupMargin = 5 * 1000; // Give extra 5 seconds of margin - - // delay a bit so regular sync sees it's up to date and sync is completed from the beginning - const genesisSlotsDelay = 16; - - const timeout = - ((epochsOfMargin + expectedEpochsToFinish) * SLOTS_PER_EPOCH + genesisSlotsDelay) * - testParams.SECONDS_PER_SLOT * - 1000; - - this.timeout(timeout + 2 * timeoutSetupMargin); - - const genesisTime = Math.floor(Date.now() / 1000) + genesisSlotsDelay * testParams.SECONDS_PER_SLOT; - - const testLoggerOpts: TestLoggerOpts = { - timestampFormat: { - format: TimestampFormatCode.EpochSlot, - genesisTime: genesisTime, - slotsPerEpoch: SLOTS_PER_EPOCH, - secondsPerSlot: testParams.SECONDS_PER_SLOT, - }, - }; - const loggerNodeA = testLogger("Node-A", testLoggerOpts); - const loggerNodeB = testLogger("Node-B", testLoggerOpts); - - const bn = await getDevBeaconNode({ - params: {...testParams, ALTAIR_FORK_EPOCH: Infinity}, - options: { - api: { - rest: {enabled: true, api: ["debug"]} as BeaconRestApiServerOpts, - }, - sync: {isSingleNode: true}, - network: {allowPublishToZeroPeers: true}, - chain: {blsVerifyAllMainThread: true}, - }, - validatorCount: 32, - logger: loggerNodeA, - genesisTime, - }); - afterEachCallbacks.push(() => bn.close()); - - const finalizedEventistener = waitForEvent(bn.chain.emitter, ChainEvent.finalized, timeout); - const {validators} = await getAndInitDevValidators({ - node: bn, - validatorsPerClient: 32, - validatorClientCount: 1, - startIndex: 0, - // At least one sim test must use the REST API for beacon <-> validator comms - useRestApi: true, - testLoggerOpts, - }); - - afterEachCallbacks.push(() => Promise.all(validators.map((v) => v.close()))); - - try { - await finalizedEventistener; - await waitForEvent(bn.chain.emitter, ChainEvent.finalized, timeout); - loggerNodeA.info("\n\nNode A finalized\n\n"); - } catch (e) { - (e as Error).message = `Node A failed to finalize: ${(e as Error).message}`; - throw e; - } - - const checkpointSyncUrl = "http://127.0.0.1:19596"; - loggerNodeB.info("Fetching weak subjectivity state ", {checkpointSyncUrl}); - const {wsState, wsCheckpoint} = await fetchWeakSubjectivityState(config, loggerNodeB, {checkpointSyncUrl}); - loggerNodeB.info("Fetched wss state"); - - const bnStartingFromWSS = await getDevBeaconNode({ - params: {...testParams, ALTAIR_FORK_EPOCH: Infinity}, - options: { - api: {rest: {enabled: true, port: 9587} as BeaconRestApiServerOpts}, - sync: {isSingleNode: true, backfillBatchSize: 64}, - chain: {blsVerifyAllMainThread: true}, - }, - validatorCount: 32, - logger: loggerNodeB, - genesisTime, - anchorState: wsState, - wsCheckpoint, - }); - afterEachCallbacks.push(() => bnStartingFromWSS.close()); - - const head = bn.chain.forkChoice.getHead(); - // eslint-disable-next-line @typescript-eslint/strict-boolean-expressions - if (!head) throw Error("First beacon node has no head block"); - if (!bnStartingFromWSS.backfillSync) throw Error("Backfill sync not started"); - const waitForSynced = waitForEvent( - bnStartingFromWSS.backfillSync, - BackfillSyncEvent.completed, - 100000, - (slot) => slot == GENESIS_SLOT - ); - - await connect(bnStartingFromWSS.network, bn.network.peerId, bn.network.localMultiaddrs); - - await waitForSynced; - }); -}); diff --git a/packages/cli/package.json b/packages/cli/package.json index da39b00832d8..d85bdc47b054 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -34,7 +34,7 @@ "pretest": "yarn run check-types", "test:unit": "nyc --cache-dir .nyc_output/.cache -e .ts mocha 'test/unit/**/*.test.ts'", "test:e2e": "mocha --timeout 30000 'test/e2e/**/*.test.ts'", - "test:sim": "LODESTAR_PRESET=minimal ts-node --esm test/simulation/multi_fork.test.ts", + "test:sim": "export LODESTAR_PRESET=minimal && ts-node --esm test/simulation/multi_fork.test.ts && ts-node --esm test/simulation/beacon_endpoints.test.ts", "test": "yarn test:unit && yarn test:e2e", "coverage": "codecov -F lodestar", "check-readme": "typescript-docs-verifier" diff --git a/packages/cli/src/options/beaconNodeOptions/sync.ts b/packages/cli/src/options/beaconNodeOptions/sync.ts index 6961ba537395..a36a726bb91b 100644 --- a/packages/cli/src/options/beaconNodeOptions/sync.ts +++ b/packages/cli/src/options/beaconNodeOptions/sync.ts @@ -4,6 +4,7 @@ import {ICliCommandOptions} from "../../util/index.js"; export interface ISyncArgs { "sync.isSingleNode": boolean; "sync.disableProcessAsChainSegment": boolean; + "sync.disableRangeSync": boolean; "sync.backfillBatchSize": number; } @@ -12,6 +13,7 @@ export function parseArgs(args: ISyncArgs): IBeaconNodeOptions["sync"] { isSingleNode: args["sync.isSingleNode"], disableProcessAsChainSegment: args["sync.disableProcessAsChainSegment"], backfillBatchSize: args["sync.backfillBatchSize"], + disableRangeSync: args["sync.disableRangeSync"], }; } @@ -26,6 +28,14 @@ Use only for local networks with a single node, can be dangerous in regular netw group: "sync", }, + "sync.disableRangeSync": { + hidden: true, + type: "boolean", + description: "Disable range sync completely. Should only be used for debugging or testing.", + defaultDescription: String(defaultOptions.sync.disableRangeSync), + group: "sync", + }, + "sync.disableProcessAsChainSegment": { hidden: true, type: "boolean", diff --git a/packages/cli/test/simulation/beacon_endpoints.test.ts b/packages/cli/test/simulation/beacon_endpoints.test.ts new file mode 100644 index 000000000000..1a0597a32174 --- /dev/null +++ b/packages/cli/test/simulation/beacon_endpoints.test.ts @@ -0,0 +1,135 @@ +/* eslint-disable @typescript-eslint/naming-convention */ +import {join} from "node:path"; +import {expect} from "chai"; +import {toHexString} from "@chainsafe/ssz"; +import {routes} from "@lodestar/api"; +import {CLClient, ELClient} from "../utils/simulation/interfaces.js"; +import {SimulationEnvironment} from "../utils/simulation/SimulationEnvironment.js"; +import {getEstimatedTimeInSecForRun, logFilesDir} from "../utils/simulation/utils/index.js"; +import {waitForSlot} from "../utils/simulation/utils/network.js"; +import {SIM_TESTS_SECONDS_PER_SLOT} from "../utils/simulation/constants.js"; + +const genesisSlotsDelay = 10; +const altairForkEpoch = 2; +const bellatrixForkEpoch = 4; +const validatorCount = 2; +const timeout = + getEstimatedTimeInSecForRun({ + genesisSlotDelay: genesisSlotsDelay, + secondsPerSlot: SIM_TESTS_SECONDS_PER_SLOT, + runTill: 2, + // After adding Nethermind its took longer to complete + graceExtraTimeFraction: 0.1, + }) * 1000; + +const env = SimulationEnvironment.initWithDefaults( + { + id: "beacon-endpoints", + logsDir: join(logFilesDir, "beacon-endpoints"), + chainConfig: { + ALTAIR_FORK_EPOCH: altairForkEpoch, + BELLATRIX_FORK_EPOCH: bellatrixForkEpoch, + GENESIS_DELAY: genesisSlotsDelay, + }, + }, + [ + { + id: "node-1", + cl: {type: CLClient.Lodestar, options: {"sync.isSingleNode": true}}, + el: ELClient.Geth, + keysCount: validatorCount, + mining: true, + }, + ] +); +await env.start(timeout); + +const node = env.nodes[0].cl; +await waitForSlot(2, env.nodes, {env, silent: true}); + +const stateValidators = (await node.api.beacon.getStateValidators("head")).data; + +await env.tracker.assert("should have correct validators count called without filters", async () => { + expect(stateValidators.length).to.be.equal(validatorCount); +}); + +await env.tracker.assert("should have correct validator index for first validator filters", async () => { + expect(stateValidators[0].index).to.be.equal(0); +}); + +await env.tracker.assert("should have correct validator index for second validator filters", async () => { + expect(stateValidators[1].index).to.be.equal(1); +}); + +await env.tracker.assert( + "should return correct number of filtered validators when getStateValidators called with filters", + async () => { + const filterPubKey = + "0xa99a76ed7796f7be22d5b7e85deeb7c5677e88e511e0b337618f8c4eb61349b4bf2d153f649f7b53359fe8b94a38e44c"; + + const response = await node.api.beacon.getStateValidators("head", { + id: [filterPubKey], + }); + + expect(response.data.length).to.be.equal(1); + } +); + +await env.tracker.assert( + "should return correct filtered validators when getStateValidators called with filters", + async () => { + const filterPubKey = + "0xa99a76ed7796f7be22d5b7e85deeb7c5677e88e511e0b337618f8c4eb61349b4bf2d153f649f7b53359fe8b94a38e44c"; + + const response = await node.api.beacon.getStateValidators("head", { + id: [filterPubKey], + }); + + expect(toHexString(response.data[0].validator.pubkey)).to.be.equal(filterPubKey); + } +); + +await env.tracker.assert( + "should return the validator when getStateValidator is called with the validator index", + async () => { + const validatorIndex = "0"; + + const response = await node.api.beacon.getStateValidator("head", validatorIndex); + + // TODO: the index in data should be a string instead of an integer + expect(response.data.index).to.be.equal(parseInt(validatorIndex)); + } +); + +await env.tracker.assert( + "should return the validator when getStateValidator is called with the hex encoded public key", + async () => { + const hexPubKey = + "0xa99a76ed7796f7be22d5b7e85deeb7c5677e88e511e0b337618f8c4eb61349b4bf2d153f649f7b53359fe8b94a38e44c"; + + const response = await node.api.beacon.getStateValidator("head", hexPubKey); + + expect(toHexString(response.data.validator.pubkey)).to.be.equal(hexPubKey); + } +); + +await env.tracker.assert("BN Not Synced", async () => { + const expectedSyncStatus: routes.node.SyncingStatus = { + headSlot: "2", + syncDistance: "0", + isSyncing: false, + isOptimistic: false, + }; + + const response = await node.api.node.getSyncingStatus(); + + expect(response.data).to.be.deep.equal(expectedSyncStatus); +}); + +await env.tracker.assert("Return READY pre genesis", async () => { + const response = await node.api.node.getHealth(); + + expect(response).to.be.equal(routes.node.NodeHealth.READY); +}); + +await env.stop(); diff --git a/packages/cli/test/simulation/multi_fork.test.ts b/packages/cli/test/simulation/multi_fork.test.ts index f768e641382b..c6c68bb0c06f 100644 --- a/packages/cli/test/simulation/multi_fork.test.ts +++ b/packages/cli/test/simulation/multi_fork.test.ts @@ -1,11 +1,18 @@ /* eslint-disable @typescript-eslint/naming-convention */ import {join} from "node:path"; import {activePreset} from "@lodestar/params"; +import {toHexString} from "@lodestar/utils"; import {CLIQUE_SEALING_PERIOD, SIM_TESTS_SECONDS_PER_SLOT} from "../utils/simulation/constants.js"; import {CLClient, ELClient} from "../utils/simulation/interfaces.js"; import {SimulationEnvironment} from "../utils/simulation/SimulationEnvironment.js"; import {getEstimatedTimeInSecForRun, getEstimatedTTD, logFilesDir} from "../utils/simulation/utils/index.js"; -import {connectAllNodes, connectNewNode} from "../utils/simulation/utils/network.js"; +import { + connectAllNodes, + connectNewNode, + waitForHead, + waitForNodeSync, + waitForSlot, +} from "../utils/simulation/utils/network.js"; import {nodeAssertion} from "../utils/simulation/assertions/nodeAssertion.js"; import {mergeAssertion} from "../utils/simulation/assertions/mergeAssertion.js"; @@ -70,18 +77,17 @@ env.tracker.register({ await env.start(timeout); await connectAllNodes(env.nodes); + // The `TTD` will be reach around `start of bellatrixForkEpoch + additionalSlotsForMerge` slot // We wait for the end of that epoch with half more epoch to make sure merge transition is complete -await env.waitForSlot( - env.clock.getLastSlotOfEpoch(bellatrixForkEpoch) + activePreset.SLOTS_PER_EPOCH / 2, - env.nodes, - true -); - -const { - data: {finalized}, -} = await env.nodes[0].cl.api.beacon.getStateFinalityCheckpoints("head"); +await waitForSlot(env.clock.getLastSlotOfEpoch(bellatrixForkEpoch) + activePreset.SLOTS_PER_EPOCH / 2, env.nodes, { + silent: true, + env, +}); +// Range Sync +// ======================================================== +const headForRangeSync = await env.nodes[0].cl.api.beacon.getBlockHeader("head"); const rangeSync = env.createNodePair({ id: "range-sync-node", cl: CLClient.Lodestar, @@ -89,12 +95,19 @@ const rangeSync = env.createNodePair({ keysCount: 0, }); +// Checkpoint sync involves Weak Subjectivity Checkpoint +// ======================================================== +const { + data: {finalized: headForCheckpointSync}, +} = await env.nodes[0].cl.api.beacon.getStateFinalityCheckpoints("head"); const checkpointSync = env.createNodePair({ id: "checkpoint-sync-node", - cl: CLClient.Lodestar, + cl: { + type: CLClient.Lodestar, + options: {wssCheckpoint: `${headForCheckpointSync.root}:${headForCheckpointSync.epoch}`}, + }, el: ELClient.Geth, keysCount: 0, - wssCheckpoint: `${finalized.root}:${finalized.epoch}`, }); await rangeSync.jobs.el.start(); @@ -105,11 +118,59 @@ await checkpointSync.jobs.el.start(); await checkpointSync.jobs.cl.start(); await connectNewNode(checkpointSync.nodePair, env.nodes); -await env.waitForNodeSync(rangeSync.nodePair); -await env.waitForNodeSync(checkpointSync.nodePair); +await Promise.all([ + await waitForNodeSync(env, rangeSync.nodePair, { + head: toHexString(headForRangeSync.data.root), + slot: headForRangeSync.data.header.message.slot, + }), + await waitForNodeSync(env, checkpointSync.nodePair, { + head: toHexString(headForCheckpointSync.root), + slot: env.clock.getLastSlotOfEpoch(headForCheckpointSync.epoch), + }), +]); await rangeSync.jobs.cl.stop(); await rangeSync.jobs.el.stop(); await checkpointSync.jobs.cl.stop(); await checkpointSync.jobs.el.stop(); + +// Unknown block sync +// ======================================================== +const unknownBlockSync = env.createNodePair({ + id: "unknown-block-sync-node", + cl: {type: CLClient.Lodestar, options: {"network.allowPublishToZeroPeers": true, "sync.disableRangeSync": true}}, + el: ELClient.Geth, + keysCount: 0, +}); +await unknownBlockSync.jobs.el.start(); +await unknownBlockSync.jobs.cl.start(); +const headForUnknownBlockSync = await env.nodes[0].cl.api.beacon.getBlockV2("head"); +await connectNewNode(unknownBlockSync.nodePair, env.nodes); + +try { + await unknownBlockSync.nodePair.cl.api.beacon.publishBlock(headForUnknownBlockSync.data); + + env.tracker.record({ + message: "Publishing unknown block should fail", + slot: env.clock.currentSlot, + assertionId: "unknownBlockParent", + }); +} catch (error) { + if (!(error as Error).message.includes("BLOCK_ERROR_PARENT_UNKNOWN")) { + env.tracker.record({ + message: `Publishing unknown block should return "BLOCK_ERROR_PARENT_UNKNOWN" got "${(error as Error).message}"`, + slot: env.clock.currentSlot, + assertionId: "unknownBlockParent", + }); + } +} +await waitForHead(env, unknownBlockSync.nodePair, { + head: toHexString( + env.forkConfig + .getForkTypes(headForUnknownBlockSync.data.message.slot) + .BeaconBlock.hashTreeRoot(headForUnknownBlockSync.data.message) + ), + slot: headForUnknownBlockSync.data.message.slot, +}); + await env.stop(); diff --git a/packages/cli/test/unit/options/beaconNodeOptions.test.ts b/packages/cli/test/unit/options/beaconNodeOptions.test.ts index 02436f6a58cf..58d3d7e31506 100644 --- a/packages/cli/test/unit/options/beaconNodeOptions.test.ts +++ b/packages/cli/test/unit/options/beaconNodeOptions.test.ts @@ -76,6 +76,7 @@ describe("options / beaconNodeOptions", () => { "sync.isSingleNode": true, "sync.disableProcessAsChainSegment": true, "sync.backfillBatchSize": 64, + "sync.disableRangeSync": false, } as IBeaconNodeArgs; const expectedOptions: RecursivePartial = { @@ -156,6 +157,7 @@ describe("options / beaconNodeOptions", () => { isSingleNode: true, disableProcessAsChainSegment: true, backfillBatchSize: 64, + disableRangeSync: false, }, }; diff --git a/packages/cli/test/utils/simulation/SimulationEnvironment.ts b/packages/cli/test/utils/simulation/SimulationEnvironment.ts index ef611e22f8cb..6d934adce3a0 100644 --- a/packages/cli/test/utils/simulation/SimulationEnvironment.ts +++ b/packages/cli/test/utils/simulation/SimulationEnvironment.ts @@ -1,51 +1,50 @@ /* eslint-disable @typescript-eslint/naming-convention */ -import {mkdir, writeFile} from "node:fs/promises"; import {EventEmitter} from "node:events"; +import {mkdir, writeFile} from "node:fs/promises"; import {join} from "node:path"; import tmp from "tmp"; -import {routes} from "@lodestar/api/beacon"; +import {fromHexString} from "@chainsafe/ssz"; import {nodeUtils} from "@lodestar/beacon-node"; import {createIChainForkConfig, IChainForkConfig} from "@lodestar/config"; import {activePreset} from "@lodestar/params"; import {BeaconStateAllForks, interopSecretKey} from "@lodestar/state-transition"; -import {Slot} from "@lodestar/types"; -import {fromHexString} from "@chainsafe/ssz"; -import {sleep} from "@lodestar/utils"; import {generateLodestarBeaconNode} from "./cl_clients/lodestar.js"; +import { + BN_P2P_BASE_PORT, + BN_REST_BASE_PORT, + CLIQUE_SEALING_PERIOD, + EL_ENGINE_BASE_PORT, + EL_ETH_BASE_PORT, + EL_P2P_BASE_PORT, + KEY_MANAGER_BASE_PORT, + SIM_TESTS_SECONDS_PER_SLOT, +} from "./constants.js"; +import {generateGethNode} from "./el_clients/geth.js"; +import {generateNethermindNode} from "./el_clients/nethermind.js"; import {EpochClock} from "./EpochClock.js"; import {ExternalSignerServer} from "./ExternalSignerServer.js"; import { AtLeast, CLClient, - CLClientOptions, + CLClientGeneratorOptions, + CLClientsOptions, CLNode, - NodePairResult, ELClient, - ELClientOptions, + ELClientsOptions, + ELGeneratorClientOptions, ELNode, ELStartMode, Job, NodePair, NodePairOptions, + NodePairResult, SimulationInitOptions, SimulationOptions, } from "./interfaces.js"; import {ChildProcessRunner} from "./runner/ChildProcessRunner.js"; +import {DockerRunner} from "./runner/DockerRunner.js"; import {SimulationTracker} from "./SimulationTracker.js"; -import { - BN_P2P_BASE_PORT, - BN_REST_BASE_PORT, - CLIQUE_SEALING_PERIOD, - EL_ENGINE_BASE_PORT, - EL_ETH_BASE_PORT, - EL_P2P_BASE_PORT, - KEY_MANAGER_BASE_PORT, - SIM_TESTS_SECONDS_PER_SLOT, -} from "./constants.js"; -import {generateGethNode} from "./el_clients/geth.js"; import {getEstimatedTTD} from "./utils/index.js"; -import {DockerRunner} from "./runner/DockerRunner.js"; -import {generateNethermindNode} from "./el_clients/nethermind.js"; export const SHARED_JWT_SECRET = "0xdc6457099f127cf0bac78de8b297df04951281909db4f58b43def7c7151e765d"; @@ -216,65 +215,21 @@ export class SimulationEnvironment { await this.dockerRunner.stop(); if (this.tracker.getErrorCount() > 0) { - this.tracker.printErrors(); + this.tracker.reporter.summary(); process.exit(this.tracker.getErrorCount() > 0 ? 1 : code); } else { process.exit(code); } } - // TODO: Add timeout support - waitForEvent(event: routes.events.EventType, node?: CLNode): Promise { - console.log(`Waiting for event "${event}" on "${node?.id ?? "any node"}"`); - - return new Promise((resolve) => { - const handler = (beaconEvent: routes.events.BeaconEvent, eventNode: CLNode): void => { - if (!node) { - this.emitter.removeListener(event, handler); - resolve(beaconEvent); - } - - if (node && eventNode === node) { - this.emitter.removeListener(event, handler); - resolve(beaconEvent); - } - }; - - this.tracker.emitter.addListener(event, handler); - }); - } - - async waitForSlot(slot: Slot, nodes?: NodePair[], silent = true): Promise { - if (!silent) { - console.log(`\nWaiting for slot on "${nodes ? nodes.map((n) => n.cl.id).join(",") : "all nodes"}"`, { - target: slot, - current: this.clock.currentSlot, - }); - } - - await Promise.all( - (nodes ?? this.nodes).map( - (node) => - new Promise((resolve) => { - this.tracker.onSlot(slot, node, resolve); - }) - ) - ); - } - - async waitForNodeSync(node: NodePair): Promise { - // eslint-disable-next-line no-constant-condition - while (true) { - const result = await node.cl.api.node.getSyncingStatus(); - if (result.data.isSyncing) { - await sleep(1000, this.options.controller.signal); - } else { - break; - } - } - } - - createNodePair({el, cl, keysCount, id, wssCheckpoint, remote, mining}: NodePairOptions): NodePairResult { + createNodePair({ + el, + cl, + keysCount, + id, + remote, + mining, + }: NodePairOptions): NodePairResult { if (this.genesisState && keysCount > 0) { throw new Error("Genesis state already initialized. Can not add more keys to it."); } @@ -290,7 +245,6 @@ export class SimulationEnvironment { id, remoteKeys: remote ? keys : [], localKeys: remote ? [] : keys, - wssCheckpoint, }); const elClient = this.createELNode(el, {id, mining}); @@ -301,15 +255,18 @@ export class SimulationEnvironment { }; } - private createCLNode( - client: CLClient, - options?: AtLeast + private createCLNode( + client: C | {type: C; options: CLClientsOptions[C]}, + options?: AtLeast ): {job: Job; node: CLNode} { - const clId = `${options?.id}-cl-${client}`; + const clientType = typeof client === "object" ? client.type : client; + const clientOptions = typeof client === "object" ? client.options : undefined; + + const clId = `${options?.id}-cl-${clientType}`; - switch (client) { + switch (clientType) { case CLClient.Lodestar: { - const opts: CLClientOptions = { + const opts: CLClientGeneratorOptions = { id: clId, dataDir: join(this.options.rootDir, clId), logFilePath: join(this.options.logsDir, `${clId}.log`), @@ -324,6 +281,7 @@ export class SimulationEnvironment { genesisTime: this.options.genesisTime, engineUrl: options?.engineUrl ?? `http://127.0.0.1:${EL_ENGINE_BASE_PORT + this.nodePairCount + 1}`, jwtSecretHex: options?.jwtSecretHex ?? SHARED_JWT_SECRET, + clientOptions: clientOptions ?? {}, }; return generateLodestarBeaconNode(opts, this.childProcessRunner); } @@ -332,10 +290,16 @@ export class SimulationEnvironment { } } - private createELNode(client: ELClient, options: AtLeast): {job: Job; node: ELNode} { - const elId = `${options.id}-el-${client}`; + private createELNode( + client: E | {type: E; options: ELClientsOptions[E]}, + options: AtLeast + ): {job: Job; node: ELNode} { + const clientType = typeof client === "object" ? client.type : client; + const clientOptions = typeof client === "object" ? client.options : undefined; + + const elId = `${options.id}-el-${clientType}`; - const opts: ELClientOptions = { + const opts: ELGeneratorClientOptions = { id: elId, mode: options?.mode ?? (this.forkConfig.BELLATRIX_FORK_EPOCH > 0 ? ELStartMode.PreMerge : ELStartMode.PostMerge), ttd: options?.ttd ?? this.forkConfig.TERMINAL_TOTAL_DIFFICULTY, @@ -348,14 +312,15 @@ export class SimulationEnvironment { port: options?.port ?? EL_P2P_BASE_PORT + this.nodePairCount + 1, address: this.dockerRunner.getNextIp(), mining: options?.mining ?? false, + clientOptions: clientOptions ?? [], }; - switch (client) { + switch (clientType) { case ELClient.Geth: { - return generateGethNode(opts, this.dockerRunner); + return generateGethNode(opts as ELGeneratorClientOptions, this.dockerRunner); } case ELClient.Nethermind: { - return generateNethermindNode(opts, this.dockerRunner); + return generateNethermindNode(opts as ELGeneratorClientOptions, this.dockerRunner); } default: throw new Error(`EL Client "${client}" not supported`); diff --git a/packages/cli/test/utils/simulation/SimulationTracker.ts b/packages/cli/test/utils/simulation/SimulationTracker.ts index d0fd2c063911..84fb4a5b4e55 100644 --- a/packages/cli/test/utils/simulation/SimulationTracker.ts +++ b/packages/cli/test/utils/simulation/SimulationTracker.ts @@ -3,28 +3,18 @@ import {routes} from "@lodestar/api/beacon"; import {IChainForkConfig} from "@lodestar/config"; import {Epoch, Slot} from "@lodestar/types"; import {EpochClock} from "./EpochClock.js"; -import {NodeId, NodePair, SimulationAssertion, SimulationAssertionError, StoreType, StoreTypes} from "./interfaces.js"; -import {arrayGroupBy, avg, getForkName, squeezeString} from "./utils/index.js"; -import {attestationsCountAssertion} from "./assertions/defaults/attestationCountAssertion.js"; -import {attestationParticipationAssertion} from "./assertions/defaults/attestationParticipationAssertion.js"; -import {connectedPeerCountAssertion} from "./assertions/defaults/connectedPeerCountAssertion.js"; -import {finalizedAssertion} from "./assertions/defaults/finalizedAssertion.js"; -import {headAssertion} from "./assertions/defaults/headAssertion.js"; -import {inclusionDelayAssertion} from "./assertions/defaults/inclusionDelayAssertion.js"; -import {missedBlocksAssertion} from "./assertions/defaults/missedBlocksAssertion.js"; -import {syncCommitteeParticipation} from "./assertions/defaults/syncCommitteeParticipation.js"; -import {TableRenderer} from "./TableRenderer.js"; - -const defaultAssertions = [ - inclusionDelayAssertion, - attestationsCountAssertion, - attestationParticipationAssertion, - connectedPeerCountAssertion, - finalizedAssertion, - headAssertion, - missedBlocksAssertion, - syncCommitteeParticipation, -]; +import { + AtLeast, + NodeId, + NodePair, + SimulationAssertion, + SimulationAssertionError, + SimulationReporter, + StoreType, + StoreTypes, +} from "./interfaces.js"; +import {defaultAssertions} from "./assertions/defaults/index.js"; +import {TableReporter} from "./TableReporter.js"; interface SimulationTrackerInitOptions { nodes: NodePair[]; @@ -33,20 +23,28 @@ interface SimulationTrackerInitOptions { signal: AbortSignal; } +export enum SimulationTrackerEvent { + Slot = "slot", + Head = "head", +} + +export type SimulationTrackerEvents = { + [SimulationTrackerEvent.Slot]: {slot: Slot}; + [SimulationTrackerEvent.Head]: routes.events.EventData[routes.events.EventType.head]; +}; + +export const getEventNameForNodePair = (nodePair: NodePair, event: SimulationTrackerEvent): string => + `sim:tracker:${event}:${nodePair.id}`; + +const eventStreamEventMap = { + [SimulationTrackerEvent.Head]: routes.events.EventType.head, + [SimulationTrackerEvent.Slot]: routes.events.EventType.block, +}; + /* eslint-disable no-console */ export class SimulationTracker { readonly emitter = new EventEmitter(); - table = new TableRenderer({ - fork: 10, - eph: 5, - slot: 4, - head: 16, - finzed: 6, - peers: 6, - attCount: 8, - incDelay: 8, - errors: 10, - }); + readonly reporter: SimulationReporter>; private lastSeenSlot: Map = new Map(); private slotCapture: Map = new Map(); @@ -54,21 +52,27 @@ export class SimulationTracker { private signal: AbortSignal; private nodes: NodePair[]; private clock: EpochClock; - private config: IChainForkConfig; + private forkConfig: IChainForkConfig; private errors: SimulationAssertionError[] = []; private stores: StoreTypes & StoreType; private assertions: SimulationAssertion[]; private assertionIdsMap: Record = {}; - private constructor({signal, nodes, clock, config}: SimulationTrackerInitOptions) { this.signal = signal; this.nodes = nodes; this.clock = clock; - this.config = config; + this.forkConfig = config; this.stores = {} as StoreTypes & StoreType; this.assertions = [] as SimulationAssertion[]; + this.reporter = new TableReporter({ + clock: this.clock, + forkConfig: this.forkConfig, + nodes: this.nodes, + stores: this.stores, + errors: this.errors, + }); } static initWithDefaultAssertions(opts: SimulationTrackerInitOptions): SimulationTracker { @@ -81,6 +85,45 @@ export class SimulationTracker { return tracker; } + once( + nodePair: NodePair, + eventName: K, + fn: (data: SimulationTrackerEvents[K]) => void + ): void { + if (this.nodes.indexOf(nodePair) < 0) { + this.initEventStreamForNode(nodePair, [eventStreamEventMap[eventName]]); + } + + this.emitter.once(getEventNameForNodePair(nodePair, eventName), fn); + } + + on( + nodePair: NodePair, + eventName: K, + fn: (data: SimulationTrackerEvents[K]) => void + ): void { + if (this.nodes.indexOf(nodePair) < 0) { + this.initEventStreamForNode(nodePair, [eventStreamEventMap[eventName]]); + } + this.emitter.on(getEventNameForNodePair(nodePair, eventName), fn); + } + + off( + nodePair: NodePair, + eventName: K, + fn: (data: SimulationTrackerEvents[K]) => void + ): void { + this.emitter.off(getEventNameForNodePair(nodePair, eventName), fn); + } + + private emit( + nodePair: NodePair, + eventName: K, + data: SimulationTrackerEvents[K] + ): void { + this.emitter.emit(getEventNameForNodePair(nodePair, eventName), data); + } + track(node: NodePair): void { this.initDataForNode(node); this.initEventStreamForNode(node); @@ -91,7 +134,7 @@ export class SimulationTracker { for (const node of this.nodes) { this.initEventStreamForNode(node); } - this.table.printHeader(); + this.reporter.bootstrap(); } async stop(): Promise { @@ -126,79 +169,19 @@ export class SimulationTracker { } } - printTrackerInfo(slot: Slot): void { - const epoch = this.clock.getEpochForSlot(slot); - const forkName = getForkName(epoch, this.config); - const epochStr = `${epoch}/${this.clock.getSlotIndexInEpoch(slot)}`; - - if (this.clock.isFirstSlotOfEpoch(slot)) { - // We are printing this info for last epoch - if (epoch - 1 < this.config.ALTAIR_FORK_EPOCH) { - this.table.addEmptyRow("Att Participation: N/A - SC Participation: N/A"); - } else { - // attestationParticipation is calculated at first slot of an epoch - const participation = this.nodes.map((node) => this.stores["attestationParticipation"][node.cl.id][slot] ?? 0); - const head = avg(participation.map((p) => p.head)).toFixed(2); - const source = avg(participation.map((p) => p.source)).toFixed(2); - const target = avg(participation.map((p) => p.target)).toFixed(2); - - // syncParticipation is calculated at last slot of an epoch so we subtract "slot -1" - const syncParticipation = avg( - this.nodes.map((node) => this.stores["syncCommitteeParticipation"][node.cl.id][slot - 1] ?? "-") - ).toFixed(2); - - this.table.addEmptyRow( - `Att Participation: H: ${head}, S: ${source}, T: ${target} - SC Participation: ${syncParticipation}` - ); - } - } - - const finalizedSlots = this.nodes.map((node) => this.stores["finalized"][node.cl.id][slot] ?? "-"); - const finalizedSlotsUnique = new Set(finalizedSlots); - - const inclusionDelay = this.nodes.map((node) => this.stores["inclusionDelay"][node.cl.id][slot] ?? "-"); - const inclusionDelayUnique = new Set(inclusionDelay); - - const attestationCount = this.nodes.map((node) => this.stores["attestationsCount"][node.cl.id][slot] ?? "-"); - const attestationCountUnique = new Set(attestationCount); - - const head = this.nodes.map((node) => this.stores["head"][node.cl.id][slot] ?? "-"); - const headUnique = new Set(head); - - const peerCount = this.nodes.map((node) => this.stores["connectedPeerCount"][node.cl.id][slot] ?? "-"); - const peerCountUnique = new Set(head); - - const errorCount = this.errors.filter((e) => e.slot === slot).length; - - this.table.addRow({ - fork: forkName, - eph: epochStr, - slot: slot, - head: headUnique.size === 1 ? squeezeString(head[0], 16, "..") : "different", - finzed: finalizedSlotsUnique.size === 1 ? finalizedSlots[0] : finalizedSlots.join(","), - peers: peerCountUnique.size === 1 ? peerCount[0] : peerCount.join(","), - attCount: attestationCountUnique.size === 1 ? attestationCount[0] : "---", - incDelay: inclusionDelayUnique.size === 1 ? inclusionDelay[0].toFixed(2) : "---", - errors: errorCount, - }); + record(error: AtLeast): void { + this.errors.push({...error, epoch: error.epoch ?? this.clock.getEpochForSlot(error.slot)}); } - printErrors(): void { - console.log(`├${"─".repeat(10)} Errors (${this.errors.length}) ${"─".repeat(10)}┤`); - - const groupBySlot = arrayGroupBy(this.errors, (e) => String(e.slot as number)); - - for (const [slot, slotErrors] of Object.entries(groupBySlot)) { - if (slotErrors.length > 0) console.log(`├─ Slot: ${slot}`); - const groupByAssertion = arrayGroupBy(slotErrors, (e) => e.assertionId); - - for (const [assertionId, assertionErrors] of Object.entries(groupByAssertion)) { - if (assertionErrors.length > 0) console.log(`├── Assertion: ${assertionId}`); - - for (const error of assertionErrors) { - console.error(`├──── ${error.message}`); - } - } + async assert(message: string, cb: () => void | Promise): Promise { + try { + await cb(); + } catch (error) { + this.record({ + assertionId: message, + message: (error as Error).message, + slot: this.clock.currentSlot, + }); } } @@ -209,7 +192,10 @@ export class SimulationTracker { } } - private async onBlock(event: routes.events.EventData[routes.events.EventType.block], node: NodePair): Promise { + private async processOnBlock( + event: routes.events.EventData[routes.events.EventType.block], + node: NodePair + ): Promise { const slot = event.slot; const epoch = this.clock.getEpochForSlot(slot); const lastSeenSlot = this.lastSeenSlot.get(node.cl.id); @@ -227,12 +213,12 @@ export class SimulationTracker { for (const assertion of this.assertions) { if (assertion.capture) { const value = await assertion.capture({ - fork: getForkName(epoch, this.config), + fork: this.forkConfig.getForkName(slot), slot, block: block.data, clock: this.clock, node, - forkConfig: this.config, + forkConfig: this.forkConfig, epoch, store: this.stores[assertion.id][node.cl.id], // TODO: Make the store safe, to filter just the dependant stores not all @@ -258,14 +244,17 @@ export class SimulationTracker { await this.applyAssertions({slot, epoch}); - this.emitter.emit(`${node.cl.id}:slot:${slot}`, slot); + this.emit(node, SimulationTrackerEvent.Slot, {slot}); } - private onHead(_event: routes.events.EventData[routes.events.EventType.head], _node: NodePair): void { - // TODO: Add head tracking + private async processOnHead( + event: routes.events.EventData[routes.events.EventType.head], + node: NodePair + ): Promise { + this.emit(node, SimulationTrackerEvent.Head, event); } - private onFinalizedCheckpoint( + private processOnFinalizedCheckpoint( _event: routes.events.EventData[routes.events.EventType.finalizedCheckpoint], _node: NodePair ): void { @@ -280,7 +269,7 @@ export class SimulationTracker { } for (const assertion of this.assertions) { - const match = assertion.match({slot, epoch, clock: this.clock, forkConfig: this.config}); + const match = assertion.match({slot, epoch, clock: this.clock, forkConfig: this.forkConfig}); if ((typeof match === "boolean" && match) || (typeof match === "object" && match.match)) { try { const errors = await assertion.assert({ @@ -288,7 +277,7 @@ export class SimulationTracker { epoch, nodes: this.nodes, clock: this.clock, - forkConfig: this.config, + forkConfig: this.forkConfig, store: this.stores[assertion.id], // TODO: Make the store safe, to filter just the dependant stores not all dependantStores: this.stores, @@ -308,7 +297,7 @@ export class SimulationTracker { } } - this.printTrackerInfo(slot); + this.reporter.progress(slot); this.processRemoveAssertionQueue(); } @@ -320,22 +309,27 @@ export class SimulationTracker { this.removeAssertionQueue = []; } - private initEventStreamForNode(node: NodePair): void { - node.cl.api.events.eventstream( - [routes.events.EventType.block, routes.events.EventType.head, routes.events.EventType.finalizedCheckpoint], - this.signal, - async (event) => { - this.emitter.emit(event.type, event, node); - - switch (event.type) { - case routes.events.EventType.block: - await this.onBlock(event.message, node); - return; - case routes.events.EventType.finalizedCheckpoint: - this.onFinalizedCheckpoint(event.message, node); - return; - } + private initEventStreamForNode( + node: NodePair, + events: routes.events.EventType[] = [ + routes.events.EventType.block, + routes.events.EventType.head, + routes.events.EventType.finalizedCheckpoint, + ], + signal?: AbortSignal + ): void { + node.cl.api.events.eventstream(events, signal ?? this.signal, async (event) => { + switch (event.type) { + case routes.events.EventType.block: + await this.processOnBlock(event.message, node); + return; + case routes.events.EventType.head: + await this.processOnHead(event.message, node); + return; + case routes.events.EventType.finalizedCheckpoint: + this.processOnFinalizedCheckpoint(event.message, node); + return; } - ); + }); } } diff --git a/packages/cli/test/utils/simulation/TableReporter.ts b/packages/cli/test/utils/simulation/TableReporter.ts new file mode 100644 index 000000000000..63a945cf58b8 --- /dev/null +++ b/packages/cli/test/utils/simulation/TableReporter.ts @@ -0,0 +1,115 @@ +/* eslint-disable no-console */ +import {Slot} from "@lodestar/types"; +import {defaultAssertions} from "./assertions/defaults/index.js"; +import {SimulationReporter} from "./interfaces.js"; +import {TableRenderer} from "./TableRenderer.js"; +import {arrayGroupBy, avg} from "./utils/index.js"; + +export class TableReporter extends SimulationReporter { + private table = new TableRenderer({ + fork: 10, + eph: 5, + slot: 4, + head: 8, + finzed: 6, + peers: 6, + attCount: 8, + incDelay: 8, + errors: 10, + }); + + bootstrap(): void { + this.table.printHeader(); + } + + progress(slot: Slot): void { + { + const {clock, forkConfig, nodes, stores, errors} = this.options; + + const epoch = clock.getEpochForSlot(slot); + const forkName = forkConfig.getForkName(slot); + const epochStr = `${epoch}/${clock.getSlotIndexInEpoch(slot)}`; + + if (clock.isFirstSlotOfEpoch(slot)) { + // We are printing this info for last epoch + if (epoch - 1 < forkConfig.ALTAIR_FORK_EPOCH) { + this.table.addEmptyRow("Att Participation: N/A - SC Participation: N/A"); + } else { + // attestationParticipation is calculated at first slot of an epoch + const participation = nodes.map((node) => stores["attestationParticipation"][node.cl.id][slot] ?? 0); + const head = avg(participation.map((p) => p.head)).toFixed(2); + const source = avg(participation.map((p) => p.source)).toFixed(2); + const target = avg(participation.map((p) => p.target)).toFixed(2); + + // As it's printed on the first slot of epoch we need to get the previous epoch + const startSlot = clock.getFirstSlotOfEpoch(epoch - 1); + const endSlot = clock.getLastSlotOfEpoch(epoch - 1); + const nodesSyncParticipationAvg: number[] = []; + for (const node of nodes) { + const syncCommitteeParticipation: number[] = []; + for (let slot = startSlot; slot <= endSlot; slot++) { + syncCommitteeParticipation.push(stores["syncCommitteeParticipation"][node.cl.id][slot]); + } + nodesSyncParticipationAvg.push(avg(syncCommitteeParticipation)); + } + + const syncParticipation = avg(nodesSyncParticipationAvg).toFixed(2); + + this.table.addEmptyRow( + `Att Participation: H: ${head}, S: ${source}, T: ${target} - SC Participation: ${syncParticipation}` + ); + } + } + + const finalizedSlots = nodes.map((node) => stores["finalized"][node.cl.id][slot] ?? "-"); + const finalizedSlotsUnique = new Set(finalizedSlots); + + const inclusionDelay = nodes.map((node) => stores["inclusionDelay"][node.cl.id][slot] ?? "-"); + const inclusionDelayUnique = new Set(inclusionDelay); + + const attestationCount = nodes.map((node) => stores["attestationsCount"][node.cl.id][slot] ?? "-"); + const attestationCountUnique = new Set(attestationCount); + + const head = nodes.map((node) => stores["head"][node.cl.id][slot] ?? "-"); + const headUnique = new Set(head); + + const peerCount = nodes.map((node) => stores["connectedPeerCount"][node.cl.id][slot] ?? "-"); + const peerCountUnique = new Set(head); + + const errorCount = errors.filter((e) => e.slot === slot).length; + + this.table.addRow({ + fork: forkName, + eph: epochStr, + slot: slot, + head: headUnique.size === 1 ? `${head[0].slice(0, 6)}..` : "different", + finzed: finalizedSlotsUnique.size === 1 ? finalizedSlots[0] : finalizedSlots.join(","), + peers: peerCountUnique.size === 1 ? peerCount[0] : peerCount.join(","), + attCount: attestationCountUnique.size === 1 ? attestationCount[0] : "---", + incDelay: inclusionDelayUnique.size === 1 ? inclusionDelay[0].toFixed(2) : "---", + errors: errorCount, + }); + } + } + + summary(): void { + const {errors} = this.options; + + console.log(`├${"─".repeat(10)} Errors (${errors.length}) ${"─".repeat(10)}┤`); + + const groupBySlot = arrayGroupBy(errors, (e) => String(e.slot as number)); + + for (const [slot, slotErrors] of Object.entries(groupBySlot)) { + if (slotErrors.length > 0) console.log(`├─ Slot: ${slot}`); + const groupByAssertion = arrayGroupBy(slotErrors, (e) => e.assertionId); + + for (const [assertionId, assertionErrors] of Object.entries(groupByAssertion)) { + if (assertionErrors.length > 0) console.log(`├── Assertion: ${assertionId}`); + + for (const error of assertionErrors) { + console.error(`├──── ${error.message}`); + } + } + } + } +} diff --git a/packages/cli/test/utils/simulation/assertions/defaults/index.ts b/packages/cli/test/utils/simulation/assertions/defaults/index.ts new file mode 100644 index 000000000000..0bc1448c6a30 --- /dev/null +++ b/packages/cli/test/utils/simulation/assertions/defaults/index.ts @@ -0,0 +1,19 @@ +import {attestationsCountAssertion} from "./attestationCountAssertion.js"; +import {attestationParticipationAssertion} from "./attestationParticipationAssertion.js"; +import {connectedPeerCountAssertion} from "./connectedPeerCountAssertion.js"; +import {finalizedAssertion} from "./finalizedAssertion.js"; +import {headAssertion} from "./headAssertion.js"; +import {inclusionDelayAssertion} from "./inclusionDelayAssertion.js"; +import {missedBlocksAssertion} from "./missedBlocksAssertion.js"; +import {syncCommitteeParticipation} from "./syncCommitteeParticipation.js"; + +export const defaultAssertions = [ + inclusionDelayAssertion, + attestationsCountAssertion, + attestationParticipationAssertion, + connectedPeerCountAssertion, + finalizedAssertion, + headAssertion, + missedBlocksAssertion, + syncCommitteeParticipation, +]; diff --git a/packages/cli/test/utils/simulation/assertions/matchers.ts b/packages/cli/test/utils/simulation/assertions/matchers.ts index 99a985e4635d..d0aa548797ee 100644 --- a/packages/cli/test/utils/simulation/assertions/matchers.ts +++ b/packages/cli/test/utils/simulation/assertions/matchers.ts @@ -3,3 +3,6 @@ import {AssertionMatcher} from "../interfaces.js"; export const everySlotMatcher: AssertionMatcher = ({slot}) => slot >= 0; export const everyEpochMatcher: AssertionMatcher = ({slot, clock}) => clock.isLastSlotOfEpoch(slot); export const neverMatcher: AssertionMatcher = () => false; +export const onceOnSlotMatcher = (userSlot: number): AssertionMatcher => ({slot}) => + slot === userSlot ? {remove: true, match: true} : false; +export const onceOnStartupMatcher = onceOnSlotMatcher(1); diff --git a/packages/cli/test/utils/simulation/cl_clients/lodestar.ts b/packages/cli/test/utils/simulation/cl_clients/lodestar.ts index 5f546e3e2ca1..8f9faff9161d 100644 --- a/packages/cli/test/utils/simulation/cl_clients/lodestar.ts +++ b/packages/cli/test/utils/simulation/cl_clients/lodestar.ts @@ -9,14 +9,11 @@ import {LogLevel} from "@lodestar/utils"; import {IBeaconArgs} from "../../../../src/cmds/beacon/options.js"; import {IValidatorCliArgs} from "../../../../src/cmds/validator/options.js"; import {IGlobalArgs} from "../../../../src/options/globalOptions.js"; -import {CLClient, CLClientGenerator, CLClientOptions, JobOptions, Runner, RunnerType} from "../interfaces.js"; +import {CLClient, CLClientGenerator, CLClientGeneratorOptions, JobOptions, Runner, RunnerType} from "../interfaces.js"; import {LODESTAR_BINARY_PATH} from "../constants.js"; import {isChildProcessRunner} from "../runner/index.js"; -export const generateLodestarBeaconNode: CLClientGenerator = ( - opts: CLClientOptions, - runner: Runner | Runner -) => { +export const generateLodestarBeaconNode: CLClientGenerator = (opts, runner) => { if (!isChildProcessRunner(runner)) { throw new Error(`Runner "${runner.type}" not yet supported.`); } @@ -28,14 +25,13 @@ export const generateLodestarBeaconNode: CLClientGenerator = ( id, config, genesisStateFilePath, - checkpointSyncUrl, remoteKeys, localKeys, - wssCheckpoint, keyManagerPort, genesisTime, engineUrl, jwtSecretHex, + clientOptions, } = opts; const jwtSecretPath = join(dataDir, "jwtsecret"); @@ -71,16 +67,9 @@ export const generateLodestarBeaconNode: CLClientGenerator = ( eth1: true, "execution.urls": [engineUrl], "jwt-secret": jwtSecretPath, + ...clientOptions, } as unknown) as IBeaconArgs & IGlobalArgs; - if (checkpointSyncUrl) { - rcConfig["checkpointSyncUrl"] = checkpointSyncUrl; - } - - if (wssCheckpoint) { - rcConfig["wssCheckpoint"] = wssCheckpoint; - } - const validatorClientsJobs: JobOptions[] = []; if (opts.localKeys.length > 0 || opts.remoteKeys.length > 0) { validatorClientsJobs.push( @@ -139,7 +128,7 @@ export const generateLodestarBeaconNode: CLClientGenerator = ( }; export const generateLodestarValidatorJobs = ( - opts: CLClientOptions, + opts: CLClientGeneratorOptions, runner: Runner | Runner ): JobOptions => { if (runner.type !== RunnerType.ChildProcess) { diff --git a/packages/cli/test/utils/simulation/el_clients/geth.ts b/packages/cli/test/utils/simulation/el_clients/geth.ts index 4581dd25a981..9ac1989a05cb 100644 --- a/packages/cli/test/utils/simulation/el_clients/geth.ts +++ b/packages/cli/test/utils/simulation/el_clients/geth.ts @@ -6,7 +6,7 @@ import {ZERO_HASH} from "@lodestar/state-transition"; import { ELClient, ELClientGenerator, - ELClientOptions, + ELGeneratorClientOptions, ELNode, ELStartMode, JobOptions, @@ -22,7 +22,7 @@ const SECRET_KEY = "45a915e4d060149eb4365960e6a7a45f334393093061116b197e3240065f const PASSWORD = "12345678"; const GENESIS_ACCOUNT = "0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b"; -export const generateGethNode: ELClientGenerator = ( +export const generateGethNode: ELClientGenerator = ( { id, mode, @@ -36,7 +36,8 @@ export const generateGethNode: ELClientGenerator = ( cliqueSealingPeriod, address, mining, - }: ELClientOptions, + clientOptions, + }: ELGeneratorClientOptions, runner: Runner | Runner ) => { if (isChildProcessRunner(runner)) { @@ -143,6 +144,7 @@ export const generateGethNode: ELClientGenerator = ( "5", ...(mining ? ["--mine"] : []), ...(mode == ELStartMode.PreMerge ? ["--nodiscover"] : []), + ...clientOptions, ], env: {}, }, diff --git a/packages/cli/test/utils/simulation/el_clients/nethermind.ts b/packages/cli/test/utils/simulation/el_clients/nethermind.ts index 9ccc45383975..ccf7058f5d9d 100644 --- a/packages/cli/test/utils/simulation/el_clients/nethermind.ts +++ b/packages/cli/test/utils/simulation/el_clients/nethermind.ts @@ -3,12 +3,12 @@ import {mkdir, writeFile} from "node:fs/promises"; import {join} from "node:path"; import got from "got"; import {ZERO_HASH} from "@lodestar/state-transition"; -import {ELClient, ELClientGenerator, ELClientOptions, ELNode, JobOptions, Runner, RunnerType} from "../interfaces.js"; +import {ELClient, ELClientGenerator, ELNode, JobOptions} from "../interfaces.js"; import {Eth1ProviderWithAdmin} from "../Eth1ProviderWithAdmin.js"; import {isDockerRunner} from "../runner/index.js"; import {getNethermindChainSpec} from "../utils/el_genesis.js"; -export const generateNethermindNode: ELClientGenerator = ( +export const generateNethermindNode: ELClientGenerator = ( { id, mode, @@ -22,8 +22,9 @@ export const generateNethermindNode: ELClientGenerator = ( cliqueSealingPeriod, address, mining, - }: ELClientOptions, - runner: Runner | Runner + clientOptions, + }, + runner ) => { if (!isDockerRunner(runner)) { throw new Error("Nethermind client only supports docker runner"); @@ -91,6 +92,7 @@ export const generateNethermindNode: ELClientGenerator = ( "--config", "none", ...(mining ? ["--Init.IsMining", "true", "--Mining.Enabled", "true"] : []), + ...clientOptions, ], env: {}, }, diff --git a/packages/cli/test/utils/simulation/interfaces.ts b/packages/cli/test/utils/simulation/interfaces.ts index dea30b249989..8888eb2a2d20 100644 --- a/packages/cli/test/utils/simulation/interfaces.ts +++ b/packages/cli/test/utils/simulation/interfaces.ts @@ -5,6 +5,8 @@ import {Api as KeyManagerApi} from "@lodestar/api/keymanager"; import {IChainConfig, IChainForkConfig} from "@lodestar/config"; import {ForkName} from "@lodestar/params"; import {Slot, allForks, Epoch} from "@lodestar/types"; +import {IBeaconArgs} from "../../../src/cmds/beacon/options.js"; +import {IGlobalArgs} from "../../../src/options/index.js"; import {EpochClock} from "./EpochClock.js"; import {Eth1ProviderWithAdmin} from "./Eth1ProviderWithAdmin.js"; @@ -38,17 +40,25 @@ export enum ELStartMode { PostMerge = "post-merge", } -export interface NodePairOptions { - el: ELClient; - cl: CLClient; +export type CLClientsOptions = { + [CLClient.Lodestar]: Partial; +}; + +export type ELClientsOptions = { + [ELClient.Geth]: string[]; + [ELClient.Nethermind]: string[]; +}; + +export interface NodePairOptions { keysCount: number; remote?: boolean; mining?: boolean; - wssCheckpoint?: string; id: string; + cl: C | {type: C; options: CLClientsOptions[C]}; + el: E | {type: E; options: ELClientsOptions[E]}; } -export interface CLClientOptions { +export interface CLClientGeneratorOptions { id: string; dataDir: string; logFilePath: string; @@ -60,19 +70,18 @@ export interface CLClientOptions { config: IChainForkConfig; localKeys: SecretKey[]; remoteKeys: SecretKey[]; - checkpointSyncUrl?: string; - wssCheckpoint?: string; genesisTime: number; engineUrl: string; jwtSecretHex: string; + clientOptions: CLClientsOptions[C]; } -export interface ELGenesisOptions { +export interface ELGeneratorGenesisOptions { ttd: bigint; cliqueSealingPeriod: number; } -export interface ELClientOptions extends ELGenesisOptions { +export interface ELGeneratorClientOptions extends ELGeneratorGenesisOptions { mode: ELStartMode; id: string; logFilePath: string; @@ -83,6 +92,7 @@ export interface ELClientOptions extends ELGenesisOptions { port: number; address: string; mining: boolean; + clientOptions: ELClientsOptions[E]; } export interface CLNode { @@ -116,12 +126,12 @@ export interface NodePairResult { jobs: {el: Job; cl: Job}; } -export type CLClientGenerator = ( - opts: CLClientOptions, +export type CLClientGenerator = ( + opts: CLClientGeneratorOptions, runner: Runner | Runner ) => {job: Job; node: CLNode}; -export type ELClientGenerator = ( - opts: ELClientOptions, +export type ELClientGenerator = ( + opts: ELGeneratorClientOptions, runner: Runner | Runner ) => {job: Job; node: ELNode}; @@ -256,3 +266,18 @@ export type Eth1GenesisBlock = { }; alloc: Record; }; + +export abstract class SimulationReporter { + constructor( + protected options: { + clock: EpochClock; + forkConfig: IChainForkConfig; + stores: StoreTypes; + nodes: NodePair[]; + errors: SimulationAssertionError[]; + } + ) {} + abstract bootstrap(): void; + abstract progress(slot: Slot): void; + abstract summary(): void; +} diff --git a/packages/cli/test/utils/simulation/runner/DockerRunner.ts b/packages/cli/test/utils/simulation/runner/DockerRunner.ts index 83188ff7e6e9..2e936ea53360 100644 --- a/packages/cli/test/utils/simulation/runner/DockerRunner.ts +++ b/packages/cli/test/utils/simulation/runner/DockerRunner.ts @@ -81,15 +81,20 @@ export class DockerRunner implements Runner { } async start(): Promise { - await startChildProcess({ - cli: { - command: "docker", - args: ["network", "create", "--subnet", `${dockerNetworkIpRange}.0/24`, dockerNetworkName], - }, - logs: { - stdoutFilePath: this.logFilePath, - }, - }); + try { + await startChildProcess({ + cli: { + command: "docker", + args: ["network", "create", "--subnet", `${dockerNetworkIpRange}.0/24`, dockerNetworkName], + }, + logs: { + stdoutFilePath: this.logFilePath, + }, + }); + } catch (e) { + // During multiple sim tests files the network might already exist + console.error(e); + } } async stop(): Promise { diff --git a/packages/cli/test/utils/simulation/utils/child_process.ts b/packages/cli/test/utils/simulation/utils/child_process.ts index 4de83d0e6752..aef1f4004132 100644 --- a/packages/cli/test/utils/simulation/utils/child_process.ts +++ b/packages/cli/test/utils/simulation/utils/child_process.ts @@ -9,7 +9,7 @@ export const stopChildProcess = async ( childProcess: ChildProcess, signal: NodeJS.Signals | number = "SIGTERM" ): Promise => { - if (childProcess.killed || childProcess.exitCode !== null || childProcess.signalCode !== undefined) { + if (childProcess.killed || childProcess.exitCode !== null || childProcess.signalCode !== null) { return; } diff --git a/packages/cli/test/utils/simulation/utils/el_genesis.ts b/packages/cli/test/utils/simulation/utils/el_genesis.ts index 2cc30db4d0a8..30818850e4e2 100644 --- a/packages/cli/test/utils/simulation/utils/el_genesis.ts +++ b/packages/cli/test/utils/simulation/utils/el_genesis.ts @@ -1,7 +1,7 @@ import {SIM_ENV_CHAIN_ID, SIM_ENV_NETWORK_ID} from "../constants.js"; -import {ELGenesisOptions, ELStartMode, Eth1GenesisBlock} from "../interfaces.js"; +import {ELGeneratorGenesisOptions, ELStartMode, Eth1GenesisBlock} from "../interfaces.js"; -export const getGethGenesisBlock = (mode: ELStartMode, options: ELGenesisOptions): Record => { +export const getGethGenesisBlock = (mode: ELStartMode, options: ELGeneratorGenesisOptions): Record => { const {ttd, cliqueSealingPeriod} = options; const genesis = { @@ -56,7 +56,10 @@ export const getGethGenesisBlock = (mode: ELStartMode, options: ELGenesisOptions return genesis; }; -export const getNethermindChainSpec = (mode: ELStartMode, options: ELGenesisOptions): Record => { +export const getNethermindChainSpec = ( + mode: ELStartMode, + options: ELGeneratorGenesisOptions +): Record => { const {ttd} = options; const genesis = getGethGenesisBlock(mode, options) as Eth1GenesisBlock; diff --git a/packages/cli/test/utils/simulation/utils/index.ts b/packages/cli/test/utils/simulation/utils/index.ts index 9e9088068a85..1f7ee704323b 100644 --- a/packages/cli/test/utils/simulation/utils/index.ts +++ b/packages/cli/test/utils/simulation/utils/index.ts @@ -1,6 +1,5 @@ import {Epoch, Slot} from "@lodestar/types"; -import {ForkName, activePreset} from "@lodestar/params"; -import {IChainForkConfig} from "@lodestar/config"; +import {activePreset} from "@lodestar/params"; import {ETH_TTD_INCREMENT} from "../constants.js"; export const logFilesDir = "test-logs"; @@ -9,16 +8,6 @@ export const avg = (arr: number[]): number => { return arr.length === 0 ? 0 : arr.reduce((p, c) => p + c, 0) / arr.length; }; -export const getForkName = (epoch: Epoch, config: IChainForkConfig): ForkName => { - if (epoch < config.ALTAIR_FORK_EPOCH) { - return ForkName.phase0; - } else if (epoch < config.BELLATRIX_FORK_EPOCH) { - return ForkName.altair; - } else { - return ForkName.bellatrix; - } -}; - export const getEstimatedTimeInSecForRun = ({ genesisSlotDelay, runTill, diff --git a/packages/cli/test/utils/simulation/utils/network.ts b/packages/cli/test/utils/simulation/utils/network.ts index a9ba9f90867f..263a642931fa 100644 --- a/packages/cli/test/utils/simulation/utils/network.ts +++ b/packages/cli/test/utils/simulation/utils/network.ts @@ -1,4 +1,9 @@ +/* eslint-disable no-console */ +import {Slot} from "@lodestar/types"; +import {sleep} from "@lodestar/utils"; import {ELClient, NodePair} from "../interfaces.js"; +import {SimulationEnvironment} from "../SimulationEnvironment.js"; +import {SimulationTrackerEvent} from "../SimulationTracker.js"; export async function connectAllNodes(nodes: NodePair[]): Promise { for (const node of nodes) { @@ -25,3 +30,92 @@ export async function connectNewNode(newNode: NodePair, nodes: NodePair[]): Prom await node.cl.api.lodestar.connectPeer(clIdentity.peerId, clIdentity.p2pAddresses); } } + +export async function waitForNodeSync( + env: SimulationEnvironment, + node: NodePair, + options?: {head: string; slot: Slot} +): Promise { + if (options) { + await Promise.all([waitForNodeSyncStatus(env, node), waitForHead(env, node, options)]); + return; + } + + return waitForNodeSyncStatus(env, node); +} + +export async function waitForNodeSyncStatus(env: SimulationEnvironment, node: NodePair): Promise { + // eslint-disable-next-line no-constant-condition + while (true) { + const result = await node.cl.api.node.getSyncingStatus(); + if (!result.data.isSyncing) { + break; + } else { + await sleep(1000, env.options.controller.signal); + } + } +} + +export async function waitForHead( + env: SimulationEnvironment, + node: NodePair, + options: {slot: Slot; head: string} +): Promise { + return new Promise((resolve) => { + let firstHeadEventSlot: number; + + const cb = (event: {block: string; slot: Slot}): void => { + if (!firstHeadEventSlot) { + firstHeadEventSlot = event.slot; + } + + // The syncing happens quickly and we already crossed the head slot + if (firstHeadEventSlot >= options.slot) { + env.tracker.off(node, SimulationTrackerEvent.Head, cb); + resolve(); + return; + } + + if (event.block === options.head) { + env.tracker.off(node, SimulationTrackerEvent.Head, cb); + resolve(); + return; + } + }; + + env.tracker.on(node, SimulationTrackerEvent.Head, cb); + }); +} + +export async function waitForSlot( + slot: Slot, + nodes: NodePair[], + {silent, env}: {silent?: boolean; env: SimulationEnvironment} +): Promise { + if (!silent) { + console.log(`\nWaiting for slot on "${nodes.map((n) => n.cl.id).join(",")}"`, { + target: slot, + current: env.clock.currentSlot, + }); + } + + await Promise.all( + nodes.map( + (node) => + new Promise((resolve, reject) => { + const cb = (event: {slot: Slot}): void => { + if (slot === event.slot) { + resolve(); + env.tracker.off(node, SimulationTrackerEvent.Slot, cb); + return; + } + + if (event.slot >= slot) { + reject(new Error(`${node.cl.id} had passed target slot ${slot}. Current slot ${event.slot}`)); + } + }; + env.tracker.on(node, SimulationTrackerEvent.Slot, cb); + }) + ) + ); +}