feat: use Uint32Array for shuffling/committees (#6475)

* feat: use Uint32Array for shuffling/committees * chore: address pr comments
ChainSafe · Mar 7, 2024 · 36f50cf · 36f50cf
1 parent 8ad2cb0
commit 36f50cf
Show file tree

Hide file tree

Showing 26 changed files with 61 additions and 54 deletions.
diff --git a/packages/api/package.json b/packages/api/package.json
@@ -68,7 +68,7 @@
   },
   "dependencies": {
     "@chainsafe/persistent-merkle-tree": "^0.6.1",
-    "@chainsafe/ssz": "^0.14.0",
+    "@chainsafe/ssz": "^0.14.3",
     "@lodestar/config": "^1.16.0",
     "@lodestar/params": "^1.16.0",
     "@lodestar/types": "^1.16.0",

diff --git a/packages/api/src/beacon/routes/beacon/state.ts b/packages/api/src/beacon/routes/beacon/state.ts
@@ -66,7 +66,7 @@ export type ValidatorBalance = {
 export type EpochCommitteeResponse = {
   index: CommitteeIndex;
   slot: Slot;
-  validators: ValidatorIndex[];
+  validators: ArrayLike<ValidatorIndex>;
 };
 
 export type EpochSyncCommitteeResponse = {

diff --git a/packages/beacon-node/package.json b/packages/beacon-node/package.json
@@ -102,7 +102,7 @@
     "@chainsafe/libp2p-noise": "^14.1.0",
     "@chainsafe/persistent-merkle-tree": "^0.6.1",
     "@chainsafe/prometheus-gc-stats": "^1.0.0",
-    "@chainsafe/ssz": "^0.14.0",
+    "@chainsafe/ssz": "^0.14.3",
     "@chainsafe/threads": "^1.11.1",
     "@ethersproject/abi": "^5.7.0",
     "@fastify/bearer-auth": "^9.0.0",

diff --git a/packages/beacon-node/src/chain/opPools/aggregatedAttestationPool.ts b/packages/beacon-node/src/chain/opPools/aggregatedAttestationPool.ts
@@ -26,7 +26,7 @@ type AttestationWithScore = {attestation: phase0.Attestation; score: number};
  * This function returns not seen participation for a given epoch and committee.
  * Return null if all validators are seen or no info to check.
  */
-type GetNotSeenValidatorsFn = (epoch: Epoch, committee: number[]) => Set<number> | null;
+type GetNotSeenValidatorsFn = (epoch: Epoch, committee: Uint32Array) => Set<number> | null;
 
 type ValidateAttestationDataFn = (attData: phase0.AttestationData) => boolean;
 
@@ -79,7 +79,7 @@ export class AggregatedAttestationPool {
     attestation: phase0.Attestation,
     dataRootHex: RootHex,
     attestingIndicesCount: number,
-    committee: ValidatorIndex[]
+    committee: Uint32Array
   ): InsertOutcome {
     const slot = attestation.data.slot;
     const lowestPermissibleSlot = this.lowestPermissibleSlot;
@@ -271,7 +271,7 @@ export class MatchingDataAttestationGroup {
 
   constructor(
     // TODO: no need committee here
-    readonly committee: ValidatorIndex[],
+    readonly committee: Uint32Array,
     readonly data: phase0.AttestationData
   ) {}
 
@@ -398,7 +398,7 @@ export function getNotSeenValidatorsFn(state: CachedBeaconStateAllForks): GetNot
       state
     );
 
-    return (epoch: Epoch, committee: number[]) => {
+    return (epoch: Epoch, committee: Uint32Array) => {
       const participants =
         epoch === stateEpoch ? currentEpochParticipants : epoch === stateEpoch - 1 ? previousEpochParticipants : null;
       if (participants === null) {
@@ -426,7 +426,7 @@ export function getNotSeenValidatorsFn(state: CachedBeaconStateAllForks): GetNot
     const currentParticipation = altairState.currentEpochParticipation.getAll();
     const stateEpoch = computeEpochAtSlot(state.slot);
 
-    return (epoch: Epoch, committee: number[]) => {
+    return (epoch: Epoch, committee: Uint32Array) => {
       const participationStatus =
         epoch === stateEpoch ? currentParticipation : epoch === stateEpoch - 1 ? previousParticipation : null;
 

diff --git a/packages/beacon-node/src/chain/seenCache/seenAttestationData.ts b/packages/beacon-node/src/chain/seenCache/seenAttestationData.ts
@@ -6,7 +6,7 @@ import {InsertOutcome} from "../opPools/types.js";
 
 export type AttestationDataCacheEntry = {
   // part of shuffling data, so this does not take memory
-  committeeIndices: number[];
+  committeeIndices: Uint32Array;
   // IndexedAttestationData signing root, 32 bytes
   signingRoot: Uint8Array;
   // to be consumed by forkchoice and oppool

diff --git a/packages/beacon-node/src/chain/validation/aggregateAndProof.ts b/packages/beacon-node/src/chain/validation/aggregateAndProof.ts
@@ -1,6 +1,6 @@
 import {toHexString} from "@chainsafe/ssz";
 import {ForkName} from "@lodestar/params";
-import {phase0, RootHex, ssz, ValidatorIndex} from "@lodestar/types";
+import {phase0, RootHex, ssz} from "@lodestar/types";
 import {
   computeEpochAtSlot,
   isAggregatorFromCommitteeLength,
@@ -21,7 +21,7 @@ import {
 
 export type AggregateAndProofValidationResult = {
   indexedAttestation: phase0.IndexedAttestation;
-  committeeIndices: ValidatorIndex[];
+  committeeIndices: Uint32Array;
   attDataRootHex: RootHex;
 };
 
@@ -148,7 +148,7 @@ async function validateAggregateAndProof(
     RegenCaller.validateGossipAttestation
   );
 
-  const committeeIndices: number[] = cachedAttData
+  const committeeIndices = cachedAttData
     ? cachedAttData.committeeIndices
     : getCommitteeIndices(shuffling, attSlot, attIndex);
 

diff --git a/packages/beacon-node/src/chain/validation/attestation.ts b/packages/beacon-node/src/chain/validation/attestation.ts
@@ -319,7 +319,7 @@ async function validateGossipAttestationNoSignatureCheck(
     });
   }
 
-  let committeeIndices: number[];
+  let committeeIndices: Uint32Array;
   let getSigningRoot: () => Uint8Array;
   let expectedSubnet: number;
   if (attestationOrCache.cache) {
@@ -702,7 +702,7 @@ export function getCommitteeIndices(
   shuffling: EpochShuffling,
   attestationSlot: Slot,
   attestationIndex: number
-): number[] {
+): Uint32Array {
   const {committees} = shuffling;
   const slotCommittees = committees[attestationSlot % SLOTS_PER_EPOCH];
 

diff --git a/packages/beacon-node/test/fixtures/phase0.ts b/packages/beacon-node/test/fixtures/phase0.ts
@@ -20,7 +20,7 @@ export function generateIndexedAttestations(
 
     for (let committeeIndex = 0; committeeIndex < committeeCount; committeeIndex++) {
       result.push({
-        attestingIndices: state.epochCtx.getBeaconCommittee(slot, committeeIndex),
+        attestingIndices: Array.from(state.epochCtx.getBeaconCommittee(slot, committeeIndex)),
         data: {
           slot: slot,
           index: committeeIndex,

diff --git a/packages/beacon-node/test/unit/chain/opPools/aggregatedAttestationPool.test.ts b/packages/beacon-node/test/unit/chain/opPools/aggregatedAttestationPool.test.ts
@@ -181,7 +181,7 @@ describe("MatchingDataAttestationGroup.add()", () => {
   ];
 
   const attestationData = ssz.phase0.AttestationData.defaultValue();
-  const committee = linspace(0, 7);
+  const committee = Uint32Array.from(linspace(0, 7));
 
   for (const {id, attestationsToAdd} of testCases) {
     it(id, () => {
@@ -251,7 +251,7 @@ describe("MatchingDataAttestationGroup.getAttestationsForBlock", () => {
   ];
 
   const attestationData = ssz.phase0.AttestationData.defaultValue();
-  const committee = linspace(0, 7);
+  const committee = Uint32Array.from(linspace(0, 7));
 
   for (const {id, notSeenAttestingBits, attestationsToAdd} of testCases) {
     it(id, () => {

diff --git a/packages/beacon-node/test/unit/chain/stateCache/fifoBlockStateCache.test.ts b/packages/beacon-node/test/unit/chain/stateCache/fifoBlockStateCache.test.ts
@@ -10,8 +10,8 @@ describe("FIFOBlockStateCache", function () {
   let cache: FIFOBlockStateCache;
   const shuffling: EpochShuffling = {
     epoch: 0,
-    activeIndices: [],
-    shuffling: [],
+    activeIndices: new Uint32Array(),
+    shuffling: new Uint32Array(),
     committees: [],
     committeesPerSlot: 1,
   };

diff --git a/packages/beacon-node/test/unit/chain/stateCache/stateContextCache.test.ts b/packages/beacon-node/test/unit/chain/stateCache/stateContextCache.test.ts
@@ -12,8 +12,8 @@ describe("StateContextCache", function () {
   let key1: Root, key2: Root;
   const shuffling: EpochShuffling = {
     epoch: 0,
-    activeIndices: [],
-    shuffling: [],
+    activeIndices: new Uint32Array(),
+    shuffling: new Uint32Array(),
     committees: [],
     committeesPerSlot: 1,
   };

diff --git a/packages/cli/package.json b/packages/cli/package.json
@@ -58,7 +58,7 @@
     "@chainsafe/discv5": "^9.0.0",
     "@chainsafe/enr": "^3.0.0",
     "@chainsafe/persistent-merkle-tree": "^0.6.1",
-    "@chainsafe/ssz": "^0.14.0",
+    "@chainsafe/ssz": "^0.14.3",
     "@chainsafe/threads": "^1.11.1",
     "@libp2p/crypto": "^3.0.4",
     "@libp2p/peer-id": "^4.0.4",

diff --git a/packages/config/package.json b/packages/config/package.json
@@ -64,7 +64,7 @@
     "blockchain"
   ],
   "dependencies": {
-    "@chainsafe/ssz": "^0.14.0",
+    "@chainsafe/ssz": "^0.14.3",
     "@lodestar/params": "^1.16.0",
     "@lodestar/types": "^1.16.0"
   }

diff --git a/packages/db/package.json b/packages/db/package.json
@@ -35,7 +35,7 @@
     "check-readme": "typescript-docs-verifier"
   },
   "dependencies": {
-    "@chainsafe/ssz": "^0.14.0",
+    "@chainsafe/ssz": "^0.14.3",
     "@lodestar/config": "^1.16.0",
     "@lodestar/utils": "^1.16.0",
     "it-all": "^3.0.4",

diff --git a/packages/fork-choice/package.json b/packages/fork-choice/package.json
@@ -36,7 +36,7 @@
     "check-readme": "typescript-docs-verifier"
   },
   "dependencies": {
-    "@chainsafe/ssz": "^0.14.0",
+    "@chainsafe/ssz": "^0.14.3",
     "@lodestar/config": "^1.16.0",
     "@lodestar/params": "^1.16.0",
     "@lodestar/state-transition": "^1.16.0",

diff --git a/packages/light-client/package.json b/packages/light-client/package.json
@@ -67,7 +67,7 @@
   "dependencies": {
     "@chainsafe/bls": "7.1.3",
     "@chainsafe/persistent-merkle-tree": "^0.6.1",
-    "@chainsafe/ssz": "^0.14.0",
+    "@chainsafe/ssz": "^0.14.3",
     "@lodestar/api": "^1.16.0",
     "@lodestar/config": "^1.16.0",
     "@lodestar/params": "^1.16.0",

diff --git a/packages/state-transition/package.json b/packages/state-transition/package.json
@@ -63,7 +63,7 @@
     "@chainsafe/blst": "^0.2.10",
     "@chainsafe/persistent-merkle-tree": "^0.6.1",
     "@chainsafe/persistent-ts": "^0.19.1",
-    "@chainsafe/ssz": "^0.14.0",
+    "@chainsafe/ssz": "^0.14.3",
     "@lodestar/config": "^1.16.0",
     "@lodestar/params": "^1.16.0",
     "@lodestar/types": "^1.16.0",

diff --git a/packages/state-transition/src/cache/epochCache.ts b/packages/state-transition/src/cache/epochCache.ts
@@ -584,7 +584,7 @@ export class EpochCache {
   /**
    * Return the beacon committee at slot for index.
    */
-  getBeaconCommittee(slot: Slot, index: CommitteeIndex): ValidatorIndex[] {
+  getBeaconCommittee(slot: Slot, index: CommitteeIndex): Uint32Array {
     const slotCommittees = this.getShufflingAtSlot(slot).committees[slot % SLOTS_PER_EPOCH];
     if (index >= slotCommittees.length) {
       throw new EpochCacheError({
@@ -745,7 +745,7 @@ export class EpochCache {
         const committee = this.getBeaconCommittee(slot, i);
         if (committee.includes(validatorIndex)) {
           return {
-            validators: committee,
+            validators: Array.from(committee),
             committeeIndex: i,
             slot,
           };

diff --git a/packages/state-transition/src/util/epochShuffling.ts b/packages/state-transition/src/util/epochShuffling.ts
@@ -30,12 +30,12 @@ export type EpochShuffling = {
   /**
    * Non-shuffled active validator indices
    */
-  activeIndices: ValidatorIndex[];
+  activeIndices: Uint32Array;
 
   /**
    * The active validator indices, shuffled into their committee
    */
-  shuffling: ValidatorIndex[];
+  shuffling: Uint32Array;
 
   /**
    * List of list of committees Committees
@@ -45,7 +45,7 @@ export type EpochShuffling = {
    * Note: With a high amount of shards, or low amount of validators,
    * some shards may not have a committee this epoch
    */
-  committees: ValidatorIndex[][][];
+  committees: Uint32Array[][];
 
   /**
    * Committees per slot, for fast attestation verification
@@ -61,38 +61,39 @@ export function computeCommitteeCount(activeValidatorCount: number): number {
 
 export function computeEpochShuffling(
   state: BeaconStateAllForks,
-  activeIndices: ValidatorIndex[],
+  activeIndices: ArrayLike<ValidatorIndex>,
   epoch: Epoch
 ): EpochShuffling {
   const seed = getSeed(state, epoch, DOMAIN_BEACON_ATTESTER);
 
   // copy
-  const shuffling = activeIndices.slice();
+  const _activeIndices = new Uint32Array(activeIndices);
+  const shuffling = _activeIndices.slice();
   unshuffleList(shuffling, seed);
 
   const activeValidatorCount = activeIndices.length;
   const committeesPerSlot = computeCommitteeCount(activeValidatorCount);
 
   const committeeCount = committeesPerSlot * SLOTS_PER_EPOCH;
 
-  const committees: ValidatorIndex[][][] = [];
+  const committees: Uint32Array[][] = [];
   for (let slot = 0; slot < SLOTS_PER_EPOCH; slot++) {
-    const slotCommittees: ValidatorIndex[][] = [];
+    const slotCommittees: Uint32Array[] = [];
     for (let committeeIndex = 0; committeeIndex < committeesPerSlot; committeeIndex++) {
       const index = slot * committeesPerSlot + committeeIndex;
       const startOffset = Math.floor((activeValidatorCount * index) / committeeCount);
       const endOffset = Math.floor((activeValidatorCount * (index + 1)) / committeeCount);
       if (!(startOffset <= endOffset)) {
         throw new Error(`Invalid offsets: start ${startOffset} must be less than or equal end ${endOffset}`);
       }
-      slotCommittees.push(shuffling.slice(startOffset, endOffset));
+      slotCommittees.push(shuffling.subarray(startOffset, endOffset));
     }
     committees.push(slotCommittees);
   }
 
   return {
     epoch,
-    activeIndices,
+    activeIndices: _activeIndices,
     shuffling,
     committees,
     committeesPerSlot,

diff --git a/packages/state-transition/src/util/seed.ts b/packages/state-transition/src/util/seed.ts
@@ -21,7 +21,7 @@ import {computeEpochAtSlot} from "./epoch.js";
  */
 export function computeProposers(
   epochSeed: Uint8Array,
-  shuffling: {epoch: Epoch; activeIndices: ValidatorIndex[]},
+  shuffling: {epoch: Epoch; activeIndices: ArrayLike<ValidatorIndex>},
   effectiveBalanceIncrements: EffectiveBalanceIncrements
 ): number[] {
   const startSlot = computeStartSlotAtEpoch(shuffling.epoch);
@@ -45,7 +45,7 @@ export function computeProposers(
  */
 export function computeProposerIndex(
   effectiveBalanceIncrements: EffectiveBalanceIncrements,
-  indices: ValidatorIndex[],
+  indices: ArrayLike<ValidatorIndex>,
   seed: Uint8Array
 ): ValidatorIndex {
   if (indices.length === 0) {
@@ -91,7 +91,7 @@ export function computeProposerIndex(
  */
 export function getNextSyncCommitteeIndices(
   state: BeaconStateAllForks,
-  activeValidatorIndices: ValidatorIndex[],
+  activeValidatorIndices: ArrayLike<ValidatorIndex>,
   effectiveBalanceIncrements: EffectiveBalanceIncrements
 ): ValidatorIndex[] {
   // TODO: Bechmark if it's necessary to inline outside of this function

diff --git a/packages/state-transition/src/util/shuffle.ts b/packages/state-transition/src/util/shuffle.ts
@@ -1,15 +1,21 @@
 import {digest} from "@chainsafe/as-sha256";
 import {SHUFFLE_ROUND_COUNT} from "@lodestar/params";
-import {ValidatorIndex, Bytes32} from "@lodestar/types";
+import {Bytes32} from "@lodestar/types";
 import {assert, bytesToBigInt} from "@lodestar/utils";
 
+// ArrayLike<number> but with settable indices
+type Shuffleable = {
+  readonly length: number;
+  [index: number]: number;
+};
+
 // ShuffleList shuffles a list, using the given seed for randomness. Mutates the input list.
-export function shuffleList(input: ValidatorIndex[], seed: Bytes32): void {
+export function shuffleList(input: Shuffleable, seed: Bytes32): void {
   innerShuffleList(input, seed, true);
 }
 
 // UnshuffleList undoes a list shuffling using the seed of the shuffling. Mutates the input list.
-export function unshuffleList(input: ValidatorIndex[], seed: Bytes32): void {
+export function unshuffleList(input: Shuffleable, seed: Bytes32): void {
   innerShuffleList(input, seed, false);
 }
 
@@ -70,7 +76,7 @@ function setPositionUint32(value: number, buf: Buffer): void {
 }
 
 // Shuffles or unshuffles, depending on the `dir` (true for shuffling, false for unshuffling
-function innerShuffleList(input: ValidatorIndex[], seed: Bytes32, dir: boolean): void {
+function innerShuffleList(input: Shuffleable, seed: Bytes32, dir: boolean): void {
   if (input.length <= 1) {
     // nothing to (un)shuffle
     return;

diff --git a/packages/state-transition/src/util/syncCommittee.ts b/packages/state-transition/src/util/syncCommittee.ts
@@ -20,7 +20,7 @@ import {getNextSyncCommitteeIndices} from "./seed.js";
  */
 export function getNextSyncCommittee(
   state: BeaconStateAllForks,
-  activeValidatorIndices: ValidatorIndex[],
+  activeValidatorIndices: ArrayLike<ValidatorIndex>,
   effectiveBalanceIncrements: EffectiveBalanceIncrements
 ): {indices: ValidatorIndex[]; syncCommittee: altair.SyncCommittee} {
   const indices = getNextSyncCommitteeIndices(state, activeValidatorIndices, effectiveBalanceIncrements);