Loop thru validators exactly 1 time during epoch transition (#2871)

packages/beacon-state-transition/src/allForks/stateTransition.ts

@@ -144,7 +144,7 @@ function processSlotsWithTransientCache(
         metrics?.registerValidatorStatuses(process.currentEpoch, process.statuses);
 
         postState.slot++;
-        rotateEpochs(postState.epochCtx, postState, process.indicesBounded);
+        rotateEpochs(postState.epochCtx, postState, process.nextEpochActiveValidatorIndices);
       } finally {
         if (timer) timer();
       }

packages/beacon-state-transition/src/allForks/util/cachedBeaconState.ts

@@ -215,7 +215,9 @@ export class BeaconStateContext<T extends allForks.BeaconState> {
     const state = (this.type.createTreeBacked(this.tree) as unknown) as TreeBacked<altair.BeaconState>;
     this.currentSyncCommittee = this.nextSyncCommittee;
     state.currentSyncCommittee = state.nextSyncCommittee;
-    const nextSyncCommittee = ssz.altair.SyncCommittee.createTreeBackedFromStruct(getNextSyncCommittee(state));
+    const nextSyncCommittee = ssz.altair.SyncCommittee.createTreeBackedFromStruct(
+      getNextSyncCommittee(state, this.epochCtx.nextShuffling.activeIndices)
+    );
     this.nextSyncCommittee = convertToIndexedSyncCommittee(nextSyncCommittee, this.epochCtx.pubkey2index);
     state.nextSyncCommittee = nextSyncCommittee;
   }

packages/beacon-state-transition/src/allForks/util/epochContext.ts

@@ -30,7 +30,8 @@ import {
   computeProposerIndex,
   computeStartSlotAtEpoch,
   getSeed,
-  getTotalActiveBalance,
+  getTotalBalance,
+  isActiveValidator,
   isAggregatorFromCommitteeLength,
   zipIndexesCommitteeBits,
 } from "../../util";
@@ -100,29 +101,38 @@ export function createEpochContext(
   const previousEpoch = currentEpoch === GENESIS_EPOCH ? GENESIS_EPOCH : currentEpoch - 1;
   const nextEpoch = currentEpoch + 1;
 
-  const indicesBounded: [ValidatorIndex, Epoch, Epoch][] = validators.map((v, i) => [
-    i,
-    v.activationEpoch,
-    v.exitEpoch,
-  ]);
+  const previousActiveIndices: ValidatorIndex[] = [];
+  const currentActiveIndices: ValidatorIndex[] = [];
+  const nextActiveIndices: ValidatorIndex[] = [];
+  validators.forEach(function processActiveIndices(v, i) {
+    if (isActiveValidator(v, previousEpoch)) {
+      previousActiveIndices.push(i);
+    }
+    if (isActiveValidator(v, currentEpoch)) {
+      currentActiveIndices.push(i);
+    }
+    if (isActiveValidator(v, nextEpoch)) {
+      nextActiveIndices.push(i);
+    }
+  });
 
-  const currentShuffling = computeEpochShuffling(state, indicesBounded, currentEpoch);
+  const currentShuffling = computeEpochShuffling(state, currentActiveIndices, currentEpoch);
   let previousShuffling;
   if (previousEpoch === currentEpoch) {
     // in case of genesis
     previousShuffling = currentShuffling;
   } else {
-    previousShuffling = computeEpochShuffling(state, indicesBounded, previousEpoch);
+    previousShuffling = computeEpochShuffling(state, previousActiveIndices, previousEpoch);
   }
-  const nextShuffling = computeEpochShuffling(state, indicesBounded, nextEpoch);
+  const nextShuffling = computeEpochShuffling(state, nextActiveIndices, nextEpoch);
 
   // Allow to create CachedBeaconState for empty states
   const proposers = state.validators.length > 0 ? computeProposers(state, currentShuffling) : [];
 
   // Only after altair, compute the indices of the current sync committee
   const onAltairFork = currentEpoch >= config.ALTAIR_FORK_EPOCH;
 
-  const totalActiveBalance = getTotalActiveBalance(state);
+  const totalActiveBalance = getTotalBalance(state, currentShuffling.activeIndices);
   const syncParticipantReward = onAltairFork ? computeSyncParticipantReward(config, totalActiveBalance) : BigInt(0);
   const syncProposerReward = onAltairFork
     ? (syncParticipantReward * PROPOSER_WEIGHT) / (WEIGHT_DENOMINATOR - PROPOSER_WEIGHT)
@@ -203,17 +213,17 @@ export function computeSyncParticipantReward(config: IBeaconConfig, totalActiveB
 export function rotateEpochs(
   epochCtx: EpochContext,
   state: allForks.BeaconState,
-  indicesBounded: [ValidatorIndex, Epoch, Epoch][]
+  activeValidatorIndices: ValidatorIndex[]
 ): void {
   epochCtx.previousShuffling = epochCtx.currentShuffling;
   epochCtx.currentShuffling = epochCtx.nextShuffling;
   const currEpoch = epochCtx.currentShuffling.epoch;
   const nextEpoch = currEpoch + 1;
-  epochCtx.nextShuffling = computeEpochShuffling(state, indicesBounded, nextEpoch);
+  epochCtx.nextShuffling = computeEpochShuffling(state, activeValidatorIndices, nextEpoch);
   epochCtx.proposers = computeProposers(state, epochCtx.currentShuffling);
 
   if (currEpoch >= epochCtx.config.ALTAIR_FORK_EPOCH) {
-    const totalActiveBalance = getTotalActiveBalance(state);
+    const totalActiveBalance = getTotalBalance(state, activeValidatorIndices);
     epochCtx.syncParticipantReward = computeSyncParticipantReward(epochCtx.config, totalActiveBalance);
     epochCtx.syncProposerReward =
       (epochCtx.syncParticipantReward * PROPOSER_WEIGHT) / (WEIGHT_DENOMINATOR - PROPOSER_WEIGHT);

packages/beacon-state-transition/src/allForks/util/epochProcess.ts

@@ -55,7 +55,7 @@ export interface IEpochProcess {
   validators: phase0.Validator[];
   balances?: BigUint64Array;
   // to be used for rotateEpochs()
-  indicesBounded: [ValidatorIndex, Epoch, Epoch][];
+  nextEpochActiveValidatorIndices: ValidatorIndex[];
 }
 
 export function createIEpochProcess(): IEpochProcess {
@@ -79,7 +79,7 @@ export function createIEpochProcess(): IEpochProcess {
     churnLimit: 0,
     statuses: [],
     validators: [],
-    indicesBounded: [],
+    nextEpochActiveValidatorIndices: [],
   };
 }
 
@@ -90,6 +90,7 @@ export function prepareEpochProcessState<T extends allForks.BeaconState>(state:
   const forkName = config.getForkName(state.slot);
   const currentEpoch = epochCtx.currentShuffling.epoch;
   const prevEpoch = epochCtx.previousShuffling.epoch;
+  const nextEpoch = currentEpoch + 1;
   out.currentEpoch = currentEpoch;
   out.prevEpoch = prevEpoch;
 
@@ -144,7 +145,9 @@ export function prepareEpochProcessState<T extends allForks.BeaconState>(state:
     }
 
     out.statuses.push(status);
-    out.indicesBounded.push([i, v.activationEpoch, v.exitEpoch]);
+    if (isActiveValidator(v, nextEpoch)) {
+      out.nextEpochActiveValidatorIndices.push(i);
+    }
   });
 
   if (out.totalActiveStake < EFFECTIVE_BALANCE_INCREMENT) {

packages/beacon-state-transition/src/allForks/util/epochShuffling.ts

@@ -57,18 +57,11 @@ export function computeCommitteeCount(activeValidatorCount: number): number {
 
 export function computeEpochShuffling(
   state: allForks.BeaconState,
-  indicesBounded: [ValidatorIndex, Epoch, Epoch][],
+  activeIndices: ValidatorIndex[],
   epoch: Epoch
 ): IEpochShuffling {
   const seed = getSeed(state, epoch, DOMAIN_BEACON_ATTESTER);
 
-  const activeIndices: ValidatorIndex[] = [];
-  for (const [index, activationEpoch, exitEpoch] of indicesBounded) {
-    if (activationEpoch <= epoch && epoch < exitEpoch) {
-      activeIndices.push(index);
-    }
-  }
-
   // copy
   const shuffling = activeIndices.slice();
   unshuffleList(shuffling, seed);

packages/beacon-state-transition/src/altair/epoch/sync_committee.ts

@@ -4,22 +4,22 @@ import {altair, ValidatorIndex, allForks} from "@chainsafe/lodestar-types";
 import {intDiv, intToBytes} from "@chainsafe/lodestar-utils";
 import {hash} from "@chainsafe/ssz";
 
-import {computeEpochAtSlot, computeShuffledIndex, getActiveValidatorIndices, getSeed} from "../../util";
+import {computeEpochAtSlot, computeShuffledIndex, getSeed} from "../../util";
 
 const MAX_RANDOM_BYTE = BigInt(2 ** 8 - 1);
 
 /**
- * TODO: NAIVE
- * 
  * Return the sync committee indices for a given state and epoch.
  * Aligns `epoch` to `baseEpoch` so the result is the same with any `epoch` within a sync period.
  *  Note: This function should only be called at sync committee period boundaries, as
     ``get_sync_committee_indices`` is not stable within a given period.
  */
-export function getNextSyncCommitteeIndices(state: allForks.BeaconState): ValidatorIndex[] {
+export function getNextSyncCommitteeIndices(
+  state: allForks.BeaconState,
+  activeValidatorIndices: ValidatorIndex[]
+): ValidatorIndex[] {
   const epoch = computeEpochAtSlot(state.slot) + 1;
 
-  const activeValidatorIndices = getActiveValidatorIndices(state, epoch);
   const activeValidatorCount = activeValidatorIndices.length;
   const seed = getSeed(state, epoch, DOMAIN_SYNC_COMMITTEE);
   let i = 0;
@@ -38,12 +38,13 @@ export function getNextSyncCommitteeIndices(state: allForks.BeaconState): Valida
 }
 
 /**
- * TODO: NAIVE
- *
  * Return the sync committee for a given state and epoch.
  */
-export function getNextSyncCommittee(state: allForks.BeaconState): altair.SyncCommittee {
-  const indices = getNextSyncCommitteeIndices(state);
+export function getNextSyncCommittee(
+  state: allForks.BeaconState,
+  activeValidatorIndices: ValidatorIndex[]
+): altair.SyncCommittee {
+  const indices = getNextSyncCommitteeIndices(state, activeValidatorIndices);
   const pubkeys = indices.map((index) => state.validators[index].pubkey);
   return {
     pubkeys,

packages/beacon-state-transition/src/altair/upgradeState.ts

@@ -1,4 +1,4 @@
-import {altair, ParticipationFlags, phase0, ssz, Uint8} from "@chainsafe/lodestar-types";
+import {altair, ParticipationFlags, phase0, ssz, Uint8, ValidatorIndex} from "@chainsafe/lodestar-types";
 import {CachedBeaconState, createCachedBeaconState} from "../allForks/util";
 import {getCurrentEpoch, newZeroedArray} from "../util";
 import {List, TreeBacked} from "@chainsafe/ssz";
@@ -13,15 +13,21 @@ import {getNextSyncCommittee} from "./epoch/sync_committee";
 export function upgradeState(state: CachedBeaconState<phase0.BeaconState>): CachedBeaconState<altair.BeaconState> {
   const {config} = state;
   const pendingAttesations = Array.from(state.previousEpochAttestations);
-  const postTreeBackedState = upgradeTreeBackedState(config, ssz.phase0.BeaconState.createTreeBacked(state.tree));
+  const nextEpochActiveIndices = state.nextShuffling.activeIndices;
+  const postTreeBackedState = upgradeTreeBackedState(
+    config,
+    ssz.phase0.BeaconState.createTreeBacked(state.tree),
+    nextEpochActiveIndices
+  );
   const postState = createCachedBeaconState(config, postTreeBackedState);
   translateParticipation(postState, pendingAttesations);
   return postState;
 }
 
 function upgradeTreeBackedState(
   config: IBeaconConfig,
-  state: TreeBacked<phase0.BeaconState>
+  state: TreeBacked<phase0.BeaconState>,
+  nextEpochActiveIndices: ValidatorIndex[]
 ): TreeBacked<altair.BeaconState> {
   const validatorCount = state.validators.length;
   const epoch = getCurrentEpoch(state);
@@ -34,7 +40,7 @@ function upgradeTreeBackedState(
   postState.previousEpochParticipation = newZeroedArray(validatorCount) as List<ParticipationFlags>;
   postState.currentEpochParticipation = newZeroedArray(validatorCount) as List<ParticipationFlags>;
   postState.inactivityScores = newZeroedArray(validatorCount) as List<Uint8>;
-  const syncCommittee = getNextSyncCommittee(state);
+  const syncCommittee = getNextSyncCommittee(state, nextEpochActiveIndices);
   postState.currentSyncCommittee = syncCommittee;
   postState.nextSyncCommittee = syncCommittee;
   return postState;

packages/beacon-state-transition/src/util/balance.ts

@@ -24,6 +24,7 @@ export function getTotalBalance(state: allForks.BeaconState, indices: ValidatorI
 }
 
 /**
+ * Call this function with care since it has to loop through validators which is expensive.
  * Return the combined effective balance of the active validators.
  * Note: `getTotalBalance` returns `EFFECTIVE_BALANCE_INCREMENT` Gwei minimum to avoid divisions by zero.
  */

packages/beacon-state-transition/src/util/genesis.ts

@@ -7,7 +7,7 @@ import {
   GENESIS_SLOT,
   MAX_EFFECTIVE_BALANCE,
 } from "@chainsafe/lodestar-params";
-import {allForks, altair, Bytes32, Number64, phase0, Root, ssz} from "@chainsafe/lodestar-types";
+import {allForks, altair, Bytes32, Number64, phase0, Root, ssz, ValidatorIndex} from "@chainsafe/lodestar-types";
 import {bigIntMin} from "@chainsafe/lodestar-utils";
 
 import {computeEpochAtSlot} from "./epoch";
@@ -112,13 +112,14 @@ export function applyTimestamp(
  * @param state BeaconState
  * @param newDeposits new deposits
  * @param fullDepositDataRootList full list of deposit data root from index 0
+ * @returns active validator indices
  */
 export function applyDeposits(
   config: IChainForkConfig,
   state: CachedBeaconState<allForks.BeaconState>,
   newDeposits: phase0.Deposit[],
   fullDepositDataRootList?: TreeBacked<List<Root>>
-): void {
+): ValidatorIndex[] {
   const depositDataRootList: Root[] = [];
   if (fullDepositDataRootList) {
     for (let index = 0; index < state.eth1Data.depositCount; index++) {
@@ -147,6 +148,7 @@ export function applyDeposits(
     processDeposit(forkName, state, deposit);
   }
 
+  const activeValidatorIndices: ValidatorIndex[] = [];
   // Process activations
   // validators are edited, so we're not iterating (read-only) through the validators
   const validatorLength = state.validators.length;
@@ -158,6 +160,7 @@ export function applyDeposits(
     if (validator.effectiveBalance === MAX_EFFECTIVE_BALANCE) {
       validator.activationEligibilityEpoch = computeEpochAtSlot(GENESIS_SLOT);
       validator.activationEpoch = computeEpochAtSlot(GENESIS_SLOT);
+      activeValidatorIndices.push(index);
     }
     // If state is a CachedBeaconState<> validator has to be re-assigned manually
     state.validators[index] = validator;
@@ -167,6 +170,7 @@ export function applyDeposits(
   state.genesisValidatorsRoot = config
     .getForkTypes(state.slot)
     .BeaconState.fields.validators.hashTreeRoot(state.validators);
+  return activeValidatorIndices;
 }
 
 /**
@@ -194,10 +198,10 @@ export function initializeBeaconStateFromEth1(
   applyEth1BlockHash(state, eth1BlockHash);
 
   // Process deposits
-  applyDeposits(config, state, deposits);
+  const activeValidatorIndices = applyDeposits(config, state, deposits);
 
   if (config.getForkName(GENESIS_SLOT) === ForkName.altair) {
-    const syncCommittees = getNextSyncCommittee(state);
+    const syncCommittees = getNextSyncCommittee(state, activeValidatorIndices);
     const altairState = state as TreeBacked<altair.BeaconState>;
     altairState.currentSyncCommittee = syncCommittees;
     altairState.nextSyncCommittee = syncCommittees;

packages/beacon-state-transition/test/perf/util.ts

@@ -181,7 +181,9 @@ export function generatePerformanceStateAltair(pubkeysArg?: Uint8Array[]): TreeB
       ParticipationFlags
     >;
     state.inactivityScores = Array.from({length: pubkeys.length}, (_, i) => i % 2) as List<ParticipationFlags>;
-    const syncCommittee = getNextSyncCommittee(state);
+    const epoch = computeEpochAtSlot(state.slot);
+    const activeValidatorIndices = getActiveValidatorIndices(state, epoch);
+    const syncCommittee = getNextSyncCommittee(state, activeValidatorIndices);
     state.currentSyncCommittee = syncCommittee;
     state.nextSyncCommittee = syncCommittee;
     altairState = ssz.altair.BeaconState.createTreeBackedFromStruct(state);

packages/lodestar/src/api/impl/beacon/state/utils.ts

@@ -6,6 +6,7 @@ import {
   createCachedBeaconState,
   computeSyncPeriodAtEpoch,
   computeEpochAtSlot,
+  isActiveValidator,
 } from "@chainsafe/lodestar-beacon-state-transition";
 import {altair, phase0} from "@chainsafe/lodestar-types";
 import {allForks} from "@chainsafe/lodestar-beacon-state-transition";
@@ -131,12 +132,13 @@ export function getEpochBeaconCommittees(
     }
   }
 
-  const indicesBounded: [ValidatorIndex, Epoch, Epoch][] = Array.from(readonlyValues(state.validators), (v, i) => [
-    i,
-    v.activationEpoch,
-    v.exitEpoch,
-  ]);
-  const shuffling = allForks.computeEpochShuffling(state, indicesBounded, epoch);
+  const activeValidatorIndices: ValidatorIndex[] = [];
+  for (const [i, v] of Array.from(readonlyValues(state.validators)).entries()) {
+    if (isActiveValidator(v, epoch)) {
+      activeValidatorIndices.push(i);
+    }
+  }
+  const shuffling = allForks.computeEpochShuffling(state, activeValidatorIndices, epoch);
   return shuffling.committees;
 }