Skip to content

Commit

Permalink
Add LightclientUpdater to BeaconChain (#2584)
Browse files Browse the repository at this point in the history 
* Add LightclientUpdater to BeaconChain

* Integrate LightClientUpdater with persistent DB

* Update MockBeaconChain

* Update LightclientFinalizedCheckpoint ssz type

* Fix dev handler arg parsing
  • Loading branch information
@dapplion
dapplion committed Jun 1, 2021
1 parent 8f4e720 commit 13bb120
Show file tree
Hide file tree
Showing 20 changed files with 273 additions and 84 deletions.
9 changes: 5 additions & 4 deletions packages/cli/src/cmds/dev/handler.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -95,16 +95,17 @@ export async function devHandler(args: IDevArgs & IGlobalArgs): Promise<void> {
if (args.startValidators) {
const secretKeys: SecretKey[] = [];
const [fromIndex, toIndex] = args.startValidators.split(":").map((s) => parseInt(s));
const maxIndex = anchorState.validators.length - 1;

if (fromIndex > toIndex) {
throw Error(`Invalid startValidators arg - fromIndex > toIndex: ${args.startValidators}`);
throw Error(`Invalid startValidators arg '${args.startValidators}' - fromIndex > toIndex`);
}

if (toIndex >= anchorState.validators.length) {
throw Error("Invalid startValidators arg - toIndex > state.validators.length");
if (toIndex > maxIndex) {
throw Error(`Invalid startValidators arg '${args.startValidators}' - state has ${maxIndex} validators`);
}

for (let i = fromIndex; i < toIndex; i++) {
for (let i = fromIndex; i <= toIndex; i++) {
secretKeys.push(interopSecretKey(i));
}

Expand Down
8 changes: 8 additions & 0 deletions packages/db/src/schema.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,14 @@ export enum Bucket {
allForks_pendingBlock = 25, // Root -> SignedBeaconBlock

index_stateArchiveRootIndex = 26, // State Root -> slot

// Lightclient server
altair_bestUpdatePerCommitteePeriod = 30, // SyncPeriod -> LightClientUpdate
// TODO: Review if it's really necessary to persist these two
altair_latestFinalizedUpdate = 31, // Single: LightClientUpdate
altair_latestNonFinalizedUpdate = 32, // Single: LightClientUpdate
// TODO: Review if it's really necessary
altair_lightclientFinalizedCheckpoint = 33, // Epoch -> FinalizedCheckpointData
}

export enum Key {
Expand Down
63 changes: 36 additions & 27 deletions packages/light-client/src/server/LightClientUpdater.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,12 +8,11 @@ import {
getForkVersion,
} from "@chainsafe/lodestar-beacon-state-transition";
import {FINALIZED_ROOT_INDEX, NEXT_SYNC_COMMITTEE_INDEX} from "@chainsafe/lodestar-params";
import {Checkpoint, Epoch, LightClientUpdate, Slot} from "@chainsafe/lodestar-types/lib/altair";
import {Checkpoint, Epoch, LightClientUpdate, Slot, SyncPeriod} from "@chainsafe/lodestar-types/lib/altair";
import {isZeroHash, sumBits, toBlockHeader} from "../utils/utils";

type CommitteePeriod = number;
type DbRepo<K, T> = {put(key: K, data: T): void; get(key: K): T | null};
type DbItem<T> = {put(data: T): void; get(): T | null};
type DbRepo<K, T> = {put(key: K, data: T): Promise<void>; get(key: K): Promise<T | null>};
type DbItem<T> = {put(data: T): Promise<void>; get(): Promise<T | null>};

export type SyncAttestedData = Pick<
LightClientUpdate,
Expand Down Expand Up @@ -42,7 +41,7 @@ export type LightClientUpdaterDb = {
*
* Must persist the best update for each committee period between the longest possible weak subjectivity epoch and now.
*/
bestUpdatePerCommitteePeriod: DbRepo<CommitteePeriod, LightClientUpdate>;
bestUpdatePerCommitteePeriod: DbRepo<SyncPeriod, LightClientUpdate>;
latestFinalizedUpdate: DbItem<LightClientUpdate>;
latestNonFinalizedUpdate: DbItem<LightClientUpdate>;
};
Expand All @@ -52,12 +51,18 @@ export type LightClientUpdaterDb = {
*/
const PREV_DATA_MAX_SIZE = 64;

export interface ILightClientUpdater {
getBestUpdates(periods: SyncPeriod[]): Promise<altair.LightClientUpdate[]>;
getLatestUpdateFinalized(): Promise<altair.LightClientUpdate | null>;
getLatestUpdateNonFinalized(): Promise<altair.LightClientUpdate | null>;
}

/**
* Compute and cache LightClientUpdate objects as the chain advances
*
* Spec v1.0.1
*/
export class LightClientUpdater {
export class LightClientUpdater implements ILightClientUpdater {
private readonly prevHeadData = new Map<string, SyncAttestedData>();
private readonly zero: Pick<
LightClientUpdate,
Expand All @@ -81,10 +86,10 @@ export class LightClientUpdater {
/**
* To be called in API route GET /eth/v1/lightclient/best_update/:periods
*/
async getBestUpdates(periods: CommitteePeriod[]): Promise<LightClientUpdate[]> {
async getBestUpdates(periods: SyncPeriod[]): Promise<LightClientUpdate[]> {
const updates: LightClientUpdate[] = [];
for (const period of periods) {
const update = this.db.bestUpdatePerCommitteePeriod.get(period);
const update = await this.db.bestUpdatePerCommitteePeriod.get(period);
if (update) updates.push(update);
}
return updates;
Expand Down Expand Up @@ -113,7 +118,7 @@ export class LightClientUpdater {
* - Consider persisting the best of the best per comittee period in another db repo
* - Consider storing two best sync aggregates: the one with most bits and the one with most full aggregate sigs
*/
onHead(block: altair.BeaconBlock, postState: TreeBacked<altair.BeaconState>): void {
async onHead(block: altair.BeaconBlock, postState: TreeBacked<altair.BeaconState>): Promise<void> {
// Store a proof expected to be attested by the sync committee in a future block
// Prove that the `finalizedCheckpointRoot` belongs in that block
this.prevHeadData.set(toHexString(this.config.types.altair.BeaconBlock.hashTreeRoot(block)), {
Expand Down Expand Up @@ -142,9 +147,9 @@ export class LightClientUpdater {
}

// Store the best finalized update per period
const committeePeriodWithFinalized = this.persistBestFinalizedUpdate(syncAttestedData, signatureData);
const committeePeriodWithFinalized = await this.persistBestFinalizedUpdate(syncAttestedData, signatureData);
// Then, store the best non finalized update per period
this.persistBestNonFinalizedUpdate(syncAttestedData, signatureData, committeePeriodWithFinalized);
await this.persistBestNonFinalizedUpdate(syncAttestedData, signatureData, committeePeriodWithFinalized);

// Prune old prevHeadData
if (this.prevHeadData.size > PREV_DATA_MAX_SIZE) {
Expand All @@ -163,9 +168,13 @@ export class LightClientUpdater {
*
* NOTE: Must be called also on start with the current finalized checkpoint (may be genesis)
*/
onFinalized(checkpoint: Checkpoint, block: altair.BeaconBlock, postState: TreeBacked<altair.BeaconState>): void {
async onFinalized(
checkpoint: Checkpoint,
block: altair.BeaconBlock,
postState: TreeBacked<altair.BeaconState>
): Promise<void> {
// Pre-compute the nextSyncCommitteeBranch for this checkpoint, it will never change
this.db.lightclientFinalizedCheckpoint.put(checkpoint.epoch, {
await this.db.lightclientFinalizedCheckpoint.put(checkpoint.epoch, {
header: toBlockHeader(this.config, block),
nextSyncCommittee: postState.nextSyncCommittee,
// Prove that the `nextSyncCommittee` is included in a finalized state "attested" by the current sync committee
Expand All @@ -179,13 +188,13 @@ export class LightClientUpdater {
/**
* Store the best syncAggregate per finalizedEpoch
*/
private persistBestFinalizedUpdate(
private async persistBestFinalizedUpdate(
syncAttestedData: SyncAttestedData,
signatureData: CommitteeSignatureData
): CommitteePeriod | null {
): Promise<SyncPeriod | null> {
// Retrieve finality branch for attested finalized checkpoint
const finalizedEpoch = syncAttestedData.finalizedCheckpoint.epoch;
const finalizedData = this.db.lightclientFinalizedCheckpoint.get(finalizedEpoch);
const finalizedData = await this.db.lightclientFinalizedCheckpoint.get(finalizedEpoch);

// If there's no finalized data available for this epoch, we can't create an update
// TODO: Review if we can recover this data from the previous best update maybe, then prune
Expand Down Expand Up @@ -215,14 +224,14 @@ export class LightClientUpdater {
forkVersion: signatureData.forkVersion,
};

const prevBestUpdate = this.db.bestUpdatePerCommitteePeriod.get(committeePeriod);
const prevBestUpdate = await this.db.bestUpdatePerCommitteePeriod.get(committeePeriod);
if (!prevBestUpdate || isBetterUpdate(prevBestUpdate, newUpdate)) {
this.db.bestUpdatePerCommitteePeriod.put(committeePeriod, newUpdate);
await this.db.bestUpdatePerCommitteePeriod.put(committeePeriod, newUpdate);
}

const prevLatestUpdate = this.db.latestFinalizedUpdate.get();
const prevLatestUpdate = await this.db.latestFinalizedUpdate.get();
if (!prevLatestUpdate || isLatestBestFinalizedUpdate(prevLatestUpdate, newUpdate)) {
this.db.latestFinalizedUpdate.put(newUpdate);
await this.db.latestFinalizedUpdate.put(newUpdate);
}

return committeePeriod;
Expand All @@ -231,11 +240,11 @@ export class LightClientUpdater {
/**
* Store the best syncAggregate per committeePeriod in case finality is not reached
*/
private persistBestNonFinalizedUpdate(
private async persistBestNonFinalizedUpdate(
syncAttestedData: SyncAttestedData,
signatureData: CommitteeSignatureData,
committeePeriodWithFinalized: CommitteePeriod | null
): void {
committeePeriodWithFinalized: SyncPeriod | null
): Promise<void> {
const committeePeriod = computeSyncPeriodAtSlot(this.config, syncAttestedData.header.slot);
const signaturePeriod = computeSyncPeriodAtSlot(this.config, signatureData.slot);
if (committeePeriod !== signaturePeriod) {
Expand All @@ -255,17 +264,17 @@ export class LightClientUpdater {

// Optimization: If there's already a finalized update for this committee period, no need to create a non-finalized update
if (committeePeriodWithFinalized !== committeePeriod) {
const prevBestUpdate = this.db.bestUpdatePerCommitteePeriod.get(committeePeriod);
const prevBestUpdate = await this.db.bestUpdatePerCommitteePeriod.get(committeePeriod);
if (!prevBestUpdate || isBetterUpdate(prevBestUpdate, newUpdate)) {
this.db.bestUpdatePerCommitteePeriod.put(committeePeriod, newUpdate);
await this.db.bestUpdatePerCommitteePeriod.put(committeePeriod, newUpdate);
}
}

// Store the latest update here overall. Not checking it's the best
const prevLatestUpdate = this.db.latestNonFinalizedUpdate.get();
const prevLatestUpdate = await this.db.latestNonFinalizedUpdate.get();
if (!prevLatestUpdate || isLatestBestNonFinalizedUpdate(prevLatestUpdate, newUpdate)) {
// TODO: Don't store nextCommittee, that can be fetched through getBestUpdates()
this.db.latestNonFinalizedUpdate.put(newUpdate);
await this.db.latestNonFinalizedUpdate.put(newUpdate);
}
}
}
Expand Down
51 changes: 30 additions & 21 deletions packages/light-client/test/lightclientMockServer.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,29 +28,30 @@ export class LightclientMockServer {
private readonly stateCache = new Map<string, TreeBacked<altair.BeaconState>>();
private finalizedCheckpoint: altair.Checkpoint | null = null;
private prevBlock: altair.BeaconBlock | null = null;
private prevState: TreeBacked<altair.BeaconState>;
private prevState: TreeBacked<altair.BeaconState> | null = null;

// API state
private apiState: ApiState = {status: ApiStatus.stopped};

constructor(
private readonly config: IBeaconConfig,
private readonly logger: ILogger,
private readonly genesisValidatorsRoot: Root,
readonly initialFinalizedCheckpoint: {
checkpoint: Checkpoint;
block: altair.BeaconBlock;
state: TreeBacked<altair.BeaconState>;
}
private readonly genesisValidatorsRoot: Root
) {
const db = getLightClientUpdaterDb();
this.lightClientUpdater = new LightClientUpdater(config, db);
this.stateRegen = new MockStateRegen(this.stateCache);
}

async initialize(initialFinalizedCheckpoint: {
checkpoint: Checkpoint;
block: altair.BeaconBlock;
state: TreeBacked<altair.BeaconState>;
}): Promise<void> {
const {checkpoint, block, state} = initialFinalizedCheckpoint;
this.lightClientUpdater.onFinalized(checkpoint, block, state);
void this.lightClientUpdater.onFinalized(checkpoint, block, state);
this.stateCache.set(toHexString(state.hashTreeRoot()), state);
this.prevState = config.types.altair.BeaconState.createTreeBackedFromStruct(state);
this.prevState = this.config.types.altair.BeaconState.createTreeBackedFromStruct(state);
}

async startApiServer(opts: ServerOpts): Promise<void> {
Expand All @@ -73,10 +74,10 @@ export class LightclientMockServer {
await this.apiState.server.close();
}

createNewBlock(slot: Slot): void {
async createNewBlock(slot: Slot): Promise<void> {
// Create a block and postState
const block = this.config.types.altair.BeaconBlock.defaultValue();
const state = this.prevState.clone();
const state = this.prevState?.clone() || this.config.types.altair.BeaconState.defaultTreeBacked();
block.slot = slot;
state.slot = slot;

Expand Down Expand Up @@ -133,7 +134,7 @@ export class LightclientMockServer {
};

// Feed new finalized block and state to the LightClientUpdater
this.lightClientUpdater.onFinalized(
await this.lightClientUpdater.onFinalized(
this.finalizedCheckpoint,
finalizedData.block,
this.config.types.altair.BeaconState.createTreeBackedFromStruct(finalizedData.state)
Expand All @@ -160,7 +161,7 @@ export class LightclientMockServer {
}

// Feed new block and state to the LightClientUpdater
this.lightClientUpdater.onHead(block, this.config.types.altair.BeaconState.createTreeBackedFromStruct(state));
await this.lightClientUpdater.onHead(block, this.config.types.altair.BeaconState.createTreeBackedFromStruct(state));
}

private getSyncCommittee(period: SyncPeriod): SyncCommitteeKeys {
Expand Down Expand Up @@ -193,20 +194,28 @@ function getLightClientUpdaterDb(): LightClientUpdaterDb {
let latestNonFinalizedUpdate: altair.LightClientUpdate | null = null;
return {
lightclientFinalizedCheckpoint: {
put: (key, data) => lightclientFinalizedCheckpoint.set(key, data),
get: (key) => lightclientFinalizedCheckpoint.get(key) ?? null,
get: async (key) => lightclientFinalizedCheckpoint.get(key) ?? null,
put: async (key, data) => {
lightclientFinalizedCheckpoint.set(key, data);
},
},
bestUpdatePerCommitteePeriod: {
put: (key, data) => bestUpdatePerCommitteePeriod.set(key, data),
get: (key) => bestUpdatePerCommitteePeriod.get(key) ?? null,
get: async (key) => bestUpdatePerCommitteePeriod.get(key) ?? null,
put: async (key, data) => {
bestUpdatePerCommitteePeriod.set(key, data);
},
},
latestFinalizedUpdate: {
put: (data) => (latestFinalizedUpdate = data),
get: () => latestFinalizedUpdate,
get: async () => latestFinalizedUpdate,
put: async (data) => {
latestFinalizedUpdate = data;
},
},
latestNonFinalizedUpdate: {
put: (data) => (latestNonFinalizedUpdate = data),
get: () => latestNonFinalizedUpdate,
get: async () => latestNonFinalizedUpdate,
put: async (data) => {
latestNonFinalizedUpdate = data;
},
},
};
}
11 changes: 4 additions & 7 deletions packages/light-client/test/sim/altairChainSimulator.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,19 +45,16 @@ async function runAltairChainSimulator(): Promise<void> {
});

const logger = new WinstonLogger();
const lightclientServer = new LightclientMockServer(config, logger, genesisValidatorsRoot, {
block: genesisBlock,
state: genesisState,
checkpoint: genesisCheckpoint,
});
const lightclientServer = new LightclientMockServer(config, logger, genesisValidatorsRoot);
await lightclientServer.initialize({block: genesisBlock, state: genesisState, checkpoint: genesisCheckpoint});

// Start API server
await lightclientServer.startApiServer(serverOpts);

// Compute all periods until currentSlot
console.log("Syncing to latest slot...");
for (let slot = 1; slot <= getCurrentSlot(config, leveGenesisTime); slot++) {
lightclientServer.createNewBlock(slot);
await lightclientServer.createNewBlock(slot);
}
console.log("Synced to latest slot");

Expand All @@ -72,7 +69,7 @@ async function runAltairChainSimulator(): Promise<void> {

const slot = getCurrentSlot(config, leveGenesisTime);
console.log("Slot", slot);
lightclientServer.createNewBlock(slot);
await lightclientServer.createNewBlock(slot);
}
}

Expand Down
9 changes: 3 additions & 6 deletions packages/light-client/test/unit/sync.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -78,14 +78,11 @@ describe("Lightclient flow with LightClientUpdater", () => {
});

const logger = new WinstonLogger();
lightclientServer = new LightclientMockServer(config, logger, genesisValidatorsRoot, {
block: genesisBlock,
state: genesisState,
checkpoint: genesisCheckpoint,
});
lightclientServer = new LightclientMockServer(config, logger, genesisValidatorsRoot);
await lightclientServer.initialize({block: genesisBlock, state: genesisState, checkpoint: genesisCheckpoint});

for (let slot = fromSlot; slot <= toSlot; slot++) {
lightclientServer.createNewBlock(slot);
await lightclientServer.createNewBlock(slot);
}

// Check the current state of updates
Expand Down
1 change: 1 addition & 0 deletions packages/lodestar/package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,7 @@
"@chainsafe/lodestar-config": "^0.23.0",
"@chainsafe/lodestar-db": "^0.23.0",
"@chainsafe/lodestar-fork-choice": "^0.23.0",
"@chainsafe/lodestar-light-client": "^0.23.0",
"@chainsafe/lodestar-params": "^0.23.0",
"@chainsafe/lodestar-types": "^0.23.0",
"@chainsafe/lodestar-utils": "^0.23.0",
Expand Down

0 comments on commit 13bb120

Please sign in to comment.