Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

FIx archive finalized states db entry key typo #4508

Merged
merged 5 commits into from
Sep 9, 2022
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
44 changes: 27 additions & 17 deletions packages/beacon-node/src/chain/archiver/archiveStates.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
import {ILogger} from "@lodestar/utils";
import {computeEpochAtSlot} from "@lodestar/state-transition";
import {SLOTS_PER_EPOCH} from "@lodestar/params";
import {Slot, Epoch} from "@lodestar/types";
import {computeEpochAtSlot, computeStartSlotAtEpoch} from "@lodestar/state-transition";
import {CheckpointWithHex} from "@lodestar/fork-choice";
import {IBeaconDb} from "../../db/index.js";
import {CheckpointStateCache} from "../stateCache/index.js";
Expand Down Expand Up @@ -45,14 +47,20 @@ export class StatesArchiver {
if (finalized.epoch - lastStoredEpoch > PERSIST_TEMP_STATE_EVERY_EPOCHS) {
await this.archiveState(finalized);

const storedEpochs = await this.db.stateArchive.keys({
lt: finalized.epoch,
// Only check the current and previous intervals
gte: Math.max(0, (Math.floor(finalized.epoch / PERSIST_STATE_EVERY_EPOCHS) - 1) * PERSIST_STATE_EVERY_EPOCHS),
// Only check the current and previous intervals
const minEpoch = Math.max(
0,
(Math.floor(finalized.epoch / PERSIST_STATE_EVERY_EPOCHS) - 1) * PERSIST_STATE_EVERY_EPOCHS
);

const storedStateSlots = await this.db.stateArchive.keys({
lt: computeStartSlotAtEpoch(finalized.epoch),
gte: computeStartSlotAtEpoch(minEpoch),
});
const statesToDelete = computeEpochsToDelete(storedEpochs, PERSIST_STATE_EVERY_EPOCHS);
if (statesToDelete.length > 0) {
await this.db.stateArchive.batchDelete(statesToDelete);

const statesSlotsToDelete = computeStateSlotsToDelete(storedStateSlots, PERSIST_STATE_EVERY_EPOCHS);
if (statesSlotsToDelete.length > 0) {
await this.db.stateArchive.batchDelete(statesSlotsToDelete);
}
}
}
Expand All @@ -75,17 +83,19 @@ export class StatesArchiver {
/**
* Keeps first epoch per interval of persistEveryEpochs, deletes the rest
*/
export function computeEpochsToDelete(storedEpochs: number[], persistEveryEpochs: number): number[] {
const epochBuckets = new Set<number>();
const toDelete = new Set<number>();
for (const epoch of storedEpochs) {
const epochBucket = epoch - (epoch % persistEveryEpochs);
if (epochBuckets.has(epochBucket)) {
toDelete.add(epoch);
export function computeStateSlotsToDelete(storedStateSlots: Slot[], persistEveryEpochs: Epoch): Slot[] {
const persistEverySlots = persistEveryEpochs * SLOTS_PER_EPOCH;
const intervalsWithStates = new Set<number>();
const stateSlotsToDelete = new Set<number>();

for (const slot of storedStateSlots) {
const interval = Math.floor(slot / persistEverySlots);
if (intervalsWithStates.has(interval)) {
stateSlotsToDelete.add(slot);
} else {
epochBuckets.add(epochBucket);
intervalsWithStates.add(interval);
}
}

return Array.from(toDelete.values());
return Array.from(stateSlotsToDelete.values());
}
Original file line number Diff line number Diff line change
@@ -1,42 +1,46 @@
import {expect} from "chai";
import {computeEpochsToDelete} from "../../../../src/chain/archiver/archiveStates.js";
import {computeStartSlotAtEpoch} from "@lodestar/state-transition";
import {computeStateSlotsToDelete} from "../../../../src/chain/archiver/archiveStates.js";

describe("state archiver task", () => {
describe("computeEpochsToDelete", () => {
describe("computeStateSlotsToDelete", () => {
const testCases: {
id: string;
storedEpochs: number[];
persistEveryEpochs?: number;
toDelete: number[];
persistEveryEpochs: number;
epochsToDelete: number[];
}[] = [
{
id: "Empty",
storedEpochs: [],
toDelete: [],
persistEveryEpochs: 8,
epochsToDelete: [],
},
{
id: "Equally spaced, delete x%8 != 0",
storedEpochs: [0, 2, 4, 6, 8, 10, 12, 14, 16],
persistEveryEpochs: 8,
toDelete: [2, 4, 6, 10, 12, 14],
epochsToDelete: [2, 4, 6, 10, 12, 14],
},
{
id: "Equally spaced with offset",
storedEpochs: [0, 3, 5, 7, 9, 11, 13, 15, 17],
persistEveryEpochs: 8,
toDelete: [3, 5, 7, 11, 13, 15],
epochsToDelete: [3, 5, 7, 11, 13, 15],
},
{
id: "Edge case with offset that causes a very large gap between epochs",
storedEpochs: [7, 8, 23, 24],
persistEveryEpochs: 8,
toDelete: [],
epochsToDelete: [],
},
];

for (const {id, storedEpochs, persistEveryEpochs, toDelete} of testCases) {
for (const {id, storedEpochs, persistEveryEpochs, epochsToDelete} of testCases) {
it(id, () => {
expect(computeEpochsToDelete(storedEpochs, persistEveryEpochs ?? 1024)).to.deep.equal(toDelete);
const storedStateSlots = storedEpochs.map((epoch) => computeStartSlotAtEpoch(epoch));
const stateSlotsToDelete = epochsToDelete.map((epoch) => computeStartSlotAtEpoch(epoch));
expect(computeStateSlotsToDelete(storedStateSlots, persistEveryEpochs)).to.deep.equal(stateSlotsToDelete);
});
}
});
Expand Down