From c0b8392c931838a78f939954daaee752d99340d2 Mon Sep 17 00:00:00 2001 From: Andy Matuschak Date: Thu, 23 Apr 2020 17:29:57 -0700 Subject: [PATCH] Closes andymatuschak/metabook#24 Write prompt state cache to local level DB --- @types/level-auto-index/index.d.ts | 16 + .../index.js | 6 +- metabook-app/package.json | 9 +- metabook-app/src/Root.tsx | 107 ++++-- .../src/{ => model}/dataRecordCache.test.ts | 0 .../src/{ => model}/dataRecordCache.ts | 34 +- .../src/{ => model}/dataRecordClient.test.ts | 1 - .../src/{ => model}/dataRecordClient.ts | 0 metabook-app/src/model/levelDBUtil.ts | 23 ++ .../src/model/promptStateClient.test.ts | 317 ++++++++++++++++++ metabook-app/src/model/promptStateClient.ts | 291 ++++++++++++++++ .../src/model/promptStateStore.test.ts | 108 ++++++ metabook-app/src/model/promptStateStore.ts | 199 +++++++++++ metabook-app/src/reviewQueue.ts | 2 +- metabook-app/src/useReviewItems.ts | 87 +++-- metabook-app/tsconfig.json | 3 +- .../firebaseClient/firebaseClient.test.ts | 157 +-------- .../firebaseClient/firebaseClient.ts | 246 ++++---------- .../src/userClient/localClient/index.ts | 1 - .../localClient/localClient.test.ts | 75 ----- .../src/userClient/localClient/localClient.ts | 95 ------ metabook-client/src/userClient/userClient.ts | 23 +- .../src/actionLogDocument.ts | 6 +- .../src/batchWriteEntries.ts | 8 +- metabook-firebase-support/src/index.ts | 1 + .../src/libraryAbstraction.ts | 5 +- .../src/promptStateCache.ts | 10 +- yarn.lock | 132 +++++++- 28 files changed, 1330 insertions(+), 632 deletions(-) create mode 100644 @types/level-auto-index/index.d.ts rename metabook-app/src/{ => model}/dataRecordCache.test.ts (100%) rename metabook-app/src/{ => model}/dataRecordCache.ts (51%) rename metabook-app/src/{ => model}/dataRecordClient.test.ts (99%) rename metabook-app/src/{ => model}/dataRecordClient.ts (100%) create mode 100644 metabook-app/src/model/levelDBUtil.ts create mode 100644 metabook-app/src/model/promptStateClient.test.ts create mode 100644 metabook-app/src/model/promptStateClient.ts create mode 100644 metabook-app/src/model/promptStateStore.test.ts create mode 100644 metabook-app/src/model/promptStateStore.ts delete mode 100644 metabook-client/src/userClient/localClient/index.ts delete mode 100644 metabook-client/src/userClient/localClient/localClient.test.ts delete mode 100644 metabook-client/src/userClient/localClient/localClient.ts diff --git a/@types/level-auto-index/index.d.ts b/@types/level-auto-index/index.d.ts new file mode 100644 index 000000000..fb2c3a9c4 --- /dev/null +++ b/@types/level-auto-index/index.d.ts @@ -0,0 +1,16 @@ +declare module "level-auto-index" { + import { LevelUp } from "levelup"; + function AutoIndex( + db: LevelUp, + idb: LevelUp, + reduce: (value: T) => string, + opts: unknown, + ): AutoIndex.IndexedDB; + + export = AutoIndex; + namespace AutoIndex { + interface IndexedDB { + get(key: string, opts: unknown): Promise; + } + } +} diff --git a/firebase-react-native-persistence-shim/index.js b/firebase-react-native-persistence-shim/index.js index b22c2b016..10f9ea7e1 100644 --- a/firebase-react-native-persistence-shim/index.js +++ b/firebase-react-native-persistence-shim/index.js @@ -12,7 +12,11 @@ export default function shimFirebasePersistence(databaseBasePath) { isShimmed = true; window.openDatabase = SQLite.openDatabase; - setGlobalVars(window, { checkOrigin: false, databaseBasePath }); + setGlobalVars(window, { + checkOrigin: false, + databaseBasePath, + useSQLiteIndexes: true, + }); window.__localStorageStore = {}; window.localStorage = { diff --git a/metabook-app/package.json b/metabook-app/package.json index 2d21ff282..46dec7f98 100644 --- a/metabook-app/package.json +++ b/metabook-app/package.json @@ -13,6 +13,8 @@ }, "dependencies": { "@tradle/react-native-http": "^2.0.1", + "@types/subleveldown": "^4.1.0", + "abstract-leveldown": "^6.3.0", "assert": "^1.5.0", "base-64": "^0.1.0", "browserify-zlib": "^0.1.4", @@ -31,8 +33,11 @@ "firebase-react-native-persistence-shim": "0.0.1", "https-browserify": "0.0.1", "inherits": "^2.0.4", + "level-auto-index": "^2.0.0", "level-js": "^5.0.2", "levelup": "^4.4.0", + "lexicographic-integer": "^1.1.0", + "lodash.isequal": "^4.5.0", "metabook-client": "0.0.1", "metabook-core": "0.0.1", "metabook-sample-data": "0.0.1", @@ -49,8 +54,8 @@ "react-native-unimodules": "^0.9.0", "react-native-web": "~0.11.7", "readable-stream": "^1.0.33", - "stream-browserify": "^1.0.0", "string_decoder": "^0.10.31", + "subleveldown": "andymatuschak/subleveldown", "timers-browserify": "^1.4.2", "tty-browserify": "0.0.0", "url": "^0.11.0", @@ -64,6 +69,8 @@ "@types/jest": "^25.2.1", "@types/level-js": "^4.0.1", "@types/levelup": "^4.3.0", + "@types/lexicographic-integer": "^1.1.0", + "@types/lodash.isequal": "^4.5.5", "@types/react": "~16.9.0", "@types/react-native": "~0.60.23", "babel-core": "^6.26.3", diff --git a/metabook-app/src/Root.tsx b/metabook-app/src/Root.tsx index f2062aefb..6d880cd52 100644 --- a/metabook-app/src/Root.tsx +++ b/metabook-app/src/Root.tsx @@ -5,22 +5,27 @@ import { MetabookFirebaseUserClient, } from "metabook-client"; import { - getActionLogFromPromptActionLog, getIDForPrompt, getIDForPromptTask, getNextTaskParameters, PromptTask, repetitionActionLogType, } from "metabook-core"; -import { ReviewArea, ReviewAreaProps } from "metabook-ui"; +import { ReviewArea, ReviewAreaProps, ReviewItem } from "metabook-ui"; import colors from "metabook-ui/dist/styles/colors"; -import "node-libs-react-native/globals"; import typography from "metabook-ui/dist/styles/typography"; -import React, { useCallback, useState } from "react"; -import { View, Text } from "react-native"; -import DataRecordCache from "./dataRecordCache"; -import DataRecordClient from "./dataRecordClient"; -import { getFirebaseApp } from "./firebase"; +import "node-libs-react-native/globals"; +import React, { useCallback, useEffect, useState } from "react"; +import { Text, View } from "react-native"; +import { + enableFirebasePersistence, + getFirebaseApp, + PersistenceStatus, +} from "./firebase"; +import DataRecordCache from "./model/dataRecordCache"; +import DataRecordClient from "./model/dataRecordClient"; +import PromptStateClient from "./model/promptStateClient"; +import PromptStateStore from "./model/promptStateStore"; import { useReviewItems } from "./useReviewItems"; async function cacheWriteHandler(name: string, data: Buffer): Promise { @@ -43,35 +48,76 @@ async function fileExistsAtURL(url: string): Promise { return info.exists; } +function usePersistenceStatus() { + const [persistenceStatus, setPersistenceStatus] = useState( + "pending", + ); + + useEffect(() => { + let hasUnmounted = false; + + function safeSetPersistenceStatus(newStatus: PersistenceStatus) { + if (!hasUnmounted) { + setPersistenceStatus(newStatus); + } + } + + enableFirebasePersistence() + .then(() => safeSetPersistenceStatus("enabled")) + .catch(() => safeSetPersistenceStatus("unavailable")); + + return () => { + hasUnmounted = true; + }; + }, []); + + return persistenceStatus; +} + export default function App() { - const [{ userClient, dataRecordClient }] = useState(() => { - const firebaseApp = getFirebaseApp(); - const dataClient = new MetabookFirebaseDataClient( - firebaseApp, - firebaseApp.functions(), - ); - const dataCache = new DataRecordCache(); - return { - userClient: new MetabookFirebaseUserClient( + const persistenceStatus = usePersistenceStatus(); + const [ + promptStateClient, + setPromptStateClient, + ] = useState(null); + const [ + dataRecordClient, + setDataRecordClient, + ] = useState(null); + + useEffect(() => { + if (persistenceStatus === "enabled") { + const firebaseApp = getFirebaseApp(); + const userClient = new MetabookFirebaseUserClient( firebaseApp.firestore(), "x5EWk2UT56URxbfrl7djoxwxiqH2", - ), - dataRecordClient: new DataRecordClient(dataClient, dataCache, { - writeFile: cacheWriteHandler, - fileExistsAtURL, - }), - }; - }); + ); + setPromptStateClient( + new PromptStateClient(userClient, new PromptStateStore()), + ); + const dataClient = new MetabookFirebaseDataClient( + firebaseApp, + firebaseApp.functions(), + ); + const dataCache = new DataRecordCache(); + setDataRecordClient( + new DataRecordClient(dataClient, dataCache, { + writeFile: cacheWriteHandler, + fileExistsAtURL, + }), + ); + } + }, [persistenceStatus]); - const items = useReviewItems(userClient, dataRecordClient); + const items = useReviewItems(promptStateClient, dataRecordClient); const onMark = useCallback( async (marking) => { console.log("[Performance] Mark prompt", Date.now() / 1000.0); - userClient - .recordActionLogs([ - getActionLogFromPromptActionLog({ + promptStateClient! + .recordPromptActionLogs([ + { actionLogType: repetitionActionLogType, parentActionLogIDs: marking.reviewItem.promptState?.headActionLogIDs ?? [], @@ -87,10 +133,9 @@ export default function App() { marking.reviewItem.prompt, marking.reviewItem.promptState?.lastReviewTaskParameters ?? null, ), - }), + }, ]) .then(() => { - console.log("Committed", marking.reviewItem.prompt); console.log( "[Performance] Log committed to server", Date.now() / 1000.0, @@ -100,7 +145,7 @@ export default function App() { console.error("Couldn't commit", marking.reviewItem.prompt, error); }); }, - [userClient], + [promptStateClient], ); console.log("[Performance] Render", Date.now() / 1000.0); diff --git a/metabook-app/src/dataRecordCache.test.ts b/metabook-app/src/model/dataRecordCache.test.ts similarity index 100% rename from metabook-app/src/dataRecordCache.test.ts rename to metabook-app/src/model/dataRecordCache.test.ts diff --git a/metabook-app/src/dataRecordCache.ts b/metabook-app/src/model/dataRecordCache.ts similarity index 51% rename from metabook-app/src/dataRecordCache.ts rename to metabook-app/src/model/dataRecordCache.ts index 2557f0086..15ba496bc 100644 --- a/metabook-app/src/dataRecordCache.ts +++ b/metabook-app/src/model/dataRecordCache.ts @@ -6,6 +6,7 @@ import { Prompt, PromptID, } from "metabook-core"; +import { getJSONRecord, saveJSONRecord } from "./levelDBUtil"; export default class DataRecordCache { private db: levelup.LevelUp; @@ -13,47 +14,34 @@ export default class DataRecordCache { this.db = LevelUp(LevelJS(cacheName)); } - private saveRecord(key: string, value: unknown): Promise { - return this.db.put(key, JSON.stringify(value)); - } - - private async getRecord(key: string): Promise { - const recordString = await this.db - .get(key) - .catch((error) => (error.notFound ? null : Promise.reject(error))); - if (recordString) { - return JSON.parse(recordString) as R; - } else { - return null; - } - } - async savePrompt(id: PromptID, prompt: Prompt): Promise { - await this.saveRecord(id, prompt); + await saveJSONRecord(this.db, id, prompt); } async getPrompt(id: PromptID): Promise { - return this.getRecord(id); + const result = await getJSONRecord(this.db, id); + return (result?.record as Prompt) ?? null; } async saveAttachmentURLReference( id: AttachmentID, reference: AttachmentURLReference, ): Promise { - await this.saveRecord(id, reference); + await saveJSONRecord(this.db, id, reference); } async getAttachmentURLReference( id: AttachmentID, ): Promise { - return this.getRecord(id); + const result = await getJSONRecord(this.db, id); + return (result?.record as AttachmentURLReference) ?? null; } - async clear() { - return this.db.clear(); + async clear(): Promise { + await this.db.clear(); } - async close() { - return this.db.close(); + async close(): Promise { + await this.db.close(); } } diff --git a/metabook-app/src/dataRecordClient.test.ts b/metabook-app/src/model/dataRecordClient.test.ts similarity index 99% rename from metabook-app/src/dataRecordClient.test.ts rename to metabook-app/src/model/dataRecordClient.test.ts index 8701351e4..589b59f58 100644 --- a/metabook-app/src/dataRecordClient.test.ts +++ b/metabook-app/src/model/dataRecordClient.test.ts @@ -46,7 +46,6 @@ class MockDataClient implements MetabookDataClient { } let cache: DataRecordCache; -let client: DataRecordClient; const testBasicPromptID = getIDForPrompt(testBasicPrompt); beforeEach(() => { cache = new DataRecordCache(); diff --git a/metabook-app/src/dataRecordClient.ts b/metabook-app/src/model/dataRecordClient.ts similarity index 100% rename from metabook-app/src/dataRecordClient.ts rename to metabook-app/src/model/dataRecordClient.ts diff --git a/metabook-app/src/model/levelDBUtil.ts b/metabook-app/src/model/levelDBUtil.ts new file mode 100644 index 000000000..a7e2cb46b --- /dev/null +++ b/metabook-app/src/model/levelDBUtil.ts @@ -0,0 +1,23 @@ +import { LevelUp } from "levelup"; + +export async function saveJSONRecord( + db: LevelUp, + key: string, + value: unknown, +): Promise { + await db.put(key, JSON.stringify(value)); +} + +export async function getJSONRecord( + db: LevelUp, + key: string, +): Promise<{ record: T } | null> { + const recordString = await db + .get(key) + .catch((error) => (error.notFound ? null : Promise.reject(error))); + if (recordString) { + return { record: JSON.parse(recordString) }; + } else { + return null; + } +} diff --git a/metabook-app/src/model/promptStateClient.test.ts b/metabook-app/src/model/promptStateClient.test.ts new file mode 100644 index 000000000..4af641574 --- /dev/null +++ b/metabook-app/src/model/promptStateClient.test.ts @@ -0,0 +1,317 @@ +import { MetabookUserClient } from "metabook-client"; +import { + applyActionLogToPromptState, + BasicPromptTaskParameters, + basicPromptType, + getActionLogFromPromptActionLog, + getIDForActionLog, + getIDForPromptTask, + ingestActionLogType, + PromptID, + PromptIngestActionLog, + PromptRepetitionActionLog, + PromptRepetitionOutcome, + PromptState, + PromptTaskID, + repetitionActionLogType, +} from "metabook-core"; +import PromptStateClient, { + computeSubscriberUpdate, + PromptStateClientUpdate, +} from "./promptStateClient"; +import PromptStateStore from "./promptStateStore"; + +export function promiseForNextCall(fn: jest.Mock): Promise { + return new Promise((resolve) => fn.mockImplementation(resolve)); +} + +describe("computeSubscriberUpdate", () => { + const mockSubscriber = { + onUpdate: jest.fn(), + dueThresholdMillis: 1000, + }; + const testTaskID = "x" as PromptTaskID; + + test("generates added event when a prompt state is added", () => { + const testPromptState = { dueTimestampMillis: 500 } as PromptState; + const updates = computeSubscriberUpdate(mockSubscriber, [ + { + taskID: testTaskID, + oldPromptState: null, + newPromptState: testPromptState, + }, + ]); + expect(updates.addedEntries).toMatchObject( + new Map([[testTaskID, testPromptState]]), + ); + expect(updates.updatedEntries.size).toEqual(0); + expect(updates.removedEntries.size).toEqual(0); + }); + + test("generates added event when a prompt becomes due", () => { + const updates = computeSubscriberUpdate(mockSubscriber, [ + { + taskID: testTaskID, + oldPromptState: { dueTimestampMillis: 2000 } as PromptState, + newPromptState: { dueTimestampMillis: 500 } as PromptState, + }, + ]); + expect(updates.addedEntries.size).toEqual(1); + expect(updates.updatedEntries.size).toEqual(0); + expect(updates.removedEntries.size).toEqual(0); + }); + + test("does not generate added event when a not-yet-due prompt is added", () => { + const updates = computeSubscriberUpdate(mockSubscriber, [ + { + taskID: testTaskID, + oldPromptState: null, + newPromptState: { dueTimestampMillis: 2000 } as PromptState, + }, + ]); + expect(updates.addedEntries.size).toEqual(0); + expect(updates.updatedEntries.size).toEqual(0); + expect(updates.removedEntries.size).toEqual(0); + }); + + test("generates changed event when a due prompt changes", () => { + const updates = computeSubscriberUpdate(mockSubscriber, [ + { + taskID: testTaskID, + oldPromptState: { dueTimestampMillis: 1000 } as PromptState, + newPromptState: { dueTimestampMillis: 500 } as PromptState, + }, + ]); + expect(updates.addedEntries.size).toEqual(0); + expect(updates.updatedEntries.size).toEqual(1); + expect(updates.removedEntries.size).toEqual(0); + }); + + test("does not generate changed event when a not-yet-due prompt changes", () => { + const updates = computeSubscriberUpdate(mockSubscriber, [ + { + taskID: testTaskID, + oldPromptState: { dueTimestampMillis: 3000 } as PromptState, + newPromptState: { dueTimestampMillis: 2000 } as PromptState, + }, + ]); + expect(updates.addedEntries.size).toEqual(0); + expect(updates.updatedEntries.size).toEqual(0); + expect(updates.removedEntries.size).toEqual(0); + }); + + test("does not generate changed event when a due identical prompt state remains the same", () => { + const testPromptState = { dueTimestampMillis: 1000 } as PromptState; + const updates = computeSubscriberUpdate(mockSubscriber, [ + { + taskID: testTaskID, + oldPromptState: testPromptState, + newPromptState: testPromptState, + }, + ]); + expect(updates.addedEntries.size).toEqual(0); + expect(updates.updatedEntries.size).toEqual(0); + expect(updates.removedEntries.size).toEqual(0); + }); + + test("generates removed event when a due prompt becomes no longer due", () => { + const updates = computeSubscriberUpdate(mockSubscriber, [ + { + taskID: testTaskID, + oldPromptState: { dueTimestampMillis: 500 } as PromptState, + newPromptState: { dueTimestampMillis: 2000 } as PromptState, + }, + ]); + expect(updates.addedEntries.size).toEqual(0); + expect(updates.updatedEntries.size).toEqual(0); + expect(updates.removedEntries.size).toEqual(1); + }); +}); + +// TODO: test nothing happens for identical updates + +describe("prompt state subscriptions", () => { + let promptStateStore: PromptStateStore; + let remoteClient: MetabookUserClient; + let promptStateClient: PromptStateClient; + + beforeEach(() => { + promptStateStore = {} as PromptStateStore; + remoteClient = {} as MetabookUserClient; + remoteClient.subscribeToActionLogs = jest.fn(); + promptStateClient = new PromptStateClient(remoteClient, promptStateStore); + }); + + test("fetches remote states when none are cached", async () => { + promptStateStore.getLatestLogServerTimestamp = jest + .fn() + .mockResolvedValue(null); + promptStateStore.savePromptStateCaches = jest.fn(); + remoteClient.getDuePromptStates = jest.fn().mockResolvedValue([{}]); + + const callbackMock = jest.fn(); + const callbackPromise = promiseForNextCall(callbackMock); + const unsubscribe = promptStateClient.subscribeToDuePromptStates( + 1000, + callbackMock, + ); + await callbackPromise; + + expect(callbackMock.mock.calls[0][0].addedEntries.size).toEqual(1); + expect(promptStateStore.savePromptStateCaches).toHaveBeenCalled(); + unsubscribe(); + }); + + test("uses cached states when available", async () => { + promptStateStore.getLatestLogServerTimestamp = jest + .fn() + .mockResolvedValue(0); + promptStateStore.getDuePromptStates = jest + .fn() + .mockResolvedValue(new Map([["x", {}]])); + + const callbackMock = jest.fn(); + const callbackPromise = promiseForNextCall(callbackMock); + const unsubscribe = promptStateClient.subscribeToDuePromptStates( + 1000, + callbackMock, + ); + await callbackPromise; + + expect(callbackMock.mock.calls[0][0].addedEntries.size).toEqual(1); + expect(promptStateStore.getDuePromptStates).toHaveBeenCalled(); + unsubscribe(); + }); + + test("uses cached states when available", async () => { + promptStateStore.getLatestLogServerTimestamp = jest + .fn() + .mockResolvedValue(0); + promptStateStore.getDuePromptStates = jest + .fn() + .mockResolvedValue(new Map([["x", {}]])); + + const callbackMock = jest.fn(); + const callbackPromise = promiseForNextCall(callbackMock); + const unsubscribe = promptStateClient.subscribeToDuePromptStates( + 1000, + callbackMock, + ); + await callbackPromise; + + expect(callbackMock.mock.calls[0][0].addedEntries.size).toEqual(1); + expect(promptStateStore.getDuePromptStates).toHaveBeenCalled(); + unsubscribe(); + }); + + describe("updating based on logs", () => { + const testPromptTaskID = getIDForPromptTask({ + promptParameters: null, + promptType: basicPromptType, + promptID: "x" as PromptID, + }); + const testIngestLog: PromptIngestActionLog = { + actionLogType: ingestActionLogType, + provenance: null, + taskID: testPromptTaskID, + timestampMillis: 0, + }; + const initialPromptState = applyActionLogToPromptState({ + basePromptState: null, + promptActionLog: testIngestLog, + schedule: "default", + }) as PromptState; + const testIngestLogID = getIDForActionLog( + getActionLogFromPromptActionLog(testIngestLog), + ); + const repetitionLog: PromptRepetitionActionLog = { + actionLogType: repetitionActionLogType, + taskID: testPromptTaskID, + timestampMillis: initialPromptState.dueTimestampMillis, + taskParameters: null, + parentActionLogIDs: [testIngestLogID], + outcome: PromptRepetitionOutcome.Remembered, + context: null, + }; + + beforeEach(() => { + promptStateStore.getLatestLogServerTimestamp = jest + .fn() + .mockResolvedValue(0); + promptStateStore.getDuePromptStates = jest + .fn() + .mockResolvedValue(new Map()); + promptStateStore.getPromptState = jest + .fn() + .mockResolvedValue(initialPromptState); + promptStateStore.savePromptStateCaches = jest.fn(); + }); + + describe("recording local logs", () => { + beforeEach(() => { + remoteClient.recordActionLogs = jest.fn(); + }); + + test("notifies subscribers when recording local log", async () => { + const callbackMock = jest.fn(); + let callbackPromise = promiseForNextCall(callbackMock); + const unsubscribe = promptStateClient.subscribeToDuePromptStates( + initialPromptState.dueTimestampMillis, + callbackMock, + ); + await callbackPromise; + + callbackPromise = promiseForNextCall(callbackMock); + promptStateClient.recordPromptActionLogs([repetitionLog]); + const update = (await callbackPromise) as PromptStateClientUpdate; + + expect(update.removedEntries.size).toEqual(1); + expect(promptStateStore.savePromptStateCaches).toHaveBeenCalled(); + expect(remoteClient.recordActionLogs).toHaveBeenCalled(); + unsubscribe(); + }); + + test("doesn't notify unsubscribed subscribers", async () => { + const callbackMock = jest.fn(); + const unsubscribe = promptStateClient.subscribeToDuePromptStates( + initialPromptState.dueTimestampMillis, + callbackMock, + ); + unsubscribe(); + await promptStateClient.recordPromptActionLogs([repetitionLog]); + expect(callbackMock).not.toHaveBeenCalled(); + }); + }); + + describe("remote logs", () => { + test("notifies subscribers when new remote log arrives", async () => { + remoteClient.subscribeToActionLogs = jest.fn((t, callback) => { + callback([ + { + log: repetitionLog, + serverTimestamp: { seconds: 10, nanoseconds: 0 }, + }, + ]); + return () => { + return; + }; + }); + + const callbackMock = jest.fn(); + let callbackPromise = promiseForNextCall(callbackMock); + const unsubscribe = promptStateClient.subscribeToDuePromptStates( + initialPromptState.dueTimestampMillis, + callbackMock, + ); + await callbackPromise; // first update will be the initial prompt states + + // second update should get the repetition log + callbackPromise = promiseForNextCall(callbackMock); + const update = (await callbackPromise) as PromptStateClientUpdate; + expect(update.removedEntries.size).toEqual(1); + expect(promptStateStore.savePromptStateCaches).toHaveBeenCalled(); + unsubscribe(); + }); + }); + }); +}); diff --git a/metabook-app/src/model/promptStateClient.ts b/metabook-app/src/model/promptStateClient.ts new file mode 100644 index 000000000..8f763d5b0 --- /dev/null +++ b/metabook-app/src/model/promptStateClient.ts @@ -0,0 +1,291 @@ +import isEqual from "lodash.isequal"; + +import { MetabookUserClient } from "metabook-client"; +import { + ActionLog, + applyActionLogToPromptState, + getActionLogFromPromptActionLog, + getPromptActionLogFromActionLog, + PromptActionLog, + promptActionLogCanBeAppliedToPromptState, + PromptState, + PromptTaskID, + PromptTaskParameters, +} from "metabook-core"; +import { ServerTimestamp } from "metabook-firebase-support"; +import PromptStateStore from "./promptStateStore"; + +function updatePromptStateForLog( + basePromptState: PromptState | null, + promptActionLog: PromptActionLog, +): PromptState | null { + // TODO: implement real resolution + if ( + promptActionLogCanBeAppliedToPromptState(promptActionLog, basePromptState) + ) { + const newPromptState = applyActionLogToPromptState({ + promptActionLog, + basePromptState, + schedule: "default", + }); + if (newPromptState instanceof Error) { + throw newPromptState; + } else { + return newPromptState; + } + } else { + return null; + } +} + +export function computeSubscriberUpdate( + subscriber: PromptStateClientSubscriber, + changes: { + taskID: PromptTaskID; + oldPromptState: PromptState | null; + newPromptState: PromptState; + }[], +): PromptStateClientUpdate { + function promptStateMatchesQuery(promptState: PromptState): boolean { + return promptState.dueTimestampMillis <= subscriber.dueThresholdMillis; + } + + const update: PromptStateClientUpdate = { + addedEntries: new Map(), + updatedEntries: new Map(), + removedEntries: new Set(), + }; + for (const { taskID, oldPromptState, newPromptState } of changes) { + if (isEqual(oldPromptState, newPromptState)) { + continue; + } + + const didMatch = oldPromptState + ? promptStateMatchesQuery(oldPromptState) + : false; + const nowMaches = promptStateMatchesQuery(newPromptState); + if (didMatch && nowMaches) { + update.updatedEntries.set(taskID, newPromptState); + } else if (!didMatch && nowMaches) { + update.addedEntries.set(taskID, newPromptState); + } else if (didMatch && !nowMaches) { + update.removedEntries.add(taskID); + } + } + + return update; +} + +export interface PromptStateClientUpdate { + addedEntries: Map; + updatedEntries: Map; + removedEntries: Set; +} + +interface PromptStateClientSubscriber { + dueThresholdMillis: number; + onUpdate: (update: PromptStateClientUpdate) => void; +} + +export default class PromptStateClient { + private remoteClient: MetabookUserClient; + private promptStateStore: PromptStateStore; + private subscribers: Set; + private remoteLogSubscription: (() => void) | null; + + constructor( + remoteClient: MetabookUserClient, + promptStateStore: PromptStateStore, + ) { + this.remoteClient = remoteClient; + this.promptStateStore = promptStateStore; + this.subscribers = new Set(); + this.remoteLogSubscription = null; + } + + subscribeToDuePromptStates( + dueThresholdMillis: number, + onUpdate: (update: PromptStateClientUpdate) => void, + ): () => void { + const subscriber: PromptStateClientSubscriber = { + dueThresholdMillis, + onUpdate, + }; + this.subscribers.add(subscriber); + + this.promptStateStore + .getLatestLogServerTimestamp() + .then(async (latestServerLogTimestamp) => { + let initialEntries: Map; + if (latestServerLogTimestamp === null) { + console.log( + "No cached prompt states. Fetching initial prompt states.", + ); + const initialPromptStateCaches = await this.remoteClient.getDuePromptStates( + dueThresholdMillis, + ); + await this.promptStateStore.savePromptStateCaches( + initialPromptStateCaches.map((cache) => { + const { taskID, lastLogServerTimestamp, ...promptState } = cache; + return { + taskID: taskID, + lastLogServerTimestamp, + promptState, + }; + }), + ); + initialEntries = new Map( + initialPromptStateCaches.map((cache) => { + const { taskID, lastLogServerTimestamp, ...promptState } = cache; + return [taskID, promptState]; + }), + ); + } else { + initialEntries = await this.promptStateStore.getDuePromptStates( + dueThresholdMillis, + ); + console.log( + "[Performance] Got stored due prompt states", + Date.now() / 1000.0, + ); + } + + if (this.subscribers.has(subscriber)) { + onUpdate({ + addedEntries: initialEntries, + updatedEntries: new Map(), + removedEntries: new Set(), + }); + } + + this.ensureSubscriptionToRemoteLogs(latestServerLogTimestamp); + }); + + return () => { + this.subscribers.delete(subscriber); + if (this.subscribers.size === 0) { + this.remoteLogSubscription?.(); + this.remoteLogSubscription = null; + } + }; + } + + private ensureSubscriptionToRemoteLogs( + latestServerLogTimestamp: ServerTimestamp | null, + ) { + if (this.remoteLogSubscription) { + return; + } + + // TODO: this timestamp should be the latest timestamp we've seen from the logs store, not the prompt state store + this.remoteLogSubscription = this.remoteClient.subscribeToActionLogs( + latestServerLogTimestamp, + async (newLogs) => { + this.updateLocalStateForLogEntries(newLogs); + }, + (error) => { + console.error(`Error with log listener`, error); // TODO + }, + ); + } + + private async updateLocalStateForLogEntries( + entries: Iterable<{ + log: ActionLog; + serverTimestamp: ServerTimestamp | null; + }>, + ): Promise { + // First, we get the current prompt states for these logs. + const promptStateGetPromises: Promise[] = []; + for (const { log } of entries) { + promptStateGetPromises.push( + this.promptStateStore.getPromptState( + getPromptActionLogFromActionLog(log).taskID, + ), + ); + } + const promptStates = await Promise.all(promptStateGetPromises); + + // Now we'll compute the new prompt states + const updates: { + oldPromptState: PromptState | null; + newPromptState: PromptState; + taskID: PromptTaskID; + lastLogServerTimestamp: null; + }[] = []; + let logIndex = 0; + for (const { log } of entries) { + const basePromptState = promptStates[logIndex]; + logIndex++; + + if (!basePromptState) { + console.warn( + `Attempting to record log ${JSON.stringify( + log, + null, + "\t", + )} for which we have no prompt state`, + ); + continue; + } + + const promptActionLog = getPromptActionLogFromActionLog(log); + const newPromptState = updatePromptStateForLog( + basePromptState, + promptActionLog, + ); + if (!newPromptState) { + throw new Error( + `Can't apply log to prompt. New log heads: ${JSON.stringify( + log, + null, + "\t", + )}. Base prompt state: ${JSON.stringify( + basePromptState, + null, + "\t", + )}`, + ); + } + + updates.push({ + oldPromptState: basePromptState, + newPromptState, + taskID: promptActionLog.taskID, + lastLogServerTimestamp: null, + }); + } + + // TODO on write, update log listener for latest server timestamp + // TODO cache logs + + const savePromise = this.promptStateStore.savePromptStateCaches( + updates.map(({ newPromptState, taskID, lastLogServerTimestamp }) => ({ + promptState: newPromptState, + taskID, + lastLogServerTimestamp, + })), + ); + + for (const subscriber of this.subscribers) { + subscriber.onUpdate(computeSubscriberUpdate(subscriber, updates)); + } + + await savePromise; + } + + async recordPromptActionLogs( + logs: Iterable>, + ): Promise { + const actionLogs = [...logs].map(getActionLogFromPromptActionLog); + // TODO: if we're still doing our initial action log sync, queue these + + await Promise.all([ + // TODO on write, update prompt state caches with latest server log timestamps + this.remoteClient.recordActionLogs(actionLogs), + this.updateLocalStateForLogEntries( + actionLogs.map((log) => ({ log, serverTimestamp: null })), + ), + ]); + } +} diff --git a/metabook-app/src/model/promptStateStore.test.ts b/metabook-app/src/model/promptStateStore.test.ts new file mode 100644 index 000000000..bf9bb0376 --- /dev/null +++ b/metabook-app/src/model/promptStateStore.test.ts @@ -0,0 +1,108 @@ +import shimFirebasePersistence from "firebase-node-persistence-shim"; +import { PromptState, PromptTaskID } from "metabook-core"; +import { ServerTimestamp } from "metabook-firebase-support"; +import PromptStateStore from "./promptStateStore"; + +beforeAll(() => { + shimFirebasePersistence(); +}); + +let store: PromptStateStore; +beforeEach(() => { + store = new PromptStateStore(); +}); + +afterEach(async () => { + await store.clear(); + await store.close(); +}); + +const testPromptState = ({ + test: true, + dueTimestampMillis: 0, +} as unknown) as PromptState; + +async function saveTestPromptState( + lastLogServerTimestamp: ServerTimestamp | null, +) { + return await store.savePromptStateCaches([ + { + promptState: testPromptState, + lastLogServerTimestamp, + taskID: "x" as PromptTaskID, + }, + ]); +} + +test("round trips data", async () => { + const saveResult = await saveTestPromptState(null); + expect(saveResult).toBeNull(); + const record = await store.getPromptState("x" as PromptTaskID); + expect(record).toMatchObject(testPromptState); +}); + +test("writes last log timestamp", async () => { + const testTimestamp = { seconds: 1000, nanoseconds: 0 }; + const timestamp = await saveTestPromptState(testTimestamp); + expect(timestamp).toMatchObject(testTimestamp); + + await store.close(); + store = new PromptStateStore(); + expect(await store.getLatestLogServerTimestamp()).toMatchObject( + testTimestamp, + ); +}); + +test("only updates timestamp if newer", async () => { + await saveTestPromptState({ seconds: 1000, nanoseconds: 0 }); + const timestamp = await saveTestPromptState({ seconds: 500, nanoseconds: 0 }); + expect(timestamp?.seconds).toEqual(1000); +}); + +test("returns null for missing keys", async () => { + const record = await store.getPromptState("foo" as PromptTaskID); + expect(record).toBeNull(); +}); + +describe("access by due timestamp", () => { + test("accesses due prompts", async () => { + const testTaskID = "x" as PromptTaskID; + const testPromptState = { dueTimestampMillis: 1000 } as PromptState; + await store.savePromptStateCaches([ + { + promptState: testPromptState, + lastLogServerTimestamp: null, + taskID: testTaskID, + }, + { + promptState: { dueTimestampMillis: 5000 } as PromptState, + lastLogServerTimestamp: null, + taskID: "another" as PromptTaskID, + }, + ]); + + expect(await store.getDuePromptStates(1000)).toMatchObject( + new Map([[testTaskID, testPromptState]]), + ); + }); + + test("indexed due times update when overwritten", async () => { + const testTaskID = "x" as PromptTaskID; + await store.savePromptStateCaches([ + { + promptState: { dueTimestampMillis: 1000 } as PromptState, + lastLogServerTimestamp: null, + taskID: testTaskID, + }, + ]); + await store.savePromptStateCaches([ + { + promptState: { dueTimestampMillis: 5000 } as PromptState, + lastLogServerTimestamp: null, + taskID: testTaskID, + }, + ]); + + expect(await store.getDuePromptStates(1000)).toMatchObject(new Map()); + }); +}); diff --git a/metabook-app/src/model/promptStateStore.ts b/metabook-app/src/model/promptStateStore.ts new file mode 100644 index 000000000..3e194cc94 --- /dev/null +++ b/metabook-app/src/model/promptStateStore.ts @@ -0,0 +1,199 @@ +import LevelJS from "level-js"; +import LevelUp, * as levelup from "levelup"; +import { Transform } from "stream"; +import * as lexi from "lexicographic-integer"; + +import { PromptState, PromptTaskID } from "metabook-core"; +import { ServerTimestamp } from "metabook-firebase-support"; +import sub from "subleveldown"; +import { getJSONRecord } from "./levelDBUtil"; + +const latestLogServerTimestampDBKey = "_latestLogServerTimestamp"; + +function getDueTimestampIndexKey( + promptState: PromptState, + taskID: string & { __promptTaskIDOpaqueType: never }, +) { + return `${lexi.pack(promptState.dueTimestampMillis, "hex")}!${taskID}`; +} + +export default class PromptStateStore { + private rootDB: levelup.LevelUp; + private promptStateDB: levelup.LevelUp; + private dueTimestampIndexDB: levelup.LevelUp; + private opQueue: (() => Promise)[]; + + private cachedLatestLogServerTimestamp: ServerTimestamp | null | undefined; + + constructor(cacheName = "PromptStateStore") { + console.log("[Performance] Opening prompt store", Date.now() / 1000.0); + this.rootDB = LevelUp(LevelJS(cacheName), () => { + console.log("[Performance] Opened database", Date.now() / 1000.0); + }); + this.promptStateDB = sub(this.rootDB, "promptStates"); + this.dueTimestampIndexDB = sub(this.rootDB, "dueTimestampMillis"); + this.cachedLatestLogServerTimestamp = undefined; + this.opQueue = []; + } + + private runOp(op: () => Promise): Promise { + return new Promise((resolve, reject) => { + const runOp = () => { + return op() + .then(resolve, reject) + .finally(() => { + const nextOp = this.opQueue.shift(); + if (nextOp) { + nextOp(); + } + }); + }; + if (this.opQueue.length === 0) { + runOp(); + } else { + this.opQueue.push(runOp); + } + }); + } + + // Can only be called from the op queue + private async _getLatestLogServerTimestamp(): Promise { + if (this.cachedLatestLogServerTimestamp === undefined) { + const result = await getJSONRecord( + this.promptStateDB, + latestLogServerTimestampDBKey, + ); + this.cachedLatestLogServerTimestamp = + (result?.record as ServerTimestamp) ?? null; + } + return this.cachedLatestLogServerTimestamp ?? null; + } + + async getLatestLogServerTimestamp(): Promise { + return this.runOp(async () => { + return this._getLatestLogServerTimestamp(); + }); + } + + async savePromptStateCaches( + entries: Iterable<{ + promptState: PromptState; + taskID: PromptTaskID; + lastLogServerTimestamp: ServerTimestamp | null; + }>, + ): Promise { + return this.runOp(async () => { + const initialLatestLogServerTimestamp = await this._getLatestLogServerTimestamp(); + let latestLogServerTimestamp = initialLatestLogServerTimestamp; + + const batch = this.promptStateDB.batch(); + const dueTimestampIndexBatch = this.dueTimestampIndexDB.batch(); + for (const { promptState, taskID, lastLogServerTimestamp } of entries) { + const encodedPromptState = JSON.stringify(promptState); + batch.put(taskID, encodedPromptState); + + dueTimestampIndexBatch.put( + getDueTimestampIndexKey(promptState, taskID), + encodedPromptState, + ); + const oldPromptState = await this._getPromptState(taskID); + if (oldPromptState) { + dueTimestampIndexBatch.del( + getDueTimestampIndexKey(oldPromptState, taskID), + ); + } + if ( + lastLogServerTimestamp && + (latestLogServerTimestamp === null || + lastLogServerTimestamp.seconds > latestLogServerTimestamp.seconds || + (lastLogServerTimestamp.seconds === + latestLogServerTimestamp.seconds && + lastLogServerTimestamp.nanoseconds > + latestLogServerTimestamp.nanoseconds)) + ) { + latestLogServerTimestamp = lastLogServerTimestamp; + } + } + + if (latestLogServerTimestamp !== initialLatestLogServerTimestamp) { + batch.put( + latestLogServerTimestampDBKey, + JSON.stringify(latestLogServerTimestamp), + ); + this.cachedLatestLogServerTimestamp = latestLogServerTimestamp; + } + + await Promise.all([batch.write(), dueTimestampIndexBatch.write()]); + return latestLogServerTimestamp; + }); + } + + private async _getPromptState( + taskID: PromptTaskID, + ): Promise { + const recordString = await this.promptStateDB + .get(taskID) + .catch((error) => (error.notFound ? null : Promise.reject(error))); + if (recordString) { + return JSON.parse(recordString) as PromptState; + } else { + return null; + } + } + + async getPromptState(taskID: PromptTaskID): Promise { + return this.runOp(async () => { + return this._getPromptState(taskID); + }); + } + + async getDuePromptStates( + dueThresholdMillis: number, + limit?: number, + ): Promise> { + return this.runOp( + () => + new Promise((resolve, reject) => { + const output: Map = new Map(); + + const indexUpdateTransformer = new Transform({ + objectMode: true, + transform: async (chunk, inc, done) => { + // console.log("[Performance] Start transform", Date.now()); + const indexKey = chunk.key; + const promptState = JSON.parse(chunk.value); + // If this is a stale index entry, ditch it. + const taskID = indexKey.split("!")[1]; + // console.log("[Performance] Finish transform", Date.now()); + done(null, { taskID, promptState }); + }, + }); + + this.dueTimestampIndexDB + .createReadStream({ + lt: `${lexi.pack(dueThresholdMillis, "hex")}~`, // i.e. the character after the due timestamp + keys: true, + values: true, + limit, + }) + .pipe(indexUpdateTransformer) + .on("data", ({ taskID, promptState }) => { + // console.log("[Performance] Start data fn", Date.now()); + output.set(taskID, promptState); + // console.log("[Performance] End data fn", Date.now()); + }) + .on("error", reject) + .on("close", () => reject(new Error(`Database unexpected closed`))) + .on("end", () => resolve(output)); + }), + ); + } + + async clear(): Promise { + await this.rootDB.clear(); + } + + async close(): Promise { + await this.rootDB.close(); + } +} diff --git a/metabook-app/src/reviewQueue.ts b/metabook-app/src/reviewQueue.ts index 7bb763175..f650cff45 100644 --- a/metabook-app/src/reviewQueue.ts +++ b/metabook-app/src/reviewQueue.ts @@ -1,6 +1,6 @@ import { MetabookDataClient } from "metabook-client"; import { ReviewItem } from "metabook-ui"; -import DataRecordCache from "./dataRecordCache"; +import DataRecordCache from "./model/dataRecordCache"; class ReviewQueue { private dataClient: MetabookDataClient; diff --git a/metabook-app/src/useReviewItems.ts b/metabook-app/src/useReviewItems.ts index 75c64178a..3c1caa902 100644 --- a/metabook-app/src/useReviewItems.ts +++ b/metabook-app/src/useReviewItems.ts @@ -1,15 +1,13 @@ import { - MetabookDataClient, MetabookDataSnapshot, MetabookPromptStateSnapshot, - MetabookUserClient, } from "metabook-client"; import { AttachmentID, AttachmentURLReference, getDuePromptTaskIDs, - getPromptTaskForID, getIDForPromptTask, + getPromptTaskForID, Prompt, PromptField, PromptID, @@ -30,40 +28,12 @@ import { useRef, useState, } from "react"; -import DataRecordClient from "./dataRecordClient"; -import { enableFirebasePersistence, PersistenceStatus } from "./firebase"; -import promptDataCache from "./dataRecordCache"; - -function usePersistenceStatus() { - const [persistenceStatus, setPersistenceStatus] = useState( - "pending", - ); - - useEffect(() => { - let hasUnmounted = false; - - function safeSetPersistenceStatus(newStatus: PersistenceStatus) { - if (!hasUnmounted) { - setPersistenceStatus(newStatus); - } - } - - enableFirebasePersistence() - .then(() => safeSetPersistenceStatus("enabled")) - .catch(() => safeSetPersistenceStatus("unavailable")); - - return () => { - hasUnmounted = true; - }; - }, []); - - return persistenceStatus; -} +import DataRecordClient from "./model/dataRecordClient"; +import PromptStateClient from "./model/promptStateClient"; function usePromptStates( - userClient: MetabookUserClient, + promptStateClient: PromptStateClient | null, ): MetabookPromptStateSnapshot | null { - const persistenceStatus = usePersistenceStatus(); const [ promptStates, setPromptStates, @@ -76,7 +46,7 @@ function usePromptStates( ); useEffect(() => { - if (persistenceStatus === "pending") { + if (!promptStateClient) { return; } @@ -85,12 +55,30 @@ function usePromptStates( "[Performance] Subscribing to prompt states", Date.now() / 1000.0, ); - return userClient.subscribeToPromptStates( - { dueBeforeTimestampMillis: Date.now() }, // TODO: use fuzzy due dates - setPromptStates, - subscriptionDidFail, + return promptStateClient.subscribeToDuePromptStates( + // TODO use fuzzy due dates + // TODO when does this change? + Date.now(), + (update) => + setPromptStates((oldPromptStates) => { + console.log( + "[Performance] Updating prompt state cache in useReviewItems", + Date.now() / 1000, + ); + const newPromptStates = new Map(oldPromptStates ?? []); + for (const [taskID, promptState] of update.addedEntries) { + newPromptStates.set(taskID, promptState); + } + for (const [taskID, promptState] of update.updatedEntries) { + newPromptStates.set(taskID, promptState); + } + for (const taskID of update.removedEntries) { + newPromptStates.delete(taskID); + } + return newPromptStates; + }), ); - }, [userClient, persistenceStatus, setPromptStates, subscriptionDidFail]); + }, [promptStateClient, setPromptStates, subscriptionDidFail]); return promptStates; } @@ -104,7 +92,7 @@ function useWeakRef(val: T): MutableRefObject { } function usePrompts( - dataRecordClient: DataRecordClient, + dataRecordClient: DataRecordClient | null, promptIDs: Set, ): MetabookDataSnapshot | null { const unsubscribeFromDataRequest = useRef<(() => void) | null>(null); @@ -115,6 +103,10 @@ function usePrompts( const weakPrompts = useWeakRef(prompts); useEffect(() => { + if (dataRecordClient === null) { + return; + } + unsubscribeFromDataRequest.current?.(); let promptIDsToFetch: Set; if (weakPrompts.current) { @@ -162,7 +154,7 @@ function usePrompts( } function useAttachments( - dataRecordClient: DataRecordClient, + dataRecordClient: DataRecordClient | null, attachmentIDs: Set, ): MetabookDataSnapshot | null { const unsubscribeFromDataRequest = useRef<(() => void) | null>(null); @@ -176,6 +168,10 @@ function useAttachments( const weakAttachmentResolutionMap = useWeakRef(attachmentResolutionMap); useEffect(() => { + if (dataRecordClient === null) { + return; + } + unsubscribeFromDataRequest.current?.(); let attachmentIDsToFetch: Set; if (weakAttachmentResolutionMap.current) { @@ -267,10 +263,10 @@ function getAttachmentIDsInPrompts( } export function useReviewItems( - userClient: MetabookUserClient, - dataRecordClient: DataRecordClient, + promptStateClient: PromptStateClient | null, + dataRecordClient: DataRecordClient | null, ): ReviewItem[] | null { - const promptStates = usePromptStates(userClient); + const promptStates = usePromptStates(promptStateClient); const orderedDuePromptTasks: PromptTask[] | null = useMemo(() => { if (promptStates === null) { @@ -320,7 +316,6 @@ export function useReviewItems( ); return useMemo(() => { - console.log("Computing review items"); console.log("[Performance] Computing review items", Date.now() / 1000.0); return ( orderedDuePromptTasks diff --git a/metabook-app/tsconfig.json b/metabook-app/tsconfig.json index 31216ff71..0e0921ef9 100644 --- a/metabook-app/tsconfig.json +++ b/metabook-app/tsconfig.json @@ -13,6 +13,7 @@ "./src/**/*.tsx", "./src/**/*.ts", "./App.tsx", - "./src/shimBase64.js" + "./src/shimBase64.js", + "../@types/**/*.d.ts" ] } diff --git a/metabook-client/src/userClient/firebaseClient/firebaseClient.test.ts b/metabook-client/src/userClient/firebaseClient/firebaseClient.test.ts index 35e6a2e40..b392ec842 100644 --- a/metabook-client/src/userClient/firebaseClient/firebaseClient.test.ts +++ b/metabook-client/src/userClient/firebaseClient/firebaseClient.test.ts @@ -4,18 +4,13 @@ import { applyActionLogToPromptState, basicPromptType, clozePromptType, - getActionLogFromPromptActionLog, - getIDForActionLog, getIDForPrompt, getIDForPromptTask, ingestActionLogType, PromptID, PromptIngestActionLog, - PromptRepetitionOutcome, PromptState, PromptTask, - PromptTaskID, - repetitionActionLogType, } from "metabook-core"; import { getTaskStateCacheReferenceForTaskID, @@ -24,7 +19,6 @@ import { import { testBasicPrompt } from "metabook-sample-data"; import { promiseForNextCall } from "../../util/tests/promiseForNextCall"; import { recordTestPromptStateUpdate } from "../../util/tests/recordTestPromptStateUpdate"; -import { MetabookLocalUserClient } from "../localClient"; import { MetabookFirebaseUserClient } from "./firebaseClient"; let testFirestore: firebase.firestore.Firestore; @@ -45,49 +39,26 @@ afterEach(() => { return firebaseTesting.clearFirestoreData({ projectId: testProjectID }); }); -test("recording a review triggers card state update", async () => { +test("recording a review triggers new log", async () => { const mockFunction = jest.fn(); - const firstMockCall = promiseForNextCall(mockFunction); - const unsubscribe = client.subscribeToPromptStates( - {}, + const unsubscribe = client.subscribeToActionLogs( + null, mockFunction, (error) => { fail(error); }, ); - await firstMockCall; - const initialPromptStates = mockFunction.mock.calls[0][0]; - expect(initialPromptStates).toMatchObject(new Map()); - const secondMockCall = promiseForNextCall(mockFunction); - const { - testPromptTaskID, - testPromptActionLog, - commit, - } = recordTestPromptStateUpdate(client); + const mockCall = promiseForNextCall(mockFunction); + const { testPromptActionLog, commit } = recordTestPromptStateUpdate(client); await commit; - const updatedPromptStates = await secondMockCall; - expect(updatedPromptStates).toMatchObject( - new Map([ - [ - testPromptTaskID, - applyActionLogToPromptState({ - promptActionLog: testPromptActionLog, - basePromptState: null, - schedule: "default", - }), - ], - ]), - ); - - // The new prompt states should be a different object. - expect(updatedPromptStates).not.toMatchObject(initialPromptStates); - + const newLogs = await mockCall; + expect(newLogs).toMatchObject([{ log: testPromptActionLog }]); unsubscribe(); }); -describe("prompt state cache", () => { +describe("prompt states", () => { test("initial prompt states", async () => { const testPromptID = getIDForPrompt(testBasicPrompt); const promptTask: PromptTask = { @@ -100,9 +71,7 @@ describe("prompt state cache", () => { testFirestore, testUserID, taskID, - ) as firebase.firestore.DocumentReference< - PromptStateCache - >; + ) as firebase.firestore.DocumentReference; const ingestLog: PromptIngestActionLog = { taskID, actionLogType: ingestActionLogType, @@ -120,120 +89,26 @@ describe("prompt state cache", () => { lastLogServerTimestamp: new firebase.firestore.Timestamp(0.5, 0), }); - const mockFunction = jest.fn(); - const firstMockCall = promiseForNextCall(mockFunction); - const unsubscribe = client.subscribeToPromptStates( - {}, - mockFunction, - (error) => { - fail(error); - }, + const initialPromptStates = await client.getDuePromptStates( + initialPromptState.dueTimestampMillis, ); - await firstMockCall; - const initialPromptStates = mockFunction.mock.calls[0][0]; - expect(initialPromptStates.get(taskID)).toEqual(initialPromptState); - - const secondMockCall = promiseForNextCall(mockFunction); - await client.recordActionLogs([ + expect(initialPromptStates).toMatchObject([ { - actionLogType: repetitionActionLogType, taskID, - taskParameters: null, - timestampMillis: 2000, - outcome: PromptRepetitionOutcome.Remembered, - parentActionLogIDs: [ - getIDForActionLog(getActionLogFromPromptActionLog(ingestLog)), - ], - context: null, + ...initialPromptState, }, ]); - - await secondMockCall; - const updatedPromptStates = mockFunction.mock.calls[1][0] as Map< - PromptTaskID, - PromptState - >; - expect(updatedPromptStates.get(taskID)!.lastReviewTimestampMillis).toEqual( - 2000, - ); - - unsubscribe(); }); }); -describe("ingesting prompt specs", () => { - test("ingesting a basic prompt spec", async () => { - const promptTask: PromptTask = { - promptID: "test" as PromptID, - promptType: basicPromptType, - promptParameters: null, - }; - await client.recordActionLogs([ - { - actionLogType: ingestActionLogType, - taskID: getIDForPromptTask(promptTask), - timestampMillis: Date.UTC(2020, 0), - metadata: null, - }, - ]); - const cardStates = await client.getPromptStates({}); - expect(cardStates.get(getIDForPromptTask(promptTask))).toBeTruthy(); - }); - - test("ingesting a cloze prompt", async () => { - const promptTask: PromptTask = { - promptID: "test" as PromptID, - promptType: clozePromptType, - promptParameters: { clozeIndex: 2 }, - }; - await client.recordActionLogs([ - { - actionLogType: ingestActionLogType, - taskID: getIDForPromptTask(promptTask), - timestampMillis: Date.UTC(2020, 0), - metadata: null, - }, - ]); - const cardStates = await client.getPromptStates({}); - expect(cardStates.get(getIDForPromptTask(promptTask))).toBeTruthy(); - }); -}); - -test("port logs from local client", async () => { - const localClient = new MetabookLocalUserClient(); - recordTestPromptStateUpdate(localClient); - const { commit, testPromptTaskID } = recordTestPromptStateUpdate(localClient); - await commit; - const localCardStates = await localClient.getPromptStates({}); - expect(localCardStates.get(testPromptTaskID)).toBeTruthy(); - - await client.recordActionLogs(localClient.getAllLogs()); - const cardStates = await client.getPromptStates({}); - expect(cardStates.get(testPromptTaskID)).toMatchObject( - localCardStates.get(testPromptTaskID)!, - ); -}); - -test("getCardStates changes after recording update", async () => { - const initialCardStates = await client.getPromptStates({}); - await recordTestPromptStateUpdate(client).commit; - const finalCardStates = await client.getPromptStates({}); - expect(initialCardStates).not.toMatchObject(finalCardStates); -}); - test("no events after unsubscribing", async () => { const mockFunction = jest.fn(); - const firstMockCall = promiseForNextCall(mockFunction); - const unsubscribe = client.subscribeToPromptStates( - {}, + const unsubscribe = client.subscribeToActionLogs( + null, mockFunction, jest.fn(), ); - await firstMockCall; - mockFunction.mockClear(); - unsubscribe(); - await recordTestPromptStateUpdate(client).commit; expect(mockFunction).not.toHaveBeenCalled(); }); @@ -249,7 +124,7 @@ describe("security rules", () => { test("can't read cards from another user", async () => { await recordTestPromptStateUpdate(client).commit; - await expect(anotherClient.getPromptStates({})).rejects.toBeInstanceOf( + await expect(anotherClient.getDuePromptStates(1e10)).rejects.toBeInstanceOf( Error, ); }); diff --git a/metabook-client/src/userClient/firebaseClient/firebaseClient.ts b/metabook-client/src/userClient/firebaseClient/firebaseClient.ts index 8b9974d4b..eadf07afc 100644 --- a/metabook-client/src/userClient/firebaseClient/firebaseClient.ts +++ b/metabook-client/src/userClient/firebaseClient/firebaseClient.ts @@ -1,15 +1,7 @@ import firebase from "firebase/app"; import "firebase/firestore"; -import { - ActionLog, - applyActionLogToPromptState, - getIDForActionLog, - getPromptActionLogFromActionLog, - promptActionLogCanBeAppliedToPromptState, - PromptState, - PromptTaskID, -} from "metabook-core"; +import { ActionLog, getIDForActionLog } from "metabook-core"; import { ActionLogDocument, batchWriteEntries, @@ -17,26 +9,17 @@ import { getReferenceForActionLogID, getTaskStateCacheCollectionReference, PromptStateCache, + ServerTimestamp, } from "metabook-firebase-support"; import { getDefaultFirebaseApp } from "../../firebase"; import { MetabookUnsubscribe } from "../../types/unsubscribe"; -import getPromptStates from "../getPromptStates"; -import { - PromptStateQuery, - MetabookPromptStateSnapshot, - MetabookUserClient, -} from "../userClient"; +import { MetabookUserClient } from "../userClient"; type Timestamp = firebase.firestore.Timestamp; -type Subscriber = Parameters< - MetabookFirebaseUserClient["subscribeToPromptStates"] ->; - export class MetabookFirebaseUserClient implements MetabookUserClient { userID: string; private database: firebase.firestore.Firestore; - private latestRecordedServerLogTimestamp: Timestamp | null; constructor( firestore: firebase.firestore.Firestore = getDefaultFirebaseApp().firestore(), @@ -44,191 +27,96 @@ export class MetabookFirebaseUserClient implements MetabookUserClient { ) { this.userID = userID; this.database = firestore; - this.latestRecordedServerLogTimestamp = null; - } - - async getPromptStates( - query: PromptStateQuery, - ): Promise { - return getPromptStates(this, query); } - private async fetchInitialCachedPromptStates( - query: PromptStateQuery, - ): Promise> { - // TODO: rationalize unwritten logs with incoming prompt states - - const promptStateCache: Map = new Map(); - console.log("Fetching prompt states with query", query); - - let ref = getTaskStateCacheCollectionReference( - this.database, - this.userID, - ).limit(1000) as firebase.firestore.Query>; - - if (query.dueBeforeTimestampMillis) { - ref = ref.where( + async getDuePromptStates( + thresholdTimestampMillis: number, + ): Promise { + const output: PromptStateCache[] = []; + const ref = getTaskStateCacheCollectionReference(this.database, this.userID) + .limit(1000) + .where( "dueTimestampMillis", - "<", - query.dueBeforeTimestampMillis, - ); - } + "<=", + thresholdTimestampMillis, + ) as firebase.firestore.Query; let startAfter: firebase.firestore.DocumentSnapshot< - PromptStateCache + PromptStateCache > | null = null; - let usingCache = true; do { - const offsetRef: firebase.firestore.Query> = startAfter ? ref.startAfter(startAfter) : ref; + const offsetRef: firebase.firestore.Query = startAfter + ? ref.startAfter(startAfter) + : ref; // We'll try from cache first, but if we don't have anything in the cache, we'll look to the server. - let snapshot = await offsetRef.get(usingCache ? { source: "cache" } : {}); - if (snapshot.docs.length === 0 && promptStateCache.size === 0) { - console.log( - "No cached prompt states; falling back to remote prompt states.", - ); - usingCache = false; - snapshot = await offsetRef.get(); - } + const snapshot = await offsetRef.get(); for (const doc of snapshot.docs) { - const { taskID, lastLogServerTimestamp, ...promptState } = doc.data(); - promptStateCache.set(taskID as PromptTaskID, promptState); - this.latestRecordedServerLogTimestamp = - this.latestRecordedServerLogTimestamp === null || - lastLogServerTimestamp > this.latestRecordedServerLogTimestamp - ? lastLogServerTimestamp - : this.latestRecordedServerLogTimestamp; + output.push(doc.data() as PromptStateCache); } startAfter = snapshot.empty ? null : snapshot.docs[snapshot.docs.length - 1]; - console.log( - `Fetched ${promptStateCache.size} caches ${ - snapshot.metadata.fromCache ? "from cache" : "" - }`, - ); + console.log(`Fetched ${output.length} prompt states`); } while (startAfter !== null); - console.log("Done fetching prompt state caches."); - console.log( - "Latest server timestamp now", - this.latestRecordedServerLogTimestamp, - ); - - return promptStateCache; + return output; } - subscribeToPromptStates( - query: PromptStateQuery, - onPromptStatesDidUpdate: ( - newCardStates: MetabookPromptStateSnapshot, + subscribeToActionLogs( + afterServerTimestamp: ServerTimestamp | null, + onNewLogs: ( + newLogs: { log: ActionLog; serverTimestamp: ServerTimestamp }[], ) => void, onError: (error: Error) => void, ): MetabookUnsubscribe { - let didUnsubscribe = false; - let unsubscribeListener: MetabookUnsubscribe | null = null; - - this.fetchInitialCachedPromptStates(query) - .then((promptStateCache) => { - if (didUnsubscribe) { - return; - } - - let userStateLogsRef = getLogCollectionReference( - this.database, - this.userID, - ).orderBy("serverTimestamp", "asc") as firebase.firestore.Query< - ActionLogDocument - >; - if (this.latestRecordedServerLogTimestamp) { - userStateLogsRef = userStateLogsRef.where( - "serverTimestamp", - ">", - this.latestRecordedServerLogTimestamp, - ); - } + let userStateLogsRef = getLogCollectionReference( + this.database, + this.userID, + ).orderBy("serverTimestamp", "asc") as firebase.firestore.Query< + ActionLogDocument + >; + if (afterServerTimestamp) { + userStateLogsRef = userStateLogsRef.where( + "serverTimestamp", + ">", + afterServerTimestamp, + ); + } - console.log("Subscribing to logs", Date.now()); - unsubscribeListener = userStateLogsRef.onSnapshot((snapshot) => { - console.log("Got log snapshot of size", snapshot.size, Date.now()); - for (const change of snapshot.docChanges()) { - const log = change.doc.data({ serverTimestamps: "none" }); - if ( - log.serverTimestamp && - (!this.latestRecordedServerLogTimestamp || - log.serverTimestamp > this.latestRecordedServerLogTimestamp) - ) { - this.latestRecordedServerLogTimestamp = log.serverTimestamp; - console.log(`Latest server timestamp now`, log.serverTimestamp); - } - switch (change.type) { - case "added": - this.updateCacheForLog(promptStateCache, log); - break; - case "modified": - console.log("Ignoring modification to log", change.doc.id, log); - break; - case "removed": - // TODO make more robust against client failures to persist / sync - throw new Error( - `Log entries should never disappear. Unsupported change type ${ - change.type - } with doc ${change.doc.data()}`, - ); - } + return userStateLogsRef.onSnapshot((snapshot) => { + const newLogs: { + log: ActionLog; + serverTimestamp: ServerTimestamp; + }[] = []; + for (const change of snapshot.docChanges()) { + if (!change.doc.metadata.hasPendingWrites) { + switch (change.type) { + case "added": + case "modified": + const { + serverTimestamp, + suppressTaskStateCacheUpdate, + ...log + } = change.doc.data({ serverTimestamps: "none" }); + newLogs.push({ log, serverTimestamp }); + break; + case "removed": + // TODO make more robust against client failures to persist / sync + throw new Error( + `Log entries shouldn't change after their creation. Unsupported change type ${ + change.type + } with doc ${change.doc.data()}`, + ); } - onPromptStatesDidUpdate(new Map(promptStateCache)); - }, onError); - }) - .catch((error) => { - onError(error); - }); - - return () => { - didUnsubscribe = true; - unsubscribeListener?.(); - }; - } - - private updateCacheForLog( - promptStateCache: Map, - log: ActionLog, - ): void { - const promptActionLog = getPromptActionLogFromActionLog(log); - const cachedPromptState = - promptStateCache.get(promptActionLog.taskID) ?? null; - if ( - promptActionLogCanBeAppliedToPromptState( - promptActionLog, - cachedPromptState, - ) - ) { - const newPromptState = applyActionLogToPromptState({ - promptActionLog, - basePromptState: cachedPromptState, - schedule: "default", - }); - if (newPromptState instanceof Error) { - throw newPromptState; + } } - promptStateCache.set(promptActionLog.taskID, newPromptState); - } else { - console.warn( - `Unsupported prompt log addition. New log heads: ${JSON.stringify( - log, - null, - "\t", - )}. Cached prompt state: ${JSON.stringify( - cachedPromptState, - null, - "\t", - )}`, - ); - } + if (newLogs.length > 0) { + onNewLogs(newLogs); + } + }, onError); } async recordActionLogs(logs: ActionLog[]): Promise { diff --git a/metabook-client/src/userClient/localClient/index.ts b/metabook-client/src/userClient/localClient/index.ts deleted file mode 100644 index 3dbe624bd..000000000 --- a/metabook-client/src/userClient/localClient/index.ts +++ /dev/null @@ -1 +0,0 @@ -export * from "./localClient"; diff --git a/metabook-client/src/userClient/localClient/localClient.test.ts b/metabook-client/src/userClient/localClient/localClient.test.ts deleted file mode 100644 index 354c036f2..000000000 --- a/metabook-client/src/userClient/localClient/localClient.test.ts +++ /dev/null @@ -1,75 +0,0 @@ -import { - applyActionLogToPromptState, - PromptTaskID, - PromptState, -} from "metabook-core"; -import { promiseForNextCall } from "../../util/tests/promiseForNextCall"; -import { recordTestPromptStateUpdate } from "../../util/tests/recordTestPromptStateUpdate"; -import { MetabookLocalUserClient } from "./localClient"; - -let client: MetabookLocalUserClient; -beforeEach(() => { - client = new MetabookLocalUserClient(); -}); - -test("recording a marking triggers card state update", async () => { - const mockFunction = jest.fn(); - const firstMockCall = promiseForNextCall(mockFunction); - client.subscribeToPromptStates({}, mockFunction, (error) => { - fail(error); - }); - await firstMockCall; - expect(mockFunction).toHaveBeenCalledWith(new Map()); - - const secondMockCall = promiseForNextCall(mockFunction); - jest.spyOn(Math, "random").mockReturnValue(0.25); - const { - commit, - testPromptActionLog, - testPromptTaskID, - } = recordTestPromptStateUpdate(client); - await commit; - - const updatedCardStates = (await secondMockCall) as Map< - PromptTaskID, - PromptState - >; - expect(updatedCardStates.get(testPromptTaskID)).toMatchObject( - applyActionLogToPromptState({ - basePromptState: null, - promptActionLog: testPromptActionLog, - schedule: "default", - }), - ); -}); - -test("getCardStates changes after recording update", async () => { - const initialCardStates = await client.getPromptStates({}); - await recordTestPromptStateUpdate(client).commit; - const finalCardStates = await client.getPromptStates({}); - expect(initialCardStates).not.toMatchObject(finalCardStates); -}); - -test("logs reflect updates", async () => { - expect(client.getAllLogs()).toHaveLength(0); - await recordTestPromptStateUpdate(client).commit; - await recordTestPromptStateUpdate(client).commit; - expect(client.getAllLogs()).toHaveLength(2); -}); - -test("no events after unsubscribing", async () => { - const mockFunction = jest.fn(); - const firstMockCall = promiseForNextCall(mockFunction); - const unsubscribe = client.subscribeToPromptStates( - {}, - mockFunction, - jest.fn(), - ); - await firstMockCall; - mockFunction.mockClear(); - - unsubscribe(); - - await recordTestPromptStateUpdate(client).commit; - expect(mockFunction).not.toHaveBeenCalled(); -}); diff --git a/metabook-client/src/userClient/localClient/localClient.ts b/metabook-client/src/userClient/localClient/localClient.ts deleted file mode 100644 index 58443edcd..000000000 --- a/metabook-client/src/userClient/localClient/localClient.ts +++ /dev/null @@ -1,95 +0,0 @@ -import { - ActionLog, - applyActionLogToPromptState, - getPromptActionLogFromActionLog, - promptActionLogCanBeAppliedToPromptState, - PromptState, - PromptTaskID, -} from "metabook-core"; -import { MetabookUnsubscribe } from "../../types/unsubscribe"; -import getPromptStates from "../getPromptStates"; -import { - PromptStateQuery, - MetabookPromptStateSnapshot, - MetabookUserClient, -} from "../userClient"; - -export class MetabookLocalUserClient implements MetabookUserClient { - private readonly latestPromptStates: Map; - private readonly cardStateSubscribers: Set; - private readonly logs: ActionLog[]; - - constructor() { - this.latestPromptStates = new Map(); - this.cardStateSubscribers = new Set(); - this.logs = []; - } - - getPromptStates( - query: PromptStateQuery, - ): Promise { - return getPromptStates(this, query); - } - - async recordActionLogs(logs: ActionLog[]): Promise { - for (const log of logs) { - const promptActionLog = getPromptActionLogFromActionLog(log); - const cachedPromptState = - this.latestPromptStates.get(promptActionLog.taskID) ?? null; - if ( - promptActionLogCanBeAppliedToPromptState( - promptActionLog, - cachedPromptState, - ) - ) { - const newPromptState = applyActionLogToPromptState({ - promptActionLog, - basePromptState: cachedPromptState, - schedule: "default", - }); - if (newPromptState instanceof Error) { - throw newPromptState; - } - this.latestPromptStates.set(promptActionLog.taskID, newPromptState); - } - } - this.logs.push(...logs); - - return new Promise((resolve) => { - // We return control to the caller before calling subscribers and resolving. - setTimeout(() => { - this.notifyCardStateSubscribers(); - resolve(); - }, 0); - }); - } - - subscribeToPromptStates( - query: PromptStateQuery, - onCardStatesDidUpdate: (newCardStates: MetabookPromptStateSnapshot) => void, - onError: (error: Error) => void, - ): MetabookUnsubscribe { - onCardStatesDidUpdate(new Map(this.latestPromptStates)); - const subscriber = { query, onCardStatesDidUpdate, onError }; - this.cardStateSubscribers.add(subscriber); - return () => { - this.cardStateSubscribers.delete(subscriber); - }; - } - - getAllLogs(): ActionLog[] { - return [...this.logs]; - } - - private notifyCardStateSubscribers() { - for (const { onCardStatesDidUpdate } of this.cardStateSubscribers) { - onCardStatesDidUpdate(new Map(this.latestPromptStates)); - } - } -} - -interface CardStateSubscriber { - query: PromptStateQuery; - onCardStatesDidUpdate: (newCardStates: MetabookPromptStateSnapshot) => void; - onError: (error: Error) => void; -} diff --git a/metabook-client/src/userClient/userClient.ts b/metabook-client/src/userClient/userClient.ts index e122ad07e..9a4e886fd 100644 --- a/metabook-client/src/userClient/userClient.ts +++ b/metabook-client/src/userClient/userClient.ts @@ -1,28 +1,23 @@ import { ActionLog, PromptState, PromptTaskID } from "metabook-core"; +import { PromptStateCache, ServerTimestamp } from "metabook-firebase-support"; import { MetabookUnsubscribe } from "../types/unsubscribe"; export interface MetabookUserClient { - subscribeToPromptStates( - query: PromptStateQuery, - onPromptStatesDidUpdate: ( - newPromptStates: MetabookPromptStateSnapshot, + getDuePromptStates( + thresholdTimestampMillis: number, + ): Promise; + + subscribeToActionLogs( + afterServerTimestamp: ServerTimestamp | null, + onNewLogs: ( + newLogs: { log: ActionLog; serverTimestamp: ServerTimestamp }[], ) => void, onError: (error: Error) => void, ): MetabookUnsubscribe; - getPromptStates( - query: PromptStateQuery, - ): Promise; - recordActionLogs(logs: ActionLog[]): Promise; } -// TODO -// eslint-disable-next-line @typescript-eslint/no-empty-interface -export interface PromptStateQuery { - dueBeforeTimestampMillis?: number; -} - export type MetabookPromptStateSnapshot = ReadonlyMap< PromptTaskID, PromptState diff --git a/metabook-firebase-support/src/actionLogDocument.ts b/metabook-firebase-support/src/actionLogDocument.ts index 05c0197b9..f5bc84469 100644 --- a/metabook-firebase-support/src/actionLogDocument.ts +++ b/metabook-firebase-support/src/actionLogDocument.ts @@ -1,7 +1,7 @@ import { ActionLog } from "metabook-core"; -import { Timestamp } from "./libraryAbstraction"; +import { ServerTimestamp } from "./libraryAbstraction"; -export type ActionLogDocument = ActionLog & { - serverTimestamp: Timestamp; +export type ActionLogDocument = ActionLog & { + serverTimestamp: ServerTimestamp; suppressTaskStateCacheUpdate?: boolean; }; diff --git a/metabook-firebase-support/src/batchWriteEntries.ts b/metabook-firebase-support/src/batchWriteEntries.ts index 635c1ae20..1b5cb760c 100644 --- a/metabook-firebase-support/src/batchWriteEntries.ts +++ b/metabook-firebase-support/src/batchWriteEntries.ts @@ -1,10 +1,14 @@ -import { Database, DocumentReference, Timestamp } from "./libraryAbstraction"; +import { + Database, + DocumentReference, + ServerTimestamp, +} from "./libraryAbstraction"; const batchSize = 250; export default async function batchWriteEntries< D extends Database, - T extends Timestamp + T extends ServerTimestamp >( // eslint-disable-next-line @typescript-eslint/no-explicit-any logEntries: [DocumentReference, any][], diff --git a/metabook-firebase-support/src/index.ts b/metabook-firebase-support/src/index.ts index 40a9e7bfe..dbb036c70 100644 --- a/metabook-firebase-support/src/index.ts +++ b/metabook-firebase-support/src/index.ts @@ -3,3 +3,4 @@ export * from "./dataRecord"; export { default as batchWriteEntries } from "./batchWriteEntries"; export type { PromptStateCache } from "./promptStateCache"; export type { ActionLogDocument } from "./actionLogDocument"; +export type { ServerTimestamp } from "./libraryAbstraction"; diff --git a/metabook-firebase-support/src/libraryAbstraction.ts b/metabook-firebase-support/src/libraryAbstraction.ts index a9ee11459..bac390ce6 100644 --- a/metabook-firebase-support/src/libraryAbstraction.ts +++ b/metabook-firebase-support/src/libraryAbstraction.ts @@ -9,4 +9,7 @@ export type DocumentReference = ReturnType< CollectionReference["doc"] >; -export type Timestamp = ClientFirestore.Timestamp | AdminFirestore.Timestamp; +export interface ServerTimestamp { + seconds: number; + nanoseconds: number; +} diff --git a/metabook-firebase-support/src/promptStateCache.ts b/metabook-firebase-support/src/promptStateCache.ts index 17a4329a0..fac2f34b5 100644 --- a/metabook-firebase-support/src/promptStateCache.ts +++ b/metabook-firebase-support/src/promptStateCache.ts @@ -1,7 +1,7 @@ -import { PromptState } from "metabook-core"; -import { Timestamp } from "./libraryAbstraction"; +import { PromptState, PromptTaskID } from "metabook-core"; +import { ServerTimestamp } from "./libraryAbstraction"; -export interface PromptStateCache extends PromptState { - taskID: string; - lastLogServerTimestamp: T; +export interface PromptStateCache extends PromptState { + taskID: PromptTaskID; + lastLogServerTimestamp: ServerTimestamp; } diff --git a/yarn.lock b/yarn.lock index 9256621c0..90faa1982 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3981,6 +3981,11 @@ resolved "https://registry.yarnpkg.com/@types/json-schema/-/json-schema-7.0.4.tgz#38fd73ddfd9b55abb1e1b2ed578cb55bd7b7d339" integrity sha512-8+KAKzEvSUdeo+kmqnKrqgeE+LcA0tjYWFY7RPProVYwnqDjukzO+3b6dLD56rYX5TdWejnEOLJYOIeh4CXKuA== +"@types/level-codec@*": + version "9.0.0" + resolved "https://registry.yarnpkg.com/@types/level-codec/-/level-codec-9.0.0.tgz#9f1dc7f9017b6fba094a450602ec0b91cc384059" + integrity sha512-SWYkVJylo1dqblkhrr7UtmsQh4wdZA9bV1y3QJSywMPSqGfW0p1w37N1EayZtKbg1dGReIIQEEOtxk4wZvGrWQ== + "@types/level-js@^4.0.1": version "4.0.1" resolved "https://registry.yarnpkg.com/@types/level-js/-/level-js-4.0.1.tgz#e094b684e546b80346b2cec0562c4216d8d29eeb" @@ -3988,7 +3993,7 @@ dependencies: "@types/abstract-leveldown" "*" -"@types/levelup@^4.3.0": +"@types/levelup@*", "@types/levelup@^4.3.0": version "4.3.0" resolved "https://registry.yarnpkg.com/@types/levelup/-/levelup-4.3.0.tgz#4f55585e05a33caa08c1439c344bbba93e947327" integrity sha512-h82BoajhjU/zwLoM4BUBX/SCodCFi1ae/ZlFOYh5Z4GbHeaXj9H709fF1LYl/StrK8KSwnJOeMRPo9lnC6sz4w== @@ -3996,6 +4001,11 @@ "@types/abstract-leveldown" "*" "@types/node" "*" +"@types/lexicographic-integer@^1.1.0": + version "1.1.0" + resolved "https://registry.yarnpkg.com/@types/lexicographic-integer/-/lexicographic-integer-1.1.0.tgz#726bddd8f38fc5e365b3cdcc421a27ae0f0e62fd" + integrity sha512-0HDnRPXHJM/qUW3qlthCj2i1DQ0nHz21hnB4z8kZGzGc4tvIOptnNQGwl1sOAayXzwG1+2LQRlwJD/tncQTgTw== + "@types/linkify-it@*": version "2.1.0" resolved "https://registry.yarnpkg.com/@types/linkify-it/-/linkify-it-2.1.0.tgz#ea3dd64c4805597311790b61e872cbd1ed2cd806" @@ -4220,6 +4230,15 @@ resolved "https://registry.yarnpkg.com/@types/stack-utils/-/stack-utils-1.0.1.tgz#0a851d3bd96498fa25c33ab7278ed3bd65f06c3e" integrity sha512-l42BggppR6zLmpfU6fq9HEa2oGPEI8yrSPL3GITjfRInppYFahObbIQOQK3UGxEnyQpltZLaPe75046NOZQikw== +"@types/subleveldown@^4.1.0": + version "4.1.0" + resolved "https://registry.yarnpkg.com/@types/subleveldown/-/subleveldown-4.1.0.tgz#9ba88db89de9e24fc4dbbd570f3b19901e0191c2" + integrity sha512-nAhIJJyrC23eDrPR29uu/G6p6TTE9J42dw6wtkjhRN+LfztTli7BDJsGIQMQma9zGpimwIA51CV/UGD/a3LhrA== + dependencies: + "@types/abstract-leveldown" "*" + "@types/level-codec" "*" + "@types/levelup" "*" + "@types/tapable@*": version "1.0.5" resolved "https://registry.yarnpkg.com/@types/tapable/-/tapable-1.0.5.tgz#9adbc12950582aa65ead76bffdf39fe0c27a3c02" @@ -4562,6 +4581,17 @@ absolute-path@^0.0.0: resolved "https://registry.yarnpkg.com/absolute-path/-/absolute-path-0.0.0.tgz#a78762fbdadfb5297be99b15d35a785b2f095bf7" integrity sha1-p4di+9rftSl76ZsV01p4Wy8JW/c= +abstract-leveldown@^6.2.1, abstract-leveldown@^6.3.0: + version "6.3.0" + resolved "https://registry.yarnpkg.com/abstract-leveldown/-/abstract-leveldown-6.3.0.tgz#d25221d1e6612f820c35963ba4bd739928f6026a" + integrity sha512-TU5nlYgta8YrBMNpc9FwQzRbiXsj49gsALsXadbGHt9CROPzX5fB0rWDR5mtdpOOKa5XqRFpbj1QroPAoPzVjQ== + dependencies: + buffer "^5.5.0" + immediate "^3.2.3" + level-concat-iterator "~2.0.0" + level-supports "~1.0.0" + xtend "~4.0.0" + abstract-leveldown@~6.2.1, abstract-leveldown@~6.2.3: version "6.2.3" resolved "https://registry.yarnpkg.com/abstract-leveldown/-/abstract-leveldown-6.2.3.tgz#036543d87e3710f2528e47040bc3261b77a9a8eb" @@ -8155,6 +8185,11 @@ define-property@^2.0.2: is-descriptor "^1.0.2" isobject "^3.0.1" +defined@~0.0.0: + version "0.0.0" + resolved "https://registry.yarnpkg.com/defined/-/defined-0.0.0.tgz#f35eea7d705e933baf13b2f03b3f83d921403b3e" + integrity sha1-817qfXBekzuvE7LwOz+D2SFAOz4= + del@^4.1.1: version "4.1.1" resolved "https://registry.yarnpkg.com/del/-/del-4.1.1.tgz#9e8f117222ea44a31ff3a156c049b99052a9f0b4" @@ -8580,6 +8615,16 @@ encodeurl@~1.0.2: resolved "https://registry.yarnpkg.com/encodeurl/-/encodeurl-1.0.2.tgz#ad3ff4c86ec2d029322f5a02c3a9a606c95b3f59" integrity sha1-rT/0yG7C0CkyL1oCw6mmBslbP1k= +encoding-down@^6.2.0: + version "6.3.0" + resolved "https://registry.yarnpkg.com/encoding-down/-/encoding-down-6.3.0.tgz#b1c4eb0e1728c146ecaef8e32963c549e76d082b" + integrity sha512-QKrV0iKR6MZVJV08QY0wp1e7vF6QbhnbQhb07bwpEyuz4uZiZgPlEGdkCROuFkUwdxlFaiPIhjyarH1ee/3vhw== + dependencies: + abstract-leveldown "^6.2.1" + inherits "^2.0.3" + level-codec "^9.0.0" + level-errors "^2.0.0" + encoding@^0.1.11: version "0.1.12" resolved "https://registry.yarnpkg.com/encoding/-/encoding-0.1.12.tgz#538b66f3ee62cd1ab51ec323829d1f9480c74beb" @@ -9124,6 +9169,11 @@ execa@^3.2.0: signal-exit "^3.0.2" strip-final-newline "^2.0.0" +existy@^1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/existy/-/existy-1.0.1.tgz#31ae2a103e658c001aed68f09cf3468dcc6d81e5" + integrity sha1-Ma4qED5ljAAa7WjwnPNGjcxtgeU= + exit@^0.1.2: version "0.1.2" resolved "https://registry.yarnpkg.com/exit/-/exit-0.1.2.tgz#0632638f8d877cc82107d30a0fff1a17cba1cd0c" @@ -14401,18 +14451,43 @@ left-pad@^1.3.0: resolved "https://registry.yarnpkg.com/left-pad/-/left-pad-1.3.0.tgz#5b8a3a7765dfe001261dde915589e782f8c94d1e" integrity sha512-XI5MPzVNApjAyhQzphX8BkmKsKUxD4LdyK24iZeQGinBN9yTQT3bFlCBy/aVx2HrNcqQGsdot8ghrjyrvMCoEA== +level-auto-index@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/level-auto-index/-/level-auto-index-2.0.0.tgz#d849be1c957de93e7dfce4224e92dd70b4d17840" + integrity sha512-sed5xe+hTI0uktAF2OvqpPJuz7JUIJ7NturXzMHDvb72WG2nId6KpB5/fzSj3ZcctHK7gB9xqbzVM6ZjpnKnlg== + dependencies: + existy "^1.0.1" + level-hookdown "^3.0.0" + readable-stream "^3.4.0" + xtend "^4.0.2" + +level-codec@^9.0.0: + version "9.0.1" + resolved "https://registry.yarnpkg.com/level-codec/-/level-codec-9.0.1.tgz#042f4aa85e56d4328ace368c950811ba802b7247" + integrity sha512-ajFP0kJ+nyq4i6kptSM+mAvJKLOg1X5FiFPtLG9M5gCEZyBmgDi3FkDrvlMkEzrUn1cWxtvVmrvoS4ASyO/q+Q== + level-concat-iterator@~2.0.0: version "2.0.1" resolved "https://registry.yarnpkg.com/level-concat-iterator/-/level-concat-iterator-2.0.1.tgz#1d1009cf108340252cb38c51f9727311193e6263" integrity sha512-OTKKOqeav2QWcERMJR7IS9CUo1sHnke2C0gkSmcR7QuEtFNLLzHQAvnMw8ykvEcv0Qtkg0p7FOwP1v9e5Smdcw== -level-errors@~2.0.0: +level-errors@^2.0.0, level-errors@~2.0.0: version "2.0.1" resolved "https://registry.yarnpkg.com/level-errors/-/level-errors-2.0.1.tgz#2132a677bf4e679ce029f517c2f17432800c05c8" integrity sha512-UVprBJXite4gPS+3VznfgDSU8PTRuVX0NXwoWW50KLxd2yw4Y1t2JUR5In1itQnudZqRMT9DlAM3Q//9NCjCFw== dependencies: errno "~0.1.1" +level-hookdown@^3.0.0: + version "3.1.2" + resolved "https://registry.yarnpkg.com/level-hookdown/-/level-hookdown-3.1.2.tgz#78d21a8a05f510cb7639535317873fad98489c86" + integrity sha512-oYR+Q/72ai+XMiM+KmM2rFUdq0yzW8sqwEwnvBvTUW+03hXkYv3zq3Cqa9aIbVyxsyG/VmYjB6hSoLz6Gg6INg== + dependencies: + run-parallel "^1.1.9" + run-parallel-limit "^1.0.5" + run-series "^1.1.8" + xtend "^4.0.2" + level-iterator-stream@~4.0.0: version "4.0.2" resolved "https://registry.yarnpkg.com/level-iterator-stream/-/level-iterator-stream-4.0.2.tgz#7ceba69b713b0d7e22fcc0d1f128ccdc8a24f79c" @@ -14432,6 +14507,13 @@ level-js@^5.0.2: inherits "^2.0.3" ltgt "^2.1.2" +level-option-wrap@^1.1.0: + version "1.1.0" + resolved "https://registry.yarnpkg.com/level-option-wrap/-/level-option-wrap-1.1.0.tgz#ad20e68d9f3c22c8897531cc6aa7af596b1ed129" + integrity sha1-rSDmjZ88IsiJdTHMaqevWWse0Sk= + dependencies: + defined "~0.0.0" + level-supports@~1.0.0: version "1.0.1" resolved "https://registry.yarnpkg.com/level-supports/-/level-supports-1.0.1.tgz#2f530a596834c7301622521988e2c36bb77d122d" @@ -14470,6 +14552,11 @@ levn@^0.3.0, levn@~0.3.0: prelude-ls "~1.1.2" type-check "~0.3.2" +lexicographic-integer@^1.1.0: + version "1.1.0" + resolved "https://registry.yarnpkg.com/lexicographic-integer/-/lexicographic-integer-1.1.0.tgz#52ca6d998a572e6322b515f5b80e396c6043e9b8" + integrity sha1-UsptmYpXLmMitRX1uA45bGBD6bg= + lines-and-columns@^1.1.6: version "1.1.6" resolved "https://registry.yarnpkg.com/lines-and-columns/-/lines-and-columns-1.1.6.tgz#1c00c743b433cd0a4e80758f7b64a57440d9ff00" @@ -18029,6 +18116,11 @@ rc@^1.2.7: minimist "^1.2.0" strip-json-comments "~2.0.1" +reachdown@^1.1.0: + version "1.1.0" + resolved "https://registry.yarnpkg.com/reachdown/-/reachdown-1.1.0.tgz#c3b85b459dbd0fe2c79782233a0a38e66a9b5454" + integrity sha512-6LsdRe4cZyOjw4NnvbhUd/rGG7WQ9HMopPr+kyL018Uci4kijtxcGR5kVb5Ln13k4PEE+fEFQbjfOvNw7cnXmA== + react-addons-create-fragment@^15.6.2: version "15.6.2" resolved "https://registry.yarnpkg.com/react-addons-create-fragment/-/react-addons-create-fragment-15.6.2.tgz#a394de7c2c7becd6b5475ba1b97ac472ce7c74f8" @@ -18666,7 +18758,7 @@ read-pkg@^5.2.0: string_decoder "^1.1.1" util-deprecate "^1.0.1" -readable-stream@^1.0.27-1, readable-stream@^1.0.33: +readable-stream@^1.0.33: version "1.1.14" resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-1.1.14.tgz#7cf4c54ef648e3813084c636dd2079e166c081d9" integrity sha1-fPTFTvZI44EwhMY23SB54WbAgdk= @@ -19285,6 +19377,16 @@ run-async@^2.2.0, run-async@^2.4.0: dependencies: is-promise "^2.1.0" +run-parallel-limit@^1.0.5: + version "1.0.5" + resolved "https://registry.yarnpkg.com/run-parallel-limit/-/run-parallel-limit-1.0.5.tgz#c29a4fd17b4df358cb52a8a697811a63c984f1b7" + integrity sha512-NsY+oDngvrvMxKB3G8ijBzIema6aYbQMD2bHOamvN52BysbIGTnEY2xsNyfrcr9GhY995/t/0nQN3R3oZvaDlg== + +run-parallel@^1.1.9: + version "1.1.9" + resolved "https://registry.yarnpkg.com/run-parallel/-/run-parallel-1.1.9.tgz#c9dd3a7cf9f4b2c4b6244e173a6ed866e61dd679" + integrity sha512-DEqnSRTDw/Tc3FXf49zedI638Z9onwUotBMiUFKmrO2sdFKIbXamXGQ3Axd4qgphxKB4kw/qP1w5kTxnfU1B9Q== + run-queue@^1.0.0, run-queue@^1.0.3: version "1.0.3" resolved "https://registry.yarnpkg.com/run-queue/-/run-queue-1.0.3.tgz#e848396f057d223f24386924618e25694161ec47" @@ -19292,6 +19394,11 @@ run-queue@^1.0.0, run-queue@^1.0.3: dependencies: aproba "^1.1.1" +run-series@^1.1.8: + version "1.1.8" + resolved "https://registry.yarnpkg.com/run-series/-/run-series-1.1.8.tgz#2c4558f49221e01cd6371ff4e0a1e203e460fc36" + integrity sha512-+GztYEPRpIsQoCSraWHDBs9WVy4eVME16zhOtDB4H9J4xN0XRhknnmLOl+4gRgZtu8dpp9N/utSPjKH/xmDzXg== + rx-lite-aggregates@^4.0.8: version "4.0.8" resolved "https://registry.yarnpkg.com/rx-lite-aggregates/-/rx-lite-aggregates-4.0.8.tgz#753b87a89a11c95467c4ac1626c4efc4e05c67be" @@ -20041,14 +20148,6 @@ store2@^2.7.1: resolved "https://registry.yarnpkg.com/store2/-/store2-2.10.0.tgz#46b82bb91878daf1b0d56dec2f1d41e54d5103cf" integrity sha512-tWEpK0snS2RPUq1i3R6OahfJNjWCQYNxq0+by1amCSuw0mXtymJpzmZIeYpA1UAa+7B0grCpNYIbDcd7AgTbFg== -stream-browserify@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/stream-browserify/-/stream-browserify-1.0.0.tgz#bf9b4abfb42b274d751479e44e0ff2656b6f1193" - integrity sha1-v5tKv7QrJ011FHnkTg/yZWtvEZM= - dependencies: - inherits "~2.0.1" - readable-stream "^1.0.27-1" - stream-browserify@^2.0.1: version "2.0.2" resolved "https://registry.yarnpkg.com/stream-browserify/-/stream-browserify-2.0.2.tgz#87521d38a44aa7ee91ce1cd2a47df0cb49dd660b" @@ -20340,6 +20439,17 @@ stylehacks@^4.0.0: postcss "^7.0.0" postcss-selector-parser "^3.0.0" +subleveldown@andymatuschak/subleveldown: + version "5.0.0" + resolved "https://codeload.github.com/andymatuschak/subleveldown/tar.gz/41dd28eb78ff03b40a533f47cc73316e6db3efb2" + dependencies: + abstract-leveldown "^6.3.0" + encoding-down "^6.2.0" + inherits "^2.0.3" + level-option-wrap "^1.1.0" + levelup "^4.4.0" + reachdown "^1.1.0" + sudo-prompt@^9.0.0: version "9.1.1" resolved "https://registry.yarnpkg.com/sudo-prompt/-/sudo-prompt-9.1.1.tgz#73853d729770392caec029e2470db9c221754db0"