From 6a03931a0d25f4c011a316e29d7508dd01bf4f63 Mon Sep 17 00:00:00 2001 From: "g. nicholas d'andrea" Date: Fri, 14 May 2021 22:26:45 -0400 Subject: [PATCH] Clarify data marshaling boundary - Use `_id` in Adapter instead of `id` - Redefine Historical to include all the PouchDB properties - Add concept of internal properties, to begin with $ - Implement marshal/unmarshal methods in AdapterWorkspace, to handle adding/removing properties, i.e. translating between Historical> and SavedInput - Move Historical generic into meta/pouch/types rather than alongside Workspace --- packages/db/src/meta/data.ts | 12 +- packages/db/src/meta/pouch/adapters/base.ts | 150 +++++++++----------- packages/db/src/meta/pouch/types.ts | 19 +-- packages/db/src/meta/pouch/workspace.ts | 118 +++++++++------ 4 files changed, 156 insertions(+), 143 deletions(-) diff --git a/packages/db/src/meta/data.ts b/packages/db/src/meta/data.ts index af441ba6297..2396d9553b4 100644 --- a/packages/db/src/meta/data.ts +++ b/packages/db/src/meta/data.ts @@ -26,7 +26,7 @@ export interface Workspace { get>( collectionName: N, id: string | undefined - ): Promise> | undefined>; + ): Promise | undefined>; add>( collectionName: N, @@ -43,13 +43,3 @@ export interface Workspace { input: MutationInput ): Promise; } - -export type History = Pick; - -export type Historical = { - [K in keyof T | keyof History]: K extends keyof History - ? History[K] - : K extends keyof T - ? T[K] - : never; -}; diff --git a/packages/db/src/meta/pouch/adapters/base.ts b/packages/db/src/meta/pouch/adapters/base.ts index 5dd8c6fd8ee..766f3c834b1 100644 --- a/packages/db/src/meta/pouch/adapters/base.ts +++ b/packages/db/src/meta/pouch/adapters/base.ts @@ -5,14 +5,11 @@ import PouchDB from "pouchdb"; import PouchDBDebug from "pouchdb-debug"; import PouchDBFind from "pouchdb-find"; -import type { - CollectionName, - Collections, -} from "@truffle/db/meta/collections"; +import type { CollectionName, Collections } from "@truffle/db/meta/collections"; import * as Id from "@truffle/db/meta/id"; -import type { Historical } from "@truffle/db/meta/data"; import type { + Historical, Adapter, Definition, Definitions @@ -81,9 +78,10 @@ export abstract class Databases implements Adapter { } } - public async every, I extends { id: string }>( - collectionName: N - ): Promise[]> { + public async every< + N extends CollectionName, + I extends PouchDB.Core.IdMeta + >(collectionName: N): Promise[]> { await this.ready; const { rows }: any = await this.collections[collectionName].allDocs({ @@ -91,26 +89,25 @@ export abstract class Databases implements Adapter { }); const result = rows - // make sure we include `id` in the response as well - .map(({ doc }) => ({ ...doc, id: doc["_id"] })) - // but filter out any views + .map(({ doc }) => doc) + // filter out any views .filter(({ views }) => !views); return result; } - public async retrieve, I extends { id: string }>( - collectionName: N, - references: (Pick | undefined)[] - ) { + public async retrieve< + N extends CollectionName, + I extends PouchDB.Core.IdMeta + >(collectionName: N, references: (Pick | undefined)[]) { await this.ready; const unordered = await this.search(collectionName, { selector: { - id: { + _id: { $in: references - .filter((obj): obj is Pick => !!obj) - .map(({ id }) => id) + .filter((obj): obj is Pick => !!obj) + .map(({ _id }) => _id) } } }); @@ -120,21 +117,21 @@ export abstract class Databases implements Adapter { savedInput ? { ...byId, - [savedInput.id as string]: savedInput + [savedInput._id as string]: savedInput } : byId, {} as { [id: string]: Historical } ); return references.map(reference => - reference ? byId[reference.id] : undefined + reference ? byId[reference._id] : undefined ); } - public async search, I extends { id: string }>( - collectionName: N, - options: PouchDB.Find.FindRequest<{}> - ) { + public async search< + N extends CollectionName, + I extends PouchDB.Core.IdMeta + >(collectionName: N, options: PouchDB.Find.FindRequest<{}>) { await this.ready; // allows searching with `id` instead of pouch's internal `_id`, @@ -153,25 +150,15 @@ export abstract class Databases implements Adapter { selector: fixIdSelector(options.selector) }); - // make sure we include `id` in the response as well - const result: Historical[] = docs.map(doc => { - const { - _id, - _rev, - ...retrievedInput - } = doc; - - return { - ...retrievedInput, - _rev, - id: _id - }; - }); + const result: Historical[] = docs; return result; } - public async record, I extends { id: string }>( + public async record< + N extends CollectionName, + I extends PouchDB.Core.IdMeta + >( collectionName: N, inputs: (I | undefined)[], options: { overwrite?: boolean } = {} @@ -181,68 +168,69 @@ export abstract class Databases implements Adapter { const { overwrite = false } = options; const inputsById: { - [id: string]: I + [id: string]: I; } = inputs .filter((input): input is I => !!input) - .map((input) => ({ - [input.id]: input + .map(input => ({ + [input._id]: input })) .reduce((a, b) => ({ ...a, ...b }), {} as { [id: string]: I }); const currentInputsById: { [id: string]: Historical; - } = (await this.retrieve( - collectionName, - Object.keys(inputsById).map(id => ({ id })) - )) + } = ( + await this.retrieve( + collectionName, + Object.keys(inputsById).map(_id => ({ _id })) + ) + ) .filter((currentInput): currentInput is Historical => !!currentInput) - .map(currentInput => ({ [currentInput.id]: currentInput })) + .map(currentInput => ({ [currentInput._id]: currentInput })) .reduce( (a, b) => ({ ...a, ...b }), {} as { [id: string]: Historical } ); const savedInputsById: { - [id: string]: Historical + [id: string]: Historical; } = ( await Promise.all( - Object.entries(inputsById) - .map(async ([id, input]) => { - const currentInput = currentInputsById[id]; + Object.entries(inputsById).map(async ([_id, input]) => { + const currentInput = currentInputsById[_id]; - if (currentInput && !overwrite) { - return currentInput; - } + if (currentInput && !overwrite) { + return currentInput; + } - const { _rev = undefined } = currentInput || {}; + const { _rev = undefined } = currentInput || {}; - const { rev } = await this.collections[collectionName].put({ - ...input, - _rev, - _id: id - }); + const { rev } = await this.collections[collectionName].put({ + ...input, + _rev, + _id + }); - return { - ...input, - _rev: rev, - id - } as Historical; - }) + return { + ...input, + _rev: rev, + _id + } as Historical; + }) ) ) - .map(savedInput => ({ [savedInput.id]: savedInput })) + .map(savedInput => ({ [savedInput._id]: savedInput })) .reduce( (a, b) => ({ ...a, ...b }), {} as { [id: string]: Historical } ); - return inputs.map(input => input && savedInputsById[input.id]); + return inputs.map(input => input && savedInputsById[input._id]); } - public async forget, I extends { id: string }>( - collectionName: N, - references: (Pick | undefined)[] - ) { + public async forget< + N extends CollectionName, + I extends PouchDB.Core.IdMeta + >(collectionName: N, references: (Pick | undefined)[]) { await this.ready; const retrievedInputs = await this.retrieve( @@ -255,7 +243,7 @@ export abstract class Databases implements Adapter { (retrievedInput): retrievedInput is Historical => !!retrievedInput ) .map(retrievedInput => ({ - [retrievedInput.id]: retrievedInput + [retrievedInput._id]: retrievedInput })) .reduce( (a, b) => ({ ...a, ...b }), @@ -263,17 +251,15 @@ export abstract class Databases implements Adapter { ); await Promise.all( - Object.values(retrievedInputsById) - .map(async ({ id, _rev }) => { - await this.collections[collectionName].put({ - _rev, - _id: id, - _deleted: true - }); - }) + Object.values(retrievedInputsById).map(async ({ _id, _rev }) => { + await this.collections[collectionName].put({ + _rev, + _id, + _deleted: true + }); + }) ); } - } type CollectionDatabases = { diff --git a/packages/db/src/meta/pouch/types.ts b/packages/db/src/meta/pouch/types.ts index 14ca4882887..977467dea1d 100644 --- a/packages/db/src/meta/pouch/types.ts +++ b/packages/db/src/meta/pouch/types.ts @@ -4,33 +4,36 @@ const debug = logger("db:meta:pouch:types"); import PouchDB from "pouchdb"; import type { Collections, CollectionName } from "@truffle/db/meta/collections"; -import type { Historical } from "@truffle/db/meta/data"; import type * as Id from "@truffle/db/meta/id"; +export type History = PouchDB.Core.IdMeta & PouchDB.Core.GetMeta; + +export type Historical = T & History; + export interface Adapter { - every, I extends { id: string }>( + every, I extends PouchDB.Core.IdMeta>( collectionName: N ): Promise[]>; - retrieve, I extends { id: string }>( + retrieve, I extends PouchDB.Core.IdMeta>( collectionName: N, - references: (Pick | undefined)[] + references: (Pick | undefined)[] ): Promise<(Historical | undefined)[]>; - search, I extends { id: string }>( + search, I extends PouchDB.Core.IdMeta>( collectionName: N, options: PouchDB.Find.FindRequest<{}> ): Promise[]>; - record, I extends { id: string }>( + record, I extends PouchDB.Core.IdMeta>( collectionName: N, inputs: (I | undefined)[], options: { overwrite?: boolean } ): Promise<(Historical | undefined)[]>; - forget, I extends { id: string }>( + forget, I extends PouchDB.Core.IdMeta>( collectionName: N, - references: (Pick | undefined)[] + references: (Pick | undefined)[] ): Promise; } diff --git a/packages/db/src/meta/pouch/workspace.ts b/packages/db/src/meta/pouch/workspace.ts index 9af4b88501e..954de7cf408 100644 --- a/packages/db/src/meta/pouch/workspace.ts +++ b/packages/db/src/meta/pouch/workspace.ts @@ -13,9 +13,13 @@ import type { SavedInput } from "@truffle/db/meta/collections"; import * as Id from "@truffle/db/meta/id"; -import type { Workspace, Historical } from "@truffle/db/meta/data"; +import type { Workspace } from "@truffle/db/meta/data"; -import type { Adapter, Definitions } from "@truffle/db/meta/pouch/types"; +import type { + Historical, + Adapter, + Definitions +} from "@truffle/db/meta/pouch/types"; export interface AdapterWorkspaceConstructorOptions { adapter: Adapter; @@ -26,10 +30,7 @@ export class AdapterWorkspace implements Workspace { private adapter: Adapter; private generateId: Id.GenerateId; - constructor({ - adapter, - definitions - }: AdapterWorkspaceConstructorOptions) { + constructor({ adapter, definitions }: AdapterWorkspaceConstructorOptions) { this.adapter = adapter; this.generateId = Id.forDefinitions(definitions); } @@ -41,10 +42,12 @@ export class AdapterWorkspace implements Workspace { log("Fetching all..."); try { - const result = await this.adapter.every>(collectionName); + const records = await this.adapter.every>>( + collectionName + ); log("Found."); - return result as SavedInput[]; + return records.map(record => this.unmarshal(collectionName, record)); } catch (error) { log("Error fetching all %s, got error: %O", collectionName, error); throw error; @@ -61,22 +64,22 @@ export class AdapterWorkspace implements Workspace { try { // handle convenient interface for getting a bunch of IDs while preserving // order of input request - if (Array.isArray(options)) { - const references = options; - - return await this.adapter.retrieve>( - collectionName, - references as (Pick, "id"> | undefined)[] - ) as (SavedInput | undefined)[]; - } - - const result = await this.adapter.search>( - collectionName, - options - ); + const records = Array.isArray(options) + ? await this.adapter.retrieve>>( + collectionName, + options.map(reference => + reference ? { _id: reference.id } : undefined + ) as (Pick>, "_id"> | undefined)[] + ) + : await this.adapter.search>>( + collectionName, + options + ); log("Found."); - return result as SavedInput[]; + return records.map(record => + record ? this.unmarshal(collectionName, record) : undefined + ); } catch (error) { log("Error fetching all %s, got error: %O", collectionName, error); throw error; @@ -86,16 +89,18 @@ export class AdapterWorkspace implements Workspace { public async get>( collectionName: N, id: string | undefined - ): Promise> | undefined> { + ): Promise | undefined> { if (typeof id === "undefined") { return; } - const [savedInput] = await this.adapter.retrieve>( - collectionName, - [{ id }] - ); + const [savedInput] = await this.adapter.retrieve< + N, + Historical> + >(collectionName, [{ _id: id } as Pick>, "_id">]); - return savedInput; + if (savedInput) { + return this.unmarshal(collectionName, savedInput); + } } public async add>( @@ -105,20 +110,24 @@ export class AdapterWorkspace implements Workspace { const log = debug.extend(`${collectionName}:add`); log("Adding..."); - const inputsWithIds = input[collectionName].map( - input => this.attachId(collectionName, input) + const inputsWithIds = input[collectionName].map(input => + input ? this.marshal(collectionName, input) : undefined ); - const addedInputs = await this.adapter.record>( + const records = await this.adapter.record>>( collectionName, inputsWithIds, { overwrite: false } ); + const addedInputs = records.map(record => + record ? this.unmarshal(collectionName, record) : undefined + ); + log( "Added ids: %o", addedInputs - .filter((input): input is Historical> => !!input) + .filter((input): input is SavedInput => !!input) .map(({ id }) => id) ); @@ -134,20 +143,24 @@ export class AdapterWorkspace implements Workspace { const log = debug.extend(`${collectionName}:update`); log("Updating..."); - const inputsWithIds = input[collectionName].map( - input => this.attachId(collectionName, input) + const inputsWithIds = input[collectionName].map(input => + input ? this.marshal(collectionName, input) : undefined ); - const updatedInputs = await this.adapter.record>( + const records = await this.adapter.record>>( collectionName, inputsWithIds, { overwrite: true } ); + const updatedInputs = records.map(record => + record ? this.unmarshal(collectionName, record) : undefined + ); + log( "Updated ids: %o", updatedInputs - .filter((input): input is Historical> => !!input) + .filter((input): input is SavedInput => !!input) .map(({ id }) => id) ); @@ -163,8 +176,8 @@ export class AdapterWorkspace implements Workspace { const log = debug.extend(`${collectionName}:remove`); log("Removing..."); - const inputsWithIds = input[collectionName].map( - input => this.attachId(collectionName, input) + const inputsWithIds = input[collectionName].map(input => + input ? this.marshal(collectionName, input) : undefined ); await this.adapter.forget(collectionName, inputsWithIds); @@ -172,16 +185,37 @@ export class AdapterWorkspace implements Workspace { log("Removed."); } - private attachId>( + private marshal>( collectionName: N, - input: Input | undefined - ) { + input: Input + ): Historical> { const id = this.generateId(collectionName, input); if (typeof id === "undefined") { - return; + throw new Error(`Unable to generate ID for "${collectionName}" resource`); } + return { + ...input, + _id: id + } as Historical>; + } + + private unmarshal>( + collectionName: N, + record: any + ): SavedInput { + const id = (record as Historical>)._id; + + // remove internal properties: + // - PouchDB properties (begin with _) + // - ours (begin with $, reserved for future use) + const fieldNamePattern = /^[^_$]/; + const input: Input = Object.entries(record) + .filter(([propertyName]) => propertyName.match(fieldNamePattern)) + .map(([fieldName, value]) => ({ [fieldName]: value })) + .reduce((a, b) => ({ ...a, ...b }), {}); + return { ...input, id