From 881325ac4c8446365c14882a3915b821b979fcbb Mon Sep 17 00:00:00 2001 From: Tom Andrews Date: Sat, 31 Oct 2020 21:49:49 +0000 Subject: [PATCH] Change schema definition (#21) * Swtich to full joi schema to allow for more control * Overriding tables does not fully work as an unknown field is treated as `Joi.any()` so may not serialize to the correct type. --- README.md | 17 ++-- src/deserializer.ts | 4 +- src/index.test.ts | 206 ++++++++++++++++++++++++++++++------------- src/index.ts | 6 +- src/joiReflection.ts | 21 +++++ src/serializer.ts | 28 ++---- src/table.ts | 43 +++++---- src/types.ts | 12 +-- 8 files changed, 216 insertions(+), 121 deletions(-) create mode 100644 src/joiReflection.ts diff --git a/README.md b/README.md index a6ab7a4..5854a94 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Omanyd -A simple but production ready dynamodb mapper. +A simple and experimental dynamodb data mapper. [![Coverage Status](https://coveralls.io/repos/github/tgandrews/omanyd/badge.svg?branch=main)](https://coveralls.io/github/tgandrews/omanyd?branch=main) @@ -12,6 +12,7 @@ A simple but production ready dynamodb mapper. - Complete typescript typings - Basic global indexes - Range key +- Lists ### Missing features @@ -19,9 +20,7 @@ A simple but production ready dynamodb mapper. - Paging - Complex querying - Number and binary sets -- Boolean types - Date types -- Lists - Local indexes ## Installation @@ -78,10 +77,10 @@ interface Tweet { const TweetStore = Omanyd.define({ name: "Tweet", hashKey: "id", - schema: { + schema: Joi.object({ id: Omanyd.types.id(), content: Joi.string(), - }, + }), }); ``` @@ -210,11 +209,11 @@ interface Document { const DocumentStore = Omanyd.define({ name: "Documents", hashKey: "id", - schema: { + schema: Joi.object({ id: Omanyd.types.id(), version: Joi.string().required(), email: Joi.string().required(), - }, + }), }); // Assuming table has been created separately @@ -255,10 +254,10 @@ interface User { const UserStore = Omanyd.define({ name: "Users", hashKey: "id", - schema: { + schema: Joi.object({ id: Omanyd.types.id(), email: Joi.string().required(), - }, + }), indexes: [ { name: "EmailIndex", diff --git a/src/deserializer.ts b/src/deserializer.ts index 83f345a..340eecd 100644 --- a/src/deserializer.ts +++ b/src/deserializer.ts @@ -1,10 +1,8 @@ -import { PlainObject, Schema } from "./types"; +import { PlainObject } from "./types"; type DynamoType = keyof AWS.DynamoDB.AttributeValue; class Deserializer { - constructor(private schema: Schema) {} - fromDynamoValue(attributeValue: AWS.DynamoDB.AttributeValue): any { const [dynamoType, dynamoValue] = Object.entries(attributeValue)[0] as [ DynamoType, diff --git a/src/index.test.ts b/src/index.test.ts index 6bac0cc..ab655f0 100644 --- a/src/index.test.ts +++ b/src/index.test.ts @@ -16,10 +16,10 @@ describe("omanyd", () => { const ThingStore = Omanyd.define({ name: "basic", hashKey: "id", - schema: { + schema: Joi.object({ id: Omanyd.types.id(), value: Joi.string().required(), - }, + }), }); await Omanyd.createTables(); @@ -41,10 +41,10 @@ describe("omanyd", () => { const ThingStore = Omanyd.define({ name: "basicNotFound", hashKey: "id", - schema: { + schema: Joi.object({ id: Omanyd.types.id(), value: Joi.string().required(), - }, + }), }); await Omanyd.createTables(); @@ -63,11 +63,11 @@ describe("omanyd", () => { const ThingStore = Omanyd.define({ name: "basicNumber", hashKey: "id", - schema: { + schema: Joi.object({ id: Omanyd.types.id(), value: Joi.number().required(), value2: Joi.number().required(), - }, + }), }); await Omanyd.createTables(); @@ -90,11 +90,11 @@ describe("omanyd", () => { const ThingStore = Omanyd.define({ name: "basicBoolean", hashKey: "id", - schema: { + schema: Joi.object({ id: Omanyd.types.id(), value: Joi.boolean().required(), value2: Joi.boolean().required(), - }, + }), }); await Omanyd.createTables(); @@ -123,14 +123,14 @@ describe("omanyd", () => { const ThingStore = Omanyd.define({ name: "basicObject", hashKey: "id", - schema: { + schema: Joi.object({ id: Omanyd.types.id(), value: Joi.object({ key1: Joi.string().required(), key2: Joi.number().required(), key3: Omanyd.types.stringSet().required(), }).required(), - }, + }), }); await Omanyd.createTables(); @@ -167,11 +167,11 @@ describe("omanyd", () => { const ThingStore = Omanyd.define({ name: "basicNull", hashKey: "id", - schema: { + schema: Joi.object({ id: Omanyd.types.id(), value: Joi.any(), value2: Joi.number().required(), - }, + }), }); await Omanyd.createTables(); @@ -201,10 +201,10 @@ describe("omanyd", () => { const ThingStore = Omanyd.define({ name: "basicUpdating", hashKey: "id", - schema: { + schema: Joi.object({ id: Omanyd.types.id(), value: Joi.string().required(), - }, + }), }); await Omanyd.createTables(); @@ -228,10 +228,10 @@ describe("omanyd", () => { const ThingStore = Omanyd.define({ name: "basicUndefined", hashKey: "id", - schema: { + schema: Joi.object({ id: Omanyd.types.id(), value: Joi.string(), - }, + }), }); await Omanyd.createTables(); @@ -255,11 +255,11 @@ describe("omanyd", () => { name: "basicRangeKey", hashKey: "id", rangeKey: "version", - schema: { + schema: Joi.object({ id: Joi.string().required(), version: Joi.string().required(), value: Joi.string().required(), - }, + }), }); await Omanyd.createTables(); @@ -285,11 +285,11 @@ describe("omanyd", () => { name: "rangeKeyItemNotFound", hashKey: "id", rangeKey: "version", - schema: { + schema: Joi.object({ id: Joi.string().required(), version: Joi.string().required(), value: Joi.string().required(), - }, + }), }); await Omanyd.createTables(); @@ -308,11 +308,11 @@ describe("omanyd", () => { const ThingStore = Omanyd.define({ name: "noRangeKeyError", hashKey: "id", - schema: { + schema: Joi.object({ id: Omanyd.types.id(), version: Joi.string().required(), value: Joi.string().required(), - }, + }), }); await Omanyd.createTables(); @@ -332,10 +332,10 @@ describe("omanyd", () => { const ThingStore = Omanyd.define({ name: "errorFunction", hashKey: "id", - schema: { + schema: Joi.object({ id: Omanyd.types.id(), value: Joi.any().required(), - }, + }), }); await Omanyd.createTables(); @@ -357,10 +357,10 @@ describe("omanyd", () => { const ThingStore = Omanyd.define({ name: "errorSymbol", hashKey: "id", - schema: { + schema: Joi.object({ id: Omanyd.types.id(), value: Joi.any().required(), - }, + }), }); await Omanyd.createTables(); @@ -380,10 +380,10 @@ describe("omanyd", () => { const ThingStore = Omanyd.define({ name: "errorBuffer", hashKey: "id", - schema: { + schema: Joi.object({ id: Omanyd.types.id(), value: Joi.any().required(), - }, + }), }); await Omanyd.createTables(); @@ -403,10 +403,10 @@ describe("omanyd", () => { const ThingStore = Omanyd.define({ name: "errorMissingSchema", hashKey: "id", - schema: { + schema: Joi.object({ id: Omanyd.types.id(), value: Joi.string().required(), - }, + }), }); await Omanyd.createTables(); @@ -429,10 +429,10 @@ describe("omanyd", () => { const ThingStore = Omanyd.define({ name: "StringSet", hashKey: "id", - schema: { + schema: Joi.object({ id: Omanyd.types.id(), value: Omanyd.types.stringSet(), - }, + }), }); await Omanyd.createTables(); @@ -455,10 +455,10 @@ describe("omanyd", () => { const ThingStore = Omanyd.define({ name: "setEmpty", hashKey: "id", - schema: { + schema: Joi.object({ id: Omanyd.types.id(), value: Omanyd.types.stringSet(), - }, + }), }); await Omanyd.createTables(); @@ -483,10 +483,10 @@ describe("omanyd", () => { const ThingStore = Omanyd.define({ name: "listAny", hashKey: "id", - schema: { + schema: Joi.object({ id: Omanyd.types.id(), value: Joi.array().items(Joi.any()).required(), - }, + }), }); await Omanyd.createTables(); @@ -515,10 +515,10 @@ describe("omanyd", () => { const ThingStore = Omanyd.define({ name: "listUpdating", hashKey: "id", - schema: { + schema: Joi.object({ id: Omanyd.types.id(), value: Joi.array().items(Joi.any()).required(), - }, + }), }); await Omanyd.createTables(); @@ -546,10 +546,10 @@ describe("omanyd", () => { const ThingStore = Omanyd.define({ name: "scanAll", hashKey: "id", - schema: { + schema: Joi.object({ id: Omanyd.types.id(), value: Joi.string().required(), - }, + }), }); await Omanyd.createTables(); @@ -584,10 +584,10 @@ describe("omanyd", () => { const ThingStore = Omanyd.define({ name: "scanEmpty", hashKey: "id", - schema: { + schema: Joi.object({ id: Omanyd.types.id(), value: Joi.string().required(), - }, + }), }); await Omanyd.createTables(); @@ -607,10 +607,10 @@ describe("omanyd", () => { const ThingStore = Omanyd.define({ name: "indexQuery", hashKey: "id", - schema: { + schema: Joi.object({ id: Omanyd.types.id(), email: Joi.string().required(), - }, + }), indexes: [ { name: "ValueIndex", @@ -640,10 +640,10 @@ describe("omanyd", () => { const ThingStore = Omanyd.define({ name: "indexQueryNotFound", hashKey: "id", - schema: { + schema: Joi.object({ id: Omanyd.types.id(), email: Joi.string().required(), - }, + }), indexes: [ { name: "ValueIndex", @@ -671,10 +671,10 @@ describe("omanyd", () => { const ThingStore = Omanyd.define({ name: "indexNotDefined", hashKey: "id", - schema: { + schema: Joi.object({ id: Omanyd.types.id(), email: Joi.string().required(), - }, + }), }); await Omanyd.createTables(); @@ -694,10 +694,10 @@ describe("omanyd", () => { const ThingStore = Omanyd.define({ name: "clearTables", hashKey: "id", - schema: { + schema: Joi.object({ id: Omanyd.types.id(), value: Joi.string().required(), - }, + }), }); await Omanyd.createTables(); @@ -715,7 +715,7 @@ describe("omanyd", () => { }); describe("multiple table defintions", () => { - it("shoudl error when defining two stores for the same table", async () => { + it("should error when defining two stores for the same table", async () => { interface Thing { id: string; value: string; @@ -723,19 +723,19 @@ describe("omanyd", () => { Omanyd.define({ name: "multipleTablesError", hashKey: "id", - schema: { + schema: Joi.object({ id: Omanyd.types.id(), value: Joi.string().required(), - }, + }), }); expect(() => { Omanyd.define({ name: "multipleTablesError", hashKey: "id", - schema: { + schema: Joi.object({ id: Omanyd.types.id(), value: Joi.string().required(), - }, + }), }); }).toThrow(/clashing table name: "multipleTablesError"/); }); @@ -748,23 +748,111 @@ describe("omanyd", () => { Omanyd.define({ name: "multipleTablesSuccess", hashKey: "id", - schema: { + schema: Joi.object({ id: Omanyd.types.id(), value: Joi.string().required(), - }, + }), }); expect(() => { Omanyd.define({ name: "multipleTablesSuccess", hashKey: "id", - schema: { + schema: Joi.object({ id: Omanyd.types.id(), value: Joi.string().required(), extras: Joi.array().items(Joi.string()), - }, + }), allowNameClash: true, }); }).not.toThrow(); }); + + it("should allow for missing definitions", async () => { + interface Thing { + id: string; + value: string; + } + const ThingStore = Omanyd.define({ + name: "multipleTablesReadWrite", + hashKey: "id", + schema: Joi.object({ + id: Omanyd.types.id(), + value: Joi.string().required(), + }).unknown(true), + }); + + interface ExtendedThing extends Thing { + extras: any[]; + } + + const ExtendedThingStore = Omanyd.define({ + name: "multipleTablesReadWrite", + hashKey: "id", + schema: Joi.object({ + id: Omanyd.types.id(), + value: Joi.string().required(), + extras: Joi.array().items(Joi.any()), + }), + allowNameClash: true, + }); + + await Omanyd.createTables(); + + const extendedThing = await ExtendedThingStore.create({ + value: "hello", + extras: ["hello", 1], + }); + + const thing = await ThingStore.getByHashKey(extendedThing.id); + + expect(thing).toStrictEqual(extendedThing); + }); + + it("should be possible for an extended store to save a read value", async () => { + interface Thing { + id: string; + value: string; + } + const ThingStore = Omanyd.define({ + name: "multipleTablesReadWriteExtended", + hashKey: "id", + schema: Joi.object({ + id: Omanyd.types.id(), + value: Joi.string().required(), + }).unknown(true), + }); + + interface ExtendedThing extends Thing { + extras: any[]; + } + + const ExtendedThingStore = Omanyd.define({ + name: "multipleTablesReadWriteExtended", + hashKey: "id", + schema: Joi.object({ + id: Omanyd.types.id(), + value: Joi.string().required(), + extras: Joi.array().items(Joi.any()), + }), + allowNameClash: true, + }); + + await Omanyd.createTables(); + + const extendedThing = await ExtendedThingStore.create({ + value: "hello", + extras: ["hello", 1], + }); + + const thing = await ThingStore.getByHashKey(extendedThing.id); + thing.value = "goodbye"; + + const updatedThing = await ThingStore.put(thing); + + expect(updatedThing).toStrictEqual({ + ...extendedThing, + value: "goodbye", + }); + }); }); }); diff --git a/src/index.ts b/src/index.ts index 51b6a6a..5653175 100644 --- a/src/index.ts +++ b/src/index.ts @@ -4,8 +4,6 @@ import { v4 } from "uuid"; import type { Options } from "./types"; import Table from "./table"; -export { Schema } from "./types"; - let STORES: { [name: string]: Table } = {}; export const types = { @@ -21,8 +19,6 @@ export const types = { export function define(options: Options) { const t = new Table(options); - const validator = Joi.object(options.schema); - const allowNameClash = options.allowNameClash ?? false; if (STORES[t.name] && !allowNameClash) { throw new Error( @@ -31,6 +27,8 @@ export function define(options: Options) { } STORES[t.name] = t; + const validator = options.schema; + return { async create(obj: Omit | T): Promise { const validated: T = await validator.validateAsync(obj); diff --git a/src/joiReflection.ts b/src/joiReflection.ts new file mode 100644 index 0000000..676923f --- /dev/null +++ b/src/joiReflection.ts @@ -0,0 +1,21 @@ +import Joi from "joi"; + +interface JoiObjectSchema extends Joi.ObjectSchema { + _ids: { + _byKey: Map; + }; +} + +export const getItemSchemaFromObjectSchema = ( + objectSchema: Joi.ObjectSchema, + key: string +): Joi.AnySchema => { + // This is a complete hack and should be opened as an issue against Joi to get a proper API + const itemSchema = (objectSchema as JoiObjectSchema)._ids._byKey.get(key) + ?.schema; + // This must mean unknown is enabled + if (!itemSchema) { + return Joi.any(); + } + return itemSchema; +}; diff --git a/src/serializer.ts b/src/serializer.ts index 1b1c2b2..e8de2aa 100644 --- a/src/serializer.ts +++ b/src/serializer.ts @@ -1,9 +1,10 @@ import AWS from "aws-sdk"; import Joi from "joi"; -import type { JoiObjectSchema, PlainObject, Schema } from "./types"; +import type { PlainObject } from "./types"; +import { getItemSchemaFromObjectSchema } from "./joiReflection"; class Serializer { - constructor(private schema: Schema) {} + constructor(private schema: Joi.ObjectSchema) {} private string(value: string): AWS.DynamoDB.AttributeValue { return { S: value }; @@ -98,7 +99,9 @@ class Serializer { schemaKey: string | Joi.AnySchema ): AWS.DynamoDB.AttributeValue | undefined { const schema = - typeof schemaKey === "string" ? this.schema[schemaKey] : schemaKey; + typeof schemaKey === "string" + ? getItemSchemaFromObjectSchema(this.schema, schemaKey) + : schemaKey; if (userValue === undefined) { return undefined; @@ -123,25 +126,10 @@ class Serializer { toDynamoMap( userObj: PlainObject, - schema?: Joi.ObjectSchema + objectSchema: Joi.ObjectSchema = this.schema ): AWS.DynamoDB.AttributeMap { - const parentSchema = schema || this.schema; - return Object.entries(userObj).reduce((dynamoObj, [key, userValue]) => { - let itemSchema: Joi.AnySchema | undefined; - if (parentSchema.type === "object") { - // This is a complete hack and should be opened as an issue against Joi to get a proper API - itemSchema = (parentSchema as JoiObjectSchema)._ids._byKey.get(key) - ?.schema; - } else { - itemSchema = (parentSchema as Schema)[key]; - } - - // Cannot recreate this state but TS is certain it exists - if (!itemSchema) { - throw new Error(`Could not find schema for "${key}"`); - } - + const itemSchema = getItemSchemaFromObjectSchema(objectSchema, key); const dynamoValue = this.toDynamoValue(userValue, itemSchema); if (dynamoValue === undefined) { return dynamoObj; diff --git a/src/table.ts b/src/table.ts index def568e..829e775 100644 --- a/src/table.ts +++ b/src/table.ts @@ -2,6 +2,7 @@ import AWS from "aws-sdk"; import Serializer from "./serializer"; import Deserializer from "./deserializer"; +import { getItemSchemaFromObjectSchema } from "./joiReflection"; import type { PlainObject, Options } from "./types"; @@ -24,7 +25,7 @@ export default class Table { this.dynamoDB = new AWS.DynamoDB(config); this.name = options.name; this.serializer = new Serializer(options.schema); - this.deserializer = new Deserializer(options.schema); + this.deserializer = new Deserializer(); } async createTable(): Promise { @@ -149,7 +150,7 @@ export default class Table { }); } - async getByHashKey(hashKey: string): Promise { + async getByHashKey(hashKeyValue: string): Promise { return new Promise((res, rej) => { this.dynamoDB.getItem( { @@ -157,8 +158,11 @@ export default class Table { ConsistentRead: true, Key: { [this.options.hashKey]: this.serializer.toDynamoValue( - hashKey, - this.options.schema[this.options.hashKey] + hashKeyValue, + getItemSchemaFromObjectSchema( + this.options.schema, + this.options.hashKey + ) )!, }, }, @@ -177,9 +181,11 @@ export default class Table { } async getByHashAndRangeKey( - hashKey: string, - rangeKey: string + hashKeyValue: string, + rangeKeyValue: string ): Promise { + const { rangeKey, hashKey } = this.options; + return new Promise((res, rej) => { this.dynamoDB.getItem( { @@ -187,12 +193,13 @@ export default class Table { ConsistentRead: true, Key: { [this.options.hashKey]: this.serializer.toDynamoValue( - hashKey, - this.options.schema[this.options.hashKey] + hashKeyValue, + getItemSchemaFromObjectSchema(this.options.schema, hashKey) )!, [this.options.rangeKey!]: this.serializer.toDynamoValue( - rangeKey, - this.options.schema[this.options.rangeKey!] + rangeKeyValue, + // Range key is guaranteed by check in store + getItemSchemaFromObjectSchema(this.options.schema, rangeKey!) )!, }, }, @@ -210,12 +217,15 @@ export default class Table { }); } - async getByIndex(name: string, hashKey: string): Promise { + async getByIndex( + indexName: string, + value: string + ): Promise { const indexDefintion = (this.options.indexes ?? []).find( - (index) => index.name === name + (index) => index.name === indexName ); if (!indexDefintion) { - throw new Error(`No index found with name: '${name}'`); + throw new Error(`No index found with name: '${indexName}'`); } return new Promise((res, rej) => { this.dynamoDB.query( @@ -225,8 +235,11 @@ export default class Table { KeyConditionExpression: `${indexDefintion.hashKey} = :hashKey`, ExpressionAttributeValues: { ":hashKey": this.serializer.toDynamoValue( - hashKey, - this.options.schema[this.options.hashKey] + value, + getItemSchemaFromObjectSchema( + this.options.schema, + indexDefintion.hashKey + ) )!, }, }, diff --git a/src/types.ts b/src/types.ts index e5bda12..a91dfc7 100644 --- a/src/types.ts +++ b/src/types.ts @@ -18,7 +18,7 @@ export interface Options { name: string; hashKey: string; rangeKey?: string; - schema: Schema; + schema: Joi.ObjectSchema; indexes?: { name: string; type: "global"; @@ -26,13 +26,3 @@ export interface Options { }[]; allowNameClash?: boolean; } - -export interface Schema { - [key: string]: Joi.AnySchema; -} - -export interface JoiObjectSchema extends Joi.ObjectSchema { - _ids: { - _byKey: Map; - }; -}