From 7849facb86118ee1d2eb19fc0479ba1ea769901f Mon Sep 17 00:00:00 2001 From: Remi Cattiau Date: Fri, 17 Nov 2023 08:45:37 -0800 Subject: [PATCH] feat: add a BinaryModel for big json --- packages/core/src/services/binary.spec.ts | 2 + packages/core/src/services/binary.ts | 7 ++ packages/runtime/src/index.ts | 1 + .../runtime/src/models/binarymodel.spec.ts | 45 ++++++++++ packages/runtime/src/models/binarymodel.ts | 83 +++++++++++++++++++ packages/runtime/webda.module.json | 55 +++++++++++- 6 files changed, 189 insertions(+), 4 deletions(-) create mode 100644 packages/runtime/src/models/binarymodel.spec.ts create mode 100644 packages/runtime/src/models/binarymodel.ts diff --git a/packages/core/src/services/binary.spec.ts b/packages/core/src/services/binary.spec.ts index ae5f10c8d..fab7f45c7 100644 --- a/packages/core/src/services/binary.spec.ts +++ b/packages/core/src/services/binary.spec.ts @@ -183,7 +183,9 @@ class BinaryTest extends WebdaTest { // cov let buf2 = await BinaryService.streamToBuffer(await binary._get(user1[map][0])); + const buf3 = await user1[map][0].getAsBuffer(); assert.strictEqual(buf1.toString(), buf2.toString()); + assert.strictEqual(buf1.toString(), buf3.toString()); this.log("DEBUG", "Delete CoreModel and ensure usage count"); await userStore.delete(user1.uuid); value = await binary.getUsageCount(hash); diff --git a/packages/core/src/services/binary.ts b/packages/core/src/services/binary.ts index 619dd6a03..3484d6b21 100644 --- a/packages/core/src/services/binary.ts +++ b/packages/core/src/services/binary.ts @@ -265,6 +265,13 @@ export class BinaryMap extends BinaryFile { return this.__store.get(this); } + /** + * Get into a buffer + */ + async getAsBuffer(): Promise { + return BinaryService.streamToBuffer(await this.get()); + } + /** * Download the binary to a path * diff --git a/packages/runtime/src/index.ts b/packages/runtime/src/index.ts index 2c0b09900..8eb91b8f4 100644 --- a/packages/runtime/src/index.ts +++ b/packages/runtime/src/index.ts @@ -1 +1,2 @@ +export * from "./models/binarymodel"; export * from "./stores/migration"; diff --git a/packages/runtime/src/models/binarymodel.spec.ts b/packages/runtime/src/models/binarymodel.spec.ts new file mode 100644 index 000000000..ef36cdf69 --- /dev/null +++ b/packages/runtime/src/models/binarymodel.spec.ts @@ -0,0 +1,45 @@ +import { suite, test } from "@testdeck/mocha"; +import { FileBinary } from "@webda/core"; +import { WebdaSimpleTest } from "@webda/core/lib/test"; +import * as assert from "assert"; +import { BinaryModel } from "./binarymodel"; + +interface TestData { + test: string; + array: number[]; +} +@suite +class BinaryModelTest extends WebdaSimpleTest { + @test + async binary() { + await this.registerService(new FileBinary(this.webda, "file", { folder: "/tmp", models: { "*": ["*"] } })) + .resolve() + .init(); + let model = new BinaryModel().load({}, true); + // @ts-ignore + model.uuid = "test"; + assert.strictEqual(model.needsUpload(), false); + model.data = { + test: "OK" + }; + assert.strictEqual(model.needsUpload(), true); + await model.save(); + assert.strictEqual(model.needsUpload(), false); + await model.save(); + model.data.test = "OK2"; + assert.strictEqual(model.needsUpload(), true); + await model.save(); + let stored = >await BinaryModel.ref("test").get(); + await stored.loadData(); + // Call it twice to check that it is not downloaded twice + await stored.loadData(); + await stored.loadData(true); + // @ts-ignore + assert.strictEqual(stored.data.test, "OK2"); + + stored.data.array = [1, 2, 3]; + stored.data.array.push(4); + delete stored.data.array[1]; + delete stored.data.test; + } +} diff --git a/packages/runtime/src/models/binarymodel.ts b/packages/runtime/src/models/binarymodel.ts new file mode 100644 index 000000000..a05956406 --- /dev/null +++ b/packages/runtime/src/models/binarymodel.ts @@ -0,0 +1,83 @@ +import { Binary, Core, CoreModel, MemoryBinaryFile, NotEnumerable } from "@webda/core"; + +export class BinaryModel extends CoreModel { + @NotEnumerable + __data: T; + + __binary: Binary; + + @NotEnumerable + __dataUpdated: boolean = false; + + public get data(): T { + const subProxier = prop => { + return { + set: (target: this, p: string | symbol, value) => { + this.__dataUpdated = true; + target[p] = value; + return true; + }, + get: (target: this, p: string | symbol) => { + if (Array.isArray(target[p]) || target[p] instanceof Object) { + return new Proxy(target[p], subProxier(prop)); + } + return target[p]; + }, + deleteProperty: (t, property) => { + delete t[property]; + this.__dataUpdated = true; + return true; + } + }; + }; + const proxier = { + deleteProperty: (t, property) => { + delete t[property]; + this.__dataUpdated = true; + return true; + }, + set: (target: this, p: string | symbol, value) => { + this.__dataUpdated = true; + target[p] = value; + return true; + }, + get: (target: this, p: string | symbol) => { + if (Array.isArray(target[p]) || target[p] instanceof Object) { + return new Proxy(target[p], subProxier(p)); + } + return target[p]; + } + }; + return new Proxy(this.__data, proxier); + } + + public set data(data: T) { + this.__data = data; + this.__dataUpdated = true; + } + + needsUpload(): boolean { + return this.__dataUpdated; + } + + async loadData(force: boolean = false) { + if (this.__data && !force) { + return; + } + this.__data = JSON.parse((await this.__binary.getAsBuffer()).toString()); + } + + async save() { + await super.save(); + await this.updateBinary(); + return this; + } + + async updateBinary() { + if (!this.__dataUpdated) { + return; + } + await this.__binary.upload(new MemoryBinaryFile(JSON.stringify(this.__data))); + this.__dataUpdated = false; + } +} diff --git a/packages/runtime/webda.module.json b/packages/runtime/webda.module.json index 8e100a871..2a532294d 100644 --- a/packages/runtime/webda.module.json +++ b/packages/runtime/webda.module.json @@ -5,13 +5,60 @@ "Webda/MigrationStore": "lib/stores/migration:MigrationStore" }, "models": { - "graph": {}, - "tree": {}, + "graph": { + "Webda/BinaryModel": { + "binaries": [ + { + "attribute": "__binary", + "cardinality": "ONE" + } + ] + } + }, + "tree": { + "Webda/BinaryModel": {} + }, "plurals": {}, - "list": {}, - "reflections": {} + "list": { + "Webda/BinaryModel": "lib/models/binarymodel:BinaryModel" + }, + "reflections": { + "Webda/BinaryModel": { + "__data": "T", + "__binary": "Binary", + "__dataUpdated": "boolean", + "__class": "CoreModelDefinition", + "__type": "string", + "__types": "string[]", + "__ctx": "OperationContext", + "__store": "Store", + "__dirty": "Set", + "_creationDate": "Date", + "_lastUpdate": "Date", + "__deleted": "boolean" + } + } }, "schemas": { + "Webda/BinaryModel": { + "type": "object", + "properties": { + "_creationDate": { + "type": "string", + "format": "date-time", + "description": "Creation date", + "readOnly": true + }, + "_lastUpdate": { + "type": "string", + "format": "date-time", + "description": "Last update date", + "readOnly": true + } + }, + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "BinaryModel" + }, "Webda/MigrationStore": { "type": "object", "properties": {