From 0bf6355a608006ad46f6448083ae41eae3167156 Mon Sep 17 00:00:00 2001 From: zzj3720 Date: Wed, 7 Feb 2024 15:37:31 +0800 Subject: [PATCH 1/4] feat(core): orm demo --- packages/frontend/core/src/index.tsx | 1 + packages/frontend/core/src/orm/index.ts | 264 ++++++++++++++++++++++++ packages/frontend/core/src/orm/test.ts | 119 +++++++++++ 3 files changed, 384 insertions(+) create mode 100644 packages/frontend/core/src/orm/index.ts create mode 100644 packages/frontend/core/src/orm/test.ts diff --git a/packages/frontend/core/src/index.tsx b/packages/frontend/core/src/index.tsx index 1a1e95584b70..76d835861cfb 100644 --- a/packages/frontend/core/src/index.tsx +++ b/packages/frontend/core/src/index.tsx @@ -1,5 +1,6 @@ import './polyfill/intl-segmenter'; import './polyfill/request-idle-callback'; +import './orm/test'; import { assertExists } from '@blocksuite/global/utils'; import { StrictMode } from 'react'; diff --git a/packages/frontend/core/src/orm/index.ts b/packages/frontend/core/src/orm/index.ts new file mode 100644 index 000000000000..6388b9e41b32 --- /dev/null +++ b/packages/frontend/core/src/orm/index.ts @@ -0,0 +1,264 @@ +import { Observable } from 'rxjs'; +import type { AbstractType as YAbstract, Array as YArray, Doc } from 'yjs'; +import { Map as YMap } from 'yjs'; + +export const table = >( + name: string, + schema: T +): TableSchema => { + return { + __name: name, + __schema: schema, + // type + __data: {} as GetSchemaDataType, + }; +}; + +type TableSchema> = { + __name: string; + __schema: T; + __data: GetSchemaDataType; +}; +type ColumnType = 'string' | 'boolean' | 'json' | 'number' | 'raw'; + +class Filed< + Type = unknown, + Required extends boolean = boolean, + Default extends boolean = boolean, +> { + constructor( + public readonly ops: { + type: ColumnType; + required: Required; + hasDefault: Default; + default?: () => Type; + } + ) {} + + required(): Filed { + return new Filed({ + ...this.ops, + required: true, + }); + } + + default(value: () => Type): Filed { + return new Filed({ + ...this.ops, + default: value, + hasDefault: true, + }); + } +} + +export const f = { + string: (): Filed => { + return new Filed({ + type: 'string', + required: false, + hasDefault: false, + }); + }, + boolean: (): Filed => { + return new Filed({ + type: 'boolean', + required: false, + hasDefault: false, + }); + }, + number: (): Filed => { + return new Filed({ + type: 'number', + required: false, + hasDefault: false, + }); + }, + json: (): Filed => { + return new Filed({ + type: 'json', + required: false, + hasDefault: false, + }); + }, + raw: >(): Filed => { + return new Filed({ + type: 'raw', + required: false, + hasDefault: false, + }); + }, +}; +type Where> = Partial< + GetSchemaDataType +>; + +type ConvertProperty = T extends Filed + ? R + : never; +type ToRequired, P> = { + [K in keyof T as T[K] extends P ? K : never]-?: ConvertProperty; +}; +type ToOptional, P> = { + [K in keyof T as T[K] extends P ? K : never]?: ConvertProperty; +}; + +type GetSchemaDataType> = Pretty< + ToRequired> & ToOptional> +>; + +type GetSchemaCreateType> = Pretty< + ToRequired> & + ToOptional> & + ToOptional> & + ToOptional> +>; +type Pretty = T extends any + ? { + [P in keyof T]: T[P]; + } + : never; + +export const createDB = (yjs: Doc) => { + const find = (arr: YArray>, where: [string, unknown][]) => { + for (const item of arr) { + const isMatch = where.every(([key, value]) => { + return item.get(key) === value; + }); + if (isMatch) { + return item; + } + } + return; + }; + const filter = (arr: YArray>, where: [string, unknown][]) => { + const result = []; + for (const item of arr) { + const isMatch = where.every(([key, value]) => { + return item.get(key) === value; + }); + if (isMatch) { + result.push(item); + } + } + return result; + }; + const toObject = (map: YMap): T => { + return Object.fromEntries(map.entries()) as T; + }; + + return { + findFirst: >( + from: TableSchema, + where: Where + ): GetSchemaDataType | undefined => { + const arr = yjs.getArray(from.__name) as YArray>; + const whereEntries = Object.entries(where); + const item = find(arr, whereEntries); + return item ? toObject>(item) : undefined; + }, + findList: >( + from: TableSchema, + where: Where + ): GetSchemaDataType[] => { + const arr = yjs.getArray(from.__name) as YArray>; + const whereEntries = Object.entries(where); + const items = filter(arr, whereEntries); + return items.map(toObject>); + }, + observeFirst: >( + from: TableSchema, + where: Where + ): Observable | undefined> => { + const arr = yjs.getArray(from.__name) as YArray>; + const whereEntries = Object.entries(where); + return new Observable(subscriber => { + const listener = () => { + const item = find(arr, whereEntries); + subscriber.next( + item ? toObject>(item) : undefined + ); + }; + arr.observe(listener); + return () => { + arr.unobserve(listener); + }; + }); + }, + observeList: >( + from: TableSchema, + where: Where + ): Observable[]> => { + const arr = yjs.getArray(from.__name) as YArray>; + const whereEntries = Object.entries(where); + return new Observable(subscriber => { + const listener = () => { + const items = filter(arr, whereEntries); + subscriber.next(items.map(toObject>)); + }; + arr.observe(listener); + return () => { + arr.unobserve(listener); + }; + }); + }, + create: >( + from: TableSchema, + value: GetSchemaCreateType + ): GetSchemaDataType => { + const data = Object.fromEntries( + Object.entries(from.__schema).map(([key, field]) => { + if (key in value) { + return [key, (value as Record)[key]]; + } + if (field.ops.default) { + return [key, field.ops.default()]; + } + return [key, undefined]; + }) + ); + const arr = yjs.getArray(from.__name) as YArray>; + arr.insert(0, [new YMap(Object.entries(data))]); + return data as GetSchemaDataType; + }, + update: >( + from: TableSchema, + where: Where, + value: ( + old: GetSchemaDataType + ) => Partial> + ) => { + const arr = yjs.getArray(from.__name) as YArray>; + const whereKeys = Object.entries(where); + const item = find(arr, whereKeys); + if (item) { + const newValue = value(item.toJSON() as GetSchemaDataType); + Object.entries(newValue).forEach(([key, value]) => { + item.set(key, value); + }); + } + }, + delete: >( + from: TableSchema, + where: Where + ) => { + const arr = yjs.getArray(from.__name) as YArray>; + const whereKeys = Object.entries(where); + const findIndex = (arr: YArray>) => { + for (let i = 0; i < arr.length; i++) { + const item = arr.get(i); + const isMatch = whereKeys.every(([key, value]) => { + return item.get(key) === value; + }); + if (isMatch) { + return i; + } + } + return -1; + }; + const index = findIndex(arr); + if (index !== -1) { + arr.delete(index, 1); + } + }, + }; +}; diff --git a/packages/frontend/core/src/orm/test.ts b/packages/frontend/core/src/orm/test.ts new file mode 100644 index 000000000000..769a5f31f93d --- /dev/null +++ b/packages/frontend/core/src/orm/test.ts @@ -0,0 +1,119 @@ +import { nanoid } from 'nanoid'; +import type { Observable } from 'rxjs'; +import { map, of, switchMap } from 'rxjs'; +import { Doc, Map as YMap, type Text as YText } from 'yjs'; + +import { createDB, f, table } from './index'; + +type Rule = { + field: string; + operator: string; + value: string; +}; + +const collectionTable = table('collection', { + id: f.string().required().default(nanoid), + title: f.string().required(), + workspaceId: f.string().required(), + rules: f + .json() + .required() + .default(() => []), +}); + +const workspaceTable = table('workspace', { + id: f.string().required().default(nanoid), + name: f.string().required(), +}); + +const pageTable = table('page', { + id: f.string().required().default(nanoid), + title: f.string().required(), + favorite: f + .boolean() + .required() + .default(() => false), + workspaceId: f.string().required(), +}); +const blockTable = table('block', { + id: f.string().required().default(nanoid), + pageId: f.string().required(), + flavor: f.string().required(), + text: f.raw(), + props: f + .raw>() + .required() + .default(() => new YMap()), +}); + +const doc = new Doc(); +const db = createDB(doc); +const workspaceId = 'a'; +const aWorkspaceObservable = db.observeFirst(workspaceTable, { + id: workspaceId, +}); + +const merge = ( + a: Observable, + bf: (a: A) => Observable, + fieldName: F +) => { + return a.pipe( + switchMap(aValue => { + return bf(aValue).pipe( + map(bValue => { + return { + ...aValue, + [fieldName]: bValue, + } as A & { [K in F]: B }; + }) + ); + }) + ); +}; +const getPages = (workspace?: { id: string }) => { + return workspace + ? db.observeList(pageTable, { + workspaceId: workspace.id, + }) + : of([]); +}; +merge(aWorkspaceObservable, getPages, 'pages').subscribe(workspace => { + console.log(JSON.stringify(workspace, null, 2)); +}); +//or +// aWorkspaceObservable.pipe(switchMap(workspace => { +// return getPages(workspace).pipe( +// map(pages => { +// return { +// ...workspace, +// pages, +// }; +// }) +// ); +// })).subscribe(workspace => { +// console.log(JSON.stringify(workspace, null, 2)); +// }); + +const workspaceA = db.create(workspaceTable, { + id: workspaceId, + name: 'first workspace', +}); +const pageA = db.create(pageTable, { + title: 'first page', + workspaceId: workspaceA.id, + favorite: false, +}); +db.create(collectionTable, { + title: 'first collection', + workspaceId: workspaceA.id, + rules: [], +}); +db.delete(pageTable, { + id: pageA.id, +}); +db.create(blockTable, { + flavor: 'text', + pageId: pageA.id, +}); +console.log(doc.toJSON()); From cc6390e2a6cb21f87aa0ee85fa47dbf1a7d4e8d4 Mon Sep 17 00:00:00 2001 From: zzj3720 Date: Wed, 7 Feb 2024 15:37:31 +0800 Subject: [PATCH 2/4] feat(core): orm demo --- packages/common/infra/src/orm/index.ts | 264 +++++++++++++++++++++++++ packages/common/infra/src/orm/test.ts | 119 +++++++++++ 2 files changed, 383 insertions(+) create mode 100644 packages/common/infra/src/orm/index.ts create mode 100644 packages/common/infra/src/orm/test.ts diff --git a/packages/common/infra/src/orm/index.ts b/packages/common/infra/src/orm/index.ts new file mode 100644 index 000000000000..6388b9e41b32 --- /dev/null +++ b/packages/common/infra/src/orm/index.ts @@ -0,0 +1,264 @@ +import { Observable } from 'rxjs'; +import type { AbstractType as YAbstract, Array as YArray, Doc } from 'yjs'; +import { Map as YMap } from 'yjs'; + +export const table = >( + name: string, + schema: T +): TableSchema => { + return { + __name: name, + __schema: schema, + // type + __data: {} as GetSchemaDataType, + }; +}; + +type TableSchema> = { + __name: string; + __schema: T; + __data: GetSchemaDataType; +}; +type ColumnType = 'string' | 'boolean' | 'json' | 'number' | 'raw'; + +class Filed< + Type = unknown, + Required extends boolean = boolean, + Default extends boolean = boolean, +> { + constructor( + public readonly ops: { + type: ColumnType; + required: Required; + hasDefault: Default; + default?: () => Type; + } + ) {} + + required(): Filed { + return new Filed({ + ...this.ops, + required: true, + }); + } + + default(value: () => Type): Filed { + return new Filed({ + ...this.ops, + default: value, + hasDefault: true, + }); + } +} + +export const f = { + string: (): Filed => { + return new Filed({ + type: 'string', + required: false, + hasDefault: false, + }); + }, + boolean: (): Filed => { + return new Filed({ + type: 'boolean', + required: false, + hasDefault: false, + }); + }, + number: (): Filed => { + return new Filed({ + type: 'number', + required: false, + hasDefault: false, + }); + }, + json: (): Filed => { + return new Filed({ + type: 'json', + required: false, + hasDefault: false, + }); + }, + raw: >(): Filed => { + return new Filed({ + type: 'raw', + required: false, + hasDefault: false, + }); + }, +}; +type Where> = Partial< + GetSchemaDataType +>; + +type ConvertProperty = T extends Filed + ? R + : never; +type ToRequired, P> = { + [K in keyof T as T[K] extends P ? K : never]-?: ConvertProperty; +}; +type ToOptional, P> = { + [K in keyof T as T[K] extends P ? K : never]?: ConvertProperty; +}; + +type GetSchemaDataType> = Pretty< + ToRequired> & ToOptional> +>; + +type GetSchemaCreateType> = Pretty< + ToRequired> & + ToOptional> & + ToOptional> & + ToOptional> +>; +type Pretty = T extends any + ? { + [P in keyof T]: T[P]; + } + : never; + +export const createDB = (yjs: Doc) => { + const find = (arr: YArray>, where: [string, unknown][]) => { + for (const item of arr) { + const isMatch = where.every(([key, value]) => { + return item.get(key) === value; + }); + if (isMatch) { + return item; + } + } + return; + }; + const filter = (arr: YArray>, where: [string, unknown][]) => { + const result = []; + for (const item of arr) { + const isMatch = where.every(([key, value]) => { + return item.get(key) === value; + }); + if (isMatch) { + result.push(item); + } + } + return result; + }; + const toObject = (map: YMap): T => { + return Object.fromEntries(map.entries()) as T; + }; + + return { + findFirst: >( + from: TableSchema, + where: Where + ): GetSchemaDataType | undefined => { + const arr = yjs.getArray(from.__name) as YArray>; + const whereEntries = Object.entries(where); + const item = find(arr, whereEntries); + return item ? toObject>(item) : undefined; + }, + findList: >( + from: TableSchema, + where: Where + ): GetSchemaDataType[] => { + const arr = yjs.getArray(from.__name) as YArray>; + const whereEntries = Object.entries(where); + const items = filter(arr, whereEntries); + return items.map(toObject>); + }, + observeFirst: >( + from: TableSchema, + where: Where + ): Observable | undefined> => { + const arr = yjs.getArray(from.__name) as YArray>; + const whereEntries = Object.entries(where); + return new Observable(subscriber => { + const listener = () => { + const item = find(arr, whereEntries); + subscriber.next( + item ? toObject>(item) : undefined + ); + }; + arr.observe(listener); + return () => { + arr.unobserve(listener); + }; + }); + }, + observeList: >( + from: TableSchema, + where: Where + ): Observable[]> => { + const arr = yjs.getArray(from.__name) as YArray>; + const whereEntries = Object.entries(where); + return new Observable(subscriber => { + const listener = () => { + const items = filter(arr, whereEntries); + subscriber.next(items.map(toObject>)); + }; + arr.observe(listener); + return () => { + arr.unobserve(listener); + }; + }); + }, + create: >( + from: TableSchema, + value: GetSchemaCreateType + ): GetSchemaDataType => { + const data = Object.fromEntries( + Object.entries(from.__schema).map(([key, field]) => { + if (key in value) { + return [key, (value as Record)[key]]; + } + if (field.ops.default) { + return [key, field.ops.default()]; + } + return [key, undefined]; + }) + ); + const arr = yjs.getArray(from.__name) as YArray>; + arr.insert(0, [new YMap(Object.entries(data))]); + return data as GetSchemaDataType; + }, + update: >( + from: TableSchema, + where: Where, + value: ( + old: GetSchemaDataType + ) => Partial> + ) => { + const arr = yjs.getArray(from.__name) as YArray>; + const whereKeys = Object.entries(where); + const item = find(arr, whereKeys); + if (item) { + const newValue = value(item.toJSON() as GetSchemaDataType); + Object.entries(newValue).forEach(([key, value]) => { + item.set(key, value); + }); + } + }, + delete: >( + from: TableSchema, + where: Where + ) => { + const arr = yjs.getArray(from.__name) as YArray>; + const whereKeys = Object.entries(where); + const findIndex = (arr: YArray>) => { + for (let i = 0; i < arr.length; i++) { + const item = arr.get(i); + const isMatch = whereKeys.every(([key, value]) => { + return item.get(key) === value; + }); + if (isMatch) { + return i; + } + } + return -1; + }; + const index = findIndex(arr); + if (index !== -1) { + arr.delete(index, 1); + } + }, + }; +}; diff --git a/packages/common/infra/src/orm/test.ts b/packages/common/infra/src/orm/test.ts new file mode 100644 index 000000000000..769a5f31f93d --- /dev/null +++ b/packages/common/infra/src/orm/test.ts @@ -0,0 +1,119 @@ +import { nanoid } from 'nanoid'; +import type { Observable } from 'rxjs'; +import { map, of, switchMap } from 'rxjs'; +import { Doc, Map as YMap, type Text as YText } from 'yjs'; + +import { createDB, f, table } from './index'; + +type Rule = { + field: string; + operator: string; + value: string; +}; + +const collectionTable = table('collection', { + id: f.string().required().default(nanoid), + title: f.string().required(), + workspaceId: f.string().required(), + rules: f + .json() + .required() + .default(() => []), +}); + +const workspaceTable = table('workspace', { + id: f.string().required().default(nanoid), + name: f.string().required(), +}); + +const pageTable = table('page', { + id: f.string().required().default(nanoid), + title: f.string().required(), + favorite: f + .boolean() + .required() + .default(() => false), + workspaceId: f.string().required(), +}); +const blockTable = table('block', { + id: f.string().required().default(nanoid), + pageId: f.string().required(), + flavor: f.string().required(), + text: f.raw(), + props: f + .raw>() + .required() + .default(() => new YMap()), +}); + +const doc = new Doc(); +const db = createDB(doc); +const workspaceId = 'a'; +const aWorkspaceObservable = db.observeFirst(workspaceTable, { + id: workspaceId, +}); + +const merge = ( + a: Observable, + bf: (a: A) => Observable, + fieldName: F +) => { + return a.pipe( + switchMap(aValue => { + return bf(aValue).pipe( + map(bValue => { + return { + ...aValue, + [fieldName]: bValue, + } as A & { [K in F]: B }; + }) + ); + }) + ); +}; +const getPages = (workspace?: { id: string }) => { + return workspace + ? db.observeList(pageTable, { + workspaceId: workspace.id, + }) + : of([]); +}; +merge(aWorkspaceObservable, getPages, 'pages').subscribe(workspace => { + console.log(JSON.stringify(workspace, null, 2)); +}); +//or +// aWorkspaceObservable.pipe(switchMap(workspace => { +// return getPages(workspace).pipe( +// map(pages => { +// return { +// ...workspace, +// pages, +// }; +// }) +// ); +// })).subscribe(workspace => { +// console.log(JSON.stringify(workspace, null, 2)); +// }); + +const workspaceA = db.create(workspaceTable, { + id: workspaceId, + name: 'first workspace', +}); +const pageA = db.create(pageTable, { + title: 'first page', + workspaceId: workspaceA.id, + favorite: false, +}); +db.create(collectionTable, { + title: 'first collection', + workspaceId: workspaceA.id, + rules: [], +}); +db.delete(pageTable, { + id: pageA.id, +}); +db.create(blockTable, { + flavor: 'text', + pageId: pageA.id, +}); +console.log(doc.toJSON()); From 8eec06f06533e58ddcde0b0f02a2d2a69691694c Mon Sep 17 00:00:00 2001 From: zzj3720 Date: Tue, 12 Mar 2024 16:41:34 +0800 Subject: [PATCH 3/4] feat(core): migration test tool --- packages/frontend/core/src/index.tsx | 1 + packages/frontend/core/src/orm/test.ts | 27 +++ .../core/src/orm/yjs-test-tool/client.ts | 75 +++++++ .../core/src/orm/yjs-test-tool/snapshot.ts | 204 ++++++++++++++++++ .../core/src/orm/yjs-test-tool/test.ts | 76 +++++++ .../core/src/orm/yjs-test-tool/timeline.ts | 21 ++ 6 files changed, 404 insertions(+) create mode 100644 packages/frontend/core/src/orm/yjs-test-tool/client.ts create mode 100644 packages/frontend/core/src/orm/yjs-test-tool/snapshot.ts create mode 100644 packages/frontend/core/src/orm/yjs-test-tool/test.ts create mode 100644 packages/frontend/core/src/orm/yjs-test-tool/timeline.ts diff --git a/packages/frontend/core/src/index.tsx b/packages/frontend/core/src/index.tsx index 76d835861cfb..d02ee24be88d 100644 --- a/packages/frontend/core/src/index.tsx +++ b/packages/frontend/core/src/index.tsx @@ -1,6 +1,7 @@ import './polyfill/intl-segmenter'; import './polyfill/request-idle-callback'; import './orm/test'; +import './orm/yjs-test-tool/test'; import { assertExists } from '@blocksuite/global/utils'; import { StrictMode } from 'react'; diff --git a/packages/frontend/core/src/orm/test.ts b/packages/frontend/core/src/orm/test.ts index 769a5f31f93d..4b6f38093f38 100644 --- a/packages/frontend/core/src/orm/test.ts +++ b/packages/frontend/core/src/orm/test.ts @@ -117,3 +117,30 @@ db.create(blockTable, { pageId: pageA.id, }); console.log(doc.toJSON()); + +const workspace = db.create(workspaceTable, { + name: 'first workspace', +}); +const workspace$ = db.observeFirst(workspaceTable, { + id: workspace.id, +}); +workspace$ + .pipe( + switchMap(workspace => { + return getPages(workspace).pipe( + map(pages => { + return { + ...workspace, + pages, + }; + }) + ); + }) + ) + .subscribe(workspace => { + console.log(JSON.stringify(workspace, null, 2)); + }); + +db.delete(pageTable, { + id: pageA.id, +}); diff --git a/packages/frontend/core/src/orm/yjs-test-tool/client.ts b/packages/frontend/core/src/orm/yjs-test-tool/client.ts new file mode 100644 index 000000000000..a702cefb735a --- /dev/null +++ b/packages/frontend/core/src/orm/yjs-test-tool/client.ts @@ -0,0 +1,75 @@ +import { applyUpdate, Doc, encodeStateAsUpdate, encodeStateVector } from 'yjs'; + +import { fromSnapshot, type Path, type Snapshot, toSnapshot } from './snapshot'; + +export class ServerClient { + public static fromDoc(doc: Doc) { + const doc1 = new Doc(); + applyUpdate(doc1, encodeStateAsUpdate(doc)); + return new ServerClient(doc1); + } + + public static fromSnapshot(snapshot: Snapshot) { + const doc = new Doc(); + fromSnapshot(doc, snapshot); + return new ServerClient(doc); + } + + constructor(public doc: Doc) {} + + forkClient() { + return new UserClient(this); + } +} + +export class UserClient { + doc: Doc; + onlineUnsub?: () => void; + + constructor(public workspace: ServerClient) { + this.doc = new Doc(); + this.online(); + } + + private syncFromServer() { + applyUpdate( + this.doc, + encodeStateAsUpdate(this.workspace.doc, encodeStateVector(this.doc)) + ); + } + private syncToServer() { + applyUpdate( + this.workspace.doc, + encodeStateAsUpdate(this.doc, encodeStateVector(this.workspace.doc)) + ); + } + + online() { + if (this.onlineUnsub) { + return; + } + this.syncFromServer(); + this.syncToServer(); + const updateSelf = () => { + this.syncFromServer(); + }; + const updateServer = () => { + this.syncToServer(); + }; + this.workspace.doc.on('update', updateSelf); + this.doc.on('update', updateServer); + this.onlineUnsub = () => { + this.workspace.doc.off('update', updateSelf); + this.doc.off('update', updateServer); + }; + } + + offline() { + this.onlineUnsub?.(); + this.onlineUnsub = undefined; + } + + snapshot(...paths: Path[]): Snapshot { + return toSnapshot(this.doc, ...paths); + } +} diff --git a/packages/frontend/core/src/orm/yjs-test-tool/snapshot.ts b/packages/frontend/core/src/orm/yjs-test-tool/snapshot.ts new file mode 100644 index 000000000000..231a6993cdf0 --- /dev/null +++ b/packages/frontend/core/src/orm/yjs-test-tool/snapshot.ts @@ -0,0 +1,204 @@ +import type { Doc } from 'yjs'; +import { Array as YArray, Map as YMap, Text as YText } from 'yjs'; + +type DataMap = { + yarray: { + list: JSONData[]; + }; + ymap: { + props: Record; + }; + ytext: { + text: string; + }; + array: { + list: JSONData[]; + }; + object: { + props: Record; + }; + literal: { + value: string | number | boolean | null; + }; +}; +export type JSONData = { + [K in keyof DataMap]: { + type: K; + } & DataMap[K]; +}[keyof DataMap]; +export type Snapshot = Record; +type Node = number | string; +export type Path = [string, ...Node[]] | string; +const getData = (doc: Doc, path: Path) => { + const [rootPath, ...pathList] = Array.isArray(path) ? path : [path]; + let data: unknown = doc.get(rootPath); + for (const path of pathList) { + if (typeof path === 'string' && data instanceof YMap) { + data = data.get(path); + } else if (typeof path === 'number' && data instanceof YArray) { + data = data.get(path); + } else if (typeof path === 'number' && Array.isArray(data)) { + data = data[path]; + } else if ( + typeof path === 'string' && + typeof data === 'object' && + data != null + ) { + data = (data as Record)[path]; + } else { + return undefined; + } + } + return [rootPath, data]; +}; +type Result = + | { + type: 'ok'; + data: T; + } + | { + type: 'error'; + message?: string; + }; +const ok = (data: T): Result => { + return { + type: 'ok', + data, + }; +}; +const next: Result = { + type: 'error', +}; +const convertMap: { + [K in keyof DataMap]: { + to: (data: unknown) => Result; + from: (json: DataMap[K]) => unknown; + }; +} = { + ymap: { + to: data => { + if (!(data instanceof YMap)) { + return next; + } + const props = Object.fromEntries( + [...data.entries()].map(([key, value]) => [key, toJSON(value)]) + ); + return ok({ props }); + }, + from: data => + new YMap( + Object.entries(data.props).map(([key, value]) => [key, fromJSON(value)]) + ), + }, + yarray: { + to: data => { + if (!(data instanceof YArray)) { + return next; + } + return ok({ list: [...data].map(toJSON) }); + }, + from: data => { + const list = data.list.map(v => fromJSON(v) as any); + console.log(list); + return YArray.from(list); + }, + }, + ytext: { + to: data => { + if (!(data instanceof YText)) { + return next; + } + return ok({ text: data.toString() }); + }, + from: data => new YText(data.text), + }, + array: { + to: data => { + if (!Array.isArray(data)) { + return next; + } + return ok({ list: data.map(toJSON) }); + }, + from: data => data.list.map(fromJSON), + }, + object: { + to: data => { + if (typeof data !== 'object' || data == null) { + return next; + } + const props = Object.fromEntries( + Object.entries(data).map(([key, value]) => [key, toJSON(value)]) + ); + return ok({ props }); + }, + from: data => + Object.fromEntries( + Object.entries(data.props).map(([key, value]) => [key, fromJSON(value)]) + ), + }, + literal: { + to: data => { + if ( + typeof data !== 'string' && + typeof data !== 'number' && + typeof data !== 'boolean' && + data !== null + ) { + return next; + } + return ok({ value: data }); + }, + from: data => data.value, + }, +}; +const toJsonList = Object.entries(convertMap).map(([key, value]) => { + return { + key, + ...value, + }; +}); +const toJSON = (data: unknown): JSONData => { + for (const v of toJsonList) { + const result = v.to(data); + if (result.type === 'ok') { + return { + type: v.key, + ...result.data, + } as JSONData; + } + } + throw new Error('Unknown type'); +}; +const fromJSON = (json: JSONData): unknown => { + const v = convertMap[json.type]; + if (v == null) { + throw new Error('Unknown type'); + } + return v.from(json as any); +}; +const notNull = (value: T | null | undefined): value is T => value != null; +export const toSnapshot = (doc: Doc, ...paths: Path[]) => { + return Object.fromEntries( + paths + .map(path => { + const result = getData(doc, path); + if (!result) { + return undefined; + } + const [rootPath, data] = result; + return [rootPath, toJSON(data)] as const; + }) + .filter(notNull) + ); +}; +export const fromSnapshot = (doc: Doc, snapshot: Record) => { + for (const [key, value] of Object.entries(snapshot)) { + if (value.type === 'yarray') { + doc.getArray(key).push(value.list.map(v => fromJSON(v))); + } else if (value.type === 'ymap') { + Object.entries(value).forEach(([k, value]) => { + doc.getMap(key).set(k, value); + }); + } + } +}; diff --git a/packages/frontend/core/src/orm/yjs-test-tool/test.ts b/packages/frontend/core/src/orm/yjs-test-tool/test.ts new file mode 100644 index 000000000000..ea6937c4fa0c --- /dev/null +++ b/packages/frontend/core/src/orm/yjs-test-tool/test.ts @@ -0,0 +1,76 @@ +import Y from 'yjs'; + +import { ServerClient } from './client'; +import { toSnapshot } from './snapshot'; +import { timeline } from './timeline'; + +const nestedExample = async () => { + const migrateCollection = (doc: Y.Doc) => { + let collection = doc + .getMap('userSetting') + .get('collection') as Y.Array; + if (!collection) { + collection = new Y.Array(); + doc.getMap('userSetting').set('collection', collection); + } + return collection; + }; + await migrateTest((doc, clientName) => { + const collection = migrateCollection(doc); + collection.insert(0, [clientName]); + }); +}; +const flatExample = async () => { + await migrateTest((doc, clientName) => { + doc.getArray('collection').insert(0, [clientName]); + }); +}; + +const migrateTest = async ( + migrate: (doc: Y.Doc, clientName: string) => void +) => { + const dataLog = () => { + console.log( + JSON.stringify(clientA.doc.toJSON(), null, 2), + JSON.stringify(clientB.doc.toJSON(), null, 2) + ); + }; + const doc1 = new Y.Doc(); + const initSnapshot = toSnapshot(doc1); + + const serverClient = ServerClient.fromSnapshot(initSnapshot); + const clientA = serverClient.forkClient(); + const clientB = serverClient.forkClient(); + dataLog(); + await timeline + .step('clientB offline', async () => { + clientB.offline(); + }) + .step('clientA migrating', async () => { + migrate(clientA.doc, 'A'); + dataLog(); + }) + .step('clientB migrating', async () => { + migrate(clientB.doc, 'B'); + dataLog(); + }) + .step('clientB online', async () => { + clientB.online(); + }) + .run(); + dataLog(); + // 在这里断言 expect(clientA.snapshot).toEqual(expectSnapshot) +}; +setTimeout(() => { + console.group('nested data example'); + nestedExample() + .then(async () => { + console.groupEnd(); + console.group('flat data example'); + await flatExample(); + console.groupEnd(); + }) + .catch(e => { + console.error(e); + }); +}, 2000); diff --git a/packages/frontend/core/src/orm/yjs-test-tool/timeline.ts b/packages/frontend/core/src/orm/yjs-test-tool/timeline.ts new file mode 100644 index 000000000000..38ba27d31f5c --- /dev/null +++ b/packages/frontend/core/src/orm/yjs-test-tool/timeline.ts @@ -0,0 +1,21 @@ +class Timeline { + constructor(private readonly tasks: (() => Promise)[] = []) {} + + step(name: string, callback: () => Promise): Timeline { + return new Timeline([ + ...this.tasks, + () => { + console.log(name); + return callback(); + }, + ]); + } + + async run() { + for (const task of this.tasks) { + await task(); + } + } +} + +export const timeline = new Timeline(); From ef3f9674313c760afacc267ae7aabc7cab6d989c Mon Sep 17 00:00:00 2001 From: zzj3720 Date: Tue, 12 Mar 2024 17:04:04 +0800 Subject: [PATCH 4/4] feat(core): migration test tool --- packages/frontend/core/src/orm/yjs-test-tool/test.ts | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/packages/frontend/core/src/orm/yjs-test-tool/test.ts b/packages/frontend/core/src/orm/yjs-test-tool/test.ts index ea6937c4fa0c..7579c88b554b 100644 --- a/packages/frontend/core/src/orm/yjs-test-tool/test.ts +++ b/packages/frontend/core/src/orm/yjs-test-tool/test.ts @@ -1,7 +1,6 @@ import Y from 'yjs'; import { ServerClient } from './client'; -import { toSnapshot } from './snapshot'; import { timeline } from './timeline'; const nestedExample = async () => { @@ -36,9 +35,8 @@ const migrateTest = async ( ); }; const doc1 = new Y.Doc(); - const initSnapshot = toSnapshot(doc1); - const serverClient = ServerClient.fromSnapshot(initSnapshot); + const serverClient = ServerClient.fromDoc(doc1); const clientA = serverClient.forkClient(); const clientB = serverClient.forkClient(); dataLog();