diff --git a/packages/blocks/src/__tests__/adapters/html.unit.spec.ts b/packages/blocks/src/__tests__/adapters/html.unit.spec.ts index cd3a63706cff..7ca630d18ab4 100644 --- a/packages/blocks/src/__tests__/adapters/html.unit.spec.ts +++ b/packages/blocks/src/__tests__/adapters/html.unit.spec.ts @@ -1,5 +1,5 @@ import type { BlockSnapshot } from '@blocksuite/store'; -import { MemoryBlobManager } from '@blocksuite/store'; +import { MemoryBlobCRUD } from '@blocksuite/store'; import { AssetsManager } from '@blocksuite/store'; import { describe, expect, test } from 'vitest'; @@ -946,10 +946,10 @@ describe('snapshot to html', () => { ); const htmlAdapter = new HtmlAdapter(); - const blobManager = new MemoryBlobManager(); + const blobManager = new MemoryBlobCRUD(); await blobManager.set( - new Blob(), - 'YXXTjRmLlNyiOUnHb8nAIvUP6V7PAXhwW9F5_tc2LGs=' + 'YXXTjRmLlNyiOUnHb8nAIvUP6V7PAXhwW9F5_tc2LGs=', + new Blob() ); const assets = new AssetsManager({ blob: blobManager }); diff --git a/packages/blocks/src/__tests__/adapters/markdown.unit.spec.ts b/packages/blocks/src/__tests__/adapters/markdown.unit.spec.ts index 1824cea54a13..a29be7eb1b4f 100644 --- a/packages/blocks/src/__tests__/adapters/markdown.unit.spec.ts +++ b/packages/blocks/src/__tests__/adapters/markdown.unit.spec.ts @@ -1,5 +1,5 @@ import type { BlockSnapshot, SliceSnapshot } from '@blocksuite/store'; -import { AssetsManager, MemoryBlobManager } from '@blocksuite/store'; +import { AssetsManager, MemoryBlobCRUD } from '@blocksuite/store'; import { describe, expect, test } from 'vitest'; import { MarkdownAdapter } from '../../_common/adapters/markdown.js'; @@ -1140,12 +1140,12 @@ hhh '![](assets/YXXTjRmLlNyiOUnHb8nAIvUP6V7PAXhwW9F5_tc2LGs=.blob "aaa")\n\n'; const mdAdapter = new MarkdownAdapter(); - const blobManager = new MemoryBlobManager(); - await blobManager.set( - new Blob(), - 'YXXTjRmLlNyiOUnHb8nAIvUP6V7PAXhwW9F5_tc2LGs=' + const blobCRUD = new MemoryBlobCRUD(); + await blobCRUD.set( + 'YXXTjRmLlNyiOUnHb8nAIvUP6V7PAXhwW9F5_tc2LGs=', + new Blob() ); - const assets = new AssetsManager({ blob: blobManager }); + const assets = new AssetsManager({ blob: blobCRUD }); const target = await mdAdapter.fromBlockSnapshot({ snapshot: blockSnapshot, diff --git a/packages/blocks/src/__tests__/adapters/notion-html.unit.spec.ts b/packages/blocks/src/__tests__/adapters/notion-html.unit.spec.ts index 24279cc2ad7a..3a2aad5c097e 100644 --- a/packages/blocks/src/__tests__/adapters/notion-html.unit.spec.ts +++ b/packages/blocks/src/__tests__/adapters/notion-html.unit.spec.ts @@ -1,7 +1,7 @@ import { AssetsManager, type BlockSnapshot, - MemoryBlobManager, + MemoryBlobCRUD, } from '@blocksuite/store'; import { describe, expect, test } from 'vitest'; @@ -1084,7 +1084,7 @@ describe('notion html to snapshot', () => { const adapter = new NotionHtmlAdapter(); const rawBlockSnapshot = await adapter.toBlockSnapshot({ file: html, - assets: new AssetsManager({ blob: new MemoryBlobManager() }), + assets: new AssetsManager({ blob: new MemoryBlobCRUD() }), }); expect(nanoidReplacement(rawBlockSnapshot)).toEqual(blockSnapshot); }); @@ -1176,9 +1176,9 @@ describe('notion html to snapshot', () => { }; const adapter = new NotionHtmlAdapter(); - const blobManager = new MemoryBlobManager(); - const key = await blobManager.set(new File([], 'README.pdf')); - const assestsManager = new AssetsManager({ blob: blobManager }); + const blobCRUD = new MemoryBlobCRUD(); + const key = await blobCRUD.set(new File([], 'README.pdf')); + const assestsManager = new AssetsManager({ blob: blobCRUD }); await assestsManager.readFromBlob(key); const rawBlockSnapshot = await adapter.toBlockSnapshot({ file: html, diff --git a/packages/blocks/src/_common/adapters/attachment.ts b/packages/blocks/src/_common/adapters/attachment.ts index 8d60cafdcd35..ab40af927860 100644 --- a/packages/blocks/src/_common/adapters/attachment.ts +++ b/packages/blocks/src/_common/adapters/attachment.ts @@ -1,3 +1,4 @@ +import { sha } from '@blocksuite/global/utils'; import type { AssetsManager } from '@blocksuite/store'; import { BaseAdapter, @@ -10,7 +11,6 @@ import { type FromSliceSnapshotPayload, type FromSliceSnapshotResult, nanoid, - sha, type SliceSnapshot, type ToBlockSnapshotPayload, type ToDocSnapshotPayload, diff --git a/packages/blocks/src/_common/adapters/html.ts b/packages/blocks/src/_common/adapters/html.ts index 09c2d09c7e64..af3d42a522fd 100644 --- a/packages/blocks/src/_common/adapters/html.ts +++ b/packages/blocks/src/_common/adapters/html.ts @@ -1,3 +1,4 @@ +import { sha } from '@blocksuite/global/utils'; import type { DeltaInsert } from '@blocksuite/inline'; import type { FromBlockSnapshotPayload, @@ -14,7 +15,6 @@ import { BlockSnapshotSchema, getAssetName, nanoid, - sha, } from '@blocksuite/store'; import { ASTWalker, BaseAdapter } from '@blocksuite/store'; import { diff --git a/packages/blocks/src/_common/adapters/image.ts b/packages/blocks/src/_common/adapters/image.ts index f18e6b6ea27f..45727208638f 100644 --- a/packages/blocks/src/_common/adapters/image.ts +++ b/packages/blocks/src/_common/adapters/image.ts @@ -1,3 +1,4 @@ +import { sha } from '@blocksuite/global/utils'; import type { AssetsManager } from '@blocksuite/store'; import { BaseAdapter, @@ -10,7 +11,6 @@ import { type FromSliceSnapshotPayload, type FromSliceSnapshotResult, nanoid, - sha, type SliceSnapshot, type ToBlockSnapshotPayload, type ToDocSnapshotPayload, diff --git a/packages/blocks/src/_common/adapters/markdown.ts b/packages/blocks/src/_common/adapters/markdown.ts index ef1b3a393be0..88bbb5a4f4e0 100644 --- a/packages/blocks/src/_common/adapters/markdown.ts +++ b/packages/blocks/src/_common/adapters/markdown.ts @@ -1,3 +1,4 @@ +import { sha } from '@blocksuite/global/utils'; import type { DeltaInsert } from '@blocksuite/inline/types'; import type { FromBlockSnapshotPayload, @@ -18,7 +19,6 @@ import { type DocSnapshot, getAssetName, nanoid, - sha, type SliceSnapshot, } from '@blocksuite/store'; import { format } from 'date-fns/format'; diff --git a/packages/blocks/src/_common/adapters/notion-html.ts b/packages/blocks/src/_common/adapters/notion-html.ts index d3708aebad2f..9994f84658dd 100644 --- a/packages/blocks/src/_common/adapters/notion-html.ts +++ b/packages/blocks/src/_common/adapters/notion-html.ts @@ -1,4 +1,4 @@ -import { isEqual } from '@blocksuite/global/utils'; +import { isEqual, sha } from '@blocksuite/global/utils'; import type { DeltaInsert } from '@blocksuite/inline'; import type { FromBlockSnapshotPayload, @@ -16,7 +16,6 @@ import { type DocSnapshot, getAssetName, nanoid, - sha, type SliceSnapshot, } from '@blocksuite/store'; import rehypeParse from 'rehype-parse'; diff --git a/packages/blocks/src/_common/export-manager/export-manager.ts b/packages/blocks/src/_common/export-manager/export-manager.ts index 8014963ae8a4..976e4a9c9bf3 100644 --- a/packages/blocks/src/_common/export-manager/export-manager.ts +++ b/packages/blocks/src/_common/export-manager/export-manager.ts @@ -242,7 +242,7 @@ export class ExportManager { if (matchFlavours(block, ['affine:image'])) { if (!block.sourceId) return; - const blob = await block.doc.blob.get(block.sourceId); + const blob = await block.doc.blobSync.get(block.sourceId); if (!blob) return; const blobToImage = (blob: Blob) => diff --git a/packages/blocks/src/_common/transformers/zip.ts b/packages/blocks/src/_common/transformers/zip.ts index d070ff6e3863..4d3a544c948c 100644 --- a/packages/blocks/src/_common/transformers/zip.ts +++ b/packages/blocks/src/_common/transformers/zip.ts @@ -1,4 +1,4 @@ -import { assertExists } from '@blocksuite/global/utils'; +import { assertExists, sha } from '@blocksuite/global/utils'; import type { CollectionInfoSnapshot, Doc, @@ -6,7 +6,7 @@ import type { DocSnapshot, JobMiddleware, } from '@blocksuite/store'; -import { extMimeMap, getAssetName, Job, sha } from '@blocksuite/store'; +import { extMimeMap, getAssetName, Job } from '@blocksuite/store'; import JSZip from 'jszip'; import { replaceIdMiddleware, titleMiddleware } from './middlewares.js'; diff --git a/packages/blocks/src/_common/utils/filesys.ts b/packages/blocks/src/_common/utils/filesys.ts index 4533bab75703..dcddbced6b42 100644 --- a/packages/blocks/src/_common/utils/filesys.ts +++ b/packages/blocks/src/_common/utils/filesys.ts @@ -1,5 +1,3 @@ -import type { BlobManager } from '@blocksuite/store'; - // Polyfill for `showOpenFilePicker` API // See https://github.com/DefinitelyTyped/DefinitelyTyped/blob/master/types/wicg-file-system-access/index.d.ts // See also https://caniuse.com/?search=showOpenFilePicker @@ -223,29 +221,6 @@ export async function getImageFilesFromLocal() { return imageFiles; } -export async function uploadImageFromLocal(storage: BlobManager) { - const imageFiles = await openFileOrFiles({ - acceptType: 'Images', - multiple: true, - }); - if (!imageFiles) return []; - return loadImages(imageFiles, storage); -} - -export async function loadImages(images: File[], storage: BlobManager) { - const res: { file: File; sourceId: string }[] = []; - for (let i = 0; i < images.length; i++) { - const file = images[i]; - const sourceId = await storage.set(file); - res.push({ file, sourceId }); - } - const { saveAttachmentData } = withTempBlobData(); - res.forEach(({ file, sourceId }) => { - saveAttachmentData(sourceId, { name: file.name }); - }); - return res; -} - export function downloadBlob(blob: Blob, name: string) { const dataURL = URL.createObjectURL(blob); const tmpLink = document.createElement('a'); diff --git a/packages/blocks/src/_common/utils/render-linked-doc.ts b/packages/blocks/src/_common/utils/render-linked-doc.ts index c4a20e3b30bc..e49816fd2d00 100644 --- a/packages/blocks/src/_common/utils/render-linked-doc.ts +++ b/packages/blocks/src/_common/utils/render-linked-doc.ts @@ -323,7 +323,7 @@ async function renderImageAbstract( const sourceId = (image as ImageBlockModel).sourceId; if (!sourceId) return; - const storage = card.linkedDoc?.blob; + const storage = card.linkedDoc?.blobSync; if (!storage) return; const blob = await storage.get(sourceId); diff --git a/packages/blocks/src/attachment-block/utils.ts b/packages/blocks/src/attachment-block/utils.ts index 296ab65ed954..9743ab6c4b70 100644 --- a/packages/blocks/src/attachment-block/utils.ts +++ b/packages/blocks/src/attachment-block/utils.ts @@ -52,7 +52,7 @@ async function uploadAttachmentBlob( try { setAttachmentUploading(blockId); - sourceId = await doc.blob.set(blob); + sourceId = await doc.blobSync.set(blob); } catch (error) { console.error(error); if (error instanceof Error) { @@ -84,7 +84,7 @@ async function getAttachmentBlob(model: AttachmentBlockModel) { } const doc = model.doc; - let blob = await doc.blob.get(sourceId); + let blob = await doc.blobSync.get(sourceId); if (blob) { blob = new Blob([blob], { type: model.type }); diff --git a/packages/blocks/src/image-block/utils.ts b/packages/blocks/src/image-block/utils.ts index 4db87f9f46c3..6edf76ed2843 100644 --- a/packages/blocks/src/image-block/utils.ts +++ b/packages/blocks/src/image-block/utils.ts @@ -39,7 +39,7 @@ export async function uploadBlobForImage( try { setImageUploaded(blockId); - sourceId = await doc.blob.set(blob); + sourceId = await doc.blobSync.set(blob); } catch (error) { console.error(error); if (error instanceof Error) { @@ -69,7 +69,7 @@ async function getImageBlob(model: ImageBlockModel) { } const doc = model.doc; - const blob = await doc.blob.get(sourceId); + const blob = await doc.blobSync.get(sourceId); if (!blob) { return null; @@ -123,7 +123,7 @@ export async function fetchImageBlob(block: ImageBlockComponent) { throw new Error('Image sourceId is missing!'); } - const blob = await doc.blob.get(sourceId); + const blob = await doc.blobSync.get(sourceId); if (!blob) { throw new Error('Image blob is missing!'); } diff --git a/packages/blocks/src/root-block/edgeless/edgeless-root-block.ts b/packages/blocks/src/root-block/edgeless/edgeless-root-block.ts index e0d6eb113096..20f5a121eaf2 100644 --- a/packages/blocks/src/root-block/edgeless/edgeless-root-block.ts +++ b/packages/blocks/src/root-block/edgeless/edgeless-root-block.ts @@ -417,7 +417,7 @@ export class EdgelessRootBlockComponent extends BlockElement< const uploadPromises = imageFiles.map(async (file, index) => { const { point, blockId } = dropInfos[index]; - const sourceId = await this.doc.blob.set(file); + const sourceId = await this.doc.blobSync.set(file); const imageSize = await readImageSize(file); const center = Vec.toVec(point); @@ -502,7 +502,7 @@ export class EdgelessRootBlockComponent extends BlockElement< let sourceId: string | undefined; try { setAttachmentUploading(blockId); - sourceId = await this.doc.blob.set(file); + sourceId = await this.doc.blobSync.set(file); } catch (error) { console.error(error); if (error instanceof Error) { diff --git a/packages/blocks/src/root-block/widgets/linked-doc/import-doc/import-doc.ts b/packages/blocks/src/root-block/widgets/linked-doc/import-doc/import-doc.ts index 0cdd8456c39d..a858851bf49e 100644 --- a/packages/blocks/src/root-block/widgets/linked-doc/import-doc/import-doc.ts +++ b/packages/blocks/src/root-block/widgets/linked-doc/import-doc/import-doc.ts @@ -1,7 +1,8 @@ import '../../../../_common/components/loader.js'; import { WithDisposable } from '@blocksuite/block-std'; -import { type DocCollection, extMimeMap, sha } from '@blocksuite/store'; +import { sha } from '@blocksuite/global/utils'; +import { type DocCollection, extMimeMap } from '@blocksuite/store'; import { Job } from '@blocksuite/store'; import JSZip from 'jszip'; import { html, LitElement, type PropertyValues } from 'lit'; diff --git a/packages/blocks/src/surface-block/mini-mindmap/mindmap-preview.ts b/packages/blocks/src/surface-block/mini-mindmap/mindmap-preview.ts index 35addd52a91d..cd4394cb4460 100644 --- a/packages/blocks/src/surface-block/mini-mindmap/mindmap-preview.ts +++ b/packages/blocks/src/surface-block/mini-mindmap/mindmap-preview.ts @@ -127,7 +127,6 @@ export class MiniMindmapPreview extends WithDisposable(LitElement) { id: 'MINI_MINDMAP_TEMPORARY', schema, idGenerator: Generator.NanoID, - blobStorages: [], awarenessSources: [], }; diff --git a/packages/framework/global/package.json b/packages/framework/global/package.json index f15c64325f36..55eb4916b72b 100644 --- a/packages/framework/global/package.json +++ b/packages/framework/global/package.json @@ -49,6 +49,7 @@ "!dist/__tests__" ], "dependencies": { + "lib0": "^0.2.93", "zod": "^3.23.8" } } diff --git a/packages/framework/store/src/persistence/blob/utils.ts b/packages/framework/global/src/utils/crypto.ts similarity index 100% rename from packages/framework/store/src/persistence/blob/utils.ts rename to packages/framework/global/src/utils/crypto.ts diff --git a/packages/framework/global/src/utils/index.ts b/packages/framework/global/src/utils/index.ts index b24cd81b3079..ee07c1c27456 100644 --- a/packages/framework/global/src/utils/index.ts +++ b/packages/framework/global/src/utils/index.ts @@ -1,4 +1,5 @@ export * from './assert.js'; +export * from './crypto.js'; export * from './disposable.js'; export * from './function.js'; export * from './logger.js'; diff --git a/packages/framework/store/src/__tests__/transformer.unit.spec.ts b/packages/framework/store/src/__tests__/transformer.unit.spec.ts index d34736ad4885..f3258ff7ed72 100644 --- a/packages/framework/store/src/__tests__/transformer.unit.spec.ts +++ b/packages/framework/store/src/__tests__/transformer.unit.spec.ts @@ -1,7 +1,7 @@ import { expect, test } from 'vitest'; import * as Y from 'yjs'; -import { MemoryBlobManager } from '../adapter/index.js'; +import { MemoryBlobCRUD } from '../adapter/index.js'; import { Text } from '../reactive/index.js'; import { type BlockModel, @@ -51,8 +51,8 @@ function createTestOptions() { } const transformer = new BaseBlockTransformer(); -const blobManager = new MemoryBlobManager(); -const assets = new AssetsManager({ blob: blobManager }); +const blobCRUD = new MemoryBlobCRUD(); +const assets = new AssetsManager({ blob: blobCRUD }); test('model to snapshot', () => { const options = createTestOptions(); diff --git a/packages/framework/store/src/adapter/assets.ts b/packages/framework/store/src/adapter/assets.ts index b1e6a5da43e4..0716206972c4 100644 --- a/packages/framework/store/src/adapter/assets.ts +++ b/packages/framework/store/src/adapter/assets.ts @@ -1,19 +1,30 @@ -import { assertExists } from '@blocksuite/global/utils'; +import { assertExists, sha } from '@blocksuite/global/utils'; -import { sha } from '../persistence/blob/utils.js'; - -export class MemoryBlobManager { +/** + * @internal just for test + */ +export class MemoryBlobCRUD { private readonly _map = new Map(); - private readonly _blobsRef = new Map(); get(key: string) { return this._map.get(key) ?? null; } - async set(value: Blob, key?: string) { - const _key = key || (await sha(await value.arrayBuffer())); - this._map.set(_key, value); - return _key; + async set(value: Blob): Promise; + async set(key: string, value: Blob): Promise; + async set(valueOrKey: string | Blob, _value?: Blob) { + const key = + typeof valueOrKey === 'string' + ? valueOrKey + : await sha(await valueOrKey.arrayBuffer()); + const value = typeof valueOrKey === 'string' ? _value : valueOrKey; + + if (!value) { + throw new Error('value is required'); + } + + this._map.set(key, value); + return key; } delete(key: string) { @@ -23,27 +34,6 @@ export class MemoryBlobManager { list() { return Array.from(this._map.keys()); } - - gc() { - const blobs = this.list(); - blobs.forEach(blobId => { - const ref = this._blobsRef.get(blobId); - if (!ref || ref <= 0) { - this.delete(blobId); - this._blobsRef.delete(blobId); - } - }); - } - - increaseRef(blobId: string) { - const ref = this._blobsRef.get(blobId) ?? 0; - this._blobsRef.set(blobId, ref + 1); - } - - decreaseRef(blobId: string) { - const ref = this._blobsRef.get(blobId) ?? 0; - this._blobsRef.set(blobId, Math.max(ref - 1, 0)); - } } export const mimeExtMap = new Map([ diff --git a/packages/framework/store/src/index.ts b/packages/framework/store/src/index.ts index f50649a5e0a3..0018b6d61ea5 100644 --- a/packages/framework/store/src/index.ts +++ b/packages/framework/store/src/index.ts @@ -5,15 +5,6 @@ export type { Y }; export * from './adapter/index.js'; export * from './migration/index.js'; -export { createIndexeddbStorage } from './persistence/blob/indexeddb-storage.js'; -export { createMemoryStorage } from './persistence/blob/memory-storage.js'; -export { createSimpleServerStorage } from './persistence/blob/mock-server-storage.js'; -export type { - BlobManager, - BlobStorage, - BlobStorageCRUD, -} from './persistence/blob/types.js'; -export { sha } from './persistence/blob/utils.js'; export * from './reactive/index.js'; export * from './schema/index.js'; export * from './store/index.js'; diff --git a/packages/framework/store/src/persistence/blob/indexeddb-storage.ts b/packages/framework/store/src/persistence/blob/indexeddb-storage.ts deleted file mode 100644 index 43e44125d090..000000000000 --- a/packages/framework/store/src/persistence/blob/indexeddb-storage.ts +++ /dev/null @@ -1,42 +0,0 @@ -import type { UseStore } from 'idb-keyval'; - -import type { BlobStorage } from './types.js'; - -export const createIndexeddbStorage = (database: string): BlobStorage => { - let db: UseStore; - let mimeTypeDb: UseStore; - // async import `idb-keyval` to avoid side effect - const idbPromise = import('idb-keyval').then(({ createStore, ...idb }) => { - // don't change the db name, it's for backward compatibility - db = createStore(`${database}_blob`, 'blob'); - mimeTypeDb = createStore(`${database}_blob_mime`, 'blob_mime'); - return idb; - }); - return { - crud: { - get: async (key: string) => { - const get = (await idbPromise).get; - const res = await get(key, db); - if (res) { - return new Blob([res], { type: await get(key, mimeTypeDb) }); - } - return null; - }, - set: async (key: string, value: Blob) => { - const set = (await idbPromise).set; - await set(key, await value.arrayBuffer(), db); - await set(key, value.type, mimeTypeDb); - return key; - }, - delete: async (key: string) => { - const del = (await idbPromise).del; - await del(key, db); - await del(key, mimeTypeDb); - }, - list: async () => { - const keys = (await idbPromise).keys; - return keys(db); - }, - }, - }; -}; diff --git a/packages/framework/store/src/persistence/blob/memory-storage.ts b/packages/framework/store/src/persistence/blob/memory-storage.ts deleted file mode 100644 index be6a5975614c..000000000000 --- a/packages/framework/store/src/persistence/blob/memory-storage.ts +++ /dev/null @@ -1,22 +0,0 @@ -import type { BlobStorage } from './types.js'; - -export const createMemoryStorage = (): BlobStorage => { - const memoryStorage = new Map(); - return { - crud: { - get: (key: string) => { - return memoryStorage.get(key) ?? null; - }, - set: (key: string, value: Blob) => { - memoryStorage.set(key, value); - return key; - }, - delete: (key: string) => { - memoryStorage.delete(key); - }, - list: () => { - return Array.from(memoryStorage.keys()); - }, - }, - }; -}; diff --git a/packages/framework/store/src/persistence/blob/mock-server-storage.ts b/packages/framework/store/src/persistence/blob/mock-server-storage.ts deleted file mode 100644 index 17fcf1dc6608..000000000000 --- a/packages/framework/store/src/persistence/blob/mock-server-storage.ts +++ /dev/null @@ -1,48 +0,0 @@ -import type { BlobStorage } from './types.js'; - -/** - * API: /api/collection/:id/blob/:key - * GET: get blob - * PUT: set blob - * DELETE: delete blob - */ -export function createSimpleServerStorage(id: string): BlobStorage { - const localCache = new Map(); - return { - crud: { - get: async (key: string) => { - if (localCache.has(key)) { - return localCache.get(key) as Blob; - } else { - const blob = await fetch(`/api/collection/${id}/blob/${key}`, { - method: 'GET', - }).then(response => { - if (!response.ok) { - throw new Error(`Failed to fetch blob ${key}`); - } - return response.blob(); - }); - localCache.set(key, blob); - return blob; - } - }, - set: async (key: string, value: Blob) => { - localCache.set(key, value); - await fetch(`/api/collection/${id}/blob/${key}`, { - method: 'PUT', - body: await value.arrayBuffer(), - }); - return key; - }, - delete: async (key: string) => { - localCache.delete(key); - await fetch(`/api/collection/${id}/blob/${key}`, { - method: 'DELETE', - }); - }, - list: () => { - return Array.from(localCache.keys()); - }, - }, - }; -} diff --git a/packages/framework/store/src/persistence/blob/types.ts b/packages/framework/store/src/persistence/blob/types.ts deleted file mode 100644 index 5153a849b881..000000000000 --- a/packages/framework/store/src/persistence/blob/types.ts +++ /dev/null @@ -1,17 +0,0 @@ -export interface BlobStorageCRUD { - get: (key: string) => Promise | Blob | null; - set: (key: string, value: Blob) => Promise | string; - delete: (key: string) => Promise | void; - list: () => Promise | string[]; -} - -export interface BlobManager { - get: (key: string) => Promise | Blob | null; - set: (value: Blob, key?: string) => Promise | string; - delete: (key: string) => Promise | void; - list: () => Promise | string[]; -} - -export interface BlobStorage { - crud: BlobStorageCRUD; -} diff --git a/packages/framework/store/src/store/addon/blob.ts b/packages/framework/store/src/store/addon/blob.ts deleted file mode 100644 index cf619a4cbe70..000000000000 --- a/packages/framework/store/src/store/addon/blob.ts +++ /dev/null @@ -1,70 +0,0 @@ -import { createMemoryStorage } from '../../persistence/blob/memory-storage.js'; -import type { BlobManager, BlobStorage } from '../../persistence/blob/types.js'; -import { sha } from '../../persistence/blob/utils.js'; -import type { DocCollectionOptions } from '../collection.js'; -import { addOnFactory } from './shared.js'; - -export interface BlobAddon { - get blob(): BlobManager; -} - -export const blob = addOnFactory( - originalClass => - class extends originalClass { - private readonly _storages: BlobStorage[] = []; - - readonly blob: BlobManager; - - constructor(storeOptions: DocCollectionOptions) { - super(storeOptions); - - this._storages = ( - storeOptions.blobStorages ?? [createMemoryStorage] - ).map(fn => fn(storeOptions.id || '')); - - this.blob = { - get: async id => { - let found = false; - let count = 0; - return new Promise(res => { - void this._storages.map(async storage => { - try { - const blob = await storage.crud.get(id); - if (blob && !found) { - found = true; - res(blob); - } - if (++count === this._storages.length && !found) { - res(null); - } - } catch (e) { - console.error(e); - if (++count === this._storages.length && !found) { - res(null); - } - } - }); - }); - }, - set: async (value, key) => { - const _key = key || (await sha(await value.arrayBuffer())); - await Promise.all(this._storages.map(s => s.crud.set(_key, value))); - return _key; - }, - delete: async key => { - await Promise.all(this._storages.map(s => s.crud.delete(key))); - }, - list: async () => { - const keys = new Set(); - await Promise.all( - this._storages.map(async s => { - const list = await s.crud.list(); - list.forEach(key => keys.add(key)); - }) - ); - return Array.from(keys); - }, - }; - } - } -); diff --git a/packages/framework/store/src/store/addon/index.ts b/packages/framework/store/src/store/addon/index.ts index 59283c542667..9a01cf0a8bcb 100644 --- a/packages/framework/store/src/store/addon/index.ts +++ b/packages/framework/store/src/store/addon/index.ts @@ -1,4 +1,3 @@ -export { blob } from './blob.js'; export { indexer } from './indexer.js'; export { test } from './test.js'; export { DocCollectionAddonType } from './type.js'; diff --git a/packages/framework/store/src/store/addon/type.ts b/packages/framework/store/src/store/addon/type.ts index 1e20488ba3da..27dcc7687fed 100644 --- a/packages/framework/store/src/store/addon/type.ts +++ b/packages/framework/store/src/store/addon/type.ts @@ -1,12 +1,7 @@ -import type { BlobAddon } from './blob.js'; import type { IndexerAddon } from './indexer.js'; import type { TestAddon } from './test.js'; -export class DocCollectionAddonType - implements BlobAddon, IndexerAddon, TestAddon -{ - blob!: BlobAddon['blob']; - +export class DocCollectionAddonType implements IndexerAddon, TestAddon { indexer!: IndexerAddon['indexer']; search!: IndexerAddon['search']; diff --git a/packages/framework/store/src/store/collection.ts b/packages/framework/store/src/store/collection.ts index f74c7ec78a1b..2966a16c22bb 100644 --- a/packages/framework/store/src/store/collection.ts +++ b/packages/framework/store/src/store/collection.ts @@ -3,7 +3,7 @@ import * as Y from 'yjs'; import type { Schema } from '../schema/index.js'; import type { AwarenessStore } from '../yjs/index.js'; -import { blob, DocCollectionAddonType, indexer, test } from './addon/index.js'; +import { DocCollectionAddonType, indexer, test } from './addon/index.js'; import { BlockCollection, defaultBlockSelector, @@ -16,7 +16,6 @@ export type DocCollectionOptions = StoreOptions & { schema: Schema; }; -@blob @indexer @test export class DocCollection extends DocCollectionAddonType { @@ -93,6 +92,10 @@ export class DocCollection extends DocCollectionAddonType { return this.store.awarenessSync; } + get blobSync() { + return this.store.blobSync; + } + private _hasDoc(docId: string) { return this.docs.has(docId); } @@ -182,6 +185,7 @@ export class DocCollection extends DocCollectionAddonType { */ start() { this.docSync.start(); + this.blobSync.start(); this.awarenessSync.connect(); } @@ -206,6 +210,7 @@ export class DocCollection extends DocCollectionAddonType { */ forceStop() { this.docSync.forceStop(); + this.blobSync.stop(); this.awarenessSync.disconnect(); } diff --git a/packages/framework/store/src/store/doc/block-collection.ts b/packages/framework/store/src/store/doc/block-collection.ts index 24a0c85dc01c..bc5a94a69ed3 100644 --- a/packages/framework/store/src/store/doc/block-collection.ts +++ b/packages/framework/store/src/store/doc/block-collection.ts @@ -141,6 +141,18 @@ export class BlockCollection extends Space { return this._collection; } + get docSync() { + return this.collection.docSync; + } + + get awarenessSync() { + return this.collection.awarenessSync; + } + + get blobSync() { + return this.collection.blobSync; + } + get schema() { return this.collection.schema; } @@ -149,10 +161,6 @@ export class BlockCollection extends Space { return this.collection.meta.getDocMeta(this.id); } - get blob() { - return this.collection.blob; - } - get isEmpty() { return this._yBlocks.size === 0; } diff --git a/packages/framework/store/src/store/doc/doc.ts b/packages/framework/store/src/store/doc/doc.ts index 505b528bf400..fb790ef1a1a2 100644 --- a/packages/framework/store/src/store/doc/doc.ts +++ b/packages/framework/store/src/store/doc/doc.ts @@ -80,12 +80,20 @@ export class Doc { return this._blockCollection.collection; } - get meta() { - return this._blockCollection.meta; + get docSync() { + return this.collection.docSync; + } + + get awarenessSync() { + return this.collection.awarenessSync; } - get blob() { - return this._blockCollection.blob; + get blobSync() { + return this.collection.blobSync; + } + + get meta() { + return this._blockCollection.meta; } get isEmpty() { diff --git a/packages/framework/store/src/store/store.ts b/packages/framework/store/src/store/store.ts index 0c4fc3b58663..a73e1f16f3f3 100644 --- a/packages/framework/store/src/store/store.ts +++ b/packages/framework/store/src/store/store.ts @@ -2,14 +2,16 @@ import { type Logger, NoopLogger } from '@blocksuite/global/utils'; import { AwarenessEngine, type AwarenessSource, + BlobEngine, + type BlobSource, DocEngine, type DocSource, + MemoryBlobSource, NoopDocSource, } from '@blocksuite/sync'; import { merge } from 'merge'; import { Awareness } from 'y-protocols/awareness.js'; -import type { BlobStorage } from '../persistence/blob/types.js'; import type { IdGenerator } from '../utils/id-generator.js'; import { createAutoIncrementIdGenerator, @@ -54,11 +56,14 @@ export interface StoreOptions< id?: string; idGenerator?: Generator | IdGenerator; defaultFlags?: Partial; - blobStorages?: ((id: string) => BlobStorage)[]; logger?: Logger; docSources?: { main: DocSource; - shadow?: DocSource[]; + shadows?: DocSource[]; + }; + blobSources?: { + main: BlobSource; + shadows?: BlobSource[]; }; awarenessSources?: AwarenessSource[]; } @@ -86,6 +91,7 @@ export class Store { readonly docSync: DocEngine; readonly awarenessSync: AwarenessEngine; + readonly blobSync: BlobEngine; constructor( { @@ -96,6 +102,9 @@ export class Store { docSources = { main: new NoopDocSource(), }, + blobSources = { + main: new MemoryBlobSource(), + }, logger = new NoopLogger(), }: StoreOptions = { id: nanoid(), @@ -116,7 +125,12 @@ export class Store { this.docSync = new DocEngine( this.doc, docSources.main, - docSources.shadow ?? [], + docSources.shadows ?? [], + logger + ); + this.blobSync = new BlobEngine( + blobSources.main, + blobSources.shadows ?? [], logger ); diff --git a/packages/framework/store/src/transformer/assets.ts b/packages/framework/store/src/transformer/assets.ts index 064c6a1973bd..a92d8d392177 100644 --- a/packages/framework/store/src/transformer/assets.ts +++ b/packages/framework/store/src/transformer/assets.ts @@ -1,14 +1,19 @@ import { assertExists } from '@blocksuite/global/utils'; -import type { BlobManager } from '../persistence/blob/types.js'; +interface BlobCRUD { + get: (key: string) => Promise | Blob | null; + set: (key: string, value: Blob) => Promise | string; + delete: (key: string) => Promise | void; + list: () => Promise | string[]; +} type AssetsManagerConfig = { - blob: BlobManager; + blob: BlobCRUD; }; export class AssetsManager { private readonly _assetsMap = new Map(); - private readonly _blob: BlobManager; + private readonly _blob: BlobCRUD; constructor(options: AssetsManagerConfig) { this._blob = options.blob; @@ -42,6 +47,6 @@ export class AssetsManager { return; } - await this._blob.set(blob, blobId); + await this._blob.set(blobId, blob); } } diff --git a/packages/framework/store/src/transformer/job.ts b/packages/framework/store/src/transformer/job.ts index 39494745bf52..e6b9d16cacb0 100644 --- a/packages/framework/store/src/transformer/job.ts +++ b/packages/framework/store/src/transformer/job.ts @@ -48,7 +48,7 @@ export class Job { constructor({ collection, middlewares = [] }: JobConfig) { this._collection = collection; - this._assetsManager = new AssetsManager({ blob: collection.blob }); + this._assetsManager = new AssetsManager({ blob: collection.blobSync }); middlewares.forEach(middleware => { middleware({ diff --git a/packages/framework/sync/package.json b/packages/framework/sync/package.json index 4a267536f735..4ce6898530aa 100644 --- a/packages/framework/sync/package.json +++ b/packages/framework/sync/package.json @@ -14,10 +14,10 @@ "license": "MPL-2.0", "dependencies": { "@blocksuite/global": "workspace:*", - "y-protocols": "^1.0.6", - "idb": "^8.0.0" + "idb": "^8.0.0", + "idb-keyval": "^6.2.1", + "y-protocols": "^1.0.6" }, - "devDependencies": {}, "peerDependencies": { "yjs": "^13.6.15" }, diff --git a/packages/framework/sync/src/__tests__/blob.unit.spec.ts b/packages/framework/sync/src/__tests__/blob.unit.spec.ts new file mode 100644 index 000000000000..69cd7266bee3 --- /dev/null +++ b/packages/framework/sync/src/__tests__/blob.unit.spec.ts @@ -0,0 +1,51 @@ +import { NoopLogger } from '@blocksuite/global/utils'; +import { beforeEach, describe, expect, it } from 'vitest'; + +import { BlobEngine } from '../blob/engine.js'; +import { MemoryBlobSource } from '../blob/impl/index.js'; + +describe('BlobEngine with MemoryBlobSource', () => { + let mainSource: MemoryBlobSource; + let shadowSource: MemoryBlobSource; + let engine: BlobEngine; + + beforeEach(() => { + mainSource = new MemoryBlobSource(); + shadowSource = new MemoryBlobSource(); + engine = new BlobEngine(mainSource, [shadowSource], new NoopLogger()); + }); + + it('should set and get blobs', async () => { + const blob = new Blob(['test'], { type: 'text/plain' }); + const key = await engine.set(blob); + const retrievedBlob = await engine.get(key); + expect(retrievedBlob).not.toBeNull(); + expect(await retrievedBlob?.text()).toBe('test'); + }); + + it('should sync blobs between main and shadow sources', async () => { + const blob = new Blob(['test'], { type: 'text/plain' }); + const key = await engine.set(blob); + await engine.sync(); + const retrievedBlob = await shadowSource.get(key); + expect(retrievedBlob).not.toBeNull(); + expect(await retrievedBlob?.text()).toBe('test'); + }); + + it('should list all blobs', async () => { + const blob1 = new Blob(['test1'], { type: 'text/plain' }); + const blob2 = new Blob(['test2'], { type: 'text/plain' }); + await engine.set(blob1); + await engine.set(blob2); + const blobList = await engine.list(); + expect(blobList.length).toBe(2); + }); + + it('should not delete blobs (unsupported feature)', async () => { + const blob = new Blob(['test'], { type: 'text/plain' }); + const key = await engine.set(blob); + await engine.delete(key); + const retrievedBlob = await engine.get(key); + expect(retrievedBlob).not.toBeNull(); + }); +}); diff --git a/packages/framework/sync/src/blob/engine.ts b/packages/framework/sync/src/blob/engine.ts new file mode 100644 index 000000000000..d349fae72359 --- /dev/null +++ b/packages/framework/sync/src/blob/engine.ts @@ -0,0 +1,196 @@ +import { type Logger, sha } from '@blocksuite/global/utils'; + +import type { BlobSource } from './source.js'; + +export interface BlobStatus { + isStorageOverCapacity: boolean; +} + +/** + * # BlobEngine + * + * sync blobs between storages in background. + * + * all operations priority use main, then use shadows. + */ +export class BlobEngine { + private _abort: AbortController | null = null; + + constructor( + readonly main: BlobSource, + readonly shadows: BlobSource[], + readonly logger: Logger + ) {} + + start() { + if (this._abort) { + return; + } + this._abort = new AbortController(); + const abortSignal = this._abort.signal; + + const sync = () => { + if (abortSignal.aborted) { + return; + } + + this.sync() + .catch(error => { + this.logger.error('sync blob error', error); + }) + .finally(() => { + // sync every 1 minute + setTimeout(sync, 60000); + }); + }; + + sync(); + } + + stop() { + this._abort?.abort(); + this._abort = null; + } + + get sources() { + return [this.main, ...this.shadows]; + } + + async sync() { + if (this.main.readonly) { + return; + } + this.logger.debug('start syncing blob...'); + for (const shadow of this.shadows) { + let mainList: string[] = []; + let shadowList: string[] = []; + + if (!shadow.readonly) { + try { + mainList = await this.main.list(); + shadowList = await shadow.list(); + } catch (err) { + this.logger.error(`error when sync`, err); + continue; + } + + const needUpload = mainList.filter(key => !shadowList.includes(key)); + for (const key of needUpload) { + try { + const data = await this.main.get(key); + if (data) { + await shadow.set(key, data); + } else { + this.logger.error( + 'data not found when trying upload from main to shadow' + ); + } + } catch (err) { + this.logger.error( + `error when sync ${key} from [${this.main.name}] to [${shadow.name}]`, + err + ); + } + } + } + + const needDownload = shadowList.filter(key => !mainList.includes(key)); + for (const key of needDownload) { + try { + const data = await shadow.get(key); + if (data) { + await this.main.set(key, data); + } else { + this.logger.error( + 'data not found when trying download from shadow to main' + ); + } + } catch (err) { + this.logger.error( + `error when sync ${key} from [${shadow.name}] to [${this.main.name}]`, + err + ); + } + } + } + + this.logger.debug('finish syncing blob'); + } + + async get(key: string) { + this.logger.debug('get blob', key); + for (const source of this.sources) { + const data = await source.get(key); + if (data) { + return data; + } + } + return null; + } + + async set(value: Blob): Promise; + async set(key: string, value: Blob): Promise; + async set(valueOrKey: string | Blob, _value?: Blob) { + if (this.main.readonly) { + throw new Error('main peer is readonly'); + } + + const key = + typeof valueOrKey === 'string' + ? valueOrKey + : await sha(await valueOrKey.arrayBuffer()); + const value = typeof valueOrKey === 'string' ? _value : valueOrKey; + + if (!value) { + throw new Error('value is empty'); + } + + // await upload to the main peer + await this.main.set(key, value); + + // uploads to other peers in the background + Promise.allSettled( + this.shadows + .filter(r => !r.readonly) + .map(peer => + peer.set(key, value).catch(err => { + this.logger.error('Error when uploading to peer', err); + }) + ) + ) + .then(result => { + if (result.some(({ status }) => status === 'rejected')) { + this.logger.error( + `blob ${key} update finish, but some peers failed to update` + ); + } else { + this.logger.debug(`blob ${key} update finish`); + } + }) + .catch(() => { + // Promise.allSettled never reject + }); + + return key; + } + + // eslint-disable-next-line @typescript-eslint/require-await + async delete(_key: string) { + this.logger.error( + 'You are trying to delete a blob. We do not support this feature yet. We need to wait until we implement the indexer, which will inform us which doc is using a particular blob so that we can safely delete it.' + ); + } + + async list() { + const blobIdSet = new Set(); + + for (const source of this.sources) { + const blobs = await source.list(); + for (const blob of blobs) { + blobIdSet.add(blob); + } + } + + return Array.from(blobIdSet); + } +} diff --git a/packages/framework/sync/src/blob/impl/index.ts b/packages/framework/sync/src/blob/impl/index.ts new file mode 100644 index 000000000000..cdb0807edab8 --- /dev/null +++ b/packages/framework/sync/src/blob/impl/index.ts @@ -0,0 +1,2 @@ +export * from './indexeddb.js'; +export * from './memory.js'; diff --git a/packages/framework/sync/src/blob/impl/indexeddb.ts b/packages/framework/sync/src/blob/impl/indexeddb.ts new file mode 100644 index 000000000000..3c151164da55 --- /dev/null +++ b/packages/framework/sync/src/blob/impl/indexeddb.ts @@ -0,0 +1,38 @@ +import { createStore, del, get, keys, set } from 'idb-keyval'; + +import type { BlobSource } from '../source.js'; + +export class IndexedDBBlobSource implements BlobSource { + readonly = false; + + readonly store = createStore(`${this.name}_blob`, 'blob'); + readonly mimeTypeStore = createStore(`${this.name}_blob_mime`, 'blob_mime'); + + constructor(readonly name: string) {} + + async get(key: string) { + const res = await get(key, this.store); + if (res) { + return new Blob([res], { + type: await get(key, this.mimeTypeStore), + }); + } + return null; + } + + async set(key: string, value: Blob) { + await set(key, value.arrayBuffer(), this.store); + await set(key, value.type, this.mimeTypeStore); + return key; + } + + async delete(key: string) { + await del(key, this.store); + await del(key, this.mimeTypeStore); + } + + async list() { + const list = await keys(this.store); + return list; + } +} diff --git a/packages/framework/sync/src/blob/impl/memory.ts b/packages/framework/sync/src/blob/impl/memory.ts new file mode 100644 index 000000000000..3c21d996b88f --- /dev/null +++ b/packages/framework/sync/src/blob/impl/memory.ts @@ -0,0 +1,23 @@ +import type { BlobSource } from '../source.js'; + +export class MemoryBlobSource implements BlobSource { + name = 'memory'; + readonly = false; + + readonly map: Map = new Map(); + + get(key: string) { + return Promise.resolve(this.map.get(key) ?? null); + } + set(key: string, value: Blob) { + this.map.set(key, value); + return Promise.resolve(key); + } + delete(key: string) { + this.map.delete(key); + return Promise.resolve(); + } + list() { + return Promise.resolve(Array.from(this.map.keys())); + } +} diff --git a/packages/framework/sync/src/blob/index.ts b/packages/framework/sync/src/blob/index.ts new file mode 100644 index 000000000000..63ae7be0f547 --- /dev/null +++ b/packages/framework/sync/src/blob/index.ts @@ -0,0 +1,3 @@ +export * from './engine.js'; +export * from './impl/index.js'; +export * from './source.js'; diff --git a/packages/framework/sync/src/blob/source.ts b/packages/framework/sync/src/blob/source.ts new file mode 100644 index 000000000000..ebb675db582e --- /dev/null +++ b/packages/framework/sync/src/blob/source.ts @@ -0,0 +1,8 @@ +export interface BlobSource { + name: string; + readonly: boolean; + get: (key: string) => Promise; + set: (key: string, value: Blob) => Promise; + delete: (key: string) => Promise; + list: () => Promise; +} diff --git a/packages/framework/sync/src/doc/engine.ts b/packages/framework/sync/src/doc/engine.ts index 4cb0cd315971..c106268afc9e 100644 --- a/packages/framework/sync/src/doc/engine.ts +++ b/packages/framework/sync/src/doc/engine.ts @@ -10,7 +10,7 @@ import { type DocSource } from './source.js'; export interface DocEngineStatus { step: DocEngineStep; main: DocPeerStatus | null; - shadow: (DocPeerStatus | null)[]; + shadows: (DocPeerStatus | null)[]; retrying: boolean; } @@ -41,7 +41,7 @@ export interface DocEngineStatus { * 1. start main sync * 2. wait for main sync complete * 3. start shadow sync - * 4. continuously sync main and shadow + * 4. continuously sync main and shadows */ export class DocEngine { get rootDocId() { @@ -66,13 +66,13 @@ export class DocEngine { constructor( readonly rootDoc: Doc, readonly main: DocSource, - readonly shadow: DocSource[], + readonly shadows: DocSource[], readonly logger: Logger ) { this._status = { step: DocEngineStep.Stopped, main: null, - shadow: shadow.map(() => null), + shadows: shadows.map(() => null), retrying: false, }; this.logger.debug(`syne-engine:${this.rootDocId} status init`, this.status); @@ -121,7 +121,7 @@ export class DocEngine { this.setStatus({ step: DocEngineStep.Stopped, main: null, - shadow: this.shadow.map(() => null), + shadows: this.shadows.map(() => null), retrying: false, }); } @@ -133,7 +133,7 @@ export class DocEngine { shadowPeers: (SyncPeer | null)[]; } = { mainPeer: null, - shadowPeers: this.shadow.map(() => null), + shadowPeers: this.shadows.map(() => null), }; const cleanUp: (() => void)[] = []; @@ -159,7 +159,7 @@ export class DocEngine { await state.mainPeer.waitForLoaded(signal); // Step 3: start shadow sync peer - state.shadowPeers = this.shadow.map(shadow => { + state.shadowPeers = this.shadows.map(shadow => { const peer = new SyncPeer( this.rootDoc, shadow, @@ -205,9 +205,9 @@ export class DocEngine { } } - updateSyncingState(local: SyncPeer | null, shadow: (SyncPeer | null)[]) { + updateSyncingState(local: SyncPeer | null, shadows: (SyncPeer | null)[]) { let step = DocEngineStep.Synced; - const allPeer = [local, ...shadow]; + const allPeer = [local, ...shadows]; for (const peer of allPeer) { if (!peer || peer.status.step !== DocPeerStep.Synced) { step = DocEngineStep.Syncing; @@ -217,7 +217,7 @@ export class DocEngine { this.setStatus({ step, main: local?.status ?? null, - shadow: shadow.map(peer => peer?.status ?? null), + shadows: shadows.map(peer => peer?.status ?? null), retrying: allPeer.some( peer => peer?.status.step === DocPeerStep.Retrying ), @@ -250,7 +250,7 @@ export class DocEngine { async waitForLoadedRootDoc(abort?: AbortSignal) { function isLoadedRootDoc(status: DocEngineStatus) { - return ![status.main, ...status.shadow].some( + return ![status.main, ...status.shadows].some( peer => !peer || peer.step <= DocPeerStep.LoadingRootDoc ); } diff --git a/packages/framework/sync/src/index.ts b/packages/framework/sync/src/index.ts index 45f959267c50..55d643391569 100644 --- a/packages/framework/sync/src/index.ts +++ b/packages/framework/sync/src/index.ts @@ -1,2 +1,3 @@ export * from './awareness/index.js'; +export * from './blob/index.js'; export * from './doc/index.js'; diff --git a/packages/framework/sync/vitest.config.ts b/packages/framework/sync/vitest.config.ts new file mode 100644 index 000000000000..fc5865ff0f21 --- /dev/null +++ b/packages/framework/sync/vitest.config.ts @@ -0,0 +1,23 @@ +import { defineConfig } from 'vitest/config'; + +export default defineConfig({ + test: { + include: ['src/__tests__/**/*.unit.spec.ts'], + testTimeout: 500, + coverage: { + provider: 'istanbul', // or 'c8' + reporter: ['lcov'], + reportsDirectory: '../../.coverage/sync', + }, + /** + * Custom handler for console.log in tests. + * + * Return `false` to ignore the log. + */ + onConsoleLog(log, type) { + console.warn(`Unexpected ${type} log`, log); + throw new Error(log); + }, + restoreMocks: true, + }, +}); diff --git a/packages/playground/apps/_common/sync/blob/mock-server.ts b/packages/playground/apps/_common/sync/blob/mock-server.ts new file mode 100644 index 000000000000..52a3ea887160 --- /dev/null +++ b/packages/playground/apps/_common/sync/blob/mock-server.ts @@ -0,0 +1,55 @@ +import type { BlobSource } from '@blocksuite/sync'; + +/** + * @internal just for test + * + * API: /api/collection/:id/blob/:key + * GET: get blob + * PUT: set blob + * DELETE: delete blob + */ +export class MockServerBlobSource implements BlobSource { + private readonly _cache = new Map(); + + constructor(readonly name: string) {} + + readonly = false; + + async get(key: string) { + if (this._cache.has(key)) { + return this._cache.get(key) as Blob; + } else { + const blob = await fetch(`/api/collection/${this.name}/blob/${key}`, { + method: 'GET', + }).then(response => { + if (!response.ok) { + throw new Error(`Failed to fetch blob ${key}`); + } + return response.blob(); + }); + this._cache.set(key, blob); + return blob; + } + } + + async set(key: string, value: Blob) { + this._cache.set(key, value); + await fetch(`/api/collection/${this.name}/blob/${key}`, { + method: 'PUT', + body: await value.arrayBuffer(), + }); + return key; + } + + async delete(key: string) { + this._cache.delete(key); + await fetch(`/api/collection/${this.name}/blob/${key}`, { + method: 'DELETE', + }); + } + + // eslint-disable-next-line @typescript-eslint/require-await + async list() { + return Array.from(this._cache.keys()); + } +} diff --git a/packages/playground/apps/default/utils/collection.ts b/packages/playground/apps/default/utils/collection.ts index dd31b982103a..a7c5645c1d5c 100644 --- a/packages/playground/apps/default/utils/collection.ts +++ b/packages/playground/apps/default/utils/collection.ts @@ -1,8 +1,6 @@ import { AffineSchemas } from '@blocksuite/blocks'; import { assertExists } from '@blocksuite/global/utils'; import { - type BlobStorage, - createIndexeddbStorage, DocCollection, type DocCollectionOptions, Generator, @@ -14,6 +12,7 @@ import { import { BroadcastChannelAwarenessSource, BroadcastChannelDocSource, + IndexedDBBlobSource, IndexedDBDocSource, } from '@blocksuite/sync'; @@ -23,9 +22,6 @@ import { WebSocketDocSource } from '../../_common/sync/websocket/doc'; const BASE_WEBSOCKET_URL = new URL(import.meta.env.PLAYGROUND_WS); export async function createDefaultDocCollection() { - const blobStorages: ((id: string) => BlobStorage)[] = [ - createIndexeddbStorage, - ]; const idGenerator: Generator = Generator.NanoID; const schema = new Schema(); schema.register(AffineSchemas); @@ -45,14 +41,14 @@ export async function createDefaultDocCollection() { .then(() => { docSources = { main: new IndexedDBDocSource(), - shadow: [new WebSocketDocSource(ws)], + shadows: [new WebSocketDocSource(ws)], }; awarenessSources = [new WebSocketAwarenessSource(ws)]; }) .catch(() => { docSources = { main: new IndexedDBDocSource(), - shadow: [new BroadcastChannelDocSource()], + shadows: [new BroadcastChannelDocSource()], }; awarenessSources = [ new BroadcastChannelAwarenessSource('quickEdgeless'), @@ -64,7 +60,9 @@ export async function createDefaultDocCollection() { id: 'quickEdgeless', schema, idGenerator, - blobStorages, + blobSources: { + main: new IndexedDBBlobSource('quickEdgeless'), + }, docSources, awarenessSources, defaultFlags: { diff --git a/packages/playground/apps/starter/utils/collection.ts b/packages/playground/apps/starter/utils/collection.ts index c0a2a8213731..1be189f71443 100644 --- a/packages/playground/apps/starter/utils/collection.ts +++ b/packages/playground/apps/starter/utils/collection.ts @@ -2,10 +2,6 @@ import { AffineSchemas, TestUtils } from '@blocksuite/blocks'; import { assertExists } from '@blocksuite/global/utils'; import type { BlockCollection } from '@blocksuite/store'; import { - type BlobStorage, - createIndexeddbStorage, - createMemoryStorage, - createSimpleServerStorage, DocCollection, type DocCollectionOptions, Generator, @@ -14,29 +10,23 @@ import { type StoreOptions, } from '@blocksuite/store'; import { + type BlobSource, BroadcastChannelAwarenessSource, BroadcastChannelDocSource, + IndexedDBBlobSource, + MemoryBlobSource, } from '@blocksuite/sync'; +import { MockServerBlobSource } from '../../_common/sync/blob/mock-server.js'; import type { InitFn } from '../data/utils.js'; const params = new URLSearchParams(location.search); const room = params.get('room'); -const blobStorageArgs = (params.get('blobStorage') ?? 'memory').split(','); const isE2E = room?.startsWith('playwright'); +const blobSourceArgs = (params.get('blobSource') ?? '').split(','); export function createStarterDocCollection() { - const blobStorages: ((id: string) => BlobStorage)[] = []; - if (blobStorageArgs.includes('memory')) { - blobStorages.push(createMemoryStorage); - } - if (blobStorageArgs.includes('idb')) { - blobStorages.push(createIndexeddbStorage); - } - if (blobStorageArgs.includes('mock')) { - blobStorages.push(createSimpleServerStorage); - } - + const collectionId = room ?? 'starter'; const schema = new Schema(); schema.register(AffineSchemas); const idGenerator = isE2E ? Generator.AutoIncrement : Generator.NanoID; @@ -49,11 +39,21 @@ export function createStarterDocCollection() { } const id = room ?? `starter-${Math.random().toString(16).slice(2, 8)}`; + const blobSources = { + main: new MemoryBlobSource(), + shadows: [] as BlobSource[], + } satisfies StoreOptions['blobSources']; + if (blobSourceArgs.includes('mock')) { + blobSources.shadows.push(new MockServerBlobSource(collectionId)); + } + if (blobSourceArgs.includes('idb')) { + blobSources.shadows.push(new IndexedDBBlobSource(collectionId)); + } + const options: DocCollectionOptions = { - id, + id: collectionId, schema, idGenerator, - blobStorages, defaultFlags: { enable_synced_doc_block: true, enable_pie_menu: true, @@ -62,6 +62,7 @@ export function createStarterDocCollection() { }, awarenessSources: [new BroadcastChannelAwarenessSource(id)], docSources, + blobSources, }; const collection = new DocCollection(options); diff --git a/packages/presets/src/__tests__/utils/setup.ts b/packages/presets/src/__tests__/utils/setup.ts index b4479aa6a146..62a37a9ca1f6 100644 --- a/packages/presets/src/__tests__/utils/setup.ts +++ b/packages/presets/src/__tests__/utils/setup.ts @@ -1,13 +1,12 @@ import { AffineSchemas } from '@blocksuite/blocks/schemas'; import { assertExists } from '@blocksuite/global/utils'; import type { BlockCollection } from '@blocksuite/store'; -import { type BlobStorage, DocCollection, Text } from '@blocksuite/store'; -import { createMemoryStorage, Generator, Schema } from '@blocksuite/store'; +import { DocCollection, Text } from '@blocksuite/store'; +import { Generator, Schema } from '@blocksuite/store'; import { AffineEditorContainer } from '../../index.js'; function createCollectionOptions() { - const blobStorages: ((id: string) => BlobStorage)[] = []; const schema = new Schema(); const room = Math.random().toString(16).slice(2, 8); @@ -15,13 +14,10 @@ function createCollectionOptions() { const idGenerator: Generator = Generator.AutoIncrement; // works only in single user mode - blobStorages.push(createMemoryStorage); - return { id: room, schema, idGenerator, - blobStorages, defaultFlags: { enable_synced_doc_block: true, enable_pie_menu: true, diff --git a/packages/presets/src/ai/entries/edgeless/actions-config.ts b/packages/presets/src/ai/entries/edgeless/actions-config.ts index 2ba8f667ac9b..8e34d7d6d6c3 100644 --- a/packages/presets/src/ai/entries/edgeless/actions-config.ts +++ b/packages/presets/src/ai/entries/edgeless/actions-config.ts @@ -77,7 +77,7 @@ const imageCustomInput = async (host: EditorHost) => { if (!(imageBlock instanceof ImageBlockModel)) return; if (!imageBlock.sourceId) return; - const blob = await host.doc.blob.get(imageBlock.sourceId); + const blob = await host.doc.blobSync.get(imageBlock.sourceId); if (!blob) return; return { diff --git a/packages/presets/src/ai/utils/selection-utils.ts b/packages/presets/src/ai/utils/selection-utils.ts index 7e98d93de01f..d0b264fc04e1 100644 --- a/packages/presets/src/ai/utils/selection-utils.ts +++ b/packages/presets/src/ai/utils/selection-utils.ts @@ -187,7 +187,7 @@ export const getSelectedImagesAsBlobs = async (host: EditorHost) => { const sourceId = (host.doc.getBlock(s.blockId)?.model as ImageBlockModel) ?.sourceId; if (!sourceId) return null; - const blob = await (sourceId ? host.doc.blob.get(sourceId) : null); + const blob = await (sourceId ? host.doc.blobSync.get(sourceId) : null); if (!blob) return null; return new File([blob], sourceId); }) ?? [] diff --git a/packages/presets/src/fragments/copilot-panel/edgeless/logic.ts b/packages/presets/src/fragments/copilot-panel/edgeless/logic.ts index 0ef1c694a338..4864c4aa8aed 100644 --- a/packages/presets/src/fragments/copilot-panel/edgeless/logic.ts +++ b/packages/presets/src/fragments/copilot-panel/edgeless/logic.ts @@ -215,7 +215,7 @@ export class AIEdgelessLogic { const surface = getSurfaceElementFromEditor(this.host); let image = getFirstImageInFrame(model, this.host); const imgFile = jpegBase64ToFile(b64, 'img'); - const sourceId = await this.host.doc.collection.blob.set(imgFile); + const sourceId = await this.host.doc.blobSync.set(imgFile); if (!image) { image = surface.edgeless.service.addBlock( 'affine:image', diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index d01de475dfc7..fbdcec8dee9d 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -352,6 +352,9 @@ importers: packages/framework/global: dependencies: + lib0: + specifier: ^0.2.93 + version: 0.2.93 zod: specifier: ^3.23.8 version: 3.23.8 @@ -426,6 +429,9 @@ importers: idb: specifier: ^8.0.0 version: 8.0.0 + idb-keyval: + specifier: ^6.2.1 + version: 6.2.1 y-protocols: specifier: ^1.0.6 version: 1.0.6(yjs@13.6.15) diff --git a/tests/basic.spec.ts b/tests/basic.spec.ts index 063e33bf01e5..840652950c61 100644 --- a/tests/basic.spec.ts +++ b/tests/basic.spec.ts @@ -103,7 +103,6 @@ test(scoped`basic multi user state`, async ({ context, page: pageA }) => { await enterPlaygroundRoom(pageB, { flags: {}, room, - blobStorage: undefined, noInit: true, }); await waitDefaultPageLoaded(pageB); @@ -127,7 +126,6 @@ test( await enterPlaygroundRoom(pageB, { flags: {}, room, - blobStorage: undefined, noInit: true, }); @@ -153,7 +151,6 @@ test(scoped`A first open, B first edit`, async ({ context, page: pageA }) => { await enterPlaygroundRoom(pageB, { flags: {}, room, - blobStorage: undefined, noInit: true, }); await focusRichText(pageB); diff --git a/tests/image.spec.ts b/tests/image.spec.ts index 63461cdfc029..9a1fb37fb59a 100644 --- a/tests/image.spec.ts +++ b/tests/image.spec.ts @@ -269,7 +269,11 @@ async function initMockImage(page: Page) { } test('image loading but failed', async ({ page }) => { - expectConsoleMessage(page, 'Error: Failed to fetch blob _e2e_test_image_id_'); + expectConsoleMessage( + page, + 'Error: Failed to fetch blob _e2e_test_image_id_', + 'warning' + ); expectConsoleMessage( page, 'Failed to load resource: the server responded with a status of 404 (Not Found)' @@ -280,7 +284,7 @@ test('image loading but failed', async ({ page }) => { 'warning' ); - const room = await enterPlaygroundRoom(page, { blobStorage: ['mock'] }); + const room = await enterPlaygroundRoom(page, { blobSource: ['mock'] }); const timeout = 2000; // block image data request, force wait 100ms for loading test, @@ -311,7 +315,11 @@ test('image loading but failed', async ({ page }) => { }); test('image loading but success', async ({ page }) => { - expectConsoleMessage(page, 'Error: Failed to fetch blob _e2e_test_image_id_'); + expectConsoleMessage( + page, + 'Error: Failed to fetch blob _e2e_test_image_id_', + 'warning' + ); expectConsoleMessage( page, 'Failed to load resource: the server responded with a status of 404 (Not Found)' @@ -322,7 +330,7 @@ test('image loading but success', async ({ page }) => { 'warning' ); - const room = await enterPlaygroundRoom(page, { blobStorage: ['mock'] }); + const room = await enterPlaygroundRoom(page, { blobSource: ['mock'] }); const imageBuffer = await readFile( fileURLToPath(new URL('./fixtures/smile.png', import.meta.url)) ); @@ -366,7 +374,7 @@ test('image loading but success', async ({ page }) => { }); test('image loaded successfully', async ({ page }) => { - const room = await enterPlaygroundRoom(page, { blobStorage: ['mock'] }); + const room = await enterPlaygroundRoom(page, { blobSource: ['mock'] }); const imageBuffer = await readFile( fileURLToPath(new URL('./fixtures/smile.png', import.meta.url)) ); diff --git a/tests/utils/actions/misc.ts b/tests/utils/actions/misc.ts index 5eba1d5f3b68..91e4d0e99a31 100644 --- a/tests/utils/actions/misc.ts +++ b/tests/utils/actions/misc.ts @@ -264,18 +264,18 @@ export async function enterPlaygroundRoom( ops?: { flags?: Partial; room?: string; - blobStorage?: ('memory' | 'idb' | 'mock')[]; + blobSource?: ('idb' | 'mock')[]; noInit?: boolean; } ) { const url = new URL(DEFAULT_PLAYGROUND); let room = ops?.room; - const blobStorage = ops?.blobStorage; + const blobSource = ops?.blobSource; if (!room) { room = generateRandomRoomId(); } url.searchParams.set('room', room); - url.searchParams.set('blobStorage', blobStorage?.join(',') || 'idb'); + url.searchParams.set('blobSource', blobSource?.join(',') || 'idb'); await page.goto(url.toString()); // const readyPromise = waitForPageReady(page); @@ -1299,7 +1299,7 @@ export async function initImageState(page: Page) { const imageBlob = await fetch(`${location.origin}/test-card-1.png`).then( response => response.blob() ); - const storage = pageRoot.doc.blob; + const storage = pageRoot.doc.blobSync; const sourceId = await storage.set(imageBlob); const imageId = doc.addBlock( 'affine:image', diff --git a/vitest.workspace.js b/vitest.workspace.js index 80dec71bb5a1..dee8351e893d 100644 --- a/vitest.workspace.js +++ b/vitest.workspace.js @@ -6,5 +6,6 @@ export default defineWorkspace([ './packages/framework/global/vitest.config.ts', './packages/framework/inline/vitest.config.ts', './packages/framework/store/vitest.config.ts', + './packages/framework/sync/vitest.config.ts', './packages/presets/vitest.config.ts', ]);