feat(sync): add blob engine

packages/blocks/src/__tests__/adapters/html.unit.spec.ts

@@ -1,5 +1,5 @@
 import type { BlockSnapshot } from '@blocksuite/store';
-import { MemoryBlobManager } from '@blocksuite/store';
+import { MemoryBlobCRUD } from '@blocksuite/store';
 import { AssetsManager } from '@blocksuite/store';
 import { describe, expect, test } from 'vitest';
 
@@ -814,10 +814,10 @@ describe('snapshot to html', () => {
     );
 
     const htmlAdapter = new HtmlAdapter();
-    const blobManager = new MemoryBlobManager();
+    const blobManager = new MemoryBlobCRUD();
     await blobManager.set(
-      new Blob(),
-      'YXXTjRmLlNyiOUnHb8nAIvUP6V7PAXhwW9F5_tc2LGs='
+      'YXXTjRmLlNyiOUnHb8nAIvUP6V7PAXhwW9F5_tc2LGs=',
+      new Blob()
     );
     const assets = new AssetsManager({ blob: blobManager });
 

packages/blocks/src/__tests__/adapters/markdown.unit.spec.ts

@@ -1,5 +1,5 @@
 import type { BlockSnapshot } from '@blocksuite/store';
-import { MemoryBlobManager } from '@blocksuite/store';
+import { MemoryBlobCRUD } from '@blocksuite/store';
 import { AssetsManager } from '@blocksuite/store';
 import { describe, expect, test } from 'vitest';
 
@@ -1141,12 +1141,12 @@ hhh
       '![](assets/YXXTjRmLlNyiOUnHb8nAIvUP6V7PAXhwW9F5_tc2LGs=.blob "aaa")\n\n';
 
     const mdAdapter = new MarkdownAdapter();
-    const blobManager = new MemoryBlobManager();
-    await blobManager.set(
-      new Blob(),
-      'YXXTjRmLlNyiOUnHb8nAIvUP6V7PAXhwW9F5_tc2LGs='
+    const blobCRUD = new MemoryBlobCRUD();
+    await blobCRUD.set(
+      'YXXTjRmLlNyiOUnHb8nAIvUP6V7PAXhwW9F5_tc2LGs=',
+      new Blob()
     );
-    const assets = new AssetsManager({ blob: blobManager });
+    const assets = new AssetsManager({ blob: blobCRUD });
 
     const target = await mdAdapter.fromBlockSnapshot({
       snapshot: blockSnapshot,

packages/blocks/src/__tests__/adapters/notion-html.unit.spec.ts

@@ -1,7 +1,7 @@
 import {
   AssetsManager,
   type BlockSnapshot,
-  MemoryBlobManager,
+  MemoryBlobCRUD,
 } from '@blocksuite/store';
 import { describe, expect, test } from 'vitest';
 
@@ -1084,7 +1084,7 @@ describe('notion html to snapshot', () => {
     const adapter = new NotionHtmlAdapter();
     const rawBlockSnapshot = await adapter.toBlockSnapshot({
       file: html,
-      assets: new AssetsManager({ blob: new MemoryBlobManager() }),
+      assets: new AssetsManager({ blob: new MemoryBlobCRUD() }),
     });
     expect(nanoidReplacement(rawBlockSnapshot)).toEqual(blockSnapshot);
   });
@@ -1176,9 +1176,9 @@ describe('notion html to snapshot', () => {
     };
 
     const adapter = new NotionHtmlAdapter();
-    const blobManager = new MemoryBlobManager();
-    const key = await blobManager.set(new File([], 'README.pdf'));
-    const assestsManager = new AssetsManager({ blob: blobManager });
+    const blobCRUD = new MemoryBlobCRUD();
+    const key = await blobCRUD.set(new File([], 'README.pdf'));
+    const assestsManager = new AssetsManager({ blob: blobCRUD });
     await assestsManager.readFromBlob(key);
     const rawBlockSnapshot = await adapter.toBlockSnapshot({
       file: html,

packages/blocks/src/_common/adapters/attachment.ts

@@ -1,3 +1,4 @@
+import { sha } from '@blocksuite/global/utils';
 import type { AssetsManager } from '@blocksuite/store';
 import {
   BaseAdapter,
@@ -10,7 +11,6 @@ import {
   type FromSliceSnapshotPayload,
   type FromSliceSnapshotResult,
   nanoid,
-  sha,
   type SliceSnapshot,
   type ToBlockSnapshotPayload,
   type ToDocSnapshotPayload,

packages/blocks/src/_common/adapters/html.ts

@@ -1,4 +1,4 @@
-import { assertEquals } from '@blocksuite/global/utils';
+import { assertEquals, sha } from '@blocksuite/global/utils';
 import type { DeltaInsert } from '@blocksuite/inline';
 import type {
   FromBlockSnapshotPayload,
@@ -15,7 +15,6 @@ import {
   BlockSnapshotSchema,
   getAssetName,
   nanoid,
-  sha,
 } from '@blocksuite/store';
 import { ASTWalker, BaseAdapter } from '@blocksuite/store';
 import {

packages/blocks/src/_common/adapters/image.ts

@@ -1,3 +1,4 @@
+import { sha } from '@blocksuite/global/utils';
 import type { AssetsManager } from '@blocksuite/store';
 import {
   BaseAdapter,
@@ -10,7 +11,6 @@ import {
   type FromSliceSnapshotPayload,
   type FromSliceSnapshotResult,
   nanoid,
-  sha,
   type SliceSnapshot,
   type ToBlockSnapshotPayload,
   type ToDocSnapshotPayload,

packages/blocks/src/_common/adapters/markdown.ts

@@ -1,3 +1,4 @@
+import { sha } from '@blocksuite/global/utils';
 import type { DeltaInsert } from '@blocksuite/inline/types';
 import type {
   FromBlockSnapshotPayload,
@@ -18,7 +19,6 @@ import {
 } from '@blocksuite/store';
 import { nanoid } from '@blocksuite/store';
 import { ASTWalker, BaseAdapter } from '@blocksuite/store';
-import { sha } from '@blocksuite/store';
 import { format } from 'date-fns/format';
 import type { Heading, Root, RootContentMap, TableRow } from 'mdast';
 import remarkParse from 'remark-parse';

packages/blocks/src/_common/adapters/notion-html.ts

@@ -1,4 +1,4 @@
-import { isEqual } from '@blocksuite/global/utils';
+import { isEqual, sha } from '@blocksuite/global/utils';
 import type { DeltaInsert } from '@blocksuite/inline';
 import type {
   FromBlockSnapshotPayload,
@@ -16,7 +16,6 @@ import {
   type DocSnapshot,
   getAssetName,
   nanoid,
-  sha,
   type SliceSnapshot,
 } from '@blocksuite/store';
 import rehypeParse from 'rehype-parse';

packages/blocks/src/_common/export-manager/export-manager.ts

@@ -242,7 +242,7 @@ export class ExportManager {
       if (matchFlavours(block, ['affine:image'])) {
         if (!block.sourceId) return;
 
-        const blob = await block.doc.blob.get(block.sourceId);
+        const blob = await block.doc.blobSync.get(block.sourceId);
         if (!blob) return;
 
         const blobToImage = (blob: Blob) =>

packages/blocks/src/_common/transformers/zip.ts

@@ -1,12 +1,12 @@
-import { assertExists } from '@blocksuite/global/utils';
+import { assertExists, sha } from '@blocksuite/global/utils';
 import type {
   CollectionInfoSnapshot,
   Doc,
   DocCollection,
   DocSnapshot,
   JobMiddleware,
 } from '@blocksuite/store';
-import { extMimeMap, getAssetName, Job, sha } from '@blocksuite/store';
+import { extMimeMap, getAssetName, Job } from '@blocksuite/store';
 import JSZip from 'jszip';
 
 import { replaceIdMiddleware } from './middlewares.js';

packages/blocks/src/_common/utils/filesys.ts

@@ -1,5 +1,3 @@
-import type { BlobManager } from '@blocksuite/store';
-
 // Polyfill for `showOpenFilePicker` API
 // See https://github.com/DefinitelyTyped/DefinitelyTyped/blob/master/types/wicg-file-system-access/index.d.ts
 // See also https://caniuse.com/?search=showOpenFilePicker
@@ -219,29 +217,6 @@ export async function getImageFilesFromLocal() {
   return imageFiles;
 }
 
-export async function uploadImageFromLocal(storage: BlobManager) {
-  const imageFiles = await openFileOrFiles({
-    acceptType: 'Images',
-    multiple: true,
-  });
-  if (!imageFiles) return [];
-  return loadImages(imageFiles, storage);
-}
-
-export async function loadImages(images: File[], storage: BlobManager) {
-  const res: { file: File; sourceId: string }[] = [];
-  for (let i = 0; i < images.length; i++) {
-    const file = images[i];
-    const sourceId = await storage.set(file);
-    res.push({ file, sourceId });
-  }
-  const { saveAttachmentData } = withTempBlobData();
-  res.forEach(({ file, sourceId }) => {
-    saveAttachmentData(sourceId, { name: file.name });
-  });
-  return res;
-}
-
 export function downloadBlob(blob: Blob, name: string) {
   const dataURL = URL.createObjectURL(blob);
   const tmpLink = document.createElement('a');

packages/blocks/src/_common/utils/render-linked-doc.ts

@@ -305,7 +305,7 @@ async function renderImageAbstract(
   const sourceId = (image as ImageBlockModel).sourceId;
   if (!sourceId) return;
 
-  const storage = card.linkedDoc?.blob;
+  const storage = card.linkedDoc?.blobSync;
   if (!storage) return;
 
   const blob = await storage.get(sourceId);

packages/blocks/src/attachment-block/utils.ts

@@ -52,7 +52,7 @@ async function uploadAttachmentBlob(
 
   try {
     setAttachmentUploading(blockId);
-    sourceId = await doc.blob.set(blob);
+    sourceId = await doc.blobSync.set(blob);
   } catch (error) {
     console.error(error);
     if (error instanceof Error) {
@@ -84,7 +84,7 @@ async function getAttachmentBlob(model: AttachmentBlockModel) {
   }
 
   const doc = model.doc;
-  let blob = await doc.blob.get(sourceId);
+  let blob = await doc.blobSync.get(sourceId);
 
   if (blob) {
     blob = new Blob([blob], { type: model.type });

packages/blocks/src/image-block/utils.ts

@@ -39,7 +39,7 @@ export async function uploadBlobForImage(
 
   try {
     setImageUploaded(blockId);
-    sourceId = await doc.blob.set(blob);
+    sourceId = await doc.blobSync.set(blob);
   } catch (error) {
     console.error(error);
     if (error instanceof Error) {
@@ -69,7 +69,7 @@ async function getImageBlob(model: ImageBlockModel) {
   }
 
   const doc = model.doc;
-  const blob = await doc.blob.get(sourceId);
+  const blob = await doc.blobSync.get(sourceId);
 
   if (!blob) {
     return null;
@@ -123,7 +123,7 @@ export async function fetchImageBlob(block: ImageBlockComponent) {
       throw new Error('Image sourceId is missing!');
     }
 
-    const blob = await doc.blob.get(sourceId);
+    const blob = await doc.blobSync.get(sourceId);
     if (!blob) {
       throw new Error('Image blob is missing!');
     }

packages/blocks/src/root-block/edgeless/edgeless-root-block.ts

@@ -416,7 +416,7 @@ export class EdgelessRootBlockComponent extends BlockElement<
     const uploadPromises = imageFiles.map(async (file, index) => {
       const { point, blockId } = dropInfos[index];
 
-      const sourceId = await this.doc.blob.set(file);
+      const sourceId = await this.doc.blobSync.set(file);
       const imageSize = await readImageSize(file);
 
       const center = Vec.toVec(point);
@@ -501,7 +501,7 @@ export class EdgelessRootBlockComponent extends BlockElement<
       let sourceId: string | undefined;
       try {
         setAttachmentUploading(blockId);
-        sourceId = await this.doc.blob.set(file);
+        sourceId = await this.doc.blobSync.set(file);
       } catch (error) {
         console.error(error);
         if (error instanceof Error) {

packages/blocks/src/root-block/widgets/linked-doc/import-doc/import-doc.ts

@@ -1,7 +1,8 @@
 import '../../../../_common/components/loader.js';
 
 import { WithDisposable } from '@blocksuite/block-std';
-import { type DocCollection, extMimeMap, sha } from '@blocksuite/store';
+import { sha } from '@blocksuite/global/utils';
+import { type DocCollection, extMimeMap } from '@blocksuite/store';
 import { Job } from '@blocksuite/store';
 import JSZip from 'jszip';
 import { html, LitElement, type PropertyValues } from 'lit';

packages/blocks/src/surface-block/mini-mindmap/mindmap-preview.ts

@@ -127,7 +127,6 @@ export class MiniMindmapPreview extends WithDisposable(LitElement) {
       id: 'MINI_MINDMAP_TEMPORARY',
       schema,
       idGenerator: Generator.NanoID,
-      blobStorages: [],
       awarenessSources: [],
     };
 

packages/framework/global/package.json

@@ -49,6 +49,7 @@
     "!dist/__tests__"
   ],
   "dependencies": {
+    "lib0": "^0.2.93",
     "zod": "^3.22.4"
   }
 }

packages/framework/global/src/utils/index.ts

@@ -1,4 +1,5 @@
 export * from './assert.js';
+export * from './crypto.js';
 export * from './disposable.js';
 export * from './function.js';
 export * from './logger.js';

packages/framework/store/src/__tests__/transformer.unit.spec.ts

@@ -1,7 +1,7 @@
 import { expect, test } from 'vitest';
 import * as Y from 'yjs';
 
-import { MemoryBlobManager } from '../adapter/index.js';
+import { MemoryBlobCRUD } from '../adapter/index.js';
 import { Text } from '../reactive/index.js';
 import {
   type BlockModel,
@@ -51,8 +51,8 @@ function createTestOptions() {
 }
 
 const transformer = new BaseBlockTransformer();
-const blobManager = new MemoryBlobManager();
-const assets = new AssetsManager({ blob: blobManager });
+const blobCRUD = new MemoryBlobCRUD();
+const assets = new AssetsManager({ blob: blobCRUD });
 
 test('model to snapshot', () => {
   const options = createTestOptions();

packages/framework/store/src/adapter/assets.ts

@@ -1,19 +1,30 @@
-import { assertExists } from '@blocksuite/global/utils';
+import { assertExists, sha } from '@blocksuite/global/utils';
 
-import { sha } from '../persistence/blob/utils.js';
-
-export class MemoryBlobManager {
+/**
+ * @internal just for test
+ */
+export class MemoryBlobCRUD {
   private readonly _map = new Map<string, Blob>();
-  private readonly _blobsRef = new Map<string, number>();
 
   get(key: string) {
     return this._map.get(key) ?? null;
   }
 
-  async set(value: Blob, key?: string) {
-    const _key = key || (await sha(await value.arrayBuffer()));
-    this._map.set(_key, value);
-    return _key;
+  async set(value: Blob): Promise<string>;
+  async set(key: string, value: Blob): Promise<string>;
+  async set(valueOrKey: string | Blob, _value?: Blob) {
+    const key =
+      typeof valueOrKey === 'string'
+        ? valueOrKey
+        : await sha(await valueOrKey.arrayBuffer());
+    const value = typeof valueOrKey === 'string' ? _value : valueOrKey;
+
+    if (!value) {
+      throw new Error('value is required');
+    }
+
+    this._map.set(key, value);
+    return key;
   }
 
   delete(key: string) {
@@ -23,27 +34,6 @@ export class MemoryBlobManager {
   list() {
     return Array.from(this._map.keys());
   }
-
-  gc() {
-    const blobs = this.list();
-    blobs.forEach(blobId => {
-      const ref = this._blobsRef.get(blobId);
-      if (!ref || ref <= 0) {
-        this.delete(blobId);
-        this._blobsRef.delete(blobId);
-      }
-    });
-  }
-
-  increaseRef(blobId: string) {
-    const ref = this._blobsRef.get(blobId) ?? 0;
-    this._blobsRef.set(blobId, ref + 1);
-  }
-
-  decreaseRef(blobId: string) {
-    const ref = this._blobsRef.get(blobId) ?? 0;
-    this._blobsRef.set(blobId, Math.max(ref - 1, 0));
-  }
 }
 
 export const mimeExtMap = new Map([