diff --git a/packages/core/package.json b/packages/core/package.json index fe33f8e784..27f22ccc4b 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -61,6 +61,7 @@ "build": "tsc && vite build" }, "peerDependencies": { + "@univerjs/protocol": "^0.1.0", "@wendellhu/redi": "^0.13.0", "rxjs": ">=7.0.0" }, @@ -71,6 +72,7 @@ }, "devDependencies": { "@types/numeral": "^2.0.5", + "@univerjs/protocol": "^0.1.0", "@univerjs/shared": "workspace:*", "@wendellhu/redi": "^0.13.0", "rxjs": "^7.8.1", diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 64a9af6791..7348ae4549 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -133,5 +133,16 @@ export * from './types/interfaces'; export { UniverInstanceService } from './services/instance/instance.service'; export { LifecycleInitializerService } from './services/lifecycle/lifecycle.service'; export { ConfigService } from './services/config/config.service'; +export { ISnapshotServerService } from './services/snapshot/snapshot-server.service'; +export { + transformSnapshotToWorkbookData, + transformWorkbookDataToSnapshot, + transformDocumentDataToSnapshot, + transformSnapshotToDocumentData, + generateTemporarySnap, +} from './services/snapshot/snapshot-transform'; +export { textEncoder } from './services/snapshot/snapshot-utils'; +export { type ILogContext } from './services/log/context'; +export { b64DecodeUnicode, b64EncodeUnicode } from './shared/coder'; installShims(); diff --git a/packages/core/src/services/log/context.ts b/packages/core/src/services/log/context.ts new file mode 100644 index 0000000000..6c5e7b12c8 --- /dev/null +++ b/packages/core/src/services/log/context.ts @@ -0,0 +1,19 @@ +/** + * Copyright 2023-present DreamNum Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export interface ILogContext { + metadata?: Record; +} diff --git a/packages/core/src/services/snapshot/__tests__/snapshot-mock.ts b/packages/core/src/services/snapshot/__tests__/snapshot-mock.ts new file mode 100644 index 0000000000..7891848f9e --- /dev/null +++ b/packages/core/src/services/snapshot/__tests__/snapshot-mock.ts @@ -0,0 +1,219 @@ +/** + * Copyright 2023-present DreamNum Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import type { IFetchMissingChangesetsRequest, IFetchMissingChangesetsResponse, IGetResourcesRequest, IGetResourcesResponse, IGetSheetBlockRequest, IGetSheetBlockResponse, IGetUnitOnRevRequest, IGetUnitOnRevResponse, ISaveChangesetRequest, ISaveChangesetResponse, ISaveSheetBlockRequest, ISaveSheetBlockResponse, ISaveSnapshotRequest, ISaveSnapshotResponse, ISheetBlock, ISnapshot } from '@univerjs/protocol'; +import { ErrorCode } from '@univerjs/protocol'; + +import { textEncoder } from '../snapshot-utils'; +import type { IWorkbookData } from '../../../types/interfaces/i-workbook-data'; +import { LocaleType } from '../../../types/enum/locale-type'; +import type { ILogContext } from '../../log/context'; +import type { ISnapshotServerService } from '../snapshot-server.service'; +import { b64DecodeUnicode } from '../../../shared/coder'; + +export const testSnapshot = (): ISnapshot => ({ + unitID: '100', + type: 2, + rev: 3, + workbook: { + unitID: '100', + rev: 3, + creator: '', + name: 'New Sheet By Univer', + sheetOrder: [ + 'sheet-1', + ], + sheets: { + 'sheet-1': { + type: 0, + id: 'sheet-1', + name: 'Sheet 1', + rowCount: 1000, + columnCount: 20, + originalMeta: textEncoder.encode(b64DecodeUnicode('eyJmcmVlemUiOiB7InhTcGxpdCI6IDAsICJ5U3BsaXQiOiAwLCAic3RhcnRSb3ciOiAtMSwgInN0YXJ0Q29sdW1uIjogLTF9LCAiaGlkZGVuIjogMCwgInJvd0RhdGEiOiB7IjAiOiB7ImgiOiAyNywgImFoIjogMjcsICJoZCI6IDB9fSwgInRhYkNvbG9yIjogInJlZCIsICJtZXJnZURhdGEiOiBbXSwgInJvd0hlYWRlciI6IHsid2lkdGgiOiA0NiwgImhpZGRlbiI6IDB9LCAic2Nyb2xsVG9wIjogMjAwLCAiem9vbVJhdGlvIjogMSwgImNvbHVtbkRhdGEiOiB7fSwgInNjcm9sbExlZnQiOiAxMDAsICJzZWxlY3Rpb25zIjogWyJBMSJdLCAicmlnaHRUb0xlZnQiOiAwLCAiY29sdW1uSGVhZGVyIjogeyJoZWlnaHQiOiAyMCwgImhpZGRlbiI6IDB9LCAic2hvd0dyaWRsaW5lcyI6IDEsICJkZWZhdWx0Um93SGVpZ2h0IjogMjcsICJkZWZhdWx0Q29sdW1uV2lkdGgiOiA5M30=')), + }, + }, + resources: [], + blockMeta: { + 'sheet-1': { + sheetID: 'sheet-1', + blocks: [ + '100100', + ], + }, + }, + originalMeta: textEncoder.encode(b64DecodeUnicode('eyJsb2NhbGUiOiAiZW5VUyIsICJzdHlsZXMiOiB7fSwgInJlc291cmNlcyI6IFtdLCAiYXBwVmVyc2lvbiI6ICIzLjAuMC1hbHBoYSJ9')), + }, + doc: undefined, +}); + +export const testSheetBlocks = (): ISheetBlock[] => [ + { + id: '100100', + startRow: 0, + endRow: 0, + data: textEncoder.encode(b64DecodeUnicode('eyIwIjogeyIwIjogeyJ0IjogMiwgInYiOiAxfSwgIjEiOiB7InQiOiAyLCAidiI6IDJ9fX0=')), + }, +]; + +export const testWorkbookData = (): IWorkbookData => ({ + id: '100', + sheetOrder: [ + 'sheet-1', + ], + name: 'New Sheet By Univer', + appVersion: '3.0.0-alpha', + locale: LocaleType.EN_US, + styles: {}, + sheets: { + 'sheet-1': { + id: 'sheet-1', + name: 'Sheet 1', + rowCount: 1000, + columnCount: 20, + freeze: { + xSplit: 0, + ySplit: 0, + startRow: -1, + startColumn: -1, + }, + hidden: 0, + rowData: { + 0: { + h: 27, + ah: 27, + hd: 0, + }, + }, + tabColor: 'red', + mergeData: [], + rowHeader: { + width: 46, + hidden: 0, + }, + scrollTop: 200, + zoomRatio: 1, + columnData: {}, + scrollLeft: 100, + selections: [ + 'A1', + ], + rightToLeft: 0, + columnHeader: { + height: 20, + hidden: 0, + }, + showGridlines: 1, + defaultRowHeight: 27, + defaultColumnWidth: 93, + cellData: { + 0: { + 0: { + t: 2, + v: 1, + }, + 1: { + t: 2, + v: 2, + }, + }, + }, + }, + }, + resources: [], + rev: 3, +}); + +export class MockSnapshotServerService implements ISnapshotServerService { + /** Load snapshot from a database. */ + getUnitOnRev(context: ILogContext, params: IGetUnitOnRevRequest): Promise { + return Promise.resolve({ + snapshot: testSnapshot(), + changesets: [], + error: { + code: ErrorCode.OK, + message: '', + }, + }); + } + + /** Load sheet block from a database. */ + getSheetBlock(context: ILogContext, params: IGetSheetBlockRequest): Promise { + return Promise.resolve({ + block: testSheetBlocks()[0], + error: { + code: ErrorCode.OK, + message: '', + }, + }); + } + + /** Fetch missing changeset */ + fetchMissingChangesets( + context: ILogContext, + params: IFetchMissingChangesetsRequest + ): Promise { + return Promise.resolve({ + changesets: [], + error: { + code: ErrorCode.OK, + message: '', + }, + }); + } + + getResourcesRequest(context: ILogContext, params: IGetResourcesRequest): Promise { + return Promise.resolve({ + resources: {}, + error: { + code: ErrorCode.OK, + message: '', + }, + }); + } + + /** Save snapshot to a database. */ + saveSnapshot(context: ILogContext, params: ISaveSnapshotRequest): Promise { + return Promise.resolve({ + error: { + code: ErrorCode.OK, + message: '', + }, + }); + }; + + /** Save sheet block to a database. */ + saveSheetBlock(context: ILogContext, params: ISaveSheetBlockRequest): Promise { + return Promise.resolve({ + error: { + code: ErrorCode.OK, + message: '', + }, + blockID: '100100', // mock block id + }); + }; + + /** Save changeset to a database. */ + saveChangeset(context: ILogContext, params: ISaveChangesetRequest): Promise { + return Promise.resolve({ + error: { + code: ErrorCode.OK, + message: '', + }, + concurrent: [], + }); + }; +} diff --git a/packages/core/src/services/snapshot/__tests__/snapshot-transform.spec.ts b/packages/core/src/services/snapshot/__tests__/snapshot-transform.spec.ts new file mode 100644 index 0000000000..58cf4ded51 --- /dev/null +++ b/packages/core/src/services/snapshot/__tests__/snapshot-transform.spec.ts @@ -0,0 +1,70 @@ +/** + * Copyright 2023-present DreamNum Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { describe, expect, it } from 'vitest'; +import type { ISnapshot } from '@univerjs/protocol'; +import { getSheetBlocksFromSnapshot, transformSnapshotToWorkbookData, transformWorkbookDataToSnapshot } from '../snapshot-transform'; + +import type { ILogContext } from '../../log/context'; +import type { ISnapshotServerService } from '../snapshot-server.service'; +import { textDecoder } from '../snapshot-utils'; +import { MockSnapshotServerService, testSheetBlocks, testSnapshot, testWorkbookData } from './snapshot-mock'; + +/** + * The Uint8Array converted from the encoded string, and the encoded string converted back is different from the original one, so objects can only be used for comparison. + * @param snapshot + * @returns + */ +function transformSnapshotMetaToObject(snapshot: ISnapshot) { + const workbook = snapshot.workbook; + if (!workbook) return snapshot; + + // Loop through sheets and convert originalMeta + Object.keys(workbook.sheets).forEach((sheetKey) => { + const sheet = workbook.sheets[sheetKey]; + sheet.originalMeta = JSON.parse(textDecoder.decode(sheet.originalMeta)); // Reassign the converted object to originalMeta + }); + + workbook.originalMeta = JSON.parse(textDecoder.decode(workbook.originalMeta)); // Reassign the converted object to originalMeta + + return snapshot; +} + +describe('Test snapshot transform', () => { + it('Function transformWorkbookDataToSnapshot', async () => { + const context: ILogContext = { + metadata: undefined, + }; + const workbookData = testWorkbookData(); + const unitID = workbookData.id; + const rev = workbookData.rev ?? 0; + + const snapshotService: ISnapshotServerService = new MockSnapshotServerService(); + + const { snapshot } = await transformWorkbookDataToSnapshot(context, workbookData, unitID, rev, snapshotService); + + const snapshotData = transformSnapshotMetaToObject(snapshot); + + expect(snapshotData).toStrictEqual(transformSnapshotMetaToObject(testSnapshot())); + + const blocks = await getSheetBlocksFromSnapshot(snapshot, snapshotService); + + expect(blocks).toStrictEqual(testSheetBlocks()); + }); + it('Function transformSnapshotToWorkbookData', () => { + expect(transformSnapshotToWorkbookData(testSnapshot(), testSheetBlocks())).toStrictEqual(testWorkbookData()); + }); +}); diff --git a/packages/core/src/services/snapshot/snapshot-server.service.ts b/packages/core/src/services/snapshot/snapshot-server.service.ts new file mode 100644 index 0000000000..a959b0bb37 --- /dev/null +++ b/packages/core/src/services/snapshot/snapshot-server.service.ts @@ -0,0 +1,185 @@ +/** + * Copyright 2023-present DreamNum Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { + ErrorCode, + UniverType, +} from '@univerjs/protocol'; +import type { + IFetchMissingChangesetsRequest, + IFetchMissingChangesetsResponse, + IGetResourcesRequest, + IGetResourcesResponse, + IGetSheetBlockRequest, + IGetSheetBlockResponse, + IGetUnitOnRevRequest, + IGetUnitOnRevResponse, + ISaveChangesetRequest, + ISaveChangesetResponse, + ISaveSheetBlockRequest, + ISaveSheetBlockResponse, + ISaveSnapshotRequest, + ISaveSnapshotResponse, + ISheetBlock, +} from '@univerjs/protocol'; +import { createIdentifier } from '@wendellhu/redi'; + +import type { ILogContext } from '../log/context'; + +/** + * It provides implementations for server side controllers to load or save + * or load snapshots. This service should be implemented by the host environment. + * And it shouldn't contain any business logic. + */ +export const ISnapshotServerService = createIdentifier( + 'univer.snapshot-server-service' +); +export interface ISnapshotServerService { + /** Load snapshot from a database. */ + getUnitOnRev: (context: ILogContext, params: IGetUnitOnRevRequest) => Promise; + /** Load sheet block from a database. */ + getSheetBlock: (context: ILogContext, params: IGetSheetBlockRequest) => Promise; + /** Fetch missing changeset */ + fetchMissingChangesets: ( + context: ILogContext, + params: IFetchMissingChangesetsRequest + ) => Promise; + + getResourcesRequest: (context: ILogContext, params: IGetResourcesRequest) => Promise; + // #region - server only methods + + // These methods should not be implemented by client code because snapshot + // saving is completed by the Apply microservice running on Node.js. + + /** Save snapshot to a database. */ + saveSnapshot: (context: ILogContext, params: ISaveSnapshotRequest) => Promise; + /** Save sheet block to a database. */ + saveSheetBlock: (context: ILogContext, params: ISaveSheetBlockRequest) => Promise; + /** Save changeset to a database. */ + saveChangeset: (context: ILogContext, params: ISaveChangesetRequest) => Promise; + + // #endregion - server only methods +} + +/** + * The server needs to fully implement all interfaces, but when used by the client, use saveSheetBlock to cache the sheet block locally, and use getSheetBlock to obtain the sheet block. + */ +export class ClientSnapshotServerService implements ISnapshotServerService { + private _sheetBlockCache: Map = new Map(); + + /** Load snapshot from a database. */ + getUnitOnRev(context: ILogContext, params: IGetUnitOnRevRequest): Promise { + return Promise.resolve({ + snapshot: { + unitID: '', + type: UniverType.UNIVER_SHEET, + rev: 0, + workbook: undefined, + doc: undefined, + }, + changesets: [], + error: { + code: ErrorCode.OK, + message: '', + }, + }); + } + + /** Load sheet block from a database. */ + getSheetBlock(context: ILogContext, params: IGetSheetBlockRequest): Promise { + // get block from cache + const block = this._sheetBlockCache.get(params.blockID); + + return Promise.resolve({ + block, + error: { + code: ErrorCode.OK, + message: '', + }, + }); + } + + /** Fetch missing changeset */ + fetchMissingChangesets( + context: ILogContext, + params: IFetchMissingChangesetsRequest + ): Promise { + return Promise.resolve({ + changesets: [], + error: { + code: ErrorCode.OK, + message: '', + }, + }); + } + + getResourcesRequest(context: ILogContext, params: IGetResourcesRequest): Promise { + return Promise.resolve({ + resources: {}, + error: { + code: ErrorCode.OK, + message: '', + }, + }); + } + + /** Save snapshot to a database. */ + saveSnapshot(context: ILogContext, params: ISaveSnapshotRequest): Promise { + return Promise.resolve({ + error: { + code: ErrorCode.OK, + message: '', + }, + }); + }; + + /** Save sheet block to a database. */ + saveSheetBlock(context: ILogContext, params: ISaveSheetBlockRequest): Promise { + const { block } = params; + + if (!block) { + return Promise.resolve({ + error: { + code: ErrorCode.UNDEFINED, + message: 'block is required', + }, + blockID: '', + }); + } + + // save block to cache + this._sheetBlockCache.set(block.id, block); + + return Promise.resolve({ + error: { + code: ErrorCode.OK, + message: '', + }, + blockID: block.id, + }); + }; + + /** Save changeset to a database. */ + saveChangeset(context: ILogContext, params: ISaveChangesetRequest): Promise { + return Promise.resolve({ + error: { + code: ErrorCode.OK, + message: '', + }, + concurrent: [], + }); + }; +} diff --git a/packages/core/src/services/snapshot/snapshot-transform.ts b/packages/core/src/services/snapshot/snapshot-transform.ts new file mode 100644 index 0000000000..35f23bd174 --- /dev/null +++ b/packages/core/src/services/snapshot/snapshot-transform.ts @@ -0,0 +1,394 @@ +/** + * Copyright 2023-present DreamNum Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import type { IDocumentMeta, IGetSheetBlockRequest, ISheetBlock, ISheetBlockMeta, ISnapshot, IWorkbookMeta, IWorksheetMeta } from '@univerjs/protocol'; +import { ErrorCode, isError, UniverType } from '@univerjs/protocol'; + +import type { ILogContext } from '../log/context'; +import type { IWorkbookData } from '../../types/interfaces/i-workbook-data'; +import { SheetTypes } from '../../types/enum/sheet-types'; +import type { IWorksheetData } from '../../types/interfaces/i-worksheet-data'; +import type { ICellData } from '../../types/interfaces/i-cell-data'; +import { LocaleType } from '../../types/enum/locale-type'; +import type { IDocumentData } from '../../types/interfaces/i-document-data'; +import { + decodeDocOriginalMeta, + decodePartOfCellData, + decodeWorksheetOtherMetas, + encodeDocOriginalMeta, + encodeWorkbookOtherMetas, + encodeWorksheetOtherMetas, + splitCellDataToBlocks, +} from './snapshot-utils'; +import type { ISnapshotServerService } from './snapshot-server.service'; + +export async function generateTemporarySnap( + context: ILogContext, + workbook: IWorkbookData, + unitID: string, + rev: number, + snapshotService: ISnapshotServerService +): Promise<{ + snapshotRes: ISnapshot; + }> { + const blockMeta: { [key: string]: ISheetBlockMeta } = {}; + + // Deal with worksheets and their blocks. + const sheetMetas: { [key: string]: IWorksheetMeta } = {}; + const blocksSaveSuccess = await Promise.all( + Object.entries(workbook.sheets).map(async ([sheetID, worksheet]) => { + const sheetMeta: IWorksheetMeta = { + id: worksheet.id!, + type: SheetTypes.GRID, + name: worksheet.name!, + rowCount: worksheet.rowCount!, + columnCount: worksheet.columnCount!, + originalMeta: encodeWorksheetOtherMetas(worksheet), + }; + + sheetMetas[sheetID] = sheetMeta; + + // Trigger RPC and store the result in sheetBlocks. + if (worksheet.cellData) { + const sheetBlocks = splitCellDataToBlocks(worksheet.cellData, worksheet.rowCount!); + const responses = await Promise.all( + sheetBlocks.map((block) => + snapshotService.saveSheetBlock(context, { + unitID, + type: UniverType.UNIVER_SHEET, + block, + }) + ) + ); + + if (responses.some((response) => response.error?.code !== ErrorCode.OK)) { + return false; + } + + blockMeta[sheetID] = { + sheetID, + blocks: responses.map((response) => response.blockID), + }; + } + + return true; + }) + ); + + if (!blocksSaveSuccess) { + throw new Error('[transformWorkbookDataToSnapshot()]: Failed to save sheet blocks.'); + } + + const originalMeta = encodeWorkbookOtherMetas(workbook); + const workbookMeta: IWorkbookMeta = { + unitID: workbook.id, + rev, + creator: '', + name: workbook.name, + sheetOrder: workbook.sheetOrder, + sheets: sheetMetas, + blockMeta, // this should not be empty + resources: workbook.resources || [], + originalMeta, + }; + const snapshotRes: ISnapshot = { + unitID, + rev: workbookMeta.rev, + type: UniverType.UNIVER_SHEET, + workbook: workbookMeta, + doc: undefined, + }; + + return { + snapshotRes, + }; +} + +// TODO@wzhudev: How to reuse blocks? + +export async function transformWorkbookDataToSnapshot( + context: ILogContext, + workbook: IWorkbookData, + unitID: string, + rev: number, + snapshotService: ISnapshotServerService +): Promise<{ + snapshot: ISnapshot; + }> { + // Gather sheet blocks info for worksheets. + const blockMeta: { [key: string]: ISheetBlockMeta } = {}; + + // Deal with worksheets and their blocks. + const sheetMetas: { [key: string]: IWorksheetMeta } = {}; + const blocksSaveSuccess = await Promise.all( + Object.entries(workbook.sheets).map(async ([sheetID, worksheet]) => { + const sheetMeta: IWorksheetMeta = { + id: worksheet.id!, + type: SheetTypes.GRID, + name: worksheet.name!, + rowCount: worksheet.rowCount!, + columnCount: worksheet.columnCount!, + originalMeta: encodeWorksheetOtherMetas(worksheet), + }; + + sheetMetas[sheetID] = sheetMeta; + + // Trigger RPC and store the result in sheetBlocks. + if (worksheet.cellData) { + const sheetBlocks = splitCellDataToBlocks(worksheet.cellData, worksheet.rowCount!); + const responses = await Promise.all( + sheetBlocks.map((block) => + snapshotService.saveSheetBlock(context, { + unitID, + type: UniverType.UNIVER_SHEET, + block, + }) + ) + ); + + if (responses.some((response) => response.error?.code !== ErrorCode.OK)) { + return false; + } + + blockMeta[sheetID] = { + sheetID, + blocks: responses.map((response) => response.blockID), + }; + } + + return true; + }) + ); + + if (!blocksSaveSuccess) { + throw new Error('[transformWorkbookDataToSnapshot()]: Failed to save sheet blocks.'); + } + + const originalMeta = encodeWorkbookOtherMetas(workbook); + const workbookMeta: IWorkbookMeta = { + unitID: workbook.id, + rev, + creator: '', + name: workbook.name, + sheetOrder: workbook.sheetOrder, + sheets: sheetMetas, + blockMeta, // this should not be empty + resources: workbook.resources || [], + originalMeta, + }; + const snapshot: ISnapshot = { + unitID, + rev: workbookMeta.rev, + type: UniverType.UNIVER_SHEET, + workbook: workbookMeta, + doc: undefined, + }; + + const saveResult = await snapshotService.saveSnapshot(context, { + unitID, + type: UniverType.UNIVER_SHEET, + snapshot, + }); + + if (isError(saveResult.error)) { + throw new Error( + `transformWorkbookDataToSnapshot(): Failed to save snapshot.\nErrorCode: ${saveResult.error?.code}:${saveResult.error?.message}` + ); + } + + return { + snapshot, + }; +} + +// NOTE: performance of this method is pretty suspicious. +/** + * Assemble a snapshot to a workbook. + * @param snapshot + * @param sheetBlocks + */ +export function transformSnapshotToWorkbookData( + snapshot: ISnapshot, + sheetBlocks: ISheetBlock[], + _context?: ILogContext +): IWorkbookData { + const workbookMeta = snapshot.workbook; + if (!workbookMeta) { + throw new Error(''); + } + + // Deal with worksheets. + const sheetMap: { [key: string]: Partial } = {}; + Object.entries(workbookMeta.sheets).forEach(([sheetID, sheetMeta]) => { + const otherMeta = decodeWorksheetOtherMetas(sheetMeta.originalMeta); + sheetMap[sheetID] = { + id: sheetMeta.id, + name: sheetMeta.name, + rowCount: sheetMeta.rowCount, + columnCount: sheetMeta.columnCount, + ...otherMeta, + }; + }); + + // Deal with sheet blocks. + const sheetBlocksMap = new Map(); + sheetBlocks.forEach((block) => { + sheetBlocksMap.set(block.id, block); + }); + + if (workbookMeta.blockMeta) { + Object.entries(workbookMeta.blockMeta).forEach(([sheetID, blocksOfSheet]) => { + const worksheetConfig = sheetMap[sheetID]; + worksheetConfig.cellData = {}; + + const blocks: ISheetBlock[] = []; + blocksOfSheet.blocks?.forEach((blockID) => { + const block = sheetBlocksMap.get(blockID); + if (block) { + blocks.push(block); + } else { + throw new Error(''); + } + }); + + blocks.forEach((block) => { + const partOfCellData = decodePartOfCellData(block.data); + Object.entries(partOfCellData).forEach(([rowNumber, rowData]) => { + const row: { [key: number]: ICellData } = (worksheetConfig.cellData![+rowNumber as number] = {}); + Object.entries(rowData).forEach(([columnNumber, cellData]) => { + row[+columnNumber as number] = cellData as ICellData; + }); + }); + }); + }); + } + + // Deal with workbook meta + const otherMeta = decodeWorksheetOtherMetas(workbookMeta.originalMeta); + const workbookData: IWorkbookData = { + id: snapshot.unitID, + rev: workbookMeta.rev, + name: workbookMeta.name, + sheetOrder: workbookMeta.sheetOrder, + appVersion: '', + locale: LocaleType.EN_US, + sheets: sheetMap, + styles: {}, + resources: workbookMeta.resources || [], + ...otherMeta, + }; + + return workbookData; +} + +export function transformSnapshotToDocumentData(snapshot: ISnapshot): IDocumentData { + const documentMeta = snapshot.doc; + if (documentMeta == null) { + throw new Error('transformSnapshotToDocumentData(): snapshot.doc is undefined.'); + } + + const { unitID, rev, name, originalMeta } = documentMeta; + + const { body, documentStyle = {}, settings = {} } = decodeDocOriginalMeta(originalMeta); + + const documentData: IDocumentData = { + id: unitID, + rev, + locale: LocaleType.EN_US, + title: name, + body, + documentStyle, + settings, + }; + + return documentData; +} +export async function transformDocumentDataToSnapshot( + context: ILogContext, + document: IDocumentData, + unitID: string, + rev: number, + snapshotService: ISnapshotServerService +): Promise<{ snapshot: ISnapshot }> { + const documentMeta: IDocumentMeta = { + unitID: document.id, + rev, + creator: '', + name: document.title ?? '', + resources: document.resources || [], + originalMeta: encodeDocOriginalMeta(document), + }; + const snapshot: ISnapshot = { + unitID, + rev: documentMeta.rev, + type: UniverType.UNIVER_DOC, + workbook: undefined, + doc: documentMeta, + }; + + const saveResult = await snapshotService.saveSnapshot(context, { + unitID, + type: UniverType.UNIVER_DOC, + snapshot, + }); + + if (isError(saveResult.error)) { + throw new Error( + `transformDocumentDataToSnapshot(): Failed to save snapshot.\nErrorCode: ${saveResult.error?.code}:${saveResult.error?.message}` + ); + } + + return { + snapshot, + }; +} + +/** + * + * @param snapshot + * @param snapshotService + * @returns + */ +export async function getSheetBlocksFromSnapshot(snapshot: ISnapshot, snapshotService: ISnapshotServerService) { + const workbookMeta = snapshot.workbook; + + if (!workbookMeta) { + throw new Error('Workbook metadata is not available'); + } + + const blocks: ISheetBlock[] = []; + const promises: Promise[] = []; + + Object.entries(workbookMeta.blockMeta).forEach(([sheetID, blocksOfSheet]) => { + const blockPromises = blocksOfSheet.blocks.map(async (blockID) => { + const params: IGetSheetBlockRequest = { + unitID: workbookMeta.unitID, + type: UniverType.UNIVER_SHEET, + blockID, + }; + const { block } = await snapshotService.getSheetBlock({}, params); + if (block) { + blocks.push(block); + } else { + throw new Error('Block not found'); + } + }); + promises.push(...blockPromises); + }); + + await Promise.all(promises); + return blocks; +} diff --git a/packages/core/src/services/snapshot/snapshot-utils.ts b/packages/core/src/services/snapshot/snapshot-utils.ts new file mode 100644 index 0000000000..7a8cf53dd2 --- /dev/null +++ b/packages/core/src/services/snapshot/snapshot-utils.ts @@ -0,0 +1,131 @@ +/** + * Copyright 2023-present DreamNum Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import type { ISheetBlock } from '@univerjs/protocol'; + +import { b64DecodeUnicode } from '../../shared/coder'; +import type { IWorksheetData } from '../../types/interfaces/i-worksheet-data'; +import { Tools } from '../../shared/tools'; +import type { IWorkbookData } from '../../types/interfaces/i-workbook-data'; +import type { IDocumentData } from '../../types/interfaces/i-document-data'; +import type { ICellData } from '../../types/interfaces/i-cell-data'; +import type { IObjectMatrixPrimitiveType } from '../../shared/object-matrix'; +import { ObjectMatrix } from '../../shared/object-matrix'; + +// Some properties are stored in the meta fields or are in sheet blocks. +// They can be found in `packages/collaboration/src/services/snapshot/snapshot-utils.ts`. + +export const textEncoder = new TextEncoder(); +export const textDecoder = new TextDecoder(); + +/** + * @param worksheet + */ +export function encodeWorksheetOtherMetas(worksheet: Partial): Uint8Array { + const cloned: Partial = Tools.deepClone(worksheet); + // Some properties are stored in the meta fields are in sheet blocks + // so we need to delete them before serializing remaining properties. + delete cloned.id; + delete cloned.name; + delete cloned.rowCount; + delete cloned.columnCount; + delete cloned.cellData; + const meta = textEncoder.encode(JSON.stringify(cloned)); + return meta; +} + +export function encodeWorkbookOtherMetas(workbook: IWorkbookData): Uint8Array { + const cloned: Partial = Tools.deepClone(workbook); + // Some properties are stored in the meta fields are in sheet blocks + // so we need to delete them before serializing remaining properties. + delete cloned.id; + delete cloned.rev; + delete cloned.name; + delete cloned.sheetOrder; + delete cloned.sheets; + const meta = textEncoder.encode(JSON.stringify(cloned)); + return meta; +} + +export function encodeDocOriginalMeta(document: IDocumentData): Uint8Array { + const cloned: Partial = Tools.deepClone(document); + // Some properties are stored in the meta fields are in sheet blocks + // so we need to delete them before serializing remaining properties. + delete cloned.id; + delete cloned.rev; + delete cloned.title; + delete cloned.resources; + const meta = textEncoder.encode(JSON.stringify(cloned)); + return meta; +} + +export function decodeWorksheetOtherMetas(buffer: Uint8Array): Partial { + return JSON.parse(textDecoder.decode(buffer)); +} + +export function decodeWorkbookOtherMetas(buffer: Uint8Array): Partial { + return JSON.parse(textDecoder.decode(buffer)); +} + +export function decodePartOfCellData(buffer: Uint8Array | string): IObjectMatrixPrimitiveType { + if (typeof buffer === 'string') { + return JSON.parse(b64DecodeUnicode(buffer)); + } + + return JSON.parse(textDecoder.decode(buffer)); +} + +export function decodeDocOriginalMeta(buffer: Uint8Array | string): Partial { + if (typeof buffer === 'string') { + return JSON.parse(b64DecodeUnicode(buffer)); + } + + return JSON.parse(textDecoder.decode(buffer)); +} + +const FRAGMENT_ROW_COUNT = 256; +export function splitCellDataToBlocks( + cellData: IObjectMatrixPrimitiveType, + maxColumn: number +): ISheetBlock[] { + const utilObjectMatrix = new ObjectMatrix(cellData); + const length = utilObjectMatrix.getLength(); + const blocks: ISheetBlock[] = []; + + // Store every 32 rows into a block even if some rows are empty. + let i = 0; + while (i < length) { + const endRow = Math.min(i + FRAGMENT_ROW_COUNT, length - 1); + const slice = utilObjectMatrix.getSlice(i, Math.min(i + FRAGMENT_ROW_COUNT, length - 1), 0, maxColumn); + const data = serializeCellDataSlice(slice); + + blocks.push({ + id: Tools.generateRandomId(19, '0123456789'), // an random ID for client, this would be changed after the block is saved in the server + startRow: i, + endRow, + data, + }); + + i += FRAGMENT_ROW_COUNT; + } + + return blocks; +} + +function serializeCellDataSlice(slice: ObjectMatrix): Uint8Array { + const data = slice.getData(); + return textEncoder.encode(JSON.stringify(data)); +} diff --git a/packages/core/src/shared/coder.ts b/packages/core/src/shared/coder.ts new file mode 100644 index 0000000000..006b4d36b6 --- /dev/null +++ b/packages/core/src/shared/coder.ts @@ -0,0 +1,27 @@ +/** + * Copyright 2023-present DreamNum Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export function b64EncodeUnicode(str: string): string { + return btoa( + encodeURIComponent(str).replace(/%([0-9A-F]{2})/g, (match, p1) => String.fromCharCode(Number.parseInt(p1, 16))) + ); +} + +export function b64DecodeUnicode(str: string): string { + return decodeURIComponent( + Array.prototype.map.call(atob(str), (c) => `%${`00${c.charCodeAt(0).toString(16)}`.slice(-2)}`).join('') + ); +} diff --git a/packages/core/src/shared/tools.ts b/packages/core/src/shared/tools.ts index 2408ae2cb4..319380e4f6 100644 --- a/packages/core/src/shared/tools.ts +++ b/packages/core/src/shared/tools.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import { nanoid } from 'nanoid'; +import { customAlphabet, nanoid } from 'nanoid'; import type { Class, IKeyValue } from './types'; @@ -165,7 +165,10 @@ export class Tools { return 'Unknown browser'; } - static generateRandomId(n: number = 21): string { + static generateRandomId(n: number = 21, alphabet?: string): string { + if (alphabet) { + return customAlphabet(alphabet, n)(); + } return nanoid(n); } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 5ef0fbe2db..61c737f831 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -314,6 +314,9 @@ importers: '@types/numeral': specifier: ^2.0.5 version: 2.0.5 + '@univerjs/protocol': + specifier: ^0.1.0 + version: 0.1.0(@grpc/grpc-js@1.10.1)(rxjs@7.8.1) '@univerjs/shared': specifier: workspace:* version: link:../../common/shared @@ -3413,6 +3416,25 @@ packages: resolution: {integrity: sha512-cEee/Z+I12mZcFJshKcCqC8tuX5hG3s+d+9nZ3LabqKF1vKdF41B92pJVCBggjAGORAeOzyyDDKrZwIkLffeOQ==} dev: false + /@grpc/grpc-js@1.10.1: + resolution: {integrity: sha512-55ONqFytZExfOIjF1RjXPcVmT/jJqFzbbDqxK9jmRV4nxiYWtL9hENSW1Jfx0SdZfrvoqd44YJ/GJTqfRrawSQ==} + engines: {node: ^8.13.0 || >=10.10.0} + dependencies: + '@grpc/proto-loader': 0.7.10 + '@types/node': 20.11.24 + dev: true + + /@grpc/proto-loader@0.7.10: + resolution: {integrity: sha512-CAqDfoaQ8ykFd9zqBDn4k6iWT9loLAlc2ETmDFS9JCD70gDcnA4L3AFEo2iV7KyAtAAHFW9ftq1Fz+Vsgq80RQ==} + engines: {node: '>=6'} + hasBin: true + dependencies: + lodash.camelcase: 4.3.0 + long: 5.2.3 + protobufjs: 7.2.6 + yargs: 17.7.2 + dev: true + /@humanwhocodes/config-array@0.11.14: resolution: {integrity: sha512-3T8LkOmg45BV5FICb15QQMsyUSWrQ8AygVfC7ZG32zOalnqrilm018ZVCw0eapXux8FtA33q8PSRSstjee3jSg==} engines: {node: '>=10.10.0'} @@ -3734,6 +3756,49 @@ packages: config-chain: 1.1.13 dev: true + /@protobufjs/aspromise@1.1.2: + resolution: {integrity: sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ==} + dev: true + + /@protobufjs/base64@1.1.2: + resolution: {integrity: sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg==} + dev: true + + /@protobufjs/codegen@2.0.4: + resolution: {integrity: sha512-YyFaikqM5sH0ziFZCN3xDC7zeGaB/d0IUb9CATugHWbd1FRFwWwt4ld4OYMPWu5a3Xe01mGAULCdqhMlPl29Jg==} + dev: true + + /@protobufjs/eventemitter@1.1.0: + resolution: {integrity: sha512-j9ednRT81vYJ9OfVuXG6ERSTdEL1xVsNgqpkxMsbIabzSo3goCjDIveeGv5d03om39ML71RdmrGNjG5SReBP/Q==} + dev: true + + /@protobufjs/fetch@1.1.0: + resolution: {integrity: sha512-lljVXpqXebpsijW71PZaCYeIcE5on1w5DlQy5WH6GLbFryLUrBD4932W/E2BSpfRJWseIL4v/KPgBFxDOIdKpQ==} + dependencies: + '@protobufjs/aspromise': 1.1.2 + '@protobufjs/inquire': 1.1.0 + dev: true + + /@protobufjs/float@1.0.2: + resolution: {integrity: sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ==} + dev: true + + /@protobufjs/inquire@1.1.0: + resolution: {integrity: sha512-kdSefcPdruJiFMVSbn801t4vFK7KB/5gd2fYvrxhuJYg8ILrmn9SKSX2tZdV6V+ksulWqS7aXjBcRXl3wHoD9Q==} + dev: true + + /@protobufjs/path@1.1.2: + resolution: {integrity: sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA==} + dev: true + + /@protobufjs/pool@1.1.0: + resolution: {integrity: sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw==} + dev: true + + /@protobufjs/utf8@1.1.0: + resolution: {integrity: sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw==} + dev: true + /@radix-ui/react-compose-refs@1.0.1(@types/react@18.2.64)(react@18.2.0): resolution: {integrity: sha512-fDSBgd44FKHa1FRMU59qBMPFcl2PZE+2nmqunj+BWFyYYjnhIDWL2ItDs3rrbJDQOtzt5nIebLCQc4QRfz6LJw==} peerDependencies: @@ -5667,6 +5732,17 @@ packages: react-dom: 18.2.0(react@18.2.0) dev: false + /@univerjs/protocol@0.1.0(@grpc/grpc-js@1.10.1)(rxjs@7.8.1): + resolution: {integrity: sha512-tta0qKO39tR+TkfYXGecmKvihCYS55Jc2YAX176IDsFGQxWUzsqx9Y+ait+qDflSJALj6ocoNf7g81xUfz6DNA==} + engines: {node: '>=16.0.0', npm: '>=8.0.0'} + peerDependencies: + '@grpc/grpc-js': ^1.9.14 + rxjs: '>=7.0.0' + dependencies: + '@grpc/grpc-js': 1.10.1 + rxjs: 7.8.1 + dev: true + /@vitejs/plugin-react@4.2.1(vite@5.1.4): resolution: {integrity: sha512-oojO9IDc4nCUUi8qIR11KoQm0XFFLIwsRBwHRR4d/88IWghn1y6ckz/bJ8GHDCsYEJee8mDzqtJxh15/cisJNQ==} engines: {node: ^14.18.0 || >=16.0.0} @@ -10795,6 +10871,10 @@ packages: wrap-ansi: 9.0.0 dev: true + /long@5.2.3: + resolution: {integrity: sha512-lcHwpNoggQTObv5apGNCTdJrO69eHOZMi4BNC+rTLER8iHAqGrUVeLh/irVIM7zTw2bOXA8T6uNPeujwOLg/2Q==} + dev: true + /loose-envify@1.4.0: resolution: {integrity: sha512-lyuxPGr/Wfhrlem2CL/UcnUc1zcqKAImBDzukY7Y5F/yQiNdko6+fRLevlw1HgMySw7f611UIY408EtxRSoK3Q==} hasBin: true @@ -11947,6 +12027,25 @@ packages: resolution: {integrity: sha512-vtK/94akxsTMhe0/cbfpR+syPuszcuwhqVjJq26CuNDgFGj682oRBXOP5MJpv2r7JtE8MsiepGIqvvOTBwn2vA==} dev: true + /protobufjs@7.2.6: + resolution: {integrity: sha512-dgJaEDDL6x8ASUZ1YqWciTRrdOuYNzoOf27oHNfdyvKqHr5i0FV7FSLU+aIeFjyFgVxrpTOtQUi0BLLBymZaBw==} + engines: {node: '>=12.0.0'} + requiresBuild: true + dependencies: + '@protobufjs/aspromise': 1.1.2 + '@protobufjs/base64': 1.1.2 + '@protobufjs/codegen': 2.0.4 + '@protobufjs/eventemitter': 1.1.0 + '@protobufjs/fetch': 1.1.0 + '@protobufjs/float': 1.0.2 + '@protobufjs/inquire': 1.1.0 + '@protobufjs/path': 1.1.2 + '@protobufjs/pool': 1.1.0 + '@protobufjs/utf8': 1.1.0 + '@types/node': 20.11.24 + long: 5.2.3 + dev: true + /protocols@2.0.1: resolution: {integrity: sha512-/XJ368cyBJ7fzLMwLKv1e4vLxOju2MNAIokcr7meSaNcVbWz/CPcW22cP04mwxOErdA5mwjA8Q6w/cdAQxVn7Q==} dev: true