From 44cbd9d81c09fc0408d479accfe4a2eb3927a8d1 Mon Sep 17 00:00:00 2001 From: Sergey Popov Date: Tue, 22 Feb 2022 14:22:04 +0300 Subject: [PATCH] Roam import WIP --- .../src/pages/VaultsPage/ImportModal.tsx | 62 ++- packages/web-core/index.ts | 1 + .../repositories/TextBlocksRepository.ts | 24 +- .../services/ImportExportService.ts | 72 ++- .../synchronizer/OnDbChangeNotifier.ts | 2 - .../web-core/src/lib/roamToHarikaJson.test.ts | 493 +++--------------- packages/web-core/src/lib/roamToHarikaJson.ts | 171 +++++- 7 files changed, 332 insertions(+), 493 deletions(-) diff --git a/packages/web-app/src/pages/VaultsPage/ImportModal.tsx b/packages/web-app/src/pages/VaultsPage/ImportModal.tsx index c8fcbc5b..4c57e46f 100644 --- a/packages/web-app/src/pages/VaultsPage/ImportModal.tsx +++ b/packages/web-app/src/pages/VaultsPage/ImportModal.tsx @@ -1,6 +1,15 @@ -import React from 'react'; +import { + generateId, + roamToHarikaJson, + VaultApplication, +} from '@harika/web-core'; +import { isArray } from 'lodash-es'; +import React, { useEffect } from 'react'; import { modalClass } from '../../components/Modal/Modal'; +import { useGetSyncToken } from '../../hooks/useGetSyncToken'; +import { useUserVaults } from '../../hooks/useUserApp'; +import { paths } from '../../paths'; import { cn } from '../../utils'; const formModalClass = cn('vault-form-modal'); @@ -12,6 +21,57 @@ export const ImportModal = ({ file: File; vaultName: string; }) => { + const userVaults = useUserVaults(); + const getSyncToken = useGetSyncToken(); + + useEffect(() => { + const callback = async () => { + let data: any | undefined = undefined; + + try { + data = JSON.parse(await file.text()); + } catch { + console.error('Failed to parse JSON!'); + + return; + } + + const dump = (() => { + if (isArray(data)) { + return roamToHarikaJson(data); + // we assume it is roam format + } else { + // Otherwise harika format + return data; + } + })(); + + const vaultId = generateId(); + const vault = await userVaults?.createVault({ + name: vaultName, + dbId: vaultId, + }); + if (!vault) { + console.error('Failed to create vault'); + + return; + } + + const vaultApp = new VaultApplication( + vaultId, + import.meta.env.VITE_PUBLIC_WS_URL as string, + getSyncToken, + ); + + await vaultApp.start(); + await vaultApp.getImportExportService().importData(dump); + + // window.location.pathname = paths.vaultDailyPath({ vaultId: vault.id }); + }; + + callback(); + }, [file, getSyncToken, userVaults, vaultName]); + return ( <>

diff --git a/packages/web-core/index.ts b/packages/web-core/index.ts index 03facaf9..dc98cc7f 100644 --- a/packages/web-core/index.ts +++ b/packages/web-core/index.ts @@ -25,3 +25,4 @@ export * from './src/apps/VaultApplication/BlocksExtension/models/BaseBlock'; export * from './src/apps/VaultApplication/BlocksExtension/models/BlocksSelection'; export * from './src/lib/blockParser/blockUtils'; export * from './src/apps/VaultApplication/BlocksExtension/selectors/getGroupedBacklinks'; +export * from './src/lib/roamToHarikaJson'; diff --git a/packages/web-core/src/apps/VaultApplication/BlocksExtension/repositories/TextBlocksRepository.ts b/packages/web-core/src/apps/VaultApplication/BlocksExtension/repositories/TextBlocksRepository.ts index 3f037311..d1703e24 100644 --- a/packages/web-core/src/apps/VaultApplication/BlocksExtension/repositories/TextBlocksRepository.ts +++ b/packages/web-core/src/apps/VaultApplication/BlocksExtension/repositories/TextBlocksRepository.ts @@ -22,24 +22,22 @@ export class TextBlocksRepository extends BaseBlockRepository< TextBlockDoc, TextBlockRow > { - bulkCreate( + async bulkCreate( attrsArray: TextBlockDoc[], ctx: ISyncCtx, e: IQueryExecuter = this.db, ) { - return e.transaction(async (t) => { - const res = await super.bulkCreate(attrsArray, ctx, t); - - await t.insertRecords( - textBlocksFTSTable, - res.map((row) => ({ - id: row.id, - textContent: row.content.toLowerCase(), - })), - ); + const res = await super.bulkCreate(attrsArray, ctx); - return res; - }); + await e.insertRecords( + textBlocksFTSTable, + res.map((row) => ({ + id: row.id, + textContent: row.content.toLowerCase(), + })), + ); + + return res; } bulkUpdate( diff --git a/packages/web-core/src/apps/VaultApplication/BlocksExtension/services/ImportExportService.ts b/packages/web-core/src/apps/VaultApplication/BlocksExtension/services/ImportExportService.ts index 6ff8febe..5baa4b78 100644 --- a/packages/web-core/src/apps/VaultApplication/BlocksExtension/services/ImportExportService.ts +++ b/packages/web-core/src/apps/VaultApplication/BlocksExtension/services/ImportExportService.ts @@ -73,51 +73,49 @@ export class ImportExportService { const links: BlockLinkRow[] = []; - await this.noteBlocksRepository.transaction(async (t) => { - for (const { tableName, rows } of dump.data) { - if (tableName === noteBlocksTable) { - // eslint-disable-next-line no-loop-func - rows.forEach((row) => { - row.linkedBlockIds.forEach((id: string) => { - links.push({ - id: generateId(), - blockId: row.id, - linkedToBlockId: id, - orderPosition: i++, - createdAt: new Date().getTime(), - updatedAt: new Date().getTime(), - }); + for (const { tableName, rows } of dump.data) { + if (tableName === noteBlocksTable) { + // eslint-disable-next-line no-loop-func + rows.forEach((row) => { + row.linkedBlockIds.forEach((id: string) => { + links.push({ + id: generateId(), + blockId: row.id, + linkedToBlockId: id, + orderPosition: i++, + createdAt: new Date().getTime(), + updatedAt: new Date().getTime(), }); - - delete row['linkedBlockIds']; }); - await this.noteBlocksRepository.bulkCreate(rows, ctx, t); - } else if (tableName === textBlocksTable) { - // eslint-disable-next-line no-loop-func - rows.forEach((row) => { - row.linkedBlockIds.forEach((id: string) => { - links.push({ - id: generateId(), - blockId: row.id, - linkedToBlockId: id, - orderPosition: i++, - createdAt: new Date().getTime(), - updatedAt: new Date().getTime(), - }); - }); + delete row['linkedBlockIds']; + }); - delete row['linkedBlockIds']; + await this.noteBlocksRepository.bulkCreate(rows, ctx); + } else if (tableName === textBlocksTable) { + // eslint-disable-next-line no-loop-func + rows.forEach((row) => { + row.linkedBlockIds.forEach((id: string) => { + links.push({ + id: generateId(), + blockId: row.id, + linkedToBlockId: id, + orderPosition: i++, + createdAt: new Date().getTime(), + updatedAt: new Date().getTime(), + }); }); - await this.textBlocksRepository.bulkCreate(rows, ctx, t); - } else if (tableName === blocksScopesTable) { - await this.blocksScopesRepository.bulkCreate(rows, ctx, t); - } + delete row['linkedBlockIds']; + }); + + await this.textBlocksRepository.bulkCreate(rows, ctx); + } else if (tableName === blocksScopesTable) { + await this.blocksScopesRepository.bulkCreate(rows, ctx); } + } - await this.blockLinksRepo.bulkCreate(links, ctx, t); - }); + await this.blockLinksRepo.bulkCreate(links, ctx); } private async secondVersionImport(dump: Dump) { diff --git a/packages/web-core/src/extensions/SyncExtension/synchronizer/OnDbChangeNotifier.ts b/packages/web-core/src/extensions/SyncExtension/synchronizer/OnDbChangeNotifier.ts index 34e14979..f7f6d65c 100644 --- a/packages/web-core/src/extensions/SyncExtension/synchronizer/OnDbChangeNotifier.ts +++ b/packages/web-core/src/extensions/SyncExtension/synchronizer/OnDbChangeNotifier.ts @@ -35,8 +35,6 @@ export class OnDbChangeNotifier { windowId !== this.currentWindowId || source === 'inDbChanges', ); - console.log('New events need to notify', JSON.stringify(evs)); - if (evs.length === 0) return; const groupedMappedData = this.getGroupedMappedData(evs); diff --git a/packages/web-core/src/lib/roamToHarikaJson.test.ts b/packages/web-core/src/lib/roamToHarikaJson.test.ts index 2770a546..c385343e 100644 --- a/packages/web-core/src/lib/roamToHarikaJson.test.ts +++ b/packages/web-core/src/lib/roamToHarikaJson.test.ts @@ -1,435 +1,72 @@ +import { expect } from 'chai'; + import { PageNode, roamToHarikaJson } from './roamToHarikaJson'; -const data = [ - { - 'create-time': 1607495147433, - title: 'December 9th, 2020', - ':create/user': { ':user/uid': 'ovuhJr9o9Qc4Sha7jQrVDBQer0c2' }, - ':log/id': 1607495147432, - children: [ - { - string: '', - uid: 'IbhmsQJSb', - 'create-time': 1607495151820, - 'edit-time': 1607495151820, - ':create/user': { ':user/uid': 'ovuhJr9o9Qc4Sha7jQrVDBQer0c2' }, - ':edit/user': { ':user/uid': 'ovuhJr9o9Qc4Sha7jQrVDBQer0c2' }, - }, - { - string: 'regwe', - uid: 'vhm-iQfXR', - 'create-time': 1607495151065, - 'edit-time': 1607495152422, - ':create/user': { ':user/uid': 'ovuhJr9o9Qc4Sha7jQrVDBQer0c2' }, - ':edit/user': { ':user/uid': 'ovuhJr9o9Qc4Sha7jQrVDBQer0c2' }, - }, - { - string: 'g', - uid: '0VJCALgZy', - 'create-time': 1607495152386, - 'edit-time': 1607495152580, - ':create/user': { ':user/uid': 'ovuhJr9o9Qc4Sha7jQrVDBQer0c2' }, - ':edit/user': { ':user/uid': 'ovuhJr9o9Qc4Sha7jQrVDBQer0c2' }, - }, - { - string: 'wer', - uid: 'ttS8WV-gY', - 'create-time': 1607495152574, - 'edit-time': 1607495152840, - ':create/user': { ':user/uid': 'ovuhJr9o9Qc4Sha7jQrVDBQer0c2' }, - ':edit/user': { ':user/uid': 'ovuhJr9o9Qc4Sha7jQrVDBQer0c2' }, - }, - { - string: 'g', - uid: 'wgGjcWwxF', - 'crate-time': 1607495152795, - 'edit-time': 1607495153031, - ':create/user': { ':user/uid': 'ovuhJr9o9Qc4Sha7jQrVDBQer0c2' }, - ':edit/user': { ':user/uid': 'ovuhJr9o9Qc4Sha7jQrVDBQer0c2' }, - }, - { - string: 'egrw', - 'create-time': 1607495153027, - ':create/user': { ':user/uid': 'ovuhJr9o9Qc4Sha7jQrVDBQer0c2' }, - children: [ - { - string: 'rggerqg', - uid: 'ij-6qNONk', - 'create-time': 1607495153436, - 'edit-time': 1607495154346, - ':create/user': { ':user/uid': 'ovuhJr9o9Qc4Sha7jQrVDBQer0c2' }, - ':edit/user': { ':user/uid': 'ovuhJr9o9Qc4Sha7jQrVDBQer0c2' }, - }, - { - string: 'q', - uid: '28yQhRCmC', - 'create-time': 1607495154342, - 'edit-time': 1607495180884, - ':create/user': { ':user/uid': 'ovuhJr9o9Qc4Sha7jQrVDBQer0c2' }, - ':edit/user': { ':user/uid': 'ovuhJr9o9Qc4Sha7jQrVDBQer0c2' }, - }, - ], - uid: 'bwUJfNNh7', - 'edit-time': 1607495153440, - ':edit/user': { ':user/uid': 'ovuhJr9o9Qc4Sha7jQrVDBQer0c2' }, - }, - ], - uid: '12-09-2020', - 'edit-time': 1607495147434, - ':edit/user': { ':user/uid': 'ovuhJr9o9Qc4Sha7jQrVDBQer0c2' }, - }, - { - 'create-time': 1607668392047, - title: 'December 11th, 2020', - ':create/user': { ':user/uid': 'ovuhJr9o9Qc4Sha7jQrVDBQer0c2' }, - ':log/id': 1607668392047, - children: [ - { - string: "'ver", - uid: 'pBqx-DzCU', - 'create-time': 1607668393077, - 'edit-time': 1607668393831, - ':create/user': { ':user/uid': 'ovuhJr9o9Qc4Sha7jQrVDBQer0c2' }, - ':edit/user': { ':user/uid': 'ovuhJr9o9Qc4Sha7jQrVDBQer0c2' }, - }, - { - string: 'er123333ffffffwefqwffqwefqwefqwefqwef wefq wef w', - uid: 'irRvKFTCO', - 'create-time': 1607668393823, - 'edit-time': 1607669749425, - ':create/user': { ':user/uid': 'ovuhJr9o9Qc4Sha7jQrVDBQer0c2' }, - ':edit/user': { ':user/uid': 'ovuhJr9o9Qc4Sha7jQrVDBQer0c2' }, - }, - { - string: 'qwfeqwfqwef', - uid: 'unrUAQvGv', - 'create-time': 1607668394277, - 'edit-time': 1607669723819, - ':create/user': { ':user/uid': 'ovuhJr9o9Qc4Sha7jQrVDBQer0c2' }, - ':edit/user': { ':user/uid': 'ovuhJr9o9Qc4Sha7jQrVDBQer0c2' }, - }, - { - string: 'erf', - uid: 'K1qemCIoF', - 'create-time': 1607668394099, - 'edit-time': 1607668394434, - ':create/user': { ':user/uid': 'ovuhJr9o9Qc4Sha7jQrVDBQer0c2' }, - ':edit/user': { ':user/uid': 'ovuhJr9o9Qc4Sha7jQrVDBQer0c2' }, - }, - { - string: '', - uid: 'WhMtK6xFD', - 'create-time': 1607669729742, - 'edit-time': 1607669729742, - ':create/user': { ':user/uid': 'ovuhJr9o9Qc4Sha7jQrVDBQer0c2' }, - ':edit/user': { ':user/uid': 'ovuhJr9o9Qc4Sha7jQrVDBQer0c2' }, - }, - { - string: '', - uid: 'j_EIHChnb', - 'create-time': 1607669729941, - 'edit-time': 1607669729941, - ':create/user': { ':user/uid': 'ovuhJr9o9Qc4Sha7jQrVDBQer0c2' }, - ':edit/user': { ':user/uid': 'ovuhJr9o9Qc4Sha7jQrVDBQer0c2' }, - }, - { - string: '', - uid: 'NnEtCqigQ', - 'create-time': 1607669730108, - 'edit-time': 1607669730108, - ':create/user': { ':user/uid': 'ovuhJr9o9Qc4Sha7jQrVDBQer0c2' }, - ':edit/user': { ':user/uid': 'ovuhJr9o9Qc4Sha7jQrVDBQer0c2' }, - }, - { - string: '', - uid: 'eVJADbXef', - 'create-time': 1607669730261, - 'edit-time': 1607669730261, - ':create/user': { ':user/uid': 'ovuhJr9o9Qc4Sha7jQrVDBQer0c2' }, - ':edit/user': { ':user/uid': 'ovuhJr9o9Qc4Sha7jQrVDBQer0c2' }, - }, - { - string: '', - uid: '7JinhQXTY', - 'create-time': 1607669730394, - 'edit-time': 1607669730394, - ':create/user': { ':user/uid': 'ovuhJr9o9Qc4Sha7jQrVDBQer0c2' }, - ':edit/user': { ':user/uid': 'ovuhJr9o9Qc4Sha7jQrVDBQer0c2' }, - }, - { - string: '', - uid: 'PSSn4ffg9', - 'create-time': 1607668394558, - 'edit-time': 1607668394558, - ':create/user': { ':user/uid': 'ovuhJr9o9Qc4Sha7jQrVDBQer0c2' }, - ':edit/user': { ':user/uid': 'ovuhJr9o9Qc4Sha7jQrVDBQer0c2' }, - }, - { - string: 'fe', - uid: 'T1hVMlPB6', - 'create-time': 1607668394430, - 'edit-time': 1607668395028, - ':create/user': { ':user/uid': 'ovuhJr9o9Qc4Sha7jQrVDBQer0c2' }, - ':edit/user': { ':user/uid': 'ovuhJr9o9Qc4Sha7jQrVDBQer0c2' }, - }, - { - 'edit-time': 1607668397808, - children: [ - { - string: 'eqr', - uid: 'XyTYHum8z', - 'create-time': 1607668400386, - 'edit-time': 1607668420199, - ':create/user': { ':user/uid': 'ovuhJr9o9Qc4Sha7jQrVDBQer0c2' }, - ':edit/user': { ':user/uid': 'ovuhJr9o9Qc4Sha7jQrVDBQer0c2' }, - }, - { - string: 'fer', - uid: 'E2G2laPrI', - 'create-time': 1607668420196, - 'edit-time': 1607668420445, - ':create/user': { ':user/uid': 'ovuhJr9o9Qc4Sha7jQrVDBQer0c2' }, - ':edit/user': { ':user/uid': 'ovuhJr9o9Qc4Sha7jQrVDBQer0c2' }, - }, - { - string: 'fwgwe', - uid: 'HjU5fbr7P', - 'create-time': 1607668420442, - 'edit-time': 1607668420847, - ':create/user': { ':user/uid': 'ovuhJr9o9Qc4Sha7jQrVDBQer0c2' }, - ':edit/user': { ':user/uid': 'ovuhJr9o9Qc4Sha7jQrVDBQer0c2' }, - }, - { - string: 'g', - uid: 'yJWnEXOC8', - 'create-time': 1607668420802, - 'edit-time': 1607668421614, - ':create/user': { ':user/uid': 'ovuhJr9o9Qc4Sha7jQrVDBQer0c2' }, - ':edit/user': { ':user/uid': 'ovuhJr9o9Qc4Sha7jQrVDBQer0c2' }, - }, - { - string: 'er', - uid: 'OFv-Hlo9G', - 'create-time': 1607668425798, - 'edit-time': 1607668426171, - ':create/user': { ':user/uid': 'ovuhJr9o9Qc4Sha7jQrVDBQer0c2' }, - ':edit/user': { ':user/uid': 'ovuhJr9o9Qc4Sha7jQrVDBQer0c2' }, - }, - { - string: 'fwe', - uid: 'oevlYT4Pl', - 'create-time': 1607668426168, - 'edit-time': 1607668426429, - ':create/user': { ':user/uid': 'ovuhJr9o9Qc4Sha7jQrVDBQer0c2' }, - ':edit/user': { ':user/uid': 'ovuhJr9o9Qc4Sha7jQrVDBQer0c2' }, - }, - { - string: 'g', - uid: 'm_A25M7Qw', - 'create-time': 1607668426426, - 'edit-time': 1607668427162, - ':create/user': { ':user/uid': 'ovuhJr9o9Qc4Sha7jQrVDBQer0c2' }, - ':edit/user': { ':user/uid': 'ovuhJr9o9Qc4Sha7jQrVDBQer0c2' }, - }, - { - 'edit-time': 1607668430396, - children: [ - { - string: 'werf', - uid: 'idahzehV5', - 'create-time': 1607668431875, - 'edit-time': 1607668433055, - ':create/user': { ':user/uid': 'ovuhJr9o9Qc4Sha7jQrVDBQer0c2' }, - ':edit/user': { ':user/uid': 'ovuhJr9o9Qc4Sha7jQrVDBQer0c2' }, - }, - { - string: '[[123testnnnfewqfwefwefwefnnnn]]', - 'create-time': 1607668459695, - ':block/refs': [{ ':block/uid': 'tzMqTUmji' }], - refs: [{ uid: 'tzMqTUmji' }], - ':create/user': { ':user/uid': 'ovuhJr9o9Qc4Sha7jQrVDBQer0c2' }, - uid: 'nMPiheLfp', - 'edit-time': 1607669776560, - ':edit/user': { ':user/uid': 'ovuhJr9o9Qc4Sha7jQrVDBQer0c2' }, - }, - ], - refs: [{ uid: '12-09-2020' }], - uid: 'Qd4hFI40g', - ':block/refs': [{ ':block/uid': '12-09-2020' }], - string: '[[December 9th, 2020]]', - ':create/user': { ':user/uid': 'ovuhJr9o9Qc4Sha7jQrVDBQer0c2' }, - 'create-time': 1607668427172, - ':edit/user': { ':user/uid': 'ovuhJr9o9Qc4Sha7jQrVDBQer0c2' }, - }, - ], - refs: [{ uid: '12-09-2020' }], - uid: 'zpIUsjcNv', - ':block/refs': [{ ':block/uid': '12-09-2020' }], - string: '[[December 9th, 2020]]', - ':create/user': { ':user/uid': 'ovuhJr9o9Qc4Sha7jQrVDBQer0c2' }, - 'create-time': 1607668395024, - ':edit/user': { ':user/uid': 'ovuhJr9o9Qc4Sha7jQrVDBQer0c2' }, - }, - ], - uid: '12-11-2020', - 'edit-time': 1607668392048, - ':edit/user': { ':user/uid': 'ovuhJr9o9Qc4Sha7jQrVDBQer0c2' }, - }, - { - 'create-time': 1607668461636, - title: '123testnnnfewqfwefwefwefnnnn', - ':create/user': { ':user/uid': 'ovuhJr9o9Qc4Sha7jQrVDBQer0c2' }, - children: [ - { - string: 'ferfffffewfqw', - uid: 'fi4wTQw9z', - 'create-time': 1607668470372, - 'edit-time': 1607669769997, - ':create/user': { ':user/uid': 'ovuhJr9o9Qc4Sha7jQrVDBQer0c2' }, - ':edit/user': { ':user/uid': 'ovuhJr9o9Qc4Sha7jQrVDBQer0c2' }, - }, - { - 'edit-time': 1607668467763, - children: [ - { - string: 'ferfeqf', - uid: 'dSbQo-dC8', - 'create-time': 1607668472146, - 'edit-time': 1607668473717, - ':create/user': { ':user/uid': 'ovuhJr9o9Qc4Sha7jQrVDBQer0c2' }, - ':edit/user': { ':user/uid': 'ovuhJr9o9Qc4Sha7jQrVDBQer0c2' }, +describe('roamToHarikaJson', () => { + const dailyNoteData = [ + { + 'create-time': 1607668461636, + title: 'test title', + ':create/user': { ':user/uid': 'ovuhJr9o9Qc4Sha7jQrVDBQer0c2' }, + uid: 'nMPiheLfp', + 'edit-time': 1607669776560, + children: [ + { + string: '[[September 23rd, 2021]] [[September 23rd, 2021]] hey!', + 'create-time': 1608721845828, + ':block/refs': [{ ':block/uid': '09-23-2021' }], + refs: [{ uid: '09-23-2021' }], + ':create/user': { + ':user/uid': 'ovuhJr9o9Qc4Sha7jQrVDBQer0c2', }, - ], - refs: [{ uid: '12-09-2020' }], - uid: 'oa5W2FU9W', - ':block/refs': [{ ':block/uid': '12-09-2020' }], - string: '[[December 9th, 2020]]', - ':create/user': { ':user/uid': 'ovuhJr9o9Qc4Sha7jQrVDBQer0c2' }, - 'create-time': 1607668464740, - ':edit/user': { ':user/uid': 'ovuhJr9o9Qc4Sha7jQrVDBQer0c2' }, - }, - ], - uid: 'tzMqTUmji', - 'edit-time': 1607669776560, - ':edit/user': { ':user/uid': 'ovuhJr9o9Qc4Sha7jQrVDBQer0c2' }, - }, - { - 'create-time': 1608721839944, - title: 'December 23rd, 2020', - ':create/user': { ':user/uid': 'ovuhJr9o9Qc4Sha7jQrVDBQer0c2' }, - ':log/id': 1608721839944, - children: [ - { - string: 'f', - uid: 'FqjWDwH9c', - 'create-time': 1608721843148, - 'edit-time': 1608721844048, - ':create/user': { ':user/uid': 'ovuhJr9o9Qc4Sha7jQrVDBQer0c2' }, - ':edit/user': { ':user/uid': 'ovuhJr9o9Qc4Sha7jQrVDBQer0c2' }, - }, - { - string: 'erf', - 'create-time': 1608721844039, - ':create/user': { ':user/uid': 'ovuhJr9o9Qc4Sha7jQrVDBQer0c2' }, - children: [ - { - string: 'ff', - 'create-time': 1608721844435, - ':create/user': { ':user/uid': 'ovuhJr9o9Qc4Sha7jQrVDBQer0c2' }, - children: [ - { - string: 'f', - 'create-time': 1608721844828, - ':create/user': { ':user/uid': 'ovuhJr9o9Qc4Sha7jQrVDBQer0c2' }, - children: [ - { - string: 'wf', - uid: 'iblb8wtRt', - 'create-time': 1608721845060, - 'edit-time': 1608721845367, - ':create/user': { - ':user/uid': 'ovuhJr9o9Qc4Sha7jQrVDBQer0c2', - }, - ':edit/user': { - ':user/uid': 'ovuhJr9o9Qc4Sha7jQrVDBQer0c2', - }, - }, - { - string: 'wef', - 'create-time': 1608721845361, - ':create/user': { - ':user/uid': 'ovuhJr9o9Qc4Sha7jQrVDBQer0c2', - }, - children: [ - { - string: '[[December 23rd, 2020]] ', - 'create-time': 1608721845828, - ':block/refs': [{ ':block/uid': '12-23-2020' }], - refs: [{ uid: '12-23-2020' }], - ':create/user': { - ':user/uid': 'ovuhJr9o9Qc4Sha7jQrVDBQer0c2', - }, - uid: 'Glefk_NSB', - 'edit-time': 1608721860004, - ':edit/user': { - ':user/uid': 'ovuhJr9o9Qc4Sha7jQrVDBQer0c2', - }, - }, - ], - uid: 'XUbK5CcXc', - 'edit-time': 1608721845836, - ':edit/user': { - ':user/uid': 'ovuhJr9o9Qc4Sha7jQrVDBQer0c2', - }, - }, - ], - uid: 'jyEq32VZB', - 'edit-time': 1608721845068, - ':edit/user': { ':user/uid': 'ovuhJr9o9Qc4Sha7jQrVDBQer0c2' }, - }, - ], - uid: 'u5-VOX7fc', - 'edit-time': 1608721844836, - ':edit/user': { ':user/uid': 'ovuhJr9o9Qc4Sha7jQrVDBQer0c2' }, + uid: 'Glefk_NSB', + 'edit-time': 1608721860004, + ':edit/user': { + ':user/uid': 'ovuhJr9o9Qc4Sha7jQrVDBQer0c2', }, - ], - uid: 'NkNhJs3UN', - 'edit-time': 1608721844441, - ':edit/user': { ':user/uid': 'ovuhJr9o9Qc4Sha7jQrVDBQer0c2' }, - }, - ], - uid: '12-23-2020', - 'edit-time': 1608721839944, - ':edit/user': { ':user/uid': 'ovuhJr9o9Qc4Sha7jQrVDBQer0c2' }, - }, - { - 'create-time': 1608721857068, - title: 'December 23rd, 200', - ':create/user': { ':user/uid': 'ovuhJr9o9Qc4Sha7jQrVDBQer0c2' }, - ':log/id': -55825034152000, - uid: '12-23-200', - 'edit-time': 1608721857069, - ':edit/user': { ':user/uid': 'ovuhJr9o9Qc4Sha7jQrVDBQer0c2' }, - }, - { - uid: 'OMRalqVh4', - 'create-time': 1632424770974, - 'edit-time': 1632424770974, - title: 'Sergey', - }, - { - 'create-time': 1632424770976, - title: 'September 23rd, 2021', - ':create/user': { ':user/uid': 'ovuhJr9o9Qc4Sha7jQrVDBQer0c2' }, - ':log/id': 1632424770976, - uid: '09-23-2021', - 'edit-time': 1632424770976, - ':edit/user': { ':user/uid': 'ovuhJr9o9Qc4Sha7jQrVDBQer0c2' }, - }, -] as PageNode[]; + }, + ], + }, + { + 'create-time': 1632424770976, + title: 'September 23rd, 2021', + uid: '09-23-2021', + 'edit-time': 1632424770976, + }, + ] as PageNode[]; -describe('roamToHarikaJson', () => { - it('works', () => { - console.log(roamToHarikaJson(data)); + it('keeps none daily note title', () => { + const res = roamToHarikaJson(dailyNoteData).data[0].rows; + + expect(res[0]).to.include({ + title: 'test title', + }); + }); + + it('fixes daily notes', () => { + const res = roamToHarikaJson(dailyNoteData).data[0].rows; + + expect(res[1]).to.include({ + title: '23 Sep 2021', + dailyNoteDate: 1632344400000, + createdAt: 1632424770976, + updatedAt: 1632424770976, + }); + // Check that new id was generated + expect(res[1].id.length).to.eq(20); }); + + it('replaces daily notes', () => { + const res = roamToHarikaJson(dailyNoteData).data[1].rows; + + expect(res[0]).to.include({ + content: '[[23 Sep 2021]] [[23 Sep 2021]] hey!', + createdAt: 1608721845828, + updatedAt: 1608721860004, + }); + + expect(res[0].id.length).to.eq(20); + }); + + // TODO: case with nested refs in note title }); diff --git a/packages/web-core/src/lib/roamToHarikaJson.ts b/packages/web-core/src/lib/roamToHarikaJson.ts index 853c9f4b..8d32c999 100644 --- a/packages/web-core/src/lib/roamToHarikaJson.ts +++ b/packages/web-core/src/lib/roamToHarikaJson.ts @@ -1,3 +1,6 @@ +import dayjs from 'dayjs'; +import { groupBy, uniqBy } from 'lodash-es'; + import { BlockLinkDoc, BlockLinkRow, @@ -12,10 +15,11 @@ import { textBlocksTable, } from '../apps/VaultApplication/BlocksExtension/repositories/TextBlocksRepository'; import { Dump } from '../apps/VaultApplication/BlocksExtension/services/ImportExportService'; +import { generateId } from './generateId'; type ICommon = { - 'create-time': number; - 'edit-time': number; + 'create-time'?: number; + 'edit-time'?: number; uid: string; refs?: { uid: string }[]; }; @@ -28,14 +32,40 @@ type TextNode = ICommon & { children?: TextNode[]; }; -export const roamToHarikaJson = (data: PageNode[]): Dump => { - const links: BlockLinkRow[] = []; - const textDocs: TextBlockDoc[] = []; - const noteDocs: NoteBlockDoc[] = []; +type Node = PageNode | TextNode; - for (const page of data) { - gatherData(page, noteDocs, textDocs, links); - } +export const roamToHarikaJson = (data: PageNode[]) => { + const links: BlockLinkDoc[] = []; + let textDocs: TextBlockDoc[] = []; + let noteDocs: NoteBlockDoc[] = []; + + const oldIdToNewIdMap: Record = {}; + const newTitleToOldTitleMap: Record = {}; + + data.forEach((page, i) => { + gatherData( + undefined, + page, + i, + noteDocs, + textDocs, + links, + oldIdToNewIdMap, + newTitleToOldTitleMap, + ); + }); + + replaceTitlesAndBlockIds( + links, + textDocs, + noteDocs, + oldIdToNewIdMap, + newTitleToOldTitleMap, + ); + + // TODO: log which blocks were discarded + noteDocs = uniqBy(noteDocs, (d) => d.id); + textDocs = uniqBy(textDocs, (d) => d.id); return { version: 2, @@ -52,13 +82,130 @@ export const roamToHarikaJson = (data: PageNode[]): Dump => { tableName: blockLinksTable, rows: links, }, - ], + ] as const, }; }; const gatherData = ( - currentNode: PageNode, + parent: Node | undefined, + currentNode: Node, + orderPosition: number, noteDocs: NoteBlockDoc[], textBlocks: TextBlockDoc[], links: BlockLinkDoc[], -) => {}; + oldIdToNewIdMap: Record, + newTitleToOldTitleMap: Record, +) => { + const getOrCreateNewId = (currentId: string) => { + const newId = oldIdToNewIdMap[currentId] || generateId(); + + oldIdToNewIdMap[currentId] = newId; + + return newId; + }; + + const newId = getOrCreateNewId(currentNode.uid); + + currentNode.refs?.forEach((ref) => { + links.push({ + id: generateId(), + blockId: newId, + linkedToBlockId: getOrCreateNewId(ref.uid), + orderPosition: + Number.MAX_SAFE_INTEGER - + (currentNode['create-time'] || new Date().getTime()), + createdAt: currentNode['create-time'] || new Date().getTime(), + updatedAt: currentNode['create-time'] || new Date().getTime(), + }); + }); + + (currentNode.children || []).forEach((child, i) => { + gatherData( + currentNode, + child, + i, + noteDocs, + textBlocks, + links, + oldIdToNewIdMap, + newTitleToOldTitleMap, + ); + }); + + const parentId = parent?.uid ? getOrCreateNewId(parent?.uid) : undefined; + + if ('title' in currentNode) { + const time = new Date(currentNode.uid).getTime(); + const newTitle = time && dayjs(time).format('D MMM YYYY'); + + if (newTitle) { + newTitleToOldTitleMap[newTitle] = currentNode.title; + } + + noteDocs.push({ + id: newId, + title: currentNode.title, + ...(!time || !newTitle + ? { dailyNoteDate: null } + : { + dailyNoteDate: time, + title: dayjs(time).format('D MMM YYYY'), + }), + type: 'noteBlock', + parentId, + orderPosition, + createdAt: currentNode['create-time'] || new Date().getTime(), + updatedAt: currentNode['edit-time'] || new Date().getTime(), + }); + } else { + textBlocks.push({ + id: newId, + content: currentNode.string, + type: 'textBlock', + parentId, + orderPosition, + createdAt: currentNode['create-time'] || new Date().getTime(), + updatedAt: currentNode['edit-time'] || new Date().getTime(), + }); + } +}; + +const replaceTitlesAndBlockIds = ( + links: BlockLinkDoc[], + textDocs: TextBlockDoc[], + noteDocs: NoteBlockDoc[], + oldIdToNewIdMap: Record, + newTitleToOldTitleMap: Record, +) => { + const textDocsMap: Record = Object.fromEntries( + textDocs.map((d) => [d.id, d]), + ); + const noteDocsMap: Record = Object.fromEntries( + noteDocs.map((d) => [d.id, d]), + ); + + const newIdToOldIdMap = Object.fromEntries( + Object.entries(oldIdToNewIdMap).map(([k, v]) => [v, k]), + ); + + links.forEach(({ linkedToBlockId, blockId }) => { + const textDoc = textDocsMap[blockId]; + + if (textDoc) { + const linkedToNoteDoc = noteDocsMap[linkedToBlockId]; + + if (linkedToNoteDoc) { + if (newTitleToOldTitleMap[linkedToNoteDoc.title]) { + textDoc.content = textDoc.content.replaceAll( + `[[${newTitleToOldTitleMap[linkedToNoteDoc.title]}]]`, + `[[${linkedToNoteDoc.title}]]`, + ); + } + } else { + const oldId = newIdToOldIdMap[linkedToBlockId]; + + textDoc.content.replaceAll(`((${oldId}))`, `((${linkedToBlockId}))`); + } + } + }); +};