diff --git a/src/json-crdt-extensions/peritext/__tests__/Peritext.render-block.spec.ts b/src/json-crdt-extensions/peritext/__tests__/Peritext.render-block.spec.ts index 92bca487d1..75e6617f6a 100644 --- a/src/json-crdt-extensions/peritext/__tests__/Peritext.render-block.spec.ts +++ b/src/json-crdt-extensions/peritext/__tests__/Peritext.render-block.spec.ts @@ -47,8 +47,6 @@ const runInlineSlicesTests = ( editor.saved.insMarker(['p'], {foo: 'bar'}); expect(view()).toMatchInlineSnapshot(` "<> - <0> - "" { }

{ foo = "bar" } "abcdefghijklmnopqrstuvwxyz" { } " diff --git a/src/json-crdt-extensions/peritext/block/Fragment.ts b/src/json-crdt-extensions/peritext/block/Fragment.ts index e8e231e53f..414358c7c3 100644 --- a/src/json-crdt-extensions/peritext/block/Fragment.ts +++ b/src/json-crdt-extensions/peritext/block/Fragment.ts @@ -80,6 +80,8 @@ export class Fragment extends Range implements Printable, Stateful { let pair: ReturnType; while ((pair = iterator())) { const [p1, p2] = pair; + const skipFirstVirtualBlock = !p1 && this.start.isAbsStart() && p2 && p2.viewPos() === 0; + if (skipFirstVirtualBlock) continue; const type = p1 ? p1.type() : CommonSliceType.p; const path = type instanceof Array ? type : [type]; const block = this.insertBlock(parent, path, p1, p2); diff --git a/src/json-crdt-extensions/peritext/block/types.ts b/src/json-crdt-extensions/peritext/block/types.ts index f848316418..f5d96e1736 100644 --- a/src/json-crdt-extensions/peritext/block/types.ts +++ b/src/json-crdt-extensions/peritext/block/types.ts @@ -1,10 +1,15 @@ +import type {SliceBehavior} from '../slice/constants'; + export type PeritextMlNode = string | PeritextMlElement; -export type PeritextMlElement = [ - tag: string | number, - attrs: null | PeritextMlAttributes, + +export type PeritextMlElement = [ + tag: Tag, + attrs: null | PeritextMlAttributes, ...children: PeritextMlNode[], ]; -export interface PeritextMlAttributes { - inline?: boolean; - data?: unknown; + +export interface PeritextMlAttributes { + data?: Data; + inline?: Inline; + behavior?: SliceBehavior; } diff --git a/src/json-crdt-extensions/peritext/lazy/__tests__/import-export-html.spec.ts b/src/json-crdt-extensions/peritext/lazy/__tests__/import-export-html.spec.ts new file mode 100644 index 0000000000..e03afe6eda --- /dev/null +++ b/src/json-crdt-extensions/peritext/lazy/__tests__/import-export-html.spec.ts @@ -0,0 +1,14 @@ +import {setupKit} from '../../__tests__/setup'; +import {CommonSliceType} from '../../slice'; +import {fromHtml, toViewRange} from '../import-html'; + +test('a single paragraph', () => { + const {peritext} = setupKit(); + const html = '

Hello world

'; + const peritextMl = fromHtml(html); + const rangeView = toViewRange(peritextMl); + peritext.editor.import(0, rangeView); + peritext.refresh(); + const json = peritext.blocks.toJson(); + expect(json).toEqual(['', null, [CommonSliceType.p, null, 'Hello world']]); +}); diff --git a/src/json-crdt-extensions/peritext/lazy/__tests__/import-html.spec.ts b/src/json-crdt-extensions/peritext/lazy/__tests__/import-html.spec.ts new file mode 100644 index 0000000000..29ca73b29f --- /dev/null +++ b/src/json-crdt-extensions/peritext/lazy/__tests__/import-html.spec.ts @@ -0,0 +1,100 @@ +import {Anchor} from '../../rga/constants'; +import {CommonSliceType} from '../../slice'; +import {SliceBehavior, SliceHeaderShift} from '../../slice/constants'; +import {fromHtml, toViewRange} from '../import-html'; + +describe('.fromHtml()', () => { + test('a single paragraph', () => { + const html = '

Hello world

'; + const peritextMl = fromHtml(html); + expect(peritextMl).toEqual(['', null, [CommonSliceType.p, null, 'Hello world']]); + }); + + test('a paragraph with trailing text', () => { + const html = '

Hello world

more text'; + const peritextMl = fromHtml(html); + expect(peritextMl).toEqual(['', null, [CommonSliceType.p, null, 'Hello world'], ' more text']); + }); + + test('text formatted as italic', () => { + const html = '

Hello world

\n

italic text, more italic

'; + const peritextMl = fromHtml(html); + expect(peritextMl).toEqual([ + '', + null, + [CommonSliceType.p, null, 'Hello world'], + '\n', + [ + CommonSliceType.p, + null, + [CommonSliceType.i, {behavior: SliceBehavior.One, inline: true}, 'italic'], + ' text, ', + [CommonSliceType.i, {behavior: SliceBehavior.One, inline: true}, 'more italic'], + ], + ]); + }); +}); + +describe('.toViewRange()', () => { + test('plain text', () => { + const html = 'this is plain text'; + const peritextMl = fromHtml(html); + const view = toViewRange(peritextMl); + expect(view).toEqual(['this is plain text', 0, []]); + }); + + test('a single paragraph', () => { + const html = '

Hello world

'; + const peritextMl = fromHtml(html); + const view = toViewRange(peritextMl); + expect(view).toEqual(['\nHello world', 0, [[0, 0, 0, 0]]]); + }); + + test('two consecutive paragraphs', () => { + const html = '

Hello world

Goodbye world

'; + const peritextMl = fromHtml(html); + const view = toViewRange(peritextMl); + expect(view).toEqual([ + '\nHello world\nGoodbye world', + 0, + [ + [0, 0, 0, 0], + [0, 12, 12, 0], + ], + ]); + }); + + test('two paragraphs with whitespace gap', () => { + const html = '

Hello world

\n

Goodbye world

'; + const peritextMl = fromHtml(html); + const view = toViewRange(peritextMl); + expect(view).toEqual([ + '\nHello world\nGoodbye world', + 0, + [ + [0, 0, 0, 0], + [0, 12, 12, 0], + ], + ]); + }); + + test('single inline annotation', () => { + const html = 'here is some italic text'; + const peritextMl = fromHtml(html); + const view = toViewRange(peritextMl); + expect(view).toEqual([ + 'here is some italic text', + 0, + [ + [ + (SliceBehavior.One << SliceHeaderShift.Behavior) + + (Anchor.Before << SliceHeaderShift.X1Anchor) + + (Anchor.After << SliceHeaderShift.X2Anchor), + 13, + 19, + CommonSliceType.i, + ], + ], + ]); + }); +}); diff --git a/src/json-crdt-extensions/peritext/lazy/export-markdown.ts b/src/json-crdt-extensions/peritext/lazy/export-markdown.ts index e9b5aa78f3..6f8d2f3637 100644 --- a/src/json-crdt-extensions/peritext/lazy/export-markdown.ts +++ b/src/json-crdt-extensions/peritext/lazy/export-markdown.ts @@ -6,9 +6,7 @@ import type {PeritextMlNode} from '../block/types'; export const toMdast = (json: PeritextMlNode): IRoot => { const hast = toHast(json); - // console.log(hast); const mdast = _toMdast(hast) as IRoot; - // console.log(mdast); return mdast; }; diff --git a/src/json-crdt-extensions/peritext/lazy/import-html.ts b/src/json-crdt-extensions/peritext/lazy/import-html.ts new file mode 100644 index 0000000000..b29be1cd45 --- /dev/null +++ b/src/json-crdt-extensions/peritext/lazy/import-html.ts @@ -0,0 +1,117 @@ +import {html as _html} from 'very-small-parser/lib/html'; +import {fromHast as _fromHast} from 'very-small-parser/lib/html/json-ml/fromHast'; +import {SliceTypeName} from '../slice'; +import {registry as defaultRegistry} from '../registry/registry'; +import {SliceBehavior, SliceHeaderShift} from '../slice/constants'; +import {Anchor} from '../rga/constants'; +import type {JsonMlNode} from 'very-small-parser/lib/html/json-ml/types'; +import type {THtmlToken} from 'very-small-parser/lib/html/types'; +import type {PeritextMlNode} from '../block/types'; +import type {SliceRegistry} from '../registry/SliceRegistry'; +import type {ViewRange, ViewSlice} from '../editor/types'; + +/** + * Flattens a {@link PeritextMlNode} tree structure into a {@link ViewRange} + * flat string with annotation ranges. + */ +class ViewRangeBuilder { + private text = ''; + private slices: ViewSlice[] = []; + + constructor(private registry: SliceRegistry) {} + + private build0(node: PeritextMlNode, depth = 0): void { + const skipWhitespace = depth < 2; + if (typeof node === 'string') { + if (skipWhitespace && !node.trim()) return; + this.text += node; + return; + } + const [type, attr] = node; + const start = this.text.length; + const length = node.length; + const inline = !!attr?.inline; + if (!!type || type === 0) { + let end: number = 0, + header: number = 0; + if (!inline) { + this.text += '\n'; + end = start; + header = + (SliceBehavior.Marker << SliceHeaderShift.Behavior) + + (Anchor.Before << SliceHeaderShift.X1Anchor) + + (Anchor.Before << SliceHeaderShift.X2Anchor); + const slice: ViewSlice = [header, start, end, type]; + const data = attr?.data; + if (data) slice.push(data); + this.slices.push(slice); + } + } + for (let i = 2; i < length; i++) this.build0(node[i] as PeritextMlNode, depth + 1); + if (!!type || type === 0) { + let end: number = 0, + header: number = 0; + if (inline) { + end = this.text.length; + const behavior: SliceBehavior = attr?.behavior ?? SliceBehavior.Many; + header = + (behavior << SliceHeaderShift.Behavior) + + (Anchor.Before << SliceHeaderShift.X1Anchor) + + (Anchor.After << SliceHeaderShift.X2Anchor); + const slice: ViewSlice = [header, start, end, type]; + const data = attr?.data; + if (data) slice.push(data); + this.slices.push(slice); + } + } + } + + public build(node: PeritextMlNode): ViewRange { + this.build0(node); + const view: ViewRange = [this.text, 0, this.slices]; + return view; + } +} + +export const toViewRange = (node: PeritextMlNode, registry: SliceRegistry = defaultRegistry): ViewRange => + new ViewRangeBuilder(registry).build(node); + +export const fromJsonMl = (jsonml: JsonMlNode, registry: SliceRegistry = defaultRegistry): PeritextMlNode => { + if (typeof jsonml === 'string') return jsonml; + const tag = jsonml[0]; + const length = jsonml.length; + const node: PeritextMlNode = [tag, null]; + for (let i = 2; i < length; i++) node.push(fromJsonMl(jsonml[i] as JsonMlNode, registry)); + const res = registry.fromHtml(jsonml); + if (res) { + node[0] = res[0]; + node[1] = res[1]; + } else { + node[0] = SliceTypeName[tag as any] ?? tag; + const attr = jsonml[1] || {}; + let data = null; + if (attr['data-attr'] !== void 0) { + try { + data = JSON.parse(attr['data-attr']); + } catch {} + } + const inline = attr['data-inline'] === 'true'; + if (data || inline) node[1] = {data, inline}; + } + if (typeof node[0] === 'number' && node[0] < 0) { + const attr = node[1] || {}; + attr.inline = true; + node[1] = attr; + } + return node; +}; + +export const fromHast = (hast: THtmlToken, registry?: SliceRegistry): PeritextMlNode => { + const jsonml = _fromHast(hast); + return fromJsonMl(jsonml, registry); +}; + +export const fromHtml = (html: string, registry?: SliceRegistry): PeritextMlNode => { + const hast = _html.parsef(html); + return fromHast(hast, registry); +}; diff --git a/src/json-crdt-extensions/peritext/lazy/import-markdown.ts b/src/json-crdt-extensions/peritext/lazy/import-markdown.ts new file mode 100644 index 0000000000..2fce411e90 --- /dev/null +++ b/src/json-crdt-extensions/peritext/lazy/import-markdown.ts @@ -0,0 +1,19 @@ +import {toHast} from 'very-small-parser/lib/markdown/block/toHast'; +import {block} from 'very-small-parser/lib/markdown/block'; +import {fromHast as _fromHast} from 'very-small-parser/lib/html/json-ml/fromHast'; +import {registry as defaultRegistry} from '../registry/registry'; +import {fromHast} from './import-html'; +import type {IRoot} from 'very-small-parser/lib/markdown/block/types'; +import type {PeritextMlNode} from '../block/types'; +import type {SliceRegistry} from '../registry/SliceRegistry'; + +export const fromMdast = (mdast: IRoot, registry: SliceRegistry = defaultRegistry): PeritextMlNode => { + const hast = toHast(mdast); + const node = fromHast(hast, registry); + return node; +}; + +export const fromMarkdown = (markdown: string, registry?: SliceRegistry): PeritextMlNode => { + const mdast = block.parsef(markdown); + return fromMdast(mdast, registry); +}; diff --git a/src/json-crdt-extensions/peritext/registry/SliceRegistry.ts b/src/json-crdt-extensions/peritext/registry/SliceRegistry.ts new file mode 100644 index 0000000000..0205b06f4f --- /dev/null +++ b/src/json-crdt-extensions/peritext/registry/SliceRegistry.ts @@ -0,0 +1,61 @@ +import type {PeritextMlElement} from '../block/types'; +import type {NodeBuilder} from '../../../json-crdt-patch'; +import {SliceBehavior} from '../slice/constants'; +import type {JsonMlElement} from 'very-small-parser/lib/html/json-ml/types'; +import type {FromHtmlConverter, SliceTypeDefinition, ToHtmlConverter} from './types'; + +export class SliceRegistry { + private map: Map> = new Map(); + private toHtmlMap: Map> = new Map(); + private fromHtmlMap: Map, converter: FromHtmlConverter][]> = + new Map(); + + public add( + def: SliceTypeDefinition, + ): void { + const {type, toHtml, fromHtml} = def; + this.map.set(type, def); + if (toHtml) this.toHtmlMap.set(type, toHtml); + if (fromHtml) { + const fromHtmlMap = this.fromHtmlMap; + for (const htmlTag in fromHtml) { + const converter = fromHtml[htmlTag]; + const converters = fromHtmlMap.get(htmlTag) ?? []; + converters.push([def, converter]); + fromHtmlMap.set(htmlTag, converters); + } + } + } + + public def( + type: Type, + schema: Schema, + behavior: SliceBehavior, + inline: boolean, + rest: Omit, 'type' | 'schema' | 'behavior' | 'inline'> = {}, + ): void { + this.add({type, schema, behavior, inline, ...rest}); + } + + public toHtml(el: PeritextMlElement): ReturnType> | undefined { + const converter = this.toHtmlMap.get(el[0]); + return converter ? converter(el) : undefined; + } + + public fromHtml(el: JsonMlElement): PeritextMlElement | undefined { + const tag = el[0] + ''; + const converters = this.fromHtmlMap.get(tag); + if (converters) { + for (const [def, converter] of converters) { + const result = converter(el); + if (result) { + const attr = result[1] ?? (result[1] = {}); + attr.inline = def.inline ?? def.type < 0; + attr.behavior = !attr.inline ? SliceBehavior.Marker : (def.behavior ?? SliceBehavior.Many); + return result; + } + } + } + return; + } +} diff --git a/src/json-crdt-extensions/peritext/registry/registry.ts b/src/json-crdt-extensions/peritext/registry/registry.ts new file mode 100644 index 0000000000..6bf610712d --- /dev/null +++ b/src/json-crdt-extensions/peritext/registry/registry.ts @@ -0,0 +1,73 @@ +import {s} from '../../../json-crdt-patch'; +import type {JsonNodeView} from '../../../json-crdt/nodes'; +import type {SchemaToJsonNode} from '../../../json-crdt/schema/types'; +import {CommonSliceType} from '../slice'; +import {SliceBehavior} from '../slice/constants'; +import {SliceRegistry} from './SliceRegistry'; + +const undefSchema = s.con(undefined); + +/** + * Default annotation type registry. + */ +export const registry = new SliceRegistry(); + +registry.def(CommonSliceType.i, undefSchema, SliceBehavior.One, true, { + fromHtml: { + i: () => [CommonSliceType.i, null], + em: () => [CommonSliceType.i, null], + }, +}); + +registry.def(CommonSliceType.b, undefSchema, SliceBehavior.One, true, { + fromHtml: { + b: () => [CommonSliceType.b, null], + strong: () => [CommonSliceType.b, null], + }, +}); + +registry.def(CommonSliceType.s, undefSchema, SliceBehavior.One, true, { + fromHtml: { + s: () => [CommonSliceType.s, null], + strike: () => [CommonSliceType.s, null], + }, +}); + +registry.def(CommonSliceType.u, undefSchema, SliceBehavior.One, true); +registry.def(CommonSliceType.code, undefSchema, SliceBehavior.One, true); +registry.def(CommonSliceType.mark, undefSchema, SliceBehavior.One, true); +registry.def(CommonSliceType.kbd, undefSchema, SliceBehavior.One, true); +registry.def(CommonSliceType.del, undefSchema, SliceBehavior.One, true); +registry.def(CommonSliceType.ins, undefSchema, SliceBehavior.One, true); +registry.def(CommonSliceType.sup, undefSchema, SliceBehavior.One, true); +registry.def(CommonSliceType.sub, undefSchema, SliceBehavior.One, true); +registry.def(CommonSliceType.math, undefSchema, SliceBehavior.One, true); + +const aSchema = s.obj({ + href: s.str(''), + title: s.str(''), +}); +registry.def(CommonSliceType.a, aSchema, SliceBehavior.Many, true, { + fromHtml: { + a: (jsonml) => { + const attr = jsonml[1] || {}; + const data: JsonNodeView> = { + href: attr.href ?? '', + title: attr.title ?? '', + }; + return [CommonSliceType.a, {data, inline: true}]; + }, + }, +}); + +// TODO: add more default annotations +// comment = SliceTypeCon.comment, +// font = SliceTypeCon.font, +// col = SliceTypeCon.col, +// bg = SliceTypeCon.bg, +// hidden = SliceTypeCon.hidden, +// footnote = SliceTypeCon.footnote, +// ref = SliceTypeCon.ref, +// iaside = SliceTypeCon.iaside, +// iembed = SliceTypeCon.iembed, +// bookmark = SliceTypeCon.bookmark, diff --git a/src/json-crdt-extensions/peritext/registry/types.ts b/src/json-crdt-extensions/peritext/registry/types.ts new file mode 100644 index 0000000000..1dd486fc31 --- /dev/null +++ b/src/json-crdt-extensions/peritext/registry/types.ts @@ -0,0 +1,29 @@ +import type {NodeBuilder} from '../../../json-crdt-patch'; +import type {JsonNodeView} from '../../../json-crdt/nodes'; +import type {SchemaToJsonNode} from '../../../json-crdt/schema/types'; +import type {PeritextMlElement} from '../block/types'; +import type {JsonMlElement} from 'very-small-parser/lib/html/json-ml/types'; +import type {SliceBehavior} from '../slice/constants'; + +export interface SliceTypeDefinition< + Type extends number | string = number | string, + Schema extends NodeBuilder = NodeBuilder, + Inline extends boolean = true, +> { + type: Type; + schema: Schema; + behavior?: SliceBehavior; + inline?: boolean; + toHtml?: ToHtmlConverter>, Inline>>; + fromHtml?: { + [htmlTag: string]: FromHtmlConverter>, Inline>>; + }; +} + +export type ToHtmlConverter< + El extends PeritextMlElement = PeritextMlElement, +> = (element: El) => [tag: string, attr: Record | null]; + +export type FromHtmlConverter< + El extends PeritextMlElement = PeritextMlElement, +> = (jsonml: JsonMlElement) => El; diff --git a/src/json-crdt-extensions/quill-delta/QuillDeltaApi.ts b/src/json-crdt-extensions/quill-delta/QuillDeltaApi.ts index 2f54e8524c..10e7b070fa 100644 --- a/src/json-crdt-extensions/quill-delta/QuillDeltaApi.ts +++ b/src/json-crdt-extensions/quill-delta/QuillDeltaApi.ts @@ -1,11 +1,11 @@ import {QuillConst} from './constants'; -import type {PathStep} from '@jsonjoy.com/json-pointer'; -import type {QuillDeltaNode} from './QuillDeltaNode'; import {NodeApi} from '../../json-crdt/model/api/nodes'; import {konst} from '../../json-crdt-patch/builder/Konst'; import {SliceBehavior} from '../peritext/slice/constants'; import {PersistedSlice} from '../peritext/slice/PersistedSlice'; import {diffAttributes, getAttributes, removeErasures} from './util'; +import type {PathStep} from '@jsonjoy.com/json-pointer'; +import type {QuillDeltaNode} from './QuillDeltaNode'; import type {ArrApi, ArrNode, ExtApi, StrApi} from '../../json-crdt'; import type { QuillDeltaAttributes, diff --git a/src/json-crdt-extensions/quill-delta/QuillDeltaNode.ts b/src/json-crdt-extensions/quill-delta/QuillDeltaNode.ts index 1f980fdcb2..df3d515739 100644 --- a/src/json-crdt-extensions/quill-delta/QuillDeltaNode.ts +++ b/src/json-crdt-extensions/quill-delta/QuillDeltaNode.ts @@ -1,16 +1,16 @@ import {isEmpty} from '@jsonjoy.com/util/lib/isEmpty'; import {deepEqual} from '@jsonjoy.com/util/lib/json-equal/deepEqual'; -import type {StrNode} from '../../json-crdt/nodes/str/StrNode'; -import type {ArrNode} from '../../json-crdt/nodes/arr/ArrNode'; import {Peritext} from '../peritext'; import {ExtensionId} from '../constants'; import {MNEMONIC, QuillConst} from './constants'; import {ExtNode} from '../../json-crdt/extensions/ExtNode'; import {getAttributes} from './util'; import {updateRga} from '../../json-crdt/hash'; -import type {QuillDataNode, QuillDeltaAttributes, QuillDeltaOp, QuillDeltaOpInsert} from './types'; +import type {StrNode} from '../../json-crdt/nodes/str/StrNode'; +import type {ArrNode} from '../../json-crdt/nodes/arr/ArrNode'; import type {StringChunk} from '../peritext/util/types'; import type {OverlayTuple} from '../peritext/overlay/types'; +import type {QuillDataNode, QuillDeltaAttributes, QuillDeltaOp, QuillDeltaOpInsert} from './types'; export class QuillDeltaNode extends ExtNode { public readonly txt: Peritext; diff --git a/src/json-crdt-patch/builder/schema.ts b/src/json-crdt-patch/builder/schema.ts index caa32a9ac5..433498f86d 100644 --- a/src/json-crdt-patch/builder/schema.ts +++ b/src/json-crdt-patch/builder/schema.ts @@ -132,6 +132,30 @@ export namespace nodes { * age: s.con(0), * }); * ``` + * + * Specify optional keys as the second argument: + * + * ```ts + * s.obj( + * { + * href: s.str('https://example.com'), + * }, + * { + * title: s.str(''), + * }, + * ) + * ``` + * + * Or, specify only the type, using the `optional` method: + * + * ```ts + * s.obj({ + * href: s.str('https://example.com'), + * }) + * .optional() + * ``` */ export class obj< T extends Record,