diff --git a/.git-cz.json b/.git-cz.json index c17a77c5fb..f2a765bbd2 100644 --- a/.git-cz.json +++ b/.git-cz.json @@ -11,6 +11,7 @@ "json-crdt-patch", "json-equal", "json-expression", + "json-ot", "json-pack", "json-patch", "json-patch-ot", diff --git a/package.json b/package.json index 321717b12d..d92cb94cf9 100644 --- a/package.json +++ b/package.json @@ -95,6 +95,7 @@ "messagepack": "^1.1.12", "msgpack-lite": "^0.1.26", "msgpack5": "^5.3.2", + "ot-text": "^1.0.2", "pako": "^2.0.4", "prettier": "^2.5.1", "pretty-quick": "^3.1.3", diff --git a/src/__tests__/msgpack-documents.ts b/src/__tests__/msgpack-documents.ts index 254f420349..344f07e683 100644 --- a/src/__tests__/msgpack-documents.ts +++ b/src/__tests__/msgpack-documents.ts @@ -1,5 +1,5 @@ -import {JsonPackExtension, JsonPackValue} from "../json-pack"; -import {encodeFull} from "../json-pack/util"; +import {JsonPackExtension, JsonPackValue} from '../json-pack'; +import {encodeFull} from '../json-pack/util'; export interface JsonDocument { name: string; @@ -24,9 +24,7 @@ export const msgPackDocuments: JsonDocument[] = [ }, { name: 'MessagePack value in array', - json: [ - new JsonPackValue(encodeFull(null)), - ], + json: [new JsonPackValue(encodeFull(null))], }, { name: 'MessagePack extension', @@ -36,7 +34,7 @@ export const msgPackDocuments: JsonDocument[] = [ name: 'MessagePack extension in object', json: { foo: new JsonPackExtension(1, new Uint8Array([1, 2, 3])), - } + }, }, { name: 'MessagePack extension in array', diff --git a/src/demo/ot-text.ts b/src/demo/ot-text.ts new file mode 100644 index 0000000000..aa58b806c6 --- /dev/null +++ b/src/demo/ot-text.ts @@ -0,0 +1,21 @@ +/* tslint:disable no-console */ + +/** + * Run this demo with: + * + * npx ts-node src/demo/ot-text.ts + */ + +const {type} = require('ot-text'); + +const op1 = [1, 'a']; +const op2 = [3, 'b']; + +// const op1 = [3, 'a']; +// const op2 = [3, 'b']; + +const op3 = type.transform(op1, op2, 'left'); +const op4 = type.transform(op2, op1, 'right'); + +console.log('op3', op3); +console.log('op4', op4); diff --git a/src/json-binary/index.ts b/src/json-binary/index.ts index c112772a9c..a7183364d5 100644 --- a/src/json-binary/index.ts +++ b/src/json-binary/index.ts @@ -33,7 +33,8 @@ const unwrapBinary = (value: unknown): unknown => { } case 'string': { if (item.length < minDataUri) continue; - if (item.substring(0, binUriStartLength) === binUriStart) value[i] = fromBase64(item.substring(binUriStartLength)); + if (item.substring(0, binUriStartLength) === binUriStart) + value[i] = fromBase64(item.substring(binUriStartLength)); else if (item.substring(0, msgPackUriStartLength) === msgPackUriStart) value[i] = new JsonPackValue(fromBase64(item.substring(msgPackUriStartLength))); else if (item.substring(0, msgPackExtStartLength) === msgPackExtStart) value[i] = parseExtDataUri(item); @@ -57,8 +58,8 @@ const unwrapBinary = (value: unknown): unknown => { (value as any)[key] = buf; } else if (item.substring(0, msgPackUriStartLength) === msgPackUriStart) { (value as any)[key] = new JsonPackValue(fromBase64(item.substring(msgPackUriStartLength))); - } - else if (item.substring(0, msgPackExtStartLength) === msgPackExtStart) (value as any)[key] = parseExtDataUri(item); + } else if (item.substring(0, msgPackExtStartLength) === msgPackExtStart) + (value as any)[key] = parseExtDataUri(item); } } } @@ -67,7 +68,8 @@ const unwrapBinary = (value: unknown): unknown => { if (typeof value === 'string') { if (value.length < minDataUri) return value; if (value.substring(0, binUriStartLength) === binUriStart) return fromBase64(value.substring(binUriStartLength)); - if (value.substring(0, msgPackUriStartLength) === msgPackUriStart) return new JsonPackValue(fromBase64(value.substring(msgPackUriStartLength))); + if (value.substring(0, msgPackUriStartLength) === msgPackUriStart) + return new JsonPackValue(fromBase64(value.substring(msgPackUriStartLength))); if (value.substring(0, msgPackExtStartLength) === msgPackExtStart) return parseExtDataUri(value); else return value; } diff --git a/src/json-block/Block.ts b/src/json-block/Block.ts index f95695d3a1..d9dc05cdcc 100644 --- a/src/json-block/Block.ts +++ b/src/json-block/Block.ts @@ -43,16 +43,12 @@ export class BasicBlock { public apply(patch: Patch): void { this.model.apply(patch); - this.v$.next(this.v$.getValue() + 1); + this.v$.next(this.v$.getValue()); } } export type PatchResponse = true | false | Patch; -export interface BranchDependencies { - readonly merge: (baseVersion: number, patches: Patch[]) => Promise>; -} - export interface BranchMergeResponse { version: number; batches: PatchResponse[][]; @@ -66,13 +62,13 @@ export class Branch { protected readonly head$: BehaviorSubject>; /** - * List of patches applied locally to the head, but not saved - * on the server. This is delta between the base and the head. + * List of patches applied locally to the head, but not yet + * added to the `batches` list. */ protected patches: Patch[]; - /** Number of patches currently being merged to the server. */ - protected merging: number = 0; + /** List of batches to be persisted on the server. */ + protected batches: Patch[][] = []; constructor(base: BasicBlock) { this.base$ = new BehaviorSubject(base); @@ -80,17 +76,9 @@ export class Branch { this.head$ = new BehaviorSubject(base.fork()); } - public async merge(opts: BranchDependencies): Promise { - try { - const base = this.base$.getValue(); - const baseVersion = base.v$.getValue(); - const batch = [...this.patches]; - this.merging = batch.length; - const res = await opts.merge(baseVersion, batch); - - } catch (error) { - this.merging = 0; - } + public cutBatch(): void { + this.batches.push(this.patches); + this.patches = []; } /** Apply a patch locally to the head. */ diff --git a/src/json-block/types.ts b/src/json-block/types.ts index 7e273126ef..56242253af 100644 --- a/src/json-block/types.ts +++ b/src/json-block/types.ts @@ -1,4 +1,4 @@ -import type {Observable} from "rxjs"; +import type {Observable} from 'rxjs'; export interface BlockServerApi { create: () => Promise; @@ -11,14 +11,14 @@ export interface BlockClientApiMergeRequest { /** Last known batch ID by the client. */ v: number; /** List of patches serialized in block-specific codec. */ - batch: unknown[]; + b: unknown[]; } /** * There are a number of scenarios that can happen * when merging changes. The possible scenarios also depend * on the collaborative editing algorithm used for the block. - * + * * 1. The batch is accepted without any conflicts (changes). * 2. The batch is accepted with conflicts, hence the batch may be * be modified to resolve the conflicts. This is relevant for diff --git a/src/json-crdt/codec/binary/Encoder.ts b/src/json-crdt/codec/binary/Encoder.ts index e82f57f56c..4477351adb 100644 --- a/src/json-crdt/codec/binary/Encoder.ts +++ b/src/json-crdt/codec/binary/Encoder.ts @@ -46,7 +46,7 @@ export class Encoder extends CrdtEncoder { delete this.literals; return this.flush(); } - + protected encodeLiteralsTable(model: Model) { const literalFrequencies = new Map(); for (const node of model.nodes.iterate()) { @@ -75,11 +75,9 @@ export class Encoder extends CrdtEncoder { } } const literals: unknown[] = []; - for (const [literal, frequency] of literalFrequencies.entries()) - if (frequency > 1) literals.push(literal); + for (const [literal, frequency] of literalFrequencies.entries()) if (frequency > 1) literals.push(literal); this.encodeArray(literals); - for (let i = 0; i < literals.length; i++) - this.literals!.set(literals[i], i); + for (let i = 0; i < literals.length; i++) this.literals!.set(literals[i], i); } protected encodeClockTable(data: Uint8Array) { diff --git a/src/json-crdt/codec/binary/__tests__/ViewDecoder.spec.ts b/src/json-crdt/codec/binary/__tests__/ViewDecoder.spec.ts index cc25ec516b..e8deff6f69 100644 --- a/src/json-crdt/codec/binary/__tests__/ViewDecoder.spec.ts +++ b/src/json-crdt/codec/binary/__tests__/ViewDecoder.spec.ts @@ -9,7 +9,10 @@ describe('logical', () => { doc1.api.root([1, 'asdf', false, {}, {foo: 'bar'}]).commit(); doc1.api.str([1]).ins(4, '!').commit(); doc1.api.str([4, 'foo']).del(1, 1).commit(); - doc1.api.arr([]).ins(1, [new Uint8Array([1, 2, 3])]).commit(); + doc1.api + .arr([]) + .ins(1, [new Uint8Array([1, 2, 3])]) + .commit(); doc1.api.bin([1]).del(2, 1).commit(); doc1.api.arr([]).ins(6, ['a', 'b', 'c']).commit(); doc1.api.arr([]).del(7, 1).commit(); diff --git a/src/json-crdt/codec/binary/__tests__/literals.spec.ts b/src/json-crdt/codec/binary/__tests__/literals.spec.ts index 6e33e4fb35..906b3d4472 100644 --- a/src/json-crdt/codec/binary/__tests__/literals.spec.ts +++ b/src/json-crdt/codec/binary/__tests__/literals.spec.ts @@ -4,17 +4,21 @@ import {Encoder} from '../Encoder'; test('encodes repeating object keys into literals table', () => { const encoder = new Encoder(); const model1 = Model.withLogicalClock(); - model1.api.root({ - foooo: { - baaar: 0, - } - }).commit(); + model1.api + .root({ + foooo: { + baaar: 0, + }, + }) + .commit(); const model2 = Model.withLogicalClock(); - model2.api.root({ - foooo: { - foooo: 0, - } - }).commit(); + model2.api + .root({ + foooo: { + foooo: 0, + }, + }) + .commit(); const encoded1 = encoder.encode(model1); const encoded2 = encoder.encode(model2); expect(encoded1.byteLength > encoded2.byteLength).toBe(true); diff --git a/src/json-crdt/json-patch/JsonPatchDraft.ts b/src/json-crdt/json-patch/JsonPatchDraft.ts index 091d8dc6e0..8a60f2b3f9 100644 --- a/src/json-crdt/json-patch/JsonPatchDraft.ts +++ b/src/json-crdt/json-patch/JsonPatchDraft.ts @@ -6,7 +6,17 @@ import {ObjectType} from '../types/lww-object/ObjectType'; import {UNDEFINED_ID} from '../../json-crdt-patch/constants'; import {toPath} from '../../json-pointer/util'; import type {Model} from '../model'; -import type {Operation, OperationAdd, OperationRemove, OperationReplace, OperationMove, OperationCopy, OperationTest, OperationStrIns, OperationStrDel} from '../../json-patch'; +import type { + Operation, + OperationAdd, + OperationRemove, + OperationReplace, + OperationMove, + OperationCopy, + OperationTest, + OperationStrIns, + OperationStrDel, +} from '../../json-patch'; export class JsonPatchDraft { public readonly draft = new Draft(); @@ -18,16 +28,33 @@ export class JsonPatchDraft { } public applyOp(op: Operation): void { - switch(op.op) { - case 'add': this.applyOpAdd(op); break; - case 'remove': this.applyRemove(op); break; - case 'replace': this.applyReplace(op); break; - case 'move': this.applyMove(op); break; - case 'copy': this.applyCopy(op); break; - case 'test': this.applyTest(op); break; - case 'str_ins': this.applyStrIns(op); break; - case 'str_del': this.applyStrDel(op); break; - default: throw new Error('UNKNOWN_OP'); + switch (op.op) { + case 'add': + this.applyOpAdd(op); + break; + case 'remove': + this.applyRemove(op); + break; + case 'replace': + this.applyReplace(op); + break; + case 'move': + this.applyMove(op); + break; + case 'copy': + this.applyCopy(op); + break; + case 'test': + this.applyTest(op); + break; + case 'str_ins': + this.applyStrIns(op); + break; + case 'str_del': + this.applyStrDel(op); + break; + default: + throw new Error('UNKNOWN_OP'); } } diff --git a/src/json-crdt/json-patch/__tests__/JsonPatch.str.spec.ts b/src/json-crdt/json-patch/__tests__/JsonPatch.str.spec.ts index 1c6ef02219..c73314135c 100644 --- a/src/json-crdt/json-patch/__tests__/JsonPatch.str.spec.ts +++ b/src/json-crdt/json-patch/__tests__/JsonPatch.str.spec.ts @@ -53,7 +53,7 @@ const testCases: TestCase[] = [ doc1: null, patches: [ [{op: 'add', path: '', value: {foo: [{bar: 'baz'}]}}], - [{op: 'str_ins', path: '/foo/0/bar', pos: 3, str: '!'}] + [{op: 'str_ins', path: '/foo/0/bar', pos: 3, str: '!'}], ], doc2: {foo: [{bar: 'baz!'}]}, }, @@ -62,73 +62,56 @@ const testCases: TestCase[] = [ doc1: null, patches: [ [{op: 'add', path: '', value: {foo: [{bar: 'baz'}]}}], - [{op: 'str_ins', path: ['foo', 0, 'bar'], pos: 3, str: '!'}] + [{op: 'str_ins', path: ['foo', 0, 'bar'], pos: 3, str: '!'}], ], doc2: {foo: [{bar: 'baz!'}]}, }, { name: 'can delete a single char', doc1: 'a', - patches: [ - [{op: 'str_del', path: [], pos: 0, len: 1}] - ], + patches: [[{op: 'str_del', path: [], pos: 0, len: 1}]], doc2: '', }, { name: 'can delete from already empty string', doc1: '', - patches: [ - [{op: 'str_del', path: [], pos: 0, len: 1}] - ], + patches: [[{op: 'str_del', path: [], pos: 0, len: 1}]], doc2: '', }, { name: 'can delete at the end of string', doc1: 'ab', - patches: [ - [{op: 'str_del', path: [], pos: 1, len: 1}] - ], + patches: [[{op: 'str_del', path: [], pos: 1, len: 1}]], doc2: 'a', }, { name: 'can delete at the beginning of string', doc1: 'ab', - patches: [ - [{op: 'str_del', path: [], pos: 0, len: 1}] - ], + patches: [[{op: 'str_del', path: [], pos: 0, len: 1}]], doc2: 'b', }, { name: 'can delete in the middle of string', doc1: 'abc', - patches: [ - [{op: 'str_del', path: [], pos: 1, len: 1}] - ], + patches: [[{op: 'str_del', path: [], pos: 1, len: 1}]], doc2: 'ac', }, { name: 'can delete multiple chars', doc1: '1234', - patches: [ - [{op: 'str_del', path: [], pos: 1, len: 2}], - [{op: 'str_del', path: [], pos: 1, len: 5}], - ], + patches: [[{op: 'str_del', path: [], pos: 1, len: 2}], [{op: 'str_del', path: [], pos: 1, len: 5}]], doc2: '1', }, { name: 'handles deletion beyond end of string', doc1: '1234', - patches: [ - [{op: 'str_del', path: [], pos: 1111, len: 2}], - ], + patches: [[{op: 'str_del', path: [], pos: 1111, len: 2}]], doc2: '1234', }, { name: 'can delete a string in object', doc1: {foo: '123'}, - patches: [ - [{op: 'str_del', path: '/foo', pos: 1, len: 2}], - ], + patches: [[{op: 'str_del', path: '/foo', pos: 1, len: 2}]], doc2: {foo: '1'}, }, ]; diff --git a/src/json-crdt/model/__tests__/Model.caching.spec.ts b/src/json-crdt/model/__tests__/Model.caching.spec.ts index 5c6eeddc46..93d1f6119e 100644 --- a/src/json-crdt/model/__tests__/Model.caching.spec.ts +++ b/src/json-crdt/model/__tests__/Model.caching.spec.ts @@ -1,4 +1,4 @@ -import {Model} from ".."; +import {Model} from '..'; test('returns cached value, when shallow object keys not modified', () => { const model = Model.withLogicalClock(); diff --git a/src/json-crdt/model/__tests__/fuzzer/Picker.ts b/src/json-crdt/model/__tests__/fuzzer/Picker.ts index 6175b73f48..1ed538dfec 100644 --- a/src/json-crdt/model/__tests__/fuzzer/Picker.ts +++ b/src/json-crdt/model/__tests__/fuzzer/Picker.ts @@ -19,14 +19,7 @@ type BinaryOp = typeof InsertBinaryDataOperation | typeof DeleteOperation; type ArrayOp = typeof InsertArrayElementsOperation | typeof DeleteOperation; type ObjectOp = typeof SetObjectKeysOperation | typeof DeleteOperation; -const commonKeys = [ - 'a', - 'op', - 'test', - 'name', - '', - '__proto__', -]; +const commonKeys = ['a', 'op', 'test', 'name', '', '__proto__']; /** * This class picks random nodes from a model and picks a random diff --git a/src/json-crdt/types/rga-array/ArrayType.ts b/src/json-crdt/types/rga-array/ArrayType.ts index 44bccac1c6..7a3a731ec0 100644 --- a/src/json-crdt/types/rga-array/ArrayType.ts +++ b/src/json-crdt/types/rga-array/ArrayType.ts @@ -171,9 +171,8 @@ export class ArrayType implements JsonNode { curr = curr.right; } const _toJson = this._toJson; - if (arr.length !== _toJson.length) return this._toJson = arr; - for (let i = 0; i < arr.length; i++) - if (arr[i] !== _toJson[i]) return this._toJson = arr; + if (arr.length !== _toJson.length) return (this._toJson = arr); + for (let i = 0; i < arr.length; i++) if (arr[i] !== _toJson[i]) return (this._toJson = arr); return _toJson; } diff --git a/src/json-crdt/types/rga-binary/BinaryType.ts b/src/json-crdt/types/rga-binary/BinaryType.ts index d3900d2bed..02b48187a4 100644 --- a/src/json-crdt/types/rga-binary/BinaryType.ts +++ b/src/json-crdt/types/rga-binary/BinaryType.ts @@ -164,7 +164,7 @@ export class BinaryType implements JsonNode { res.set(buf, offset); offset += buf.length; } - return this._toJson = res; + return (this._toJson = res); } public clone(doc: Model): BinaryType { diff --git a/src/json-crdt/types/rga-string/StringType.ts b/src/json-crdt/types/rga-string/StringType.ts index 3e29400d67..d3eab82c67 100644 --- a/src/json-crdt/types/rga-string/StringType.ts +++ b/src/json-crdt/types/rga-string/StringType.ts @@ -150,7 +150,7 @@ export class StringType implements JsonNode { let str: string = ''; let curr: StringChunk | null = this.start; while ((curr = curr.right)) if (curr.str) str += curr.str; - return this._toString = str; + return (this._toString = str); } public clone(doc: Model): StringType { diff --git a/src/json-ot/index.ts b/src/json-ot/index.ts new file mode 100644 index 0000000000..e69de29bb2 diff --git a/src/json-ot/types/ot-string/StringType.ts b/src/json-ot/types/ot-string/StringType.ts new file mode 100644 index 0000000000..7685300c8c --- /dev/null +++ b/src/json-ot/types/ot-string/StringType.ts @@ -0,0 +1,336 @@ +import type {StringTypeOp, StringTypeOpComponent} from './types'; + +export const enum VALIDATE_RESULT { + SUCCESS = 0, + INVALID_OP, + INVALID_COMPONENT, + ADJACENT_SAME_TYPE, + NO_TRAILING_RETAIN, +} + +export const validate = (op: StringTypeOp): VALIDATE_RESULT => { + if (!(op instanceof Array)) return VALIDATE_RESULT.INVALID_OP; + if (op.length === 0) return VALIDATE_RESULT.INVALID_OP; + let last: StringTypeOpComponent | undefined; + for (let i = 0; i < op.length; i++) { + const component = op[i]; + switch (typeof component) { + case 'number': { + if (!component) return VALIDATE_RESULT.INVALID_COMPONENT; + if (component !== Math.round(component)) return VALIDATE_RESULT.INVALID_COMPONENT; + if (component > 0) { + const lastComponentIsRetain = typeof last === 'number' && last > 0; + if (lastComponentIsRetain) return VALIDATE_RESULT.ADJACENT_SAME_TYPE; + } else { + const lastComponentIsDelete = typeof last === 'number' && last < 0; + if (lastComponentIsDelete) return VALIDATE_RESULT.ADJACENT_SAME_TYPE; + } + break; + } + case 'string': { + if (!component.length) return VALIDATE_RESULT.INVALID_COMPONENT; + const lastComponentIsInsert = typeof last === 'string'; + if (lastComponentIsInsert) return VALIDATE_RESULT.ADJACENT_SAME_TYPE; + break; + } + case 'object': { + if (!(component instanceof Array)) return VALIDATE_RESULT.INVALID_COMPONENT; + if (component.length !== 1) return VALIDATE_RESULT.INVALID_COMPONENT; + const lastComponentIsRetainedDelete = last instanceof Array; + if (lastComponentIsRetainedDelete) return VALIDATE_RESULT.ADJACENT_SAME_TYPE; + break; + } + default: + return VALIDATE_RESULT.INVALID_COMPONENT; + } + last = component; + } + const isLastRetain = typeof last === 'number' && last > 0; + if (isLastRetain) return VALIDATE_RESULT.NO_TRAILING_RETAIN; + return VALIDATE_RESULT.SUCCESS; +}; + +export const append = (op: StringTypeOp, component: StringTypeOpComponent): void => { + if (!component) return; + if (!op.length) { + op.push(component); + return; + } + const lastIndex = op.length - 1; + const last = op[lastIndex]; + switch (typeof component) { + case 'number': { + if (typeof last === 'number') { + if (component > 0 && last > 0) op[lastIndex] = last + component; + else if (component < 0 && last < 0) op[lastIndex] = last + component; + else op.push(component); + } else op.push(component); + break; + } + case 'string': { + if (typeof last === 'string') op[lastIndex] = last + component; + else op.push(component); + break; + } + case 'object': { + if (last instanceof Array) last[0] = last + component[0]; + else op.push(component); + break; + } + } +}; + +const componentLength = (component: StringTypeOpComponent): number => { + switch (typeof component) { + case 'number': + return Math.abs(component); + case 'string': + return component.length; + default: + return component[0].length; + } +}; + +const idDeleteComponent = (component: StringTypeOpComponent): boolean => { + switch (typeof component) { + case 'number': + return component < 0; + case 'object': + return true; + default: + return false; + } +}; + +const trim = (op: StringTypeOp): void => { + if (!op.length) return; + const last = op[op.length - 1]; + const isLastRetain = typeof last === 'number' && last > 0; + if (isLastRetain) op.pop(); +}; + +export const normalize = (op: StringTypeOp): StringTypeOp => { + const op2: StringTypeOp = []; + const length = op.length; + for (let i = 0; i < length; i++) append(op2, op[i]); + trim(op2); + return op2; +}; + +export const apply = (str: string, op: StringTypeOp): string => { + const length = op.length; + let res = ''; + let offset = 0; + for (let i = 0; i < length; i++) { + const component = op[i]; + switch (typeof component) { + case 'number': { + if (component > 0) { + const end = offset + component; + res += str.substring(offset, end); + offset = end; + } else offset -= component; + break; + } + case 'string': + res += component; + break; + case 'object': + offset += component[0].length; + break; + } + } + return res + str.substring(offset); +}; + +/** + * Extracts a full or a part of a component from an operation. + * + * @param component Component from which to extract a chunk. + * @param offset Position within the component to start from. + * @param maxLength Maximum length of the component to extract. + * @returns Full or partial component at index `index` of operation `op`. + */ +const chunk = (component: StringTypeOpComponent, offset: number, maxLength: number): StringTypeOpComponent => { + switch (typeof component) { + case 'number': { + return component > 0 + ? Math.min(component - offset, maxLength) + : -Math.min(-component - offset, maxLength); + } + case 'string': { + const end = Math.min(offset + maxLength, component.length); + return component.substring(offset, end); + } + case 'object': { + const str = component[0]; + const end = Math.min(offset + maxLength, str.length); + return [str.substring(offset, end)]; + } + } +}; + +/** + * Combine two operations into one, such that the changes produced by the + * by the single operation are the same as if the two operations were applied + * in sequence. + * + * ``` + * apply(str, combine(op1, op2)) === apply(apply(str, op1), op2) + * ``` + * + * @param op1 First operation. + * @param op2 Second operation. + * @returns A combined operation. + */ +export const compose = (op1: StringTypeOp, op2: StringTypeOp): StringTypeOp => { + const op3: StringTypeOp = []; + const len1 = op1.length; + const len2 = op2.length; + let off1 = 0; + let i1 = 0; + for (let i2 = 0; i2 < len2; i2++) { + const comp2 = op2[i2]; + let doDelete = false; + switch (typeof comp2) { + case 'number': { + if (comp2 > 0) { + let length2 = comp2; + while (length2 > 0) { + const comp1 = op1[i1]; + const comp = i1 >= len1 ? length2 : chunk(comp1, off1, length2); + const compLength = componentLength(comp); + const isDelete = idDeleteComponent(comp); + const length1 = componentLength(comp1 || comp); + append(op3, comp); + off1 += compLength; + if (off1 >= length1) { + i1++; + off1 = 0; + } + if (!isDelete) length2 -= compLength; + } + } else doDelete = true; + break; + } + case 'string': { + append(op3, comp2); + break; + } + case 'object': { + doDelete = true; + break; + } + } + if (doDelete) { + const isReversible = comp2 instanceof Array; + const length2 = isReversible ? comp2[0].length : -comp2; + let off2 = 0; + while (off2 < length2) { + const remaining = length2 - off2; + const comp1 = op1[i1]; + const comp = i1 >= len1 ? remaining : chunk(comp1, off1, remaining); + const compLength = componentLength(comp); + const isDelete = idDeleteComponent(comp); + const length1 = componentLength(comp1 || comp); + if (isDelete) append(op3, comp); + else if (typeof comp === 'number') + append(op3, isReversible ? [comp2[0].substring(off2, off2 + compLength)] : -compLength); + off1 += compLength; + if (off1 >= length1) { + i1++; + off1 = 0; + } + if (!isDelete) off2 += compLength; + } + } + } + if (i1 < len1 && off1) append(op3, chunk(op1[i1++], off1, Infinity)); + for (; i1 < len1; i1++) append(op3, op1[i1]); + trim(op3); + return op3; +}; + +/** + * Transforms an operation such that the transformed operations can be + * applied to a string in reverse order. + * + * ``` + * apply(apply(doc, op1), transform(op2, op1)) === apply(apply(doc, op2), transform(op1, op2)) + * ``` + * + * @param op1 The operation to transform. + * @param op2 The operation to transform against. + * @returns A new operation with user intentions preserved. + */ + export const transform = (op1: StringTypeOp, op2: StringTypeOp, leftInsertFirst: boolean): StringTypeOp => { + const op3: StringTypeOp = []; + const len1 = op1.length; + const len2 = op2.length; + let i1 = 0; + let i2 = 0; + let off1 = 0; + for (; i2 < len2; i2++) { + const comp2 = op2[i2]; + let doDelete = false; + switch (typeof comp2) { + case 'number': { + if (comp2 > 0) { + let length2 = comp2; + while (length2 > 0) { + const comp1 = op1[i1]; + const comp = i1 >= len1 ? length2 : chunk(comp1, off1, length2); + const compLength = componentLength(comp); + const length1 = componentLength(comp1 || comp); + append(op3, comp); + off1 += compLength; + if (off1 >= length1) { + i1++; + off1 = 0; + } + if (typeof comp !== 'string') length2 -= compLength; + } + } else doDelete = true; + break; + } + case 'string': { + if (leftInsertFirst) { + if (typeof op1[i1] === 'string') { + const comp = chunk(op1[i1++], off1, Infinity); + off1 = 0; + append(op3, comp) + } + } + append(op3, comp2.length); + break; + } + case 'object': { + doDelete = true; + break; + } + } + if (doDelete) { + const isReversible = comp2 instanceof Array; + const length2 = isReversible ? comp2[0].length : -comp2; + let off2 = 0; + while (off2 < length2) { + const remaining = length2 - off2; + const comp1 = op1[i1]; + const comp = i1 >= len1 ? remaining : chunk(comp1, off1, remaining); + const compLength = componentLength(comp); + const length1 = componentLength(comp1 || comp); + if (typeof comp === 'string') append(op3, comp); + else off2 += compLength; + off1 += compLength; + if (off1 >= length1) { + i1++; + off1 = 0; + } + } + } + } + if (i1 < len1 && off1) append(op3, chunk(op1[i1++], off1, Infinity)); + for (; i1 < len1; i1++) append(op3, op1[i1]); + trim(op3); + return op3; +}; diff --git a/src/json-ot/types/ot-string/__tests__/StringOtFuzzer.ts b/src/json-ot/types/ot-string/__tests__/StringOtFuzzer.ts new file mode 100644 index 0000000000..de190c4d20 --- /dev/null +++ b/src/json-ot/types/ot-string/__tests__/StringOtFuzzer.ts @@ -0,0 +1,42 @@ +import {RandomJson} from '../../../../json-random'; +import {Fuzzer} from '../../../../util/Fuzzer'; +import {append, normalize} from '../StringType'; +import {StringTypeOp} from '../types'; + +export class StringOtFuzzer extends Fuzzer { + genString(): string { + return RandomJson.genString(20); + } + + genOp(str: string): StringTypeOp { + if (!str) return [this.genString()]; + let op: StringTypeOp = []; + let off = 0; + let remaining = str.length; + while (remaining > 0) { + const len = Fuzzer.generateInteger(1, remaining); + const fn = Fuzzer.pick([ + () => { + append(op, len); + off += len; + }, + () => { + if (Math.random() < 0.5) { + append(op, -len); + } else { + append(op, [str.substring(off, off + len)]); + } + off += len; + }, + () => { + append(op, RandomJson.genString(len)); + }, + ]); + fn(); + remaining = str.length - off; + } + op = normalize(op); + if (op.length === 1 && typeof op[0] === 'number' && op[0] > 0) return [this.genString()]; + return op; + } +} diff --git a/src/json-ot/types/ot-string/__tests__/StringType.spec.ts b/src/json-ot/types/ot-string/__tests__/StringType.spec.ts new file mode 100644 index 0000000000..f1cf44dc8a --- /dev/null +++ b/src/json-ot/types/ot-string/__tests__/StringType.spec.ts @@ -0,0 +1,186 @@ +import {validate, append, normalize, apply, compose, transform} from '../StringType'; +import {StringTypeOp} from '../types'; + +describe('validate()', () => { + test('returns 0 on valid op', () => { + expect(validate(['a'])).toBe(0); + expect(validate([1, 'a'])).toBe(0); + expect(validate([1, -1, 'a'])).toBe(0); + expect(validate([1, -1, ['b'], 'a'])).toBe(0); + }); + + test('returns non-zero integer on invalid operation', () => { + expect(validate([1])).not.toBe(0); + expect(validate([0])).not.toBe(0); + expect(validate([5])).not.toBe(0); + expect(validate([1, 'a', 11])).not.toBe(0); + expect(validate([1, -1, 'a', 'b'])).not.toBe(0); + expect(validate([1, -1, 'a', ''])).not.toBe(0); + expect(validate([''])).not.toBe(0); + expect(validate([1, 2, -1, ['b'], 'a'])).not.toBe(0); + expect(validate([1, -1, -3, ['b'], 'a'])).not.toBe(0); + expect(validate([1, 0.3, ['b'], 'a'])).not.toBe(0); + expect(validate([1, 0.3])).not.toBe(0); + expect(validate([1, ''])).not.toBe(0); + expect(validate([''])).not.toBe(0); + }); +}); + +describe('append()', () => { + test('adds components to operation', () => { + const op: StringTypeOp = []; + append(op, 1); + expect(op).toStrictEqual([1]); + append(op, 0); + expect(op).toStrictEqual([1]); + append(op, 4); + expect(op).toStrictEqual([5]); + append(op, 'asdf'); + expect(op).toStrictEqual([5, 'asdf']); + append(op, 'asdf'); + expect(op).toStrictEqual([5, 'asdfasdf']); + append(op, -4); + expect(op).toStrictEqual([5, 'asdfasdf', -4]); + append(op, 0); + expect(op).toStrictEqual([5, 'asdfasdf', -4]); + append(op, -3); + expect(op).toStrictEqual([5, 'asdfasdf', -7]); + append(op, ['a']); + expect(op).toStrictEqual([5, 'asdfasdf', -7, ['a']]); + append(op, ['b']); + expect(op).toStrictEqual([5, 'asdfasdf', -7, ['ab']]); + }); +}); + +describe('normalize()', () => { + test('normalizes operation', () => { + expect(normalize(['asdf'])).toStrictEqual(['asdf']); + expect(normalize(['asdf', 'e'])).toStrictEqual(['asdfe']); + expect(normalize(['asdf', 'e', 1, 2])).toStrictEqual(['asdfe']); + expect(normalize(['asdf', 'e', 1, 2, -3])).toStrictEqual(['asdfe', 3, -3]); + expect(normalize(['asdf', 'e', 1, 2, -3, -1, -1])).toStrictEqual(['asdfe', 3, -5]); + expect(normalize(['asdf', 'e', 1, 2, -3, -1, -1, ['asdf']])).toStrictEqual(['asdfe', 3, -5, ['asdf']]); + expect(normalize(['asdf', 'e', 1, 2, -3, -1, -1, ['asdf'], ['a']])).toStrictEqual(['asdfe', 3, -5, ['asdfa']]); + expect(normalize(['asdf', 'e', 1, 2, -3, -1, -1, ['asdf'], 3, ['a']])).toStrictEqual([ + 'asdfe', + 3, + -5, + ['asdf'], + 3, + ['a'], + ]); + }); +}); + +describe('apply()', () => { + test('can apply operation', () => { + expect(apply('', ['abc'])).toBe('abc'); + expect(apply('13', [1, '2'])).toBe('123'); + expect(apply('13', [1, '2', '4'])).toBe('1243'); + expect(apply('13', [1, '2', 1, '4'])).toBe('1234'); + expect(apply('13', [1, '2', 2, '4'])).toBe('1234'); + expect(apply('13', [1, '2', 3, '4'])).toBe('1234'); + expect(apply('123', [1, -1])).toBe('13'); + expect(apply('123', [1, -1, 1, 'a'])).toBe('13a'); + expect(apply('123', [1, -1, 1])).toBe('13'); + expect(apply('123', [1, ['2'], 1, 'a'])).toBe('13a'); + }); +}); + +describe('compose()', () => { + test('can combine two ops', () => { + const op1: StringTypeOp = [1, 'a']; + const op2: StringTypeOp = [1, 'b']; + const op3 = compose(op1, op2); + expect(op3).toStrictEqual([1, 'ba']); + }); + + test('can delete insert of op1', () => { + const op1: StringTypeOp = [1, 'a']; + const op2: StringTypeOp = [1, -1, 'b']; + const op3 = compose(op1, op2); + expect(op3).toStrictEqual([1, 'b']); + }); + + type TestCase = [name: string, str: string, op1: StringTypeOp, op2: StringTypeOp, expected: string, only?: boolean]; + + const testCases: TestCase[] = [ + ['insert-insert', 'abc', [1, 'a'], [1, 'b'], 'ababc'], + ['insert-delete', 'abc', [1, 'a'], [1, -1], 'abc'], + ['insert-delete-2', 'abc', [1, 'a'], [2, -1], 'aac'], + ['insert in previous insert', 'aabb', [2, '1111'], [4, '22'], 'aa112211bb'], + ['fuzzer bug #1', 'd6', ['}B'], [['}'], ';0q', 2, ['6']], ';0qBd'], + ['fuzzer bug #2', 'Ai', [['A'], '#', -1], [-1], ''], + ['fuzzer bug #3', 'M}', ['!y1'], ["'/*s", 2, ',/@', -2, ['}']], "'/*s!y,/@"], + ['fuzzer bug #4', '8sL', [-2, 'w', ['L']], [['w']], ''], + ['fuzzer bug #5', '%V=', [2, ';'], ['3O"', 1, 'J', -2], '3O"%J='], + ]; + + describe('can compose', () => { + for (const [name, str, op1, op2, expected, only] of testCases) { + (only ? test.only : test)(`${name}`, () => { + const res1 = apply(apply(str, op1), op2); + // console.log('res1', res1); + const op3 = compose(op1, op2); + // console.log('op3', op3); + const res2 = apply(str, op3); + // console.log('res2', res2); + expect(res2).toStrictEqual(res1); + expect(res2).toStrictEqual(expected); + }); + } + }); +}); + +describe('transform()', () => { + test('can transform two inserts', () => { + const op1: StringTypeOp = [1, 'a']; + const op2: StringTypeOp = [3, 'b']; + const op3 = transform(op1, op2, true); + const op4 = transform(op2, op1, false); + expect(op3).toStrictEqual([1, 'a']); + expect(op4).toStrictEqual([4, 'b']); + }); + + test('insert at the same place', () => { + const op1: StringTypeOp = [3, 'a']; + const op2: StringTypeOp = [3, 'b']; + const op3 = transform(op1, op2, true); + const op4 = transform(op2, op1, false); + expect(op3).toStrictEqual([3, 'a']); + expect(op4).toStrictEqual([4, 'b']); + }); + + test('can transform two deletes', () => { + const op1: StringTypeOp = [1, -1]; + const op2: StringTypeOp = [3, -1]; + const op3 = transform(op1, op2, true); + const op4 = transform(op2, op1, false); + expect(op3).toStrictEqual([1, -1]); + expect(op4).toStrictEqual([2, -1]); + }); + + type TestCase = [name: string, str: string, op1: StringTypeOp, op2: StringTypeOp, expected: string, only?: boolean]; + + const testCases: TestCase[] = [ + ['delete-delete', 'abc', [1, -1], [2, -1], 'a'], + ['insert-insert', '12345', [1, 'one', 2, 'three'], [2, 'two', 2, 'four'], '1one2two3three4four5'], + ]; + + describe('can transform', () => { + for (const [name, str, op1, op2, expected, only] of testCases) { + (only ? test.only : test)(`${name}`, () => { + const op11 = transform(op1, op2, true); + const op22 = transform(op2, op1, false); + const res1 = apply(apply(str, op1), op22); + const res2 = apply(apply(str, op2), op11); + // console.log('op11', op11); + // console.log('op22', op22); + // console.log('res1', res1); + // console.log('res2', res2); + expect(res2).toStrictEqual(res1); + expect(res2).toStrictEqual(expected); + }); + } + }); +}); diff --git a/src/json-ot/types/ot-string/__tests__/fuzzer.compose.spec.ts b/src/json-ot/types/ot-string/__tests__/fuzzer.compose.spec.ts new file mode 100644 index 0000000000..7d8c97d3b0 --- /dev/null +++ b/src/json-ot/types/ot-string/__tests__/fuzzer.compose.spec.ts @@ -0,0 +1,23 @@ +import {apply, compose} from '../StringType'; +import {StringOtFuzzer} from './StringOtFuzzer'; + +const fuzzer = new StringOtFuzzer(); + +test('works', () => { + for (let i = 0; i < 10000; i++) { + const str1 = fuzzer.genString(); + const op1 = fuzzer.genOp(str1); + const str2 = apply(str1, op1); + const op2 = fuzzer.genOp(str2); + const str3 = apply(str2, op2); + const op3 = compose(op1, op2); + const str4 = apply(str1, op3); + try { + expect(str4).toBe(str3); + } catch (error) { + // tslint:disable-next-line no-console + console.log([str1, op1, str2, op2, str3, op3, str4]); + throw error; + } + } +}); diff --git a/src/json-ot/types/ot-string/__tests__/fuzzer.transform.spec.ts b/src/json-ot/types/ot-string/__tests__/fuzzer.transform.spec.ts new file mode 100644 index 0000000000..8dab60a26c --- /dev/null +++ b/src/json-ot/types/ot-string/__tests__/fuzzer.transform.spec.ts @@ -0,0 +1,24 @@ +import {apply, transform} from '../StringType'; +import {StringOtFuzzer} from './StringOtFuzzer'; + +const fuzzer = new StringOtFuzzer(); + +test('works', () => { + for (let i = 0; i < 1000; i++) { + const str1 = fuzzer.genString(); + const op1 = fuzzer.genOp(str1); + const op2 = fuzzer.genOp(str1); + const op11 = transform(op1, op2, true); + const op22 = transform(op2, op1, false); + const str2 = apply(apply(str1, op1), op22); + const str3 = apply(apply(str1, op2), op11); + try { + expect(str3).toBe(str2); + // console.log([str1, op1, op2, op11, op22, str2, str3]); + } catch (error) { + // tslint:disable-next-line no-console + console.log([str1, op1, op2, op11, op22, str2, str3]); + throw error; + } + } +}); diff --git a/src/json-ot/types/ot-string/types.ts b/src/json-ot/types/ot-string/types.ts new file mode 100644 index 0000000000..7495511c38 --- /dev/null +++ b/src/json-ot/types/ot-string/types.ts @@ -0,0 +1,24 @@ +/** + * - Positive number specifies how many chars to retain. + * - Negative number specifies how many chars to remove. + * - String in an array specifies a substring deletion. + * - String is a substring insertion. + */ +export type StringTypeOpComponent = number | [string] | string; + +/** + * This form of operation encoding results in most efficient + * binary encoding using MessagePack. + * + * Consider operation: + * + * ``` + * [5, "hello", -4] + * ``` + * + * - Array is encoded as one byte fixarr. + * - 5 is encoded as one byte fixint. + * - String header is encoded as one byte fixstr followed by 5 bytes of UTF-8 string. + * - -4 is encoded as one byte fixint. + */ +export type StringTypeOp = StringTypeOpComponent[]; diff --git a/src/json-ot/types/types.ts b/src/json-ot/types/types.ts new file mode 100644 index 0000000000..e69de29bb2 diff --git a/src/json-pack/EncoderStable/__tests__/EncoderStable.spec.ts b/src/json-pack/EncoderStable/__tests__/EncoderStable.spec.ts index b5a68ee41d..5e30bf93af 100644 --- a/src/json-pack/EncoderStable/__tests__/EncoderStable.spec.ts +++ b/src/json-pack/EncoderStable/__tests__/EncoderStable.spec.ts @@ -7,8 +7,8 @@ const decoder = new Decoder(); const decode = (a: Uint8Array) => decoder.decode(a); test('encodes object the same regardless of key order', () => { - const data1 = {a: 1, b: 2}; - const data2 = {b: 2, a: 1}; + const data1 = {a: 1, b: 2}; + const data2 = {b: 2, a: 1}; const arr1 = encode(data1); const arr2 = encode(data2); expect(arr1).toStrictEqual(arr2); diff --git a/src/json-pack/EncoderStable/index.ts b/src/json-pack/EncoderStable/index.ts index 079b961bab..fb8f28aa60 100644 --- a/src/json-pack/EncoderStable/index.ts +++ b/src/json-pack/EncoderStable/index.ts @@ -5,7 +5,7 @@ import {Encoder} from '../Encoder'; */ export class EncoderStable extends Encoder { public encodeObject(obj: Record): void { - const keys = Object.keys(obj).sort((a, b) => a > b ? 1 : -1); + const keys = Object.keys(obj).sort((a, b) => (a > b ? 1 : -1)); const length = keys.length; this.encodeObjectHeader(length); for (let i = 0; i < length; i++) { diff --git a/src/util/base64/decode.ts b/src/util/base64/decode.ts index 34a8c5c92e..7ed00500e1 100644 --- a/src/util/base64/decode.ts +++ b/src/util/base64/decode.ts @@ -35,7 +35,8 @@ export const createFromBase64 = (chars: string = alphabet) => { const sextet1 = table[c1]; const sextet2 = table[c2]; const sextet3 = table[c3]; - if ((sextet0 === undefined) || (sextet1 === undefined) || (sextet2 === undefined) || (sextet3 === undefined)) throw new Error('INVALID_BASE64_STRING'); + if (sextet0 === undefined || sextet1 === undefined || sextet2 === undefined || sextet3 === undefined) + throw new Error('INVALID_BASE64_STRING'); buf[j] = (sextet0 << 2) | (sextet1 >> 4); buf[j + 1] = (sextet1 << 4) | (sextet2 >> 2); buf[j + 2] = (sextet2 << 6) | sextet3; @@ -46,7 +47,7 @@ export const createFromBase64 = (chars: string = alphabet) => { const c1 = encoded[mainLength + 1]; const sextet0 = table[c0]; const sextet1 = table[c1]; - if ((sextet0 === undefined) || (sextet1 === undefined)) throw new Error('INVALID_BASE64_STRING'); + if (sextet0 === undefined || sextet1 === undefined) throw new Error('INVALID_BASE64_STRING'); buf[j] = (sextet0 << 2) | (sextet1 >> 4); } else if (padding === 1) { const c0 = encoded[mainLength]; @@ -55,7 +56,8 @@ export const createFromBase64 = (chars: string = alphabet) => { const sextet0 = table[c0]; const sextet1 = table[c1]; const sextet2 = table[c2]; - if ((sextet0 === undefined) || (sextet1 === undefined) || (sextet2 === undefined)) throw new Error('INVALID_BASE64_STRING'); + if (sextet0 === undefined || sextet1 === undefined || sextet2 === undefined) + throw new Error('INVALID_BASE64_STRING'); buf[j] = (sextet0 << 2) | (sextet1 >> 4); buf[j + 1] = (sextet1 << 4) | (sextet2 >> 2); } diff --git a/yarn.lock b/yarn.lock index 9faa4a75d5..fb594c2f99 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4726,6 +4726,11 @@ original@^1.0.0: dependencies: url-parse "^1.4.3" +ot-text@^1.0.2: + version "1.0.2" + resolved "https://registry.yarnpkg.com/ot-text/-/ot-text-1.0.2.tgz#fc27c12b07a93b13c328c833bf94c9c24ac46231" + integrity sha512-1xjcAjB57tYtv722j8J+IaYe4fKZOeaQ3zDa0clUlkhboawrq+dk1cpDfe4+TBSb9NNe7Yra9+tJKsw+8q4H9A== + p-each-series@^2.1.0: version "2.2.0" resolved "https://registry.yarnpkg.com/p-each-series/-/p-each-series-2.2.0.tgz#105ab0357ce72b202a8a8b94933672657b5e2a9a"