From de774c94623dda5ead34928230acb1b1ae1447dd Mon Sep 17 00:00:00 2001 From: streamich Date: Sun, 10 Mar 2024 19:19:32 +0100 Subject: [PATCH 01/16] =?UTF-8?q?feat:=20=F0=9F=8E=B8=20start=20HAMT=20CRD?= =?UTF-8?q?T=20implementation?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/web3/adl/hamt-crdt/README.md | 10 ++++++ src/web3/adl/hamt-crdt/types.ts | 62 ++++++++++++++++++++++++++++++++ 2 files changed, 72 insertions(+) create mode 100644 src/web3/adl/hamt-crdt/README.md create mode 100644 src/web3/adl/hamt-crdt/types.ts diff --git a/src/web3/adl/hamt-crdt/README.md b/src/web3/adl/hamt-crdt/README.md new file mode 100644 index 0000000000..eb4d07b59b --- /dev/null +++ b/src/web3/adl/hamt-crdt/README.md @@ -0,0 +1,10 @@ +# HAMT CRDT + +HAMT CRDT is an infinitely scalable key-value store that is implemented using as +a HAMT data structure as a CRDT. Each HAMT node stores up to 16 *key-value-time* +3-tuple entries and up to 17 links to child nodes. + +It supports only a single `.put()` operation. To delete a key, one puts an +`undefined` value. Each operation is accompanied by an ID (timestamp), expressed +as hybrid logical clock (HLC) value. Each key is a single value LWW register, +where the HLC clock is used for determining the winner. diff --git a/src/web3/adl/hamt-crdt/types.ts b/src/web3/adl/hamt-crdt/types.ts new file mode 100644 index 0000000000..7f5546172a --- /dev/null +++ b/src/web3/adl/hamt-crdt/types.ts @@ -0,0 +1,62 @@ +import type {HlcDto} from '../../hlc'; +import type {Cid} from '../../multiformats'; + +export interface HamtApi { + put(key: Uint8Array | string, val: unknown): Promise; + get(key: Uint8Array | string): Promise; + has(key: Uint8Array | string): Promise; + del(key: Uint8Array | string): Promise; +} + +/** Data of the root node of the HAMT. */ +export type HamtRootFrameDto = [ + /** + * CID of the previous state, previous root node. Zero, if there is no + * previous state. + */ + prev: Cid | null, + + /** + * Monotonically incrementing sequence number of the current state + * (increments with each new state). + */ + seq: number, + + /** + * An ordered list of operations which were performed on previous state to + * create the current state. Sorted, where the first operation is the oldest. + */ + ops: HamtOp[], + + /** + * Root level data of the HAMT. + */ + data: HamtFrameDto, +]; + +export type HamtFrameDto = [ + /** + * List of key value pairs stored in this node. + */ + entries: HamtFrameEntry[], + + /** + * Links to child nodes. This array must always be exactly one element larger + * than the `entries` array. Gaps are filled with nulls. + */ + children: (Cid | null)[], +]; + +export type HamtFrameEntry = [key: Uint8Array, val: unknown, id: HlcDto]; + +/** + * Key update operation. + */ +export type HamtOp = [ + /** Key that was updated. */ + key: Uint8Array, + /** New value of the key. */ + val: unknown, + /** ID of the operation as hybrid logical clock. */ + id: HlcDto, +]; From 9a54500657c0125d042369338274529be3c08d84 Mon Sep 17 00:00:00 2001 From: streamich Date: Sun, 10 Mar 2024 22:23:47 +0100 Subject: [PATCH 02/16] =?UTF-8?q?feat:=20=F0=9F=8E=B8=20add=20DAG-CBOR=20c?= =?UTF-8?q?odec?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/json-pack/cbor/CborDecoderBase.ts | 3 +-- src/json-pack/types.ts | 2 ++ src/web3/codec/__tests__/cbor.spec.ts | 10 ++++++++++ src/web3/codec/cbor.ts | 28 +++++++++++++++++++++++++++ src/web3/codec/writer.ts | 3 +++ 5 files changed, 44 insertions(+), 2 deletions(-) create mode 100644 src/web3/codec/__tests__/cbor.spec.ts create mode 100644 src/web3/codec/cbor.ts create mode 100644 src/web3/codec/writer.ts diff --git a/src/json-pack/cbor/CborDecoderBase.ts b/src/json-pack/cbor/CborDecoderBase.ts index 3b27a6ea8d..f99bf7f0a5 100644 --- a/src/json-pack/cbor/CborDecoderBase.ts +++ b/src/json-pack/cbor/CborDecoderBase.ts @@ -13,7 +13,7 @@ export class CborDecoderBase; export interface BinaryJsonEncoder { + encode(value: unknown): Uint8Array; writer: IWriter & IWriterGrowable; writeAny(value: unknown): void; writeNull(): void; @@ -52,6 +53,7 @@ export interface TlvBinaryJsonEncoder { } export interface BinaryJsonDecoder { + decode(uint8: Uint8Array): unknown; reader: IReader & IReaderResettable; read(uint8: Uint8Array): PackValue; } diff --git a/src/web3/codec/__tests__/cbor.spec.ts b/src/web3/codec/__tests__/cbor.spec.ts new file mode 100644 index 0000000000..cac0dd8e72 --- /dev/null +++ b/src/web3/codec/__tests__/cbor.spec.ts @@ -0,0 +1,10 @@ +import {Cid} from '../../multiformats'; +import {cbor} from '../cbor'; + +test('can encode and decode CID', async () => { + const cid = await Cid.fromData(new Uint8Array([1, 2, 3, 4])); + const data = {foo: cid}; + const encoded = cbor.encoder.encode(data); + const decoded = cbor.decoder.decode(encoded); + expect(decoded).toStrictEqual(data); +}); diff --git a/src/web3/codec/cbor.ts b/src/web3/codec/cbor.ts new file mode 100644 index 0000000000..d35a5fe7e8 --- /dev/null +++ b/src/web3/codec/cbor.ts @@ -0,0 +1,28 @@ +import {CborEncoderDag} from '../../json-pack/cbor/CborEncoderDag'; +import {CborDecoderDag} from '../../json-pack/cbor/CborDecoderDag'; +import {Cid} from '../multiformats'; +import {JsonValueCodec} from '../../json-pack/codecs/types'; +import {EncodingFormat} from '../../json-pack/constants'; +import {writer} from './writer'; + +const encoder = new (class extends CborEncoderDag { + public writeUnknown(val: unknown): void { + if (val instanceof Cid) encoder.writeTag(42, val.toBinary()); + else throw new Error('UNKNOWN_VALUE'); + } +})(writer); + +const decoder = new class extends CborDecoderDag { + public readTagRaw(tag: number): Cid | unknown { + const value = this.val(); + if (tag === 42) return Cid.fromBinary(value as Uint8Array); + throw new Error('UNKNOWN_TAG'); + } +} + +export const cbor: JsonValueCodec = { + id: 'DAG-CBOR', + format: EncodingFormat.Cbor, + encoder, + decoder, +}; diff --git a/src/web3/codec/writer.ts b/src/web3/codec/writer.ts new file mode 100644 index 0000000000..072e3b344e --- /dev/null +++ b/src/web3/codec/writer.ts @@ -0,0 +1,3 @@ +import {Writer} from "../../util/buffers/Writer"; + +export const writer = new Writer(); From 4bfa363cbe7bfa95ac5cca9e5b703b1bd97ddd62 Mon Sep 17 00:00:00 2001 From: streamich Date: Sun, 10 Mar 2024 22:25:03 +0100 Subject: [PATCH 03/16] =?UTF-8?q?feat(json-pack):=20=F0=9F=8E=B8=20add=20.?= =?UTF-8?q?decode()=20methods=20to=20decoders?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/json-pack/json/JsonDecoder.ts | 5 +++++ src/json-pack/ubjson/UbjsonDecoder.ts | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/src/json-pack/json/JsonDecoder.ts b/src/json-pack/json/JsonDecoder.ts index 79d0a8d7f1..f2c0b8ec9a 100644 --- a/src/json-pack/json/JsonDecoder.ts +++ b/src/json-pack/json/JsonDecoder.ts @@ -203,6 +203,11 @@ export class JsonDecoder implements BinaryJsonDecoder { return this.readAny(); } + public decode(uint8: Uint8Array): unknown { + this.reader.reset(uint8); + return this.readAny(); + } + public readAny(): PackValue { this.skipWhitespace(); const reader = this.reader; diff --git a/src/json-pack/ubjson/UbjsonDecoder.ts b/src/json-pack/ubjson/UbjsonDecoder.ts index 73c87ebf4f..927eb47120 100644 --- a/src/json-pack/ubjson/UbjsonDecoder.ts +++ b/src/json-pack/ubjson/UbjsonDecoder.ts @@ -11,6 +11,11 @@ export class UbjsonDecoder implements BinaryJsonDecoder { return this.readAny(); } + public decode(uint8: Uint8Array): unknown { + this.reader.reset(uint8); + return this.readAny(); + } + public readAny(): PackValue { const reader = this.reader; const octet = reader.u8(); From 1f0bee84dd73829be0a1b5911334714026a55dc0 Mon Sep 17 00:00:00 2001 From: streamich Date: Sun, 10 Mar 2024 23:42:57 +0100 Subject: [PATCH 04/16] =?UTF-8?q?test:=20=F0=9F=92=8D=20add=20CBOR=20code?= =?UTF-8?q?=20fixture=20test?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/web3/codec/__tests__/cbor.spec.ts | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/web3/codec/__tests__/cbor.spec.ts b/src/web3/codec/__tests__/cbor.spec.ts index cac0dd8e72..7eda57823d 100644 --- a/src/web3/codec/__tests__/cbor.spec.ts +++ b/src/web3/codec/__tests__/cbor.spec.ts @@ -8,3 +8,15 @@ test('can encode and decode CID', async () => { const decoded = cbor.decoder.decode(encoded); expect(decoded).toStrictEqual(data); }); + +test('can encode simplest fixture', async () => { + const data = [2]; + const encoded = cbor.encoder.encode(data); + const decoded = cbor.decoder.decode(encoded); + expect(decoded).toStrictEqual(data); + expect(encoded.length).toBe(2); + expect(encoded[0]).toBe(0x81); + expect(encoded[1]).toBe(0x02); + const cid = await Cid.fromDagCbor(encoded); + expect(cid.toText('base32')).toBe('bafyreihdb57fdysx5h35urvxz64ros7zvywshber7id6t6c6fek37jgyfe'); +}); From e4fb7882eaff009fe47b389b6cf14fd272263461 Mon Sep 17 00:00:00 2001 From: streamich Date: Mon, 11 Mar 2024 00:00:10 +0100 Subject: [PATCH 05/16] =?UTF-8?q?feat:=20=F0=9F=8E=B8=20implement=20IPLD?= =?UTF-8?q?=20codecs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/web3/codec/Codecs.ts | 20 +++++++++++++++++++ .../codec/{ => codecs}/__tests__/cbor.spec.ts | 0 src/web3/codec/{ => codecs}/cbor.ts | 14 ++++++------- src/web3/codec/codecs/raw.ts | 19 ++++++++++++++++++ src/web3/codec/codecs/writer.ts | 3 +++ src/web3/codec/index.ts | 13 ++++++++++++ src/web3/codec/types.ts | 7 +++++++ src/web3/codec/writer.ts | 3 --- 8 files changed, 68 insertions(+), 11 deletions(-) create mode 100644 src/web3/codec/Codecs.ts rename src/web3/codec/{ => codecs}/__tests__/cbor.spec.ts (100%) rename src/web3/codec/{ => codecs}/cbor.ts (57%) create mode 100644 src/web3/codec/codecs/raw.ts create mode 100644 src/web3/codec/codecs/writer.ts create mode 100644 src/web3/codec/index.ts create mode 100644 src/web3/codec/types.ts delete mode 100644 src/web3/codec/writer.ts diff --git a/src/web3/codec/Codecs.ts b/src/web3/codec/Codecs.ts new file mode 100644 index 0000000000..1e4ecf5fbd --- /dev/null +++ b/src/web3/codec/Codecs.ts @@ -0,0 +1,20 @@ +import {MulticodecIpld} from "../multiformats"; +import type {IpldCodec} from "./types"; + +export class Codecs { + protected readonly map = new Map(); + + public set(codec: MulticodecIpld, jsonCodec: IpldCodec): void { + this.map.set(codec, jsonCodec); + } + + public get(codec: MulticodecIpld): IpldCodec | undefined { + return this.map.get(codec); + } + + public getOrThrow(codec: MulticodecIpld): IpldCodec { + const jsonCodec = this.get(codec); + if (!jsonCodec) throw new Error(`Codec ${codec} (0x${codec.toString(16)}) not found`); + return jsonCodec; + } +} diff --git a/src/web3/codec/__tests__/cbor.spec.ts b/src/web3/codec/codecs/__tests__/cbor.spec.ts similarity index 100% rename from src/web3/codec/__tests__/cbor.spec.ts rename to src/web3/codec/codecs/__tests__/cbor.spec.ts diff --git a/src/web3/codec/cbor.ts b/src/web3/codec/codecs/cbor.ts similarity index 57% rename from src/web3/codec/cbor.ts rename to src/web3/codec/codecs/cbor.ts index d35a5fe7e8..9c3387c93b 100644 --- a/src/web3/codec/cbor.ts +++ b/src/web3/codec/codecs/cbor.ts @@ -1,9 +1,8 @@ -import {CborEncoderDag} from '../../json-pack/cbor/CborEncoderDag'; -import {CborDecoderDag} from '../../json-pack/cbor/CborDecoderDag'; -import {Cid} from '../multiformats'; -import {JsonValueCodec} from '../../json-pack/codecs/types'; -import {EncodingFormat} from '../../json-pack/constants'; +import {CborEncoderDag} from '../../../json-pack/cbor/CborEncoderDag'; +import {CborDecoderDag} from '../../../json-pack/cbor/CborDecoderDag'; +import {Cid} from '../../multiformats'; import {writer} from './writer'; +import type {IpldCodec} from '../types'; const encoder = new (class extends CborEncoderDag { public writeUnknown(val: unknown): void { @@ -20,9 +19,8 @@ const decoder = new class extends CborDecoderDag { } } -export const cbor: JsonValueCodec = { - id: 'DAG-CBOR', - format: EncodingFormat.Cbor, +export const cbor: IpldCodec = { + name: 'DAG-CBOR', encoder, decoder, }; diff --git a/src/web3/codec/codecs/raw.ts b/src/web3/codec/codecs/raw.ts new file mode 100644 index 0000000000..18101aa0bc --- /dev/null +++ b/src/web3/codec/codecs/raw.ts @@ -0,0 +1,19 @@ +// import {bufferToUint8Array} from '../../../util/buffers/bufferToUint8Array'; +import type {IpldCodec} from '../types'; + +export const raw: IpldCodec = { + name: 'Raw', + encoder: { + encode: (value: unknown): Uint8Array => { + if (value instanceof Uint8Array) return value; + // if (typeof Buffer !== 'undefined') { + // if(Buffer.isBuffer(value)) return bufferToUint8Array(value as Buffer); + // return bufferToUint8Array(Buffer.from(String(value))); + // } + throw new Error('VALUE_NOT_SUPPORTED'); + }, + }, + decoder: { + decode: (data: Uint8Array): unknown => data, + }, +}; diff --git a/src/web3/codec/codecs/writer.ts b/src/web3/codec/codecs/writer.ts new file mode 100644 index 0000000000..2d093e6085 --- /dev/null +++ b/src/web3/codec/codecs/writer.ts @@ -0,0 +1,3 @@ +import {Writer} from "../../../util/buffers/Writer"; + +export const writer = new Writer(); diff --git a/src/web3/codec/index.ts b/src/web3/codec/index.ts new file mode 100644 index 0000000000..b373fe712c --- /dev/null +++ b/src/web3/codec/index.ts @@ -0,0 +1,13 @@ +import {MulticodecIpld} from "../multiformats"; +import {Codecs} from "./Codecs"; +import {raw} from "./codecs/raw"; +import {cbor} from "./codecs/cbor"; + +export * from './types'; +export * from './Codecs'; + +export const codecs = new Codecs(); + +codecs.set(MulticodecIpld.Raw, raw); +codecs.set(MulticodecIpld.Cbor, cbor); +codecs.set(MulticodecIpld.DagCbor, cbor); diff --git a/src/web3/codec/types.ts b/src/web3/codec/types.ts new file mode 100644 index 0000000000..4d38042ca3 --- /dev/null +++ b/src/web3/codec/types.ts @@ -0,0 +1,7 @@ +import type {BinaryJsonDecoder, BinaryJsonEncoder} from "../../json-pack/types"; + +export interface IpldCodec { + name: string; + encoder: Pick; + decoder: Pick; +} diff --git a/src/web3/codec/writer.ts b/src/web3/codec/writer.ts deleted file mode 100644 index 072e3b344e..0000000000 --- a/src/web3/codec/writer.ts +++ /dev/null @@ -1,3 +0,0 @@ -import {Writer} from "../../util/buffers/Writer"; - -export const writer = new Writer(); From 6c6927e1aabbe64f88897d94640318574a444707 Mon Sep 17 00:00:00 2001 From: streamich Date: Mon, 11 Mar 2024 00:15:00 +0100 Subject: [PATCH 06/16] =?UTF-8?q?feat:=20=F0=9F=8E=B8=20add=20sha256()=20m?= =?UTF-8?q?ethod?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/web3/crypto/index.ts | 10 ++-------- src/web3/crypto/sha256.ts | 6 ++++++ src/web3/crypto/webcrypto.ts | 8 ++++++++ 3 files changed, 16 insertions(+), 8 deletions(-) create mode 100644 src/web3/crypto/sha256.ts create mode 100644 src/web3/crypto/webcrypto.ts diff --git a/src/web3/crypto/index.ts b/src/web3/crypto/index.ts index 51f41f6428..c7c41c22e3 100644 --- a/src/web3/crypto/index.ts +++ b/src/web3/crypto/index.ts @@ -1,8 +1,2 @@ -import {isNode} from '../constants'; - -/** - * Universal Node.js/browser Web Crypto API reference. - * - * @todo Maybe create an isomorphic package for this? - */ -export const crypto: Crypto = isNode ? require('node:crypto').webcrypto : window.crypto; +export * from './webcrypto'; +export * from './sha256'; diff --git a/src/web3/crypto/sha256.ts b/src/web3/crypto/sha256.ts new file mode 100644 index 0000000000..630abd2cb4 --- /dev/null +++ b/src/web3/crypto/sha256.ts @@ -0,0 +1,6 @@ +import {crypto} from './webcrypto'; + +export const sha256 = async (buf: Uint8Array): Promise => { + const ab = await crypto.subtle.digest('SHA-256', buf); + return new Uint8Array(ab); +}; diff --git a/src/web3/crypto/webcrypto.ts b/src/web3/crypto/webcrypto.ts new file mode 100644 index 0000000000..51f41f6428 --- /dev/null +++ b/src/web3/crypto/webcrypto.ts @@ -0,0 +1,8 @@ +import {isNode} from '../constants'; + +/** + * Universal Node.js/browser Web Crypto API reference. + * + * @todo Maybe create an isomorphic package for this? + */ +export const crypto: Crypto = isNode ? require('node:crypto').webcrypto : window.crypto; From ae3282f5316b59944488a94f2774f9b43a022d26 Mon Sep 17 00:00:00 2001 From: streamich Date: Mon, 11 Mar 2024 00:15:20 +0100 Subject: [PATCH 07/16] =?UTF-8?q?fix:=20=F0=9F=90=9B=20add=20extra=20long?= =?UTF-8?q?=20content=20type=20check?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/web3/multiformats/Cid.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/web3/multiformats/Cid.ts b/src/web3/multiformats/Cid.ts index 80c0241d22..7391a6bc1b 100644 --- a/src/web3/multiformats/Cid.ts +++ b/src/web3/multiformats/Cid.ts @@ -90,6 +90,7 @@ export class Cid { const contentType = this.ipldType; if (contentType >= 0b10000000) size += 1; if (contentType >= 0b10000000_0000000) size += 1; + if (contentType >= 0b10000000_0000000_0000000) throw new Error('UNSUPPORTED_IPLD_TYPE'); const hash = this.hash; const hashBuf = hash.buf; size += hashBuf.length; From eb59c09261266d70a567baf42d3097aa64958b4f Mon Sep 17 00:00:00 2001 From: streamich Date: Mon, 11 Mar 2024 00:23:18 +0100 Subject: [PATCH 08/16] =?UTF-8?q?feat:=20=F0=9F=8E=B8=20progress=20on=20HA?= =?UTF-8?q?MT=20implementation?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/util/buffers/cmpUint8Array2.ts | 10 + src/util/buffers/toBuf.ts | 8 + src/web3/adl/hamt-crdt/Hamt.ts | 101 +++++++ src/web3/adl/hamt-crdt/HamtFactory.ts | 12 + src/web3/adl/hamt-crdt/HamtFrame.ts | 207 +++++++++++++ .../adl/hamt-crdt/__tests__/HamtCrdt.spec.ts | 275 ++++++++++++++++++ src/web3/adl/hamt-crdt/constants.ts | 3 + src/web3/adl/hamt-crdt/types.ts | 2 + 8 files changed, 618 insertions(+) create mode 100644 src/util/buffers/cmpUint8Array2.ts create mode 100644 src/util/buffers/toBuf.ts create mode 100644 src/web3/adl/hamt-crdt/Hamt.ts create mode 100644 src/web3/adl/hamt-crdt/HamtFactory.ts create mode 100644 src/web3/adl/hamt-crdt/HamtFrame.ts create mode 100644 src/web3/adl/hamt-crdt/__tests__/HamtCrdt.spec.ts create mode 100644 src/web3/adl/hamt-crdt/constants.ts diff --git a/src/util/buffers/cmpUint8Array2.ts b/src/util/buffers/cmpUint8Array2.ts new file mode 100644 index 0000000000..6bfcbcc38d --- /dev/null +++ b/src/util/buffers/cmpUint8Array2.ts @@ -0,0 +1,10 @@ +export const cmpUint8Array2 = (a: Uint8Array, b: Uint8Array): number => { + const len1 = a.length; + const len2 = b.length; + const len = Math.min(len1, len2); + for (let i = 0; i < len; i++) { + const diffChar = a[i] - b[i]; + if (diffChar !== 0) return diffChar; + } + return len1 - len2; +}; diff --git a/src/util/buffers/toBuf.ts b/src/util/buffers/toBuf.ts new file mode 100644 index 0000000000..576f8a517f --- /dev/null +++ b/src/util/buffers/toBuf.ts @@ -0,0 +1,8 @@ +import {encode} from './utf8/encode'; + +export const toBuf = (str: string): Uint8Array => { + const maxLength = str.length * 4; + const arr = new Uint8Array(maxLength); + const strBufferLength = encode(arr, str, 0, maxLength); + return arr.slice(0, strBufferLength); +}; diff --git a/src/web3/adl/hamt-crdt/Hamt.ts b/src/web3/adl/hamt-crdt/Hamt.ts new file mode 100644 index 0000000000..212cf802d0 --- /dev/null +++ b/src/web3/adl/hamt-crdt/Hamt.ts @@ -0,0 +1,101 @@ +import {HamtFrame} from './HamtFrame'; +import {toDto, type HlcFactory} from '../../hlc'; +import {Cid} from '../../multiformats'; +import {sha256} from '../../crypto'; +import {toBuf} from '../../../util/buffers/toBuf'; +import type {CidCasStruct} from '../../store/cas/CidCasStruct'; +import type * as types from './types'; + +export interface HamtDependencies { + cas: CidCasStruct; + hlcs: HlcFactory; +} + +export class Hamt implements types.HamtApi { + protected _dirty: boolean = false; + protected _loading: Promise | null = null; + protected _root: HamtFrame; + + public prevSeq: number = 0; + public prevId: Cid | null = null; + public ops: types.HamtOp[] = []; + + constructor(protected readonly deps: HamtDependencies) { + this._root = new HamtFrame(deps.cas, null); + } + + public hasChanges(): boolean { + return this._dirty; + } + + public toDto(): types.HamtRootFrameDto { + const id = this.prevId ? this.prevId.bytes : 0; + const [entries, children] = this._root.toDto(); + const ops: HamtOp[] = []; + const dto: HamtRootFrameDto = [entries, children, this.prevSeq + 1, id, ops]; + return dto; + } + + /** Convert any key to buffer and prefix with 4-byte hash. */ + protected async _key(key: Uint8Array | string): Promise { + const keyBuf = typeof key === 'string' ? toBuf(key) : key; + const hash = await sha256(keyBuf); + const buf = new Uint8Array(4 + keyBuf.length); + buf.set(hash.subarray(0, 4), 0); + buf.set(keyBuf, 4); + return buf; + } + + // ------------------------------------------------------------------ HamtApi + + public async load(id: Cid): Promise { + this.prevId = id; + this._loading = this.deps.cas + .get(id) + .then(async (data) => { + const [entries, children, seq] = data as types.HamtRootFrameDto; + this.prevSeq = seq; + this._root.loadData([entries, children], null); + this._loading = null; + }) + .catch(() => {}); + } + + public async put(key: Uint8Array | string, val: unknown): Promise { + if (this._loading) await this._loading; + const hashedKey = await this._key(key); + const id = this.deps.hlcs.inc(); + const idDto = toDto(id); + const op: types.HamtOp = [hashedKey, val, idDto]; + const success = await this._root.put(op); + if (success) this.ops.push(op); + return success; + } + + public async get(key: Uint8Array | string): Promise { + if (this._loading) await this._loading; + const hashedKey = await this._key(key); + return await this._root.get(hashedKey); + } + + public async has(key: Uint8Array | string): Promise { + if (this._loading) await this._loading; + return (await this.get(key)) !== undefined; + } + + public async del(key: Uint8Array | string): Promise { + if (this._loading) await this._loading; + return await this.put(key, undefined); + } + + public async save(): Promise<[head: Cid, affectedIds: Cid[]]> { + const [ids] = await this._root.saveChildren(); + const dto = this.toDto(); + const cid = await this.deps.cas.put(dto); + this.prevId = cid; + this.prevSeq = this.prevSeq + 1; + this.ops = []; + ids.push(cid); + return [cid, ids]; + } +} diff --git a/src/web3/adl/hamt-crdt/HamtFactory.ts b/src/web3/adl/hamt-crdt/HamtFactory.ts new file mode 100644 index 0000000000..15397c35f6 --- /dev/null +++ b/src/web3/adl/hamt-crdt/HamtFactory.ts @@ -0,0 +1,12 @@ +import {Hamt, type HamtDependencies} from './Hamt'; + +export interface HamtFactoryDependencies extends HamtDependencies {} + +export class HamtFactory { + constructor(protected readonly deps: HamtFactoryDependencies) {} + + public make(): Hamt { + const hamt = new Hamt(this.deps); + return hamt; + } +} diff --git a/src/web3/adl/hamt-crdt/HamtFrame.ts b/src/web3/adl/hamt-crdt/HamtFrame.ts new file mode 100644 index 0000000000..280c243a5c --- /dev/null +++ b/src/web3/adl/hamt-crdt/HamtFrame.ts @@ -0,0 +1,207 @@ +import {cmpUint8Array2} from '../../../util/buffers/cmpUint8Array2'; +import {Defer} from 'thingies/lib/Defer'; +import {cmpDto} from '../../hlc'; +import {CidCasStruct} from '../../store/cas/CidCasStruct'; +import {Cid} from '../../multiformats'; +import {HamtConstraints} from './constants'; +import type * as types from './types'; + +export class HamtFrame { + protected _entries: types.HamtFrameEntry[] = []; + protected _children: (HamtFrame | null)[] = [null]; + protected _loaded: boolean = false; + protected _loading: Defer | null = null; + protected _dirty: boolean = false; + + constructor(protected readonly cas: CidCasStruct, public id: Cid | null) {} + + /** + * Recursively find a key value from current node or any of its children. + * + * @param key The key to fetch. + * @returns Returns the value if found, otherwise undefined. + */ + public async get(key: Uint8Array): Promise { + if (!this._loaded) await this.ensureLoaded(); + const entries = this._entries; + const length = entries.length; + for (let i = 0; i < length; i++) { + const entry = entries[i]; + const currentKey = entry[0]; + const comparison = cmpUint8Array2(currentKey, key); + if (comparison === 0) return entry[1]; + const isKeySmallerThanCurrentKey = comparison > 0; + if (isKeySmallerThanCurrentKey) { + const child = this._children[i]; + if (!child) return undefined; + return await child.get(key); + } + } + const lastChild = this._children[length]; + if (!lastChild) return undefined; + return await lastChild.get(key); + } + + public async has(key: Uint8Array): Promise { + return (await this.get(key)) !== undefined; + } + + /** + * Insert or overwrite a key value pair in current node or any of its children. + * + * @param id HLC ID of the key. + * @param key Key to put. + * @param val Key value to put. + * @returns Returns true if the key was inserted. Insertion can fail if the + * ID of the insert operation is lower than the ID of the last write. + */ + public async put(op: types.HamtOp): Promise { + if (!this._loaded) await this.ensureLoaded(); + const [key, , id] = op; + const entries = this._entries; + const length = entries.length; + const insertInChild = length >= HamtConstraints.MaxEntriesPerFrame; + for (let i = 0; i < length; i++) { + const entry = entries[i]; + const currentKey = entry[0]; + const comparison = cmpUint8Array2(currentKey, key); + + // Replace existing entry if keys are equal. + if (comparison === 0) { + const oldId = entry[2]; + if (cmpDto(oldId, id) >= 0) return false; + this._entries[i] = op; + this._markDirty(); + return true; + } + const isKeySmallerThanCurrentKey = comparison > 0; + if (isKeySmallerThanCurrentKey) { + if (insertInChild) { + // Insert at child node. + const wasInserted = await this._putAtChild(i, op); + if (wasInserted) this._markDirty(); + return wasInserted; + } else { + // Insert at current node, but shifting entries to the right. + this._entries.splice(i, 0, op); + this._children.splice(i, 0, null); + this._markDirty(); + return true; + } + } + } + + // Insert into the last child. + if (insertInChild) { + const wasInserted = await this._putAtChild(length, op); + if (wasInserted) this._markDirty(); + return wasInserted; + } + + // Append entry at the end of current block. + this._entries.push(op); + this._children.push(null); + this._markDirty(); + return true; + } + + protected _markDirty() { + this._dirty = true; + this.id = null; + } + + private async _putAtChild(i: number, op: HamtOp): Promise { + let child = this._children[i]; + if (!child) child = this._children[i] = new HamtFrame(this.cas, null); + return await child.put(op); + } + + /** + * Save current node and all of its children. + * + * @returns Returns CID of current node, and a list of all affected CIDs, + * including the current CID. + */ + public async save(): Promise<[id: Cid, affected: Cid[]]> { + if (!this._loaded) await this.ensureLoaded(); + if (this.id && !this._dirty) return [this.id, []]; + const [ids, children] = await this.saveChildren(); + const data: types.HamtFrameDto = [this._entries, children]; + const encoded = encoder.encode(data); + const cid = await this.cas.put(encoded); + this.id = cid; + ids.push(cid); + return [cid, ids]; + } + + /** + * Saves all "dirty" children and returns a lift of all children. + * + * @returns Returns a list of stored CIDs and a all children of the current node, + * even the children which were not saved. + */ + public async saveChildren(): Promise<[affectedIds: Cid[], children: (Cid | null)[]]> { + const ids: Cid[] = []; + const children: (Cid | null)[] = []; + const length = this._children.length; + for (let i = 0; i < length; i++) { + const child = this._children[i]; + if (!child) { + children.push(0); + continue; + } + const [childId, allIds] = await child.save(); + ids.push(...allIds); + children.push(childId.bytes); + } + return [ids, children]; + } + + public ensureLoaded(): Promise { + if (this._loading) return this._loading.promise; + if (this._loaded) return Promise.resolve(); + if (!this.id) return Promise.resolve(); + return this.load(this.id); + } + + /** + * Load the current node by CID from CAS. + * + * @param id CID of the node to load. + */ + public async load(id: Cid): Promise { + this._loading = new Defer(); + const data = await this.cas.get(id) as types.HamtFrameDto; + this.loadData(data, id); + this._loading.resolve(); + this._loading = null; + } + + /** + * Load the current node from known data. Provided data will be mutated + * internally, so it should not be used after this method is called. + * + * @param data Serialized data of the node to load. + * @param id CID of the node to load, or null if CID is not known. + */ + public loadData(data: types.HamtFrameDto, id: Cid | null) { + this.id = id; + const [entries, children] = data; + this._entries = entries; + this._children = []; + const length = children.length; + for (let i = 0; i < length; i++) { + const childId = children[i]; + const child = childId ? new HamtFrame(this.cas, CID.decode(childId)) : null; + this._children.push(child); + } + this._loaded = true; + } + + public toDto(): types.HamtFrameDto { + const children: (Cid | null)[] = []; + for (const child of this._children) children.push(child && child.id ? child.id.bytes : 0); + const dto: types.HamtFrameDto = [this._entries, children]; + return dto; + } +} diff --git a/src/web3/adl/hamt-crdt/__tests__/HamtCrdt.spec.ts b/src/web3/adl/hamt-crdt/__tests__/HamtCrdt.spec.ts new file mode 100644 index 0000000000..a123874030 --- /dev/null +++ b/src/web3/adl/hamt-crdt/__tests__/HamtCrdt.spec.ts @@ -0,0 +1,275 @@ +/** @jest-environment node */ +import {MemoryLevel} from 'memory-level'; +import {b} from '../../../../util/buffer/b'; +import {HlcFactory} from '../../../hlc'; +import {LevelCAS} from '../../LevelCAS'; +import {MemoryCAS} from '../../MemoryCAS'; +import {HamtCrdt} from '../HamtCrdt'; + +const setup = () => { + const hlc = new HlcFactory({ + getTs: () => Math.floor(Date.now() / 1000), + processId: 123, + }); + const db = new MemoryLevel({ + keyEncoding: 'view', + valueEncoding: 'view', + }); + const cas = new LevelCAS({db}); + const hamt = new HamtCrdt({ + hlc, + cas, + }); + return { + hlc, + db, + cas, + hamt, + }; +}; + +const toArr = (buf: Uint8Array): number[] => { + const arr: number[] = []; + for (let i = 0; i < buf.length; i++) arr.push(buf[i]); + return arr; +}; + +test('can store file in IPFS', async () => { + const {cas} = setup(); + const cid = await cas.put(b(1, 2, 3)); + const data = await cas.get(cid); + expect(data).toStrictEqual(b(1, 2, 3)); +}); + +describe('HamtCrdt', () => { + test('new database has no changes', async () => { + const {hamt} = setup(); + const res = hamt.hasChanges(); + expect(res).toBe(false); + }); + + describe('.get()', () => { + test('returns undefined in empty database', async () => { + const {hamt} = setup(); + const res1 = await hamt.get(b(1, 2, 3)); + const res2 = await hamt.get('test'); + expect(res1).toBe(undefined); + expect(res2).toBe(undefined); + }); + + test('returns undefined in empty database', async () => { + const {hamt} = setup(); + const res1 = await hamt.get(b(1, 2, 3)); + const res2 = await hamt.get('test'); + expect(res1).toBe(undefined); + expect(res2).toBe(undefined); + }); + }); + + describe('.put()', () => { + test('can store a string key', async () => { + const {hamt} = setup(); + const res1 = await hamt.put('test', b(1, 2, 3)); + expect(res1).toBe(true); + const res2 = await hamt.get('test'); + expect(res2).toStrictEqual(b(1, 2, 3)); + }); + + test('can store a multiple keys', async () => { + const {hamt} = setup(); + const res1 = await hamt.put('/@user1', b(1, 2, 3)); + const res2 = await hamt.put('/@user2', b(4, 5, 6)); + const res3 = await hamt.put('/@user3', b(7, 7, 7)); + expect(res1).toBe(true); + expect(res2).toBe(true); + expect(res3).toBe(true); + const res4 = await hamt.get('/@user1'); + const res5 = await hamt.get('/@user2'); + const res6 = await hamt.get('/@user3'); + expect(res4).toStrictEqual(b(1, 2, 3)); + expect(res5).toStrictEqual(b(4, 5, 6)); + expect(res6).toStrictEqual(b(7, 7, 7)); + }); + + test('can store into a binary key', async () => { + const {hamt} = setup(); + const res1 = await hamt.put(b(69), b(1, 2, 3)); + expect(res1).toBe(true); + const res2 = await hamt.get(b(69)); + expect(res2).toStrictEqual(b(1, 2, 3)); + }); + + test('can store into an empty key', async () => { + const {hamt} = setup(); + const res1 = await hamt.put(b(), b(1, 2, 3)); + expect(res1).toBe(true); + const res2 = await hamt.get(b()); + expect(res2).toStrictEqual(b(1, 2, 3)); + }); + + test('can overwrite a key', async () => { + const {hamt} = setup(); + await hamt.put('foo', b(1, 2, 3)); + await hamt.put('foo', b(4, 5, 6)); + const res2 = await hamt.get('foo'); + expect(res2).toStrictEqual(b(4, 5, 6)); + }); + + test('can add more than 16 keys', async () => { + const {hamt} = setup(); + for (let i = 0; i < 30; i++) { + await hamt.put('foo-' + i, b(i)); + } + for (let i = 0; i < 30; i++) { + const res = await hamt.get('foo-' + i); + expect(res).toStrictEqual(b(i)); + } + }); + + test('can store any serializable value', async () => { + const {hamt} = setup(); + const res1 = await hamt.put(b(), {foo: 123, bar: [true, false]}); + expect(res1).toBe(true); + const res2 = await hamt.get(b()); + expect(res2).toStrictEqual({foo: 123, bar: [true, false]}); + }); + }); + + describe('.has()', () => { + test('returns false for missing keys', async () => { + const {hamt} = setup(); + const res1 = await hamt.has('a'); + const res2 = await hamt.has('b'); + expect(res1).toBe(false); + expect(res2).toBe(false); + }); + + test('returns true for existing keys', async () => { + const {hamt} = setup(); + await hamt.put('a', b()); + await hamt.put('b', b(1, 2, 3)); + const res1 = await hamt.has('a'); + const res2 = await hamt.has('b'); + expect(res1).toBe(true); + expect(res2).toBe(true); + }); + }); + + describe('.del()', () => { + test('can delete non-existing keys', async () => { + const {hamt} = setup(); + const res1 = await hamt.del('a'); + const res2 = await hamt.del('b'); + expect(res1).toBe(true); + expect(res2).toBe(true); + const res3 = await hamt.get('a'); + const res4 = await hamt.get('b'); + expect(res3).toBe(undefined); + expect(res4).toBe(undefined); + }); + + test('can delete existing key', async () => { + const {hamt} = setup(); + await hamt.put('a', b()); + await hamt.put('b', b(1, 2, 3)); + const res1 = await hamt.del('a'); + const res2 = await hamt.del('b'); + expect(res1).toBe(true); + expect(res2).toBe(true); + const res3 = await hamt.get('a'); + const res4 = await hamt.get('b'); + expect(res3).toBe(undefined); + expect(res4).toBe(undefined); + }); + }); + + describe('.save()', () => { + test('can persist empty HAMT', async () => { + const {hamt, hlc, cas} = setup(); + const [cid] = await hamt.save(); + expect(cid).toBeDefined(); + const hamt2 = new HamtCrdt({ + hlc, + cas, + }); + await hamt2.load(cid); + }); + + test('can save a single key', async () => { + const hlc = new HlcFactory({ + getTs: () => Math.floor(Date.now() / 1000), + processId: 123, + }); + const cas = new MemoryCAS(); + const hamt = new HamtCrdt({ + hlc, + cas, + }); + const data = 111; + await hamt.put('a', b(data)); + const size = cas._map.size; + expect(size).toBe(0); + const [cid] = await hamt.save(); + expect(cas._map.size).toBe(size + 1); + const blob = await cas.get(cid); + const found = toArr(blob).findIndex(octet => octet === data); + expect(found > -1).toBe(true); + }); + + test('can load saved data', async () => { + const {hamt, cas, hlc} = setup(); + await hamt.put('a', b(123)); + const [cid] = await hamt.save(); + const hamt2 = new HamtCrdt({ + hlc, + cas, + }); + const res1 = await hamt2.get('a'); + expect(res1).toBe(undefined); + await hamt2.load(cid); + const res2 = await hamt2.get('a'); + expect(res2).toStrictEqual(b(123)); + }); + + test('can save and load more than 16 keys of data', async () => { + const {hamt, cas, hlc} = setup(); + const keys = 1111; + for (let i = 0; i < keys; i++) { + await hamt.put('a:' + i, b(i, i + 1, i + 2)); + } + const [cid, all] = await hamt.save(); + const hamt2 = new HamtCrdt({ + hlc, + cas, + }); + await hamt2.load(cid); + for (let i = 0; i < keys; i++) { + const res = await hamt2.get('a:' + i); + expect(res).toStrictEqual(b(i, i + 1, i + 2)); + } + }); + }); + + describe('operations', () => { + test('stores operations as keys are edited', async () => { + const {hamt} = setup(); + expect(hamt.ops.length).toBe(0); + await hamt.put(b(0), 0); + expect(hamt.ops.length).toBe(1); + expect(hamt.ops).toStrictEqual([[expect.any(Uint8Array), 0, expect.any(Array)]]); + await hamt.put(b(1), 1); + expect(hamt.ops.length).toBe(2); + expect(hamt.ops).toStrictEqual([ + [expect.any(Uint8Array), 0, expect.any(Array)], + [expect.any(Uint8Array), 1, expect.any(Array)], + ]); + await hamt.del(b(0)); + expect(hamt.ops.length).toBe(3); + expect(hamt.ops).toStrictEqual([ + [expect.any(Uint8Array), 0, expect.any(Array)], + [expect.any(Uint8Array), 1, expect.any(Array)], + [hamt.ops[0][0], undefined, expect.any(Array)], + ]); + }); + }); +}); diff --git a/src/web3/adl/hamt-crdt/constants.ts b/src/web3/adl/hamt-crdt/constants.ts new file mode 100644 index 0000000000..c12b921b13 --- /dev/null +++ b/src/web3/adl/hamt-crdt/constants.ts @@ -0,0 +1,3 @@ +export const enum HamtConstraints { + MaxEntriesPerFrame = 16, +} diff --git a/src/web3/adl/hamt-crdt/types.ts b/src/web3/adl/hamt-crdt/types.ts index 7f5546172a..ee9e54d8d8 100644 --- a/src/web3/adl/hamt-crdt/types.ts +++ b/src/web3/adl/hamt-crdt/types.ts @@ -2,10 +2,12 @@ import type {HlcDto} from '../../hlc'; import type {Cid} from '../../multiformats'; export interface HamtApi { + load(id: Cid): Promise; put(key: Uint8Array | string, val: unknown): Promise; get(key: Uint8Array | string): Promise; has(key: Uint8Array | string): Promise; del(key: Uint8Array | string): Promise; + save(): Promise<[head: Cid, affected: Cid[]]>; } /** Data of the root node of the HAMT. */ From bb177f4a931d762d397b82d6bef74cfea5bf0430 Mon Sep 17 00:00:00 2001 From: streamich Date: Mon, 11 Mar 2024 08:39:27 +0100 Subject: [PATCH 09/16] =?UTF-8?q?refactor:=20=F0=9F=92=A1=20cleanup=20Hamt?= =?UTF-8?q?Frame=20code?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/web3/adl/hamt-crdt/HamtFrame.ts | 141 +++++++++++++++------------- 1 file changed, 74 insertions(+), 67 deletions(-) diff --git a/src/web3/adl/hamt-crdt/HamtFrame.ts b/src/web3/adl/hamt-crdt/HamtFrame.ts index 280c243a5c..0f39f3fa7f 100644 --- a/src/web3/adl/hamt-crdt/HamtFrame.ts +++ b/src/web3/adl/hamt-crdt/HamtFrame.ts @@ -1,9 +1,10 @@ +import {Defer} from 'thingies/es2020/Defer'; import {cmpUint8Array2} from '../../../util/buffers/cmpUint8Array2'; -import {Defer} from 'thingies/lib/Defer'; import {cmpDto} from '../../hlc'; import {CidCasStruct} from '../../store/cas/CidCasStruct'; import {Cid} from '../../multiformats'; import {HamtConstraints} from './constants'; +import {mutex} from 'thingies/es2020/mutex'; import type * as types from './types'; export class HamtFrame { @@ -11,17 +12,56 @@ export class HamtFrame { protected _children: (HamtFrame | null)[] = [null]; protected _loaded: boolean = false; protected _loading: Defer | null = null; + /** Maybe instead of `_dirty`, just consider `id` === `null` to mean there are unsaved changes. */ protected _dirty: boolean = false; constructor(protected readonly cas: CidCasStruct, public id: Cid | null) {} + protected async ensureLoaded(): Promise { + if (this._loading) return this._loading.promise; + if (this._loaded) return; + if (!this.id) return; + if (this.id) await this.load(); + } + /** - * Recursively find a key value from current node or any of its children. + * Load the current node by CID from CAS. * - * @param key The key to fetch. - * @returns Returns the value if found, otherwise undefined. + * @param id CID of the node to load. */ - public async get(key: Uint8Array): Promise { + @mutex + protected async load(): Promise { + const id = this.id; + if (!id) throw new Error('ID_NOT_SET'); + this._loading = new Defer(); + const data = await this.cas.get(id) as types.HamtFrameDto; + this.loadDto(data, id); + this._loading.resolve(); + this._loading = null; + } + + /** + * Load the current node from known data. Provided data will be mutated + * internally, so it MUST not be used after this method is called. + * + * @param data Serialized data of the node to load. + * @param id CID of the node to load, or null if CID is not known. + */ + protected loadDto(data: types.HamtFrameDto, id: Cid | null) { + this.id = id; + const [entries, children] = data; + this._entries = entries; + this._children = []; + const length = children.length; + for (let i = 0; i < length; i++) { + const childCid = children[i]; + const child = childCid ? new HamtFrame(this.cas, childCid) : null; + this._children.push(child); + } + this._loaded = true; + } + + public async getEntry(key: Uint8Array): Promise { if (!this._loaded) await this.ensureLoaded(); const entries = this._entries; const length = entries.length; @@ -29,17 +69,29 @@ export class HamtFrame { const entry = entries[i]; const currentKey = entry[0]; const comparison = cmpUint8Array2(currentKey, key); - if (comparison === 0) return entry[1]; + if (comparison === 0) return entry; const isKeySmallerThanCurrentKey = comparison > 0; if (isKeySmallerThanCurrentKey) { const child = this._children[i]; if (!child) return undefined; - return await child.get(key); + return await child.getEntry(key); } } const lastChild = this._children[length]; if (!lastChild) return undefined; - return await lastChild.get(key); + return await lastChild.getEntry(key); + } + + /** + * Recursively find a key value from current node or any of its children. + * + * @param key The key to fetch. + * @returns Returns the value if found, otherwise undefined. + */ + public async get(key: Uint8Array): Promise { + const entry = await this.getEntry(key); + if (!entry) return undefined; + return entry[1]; } public async has(key: Uint8Array): Promise { @@ -65,7 +117,6 @@ export class HamtFrame { const entry = entries[i]; const currentKey = entry[0]; const comparison = cmpUint8Array2(currentKey, key); - // Replace existing entry if keys are equal. if (comparison === 0) { const oldId = entry[2]; @@ -90,14 +141,12 @@ export class HamtFrame { } } } - // Insert into the last child. if (insertInChild) { const wasInserted = await this._putAtChild(length, op); if (wasInserted) this._markDirty(); return wasInserted; } - // Append entry at the end of current block. this._entries.push(op); this._children.push(null); @@ -110,7 +159,7 @@ export class HamtFrame { this.id = null; } - private async _putAtChild(i: number, op: HamtOp): Promise { + private async _putAtChild(i: number, op: types.HamtOp): Promise { let child = this._children[i]; if (!child) child = this._children[i] = new HamtFrame(this.cas, null); return await child.put(op); @@ -124,83 +173,41 @@ export class HamtFrame { */ public async save(): Promise<[id: Cid, affected: Cid[]]> { if (!this._loaded) await this.ensureLoaded(); + // TODO: Maybe throw if there are no changes. if (this.id && !this._dirty) return [this.id, []]; - const [ids, children] = await this.saveChildren(); + const [children, affected] = await this.saveChildren(); const data: types.HamtFrameDto = [this._entries, children]; - const encoded = encoder.encode(data); - const cid = await this.cas.put(encoded); + const cid = await this.cas.put(data); this.id = cid; - ids.push(cid); - return [cid, ids]; + affected.push(cid); + return [cid, affected]; } /** - * Saves all "dirty" children and returns a lift of all children. + * Saves all "dirty" children and returns a list of all children. * * @returns Returns a list of stored CIDs and a all children of the current node, * even the children which were not saved. */ - public async saveChildren(): Promise<[affectedIds: Cid[], children: (Cid | null)[]]> { + public async saveChildren(): Promise<[children: (Cid | null)[], affected: Cid[]]> { const ids: Cid[] = []; const children: (Cid | null)[] = []; const length = this._children.length; for (let i = 0; i < length; i++) { const child = this._children[i]; if (!child) { - children.push(0); + children.push(null); continue; } - const [childId, allIds] = await child.save(); - ids.push(...allIds); - children.push(childId.bytes); - } - return [ids, children]; - } - - public ensureLoaded(): Promise { - if (this._loading) return this._loading.promise; - if (this._loaded) return Promise.resolve(); - if (!this.id) return Promise.resolve(); - return this.load(this.id); - } - - /** - * Load the current node by CID from CAS. - * - * @param id CID of the node to load. - */ - public async load(id: Cid): Promise { - this._loading = new Defer(); - const data = await this.cas.get(id) as types.HamtFrameDto; - this.loadData(data, id); - this._loading.resolve(); - this._loading = null; - } - - /** - * Load the current node from known data. Provided data will be mutated - * internally, so it should not be used after this method is called. - * - * @param data Serialized data of the node to load. - * @param id CID of the node to load, or null if CID is not known. - */ - public loadData(data: types.HamtFrameDto, id: Cid | null) { - this.id = id; - const [entries, children] = data; - this._entries = entries; - this._children = []; - const length = children.length; - for (let i = 0; i < length; i++) { - const childId = children[i]; - const child = childId ? new HamtFrame(this.cas, CID.decode(childId)) : null; - this._children.push(child); + const [childCid, affected] = await child.save(); + ids.push(...affected); + children.push(childCid); } - this._loaded = true; + return [children, ids]; } public toDto(): types.HamtFrameDto { - const children: (Cid | null)[] = []; - for (const child of this._children) children.push(child && child.id ? child.id.bytes : 0); + const children = this._children.map((child) => child ? child.id : null); const dto: types.HamtFrameDto = [this._entries, children]; return dto; } From 84bd0517fd7e2f32a0f3945a440e63243d0288e7 Mon Sep 17 00:00:00 2001 From: streamich Date: Mon, 11 Mar 2024 10:05:09 +0100 Subject: [PATCH 10/16] =?UTF-8?q?feat:=20=F0=9F=8E=B8=20cleanup=20Hamt=20C?= =?UTF-8?q?RDT=20code?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- package.json | 2 +- src/web3/adl/hamt-crdt/Hamt.ts | 88 +++++++++++++++-------------- src/web3/adl/hamt-crdt/HamtFrame.ts | 22 ++++---- yarn.lock | 8 +-- 4 files changed, 62 insertions(+), 58 deletions(-) diff --git a/package.json b/package.json index b4c89b3eac..ae954ab3fe 100644 --- a/package.json +++ b/package.json @@ -83,7 +83,7 @@ "arg": "^5.0.2", "hyperdyperid": "^1.2.0", "multibase": "^4.0.6", - "thingies": "^1.17.0" + "thingies": "^1.18.0" }, "devDependencies": { "@automerge/automerge": "2.1.7", diff --git a/src/web3/adl/hamt-crdt/Hamt.ts b/src/web3/adl/hamt-crdt/Hamt.ts index 212cf802d0..9a978d40c8 100644 --- a/src/web3/adl/hamt-crdt/Hamt.ts +++ b/src/web3/adl/hamt-crdt/Hamt.ts @@ -1,5 +1,7 @@ +import {Defer} from 'thingies/es2020/Defer'; +import {concurrency} from 'thingies/es2020/concurrencyDecorator'; import {HamtFrame} from './HamtFrame'; -import {toDto, type HlcFactory} from '../../hlc'; +import * as hlc from '../../hlc'; import {Cid} from '../../multiformats'; import {sha256} from '../../crypto'; import {toBuf} from '../../../util/buffers/toBuf'; @@ -8,16 +10,17 @@ import type * as types from './types'; export interface HamtDependencies { cas: CidCasStruct; - hlcs: HlcFactory; + hlcs: hlc.HlcFactory; } export class Hamt implements types.HamtApi { + protected _root: HamtFrame; protected _dirty: boolean = false; protected _loading: Promise | null = null; - protected _root: HamtFrame; - public prevSeq: number = 0; - public prevId: Cid | null = null; + public cid: Cid | null = null; + public prev: Cid | null = null; + public seq: number = 0; public ops: types.HamtOp[] = []; constructor(protected readonly deps: HamtDependencies) { @@ -28,44 +31,32 @@ export class Hamt implements types.HamtApi { return this._dirty; } - public toDto(): types.HamtRootFrameDto { - const id = this.prevId ? this.prevId.bytes : 0; - const [entries, children] = this._root.toDto(); - const ops: HamtOp[] = []; - const dto: HamtRootFrameDto = [entries, children, this.prevSeq + 1, id, ops]; - return dto; - } - - /** Convert any key to buffer and prefix with 4-byte hash. */ - protected async _key(key: Uint8Array | string): Promise { - const keyBuf = typeof key === 'string' ? toBuf(key) : key; - const hash = await sha256(keyBuf); - const buf = new Uint8Array(4 + keyBuf.length); - buf.set(hash.subarray(0, 4), 0); - buf.set(keyBuf, 4); - return buf; - } - // ------------------------------------------------------------------ HamtApi - public async load(id: Cid): Promise { - this.prevId = id; - this._loading = this.deps.cas - .get(id) - .then(async (data) => { - const [entries, children, seq] = data as types.HamtRootFrameDto; - this.prevSeq = seq; - this._root.loadData([entries, children], null); - this._loading = null; - }) - .catch(() => {}); + public async load(cid: Cid): Promise { + this.cid = cid; + const future = new Defer(); + this._loading = future.promise; + try { + const [prev, seq, ops, data] = await this.deps.cas.get(cid) as types.HamtRootFrameDto; + this.prev = prev; + this.seq = seq; + this._root.loadDto(data, null); + future.resolve(); + } catch (err) { + future.reject(err); + } finally { + this._loading = null; + } + return future.promise; } + @concurrency(1) public async put(key: Uint8Array | string, val: unknown): Promise { if (this._loading) await this._loading; const hashedKey = await this._key(key); const id = this.deps.hlcs.inc(); - const idDto = toDto(id); + const idDto = hlc.toDto(id); const op: types.HamtOp = [hashedKey, val, idDto]; const success = await this._root.put(op); if (success) this.ops.push(op); @@ -78,6 +69,16 @@ export class Hamt implements types.HamtApi { return await this._root.get(hashedKey); } + /** Convert any key to buffer and prefix with 4-byte hash. */ + protected async _key(key: Uint8Array | string): Promise { + const keyBuf = typeof key === 'string' ? toBuf(key) : key; + const hash = await sha256(keyBuf); + const buf = new Uint8Array(4 + keyBuf.length); + buf.set(hash.subarray(0, 4), 0); + buf.set(keyBuf, 4); + return buf; + } + public async has(key: Uint8Array | string): Promise { if (this._loading) await this._loading; return (await this.get(key)) !== undefined; @@ -88,14 +89,17 @@ export class Hamt implements types.HamtApi { return await this.put(key, undefined); } - public async save(): Promise<[head: Cid, affectedIds: Cid[]]> { - const [ids] = await this._root.saveChildren(); - const dto = this.toDto(); + public async save(): Promise<[head: Cid, affected: Cid[]]> { + const [, affected] = await this._root.saveChildren(); + const prev = this.cid; + const seq = this.seq + 1; + const dto: types.HamtRootFrameDto = [prev, seq, this.ops, this._root.toDto()]; const cid = await this.deps.cas.put(dto); - this.prevId = cid; - this.prevSeq = this.prevSeq + 1; + this.cid = cid; + this.prev = prev; + this.seq = seq; this.ops = []; - ids.push(cid); - return [cid, ids]; + affected.push(cid); + return [cid, affected]; } } diff --git a/src/web3/adl/hamt-crdt/HamtFrame.ts b/src/web3/adl/hamt-crdt/HamtFrame.ts index 0f39f3fa7f..01d1c999ef 100644 --- a/src/web3/adl/hamt-crdt/HamtFrame.ts +++ b/src/web3/adl/hamt-crdt/HamtFrame.ts @@ -15,13 +15,13 @@ export class HamtFrame { /** Maybe instead of `_dirty`, just consider `id` === `null` to mean there are unsaved changes. */ protected _dirty: boolean = false; - constructor(protected readonly cas: CidCasStruct, public id: Cid | null) {} + constructor(protected readonly cas: CidCasStruct, public cid: Cid | null) {} protected async ensureLoaded(): Promise { if (this._loading) return this._loading.promise; if (this._loaded) return; - if (!this.id) return; - if (this.id) await this.load(); + if (!this.cid) return; + if (this.cid) await this.load(); } /** @@ -31,7 +31,7 @@ export class HamtFrame { */ @mutex protected async load(): Promise { - const id = this.id; + const id = this.cid; if (!id) throw new Error('ID_NOT_SET'); this._loading = new Defer(); const data = await this.cas.get(id) as types.HamtFrameDto; @@ -45,10 +45,10 @@ export class HamtFrame { * internally, so it MUST not be used after this method is called. * * @param data Serialized data of the node to load. - * @param id CID of the node to load, or null if CID is not known. + * @param cid CID of the node to load, or null if CID is not known. */ - protected loadDto(data: types.HamtFrameDto, id: Cid | null) { - this.id = id; + public loadDto(data: types.HamtFrameDto, cid: Cid | null) { + this.cid = cid; const [entries, children] = data; this._entries = entries; this._children = []; @@ -156,7 +156,7 @@ export class HamtFrame { protected _markDirty() { this._dirty = true; - this.id = null; + this.cid = null; } private async _putAtChild(i: number, op: types.HamtOp): Promise { @@ -174,11 +174,11 @@ export class HamtFrame { public async save(): Promise<[id: Cid, affected: Cid[]]> { if (!this._loaded) await this.ensureLoaded(); // TODO: Maybe throw if there are no changes. - if (this.id && !this._dirty) return [this.id, []]; + if (this.cid && !this._dirty) return [this.cid, []]; const [children, affected] = await this.saveChildren(); const data: types.HamtFrameDto = [this._entries, children]; const cid = await this.cas.put(data); - this.id = cid; + this.cid = cid; affected.push(cid); return [cid, affected]; } @@ -207,7 +207,7 @@ export class HamtFrame { } public toDto(): types.HamtFrameDto { - const children = this._children.map((child) => child ? child.id : null); + const children = this._children.map((child) => child ? child.cid : null); const dto: types.HamtFrameDto = [this._entries, children]; return dto; } diff --git a/yarn.lock b/yarn.lock index f3ea9152e2..9643b61e24 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5318,10 +5318,10 @@ test-exclude@^6.0.0: glob "^7.1.4" minimatch "^3.0.4" -thingies@^1.17.0: - version "1.17.0" - resolved "https://registry.yarnpkg.com/thingies/-/thingies-1.17.0.tgz#c172c423e5d3000a44666fc03723a29004ef0dcc" - integrity sha512-cpwd9eT9b7vcWDlJrIsfS+kBrcHEMST6DG08pOJoDkl/ToSwKYkW5x8RfBp5+k+iz1bT/dqEpoZOewg8I2lycw== +thingies@^1.18.0: + version "1.18.0" + resolved "https://registry.yarnpkg.com/thingies/-/thingies-1.18.0.tgz#827141872d96f3c3c2c0b432ab0dfdb581b4b4ac" + integrity sha512-WiB26BQP0MF47Bbvbq0P19KpyfrvdTK07L8xnltobpZ/aJPmu52CBGhYjLsnFgjyawmusJ0gVkTplnnoz2hBkQ== thunky@^1.0.2: version "1.1.0" From c836f8a0b52b1a6ba4c2998a136bbe8e9248e810 Mon Sep 17 00:00:00 2001 From: streamich Date: Mon, 11 Mar 2024 10:05:51 +0100 Subject: [PATCH 11/16] =?UTF-8?q?style:=20=F0=9F=92=84=20run=20Prettier?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/web3/adl/hamt-crdt/Hamt.ts | 2 +- src/web3/adl/hamt-crdt/HamtFrame.ts | 9 ++++++--- src/web3/adl/hamt-crdt/__tests__/HamtCrdt.spec.ts | 2 +- src/web3/codec/Codecs.ts | 4 ++-- src/web3/codec/codecs/cbor.ts | 4 ++-- src/web3/codec/codecs/writer.ts | 2 +- src/web3/codec/index.ts | 8 ++++---- src/web3/codec/types.ts | 2 +- 8 files changed, 18 insertions(+), 15 deletions(-) diff --git a/src/web3/adl/hamt-crdt/Hamt.ts b/src/web3/adl/hamt-crdt/Hamt.ts index 9a978d40c8..bd8245361e 100644 --- a/src/web3/adl/hamt-crdt/Hamt.ts +++ b/src/web3/adl/hamt-crdt/Hamt.ts @@ -38,7 +38,7 @@ export class Hamt implements types.HamtApi { const future = new Defer(); this._loading = future.promise; try { - const [prev, seq, ops, data] = await this.deps.cas.get(cid) as types.HamtRootFrameDto; + const [prev, seq, ops, data] = (await this.deps.cas.get(cid)) as types.HamtRootFrameDto; this.prev = prev; this.seq = seq; this._root.loadDto(data, null); diff --git a/src/web3/adl/hamt-crdt/HamtFrame.ts b/src/web3/adl/hamt-crdt/HamtFrame.ts index 01d1c999ef..d7db93f788 100644 --- a/src/web3/adl/hamt-crdt/HamtFrame.ts +++ b/src/web3/adl/hamt-crdt/HamtFrame.ts @@ -15,7 +15,10 @@ export class HamtFrame { /** Maybe instead of `_dirty`, just consider `id` === `null` to mean there are unsaved changes. */ protected _dirty: boolean = false; - constructor(protected readonly cas: CidCasStruct, public cid: Cid | null) {} + constructor( + protected readonly cas: CidCasStruct, + public cid: Cid | null, + ) {} protected async ensureLoaded(): Promise { if (this._loading) return this._loading.promise; @@ -34,7 +37,7 @@ export class HamtFrame { const id = this.cid; if (!id) throw new Error('ID_NOT_SET'); this._loading = new Defer(); - const data = await this.cas.get(id) as types.HamtFrameDto; + const data = (await this.cas.get(id)) as types.HamtFrameDto; this.loadDto(data, id); this._loading.resolve(); this._loading = null; @@ -207,7 +210,7 @@ export class HamtFrame { } public toDto(): types.HamtFrameDto { - const children = this._children.map((child) => child ? child.cid : null); + const children = this._children.map((child) => (child ? child.cid : null)); const dto: types.HamtFrameDto = [this._entries, children]; return dto; } diff --git a/src/web3/adl/hamt-crdt/__tests__/HamtCrdt.spec.ts b/src/web3/adl/hamt-crdt/__tests__/HamtCrdt.spec.ts index a123874030..bc3089efa2 100644 --- a/src/web3/adl/hamt-crdt/__tests__/HamtCrdt.spec.ts +++ b/src/web3/adl/hamt-crdt/__tests__/HamtCrdt.spec.ts @@ -212,7 +212,7 @@ describe('HamtCrdt', () => { const [cid] = await hamt.save(); expect(cas._map.size).toBe(size + 1); const blob = await cas.get(cid); - const found = toArr(blob).findIndex(octet => octet === data); + const found = toArr(blob).findIndex((octet) => octet === data); expect(found > -1).toBe(true); }); diff --git a/src/web3/codec/Codecs.ts b/src/web3/codec/Codecs.ts index 1e4ecf5fbd..7229348474 100644 --- a/src/web3/codec/Codecs.ts +++ b/src/web3/codec/Codecs.ts @@ -1,5 +1,5 @@ -import {MulticodecIpld} from "../multiformats"; -import type {IpldCodec} from "./types"; +import {MulticodecIpld} from '../multiformats'; +import type {IpldCodec} from './types'; export class Codecs { protected readonly map = new Map(); diff --git a/src/web3/codec/codecs/cbor.ts b/src/web3/codec/codecs/cbor.ts index 9c3387c93b..4fc3bcc0f7 100644 --- a/src/web3/codec/codecs/cbor.ts +++ b/src/web3/codec/codecs/cbor.ts @@ -11,13 +11,13 @@ const encoder = new (class extends CborEncoderDag { } })(writer); -const decoder = new class extends CborDecoderDag { +const decoder = new (class extends CborDecoderDag { public readTagRaw(tag: number): Cid | unknown { const value = this.val(); if (tag === 42) return Cid.fromBinary(value as Uint8Array); throw new Error('UNKNOWN_TAG'); } -} +})(); export const cbor: IpldCodec = { name: 'DAG-CBOR', diff --git a/src/web3/codec/codecs/writer.ts b/src/web3/codec/codecs/writer.ts index 2d093e6085..fd58d8bb9f 100644 --- a/src/web3/codec/codecs/writer.ts +++ b/src/web3/codec/codecs/writer.ts @@ -1,3 +1,3 @@ -import {Writer} from "../../../util/buffers/Writer"; +import {Writer} from '../../../util/buffers/Writer'; export const writer = new Writer(); diff --git a/src/web3/codec/index.ts b/src/web3/codec/index.ts index b373fe712c..77f5672358 100644 --- a/src/web3/codec/index.ts +++ b/src/web3/codec/index.ts @@ -1,7 +1,7 @@ -import {MulticodecIpld} from "../multiformats"; -import {Codecs} from "./Codecs"; -import {raw} from "./codecs/raw"; -import {cbor} from "./codecs/cbor"; +import {MulticodecIpld} from '../multiformats'; +import {Codecs} from './Codecs'; +import {raw} from './codecs/raw'; +import {cbor} from './codecs/cbor'; export * from './types'; export * from './Codecs'; diff --git a/src/web3/codec/types.ts b/src/web3/codec/types.ts index 4d38042ca3..c02c6ae304 100644 --- a/src/web3/codec/types.ts +++ b/src/web3/codec/types.ts @@ -1,4 +1,4 @@ -import type {BinaryJsonDecoder, BinaryJsonEncoder} from "../../json-pack/types"; +import type {BinaryJsonDecoder, BinaryJsonEncoder} from '../../json-pack/types'; export interface IpldCodec { name: string; From a697deb809128ff3d9d7daac8a1345c7547cc9f4 Mon Sep 17 00:00:00 2001 From: streamich Date: Mon, 11 Mar 2024 17:18:12 +0100 Subject: [PATCH 12/16] =?UTF-8?q?test(json-pack):=20=F0=9F=92=8D=20add=20e?= =?UTF-8?q?xtra=20CborEncoderDag=20test?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../cbor/__tests__/CborEncoderDag.spec.ts | 58 ++++++++++++------- 1 file changed, 36 insertions(+), 22 deletions(-) diff --git a/src/json-pack/cbor/__tests__/CborEncoderDag.spec.ts b/src/json-pack/cbor/__tests__/CborEncoderDag.spec.ts index f71521e255..dfe22e8aaf 100644 --- a/src/json-pack/cbor/__tests__/CborEncoderDag.spec.ts +++ b/src/json-pack/cbor/__tests__/CborEncoderDag.spec.ts @@ -2,6 +2,7 @@ import {Writer} from '../../../util/buffers/Writer'; import {CborEncoderDag} from '../CborEncoderDag'; import {CborDecoder} from '../CborDecoder'; import {JsonPackExtension} from '../../JsonPackExtension'; +import {CborDecoderDag} from '../CborDecoderDag'; const writer = new Writer(1); const encoder = new CborEncoderDag(writer); @@ -50,37 +51,50 @@ describe('only extension = 42 is permitted', () => { expect(val).toStrictEqual({a: 'a', b: 'b'}); }); - test('can encode CID using inlined custom class', () => { - class CID { - constructor(public readonly value: string) {} + class CID { + constructor(public readonly value: string) {} + } + class NotCID { + constructor(public readonly value: string) {} + } + + class IpfsCborEncoder extends CborEncoderDag { + public writeUnknown(val: unknown): void { + if (val instanceof CID) this.writeTag(42, val.value); + else throw new Error('Unknown value type'); } - const encoder = new (class extends CborEncoderDag { - public writeUnknown(val: unknown): void { - if (val instanceof CID) encoder.writeTag(42, val.value); - else throw new Error('Unknown value type'); - } - })(); + } + + class IpfsCborDecoder extends CborDecoderDag { + public readTagRaw(tag: number): CID | unknown { + if (tag === 42) return new CID(this.val() as any); + throw new Error('UNKNOWN_TAG'); + } + } + + test('can encode CID using inlined custom class', () => { + const encoder = new IpfsCborEncoder(); const encoded = encoder.encode({a: 'a', b: new JsonPackExtension(42, 'b')}); const val = decoder.read(encoded); expect(val).toStrictEqual({a: 'a', b: new JsonPackExtension(42, 'b')}); const encoded2 = encoder.encode({a: 'a', b: new CID('b')}); - const val2 = decoder.read(encoded2); + const val2 = decoder.decode(encoded2); expect(val).toStrictEqual({a: 'a', b: new JsonPackExtension(42, 'b')}); + expect(val2).toStrictEqual({a: 'a', b: new JsonPackExtension(42, 'b')}); + }); + + test('can encode CID inside a nested array', () => { + const encoder = new IpfsCborEncoder(); + const decoder = new IpfsCborDecoder(); + const cid = new CID('my-cid'); + const data = [1, [2, [3, cid, 4], 5], 6]; + const encoded = encoder.encode(data); + const decoded = decoder.decode(encoded); + expect(decoded).toStrictEqual(data); }); test('can throw on unknown custom class', () => { - class CID { - constructor(public readonly value: string) {} - } - class NotCID { - constructor(public readonly value: string) {} - } - const encoder = new (class extends CborEncoderDag { - public writeUnknown(val: unknown): void { - if (val instanceof CID) encoder.writeTag(42, val.value); - else throw new Error('Unknown value type'); - } - })(); + const encoder = new IpfsCborEncoder(); const encoded1 = encoder.encode({a: 'a', b: new CID('b')}); expect(() => encoder.encode({a: 'a', b: new NotCID('b')})).toThrowError(new Error('Unknown value type')); }); From b942d03e09ea89c35be30706a43d3a6b246885c8 Mon Sep 17 00:00:00 2001 From: streamich Date: Mon, 11 Mar 2024 17:45:52 +0100 Subject: [PATCH 13/16] =?UTF-8?q?feat:=20=F0=9F=8E=B8=20improve=20CAS=20im?= =?UTF-8?q?plementations?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/web3/store/cas/CidCasMemory.ts | 12 ++++++++---- src/web3/store/cas/CidCasStruct.ts | 10 ++++------ src/web3/store/cas/CidCasStructCbor.ts | 6 +++--- 3 files changed, 15 insertions(+), 13 deletions(-) diff --git a/src/web3/store/cas/CidCasMemory.ts b/src/web3/store/cas/CidCasMemory.ts index 8d702f0634..dfec3d53ed 100644 --- a/src/web3/store/cas/CidCasMemory.ts +++ b/src/web3/store/cas/CidCasMemory.ts @@ -7,7 +7,7 @@ export class CidCasMemory implements CidCas { public async get(cid: Cid): Promise { const key = cid.toText(); - await new Promise((resolve) => setTimeout(resolve, 1)); + await new Promise((resolve) => setImmediate(resolve)); const value = this.store.get(key); if (!value) throw new Error(`No value for CID: ${key}`); return value; @@ -15,21 +15,25 @@ export class CidCasMemory implements CidCas { public async has(cid: Cid): Promise { const key = cid.toText(); - await new Promise((resolve) => setTimeout(resolve, 1)); + await new Promise((resolve) => setImmediate(resolve)); return this.store.has(key); } public async del(cid: Cid): Promise { const key = cid.toText(); - await new Promise((resolve) => setTimeout(resolve, 1)); + await new Promise((resolve) => setImmediate(resolve)); this.store.delete(key); } public async put(value: Uint8Array, ipldType: MulticodecIpld = MulticodecIpld.Raw): Promise { const cid = await Cid.fromData(value, ipldType); - await new Promise((resolve) => setTimeout(resolve, 1)); + await new Promise((resolve) => setImmediate(resolve)); const key = cid.toText(); this.store.set(key, value); return cid; } + + public async size(): Promise { + return this.store.size; + } } diff --git a/src/web3/store/cas/CidCasStruct.ts b/src/web3/store/cas/CidCasStruct.ts index d0a2238a63..a8fdcbb918 100644 --- a/src/web3/store/cas/CidCasStruct.ts +++ b/src/web3/store/cas/CidCasStruct.ts @@ -1,17 +1,17 @@ import {Cid, MulticodecIpld} from '../../multiformats'; -import type {JsonValueCodec} from '../../../json-pack/codecs/types'; import type {CidCas} from './CidCas'; +import type {IpldCodec} from '../../codec'; export class CidCasStruct { constructor( protected readonly cas: CidCas, protected readonly ipldType: MulticodecIpld, - protected readonly codec: JsonValueCodec, + protected readonly codec: IpldCodec, ) {} public async get(cid: Cid): Promise { const blob = await this.cas.get(cid); - return this.codec.decoder.read(blob); + return this.codec.decoder.decode(blob); } public has(cid: Cid): Promise { @@ -23,9 +23,7 @@ export class CidCasStruct { } public async put(value: unknown): Promise { - const encoder = this.codec.encoder; - encoder.writeAny(value); - const blob = encoder.writer.flush(); + const blob = this.codec.encoder.encode(value); return this.cas.put(blob, this.ipldType); } } diff --git a/src/web3/store/cas/CidCasStructCbor.ts b/src/web3/store/cas/CidCasStructCbor.ts index ed1333e59f..08f7cd4f8e 100644 --- a/src/web3/store/cas/CidCasStructCbor.ts +++ b/src/web3/store/cas/CidCasStructCbor.ts @@ -1,13 +1,13 @@ import {MulticodecIpld} from '../../multiformats'; -import {CborJsonValueCodec} from '../../../json-pack/codecs/cbor'; -import {Writer} from '../../../util/buffers/Writer'; import {CidCasStruct} from './CidCasStruct'; +import {cbor} from '../../codec/codecs/cbor'; import type {CidCas} from './CidCas'; +import type {IpldCodec} from '../../codec'; export class CidCasStructCbor extends CidCasStruct { constructor( protected readonly cas: CidCas, - protected readonly codec: CborJsonValueCodec = new CborJsonValueCodec(new Writer(4096)), + protected readonly codec: IpldCodec = cbor, ) { super(cas, MulticodecIpld.Cbor, codec); } From 7bd51172ab6d2e474e7eccaf479981253b636fdc Mon Sep 17 00:00:00 2001 From: streamich Date: Mon, 11 Mar 2024 17:46:13 +0100 Subject: [PATCH 14/16] =?UTF-8?q?fix:=20=F0=9F=90=9B=20correctly=20encode?= =?UTF-8?q?=20CID=20tags?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/web3/codec/codecs/__tests__/cbor.spec.ts | 2 +- src/web3/codec/codecs/cbor.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/web3/codec/codecs/__tests__/cbor.spec.ts b/src/web3/codec/codecs/__tests__/cbor.spec.ts index 7eda57823d..bcfc278429 100644 --- a/src/web3/codec/codecs/__tests__/cbor.spec.ts +++ b/src/web3/codec/codecs/__tests__/cbor.spec.ts @@ -1,4 +1,4 @@ -import {Cid} from '../../multiformats'; +import {Cid} from '../../../multiformats'; import {cbor} from '../cbor'; test('can encode and decode CID', async () => { diff --git a/src/web3/codec/codecs/cbor.ts b/src/web3/codec/codecs/cbor.ts index 4fc3bcc0f7..b5a6dbdcf4 100644 --- a/src/web3/codec/codecs/cbor.ts +++ b/src/web3/codec/codecs/cbor.ts @@ -6,7 +6,7 @@ import type {IpldCodec} from '../types'; const encoder = new (class extends CborEncoderDag { public writeUnknown(val: unknown): void { - if (val instanceof Cid) encoder.writeTag(42, val.toBinary()); + if (val instanceof Cid) this.writeTag(42, val.toBinary()); else throw new Error('UNKNOWN_VALUE'); } })(writer); From a6d5c7faf4be63662b75be2890f37b7c08074cdd Mon Sep 17 00:00:00 2001 From: streamich Date: Mon, 11 Mar 2024 17:46:39 +0100 Subject: [PATCH 15/16] =?UTF-8?q?test:=20=F0=9F=92=8D=20cleanup=20basic=20?= =?UTF-8?q?HAMT=20CRDT=20tests?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/util/buffers/b.ts | 1 + src/web3/adl/hamt-crdt/Hamt.ts | 3 +- .../{HamtCrdt.spec.ts => Hamt.spec.ts} | 100 ++++++++---------- 3 files changed, 48 insertions(+), 56 deletions(-) create mode 100644 src/util/buffers/b.ts rename src/web3/adl/hamt-crdt/__tests__/{HamtCrdt.spec.ts => Hamt.spec.ts} (80%) diff --git a/src/util/buffers/b.ts b/src/util/buffers/b.ts new file mode 100644 index 0000000000..003f912314 --- /dev/null +++ b/src/util/buffers/b.ts @@ -0,0 +1 @@ +export const b = (...args: number[]) => new Uint8Array(args); diff --git a/src/web3/adl/hamt-crdt/Hamt.ts b/src/web3/adl/hamt-crdt/Hamt.ts index bd8245361e..1097349c54 100644 --- a/src/web3/adl/hamt-crdt/Hamt.ts +++ b/src/web3/adl/hamt-crdt/Hamt.ts @@ -93,7 +93,8 @@ export class Hamt implements types.HamtApi { const [, affected] = await this._root.saveChildren(); const prev = this.cid; const seq = this.seq + 1; - const dto: types.HamtRootFrameDto = [prev, seq, this.ops, this._root.toDto()]; + const frameDto = this._root.toDto(); + const dto: types.HamtRootFrameDto = [prev, seq, this.ops, frameDto]; const cid = await this.deps.cas.put(dto); this.cid = cid; this.prev = prev; diff --git a/src/web3/adl/hamt-crdt/__tests__/HamtCrdt.spec.ts b/src/web3/adl/hamt-crdt/__tests__/Hamt.spec.ts similarity index 80% rename from src/web3/adl/hamt-crdt/__tests__/HamtCrdt.spec.ts rename to src/web3/adl/hamt-crdt/__tests__/Hamt.spec.ts index bc3089efa2..bb000ffdc3 100644 --- a/src/web3/adl/hamt-crdt/__tests__/HamtCrdt.spec.ts +++ b/src/web3/adl/hamt-crdt/__tests__/Hamt.spec.ts @@ -1,29 +1,21 @@ -/** @jest-environment node */ -import {MemoryLevel} from 'memory-level'; -import {b} from '../../../../util/buffer/b'; +import {b} from '../../../../util/buffers/b'; import {HlcFactory} from '../../../hlc'; -import {LevelCAS} from '../../LevelCAS'; -import {MemoryCAS} from '../../MemoryCAS'; -import {HamtCrdt} from '../HamtCrdt'; +import {CidCasMemory} from '../../../store/cas/CidCasMemory'; +import {CidCasStructCbor} from '../../../store/cas/CidCasStructCbor'; +import {HamtFactory} from '../HamtFactory'; +import {HamtRootFrameDto} from '../types'; const setup = () => { - const hlc = new HlcFactory({ - getTs: () => Math.floor(Date.now() / 1000), - processId: 123, - }); - const db = new MemoryLevel({ - keyEncoding: 'view', - valueEncoding: 'view', - }); - const cas = new LevelCAS({db}); - const hamt = new HamtCrdt({ - hlc, - cas, - }); + const hlcs = new HlcFactory({}); + const cas0 = new CidCasMemory(); + const cas = new CidCasStructCbor(cas0); + const hamts = new HamtFactory({hlcs, cas}); + const hamt = hamts.make(); return { - hlc, - db, + hlcs, + cas0, cas, + hamts, hamt, }; }; @@ -34,13 +26,6 @@ const toArr = (buf: Uint8Array): number[] => { return arr; }; -test('can store file in IPFS', async () => { - const {cas} = setup(); - const cid = await cas.put(b(1, 2, 3)); - const data = await cas.get(cid); - expect(data).toStrictEqual(b(1, 2, 3)); -}); - describe('HamtCrdt', () => { test('new database has no changes', async () => { const {hamt} = setup(); @@ -185,45 +170,31 @@ describe('HamtCrdt', () => { describe('.save()', () => { test('can persist empty HAMT', async () => { - const {hamt, hlc, cas} = setup(); + const {hamt, hamts} = setup(); const [cid] = await hamt.save(); expect(cid).toBeDefined(); - const hamt2 = new HamtCrdt({ - hlc, - cas, - }); + const hamt2 = hamts.make(); await hamt2.load(cid); }); test('can save a single key', async () => { - const hlc = new HlcFactory({ - getTs: () => Math.floor(Date.now() / 1000), - processId: 123, - }); - const cas = new MemoryCAS(); - const hamt = new HamtCrdt({ - hlc, - cas, - }); + const {hamt, cas0} = setup(); const data = 111; await hamt.put('a', b(data)); - const size = cas._map.size; + const size = await (cas0 as CidCasMemory).size(); expect(size).toBe(0); const [cid] = await hamt.save(); - expect(cas._map.size).toBe(size + 1); - const blob = await cas.get(cid); + expect(await (cas0 as CidCasMemory).size()).toBe(size + 1); + const blob = await cas0.get(cid); const found = toArr(blob).findIndex((octet) => octet === data); expect(found > -1).toBe(true); }); test('can load saved data', async () => { - const {hamt, cas, hlc} = setup(); + const {hamt, hamts} = setup(); await hamt.put('a', b(123)); const [cid] = await hamt.save(); - const hamt2 = new HamtCrdt({ - hlc, - cas, - }); + const hamt2 = hamts.make(); const res1 = await hamt2.get('a'); expect(res1).toBe(undefined); await hamt2.load(cid); @@ -232,22 +203,41 @@ describe('HamtCrdt', () => { }); test('can save and load more than 16 keys of data', async () => { - const {hamt, cas, hlc} = setup(); + const {hamt, hamts} = setup(); const keys = 1111; for (let i = 0; i < keys; i++) { await hamt.put('a:' + i, b(i, i + 1, i + 2)); } const [cid, all] = await hamt.save(); - const hamt2 = new HamtCrdt({ - hlc, - cas, - }); + const hamt2 = hamts.make(); await hamt2.load(cid); for (let i = 0; i < keys; i++) { const res = await hamt2.get('a:' + i); expect(res).toStrictEqual(b(i, i + 1, i + 2)); } }); + + test('can save and load more than 16 keys .save()"ed at periodic intervals', async () => { + const {hamt, hamts} = setup(); + const keysPerSave = 10; + const saves = 20; + for (let j = 0; j < saves; j++) { + for (let i = 0; i < keysPerSave; i++) { + const key = (j * keysPerSave + i); + await hamt.put('abc:' + key, b(key, key + 1, key + 2)); + } + await hamt.save(); + } + const hamt2 = hamts.make(); + await hamt2.load(hamt.cid!); + for (let j = 0; j < saves; j++) { + for (let i = 0; i < keysPerSave; i++) { + const key = (j * keysPerSave + i); + const res = await hamt2.get('abc:' + key); + expect(res).toStrictEqual(b(key, key + 1, key + 2)); + } + } + }); }); describe('operations', () => { From f6107c826a632049f2957cd3ecb5c1dcede708b1 Mon Sep 17 00:00:00 2001 From: streamich Date: Mon, 11 Mar 2024 17:47:53 +0100 Subject: [PATCH 16/16] =?UTF-8?q?style:=20=F0=9F=92=84=20run=20Prettier?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/web3/adl/hamt-crdt/__tests__/Hamt.spec.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/web3/adl/hamt-crdt/__tests__/Hamt.spec.ts b/src/web3/adl/hamt-crdt/__tests__/Hamt.spec.ts index bb000ffdc3..0fb09c947d 100644 --- a/src/web3/adl/hamt-crdt/__tests__/Hamt.spec.ts +++ b/src/web3/adl/hamt-crdt/__tests__/Hamt.spec.ts @@ -223,7 +223,7 @@ describe('HamtCrdt', () => { const saves = 20; for (let j = 0; j < saves; j++) { for (let i = 0; i < keysPerSave; i++) { - const key = (j * keysPerSave + i); + const key = j * keysPerSave + i; await hamt.put('abc:' + key, b(key, key + 1, key + 2)); } await hamt.save(); @@ -232,7 +232,7 @@ describe('HamtCrdt', () => { await hamt2.load(hamt.cid!); for (let j = 0; j < saves; j++) { for (let i = 0; i < keysPerSave; i++) { - const key = (j * keysPerSave + i); + const key = j * keysPerSave + i; const res = await hamt2.get('abc:' + key); expect(res).toStrictEqual(b(key, key + 1, key + 2)); }