diff --git a/docs/design/note-identity-and-crdt.md b/docs/design/note-identity-and-crdt.md index 9ecb2ab..27468a5 100644 --- a/docs/design/note-identity-and-crdt.md +++ b/docs/design/note-identity-and-crdt.md @@ -225,7 +225,7 @@ identity**. The markdown is on disk, and the identity map is in the engram, so a rebuilt database re-adopts the same ULIDs it had before. That is the whole diagnostic tree for this design, and it is why the human-relevant tables — `path`, `ulid`, `merge_policy` — must be plain columns rather than blobs, so -`sqlite3 metadata.db 'select path, ulid from catalog'` answers a question in +`sqlite3 metadata.db 'select path, ulid from bf_catalog'` answers a question in any SQLite browser. Operation payloads stay opaque; nothing a person needs to read does. diff --git a/lib/engram/crdt/catalog.dart b/lib/engram/crdt/catalog.dart new file mode 100644 index 0000000..2de1da8 --- /dev/null +++ b/lib/engram/crdt/catalog.dart @@ -0,0 +1,236 @@ +/// The note catalog's value types: one typed row per note, and the enums that +/// row carries. +/// +/// Deliberately free of `dart:io`, `sqlite3`, and `crdt_lf_sqlite`. These +/// describe a *row*, not where it is stored, and `crdt_lf` itself is pure Dart +/// — so the types compile everywhere even though the table cannot. The table +/// and its queries live in [catalog_io.dart](catalog_io.dart), which is +/// `dart:io`-only because SQLite is. +/// +/// Parsing here raises [FormatException], the way `PeerId.parse` does. The +/// storage layer is what turns that into a `MetadataDatabaseException`, so the +/// device-local store's strictness stance is expressed in exactly one place. +library; + +import 'dart:typed_data'; + +import 'package:crdt_lf/crdt_lf.dart'; + +/// How concurrent writes to one note are reconciled (design Decision 3). +/// +/// **An open enum, not a boolean.** Vector ink — atomic, immutable, +/// add/delete-only strokes — is a third policy rather than a variation of +/// either of these, and lands as one when #49's deferred half does. Anything +/// switching on this value must stay correct when a third arrives. +/// +/// Policy is a property of the *note*, not of the device: two devices that +/// disagreed would apply incompatible semantics to one op-log, which is +/// corruption rather than divergence. It therefore travels in the shared +/// identity map (design Decision 9), and the catalog holds this device's copy. +enum MergePolicy { + /// The whole file is one `CRDTFugueTextHandler` sequence — the locked model. + fugueText, + + /// The whole file is one opaque value; concurrent writes resolve by the + /// locked tiebreak comparator. + /// + /// Stores a register, never the bytes: the op-log carries the content hash, + /// size, and stamp the comparator needs, and the file stays in the engram as + /// the ordinary file it already is. + blobLww; + + /// Parses the stored spelling, which is the enum's own [name]. + /// + /// Throws [FormatException] for anything else rather than falling back to a + /// default — a value we do not recognise means the row was written by + /// something that is not this build, and guessing at its semantics is how a + /// PNG gets character-merged. + static MergePolicy parse(String value) => values.firstWhere( + (policy) => policy.name == value, + orElse: () => throw FormatException('unknown merge policy: "$value"'), + ); +} + +/// Extensions that get [MergePolicy.fugueText]; everything else is a blob. +/// +/// Compared case-insensitively. Deliberately short: an extension belongs here +/// only once character-level merging of its contents is known to be +/// meaningful, and the cost of leaving one off is a lost concurrent edit that +/// still exists in the loser's history — recoverable, unlike the reverse. +const Set fugueTextExtensions = {'md', 'markdown', 'txt', 'text'}; + +/// The policy [path] gets at creation, derived from its extension. +/// +/// **Unrecognised extensions default to [MergePolicy.blobLww]**, and the +/// asymmetry is the point: character-merging two versions of a PNG produces a +/// corrupt file nobody can recover, while last-writer-wins on text loses one +/// edit that still exists in the loser's history. Default toward the +/// recoverable failure. +/// +/// A name with no dot after its last separator — `LICENSE` — and a dotfile +/// with no further dot — `.gitignore` — both have no extension and are +/// therefore blobs. Derivation is v1's rule only: policy is fixed at note +/// creation, and the column's ability to change is reserved, not built. +MergePolicy mergePolicyForPath(String path) => + fugueTextExtensions.contains(_extensionOf(path)) + ? MergePolicy.fugueText + : MergePolicy.blobLww; + +/// The lowercase extension of [path] without its dot, or `''` if it has none. +String _extensionOf(String path) { + final separator = path.lastIndexOf('/'); + final dot = path.lastIndexOf('.'); + // `> separator + 1` rather than `>=`: a leading dot names a hidden file, it + // does not introduce an extension. + if (dot <= separator + 1) return ''; + return path.substring(dot + 1).toLowerCase(); +} + +/// What this device currently believes about a note's existence. +enum NoteState { + /// Present on disk, with an op-log this device can build on. + live, + + /// The ULID was adopted from the identity map, but no op-log has arrived yet + /// (design Decision 7). + /// + /// Readable and editable as an ordinary file — the one bounded exception to + /// "only the materializer writes the file" — until a log lands over #67, at + /// which point the local file reconciles against it as ordinary drift. + historyPending, + + /// Deleted: the scan found the path gone with no move or rename candidate. + tombstoned, + + /// The file should exist but cannot be read right now — an unmounted drive, + /// a network engram that is offline, an iCloud placeholder not yet + /// materialized. + /// + /// Distinct from [tombstoned] on purpose, mirroring the distinction the + /// storage design already draws for whole engrams: a file missing because a + /// drive is unmounted is not a deleted file, and tombstoning one would + /// destroy a note that is merely out of reach. + unavailable; + + /// Parses the stored spelling, which is the enum's own [name]. Throws + /// [FormatException] for anything else. + static NoteState parse(String value) => values.firstWhere( + (state) => state.name == value, + orElse: () => throw FormatException('unknown note state: "$value"'), + ); +} + +/// One note, as this device currently understands it. +/// +/// Mixes state that travels with the note ([ulid], [path], [mergePolicy], and +/// [seedClaim], all mirrored from the shared identity map) with state that is +/// **device-local and must never be shared** ([materializedHash], [size], +/// [mtimeUtc], [sketch]). Decision 5 explains why sharing the hash converts +/// drift detection into silent data loss; the size, mtime, and sketch beside +/// it describe this device's copy for the same reason. +class CatalogRow { + const CatalogRow({ + required this.ulid, + required this.path, + required this.mergePolicy, + required this.state, + this.materializedHash, + this.size, + this.mtimeUtc, + this.sketch, + this.seedClaim, + }); + + /// The note's stable identity, minted once and never changed. + final String ulid; + + /// Engram-relative path, `/`-separated. + final String path; + + /// How concurrent writes to this note are reconciled. + final MergePolicy mergePolicy; + + /// What this device believes about the note's existence. + final NoteState state; + + /// Hash of the exact bytes the materializer last wrote, or `null` if this + /// device has never written the file. + /// + /// Drift detection is one comparison against this value. It is **never** + /// shared: it records what *this device's* materializer last wrote, and two + /// devices legitimately hold different values at the same instant. + final String? materializedHash; + + /// Size in bytes of the file as this device last saw it, or `null` if it has + /// not seen it. + final int? size; + + /// Modification time of the file as this device last saw it, in UTC. + final DateTime? mtimeUtc; + + /// Content sketch — shingled MinHash or equivalent — used to re-associate a + /// note renamed *and* edited in one offline window, or `null` if not yet + /// computed. + /// + /// Device-local derived data: rebuilt by a scan, never shared. It exists to + /// make the delete-plus-create cell of Decision 7 rare, not to eliminate it. + final Uint8List? sketch; + + /// Who seeded this note's first history and when, or `null` for an + /// **unclaimed** seed — a note the identity map knows but no surviving + /// op-log ever backed. + /// + /// An [OperationId] is exactly the pair the design names `seeded_by` and + /// `seed_hlc`, and its `compareTo` is exactly the locked tiebreak comparator + /// (HLC first, peerID second) — so contested claims resolve through the + /// library's ordering rather than a second one written here. + /// + /// Only the holder may seed. An adopting device does not; it may *take* an + /// unclaimed seed on the user's first edit, recording the claim as it seeds. + final OperationId? seedClaim; + + /// The peer that seeded this note, or `null` if the seed is unclaimed. + PeerId? get seededBy => seedClaim?.peerId; + + @override + bool operator ==(Object other) => + identical(this, other) || + other is CatalogRow && + other.ulid == ulid && + other.path == path && + other.mergePolicy == mergePolicy && + other.state == state && + other.materializedHash == materializedHash && + other.size == size && + other.mtimeUtc == mtimeUtc && + _sketchesEqual(other.sketch, sketch) && + other.seedClaim == seedClaim; + + @override + int get hashCode => Object.hash( + ulid, + path, + mergePolicy, + state, + materializedHash, + size, + mtimeUtc, + sketch == null ? null : Object.hashAll(sketch!), + seedClaim, + ); + + @override + String toString() => + 'CatalogRow($ulid, $path, ${mergePolicy.name}, ${state.name})'; +} + +/// Byte-wise sketch comparison. `Uint8List` equality is identity otherwise, so +/// a row read back from the database would never equal the one written. +bool _sketchesEqual(Uint8List? a, Uint8List? b) { + if (identical(a, b)) return true; + if (a == null || b == null || a.length != b.length) return false; + for (var i = 0; i < a.length; i++) { + if (a[i] != b[i]) return false; + } + return true; +} diff --git a/lib/engram/crdt/catalog_io.dart b/lib/engram/crdt/catalog_io.dart new file mode 100644 index 0000000..d90b32c --- /dev/null +++ b/lib/engram/crdt/catalog_io.dart @@ -0,0 +1,209 @@ +/// The catalog table and its queries, over a connection someone else owns. +/// +/// `dart:io`-only because SQLite is. The row types it reads and writes are +/// platform-neutral and live in [catalog.dart](catalog.dart); this file is +/// only the storage. +library; + +import 'dart:typed_data'; + +import 'package:crdt_lf/crdt_lf.dart'; +import 'package:sqlite3/sqlite3.dart' as sq; + +import 'catalog.dart'; +import 'store_exceptions.dart'; + +/// One row per note, keyed by ULID: the catalog. +/// +/// **The diagnostic surface for this whole design.** Deleting `metadata.db` +/// costs history, never content and never identity, and recovery starts in a +/// plain SQLite browser with no application code — so `select path, ulid from +/// bf_catalog` has to answer "which note is this?" on its own. That is why the +/// human-relevant fields are plain columns rather than a serialized blob, and +/// why the enums are stored as their names rather than as ordinals: an ordinal +/// re-numbers itself the moment a third [MergePolicy] is added, silently +/// reinterpreting every row already on disk. +/// +/// Does not own the connection. [MetadataDatabase] opens it, injects the +/// op-log's schema into it, and closes it; this reads and writes one table on +/// it, so a catalog write and an op-log write share one transaction boundary. +class NoteCatalog { + const NoteCatalog(this.database); + + /// The table, `bf_`-prefixed like every table BrainFrame creates. + /// + /// Every column beyond the identity quartet is nullable, and each null means + /// something specific rather than "missing": no [CatalogRow.materializedHash] + /// is a note this device has never written, and no seed claim is a note whose + /// first history nobody has created yet. + static const String createSchemaSql = ''' +CREATE TABLE IF NOT EXISTS bf_catalog ( + ulid TEXT PRIMARY KEY, + path TEXT NOT NULL, + merge_policy TEXT NOT NULL, + state TEXT NOT NULL, + materialized_hash TEXT, + size INTEGER, + mtime_utc INTEGER, + sketch BLOB, + seeded_by TEXT, + seed_hlc TEXT +); +'''; + + /// At most one *findable* note per path, enforced by the database. + /// + /// Partial rather than a plain `UNIQUE` on the column, because a tombstoned + /// row keeps the path it died at: deleting `inbox/today.md` and later + /// creating a new note at that same path is ordinary use, and a total + /// uniqueness constraint would reject the second note outright. Restricting + /// the index to non-tombstoned rows says what is actually true — a live, + /// history-pending, or unavailable note owns its path exclusively — and is + /// what makes [byPath] able to return a single row rather than a list. + /// + /// Built from the enum's own [NoteState.tombstoned] name so the predicate + /// cannot drift away from the values the rows are written with. + static final String createIndexSql = + ''' +CREATE UNIQUE INDEX IF NOT EXISTS bf_catalog_findable_path + ON bf_catalog (path) WHERE state <> '${NoteState.tombstoned.name}'; +'''; + + /// The connection this catalog reads and writes. Owned by [MetadataDatabase]. + final sq.Database database; + + /// Creates the table and its index if they are not already there. Idempotent, + /// so it is safe to re-run on every open. + static void createSchema(sq.Database database) { + database + ..execute(createSchemaSql) + ..execute(createIndexSql); + } + + /// The note at [path], or `null` if no findable note holds it. + /// + /// Tombstoned rows are excluded: a path a dead note used to occupy is a free + /// path, and returning the tombstone would make the next scan resurrect its + /// history under unrelated content. + CatalogRow? byPath(String path) => _one( + 'SELECT * FROM bf_catalog WHERE path = ? AND state <> ?', + [path, NoteState.tombstoned.name], + ); + + /// The note identified by [ulid], whatever its state, or `null` if this + /// device has no row for it. + /// + /// Unlike [byPath] this does find tombstones, and must: a note's identity + /// outlives its file, and a peer's operations arrive keyed by ULID long after + /// the local scan concluded the file was gone. + CatalogRow? byUlid(String ulid) => + _one('SELECT * FROM bf_catalog WHERE ulid = ?', [ulid]); + + /// Writes [row], replacing any existing row with the same ULID. + /// + /// Whole-row, never a delta — the same discipline the shared identity map + /// requires — so a caller that changed one field must carry the rest forward. + /// A partial write would silently blank whatever its author did not know + /// about. + void upsert(CatalogRow row) { + database.execute( + 'INSERT INTO bf_catalog ' + '(ulid, path, merge_policy, state, materialized_hash, size, mtime_utc, ' + 'sketch, seeded_by, seed_hlc) ' + 'VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ' + 'ON CONFLICT(ulid) DO UPDATE SET ' + 'path = excluded.path, ' + 'merge_policy = excluded.merge_policy, ' + 'state = excluded.state, ' + 'materialized_hash = excluded.materialized_hash, ' + 'size = excluded.size, ' + 'mtime_utc = excluded.mtime_utc, ' + 'sketch = excluded.sketch, ' + 'seeded_by = excluded.seeded_by, ' + 'seed_hlc = excluded.seed_hlc', + [ + row.ulid, + row.path, + row.mergePolicy.name, + row.state.name, + row.materializedHash, + row.size, + row.mtimeUtc?.toUtc().millisecondsSinceEpoch, + row.sketch, + row.seedClaim?.peerId.toString(), + row.seedClaim?.hlc.toString(), + ], + ); + } + + CatalogRow? _one(String sql, List parameters) { + final rows = database.select(sql, parameters); + return rows.isEmpty ? null : _rowFrom(rows.first); + } + + /// Rebuilds a [CatalogRow] from one database row. + /// + /// Every parse failure becomes a [MetadataDatabaseException] naming the + /// column and the value, matching how the store treats a schema version or a + /// peer identity it cannot read: a row we cannot interpret is surfaced, never + /// half-read into a default that would then be written back as truth. + CatalogRow _rowFrom(sq.Row row) { + final ulid = row['ulid'] as String; + final seededBy = row['seeded_by'] as String?; + final seedHlc = row['seed_hlc'] as String?; + final mtime = row['mtime_utc'] as int?; + return CatalogRow( + ulid: ulid, + path: row['path'] as String, + mergePolicy: _parse( + () => MergePolicy.parse(row['merge_policy'] as String), + 'merge_policy', + ulid, + ), + state: _parse( + () => NoteState.parse(row['state'] as String), + 'state', + ulid, + ), + materializedHash: row['materialized_hash'] as String?, + size: row['size'] as int?, + mtimeUtc: mtime == null + ? null + : DateTime.fromMillisecondsSinceEpoch(mtime, isUtc: true), + sketch: row['sketch'] as Uint8List?, + seedClaim: _seedClaim(seededBy, seedHlc, ulid), + ); + } + + /// The seed claim, or `null` when unclaimed. + /// + /// Half a claim is not an unclaimed seed, it is a corrupt row: the pair is + /// written together and means nothing apart, so a lone peer or a lone clock + /// is refused rather than quietly read as "nobody has seeded this", which + /// would invite a second device to seed a document that already has a + /// history. + OperationId? _seedClaim(String? peer, String? hlc, String ulid) { + if (peer == null && hlc == null) return null; + if (peer == null || hlc == null) { + throw MetadataDatabaseException( + 'catalog row "$ulid" has half a seed claim ' + '(seeded_by: ${peer ?? 'null'}, seed_hlc: ${hlc ?? 'null'})', + ); + } + return _parse( + () => OperationId.parse('$peer@$hlc'), + 'seeded_by/seed_hlc', + ulid, + ); + } + + T _parse(T Function() parse, String column, String ulid) { + try { + return parse(); + } on FormatException catch (error) { + throw MetadataDatabaseException( + 'catalog row "$ulid" has an unreadable $column: ${error.message}', + ); + } + } +} diff --git a/lib/engram/crdt/metadata_db.dart b/lib/engram/crdt/metadata_db.dart index f2445a6..9238a20 100644 --- a/lib/engram/crdt/metadata_db.dart +++ b/lib/engram/crdt/metadata_db.dart @@ -9,5 +9,9 @@ library; export 'app_data_resolver.dart'; +// The catalog's row types, unconditionally: they are pure Dart, so code that +// reasons about a note's merge policy or state compiles on web even though the +// table backing them cannot exist there. +export 'catalog.dart'; export 'metadata_db_stub.dart' if (dart.library.io) 'metadata_db_io.dart'; export 'schema.dart'; diff --git a/lib/engram/crdt/metadata_db_io.dart b/lib/engram/crdt/metadata_db_io.dart index 33d4134..77030ad 100644 --- a/lib/engram/crdt/metadata_db_io.dart +++ b/lib/engram/crdt/metadata_db_io.dart @@ -5,39 +5,14 @@ import 'package:crdt_lf_sqlite/crdt_lf_sqlite.dart'; import 'package:sqlite3/sqlite3.dart' as sq; import 'app_data_resolver.dart'; +import 'catalog_io.dart'; +import 'store_exceptions.dart'; -/// Thrown when `metadata.db` cannot be opened into a usable state. -/// -/// Strict by design, the way [EngramMetadata] is: a database written by a -/// newer build, or holding a value we cannot make sense of, raises rather than -/// being half-read. Recovery is cheap and documented — deleting `metadata.db` -/// costs history, never content and never identity — so failing loudly beats -/// operating on a schema we do not understand. -class MetadataDatabaseException implements Exception { - const MetadataDatabaseException(this.message); - - final String message; - - @override - String toString() => 'MetadataDatabaseException: $message'; -} - -/// Thrown when relocating a store would overwrite one that already exists. -/// -/// This device holds two stores for what is now one engram, which means it -/// adopted the same folder twice locally. Surfaced rather than resolved: a -/// silent merge would interleave two op-logs, and a silent clobber would -/// discard one wholesale. -class EngramStoreCollisionException implements Exception { - const EngramStoreCollisionException(this.path); - - /// The occupied destination directory. - final String path; - - @override - String toString() => - 'EngramStoreCollisionException: a store already exists at $path'; -} +// The failure types moved to store_exceptions.dart so catalog_io.dart can +// raise them without importing this file, which imports it. Re-exported here +// so every existing `import 'metadata_db_io.dart'` still sees them. +export 'catalog_io.dart'; +export 'store_exceptions.dart'; /// The device-local database for one engram: the CRDT op-log, and /// BrainFrame's own tables, sharing one connection and one transaction @@ -50,25 +25,32 @@ class EngramStoreCollisionException implements Exception { /// [sqlite_shared_database_test.dart](../../../test/crdt/sqlite_shared_database_test.dart). /// /// **Everything in here is device-local and none of it is shared.** The -/// op-log, the peer identity, and (from step 3) the catalog's content hashes -/// all describe *this install*, not the note. Nothing here may be copied into +/// op-log, the peer identity, and the catalog's content hashes all describe +/// *this install*, not the note. Nothing here may be copied into /// the engram folder; a live database in a synced folder is corrupted rather /// than merely stale, and a shared content hash converts drift detection into /// silent data loss. class MetadataDatabase { - MetadataDatabase._(this.database, this.crdt, this.peerId, this.schemaVersion); + MetadataDatabase._( + this.database, + this.crdt, + this.catalog, + this.peerId, + this.schemaVersion, + ); /// The schema version this build writes and is the newest it can read. static const int currentSchemaVersion = 1; - /// BrainFrame's own tables, all `bf_`-prefixed (see [brainframeTablePrefix]). + /// The store's own key/value table, `bf_`-prefixed like every table + /// BrainFrame creates (see [brainframeTablePrefix]). /// - /// One key/value table for now. It stays readable in any SQLite browser — - /// `select * from bf_meta` answers "what version, and which peer is this?" — - /// which matters because this is the whole diagnostic surface for a store - /// whose contents are otherwise opaque operation payloads. - static const String createSchemaSql = - ''' + /// It stays readable in any SQLite browser — `select * from bf_meta` answers + /// "what version, and which peer is this?" — which matters because this and + /// `bf_catalog` are the whole diagnostic surface for a store whose contents + /// are otherwise opaque operation payloads. The catalog's own DDL lives with + /// the catalog, in [NoteCatalog.createSchemaSql]. + static const String createSchemaSql = ''' CREATE TABLE IF NOT EXISTS bf_meta ( key TEXT PRIMARY KEY, value TEXT NOT NULL @@ -86,6 +68,10 @@ CREATE TABLE IF NOT EXISTS bf_meta ( /// The op-log's storage, over the same connection. final CRDTSqlite crdt; + /// One row per note, over the same connection — so a catalog write and the + /// op-log write it accompanies commit together or not at all. + final NoteCatalog catalog; + /// This install's identity for this engram, minted on first open. /// /// Scoped per engram rather than one per device, which keeps engrams @@ -132,6 +118,7 @@ CREATE TABLE IF NOT EXISTS bf_meta ( static MetadataDatabase _initialize(sq.Database database) { database.execute(createSchemaSql); + NoteCatalog.createSchema(database); // Injected into the connection BrainFrame already owns, so the catalog and // the op-log commit together. Re-run on every open, which its // IF NOT EXISTS DDL makes idempotent. @@ -139,7 +126,13 @@ CREATE TABLE IF NOT EXISTS bf_meta ( final version = _readVersion(database); final peerId = _readOrMintPeerId(database); - return MetadataDatabase._(database, crdt, peerId, version); + return MetadataDatabase._( + database, + crdt, + NoteCatalog(database), + peerId, + version, + ); } static int _readVersion(sq.Database database) { @@ -180,10 +173,9 @@ CREATE TABLE IF NOT EXISTS bf_meta ( } static String? _readMeta(sq.Database database, String key) { - final rows = database.select( - 'SELECT value FROM bf_meta WHERE key = ?', - [key], - ); + final rows = database.select('SELECT value FROM bf_meta WHERE key = ?', [ + key, + ]); return rows.isEmpty ? null : rows.first['value'] as String; } diff --git a/lib/engram/crdt/metadata_db_stub.dart b/lib/engram/crdt/metadata_db_stub.dart index 0008c5c..fbecb2a 100644 --- a/lib/engram/crdt/metadata_db_stub.dart +++ b/lib/engram/crdt/metadata_db_stub.dart @@ -18,29 +18,14 @@ library; import 'app_data_resolver.dart'; +/// The failure types are shared with the `dart:io` build rather than mirrored +/// here, so a `catch` clause names one class on every platform. They are pure +/// Dart and carry no storage of their own. +export 'store_exceptions.dart'; + const String _unsupported = 'Device-local engram storage is not supported on this platform.'; -/// Signature parity with the `dart:io` build; never constructible here. -class MetadataDatabaseException implements Exception { - const MetadataDatabaseException(this.message); - - final String message; - - @override - String toString() => 'MetadataDatabaseException: $message'; -} - -class EngramStoreCollisionException implements Exception { - const EngramStoreCollisionException(this.path); - - final String path; - - @override - String toString() => - 'EngramStoreCollisionException: a store already exists at $path'; -} - abstract final class MetadataDatabase { static const int currentSchemaVersion = 1; diff --git a/lib/engram/crdt/store_exceptions.dart b/lib/engram/crdt/store_exceptions.dart new file mode 100644 index 0000000..16b64a0 --- /dev/null +++ b/lib/engram/crdt/store_exceptions.dart @@ -0,0 +1,42 @@ +/// The device-local store's failure types, shared by every platform build. +/// +/// Pure Dart on purpose. Both `metadata_db_io.dart` and `metadata_db_stub.dart` +/// re-export these, so the seam presents the *same* classes everywhere rather +/// than two look-alikes that a `catch` clause would distinguish. Keeping them +/// here also lets `catalog_io.dart` raise them without importing the file that +/// imports it. +library; + +/// Thrown when the device-local store cannot be opened or read into a usable +/// state. +/// +/// Strict by design, the way `EngramMetadata` is: a database written by a newer +/// build, or holding a value we cannot make sense of, raises rather than being +/// half-read. Recovery is cheap and documented — deleting `metadata.db` costs +/// history, never content and never identity — so failing loudly beats +/// operating on a schema we do not understand. +class MetadataDatabaseException implements Exception { + const MetadataDatabaseException(this.message); + + final String message; + + @override + String toString() => 'MetadataDatabaseException: $message'; +} + +/// Thrown when relocating a store would overwrite one that already exists. +/// +/// This device holds two stores for what is now one engram, which means it +/// adopted the same folder twice locally. Surfaced rather than resolved: a +/// silent merge would interleave two op-logs, and a silent clobber would +/// discard one wholesale. +class EngramStoreCollisionException implements Exception { + const EngramStoreCollisionException(this.path); + + /// The occupied destination directory. + final String path; + + @override + String toString() => + 'EngramStoreCollisionException: a store already exists at $path'; +} diff --git a/test/coverage/all_files_test.dart b/test/coverage/all_files_test.dart index 6d7dfba..bb51dbb 100644 --- a/test/coverage/all_files_test.dart +++ b/test/coverage/all_files_test.dart @@ -22,10 +22,13 @@ import 'package:brainframe/engram/crdt/app_data_resolver.dart'; import 'package:brainframe/engram/crdt/app_data_resolver_io.dart'; import 'package:brainframe/engram/crdt/app_data_resolver_stub.dart'; import 'package:brainframe/engram/crdt/app_data_source.dart'; +import 'package:brainframe/engram/crdt/catalog.dart'; +import 'package:brainframe/engram/crdt/catalog_io.dart'; import 'package:brainframe/engram/crdt/metadata_db.dart'; import 'package:brainframe/engram/crdt/metadata_db_io.dart'; import 'package:brainframe/engram/crdt/metadata_db_stub.dart'; import 'package:brainframe/engram/crdt/schema.dart'; +import 'package:brainframe/engram/crdt/store_exceptions.dart'; import 'package:brainframe/engram/desktop_folder_adoption.dart'; import 'package:brainframe/engram/engram.dart'; import 'package:brainframe/engram/engram_file_ops.dart'; diff --git a/test/engram/crdt/catalog_io_test.dart b/test/engram/crdt/catalog_io_test.dart new file mode 100644 index 0000000..68590f2 --- /dev/null +++ b/test/engram/crdt/catalog_io_test.dart @@ -0,0 +1,384 @@ +import 'dart:typed_data'; + +import 'package:brainframe/engram/crdt/catalog.dart'; +// NoteCatalog and MetadataDatabaseException both arrive through here: +// metadata_db_io.dart re-exports catalog_io.dart, so importing that directly +// as well would be redundant. +import 'package:brainframe/engram/crdt/metadata_db_io.dart'; +import 'package:brainframe/engram/id.dart'; +import 'package:crdt_lf/crdt_lf.dart'; +import 'package:flutter_test/flutter_test.dart'; +import 'package:hlc_dart/hlc_dart.dart'; +import 'package:sqlite3/sqlite3.dart' as sq; + +import '../../crdt/support/peer_ids.dart'; + +/// The catalog table: what it stores, what it refuses, and how it is queried. +/// +/// Runs against an in-memory store rather than a temporary directory — the +/// table is the subject here, and where the file lives is +/// metadata_db_io_test.dart's business. +void main() { + late MetadataDatabase store; + late NoteCatalog catalog; + + setUp(() { + store = MetadataDatabase.openInMemory(); + catalog = store.catalog; + }); + + tearDown(() => store.close()); + + OperationId claim(PeerId peer, int millis) => + OperationId(peer, HybridLogicalClock(l: millis, c: 0)); + + CatalogRow row({ + String? ulid, + String path = 'inbox/today.md', + MergePolicy mergePolicy = MergePolicy.fugueText, + NoteState state = NoteState.live, + String? materializedHash, + int? size, + DateTime? mtimeUtc, + Uint8List? sketch, + OperationId? seedClaim, + }) => CatalogRow( + ulid: ulid ?? newUlid(), + path: path, + mergePolicy: mergePolicy, + state: state, + materializedHash: materializedHash, + size: size, + mtimeUtc: mtimeUtc, + sketch: sketch, + seedClaim: seedClaim, + ); + + /// Column names of [table], in declaration order. + List columnsOf(String table) => store.database + .select('SELECT name FROM pragma_table_info(?) ORDER BY cid', [table]) + .map((row) => row['name'] as String) + .toList(); + + group('schema', () { + test('the catalog holds every column the design names', () { + expect(columnsOf('bf_catalog'), [ + 'ulid', + 'path', + 'merge_policy', + 'state', + 'materialized_hash', + 'size', + 'mtime_utc', + 'sketch', + 'seeded_by', + 'seed_hlc', + ]); + }); + + test('creating the schema twice is a no-op', () { + // Every open re-runs the DDL; the app cannot know whether this engram + // has been opened before, so re-creation must never reset anything. + catalog.upsert(row(path: 'kept.md')); + NoteCatalog.createSchema(store.database); + + expect(catalog.byPath('kept.md'), isNotNull); + }); + }); + + group('the human-readable claim', () { + test('select path, ulid from bf_catalog answers on its own', () { + // The whole diagnostic tree for this design: recovery from a damaged + // store starts in a plain SQLite browser with no application code, so + // the human-relevant fields must be plain columns rather than a blob. + final ulid = newUlid(); + catalog.upsert(row(ulid: ulid, path: 'refs/crdt.md')); + + final rows = store.database.select( + 'SELECT path, ulid FROM bf_catalog ORDER BY path', + ); + + expect(rows.single['path'], 'refs/crdt.md'); + expect(rows.single['ulid'], ulid); + }); + + test('the enums are stored as their names, not as ordinals', () { + // An ordinal re-numbers itself the moment a third MergePolicy is added, + // silently reinterpreting every row already on disk. + catalog.upsert( + row(mergePolicy: MergePolicy.blobLww, state: NoteState.historyPending), + ); + + final stored = store.database + .select('SELECT merge_policy, state FROM bf_catalog') + .single; + + expect(stored['merge_policy'], 'blobLww'); + expect(stored['state'], 'historyPending'); + }); + }); + + group('queries', () { + test('a path with no note returns null', () { + expect(catalog.byPath('nothing/here.md'), isNull); + }); + + test('a ULID with no note returns null', () { + expect(catalog.byUlid(newUlid()), isNull); + }); + + test('byPath and byUlid find the same row', () { + final written = row(); + catalog.upsert(written); + + expect(catalog.byPath(written.path), written); + expect(catalog.byUlid(written.ulid), written); + }); + + test('every column round-trips through the database', () { + final written = row( + mergePolicy: MergePolicy.blobLww, + state: NoteState.unavailable, + materializedHash: 'sha256:abc', + size: 4096, + mtimeUtc: DateTime.utc(2026, 9, 5, 14, 49, 32), + sketch: Uint8List.fromList([0, 1, 2, 253, 254, 255]), + seedClaim: claim(peerB, 1234), + ); + + catalog.upsert(written); + + expect(catalog.byUlid(written.ulid), written); + }); + + test('an mtime read back is still UTC', () { + // Stored as epoch milliseconds, which carry no zone; reconstructing in + // local time would shift every timestamp by the reader's offset. + final written = row(mtimeUtc: DateTime.utc(2026, 9, 5, 14, 49, 32)); + catalog.upsert(written); + + final read = catalog.byUlid(written.ulid)!; + + expect(read.mtimeUtc!.isUtc, isTrue); + expect(read.mtimeUtc, written.mtimeUtc); + }); + + test('a local mtime is normalised to UTC on the way in', () { + final local = DateTime(2026, 9, 5, 14, 49, 32); + final written = row(mtimeUtc: local); + catalog.upsert(written); + + expect(catalog.byUlid(written.ulid)!.mtimeUtc, local.toUtc()); + }); + + test('an unclaimed seed reads back as no claim', () { + final written = row(); + catalog.upsert(written); + + expect(catalog.byUlid(written.ulid)!.seedClaim, isNull); + }); + }); + + group('upsert', () { + test('a second write to one ULID replaces the row', () { + final ulid = newUlid(); + catalog.upsert(row(ulid: ulid, path: 'inbox/today.md')); + catalog.upsert(row(ulid: ulid, path: 'journal/today.md')); + + expect(catalog.byUlid(ulid)!.path, 'journal/today.md'); + expect(catalog.byPath('inbox/today.md'), isNull); + }); + + test('a rename keeps the identity and the history it points at', () { + // Exactly how the scan records a move: keep the id, update the path. + final written = row(seedClaim: claim(peerA, 10)); + catalog.upsert(written); + catalog.upsert( + row( + ulid: written.ulid, + path: 'journal/today.md', + seedClaim: written.seedClaim, + ), + ); + + final moved = catalog.byUlid(written.ulid)!; + + expect(moved.path, 'journal/today.md'); + expect(moved.seedClaim, written.seedClaim); + }); + + test('a whole-row write clears a field the caller dropped', () { + // The write is whole-row, never a delta, so this is the documented + // behaviour rather than a bug: a caller that changed one field must + // carry the rest forward. + final ulid = newUlid(); + catalog.upsert(row(ulid: ulid, materializedHash: 'sha256:abc')); + catalog.upsert(row(ulid: ulid)); + + expect(catalog.byUlid(ulid)!.materializedHash, isNull); + }); + }); + + group('one findable note per path', () { + test('two findable notes cannot claim one path', () { + catalog.upsert(row(path: 'inbox/today.md')); + + expect( + () => catalog.upsert(row(path: 'inbox/today.md')), + throwsA(isA()), + ); + }); + + test('a history-pending note still owns its path exclusively', () { + // It is unwritten, not absent — the file is on disk and readable. + catalog.upsert(row(path: 'inbox/today.md', state: NoteState.live)); + + expect( + () => catalog.upsert( + row(path: 'inbox/today.md', state: NoteState.historyPending), + ), + throwsA(isA()), + ); + }); + + test('an unavailable note still owns its path exclusively', () { + // An unmounted drive must not let a second note take the path, or + // remounting would produce two notes for one file. + catalog.upsert(row(path: 'inbox/today.md', state: NoteState.unavailable)); + + expect( + () => catalog.upsert(row(path: 'inbox/today.md')), + throwsA(isA()), + ); + }); + + test('a tombstone frees its path for a new note', () { + // Deleting a file and later creating another at the same path is + // ordinary use; a total UNIQUE on path would reject the second note. + final dead = row(path: 'inbox/today.md', state: NoteState.tombstoned); + catalog.upsert(dead); + + final reborn = row(path: 'inbox/today.md'); + catalog.upsert(reborn); + + expect(catalog.byPath('inbox/today.md')!.ulid, reborn.ulid); + expect(catalog.byUlid(dead.ulid)!.state, NoteState.tombstoned); + }); + + test('any number of tombstones may share one path', () { + // A path repeatedly created and deleted accumulates them, and none of + // that history may block the next note. + for (var i = 0; i < 3; i++) { + catalog.upsert( + row(path: 'inbox/today.md', state: NoteState.tombstoned), + ); + } + + expect(catalog.byPath('inbox/today.md'), isNull); + }); + + test('byPath ignores tombstones; byUlid does not', () { + // Identity outlives the file: a peer's operations arrive keyed by ULID + // long after the local scan concluded the file was gone. + final dead = row(state: NoteState.tombstoned); + catalog.upsert(dead); + + expect(catalog.byPath(dead.path), isNull); + expect(catalog.byUlid(dead.ulid), dead); + }); + }); + + group('a row it cannot read is refused, not half-read', () { + /// Writes [value] straight into a column, standing in for a row another + /// build — or a person with a SQLite browser — left behind. + String forceColumn(String column, Object? value) { + final written = row(); + catalog.upsert(written); + store.database.execute( + 'UPDATE bf_catalog SET $column = ? WHERE ulid = ?', + [value, written.ulid], + ); + return written.ulid; + } + + test('an unknown merge policy', () { + final ulid = forceColumn('merge_policy', 'vectorInk'); + + expect( + () => catalog.byUlid(ulid), + throwsA( + isA().having( + (e) => e.toString(), + 'toString', + allOf(contains('merge_policy'), contains(ulid)), + ), + ), + ); + }); + + test('an unknown state', () { + final ulid = forceColumn('state', 'archived'); + + expect( + () => catalog.byUlid(ulid), + throwsA(isA()), + ); + }); + + test('a corrupt seed claim', () { + final written = row(seedClaim: claim(peerA, 10)); + catalog.upsert(written); + store.database.execute( + "UPDATE bf_catalog SET seeded_by = 'not-a-uuid' WHERE ulid = ?", + [written.ulid], + ); + + expect( + () => catalog.byUlid(written.ulid), + throwsA(isA()), + ); + }); + + test('half a seed claim is corruption, not an unclaimed seed', () { + // Reading it as "nobody has seeded this" would invite a second device to + // seed a document that already has a history — the one thing Decision 7 + // says must never happen. + final ulid = forceColumn('seed_hlc', '10.0'); + + expect( + () => catalog.byUlid(ulid), + throwsA( + isA().having( + (e) => e.toString(), + 'toString', + contains('half a seed claim'), + ), + ), + ); + }); + + test('a seeder with no clock is refused too', () { + final ulid = forceColumn('seeded_by', peerA.toString()); + + expect( + () => catalog.byUlid(ulid), + throwsA(isA()), + ); + }); + + test('the failure names the row, so a person can go look at it', () { + final ulid = forceColumn('state', 'archived'); + + expect( + () => catalog.byUlid(ulid), + throwsA( + isA().having( + (e) => e.toString(), + 'toString', + allOf(contains(ulid), contains('state')), + ), + ), + ); + }); + }); +} diff --git a/test/engram/crdt/catalog_test.dart b/test/engram/crdt/catalog_test.dart new file mode 100644 index 0000000..993f616 --- /dev/null +++ b/test/engram/crdt/catalog_test.dart @@ -0,0 +1,226 @@ +import 'dart:typed_data'; + +import 'package:brainframe/engram/crdt/catalog.dart'; +import 'package:crdt_lf/crdt_lf.dart'; +import 'package:flutter_test/flutter_test.dart'; +import 'package:hlc_dart/hlc_dart.dart'; + +import '../../crdt/support/peer_ids.dart'; + +/// The catalog's value types, which are platform-neutral and hold no storage. +void main() { + OperationId claim(PeerId peer, int millis) => + OperationId(peer, HybridLogicalClock(l: millis, c: 0)); + + CatalogRow row({ + String ulid = '01JBQ9YQ7C8VF9YB0X5H3TQ2ZK', + String path = 'inbox/today.md', + MergePolicy mergePolicy = MergePolicy.fugueText, + NoteState state = NoteState.live, + String? materializedHash, + int? size, + DateTime? mtimeUtc, + Uint8List? sketch, + OperationId? seedClaim, + }) => CatalogRow( + ulid: ulid, + path: path, + mergePolicy: mergePolicy, + state: state, + materializedHash: materializedHash, + size: size, + mtimeUtc: mtimeUtc, + sketch: sketch, + seedClaim: seedClaim, + ); + + group('MergePolicy', () { + test('round-trips through its stored spelling', () { + for (final policy in MergePolicy.values) { + expect(MergePolicy.parse(policy.name), policy); + } + }); + + test('an unknown policy is refused, not defaulted', () { + // Guessing at semantics we do not recognise is how a PNG would get + // character-merged; the store turns this into a surfaced failure. + expect( + () => MergePolicy.parse('vectorInk'), + throwsA(isA()), + ); + }); + + test('is an open enum, so nothing may assume it holds exactly two', () { + // Pins the shape rather than the count: a third policy (vector ink) is + // designed for, so this must not become `expect(values.length, 2)`. + expect(MergePolicy.values, contains(MergePolicy.fugueText)); + expect(MergePolicy.values, contains(MergePolicy.blobLww)); + }); + }); + + group('mergePolicyForPath', () { + test('text extensions get fugueText', () { + for (final path in ['a.md', 'a.markdown', 'a.txt', 'a.text']) { + expect(mergePolicyForPath(path), MergePolicy.fugueText, reason: path); + } + }); + + test('the extension is matched case-insensitively', () { + expect(mergePolicyForPath('NOTES.MD'), MergePolicy.fugueText); + expect(mergePolicyForPath('Notes.Md'), MergePolicy.fugueText); + }); + + test('binary and unknown extensions get blobLww', () { + for (final path in ['a.png', 'a.pdf', 'a.epub', 'a.zip', 'a.wat']) { + expect(mergePolicyForPath(path), MergePolicy.blobLww, reason: path); + } + }); + + test('an unrecognised extension defaults to the recoverable failure', () { + // The asymmetry in Decision 3: last-writer-wins on text loses an edit + // that still exists in the loser's history, while character-merging a + // binary produces a file nobody can recover. + expect(mergePolicyForPath('a.unheard-of'), MergePolicy.blobLww); + }); + + test('a name with no extension is a blob', () { + expect(mergePolicyForPath('LICENSE'), MergePolicy.blobLww); + expect(mergePolicyForPath('notes/LICENSE'), MergePolicy.blobLww); + }); + + test('a dotfile has no extension', () { + // The leading dot names a hidden file; it does not introduce one. + expect(mergePolicyForPath('.gitignore'), MergePolicy.blobLww); + expect(mergePolicyForPath('notes/.gitignore'), MergePolicy.blobLww); + }); + + test('a dotfile that also has an extension keeps it', () { + expect(mergePolicyForPath('.hidden.md'), MergePolicy.fugueText); + }); + + test('only the last extension counts', () { + expect(mergePolicyForPath('archive.md.zip'), MergePolicy.blobLww); + expect(mergePolicyForPath('archive.zip.md'), MergePolicy.fugueText); + }); + + test('a dot in a directory name is not the note\'s extension', () { + // Otherwise every note under `v1.0/` would be typed by its folder. + expect(mergePolicyForPath('v1.0/notes'), MergePolicy.blobLww); + expect(mergePolicyForPath('v1.0/notes.md'), MergePolicy.fugueText); + }); + }); + + group('NoteState', () { + test('round-trips through its stored spelling', () { + for (final state in NoteState.values) { + expect(NoteState.parse(state.name), state); + } + }); + + test('an unknown state is refused', () { + expect( + () => NoteState.parse('archived'), + throwsA(isA()), + ); + }); + + test('unavailable is distinct from tombstoned', () { + // A file missing because a drive is unmounted is not a deleted file, and + // collapsing the two would destroy a note that is merely out of reach. + expect(NoteState.unavailable, isNot(NoteState.tombstoned)); + }); + }); + + group('CatalogRow', () { + test('a bare row carries no device-local state yet', () { + final bare = row(); + + expect(bare.materializedHash, isNull); + expect(bare.size, isNull); + expect(bare.mtimeUtc, isNull); + expect(bare.sketch, isNull); + }); + + test('no seed claim means no seeder', () { + // An unclaimed seed: the identity map knows the note, but no surviving + // op-log ever backed it. + expect(row().seedClaim, isNull); + expect(row().seededBy, isNull); + }); + + test('a seed claim exposes the peer that took it', () { + expect(row(seedClaim: claim(peerA, 10)).seededBy, peerA); + }); + + test('seed claims order by the locked comparator', () { + // HLC first, peerID second — the library's ordering, reused rather than + // reimplemented, so the catalog cannot drift from the op-log. + expect(claim(peerA, 10).compareTo(claim(peerB, 20)), lessThan(0)); + expect(claim(peerC, 20).compareTo(claim(peerA, 10)), greaterThan(0)); + // Equal clocks fall through to the peerID. + expect(claim(peerA, 10).compareTo(claim(peerB, 10)), lessThan(0)); + }); + + test('equal rows are equal, field by field', () { + final mtime = DateTime.utc(2026, 9, 5, 12); + final sketch = Uint8List.fromList([1, 2, 3]); + final a = row( + materializedHash: 'h', + size: 12, + mtimeUtc: mtime, + sketch: sketch, + seedClaim: claim(peerA, 10), + ); + final b = row( + materializedHash: 'h', + size: 12, + mtimeUtc: mtime, + // A different list with the same bytes: Uint8List equality is + // identity, so this is what would break a round-trip comparison. + sketch: Uint8List.fromList([1, 2, 3]), + seedClaim: claim(peerA, 10), + ); + + expect(a, b); + expect(a.hashCode, b.hashCode); + }); + + test('each field participates in equality', () { + final base = row(); + + expect(base, isNot(row(ulid: '01JBQ9YQ7C8VF9YB0X5H3TQ2ZL'))); + expect(base, isNot(row(path: 'inbox/other.md'))); + expect(base, isNot(row(mergePolicy: MergePolicy.blobLww))); + expect(base, isNot(row(state: NoteState.tombstoned))); + expect(base, isNot(row(materializedHash: 'h'))); + expect(base, isNot(row(size: 1))); + expect(base, isNot(row(mtimeUtc: DateTime.utc(2026)))); + expect(base, isNot(row(sketch: Uint8List.fromList([1])))); + expect(base, isNot(row(seedClaim: claim(peerA, 10)))); + }); + + test('sketches of different lengths are not equal', () { + expect( + row(sketch: Uint8List.fromList([1, 2])), + isNot(row(sketch: Uint8List.fromList([1, 2, 3]))), + ); + }); + + test('sketches of equal length but different bytes are not equal', () { + expect( + row(sketch: Uint8List.fromList([1, 2, 3])), + isNot(row(sketch: Uint8List.fromList([1, 2, 4]))), + ); + }); + + test('toString names the note without dumping its device-local state', () { + final text = row(materializedHash: 'secret-ish').toString(); + + expect(text, contains('01JBQ9YQ7C8VF9YB0X5H3TQ2ZK')); + expect(text, contains('inbox/today.md')); + expect(text, contains('fugueText')); + expect(text, contains('live')); + expect(text, isNot(contains('secret-ish'))); + }); + }); +} diff --git a/test/engram/crdt/metadata_db_io_test.dart b/test/engram/crdt/metadata_db_io_test.dart index 34af58c..09bd162 100644 --- a/test/engram/crdt/metadata_db_io_test.dart +++ b/test/engram/crdt/metadata_db_io_test.dart @@ -40,10 +40,7 @@ void main() { final store = await MetadataDatabase.open(id, resolveRoot: resolveRoot); addTearDown(store.close); - expect( - File('${root.path}/engrams/$id/metadata.db').existsSync(), - isTrue, - ); + expect(File('${root.path}/engrams/$id/metadata.db').existsSync(), isTrue); }); test('holds our tables and the op-log in one file', () async { @@ -53,7 +50,12 @@ void main() { ); addTearDown(store.close); - expect(tableNames(store.database), {'bf_meta', 'changes', 'snapshots'}); + expect(tableNames(store.database), { + 'bf_catalog', + 'bf_meta', + 'changes', + 'snapshots', + }); }); test('stamps the current schema version', () async { @@ -160,9 +162,9 @@ void main() { // Guards the test above from passing vacuously if our schema were ever // reduced to nothing. expect( - tableNames(store.database).where( - (name) => name.startsWith(brainframeTablePrefix), - ), + tableNames( + store.database, + ).where((name) => name.startsWith(brainframeTablePrefix)), isNotEmpty, ); }); @@ -173,8 +175,13 @@ void main() { final store = MetadataDatabase.openInMemory(); addTearDown(store.close); + // Subtracting by prefix rather than by an enumerated list of our tables, + // so adding one to BrainFrame's schema never needs an edit here — while + // a table appearing from the library still fails, which is the point. expect( - tableNames(store.database).difference({'bf_meta'}), + tableNames( + store.database, + ).where((name) => !name.startsWith(brainframeTablePrefix)).toSet(), crdtTableNames, ); }); @@ -288,7 +295,10 @@ void main() { // Populate the store with everything that must survive: the peer // identity, the schema stamp, and a real op-log entry. - final before = await MetadataDatabase.open(from, resolveRoot: resolveRoot); + final before = await MetadataDatabase.open( + from, + resolveRoot: resolveRoot, + ); final peer = before.peerId; final author = Replica.named(peerA, label: 'author'); author.note.insert(0, 'history that must survive a rename');