From 54fa92e627319cbf48b72b5218dbb28859e7b279 Mon Sep 17 00:00:00 2001 From: epi13 Date: Tue, 4 Aug 2026 08:33:56 -0800 Subject: [PATCH 01/10] Add Python package scaffold --- pyproject.toml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 pyproject.toml diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..c26f269 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,17 @@ +[build-system] +requires = ["setuptools>=68"] +build-backend = "setuptools.build_meta" + +[project] +name = "ravel-memory" +version = "0.1.0" +description = "Provenance-preserving memory consolidation primitives for RAVEL" +requires-python = ">=3.11" +authors = [{name = "RAVEL contributors"}] +dependencies = [] + +[tool.setuptools.packages.find] +where = ["src"] + +[tool.pytest.ini_options] +testpaths = ["tests"] From 994a81e921f9ed8b7437fbdf814b5aabe1ed8928 Mon Sep 17 00:00:00 2001 From: epi13 Date: Tue, 4 Aug 2026 08:34:01 -0800 Subject: [PATCH 02/10] Add RAVEL package namespace --- src/ravel/__init__.py | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 src/ravel/__init__.py diff --git a/src/ravel/__init__.py b/src/ravel/__init__.py new file mode 100644 index 0000000..cc1fbb6 --- /dev/null +++ b/src/ravel/__init__.py @@ -0,0 +1,3 @@ +"""RAVEL executable research components.""" + +__all__ = ["memory"] From 7ef17e00560738b227d4656cfd65608d37bf0240 Mon Sep 17 00:00:00 2001 From: epi13 Date: Tue, 4 Aug 2026 08:34:08 -0800 Subject: [PATCH 03/10] Expose memory consolidation API --- src/ravel/memory/__init__.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 src/ravel/memory/__init__.py diff --git a/src/ravel/memory/__init__.py b/src/ravel/memory/__init__.py new file mode 100644 index 0000000..9760d8e --- /dev/null +++ b/src/ravel/memory/__init__.py @@ -0,0 +1,28 @@ +"""Memory records, immutable storage, consolidation, and retrieval planning.""" + +from .consolidation import ( + ConsolidationPolicy, + MemoryConsolidator, + RetrievalLayoutPlanner, +) +from .models import ( + AccessEvent, + ConsolidationProposal, + MemoryClass, + MemoryRecord, + RetrievalBucket, +) +from .store import ImmutableRecordError, SQLiteMemoryStore + +__all__ = [ + "AccessEvent", + "ConsolidationPolicy", + "ConsolidationProposal", + "ImmutableRecordError", + "MemoryClass", + "MemoryConsolidator", + "MemoryRecord", + "RetrievalBucket", + "RetrievalLayoutPlanner", + "SQLiteMemoryStore", +] From 4cb9dfb57995d21573fb604c305e18a6dfba4b6c Mon Sep 17 00:00:00 2001 From: epi13 Date: Tue, 4 Aug 2026 08:34:26 -0800 Subject: [PATCH 04/10] Define immutable memory records --- src/ravel/memory/models.py | 156 +++++++++++++++++++++++++++++++++++++ 1 file changed, 156 insertions(+) create mode 100644 src/ravel/memory/models.py diff --git a/src/ravel/memory/models.py b/src/ravel/memory/models.py new file mode 100644 index 0000000..93bb37f --- /dev/null +++ b/src/ravel/memory/models.py @@ -0,0 +1,156 @@ +"""Typed, deterministic records for the RAVEL memory prototype. + +The records in this module deliberately separate authoritative source memory from +replaceable consolidation and retrieval projections. Consolidation proposals +never overwrite or delete their source records. +""" + +from __future__ import annotations + +from dataclasses import asdict, dataclass, field +from datetime import datetime, timezone +from enum import Enum +import hashlib +import json +from typing import Any, Mapping + + +def utc_now() -> str: + """Return a stable RFC 3339 UTC timestamp.""" + + return datetime.now(timezone.utc).isoformat().replace("+00:00", "Z") + + +def canonical_json(value: Mapping[str, Any]) -> str: + """Serialize a mapping deterministically for hashing and export.""" + + return json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=False) + + +def digest_mapping(value: Mapping[str, Any]) -> str: + """Return a SHA-256 content digest for a canonical mapping.""" + + payload = canonical_json(value).encode("utf-8") + return f"sha256:{hashlib.sha256(payload).hexdigest()}" + + +class MemoryClass(str, Enum): + EPISODIC = "episodic" + CAUSAL = "causal" + SEMANTIC = "semantic" + PROCEDURAL = "procedural" + NEGATIVE = "negative" + + +@dataclass(frozen=True, slots=True) +class MemoryRecord: + """An immutable source memory record. + + ``scope`` is intentionally structured. Records with incompatible scope are + never automatically consolidated, even when their text is very similar. + ``relations`` supports explicit links such as ``contradicts`` and + ``supersedes`` without asking the consolidator to invent causal meaning. + """ + + record_id: str + memory_class: MemoryClass + statement: str + scope: Mapping[str, str] + created_at: str + producer_id: str + authority_class: str = "advisory" + status: str = "active" + tags: tuple[str, ...] = () + source_ids: tuple[str, ...] = () + relations: Mapping[str, tuple[str, ...]] = field(default_factory=dict) + metadata: Mapping[str, Any] = field(default_factory=dict) + + def __post_init__(self) -> None: + if not self.record_id.strip(): + raise ValueError("record_id must not be empty") + if not self.statement.strip(): + raise ValueError("statement must not be empty") + if not self.scope: + raise ValueError("scope must declare at least one boundary") + if not self.producer_id.strip(): + raise ValueError("producer_id must not be empty") + + def to_dict(self) -> dict[str, Any]: + payload = asdict(self) + payload["memory_class"] = self.memory_class.value + payload["tags"] = list(self.tags) + payload["source_ids"] = list(self.source_ids) + payload["relations"] = { + key: list(values) for key, values in sorted(self.relations.items()) + } + payload["scope"] = dict(sorted(self.scope.items())) + payload["metadata"] = dict(self.metadata) + return payload + + @property + def digest(self) -> str: + return digest_mapping(self.to_dict()) + + @property + def scope_signature(self) -> tuple[tuple[str, str], ...]: + return tuple(sorted(self.scope.items())) + + +@dataclass(frozen=True, slots=True) +class ConsolidationProposal: + """A derived, challengeable summary over source records.""" + + proposal_id: str + method_version: str + created_at: str + memory_class: MemoryClass + canonical_statement: str + scope: Mapping[str, str] + member_ids: tuple[str, ...] + supporting_ids: tuple[str, ...] + contradicting_ids: tuple[str, ...] + superseded_ids: tuple[str, ...] + retrieval_keys: tuple[str, ...] + clustering_confidence: float + status: str = "proposed" + limitations: tuple[str, ...] = ( + "Derived projection only; does not alter source status or authority.", + ) + + def to_dict(self) -> dict[str, Any]: + payload = asdict(self) + payload["memory_class"] = self.memory_class.value + payload["scope"] = dict(sorted(self.scope.items())) + for key in ( + "member_ids", + "supporting_ids", + "contradicting_ids", + "superseded_ids", + "retrieval_keys", + "limitations", + ): + payload[key] = list(payload[key]) + return payload + + @property + def digest(self) -> str: + return digest_mapping(self.to_dict()) + + +@dataclass(frozen=True, slots=True) +class AccessEvent: + """One query-time access observation for a replaceable retrieval projection.""" + + query_id: str + retrieved_ids: tuple[str, ...] + selected_ids: tuple[str, ...] = () + + +@dataclass(frozen=True, slots=True) +class RetrievalBucket: + """A suggested co-location bucket based only on observed co-access.""" + + bucket_id: str + member_ids: tuple[str, ...] + weighted_edges: tuple[tuple[str, str, int], ...] + reason: str = "frequent-co-access" From 79f1fd418c21743090ec3e1bbe6b6236d64481de Mon Sep 17 00:00:00 2001 From: epi13 Date: Tue, 4 Aug 2026 08:35:04 -0800 Subject: [PATCH 05/10] Implement semantic consolidation and retrieval planning --- src/ravel/memory/consolidation.py | 338 ++++++++++++++++++++++++++++++ 1 file changed, 338 insertions(+) create mode 100644 src/ravel/memory/consolidation.py diff --git a/src/ravel/memory/consolidation.py b/src/ravel/memory/consolidation.py new file mode 100644 index 0000000..eea58a6 --- /dev/null +++ b/src/ravel/memory/consolidation.py @@ -0,0 +1,338 @@ +"""Deterministic semantic consolidation and retrieval-layout planning. + +This module is intentionally conservative. It creates proposals over compatible +records; it does not rewrite history, promote knowledge, infer formal status, or +modify MNCS/MNCDS-governed evidence. +""" + +from __future__ import annotations + +from collections import Counter, defaultdict +from dataclasses import dataclass +import hashlib +import re +from typing import Iterable, Sequence + +from .models import ( + AccessEvent, + ConsolidationProposal, + MemoryRecord, + RetrievalBucket, + utc_now, +) + +_TOKEN_RE = re.compile(r"[a-z0-9][a-z0-9_\-]*") +_STOP_WORDS = frozenset( + { + "a", + "an", + "and", + "are", + "as", + "at", + "be", + "by", + "for", + "from", + "in", + "is", + "it", + "of", + "on", + "or", + "that", + "the", + "to", + "was", + "were", + "with", + } +) + + +def _normalize_token(token: str) -> str: + # A deliberately small, deterministic normalization baseline. This handles + # common inflectional duplicates such as preserve/preserves without adding a + # language-model or stemming dependency. + if len(token) > 4 and token.endswith("s") and not token.endswith("ss"): + return token[:-1] + return token + + +def _tokens(text: str) -> frozenset[str]: + return frozenset( + _normalize_token(token) + for token in _TOKEN_RE.findall(text.casefold()) + if token not in _STOP_WORDS + ) + + +def _jaccard(left: frozenset[str], right: frozenset[str]) -> float: + if not left and not right: + return 1.0 + union = left | right + return len(left & right) / len(union) if union else 0.0 + + +def _stable_id(prefix: str, values: Sequence[str]) -> str: + material = "\x1f".join(sorted(values)).encode("utf-8") + return f"{prefix}:{hashlib.sha256(material).hexdigest()[:24]}" + + +class _UnionFind: + def __init__(self, values: Iterable[str]) -> None: + self.parent = {value: value for value in values} + + def find(self, value: str) -> str: + parent = self.parent[value] + if parent != value: + self.parent[value] = self.find(parent) + return self.parent[value] + + def union(self, left: str, right: str) -> None: + left_root = self.find(left) + right_root = self.find(right) + if left_root == right_root: + return + # Stable root selection makes output independent of input order. + low, high = sorted((left_root, right_root)) + self.parent[high] = low + + +@dataclass(frozen=True, slots=True) +class ConsolidationPolicy: + similarity_threshold: float = 0.72 + minimum_cluster_size: int = 2 + maximum_cluster_size: int = 64 + retrieval_key_count: int = 8 + method_version: str = "ravel-semantic-consolidation/0.1" + + def __post_init__(self) -> None: + if not 0.0 <= self.similarity_threshold <= 1.0: + raise ValueError("similarity_threshold must be between 0 and 1") + if self.minimum_cluster_size < 2: + raise ValueError("minimum_cluster_size must be at least 2") + if self.maximum_cluster_size < self.minimum_cluster_size: + raise ValueError("maximum_cluster_size must not be smaller than minimum") + if self.retrieval_key_count < 1: + raise ValueError("retrieval_key_count must be positive") + + +class MemoryConsolidator: + """Create provenance-preserving consolidation proposals.""" + + def __init__(self, policy: ConsolidationPolicy | None = None) -> None: + self.policy = policy or ConsolidationPolicy() + + def propose( + self, + records: Iterable[MemoryRecord], + *, + created_at: str | None = None, + ) -> tuple[ConsolidationProposal, ...]: + ordered = sorted(records, key=lambda record: record.record_id) + if not ordered: + return () + + record_by_id = {record.record_id: record for record in ordered} + if len(record_by_id) != len(ordered): + raise ValueError("record_id values must be unique") + + groups: dict[tuple[object, ...], list[MemoryRecord]] = defaultdict(list) + for record in ordered: + groups[(record.memory_class, record.scope_signature)].append(record) + + proposals: list[ConsolidationProposal] = [] + timestamp = created_at or utc_now() + for scoped_records in groups.values(): + proposals.extend( + self._propose_group(scoped_records, record_by_id, created_at=timestamp) + ) + return tuple(sorted(proposals, key=lambda item: item.proposal_id)) + + def _propose_group( + self, + records: Sequence[MemoryRecord], + record_by_id: dict[str, MemoryRecord], + *, + created_at: str, + ) -> list[ConsolidationProposal]: + if len(records) < self.policy.minimum_cluster_size: + return [] + + token_map = {record.record_id: _tokens(record.statement) for record in records} + forest = _UnionFind(token_map) + + for index, left in enumerate(records): + for right in records[index + 1 :]: + similarity = _jaccard(token_map[left.record_id], token_map[right.record_id]) + if similarity >= self.policy.similarity_threshold: + forest.union(left.record_id, right.record_id) + + components: dict[str, list[MemoryRecord]] = defaultdict(list) + for record in records: + components[forest.find(record.record_id)].append(record) + + proposals: list[ConsolidationProposal] = [] + for component in components.values(): + if len(component) < self.policy.minimum_cluster_size: + continue + if len(component) > self.policy.maximum_cluster_size: + component = sorted(component, key=lambda item: item.record_id)[ + : self.policy.maximum_cluster_size + ] + proposals.append( + self._make_proposal(component, record_by_id, token_map, created_at) + ) + return proposals + + def _make_proposal( + self, + component: Sequence[MemoryRecord], + record_by_id: dict[str, MemoryRecord], + token_map: dict[str, frozenset[str]], + created_at: str, + ) -> ConsolidationProposal: + members = sorted(component, key=lambda item: item.record_id) + member_ids = tuple(record.record_id for record in members) + + contradicted: set[str] = set() + superseded: set[str] = set() + for record in members: + contradicted.update(record.relations.get("contradicts", ())) + superseded.update(record.relations.get("supersedes", ())) + + # Keep only valid source identities; unresolved links remain in raw records. + contradiction_ids = tuple( + sorted(item for item in contradicted if item in record_by_id) + ) + superseded_ids = tuple( + sorted(item for item in superseded if item in record_by_id) + ) + supporting_ids = tuple( + record_id + for record_id in member_ids + if record_id not in contradicted and record_id not in superseded + ) + + representative = max(members, key=self._representative_rank) + keys = self._retrieval_keys(members) + confidence = self._cluster_confidence(members, token_map) + + return ConsolidationProposal( + proposal_id=_stable_id("consolidation", member_ids), + method_version=self.policy.method_version, + created_at=created_at, + memory_class=representative.memory_class, + canonical_statement=representative.statement.strip(), + scope=dict(representative.scope), + member_ids=member_ids, + supporting_ids=supporting_ids, + contradicting_ids=contradiction_ids, + superseded_ids=superseded_ids, + retrieval_keys=keys, + clustering_confidence=round(confidence, 6), + ) + + @staticmethod + def _representative_rank(record: MemoryRecord) -> tuple[int, int, int, str, str]: + status_rank = 0 if record.status in {"retired", "rejected"} else 1 + authority_rank = { + "advisory": 0, + "repository-local": 1, + "governed-evaluation": 2, + "protected": 3, + }.get(record.authority_class, 0) + return ( + status_rank, + authority_rank, + len(record.source_ids), + record.created_at, + record.record_id, + ) + + def _retrieval_keys(self, records: Sequence[MemoryRecord]) -> tuple[str, ...]: + counts: Counter[str] = Counter() + for record in records: + counts.update(_tokens(record.statement)) + counts.update(tag.casefold() for tag in record.tags) + ordered = sorted(counts.items(), key=lambda item: (-item[1], item[0])) + return tuple(token for token, _ in ordered[: self.policy.retrieval_key_count]) + + @staticmethod + def _cluster_confidence( + records: Sequence[MemoryRecord], + token_map: dict[str, frozenset[str]], + ) -> float: + if len(records) < 2: + return 0.0 + similarities: list[float] = [] + for index, left in enumerate(records): + for right in records[index + 1 :]: + similarities.append( + _jaccard(token_map[left.record_id], token_map[right.record_id]) + ) + return sum(similarities) / len(similarities) + + +class RetrievalLayoutPlanner: + """Suggest record co-location from observed access patterns. + + This planner changes no source record and performs no physical write. It + emits rebuildable layout buckets that a future storage adapter may choose to + use for pages, shards, caches, graph entry points, or prefetch groups. + """ + + def plan( + self, + events: Iterable[AccessEvent], + *, + minimum_coaccess: int = 2, + ) -> tuple[RetrievalBucket, ...]: + if minimum_coaccess < 1: + raise ValueError("minimum_coaccess must be positive") + + weights: Counter[tuple[str, str]] = Counter() + all_ids: set[str] = set() + for event in events: + chosen = tuple(sorted(set(event.selected_ids or event.retrieved_ids))) + all_ids.update(chosen) + for index, left in enumerate(chosen): + for right in chosen[index + 1 :]: + weights[(left, right)] += 1 + + eligible = { + edge: weight for edge, weight in weights.items() if weight >= minimum_coaccess + } + if not eligible: + return () + + forest = _UnionFind(all_ids) + for left, right in eligible: + forest.union(left, right) + + components: dict[str, set[str]] = defaultdict(set) + for record_id in all_ids: + components[forest.find(record_id)].add(record_id) + + buckets: list[RetrievalBucket] = [] + for members in components.values(): + if len(members) < 2: + continue + member_ids = tuple(sorted(members)) + member_set = set(member_ids) + edges = tuple( + sorted( + (left, right, weight) + for (left, right), weight in eligible.items() + if left in member_set and right in member_set + ) + ) + buckets.append( + RetrievalBucket( + bucket_id=_stable_id("retrieval-bucket", member_ids), + member_ids=member_ids, + weighted_edges=edges, + ) + ) + return tuple(sorted(buckets, key=lambda bucket: bucket.bucket_id)) From 14403f4c91fd735b721403e87bbef4f202c639e1 Mon Sep 17 00:00:00 2001 From: epi13 Date: Tue, 4 Aug 2026 08:35:29 -0800 Subject: [PATCH 06/10] Add append-only SQLite memory store --- src/ravel/memory/store.py | 224 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 224 insertions(+) create mode 100644 src/ravel/memory/store.py diff --git a/src/ravel/memory/store.py b/src/ravel/memory/store.py new file mode 100644 index 0000000..fdf720f --- /dev/null +++ b/src/ravel/memory/store.py @@ -0,0 +1,224 @@ +"""Small SQLite-backed append-only store for RAVEL memory records.""" + +from __future__ import annotations + +import json +from pathlib import Path +import sqlite3 +from typing import Iterable + +from .models import ConsolidationProposal, MemoryClass, MemoryRecord, canonical_json + + +class ImmutableRecordError(RuntimeError): + """Raised when an existing logical identity is reused with different bytes.""" + + +class SQLiteMemoryStore: + """Persist source records and derived proposals without mutating history.""" + + def __init__(self, path: str | Path) -> None: + self.path = str(path) + self._connection = sqlite3.connect(self.path) + self._connection.row_factory = sqlite3.Row + self._connection.execute("PRAGMA foreign_keys = ON") + self._create_schema() + + def __enter__(self) -> "SQLiteMemoryStore": + return self + + def __exit__(self, exc_type: object, exc: object, traceback: object) -> None: + self.close() + + def close(self) -> None: + self._connection.close() + + def _create_schema(self) -> None: + with self._connection: + self._connection.executescript( + """ + CREATE TABLE IF NOT EXISTS source_records ( + sequence INTEGER PRIMARY KEY AUTOINCREMENT, + record_id TEXT NOT NULL UNIQUE, + digest TEXT NOT NULL, + memory_class TEXT NOT NULL, + payload_json TEXT NOT NULL, + created_at TEXT NOT NULL + ); + + CREATE TABLE IF NOT EXISTS consolidation_proposals ( + sequence INTEGER PRIMARY KEY AUTOINCREMENT, + proposal_id TEXT NOT NULL UNIQUE, + digest TEXT NOT NULL, + payload_json TEXT NOT NULL, + created_at TEXT NOT NULL, + status TEXT NOT NULL + ); + + CREATE TABLE IF NOT EXISTS consolidation_members ( + proposal_id TEXT NOT NULL, + record_id TEXT NOT NULL, + relation TEXT NOT NULL, + PRIMARY KEY (proposal_id, record_id, relation), + FOREIGN KEY (proposal_id) + REFERENCES consolidation_proposals(proposal_id), + FOREIGN KEY (record_id) + REFERENCES source_records(record_id) + ); + + CREATE INDEX IF NOT EXISTS source_records_class_idx + ON source_records(memory_class, sequence); + CREATE INDEX IF NOT EXISTS proposal_status_idx + ON consolidation_proposals(status, sequence); + """ + ) + + def insert_record(self, record: MemoryRecord) -> None: + payload = canonical_json(record.to_dict()) + existing = self._connection.execute( + "SELECT digest FROM source_records WHERE record_id = ?", (record.record_id,) + ).fetchone() + if existing is not None: + if existing["digest"] == record.digest: + return + raise ImmutableRecordError( + f"record {record.record_id!r} already exists with different content" + ) + with self._connection: + self._connection.execute( + """ + INSERT INTO source_records + (record_id, digest, memory_class, payload_json, created_at) + VALUES (?, ?, ?, ?, ?) + """, + ( + record.record_id, + record.digest, + record.memory_class.value, + payload, + record.created_at, + ), + ) + + def insert_records(self, records: Iterable[MemoryRecord]) -> None: + for record in records: + self.insert_record(record) + + def get_record(self, record_id: str) -> MemoryRecord | None: + row = self._connection.execute( + "SELECT payload_json FROM source_records WHERE record_id = ?", (record_id,) + ).fetchone() + return self._decode_record(row["payload_json"]) if row else None + + def iter_records( + self, memory_class: MemoryClass | None = None + ) -> tuple[MemoryRecord, ...]: + if memory_class is None: + rows = self._connection.execute( + "SELECT payload_json FROM source_records ORDER BY sequence" + ).fetchall() + else: + rows = self._connection.execute( + """ + SELECT payload_json FROM source_records + WHERE memory_class = ? ORDER BY sequence + """, + (memory_class.value,), + ).fetchall() + return tuple(self._decode_record(row["payload_json"]) for row in rows) + + def insert_proposal(self, proposal: ConsolidationProposal) -> None: + missing = [ + record_id + for record_id in proposal.member_ids + if self.get_record(record_id) is None + ] + if missing: + raise ValueError(f"proposal references missing records: {missing}") + + payload = canonical_json(proposal.to_dict()) + existing = self._connection.execute( + """ + SELECT digest FROM consolidation_proposals WHERE proposal_id = ? + """, + (proposal.proposal_id,), + ).fetchone() + if existing is not None: + if existing["digest"] == proposal.digest: + return + raise ImmutableRecordError( + f"proposal {proposal.proposal_id!r} already exists with different content" + ) + + relations: list[tuple[str, str, str]] = [] + support = set(proposal.supporting_ids) + contradictions = set(proposal.contradicting_ids) + superseded = set(proposal.superseded_ids) + for record_id in proposal.member_ids: + relation = "member" + if record_id in support: + relation = "supporting" + elif record_id in contradictions: + relation = "contradicting" + elif record_id in superseded: + relation = "superseded" + relations.append((proposal.proposal_id, record_id, relation)) + + with self._connection: + self._connection.execute( + """ + INSERT INTO consolidation_proposals + (proposal_id, digest, payload_json, created_at, status) + VALUES (?, ?, ?, ?, ?) + """, + ( + proposal.proposal_id, + proposal.digest, + payload, + proposal.created_at, + proposal.status, + ), + ) + self._connection.executemany( + """ + INSERT INTO consolidation_members + (proposal_id, record_id, relation) + VALUES (?, ?, ?) + """, + relations, + ) + + def export_jsonl(self) -> str: + """Return a deterministic source-first replay stream.""" + + lines: list[str] = [] + for row in self._connection.execute( + "SELECT payload_json FROM source_records ORDER BY sequence" + ): + lines.append(row["payload_json"]) + for row in self._connection.execute( + "SELECT payload_json FROM consolidation_proposals ORDER BY sequence" + ): + lines.append(row["payload_json"]) + return "\n".join(lines) + ("\n" if lines else "") + + @staticmethod + def _decode_record(payload_json: str) -> MemoryRecord: + payload = json.loads(payload_json) + return MemoryRecord( + record_id=payload["record_id"], + memory_class=MemoryClass(payload["memory_class"]), + statement=payload["statement"], + scope=payload["scope"], + created_at=payload["created_at"], + producer_id=payload["producer_id"], + authority_class=payload.get("authority_class", "advisory"), + status=payload.get("status", "active"), + tags=tuple(payload.get("tags", ())), + source_ids=tuple(payload.get("source_ids", ())), + relations={ + key: tuple(values) + for key, values in payload.get("relations", {}).items() + }, + metadata=payload.get("metadata", {}), + ) From a55fabfe5bce63862dbb1ad18da6a91f8b8348e4 Mon Sep 17 00:00:00 2001 From: epi13 Date: Tue, 4 Aug 2026 08:35:55 -0800 Subject: [PATCH 07/10] Test memory consolidation invariants --- tests/test_consolidation.py | 166 ++++++++++++++++++++++++++++++++++++ 1 file changed, 166 insertions(+) create mode 100644 tests/test_consolidation.py diff --git a/tests/test_consolidation.py b/tests/test_consolidation.py new file mode 100644 index 0000000..e81b89d --- /dev/null +++ b/tests/test_consolidation.py @@ -0,0 +1,166 @@ +from __future__ import annotations + +import tempfile +import unittest + +from ravel.memory import ( + AccessEvent, + ConsolidationPolicy, + ImmutableRecordError, + MemoryClass, + MemoryConsolidator, + MemoryRecord, + RetrievalLayoutPlanner, + SQLiteMemoryStore, +) + + +SCOPE = {"repository": "epi13/RAVEL", "contract": "mncs-memory-v1"} + + +def record( + record_id: str, + statement: str, + *, + scope: dict[str, str] | None = None, + relations: dict[str, tuple[str, ...]] | None = None, + status: str = "active", + authority_class: str = "repository-local", +) -> MemoryRecord: + return MemoryRecord( + record_id=record_id, + memory_class=MemoryClass.SEMANTIC, + statement=statement, + scope=scope or SCOPE, + created_at="2026-08-04T16:00:00Z", + producer_id="test-suite", + authority_class=authority_class, + status=status, + relations=relations or {}, + ) + + +class ConsolidationTests(unittest.TestCase): + def test_near_duplicates_produce_provenance_preserving_proposal(self) -> None: + records = [ + record("memory:1", "RAVEL preserves negative memory during retrieval."), + record("memory:2", "During retrieval RAVEL must preserve negative memory."), + ] + proposals = MemoryConsolidator( + ConsolidationPolicy(similarity_threshold=0.65) + ).propose(records, created_at="2026-08-04T17:00:00Z") + + self.assertEqual(len(proposals), 1) + proposal = proposals[0] + self.assertEqual(proposal.member_ids, ("memory:1", "memory:2")) + self.assertEqual(proposal.supporting_ids, proposal.member_ids) + self.assertEqual(proposal.status, "proposed") + self.assertIn("negative", proposal.retrieval_keys) + self.assertEqual( + records[0].statement, + "RAVEL preserves negative memory during retrieval.", + ) + + def test_scope_boundary_prevents_false_consolidation(self) -> None: + records = [ + record("memory:1", "The verifier result remains UNKNOWN."), + record( + "memory:2", + "The verifier result remains UNKNOWN.", + scope={"repository": "other/project", "contract": "mncs-memory-v1"}, + ), + ] + proposals = MemoryConsolidator().propose( + records, created_at="2026-08-04T17:00:00Z" + ) + self.assertEqual(proposals, ()) + + def test_explicit_contradiction_is_retained(self) -> None: + records = [ + record("memory:old", "The routing policy uses a static verifier order."), + record( + "memory:new", + "The routing policy uses a static verifier order.", + relations={"contradicts": ("memory:old",)}, + authority_class="governed-evaluation", + ), + ] + proposal = MemoryConsolidator().propose( + records, created_at="2026-08-04T17:00:00Z" + )[0] + self.assertEqual(proposal.contradicting_ids, ("memory:old",)) + self.assertEqual(proposal.supporting_ids, ("memory:new",)) + + def test_output_is_independent_of_input_order(self) -> None: + records = [ + record("memory:a", "RAVEL stores immutable source records."), + record("memory:b", "RAVEL keeps immutable source records."), + ] + consolidator = MemoryConsolidator( + ConsolidationPolicy(similarity_threshold=0.6) + ) + forward = consolidator.propose( + records, created_at="2026-08-04T17:00:00Z" + ) + reverse = consolidator.propose( + reversed(records), created_at="2026-08-04T17:00:00Z" + ) + self.assertEqual(forward, reverse) + + +class StoreTests(unittest.TestCase): + def test_store_rejects_identity_reuse_with_different_content(self) -> None: + with tempfile.TemporaryDirectory() as directory: + with SQLiteMemoryStore(f"{directory}/memory.sqlite3") as store: + store.insert_record(record("memory:1", "Original statement.")) + with self.assertRaises(ImmutableRecordError): + store.insert_record(record("memory:1", "Changed statement.")) + + def test_store_persists_sources_before_proposals(self) -> None: + records = [ + record("memory:1", "RAVEL preserves negative memory during retrieval."), + record("memory:2", "During retrieval RAVEL preserves negative memory."), + ] + proposal = MemoryConsolidator( + ConsolidationPolicy(similarity_threshold=0.65) + ).propose(records, created_at="2026-08-04T17:00:00Z")[0] + with tempfile.TemporaryDirectory() as directory: + with SQLiteMemoryStore(f"{directory}/memory.sqlite3") as store: + store.insert_records(records) + store.insert_proposal(proposal) + replay = store.export_jsonl().splitlines() + self.assertEqual(len(replay), 3) + self.assertIn('"record_id":"memory:1"', replay[0]) + self.assertIn('"proposal_id":', replay[2]) + + +class RetrievalLayoutTests(unittest.TestCase): + def test_frequent_coaccess_forms_rebuildable_bucket(self) -> None: + events = [ + AccessEvent( + "query:1", + ("memory:a", "memory:b", "memory:c"), + ("memory:a", "memory:b"), + ), + AccessEvent( + "query:2", + ("memory:a", "memory:b"), + ("memory:a", "memory:b"), + ), + AccessEvent( + "query:3", + ("memory:a", "memory:c"), + ("memory:a", "memory:c"), + ), + ] + buckets = RetrievalLayoutPlanner().plan(events, minimum_coaccess=2) + self.assertEqual(len(buckets), 1) + self.assertEqual(buckets[0].member_ids, ("memory:a", "memory:b")) + self.assertEqual( + buckets[0].weighted_edges, + (("memory:a", "memory:b", 2),), + ) + + +if __name__ == "__main__": + unittest.main() From 091bdfab449f49ab4a4e9079d4c308d4aaab6b19 Mon Sep 17 00:00:00 2001 From: epi13 Date: Tue, 4 Aug 2026 08:36:18 -0800 Subject: [PATCH 08/10] Document semantic consolidation architecture --- docs/SEMANTIC_CONSOLIDATION.md | 169 +++++++++++++++++++++++++++++++++ 1 file changed, 169 insertions(+) create mode 100644 docs/SEMANTIC_CONSOLIDATION.md diff --git a/docs/SEMANTIC_CONSOLIDATION.md b/docs/SEMANTIC_CONSOLIDATION.md new file mode 100644 index 0000000..6a4c510 --- /dev/null +++ b/docs/SEMANTIC_CONSOLIDATION.md @@ -0,0 +1,169 @@ +# Semantic consolidation and retrieval defragmentation + +## Status + +This document specifies a development prototype. It does not establish MNCS or MNCDS conformance, protected custody, production safety, or validated recursive self-improvement. + +## Motivation + +Long-running memory systems accumulate duplicated statements, fragmented episodes, superseded conclusions, weak cross-links, and retrieval layouts that no longer match actual access patterns. A vector index can still locate approximate neighbors, but similarity alone does not preserve scope, authority, contradictions, negative evidence, or source identity. + +RAVEL therefore separates two related maintenance operations: + +1. **Semantic consolidation** reorganizes the logical representation of memory by proposing canonical summaries over compatible records. +2. **Retrieval defragmentation** reorganizes replaceable indexes, caches, shards, graph entry points, or storage pages according to observed co-access. + +Neither operation may rewrite authoritative history. + +## Invariants + +The initial implementation enforces these invariants: + +- Source memories are immutable and append-only. +- A consolidation is a new derived record, never an edit to a source record. +- Every proposal lists all member, supporting, contradicting, and superseded identities. +- Similarity cannot cross an incompatible scope boundary. +- Clustering confidence is a retrieval-quality estimate, not evidence authority or formal status. +- Explicit contradictions and negative memories remain retrievable. +- Retrieval layout plans are disposable projections and may be rebuilt from access events. +- No consolidation automatically promotes a principle or strategy. +- No memory operation can convert `UNKNOWN` into `PASS`. + +## Architecture + +```text +append-only source records + | + +------------------------------+ + | | + v v +semantic consolidation access telemetry +cluster compatible records selected/retrieved IDs +preserve contradictions | +propose canonical view v + | co-access planner + v | +consolidation proposal v + | retrieval layout plan + +--------------+---------------+ + | + v + replaceable retrieval layer + text / vector / graph / cache / page layout +``` + +The source store remains authoritative. Consolidation proposals and layout plans are projections. + +## Semantic consolidation pipeline + +### 1. Candidate partitioning + +Records are first partitioned by memory class and exact declared scope. The prototype does not infer that two scopes are compatible. Future scope adapters may implement version-aware compatibility rules, but those rules must be explicit and testable. + +### 2. Similarity grouping + +The reference implementation uses deterministic token-set Jaccard similarity. This is deliberately simple: + +- it is inspectable; +- it has no model dependency; +- it creates a measurable baseline; and +- it cannot silently change when an embedding provider changes. + +Future embeddings may add candidate edges, but structured scope filters and provenance rules remain mandatory. + +### 3. Representative selection + +The canonical statement is selected deterministically using status, authority class, source support, timestamp, and logical identity. It is a representative statement, not an assertion that the other members have been disproven or deleted. + +### 4. Contradiction preservation + +The prototype uses explicit `contradicts` and `supersedes` relationships. It does not guess contradiction from language. A later contradiction detector may propose links, but those links must remain challengeable derived records until governed evaluation accepts them. + +### 5. Retrieval keys + +High-frequency non-stopword tokens and record tags become deterministic retrieval keys. These keys can seed exact search, graph entry points, or a semantic search query, but they do not establish applicability. + +## Retrieval defragmentation + +Physical disk defragmentation moved blocks to reduce seek cost. RAVEL's analogous operation is broader because memory can be physically and logically fragmented. + +The prototype records query-level access events and counts how often selected memories are used together. Frequently co-selected records become a `RetrievalBucket` suggestion. A future adapter may use those buckets to: + +- co-locate rows or blobs on storage pages; +- create cache or prefetch groups; +- choose graph entry points; +- produce shard-affinity hints; +- materialize joint summaries; or +- optimize batch vector reads. + +The access planner never changes source content. A layout benchmark must compare candidate layouts under the same workload, cache state, storage medium, and resource budget. + +## Record roles + +### Source memory + +A source record is an authoritative historical object within its declared authority class. It contains identity, memory class, statement, scope, producer, status, provenance, relations, and metadata. + +### Consolidation proposal + +A proposal contains: + +- deterministic proposal identity; +- method version; +- memory class and scope; +- representative statement; +- member identities; +- supporting identities; +- contradicting identities; +- superseded identities; +- retrieval keys; +- clustering confidence; and +- explicit limitations. + +### Retrieval layout plan + +A layout bucket contains member identities and weighted co-access edges. It is replaceable and should be invalidated or rebuilt when workload behavior changes materially. + +## Validation plan + +The prototype should be evaluated against a preregistered workload with at least these comparisons: + +1. raw chronological retrieval; +2. structured filtering only; +3. structured filtering plus semantic consolidation; +4. structured filtering plus access-layout planning; and +5. the combined approach. + +Measure: + +- retrieval latency and reads; +- relevant-record recall; +- negative-memory recall; +- contradiction recall; +- incorrect cross-scope merges; +- stale-summary selection; +- index rebuild cost; +- storage overhead; +- downstream verifier choice; and +- downstream task outcome under equal budgets. + +A faster result is not sufficient if it hides negative evidence, increases false applicability, or weakens provenance. + +## Implementation map + +- `src/ravel/memory/models.py` defines source and derived records. +- `src/ravel/memory/store.py` provides an append-only SQLite prototype. +- `src/ravel/memory/consolidation.py` creates deterministic consolidation proposals and access-based layout plans. +- `schemas/consolidation-proposal.schema.json` specifies the portable proposal format. +- `tests/test_consolidation.py` verifies immutability, scope isolation, contradiction retention, determinism, and co-access planning. + +## Next steps + +1. Bind records to the versioned RAVEL evidence and experience schemas. +2. Add explicit scope-compatibility contracts instead of exact scope equality. +3. Add full-text retrieval and a benchmark corpus before embeddings. +4. Record query events in the experience store with privacy and retention controls. +5. Implement proposal review, acceptance, challenge, and supersession records. +6. Add graph projection rebuilds from the append-only source stream. +7. Benchmark physical page, cache, and shard layouts on the planned local and distributed RAVEL environments. +8. Add embedding-assisted candidate generation only after the deterministic baseline is measured. From 9060e2f7365091ac5dd89d6be656fffd7e499840 Mon Sep 17 00:00:00 2001 From: epi13 Date: Tue, 4 Aug 2026 08:36:33 -0800 Subject: [PATCH 09/10] Add consolidation proposal schema --- schemas/consolidation-proposal.schema.json | 77 ++++++++++++++++++++++ 1 file changed, 77 insertions(+) create mode 100644 schemas/consolidation-proposal.schema.json diff --git a/schemas/consolidation-proposal.schema.json b/schemas/consolidation-proposal.schema.json new file mode 100644 index 0000000..6df6665 --- /dev/null +++ b/schemas/consolidation-proposal.schema.json @@ -0,0 +1,77 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/epi13/RAVEL/schemas/consolidation-proposal.schema.json", + "title": "RAVEL consolidation proposal", + "type": "object", + "additionalProperties": false, + "required": [ + "proposal_id", + "method_version", + "created_at", + "memory_class", + "canonical_statement", + "scope", + "member_ids", + "supporting_ids", + "contradicting_ids", + "superseded_ids", + "retrieval_keys", + "clustering_confidence", + "status", + "limitations" + ], + "properties": { + "proposal_id": { + "type": "string", + "pattern": "^consolidation:[a-f0-9]{24}$" + }, + "method_version": {"type": "string", "minLength": 1}, + "created_at": {"type": "string", "format": "date-time"}, + "memory_class": { + "enum": ["episodic", "causal", "semantic", "procedural", "negative"] + }, + "canonical_statement": {"type": "string", "minLength": 1}, + "scope": { + "type": "object", + "minProperties": 1, + "additionalProperties": {"type": "string"} + }, + "member_ids": { + "type": "array", + "minItems": 2, + "uniqueItems": true, + "items": {"type": "string", "minLength": 1} + }, + "supporting_ids": { + "type": "array", + "uniqueItems": true, + "items": {"type": "string", "minLength": 1} + }, + "contradicting_ids": { + "type": "array", + "uniqueItems": true, + "items": {"type": "string", "minLength": 1} + }, + "superseded_ids": { + "type": "array", + "uniqueItems": true, + "items": {"type": "string", "minLength": 1} + }, + "retrieval_keys": { + "type": "array", + "uniqueItems": true, + "items": {"type": "string", "minLength": 1} + }, + "clustering_confidence": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "status": {"const": "proposed"}, + "limitations": { + "type": "array", + "minItems": 1, + "items": {"type": "string", "minLength": 1} + } + } +} From 9427fd5e0b0a8062d15bfbafa450f05d8bba5e3d Mon Sep 17 00:00:00 2001 From: epi13 Date: Tue, 4 Aug 2026 08:36:41 -0800 Subject: [PATCH 10/10] Add memory prototype CI --- .github/workflows/tests.yml | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 .github/workflows/tests.yml diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml new file mode 100644 index 0000000..603ef6c --- /dev/null +++ b/.github/workflows/tests.yml @@ -0,0 +1,24 @@ +name: tests + +on: + pull_request: + push: + branches: [main] + +permissions: + contents: read + +jobs: + unit: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ["3.11", "3.12", "3.13"] + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + - run: python -m pip install --upgrade pip + - run: python -m pip install -e . + - run: python -m unittest discover -s tests -v