diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9eef801..68c3593 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -27,14 +27,14 @@ jobs: grep -Eq '^Signed-off-by: .+ <[^>]+>$' done - publication-scan: + hygiene: runs-on: ubuntu-latest steps: - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 with: fetch-depth: 0 - - name: Scan pristine tracked, untracked, ignored, and reachable Git content - run: python scripts/verify_public_plugin_candidate.py --root . --layout destination + - name: Check the tree for real secrets and unexpected endpoints + run: python3 scripts/check_public_hygiene.py --root . test: strategy: @@ -56,65 +56,10 @@ jobs: cache-dependency-glob: uv.lock - run: uv sync --frozen --extra dev - run: uv run --frozen --extra dev ruff check . - - run: uv run --frozen --extra dev python -m compileall -q src scripts + - run: uv run --frozen --extra dev python -m compileall -q plugins scripts - run: uv run --frozen --extra dev python -m pytest -q - - - release-artifact: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 - with: - fetch-depth: 0 - - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 - with: - python-version: "3.11" - - uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0 - with: - version: "0.11.32" - enable-cache: true - cache-dependency-glob: uv.lock - - run: uv sync --frozen --extra dev - - name: Re-run the complete privacy-safe migration matrix - run: | - set -euo pipefail - uv run --frozen --extra dev python scripts/benchmark_migration.py \ - --manifest benchmarks/hermes-migration-manifest.json \ - --profile ci \ - --output "$RUNNER_TEMP/hermes-migration-baseline.json" - uv run --frozen --extra dev python scripts/verify_fresh_migration_run.py \ - --receipt "$RUNNER_TEMP/hermes-migration-baseline.json" - - name: Build twice and verify deterministic bytes - env: - HERMES_PLUGIN_SOURCE_COMMIT: ${{ github.sha }} - run: | - set -euo pipefail - uv run --frozen --extra dev python scripts/build_plugin.py - cp dist/substrate_wiki.zip "$RUNNER_TEMP/first-substrate_wiki.zip" - rm -rf dist release-assets - uv run --frozen --extra dev python scripts/build_plugin.py - cmp "$RUNNER_TEMP/first-substrate_wiki.zip" dist/substrate_wiki.zip - uv run --frozen --extra dev python scripts/build_plugin.py --check - - name: Install exact artifact and reject a wrong digest - run: | - set -euo pipefail - archive_sha="$(sha256sum dist/substrate_wiki.zip | cut -d ' ' -f1)" - uv run --frozen --extra dev python scripts/install_hermes_plugin.py \ - --archive dist/substrate_wiki.zip \ - --hermes-home "$RUNNER_TEMP/hermes" \ - --sha256 "$archive_sha" \ - --no-activate --no-onboard \ - --yes --json - test -f "$RUNNER_TEMP/hermes/plugins/substrate_wiki/plugin.yaml" - if uv run --frozen --extra dev python scripts/install_hermes_plugin.py \ - --archive dist/substrate_wiki.zip \ - --hermes-home "$RUNNER_TEMP/tamper-canary" \ - --sha256 "$(printf '0%.0s' {1..64})" \ - --no-activate --no-onboard \ - --yes --json; then - echo "installer accepted a wrong digest" >&2 - exit 1 - fi + - run: uv run --frozen --extra dev python scripts/build_release.py + - run: uv run --frozen --extra dev python scripts/build_release.py --check dependency-review: if: github.event_name == 'pull_request' diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index a9fa377..45a0656 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -20,8 +20,6 @@ jobs: outputs: release_version: ${{ steps.metadata.outputs.release_version }} artifact_digest: ${{ steps.release_artifact.outputs.artifact-digest }} - env: - PYTHONPATH: src steps: - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 with: @@ -44,7 +42,7 @@ jobs: - name: Scan the pristine candidate before installing dependencies run: | set -euo pipefail - python scripts/verify_public_plugin_candidate.py --root . --layout destination + python3 scripts/check_public_hygiene.py --root . - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 with: python-version: "3.11" @@ -59,31 +57,24 @@ jobs: id: metadata run: | set -euo pipefail - version="$(python -c 'from pathlib import Path; print(next(line.split(":", 1)[1].strip() for line in Path("src/substrate_wiki/plugin.yaml").read_text().splitlines() if line.startswith("version: ")))')" + version="$(python -c 'from pathlib import Path; print(next(line.split(":", 1)[1].strip() for line in Path("plugins/substrate/plugin.yaml").read_text().splitlines() if line.startswith("version: ")))')" test -n "$version" + test "$version" = "0.3.0" uv run --no-sync ruff check . - uv run --no-sync python -m compileall -q src scripts + uv run --no-sync python -m compileall -q plugins scripts uv run --no-sync python -m pytest -q - uv run --no-sync python scripts/verify_migration_baseline.py - uv run --no-sync python scripts/benchmark_migration.py \ - --profile ci \ - --output "$RUNNER_TEMP/hermes-migration-baseline.json" - uv run --no-sync python scripts/verify_fresh_migration_run.py \ - --receipt "$RUNNER_TEMP/hermes-migration-baseline.json" printf 'release_version=%s\n' "$version" >> "$GITHUB_OUTPUT" - name: Build and verify exact release bytes twice env: HERMES_PLUGIN_SOURCE_COMMIT: ${{ inputs.candidate_sha }} run: | set -euo pipefail - uv run --no-sync python scripts/build_plugin.py - cp dist/substrate_wiki.zip "$RUNNER_TEMP/first-substrate_wiki.zip" - rm -rf dist release-assets - uv run --no-sync python scripts/build_plugin.py - cmp "$RUNNER_TEMP/first-substrate_wiki.zip" dist/substrate_wiki.zip - uv run --no-sync python scripts/build_plugin.py --check - cp scripts/install_hermes_plugin.py dist/install_hermes_plugin.py - (cd dist && sha256sum install_hermes_plugin.py substrate_wiki.zip > SHA256SUMS) + uv run --no-sync python scripts/build_release.py + cp dist/substrate.zip "$RUNNER_TEMP/first-substrate.zip" + rm -rf dist + uv run --no-sync python scripts/build_release.py + cmp "$RUNNER_TEMP/first-substrate.zip" dist/substrate.zip + uv run --no-sync python scripts/build_release.py --check - name: Upload exact verified release bytes id: release_artifact uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 @@ -115,9 +106,9 @@ jobs: set -euo pipefail test -n "${{ needs.verify.outputs.artifact_digest }}" test "$(find dist -maxdepth 1 -type f -printf '%f\n' | sort | tr '\n' ' ')" = \ - "SHA256SUMS install_hermes_plugin.py substrate_wiki.zip " + "SHA256SUMS substrate.zip " (cd dist && sha256sum -c SHA256SUMS) - - uses: actions/attest-build-provenance@e3fe62ef559997059fe8380e7d2b4c909e2d65f4 # pinned + - uses: actions/attest-build-provenance@e3fe62ef559997059fe8380e7d2b64c4f677262 # pinned with: subject-path: "dist/*" - name: Create immutable tag and GitHub release @@ -141,10 +132,9 @@ jobs: exit 1 fi gh release create "$tag" \ - dist/substrate_wiki.zip \ - dist/install_hermes_plugin.py \ + dist/substrate.zip \ dist/SHA256SUMS \ --repo "$GITHUB_REPOSITORY" \ --verify-tag \ - --title "substrate_wiki $tag" \ - --notes "Patch release for explicit post-approval history consent, reliable device-poll completion, and the Substrate v5 history-only handshake. Targets Hermes 0.20.x. See COMPATIBILITY.md and SECURITY.md." + --title "substrate $tag" \ + --notes "Substrate retrieval plugin for Hermes 0.21.x: one-prompt install, self-contained RFC 8628 device onboarding, memory_search/memory_expand/memory_evidence. See README.md and COMPATIBILITY.md." diff --git a/BOUNDARY.md b/BOUNDARY.md index 6efe925..27fd2cf 100644 --- a/BOUNDARY.md +++ b/BOUNDARY.md @@ -13,7 +13,10 @@ This boundary is declared on day one and will not be moved later. Substrate asks ## Current implementation status -The pending `v1.5.0` candidate contains the Hermes `substrate_wiki` plugin/client, local spool and checkpoint machinery, client-side credential redaction, history replay, deterministic packaging, and installation tooling. It requires a configured Substrate server. No `v1.5.0` tag or release exists yet. +Release `v0.3.0` contains the Hermes `substrate` retrieval plugin: hook and tool +registration, verified transport, device onboarding with profile-private credential +custody, session-completion capture, client-side redaction, deterministic packaging, and +installation tooling. It requires a configured Substrate server. The local runtime, local entity model, privacy-deletion implementation, and policy compiler are on the open side of the permanent boundary but are **not implemented in this candidate**. The candidate does not authorize agent actions and does not provide a no-server mode. Those absences are explicit product gaps, not held commercial features. diff --git a/CHANGELOG.md b/CHANGELOG.md index 0b4dec9..8d86470 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,8 +1,18 @@ # Changelog -## Unreleased - -- Add the current `substrate` plugin (`plugins/substrate`, version 0.2.x) for Hermes 0.21.x: one-prompt `hermes plugins install` of the subdirectory, self-contained RFC 8628 device onboarding on first use (browser approval link, background polling, key stored privately in the active profile's `.env`), `memory_search`/`memory_expand`/`memory_evidence` tools, verified TLS with bundled public ISRG root anchors, and health-gated legacy cutover. The legacy `substrate_wiki` 2.0.x provider below remains only for existing Hermes 0.20.x installations. +## 0.3.0 + +- First full release of the `substrate` plugin for Hermes 0.21.x: one-prompt install + from the `v0.3.0` tag, self-contained RFC 8628 device onboarding with agent display + names, `memory_search`/`memory_expand`/`memory_evidence`, live session completion + markers, pinned public ISRG trust roots, and health-gated legacy cutover. +- Remove the legacy `substrate_wiki` provider, its installer/builder, migration + benchmarks, release assets, and publication-policy machinery. The repository now + contains only the current plugin, its tests, and a deterministic release builder. +- Replace the closed-inventory publication scanner with a dependency-free public + hygiene check (`scripts/check_public_hygiene.py`). + +- Add the current `substrate` plugin (`plugins/substrate`, now version 0.3.0) for Hermes 0.21.x: one-prompt `hermes plugins install` of the subdirectory, self-contained RFC 8628 device onboarding on first use (browser approval link, background polling, key stored privately in the active profile's `.env`), `memory_search`/`memory_expand`/`memory_evidence` tools, verified TLS with bundled public ISRG root anchors, and health-gated legacy cutover. The legacy `substrate_wiki` 2.0.x provider below remains only for existing Hermes 0.20.x installations. - Let hosted onboarding validate only the history replay contract, so a setup-and-upload MVP does not need to advertise entity retrieval features. - Add an explicit `SUBSTRATE_WIKI_ORIGIN` override for isolated v5 import tests; the hosted origin remains the default. @@ -64,4 +74,4 @@ All notable changes to the public `substrate_wiki` plugin are recorded here. - Imported byte-for-byte from Substrate-v2. Archive SHA-256: `877ccf9b0212792b699d9c98912a26980675a6050df3bd319e927639e3d901f1`. - Original source commit: `a3953b0512bbb84fb62b48a75bab04cbcb845c78`. -Earlier immutable artifacts remain under `legacy-assets/` with their original provenance. +Earlier immutable artifacts remain under `legacy-assets/` with their original provenance. \ No newline at end of file diff --git a/COMPATIBILITY.md b/COMPATIBILITY.md index 9f2083b..d170a65 100644 --- a/COMPATIBILITY.md +++ b/COMPATIBILITY.md @@ -2,45 +2,27 @@ Compatibility has three independent axes: Hermes host, hosted Substrate capability contract, and plugin version. -## Current install target - -The current Hermes plugin is [`plugins/substrate`](plugins/substrate) (`substrate`, version -0.2.x). It is the only path an installing agent should select from this repository, and it is -certified for Hermes **0.21.x**. It installs with -`hermes plugins install Substrate-memory/Substrate-memory-plugins/plugins/substrate --no-enable`, -runs RFC 8628 device onboarding by itself, and registers `memory_search`, `memory_expand`, -and `memory_evidence`. It never requires a pre-existing API key. - -The table below is the **legacy** `substrate_wiki` provider record. It does not apply to the -current `substrate` plugin, and it must not be used to reject a Hermes 0.21.x install: the -legacy rows exist only for existing 0.20.x installations and migration testing. - -## Legacy certified matrix (substrate_wiki) +## Certified matrix | Plugin | Hermes | Required hosted capabilities | Status | |---|---|---|---| -| 2.0.5 | 0.20.x | Same as 2.0.4; explicit post-approval history consent and v5 handshake | Contract and lifecycle integration tested | -| 2.0.4 | 0.20.x | Same as 2.0.3; compact user/assistant capture and batched history upload | Contract and lifecycle integration tested | -| 2.0.3 | 0.20.x | Same as 2.0.2; resilient approved-device polling across hosted stalls | Contract and lifecycle integration tested | -| 2.0.2 | 0.20.x | Same as 2.0.1; bounded retry for hosted tenant cold starts | Contract and lifecycle integration tested | -| 2.0.1 | 0.20.x | Same as 2.0.0; complete email authorization URL in agent/headless prompts | Contract and lifecycle integration tested | -| 2.0.0 | 0.20.x | `stream-v2`, `entity-wiki-v1`, `entity-quality-v2`, hosted device authorization | Contract and lifecycle integration tested | -| 1.4.1 | 0.18.2 | Imported immutable historical release | Historical compatibility record only | +| 0.3.x | 0.21.x | contract v1 capabilities, device authorization, memory turn-context/search/expand/evidence, session completion | Released | -Plugin 2.0.x uses the current 0.20 user-plugin discovery path and native `post_setup` hook. -Older Hermes lifecycle conventions are not supported by this release. +The `substrate` plugin installs with +`hermes plugins install Substrate-memory/Substrate-memory-plugins/plugins/substrate --ref v0.3.0 --no-enable`, +runs RFC 8628 device onboarding by itself, and registers `memory_search`, `memory_expand`, +and `memory_evidence`. It never requires a pre-existing API key. ## Failure behavior -The plugin fails closed when the hosted origin, capabilities, credential custody, or response -contracts are invalid. It does not fall back to a local/self-hosted server, legacy history -transmission, or another memory provider. Expired or revoked credentials start repairable -hosted onboarding while durable events remain queued. +The plugin fails closed when the configured origin, capabilities, credential custody, or response +contracts are invalid. It does not fall back to a local/self-hosted server or another memory +provider. Expired or revoked credentials restart repairable device onboarding while durable +capture events remain queued. ## Versioning - Plugin behavior follows semantic versioning. -- The hosted origin is fixed at `https://app.trysubstrate.co`. +- Releases are immutable Git tags (`v0.3.0`) with attested `substrate.zip` and `SHA256SUMS` assets. - Breaking server behavior requires a new protocol/schema identifier. -- Installation activates `memory.provider: substrate_wiki`; there is no silent auto-update. -- Rollback preserves profile-local spool, checkpoints, consent state, and credential custody. +- Rollback preserves profile-local state and credential custody. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 8bf45fe..32fdf19 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -2,7 +2,10 @@ ## Scope -This repository owns the Hermes `substrate_wiki` plugin, its local state machinery, build/installer tooling, tests, release artifacts, and user documentation. Server routes, persistence, deployment, and production data belong in `Substrate-memory/Substrate-v2`. +This repository owns the Hermes `substrate` retrieval plugin (`plugins/substrate`), its +device onboarding, session-completion capture, response validation, release builder, +tests, and user documentation. Server routes, persistence, deployment, and production +data belong in the Substrate server repository. ## Workflow @@ -12,19 +15,22 @@ This repository owns the Hermes `substrate_wiki` plugin, its local state machine 4. Run: ```bash - python -m pip install -e '.[dev]' - pytest -q - python -m compileall -q src scripts - python scripts/verify_public_plugin_candidate.py --root . --layout destination + uv sync --frozen --extra dev + uv run --frozen --extra dev ruff check . + uv run --frozen --extra dev python -m pytest -q + python3 scripts/check_public_hygiene.py --root . + uv run --frozen --extra dev python scripts/build_release.py --check ``` 5. Update `CHANGELOG.md` and compatibility/security docs when behavior changes. 6. Sign every commit with the [Developer Certificate of Origin 1.1](https://developercertificate.org/) by adding `Signed-off-by: Name ` (`git commit -s`). This repository uses DCO sign-off, not a contributor license agreement. -7. Open a PR. Every PR requires an adversarial review and passing CI. +7. Open a PR. Every PR requires passing CI. ## Compatibility -Do not claim a Hermes or server version based on import success alone. Provider discovery, lifecycle hooks, tool schemas, profile isolation, import service, configuration, replay, rollback, and failure behavior must all pass. +Do not claim a Hermes or server version based on import success alone. Hook registration, +tool schemas, profile isolation, onboarding, session capture, configuration, and failure +behavior must all pass. Breaking server behavior requires a new protocol/schema identifier with concurrent old-version support. Never silently rewrite user configuration or state. diff --git a/README.md b/README.md index 2f9cf42..bfcd450 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,8 @@ # Substrate Memory Plugins -This is the official public repository and canonical editable source for Substrate memory integrations. The current -Hermes plugin is [`plugins/substrate`](plugins/substrate) and supports Hermes 0.21.x. +This is the official public repository for the Substrate memory plugin for +Hermes 0.21.x. The current release is +[`v0.3.0`](https://github.com/Substrate-memory/Substrate-memory-plugins/releases/tag/v0.3.0). ## Install with a Hermes agent @@ -18,14 +19,12 @@ for the active Hermes profile. Do not ask the user for repository paths, command hashes, API keys, or TLS workarounds. 1. Confirm `hermes --version` is 0.21.x. Do not upgrade Hermes automatically. -2. Install the plugin subdirectory, not the repository root: +2. Install the plugin subdirectory at the released tag, not the repository root: ```sh - substrate_ref="$(git ls-remote https://github.com/Substrate-memory/Substrate-memory-plugins.git refs/heads/main | awk '{print $1}')" - printf '%s\n' "$substrate_ref" | grep -Eq '^[0-9a-f]{40}$' hermes plugins install \ Substrate-memory/Substrate-memory-plugins/plugins/substrate \ - --ref "$substrate_ref" --no-enable + --ref v0.3.0 --no-enable ``` 3. Run `hermes plugins doctor substrate --ci`. Stop without changing the legacy provider @@ -36,9 +35,10 @@ hashes, API keys, or TLS workarounds. the agent must never approve it for them or ask for a pasted key. 5. The plugin stores the tenant-scoped key privately in the active profile's `.env` and retries automatically. No manual credential migration is required. -6. Only after a memory call succeeds, disable `substrate_wiki`, clear an old - `memory.provider: substrate_wiki` selection with `hermes memory off`, and restart the - gateway. This prevents duplicate capture while preserving built-in local memory. +6. Only after a memory call succeeds, disable `substrate_wiki` (if an old provider is + installed), clear an old `memory.provider: substrate_wiki` selection with + `hermes memory off`, and restart the gateway. This prevents duplicate capture while + preserving built-in local memory. 7. The optional `setup.py` can pre-connect during installation, but it is not required. 8. Verify the new `memory_search` tool in a new turn. `wiki_search` is legacy and is not a valid verification of the current plugin. @@ -55,34 +55,24 @@ The `substrate` plugin is a thin, standard-library-only adapter. It provides: - bounded next-turn memory context through `pre_llm_call`; - nonblocking completed-turn capture through `post_llm_call`; +- session completion markers through `on_session_reset` and `on_session_finalize`, so + ended sessions are materialized into the extraction pipeline automatically; - `memory_search`, `memory_expand`, and `memory_evidence`; -- active-profile credential migration and passwordless device authorization; +- active-profile credential reuse and passwordless device authorization; +- agent display names, selectable on the approval page and visible in the agent pane; - additive public-root TLS trust without disabling certificate or hostname checks. The Substrate server remains the source of truth for storage, ranking, the associative editor, and evidence. -## Legacy compatibility - -Hermes 0.21.x is certified for the current `substrate` plugin. The legacy -`src/substrate_wiki` provider and its release tooling are retained for existing Hermes 0.20.x -installations and migration tests. It is not the current install target, and its -compatibility rows in `COMPATIBILITY.md` apply only to 0.20.x. Do not run -`scripts/install_hermes_plugin.py` for the prompt above. Legacy packaging state remains -unchanged: `v2.0.5` is not published yet, its historical URL would contain -`releases/download/v2.0.5`, and its unpublished checksum marker remains -`PLUGIN_SHA256_PENDING`. These strings document legacy release truth; they are not install -instructions. Immutable historical releases remain under `legacy-assets` and Git tags. - ## Privacy boundary Visible prompts and assistant output may be sent to the configured Substrate server after redaction. The current plugin does not upload tool results or system messages. It caches no retrieved memory locally and returns empty context on any retrieval failure. -Read [SECURITY.md](SECURITY.md), [the threat model](docs/threat-model.md), -[the source-ownership boundary](docs/source-of-truth.md), and [BOUNDARY.md](BOUNDARY.md) -before deployment. +Read [SECURITY.md](SECURITY.md) and [the threat model](docs/threat-model.md) before +deployment. ## Development @@ -90,17 +80,16 @@ before deployment. uv sync --frozen --extra dev uv run --frozen --extra dev ruff check . uv run --frozen --extra dev python -m pytest -q -python3 scripts/verify_public_plugin_candidate.py --root . --layout destination +python3 scripts/check_public_hygiene.py --root . +uv run --frozen --extra dev python scripts/build_release.py --check ``` ## Repository map -- `plugins/substrate/` — current Hermes 0.21.x retrieval plugin and setup flow. -- `src/substrate_wiki/` — legacy provider retained for migration compatibility. -- `scripts/` — deterministic legacy builder, installer, and publication scanner. -- `tests/` — provider, privacy, replay, packaging, retrieval, and setup tests. -- `legacy-assets/` — immutable historical releases. -- `docs/` — architecture, compatibility, ownership, and threat-model contracts. +- `plugins/substrate/` — the Hermes 0.21.x retrieval plugin and setup flow. +- `scripts/` — deterministic release builder and public-hygiene check. +- `tests/` — contract, behavior, transport, onboarding, and packaging tests. +- `docs/` — architecture, ownership, and threat-model contracts. ## License diff --git a/SECURITY.md b/SECURITY.md index 749f849..acc7fbe 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -2,35 +2,27 @@ ## Supported versions -The current supported install target is the `substrate` plugin (0.2.x) in -[`plugins/substrate`](plugins/substrate), certified for Hermes 0.21.x. It is the plugin an -agent should install from this repository; it self-onboards and never requires a pasted API -key. - -The standalone `v2.0.0` through `v2.0.5` releases listed below are the **legacy** -`substrate_wiki` provider for Hermes 0.20.x and the hosted service at -`https://app.trysubstrate.co`. They remain documented for existing installations only. -Candidate CI artifacts are not supported releases. +The current supported install target is the `substrate` plugin (0.3.x) in +[`plugins/substrate`](plugins/substrate), certified for Hermes 0.21.x and installed from +the `v0.3.0` release tag. It self-onboards and never requires a pasted API key. ## Report a vulnerability -Use GitHub's private **Security advisories → Report a vulnerability** flow. Do not open a -public issue containing a credential, private history, spool contents, traceback with user -content, or exploit details. Include only content-free diagnostics: plugin/Hermes versions, -operating system, lifecycle operation, failure category, and a synthetic reproduction. +Use GitHub's private **Security advisories → Report a vulnerability** flow for this +repository. Do not open a public issue for a suspected secret leak, auth bypass, or +remote-execution vector. -## Security boundary +## Credential handling -The plugin: +- Tenant-scoped keys live only in the active profile's `.env` (`SUBSTRATE_API_KEY`) and + the issuing server. They never enter source, artifacts, logs, errors, chat, or process + arguments. +- Device codes live only in an owner-private profile file and are deleted on terminal + states. Onboarding state files never contain access tokens. +- The plugin verifies TLS with the system trust store plus pinned public ISRG Root X1 + and X2 anchors. Verification is never disabled and leaf certificates are never pinned. -- accepts hosted tenant credentials only through onboarding credential custody; -- fixes the network origin to `https://app.trysubstrate.co` and rejects unsafe overrides; -- stores credentials in a native vault when available, with a fail-closed owner-private fallback; -- never puts credentials in normal config, logs, errors, receipts, or command arguments; -- redacts before queueing, persistence, or network transfer; -- bounds requests, responses, spool size, retries, and import resources; -- keeps profile state beneath the active `$HERMES_HOME`; -- verifies immutable checksums, provenance, and packaged source hashes during installation. +## Response validation -Pattern redaction cannot identify every sensitive statement. Historical upload requires -explicit consent; declining it leaves future capture enabled. See `docs/threat-model.md`. +Every server response is schema-validated before use. Unknown or oversized payloads fail +closed. Tool results are bounded and redacted before they reach the model. diff --git a/benchmarks/evidence/hermes-migration-baseline.json b/benchmarks/evidence/hermes-migration-baseline.json deleted file mode 100644 index efe296d..0000000 --- a/benchmarks/evidence/hermes-migration-baseline.json +++ /dev/null @@ -1,3040 +0,0 @@ -{ - "harness_sha256": "ef57926174b22c2c22e73843a08a5ea9b7b45481bf51ef08dac21129ea9c929a", - "hosted_calls": 0, - "manifest_sha256": "d0508dd3586723f8d8ff81ee6ae8a2f4aca381905c45b3ceac42b2770cabc266", - "profile": "ci", - "protocol": "stream-v2", - "representative_provider": { - "cost_usd_per_request": 0.00025, - "mode": "deterministic_simulation", - "network_access": false, - "quota_requests_per_minute": 60 - }, - "runs": [ - { - "case_id": "small-sqlite", - "concurrency": 1, - "cpu_seconds": 0.027186, - "events": 25, - "events_per_second": 62.823, - "fixture_kind": "sqlite", - "fixture_sha256": "d3a7ad1f5e08f5d2d3bd31ccccb957b66f36533c6ce9c1d6fbd7d869848ce235", - "input_bytes": 16384, - "integrity": { - "complete": true, - "duplicate_acks": 1, - "duplicate_side_effects": 0, - "expected_windows": 8, - "idempotency_replay_kind": "session_end", - "idempotency_replays": 1, - "idempotency_state_components": [ - "duplicate_acks", - "encoded_bytes", - "max_queue_depth", - "projected_digest", - "projected_rows", - "queue_items", - "received_digest", - "received_rows", - "redacted_events", - "redaction_failures", - "requests", - "work_queue_items" - ], - "projected_windows": 8, - "redacted_events": 16, - "redaction_failures": 0 - }, - "lifecycle_seconds": { - "first_usable": 0.120124, - "fully_ready": 0.399896, - "transferred": 0.397943 - }, - "mib_per_second": 0.039, - "peak_rss_bytes": 33718272, - "phase_seconds": { - "discovery": 0.004802, - "entity_resolution": 3.1e-05, - "network_wait": 0.131823, - "normalization_redaction": 0.004659, - "projection_indexing": 0.060824, - "provider_extraction": 0.040517, - "queue_delay": 0.000588, - "request_encoding": 0.000249, - "server_persistence": 0.102529, - "source_reading": 0.002089, - "summary_reduction": 6e-06 - }, - "provider": { - "max_in_flight": 1, - "modeled_cost_usd": 0.002, - "modeled_quota_seconds": 8.0, - "observed_time_seconds": 0.040517, - "quota_requests_per_minute": 60, - "requests": 8, - "retries": 0, - "worker_threads_used": 1 - }, - "quality": { - "projection_failures": 0, - "terminal_failures": 0 - }, - "queue": { - "max_age_seconds": 0.000164, - "max_depth": 1 - }, - "requests": 25, - "retries": 0, - "source_file_bytes": 45056, - "terminal_failures": 0, - "traces": [ - { - "phase": "discovery", - "seconds": 0.004802 - }, - { - "phase": "source_reading", - "seconds": 0.002089 - }, - { - "phase": "normalization_redaction", - "seconds": 0.004659 - }, - { - "phase": "request_encoding", - "seconds": 0.000249 - }, - { - "phase": "network_wait", - "seconds": 0.131823 - }, - { - "phase": "server_persistence", - "seconds": 0.102529 - }, - { - "phase": "queue_delay", - "seconds": 0.000588 - }, - { - "phase": "provider_extraction", - "seconds": 0.040517 - }, - { - "phase": "entity_resolution", - "seconds": 3.1e-05 - }, - { - "phase": "projection_indexing", - "seconds": 0.060824 - }, - { - "phase": "summary_reduction", - "seconds": 6e-06 - } - ], - "wall_time_seconds": 0.399896, - "windows": 8, - "windows_per_minute": 1200.312 - }, - { - "case_id": "small-sqlite", - "concurrency": 2, - "cpu_seconds": 0.026553, - "events": 25, - "events_per_second": 73.242, - "fixture_kind": "sqlite", - "fixture_sha256": "d3a7ad1f5e08f5d2d3bd31ccccb957b66f36533c6ce9c1d6fbd7d869848ce235", - "input_bytes": 16384, - "integrity": { - "complete": true, - "duplicate_acks": 1, - "duplicate_side_effects": 0, - "expected_windows": 8, - "idempotency_replay_kind": "session_end", - "idempotency_replays": 1, - "idempotency_state_components": [ - "duplicate_acks", - "encoded_bytes", - "max_queue_depth", - "projected_digest", - "projected_rows", - "queue_items", - "received_digest", - "received_rows", - "redacted_events", - "redaction_failures", - "requests", - "work_queue_items" - ], - "projected_windows": 8, - "redacted_events": 16, - "redaction_failures": 0 - }, - "lifecycle_seconds": { - "first_usable": 0.121783, - "fully_ready": 0.344191, - "transferred": 0.341334 - }, - "mib_per_second": 0.046, - "peak_rss_bytes": 33619968, - "phase_seconds": { - "discovery": 0.004819, - "entity_resolution": 3.2e-05, - "network_wait": 0.131736, - "normalization_redaction": 0.005367, - "projection_indexing": 0.03197, - "provider_extraction": 0.040528, - "queue_delay": 0.04194, - "request_encoding": 0.000251, - "server_persistence": 0.07435, - "source_reading": 0.002122, - "summary_reduction": 5e-06 - }, - "provider": { - "max_in_flight": 2, - "modeled_cost_usd": 0.002, - "modeled_quota_seconds": 8.0, - "observed_time_seconds": 0.040528, - "quota_requests_per_minute": 60, - "requests": 8, - "retries": 0, - "worker_threads_used": 2 - }, - "quality": { - "projection_failures": 0, - "terminal_failures": 0 - }, - "queue": { - "max_age_seconds": 0.04144, - "max_depth": 1 - }, - "requests": 25, - "retries": 0, - "source_file_bytes": 45056, - "terminal_failures": 0, - "traces": [ - { - "phase": "discovery", - "seconds": 0.004819 - }, - { - "phase": "source_reading", - "seconds": 0.002122 - }, - { - "phase": "normalization_redaction", - "seconds": 0.005367 - }, - { - "phase": "request_encoding", - "seconds": 0.000251 - }, - { - "phase": "network_wait", - "seconds": 0.131736 - }, - { - "phase": "server_persistence", - "seconds": 0.07435 - }, - { - "phase": "queue_delay", - "seconds": 0.04194 - }, - { - "phase": "provider_extraction", - "seconds": 0.040528 - }, - { - "phase": "entity_resolution", - "seconds": 3.2e-05 - }, - { - "phase": "projection_indexing", - "seconds": 0.03197 - }, - { - "phase": "summary_reduction", - "seconds": 5e-06 - } - ], - "wall_time_seconds": 0.344191, - "windows": 8, - "windows_per_minute": 1394.573 - }, - { - "case_id": "small-sqlite", - "concurrency": 3, - "cpu_seconds": 0.025833, - "events": 25, - "events_per_second": 74.101, - "fixture_kind": "sqlite", - "fixture_sha256": "d3a7ad1f5e08f5d2d3bd31ccccb957b66f36533c6ce9c1d6fbd7d869848ce235", - "input_bytes": 16384, - "integrity": { - "complete": true, - "duplicate_acks": 1, - "duplicate_side_effects": 0, - "expected_windows": 8, - "idempotency_replay_kind": "session_end", - "idempotency_replays": 1, - "idempotency_state_components": [ - "duplicate_acks", - "encoded_bytes", - "max_queue_depth", - "projected_digest", - "projected_rows", - "queue_items", - "received_digest", - "received_rows", - "redacted_events", - "redaction_failures", - "requests", - "work_queue_items" - ], - "projected_windows": 8, - "redacted_events": 16, - "redaction_failures": 0 - }, - "lifecycle_seconds": { - "first_usable": 0.162266, - "fully_ready": 0.341269, - "transferred": 0.337376 - }, - "mib_per_second": 0.046, - "peak_rss_bytes": 33607680, - "phase_seconds": { - "discovery": 0.004911, - "entity_resolution": 3.1e-05, - "network_wait": 0.131708, - "normalization_redaction": 0.005311, - "projection_indexing": 0.059119, - "provider_extraction": 0.040538, - "queue_delay": 0.114514, - "request_encoding": 0.000272, - "server_persistence": 0.074881, - "source_reading": 0.002207, - "summary_reduction": 4e-06 - }, - "provider": { - "max_in_flight": 3, - "modeled_cost_usd": 0.002, - "modeled_quota_seconds": 8.0, - "observed_time_seconds": 0.040538, - "quota_requests_per_minute": 60, - "requests": 8, - "retries": 0, - "worker_threads_used": 3 - }, - "quality": { - "projection_failures": 0, - "terminal_failures": 0 - }, - "queue": { - "max_age_seconds": 0.076707, - "max_depth": 1 - }, - "requests": 25, - "retries": 0, - "source_file_bytes": 45056, - "terminal_failures": 0, - "traces": [ - { - "phase": "discovery", - "seconds": 0.004911 - }, - { - "phase": "source_reading", - "seconds": 0.002207 - }, - { - "phase": "normalization_redaction", - "seconds": 0.005311 - }, - { - "phase": "request_encoding", - "seconds": 0.000272 - }, - { - "phase": "network_wait", - "seconds": 0.131708 - }, - { - "phase": "server_persistence", - "seconds": 0.074881 - }, - { - "phase": "queue_delay", - "seconds": 0.114514 - }, - { - "phase": "provider_extraction", - "seconds": 0.040538 - }, - { - "phase": "entity_resolution", - "seconds": 3.1e-05 - }, - { - "phase": "projection_indexing", - "seconds": 0.059119 - }, - { - "phase": "summary_reduction", - "seconds": 4e-06 - } - ], - "wall_time_seconds": 0.341269, - "windows": 8, - "windows_per_minute": 1406.517 - }, - { - "case_id": "small-sqlite", - "concurrency": 4, - "cpu_seconds": 0.025842, - "events": 25, - "events_per_second": 73.437, - "fixture_kind": "sqlite", - "fixture_sha256": "d3a7ad1f5e08f5d2d3bd31ccccb957b66f36533c6ce9c1d6fbd7d869848ce235", - "input_bytes": 16384, - "integrity": { - "complete": true, - "duplicate_acks": 1, - "duplicate_side_effects": 0, - "expected_windows": 8, - "idempotency_replay_kind": "session_end", - "idempotency_replays": 1, - "idempotency_state_components": [ - "duplicate_acks", - "encoded_bytes", - "max_queue_depth", - "projected_digest", - "projected_rows", - "queue_items", - "received_digest", - "received_rows", - "redacted_events", - "redaction_failures", - "requests", - "work_queue_items" - ], - "projected_windows": 8, - "redacted_events": 16, - "redaction_failures": 0 - }, - "lifecycle_seconds": { - "first_usable": 0.197046, - "fully_ready": 0.342689, - "transferred": 0.340426 - }, - "mib_per_second": 0.046, - "peak_rss_bytes": 33533952, - "phase_seconds": { - "discovery": 0.004533, - "entity_resolution": 2.7e-05, - "network_wait": 0.131629, - "normalization_redaction": 0.004622, - "projection_indexing": 0.05986, - "provider_extraction": 0.04051, - "queue_delay": 0.231263, - "request_encoding": 0.00024, - "server_persistence": 0.070147, - "source_reading": 0.002054, - "summary_reduction": 4e-06 - }, - "provider": { - "max_in_flight": 4, - "modeled_cost_usd": 0.002, - "modeled_quota_seconds": 8.0, - "observed_time_seconds": 0.04051, - "quota_requests_per_minute": 60, - "requests": 8, - "retries": 0, - "worker_threads_used": 4 - }, - "quality": { - "projection_failures": 0, - "terminal_failures": 0 - }, - "queue": { - "max_age_seconds": 0.114247, - "max_depth": 1 - }, - "requests": 25, - "retries": 0, - "source_file_bytes": 45056, - "terminal_failures": 0, - "traces": [ - { - "phase": "discovery", - "seconds": 0.004533 - }, - { - "phase": "source_reading", - "seconds": 0.002054 - }, - { - "phase": "normalization_redaction", - "seconds": 0.004622 - }, - { - "phase": "request_encoding", - "seconds": 0.00024 - }, - { - "phase": "network_wait", - "seconds": 0.131629 - }, - { - "phase": "server_persistence", - "seconds": 0.070147 - }, - { - "phase": "queue_delay", - "seconds": 0.231263 - }, - { - "phase": "provider_extraction", - "seconds": 0.04051 - }, - { - "phase": "entity_resolution", - "seconds": 2.7e-05 - }, - { - "phase": "projection_indexing", - "seconds": 0.05986 - }, - { - "phase": "summary_reduction", - "seconds": 4e-06 - } - ], - "wall_time_seconds": 0.342689, - "windows": 8, - "windows_per_minute": 1400.686 - }, - { - "case_id": "current-production-jsonl", - "concurrency": 1, - "cpu_seconds": 2.473446, - "events": 2425, - "events_per_second": 67.136, - "fixture_kind": "official_export_jsonl", - "fixture_sha256": "6eec2b2216ef77459d8517eeceb82381d8ec2844762956339e19545799eec45c", - "input_bytes": 310272, - "integrity": { - "complete": true, - "duplicate_acks": 1, - "duplicate_side_effects": 0, - "expected_windows": 1212, - "idempotency_replay_kind": "session_end", - "idempotency_replays": 1, - "idempotency_state_components": [ - "duplicate_acks", - "encoded_bytes", - "max_queue_depth", - "projected_digest", - "projected_rows", - "queue_items", - "received_digest", - "received_rows", - "redacted_events", - "redaction_failures", - "requests", - "work_queue_items" - ], - "projected_windows": 1212, - "redacted_events": 1212, - "redaction_failures": 0 - }, - "lifecycle_seconds": { - "first_usable": 0.218208, - "fully_ready": 36.121958, - "transferred": 36.120878 - }, - "mib_per_second": 0.008, - "peak_rss_bytes": 38121472, - "phase_seconds": { - "discovery": 0.146899, - "entity_resolution": 0.003745, - "network_wait": 12.293947, - "normalization_redaction": 0.259365, - "projection_indexing": 6.36737, - "provider_extraction": 6.221289, - "queue_delay": 0.05404, - "request_encoding": 0.021554, - "server_persistence": 8.744172, - "source_reading": 0.597774, - "summary_reduction": 6e-06 - }, - "provider": { - "max_in_flight": 1, - "modeled_cost_usd": 0.303, - "modeled_quota_seconds": 1212.0, - "observed_time_seconds": 6.221289, - "quota_requests_per_minute": 60, - "requests": 1212, - "retries": 5, - "worker_threads_used": 1 - }, - "quality": { - "projection_failures": 0, - "terminal_failures": 0 - }, - "queue": { - "max_age_seconds": 0.000294, - "max_depth": 1 - }, - "requests": 2425, - "retries": 5, - "source_file_bytes": 461772, - "terminal_failures": 0, - "traces": [ - { - "phase": "discovery", - "seconds": 0.146899 - }, - { - "phase": "source_reading", - "seconds": 0.597774 - }, - { - "phase": "normalization_redaction", - "seconds": 0.259365 - }, - { - "phase": "request_encoding", - "seconds": 0.021554 - }, - { - "phase": "network_wait", - "seconds": 12.293947 - }, - { - "phase": "server_persistence", - "seconds": 8.744172 - }, - { - "phase": "queue_delay", - "seconds": 0.05404 - }, - { - "phase": "provider_extraction", - "seconds": 6.221289 - }, - { - "phase": "entity_resolution", - "seconds": 0.003745 - }, - { - "phase": "projection_indexing", - "seconds": 6.36737 - }, - { - "phase": "summary_reduction", - "seconds": 6e-06 - } - ], - "wall_time_seconds": 36.121958, - "windows": 1212, - "windows_per_minute": 2013.18 - }, - { - "case_id": "current-production-jsonl", - "concurrency": 2, - "cpu_seconds": 2.539915, - "events": 2425, - "events_per_second": 67.653, - "fixture_kind": "official_export_jsonl", - "fixture_sha256": "6eec2b2216ef77459d8517eeceb82381d8ec2844762956339e19545799eec45c", - "input_bytes": 310272, - "integrity": { - "complete": true, - "duplicate_acks": 1, - "duplicate_side_effects": 0, - "expected_windows": 1212, - "idempotency_replay_kind": "session_end", - "idempotency_replays": 1, - "idempotency_state_components": [ - "duplicate_acks", - "encoded_bytes", - "max_queue_depth", - "projected_digest", - "projected_rows", - "queue_items", - "received_digest", - "received_rows", - "redacted_events", - "redaction_failures", - "requests", - "work_queue_items" - ], - "projected_windows": 1212, - "redacted_events": 1212, - "redaction_failures": 0 - }, - "lifecycle_seconds": { - "first_usable": 0.228792, - "fully_ready": 35.84662, - "transferred": 35.844882 - }, - "mib_per_second": 0.008, - "peak_rss_bytes": 37748736, - "phase_seconds": { - "discovery": 0.142437, - "entity_resolution": 0.003896, - "network_wait": 12.294398, - "normalization_redaction": 0.267723, - "projection_indexing": 6.042208, - "provider_extraction": 6.236205, - "queue_delay": 0.080014, - "request_encoding": 0.021736, - "server_persistence": 8.91106, - "source_reading": 0.591201, - "summary_reduction": 6e-06 - }, - "provider": { - "max_in_flight": 2, - "modeled_cost_usd": 0.303, - "modeled_quota_seconds": 1212.0, - "observed_time_seconds": 6.236205, - "quota_requests_per_minute": 60, - "requests": 1212, - "retries": 5, - "worker_threads_used": 2 - }, - "quality": { - "projection_failures": 0, - "terminal_failures": 0 - }, - "queue": { - "max_age_seconds": 0.021806, - "max_depth": 1 - }, - "requests": 2425, - "retries": 5, - "source_file_bytes": 461772, - "terminal_failures": 0, - "traces": [ - { - "phase": "discovery", - "seconds": 0.142437 - }, - { - "phase": "source_reading", - "seconds": 0.591201 - }, - { - "phase": "normalization_redaction", - "seconds": 0.267723 - }, - { - "phase": "request_encoding", - "seconds": 0.021736 - }, - { - "phase": "network_wait", - "seconds": 12.294398 - }, - { - "phase": "server_persistence", - "seconds": 8.91106 - }, - { - "phase": "queue_delay", - "seconds": 0.080014 - }, - { - "phase": "provider_extraction", - "seconds": 6.236205 - }, - { - "phase": "entity_resolution", - "seconds": 0.003896 - }, - { - "phase": "projection_indexing", - "seconds": 6.042208 - }, - { - "phase": "summary_reduction", - "seconds": 6e-06 - } - ], - "wall_time_seconds": 35.84662, - "windows": 1212, - "windows_per_minute": 2028.643 - }, - { - "case_id": "current-production-jsonl", - "concurrency": 3, - "cpu_seconds": 2.516128, - "events": 2425, - "events_per_second": 67.729, - "fixture_kind": "official_export_jsonl", - "fixture_sha256": "6eec2b2216ef77459d8517eeceb82381d8ec2844762956339e19545799eec45c", - "input_bytes": 310272, - "integrity": { - "complete": true, - "duplicate_acks": 1, - "duplicate_side_effects": 0, - "expected_windows": 1212, - "idempotency_replay_kind": "session_end", - "idempotency_replays": 1, - "idempotency_state_components": [ - "duplicate_acks", - "encoded_bytes", - "max_queue_depth", - "projected_digest", - "projected_rows", - "queue_items", - "received_digest", - "received_rows", - "redacted_events", - "redaction_failures", - "requests", - "work_queue_items" - ], - "projected_windows": 1212, - "redacted_events": 1212, - "redaction_failures": 0 - }, - "lifecycle_seconds": { - "first_usable": 0.252446, - "fully_ready": 35.818532, - "transferred": 35.80448 - }, - "mib_per_second": 0.008, - "peak_rss_bytes": 38375424, - "phase_seconds": { - "discovery": 0.138282, - "entity_resolution": 0.003772, - "network_wait": 12.294948, - "normalization_redaction": 0.26665, - "projection_indexing": 6.437637, - "provider_extraction": 6.224627, - "queue_delay": 0.124457, - "request_encoding": 0.021809, - "server_persistence": 8.399041, - "source_reading": 0.558591, - "summary_reduction": 6e-06 - }, - "provider": { - "max_in_flight": 3, - "modeled_cost_usd": 0.303, - "modeled_quota_seconds": 1212.0, - "observed_time_seconds": 6.224627, - "quota_requests_per_minute": 60, - "requests": 1212, - "retries": 5, - "worker_threads_used": 3 - }, - "quality": { - "projection_failures": 0, - "terminal_failures": 0 - }, - "queue": { - "max_age_seconds": 0.044288, - "max_depth": 1 - }, - "requests": 2425, - "retries": 5, - "source_file_bytes": 461772, - "terminal_failures": 0, - "traces": [ - { - "phase": "discovery", - "seconds": 0.138282 - }, - { - "phase": "source_reading", - "seconds": 0.558591 - }, - { - "phase": "normalization_redaction", - "seconds": 0.26665 - }, - { - "phase": "request_encoding", - "seconds": 0.021809 - }, - { - "phase": "network_wait", - "seconds": 12.294948 - }, - { - "phase": "server_persistence", - "seconds": 8.399041 - }, - { - "phase": "queue_delay", - "seconds": 0.124457 - }, - { - "phase": "provider_extraction", - "seconds": 6.224627 - }, - { - "phase": "entity_resolution", - "seconds": 0.003772 - }, - { - "phase": "projection_indexing", - "seconds": 6.437637 - }, - { - "phase": "summary_reduction", - "seconds": 6e-06 - } - ], - "wall_time_seconds": 35.818532, - "windows": 1212, - "windows_per_minute": 2030.234 - }, - { - "case_id": "current-production-jsonl", - "concurrency": 4, - "cpu_seconds": 2.541121, - "events": 2425, - "events_per_second": 67.196, - "fixture_kind": "official_export_jsonl", - "fixture_sha256": "6eec2b2216ef77459d8517eeceb82381d8ec2844762956339e19545799eec45c", - "input_bytes": 310272, - "integrity": { - "complete": true, - "duplicate_acks": 1, - "duplicate_side_effects": 0, - "expected_windows": 1212, - "idempotency_replay_kind": "session_end", - "idempotency_replays": 1, - "idempotency_state_components": [ - "duplicate_acks", - "encoded_bytes", - "max_queue_depth", - "projected_digest", - "projected_rows", - "queue_items", - "received_digest", - "received_rows", - "redacted_events", - "redaction_failures", - "requests", - "work_queue_items" - ], - "projected_windows": 1212, - "redacted_events": 1212, - "redaction_failures": 0 - }, - "lifecycle_seconds": { - "first_usable": 0.316931, - "fully_ready": 36.11923, - "transferred": 36.088678 - }, - "mib_per_second": 0.008, - "peak_rss_bytes": 38158336, - "phase_seconds": { - "discovery": 0.143977, - "entity_resolution": 0.003787, - "network_wait": 12.294237, - "normalization_redaction": 0.27046, - "projection_indexing": 5.978563, - "provider_extraction": 6.23754, - "queue_delay": 0.222514, - "request_encoding": 0.022182, - "server_persistence": 8.778625, - "source_reading": 0.561236, - "summary_reduction": 7e-06 - }, - "provider": { - "max_in_flight": 4, - "modeled_cost_usd": 0.303, - "modeled_quota_seconds": 1212.0, - "observed_time_seconds": 6.23754, - "quota_requests_per_minute": 60, - "requests": 1212, - "retries": 5, - "worker_threads_used": 4 - }, - "quality": { - "projection_failures": 0, - "terminal_failures": 0 - }, - "queue": { - "max_age_seconds": 0.088518, - "max_depth": 1 - }, - "requests": 2425, - "retries": 5, - "source_file_bytes": 461772, - "terminal_failures": 0, - "traces": [ - { - "phase": "discovery", - "seconds": 0.143977 - }, - { - "phase": "source_reading", - "seconds": 0.561236 - }, - { - "phase": "normalization_redaction", - "seconds": 0.27046 - }, - { - "phase": "request_encoding", - "seconds": 0.022182 - }, - { - "phase": "network_wait", - "seconds": 12.294237 - }, - { - "phase": "server_persistence", - "seconds": 8.778625 - }, - { - "phase": "queue_delay", - "seconds": 0.222514 - }, - { - "phase": "provider_extraction", - "seconds": 6.23754 - }, - { - "phase": "entity_resolution", - "seconds": 0.003787 - }, - { - "phase": "projection_indexing", - "seconds": 5.978563 - }, - { - "phase": "summary_reduction", - "seconds": 7e-06 - } - ], - "wall_time_seconds": 36.11923, - "windows": 1212, - "windows_per_minute": 2013.332 - }, - { - "case_id": "large-sqlite", - "concurrency": 1, - "cpu_seconds": 4.411825, - "events": 4097, - "events_per_second": 62.356, - "fixture_kind": "sqlite", - "fixture_sha256": "aba188af3961fa12a7d4ddfd591e0ca099cef40cd0e65bf61dd337602bb1c858", - "input_bytes": 524288, - "integrity": { - "complete": true, - "duplicate_acks": 1, - "duplicate_side_effects": 0, - "expected_windows": 2048, - "idempotency_replay_kind": "session_end", - "idempotency_replays": 1, - "idempotency_state_components": [ - "duplicate_acks", - "encoded_bytes", - "max_queue_depth", - "projected_digest", - "projected_rows", - "queue_items", - "received_digest", - "received_rows", - "redacted_events", - "redaction_failures", - "requests", - "work_queue_items" - ], - "projected_windows": 2048, - "redacted_events": 2048, - "redaction_failures": 0 - }, - "lifecycle_seconds": { - "first_usable": 0.185375, - "fully_ready": 65.705342, - "transferred": 65.703417 - }, - "mib_per_second": 0.008, - "peak_rss_bytes": 41222144, - "phase_seconds": { - "discovery": 0.120288, - "entity_resolution": 0.006413, - "network_wait": 20.747029, - "normalization_redaction": 0.441244, - "projection_indexing": 11.824323, - "provider_extraction": 10.447082, - "queue_delay": 0.090012, - "request_encoding": 0.03615, - "server_persistence": 17.25631, - "source_reading": 1.252442, - "summary_reduction": 5e-06 - }, - "provider": { - "max_in_flight": 1, - "modeled_cost_usd": 0.512, - "modeled_quota_seconds": 2048.0, - "observed_time_seconds": 10.447082, - "quota_requests_per_minute": 60, - "requests": 2048, - "retries": 8, - "worker_threads_used": 1 - }, - "quality": { - "projection_failures": 0, - "terminal_failures": 0 - }, - "queue": { - "max_age_seconds": 0.000315, - "max_depth": 1 - }, - "requests": 4097, - "retries": 8, - "source_file_bytes": 856064, - "terminal_failures": 0, - "traces": [ - { - "phase": "discovery", - "seconds": 0.120288 - }, - { - "phase": "source_reading", - "seconds": 1.252442 - }, - { - "phase": "normalization_redaction", - "seconds": 0.441244 - }, - { - "phase": "request_encoding", - "seconds": 0.03615 - }, - { - "phase": "network_wait", - "seconds": 20.747029 - }, - { - "phase": "server_persistence", - "seconds": 17.25631 - }, - { - "phase": "queue_delay", - "seconds": 0.090012 - }, - { - "phase": "provider_extraction", - "seconds": 10.447082 - }, - { - "phase": "entity_resolution", - "seconds": 0.006413 - }, - { - "phase": "projection_indexing", - "seconds": 11.824323 - }, - { - "phase": "summary_reduction", - "seconds": 5e-06 - } - ], - "wall_time_seconds": 65.705342, - "windows": 2048, - "windows_per_minute": 1870.168 - }, - { - "case_id": "large-sqlite", - "concurrency": 2, - "cpu_seconds": 4.48692, - "events": 4097, - "events_per_second": 60.516, - "fixture_kind": "sqlite", - "fixture_sha256": "aba188af3961fa12a7d4ddfd591e0ca099cef40cd0e65bf61dd337602bb1c858", - "input_bytes": 524288, - "integrity": { - "complete": true, - "duplicate_acks": 1, - "duplicate_side_effects": 0, - "expected_windows": 2048, - "idempotency_replay_kind": "session_end", - "idempotency_replays": 1, - "idempotency_state_components": [ - "duplicate_acks", - "encoded_bytes", - "max_queue_depth", - "projected_digest", - "projected_rows", - "queue_items", - "received_digest", - "received_rows", - "redacted_events", - "redaction_failures", - "requests", - "work_queue_items" - ], - "projected_windows": 2048, - "redacted_events": 2048, - "redaction_failures": 0 - }, - "lifecycle_seconds": { - "first_usable": 0.32557, - "fully_ready": 67.702985, - "transferred": 67.701039 - }, - "mib_per_second": 0.007, - "peak_rss_bytes": 41242624, - "phase_seconds": { - "discovery": 0.224945, - "entity_resolution": 0.007274, - "network_wait": 20.748706, - "normalization_redaction": 0.441318, - "projection_indexing": 12.757385, - "provider_extraction": 10.445039, - "queue_delay": 0.122266, - "request_encoding": 0.036102, - "server_persistence": 18.224253, - "source_reading": 1.234864, - "summary_reduction": 6e-06 - }, - "provider": { - "max_in_flight": 2, - "modeled_cost_usd": 0.512, - "modeled_quota_seconds": 2048.0, - "observed_time_seconds": 10.445039, - "quota_requests_per_minute": 60, - "requests": 2048, - "retries": 8, - "worker_threads_used": 2 - }, - "quality": { - "projection_failures": 0, - "terminal_failures": 0 - }, - "queue": { - "max_age_seconds": 0.024455, - "max_depth": 1 - }, - "requests": 4097, - "retries": 8, - "source_file_bytes": 856064, - "terminal_failures": 0, - "traces": [ - { - "phase": "discovery", - "seconds": 0.224945 - }, - { - "phase": "source_reading", - "seconds": 1.234864 - }, - { - "phase": "normalization_redaction", - "seconds": 0.441318 - }, - { - "phase": "request_encoding", - "seconds": 0.036102 - }, - { - "phase": "network_wait", - "seconds": 20.748706 - }, - { - "phase": "server_persistence", - "seconds": 18.224253 - }, - { - "phase": "queue_delay", - "seconds": 0.122266 - }, - { - "phase": "provider_extraction", - "seconds": 10.445039 - }, - { - "phase": "entity_resolution", - "seconds": 0.007274 - }, - { - "phase": "projection_indexing", - "seconds": 12.757385 - }, - { - "phase": "summary_reduction", - "seconds": 6e-06 - } - ], - "wall_time_seconds": 67.702985, - "windows": 2048, - "windows_per_minute": 1814.986 - }, - { - "case_id": "large-sqlite", - "concurrency": 3, - "cpu_seconds": 4.33097, - "events": 4097, - "events_per_second": 67.1, - "fixture_kind": "sqlite", - "fixture_sha256": "aba188af3961fa12a7d4ddfd591e0ca099cef40cd0e65bf61dd337602bb1c858", - "input_bytes": 524288, - "integrity": { - "complete": true, - "duplicate_acks": 1, - "duplicate_side_effects": 0, - "expected_windows": 2048, - "idempotency_replay_kind": "session_end", - "idempotency_replays": 1, - "idempotency_state_components": [ - "duplicate_acks", - "encoded_bytes", - "max_queue_depth", - "projected_digest", - "projected_rows", - "queue_items", - "received_digest", - "received_rows", - "redacted_events", - "redaction_failures", - "requests", - "work_queue_items" - ], - "projected_windows": 2048, - "redacted_events": 2048, - "redaction_failures": 0 - }, - "lifecycle_seconds": { - "first_usable": 0.249121, - "fully_ready": 61.060444, - "transferred": 61.058413 - }, - "mib_per_second": 0.008, - "peak_rss_bytes": 40964096, - "phase_seconds": { - "discovery": 0.126012, - "entity_resolution": 0.006716, - "network_wait": 20.746407, - "normalization_redaction": 0.434627, - "projection_indexing": 10.3981, - "provider_extraction": 10.446778, - "queue_delay": 0.166578, - "request_encoding": 0.035807, - "server_persistence": 15.224349, - "source_reading": 1.208577, - "summary_reduction": 6e-06 - }, - "provider": { - "max_in_flight": 3, - "modeled_cost_usd": 0.512, - "modeled_quota_seconds": 2048.0, - "observed_time_seconds": 10.446778, - "quota_requests_per_minute": 60, - "requests": 2048, - "retries": 8, - "worker_threads_used": 3 - }, - "quality": { - "projection_failures": 0, - "terminal_failures": 0 - }, - "queue": { - "max_age_seconds": 0.048605, - "max_depth": 1 - }, - "requests": 4097, - "retries": 8, - "source_file_bytes": 856064, - "terminal_failures": 0, - "traces": [ - { - "phase": "discovery", - "seconds": 0.126012 - }, - { - "phase": "source_reading", - "seconds": 1.208577 - }, - { - "phase": "normalization_redaction", - "seconds": 0.434627 - }, - { - "phase": "request_encoding", - "seconds": 0.035807 - }, - { - "phase": "network_wait", - "seconds": 20.746407 - }, - { - "phase": "server_persistence", - "seconds": 15.224349 - }, - { - "phase": "queue_delay", - "seconds": 0.166578 - }, - { - "phase": "provider_extraction", - "seconds": 10.446778 - }, - { - "phase": "entity_resolution", - "seconds": 0.006716 - }, - { - "phase": "projection_indexing", - "seconds": 10.3981 - }, - { - "phase": "summary_reduction", - "seconds": 6e-06 - } - ], - "wall_time_seconds": 61.060444, - "windows": 2048, - "windows_per_minute": 2012.432 - }, - { - "case_id": "large-sqlite", - "concurrency": 4, - "cpu_seconds": 4.377926, - "events": 4097, - "events_per_second": 66.948, - "fixture_kind": "sqlite", - "fixture_sha256": "aba188af3961fa12a7d4ddfd591e0ca099cef40cd0e65bf61dd337602bb1c858", - "input_bytes": 524288, - "integrity": { - "complete": true, - "duplicate_acks": 1, - "duplicate_side_effects": 0, - "expected_windows": 2048, - "idempotency_replay_kind": "session_end", - "idempotency_replays": 1, - "idempotency_state_components": [ - "duplicate_acks", - "encoded_bytes", - "max_queue_depth", - "projected_digest", - "projected_rows", - "queue_items", - "received_digest", - "received_rows", - "redacted_events", - "redaction_failures", - "requests", - "work_queue_items" - ], - "projected_windows": 2048, - "redacted_events": 2048, - "redaction_failures": 0 - }, - "lifecycle_seconds": { - "first_usable": 0.263034, - "fully_ready": 61.198449, - "transferred": 61.196376 - }, - "mib_per_second": 0.008, - "peak_rss_bytes": 41148416, - "phase_seconds": { - "discovery": 0.124265, - "entity_resolution": 0.00688, - "network_wait": 20.74787, - "normalization_redaction": 0.43427, - "projection_indexing": 9.783309, - "provider_extraction": 10.450529, - "queue_delay": 0.247057, - "request_encoding": 0.037076, - "server_persistence": 15.494892, - "source_reading": 1.22607, - "summary_reduction": 6e-06 - }, - "provider": { - "max_in_flight": 4, - "modeled_cost_usd": 0.512, - "modeled_quota_seconds": 2048.0, - "observed_time_seconds": 10.450529, - "quota_requests_per_minute": 60, - "requests": 2048, - "retries": 8, - "worker_threads_used": 4 - }, - "quality": { - "projection_failures": 0, - "terminal_failures": 0 - }, - "queue": { - "max_age_seconds": 0.073207, - "max_depth": 1 - }, - "requests": 4097, - "retries": 8, - "source_file_bytes": 856064, - "terminal_failures": 0, - "traces": [ - { - "phase": "discovery", - "seconds": 0.124265 - }, - { - "phase": "source_reading", - "seconds": 1.22607 - }, - { - "phase": "normalization_redaction", - "seconds": 0.43427 - }, - { - "phase": "request_encoding", - "seconds": 0.037076 - }, - { - "phase": "network_wait", - "seconds": 20.74787 - }, - { - "phase": "server_persistence", - "seconds": 15.494892 - }, - { - "phase": "queue_delay", - "seconds": 0.247057 - }, - { - "phase": "provider_extraction", - "seconds": 10.450529 - }, - { - "phase": "entity_resolution", - "seconds": 0.00688 - }, - { - "phase": "projection_indexing", - "seconds": 9.783309 - }, - { - "phase": "summary_reduction", - "seconds": 6e-06 - } - ], - "wall_time_seconds": 61.198449, - "windows": 2048, - "windows_per_minute": 2007.894 - }, - { - "case_id": "oversized-message-jsonl", - "concurrency": 1, - "cpu_seconds": 2.114711, - "events": 13, - "events_per_second": 5.564, - "fixture_kind": "official_export_jsonl", - "fixture_sha256": "71c48fb839362d1d62f20e30c05ba2a90b70dca5406b21da4434b4feddb9a926", - "input_bytes": 2097152, - "integrity": { - "complete": true, - "duplicate_acks": 1, - "duplicate_side_effects": 0, - "expected_windows": 1, - "idempotency_replay_kind": "session_end", - "idempotency_replays": 1, - "idempotency_state_components": [ - "duplicate_acks", - "encoded_bytes", - "max_queue_depth", - "projected_digest", - "projected_rows", - "queue_items", - "received_digest", - "received_rows", - "redacted_events", - "redaction_failures", - "requests", - "work_queue_items" - ], - "projected_windows": 1, - "redacted_events": 1, - "redaction_failures": 0 - }, - "lifecycle_seconds": { - "first_usable": 2.339924, - "fully_ready": 2.340017, - "transferred": 2.33645 - }, - "mib_per_second": 0.856, - "peak_rss_bytes": 38264832, - "phase_seconds": { - "discovery": 0.590811, - "entity_resolution": 5e-06, - "network_wait": 0.070886, - "normalization_redaction": 0.473388, - "projection_indexing": 0.006478, - "provider_extraction": 0.005062, - "queue_delay": 0.000244, - "request_encoding": 0.003293, - "server_persistence": 0.048931, - "source_reading": 0.864336, - "summary_reduction": 4e-06 - }, - "provider": { - "max_in_flight": 1, - "modeled_cost_usd": 0.00025, - "modeled_quota_seconds": 1.0, - "observed_time_seconds": 0.005062, - "quota_requests_per_minute": 60, - "requests": 1, - "retries": 0, - "worker_threads_used": 1 - }, - "quality": { - "projection_failures": 0, - "terminal_failures": 0 - }, - "queue": { - "max_age_seconds": 0.000244, - "max_depth": 1 - }, - "requests": 13, - "retries": 0, - "source_file_bytes": 2097276, - "terminal_failures": 0, - "traces": [ - { - "phase": "discovery", - "seconds": 0.590811 - }, - { - "phase": "source_reading", - "seconds": 0.864336 - }, - { - "phase": "normalization_redaction", - "seconds": 0.473388 - }, - { - "phase": "request_encoding", - "seconds": 0.003293 - }, - { - "phase": "network_wait", - "seconds": 0.070886 - }, - { - "phase": "server_persistence", - "seconds": 0.048931 - }, - { - "phase": "queue_delay", - "seconds": 0.000244 - }, - { - "phase": "provider_extraction", - "seconds": 0.005062 - }, - { - "phase": "entity_resolution", - "seconds": 5e-06 - }, - { - "phase": "projection_indexing", - "seconds": 0.006478 - }, - { - "phase": "summary_reduction", - "seconds": 4e-06 - } - ], - "wall_time_seconds": 2.340017, - "windows": 1, - "windows_per_minute": 25.641 - }, - { - "case_id": "oversized-message-jsonl", - "concurrency": 2, - "cpu_seconds": 2.094711, - "events": 13, - "events_per_second": 5.624, - "fixture_kind": "official_export_jsonl", - "fixture_sha256": "71c48fb839362d1d62f20e30c05ba2a90b70dca5406b21da4434b4feddb9a926", - "input_bytes": 2097152, - "integrity": { - "complete": true, - "duplicate_acks": 1, - "duplicate_side_effects": 0, - "expected_windows": 1, - "idempotency_replay_kind": "session_end", - "idempotency_replays": 1, - "idempotency_state_components": [ - "duplicate_acks", - "encoded_bytes", - "max_queue_depth", - "projected_digest", - "projected_rows", - "queue_items", - "received_digest", - "received_rows", - "redacted_events", - "redaction_failures", - "requests", - "work_queue_items" - ], - "projected_windows": 1, - "redacted_events": 1, - "redaction_failures": 0 - }, - "lifecycle_seconds": { - "first_usable": 2.31569, - "fully_ready": 2.315793, - "transferred": 2.311649 - }, - "mib_per_second": 0.865, - "peak_rss_bytes": 38477824, - "phase_seconds": { - "discovery": 0.573911, - "entity_resolution": 5e-06, - "network_wait": 0.070885, - "normalization_redaction": 0.465998, - "projection_indexing": 0.006461, - "provider_extraction": 0.00506, - "queue_delay": 0.000224, - "request_encoding": 0.003413, - "server_persistence": 0.040214, - "source_reading": 0.863854, - "summary_reduction": 4e-06 - }, - "provider": { - "max_in_flight": 1, - "modeled_cost_usd": 0.00025, - "modeled_quota_seconds": 1.0, - "observed_time_seconds": 0.00506, - "quota_requests_per_minute": 60, - "requests": 1, - "retries": 0, - "worker_threads_used": 1 - }, - "quality": { - "projection_failures": 0, - "terminal_failures": 0 - }, - "queue": { - "max_age_seconds": 0.000224, - "max_depth": 1 - }, - "requests": 13, - "retries": 0, - "source_file_bytes": 2097276, - "terminal_failures": 0, - "traces": [ - { - "phase": "discovery", - "seconds": 0.573911 - }, - { - "phase": "source_reading", - "seconds": 0.863854 - }, - { - "phase": "normalization_redaction", - "seconds": 0.465998 - }, - { - "phase": "request_encoding", - "seconds": 0.003413 - }, - { - "phase": "network_wait", - "seconds": 0.070885 - }, - { - "phase": "server_persistence", - "seconds": 0.040214 - }, - { - "phase": "queue_delay", - "seconds": 0.000224 - }, - { - "phase": "provider_extraction", - "seconds": 0.00506 - }, - { - "phase": "entity_resolution", - "seconds": 5e-06 - }, - { - "phase": "projection_indexing", - "seconds": 0.006461 - }, - { - "phase": "summary_reduction", - "seconds": 4e-06 - } - ], - "wall_time_seconds": 2.315793, - "windows": 1, - "windows_per_minute": 25.909 - }, - { - "case_id": "oversized-message-jsonl", - "concurrency": 3, - "cpu_seconds": 2.095022, - "events": 13, - "events_per_second": 5.698, - "fixture_kind": "official_export_jsonl", - "fixture_sha256": "71c48fb839362d1d62f20e30c05ba2a90b70dca5406b21da4434b4feddb9a926", - "input_bytes": 2097152, - "integrity": { - "complete": true, - "duplicate_acks": 1, - "duplicate_side_effects": 0, - "expected_windows": 1, - "idempotency_replay_kind": "session_end", - "idempotency_replays": 1, - "idempotency_state_components": [ - "duplicate_acks", - "encoded_bytes", - "max_queue_depth", - "projected_digest", - "projected_rows", - "queue_items", - "received_digest", - "received_rows", - "redacted_events", - "redaction_failures", - "requests", - "work_queue_items" - ], - "projected_windows": 1, - "redacted_events": 1, - "redaction_failures": 0 - }, - "lifecycle_seconds": { - "first_usable": 2.285141, - "fully_ready": 2.28524, - "transferred": 2.281383 - }, - "mib_per_second": 0.877, - "peak_rss_bytes": 38359040, - "phase_seconds": { - "discovery": 0.574473, - "entity_resolution": 5e-06, - "network_wait": 0.070894, - "normalization_redaction": 0.468038, - "projection_indexing": 0.006857, - "provider_extraction": 0.00506, - "queue_delay": 0.000225, - "request_encoding": 0.003427, - "server_persistence": 0.043712, - "source_reading": 0.858745, - "summary_reduction": 5e-06 - }, - "provider": { - "max_in_flight": 1, - "modeled_cost_usd": 0.00025, - "modeled_quota_seconds": 1.0, - "observed_time_seconds": 0.00506, - "quota_requests_per_minute": 60, - "requests": 1, - "retries": 0, - "worker_threads_used": 1 - }, - "quality": { - "projection_failures": 0, - "terminal_failures": 0 - }, - "queue": { - "max_age_seconds": 0.000225, - "max_depth": 1 - }, - "requests": 13, - "retries": 0, - "source_file_bytes": 2097276, - "terminal_failures": 0, - "traces": [ - { - "phase": "discovery", - "seconds": 0.574473 - }, - { - "phase": "source_reading", - "seconds": 0.858745 - }, - { - "phase": "normalization_redaction", - "seconds": 0.468038 - }, - { - "phase": "request_encoding", - "seconds": 0.003427 - }, - { - "phase": "network_wait", - "seconds": 0.070894 - }, - { - "phase": "server_persistence", - "seconds": 0.043712 - }, - { - "phase": "queue_delay", - "seconds": 0.000225 - }, - { - "phase": "provider_extraction", - "seconds": 0.00506 - }, - { - "phase": "entity_resolution", - "seconds": 5e-06 - }, - { - "phase": "projection_indexing", - "seconds": 0.006857 - }, - { - "phase": "summary_reduction", - "seconds": 5e-06 - } - ], - "wall_time_seconds": 2.28524, - "windows": 1, - "windows_per_minute": 26.255 - }, - { - "case_id": "oversized-message-jsonl", - "concurrency": 4, - "cpu_seconds": 2.079808, - "events": 13, - "events_per_second": 5.777, - "fixture_kind": "official_export_jsonl", - "fixture_sha256": "71c48fb839362d1d62f20e30c05ba2a90b70dca5406b21da4434b4feddb9a926", - "input_bytes": 2097152, - "integrity": { - "complete": true, - "duplicate_acks": 1, - "duplicate_side_effects": 0, - "expected_windows": 1, - "idempotency_replay_kind": "session_end", - "idempotency_replays": 1, - "idempotency_state_components": [ - "duplicate_acks", - "encoded_bytes", - "max_queue_depth", - "projected_digest", - "projected_rows", - "queue_items", - "received_digest", - "received_rows", - "redacted_events", - "redaction_failures", - "requests", - "work_queue_items" - ], - "projected_windows": 1, - "redacted_events": 1, - "redaction_failures": 0 - }, - "lifecycle_seconds": { - "first_usable": 2.252126, - "fully_ready": 2.25221, - "transferred": 2.250348 - }, - "mib_per_second": 0.889, - "peak_rss_bytes": 38334464, - "phase_seconds": { - "discovery": 0.579623, - "entity_resolution": 6e-06, - "network_wait": 0.07089, - "normalization_redaction": 0.466827, - "projection_indexing": 0.002559, - "provider_extraction": 0.005069, - "queue_delay": 0.000221, - "request_encoding": 0.003494, - "server_persistence": 0.038854, - "source_reading": 0.841872, - "summary_reduction": 3e-06 - }, - "provider": { - "max_in_flight": 1, - "modeled_cost_usd": 0.00025, - "modeled_quota_seconds": 1.0, - "observed_time_seconds": 0.005069, - "quota_requests_per_minute": 60, - "requests": 1, - "retries": 0, - "worker_threads_used": 1 - }, - "quality": { - "projection_failures": 0, - "terminal_failures": 0 - }, - "queue": { - "max_age_seconds": 0.000221, - "max_depth": 1 - }, - "requests": 13, - "retries": 0, - "source_file_bytes": 2097276, - "terminal_failures": 0, - "traces": [ - { - "phase": "discovery", - "seconds": 0.579623 - }, - { - "phase": "source_reading", - "seconds": 0.841872 - }, - { - "phase": "normalization_redaction", - "seconds": 0.466827 - }, - { - "phase": "request_encoding", - "seconds": 0.003494 - }, - { - "phase": "network_wait", - "seconds": 0.07089 - }, - { - "phase": "server_persistence", - "seconds": 0.038854 - }, - { - "phase": "queue_delay", - "seconds": 0.000221 - }, - { - "phase": "provider_extraction", - "seconds": 0.005069 - }, - { - "phase": "entity_resolution", - "seconds": 6e-06 - }, - { - "phase": "projection_indexing", - "seconds": 0.002559 - }, - { - "phase": "summary_reduction", - "seconds": 3e-06 - } - ], - "wall_time_seconds": 2.25221, - "windows": 1, - "windows_per_minute": 26.641 - }, - { - "case_id": "sqlite-adapter", - "concurrency": 1, - "cpu_seconds": 0.139974, - "events": 161, - "events_per_second": 82.727, - "fixture_kind": "sqlite", - "fixture_sha256": "f037384ecce7e367fcb03640af51c7169aa4d77516a9e21ee840311b1eca6f1a", - "input_bytes": 262144, - "integrity": { - "complete": true, - "duplicate_acks": 1, - "duplicate_side_effects": 0, - "expected_windows": 32, - "idempotency_replay_kind": "session_end", - "idempotency_replays": 1, - "idempotency_state_components": [ - "duplicate_acks", - "encoded_bytes", - "max_queue_depth", - "projected_digest", - "projected_rows", - "queue_items", - "received_digest", - "received_rows", - "redacted_events", - "redaction_failures", - "requests", - "work_queue_items" - ], - "projected_windows": 32, - "redacted_events": 128, - "redaction_failures": 0 - }, - "lifecycle_seconds": { - "first_usable": 0.1068, - "fully_ready": 1.94829, - "transferred": 1.946168 - }, - "mib_per_second": 0.128, - "peak_rss_bytes": 34295808, - "phase_seconds": { - "discovery": 0.006441, - "entity_resolution": 0.000107, - "network_wait": 0.820954, - "normalization_redaction": 0.049537, - "projection_indexing": 0.096611, - "provider_extraction": 0.164568, - "queue_delay": 0.001449, - "request_encoding": 0.001752, - "server_persistence": 0.449776, - "source_reading": 0.012622, - "summary_reduction": 9e-06 - }, - "provider": { - "max_in_flight": 1, - "modeled_cost_usd": 0.008, - "modeled_quota_seconds": 32.0, - "observed_time_seconds": 0.164568, - "quota_requests_per_minute": 60, - "requests": 32, - "retries": 0, - "worker_threads_used": 1 - }, - "quality": { - "projection_failures": 0, - "terminal_failures": 0 - }, - "queue": { - "max_age_seconds": 0.000164, - "max_depth": 1 - }, - "requests": 161, - "retries": 0, - "source_file_bytes": 544768, - "terminal_failures": 0, - "traces": [ - { - "phase": "discovery", - "seconds": 0.006441 - }, - { - "phase": "source_reading", - "seconds": 0.012622 - }, - { - "phase": "normalization_redaction", - "seconds": 0.049537 - }, - { - "phase": "request_encoding", - "seconds": 0.001752 - }, - { - "phase": "network_wait", - "seconds": 0.820954 - }, - { - "phase": "server_persistence", - "seconds": 0.449776 - }, - { - "phase": "queue_delay", - "seconds": 0.001449 - }, - { - "phase": "provider_extraction", - "seconds": 0.164568 - }, - { - "phase": "entity_resolution", - "seconds": 0.000107 - }, - { - "phase": "projection_indexing", - "seconds": 0.096611 - }, - { - "phase": "summary_reduction", - "seconds": 9e-06 - } - ], - "wall_time_seconds": 1.94829, - "windows": 32, - "windows_per_minute": 985.48 - }, - { - "case_id": "sqlite-adapter", - "concurrency": 2, - "cpu_seconds": 0.143625, - "events": 161, - "events_per_second": 80.809, - "fixture_kind": "sqlite", - "fixture_sha256": "f037384ecce7e367fcb03640af51c7169aa4d77516a9e21ee840311b1eca6f1a", - "input_bytes": 262144, - "integrity": { - "complete": true, - "duplicate_acks": 1, - "duplicate_side_effects": 0, - "expected_windows": 32, - "idempotency_replay_kind": "session_end", - "idempotency_replays": 1, - "idempotency_state_components": [ - "duplicate_acks", - "encoded_bytes", - "max_queue_depth", - "projected_digest", - "projected_rows", - "queue_items", - "received_digest", - "received_rows", - "redacted_events", - "redaction_failures", - "requests", - "work_queue_items" - ], - "projected_windows": 32, - "redacted_events": 128, - "redaction_failures": 0 - }, - "lifecycle_seconds": { - "first_usable": 0.196707, - "fully_ready": 1.994948, - "transferred": 1.992354 - }, - "mib_per_second": 0.125, - "peak_rss_bytes": 34385920, - "phase_seconds": { - "discovery": 0.006142, - "entity_resolution": 0.000118, - "network_wait": 0.820598, - "normalization_redaction": 0.049048, - "projection_indexing": 0.101511, - "provider_extraction": 0.162942, - "queue_delay": 0.063803, - "request_encoding": 0.001833, - "server_persistence": 0.461629, - "source_reading": 0.01259, - "summary_reduction": 9e-06 - }, - "provider": { - "max_in_flight": 2, - "modeled_cost_usd": 0.008, - "modeled_quota_seconds": 32.0, - "observed_time_seconds": 0.162942, - "quota_requests_per_minute": 60, - "requests": 32, - "retries": 0, - "worker_threads_used": 2 - }, - "quality": { - "projection_failures": 0, - "terminal_failures": 0 - }, - "queue": { - "max_age_seconds": 0.062151, - "max_depth": 1 - }, - "requests": 161, - "retries": 0, - "source_file_bytes": 544768, - "terminal_failures": 0, - "traces": [ - { - "phase": "discovery", - "seconds": 0.006142 - }, - { - "phase": "source_reading", - "seconds": 0.01259 - }, - { - "phase": "normalization_redaction", - "seconds": 0.049048 - }, - { - "phase": "request_encoding", - "seconds": 0.001833 - }, - { - "phase": "network_wait", - "seconds": 0.820598 - }, - { - "phase": "server_persistence", - "seconds": 0.461629 - }, - { - "phase": "queue_delay", - "seconds": 0.063803 - }, - { - "phase": "provider_extraction", - "seconds": 0.162942 - }, - { - "phase": "entity_resolution", - "seconds": 0.000118 - }, - { - "phase": "projection_indexing", - "seconds": 0.101511 - }, - { - "phase": "summary_reduction", - "seconds": 9e-06 - } - ], - "wall_time_seconds": 1.994948, - "windows": 32, - "windows_per_minute": 962.431 - }, - { - "case_id": "sqlite-adapter", - "concurrency": 3, - "cpu_seconds": 0.142384, - "events": 161, - "events_per_second": 83.581, - "fixture_kind": "sqlite", - "fixture_sha256": "f037384ecce7e367fcb03640af51c7169aa4d77516a9e21ee840311b1eca6f1a", - "input_bytes": 262144, - "integrity": { - "complete": true, - "duplicate_acks": 1, - "duplicate_side_effects": 0, - "expected_windows": 32, - "idempotency_replay_kind": "session_end", - "idempotency_replays": 1, - "idempotency_state_components": [ - "duplicate_acks", - "encoded_bytes", - "max_queue_depth", - "projected_digest", - "projected_rows", - "queue_items", - "received_digest", - "received_rows", - "redacted_events", - "redaction_failures", - "requests", - "work_queue_items" - ], - "projected_windows": 32, - "redacted_events": 128, - "redaction_failures": 0 - }, - "lifecycle_seconds": { - "first_usable": 0.258056, - "fully_ready": 1.92831, - "transferred": 1.926283 - }, - "mib_per_second": 0.13, - "peak_rss_bytes": 34254848, - "phase_seconds": { - "discovery": 0.006847, - "entity_resolution": 0.000102, - "network_wait": 0.821013, - "normalization_redaction": 0.050055, - "projection_indexing": 0.145336, - "provider_extraction": 0.163812, - "queue_delay": 0.182578, - "request_encoding": 0.001804, - "server_persistence": 0.452874, - "source_reading": 0.01229, - "summary_reduction": 9e-06 - }, - "provider": { - "max_in_flight": 3, - "modeled_cost_usd": 0.008, - "modeled_quota_seconds": 32.0, - "observed_time_seconds": 0.163812, - "quota_requests_per_minute": 60, - "requests": 32, - "retries": 0, - "worker_threads_used": 3 - }, - "quality": { - "projection_failures": 0, - "terminal_failures": 0 - }, - "queue": { - "max_age_seconds": 0.117674, - "max_depth": 1 - }, - "requests": 161, - "retries": 0, - "source_file_bytes": 544768, - "terminal_failures": 0, - "traces": [ - { - "phase": "discovery", - "seconds": 0.006847 - }, - { - "phase": "source_reading", - "seconds": 0.01229 - }, - { - "phase": "normalization_redaction", - "seconds": 0.050055 - }, - { - "phase": "request_encoding", - "seconds": 0.001804 - }, - { - "phase": "network_wait", - "seconds": 0.821013 - }, - { - "phase": "server_persistence", - "seconds": 0.452874 - }, - { - "phase": "queue_delay", - "seconds": 0.182578 - }, - { - "phase": "provider_extraction", - "seconds": 0.163812 - }, - { - "phase": "entity_resolution", - "seconds": 0.000102 - }, - { - "phase": "projection_indexing", - "seconds": 0.145336 - }, - { - "phase": "summary_reduction", - "seconds": 9e-06 - } - ], - "wall_time_seconds": 1.92831, - "windows": 32, - "windows_per_minute": 995.691 - }, - { - "case_id": "sqlite-adapter", - "concurrency": 4, - "cpu_seconds": 0.154681, - "events": 161, - "events_per_second": 66.033, - "fixture_kind": "sqlite", - "fixture_sha256": "f037384ecce7e367fcb03640af51c7169aa4d77516a9e21ee840311b1eca6f1a", - "input_bytes": 262144, - "integrity": { - "complete": true, - "duplicate_acks": 1, - "duplicate_side_effects": 0, - "expected_windows": 32, - "idempotency_replay_kind": "session_end", - "idempotency_replays": 1, - "idempotency_state_components": [ - "duplicate_acks", - "encoded_bytes", - "max_queue_depth", - "projected_digest", - "projected_rows", - "queue_items", - "received_digest", - "received_rows", - "redacted_events", - "redaction_failures", - "requests", - "work_queue_items" - ], - "projected_windows": 32, - "redacted_events": 128, - "redaction_failures": 0 - }, - "lifecycle_seconds": { - "first_usable": 0.304478, - "fully_ready": 2.440497, - "transferred": 2.438164 - }, - "mib_per_second": 0.103, - "peak_rss_bytes": 34488320, - "phase_seconds": { - "discovery": 0.006913, - "entity_resolution": 0.000117, - "network_wait": 0.820464, - "normalization_redaction": 0.049914, - "projection_indexing": 0.16229, - "provider_extraction": 0.163729, - "queue_delay": 0.395561, - "request_encoding": 0.001809, - "server_persistence": 0.495887, - "source_reading": 0.013074, - "summary_reduction": 8e-06 - }, - "provider": { - "max_in_flight": 4, - "modeled_cost_usd": 0.008, - "modeled_quota_seconds": 32.0, - "observed_time_seconds": 0.163729, - "quota_requests_per_minute": 60, - "requests": 32, - "retries": 0, - "worker_threads_used": 4 - }, - "quality": { - "projection_failures": 0, - "terminal_failures": 0 - }, - "queue": { - "max_age_seconds": 0.191136, - "max_depth": 1 - }, - "requests": 161, - "retries": 0, - "source_file_bytes": 544768, - "terminal_failures": 0, - "traces": [ - { - "phase": "discovery", - "seconds": 0.006913 - }, - { - "phase": "source_reading", - "seconds": 0.013074 - }, - { - "phase": "normalization_redaction", - "seconds": 0.049914 - }, - { - "phase": "request_encoding", - "seconds": 0.001809 - }, - { - "phase": "network_wait", - "seconds": 0.820464 - }, - { - "phase": "server_persistence", - "seconds": 0.495887 - }, - { - "phase": "queue_delay", - "seconds": 0.395561 - }, - { - "phase": "provider_extraction", - "seconds": 0.163729 - }, - { - "phase": "entity_resolution", - "seconds": 0.000117 - }, - { - "phase": "projection_indexing", - "seconds": 0.16229 - }, - { - "phase": "summary_reduction", - "seconds": 8e-06 - } - ], - "wall_time_seconds": 2.440497, - "windows": 32, - "windows_per_minute": 786.725 - }, - { - "case_id": "official-export-jsonl", - "concurrency": 1, - "cpu_seconds": 0.289188, - "events": 161, - "events_per_second": 76.95, - "fixture_kind": "official_export_jsonl", - "fixture_sha256": "895521c0e48bb9e7d40a8caddec35fecaf797f14738dc8e247740d22cf415b98", - "input_bytes": 262144, - "integrity": { - "complete": true, - "duplicate_acks": 1, - "duplicate_side_effects": 0, - "expected_windows": 32, - "idempotency_replay_kind": "session_end", - "idempotency_replays": 1, - "idempotency_state_components": [ - "duplicate_acks", - "encoded_bytes", - "max_queue_depth", - "projected_digest", - "projected_rows", - "queue_items", - "received_digest", - "received_rows", - "redacted_events", - "redaction_failures", - "requests", - "work_queue_items" - ], - "projected_windows": 32, - "redacted_events": 128, - "redaction_failures": 0 - }, - "lifecycle_seconds": { - "first_usable": 0.198646, - "fully_ready": 2.095154, - "transferred": 2.092279 - }, - "mib_per_second": 0.119, - "peak_rss_bytes": 33878016, - "phase_seconds": { - "discovery": 0.083113, - "entity_resolution": 0.000115, - "network_wait": 0.821027, - "normalization_redaction": 0.051809, - "projection_indexing": 0.110167, - "provider_extraction": 0.176234, - "queue_delay": 0.001614, - "request_encoding": 0.001845, - "server_persistence": 0.480738, - "source_reading": 0.080368, - "summary_reduction": 9e-06 - }, - "provider": { - "max_in_flight": 1, - "modeled_cost_usd": 0.008, - "modeled_quota_seconds": 32.0, - "observed_time_seconds": 0.176234, - "quota_requests_per_minute": 60, - "requests": 32, - "retries": 0, - "worker_threads_used": 1 - }, - "quality": { - "projection_failures": 0, - "terminal_failures": 0 - }, - "queue": { - "max_age_seconds": 0.000234, - "max_depth": 1 - }, - "requests": 161, - "retries": 0, - "source_file_bytes": 270688, - "terminal_failures": 0, - "traces": [ - { - "phase": "discovery", - "seconds": 0.083113 - }, - { - "phase": "source_reading", - "seconds": 0.080368 - }, - { - "phase": "normalization_redaction", - "seconds": 0.051809 - }, - { - "phase": "request_encoding", - "seconds": 0.001845 - }, - { - "phase": "network_wait", - "seconds": 0.821027 - }, - { - "phase": "server_persistence", - "seconds": 0.480738 - }, - { - "phase": "queue_delay", - "seconds": 0.001614 - }, - { - "phase": "provider_extraction", - "seconds": 0.176234 - }, - { - "phase": "entity_resolution", - "seconds": 0.000115 - }, - { - "phase": "projection_indexing", - "seconds": 0.110167 - }, - { - "phase": "summary_reduction", - "seconds": 9e-06 - } - ], - "wall_time_seconds": 2.095154, - "windows": 32, - "windows_per_minute": 916.4 - }, - { - "case_id": "official-export-jsonl", - "concurrency": 2, - "cpu_seconds": 0.289747, - "events": 161, - "events_per_second": 75.613, - "fixture_kind": "official_export_jsonl", - "fixture_sha256": "895521c0e48bb9e7d40a8caddec35fecaf797f14738dc8e247740d22cf415b98", - "input_bytes": 262144, - "integrity": { - "complete": true, - "duplicate_acks": 1, - "duplicate_side_effects": 0, - "expected_windows": 32, - "idempotency_replay_kind": "session_end", - "idempotency_replays": 1, - "idempotency_state_components": [ - "duplicate_acks", - "encoded_bytes", - "max_queue_depth", - "projected_digest", - "projected_rows", - "queue_items", - "received_digest", - "received_rows", - "redacted_events", - "redaction_failures", - "requests", - "work_queue_items" - ], - "projected_windows": 32, - "redacted_events": 128, - "redaction_failures": 0 - }, - "lifecycle_seconds": { - "first_usable": 0.238421, - "fully_ready": 2.129478, - "transferred": 2.129269 - }, - "mib_per_second": 0.117, - "peak_rss_bytes": 33738752, - "phase_seconds": { - "discovery": 0.078693, - "entity_resolution": 0.000122, - "network_wait": 0.820727, - "normalization_redaction": 0.052274, - "projection_indexing": 0.151181, - "provider_extraction": 0.1761, - "queue_delay": 0.057336, - "request_encoding": 0.00183, - "server_persistence": 0.487409, - "source_reading": 0.080218, - "summary_reduction": 2.2e-05 - }, - "provider": { - "max_in_flight": 2, - "modeled_cost_usd": 0.008, - "modeled_quota_seconds": 32.0, - "observed_time_seconds": 0.1761, - "quota_requests_per_minute": 60, - "requests": 32, - "retries": 0, - "worker_threads_used": 2 - }, - "quality": { - "projection_failures": 0, - "terminal_failures": 0 - }, - "queue": { - "max_age_seconds": 0.055693, - "max_depth": 1 - }, - "requests": 161, - "retries": 0, - "source_file_bytes": 270688, - "terminal_failures": 0, - "traces": [ - { - "phase": "discovery", - "seconds": 0.078693 - }, - { - "phase": "source_reading", - "seconds": 0.080218 - }, - { - "phase": "normalization_redaction", - "seconds": 0.052274 - }, - { - "phase": "request_encoding", - "seconds": 0.00183 - }, - { - "phase": "network_wait", - "seconds": 0.820727 - }, - { - "phase": "server_persistence", - "seconds": 0.487409 - }, - { - "phase": "queue_delay", - "seconds": 0.057336 - }, - { - "phase": "provider_extraction", - "seconds": 0.1761 - }, - { - "phase": "entity_resolution", - "seconds": 0.000122 - }, - { - "phase": "projection_indexing", - "seconds": 0.151181 - }, - { - "phase": "summary_reduction", - "seconds": 2.2e-05 - } - ], - "wall_time_seconds": 2.129478, - "windows": 32, - "windows_per_minute": 901.63 - }, - { - "case_id": "official-export-jsonl", - "concurrency": 3, - "cpu_seconds": 0.289825, - "events": 161, - "events_per_second": 73.662, - "fixture_kind": "official_export_jsonl", - "fixture_sha256": "895521c0e48bb9e7d40a8caddec35fecaf797f14738dc8e247740d22cf415b98", - "input_bytes": 262144, - "integrity": { - "complete": true, - "duplicate_acks": 1, - "duplicate_side_effects": 0, - "expected_windows": 32, - "idempotency_replay_kind": "session_end", - "idempotency_replays": 1, - "idempotency_state_components": [ - "duplicate_acks", - "encoded_bytes", - "max_queue_depth", - "projected_digest", - "projected_rows", - "queue_items", - "received_digest", - "received_rows", - "redacted_events", - "redaction_failures", - "requests", - "work_queue_items" - ], - "projected_windows": 32, - "redacted_events": 128, - "redaction_failures": 0 - }, - "lifecycle_seconds": { - "first_usable": 0.411145, - "fully_ready": 2.188985, - "transferred": 2.185652 - }, - "mib_per_second": 0.114, - "peak_rss_bytes": 33804288, - "phase_seconds": { - "discovery": 0.077842, - "entity_resolution": 0.000122, - "network_wait": 0.82078, - "normalization_redaction": 0.052691, - "projection_indexing": 0.117177, - "provider_extraction": 0.177924, - "queue_delay": 0.373514, - "request_encoding": 0.001863, - "server_persistence": 0.539518, - "source_reading": 0.081001, - "summary_reduction": 8e-06 - }, - "provider": { - "max_in_flight": 3, - "modeled_cost_usd": 0.008, - "modeled_quota_seconds": 32.0, - "observed_time_seconds": 0.177924, - "quota_requests_per_minute": 60, - "requests": 32, - "retries": 0, - "worker_threads_used": 3 - }, - "quality": { - "projection_failures": 0, - "terminal_failures": 0 - }, - "queue": { - "max_age_seconds": 0.215772, - "max_depth": 1 - }, - "requests": 161, - "retries": 0, - "source_file_bytes": 270688, - "terminal_failures": 0, - "traces": [ - { - "phase": "discovery", - "seconds": 0.077842 - }, - { - "phase": "source_reading", - "seconds": 0.081001 - }, - { - "phase": "normalization_redaction", - "seconds": 0.052691 - }, - { - "phase": "request_encoding", - "seconds": 0.001863 - }, - { - "phase": "network_wait", - "seconds": 0.82078 - }, - { - "phase": "server_persistence", - "seconds": 0.539518 - }, - { - "phase": "queue_delay", - "seconds": 0.373514 - }, - { - "phase": "provider_extraction", - "seconds": 0.177924 - }, - { - "phase": "entity_resolution", - "seconds": 0.000122 - }, - { - "phase": "projection_indexing", - "seconds": 0.117177 - }, - { - "phase": "summary_reduction", - "seconds": 8e-06 - } - ], - "wall_time_seconds": 2.188985, - "windows": 32, - "windows_per_minute": 877.119 - }, - { - "case_id": "official-export-jsonl", - "concurrency": 4, - "cpu_seconds": 0.301907, - "events": 161, - "events_per_second": 72.218, - "fixture_kind": "official_export_jsonl", - "fixture_sha256": "895521c0e48bb9e7d40a8caddec35fecaf797f14738dc8e247740d22cf415b98", - "input_bytes": 262144, - "integrity": { - "complete": true, - "duplicate_acks": 1, - "duplicate_side_effects": 0, - "expected_windows": 32, - "idempotency_replay_kind": "session_end", - "idempotency_replays": 1, - "idempotency_state_components": [ - "duplicate_acks", - "encoded_bytes", - "max_queue_depth", - "projected_digest", - "projected_rows", - "queue_items", - "received_digest", - "received_rows", - "redacted_events", - "redaction_failures", - "requests", - "work_queue_items" - ], - "projected_windows": 32, - "redacted_events": 128, - "redaction_failures": 0 - }, - "lifecycle_seconds": { - "first_usable": 0.418488, - "fully_ready": 2.23256, - "transferred": 2.229374 - }, - "mib_per_second": 0.112, - "peak_rss_bytes": 33910784, - "phase_seconds": { - "discovery": 0.081098, - "entity_resolution": 0.000115, - "network_wait": 0.820503, - "normalization_redaction": 0.053984, - "projection_indexing": 0.229123, - "provider_extraction": 0.177425, - "queue_delay": 0.437055, - "request_encoding": 0.00189, - "server_persistence": 0.511794, - "source_reading": 0.084947, - "summary_reduction": 9e-06 - }, - "provider": { - "max_in_flight": 4, - "modeled_cost_usd": 0.008, - "modeled_quota_seconds": 32.0, - "observed_time_seconds": 0.177425, - "quota_requests_per_minute": 60, - "requests": 32, - "retries": 0, - "worker_threads_used": 4 - }, - "quality": { - "projection_failures": 0, - "terminal_failures": 0 - }, - "queue": { - "max_age_seconds": 0.21304, - "max_depth": 1 - }, - "requests": 161, - "retries": 0, - "source_file_bytes": 270688, - "terminal_failures": 0, - "traces": [ - { - "phase": "discovery", - "seconds": 0.081098 - }, - { - "phase": "source_reading", - "seconds": 0.084947 - }, - { - "phase": "normalization_redaction", - "seconds": 0.053984 - }, - { - "phase": "request_encoding", - "seconds": 0.00189 - }, - { - "phase": "network_wait", - "seconds": 0.820503 - }, - { - "phase": "server_persistence", - "seconds": 0.511794 - }, - { - "phase": "queue_delay", - "seconds": 0.437055 - }, - { - "phase": "provider_extraction", - "seconds": 0.177425 - }, - { - "phase": "entity_resolution", - "seconds": 0.000115 - }, - { - "phase": "projection_indexing", - "seconds": 0.229123 - }, - { - "phase": "summary_reduction", - "seconds": 9e-06 - } - ], - "wall_time_seconds": 2.23256, - "windows": 32, - "windows_per_minute": 859.999 - } - ], - "schema_version": 1 -} diff --git a/benchmarks/hermes-migration-budget.json b/benchmarks/hermes-migration-budget.json deleted file mode 100644 index 757a4b8..0000000 --- a/benchmarks/hermes-migration-budget.json +++ /dev/null @@ -1,130 +0,0 @@ -{ - "cases": { - "current-production-jsonl": { - "evidence_runs": 4, - "max_cpu_seconds": 5.811681, - "max_duplicate_side_effects": 0, - "max_first_usable_seconds": 2.475396, - "max_fully_ready_seconds": 56.182937, - "max_modeled_cost_usd": 0.303, - "max_modeled_quota_seconds": 1212.0, - "max_peak_rss_bytes": 268435456, - "max_projection_failures": 0, - "max_provider_retries": 5, - "max_redaction_failures": 0, - "max_terminal_failures": 0, - "max_transferred_seconds": 56.181317, - "min_duplicate_acks": 1, - "min_integrity_ratio": 1.0, - "min_redacted_events": 1212 - }, - "large-sqlite": { - "evidence_runs": 4, - "max_cpu_seconds": 8.73038, - "max_duplicate_side_effects": 0, - "max_first_usable_seconds": 2.488355, - "max_fully_ready_seconds": 103.554477, - "max_modeled_cost_usd": 0.512, - "max_modeled_quota_seconds": 2048.0, - "max_peak_rss_bytes": 268435456, - "max_projection_failures": 0, - "max_provider_retries": 8, - "max_redaction_failures": 0, - "max_terminal_failures": 0, - "max_transferred_seconds": 103.551558, - "min_duplicate_acks": 1, - "min_integrity_ratio": 1.0, - "min_redacted_events": 2048 - }, - "official-export-jsonl": { - "evidence_runs": 4, - "max_cpu_seconds": 2.45286, - "max_duplicate_side_effects": 0, - "max_first_usable_seconds": 2.627732, - "max_fully_ready_seconds": 5.34884, - "max_modeled_cost_usd": 0.008, - "max_modeled_quota_seconds": 32.0, - "max_peak_rss_bytes": 268435456, - "max_projection_failures": 0, - "max_provider_retries": 0, - "max_redaction_failures": 0, - "max_terminal_failures": 0, - "max_transferred_seconds": 5.344061, - "min_duplicate_acks": 1, - "min_integrity_ratio": 1.0, - "min_redacted_events": 128 - }, - "oversized-message-jsonl": { - "evidence_runs": 4, - "max_cpu_seconds": 5.172066, - "max_duplicate_side_effects": 0, - "max_first_usable_seconds": 5.509886, - "max_fully_ready_seconds": 5.510026, - "max_modeled_cost_usd": 0.00025, - "max_modeled_quota_seconds": 1.0, - "max_peak_rss_bytes": 268435456, - "max_projection_failures": 0, - "max_provider_retries": 0, - "max_redaction_failures": 0, - "max_terminal_failures": 0, - "max_transferred_seconds": 5.504675, - "min_duplicate_acks": 1, - "min_integrity_ratio": 1.0, - "min_redacted_events": 1 - }, - "small-sqlite": { - "evidence_runs": 4, - "max_cpu_seconds": 2.040779, - "max_duplicate_side_effects": 0, - "max_first_usable_seconds": 2.295569, - "max_fully_ready_seconds": 2.599844, - "max_modeled_cost_usd": 0.002, - "max_modeled_quota_seconds": 8.0, - "max_peak_rss_bytes": 268435456, - "max_projection_failures": 0, - "max_provider_retries": 0, - "max_redaction_failures": 0, - "max_terminal_failures": 0, - "max_transferred_seconds": 2.596915, - "min_duplicate_acks": 1, - "min_integrity_ratio": 1.0, - "min_redacted_events": 16 - }, - "sqlite-adapter": { - "evidence_runs": 4, - "max_cpu_seconds": 2.232022, - "max_duplicate_side_effects": 0, - "max_first_usable_seconds": 2.456717, - "max_fully_ready_seconds": 5.660745, - "max_modeled_cost_usd": 0.008, - "max_modeled_quota_seconds": 32.0, - "max_peak_rss_bytes": 268435456, - "max_projection_failures": 0, - "max_provider_retries": 0, - "max_redaction_failures": 0, - "max_terminal_failures": 0, - "max_transferred_seconds": 5.657246, - "min_duplicate_acks": 1, - "min_integrity_ratio": 1.0, - "min_redacted_events": 128 - } - }, - "derivation": { - "evidence_runs": 24, - "host_jitter_seconds": 2.0, - "latency_cpu_multiplier": 1.5, - "peak_rss_ceiling_bytes": 268435456, - "required_concurrency": [ - 1, - 2, - 3, - 4 - ] - }, - "harness_sha256": "ef57926174b22c2c22e73843a08a5ea9b7b45481bf51ef08dac21129ea9c929a", - "manifest_sha256": "d0508dd3586723f8d8ff81ee6ae8a2f4aca381905c45b3ceac42b2770cabc266", - "profile": "ci", - "protocol": "stream-v2", - "receipt_sha256": "a4539746513fa2fafc3c818cdc06ec27b266a3ab198715289c656697b0dadf97", - "schema_version": 1 -} diff --git a/benchmarks/hermes-migration-manifest.json b/benchmarks/hermes-migration-manifest.json deleted file mode 100644 index f03e655..0000000 --- a/benchmarks/hermes-migration-manifest.json +++ /dev/null @@ -1,89 +0,0 @@ -{ - "schema_version": 1, - "protocol": "stream-v2", - "seed": 75, - "concurrency": [1, 2, 3, 4], - "representative_provider": { - "mode": "deterministic_simulation", - "observed_latency_seconds": 0.005, - "modeled_requests_per_minute": 60, - "modeled_cost_usd_per_request": 0.00025, - "retry_every_requests": 257, - "summary_every_windows": 32 - }, - "profiles": { - "contract": ["small-sqlite"], - "ci": [ - "small-sqlite", - "current-production-jsonl", - "large-sqlite", - "oversized-message-jsonl", - "sqlite-adapter", - "official-export-jsonl" - ], - "release-candidate": [ - "small-sqlite", - "current-production-jsonl", - "large-sqlite", - "oversized-message-jsonl", - "sqlite-adapter", - "official-export-jsonl" - ] - }, - "cases": [ - { - "id": "small-sqlite", - "source": "sqlite", - "fixture_kind": "sqlite", - "sessions": 8, - "messages_per_session": 2, - "message_bytes": 1024, - "oversized_message_bytes": 0 - }, - { - "id": "current-production-jsonl", - "source": "jsonl", - "fixture_kind": "official_export_jsonl", - "sessions": 1212, - "messages_per_session": 1, - "message_bytes": 256, - "oversized_message_bytes": 0 - }, - { - "id": "large-sqlite", - "source": "sqlite", - "fixture_kind": "sqlite", - "sessions": 2048, - "messages_per_session": 1, - "message_bytes": 256, - "oversized_message_bytes": 0 - }, - { - "id": "oversized-message-jsonl", - "source": "jsonl", - "fixture_kind": "official_export_jsonl", - "sessions": 1, - "messages_per_session": 1, - "message_bytes": 256, - "oversized_message_bytes": 2097152 - }, - { - "id": "sqlite-adapter", - "source": "sqlite", - "fixture_kind": "sqlite", - "sessions": 32, - "messages_per_session": 4, - "message_bytes": 2048, - "oversized_message_bytes": 0 - }, - { - "id": "official-export-jsonl", - "source": "jsonl", - "fixture_kind": "official_export_jsonl", - "sessions": 32, - "messages_per_session": 4, - "message_bytes": 2048, - "oversized_message_bytes": 0 - } - ] -} diff --git a/benchmarks/hermes-migration-receipt.schema.json b/benchmarks/hermes-migration-receipt.schema.json deleted file mode 100644 index aa61e95..0000000 --- a/benchmarks/hermes-migration-receipt.schema.json +++ /dev/null @@ -1,267 +0,0 @@ -{ - "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "https://substrate.invalid/schemas/hermes-migration-receipt-v1.json", - "title": "Hermes migration aggregate benchmark receipt", - "type": "object", - "additionalProperties": false, - "required": [ - "schema_version", - "protocol", - "profile", - "manifest_sha256", - "harness_sha256", - "hosted_calls", - "representative_provider", - "runs" - ], - "properties": { - "schema_version": {"const": 1}, - "protocol": {"const": "stream-v2"}, - "profile": {"type": "string", "minLength": 1}, - "manifest_sha256": {"type": "string", "pattern": "^[a-f0-9]{64}$"}, - "harness_sha256": {"type": "string", "pattern": "^[a-f0-9]{64}$"}, - "hosted_calls": {"const": 0}, - "representative_provider": { - "type": "object", - "additionalProperties": false, - "required": [ - "mode", - "network_access", - "quota_requests_per_minute", - "cost_usd_per_request" - ], - "properties": { - "mode": {"const": "deterministic_simulation"}, - "network_access": {"const": false}, - "quota_requests_per_minute": {"type": "integer", "minimum": 1}, - "cost_usd_per_request": {"type": "number", "minimum": 0} - } - }, - "runs": { - "type": "array", - "minItems": 1, - "items": {"$ref": "#/$defs/run"} - } - }, - "$defs": { - "nonnegative": {"type": "number", "minimum": 0}, - "nonnegativeInteger": {"type": "integer", "minimum": 0}, - "phaseMap": { - "type": "object", - "additionalProperties": false, - "required": [ - "discovery", - "source_reading", - "normalization_redaction", - "request_encoding", - "network_wait", - "server_persistence", - "queue_delay", - "provider_extraction", - "entity_resolution", - "projection_indexing", - "summary_reduction" - ], - "properties": { - "discovery": {"$ref": "#/$defs/nonnegative"}, - "source_reading": {"$ref": "#/$defs/nonnegative"}, - "normalization_redaction": {"$ref": "#/$defs/nonnegative"}, - "request_encoding": {"$ref": "#/$defs/nonnegative"}, - "network_wait": {"$ref": "#/$defs/nonnegative"}, - "server_persistence": {"$ref": "#/$defs/nonnegative"}, - "queue_delay": {"$ref": "#/$defs/nonnegative"}, - "provider_extraction": {"$ref": "#/$defs/nonnegative"}, - "entity_resolution": {"$ref": "#/$defs/nonnegative"}, - "projection_indexing": {"$ref": "#/$defs/nonnegative"}, - "summary_reduction": {"$ref": "#/$defs/nonnegative"} - } - }, - "run": { - "type": "object", - "additionalProperties": false, - "required": [ - "case_id", - "fixture_kind", - "concurrency", - "fixture_sha256", - "source_file_bytes", - "input_bytes", - "windows", - "events", - "wall_time_seconds", - "cpu_seconds", - "peak_rss_bytes", - "events_per_second", - "mib_per_second", - "windows_per_minute", - "requests", - "retries", - "terminal_failures", - "phase_seconds", - "traces", - "lifecycle_seconds", - "queue", - "provider", - "integrity", - "quality" - ], - "properties": { - "case_id": {"type": "string", "minLength": 1}, - "fixture_kind": {"enum": ["sqlite", "official_export_jsonl"]}, - "concurrency": {"type": "integer", "minimum": 1, "maximum": 4}, - "fixture_sha256": {"type": "string", "pattern": "^[a-f0-9]{64}$"}, - "source_file_bytes": {"$ref": "#/$defs/nonnegativeInteger"}, - "input_bytes": {"$ref": "#/$defs/nonnegativeInteger"}, - "windows": {"type": "integer", "minimum": 1}, - "events": {"type": "integer", "minimum": 1}, - "wall_time_seconds": {"$ref": "#/$defs/nonnegative"}, - "cpu_seconds": {"$ref": "#/$defs/nonnegative"}, - "peak_rss_bytes": {"$ref": "#/$defs/nonnegativeInteger"}, - "events_per_second": {"$ref": "#/$defs/nonnegative"}, - "mib_per_second": {"$ref": "#/$defs/nonnegative"}, - "windows_per_minute": {"$ref": "#/$defs/nonnegative"}, - "requests": {"$ref": "#/$defs/nonnegativeInteger"}, - "retries": {"$ref": "#/$defs/nonnegativeInteger"}, - "terminal_failures": {"$ref": "#/$defs/nonnegativeInteger"}, - "phase_seconds": { - "$ref": "#/$defs/phaseMap", - "required": [ - "discovery", - "source_reading", - "normalization_redaction", - "request_encoding", - "network_wait", - "server_persistence", - "queue_delay", - "provider_extraction", - "entity_resolution", - "projection_indexing", - "summary_reduction" - ] - }, - "traces": { - "type": "array", - "minItems": 11, - "maxItems": 11, - "items": { - "type": "object", - "additionalProperties": false, - "required": ["phase", "seconds"], - "properties": { - "phase": {"enum": [ - "discovery", - "source_reading", - "normalization_redaction", - "request_encoding", - "network_wait", - "server_persistence", - "queue_delay", - "provider_extraction", - "entity_resolution", - "projection_indexing", - "summary_reduction" - ]}, - "seconds": {"$ref": "#/$defs/nonnegative"} - } - } - }, - "lifecycle_seconds": { - "type": "object", - "additionalProperties": false, - "required": ["transferred", "first_usable", "fully_ready"], - "properties": { - "transferred": {"$ref": "#/$defs/nonnegative"}, - "first_usable": {"$ref": "#/$defs/nonnegative"}, - "fully_ready": {"$ref": "#/$defs/nonnegative"} - } - }, - "queue": { - "type": "object", - "additionalProperties": false, - "required": ["max_depth", "max_age_seconds"], - "properties": { - "max_depth": {"$ref": "#/$defs/nonnegativeInteger"}, - "max_age_seconds": {"$ref": "#/$defs/nonnegative"} - } - }, - "provider": { - "type": "object", - "additionalProperties": false, - "required": [ - "requests", - "retries", - "observed_time_seconds", - "modeled_quota_seconds", - "modeled_cost_usd", - "quota_requests_per_minute", - "worker_threads_used", - "max_in_flight" - ], - "properties": { - "requests": {"$ref": "#/$defs/nonnegativeInteger"}, - "retries": {"$ref": "#/$defs/nonnegativeInteger"}, - "observed_time_seconds": {"$ref": "#/$defs/nonnegative"}, - "modeled_quota_seconds": {"$ref": "#/$defs/nonnegative"}, - "modeled_cost_usd": {"$ref": "#/$defs/nonnegative"}, - "quota_requests_per_minute": {"type": "integer", "minimum": 1}, - "worker_threads_used": {"type": "integer", "minimum": 1, "maximum": 4}, - "max_in_flight": {"type": "integer", "minimum": 1, "maximum": 4} - } - }, - "integrity": { - "type": "object", - "additionalProperties": false, - "required": [ - "expected_windows", - "projected_windows", - "redacted_events", - "redaction_failures", - "idempotency_replays", - "idempotency_replay_kind", - "idempotency_state_components", - "duplicate_acks", - "duplicate_side_effects", - "complete" - ], - "properties": { - "expected_windows": {"type": "integer", "minimum": 1}, - "projected_windows": {"$ref": "#/$defs/nonnegativeInteger"}, - "redacted_events": {"type": "integer", "minimum": 1}, - "redaction_failures": {"$ref": "#/$defs/nonnegativeInteger"}, - "idempotency_replays": {"type": "integer", "const": 1}, - "idempotency_replay_kind": {"type": "string", "const": "session_end"}, - "idempotency_state_components": { - "type": "array", - "const": [ - "duplicate_acks", - "encoded_bytes", - "max_queue_depth", - "projected_digest", - "projected_rows", - "queue_items", - "received_digest", - "received_rows", - "redacted_events", - "redaction_failures", - "requests", - "work_queue_items" - ] - }, - "duplicate_acks": {"type": "integer", "const": 1}, - "duplicate_side_effects": {"type": "integer", "const": 0}, - "complete": {"type": "boolean"} - } - }, - "quality": { - "type": "object", - "additionalProperties": false, - "required": ["projection_failures", "terminal_failures"], - "properties": { - "projection_failures": {"$ref": "#/$defs/nonnegativeInteger"}, - "terminal_failures": {"$ref": "#/$defs/nonnegativeInteger"} - } - } - } - } - } -} diff --git a/docs/api-ownership.json b/docs/api-ownership.json deleted file mode 100644 index f436b87..0000000 --- a/docs/api-ownership.json +++ /dev/null @@ -1,339 +0,0 @@ -{ - "authentication": { - "owner": "Substrate-v2 hosted server", - "plugin_obligation": "send credentials only to fixed https://app.trysubstrate.co origin and keep them in secure custody", - "scheme": "RFC 8628 device authorization issuing a tenant-scoped Bearer credential", - "server_obligation": "authenticate the live tenant/account, issue one-time revocable credentials, enforce scope and content-free failures" - }, - "capabilities": { - "breaking_change": "new schema or protocol identifier with concurrent server support", - "conditional": { - "entity_quality": { - "canonical_redirects": true, - "memory_card": true, - "min_plugin_version": "1.4.0", - "protocol": "entity-quality-v2", - "quality_version": 2 - }, - "server_commit": "40-character lowercase Git SHA when deployed" - }, - "consumer": "substrate_wiki plugin", - "endpoint": "GET /api/v1/hermes/capabilities", - "failure": "plugin fails closed with server_upgrade_required", - "owner": "Substrate-v2 server", - "required": { - "capture_schema_versions": [ - 2 - ], - "entity_memory": { - "canonical_wiki_pages": true, - "entity_page_type": "entity", - "min_plugin_version": "1.3.0", - "protocol": "entity-wiki-v1", - "search_endpoint": "/api/v1/hermes/memory/search" - }, - "history_replay": { - "content_free_completion": true, - "incremental_windows": true, - "min_plugin_version": "1.2.0", - "protocol": "stream-v2", - "status_version": 2 - }, - "max_event_bytes": 262144, - "provider": "substrate_wiki" - } - }, - "contract_status": "boundary-approved; versioned schemas and fixtures follow in SUB-47", - "onboarding": { - "approval_endpoint": "POST /oauth/device/approve", - "client_id": "substrate-hermes", - "credential_management": "owner-only /api/v1/plugin-credentials", - "device_authorization_endpoint": "POST /oauth/device_authorization", - "scopes": [ - "capture", - "retrieve" - ], - "token_endpoint": "POST /oauth/token", - "verification_endpoint": "GET /oauth/device" - }, - "operations": [ - { - "compatibility": "required", - "consumer": "substrate_wiki plugin", - "contract_version": 1, - "method": "GET", - "owner": "Substrate-v2 server", - "path": "/api/v1/hermes/capabilities", - "request_schema": "Bearer authentication; no body", - "response_schema": "HermesCapabilities-v1" - }, - { - "compatibility": "required", - "consumer": "substrate_wiki plugin", - "contract_version": 1, - "method": "POST", - "owner": "Substrate-v2 server", - "path": "/api/v1/hermes/turns", - "request_schema": "HermesCaptureEvent-v2 kind=turns; max 262144 bytes", - "response_schema": "HermesEventAck" - }, - { - "compatibility": "required", - "consumer": "substrate_wiki plugin", - "contract_version": 1, - "method": "POST", - "owner": "Substrate-v2 server", - "path": "/api/v1/hermes/completed-sessions", - "request_schema": "HermesCaptureEvent-v2 kind=completed-sessions; stream-v2 completion rules", - "response_schema": "HermesEventAck" - }, - { - "compatibility": "required", - "consumer": "substrate_wiki plugin", - "contract_version": 1, - "method": "POST", - "owner": "Substrate-v2 server", - "path": "/api/v1/hermes/memory-write-events", - "request_schema": "HermesCaptureEvent-v2 kind=memory-write-events; max 262144 bytes", - "response_schema": "HermesEventAck" - }, - { - "compatibility": "deprecated-alias", - "consumer": "legacy integrations", - "contract_version": 1, - "method": "POST", - "owner": "Substrate-v2 server", - "path": "/api/v1/hermes/sessions", - "request_schema": "HermesCaptureEvent-v2 alias for completed-sessions", - "response_schema": "HermesEventAck" - }, - { - "compatibility": "deprecated-alias", - "consumer": "legacy integrations", - "contract_version": 1, - "method": "POST", - "owner": "Substrate-v2 server", - "path": "/api/v1/hermes/memory-writes", - "request_schema": "HermesCaptureEvent-v2 alias for memory-write-events", - "response_schema": "HermesEventAck" - }, - { - "compatibility": "required", - "consumer": "substrate_wiki plugin", - "contract_version": 1, - "method": "GET", - "owner": "Substrate-v2 server", - "path": "/api/v1/hermes/import-status", - "request_schema": "query batch_id length 1..128", - "response_schema": "ImportStatus-v2 content-free aggregate" - }, - { - "compatibility": "deprecated-alias", - "consumer": "legacy substrate_wiki plugin", - "contract_version": 1, - "method": "GET", - "owner": "Substrate-v2 server", - "path": "/api/v1/hermes/representation-context", - "request_schema": "bounded query and optional scope coordinates", - "response_schema": "EntityMemorySearch-v1" - }, - { - "compatibility": "deprecated-alias", - "consumer": "legacy substrate_wiki plugin", - "contract_version": 1, - "method": "GET", - "owner": "Substrate-v2 server", - "path": "/api/v1/hermes/memory/search", - "request_schema": "bounded query and optional scope coordinates", - "response_schema": "EntityMemorySearch-v1" - }, - { - "compatibility": "deprecated-alias", - "consumer": "substrate_wiki plugin", - "contract_version": 1, - "method": "POST", - "owner": "Substrate-v2 server", - "path": "/api/v1/hermes/representation-context", - "request_schema": "HermesMemorySearchRequest; max 16384 bytes", - "response_schema": "EntityMemorySearch-v1" - }, - { - "compatibility": "required", - "consumer": "substrate_wiki plugin", - "contract_version": 1, - "method": "POST", - "owner": "Substrate-v2 server", - "path": "/api/v1/hermes/memory/search", - "request_schema": "HermesMemorySearchRequest; max 16384 bytes", - "response_schema": "EntityMemorySearch-v1" - }, - { - "compatibility": "deprecated-alias", - "consumer": "legacy substrate_wiki plugin", - "contract_version": 1, - "method": "GET", - "owner": "Substrate-v2 server", - "path": "/api/v1/hermes/wiki/search", - "request_schema": "bounded q, limit, and page_type query", - "response_schema": "list[SearchResultView]" - }, - { - "compatibility": "required", - "consumer": "substrate_wiki plugin", - "contract_version": 1, - "method": "POST", - "owner": "Substrate-v2 server", - "path": "/api/v1/hermes/wiki/search", - "request_schema": "PrivateWikiSearchRequest; max 16384 bytes", - "response_schema": "list[SearchResultView]" - }, - { - "compatibility": "deprecated-alias", - "consumer": "legacy substrate_wiki plugin", - "contract_version": 1, - "method": "GET", - "owner": "Substrate-v2 server", - "path": "/api/v1/hermes/wiki/read", - "request_schema": "query path", - "response_schema": "WikiPageView" - }, - { - "compatibility": "required", - "consumer": "substrate_wiki plugin", - "contract_version": 1, - "method": "POST", - "owner": "Substrate-v2 server", - "path": "/api/v1/hermes/wiki/read", - "request_schema": "PrivateWikiReadRequest exact path; max 8192 bytes", - "response_schema": "WikiPageView" - }, - { - "compatibility": "required", - "consumer": "substrate_wiki plugin", - "contract_version": 1, - "method": "POST", - "owner": "Substrate-v2 server", - "path": "/api/v1/hermes/wiki/query", - "request_schema": "QueryRequest", - "response_schema": "QueryResponse" - }, - { - "compatibility": "required", - "consumer": "substrate_wiki plugin", - "contract_version": 1, - "method": "POST", - "owner": "Substrate-v2 server", - "path": "/api/v1/hermes/wiki/ingest", - "request_schema": "IngestRequest plus optional Idempotency-Key", - "response_schema": "JobAccepted" - }, - { - "compatibility": "required", - "consumer": "substrate_wiki plugin", - "contract_version": 1, - "method": "GET", - "owner": "Substrate-v2 server", - "path": "/api/v1/hermes/wiki/job-status", - "request_schema": "query job_id", - "response_schema": "JobView" - }, - { - "compatibility": "required", - "consumer": "operator and validation tooling", - "contract_version": 1, - "method": "GET", - "owner": "Substrate-v2 server", - "path": "/api/v1/hermes/memory/quality/audit", - "request_schema": "Bearer authentication; no body", - "response_schema": "QualityAuditSummary-v1 aggregate only" - }, - { - "compatibility": "required", - "consumer": "operator and validation tooling", - "contract_version": 1, - "method": "GET", - "owner": "Substrate-v2 server", - "path": "/api/v1/hermes/memory/quality/migration", - "request_schema": "Bearer authentication; no body", - "response_schema": "QualityMigrationStatus-v1 aggregate only" - }, - { - "compatibility": "required", - "consumer": "operator and validation tooling", - "contract_version": 1, - "method": "GET", - "owner": "Substrate-v2 server", - "path": "/api/v1/hermes/memory/quality/clusters", - "request_schema": "bounded kind, state, q, limit, and offset query", - "response_schema": "QualityClusterPage-v1" - }, - { - "compatibility": "required", - "consumer": "operator and validation tooling", - "contract_version": 1, - "method": "POST", - "owner": "Substrate-v2 server", - "path": "/api/v1/hermes/memory/quality/actions/preview", - "request_schema": "QualityActionRequest", - "response_schema": "QualityActionPreview-v1" - }, - { - "compatibility": "required", - "consumer": "operator and validation tooling", - "contract_version": 1, - "method": "POST", - "owner": "Substrate-v2 server", - "path": "/api/v1/hermes/memory/quality/batches/apply", - "request_schema": "QualityBatchRequest plus required Idempotency-Key", - "response_schema": "QualityBatchResult-v1" - }, - { - "compatibility": "required", - "consumer": "operator and validation tooling", - "contract_version": 1, - "method": "POST", - "owner": "Substrate-v2 server", - "path": "/api/v1/hermes/memory/quality/batches/{batch_id}/undo", - "request_schema": "path batch_id plus required Idempotency-Key; no body", - "response_schema": "QualityBatchUndo-v1" - }, - { - "compatibility": "required", - "consumer": "operator and validation tooling", - "contract_version": 1, - "method": "POST", - "owner": "Substrate-v2 server", - "path": "/api/v1/hermes/memory/quality/redirects/resolve", - "request_schema": "QualityRedirectRequest", - "response_schema": "QualityRedirect-v1" - }, - { - "compatibility": "required", - "consumer": "operator and validation tooling", - "contract_version": 1, - "method": "GET", - "owner": "Substrate-v2 server", - "path": "/api/v1/hermes/memory/quality/projections/{entity_id}", - "request_schema": "path entity_id; no body", - "response_schema": "QualityProjectionState-v1" - }, - { - "compatibility": "required", - "consumer": "operator and release validation tooling", - "contract_version": 1, - "method": "POST", - "owner": "Substrate-v2 server", - "path": "/api/v1/hermes/validation/canary", - "request_schema": "Bearer authentication; empty body required", - "response_schema": "ValidationCanaryView-v1 aggregate only" - } - ], - "plugin_repository": "https://github.com/Substrate-memory/Substrate-memory-plugins", - "repository_obligations": { - "Substrate-v2": "owns implementation, authentication, validation, persistence, compatibility aliases, and concurrent support", - "hermes-substrate-wiki": "owns client shaping, origin validation, bounded requests, capability enforcement, fallback, and user-facing errors", - "shared_in_SUB-47": "versioned schemas and fixtures consumed independently by both repositories" - }, - "schema_version": 1, - "source_of_truth": "plugin consumer contract snapshot; server implementation remains Substrate-v2" -} diff --git a/docs/architecture.md b/docs/architecture.md index b113adb..eb52f8e 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -1,35 +1,29 @@ # Architecture ```text -Hermes lifecycle / tools +Hermes 0.21.x lifecycle / tools | v -substrate_wiki provider - - bounded prefetch cache - - capture + redaction - - durable spool/checkpoint - - verified HTTPS client +substrate plugin (plugins/substrate, stdlib only) + - pre_llm_call: bounded turn-context recall injection + - post_llm_call: nonblocking completed-turn capture + - on_session_reset / on_session_finalize: session-completion markers + - memory_search / memory_expand / memory_evidence tools + - onboarding.py: RFC 8628 device flow, key stored in profile .env + - verified HTTPS client with pinned public ISRG roots | v -versioned Substrate HTTP capabilities +versioned Substrate HTTP API (capabilities, ledger, memory, agents) ``` ## Recall -Hermes queues retrieval asynchronously. `prefetch()` reads only a bounded session/query cache and never blocks the model turn on network I/O. Accepted automatic-recall results are bounded cited `memory_card` values from canonical entity pages. +Retrieval is ranked server-side over projected fact units. The plugin caches no memory; +every failure injects nothing and every tool result is bounded and redacted. ## Capture -Completed turns, pre-compression boundaries, memory writes, and session boundaries become versioned events. Credential-shaped values are redacted before admission to the durable spool. Delivery uses deterministic event IDs, idempotency keys, bounded retries, and durable acknowledgement checkpoints. - -## History replay - -The importer reads the active Hermes profile's canonical SQLite or supported export source through bounded cursors. It resumes exact acknowledged progress after interruption and sends a content-free completion boundary. It refuses servers without `stream-v2`. - -## Local state - -All writable plugin state is rooted beneath the active `$HERMES_HOME/substrate_wiki` directory. No state is shared across profiles. Spool files and checkpoints are owner-private on POSIX systems. - -## Dependency boundary - -The runtime uses the Python standard library and Hermes host interfaces. The Substrate server never imports plugin Python; this repository never imports Substrate-v2 server code. +Completed turns are posted as live ledger events. Session boundaries post +`capture_session` envelopes so the server can materialize ended sessions into the +extraction pipeline. Credential-shaped values are redacted before admission to the +network path. diff --git a/docs/extraction-manifest.json b/docs/extraction-manifest.json deleted file mode 100644 index e8abf53..0000000 --- a/docs/extraction-manifest.json +++ /dev/null @@ -1,666 +0,0 @@ -{ - "copy_first": true, - "destination_only": [ - { - "class": "standalone_repository_policy_or_test", - "path": ".github/CODEOWNERS", - "reason": "Required only by the independent public repository.", - "sha256": "78e03d9bf8dee9f295260abe9976486d220615a4f0522f771cf4955026803d65" - }, - { - "class": "standalone_repository_policy_or_test", - "path": ".github/ISSUE_TEMPLATE/bug.yml", - "reason": "Required only by the independent public repository.", - "sha256": "f4bc5ff9027cb59c33bfbcef11991a2f452886e6c37403583f0b84ac66fc0449" - }, - { - "class": "standalone_repository_policy_or_test", - "path": ".github/ISSUE_TEMPLATE/config.yml", - "reason": "Required only by the independent public repository.", - "sha256": "51e4ca6236c169db161c0f6924ad41994fd708f82bd01022723982dfa062abc0" - }, - { - "class": "standalone_repository_policy_or_test", - "path": ".github/PULL_REQUEST_TEMPLATE.md", - "reason": "Required only by the independent public repository.", - "sha256": "0de24ee2fb0922911d1b35a323c3b1c5045934648950ff72b000ac87780ff242" - }, - { - "class": "standalone_repository_policy_or_test", - "path": ".github/dependabot.yml", - "reason": "Required only by the independent public repository.", - "sha256": "b69145d17fec231f5e60ba82ac7f1e2700a01fcb1fecaec860c67f0a414a027d" - }, - { - "class": "standalone_repository_policy_or_test", - "path": ".github/workflows/ci.yml", - "reason": "Required only by the independent public repository.", - "sha256": "a4a80e646d81f3b3b58a1d1c374bbbc8dbfb6dfc86ed6d36e2feaacfc8f8b325" - }, - { - "class": "standalone_repository_policy_or_test", - "path": ".github/workflows/release.yml", - "reason": "Required only by the independent public repository.", - "sha256": "1c2b887ba36fd9ecd1dd0d76ce5240e9bc87f0fd6683a5bbf2468ed3073ad201" - }, - { - "class": "standalone_repository_policy_or_test", - "path": ".gitignore", - "reason": "Required only by the independent public repository.", - "sha256": "30fda153dadc75bab33c149428d9a2bfd700e9d3777270fdc5bb8bb54f14deb9" - }, - { - "class": "standalone_repository_policy_or_test", - "path": "BOUNDARY.md", - "reason": "Required only by the independent public repository.", - "sha256": "2eddde2695fcdda7c98c38eb1036c85ce2afd5820af09873a4b496048b399698" - }, - { - "class": "standalone_repository_policy_or_test", - "path": "CHANGELOG.md", - "reason": "Required only by the independent public repository.", - "sha256": "46459da2eddfa84cae2298ef5b61cfbb6298ce9f6e9048173a7577a33fb0a5a8" - }, - { - "class": "standalone_repository_policy_or_test", - "path": "COMPATIBILITY.md", - "reason": "Required only by the independent public repository.", - "sha256": "ba5e0559b396a1618f296b6ac8824e78431cde706a8fc8774c5e16273e121768" - }, - { - "class": "standalone_repository_policy_or_test", - "path": "CONTRIBUTING.md", - "reason": "Required only by the independent public repository.", - "sha256": "187b627d72fd749ecf666c8c0489a5386e873c6b98c7f2f6f2aeeeefc29e4ab3" - }, - { - "class": "standalone_repository_policy_or_test", - "path": "README.md", - "reason": "Required only by the independent public repository.", - "sha256": "7187a43e2c9936018e4953ae805715f49fc2c0d94fa1aa708263795d7f8d590b" - }, - { - "class": "standalone_repository_policy_or_test", - "path": "SECURITY.md", - "reason": "Required only by the independent public repository.", - "sha256": "7b819f70e96bc4c07db083b416584dc66f6773c80746874a17f0bc5904e4fb2e" - }, - { - "class": "standalone_repository_policy_or_test", - "path": "docs/architecture.md", - "reason": "Required only by the independent public repository.", - "sha256": "61de4c70da5b0aeefe135c6b1600bb1cb160d5c5e535170c4db9ee2caf10e4eb" - }, - { - "class": "standalone_repository_policy_or_test", - "path": "docs/operation.md", - "reason": "Required only by the independent public repository.", - "sha256": "fadd72791e097d878bbfbf338922d568f3c3d47958f26a41f53d3597471988fa" - }, - { - "class": "standalone_repository_policy_or_test", - "path": "docs/releasing.md", - "reason": "Required only by the independent public repository.", - "sha256": "3fa20b3e11f98bfed3f2316b272dbab2b5a46bbb7db30d8745d7bd16286db22d" - }, - { - "class": "standalone_repository_policy_or_test", - "path": "docs/source-of-truth.md", - "reason": "Required only by the independent public repository.", - "sha256": "d89ad302d9f7bef9657e975abf6b419397db62d997be39462688e508c47e29b9" - }, - { - "class": "standalone_repository_policy_or_test", - "path": "docs/threat-model.md", - "reason": "Required only by the independent public repository.", - "sha256": "84c3f829b2d854690c35c6fd553ecfd25bf1ebdec428a2c167c75a1b1e0857b3" - }, - { - "class": "standalone_repository_policy_or_test", - "path": "plugins/substrate/CONTRACT.md", - "reason": "Wire contract for the independent Substrate retrieval plugin.", - "sha256": "e3137a5852c16b484c55f61da8c6048d8bc29f860e3c5fd04bddc77b97cb539c" - }, - { - "class": "standalone_repository_policy_or_test", - "path": "plugins/substrate/README.md", - "reason": "Usage documentation for the independent Substrate retrieval plugin.", - "sha256": "547e6b106337dd2fed2148fc4273fd90360822652dad4941533d85bef951838d" - }, - { - "class": "standalone_plugin_implementation", - "path": "plugins/substrate/__init__.py", - "reason": "Hermes directory-plugin entry point for Substrate retrieval.", - "sha256": "df55769a899af189af4fbc78e72a5f17cc62cae27db51441b2f4bd4311bbc543" - }, - { - "class": "standalone_repository_policy_or_test", - "path": "plugins/substrate/after-install.md", - "reason": "Health-gated active-profile setup and cutover instructions.", - "sha256": "00970176f8081afeb43b8dba40fb68683bfe040d30424c6b6edae236d30496eb" - }, - { - "class": "standalone_plugin_implementation", - "path": "plugins/substrate/ca/isrg-root-x1.pem", - "reason": "Unmodified public ISRG Root X1 trust anchor.", - "sha256": "22b557a27055b33606b6559f37703928d3e4ad79f110b407d04986e1843543d1" - }, - { - "class": "standalone_plugin_implementation", - "path": "plugins/substrate/ca/isrg-root-x2.pem", - "reason": "Unmodified public ISRG Root X2 trust anchor.", - "sha256": "a13d881e11fe6df181b53841f9fa738a2d7ca9ae7be3d53c866f722b4242b013" - }, - { - "class": "standalone_plugin_implementation", - "path": "plugins/substrate/client.py", - "reason": "Standard-library Substrate retrieval API client and active-profile credential resolver.", - "sha256": "61ece2e3c21517d34341a1df13fb6782e844100c25c1020c679b62e28c22885b" - }, - { - "class": "standalone_plugin_implementation", - "path": "plugins/substrate/contract.py", - "reason": "Validated Substrate retrieval wire contract.", - "sha256": "f8dabf7d1e8f0c8ce5b6f1dc3529ef5d751b6b98dec1f59705392ff099e24346" - }, - { - "class": "standalone_repository_policy_or_test", - "path": "plugins/substrate/contract/envelope-fixtures.json", - "reason": "Shared retrieval wire-contract fixtures.", - "sha256": "627615398b726d04f32b5bab58b480b00ba85ca80c65d66864d7e8ea1a30ab85" - }, - { - "class": "standalone_plugin_implementation", - "path": "plugins/substrate/onboarding.py", - "reason": "Automatic RFC 8628 device onboarding for the Substrate retrieval plugin.", - "sha256": "5cb19fd1bcdac20ef61091329e20c47c381212bac07578c6da4d480583978a31" - }, - { - "class": "standalone_plugin_implementation", - "path": "plugins/substrate/plugin.py", - "reason": "Hermes hooks and tools for Substrate retrieval.", - "sha256": "7289109843f5325713e7a02e311078d4dd9ce1fc18d481124b343c50b952abdf" - }, - { - "class": "standalone_plugin_implementation", - "path": "plugins/substrate/plugin.yaml", - "reason": "Native Hermes manifest for Substrate retrieval.", - "sha256": "5c52cf65a93525d3569a0935ca56a6398ffd8f413af289b19bcfe5902a10a03b" - }, - { - "class": "standalone_plugin_implementation", - "path": "plugins/substrate/setup.py", - "reason": "Passwordless active-profile setup and authenticated preflight.", - "sha256": "488a0c51bf0ea534aa75c968416c40baec09f8dc8eef10fd9f112010b089ac4a" - }, - { - "class": "standalone_repository_policy_or_test", - "path": "pyproject.toml", - "reason": "Required only by the independent public repository.", - "sha256": "460595d6d635cedb7b7e16cfc00e8d054732749df8310452102d357fabf5fde6" - }, - { - "class": "standalone_repository_policy_or_test", - "path": "scripts/verify_fresh_migration_run.py", - "reason": "Required only by the independent public repository.", - "sha256": "e185b0328bb7d684a53a57ce1029d8f2a845a608395b087ecff36d372a3ef210" - }, - { - "class": "standalone_plugin_implementation", - "path": "src/substrate_wiki/credentials.py", - "reason": "Added for hosted Hermes 0.20 automatic onboarding.", - "sha256": "0f6845aaa8f87bf6cb9306b4614959070ea7a944e327e299a6d1f35e4c9c7e9a" - }, - { - "class": "standalone_plugin_implementation", - "path": "src/substrate_wiki/onboarding.py", - "reason": "Added for hosted Hermes 0.20 automatic onboarding.", - "sha256": "ac9934df2970093eb94d1fe5fb820c025d5f3fec2a32fdc5346e77ad9df66626" - }, - { - "class": "standalone_repository_policy_or_test", - "path": "tests/test_onboarding.py", - "reason": "Added for hosted Hermes 0.20 automatic onboarding.", - "sha256": "8be0b93434e1a011cb74ecbeee92ae070f25344af73cbdc2d162872695d57186" - }, - { - "class": "standalone_repository_policy_or_test", - "path": "tests/test_publication_scanner.py", - "reason": "Required only by the independent public repository.", - "sha256": "5cb25cc2c84f8aab3de8532141aee25eb0ca5ae3ce8594fc0fb146cf46979f50" - }, - { - "class": "standalone_repository_policy_or_test", - "path": "tests/test_redaction.py", - "reason": "Required only by the independent public repository.", - "sha256": "28c3cfb895bd3e4b556aa4580da7612609d8bc3df42f4132000a9af000008d5d" - }, - { - "class": "standalone_repository_policy_or_test", - "path": "tests/test_retrieval_contract.py", - "reason": "Contract tests for the independent Substrate retrieval plugin.", - "sha256": "554679f0272b59d2380d54d398ffb4f2cf23c4f1b5bf2052085f3c77e2d7e225" - }, - { - "class": "standalone_repository_policy_or_test", - "path": "tests/test_retrieval_onboarding.py", - "reason": "Onboarding automation tests for the independent Substrate retrieval plugin.", - "sha256": "87a4b15e34f809160b33d4c94362a94a82fa93bd46e2b6ba6aa68aea10ef07b0" - }, - { - "class": "standalone_repository_policy_or_test", - "path": "tests/test_retrieval_plugin.py", - "reason": "Behavior and transport tests for the independent Substrate retrieval plugin.", - "sha256": "206fa1b00515372ccd9827b1bd2ce46c1bb08bb3e53637104970c216e09d17ea" - }, - { - "class": "standalone_repository_policy_or_test", - "path": "tests/test_retrieval_session_capture.py", - "reason": "Updated by PR #33: live session capture and agent display names.", - "sha256": "674b25023225ba72bb34aff83b971f5f70c1ce9c0739463fd8077e7077e0b70e" - }, - { - "class": "standalone_repository_policy_or_test", - "path": "tests/test_retrieval_setup.py", - "reason": "Credential, TLS, and setup regression tests.", - "sha256": "109b1721f12223493aec3528703c7c65db1b58355efa6c06f04821ddb9c0fe33" - }, - { - "class": "standalone_repository_policy_or_test", - "path": "uv.lock", - "reason": "Required only by the independent public repository.", - "sha256": "52eadc5cea9450a6348a3daa5370468ffaab9012439e15a2a61dce60520fccb6" - } - ], - "destination_repository": "Substrate-memory/Substrate-memory-plugins", - "entries": [ - { - "class": "boundary", - "destination": "LICENSE", - "destination_sha256": "3b47755449b46445ad5724111b18db9d6b9e7c53395bca79c7cb3a0920ce1037", - "source": "hermes-plugin/LICENSE", - "source_sha256": "3b47755449b46445ad5724111b18db9d6b9e7c53395bca79c7cb3a0920ce1037", - "transformation": "copied" - }, - { - "class": "plugin-benchmark", - "destination": "benchmarks/evidence/hermes-migration-baseline.json", - "destination_sha256": "a4539746513fa2fafc3c818cdc06ec27b266a3ab198715289c656697b0dadf97", - "source": "benchmarks/evidence/hermes-migration-baseline.json", - "source_sha256": "e7988deac35d97638e154e3834c1eaf20ecef7ada48729227ed1dcf1f048df63", - "transformation": "modified_for_standalone" - }, - { - "class": "plugin-benchmark", - "destination": "benchmarks/hermes-migration-budget.json", - "destination_sha256": "aa8bc454dcecb6f8728a82de9fc5cf93415d4a071908f24d1fd11d4f0ddfc719", - "source": "benchmarks/hermes-migration-budget.json", - "source_sha256": "e503c387d33ce2c2edd1c33c09130b610a159aa914df604a069831334a3d23ea", - "transformation": "modified_for_standalone" - }, - { - "class": "plugin-benchmark", - "destination": "benchmarks/hermes-migration-manifest.json", - "destination_sha256": "d0508dd3586723f8d8ff81ee6ae8a2f4aca381905c45b3ceac42b2770cabc266", - "source": "benchmarks/hermes-migration-manifest.json", - "source_sha256": "d0508dd3586723f8d8ff81ee6ae8a2f4aca381905c45b3ceac42b2770cabc266", - "transformation": "copied" - }, - { - "class": "plugin-benchmark", - "destination": "benchmarks/hermes-migration-receipt.schema.json", - "destination_sha256": "a7dd0981c1e7497fe54ac892b3a9d79065efea93d30e9660fcd6cd27bd6ccaee", - "source": "benchmarks/hermes-migration-receipt.schema.json", - "source_sha256": "a7dd0981c1e7497fe54ac892b3a9d79065efea93d30e9660fcd6cd27bd6ccaee", - "transformation": "copied" - }, - { - "class": "boundary", - "destination": "docs/api-ownership.json", - "destination_sha256": "2fc837c1f89ef9948c9d542f4912f39414a9cd43eb650b1f73f9a2dc49b3ad2e", - "source": "docs/plugin/api-ownership.json", - "source_sha256": "4e5dba6249b26ce8b90a8c483125e6fd2c5e7cec94b3770be3e92a7aa04347dc", - "transformation": "modified_for_standalone" - }, - { - "class": "plugin-benchmark", - "destination": "docs/migration-baseline.md", - "destination_sha256": "680aafaf19059d3650209ee02cec9279fd7602b447997effc375551dcbb74bb6", - "source": "docs/hermes-migration-baseline.md", - "source_sha256": "ed1313dc1ef9a921334ee0d28e8d181b1f9ba28180b891b809af323573a88073", - "transformation": "modified_for_standalone" - }, - { - "class": "boundary", - "destination": "docs/public-boundary.json", - "destination_sha256": "b3c24d1ccff43c274be683020f23661d45faaeb16b842af384d828c6b7968390", - "source": "docs/plugin/public-boundary.json", - "source_sha256": "5f4b5a663dd6590480ed432b22ae68a88a0fd5ab4704b705f60617332edc75bc", - "transformation": "modified_for_standalone" - }, - { - "class": "boundary", - "destination": "docs/public-boundary.md", - "destination_sha256": "fd5e6965cda0decaf7ef539ec53e24bcbca4de16b591d029031402ef2217b62a", - "source": "docs/plugin/public-boundary.md", - "source_sha256": "917feee8c785f9ae14a474e9e948079f52084700b1dfff6fe479cc3627adc0c1", - "transformation": "modified_for_standalone" - }, - { - "class": "boundary", - "destination": "docs/publication-checklist.md", - "destination_sha256": "dee43af6c9d07badb032a215d1764e0a16ad97c8f7230267a96178f3b00aaf8b", - "source": "docs/plugin/no-secret-publication-checklist.md", - "source_sha256": "f0f640038017860bccc2bc19d165dd22179bf7138a51c0edcf5ea246da67cc46", - "transformation": "modified_for_standalone" - }, - { - "class": "immutable-legacy-assets", - "destination": "legacy-assets/1.2.0/install_hermes_plugin.py", - "destination_sha256": "bb9a8483d3d623528f573593eacebffa9483f52ecf84a452c01fa9c362b6879e", - "source": "hermes-plugin/releases/1.2.0/install_hermes_plugin.py", - "source_sha256": "bb9a8483d3d623528f573593eacebffa9483f52ecf84a452c01fa9c362b6879e", - "transformation": "copied" - }, - { - "class": "immutable-legacy-assets", - "destination": "legacy-assets/1.2.0/substrate_wiki.zip", - "destination_sha256": "2cbf504ec83352f23a1157777d24272b62e4b7300ad0ca991a0c4bc2e2df30b5", - "source": "hermes-plugin/releases/1.2.0/substrate_wiki.zip", - "source_sha256": "2cbf504ec83352f23a1157777d24272b62e4b7300ad0ca991a0c4bc2e2df30b5", - "transformation": "copied" - }, - { - "class": "immutable-legacy-assets", - "destination": "legacy-assets/1.3.0/install_hermes_plugin.py", - "destination_sha256": "59d4d0b8557a49ec18160f4245a533465f8c4c0eb344d235af155afb8845d1b1", - "source": "hermes-plugin/releases/1.3.0/install_hermes_plugin.py", - "source_sha256": "59d4d0b8557a49ec18160f4245a533465f8c4c0eb344d235af155afb8845d1b1", - "transformation": "copied" - }, - { - "class": "immutable-legacy-assets", - "destination": "legacy-assets/1.3.0/substrate_wiki.zip", - "destination_sha256": "6827c00444c799c085ac7a3669721d672c0d7e1e703a8a491397bb16d0655c02", - "source": "hermes-plugin/releases/1.3.0/substrate_wiki.zip", - "source_sha256": "6827c00444c799c085ac7a3669721d672c0d7e1e703a8a491397bb16d0655c02", - "transformation": "copied" - }, - { - "class": "immutable-legacy-assets", - "destination": "legacy-assets/1.4.0/install_hermes_plugin.py", - "destination_sha256": "13a05be49a83fab4c75171356d575dd85e00b27b8e09ce1602b87ae903741608", - "source": "hermes-plugin/releases/1.4.0/install_hermes_plugin.py", - "source_sha256": "13a05be49a83fab4c75171356d575dd85e00b27b8e09ce1602b87ae903741608", - "transformation": "copied" - }, - { - "class": "immutable-legacy-assets", - "destination": "legacy-assets/1.4.0/substrate_wiki.zip", - "destination_sha256": "df872d60dfc53668a0e6d30fd024e8d2f533306375980c3815ef1a483676c667", - "source": "hermes-plugin/releases/1.4.0/substrate_wiki.zip", - "source_sha256": "df872d60dfc53668a0e6d30fd024e8d2f533306375980c3815ef1a483676c667", - "transformation": "copied" - }, - { - "class": "immutable-legacy-assets", - "destination": "legacy-assets/1.4.1/install_hermes_plugin.py", - "destination_sha256": "7600b2681c3aebcb1b1492b0a04be38bbbec637089cbbcfb1cc26e8c10865b8d", - "source": "hermes-plugin/releases/1.4.1/install_hermes_plugin.py", - "source_sha256": "7600b2681c3aebcb1b1492b0a04be38bbbec637089cbbcfb1cc26e8c10865b8d", - "transformation": "copied" - }, - { - "class": "immutable-legacy-assets", - "destination": "legacy-assets/1.4.1/substrate_wiki.zip", - "destination_sha256": "877ccf9b0212792b699d9c98912a26980675a6050df3bd319e927639e3d901f1", - "source": "hermes-plugin/releases/1.4.1/substrate_wiki.zip", - "source_sha256": "877ccf9b0212792b699d9c98912a26980675a6050df3bd319e927639e3d901f1", - "transformation": "copied" - }, - { - "class": "build-and-install", - "destination": "scripts/benchmark_import_memory.py", - "destination_sha256": "2e91630f2833eae081d9ad0b8fedaf3906233d37828f5d1f85a413267a0ce441", - "source": "scripts/benchmark_hermes_import_memory.py", - "source_sha256": "09572eb1fa8c75016a293a82b77d0c75f238aa021a088d4c2b30f8bce97b126a", - "transformation": "modified_for_standalone" - }, - { - "class": "plugin-benchmark", - "destination": "scripts/benchmark_migration.py", - "destination_sha256": "ef57926174b22c2c22e73843a08a5ea9b7b45481bf51ef08dac21129ea9c929a", - "source": "scripts/benchmark_hermes_migration.py", - "source_sha256": "176a628acb33562afae1a99b30cc5f55793f40d8f1fa7a6ec83e8c84180a315b", - "transformation": "modified_for_standalone" - }, - { - "class": "build-and-install", - "destination": "scripts/build_plugin.py", - "destination_sha256": "80c9d3d926e485202fe0df66e89713134f4b701622b42128a458a7fb20db80c1", - "source": "scripts/build_hermes_plugin.py", - "source_sha256": "8c81741239dcfd78afeb1a66493552bb5febbdb6b3984c143b2bdb3a2dd227a4", - "transformation": "modified_for_standalone" - }, - { - "class": "build-and-install", - "destination": "scripts/install_hermes_plugin.py", - "destination_sha256": "9f157d5e7e3b8921a01d46f975f574a8955a53c5f554ed61b306a7bdfddd7309", - "source": "scripts/install_hermes_plugin.py", - "source_sha256": "7600b2681c3aebcb1b1492b0a04be38bbbec637089cbbcfb1cc26e8c10865b8d", - "transformation": "modified_for_standalone" - }, - { - "class": "plugin-benchmark", - "destination": "scripts/verify_migration_baseline.py", - "destination_sha256": "88ec8929ae28e5387ab0071f1cea78a9789f4efd27d409df187b69c76396f376", - "source": "scripts/verify_hermes_migration_baseline.py", - "source_sha256": "34545218b12681b86c336f3a64f66fef3844258a821ef657d5ab8cb4779e2934", - "transformation": "modified_for_standalone" - }, - { - "class": "build-and-install", - "destination": "scripts/verify_public_plugin_candidate.py", - "destination_sha256": "5f53e191cab7045b83e38645ca81a24cf04f0610f84f155126b2d17e89f9cc6e", - "source": "scripts/verify_public_plugin_candidate.py", - "source_sha256": "4130935d530075fce1758e2e89bd5d973a722e2293b5b1058cfe0d17f326172b", - "transformation": "modified_for_standalone" - }, - { - "class": "plugin-package", - "destination": "src/substrate_wiki/README.md", - "destination_sha256": "72195739860b3d64338cff133e31a1cdaefe6cd6e09bcebc9d8c5e1f27e987b3", - "source": "hermes-plugin/substrate_wiki/README.md", - "source_sha256": "21b6e60ef34e408fd87570334ba6d2811d2a9ea3c95d512e4d625f958b1cb0dc", - "transformation": "modified_for_standalone" - }, - { - "class": "plugin-package", - "destination": "src/substrate_wiki/__init__.py", - "destination_sha256": "71c0c19d366759e67c7b999029777295cc8c25503d35fe9cfddc9460bcb34d5a", - "source": "hermes-plugin/substrate_wiki/__init__.py", - "source_sha256": "a4143022e05a7b93d3aa5799f4319601292e67069ad85d172770c3effe0e5e9d", - "transformation": "copied" - }, - { - "class": "plugin-package", - "destination": "src/substrate_wiki/checkpoint.py", - "destination_sha256": "9b17861d9d775e4136ff4eeea7614a1a437d69ed896da6cad8dfa8dbfc82e165", - "source": "hermes-plugin/substrate_wiki/checkpoint.py", - "source_sha256": "9b17861d9d775e4136ff4eeea7614a1a437d69ed896da6cad8dfa8dbfc82e165", - "transformation": "copied" - }, - { - "class": "plugin-package", - "destination": "src/substrate_wiki/cli.py", - "destination_sha256": "6214bed9894c7195f22144701dc468b717faafddb5b79e1227445d3a8033fe4a", - "source": "hermes-plugin/substrate_wiki/cli.py", - "source_sha256": "454bffeb76fcde508b7a16cc7597b444611ecb0028e8d43796c57c500cf75d3c", - "transformation": "modified_for_standalone" - }, - { - "class": "plugin-package", - "destination": "src/substrate_wiki/client.py", - "destination_sha256": "dd82bf56371538a8c6fe8eb39787c5a0fb4fe1232e11fcb29b2d4ac11f0bd9ed", - "source": "hermes-plugin/substrate_wiki/client.py", - "source_sha256": "8684b3ecc788ee058c630747b1c72578177cfc948c72ac539ddcc4a1df9e791b", - "transformation": "modified_for_standalone" - }, - { - "class": "plugin-package", - "destination": "src/substrate_wiki/events.py", - "destination_sha256": "f70c743866c88e3aa9add5746ae558adab175175c31918109fb3ea90dd52c662", - "source": "hermes-plugin/substrate_wiki/events.py", - "source_sha256": "675ba567a391f6b86688876ae26d1cfd9b567fe3312d9ddc03620775596c88e9", - "transformation": "copied" - }, - { - "class": "plugin-package", - "destination": "src/substrate_wiki/history.py", - "destination_sha256": "aa4216c382ecd6816aa6b5633e7143a71fb8af66a218a0fafba9837a6d6da28d", - "source": "hermes-plugin/substrate_wiki/history.py", - "source_sha256": "d758d1b89dad334fc8c352b4e30f6e8e3c0788821c82ecc11bc7e7fcefdef16c", - "transformation": "copied" - }, - { - "class": "plugin-package", - "destination": "src/substrate_wiki/plugin.yaml", - "destination_sha256": "124f058b164fe7c5beeb7ffc0c066e6f4530aa361dc0988f753267202776f083", - "source": "hermes-plugin/substrate_wiki/plugin.yaml", - "source_sha256": "5bfc8b20bfe99b8dd4fcb35616724e9a4a93bdfd1f3b270cfea1ccff18eaa116", - "transformation": "modified_for_standalone" - }, - { - "class": "plugin-package", - "destination": "src/substrate_wiki/py.typed", - "destination_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", - "source": "hermes-plugin/substrate_wiki/py.typed", - "source_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", - "transformation": "copied" - }, - { - "class": "plugin-package", - "destination": "src/substrate_wiki/redaction.py", - "destination_sha256": "e9bec198aa7ad018da359d2e9aa6df1dab717881bb41b1001348911b23e6439b", - "source": "hermes-plugin/substrate_wiki/redaction.py", - "source_sha256": "6daef16e8ac7150f6f16a88bdd6b5d3e8ff20b46902e0bec8f4913529ee92476", - "transformation": "modified_for_standalone" - }, - { - "class": "plugin-package", - "destination": "src/substrate_wiki/spool.py", - "destination_sha256": "b3c31c6f124d0f57d23c11e9a0b76626921ef0d48846cfbdec59ac117ae402db", - "source": "hermes-plugin/substrate_wiki/spool.py", - "source_sha256": "b3c31c6f124d0f57d23c11e9a0b76626921ef0d48846cfbdec59ac117ae402db", - "transformation": "copied" - }, - { - "class": "plugin-package", - "destination": "src/substrate_wiki/supervisor.py", - "destination_sha256": "0b083c2aaadfd7a3525657861fcfcaf081debaf8fa19bd713209ed72f0be6510", - "source": "hermes-plugin/substrate_wiki/supervisor.py", - "source_sha256": "7a621bc474a1f7337e2998f398b04d079270f46ba1aec4863fefdc4075ba7fb8", - "transformation": "copied" - }, - { - "class": "plugin-package", - "destination": "src/substrate_wiki/worker.py", - "destination_sha256": "4e2759852e4808b84d810c5503d66c0f6c1c85406c47ad9d2f4db1957c7eefd2", - "source": "hermes-plugin/substrate_wiki/worker.py", - "source_sha256": "85abc4f4a467a8ab677b58895a865fd7a9771d4eefa07258e72ca25243d93c08", - "transformation": "copied" - }, - { - "class": "plugin-tests", - "destination": "tests/fixtures/credential_redaction_vectors.json", - "destination_sha256": "0cf55fa5cf91acdc164f2eb6936eb49af19ed9c432d060c475db4a9090cd169b", - "source": "tests/fixtures/credential_redaction_vectors.json", - "source_sha256": "0cf55fa5cf91acdc164f2eb6936eb49af19ed9c432d060c475db4a9090cd169b", - "transformation": "copied" - }, - { - "class": "plugin-tests", - "destination": "tests/fixtures/public-plugin-secret-sentinels.json", - "destination_sha256": "7dc34d5d5234f10d3f9fbe6a71a2f65723472570ad3ebfef3710707ddc824e50", - "source": "tests/fixtures/public-plugin-secret-sentinels.json", - "source_sha256": "7dc34d5d5234f10d3f9fbe6a71a2f65723472570ad3ebfef3710707ddc824e50", - "transformation": "copied" - }, - { - "class": "plugin-tests", - "destination": "tests/test_entity_memory.py", - "destination_sha256": "e9d869d2389bdfd84e7912586284ff318f65b8841ade19c0c9bd5a03a1a4639d", - "source": "tests/contract/test_hermes_entity_memory_v13.py", - "source_sha256": "297d7dffa2a53dd08cb1737ac024a97eb190c4ce42b338a952d8bec9aeba37f4", - "transformation": "modified_for_standalone" - }, - { - "class": "plugin-tests", - "destination": "tests/test_hardening.py", - "destination_sha256": "bb9825c1889d919e29a90e43c2be48b84d846fcad813fa3896d194491cc8f386", - "source": "tests/contract/test_hermes_plugin_hardening.py", - "source_sha256": "6066ff88351167647dc6ffd368fcb2ad666ee722fcd16d2fc04600f274151b17", - "transformation": "modified_for_standalone" - }, - { - "class": "plugin-tests", - "destination": "tests/test_history.py", - "destination_sha256": "15a6e7778a92e0d826ba1af61032f222dbf926b5ec2e189c48d7985a70824f19", - "source": "tests/contract/test_hermes_history_v12.py", - "source_sha256": "dc5cf0ceca8b8c0156bba330f51cc2186fdd2c409861752a7465f1b625228d69", - "transformation": "modified_for_standalone" - }, - { - "class": "plugin-tests", - "destination": "tests/test_history_replay.py", - "destination_sha256": "226d30b1ad98ba7603bf3d8f7c86ac0857bf596bb75453d0dd57d677ec1405bd", - "source": "tests/contract/test_hermes_history_replay.py", - "source_sha256": "ef7636f612cc2d1023856fb8ff39dd7de2cca13336d20b86db5d25d944a3a212", - "transformation": "modified_for_standalone" - }, - { - "class": "plugin-tests", - "destination": "tests/test_import_memory.py", - "destination_sha256": "d59c7307d853bcc3491cc22019db0a3123639d4d8c2cd6c37440bee7a121e7d2", - "source": "tests/contract/test_hermes_import_memory.py", - "source_sha256": "33ead031eecd76965c7ca08dcfb8e89b82fe5c0787f53276ca6135ff2fddf74b", - "transformation": "modified_for_standalone" - }, - { - "class": "plugin-tests", - "destination": "tests/test_memory_provider.py", - "destination_sha256": "cc967199b8e877a8088937a0c951e80f188ad4c7f29a98067bf649a2f8500b72", - "source": "tests/contract/test_hermes_memory_provider.py", - "source_sha256": "d95c3c68592e7c45d4eab3c96c8a23726e8ba54116b205b33f78c66239a4ba6e", - "transformation": "modified_for_standalone" - }, - { - "class": "plugin-benchmark", - "destination": "tests/test_migration_baseline.py", - "destination_sha256": "0a3c7af6a761a1b22c5b94f7e1738d5d420e5f085f775d792728ed9ca684ac13", - "source": "tests/contract/test_hermes_migration_baseline.py", - "source_sha256": "1cd65a8d3f6e1355275389528628c8af209345e6757536510a3640698f53940a", - "transformation": "modified_for_standalone" - }, - { - "class": "plugin-tests", - "destination": "tests/test_packaging.py", - "destination_sha256": "96f2d25b7fa3159d31cb6274cc39ae279909a2e90a694f19779a795b86076243", - "source": "tests/contract/test_hermes_plugin_packaging.py", - "source_sha256": "c54967830788555b8d7bd5d2f871baeb6b98d7804a2bc7410ab297078bc20ee7", - "transformation": "modified_for_standalone" - } - ], - "inventory_contract": "Every tracked destination byte is hash-bound except this self-referential manifest.", - "never_move": [ - "application and server implementation", - "server tests except published contract fixtures", - "infrastructure and deployment authority", - "production configuration, credentials, data, or evidence", - "private operational or customer records" - ], - "schema_version": 2, - "self_excluded_path": "docs/extraction-manifest.json", - "source_commit": "39e7bd8f650401c4a53004dc34e06c4b47e77c28", - "source_repository": "https://github.com/Substrate-memory/Substrate-v2" -} diff --git a/docs/migration-baseline.md b/docs/migration-baseline.md deleted file mode 100644 index e619c56..0000000 --- a/docs/migration-baseline.md +++ /dev/null @@ -1,140 +0,0 @@ -# Hermes migration baseline and closed-beta budget (SUB-75) - -This baseline measures the unchanged `stream-v2` transfer path and a deterministic, -content-free downstream model. It is evidence for later protocol and tuning work; it -is **not** a protocol change, a production tuning claim, or hosted-provider evidence. - -## Reproduce - -Python 3.12 and the repository's frozen development environment are required. - -```bash -uv sync --frozen --extra dev -uv run --frozen --extra dev python scripts/benchmark_migration.py \ - --manifest benchmarks/hermes-migration-manifest.json \ - --profile ci \ - --output benchmarks/evidence/hermes-migration-baseline.json -uv run --frozen --extra dev python scripts/verify_migration_baseline.py --write-budget -uv run --frozen --extra dev python scripts/verify_migration_baseline.py -uv run --frozen --extra dev pytest -q tests/test_migration_baseline.py -``` - -The `contract` profile is the bounded CI canary and runs through the canonical -`import-memory` pytest shard. The `ci` and `release-candidate` profiles currently -select all six deterministic fixture classes and are suitable for an independently -provisioned preview/reviewer host. No command opens a hosted connection. - -Canonical inputs and retained evidence: - -- workload: `benchmarks/hermes-migration-manifest.json`; -- receipt contract: `benchmarks/hermes-migration-receipt.schema.json`; -- retained canonical aggregate: `benchmarks/evidence/hermes-migration-baseline.json`; -- measured budget: `benchmarks/hermes-migration-budget.json`. - -Fixture generation runs in a child process before each measured run. SQLite and -JSONL fixtures are deterministic for a manifest revision. The official-export case -uses the bounded JSONL shape produced by the supported export adapter; it does not -invoke a local Hermes runtime export API. The current-production case fixes the -observed retained-window cardinality (1,212), while the large case fixes 2,048 -windows. The oversized case is 2 MiB and therefore exercises stream-v2 fragmentation -above the 262,144-byte event ceiling. - -Each receipt binds the exact manifest and benchmark implementation with SHA-256 -digests. The verifier recomputes both before accepting schema, matrix, or budget -evidence; the budget then binds the canonical receipt bytes. - -## What each measurement means - -| Field | Boundary / method | -| --- | --- | -| `discovery` | Production SQLite/JSONL source discovery and content-free checkpoint inventory. | -| `source_reading` | Time spent advancing the production bounded source iterator. | -| `normalization_redaction` | From source yield to sink entry: production event construction, normalization, fragmentation and redaction. | -| `request_encoding` | Deterministic JSON wire serialization at the local sink boundary. | -| `network_wait` | Fixed local sleep only; explicitly simulated and opens no network connection. | -| `server_persistence` | Durable content-free SQLite acknowledgement commit. | -| `queue_delay` | Enqueue-to-worker-start delay; summed worker time may exceed wall time. | -| `provider_extraction` | Fixed deterministic provider double latency, including configured synthetic retries. | -| `entity_resolution` | Content-free deterministic digest work standing in for resolution. | -| `projection_indexing` | Durable content-free SQLite projection commit. | -| `summary_reduction` | Periodic and terminal deterministic reduction digest. | - -`transferred` is reached only after the production importer has received durable -acknowledgements for every event. `first_usable` is the first completed synthetic -projection and may precede transfer completion because local workers consume the -queue concurrently. `fully_ready` is reached only after transfer and every queued -projection completes. Validation requires both earlier terminals to be strictly before -`fully_ready` and requires `transferred` and `first_usable` to remain distinct; their -relative ordering is measured rather than assumed. These labels are never aliases. - -Rates use aggregate wall boundaries: events/s and MiB/s use `transferred`; windows/minute -uses `fully_ready`. CPU is process CPU time. Peak RSS is the maximum current RSS sampled -from `/proc/self/status` (with `/proc/self/statm` fallback) every 5 ms inside a fresh -per-case worker, excluding fixture generation and inherited `ru_maxrss`. Phase values are aggregate phase time, not a -partition of wall time; concurrent phase sums and queue age sums can exceed wall time. - -## Provider model and limits of the evidence - -The representative provider mode is an honest deterministic simulation: 5 ms of -observed local latency per extraction request, one retry opportunity every 257th -request, a modeled quota of 60 requests/minute, and modeled cost of $0.00025/request. -The receipt records both actually slept provider time and quota-equivalent time. It -never claims the accelerated local sleep is provider wall time. - -The harness installs socket-construction denial in the orchestrator, every fixture child, -and every measured worker. A network attempt therefore fails the run rather than silently changing the -`hosted_calls: 0` claim. `worker_threads_used` and `max_in_flight` are measured at the -processing seam; the verifier requires retained evidence to demonstrate each configured -concurrency from one through four. - -The harness uses the production importer, checkpoint, redaction, fragmentation, -SQLite reader and JSONL reader. Server persistence and all phases after persistence -are benchmark doubles. It therefore establishes a reproducible bottleneck baseline -and target budget, but does not predict hosted-provider variance, production disk -latency, model quality, Azure worker contention, or protocol speedups. No prompts, -transcripts, source payloads, credentials, event identifiers, or memory material are -written to traces or receipts; only aggregate counts, timings, sizes and synthetic -fixture digests are retained. - -## Measured result and closed-beta budget - -The retained receipt is the numeric source of truth. The budget file is derived from -that receipt with these explicit policies: - -- integrity ratio 1.0, zero redaction/projection/terminal failures; -- one measured duplicate replay acknowledgement and zero duplicate side effects per run; -- peak RSS remains below the unchanged 256 MiB importer ceiling; -- per-case transfer, first-usable, fully-ready and CPU ceilings use the worst measured - concurrency plus a 50% tolerance and a 2-second host-jitter allowance; -- provider quota-equivalent seconds, deterministic retry count and cost are fixed to - the workload's measured/modelled maximum, not traded for speed; -- no quality or integrity constraint may be relaxed to satisfy a latency ceiling. - -Numeric findings and the dominant phases are summarized below after the retained -receipt generated on the candidate host; reviewers should recompute them directly -from the JSON rather than relying only on rounded prose. - - - -| Case | Transfer max (s) | First usable max (s) | Fully ready max (s) | Peak RSS max (MiB) | CPU max (s) | Integrity | Dominant aggregate phase | -| --- | ---: | ---: | ---: | ---: | ---: | ---: | --- | -| `current-production-jsonl` | 52.509 | 0.275 | 52.511 | 36.5 | 3.007 | 4/4 | `network_wait` | -| `large-sqlite` | 69.322 | 0.448 | 69.324 | 39.2 | 5.316 | 4/4 | `network_wait` | -| `official-export-jsonl` | 2.307 | 0.359 | 2.311 | 32.7 | 0.289 | 4/4 | `network_wait` | -| `oversized-message-jsonl` | 2.457 | 2.455 | 2.458 | 36.8 | 1.867 | 4/4 | `source_reading` | -| `small-sqlite` | 0.343 | 0.200 | 0.346 | 32.6 | 0.028 | 4/4 | `network_wait` | -| `sqlite-adapter` | 2.322 | 0.752 | 2.325 | 33.2 | 0.172 | 4/4 | `network_wait` | - -All 24 runs completed with `hosted_calls: 0`, exactly one duplicate acknowledgement, -zero duplicate side effects, zero redaction/projection/terminal failures, and measured -worker concurrency matching each configured level from one through four. - -## Implications for SUB-76 and SUB-77 - -The baseline keeps protocol and tuning deliberately unchanged. A follow-up must -preserve all receipt integrity, quality, redaction, RSS, CPU, quota and cost gates, -then compare the three lifecycle boundaries independently. Transfer work belongs in -SUB-76 only if source/normalization/encoding/network/persistence evidence dominates. -Bounded worker/backpressure work belongs in SUB-77 only if queue/provider/resolution/ -projection/reduction evidence dominates. A faster `transferred` result cannot be -reported as faster `first_usable` or `fully_ready` without separately measured proof. diff --git a/docs/operation.md b/docs/operation.md deleted file mode 100644 index 973ea1e..0000000 --- a/docs/operation.md +++ /dev/null @@ -1,44 +0,0 @@ -# Operation - -## Connection and credentials - -The plugin connects only to `https://app.trysubstrate.co`. Installation and Hermes' native -`post_setup` hook start device onboarding automatically. Desktop browsers open the hosted -verification page; headless users receive the same URL and one-time code. - -Do not configure `HERMES_API_URL` or `HERMES_API_KEY`. The issued tenant-scoped credential -is revocable and stored by native credential custody where available, with an owner-private -profile file fallback. It never belongs in plugin config, logs, or arguments. Optional -non-secret tuning lives at `$HERMES_HOME/substrate_wiki/config.json`. - -Use `hermes substrate_wiki onboarding-status --json` to inspect content-free connection and -consent state. Expired or revoked credentials trigger automatic reconnect attempts. - -## Optional history import - -The consent prompt is only for past history; future capture is enabled whether history is -approved or declined. Approval creates or attaches to one durable import: - -```bash -hermes substrate_wiki import-status --json -hermes substrate_wiki import-resume --job-id --yes --wait --json -``` - -The worker is cross-platform and profile-scoped. It resumes the same checkpoint after -process interruption, crash, or reboot. Before upgrading during an active import, record the -content-free job and batch identifiers and resume that job afterward rather than creating a -replacement. - -## Tools - -- `wiki_search` — lexical search over permitted published pages. -- `wiki_read` — read a returned canonical page path. -- `wiki_query` — cited synthesis, optionally saved by the hosted service. -- `wiki_ingest` — submit text or a public URL to hosted ingestion. -- `wiki_job_status` — inspect asynchronous ingestion. - -## Recovery - -The installer activates the provider atomically and retains a rollback directory on upgrade. -Automatic retries use bounded backoff. Rolling back plugin code does not delete spool items, -checkpoints, credential custody, or non-secret configuration. diff --git a/docs/public-boundary.json b/docs/public-boundary.json deleted file mode 100644 index df60f91..0000000 --- a/docs/public-boundary.json +++ /dev/null @@ -1,169 +0,0 @@ -{ - "compatibility": { - "axes": [ - "Hermes host", - "Substrate server capability contract", - "plugin semver" - ], - "breaking_change_rule": "new schema or protocol identifier", - "candidate_hermes_versions": [ - "0.20.x" - ], - "candidate_requires": [ - "user plugin discovery", - "provider lifecycle", - "post_setup", - "tool schemas", - "profile isolation", - "portable import supervision", - "hosted device onboarding" - ], - "certified_hermes_versions": [], - "contract_tested_hermes_versions": [ - "0.20.x" - ] - }, - "data": { - "bounded_private_spool": true, - "content_free_status_and_errors": true, - "raw_history_stays_local_until_explicit_capture_or_replay": true, - "retention_and_deletion_must_be_explicit": true, - "server_receives_only_versioned_redacted_events": true - }, - "deprecation": { - "breaking_changes_require_new_contract_version": true, - "no_silent_auto_update": true, - "old_protocol_remains_supported_during_migration": false, - "rollback_preserves_configuration_and_state": true - }, - "legal": { - "attribution_reviewed": true, - "bundled_third_party_attribution": [], - "copyright_owner": "Sightline Technologies Inc", - "license": "MIT", - "license_file": "LICENSE", - "license_scope": "independently released substrate_wiki plugin", - "maintainer_approved": true, - "publication_allowed": true, - "publication_authority": "Pavel instruction, 2026-08-03", - "status": "published" - }, - "non_goals": [ - "local or self-hosted Substrate operation", - "publishing the private server", - "moving server authority into the plugin", - "granting production access or copying production credentials" - ], - "ownership": { - "meeting_point": "immutable plugin releases plus a versioned HTTP capability contract", - "plugin_repository": [ - "plugin package and Hermes lifecycle integration", - "local spool and checkpoints", - "client-side redaction", - "plugin build, installer, packaging, tests, and user documentation" - ], - "server_must_not_import_plugin_python": true, - "server_repository": [ - "HTTP endpoints and authentication", - "idempotency and request limits", - "server persistence, queues, deletion, triage, projection, indexing, and search", - "producer integration tests and deployment" - ] - }, - "preserved_identity": { - "archive": "substrate_wiki.zip", - "credential_custody": "profile-scoped native vault or owner-private fallback", - "hosted_origin": "https://app.trysubstrate.co", - "installer": "install_hermes_plugin.py", - "package": "substrate_wiki", - "plugin_directory": "$HERMES_HOME/plugins/substrate_wiki", - "provider": "substrate_wiki", - "provider_config": "memory.provider", - "state_directory": "$HERMES_HOME/substrate_wiki" - }, - "repository": { - "candidate_source_of_truth": false, - "name": "hermes-substrate-wiki", - "organization": "Substrate-memory", - "source_of_truth": true, - "url": "https://github.com/Substrate-memory/Substrate-memory-plugins", - "visibility": "public" - }, - "schema_version": 1, - "secrets": { - "allowed_names": [], - "production_credentials_may_be_copied": false, - "source": "hosted device onboarding credential custody", - "synthetic_sentinels_in_exact_adversarial_fixtures": [ - "tests/fixtures/public-plugin-secret-sentinels.json" - ], - "values_in_logs_errors_or_receipts": false, - "values_in_source_or_artifacts": false - }, - "threat_model": [ - { - "controls": [ - "profile-scoped native credential vault with owner-private fallback", - "no secret values in config, source, package, logs, arguments, errors, tests, or receipts", - "redaction and secret scanning before release" - ], - "id": "credential-disclosure" - }, - { - "controls": [ - "explicit capture or replay only", - "client redaction before network transfer", - "content-free status, logs, and receipts" - ], - "id": "private-history-disclosure" - }, - { - "controls": [ - "private user-only storage", - "bounded bytes and age", - "explicit retention and deletion behavior" - ], - "id": "spool-exposure" - }, - { - "controls": [ - "fixed https://app.trysubstrate.co origin", - "unsafe override and redirect rejection", - "capability and provider identity verification" - ], - "id": "server-impersonation-or-redirect" - }, - { - "controls": [ - "deterministic event identifiers", - "idempotency keys", - "durable acknowledgement checkpoints and exact resume" - ], - "id": "replay-duplication-or-loss" - }, - { - "controls": [ - "event and request byte limits", - "bounded spool and retries", - "backpressure and cancellation" - ], - "id": "resource-exhaustion" - }, - { - "controls": [ - "pinned dependencies", - "deterministic allowlisted archives", - "checksums, provenance, and secret scan" - ], - "id": "dependency-or-release-tampering" - }, - { - "controls": [ - "state and credential slots rooted in the active Hermes profile", - "no shared writable profile state", - "fixed hosted server identity and lifecycle tests" - ], - "id": "cross-profile-or-cross-environment-leakage" - } - ] -} diff --git a/docs/public-boundary.md b/docs/public-boundary.md deleted file mode 100644 index 1a50bc9..0000000 --- a/docs/public-boundary.md +++ /dev/null @@ -1,55 +0,0 @@ -# Public plugin boundary - -This repository publishes the current Hermes 0.21.x `substrate` plugin in -[`plugins/substrate`](../plugins/substrate) and retains the MIT-licensed legacy -`substrate_wiki` source below. Public/release status becomes factual only after GitHub -readback. - -## Identity (current substrate plugin) - -- package/plugin: `substrate` in `plugins/substrate`; -- Hermes host: 0.21.x with native plugin discovery; -- hosted origin: the configured Substrate API origin (default `https://vm-substrate-ar-01.taile961d2.ts.net:10000`); -- credential source: in-plugin RFC 8628 device onboarding storing `SUBSTRATE_API_KEY` in the active profile's `.env`; -- profile state: `$HERMES_HOME/substrate`; -- user-plugin directory: `$HERMES_HOME/plugins/substrate`. - -## Identity (legacy substrate_wiki) - -- package/provider: `substrate_wiki`; -- Hermes configuration: `memory.provider: substrate_wiki`; -- hosted origin: `https://app.trysubstrate.co`; -- credential source: hosted device onboarding and profile-scoped secure custody; -- profile state: `$HERMES_HOME/substrate_wiki`; -- user-plugin directory: `$HERMES_HOME/plugins/substrate_wiki`; -- assets: `substrate_wiki.zip`, `install_hermes_plugin.py`. - -## Ownership - -This repository owns Hermes lifecycle integration, transport, credential custody, local -spool/checkpoints, redaction, history consent/replay, build/install tooling, tests, -documentation, and releases. `Substrate-v2` owns hosted OAuth/account routes, tenant -credentials, content APIs, idempotency, persistence, queues, deletion, projection, -indexing/search, infrastructure, and deployment. Neither repository imports the other's -runtime source. - -## Compatibility and data boundary - -Hermes 0.21.x with the current `substrate` plugin in -[`plugins/substrate`](../plugins/substrate) is the current install target; the legacy -`substrate_wiki` rows below document the retained 0.20.x boundary. The plugin connects only -to the configured Substrate origin and fails closed on origin or capability mismatch. -Credentials may not enter source, artifacts, normal config, logs, errors, receipts, or -support material. - -Future capture begins after account connection. Past direct conversations and explicit saved -memories remain local until the user approves historical upload. Group sessions, -cron/webhooks, hidden reasoning, unrelated files, secrets, and binary bodies are excluded. -Status and receipts remain content-free. - -## Legal - -- license: MIT; -- copyright: Sightline Technologies Inc; -- scope: this plugin repository only; -- publication: authorized; public only after GitHub readback. diff --git a/docs/publication-checklist.md b/docs/publication-checklist.md deleted file mode 100644 index 046086c..0000000 --- a/docs/publication-checklist.md +++ /dev/null @@ -1,36 +0,0 @@ -# Publication checklist - -Run against the exact candidate and release assets. - -## Source boundary - -- [ ] Only plugin-owned source, tooling, tests, docs, and immutable imported assets are present. -- [ ] No Substrate server implementation, infrastructure, production configuration, evidence, or customer records. -- [ ] Runtime imports only the Python standard library and Hermes host interfaces. -- [ ] `Substrate-v2` does not remain an editable plugin source. - -## Secrets and privacy - -- [ ] Publication scanner passes with content-free output. -- [ ] GitHub secret scanning is enabled. -- [ ] No real credentials, private endpoints, history, spool contents, or production evidence. -- [ ] Synthetic credential fixtures are clearly non-routable. -- [ ] README and security policy explain capture and retention boundaries. - -## Compatibility and behavior - -- [ ] Provider, entity-memory, replay, hardening, redaction, packaging, and importer tests pass. -- [ ] Hermes compatibility claim names an exact version. -- [ ] Server capability requirements and fail-closed behavior are documented. -- [ ] Rollback preserves configuration, spool, and checkpoints. - -## Artifact custody - -- [ ] Version is not reused. -- [ ] Source is committed before building. -- [ ] Archive provenance names the exact source commit and every file digest. -- [ ] Archive and installer checksums are verified independently. -- [ ] Adversarial review covers source and exact artifacts. -- [ ] GitHub release assets are read back and match local bytes. - -Version 1.4.1 is an imported immutable exception: its source provenance correctly points to the historical Substrate-v2 commit. It must never be rebuilt or replaced. diff --git a/docs/releasing.md b/docs/releasing.md index 92c08a1..0f5c1fd 100644 --- a/docs/releasing.md +++ b/docs/releasing.md @@ -1,22 +1,13 @@ # Releasing -1. Choose a new semantic version. Never reuse an existing tag or immutable asset path. -2. Update `src/substrate_wiki/plugin.yaml`, installer `EXPECTED_VERSION`, client version, README, compatibility matrix, and changelog. -3. Run all tests, compile checks, publication scanner, and bounded importer benchmark. -4. Commit the release-clean source. -5. Build with exact provenance: - - ```bash - HERMES_PLUGIN_SOURCE_COMMIT="$(git rev-parse HEAD)" python scripts/build_plugin.py - python scripts/build_plugin.py --check - ``` - -6. Verify archive and installer SHA-256 values independently. -7. Run an adversarial review against the exact commit and artifacts. -8. Tag the reviewed commit and publish the exact generated archive and installer once. -9. Read the GitHub release assets back; verify byte hashes and provenance. -10. Update downstream pinned-release references in Substrate-v2 through a separate PR. - -A release must not contain credentials, private history, private server endpoints, production evidence, or server implementation. The runtime source allowlist is closed by the builder. - -Version 1.4.1 is an imported immutable release. Its provenance points to its original Substrate-v2 commit; do not rebuild it from this repository. +Releases are immutable Git tags cut from protected `main` through the Release workflow. + +1. Land the release contents on `main` through a reviewed pull request. Every commit + must carry a DCO sign-off and pass all CI checks. +2. Dispatch the **Release** workflow with `candidate_sha` set to the exact reviewed + `main` commit. +3. The workflow re-verifies the candidate (hygiene scan, tests, deterministic + double-build of `substrate.zip`) and publishes tag `v` with + `substrate.zip` and `SHA256SUMS` assets plus build attestation. +4. Never mutate a published tag or its assets. Fixes ship as a new reviewed commit + and a new tag. diff --git a/docs/source-of-truth.md b/docs/source-of-truth.md index f1f6cba..9e83555 100644 --- a/docs/source-of-truth.md +++ b/docs/source-of-truth.md @@ -2,37 +2,19 @@ ## Decision -`Substrate-memory/Substrate-memory-plugins` is the sole editable source for the Hermes `substrate_wiki` plugin. The protected default branch and immutable releases `v1.5.0`, `v2.0.0`, `v2.0.1`, `v2.0.3`, and `v2.0.4` have been read back successfully. +`Substrate-memory/Substrate-memory-plugins` is the sole editable source for the Hermes +`substrate` retrieval plugin. The protected default branch and immutable `v0.3.0` release +(tag plus attested `substrate.zip` and `SHA256SUMS`) are the only published artifacts. -Substrate-v2 owns only the server and pinned public release references; it must not vendor or modify plugin source. +The server repository owns only the Substrate API, persistence, and deployment; it must +not vendor or modify plugin source. ## Ownership ### This repository -- Hermes provider lifecycle and tool registration. +- Hermes hook and tool registration. - Client transport and response validation. -- Client-side redaction. -- Profile-local spool and checkpoints. -- Durable history replay and import service. -- Build, installer, tests, documentation, and releases. - -### Substrate-v2 - -- HTTP endpoints and authentication. -- Capability negotiation and versioned request/response behavior. -- Idempotency, request limits, persistence, queues, deletion, triage, projection, indexing, and search. -- Infrastructure, deployment, and server-side producer tests. - -The repositories meet at immutable plugin release assets and the versioned HTTP capability contract. Neither repository imports the other's runtime source. - -## Extraction provenance - -- Source repository: `Substrate-memory/Substrate-v2`. -- Extraction base: `39e7bd8f650401c4a53004dc34e06c4b47e77c28`. -- Current imported plugin release: 1.4.1. -- Original 1.4.1 source commit: `a3953b0512bbb84fb62b48a75bab04cbcb845c78`. -- Archive SHA-256: `877ccf9b0212792b699d9c98912a26980675a6050df3bd319e927639e3d901f1`. -- Installer SHA-256: `7600b2681c3aebcb1b1492b0a04be38bbbec637089cbbcfb1cc26e8c10865b8d`. - -The imported 1.4.1 artifacts remain immutable. Repository-native releases use new versions and provenance commits from this repository. +- Device onboarding and credential custody. +- Session-completion capture. +- Build, tests, documentation, and releases. diff --git a/docs/threat-model.md b/docs/threat-model.md index a20e222..da246fc 100644 --- a/docs/threat-model.md +++ b/docs/threat-model.md @@ -2,29 +2,9 @@ | Threat | Primary controls | |---|---| -| Credential disclosure | Native/profile-private custody; no config/log/argument secrets; pre-persistence redaction | -| Origin substitution | Fixed hosted HTTPS origin; unsafe environment override rejection; redirect rejection | -| Private-history disclosure | Explicit historical consent; source eligibility filters; redaction before spool/network | -| Local spool exposure | Profile-rooted owner-private files; bounded bytes; explicit deletion | -| Tenant crossover | Tenant-scoped revocable server keys; live account/tenant checks; tenant-bound API resolution | -| Replay duplication or loss | Stable batch/event IDs; idempotency keys; durable acknowledgements and exact resume | -| Resource exhaustion | Bounded bodies/events/responses/spool; poll throttling; capped retry backoff | -| Build or release tampering | Allowlisted deterministic ZIP; commit and file hashes; immutable checksums | -| Cross-profile leakage | Profile-derived credential slots and state; no shared writable profile state | -| PID reuse during import | Content-free runtime nonce and process-identity verification | -| Prompt injection in recall | Bounded cited memory cards treated as untrusted context; no authority from memory | - -## Trust assumptions - -- Hosted account authentication and tenant isolation are operated at `app.trysubstrate.co`. -- Hermes 0.21.x invokes the current `substrate` plugin's discovery path, `pre_llm_call`/`post_llm_call` hooks, and native tool registration (see [`plugins/substrate`](../plugins/substrate)). -- Hermes 0.20.x invokes the legacy `substrate_wiki` user-plugin discovery and provider lifecycle hooks. -- The operating-system user and `$HERMES_HOME` permissions protect local state. -- Published checksums and release custody are verified before installation. - -## Explicit limitations - -- Pattern redaction cannot prove arbitrary sensitive prose is absent. -- Compromise of the hosted service is outside the plugin's local security boundary. -- Memory content does not grant send, action, or authorization authority. -- Portable workers are restarted when Hermes next initializes the provider; systemd units can restart independently. +| Credential disclosure | Profile-private `.env` custody; no config/log/argument/chat secrets; pre-network redaction | +| Device-code disclosure | Owner-private profile file; deleted on terminal states; never in onboarding state | +| Origin substitution | Configured HTTPS origin allowlist; unsafe override rejection; redirect rejection | +| TLS interception | System trust plus pinned public ISRG roots; verification never disabled | +| Tenant crossover | Tenant-scoped revocable server keys; per-request agent resolution | +| Replay duplication or loss | Stable event IDs; idempotency keys; durable capture queue | diff --git a/legacy-assets/1.2.0/install_hermes_plugin.py b/legacy-assets/1.2.0/install_hermes_plugin.py deleted file mode 100644 index 38fe0fd..0000000 --- a/legacy-assets/1.2.0/install_hermes_plugin.py +++ /dev/null @@ -1,323 +0,0 @@ -#!/usr/bin/env python3 -"""Verify and atomically install or upgrade the Substrate Hermes plugin.""" - -from __future__ import annotations - -import argparse -import hashlib -import json -import os -import re -import stat -import subprocess -import sys -import tempfile -import time -import zipfile -from pathlib import Path, PurePosixPath -from typing import Any - -PLUGIN_NAME = "substrate_wiki" -EXPECTED_VERSION = "1.2.0" -EXPECTED_HERMES_VERSION = "0.18.2" -REQUIRED_FILES = { - "README.md", - "__init__.py", - "cli.py", - "client.py", - "checkpoint.py", - "events.py", - "history.py", - "plugin.yaml", - "py.typed", - "redaction.py", - "spool.py", - "supervisor.py", - "worker.py", -} -MAX_UNCOMPRESSED_BYTES = 2 * 1024 * 1024 - - -def _digest(data: bytes) -> str: - return hashlib.sha256(data).hexdigest() - - -def verify_archive(path: Path, expected_sha256: str = "") -> dict[str, Any]: - data = path.read_bytes() - digest = _digest(data) - if expected_sha256 and digest != expected_sha256.lower(): - raise ValueError("archive SHA-256 mismatch") - with zipfile.ZipFile(path) as archive: - names = archive.namelist() - if len(names) != len(set(names)): - raise ValueError("archive contains duplicate paths") - if sum(info.file_size for info in archive.infolist()) > MAX_UNCOMPRESSED_BYTES: - raise ValueError("archive is too large") - for info in archive.infolist(): - name = info.filename - candidate = PurePosixPath(name) - if candidate.is_absolute() or ".." in candidate.parts: - raise ValueError("archive contains an unsafe path") - if not candidate.parts or candidate.parts[0] != PLUGIN_NAME: - raise ValueError("archive has an unexpected root directory") - mode = info.external_attr >> 16 - if stat.S_ISLNK(mode): - raise ValueError("archive contains a symbolic link") - provenance = json.loads(archive.read(f"{PLUGIN_NAME}/PROVENANCE.json").decode("utf-8")) - if provenance.get("build_format_version") != 2: - raise ValueError("unexpected plugin archive build format") - if provenance.get("plugin_version") != EXPECTED_VERSION: - raise ValueError("unexpected plugin version") - if provenance.get("target_hermes_version") != EXPECTED_HERMES_VERSION: - raise ValueError("unexpected Hermes target version") - source_files = provenance.get("source_files") - if not isinstance(source_files, dict): - raise ValueError("archive provenance has no source file manifest") - if set(source_files) != REQUIRED_FILES: - raise ValueError("archive provenance has an unexpected source file set") - source_commit = provenance.get("source_commit") - if ( - not isinstance(source_commit, str) - or re.fullmatch(r"[0-9a-f]{40}", source_commit) is None - ): - raise ValueError("archive provenance has no immutable source commit") - for relative, expected in source_files.items(): - actual = _digest(archive.read(f"{PLUGIN_NAME}/{relative}")) - if actual != expected: - raise ValueError(f"source digest mismatch: {relative}") - return { - "archive_sha256": digest, - "plugin_version": EXPECTED_VERSION, - "source_commit": source_commit, - } - - -def _systemd_quote(value: Path) -> str: - text = os.fspath(value.resolve()) - if any(character in text for character in ('\n', '\r', '"')): - raise ValueError("service path cannot be represented safely") - return f'"{text}"' - - -def _resolve_env_path(explicit: Path | None = None) -> Path: - if explicit is not None: - path = explicit.resolve() - else: - result = subprocess.run( - ("hermes", "config", "env-path"), - check=True, - stdin=subprocess.DEVNULL, - stdout=subprocess.PIPE, - stderr=subprocess.DEVNULL, - text=True, - timeout=30, - ) - path = Path(result.stdout.strip()).resolve() - if not path.is_file() or path.is_symlink(): - raise ValueError("Hermes environment path must be a regular non-symlink file") - info = path.stat(follow_symlinks=False) - if os.name == "posix": - if info.st_uid != os.getuid() or stat.S_IMODE(info.st_mode) & 0o077: - raise ValueError("Hermes environment path must be owner-only") - names: set[str] = set() - with path.open("r", encoding="utf-8") as stream: - for raw in stream: - line = raw.strip() - if line and not line.startswith("#") and "=" in line: - names.add(line.split("=", 1)[0].removeprefix("export ").strip()) - if not {"HERMES_API_URL", "HERMES_API_KEY"} <= names: - raise ValueError("Hermes environment is missing required Substrate variables") - return path - - -def install_import_service( - hermes_home: Path, - plugin_target: Path, - *, - env_path: Path | None = None, -) -> dict[str, Any]: - if os.name != "posix": - raise ValueError("the import service can only be installed on Linux") - from hashlib import sha256 - - resolved_env = _resolve_env_path(env_path) - home_hash = sha256(os.fspath(hermes_home.resolve()).encode()).hexdigest()[:12] - unit_name = f"substrate-wiki-import-{home_hash}@.service" - user_units = Path.home() / ".config" / "systemd" / "user" - if user_units.exists() and user_units.is_symlink(): - raise ValueError("systemd user unit directory must not be a symlink") - user_units.mkdir(parents=True, exist_ok=True, mode=0o700) - unit_path = user_units / unit_name - if unit_path.is_symlink(): - raise ValueError("existing import unit must not be a symlink") - rollback: Path | None = None - if unit_path.exists(): - rollback = unit_path.with_name(f"{unit_name}.rollback-{int(time.time())}") - os.replace(unit_path, rollback) - python = Path(sys.executable).resolve() - supervisor = plugin_target / "supervisor.py" - unit = "\n".join( - ( - "[Unit]", - "Description=Substrate Wiki durable history import %i", - "After=network-online.target", - "Wants=network-online.target", - "StartLimitIntervalSec=600", - "StartLimitBurst=8", - "", - "[Service]", - "Type=simple", - f"EnvironmentFile={_systemd_quote(resolved_env)}", - f"ExecStart={_systemd_quote(python)} {_systemd_quote(supervisor)} " - f"--hermes-home {_systemd_quote(hermes_home)} --job-id %i", - "Restart=on-failure", - "RestartSec=15s", - "MemoryHigh=224M", - "MemoryMax=256M", - "OOMPolicy=stop", - "NoNewPrivileges=true", - "PrivateTmp=true", - "ProtectSystem=strict", - "ProtectHome=read-only", - f"ReadWritePaths={_systemd_quote(hermes_home)}", - "RestrictSUIDSGID=true", - "LockPersonality=true", - "", - "[Install]", - "WantedBy=default.target", - "", - ) - ).encode("utf-8") - temporary = unit_path.with_name(f".{unit_name}.{os.getpid()}.tmp") - descriptor = os.open(temporary, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) - try: - with os.fdopen(descriptor, "wb") as stream: - stream.write(unit) - stream.flush() - os.fsync(stream.fileno()) - os.replace(temporary, unit_path) - os.chmod(unit_path, 0o600) - except Exception: - if rollback is not None and rollback.exists() and not unit_path.exists(): - os.replace(rollback, unit_path) - raise - finally: - try: - temporary.unlink() - except FileNotFoundError: - pass - subprocess.run( - ("systemctl", "--user", "daemon-reload"), - check=True, - stdin=subprocess.DEVNULL, - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, - timeout=30, - ) - return { - "import_service": unit_name, - "import_service_path": os.fspath(unit_path), - "import_service_rollback": os.fspath(rollback) if rollback else None, - "memory_high_bytes": 224 * 1024 * 1024, - "memory_max_bytes": 256 * 1024 * 1024, - } - - -def install( - archive: Path, - hermes_home: Path, - *, - expected_sha256: str = "", - install_service: bool = False, - env_path: Path | None = None, -) -> dict[str, Any]: - verified = verify_archive(archive, expected_sha256) - plugins = hermes_home / "plugins" - plugins.mkdir(parents=True, exist_ok=True, mode=0o700) - if plugins.is_symlink(): - raise ValueError("Hermes plugins directory must not be a symlink") - target = plugins / PLUGIN_NAME - if target.is_symlink(): - raise ValueError("existing plugin directory must not be a symlink") - with tempfile.TemporaryDirectory(prefix="substrate-plugin-", dir=plugins) as directory: - staging_root = Path(directory) - with zipfile.ZipFile(archive) as bundle: - bundle.extractall(staging_root) - staged = staging_root / PLUGIN_NAME - if not (staged / "plugin.yaml").is_file() or not (staged / "__init__.py").is_file(): - raise ValueError("archive is missing required plugin files") - installed = not target.exists() - rollback: Path | None = None - if target.exists(): - rollback = plugins / f"{PLUGIN_NAME}.rollback-{int(time.time())}" - os.replace(target, rollback) - try: - os.replace(staged, target) - except Exception: - if rollback is not None and rollback.exists() and not target.exists(): - os.replace(rollback, target) - raise - if os.name == "posix": - os.chmod(target, 0o700) - for child in target.rglob("*"): - os.chmod(child, 0o600 if child.is_file() else 0o700) - result = { - **verified, - "action": "installed" if installed else "upgraded", - "target": os.fspath(target), - "rollback": os.fspath(rollback) if rollback is not None else None, - } - if install_service: - try: - result.update(install_import_service(hermes_home, target, env_path=env_path)) - except Exception: - if rollback is not None and rollback.exists() and target.exists(): - failed = plugins / f"{PLUGIN_NAME}.failed-{int(time.time())}" - os.replace(target, failed) - os.replace(rollback, target) - raise - return result - - -def main() -> int: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--archive", type=Path, required=True) - parser.add_argument( - "--hermes-home", - type=Path, - default=Path(os.environ.get("HERMES_HOME") or Path.home() / ".hermes"), - ) - parser.add_argument("--sha256", default="") - parser.add_argument("--install-import-service", action="store_true") - parser.add_argument("--env-path", type=Path) - parser.add_argument("--yes", action="store_true") - parser.add_argument("--json", action="store_true") - args = parser.parse_args() - if not args.yes: - parser.error("--yes is required for installation") - try: - result = install( - args.archive.resolve(), - args.hermes_home.resolve(), - expected_sha256=args.sha256, - install_service=args.install_import_service, - env_path=args.env_path, - ) - except (OSError, ValueError, zipfile.BadZipFile, json.JSONDecodeError) as exc: - message = {"error": type(exc).__name__, "installed": False} - print( - json.dumps(message, sort_keys=True) - if args.json - else f"Installation failed: {type(exc).__name__}" - ) - return 1 - print( - json.dumps(result, sort_keys=True) - if args.json - else f"Substrate plugin {result['action']}: {result['target']}" - ) - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/legacy-assets/1.2.0/substrate_wiki.zip b/legacy-assets/1.2.0/substrate_wiki.zip deleted file mode 100644 index 050676c..0000000 Binary files a/legacy-assets/1.2.0/substrate_wiki.zip and /dev/null differ diff --git a/legacy-assets/1.3.0/install_hermes_plugin.py b/legacy-assets/1.3.0/install_hermes_plugin.py deleted file mode 100644 index 5437259..0000000 --- a/legacy-assets/1.3.0/install_hermes_plugin.py +++ /dev/null @@ -1,354 +0,0 @@ -#!/usr/bin/env python3 -"""Verify and atomically install or upgrade the Substrate Hermes plugin.""" - -from __future__ import annotations - -import argparse -import hashlib -import json -import os -import re -import stat -import subprocess -import sys -import tempfile -import time -import zipfile -from pathlib import Path, PurePosixPath -from typing import Any, cast - -PLUGIN_NAME = "substrate_wiki" -EXPECTED_VERSION = "1.3.0" -EXPECTED_HERMES_VERSION = "0.18.2" -REQUIRED_FILES = { - "README.md", - "__init__.py", - "cli.py", - "client.py", - "checkpoint.py", - "events.py", - "history.py", - "plugin.yaml", - "py.typed", - "redaction.py", - "spool.py", - "supervisor.py", - "worker.py", -} -MAX_UNCOMPRESSED_BYTES = 2 * 1024 * 1024 -MAX_ARCHIVE_BYTES = 4 * 1024 * 1024 - - -def _digest(data: bytes) -> str: - return hashlib.sha256(data).hexdigest() - - -def verify_archive(path: Path, expected_sha256: str = "") -> dict[str, Any]: - if not path.is_file() or path.stat().st_size > MAX_ARCHIVE_BYTES: - raise ValueError("archive is missing or too large") - data = path.read_bytes() - digest = _digest(data) - if expected_sha256 and digest != expected_sha256.lower(): - raise ValueError("archive SHA-256 mismatch") - with zipfile.ZipFile(path) as archive: - names = archive.namelist() - if len(names) != len(set(names)): - raise ValueError("archive contains duplicate paths") - expected_members = { - f"{PLUGIN_NAME}/", - f"{PLUGIN_NAME}/PROVENANCE.json", - *(f"{PLUGIN_NAME}/{name}" for name in REQUIRED_FILES), - } - if set(names) != expected_members: - raise ValueError("archive contains an unexpected file set") - if sum(info.file_size for info in archive.infolist()) > MAX_UNCOMPRESSED_BYTES: - raise ValueError("archive is too large") - for info in archive.infolist(): - name = info.filename - candidate = PurePosixPath(name) - if candidate.is_absolute() or ".." in candidate.parts: - raise ValueError("archive contains an unsafe path") - if not candidate.parts or candidate.parts[0] != PLUGIN_NAME: - raise ValueError("archive has an unexpected root directory") - mode = info.external_attr >> 16 - if stat.S_ISLNK(mode): - raise ValueError("archive contains a symbolic link") - provenance = json.loads(archive.read(f"{PLUGIN_NAME}/PROVENANCE.json").decode("utf-8")) - if provenance.get("build_format_version") != 2: - raise ValueError("unexpected plugin archive build format") - if provenance.get("provider_id") != PLUGIN_NAME: - raise ValueError("unexpected plugin provider identity") - if provenance.get("plugin_version") != EXPECTED_VERSION: - raise ValueError("unexpected plugin version") - if provenance.get("target_hermes_version") != EXPECTED_HERMES_VERSION: - raise ValueError("unexpected Hermes target version") - source_files = provenance.get("source_files") - if not isinstance(source_files, dict): - raise ValueError("archive provenance has no source file manifest") - if set(source_files) != REQUIRED_FILES: - raise ValueError("archive provenance has an unexpected source file set") - source_commit = provenance.get("source_commit") - if ( - not isinstance(source_commit, str) - or re.fullmatch(r"[0-9a-f]{40}", source_commit) is None - ): - raise ValueError("archive provenance has no immutable source commit") - for relative, expected in source_files.items(): - if not isinstance(expected, str) or re.fullmatch(r"[0-9a-f]{64}", expected) is None: - raise ValueError(f"invalid source digest: {relative}") - actual = _digest(archive.read(f"{PLUGIN_NAME}/{relative}")) - if actual != expected: - raise ValueError(f"source digest mismatch: {relative}") - manifest = archive.read(f"{PLUGIN_NAME}/plugin.yaml").decode("utf-8") - if re.search(r"(?m)^name:\s*substrate_wiki\s*$", manifest) is None: - raise ValueError("plugin manifest identity mismatch") - if re.search(r"(?m)^version:\s*1\.3\.0\s*$", manifest) is None: - raise ValueError("plugin manifest version mismatch") - return { - "archive_sha256": digest, - "plugin_version": EXPECTED_VERSION, - "source_commit": source_commit, - } - - -def _systemd_quote(value: Path) -> str: - text = os.fspath(value.resolve()) - if any(character in text for character in ('\n', '\r', '"')): - raise ValueError("service path cannot be represented safely") - return f'"{text}"' - - -def _resolve_env_path(explicit: Path | None = None) -> Path: - if explicit is not None: - path = explicit.resolve() - else: - result = subprocess.run( - ("hermes", "config", "env-path"), - check=True, - stdin=subprocess.DEVNULL, - stdout=subprocess.PIPE, - stderr=subprocess.DEVNULL, - text=True, - timeout=30, - ) - path = Path(result.stdout.strip()).resolve() - if not path.is_file() or path.is_symlink(): - raise ValueError("Hermes environment path must be a regular non-symlink file") - info = path.stat(follow_symlinks=False) - if os.name == "posix": - current_uid = int(cast(Any, os).getuid()) - if info.st_uid != current_uid or stat.S_IMODE(info.st_mode) & 0o077: - raise ValueError("Hermes environment path must be owner-only") - names: set[str] = set() - with path.open("r", encoding="utf-8") as stream: - for raw in stream: - line = raw.strip() - if line and not line.startswith("#") and "=" in line: - names.add(line.split("=", 1)[0].removeprefix("export ").strip()) - if not {"HERMES_API_URL", "HERMES_API_KEY"} <= names: - raise ValueError("Hermes environment is missing required Substrate variables") - return path - - -def install_import_service( - hermes_home: Path, - plugin_target: Path, - *, - env_path: Path | None = None, -) -> dict[str, Any]: - if os.name != "posix": - raise ValueError("the import service can only be installed on Linux") - from hashlib import sha256 - - resolved_env = _resolve_env_path(env_path) - home_hash = sha256(os.fspath(hermes_home.resolve()).encode()).hexdigest()[:12] - unit_name = f"substrate-wiki-import-{home_hash}@.service" - user_units = Path.home() / ".config" / "systemd" / "user" - if user_units.exists() and user_units.is_symlink(): - raise ValueError("systemd user unit directory must not be a symlink") - user_units.mkdir(parents=True, exist_ok=True, mode=0o700) - unit_path = user_units / unit_name - if unit_path.is_symlink(): - raise ValueError("existing import unit must not be a symlink") - rollback: Path | None = None - if unit_path.exists(): - rollback = unit_path.with_name(f"{unit_name}.rollback-{time.time_ns()}") - os.replace(unit_path, rollback) - python = Path(sys.executable).resolve() - supervisor = plugin_target / "supervisor.py" - unit = "\n".join( - ( - "[Unit]", - "Description=Substrate Wiki durable history import %i", - "After=network-online.target", - "Wants=network-online.target", - "StartLimitIntervalSec=600", - "StartLimitBurst=8", - "", - "[Service]", - "Type=simple", - f"EnvironmentFile={_systemd_quote(resolved_env)}", - f"ExecStart={_systemd_quote(python)} {_systemd_quote(supervisor)} " - f"--hermes-home {_systemd_quote(hermes_home)} --job-id %i", - "Restart=on-failure", - "RestartSec=15s", - "MemoryHigh=224M", - "MemoryMax=256M", - "OOMPolicy=stop", - "NoNewPrivileges=true", - "PrivateTmp=true", - "ProtectSystem=strict", - "ProtectHome=read-only", - f"ReadWritePaths={_systemd_quote(hermes_home)}", - "RestrictSUIDSGID=true", - "LockPersonality=true", - "", - "[Install]", - "WantedBy=default.target", - "", - ) - ).encode("utf-8") - temporary = unit_path.with_name(f".{unit_name}.{os.getpid()}.tmp") - descriptor = os.open(temporary, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) - try: - with os.fdopen(descriptor, "wb") as stream: - stream.write(unit) - stream.flush() - os.fsync(stream.fileno()) - os.replace(temporary, unit_path) - os.chmod(unit_path, 0o600) - subprocess.run( - ("systemctl", "--user", "daemon-reload"), - check=True, - stdin=subprocess.DEVNULL, - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, - timeout=30, - ) - except Exception: - if unit_path.exists() and not unit_path.is_symlink(): - unit_path.unlink() - if rollback is not None and rollback.exists(): - os.replace(rollback, unit_path) - subprocess.run( - ("systemctl", "--user", "daemon-reload"), - check=False, - stdin=subprocess.DEVNULL, - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, - timeout=30, - ) - raise - finally: - try: - temporary.unlink() - except FileNotFoundError: - pass - return { - "import_service": unit_name, - "import_service_path": os.fspath(unit_path), - "import_service_rollback": os.fspath(rollback) if rollback else None, - "memory_high_bytes": 224 * 1024 * 1024, - "memory_max_bytes": 256 * 1024 * 1024, - } - - -def install( - archive: Path, - hermes_home: Path, - *, - expected_sha256: str = "", - install_service: bool = False, - env_path: Path | None = None, -) -> dict[str, Any]: - verified = verify_archive(archive, expected_sha256) - plugins = hermes_home / "plugins" - plugins.mkdir(parents=True, exist_ok=True, mode=0o700) - if plugins.is_symlink(): - raise ValueError("Hermes plugins directory must not be a symlink") - target = plugins / PLUGIN_NAME - if target.is_symlink(): - raise ValueError("existing plugin directory must not be a symlink") - with tempfile.TemporaryDirectory(prefix="substrate-plugin-", dir=plugins) as directory: - staging_root = Path(directory) - with zipfile.ZipFile(archive) as bundle: - bundle.extractall(staging_root) - staged = staging_root / PLUGIN_NAME - if not (staged / "plugin.yaml").is_file() or not (staged / "__init__.py").is_file(): - raise ValueError("archive is missing required plugin files") - installed = not target.exists() - rollback: Path | None = None - if target.exists(): - rollback = plugins / f"{PLUGIN_NAME}.rollback-{time.time_ns()}" - os.replace(target, rollback) - try: - os.replace(staged, target) - except Exception: - if rollback is not None and rollback.exists() and not target.exists(): - os.replace(rollback, target) - raise - if os.name == "posix": - os.chmod(target, 0o700) - for child in target.rglob("*"): - os.chmod(child, 0o600 if child.is_file() else 0o700) - result = { - **verified, - "action": "installed" if installed else "upgraded", - "target": os.fspath(target), - "rollback": os.fspath(rollback) if rollback is not None else None, - } - if install_service: - try: - result.update(install_import_service(hermes_home, target, env_path=env_path)) - except Exception: - if target.exists(): - failed = plugins / f"{PLUGIN_NAME}.failed-{time.time_ns()}" - os.replace(target, failed) - if rollback is not None and rollback.exists() and not target.exists(): - os.replace(rollback, target) - raise - return result - - -def main() -> int: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--archive", type=Path, required=True) - parser.add_argument( - "--hermes-home", - type=Path, - default=Path(os.environ.get("HERMES_HOME") or Path.home() / ".hermes"), - ) - parser.add_argument("--sha256", default="") - parser.add_argument("--install-import-service", action="store_true") - parser.add_argument("--env-path", type=Path) - parser.add_argument("--yes", action="store_true") - parser.add_argument("--json", action="store_true") - args = parser.parse_args() - if not args.yes: - parser.error("--yes is required for installation") - try: - result = install( - args.archive.resolve(), - args.hermes_home.resolve(), - expected_sha256=args.sha256, - install_service=args.install_import_service, - env_path=args.env_path, - ) - except (OSError, ValueError, zipfile.BadZipFile, json.JSONDecodeError) as exc: - message = {"error": type(exc).__name__, "installed": False} - print( - json.dumps(message, sort_keys=True) - if args.json - else f"Installation failed: {type(exc).__name__}" - ) - return 1 - print( - json.dumps(result, sort_keys=True) - if args.json - else f"Substrate plugin {result['action']}: {result['target']}" - ) - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/legacy-assets/1.3.0/substrate_wiki.zip b/legacy-assets/1.3.0/substrate_wiki.zip deleted file mode 100644 index a9f2282..0000000 Binary files a/legacy-assets/1.3.0/substrate_wiki.zip and /dev/null differ diff --git a/legacy-assets/1.4.0/install_hermes_plugin.py b/legacy-assets/1.4.0/install_hermes_plugin.py deleted file mode 100644 index cff3b54..0000000 --- a/legacy-assets/1.4.0/install_hermes_plugin.py +++ /dev/null @@ -1,432 +0,0 @@ -#!/usr/bin/env python3 -"""Verify and atomically install or upgrade the Substrate Hermes plugin.""" - -from __future__ import annotations - -import argparse -import hashlib -import io -import json -import os -import re -import stat -import subprocess -import sys -import tempfile -import time -import zipfile -from pathlib import Path, PurePosixPath -from typing import Any, cast - -PLUGIN_NAME = "substrate_wiki" -EXPECTED_VERSION = "1.4.0" -EXPECTED_HERMES_VERSION = "0.18.2" -REQUIRED_FILES = { - "README.md", - "__init__.py", - "cli.py", - "client.py", - "checkpoint.py", - "events.py", - "history.py", - "plugin.yaml", - "py.typed", - "redaction.py", - "spool.py", - "supervisor.py", - "worker.py", -} -MAX_UNCOMPRESSED_BYTES = 2 * 1024 * 1024 -MAX_ARCHIVE_BYTES = 4 * 1024 * 1024 - - -def _digest(data: bytes) -> str: - return hashlib.sha256(data).hexdigest() - - -def _normalize_expected_sha256(value: str, *, required: bool = False) -> str: - normalized = value.strip().lower() if isinstance(value, str) else "" - if not normalized: - if required: - raise ValueError("a pinned archive SHA-256 is required") - return "" - if re.fullmatch(r"[0-9a-f]{64}", normalized) is None: - raise ValueError("pinned archive SHA-256 is malformed") - return normalized - - -def _read_archive_bytes(path: Path) -> bytes: - if not path.is_file() or path.is_symlink(): - raise ValueError("archive is missing or unsafe") - with path.open("rb") as stream: - data = stream.read(MAX_ARCHIVE_BYTES + 1) - if len(data) > MAX_ARCHIVE_BYTES: - raise ValueError("archive is too large") - return data - - -def verify_archive(path: Path, expected_sha256: str = "") -> dict[str, Any]: - data = _read_archive_bytes(path) - digest = _digest(data) - expected_sha256 = _normalize_expected_sha256(expected_sha256) - if expected_sha256 and digest != expected_sha256: - raise ValueError("archive SHA-256 mismatch") - # Validate the exact bytes whose digest we report. Reopening ``path`` here - # would leave a swap window between hashing and archive inspection. - with zipfile.ZipFile(io.BytesIO(data)) as archive: - names = archive.namelist() - if len(names) != len(set(names)): - raise ValueError("archive contains duplicate paths") - expected_members = { - f"{PLUGIN_NAME}/", - f"{PLUGIN_NAME}/PROVENANCE.json", - *(f"{PLUGIN_NAME}/{name}" for name in REQUIRED_FILES), - } - if set(names) != expected_members: - raise ValueError("archive contains an unexpected file set") - if sum(info.file_size for info in archive.infolist()) > MAX_UNCOMPRESSED_BYTES: - raise ValueError("archive is too large") - for info in archive.infolist(): - name = info.filename - candidate = PurePosixPath(name) - if candidate.is_absolute() or ".." in candidate.parts: - raise ValueError("archive contains an unsafe path") - if not candidate.parts or candidate.parts[0] != PLUGIN_NAME: - raise ValueError("archive has an unexpected root directory") - mode = info.external_attr >> 16 - if stat.S_ISLNK(mode): - raise ValueError("archive contains a symbolic link") - if name == f"{PLUGIN_NAME}/": - if not info.is_dir() or not stat.S_ISDIR(mode): - raise ValueError("archive root is not a directory") - elif info.is_dir() or not stat.S_ISREG(mode): - raise ValueError("archive contains a non-regular plugin file") - if info.flag_bits & 0x1: - raise ValueError("archive contains an encrypted member") - provenance = json.loads(archive.read(f"{PLUGIN_NAME}/PROVENANCE.json").decode("utf-8")) - if provenance.get("build_format_version") != 2: - raise ValueError("unexpected plugin archive build format") - if provenance.get("provider_id") != PLUGIN_NAME: - raise ValueError("unexpected plugin provider identity") - if provenance.get("plugin_version") != EXPECTED_VERSION: - raise ValueError("unexpected plugin version") - if provenance.get("target_hermes_version") != EXPECTED_HERMES_VERSION: - raise ValueError("unexpected Hermes target version") - source_files = provenance.get("source_files") - if not isinstance(source_files, dict): - raise ValueError("archive provenance has no source file manifest") - if set(source_files) != REQUIRED_FILES: - raise ValueError("archive provenance has an unexpected source file set") - source_commit = provenance.get("source_commit") - if ( - not isinstance(source_commit, str) - or re.fullmatch(r"[0-9a-f]{40}", source_commit) is None - ): - raise ValueError("archive provenance has no immutable source commit") - for relative, expected in source_files.items(): - if not isinstance(expected, str) or re.fullmatch(r"[0-9a-f]{64}", expected) is None: - raise ValueError(f"invalid source digest: {relative}") - actual = _digest(archive.read(f"{PLUGIN_NAME}/{relative}")) - if actual != expected: - raise ValueError(f"source digest mismatch: {relative}") - manifest = archive.read(f"{PLUGIN_NAME}/plugin.yaml").decode("utf-8") - if re.search(r"(?m)^name:\s*substrate_wiki\s*$", manifest) is None: - raise ValueError("plugin manifest identity mismatch") - if re.search(r"(?m)^version:\s*1\.4\.0\s*$", manifest) is None: - raise ValueError("plugin manifest version mismatch") - return { - "archive_sha256": digest, - "plugin_version": EXPECTED_VERSION, - "source_commit": source_commit, - } - - -def _systemd_quote(value: Path) -> str: - text = os.fspath(value.resolve()) - if any(character in text for character in ("\n", "\r", '"')): - raise ValueError("service path cannot be represented safely") - return f'"{text}"' - - -def _resolve_env_path(explicit: Path | None = None) -> Path: - if explicit is not None: - candidate = explicit.expanduser().absolute() - else: - result = subprocess.run( - ("hermes", "config", "env-path"), - check=True, - stdin=subprocess.DEVNULL, - stdout=subprocess.PIPE, - stderr=subprocess.DEVNULL, - text=True, - timeout=30, - ) - candidate = Path(result.stdout.strip()).expanduser().absolute() - # Check the path entry before resolving it. Calling ``resolve`` first would - # erase the evidence that the configured environment file is a symlink. - if candidate.is_symlink(): - raise ValueError("Hermes environment path must be a regular non-symlink file") - path = candidate.resolve() - if not path.is_file(): - raise ValueError("Hermes environment path must be a regular non-symlink file") - info = path.stat(follow_symlinks=False) - if info.st_size > 1024 * 1024: - raise ValueError("Hermes environment file is unexpectedly large") - if os.name == "posix": - current_uid = int(cast(Any, os).getuid()) - if info.st_uid != current_uid or stat.S_IMODE(info.st_mode) & 0o077: - raise ValueError("Hermes environment path must be owner-only") - names: set[str] = set() - with path.open("r", encoding="utf-8") as stream: - for raw in stream: - line = raw.strip() - if line and not line.startswith("#") and "=" in line: - names.add(line.split("=", 1)[0].removeprefix("export ").strip()) - if not {"HERMES_API_URL", "HERMES_API_KEY"} <= names: - raise ValueError("Hermes environment is missing required Substrate variables") - return path - - -def install_import_service( - hermes_home: Path, - plugin_target: Path, - *, - env_path: Path | None = None, -) -> dict[str, Any]: - if os.name != "posix": - raise ValueError("the import service can only be installed on Linux") - from hashlib import sha256 - - resolved_env = _resolve_env_path(env_path) - home_hash = sha256(os.fspath(hermes_home.resolve()).encode()).hexdigest()[:12] - unit_name = f"substrate-wiki-import-{home_hash}@.service" - user_units = Path.home() / ".config" / "systemd" / "user" - if user_units.exists() and user_units.is_symlink(): - raise ValueError("systemd user unit directory must not be a symlink") - user_units.mkdir(parents=True, exist_ok=True, mode=0o700) - unit_path = user_units / unit_name - if unit_path.is_symlink(): - raise ValueError("existing import unit must not be a symlink") - rollback: Path | None = None - python = Path(sys.executable).resolve() - supervisor = plugin_target / "supervisor.py" - unit = "\n".join( - ( - "[Unit]", - "Description=Substrate Wiki durable history import %i", - "After=network-online.target", - "Wants=network-online.target", - "StartLimitIntervalSec=600", - "StartLimitBurst=8", - "", - "[Service]", - "Type=simple", - f"EnvironmentFile={_systemd_quote(resolved_env)}", - f"ExecStart={_systemd_quote(python)} {_systemd_quote(supervisor)} " - f"--hermes-home {_systemd_quote(hermes_home)} --job-id %i", - "Restart=on-failure", - "RestartSec=15s", - "MemoryHigh=224M", - "MemoryMax=256M", - "OOMPolicy=stop", - "NoNewPrivileges=true", - "PrivateTmp=true", - "ProtectSystem=strict", - "ProtectHome=read-only", - f"ReadWritePaths={_systemd_quote(hermes_home)}", - "RestrictSUIDSGID=true", - "LockPersonality=true", - "", - "[Install]", - "WantedBy=default.target", - "", - ) - ).encode("utf-8") - temporary = unit_path.with_name(f".{unit_name}.{os.getpid()}.tmp") - unit_replaced = False - try: - descriptor = os.open(temporary, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) - with os.fdopen(descriptor, "wb") as stream: - stream.write(unit) - stream.flush() - os.fsync(stream.fileno()) - if unit_path.exists() or unit_path.is_symlink(): - if unit_path.is_symlink(): - raise ValueError("existing import unit must not be a symlink") - rollback = unit_path.with_name(f"{unit_name}.rollback-{time.time_ns()}") - if rollback.exists() or rollback.is_symlink(): - raise OSError("import-unit rollback path already exists") - os.replace(unit_path, rollback) - os.replace(temporary, unit_path) - unit_replaced = True - os.chmod(unit_path, 0o600) - subprocess.run( - ("systemctl", "--user", "daemon-reload"), - check=True, - stdin=subprocess.DEVNULL, - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, - timeout=30, - ) - except Exception: - if unit_replaced and (unit_path.exists() or unit_path.is_symlink()): - unit_path.unlink() - if rollback is not None and (rollback.exists() or rollback.is_symlink()): - os.replace(rollback, unit_path) - if unit_replaced or rollback is not None: - subprocess.run( - ("systemctl", "--user", "daemon-reload"), - check=False, - stdin=subprocess.DEVNULL, - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, - timeout=30, - ) - raise - finally: - try: - temporary.unlink() - except FileNotFoundError: - pass - return { - "import_service": unit_name, - "import_service_path": os.fspath(unit_path), - "import_service_rollback": os.fspath(rollback) if rollback else None, - "memory_high_bytes": 224 * 1024 * 1024, - "memory_max_bytes": 256 * 1024 * 1024, - } - - -def _entry_exists(path: Path) -> bool: - """Return whether a directory entry exists without following dangling links.""" - return path.exists() or path.is_symlink() - - -def _restore_plugin_after_failed_install( - plugins: Path, - target: Path, - rollback: Path | None, -) -> None: - """Move a failed new install aside and atomically restore the prior plugin.""" - if _entry_exists(target): - failed = plugins / f"{PLUGIN_NAME}.failed-{time.time_ns()}" - if _entry_exists(failed): - raise OSError("failed-plugin rollback path already exists") - os.replace(target, failed) - if rollback is not None and _entry_exists(rollback) and not _entry_exists(target): - os.replace(rollback, target) - - -def _harden_plugin_permissions(target: Path) -> None: - if os.name != "posix": - return - os.chmod(target, 0o700) - for child in target.rglob("*"): - os.chmod(child, 0o600 if child.is_file() else 0o700) - - -def install( - archive: Path, - hermes_home: Path, - *, - expected_sha256: str = "", - install_service: bool = False, - env_path: Path | None = None, -) -> dict[str, Any]: - expected_sha256 = _normalize_expected_sha256(expected_sha256, required=True) - verified = verify_archive(archive, expected_sha256) - # Extract only bytes tied to the verified digest. A downloaded path can be - # replaced by another local process after verification but before install. - archive_bytes = _read_archive_bytes(archive) - if _digest(archive_bytes) != verified["archive_sha256"]: - raise ValueError("archive changed after verification") - plugins = hermes_home / "plugins" - plugins.mkdir(parents=True, exist_ok=True, mode=0o700) - if plugins.is_symlink(): - raise ValueError("Hermes plugins directory must not be a symlink") - target = plugins / PLUGIN_NAME - if target.is_symlink(): - raise ValueError("existing plugin directory must not be a symlink") - with tempfile.TemporaryDirectory(prefix="substrate-plugin-", dir=plugins) as directory: - staging_root = Path(directory) - with zipfile.ZipFile(io.BytesIO(archive_bytes)) as bundle: - bundle.extractall(staging_root) - staged = staging_root / PLUGIN_NAME - if not (staged / "plugin.yaml").is_file() or not (staged / "__init__.py").is_file(): - raise ValueError("archive is missing required plugin files") - installed = not target.exists() - rollback: Path | None = None - if target.exists(): - rollback = plugins / f"{PLUGIN_NAME}.rollback-{time.time_ns()}" - if _entry_exists(rollback): - raise OSError("plugin rollback path already exists") - os.replace(target, rollback) - try: - os.replace(staged, target) - _harden_plugin_permissions(target) - except Exception: - _restore_plugin_after_failed_install(plugins, target, rollback) - raise - result = { - **verified, - "action": "installed" if installed else "upgraded", - "target": os.fspath(target), - "rollback": os.fspath(rollback) if rollback is not None else None, - } - if install_service: - try: - result.update(install_import_service(hermes_home, target, env_path=env_path)) - except Exception: - _restore_plugin_after_failed_install(plugins, target, rollback) - raise - return result - - -def main() -> int: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--archive", type=Path, required=True) - parser.add_argument( - "--hermes-home", - type=Path, - default=Path(os.environ.get("HERMES_HOME") or Path.home() / ".hermes"), - ) - parser.add_argument("--sha256", required=True) - parser.add_argument("--install-import-service", action="store_true") - parser.add_argument("--env-path", type=Path) - parser.add_argument("--yes", action="store_true") - parser.add_argument("--json", action="store_true") - args = parser.parse_args() - if not args.yes: - parser.error("--yes is required for installation") - try: - result = install( - args.archive.expanduser().absolute(), - args.hermes_home.expanduser().absolute(), - expected_sha256=args.sha256, - install_service=args.install_import_service, - env_path=args.env_path, - ) - except ( - OSError, - ValueError, - zipfile.BadZipFile, - json.JSONDecodeError, - subprocess.SubprocessError, - ) as exc: - message = {"error": type(exc).__name__, "installed": False} - print( - json.dumps(message, sort_keys=True) - if args.json - else f"Installation failed: {type(exc).__name__}" - ) - return 1 - print( - json.dumps(result, sort_keys=True) - if args.json - else f"Substrate plugin {result['action']}: {result['target']}" - ) - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/legacy-assets/1.4.0/substrate_wiki.zip b/legacy-assets/1.4.0/substrate_wiki.zip deleted file mode 100644 index 5a85e3c..0000000 Binary files a/legacy-assets/1.4.0/substrate_wiki.zip and /dev/null differ diff --git a/legacy-assets/1.4.1/install_hermes_plugin.py b/legacy-assets/1.4.1/install_hermes_plugin.py deleted file mode 100644 index e46ab90..0000000 --- a/legacy-assets/1.4.1/install_hermes_plugin.py +++ /dev/null @@ -1,434 +0,0 @@ -#!/usr/bin/env python3 -"""Verify and atomically install or upgrade the Substrate Hermes plugin.""" - -from __future__ import annotations - -import argparse -import hashlib -import io -import json -import os -import re -import stat -import subprocess -import sys -import tempfile -import time -import zipfile -from pathlib import Path, PurePosixPath -from typing import Any, cast - -PLUGIN_NAME = "substrate_wiki" -EXPECTED_VERSION = "1.4.1" -EXPECTED_HERMES_VERSION = "0.18.2" -REQUIRED_FILES = { - "README.md", - "__init__.py", - "cli.py", - "client.py", - "checkpoint.py", - "events.py", - "history.py", - "plugin.yaml", - "py.typed", - "redaction.py", - "spool.py", - "supervisor.py", - "worker.py", -} -MAX_UNCOMPRESSED_BYTES = 2 * 1024 * 1024 -MAX_ARCHIVE_BYTES = 4 * 1024 * 1024 - - -def _digest(data: bytes) -> str: - return hashlib.sha256(data).hexdigest() - - -def _normalize_expected_sha256(value: str, *, required: bool = False) -> str: - normalized = value.strip().lower() if isinstance(value, str) else "" - if not normalized: - if required: - raise ValueError("a pinned archive SHA-256 is required") - return "" - if re.fullmatch(r"[0-9a-f]{64}", normalized) is None: - raise ValueError("pinned archive SHA-256 is malformed") - return normalized - - -def _read_archive_bytes(path: Path) -> bytes: - if not path.is_file() or path.is_symlink(): - raise ValueError("archive is missing or unsafe") - with path.open("rb") as stream: - data = stream.read(MAX_ARCHIVE_BYTES + 1) - if len(data) > MAX_ARCHIVE_BYTES: - raise ValueError("archive is too large") - return data - - -def verify_archive(path: Path, expected_sha256: str = "") -> dict[str, Any]: - data = _read_archive_bytes(path) - digest = _digest(data) - expected_sha256 = _normalize_expected_sha256(expected_sha256) - if expected_sha256 and digest != expected_sha256: - raise ValueError("archive SHA-256 mismatch") - # Validate the exact bytes whose digest we report. Reopening ``path`` here - # would leave a swap window between hashing and archive inspection. - with zipfile.ZipFile(io.BytesIO(data)) as archive: - names = archive.namelist() - if len(names) != len(set(names)): - raise ValueError("archive contains duplicate paths") - expected_members = { - f"{PLUGIN_NAME}/", - f"{PLUGIN_NAME}/PROVENANCE.json", - *(f"{PLUGIN_NAME}/{name}" for name in REQUIRED_FILES), - } - if set(names) != expected_members: - raise ValueError("archive contains an unexpected file set") - if sum(info.file_size for info in archive.infolist()) > MAX_UNCOMPRESSED_BYTES: - raise ValueError("archive is too large") - for info in archive.infolist(): - name = info.filename - candidate = PurePosixPath(name) - if candidate.is_absolute() or ".." in candidate.parts: - raise ValueError("archive contains an unsafe path") - if not candidate.parts or candidate.parts[0] != PLUGIN_NAME: - raise ValueError("archive has an unexpected root directory") - mode = info.external_attr >> 16 - if stat.S_ISLNK(mode): - raise ValueError("archive contains a symbolic link") - if name == f"{PLUGIN_NAME}/": - if not info.is_dir() or not stat.S_ISDIR(mode): - raise ValueError("archive root is not a directory") - elif info.is_dir() or not stat.S_ISREG(mode): - raise ValueError("archive contains a non-regular plugin file") - if info.flag_bits & 0x1: - raise ValueError("archive contains an encrypted member") - provenance = json.loads(archive.read(f"{PLUGIN_NAME}/PROVENANCE.json").decode("utf-8")) - if provenance.get("build_format_version") != 2: - raise ValueError("unexpected plugin archive build format") - if provenance.get("provider_id") != PLUGIN_NAME: - raise ValueError("unexpected plugin provider identity") - if provenance.get("plugin_version") != EXPECTED_VERSION: - raise ValueError("unexpected plugin version") - if provenance.get("target_hermes_version") != EXPECTED_HERMES_VERSION: - raise ValueError("unexpected Hermes target version") - source_files = provenance.get("source_files") - if not isinstance(source_files, dict): - raise ValueError("archive provenance has no source file manifest") - if set(source_files) != REQUIRED_FILES: - raise ValueError("archive provenance has an unexpected source file set") - source_commit = provenance.get("source_commit") - if ( - not isinstance(source_commit, str) - or re.fullmatch(r"[0-9a-f]{40}", source_commit) is None - ): - raise ValueError("archive provenance has no immutable source commit") - for relative, expected in source_files.items(): - if not isinstance(expected, str) or re.fullmatch(r"[0-9a-f]{64}", expected) is None: - raise ValueError(f"invalid source digest: {relative}") - actual = _digest(archive.read(f"{PLUGIN_NAME}/{relative}")) - if actual != expected: - raise ValueError(f"source digest mismatch: {relative}") - manifest = archive.read(f"{PLUGIN_NAME}/plugin.yaml").decode("utf-8") - if re.search(r"(?m)^name:\s*substrate_wiki\s*$", manifest) is None: - raise ValueError("plugin manifest identity mismatch") - if re.search( - rf"(?m)^version:\s*{re.escape(EXPECTED_VERSION)}\s*$", manifest - ) is None: - raise ValueError("plugin manifest version mismatch") - return { - "archive_sha256": digest, - "plugin_version": EXPECTED_VERSION, - "source_commit": source_commit, - } - - -def _systemd_quote(value: Path) -> str: - text = os.fspath(value.resolve()) - if any(character in text for character in ("\n", "\r", '"')): - raise ValueError("service path cannot be represented safely") - return f'"{text}"' - - -def _resolve_env_path(explicit: Path | None = None) -> Path: - if explicit is not None: - candidate = explicit.expanduser().absolute() - else: - result = subprocess.run( - ("hermes", "config", "env-path"), - check=True, - stdin=subprocess.DEVNULL, - stdout=subprocess.PIPE, - stderr=subprocess.DEVNULL, - text=True, - timeout=30, - ) - candidate = Path(result.stdout.strip()).expanduser().absolute() - # Check the path entry before resolving it. Calling ``resolve`` first would - # erase the evidence that the configured environment file is a symlink. - if candidate.is_symlink(): - raise ValueError("Hermes environment path must be a regular non-symlink file") - path = candidate.resolve() - if not path.is_file(): - raise ValueError("Hermes environment path must be a regular non-symlink file") - info = path.stat(follow_symlinks=False) - if info.st_size > 1024 * 1024: - raise ValueError("Hermes environment file is unexpectedly large") - if os.name == "posix": - current_uid = int(cast(Any, os).getuid()) - if info.st_uid != current_uid or stat.S_IMODE(info.st_mode) & 0o077: - raise ValueError("Hermes environment path must be owner-only") - names: set[str] = set() - with path.open("r", encoding="utf-8") as stream: - for raw in stream: - line = raw.strip() - if line and not line.startswith("#") and "=" in line: - names.add(line.split("=", 1)[0].removeprefix("export ").strip()) - if not {"HERMES_API_URL", "HERMES_API_KEY"} <= names: - raise ValueError("Hermes environment is missing required Substrate variables") - return path - - -def install_import_service( - hermes_home: Path, - plugin_target: Path, - *, - env_path: Path | None = None, -) -> dict[str, Any]: - if os.name != "posix": - raise ValueError("the import service can only be installed on Linux") - from hashlib import sha256 - - resolved_env = _resolve_env_path(env_path) - home_hash = sha256(os.fspath(hermes_home.resolve()).encode()).hexdigest()[:12] - unit_name = f"substrate-wiki-import-{home_hash}@.service" - user_units = Path.home() / ".config" / "systemd" / "user" - if user_units.exists() and user_units.is_symlink(): - raise ValueError("systemd user unit directory must not be a symlink") - user_units.mkdir(parents=True, exist_ok=True, mode=0o700) - unit_path = user_units / unit_name - if unit_path.is_symlink(): - raise ValueError("existing import unit must not be a symlink") - rollback: Path | None = None - python = Path(sys.executable).resolve() - supervisor = plugin_target / "supervisor.py" - unit = "\n".join( - ( - "[Unit]", - "Description=Substrate Wiki durable history import %i", - "After=network-online.target", - "Wants=network-online.target", - "StartLimitIntervalSec=600", - "StartLimitBurst=8", - "", - "[Service]", - "Type=simple", - f"EnvironmentFile={_systemd_quote(resolved_env)}", - f"ExecStart={_systemd_quote(python)} {_systemd_quote(supervisor)} " - f"--hermes-home {_systemd_quote(hermes_home)} --job-id %i", - "Restart=on-failure", - "RestartSec=15s", - "MemoryHigh=224M", - "MemoryMax=256M", - "OOMPolicy=stop", - "NoNewPrivileges=true", - "PrivateTmp=true", - "ProtectSystem=strict", - "ProtectHome=read-only", - f"ReadWritePaths={_systemd_quote(hermes_home)}", - "RestrictSUIDSGID=true", - "LockPersonality=true", - "", - "[Install]", - "WantedBy=default.target", - "", - ) - ).encode("utf-8") - temporary = unit_path.with_name(f".{unit_name}.{os.getpid()}.tmp") - unit_replaced = False - try: - descriptor = os.open(temporary, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) - with os.fdopen(descriptor, "wb") as stream: - stream.write(unit) - stream.flush() - os.fsync(stream.fileno()) - if unit_path.exists() or unit_path.is_symlink(): - if unit_path.is_symlink(): - raise ValueError("existing import unit must not be a symlink") - rollback = unit_path.with_name(f"{unit_name}.rollback-{time.time_ns()}") - if rollback.exists() or rollback.is_symlink(): - raise OSError("import-unit rollback path already exists") - os.replace(unit_path, rollback) - os.replace(temporary, unit_path) - unit_replaced = True - os.chmod(unit_path, 0o600) - subprocess.run( - ("systemctl", "--user", "daemon-reload"), - check=True, - stdin=subprocess.DEVNULL, - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, - timeout=30, - ) - except Exception: - if unit_replaced and (unit_path.exists() or unit_path.is_symlink()): - unit_path.unlink() - if rollback is not None and (rollback.exists() or rollback.is_symlink()): - os.replace(rollback, unit_path) - if unit_replaced or rollback is not None: - subprocess.run( - ("systemctl", "--user", "daemon-reload"), - check=False, - stdin=subprocess.DEVNULL, - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, - timeout=30, - ) - raise - finally: - try: - temporary.unlink() - except FileNotFoundError: - pass - return { - "import_service": unit_name, - "import_service_path": os.fspath(unit_path), - "import_service_rollback": os.fspath(rollback) if rollback else None, - "memory_high_bytes": 224 * 1024 * 1024, - "memory_max_bytes": 256 * 1024 * 1024, - } - - -def _entry_exists(path: Path) -> bool: - """Return whether a directory entry exists without following dangling links.""" - return path.exists() or path.is_symlink() - - -def _restore_plugin_after_failed_install( - plugins: Path, - target: Path, - rollback: Path | None, -) -> None: - """Move a failed new install aside and atomically restore the prior plugin.""" - if _entry_exists(target): - failed = plugins / f"{PLUGIN_NAME}.failed-{time.time_ns()}" - if _entry_exists(failed): - raise OSError("failed-plugin rollback path already exists") - os.replace(target, failed) - if rollback is not None and _entry_exists(rollback) and not _entry_exists(target): - os.replace(rollback, target) - - -def _harden_plugin_permissions(target: Path) -> None: - if os.name != "posix": - return - os.chmod(target, 0o700) - for child in target.rglob("*"): - os.chmod(child, 0o600 if child.is_file() else 0o700) - - -def install( - archive: Path, - hermes_home: Path, - *, - expected_sha256: str = "", - install_service: bool = False, - env_path: Path | None = None, -) -> dict[str, Any]: - expected_sha256 = _normalize_expected_sha256(expected_sha256, required=True) - verified = verify_archive(archive, expected_sha256) - # Extract only bytes tied to the verified digest. A downloaded path can be - # replaced by another local process after verification but before install. - archive_bytes = _read_archive_bytes(archive) - if _digest(archive_bytes) != verified["archive_sha256"]: - raise ValueError("archive changed after verification") - plugins = hermes_home / "plugins" - plugins.mkdir(parents=True, exist_ok=True, mode=0o700) - if plugins.is_symlink(): - raise ValueError("Hermes plugins directory must not be a symlink") - target = plugins / PLUGIN_NAME - if target.is_symlink(): - raise ValueError("existing plugin directory must not be a symlink") - with tempfile.TemporaryDirectory(prefix="substrate-plugin-", dir=plugins) as directory: - staging_root = Path(directory) - with zipfile.ZipFile(io.BytesIO(archive_bytes)) as bundle: - bundle.extractall(staging_root) - staged = staging_root / PLUGIN_NAME - if not (staged / "plugin.yaml").is_file() or not (staged / "__init__.py").is_file(): - raise ValueError("archive is missing required plugin files") - installed = not target.exists() - rollback: Path | None = None - if target.exists(): - rollback = plugins / f"{PLUGIN_NAME}.rollback-{time.time_ns()}" - if _entry_exists(rollback): - raise OSError("plugin rollback path already exists") - os.replace(target, rollback) - try: - os.replace(staged, target) - _harden_plugin_permissions(target) - except Exception: - _restore_plugin_after_failed_install(plugins, target, rollback) - raise - result = { - **verified, - "action": "installed" if installed else "upgraded", - "target": os.fspath(target), - "rollback": os.fspath(rollback) if rollback is not None else None, - } - if install_service: - try: - result.update(install_import_service(hermes_home, target, env_path=env_path)) - except Exception: - _restore_plugin_after_failed_install(plugins, target, rollback) - raise - return result - - -def main() -> int: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--archive", type=Path, required=True) - parser.add_argument( - "--hermes-home", - type=Path, - default=Path(os.environ.get("HERMES_HOME") or Path.home() / ".hermes"), - ) - parser.add_argument("--sha256", required=True) - parser.add_argument("--install-import-service", action="store_true") - parser.add_argument("--env-path", type=Path) - parser.add_argument("--yes", action="store_true") - parser.add_argument("--json", action="store_true") - args = parser.parse_args() - if not args.yes: - parser.error("--yes is required for installation") - try: - result = install( - args.archive.expanduser().absolute(), - args.hermes_home.expanduser().absolute(), - expected_sha256=args.sha256, - install_service=args.install_import_service, - env_path=args.env_path, - ) - except ( - OSError, - ValueError, - zipfile.BadZipFile, - json.JSONDecodeError, - subprocess.SubprocessError, - ) as exc: - message = {"error": type(exc).__name__, "installed": False} - print( - json.dumps(message, sort_keys=True) - if args.json - else f"Installation failed: {type(exc).__name__}" - ) - return 1 - print( - json.dumps(result, sort_keys=True) - if args.json - else f"Substrate plugin {result['action']}: {result['target']}" - ) - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/legacy-assets/1.4.1/substrate_wiki.zip b/legacy-assets/1.4.1/substrate_wiki.zip deleted file mode 100644 index 93564ea..0000000 Binary files a/legacy-assets/1.4.1/substrate_wiki.zip and /dev/null differ diff --git a/plugins/substrate/README.md b/plugins/substrate/README.md index d501e97..a688d69 100644 --- a/plugins/substrate/README.md +++ b/plugins/substrate/README.md @@ -8,11 +8,9 @@ the associative editor, and evidence. The plugin caches no memory and fails clos Install this directory, not the repository root: ```sh -substrate_ref="$(git ls-remote https://github.com/Substrate-memory/Substrate-memory-plugins.git refs/heads/main | awk '{print $1}')" -printf '%s\n' "$substrate_ref" | grep -Eq '^[0-9a-f]{40}$' hermes plugins install \ Substrate-memory/Substrate-memory-plugins/plugins/substrate \ - --ref "$substrate_ref" --no-enable + --ref v0.3.0 --no-enable ``` No manual credential setup is required. The plugin connects itself on first use: diff --git a/plugins/substrate/after-install.md b/plugins/substrate/after-install.md index 08856bd..ce6bc77 100644 --- a/plugins/substrate/after-install.md +++ b/plugins/substrate/after-install.md @@ -1,8 +1,11 @@ # Complete Substrate setup -The plugin files are installed. The installing agent must finish setup for the same active -Hermes profile. Run this block and keep the setup process open while authorization is -pending: +No manual credential step is required. The plugin runs device onboarding by itself +on the first turn or first `memory_search` call: it shows a browser approval link +through the agent, and after approval the tenant-scoped key is stored privately in +the active profile's `.env`. Never ask the user to paste a key into chat. + +Optionally, to connect during installation instead of on first use, run: ```sh profile_home="${HERMES_HOME:-$(dirname "$(hermes config path)")}" @@ -10,11 +13,10 @@ HERMES_HOME="$profile_home" python3 "$profile_home/plugins/substrate/setup.py" \ --hermes-home "$profile_home" ``` -If setup prints `verification_uri_complete`, show that exact URL to the user and ask them -to approve it. Do not expose or request an access token. Continue only after setup prints -`"status": "ready"`. +If it prints `verification_uri_complete`, show that exact URL to the user and ask them +to approve it in a browser. Do not expose or request an access token. -Then complete the cutover: +Then complete the cutover (only after a memory call succeeds): ```sh config_path="$(hermes config path)" @@ -42,4 +44,5 @@ In a new agent turn, call `memory_search`. Do not use the legacy `wiki_search` t Report success only when `memory_search` reaches the authenticated Substrate API. Do not disable TLS verification, install a private certificate, or inspect a different -Hermes profile. `setup.py` adds only the bundled public ISRG roots to normal system trust. +Hermes profile. The plugin supplements system trust only with the bundled public ISRG +roots. diff --git a/pyproject.toml b/pyproject.toml index c2d3aa5..e80c20b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,11 +1,7 @@ -[build-system] -requires = ["setuptools==83.0.0"] -build-backend = "setuptools.build_meta" - [project] -name = "hermes-substrate-wiki" -version = "2.0.5" -description = "Substrate memory-provider plugin for Hermes Agent" +name = "substrate-retrieval" +version = "0.3.0" +description = "Substrate retrieval plugin for Hermes Agent" readme = "README.md" requires-python = ">=3.11" license = { file = "LICENSE" } @@ -23,15 +19,8 @@ dev = [ "pytest==9.0.3", "pytest-timeout==2.3.1", "ruff==0.9.10", - "setuptools==83.0.0", ] -[tool.setuptools] -package-dir = {"" = "src"} - -[tool.setuptools.package-data] -substrate_wiki = ["README.md", "plugin.yaml", "py.typed"] - [tool.pytest.ini_options] testpaths = ["tests"] addopts = "-ra" diff --git a/scripts/benchmark_import_memory.py b/scripts/benchmark_import_memory.py deleted file mode 100644 index 4505c16..0000000 --- a/scripts/benchmark_import_memory.py +++ /dev/null @@ -1,228 +0,0 @@ -#!/usr/bin/env python3 -"""Generate history larger than RAM budgets and verify bounded importer RSS.""" - -from __future__ import annotations - -import argparse -import ctypes -import json -import os -import sqlite3 -import subprocess -import sys -import tempfile -from pathlib import Path -from typing import Any - -REPOSITORY_ROOT = Path(__file__).resolve().parents[1] -sys.path.insert(0, os.fspath(REPOSITORY_ROOT / "src")) - -from substrate_wiki.history import ( # noqa: E402 - HermesHistoryImporter, - HermesJSONLHistorySource, - HermesSQLiteHistorySource, -) - - -class SinkClient: - def capabilities(self) -> dict[str, Any]: - return { - "provider": "substrate_wiki", - "capture_schema_versions": [2], - "max_event_bytes": 262_144, - "history_replay": { - "protocol": "stream-v2", - "min_plugin_version": "1.2.0", - "content_free_completion": True, - "incremental_windows": True, - "status_version": 2, - }, - } - - def request(self, method: str, path: str, **kwargs: Any) -> dict[str, Any]: - del method, path, kwargs - return {"duplicate": False} - - def import_status(self, batch_id: str) -> dict[str, Any]: - return { - "batch_id": batch_id, - "processed_windows": 1, - "processed": 1, - "pending_review": 0, - "failed": 0, - "complete": True, - } - - -def _create_history(path: Path, *, size_mib: int, single_message_mib: int) -> None: - connection = sqlite3.connect(path) - try: - connection.executescript( - """ - CREATE TABLE sessions ( - id TEXT PRIMARY KEY, source TEXT NOT NULL, user_id TEXT, - chat_type TEXT, started_at REAL NOT NULL - ); - CREATE TABLE messages ( - id INTEGER PRIMARY KEY AUTOINCREMENT, session_id TEXT NOT NULL, - role TEXT NOT NULL, content TEXT NOT NULL, timestamp REAL NOT NULL, - active INTEGER NOT NULL DEFAULT 1 - ); - INSERT INTO sessions VALUES ('memory-benchmark', 'cli', NULL, NULL, 1.0); - """ - ) - if single_message_mib: - content = "x" * (single_message_mib * 1024 * 1024) - connection.execute( - "INSERT INTO messages(session_id, role, content, timestamp) VALUES (?, ?, ?, ?)", - ("memory-benchmark", "user", content, 1.0), - ) - else: - content = "x" * (1024 * 1024) - connection.executemany( - "INSERT INTO messages(session_id, role, content, timestamp) VALUES (?, ?, ?, ?)", - ( - ("memory-benchmark", "user" if index % 2 == 0 else "assistant", content, index) - for index in range(size_mib) - ), - ) - connection.commit() - finally: - connection.close() - - -def _create_jsonl_history(path: Path, *, size_mib: int, single_message_mib: int) -> None: - message_mib = single_message_mib or 1 - message_count = 1 if single_message_mib else size_mib - content = "x" * (message_mib * 1024 * 1024) - with path.open("w", encoding="utf-8", newline="\n") as stream: - stream.write('{"id":"memory-benchmark","source":"cli","messages":[') - for index in range(message_count): - if index: - stream.write(",") - role = "user" if index % 2 == 0 else "assistant" - stream.write(f'{{"role":"{role}","content":"') - stream.write(content) - stream.write(f'","timestamp":{index}}}') - stream.write("]}\n") - - -def _peak_rss_bytes() -> int: - if sys.platform == "win32": - class ProcessMemoryCounters(ctypes.Structure): - _fields_ = [ - ("cb", ctypes.c_ulong), - ("page_fault_count", ctypes.c_ulong), - ("peak_working_set_size", ctypes.c_size_t), - ("working_set_size", ctypes.c_size_t), - ("quota_peak_paged_pool_usage", ctypes.c_size_t), - ("quota_paged_pool_usage", ctypes.c_size_t), - ("quota_peak_non_paged_pool_usage", ctypes.c_size_t), - ("quota_non_paged_pool_usage", ctypes.c_size_t), - ("pagefile_usage", ctypes.c_size_t), - ("peak_pagefile_usage", ctypes.c_size_t), - ] - - counters = ProcessMemoryCounters() - counters.cb = ctypes.sizeof(counters) - get_current_process = ctypes.windll.kernel32.GetCurrentProcess # type: ignore[attr-defined] - get_current_process.argtypes = [] - get_current_process.restype = ctypes.c_void_p - get_process_memory_info = ctypes.windll.psapi.GetProcessMemoryInfo # type: ignore[attr-defined] - get_process_memory_info.argtypes = [ - ctypes.c_void_p, - ctypes.POINTER(ProcessMemoryCounters), - ctypes.c_ulong, - ] - get_process_memory_info.restype = ctypes.c_int - success = get_process_memory_info( - get_current_process(), - ctypes.byref(counters), - counters.cb, - ) - return int(counters.peak_working_set_size) if success else 0 - try: - import resource - - getrusage = getattr(resource, "getrusage", None) - rusage_self = getattr(resource, "RUSAGE_SELF", None) - if not callable(getrusage) or rusage_self is None: - return 0 - value = int(getrusage(rusage_self).ru_maxrss) - return value if sys.platform == "darwin" else value * 1024 - except ImportError: - return 0 - - -def main(argv: list[str] | None = None) -> int: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--size-mib", type=int, default=2048) - parser.add_argument("--single-message-mib", type=int, default=0) - parser.add_argument("--limit-mib", type=int, default=256) - parser.add_argument("--source", choices=("sqlite", "jsonl"), default="sqlite") - parser.add_argument("--generate-only", type=Path, help=argparse.SUPPRESS) - args = parser.parse_args(argv) - if args.generate_only is not None: - creator = _create_history if args.source == "sqlite" else _create_jsonl_history - creator( - args.generate_only, - size_mib=max(1, args.size_mib), - single_message_mib=max(0, args.single_message_mib), - ) - return 0 - with tempfile.TemporaryDirectory(prefix="substrate-memory-benchmark-") as directory: - home = Path(directory) / "hermes" - home.mkdir() - history_path = home / ("state.db" if args.source == "sqlite" else "history.jsonl") - # Fixture creation can transiently copy a large SQLite parameter. Run - # it in a short-lived child so this process's ru_maxrss measures the - # importer itself rather than test-data generation. - subprocess.run( - [ - sys.executable, - os.fspath(Path(__file__).resolve()), - "--generate-only", - os.fspath(history_path), - "--size-mib", - str(max(1, args.size_mib)), - "--single-message-mib", - str(max(0, args.single_message_mib)), - "--source", - args.source, - ], - check=True, - ) - source = ( - HermesSQLiteHistorySource(history_path) - if args.source == "sqlite" - else HermesJSONLHistorySource(history_path) - ) - importer = HermesHistoryImporter( - hermes_home=home, - client=SinkClient(), # type: ignore[arg-type] - source=source, - ) - try: - status = importer.run(wait=True) - peak = _peak_rss_bytes() - finally: - if importer.checkpoint is not None: - importer.checkpoint.close() - result = { - "complete": bool(status.get("complete")), - "peak_rss_bytes": peak, - "limit_bytes": args.limit_mib * 1024 * 1024, - "history_mib": args.single_message_mib or args.size_mib, - "single_message": bool(args.single_message_mib), - "source": args.source, - } - print(json.dumps(result, sort_keys=True)) - if not result["complete"]: - return 1 - if peak and peak > result["limit_bytes"]: - return 2 - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/scripts/benchmark_migration.py b/scripts/benchmark_migration.py deleted file mode 100644 index 8b86d8a..0000000 --- a/scripts/benchmark_migration.py +++ /dev/null @@ -1,1052 +0,0 @@ -#!/usr/bin/env python3 -"""Measure a privacy-safe stream-v2 Hermes migration baseline. - -The transfer path uses the production importer and SQLite/JSONL readers. Hosted -provider behavior is never invoked: downstream provider, queue, resolution, -projection and summary work is an explicitly labeled deterministic simulator. -Only aggregate counters and timings are retained. -""" - -from __future__ import annotations - -import argparse -import concurrent.futures -import hashlib -import json -import os -import queue -import re -import socket -import sqlite3 -import subprocess -import sys -import tempfile -import threading -import time -from collections.abc import Iterator, Mapping -from pathlib import Path -from typing import Any -from unittest import mock - -from substrate_wiki.history import ( - HermesHistoryImporter, - HermesJSONLHistorySource, - HermesSQLiteHistorySource, -) - -ROOT = Path(__file__).resolve().parents[1] - -PHASE_NAMES = ( - "discovery", - "source_reading", - "normalization_redaction", - "request_encoding", - "network_wait", - "server_persistence", - "queue_delay", - "provider_extraction", - "entity_resolution", - "projection_indexing", - "summary_reduction", -) -_CONTENT_FIELDS = { - "content", - "messages", - "message", - "prompt", - "prompts", - "transcript", - "transcripts", - "secret", - "secrets", - "password", - "api_key", -} -_IDENTIFIER_FIELDS = { - "agent_id", - "batch_id", - "event_id", - "job_id", - "message_id", - "session_id", - "source_id", - "source_locator", - "user_id", -} -_SECRET_VALUE_MARKERS = ( - "api_key=", - "authorization:", - "bearer ", - "ghp_", - "password=", - "sk-", -) -_CANARY = "sk-BENCHMARKCANARY1234567890" - - -class PhaseMetrics: - def __init__(self) -> None: - self.values = {name: 0.0 for name in PHASE_NAMES} - self._lock = threading.Lock() - - def add(self, name: str, elapsed: float) -> None: - with self._lock: - self.values[name] += max(0.0, elapsed) - - def measured(self, name: str): # type: ignore[no-untyped-def] - metrics = self - - class Timer: - def __enter__(self) -> None: - self.started = time.perf_counter() - - def __exit__(self, *_args: object) -> None: - metrics.add(name, time.perf_counter() - self.started) - - return Timer() - - -class InstrumentedSource: - def __init__(self, source: Any, metrics: PhaseMetrics) -> None: - self.source = source - self.metrics = metrics - self.source_kind = str(source.source_kind) - self.source_locator = str(source.source_locator) - self._yielded_at: float | None = None - self._lock = threading.Lock() - - def discover(self, checkpoint: Any) -> dict[str, int]: - with self.metrics.measured("discovery"): - return self.source.discover(checkpoint) - - def iter_messages(self, session: Any, *, start: int) -> Iterator[dict[str, Any]]: - iterator = iter(self.source.iter_messages(session, start=start)) - while True: - began = time.perf_counter() - try: - value = next(iterator) - except StopIteration: - self.metrics.add("source_reading", time.perf_counter() - began) - return - self.metrics.add("source_reading", time.perf_counter() - began) - with self._lock: - self._yielded_at = time.perf_counter() - yield value - - def take_normalization_elapsed(self) -> float: - with self._lock: - yielded_at, self._yielded_at = self._yielded_at, None - return 0.0 if yielded_at is None else time.perf_counter() - yielded_at - - -class AggregateSink: - """Content-free durable sink plus deterministic downstream simulator queue.""" - - def __init__( - self, - database: Path, - source: InstrumentedSource, - metrics: PhaseMetrics, - *, - network_latency: float, - ) -> None: - self.source = source - self.metrics = metrics - self.network_latency = network_latency - self.connection = sqlite3.connect(database, check_same_thread=False) - self.connection.execute("PRAGMA journal_mode=WAL") - self.connection.execute("PRAGMA synchronous=FULL") - self.connection.execute( - "CREATE TABLE received(sequence INTEGER PRIMARY KEY, event_id TEXT NOT NULL UNIQUE, kind TEXT NOT NULL, encoded_bytes INTEGER NOT NULL)" - ) - self.connection.execute( - "CREATE TABLE projected(sequence INTEGER PRIMARY KEY, resolution_digest TEXT NOT NULL)" - ) - self.connection.commit() - self.requests = 0 - self.retries = 0 - self.encoded_bytes = 0 - self.redaction_failures = 0 - self.redacted_events = 0 - self.duplicate_acks = 0 - self.replay_event: dict[str, Any] | None = None - self.queue: list[tuple[int, float]] = [] - self.work_queue: queue.Queue[tuple[int, float] | None] = queue.Queue() - self.max_queue_depth = 0 - self._lock = threading.Lock() - - def capabilities(self) -> dict[str, Any]: - return { - "provider": "substrate_wiki", - "capture_schema_versions": [2], - "max_event_bytes": 262_144, - "history_replay": { - "protocol": "stream-v2", - "min_plugin_version": "1.2.0", - "content_free_completion": True, - "incremental_windows": True, - "status_version": 2, - }, - } - - def request(self, method: str, path: str, **kwargs: Any) -> dict[str, Any]: - del method, path - self.metrics.add("normalization_redaction", self.source.take_normalization_elapsed()) - event = kwargs.get("body") - if not isinstance(event, dict): - raise TypeError("benchmark sink requires a mapping event") - with self.metrics.measured("request_encoding"): - encoded = json.dumps( - event, ensure_ascii=False, separators=(",", ":"), sort_keys=True - ).encode("utf-8") - with self.metrics.measured("network_wait"): - if self.network_latency: - time.sleep(self.network_latency) - kind = str(event.get("kind") or "unknown") - event_id = str(event.get("event_id") or hashlib.sha256(encoded).hexdigest()) - with self.metrics.measured("server_persistence"): - with self._lock: - if self.connection.execute( - "SELECT 1 FROM received WHERE event_id = ?", (event_id,) - ).fetchone(): - self.duplicate_acks += 1 - return {"duplicate": True} - if _CANARY.encode() in encoded: - self.redaction_failures += 1 - if b"[REDACTED]" in encoded: - self.redacted_events += 1 - self.requests += 1 - sequence = self.requests - self.encoded_bytes += len(encoded) - self.connection.execute( - "INSERT INTO received(sequence, event_id, kind, encoded_bytes) VALUES (?, ?, ?, ?)", - (sequence, event_id, kind, len(encoded)), - ) - self.connection.commit() - if kind == "session_end": - if self.replay_event is None: - self.replay_event = dict(event) - item = (sequence, time.perf_counter()) - self.queue.append(item) - self.work_queue.put(item) - self.max_queue_depth = max(self.max_queue_depth, self.work_queue.qsize()) - return {"duplicate": False} - - def import_status(self, batch_id: str) -> dict[str, Any]: - return { - "batch_id": batch_id, - "processed": len(self.queue), - "processed_windows": len(self.queue), - "pending_review": 0, - "failed": 0, - "complete": True, - } - - def close(self) -> None: - self.connection.close() - - def finish_transfer(self) -> None: - self.work_queue.put(None) - - def durable_state(self) -> dict[str, Any]: - """Fingerprint every durable or queued side effect after workers settle.""" - with self._lock: - received = self.connection.execute( - "SELECT sequence, event_id, kind, encoded_bytes FROM received ORDER BY sequence" - ).fetchall() - projected = self.connection.execute( - "SELECT sequence, resolution_digest FROM projected ORDER BY sequence" - ).fetchall() - received_digest = hashlib.sha256( - json.dumps(received, separators=(",", ":")).encode("utf-8") - ).hexdigest() - projected_digest = hashlib.sha256( - json.dumps(projected, separators=(",", ":")).encode("utf-8") - ).hexdigest() - with self.work_queue.mutex: - work_queue_items = tuple( - None if item is None else item[0] for item in self.work_queue.queue - ) - return { - "duplicate_acks": self.duplicate_acks, - "encoded_bytes": self.encoded_bytes, - "max_queue_depth": self.max_queue_depth, - "projected_digest": projected_digest, - "projected_rows": len(projected), - "queue_items": tuple(sequence for sequence, _created_at in self.queue), - "received_digest": received_digest, - "received_rows": len(received), - "redacted_events": self.redacted_events, - "redaction_failures": self.redaction_failures, - "requests": self.requests, - "work_queue_items": work_queue_items, - } - - -def _message(case_id: str, session_index: int, message_index: int, size: int) -> str: - prefix = ( - f"synthetic fixture={case_id} session={session_index:08d} item={message_index:04d} " - f"api_key={_CANARY} " - ) - if len(prefix) >= size: - return prefix[:size] - return prefix + ("x" * (size - len(prefix))) - - -def _generate_fixture(case: Mapping[str, Any], output: Path) -> None: - sessions = int(case["sessions"]) - messages_per_session = int(case["messages_per_session"]) - ordinary_size = int(case["message_bytes"]) - oversized = int(case.get("oversized_message_bytes", 0)) - case_id = str(case["id"]) - if case["source"] == "sqlite": - connection = sqlite3.connect(output) - try: - connection.executescript( - """ - CREATE TABLE sessions ( - id TEXT PRIMARY KEY, source TEXT NOT NULL, user_id TEXT, - chat_type TEXT, started_at REAL NOT NULL - ); - CREATE TABLE messages ( - id INTEGER PRIMARY KEY AUTOINCREMENT, session_id TEXT NOT NULL, - role TEXT NOT NULL, content TEXT NOT NULL, timestamp REAL NOT NULL, - active INTEGER NOT NULL DEFAULT 1 - ); - """ - ) - for session_index in range(sessions): - session_id = f"baseline-{case_id}-{session_index:08d}" - connection.execute( - "INSERT INTO sessions VALUES (?, 'cli', NULL, NULL, ?)", - (session_id, float(session_index)), - ) - for message_index in range(messages_per_session): - size = ( - oversized - if oversized and session_index == message_index == 0 - else ordinary_size - ) - connection.execute( - "INSERT INTO messages(session_id, role, content, timestamp) VALUES (?, ?, ?, ?)", - ( - session_id, - "user" if message_index % 2 == 0 else "assistant", - _message(case_id, session_index, message_index, size), - float(message_index), - ), - ) - connection.commit() - finally: - connection.close() - return - with output.open("w", encoding="utf-8", newline="\n") as stream: - for session_index in range(sessions): - messages = [] - for message_index in range(messages_per_session): - size = ( - oversized - if oversized and session_index == message_index == 0 - else ordinary_size - ) - messages.append( - { - "role": "user" if message_index % 2 == 0 else "assistant", - "content": _message(case_id, session_index, message_index, size), - "timestamp": float(message_index), - } - ) - json.dump( - { - "id": f"baseline-{case_id}-{session_index:08d}", - "source": "cli", - "messages": messages, - }, - stream, - ensure_ascii=False, - separators=(",", ":"), - sort_keys=True, - ) - stream.write("\n") - - -def _current_rss_bytes() -> int: - """Return current process RSS, avoiding inherited lifetime high-water marks.""" - - status_path = Path("/proc/self/status") - try: - for line in status_path.read_text(encoding="ascii").splitlines(): - if line.startswith("VmRSS:"): - return int(line.split()[1]) * 1024 - except (IndexError, OSError, ValueError): - pass - try: - page_size = os.sysconf("SC_PAGE_SIZE") - resident_pages = int(Path("/proc/self/statm").read_text(encoding="ascii").split()[1]) - return resident_pages * page_size - except (IndexError, OSError, ValueError): - return 0 - - -class _PeakRSSSampler: - """Sample current RSS for one isolated case worker.""" - - def __init__(self, *, interval_seconds: float = 0.005) -> None: - self._interval_seconds = interval_seconds - self._stop = threading.Event() - self._thread = threading.Thread(target=self._sample_until_stopped, daemon=True) - self.peak_bytes = _current_rss_bytes() - - def _sample(self) -> None: - self.peak_bytes = max(self.peak_bytes, _current_rss_bytes()) - - def _sample_until_stopped(self) -> None: - while not self._stop.wait(self._interval_seconds): - self._sample() - - def start(self) -> None: - self._thread.start() - - def stop(self) -> int: - self._sample() - self._stop.set() - self._thread.join(timeout=1.0) - self._sample() - return self.peak_bytes - - -def _payload_bytes(case: Mapping[str, Any]) -> int: - sessions = int(case["sessions"]) - messages = int(case["messages_per_session"]) - ordinary = sessions * messages * int(case["message_bytes"]) - oversized = int(case.get("oversized_message_bytes", 0)) - return ordinary - (int(case["message_bytes"]) if oversized else 0) + oversized - - -def _run_downstream( - sink: AggregateSink, - metrics: PhaseMetrics, - *, - concurrency: int, - expected_windows: int, - provider: Mapping[str, Any], - started: float, -) -> tuple[float, float, dict[str, Any]]: - first_usable: list[float] = [] - queue_ages: list[float] = [] - provider_requests = 0 - provider_retries = 0 - provider_lock = threading.Lock() - worker_threads: set[int] = set() - active_workers = 0 - max_in_flight = 0 - startup_parties = min(concurrency, expected_windows) - startup_barrier = threading.Barrier(startup_parties) - tasks_started = 0 - latency = float(provider["observed_latency_seconds"]) - retry_every = int(provider["retry_every_requests"]) - summary_every = int(provider["summary_every_windows"]) - - def process(item: tuple[int, float]) -> None: - nonlocal active_workers, max_in_flight, provider_requests, provider_retries, tasks_started - sequence, enqueued_at = item - with provider_lock: - worker_threads.add(threading.get_ident()) - active_workers += 1 - max_in_flight = max(max_in_flight, active_workers) - task_number = tasks_started - tasks_started += 1 - try: - if task_number < startup_parties: - try: - startup_barrier.wait(timeout=1.0) - except threading.BrokenBarrierError: - pass - age = time.perf_counter() - enqueued_at - with provider_lock: - queue_ages.append(age) - metrics.add("queue_delay", age) - with metrics.measured("provider_extraction"): - time.sleep(latency) - retry = retry_every > 0 and sequence % retry_every == 0 - if retry: - time.sleep(latency) - with provider_lock: - provider_requests += 1 - provider_retries += int(retry) - with metrics.measured("entity_resolution"): - digest = hashlib.sha256(f"synthetic-resolution:{sequence}".encode()).hexdigest() - with metrics.measured("projection_indexing"): - with sink._lock: - sink.connection.execute( - "INSERT INTO projected(sequence, resolution_digest) VALUES (?, ?)", - (sequence, digest), - ) - sink.connection.commit() - if sequence % summary_every == 0: - with metrics.measured("summary_reduction"): - hashlib.sha256(f"synthetic-summary:{sequence}".encode()).digest() - with provider_lock: - if not first_usable: - first_usable.append(time.perf_counter() - started) - finally: - with provider_lock: - active_workers -= 1 - - futures: list[concurrent.futures.Future[None]] = [] - with concurrent.futures.ThreadPoolExecutor(max_workers=concurrency) as executor: - while True: - item = sink.work_queue.get() - if item is None: - break - futures.append(executor.submit(process, item)) - for future in futures: - future.result() - if provider_requests: - with metrics.measured("summary_reduction"): - hashlib.sha256(f"synthetic-final-summary:{provider_requests}".encode()).digest() - fully_ready = time.perf_counter() - started - rpm = int(provider["modeled_requests_per_minute"]) - cost = float(provider["modeled_cost_usd_per_request"]) - modeled_provider_seconds = provider_requests * 60.0 / rpm if rpm else 0.0 - return ( - first_usable[0] if first_usable else fully_ready, - fully_ready, - { - "requests": provider_requests, - "retries": provider_retries, - "observed_time_seconds": round(metrics.values["provider_extraction"], 6), - "modeled_quota_seconds": round(modeled_provider_seconds, 6), - "modeled_cost_usd": round(provider_requests * cost, 6), - "quota_requests_per_minute": rpm, - "max_queue_depth": sink.max_queue_depth, - "max_queue_age_seconds": round(max(queue_ages, default=0.0), 6), - "worker_threads_used": len(worker_threads), - "max_in_flight": max_in_flight, - }, - ) - - -def _run_case( - case: Mapping[str, Any], - concurrency: int, - provider: Mapping[str, Any], - fixture: Path, -) -> dict[str, Any]: - rss_sampler = _PeakRSSSampler() - rss_sampler.start() - metrics = PhaseMetrics() - base_source = ( - HermesSQLiteHistorySource(fixture) - if case["source"] == "sqlite" - else HermesJSONLHistorySource(fixture) - ) - source = InstrumentedSource(base_source, metrics) - sink = AggregateSink( - fixture.parent / "server.db", - source, - metrics, - network_latency=float(provider["observed_latency_seconds"]), - ) - importer = HermesHistoryImporter( - hermes_home=fixture.parent / "hermes-home", - client=sink, # type: ignore[arg-type] - source=source, - agent_id="baseline-agent", - ) - wall_started = time.perf_counter() - cpu_started = time.process_time() - terminal_failures = 0 - try: - with concurrent.futures.ThreadPoolExecutor(max_workers=1) as controller: - downstream_future = controller.submit( - _run_downstream, - sink, - metrics, - concurrency=concurrency, - expected_windows=int(case["sessions"]), - provider=provider, - started=wall_started, - ) - try: - importer.run(wait=False) - transferred = time.perf_counter() - wall_started - if sink.replay_event is None: - raise RuntimeError("benchmark importer emitted no replayable session_end event") - finally: - sink.finish_transfer() - first_usable, fully_ready, downstream = downstream_future.result() - before_replay = sink.durable_state() - replay = sink.request("POST", "/api/v1/hermes/replay-probe", body=sink.replay_event) - after_replay = sink.durable_state() - expected_after_replay = dict(before_replay) - expected_after_replay["duplicate_acks"] += 1 - duplicate_side_effects = int(expected_after_replay != after_replay) - duplicate_acks = int( - replay.get("duplicate") is True - and after_replay["duplicate_acks"] - before_replay["duplicate_acks"] == 1 - ) - except Exception: - terminal_failures = 1 - raise - finally: - if importer.checkpoint is not None: - importer.checkpoint.close() - peak_rss_bytes = rss_sampler.stop() - cpu_seconds = time.process_time() - cpu_started - source_bytes = fixture.stat().st_size - payload_bytes = _payload_bytes(case) - windows = int(case["sessions"]) - wall_seconds = fully_ready - projected = int(sink.connection.execute("SELECT COUNT(*) FROM projected").fetchone()[0]) - sink.close() - events_per_second = sink.requests / transferred if transferred else 0.0 - mib_per_second = payload_bytes / (1024 * 1024) / transferred if transferred else 0.0 - windows_per_minute = windows * 60.0 / fully_ready if fully_ready else 0.0 - result = { - "case_id": str(case["id"]), - "fixture_kind": str(case["fixture_kind"]), - "concurrency": concurrency, - "fixture_sha256": hashlib.sha256(fixture.read_bytes()).hexdigest(), - "source_file_bytes": source_bytes, - "input_bytes": payload_bytes, - "windows": windows, - "events": sink.requests, - "wall_time_seconds": round(wall_seconds, 6), - "cpu_seconds": round(cpu_seconds, 6), - "peak_rss_bytes": peak_rss_bytes, - "events_per_second": round(events_per_second, 3), - "mib_per_second": round(mib_per_second, 3), - "windows_per_minute": round(windows_per_minute, 3), - "requests": sink.requests, - "retries": sink.retries + int(downstream["retries"]), - "terminal_failures": terminal_failures, - "phase_seconds": {name: round(metrics.values[name], 6) for name in PHASE_NAMES}, - "traces": [ - {"phase": name, "seconds": round(metrics.values[name], 6)} for name in PHASE_NAMES - ], - "lifecycle_seconds": { - "transferred": round(transferred, 6), - "first_usable": round(first_usable, 6), - "fully_ready": round(fully_ready, 6), - }, - "queue": { - "max_depth": int(downstream["max_queue_depth"]), - "max_age_seconds": downstream["max_queue_age_seconds"], - }, - "provider": { - key: downstream[key] - for key in ( - "requests", - "retries", - "observed_time_seconds", - "modeled_quota_seconds", - "modeled_cost_usd", - "quota_requests_per_minute", - "worker_threads_used", - "max_in_flight", - ) - }, - "integrity": { - "expected_windows": windows, - "projected_windows": projected, - "redacted_events": sink.redacted_events, - "redaction_failures": sink.redaction_failures, - "idempotency_replays": 1, - "idempotency_replay_kind": "session_end", - "idempotency_state_components": sorted(before_replay), - "duplicate_acks": duplicate_acks, - "duplicate_side_effects": duplicate_side_effects, - "complete": ( - projected == windows - and sink.redacted_events > 0 - and sink.redaction_failures == 0 - and duplicate_acks == 1 - and duplicate_side_effects == 0 - ), - }, - "quality": { - "projection_failures": 0, - "terminal_failures": terminal_failures, - }, - } - return result - - -def evaluate_budget(run: Mapping[str, Any], budget: Mapping[str, Any]) -> list[str]: - """Return content-free failures for one measured run and one explicit budget.""" - - lifecycle = run["lifecycle_seconds"] - integrity = run["integrity"] - quality = run["quality"] - provider = run["provider"] - expected = max(1, int(integrity["expected_windows"])) - integrity_ratio = int(integrity["projected_windows"]) / expected - observed = { - "transferred_seconds": float(lifecycle["transferred"]), - "first_usable_seconds": float(lifecycle["first_usable"]), - "fully_ready_seconds": float(lifecycle["fully_ready"]), - "peak_rss_bytes": float(run["peak_rss_bytes"]), - "cpu_seconds": float(run["cpu_seconds"]), - "terminal_failures": float(run["terminal_failures"]), - "projection_failures": float(quality["projection_failures"]), - "redaction_failures": float(integrity["redaction_failures"]), - "provider_retries": float(provider["retries"]), - "modeled_cost_usd": float(provider["modeled_cost_usd"]), - "modeled_quota_seconds": float(provider["modeled_quota_seconds"]), - } - failures = [ - f"{name}={value} exceeds {budget['max_' + name]}" - for name, value in observed.items() - if value > float(budget["max_" + name]) - ] - if integrity_ratio < float(budget["min_integrity_ratio"]): - failures.append( - f"integrity_ratio={integrity_ratio:.6f} below {budget['min_integrity_ratio']}" - ) - if int(integrity.get("redacted_events", 0)) < int(budget.get("min_redacted_events", 1)): - failures.append( - f"redacted_events={integrity.get('redacted_events', 0)} below " - f"{budget.get('min_redacted_events', 1)}" - ) - if int(integrity.get("duplicate_acks", 0)) < int(budget.get("min_duplicate_acks", 1)): - failures.append("duplicate_acks below required idempotency proof") - if int(integrity.get("duplicate_side_effects", 1)) > int( - budget.get("max_duplicate_side_effects", 0) - ): - failures.append("duplicate_side_effects exceeds zero") - if not integrity.get("complete"): - failures.append("integrity_complete=false") - return failures - - -def load_manifest(path: Path) -> dict[str, Any]: - manifest = json.loads(path.read_text(encoding="utf-8")) - if not isinstance(manifest, dict) or manifest.get("schema_version") != 1: - raise ValueError("unsupported benchmark manifest") - if manifest.get("protocol") != "stream-v2": - raise ValueError("benchmark manifest must select stream-v2") - if manifest.get("concurrency") != [1, 2, 3, 4]: - raise ValueError("benchmark manifest must cover concurrency 1-4") - cases = manifest.get("cases") - if not isinstance(cases, list) or not cases: - raise ValueError("benchmark manifest must define cases") - case_id_pattern = re.compile(r"^[a-z0-9][a-z0-9-]{0,63}$") - for case in cases: - if not isinstance(case, dict) or not case_id_pattern.fullmatch(str(case.get("id", ""))): - raise ValueError("benchmark case id must be a safe lowercase slug") - return manifest - - -def validate_receipt(receipt: Any) -> None: - def visit(value: Any) -> None: - if isinstance(value, dict): - for key, child in value.items(): - normalized_key = str(key).casefold() - if normalized_key in _CONTENT_FIELDS or normalized_key in _IDENTIFIER_FIELDS: - raise ValueError("benchmark receipts must remain content-free") - visit(child) - elif isinstance(value, list): - for child in value: - visit(child) - elif isinstance(value, str): - normalized_value = value.casefold() - if _CANARY.casefold() in normalized_value or any( - marker in normalized_value for marker in _SECRET_VALUE_MARKERS - ): - raise ValueError("benchmark receipts must remain content-free") - - visit(receipt) - if not isinstance(receipt, dict) or "runs" not in receipt: - raise ValueError("benchmark receipt is incomplete") - for run in receipt["runs"]: - if set(run.get("phase_seconds", {})) != set(PHASE_NAMES): - raise ValueError("benchmark receipt has incomplete phase instrumentation") - lifecycle = run.get("lifecycle_seconds", {}) - if not ( - 0 <= lifecycle.get("transferred", -1) < lifecycle.get("fully_ready", -1) - and 0 <= lifecycle.get("first_usable", -1) < lifecycle.get("fully_ready", -1) - and lifecycle.get("transferred") != lifecycle.get("first_usable") - ): - raise ValueError("benchmark lifecycle boundaries are invalid") - - -def _tolerant_max(runs: list[Mapping[str, Any]], path: tuple[str, ...]) -> float: - values = [] - for run in runs: - value: Any = run - for component in path: - value = value[component] - values.append(float(value)) - return round(max(values) * 1.5 + 2.0, 6) - - -def derive_budget(receipt: Mapping[str, Any]) -> dict[str, Any]: - """Derive per-case beta ceilings from the complete retained concurrency matrix.""" - - validate_receipt(receipt) - grouped: dict[str, list[Mapping[str, Any]]] = {} - for run in receipt["runs"]: - grouped.setdefault(str(run["case_id"]), []).append(run) - if not grouped: - raise ValueError("cannot derive a budget from an empty receipt") - case_budgets: dict[str, dict[str, Any]] = {} - for case_id, runs in sorted(grouped.items()): - if sorted(int(run["concurrency"]) for run in runs) != [1, 2, 3, 4]: - raise ValueError(f"budget evidence must cover concurrency 1-4 exactly: {case_id}") - - case_budgets[case_id] = { - "evidence_runs": 4, - "max_transferred_seconds": _tolerant_max(runs, ("lifecycle_seconds", "transferred")), - "max_first_usable_seconds": _tolerant_max(runs, ("lifecycle_seconds", "first_usable")), - "max_fully_ready_seconds": _tolerant_max(runs, ("lifecycle_seconds", "fully_ready")), - "max_peak_rss_bytes": 256 * 1024 * 1024, - "max_cpu_seconds": _tolerant_max(runs, ("cpu_seconds",)), - "max_terminal_failures": 0, - "max_projection_failures": 0, - "max_redaction_failures": 0, - "min_redacted_events": min(int(run["integrity"]["redacted_events"]) for run in runs), - "min_duplicate_acks": 1, - "max_duplicate_side_effects": 0, - "min_integrity_ratio": 1.0, - "max_provider_retries": max(int(run["provider"]["retries"]) for run in runs), - "max_modeled_cost_usd": max(float(run["provider"]["modeled_cost_usd"]) for run in runs), - "max_modeled_quota_seconds": max( - float(run["provider"]["modeled_quota_seconds"]) for run in runs - ), - } - return { - "schema_version": 1, - "protocol": receipt["protocol"], - "profile": receipt["profile"], - "manifest_sha256": receipt["manifest_sha256"], - "harness_sha256": receipt["harness_sha256"], - "receipt_sha256": "", - "derivation": { - "evidence_runs": len(receipt["runs"]), - "required_concurrency": [1, 2, 3, 4], - "latency_cpu_multiplier": 1.5, - "host_jitter_seconds": 2.0, - "peak_rss_ceiling_bytes": 256 * 1024 * 1024, - }, - "cases": case_budgets, - } - - -def verify_budget(receipt: Mapping[str, Any], budget: Mapping[str, Any]) -> list[str]: - """Verify derivation custody and every measured run against its case budget.""" - - expected = derive_budget(receipt) - receipt_bytes = (json.dumps(receipt, indent=2, sort_keys=True) + "\n").encode() - expected["receipt_sha256"] = hashlib.sha256(receipt_bytes).hexdigest() - failures = [] if budget == expected else ["budget artifact does not match canonical derivation"] - case_budgets = budget.get("cases", {}) - if not isinstance(case_budgets, dict): - return failures + ["budget cases must be a mapping"] - for run in receipt["runs"]: - case_budget = case_budgets.get(run["case_id"]) - if not isinstance(case_budget, dict): - failures.append(f"missing budget for case {run['case_id']}") - continue - failures.extend( - f"{run['case_id']}/c{run['concurrency']}: {failure}" - for failure in evaluate_budget(run, case_budget) - ) - return failures - - -def _receipt(manifest: Mapping[str, Any], manifest_path: Path, profile: str) -> dict[str, Any]: - cases = {str(case["id"]): case for case in manifest["cases"]} - selected = manifest["profiles"].get(profile) - if not isinstance(selected, list) or not selected: - raise ValueError(f"unknown or empty benchmark profile: {profile}") - runs: list[dict[str, Any]] = [] - with ( - tempfile.TemporaryDirectory(prefix="substrate-migration-baseline-") as directory, - mock.patch( - "socket.socket", - side_effect=RuntimeError("network access is disabled for this benchmark"), - ), - ): - root = Path(directory) - child_env = dict(os.environ) - child_env["SUBSTRATE_BENCHMARK_NETWORK_DENIED"] = "1" - for case_id in selected: - case = cases[str(case_id)] - for concurrency in manifest["concurrency"]: - case_root = root / f"{case_id}-c{concurrency}" - case_root.mkdir() - fixture = case_root / ( - "state.db" if case["source"] == "sqlite" else "official-export.jsonl" - ) - subprocess.run( - [ - sys.executable, - os.fspath(Path(__file__).resolve()), - "--generate-case", - str(case_id), - "--manifest", - os.fspath(manifest_path), - "--output", - os.fspath(fixture), - ], - check=True, - env=child_env, - ) - row_path = case_root / "run.json" - worker = subprocess.run( - [ - sys.executable, - os.fspath(Path(__file__).resolve()), - "--run-case", - str(case_id), - "--concurrency", - str(concurrency), - "--manifest", - os.fspath(manifest_path), - "--fixture", - os.fspath(fixture), - "--output", - os.fspath(row_path), - ], - check=False, - capture_output=True, - text=True, - env=child_env, - ) - if worker.returncode != 0: - raise RuntimeError(f"case worker failed: {case_id}/c{concurrency}") - row = json.loads(row_path.read_text(encoding="utf-8")) - if not isinstance(row, dict): - raise ValueError(f"case worker returned invalid row: {case_id}/c{concurrency}") - runs.append(row) - return { - "schema_version": 1, - "protocol": "stream-v2", - "profile": profile, - "manifest_sha256": hashlib.sha256(manifest_path.read_bytes()).hexdigest(), - "harness_sha256": hashlib.sha256(Path(__file__).resolve().read_bytes()).hexdigest(), - "hosted_calls": 0, - "representative_provider": { - "mode": "deterministic_simulation", - "network_access": False, - "quota_requests_per_minute": manifest["representative_provider"][ - "modeled_requests_per_minute" - ], - "cost_usd_per_request": manifest["representative_provider"][ - "modeled_cost_usd_per_request" - ], - }, - "runs": runs, - } - - -def _install_network_denial() -> None: - def deny_socket(*_args: Any, **_kwargs: Any) -> Any: - raise RuntimeError("network access is disabled for this benchmark") - - socket.socket = deny_socket # type: ignore[assignment,misc] - - -def _probe_partial_failure() -> None: - class ProbeSink: - def __init__(self) -> None: - self.work_queue: queue.Queue[tuple[int, float] | None] = queue.Queue() - self.work_queue.put((1, time.perf_counter())) - self.work_queue.put(None) - self.max_queue_depth = 1 - self._lock = threading.Lock() - self.connection = sqlite3.connect(":memory:", check_same_thread=False) - self.connection.execute( - "CREATE TABLE projected(sequence INTEGER PRIMARY KEY, resolution_digest TEXT NOT NULL)" - ) - - sink = ProbeSink() - _run_downstream( - sink, # type: ignore[arg-type] - PhaseMetrics(), - concurrency=4, - expected_windows=4, - provider={ - "observed_latency_seconds": 0.0, - "retry_every_requests": 0, - "summary_every_windows": 100, - "modeled_requests_per_minute": 60, - "modeled_cost_usd_per_request": 0.0, - }, - started=time.perf_counter(), - ) - sink.connection.close() - - -def main(argv: list[str] | None = None) -> int: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument( - "--manifest", - type=Path, - default=ROOT / "benchmarks" / "hermes-migration-manifest.json", - ) - parser.add_argument("--profile", default="ci") - parser.add_argument("--output", type=Path, required=True) - parser.add_argument("--generate-case", help=argparse.SUPPRESS) - parser.add_argument("--run-case", help=argparse.SUPPRESS) - parser.add_argument("--concurrency", type=int, help=argparse.SUPPRESS) - parser.add_argument("--fixture", type=Path, help=argparse.SUPPRESS) - parser.add_argument("--probe-network", action="store_true", help=argparse.SUPPRESS) - parser.add_argument("--probe-partial-failure", action="store_true", help=argparse.SUPPRESS) - args = parser.parse_args(argv) - if os.environ.get("SUBSTRATE_BENCHMARK_NETWORK_DENIED") == "1": - _install_network_denial() - if args.probe_network: - socket.socket() - return 0 - if args.probe_partial_failure: - _probe_partial_failure() - return 0 - manifest = load_manifest(args.manifest) - if args.generate_case: - cases = {str(case["id"]): case for case in manifest["cases"]} - case = cases.get(args.generate_case) - if case is None: - parser.error("unknown fixture case") - args.output.parent.mkdir(parents=True, exist_ok=True) - _generate_fixture(case, args.output) - return 0 - if args.run_case: - cases = {str(case["id"]): case for case in manifest["cases"]} - case = cases.get(args.run_case) - if case is None: - parser.error("unknown fixture case") - if args.concurrency not in manifest["concurrency"]: - parser.error("unsupported concurrency") - if args.fixture is None or not args.fixture.is_file(): - parser.error("fixture is required") - row = _run_case( - case, - int(args.concurrency), - manifest["representative_provider"], - args.fixture, - ) - args.output.parent.mkdir(parents=True, exist_ok=True) - args.output.write_text(json.dumps(row, sort_keys=True) + "\n", encoding="utf-8") - return 0 - receipt = _receipt(manifest, args.manifest, args.profile) - validate_receipt(receipt) - args.output.parent.mkdir(parents=True, exist_ok=True) - args.output.write_text(json.dumps(receipt, indent=2, sort_keys=True) + "\n", encoding="utf-8") - print( - json.dumps( - { - "output": os.fspath(args.output), - "profile": args.profile, - "runs": len(receipt["runs"]), - "hosted_calls": 0, - }, - sort_keys=True, - ) - ) - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/scripts/build_plugin.py b/scripts/build_plugin.py deleted file mode 100644 index 7a63ae9..0000000 --- a/scripts/build_plugin.py +++ /dev/null @@ -1,449 +0,0 @@ -#!/usr/bin/env python3 -"""Build and verify the deterministic Hermes plugin archive.""" - -from __future__ import annotations - -import argparse -import hashlib -import io -import json -import os -import re -import subprocess -import sys -import zipfile -from pathlib import Path - -REPOSITORY_ROOT = Path(__file__).resolve().parents[1] -PLUGIN_NAME = "substrate_wiki" -PLUGIN_SOURCE = REPOSITORY_ROOT / "src" / PLUGIN_NAME -ARCHIVE_PATH = REPOSITORY_ROOT / "dist" / f"{PLUGIN_NAME}.zip" -INSTALLER_PATH = REPOSITORY_ROOT / "scripts" / "install_hermes_plugin.py" -LICENSE_PATH = REPOSITORY_ROOT / "LICENSE" -RELEASES_PATH = REPOSITORY_ROOT / "release-assets" -PROVENANCE_FILENAME = "PROVENANCE.json" -LICENSE_FILENAME = "LICENSE" -TARGET_HERMES_VERSION = "0.20.0" -BUILD_FORMAT_VERSION = 3 -SOURCE_COMMIT_ENVIRONMENT_VARIABLE = "HERMES_PLUGIN_SOURCE_COMMIT" -PLUGIN_REDACTION_FILENAME = "redaction.py" -GENERATED_DETECTOR_BLOCK_START = "# BEGIN GENERATED SERVER PROVIDER DETECTORS\n" -GENERATED_DETECTOR_BLOCK_END = "# END GENERATED SERVER PROVIDER DETECTORS\n" -REQUIRED_FILES = ( - "README.md", - "__init__.py", - "cli.py", - "client.py", - "credentials.py", - "checkpoint.py", - "events.py", - "history.py", - "onboarding.py", - "plugin.yaml", - "py.typed", - "redaction.py", - "spool.py", - "supervisor.py", - "worker.py", -) -FIXED_TIMESTAMP = (1980, 1, 1, 0, 0, 0) -FILE_MODE = 0o100644 -DIRECTORY_MODE = 0o40755 - - -def _is_generated(relative_path: Path) -> bool: - return "__pycache__" in relative_path.parts or relative_path.suffix in {".pyc", ".pyo"} - - -def _require_release_clean_redaction(source: Path = PLUGIN_SOURCE) -> None: - """Reject missing or multiply generated detector blocks without server coupling.""" - - plugin = (source / PLUGIN_REDACTION_FILENAME).read_text(encoding="utf-8") - if plugin.count(GENERATED_DETECTOR_BLOCK_START) != 1 or plugin.count( - GENERATED_DETECTOR_BLOCK_END - ) != 1: - raise ValueError("plugin redaction detector block is missing or duplicated") - if "_PROVIDER_TOKEN = re.compile(" not in plugin or ( - "_PERCENT_ENCODED_PROVIDER_TOKEN = re.compile(" not in plugin - ): - raise ValueError("plugin redaction provider detectors are missing") - - -def _source_files(source: Path = PLUGIN_SOURCE) -> tuple[Path, ...]: - files = tuple( - sorted( - ( - path.relative_to(source) - for path in source.rglob("*") - if path.is_file() - and not path.is_symlink() - and not _is_generated(path.relative_to(source)) - ), - key=lambda path: path.as_posix(), - ) - ) - expected = tuple( - sorted((Path(name) for name in REQUIRED_FILES), key=lambda path: path.as_posix()) - ) - if files != expected: - missing = sorted(set(expected) - set(files), key=lambda path: path.as_posix()) - unexpected = sorted(set(files) - set(expected), key=lambda path: path.as_posix()) - details = [] - if missing: - details.append("missing: " + ", ".join(path.as_posix() for path in missing)) - if unexpected: - details.append("unexpected: " + ", ".join(path.as_posix() for path in unexpected)) - raise ValueError("plugin source is not release-clean (" + "; ".join(details) + ")") - return files - - -def _manifest_value(manifest: str, key: str) -> str: - match = re.search(rf"(?m)^{re.escape(key)}:\s*([^#\s]+)\s*$", manifest) - if match is None: - raise ValueError(f"plugin.yaml must define {key!r}") - return match.group(1) - - -def _plugin_version(source: Path = PLUGIN_SOURCE) -> str: - manifest = (source / "plugin.yaml").read_text(encoding="utf-8") - identity = _manifest_value(manifest, "name") - if identity != PLUGIN_NAME: - raise ValueError( - f"plugin.yaml name must be {PLUGIN_NAME!r} to match the package directory; got {identity!r}" - ) - return _manifest_value(manifest, "version") - - -def _validate_source_commit(source_commit: object) -> str: - if source_commit == "unknown": - return "unknown" - if isinstance(source_commit, str) and re.fullmatch(r"[0-9a-f]{40}", source_commit): - return source_commit - raise ValueError("source_commit must be 'unknown' or a full 40-character lowercase Git SHA") - - -def _source_commit() -> str: - """Return an explicitly supplied immutable commit, or a stable unknown marker.""" - candidate = os.environ.get(SOURCE_COMMIT_ENVIRONMENT_VARIABLE, "").strip().lower() - if not candidate: - return "unknown" - try: - return _validate_source_commit(candidate) - except ValueError as exc: - raise ValueError( - f"{SOURCE_COMMIT_ENVIRONMENT_VARIABLE} must be a full 40-character Git SHA" - ) from exc - - -def _require_commit_contains_source( - source_commit: str, - source: Path = PLUGIN_SOURCE, -) -> None: - """Prove that every packaged source byte exists at the claimed Git commit.""" - source = source.resolve() - try: - relative_root = source.relative_to(REPOSITORY_ROOT.resolve()) - except ValueError as exc: - raise ValueError("plugin source is outside the repository") from exc - files = _source_files(source) - expected_paths = tuple( - f"{relative_root.as_posix()}/{relative.as_posix()}" for relative in files - ) - tree = subprocess.run( - ( - "git", - "-C", - os.fspath(REPOSITORY_ROOT), - "ls-tree", - "-r", - "--name-only", - source_commit, - "--", - relative_root.as_posix(), - ), - check=False, - capture_output=True, - text=True, - ) - committed_paths = tuple(line for line in tree.stdout.splitlines() if line) - if tree.returncode != 0 or committed_paths != expected_paths: - raise ValueError("source_commit does not contain the release-clean plugin file set") - for relative, committed_path in zip(files, committed_paths, strict=True): - blob = subprocess.run( - ( - "git", - "-C", - os.fspath(REPOSITORY_ROOT), - "show", - f"{source_commit}:{committed_path}", - ), - check=False, - capture_output=True, - ) - if blob.returncode != 0 or blob.stdout != (source / relative).read_bytes(): - raise ValueError(f"plugin source differs from source_commit: {relative.as_posix()}") - license_blob = subprocess.run( - ("git", "-C", os.fspath(REPOSITORY_ROOT), "show", f"{source_commit}:LICENSE"), - check=False, - capture_output=True, - ) - if ( - license_blob.returncode != 0 - or not LICENSE_PATH.is_file() - or LICENSE_PATH.is_symlink() - or license_blob.stdout != LICENSE_PATH.read_bytes() - ): - raise ValueError("LICENSE differs from source_commit") - - -def _provenance_bytes( - source: Path, files: tuple[Path, ...], *, source_commit: str | None = None -) -> bytes: - provenance = { - "build_format_version": BUILD_FORMAT_VERSION, - "license_sha256": hashlib.sha256(LICENSE_PATH.read_bytes()).hexdigest(), - "plugin_version": _plugin_version(source), - "provider_id": PLUGIN_NAME, - "source_commit": _source_commit() - if source_commit is None - else _validate_source_commit(source_commit), - "source_files": { - path.as_posix(): hashlib.sha256((source / path).read_bytes()).hexdigest() - for path in files - }, - "target_hermes_version": TARGET_HERMES_VERSION, - } - return (json.dumps(provenance, indent=2, sort_keys=True) + "\n").encode("utf-8") - - -def _zip_info(name: str, *, directory: bool = False) -> zipfile.ZipInfo: - info = zipfile.ZipInfo(name, date_time=FIXED_TIMESTAMP) - info.compress_type = zipfile.ZIP_DEFLATED - info.create_system = 3 - info.external_attr = (DIRECTORY_MODE if directory else FILE_MODE) << 16 - if directory: - info.external_attr |= 0x10 - return info - - -def build_archive_bytes(source: Path = PLUGIN_SOURCE, *, source_commit: str | None = None) -> bytes: - """Return canonical archive bytes for the release-clean plugin source.""" - source = source.resolve() - _require_release_clean_redaction(source) - files = _source_files(source) - provenance = _provenance_bytes(source, files, source_commit=source_commit) - - output = io.BytesIO() - with zipfile.ZipFile(output, "w", compression=zipfile.ZIP_DEFLATED, compresslevel=9) as archive: - archive.writestr(_zip_info(f"{PLUGIN_NAME}/", directory=True), b"", compresslevel=9) - archive.writestr( - _zip_info(f"{PLUGIN_NAME}/{PROVENANCE_FILENAME}"), - provenance, - compresslevel=9, - ) - archive.writestr( - _zip_info(f"{PLUGIN_NAME}/{LICENSE_FILENAME}"), - LICENSE_PATH.read_bytes(), - compresslevel=9, - ) - for relative_path in files: - archive.writestr( - _zip_info(f"{PLUGIN_NAME}/{relative_path.as_posix()}"), - (source / relative_path).read_bytes(), - compresslevel=9, - ) - return output.getvalue() - - -def _archive_source_commit(archive_path: Path) -> str: - """Extract and validate the provenance commit embedded in an archive.""" - try: - with zipfile.ZipFile(archive_path) as archive: - provenance = json.loads( - archive.read(f"{PLUGIN_NAME}/{PROVENANCE_FILENAME}").decode("utf-8") - ) - except (OSError, UnicodeDecodeError, json.JSONDecodeError, KeyError, zipfile.BadZipFile) as exc: - raise ValueError(f"archive has malformed provenance: {archive_path}") from exc - if not isinstance(provenance, dict): - raise ValueError(f"archive has malformed provenance: {archive_path}") - try: - return _validate_source_commit(provenance["source_commit"]) - except (KeyError, ValueError) as exc: - raise ValueError(f"archive has malformed provenance: {archive_path}") from exc - - -def check_archive(archive_path: Path = ARCHIVE_PATH, source: Path = PLUGIN_SOURCE) -> bool: - """Return whether an archive exactly matches a canonical rebuild.""" - if not archive_path.is_file() or archive_path.is_symlink(): - return False - source_commit = _archive_source_commit(archive_path) - return archive_path.read_bytes() == build_archive_bytes(source, source_commit=source_commit) - - -def _installer_bytes(version: str, installer_path: Path = INSTALLER_PATH) -> bytes: - """Return the installer only when it is pinned to the packaged plugin version.""" - content = installer_path.read_bytes() - marker = f'EXPECTED_VERSION = "{version}"'.encode() - if marker not in content: - raise ValueError(f"installer does not target plugin version {version}") - return content - - -def _replace_file(path: Path, content: bytes) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - if any(parent.is_symlink() for parent in path.parents): - raise ValueError(f"artifact parent must not be a symlink: {path.parent}") - temporary_path = path.with_name(f".{path.name}.{os.getpid()}.tmp") - try: - temporary_path.write_bytes(content) - temporary_path.replace(path) - finally: - try: - temporary_path.unlink() - except FileNotFoundError: - pass - - -def _validate_immutable_slot(path: Path, content: bytes) -> None: - """Reject unsafe or conflicting versioned artifact paths before publication.""" - path.parent.mkdir(parents=True, exist_ok=True) - if path.is_symlink() or any(parent.is_symlink() for parent in path.parents): - raise ValueError(f"immutable release artifact path is unsafe: {path}") - if path.exists(): - if not path.is_file() or path.read_bytes() != content: - raise ValueError(f"immutable release artifact differs: {path}") - - -def _write_immutable(path: Path, content: bytes) -> None: - """Create a versioned release file once, accepting only byte-identical reruns.""" - _validate_immutable_slot(path, content) - if path.exists(): - return - try: - with path.open("xb") as stream: - stream.write(content) - stream.flush() - os.fsync(stream.fileno()) - except FileExistsError: - if path.is_symlink() or not path.is_file() or path.read_bytes() != content: - raise ValueError(f"immutable release artifact differs: {path}") from None - - -def publish_release( - archive: bytes, - *, - source: Path = PLUGIN_SOURCE, - archive_path: Path = ARCHIVE_PATH, - installer_path: Path = INSTALLER_PATH, - releases_path: Path = RELEASES_PATH, -) -> tuple[Path, Path]: - """Publish immutable versioned artifacts and refresh the current aliases.""" - version = _plugin_version(source) - installer = _installer_bytes(version, installer_path) - release_directory = releases_path / version - release_archive = release_directory / archive_path.name - release_installer = release_directory / installer_path.name - - # Versioned paths are write-once. Unversioned paths are byte-for-byte aliases - # to the current release and may move only as part of a new version release. - # Preflight both files so a pre-existing conflict cannot leave a half-published - # version directory behind. - _validate_immutable_slot(release_archive, archive) - _validate_immutable_slot(release_installer, installer) - _write_immutable(release_archive, archive) - _write_immutable(release_installer, installer) - _replace_file(archive_path, archive) - return release_archive, release_installer - - -def check_release( - *, - source: Path = PLUGIN_SOURCE, - archive_path: Path = ARCHIVE_PATH, - installer_path: Path = INSTALLER_PATH, - releases_path: Path = RELEASES_PATH, -) -> bool: - """Verify canonical current bytes and their immutable versioned copies.""" - if not check_archive(archive_path, source): - return False - source_commit = _archive_source_commit(archive_path) - if source_commit == "unknown": - return False - version = _plugin_version(source) - release_directory = releases_path / version - release_archive = release_directory / archive_path.name - release_installer = release_directory / installer_path.name - if ( - release_directory.is_symlink() - or not release_archive.is_file() - or release_archive.is_symlink() - or release_archive.read_bytes() != archive_path.read_bytes() - ): - return False - if not release_installer.is_file() or release_installer.is_symlink(): - return False - installer = _installer_bytes(version, installer_path) - return release_installer.read_bytes() == installer - - -def main(argv: list[str] | None = None) -> int: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument( - "--check", - action="store_true", - help=( - "verify the checked-in current and immutable versioned release artifacts " - "without rewriting them" - ), - ) - args = parser.parse_args(argv) - - if args.check and not ARCHIVE_PATH.is_file(): - print(f"error: archive is missing: {ARCHIVE_PATH}", file=sys.stderr) - return 1 - - try: - source_commit = _archive_source_commit(ARCHIVE_PATH) if args.check else _source_commit() - if source_commit == "unknown": - raise ValueError( - f"{SOURCE_COMMIT_ENVIRONMENT_VARIABLE} must identify the committed plugin source" - ) - _require_commit_contains_source(source_commit) - expected = build_archive_bytes(source_commit=source_commit) - except (OSError, ValueError) as exc: - print(f"error: {exc}", file=sys.stderr) - return 1 - - if args.check: - if not check_release( - source=PLUGIN_SOURCE, - archive_path=ARCHIVE_PATH, - installer_path=INSTALLER_PATH, - releases_path=RELEASES_PATH, - ): - print( - "error: Hermes plugin release is stale, non-canonical, or missing its " - "immutable versioned copy; " - "run `python scripts/build_plugin.py`", - file=sys.stderr, - ) - return 1 - print(f"Hermes plugin release is current: {ARCHIVE_PATH}") - return 0 - - release_archive, release_installer = publish_release( - expected, - source=PLUGIN_SOURCE, - archive_path=ARCHIVE_PATH, - installer_path=INSTALLER_PATH, - releases_path=RELEASES_PATH, - ) - print( - f"Built {ARCHIVE_PATH} ({len(expected)} bytes) and published immutable artifacts " - f"at {release_archive.parent} ({release_installer.name})" - ) - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/scripts/build_release.py b/scripts/build_release.py new file mode 100644 index 0000000..69efb0c --- /dev/null +++ b/scripts/build_release.py @@ -0,0 +1,97 @@ +"""Build the deterministic Substrate plugin release archive. + +Packages ``plugins/substrate`` as ``dist/substrate.zip`` with fixed +timestamps and sorted members so two builds of the same tree are +byte-identical, plus ``dist/SHA256SUMS``. ``--check`` rebuilds in memory and +compares against the on-disk archive. +""" + +from __future__ import annotations + +import argparse +import hashlib +import zipfile +from pathlib import Path + +REPOSITORY_ROOT = Path(__file__).resolve().parents[1] +SOURCE_DIR = REPOSITORY_ROOT / "plugins" / "substrate" +DIST_DIR = REPOSITORY_ROOT / "dist" +ARCHIVE_NAME = "substrate.zip" +PREFIX = "substrate/" +FIXED_TIMESTAMP = (2020, 1, 1, 0, 0, 0) + + +def collect_members() -> list[tuple[str, bytes]]: + members: list[tuple[str, bytes]] = [] + license_path = REPOSITORY_ROOT / "LICENSE" + if license_path.is_file(): + members.append((PREFIX + "LICENSE", license_path.read_bytes())) + for path in sorted(SOURCE_DIR.rglob("*")): + if not path.is_file(): + continue + relative = path.relative_to(SOURCE_DIR).as_posix() + if "__pycache__" in path.parts or relative.endswith((".pyc", ".pyo")): + continue + members.append((PREFIX + relative, path.read_bytes())) + members.sort(key=lambda item: item[0]) + return members + + +def build_archive_bytes() -> bytes: + import io + + buffer = io.BytesIO() + with zipfile.ZipFile(buffer, "w", compression=zipfile.ZIP_DEFLATED, compresslevel=9) as archive: + for name, payload in collect_members(): + info = zipfile.ZipInfo(name, date_time=FIXED_TIMESTAMP) + info.create_system = 3 + info.external_attr = 0o644 << 16 + archive.writestr(info, payload) + return buffer.getvalue() + + +def digest(data: bytes) -> str: + return hashlib.sha256(data).hexdigest() + + +def build() -> tuple[Path, str]: + DIST_DIR.mkdir(parents=True, exist_ok=True) + archive_path = DIST_DIR / ARCHIVE_NAME + data = build_archive_bytes() + archive_path.write_bytes(data) + sums = f"{digest(data)} {ARCHIVE_NAME}\n" + (DIST_DIR / "SHA256SUMS").write_text(sums, encoding="utf-8") + return archive_path, digest(data) + + +def check() -> bool: + archive_path = DIST_DIR / ARCHIVE_NAME + sums_path = DIST_DIR / "SHA256SUMS" + if not archive_path.is_file() or not sums_path.is_file(): + print("dist archive or SHA256SUMS missing; run without --check first") + return False + expected = build_archive_bytes() + if archive_path.read_bytes() != expected: + print("dist archive is not byte-identical to a fresh build") + return False + recorded = sums_path.read_text(encoding="utf-8").strip() + if recorded != f"{digest(expected)} {ARCHIVE_NAME}": + print("SHA256SUMS does not match a fresh build") + return False + print(f"OK: {ARCHIVE_NAME} {digest(expected)}") + return True + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--check", action="store_true", help="verify dist against a fresh build") + args = parser.parse_args(argv) + if args.check: + return 0 if check() else 1 + path, sha = build() + print(f"{path} {sha}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/check_public_hygiene.py b/scripts/check_public_hygiene.py new file mode 100644 index 0000000..6107ac0 --- /dev/null +++ b/scripts/check_public_hygiene.py @@ -0,0 +1,111 @@ +"""Public-hygiene check for the Substrate plugin repository. + +Fails closed on real-secret shapes (private keys, credential-like tokens, +bearer assignments) and on production endpoints outside the explicit +allowlist. Findings name only the path and class, never the matched value. +Standard library only. +""" + +from __future__ import annotations + +import argparse +import re +from pathlib import Path + +SECRET_PATTERNS = ( + ("private-key", re.compile(r"-----BEGIN (?:RSA |EC |OPENSSH )?PRIVATE KEY-----")), + ("credential", re.compile(r"\bsk-(?:proj-)?[A-Za-z0-9_-]{20,}\b")), + ("credential", re.compile(r"\bnvapi-[A-Za-z0-9_-]{20,}\b")), + ("credential", re.compile(r"\b(?:ghp|github_pat)_[A-Za-z0-9_]{20,}\b")), + ( + "credential", + re.compile(r"Authorization\s*:\s*Bearer\s+[A-Za-z0-9._~+/-]{20,}", re.IGNORECASE), + ), + ( + "credential", + re.compile(r"(? ()]+", re.IGNORECASE) + +# Hosts the repository may legitimately reference: the configured Substrate +# origins, the public code host, the Python package registry, the DCO +# reference, loopback test fixtures, and RFC 2606 documentation domains. +ALLOWED_HOSTS = frozenset({ + "github.com", + "vm-substrate-ar-01.taile961d2.ts.net", + "app.trysubstrate.co", + "pypi.org", + "files.pythonhosted.org", + "developercertificate.org", + "127.0.0.1", + "localhost", +}) + +ALLOWED_SUFFIXES = (".example",) + + +def host_allowed(host: str) -> bool: + host = host.strip().strip("`.,;:!?)]}").lower() + if not host or "." not in host: + return True + if host in ALLOWED_HOSTS: + return True + return host.endswith(ALLOWED_SUFFIXES) + +SKIP_DIRS = frozenset({".git", ".venv", "__pycache__", ".pytest_cache", ".ruff_cache", "dist", "htmlcov"}) +SKIP_SUFFIXES = (".pyc", ".pyo") + + +def iter_files(root: Path): + for directory, dirnames, filenames in __import__("os").walk(root): + dirnames[:] = sorted(name for name in dirnames if name not in SKIP_DIRS) + for name in sorted(filenames): + if name.endswith(SKIP_SUFFIXES): + continue + path = Path(directory) / name + if path.is_symlink() or not path.is_file(): + yield path, "unsafe-or-missing" + continue + yield path, None + + +def scan(root: Path) -> list[str]: + findings: list[str] = [] + for path, early in iter_files(root): + relative = path.relative_to(root).as_posix() + if early: + findings.append(f"{relative}: {early}") + continue + try: + text = path.read_text(encoding="utf-8") + except (OSError, UnicodeError): + continue + for _class, pattern in SECRET_PATTERNS: + if pattern.search(text): + findings.append(f"{relative}: {_class}") + break + else: + for match in URL_PATTERN.finditer(text): + host = match.group(0).split("://", 1)[1].split("/", 1)[0].split("@")[-1].split(":")[0] + if not host_allowed(host): + findings.append(f"{relative}: endpoint") + break + return sorted(findings) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--root", type=Path, default=Path(".")) + args = parser.parse_args(argv) + findings = scan(args.root.resolve()) + if findings: + print({"status": "fail", "findings": findings}) + return 1 + print({"status": "pass", "findings": []}) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/install_hermes_plugin.py b/scripts/install_hermes_plugin.py deleted file mode 100644 index a9c780e..0000000 --- a/scripts/install_hermes_plugin.py +++ /dev/null @@ -1,538 +0,0 @@ -#!/usr/bin/env python3 -"""Verify and atomically install or upgrade the Substrate Hermes plugin.""" - -from __future__ import annotations - -import argparse -import hashlib -import io -import json -import os -import re -import stat -import subprocess -import sys -import tempfile -import time -import zipfile -from pathlib import Path, PurePosixPath -from typing import Any, cast - -PLUGIN_NAME = "substrate_wiki" -EXPECTED_VERSION = "2.0.5" -EXPECTED_HERMES_VERSION = "0.20.0" -LICENSE_FILENAME = "LICENSE" -REQUIRED_FILES = { - "README.md", - "__init__.py", - "cli.py", - "client.py", - "credentials.py", - "checkpoint.py", - "events.py", - "history.py", - "onboarding.py", - "plugin.yaml", - "py.typed", - "redaction.py", - "spool.py", - "supervisor.py", - "worker.py", -} -MAX_UNCOMPRESSED_BYTES = 2 * 1024 * 1024 -MAX_ARCHIVE_BYTES = 4 * 1024 * 1024 - - -def _digest(data: bytes) -> str: - return hashlib.sha256(data).hexdigest() - - -def _normalize_expected_sha256(value: str, *, required: bool = False) -> str: - normalized = value.strip().lower() if isinstance(value, str) else "" - if not normalized: - if required: - raise ValueError("a pinned archive SHA-256 is required") - return "" - if re.fullmatch(r"[0-9a-f]{64}", normalized) is None: - raise ValueError("pinned archive SHA-256 is malformed") - return normalized - - -def _read_archive_bytes(path: Path) -> bytes: - if not path.is_file() or path.is_symlink(): - raise ValueError("archive is missing or unsafe") - with path.open("rb") as stream: - data = stream.read(MAX_ARCHIVE_BYTES + 1) - if len(data) > MAX_ARCHIVE_BYTES: - raise ValueError("archive is too large") - return data - - -def verify_archive(path: Path, expected_sha256: str = "") -> dict[str, Any]: - data = _read_archive_bytes(path) - digest = _digest(data) - expected_sha256 = _normalize_expected_sha256(expected_sha256) - if expected_sha256 and digest != expected_sha256: - raise ValueError("archive SHA-256 mismatch") - # Validate the exact bytes whose digest we report. Reopening ``path`` here - # would leave a swap window between hashing and archive inspection. - with zipfile.ZipFile(io.BytesIO(data)) as archive: - names = archive.namelist() - if len(names) != len(set(names)): - raise ValueError("archive contains duplicate paths") - expected_members = { - f"{PLUGIN_NAME}/", - f"{PLUGIN_NAME}/PROVENANCE.json", - f"{PLUGIN_NAME}/{LICENSE_FILENAME}", - *(f"{PLUGIN_NAME}/{name}" for name in REQUIRED_FILES), - } - if set(names) != expected_members: - raise ValueError("archive contains an unexpected file set") - if sum(info.file_size for info in archive.infolist()) > MAX_UNCOMPRESSED_BYTES: - raise ValueError("archive is too large") - for info in archive.infolist(): - name = info.filename - candidate = PurePosixPath(name) - if candidate.is_absolute() or ".." in candidate.parts: - raise ValueError("archive contains an unsafe path") - if not candidate.parts or candidate.parts[0] != PLUGIN_NAME: - raise ValueError("archive has an unexpected root directory") - mode = info.external_attr >> 16 - if stat.S_ISLNK(mode): - raise ValueError("archive contains a symbolic link") - if name == f"{PLUGIN_NAME}/": - if not info.is_dir() or not stat.S_ISDIR(mode): - raise ValueError("archive root is not a directory") - elif info.is_dir() or not stat.S_ISREG(mode): - raise ValueError("archive contains a non-regular plugin file") - if info.flag_bits & 0x1: - raise ValueError("archive contains an encrypted member") - provenance = json.loads(archive.read(f"{PLUGIN_NAME}/PROVENANCE.json").decode("utf-8")) - if provenance.get("build_format_version") != 3: - raise ValueError("unexpected plugin archive build format") - if provenance.get("provider_id") != PLUGIN_NAME: - raise ValueError("unexpected plugin provider identity") - if provenance.get("plugin_version") != EXPECTED_VERSION: - raise ValueError("unexpected plugin version") - if provenance.get("target_hermes_version") != EXPECTED_HERMES_VERSION: - raise ValueError("unexpected Hermes target version") - license_sha256 = provenance.get("license_sha256") - if ( - not isinstance(license_sha256, str) - or re.fullmatch(r"[0-9a-f]{64}", license_sha256) is None - or _digest(archive.read(f"{PLUGIN_NAME}/{LICENSE_FILENAME}")) != license_sha256 - ): - raise ValueError("archive license digest mismatch") - source_files = provenance.get("source_files") - if not isinstance(source_files, dict): - raise ValueError("archive provenance has no source file manifest") - if set(source_files) != REQUIRED_FILES: - raise ValueError("archive provenance has an unexpected source file set") - source_commit = provenance.get("source_commit") - if ( - not isinstance(source_commit, str) - or re.fullmatch(r"[0-9a-f]{40}", source_commit) is None - ): - raise ValueError("archive provenance has no immutable source commit") - for relative, expected in source_files.items(): - if not isinstance(expected, str) or re.fullmatch(r"[0-9a-f]{64}", expected) is None: - raise ValueError(f"invalid source digest: {relative}") - actual = _digest(archive.read(f"{PLUGIN_NAME}/{relative}")) - if actual != expected: - raise ValueError(f"source digest mismatch: {relative}") - manifest = archive.read(f"{PLUGIN_NAME}/plugin.yaml").decode("utf-8") - if re.search(r"(?m)^name:\s*substrate_wiki\s*$", manifest) is None: - raise ValueError("plugin manifest identity mismatch") - if re.search( - rf"(?m)^version:\s*{re.escape(EXPECTED_VERSION)}\s*$", manifest - ) is None: - raise ValueError("plugin manifest version mismatch") - return { - "archive_sha256": digest, - "plugin_version": EXPECTED_VERSION, - "source_commit": source_commit, - } - - -def _systemd_quote(value: Path) -> str: - text = os.fspath(value.resolve()) - if any(character in text for character in ("\n", "\r", '"')): - raise ValueError("service path cannot be represented safely") - return f'"{text}"' - - -def _resolve_env_path(explicit: Path | None = None) -> Path: - if explicit is not None: - candidate = explicit.expanduser().absolute() - else: - result = subprocess.run( - ("hermes", "config", "env-path"), - check=True, - stdin=subprocess.DEVNULL, - stdout=subprocess.PIPE, - stderr=subprocess.DEVNULL, - text=True, - timeout=30, - ) - candidate = Path(result.stdout.strip()).expanduser().absolute() - # Check the path entry before resolving it. Calling ``resolve`` first would - # erase the evidence that the configured environment file is a symlink. - if candidate.is_symlink(): - raise ValueError("Hermes environment path must be a regular non-symlink file") - path = candidate.resolve() - if not path.is_file(): - raise ValueError("Hermes environment path must be a regular non-symlink file") - info = path.stat(follow_symlinks=False) - if info.st_size > 1024 * 1024: - raise ValueError("Hermes environment file is unexpectedly large") - if os.name == "posix": - current_uid = int(cast(Any, os).getuid()) - if info.st_uid != current_uid or stat.S_IMODE(info.st_mode) & 0o077: - raise ValueError("Hermes environment path must be owner-only") - names: set[str] = set() - with path.open("r", encoding="utf-8") as stream: - for raw in stream: - line = raw.strip() - if line and not line.startswith("#") and "=" in line: - names.add(line.split("=", 1)[0].removeprefix("export ").strip()) - if not {"HERMES_API_URL", "HERMES_API_KEY"} <= names: - raise ValueError("Hermes environment is missing required Substrate variables") - return path - - -def install_import_service( - hermes_home: Path, - plugin_target: Path, - *, - env_path: Path | None = None, -) -> dict[str, Any]: - if os.name != "posix": - raise ValueError("the import service can only be installed on Linux") - from hashlib import sha256 - - resolved_env = _resolve_env_path(env_path) if env_path is not None else None - home_hash = sha256(os.fspath(hermes_home.resolve()).encode()).hexdigest()[:12] - unit_name = f"substrate-wiki-import-{home_hash}@.service" - user_units = Path.home() / ".config" / "systemd" / "user" - if user_units.exists() and user_units.is_symlink(): - raise ValueError("systemd user unit directory must not be a symlink") - user_units.mkdir(parents=True, exist_ok=True, mode=0o700) - unit_path = user_units / unit_name - if unit_path.is_symlink(): - raise ValueError("existing import unit must not be a symlink") - rollback: Path | None = None - python = Path(sys.executable).resolve() - supervisor = plugin_target / "supervisor.py" - unit = "\n".join( - ( - "[Unit]", - "Description=Substrate Wiki durable history import %i", - "After=network-online.target", - "Wants=network-online.target", - "StartLimitIntervalSec=600", - "StartLimitBurst=8", - "", - "[Service]", - "Type=simple", - *( - (f"EnvironmentFile={_systemd_quote(resolved_env)}",) - if resolved_env is not None else () - ), - f"ExecStart={_systemd_quote(python)} {_systemd_quote(supervisor)} " - f"--hermes-home {_systemd_quote(hermes_home)} --job-id %i", - "Restart=on-failure", - "RestartSec=15s", - "MemoryHigh=224M", - "MemoryMax=256M", - "OOMPolicy=stop", - "NoNewPrivileges=true", - "PrivateTmp=true", - "ProtectSystem=strict", - "ProtectHome=read-only", - f"ReadWritePaths={_systemd_quote(hermes_home)}", - "RestrictSUIDSGID=true", - "LockPersonality=true", - "", - "[Install]", - "WantedBy=default.target", - "", - ) - ).encode("utf-8") - temporary = unit_path.with_name(f".{unit_name}.{os.getpid()}.tmp") - unit_replaced = False - try: - descriptor = os.open(temporary, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) - with os.fdopen(descriptor, "wb") as stream: - stream.write(unit) - stream.flush() - os.fsync(stream.fileno()) - if unit_path.exists() or unit_path.is_symlink(): - if unit_path.is_symlink(): - raise ValueError("existing import unit must not be a symlink") - rollback = unit_path.with_name(f"{unit_name}.rollback-{time.time_ns()}") - if rollback.exists() or rollback.is_symlink(): - raise OSError("import-unit rollback path already exists") - os.replace(unit_path, rollback) - os.replace(temporary, unit_path) - unit_replaced = True - os.chmod(unit_path, 0o600) - subprocess.run( - ("systemctl", "--user", "daemon-reload"), - check=True, - stdin=subprocess.DEVNULL, - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, - timeout=30, - ) - except Exception: - if unit_replaced and (unit_path.exists() or unit_path.is_symlink()): - unit_path.unlink() - if rollback is not None and (rollback.exists() or rollback.is_symlink()): - os.replace(rollback, unit_path) - if unit_replaced or rollback is not None: - subprocess.run( - ("systemctl", "--user", "daemon-reload"), - check=False, - stdin=subprocess.DEVNULL, - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, - timeout=30, - ) - raise - finally: - try: - temporary.unlink() - except FileNotFoundError: - pass - return { - "import_service": unit_name, - "import_service_path": os.fspath(unit_path), - "import_service_rollback": os.fspath(rollback) if rollback else None, - "memory_high_bytes": 224 * 1024 * 1024, - "memory_max_bytes": 256 * 1024 * 1024, - } - - - - -def _verify_installed_hermes() -> str: - """Fail before mutation unless the current supported Hermes line is installed.""" - try: - result = subprocess.run( - ("hermes", "--version"), - check=True, - stdin=subprocess.DEVNULL, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - text=True, - timeout=30, - ) - except (OSError, subprocess.SubprocessError) as exc: - raise ValueError("Hermes 0.20.x is required and must be on PATH") from exc - match = re.search(r"(? bool: - """Return whether a directory entry exists without following dangling links.""" - return path.exists() or path.is_symlink() - - -def _restore_plugin_after_failed_install( - plugins: Path, - target: Path, - rollback: Path | None, -) -> None: - """Move a failed new install aside and atomically restore the prior plugin.""" - if _entry_exists(target): - failed = plugins / f"{PLUGIN_NAME}.failed-{time.time_ns()}" - if _entry_exists(failed): - raise OSError("failed-plugin rollback path already exists") - os.replace(target, failed) - if rollback is not None and _entry_exists(rollback) and not _entry_exists(target): - os.replace(rollback, target) - - -def _harden_plugin_permissions(target: Path) -> None: - if os.name != "posix": - return - os.chmod(target, 0o700) - for child in target.rglob("*"): - os.chmod(child, 0o600 if child.is_file() else 0o700) - - -def install( - archive: Path, - hermes_home: Path, - *, - expected_sha256: str = "", - install_service: bool = False, - env_path: Path | None = None, - activate: bool = False, - onboard: bool = False, - headless: bool = False, -) -> dict[str, Any]: - expected_sha256 = _normalize_expected_sha256(expected_sha256, required=True) - hermes_version = _verify_installed_hermes() if activate else None - verified = verify_archive(archive, expected_sha256) - # Extract only bytes tied to the verified digest. A downloaded path can be - # replaced by another local process after verification but before install. - archive_bytes = _read_archive_bytes(archive) - if _digest(archive_bytes) != verified["archive_sha256"]: - raise ValueError("archive changed after verification") - plugins = hermes_home / "plugins" - plugins.mkdir(parents=True, exist_ok=True, mode=0o700) - if plugins.is_symlink(): - raise ValueError("Hermes plugins directory must not be a symlink") - target = plugins / PLUGIN_NAME - if target.is_symlink(): - raise ValueError("existing plugin directory must not be a symlink") - with tempfile.TemporaryDirectory(prefix="substrate-plugin-", dir=plugins) as directory: - staging_root = Path(directory) - with zipfile.ZipFile(io.BytesIO(archive_bytes)) as bundle: - bundle.extractall(staging_root) - staged = staging_root / PLUGIN_NAME - if not (staged / "plugin.yaml").is_file() or not (staged / "__init__.py").is_file(): - raise ValueError("archive is missing required plugin files") - installed = not target.exists() - rollback: Path | None = None - if target.exists(): - rollback = plugins / f"{PLUGIN_NAME}.rollback-{time.time_ns()}" - if _entry_exists(rollback): - raise OSError("plugin rollback path already exists") - os.replace(target, rollback) - try: - os.replace(staged, target) - _harden_plugin_permissions(target) - except Exception: - _restore_plugin_after_failed_install(plugins, target, rollback) - raise - result = { - **verified, - "action": "installed" if installed else "upgraded", - "target": os.fspath(target), - "rollback": os.fspath(rollback) if rollback is not None else None, - "hermes_version": hermes_version, - } - if install_service: - try: - result.update(install_import_service(hermes_home, target, env_path=env_path)) - except Exception: - _restore_plugin_after_failed_install(plugins, target, rollback) - raise - if activate: - command = ("hermes", "config", "set", "memory.provider", PLUGIN_NAME) - try: - subprocess.run( - command, - check=True, - stdin=subprocess.DEVNULL, - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, - timeout=30, - ) - except Exception: - _restore_plugin_after_failed_install(plugins, target, rollback) - raise - result["memory_provider"] = PLUGIN_NAME - if onboard: - code = ( - "import sys; from pathlib import Path; " - f"sys.path.insert(0, {os.fspath(target.parent)!r}); " - "from substrate_wiki.onboarding import main; " - "raise SystemExit(main(sys.argv[1:]))" - ) - command = [ - sys.executable, "-c", code, "--hermes-home", os.fspath(hermes_home), - "--mode", "device" if headless else "auto", "--wait", "--history", "ask", "--json", - ] - completed = subprocess.run( - command, - check=False, - stdout=subprocess.PIPE, - stderr=None, - text=True, - timeout=930, - ) - try: - onboarding = json.loads(completed.stdout) - except json.JSONDecodeError: - onboarding = {"complete": False, "error_class": "OnboardingLaunchError"} - if completed.returncode != 0: - onboarding = { - "complete": False, - "error_class": str(onboarding.get("error_class") or "OnboardingLaunchError"), - } - if onboarding.get("phase") == "awaiting_history_consent": - onboarding["action_required"] = "history_consent" - result["action_required"] = "history_consent" - result["onboarding"] = onboarding - result["onboarding_pending"] = not bool(onboarding.get("complete")) - return result - - -def main() -> int: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--archive", type=Path, required=True) - parser.add_argument( - "--hermes-home", - type=Path, - default=Path(os.environ.get("HERMES_HOME") or Path.home() / ".hermes"), - ) - parser.add_argument("--sha256", required=True) - parser.add_argument("--install-import-service", action="store_true") - parser.add_argument("--env-path", type=Path) - parser.add_argument("--no-activate", action="store_true", - help="Do not select substrate_wiki as memory.provider") - parser.add_argument("--no-onboard", action="store_true", - help="Install only; first activation will resume onboarding") - parser.add_argument("--headless", action="store_true", - help="Use the device-code flow without opening a browser") - parser.add_argument("--yes", action="store_true") - parser.add_argument("--json", action="store_true") - args = parser.parse_args() - if not args.yes: - parser.error("--yes is required for installation") - try: - result = install( - args.archive.expanduser().absolute(), - args.hermes_home.expanduser().absolute(), - expected_sha256=args.sha256, - install_service=args.install_import_service, - env_path=args.env_path, - activate=not args.no_activate, - onboard=not args.no_onboard, - headless=args.headless, - ) - except ( - OSError, - ValueError, - zipfile.BadZipFile, - json.JSONDecodeError, - subprocess.SubprocessError, - ) as exc: - message = {"error": type(exc).__name__, "installed": False} - print( - json.dumps(message, sort_keys=True) - if args.json - else f"Installation failed: {type(exc).__name__}" - ) - return 1 - if result.get("action_required") == "history_consent": - print( - "Substrate connection successful. Return to the same conversation and ask " - "whether the user wants to import existing Hermes history.", - file=sys.stderr, - flush=True, - ) - print( - json.dumps(result, sort_keys=True) - if args.json - else f"Substrate plugin {result['action']}: {result['target']}" - ) - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/scripts/verify_fresh_migration_run.py b/scripts/verify_fresh_migration_run.py deleted file mode 100644 index 974874e..0000000 --- a/scripts/verify_fresh_migration_run.py +++ /dev/null @@ -1,86 +0,0 @@ -#!/usr/bin/env python3 -"""Validate a fresh privacy-safe migration matrix against retained beta ceilings.""" - -from __future__ import annotations - -import argparse -import json -import os -import sys -from pathlib import Path -from typing import Any - -ROOT = Path(__file__).resolve().parents[1] -sys.path.insert(0, os.fspath(ROOT / "scripts")) - -from benchmark_migration import evaluate_budget, load_manifest, validate_receipt # noqa: E402 -from verify_migration_baseline import ( # noqa: E402 - DEFAULT_BUDGET, - DEFAULT_MANIFEST, - DEFAULT_SCHEMA, - validate_schema_document, - verify_receipt_matrix, -) - - -def load_object(path: Path) -> dict[str, Any]: - value = json.loads(path.read_text(encoding="utf-8")) - if not isinstance(value, dict): - raise ValueError(f"JSON root must be an object: {path}") - return value - - -def main() -> int: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--manifest", type=Path, default=DEFAULT_MANIFEST) - parser.add_argument("--schema", type=Path, default=DEFAULT_SCHEMA) - parser.add_argument("--receipt", type=Path, required=True) - parser.add_argument("--budget", type=Path, default=DEFAULT_BUDGET) - args = parser.parse_args() - - manifest = load_manifest(args.manifest) - schema = load_object(args.schema) - receipt = load_object(args.receipt) - budget = load_object(args.budget) - validate_schema_document(schema, receipt) - validate_receipt(receipt) - verify_receipt_matrix(receipt, manifest, args.manifest) - - case_budgets = budget.get("cases") - if not isinstance(case_budgets, dict): - raise ValueError("retained budget cases must be a mapping") - failures: list[str] = [] - for run in receipt["runs"]: - case_id = str(run["case_id"]) - case_budget = case_budgets.get(case_id) - if not isinstance(case_budget, dict): - failures.append(f"missing retained budget for case {case_id}") - continue - # The retained count reflects the former all-message payload. With the - # minimal user/assistant contract, redaction remains an integrity - # presence check rather than a historical message-volume floor. - current_budget = dict(case_budget) - current_budget["min_redacted_events"] = 1 - failures.extend( - f"{case_id}/c{run['concurrency']}: {failure}" - for failure in evaluate_budget(run, current_budget) - ) - if failures: - raise ValueError("; ".join(failures)) - print( - json.dumps( - { - "budget": os.fspath(args.budget), - "receipt": os.fspath(args.receipt), - "runs": len(receipt["runs"]), - "status": "pass", - "verification": "fresh_run_against_retained_ceilings", - }, - sort_keys=True, - ) - ) - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/scripts/verify_migration_baseline.py b/scripts/verify_migration_baseline.py deleted file mode 100644 index 5325709..0000000 --- a/scripts/verify_migration_baseline.py +++ /dev/null @@ -1,208 +0,0 @@ -#!/usr/bin/env python3 -"""Validate the canonical SUB-75 receipt and its mechanically derived budget.""" - -from __future__ import annotations - -import argparse -import hashlib -import json -import os -import re -import sys -from pathlib import Path -from typing import Any - -ROOT = Path(__file__).resolve().parents[1] -sys.path.insert(0, os.fspath(ROOT / "scripts")) - -from benchmark_migration import ( # noqa: E402 - derive_budget, - load_manifest, - validate_receipt, - verify_budget, -) - -DEFAULT_MANIFEST = ROOT / "benchmarks" / "hermes-migration-manifest.json" -DEFAULT_SCHEMA = ROOT / "benchmarks" / "hermes-migration-receipt.schema.json" -DEFAULT_RECEIPT = ROOT / "benchmarks" / "evidence" / "hermes-migration-baseline.json" -DEFAULT_BUDGET = ROOT / "benchmarks" / "hermes-migration-budget.json" -BENCHMARK = ROOT / "scripts" / "benchmark_migration.py" - - -def _load_object(path: Path) -> dict[str, Any]: - value = json.loads(path.read_text(encoding="utf-8")) - if not isinstance(value, dict): - raise ValueError(f"JSON root must be an object: {path}") - return value - - -def _resolve_reference(root: dict[str, Any], reference: str) -> dict[str, Any]: - if not reference.startswith("#/"): - raise ValueError(f"only local schema references are supported: {reference}") - value: Any = root - for component in reference[2:].split("/"): - value = value[component.replace("~1", "/").replace("~0", "~")] - if not isinstance(value, dict): - raise ValueError(f"schema reference does not resolve to an object: {reference}") - return value - - -def _schema_errors( - value: Any, schema: dict[str, Any], root: dict[str, Any], path: str = "" -) -> list[str]: - errors: list[str] = [] - if "$ref" in schema: - errors.extend(_schema_errors(value, _resolve_reference(root, schema["$ref"]), root, path)) - expected_type = schema.get("type") - type_matches = { - "object": isinstance(value, dict), - "array": isinstance(value, list), - "string": isinstance(value, str), - "integer": isinstance(value, int) and not isinstance(value, bool), - "number": isinstance(value, (int, float)) and not isinstance(value, bool), - "boolean": isinstance(value, bool), - } - if expected_type is not None and not type_matches.get(expected_type, False): - return errors + [f"{path}: expected {expected_type}"] - if "const" in schema and value != schema["const"]: - errors.append(f"{path}: value does not match const") - if "enum" in schema and value not in schema["enum"]: - errors.append(f"{path}: value is not in enum") - if isinstance(value, dict): - required = schema.get("required", []) - errors.extend( - f"{path}: missing required property {key}" for key in required if key not in value - ) - properties = schema.get("properties", {}) - if schema.get("additionalProperties") is False: - errors.extend( - f"{path}: unexpected property {key}" for key in value if key not in properties - ) - for key, child in value.items(): - child_schema = properties.get(key) - if isinstance(child_schema, dict): - errors.extend(_schema_errors(child, child_schema, root, f"{path}/{key}")) - if isinstance(value, list): - if len(value) < int(schema.get("minItems", 0)): - errors.append(f"{path}: too few items") - if "maxItems" in schema and len(value) > int(schema["maxItems"]): - errors.append(f"{path}: too many items") - item_schema = schema.get("items") - if isinstance(item_schema, dict): - for index, child in enumerate(value): - errors.extend(_schema_errors(child, item_schema, root, f"{path}/{index}")) - if isinstance(value, str): - if len(value) < int(schema.get("minLength", 0)): - errors.append(f"{path}: string is too short") - if "pattern" in schema and re.fullmatch(schema["pattern"], value) is None: - errors.append(f"{path}: string does not match pattern") - if isinstance(value, (int, float)) and not isinstance(value, bool): - if "minimum" in schema and value < schema["minimum"]: - errors.append(f"{path}: value is below minimum") - if "maximum" in schema and value > schema["maximum"]: - errors.append(f"{path}: value is above maximum") - return errors - - -def validate_schema_document(schema: dict[str, Any], receipt: dict[str, Any]) -> None: - if schema.get("$schema") != "https://json-schema.org/draft/2020-12/schema": - raise ValueError("receipt schema must declare JSON Schema draft 2020-12") - errors = _schema_errors(receipt, schema, schema) - if errors: - raise ValueError("receipt schema validation failed: " + "; ".join(sorted(errors))) - - -def verify_receipt_matrix( - receipt: dict[str, Any], manifest: dict[str, Any], manifest_path: Path -) -> None: - expected_manifest_digest = hashlib.sha256(manifest_path.read_bytes()).hexdigest() - if receipt.get("manifest_sha256") != expected_manifest_digest: - raise ValueError("receipt manifest digest does not match canonical manifest bytes") - expected_harness_digest = hashlib.sha256(BENCHMARK.read_bytes()).hexdigest() - if receipt.get("harness_sha256") != expected_harness_digest: - raise ValueError("receipt harness digest does not match canonical benchmark bytes") - profile = receipt.get("profile") - selected = manifest["profiles"].get(profile) - if not isinstance(selected, list) or not selected: - raise ValueError("receipt profile is not selected by the manifest") - expected = {(str(case_id), concurrency) for case_id in selected for concurrency in (1, 2, 3, 4)} - observed = {(run["case_id"], run["concurrency"]) for run in receipt["runs"]} - if len(receipt["runs"]) != len(expected) or observed != expected: - raise ValueError("receipt does not contain the exact case/concurrency matrix") - if receipt.get("hosted_calls") != 0: - raise ValueError("hosted calls are forbidden in canonical baseline evidence") - provider = receipt.get("representative_provider", {}) - if ( - provider.get("mode") != "deterministic_simulation" - or provider.get("network_access") is not False - ): - raise ValueError( - "receipt must label provider behavior as an offline deterministic simulation" - ) - for run in receipt["runs"]: - concurrency = int(run["concurrency"]) - measured = int(run["provider"]["max_in_flight"]) - threads = int(run["provider"]["worker_threads_used"]) - if not (1 <= measured <= concurrency and 1 <= threads <= concurrency): - raise ValueError("measured worker concurrency exceeds its configured seam") - for concurrency in (1, 2, 3, 4): - if ( - max( - int(run["provider"]["max_in_flight"]) - for run in receipt["runs"] - if run["concurrency"] == concurrency - ) - != concurrency - ): - raise ValueError(f"receipt does not demonstrate measured concurrency {concurrency}") - - -def main(argv: list[str] | None = None) -> int: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--manifest", type=Path, default=DEFAULT_MANIFEST) - parser.add_argument("--schema", type=Path, default=DEFAULT_SCHEMA) - parser.add_argument("--receipt", type=Path, default=DEFAULT_RECEIPT) - parser.add_argument("--budget", type=Path, default=DEFAULT_BUDGET) - parser.add_argument( - "--write-budget", - action="store_true", - help="replace --budget with the canonical derivation from --receipt", - ) - args = parser.parse_args(argv) - - manifest = load_manifest(args.manifest) - schema = _load_object(args.schema) - receipt = _load_object(args.receipt) - validate_schema_document(schema, receipt) - validate_receipt(receipt) - verify_receipt_matrix(receipt, manifest, args.manifest) - - if args.write_budget: - budget = derive_budget(receipt) - budget["receipt_sha256"] = hashlib.sha256(args.receipt.read_bytes()).hexdigest() - args.budget.parent.mkdir(parents=True, exist_ok=True) - args.budget.write_text( - json.dumps(budget, indent=2, sort_keys=True) + "\n", encoding="utf-8" - ) - budget = _load_object(args.budget) - failures = verify_budget(receipt, budget) - if failures: - raise ValueError("; ".join(failures)) - print( - json.dumps( - { - "budget": os.fspath(args.budget), - "cases": len(budget["cases"]), - "receipt": os.fspath(args.receipt), - "runs": len(receipt["runs"]), - "schema": "draft-2020-12", - "status": "pass", - }, - sort_keys=True, - ) - ) - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/scripts/verify_public_plugin_candidate.py b/scripts/verify_public_plugin_candidate.py deleted file mode 100644 index b5dff17..0000000 --- a/scripts/verify_public_plugin_candidate.py +++ /dev/null @@ -1,842 +0,0 @@ -"""Verify the exact future public-plugin candidate is free of real secrets and endpoints. - -The verifier is intentionally content-free: findings contain only a relative path and -finding class, never the matched value. -""" - -from __future__ import annotations - -import argparse -import ast -import hashlib -import json -import os -import re -import stat -import subprocess -import tempfile -import zipfile -from pathlib import Path, PurePosixPath -from typing import Any, Literal -from urllib.parse import urlsplit - -SYNTHETIC_SENTINEL_PREFIX = "SUBSTRATE_SYNTHETIC_SECRET_DO_NOT_USE_" -SYNTHETIC_SENTINEL_PATTERN = re.compile( - rf"(?()]+", re.IGNORECASE) -DOCUMENTATION_HOST_SUFFIXES = ( - ".invalid", - ".test", - ".example", - ".example.com", - ".example.net", - ".example.org", -) -DOCUMENTATION_HOSTS = { - "example.com", - "example.net", - "example.org", - "github.com", - "developercertificate.org", - "hermes-agent.nousresearch.com", - "json-schema.org", - "pypi.org", - "files.pythonhosted.org", -} -PRODUCTION_PATTERNS = ( - ("OpenAI-shaped credential", re.compile(r"\bsk-(?:proj-)?[A-Za-z0-9_-]{20,}\b")), - ("NVIDIA-shaped credential", re.compile(r"\bnvapi-[A-Za-z0-9_-]{20,}\b")), - ("GitHub-shaped credential", re.compile(r"\b(?:ghp|github_pat)_[A-Za-z0-9_]{20,}\b")), - ( - "Bearer credential", - re.compile(r"Authorization\s*:\s*Bearer\s+[A-Za-z0-9._~+/-]{20,}", re.IGNORECASE), - ), - ( - "Hermes API credential assignment", - re.compile( - r"(? bytes: - """Remove the history policy's intentional self-reference before hashing.""" - - if relative_path == SCANNER_PATH: - text = payload.decode("utf-8") - text, count = re.subn( - r'(?s)(TRUSTED_HISTORICAL_BLOB_POLICY_SHA256\s*=\s*\(\s*")[0-9a-f]{64}("\s*\))', - lambda match: match.group(1) + "0" * 64 + match.group(2), - text, - count=1, - ) - return text.encode("utf-8") if count else payload - if relative_path == DESTINATION_MANIFEST_PATH: - try: - value = json.loads(payload.decode("utf-8")) - except (UnicodeDecodeError, json.JSONDecodeError): - return payload - normalized = 0 - for item in value.get("entries", []): - if isinstance(item, dict) and item.get("destination") == SCANNER_PATH: - item["destination_sha256"] = "0" * 64 - normalized += 1 - for item in value.get("destination_only", []): - if isinstance(item, dict) and item.get("path") == SCANNER_PATH: - item["sha256"] = "0" * 64 - normalized += 1 - if normalized == 1: - return json.dumps(value, sort_keys=True, separators=(",", ":")).encode("utf-8") - return payload - - -def _publication_tree_roots(root: Path) -> list[tuple[str, str]]: - """Return every commit tree and every ref recursively peeled to a tree.""" - - roots = { - (f"commit:{commit}", commit) - for commit in _git(root, "rev-list", "--all").decode("ascii").splitlines() - } - refs = _git(root, "for-each-ref", "--format=%(refname)").decode("utf-8").splitlines() - for ref in refs: - object_id = _git(root, "rev-parse", f"{ref}^{{}}").decode("ascii").strip() - object_type = _git(root, "cat-file", "-t", object_id).decode("ascii").strip() - if object_type == "tree": - roots.add((f"tree:{object_id}", object_id)) - return sorted(roots) - - -def _historical_blob_policy_sha256(root: Path, manifest: dict[str, Any]) -> str: - """Hash every distinct path/class/mode/content tuple across published refs.""" - - path_classes = { - item["destination"]: item["class"] for item in manifest["entries"] - } - path_classes.update( - {item["path"]: item["class"] for item in manifest["destination_only"]} - ) - path_classes[manifest["self_excluded_path"]] = "closed-inventory-manifest" - projection: set[tuple[str, str, str, str]] = set() - for _, treeish in _publication_tree_roots(root): - for raw_entry in ( - entry for entry in _git(root, "ls-tree", "-r", "-z", treeish).split(b"\0") if entry - ): - metadata, raw_path = raw_entry.split(b"\t", 1) - mode, object_type, raw_object_id = metadata.split(b" ", 2) - relative_path = raw_path.decode("utf-8") - object_id = raw_object_id.decode("ascii") - if object_type == b"blob": - payload = _git(root, "cat-file", "blob", object_id) - normalized_digest = hashlib.sha256( - _normalize_historical_policy_payload(relative_path, payload) - ).hexdigest() - else: - normalized_digest = object_id - projection.add( - ( - relative_path, - path_classes.get(relative_path, "outside-closed-inventory"), - mode.decode("ascii"), - f"{object_type.decode('ascii')}:{normalized_digest}", - ) - ) - payload = json.dumps(sorted(projection), separators=(",", ":")).encode("utf-8") - return hashlib.sha256(payload).hexdigest() - - -def _resolved_python_strings(text: str) -> set[str]: - """Resolve bounded constant string assignments without executing candidate code.""" - - try: - tree = ast.parse(text) - except (SyntaxError, ValueError): - return set() - assignments = sorted( - (node for node in ast.walk(tree) if isinstance(node, (ast.Assign, ast.AnnAssign))), - key=lambda node: (getattr(node, "lineno", 0), getattr(node, "col_offset", 0)), - ) - values: dict[str, str] = {} - - def resolve(node: ast.AST | None) -> str | None: - if isinstance(node, ast.Constant) and isinstance(node.value, str): - return node.value - if isinstance(node, ast.Name): - return values.get(node.id) - if isinstance(node, ast.BinOp) and isinstance(node.op, ast.Add): - left = resolve(node.left) - right = resolve(node.right) - if left is not None and right is not None and len(left) + len(right) <= 256: - return left + right - return None - - for _ in range(min(len(assignments) + 1, 64)): - changed = False - for assignment in assignments: - value = resolve(assignment.value) - if value is None: - continue - targets = assignment.targets if isinstance(assignment, ast.Assign) else [assignment.target] - for target in targets: - if isinstance(target, ast.Name) and values.get(target.id) != value: - values[target.id] = value - changed = True - if not changed: - break - return set(values.values()) - - -def scan_text(relative_path: str, text: str) -> list[str]: - """Return content-free findings for one candidate text file.""" - - findings: list[str] = [] - sentinels = SYNTHETIC_SENTINEL_PATTERN.findall(text) - sentinel_candidates = SYNTHETIC_SENTINEL_CANDIDATE_PATTERN.findall(text) - if len(sentinel_candidates) != len(sentinels): - findings.append(f"{relative_path}: malformed synthetic sentinel") - if sentinels and relative_path not in SYNTHETIC_FIXTURE_ALLOWLIST: - findings.append(f"{relative_path}: synthetic sentinel outside exact fixture allowlist") - production_text = SYNTHETIC_SENTINEL_PATTERN.sub("[SYNTHETIC-SENTINEL]", text) - production_text = re.sub( - r"\\(?:u([0-9a-fA-F]{4})|x([0-9a-fA-F]{2}))", - lambda match: chr(int(match.group(1) or match.group(2), 16)) - if int(match.group(1) or match.group(2), 16) < 128 - else match.group(0), - production_text, - ) - for label, pattern in PRODUCTION_PATTERNS: - if relative_path == SCANNER_PATH and label == "Hermes API credential assignment": - continue - if pattern.search(production_text): - findings.append(f"{relative_path}: {label}") - # The scanner necessarily contains protected-name detector fixtures. Its exact bytes - # are bound separately by the manifest and normalized all-public-root policy. - if relative_path != SCANNER_PATH and any( - value.casefold() == "hermes_api_key" for value in _resolved_python_strings(text) - ): - findings.append(f"{relative_path}: Hermes API credential assignment") - for value in URL_PATTERN.findall(production_text): - try: - hostname = (urlsplit(value).hostname or "").casefold().rstrip(".") - except ValueError: - findings.append(f"{relative_path}: production-shaped HTTP(S) endpoint") - continue - allowed = ( - hostname in {"localhost", "127.0.0.1", "::1"} - or hostname in DOCUMENTATION_HOSTS - or hostname.endswith(DOCUMENTATION_HOST_SUFFIXES) - ) - if not allowed: - findings.append(f"{relative_path}: production-shaped HTTP(S) endpoint") - return findings - - -def scan_archive(relative_path: str, archive: Path) -> list[str]: - """Inspect every bounded UTF-8 member of a ZIP candidate.""" - - findings: list[str] = [] - try: - with zipfile.ZipFile(archive) as bundle: - members = bundle.infolist() - if len(members) > MAX_ARCHIVE_MEMBERS: - return [f"{relative_path}: archive member count exceeds bound"] - total_bytes = 0 - for member in members: - member_path = PurePosixPath(member.filename) - target = f"{relative_path}!/{member.filename}" - if member.flag_bits & 0x1: - findings.append(f"{target}: encrypted archive member") - continue - if member_path.is_absolute() or ".." in member_path.parts: - findings.append(f"{target}: unsafe archive path") - continue - mode = member.external_attr >> 16 - if stat.S_ISLNK(mode): - findings.append(f"{target}: archive symlink is forbidden") - continue - if member.is_dir(): - continue - total_bytes += member.file_size - if total_bytes > MAX_ARCHIVE_TOTAL_BYTES: - return sorted(findings + [f"{relative_path}: archive size exceeds bound"]) - if member.file_size > MAX_ARCHIVE_MEMBER_BYTES: - findings.append(f"{target}: archive member size exceeds bound") - continue - ratio = member.file_size / max(member.compress_size, 1) - if ratio > MAX_ARCHIVE_COMPRESSION_RATIO: - findings.append(f"{target}: archive compression ratio exceeds bound") - continue - try: - payload = bundle.read(member) - except RuntimeError: - findings.append(f"{target}: unreadable archive member") - continue - if member_path.suffix.casefold() in BINARY_SUFFIXES: - text = payload.decode("latin-1") - else: - try: - text = payload.decode("utf-8") - except UnicodeDecodeError: - findings.append(f"{target}: unexpected non-UTF-8 archive member") - text = payload.decode("latin-1") - findings.extend(scan_text(target, text)) - except (OSError, zipfile.BadZipFile): - return [f"{relative_path}: invalid ZIP archive"] - return sorted(findings) - - -def _scan_bytes(relative_path: str, payload: bytes) -> list[str]: - """Scan one ordinary file or bounded ZIP payload.""" - - digest = hashlib.sha256(payload).hexdigest() - allowed_digests = SYNTHETIC_FILE_SHA256_ALLOWLIST.get( - relative_path, frozenset() - ) | EXACT_PROTECTED_KEY_REFERENCE_SHA256_ALLOWLIST.get(relative_path, frozenset()) - if digest in allowed_digests: - return [] - if Path(relative_path).suffix.casefold() == ".zip": - with tempfile.NamedTemporaryFile(suffix=".zip") as stream: - stream.write(payload) - stream.flush() - return scan_archive(relative_path, Path(stream.name)) - if Path(relative_path).suffix.casefold() in BINARY_SUFFIXES: - text = payload.decode("latin-1") - else: - try: - text = payload.decode("utf-8") - except UnicodeDecodeError: - return [f"{relative_path}: unexpected non-UTF-8 candidate file"] - return scan_text(relative_path, text) - - -def _git(root: Path, *args: str) -> bytes: - try: - return subprocess.run( - ("git", "-C", str(root), *args), - check=True, - capture_output=True, - ).stdout - except (OSError, subprocess.CalledProcessError) as exc: - raise ValueError("candidate Git inventory is unavailable") from exc - - -def _nul_paths(payload: bytes) -> list[str]: - return [value.decode("utf-8") for value in payload.split(b"\0") if value] - - -def _all_candidate_paths(root: Path) -> set[str]: - """Enumerate every file/symlink outside Git metadata, including ignored files.""" - - paths: set[str] = set() - for directory, names, files in os.walk(root, followlinks=False): - directory_path = Path(directory) - if directory_path == root and ".git" in names: - names.remove(".git") - for name in (*names, *files): - path = directory_path / name - if path.is_file() or path.is_symlink(): - paths.add(path.relative_to(root).as_posix()) - return paths - - -def _scan_destination_repository(root: Path, manifest: dict[str, Any]) -> list[str]: - """Scan the closed working tree plus every blob reachable from a Git ref.""" - - findings: list[str] = [] - try: - inventory_policy = { - "entries": sorted( - ( - { - "source": item["source"], - "destination": item["destination"], - "class": item["class"], - } - for item in manifest["entries"] - ), - key=lambda item: item["destination"], - ), - "destination_only": sorted( - ( - {"path": item["path"], "class": item["class"]} - for item in manifest["destination_only"] - ), - key=lambda item: item["path"], - ), - } - policy_payload = json.dumps( - inventory_policy, sort_keys=True, separators=(",", ":") - ).encode("utf-8") - except (KeyError, TypeError): - findings.append("candidate: invalid closed inventory path/class policy") - else: - if hashlib.sha256(policy_payload).hexdigest() != TRUSTED_INVENTORY_POLICY_SHA256: - findings.append("candidate: closed inventory path/class policy mismatch") - tracked = set(_nul_paths(_git(root, "ls-files", "-z"))) - unexpected = _all_candidate_paths(root) - tracked - for relative_path in sorted(unexpected): - findings.append(f"{relative_path}: unexpected untracked candidate file") - path = root / relative_path - if path.is_symlink() or not path.is_file(): - findings.append(f"{relative_path}: unsafe or missing untracked file") - continue - findings.extend(_scan_bytes(relative_path, path.read_bytes())) - - sources: set[str] = set() - destinations: set[str] = set() - for item in manifest["entries"]: - if ( - not isinstance(item, dict) - or not isinstance(item.get("source"), str) - or not isinstance(item.get("destination"), str) - ): - raise ValueError("invalid destination inventory entry") - source_path = item["source"] - relative_path = item["destination"] - if source_path in sources: - findings.append(f"{source_path}: duplicate source inventory entry") - if relative_path in destinations: - findings.append(f"{relative_path}: duplicate destination inventory entry") - sources.add(source_path) - destinations.add(relative_path) - expected_digest = item.get("destination_sha256") - if not isinstance(expected_digest, str) or not re.fullmatch(r"[0-9a-f]{64}", expected_digest): - findings.append(f"{relative_path}: missing destination SHA-256") - elif (root / relative_path).is_file() and hashlib.sha256( - (root / relative_path).read_bytes() - ).hexdigest() != expected_digest: - findings.append(f"{relative_path}: destination SHA-256 mismatch") - - destination_only = manifest.get("destination_only") - if not isinstance(destination_only, list): - raise ValueError("destination-only inventory is missing") - destination_only_paths: set[str] = set() - for item in destination_only: - if not isinstance(item, dict) or not isinstance(item.get("path"), str): - raise ValueError("invalid destination-only inventory entry") - relative_path = item["path"] - if relative_path in destination_only_paths: - findings.append(f"{relative_path}: duplicate destination-only inventory entry") - if relative_path in destinations: - findings.append(f"{relative_path}: path appears in both destination inventories") - destination_only_paths.add(relative_path) - expected_digest = item.get("sha256") - if not isinstance(expected_digest, str) or not re.fullmatch(r"[0-9a-f]{64}", expected_digest): - findings.append(f"{relative_path}: missing destination-only SHA-256") - elif (root / relative_path).is_file() and hashlib.sha256( - (root / relative_path).read_bytes() - ).hexdigest() != expected_digest: - findings.append(f"{relative_path}: destination-only SHA-256 mismatch") - - self_path = manifest.get("self_excluded_path") - if not isinstance(self_path, str): - raise ValueError("manifest self-exclusion is missing") - expected_tracked = destinations | destination_only_paths | {self_path} - findings.extend( - f"{path}: tracked candidate path is absent from closed inventory" - for path in sorted(tracked - expected_tracked) - ) - findings.extend( - f"{path}: closed inventory path is not tracked" - for path in sorted(expected_tracked - tracked) - ) - - for relative_path in sorted(tracked): - path = root / relative_path - if path.is_symlink() or not path.is_file(): - findings.append(f"{relative_path}: unsafe or missing tracked file") - continue - findings.extend(_scan_bytes(relative_path, path.read_bytes())) - - tree_blob_ids: set[str] = set() - for root_label, treeish in _publication_tree_roots(root): - tree_entries = _git(root, "ls-tree", "-r", "-z", treeish).split(b"\0") - for raw_entry in (entry for entry in tree_entries if entry): - metadata, raw_path = raw_entry.split(b"\t", 1) - mode, object_type, raw_object_id = metadata.split(b" ", 2) - relative_path = raw_path.decode("utf-8") - object_id = raw_object_id.decode("ascii") - tree_blob_ids.add(object_id) - if relative_path not in expected_tracked: - findings.append( - f"git:{root_label}:{relative_path}: historical path outside closed inventory" - ) - if object_type != b"blob" or mode not in {b"100644", b"100755"}: - findings.append( - f"git:{root_label}:{relative_path}: historical entry is not a regular file" - ) - for finding in scan_text(f"git-path:{root_label}", relative_path): - _, label = finding.rsplit(": ", 1) - findings.append(f"git:{root_label}:{relative_path}: {label}") - if object_type != b"blob": - continue - payload = _git(root, "cat-file", "blob", object_id) - for finding in _scan_bytes(relative_path, payload): - _, label = finding.split(": ", 1) - findings.append(f"git:{root_label}:{relative_path}: {label}") - - try: - historical_policy_digest = _historical_blob_policy_sha256(root, manifest) - except (KeyError, TypeError, ValueError): - findings.append("candidate: invalid historical blob policy") - else: - if historical_policy_digest != TRUSTED_HISTORICAL_BLOB_POLICY_SHA256: - findings.append("candidate: historical blob policy mismatch") - - reachable_objects = { - line.split(b" ", 1)[0].decode("ascii") - for line in _git(root, "rev-list", "--objects", "--all").splitlines() - if line - } - for object_id in sorted(reachable_objects): - object_type = _git(root, "cat-file", "-t", object_id).decode("ascii").strip() - if object_type not in {"blob", "commit", "tag"}: - continue - if object_type == "blob" and object_id in tree_blob_ids: - continue - payload = _git(root, "cat-file", object_type, object_id) - for finding in _scan_bytes(f"git-object:{object_id}:{object_type}.txt", payload): - _, label = finding.rsplit(": ", 1) - findings.append(f"git-object:{object_id}:{object_type}: {label}") - return sorted(set(findings)) - - -def _load_manifest(path: Path) -> dict[str, Any]: - value = json.loads(path.read_text(encoding="utf-8")) - if not isinstance(value, dict) or not isinstance(value.get("entries"), list): - raise ValueError("invalid public plugin move manifest") - return value - - -def scan_candidate( - root: Path, - manifest_path: Path, - *, - layout: Literal["auto", "source", "destination"] = "auto", -) -> list[str]: - """Scan the closed candidate repository or the exact embedded source manifest.""" - - manifest = _load_manifest(manifest_path) - findings: list[str] = [] - seen: set[str] = set() - entries = manifest["entries"] - if layout == "auto": - if all( - isinstance(item, dict) and (root / str(item.get("source"))).is_file() - for item in entries - ): - selected_layout = "source" - elif all( - isinstance(item, dict) and (root / str(item.get("destination"))).is_file() - for item in entries - ): - selected_layout = "destination" - else: - return ["candidate: neither source nor destination layout is complete"] - else: - selected_layout = layout - if selected_layout == "destination": - return _scan_destination_repository(root, manifest) - for item in entries: - if ( - not isinstance(item, dict) - or not isinstance(item.get("source"), str) - or not isinstance(item.get("destination"), str) - ): - raise ValueError("invalid public plugin move entry") - relative_path = item[selected_layout] - if relative_path in seen: - raise ValueError("duplicate public plugin move source") - seen.add(relative_path) - source = root / relative_path - if not source.is_file(): - findings.append(f"{relative_path}: missing manifest source") - continue - expected_digest = item.get("source_sha256") - if not isinstance(expected_digest, str) or not re.fullmatch(r"[0-9a-f]{64}", expected_digest): - findings.append(f"{relative_path}: missing source SHA-256") - elif hashlib.sha256(source.read_bytes()).hexdigest() != expected_digest: - findings.append(f"{relative_path}: source SHA-256 mismatch") - findings.extend(_scan_bytes(relative_path, source.read_bytes())) - return sorted(findings) - - -def main(argv: list[str] | None = None) -> int: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--root", type=Path, default=Path.cwd()) - parser.add_argument("--manifest", type=Path) - parser.add_argument("--layout", choices=("auto", "source", "destination"), default="auto") - args = parser.parse_args(argv) - root = args.root.resolve() - if args.manifest is None: - candidates = ( - root / "docs" / "extraction-manifest.json", - root / "docs" / "file-move-manifest.json", - ) - manifest = next( - (candidate for candidate in candidates if candidate.is_file()), candidates[0] - ) - else: - manifest = args.manifest if args.manifest.is_absolute() else root / args.manifest - findings = scan_candidate(root, manifest, layout=args.layout) - print(json.dumps({"findings": findings, "status": "pass" if not findings else "fail"})) - return 0 if not findings else 1 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/src/substrate_wiki/README.md b/src/substrate_wiki/README.md deleted file mode 100644 index fc66c69..0000000 --- a/src/substrate_wiki/README.md +++ /dev/null @@ -1,77 +0,0 @@ -# Substrate Wiki memory provider - -> **Legacy — not the current install target.** This provider supports Hermes 0.20.x only and -> its v2.0.5 release is not published. The current Hermes 0.21.x plugin is -> [`plugins/substrate`](../../plugins/substrate); install that instead. Do not use this -> document to reject a Hermes 0.21.x installation. - -`substrate_wiki` connects Hermes Agent 0.20.x to the hosted Substrate service at -`https://app.trysubstrate.co`. It provides cited recall, automatic future capture, and an -optional, resumable import of eligible prior conversations and explicitly saved memories. - -## Install and connect - -Use the verified release installer. It installs beneath -`$HERMES_HOME/plugins/substrate_wiki`, activates `memory.provider: substrate_wiki`, and -starts browser/device onboarding automatically: - -```bash -python3 install_hermes_plugin.py \ - --archive substrate_wiki.zip \ - --sha256 \ - --yes --json -``` - -Hermes 0.20.x discovers user plugins directly beneath `$HERMES_HOME/plugins`; do not add -another directory level or install under `plugins/memory`. The plugin has no third-party -runtime dependencies. - -On a desktop the verification page opens automatically. In a headless environment the -installer prints the hosted URL and one-time user code. Sign in with the hosted magic-link -flow and approve the device. The resulting tenant-scoped, revocable credential is stored in -the OS credential store when available, with an owner-private profile file as the fallback. -It is never written to `plugin.yaml`, `config.json`, command arguments, or logs. - -The hosted origin is fixed. `HERMES_API_URL` and `HERMES_API_KEY` are not normal setup -options. Legacy shared bearer configuration exists only on the server as a temporary -migration path. - -## History consent - -After browser approval, the polling installer returns control to the same Hermes -conversation with `action_required: history_consent`. The agent must report that the -connection succeeded and ask whether to upload eligible history. It must not infer or -pre-authorize the answer. Blank or interrupted input leaves consent pending. - -This is the only optional step. Declining history upload leaves future capture and recall -enabled. Approval starts exactly one durable profile-scoped job. Its content-free status reports discovered, -uploaded, processed, duplicate, retry, and error counts; disconnects and restarts resume the -same checkpoint and stable batch/event IDs. - -Eligible history includes direct/one-to-one conversations and explicit saved memories. -Group chats, cron/webhook sessions, unrelated files, hidden reasoning, secrets, and binary -bodies are excluded. Content is bounded and redacted before local spooling or network -transfer. - -Management commands include: - -```bash -hermes substrate_wiki onboarding-status --json -hermes substrate_wiki import-status --json -hermes substrate_wiki import-resume --yes --wait --json -hermes substrate_wiki import-cancel --job-id --yes --json -``` - -## Operation and privacy - -Live completed user/assistant turns are delivered asynchronously. Failed -transient deliveries stay in a bounded owner-private spool and retry with capped backoff. -Authentication failures trigger automatic reconnect onboarding rather than exposing or -logging credentials. Status, receipts, checkpoints, and diagnostics are content-free. - -Visible prompts and assistant output can be sent after redaction. Tool calls, tool results, -system messages, memory-write events, and provider/session metadata are excluded. Redaction is -defense in depth, not proof that arbitrary sensitive prose is absent. - -The provider exposes bounded cited wiki search/read/query/ingest/job tools and automatic -memory-card prefetch. It exposes no arbitrary filesystem-write tool. diff --git a/src/substrate_wiki/__init__.py b/src/substrate_wiki/__init__.py deleted file mode 100644 index 8d75537..0000000 --- a/src/substrate_wiki/__init__.py +++ /dev/null @@ -1,988 +0,0 @@ -"""Substrate Wiki external memory provider for Hermes Agent v0.20.0.""" - -from __future__ import annotations - -import hashlib -import json -import math -import os -import sys -import queue -import random -import re -import threading -import time -from collections import OrderedDict -from pathlib import Path -from typing import Any - -try: - from agent.memory_provider import MemoryProvider -except ModuleNotFoundError as exc: # Allows contract tests without Hermes installed. - if exc.name not in {"agent", "agent.memory_provider"}: - raise - - class MemoryProvider: # type: ignore[no-redef] - pass - - -from .client import SubstrateAPIError, SubstrateClient -from .events import CaptureEventBuilder -from .redaction import configured_secret_values -from .spool import DurableSpool, secure_atomic_json_write - -__all__ = ["SubstrateWikiProvider", "register"] - -_PROVIDER_ID = "substrate_wiki" -_MAX_TOOL_RESULT_BYTES = 64 * 1024 -_MAX_SESSIONS = 32 -_MAX_PREFETCH_BYTES = 16 * 1024 -_MAX_PREFETCH_ENTRIES = 128 -_MIN_SPOOL_BYTES = 16 * 1024 -_RETRY_BASE_SECONDS = 1.0 -_AUTH_RETRY_BASE_SECONDS = 30.0 -_MAX_RETRY_SECONDS = 300.0 -_MAX_RETRY_EXPONENT = 8 -_RETRY_JITTER_FRACTION = 0.2 -_ENTITY_TYPES = frozenset( - { - "person", - "agent", - "organization", - "project", - "product", - "place", - "event", - "other", - "system", - "service", - "automation", - } -) -_ENTITY_FILENAME = re.compile(r"[a-z0-9](?:[a-z0-9-]{0,79})--[0-9a-f]{8}\.md") -_SCOPE_KWARGS = { - "agent_identity": "agent_identity", - "agent_workspace": "agent_workspace", - "user_id": "user_id", - "platform": "platform", - "agent_id": "agent_id", - "agent_name": "agent_name", - "workspace": "workspace", - "profile": "profile", - "user": "user", - "chat_type": "chat_type", -} -_CONTEXT_SCOPE_FIELDS = ( - "agent_id", - "agent_name", - "workspace", - "profile", - "user", - "platform", - "chat_type", -) -_NON_PRIMARY_ROLES = {"cron", "subagent", "child", "secondary", "worker", "flush", "gateway"} - -# Hermes v0.20.0 consumes direct function schemas, not OpenAI's {type,function} envelope. -_TOOL_SCHEMAS = [ - { - "name": "wiki_search", - "description": "Search maintained Substrate wiki pages and return matching cited material.", - "parameters": { - "type": "object", - "properties": { - "query": {"type": "string", "description": "Search terms.", "maxLength": 4096}, - "limit": {"type": "integer", "minimum": 1, "maximum": 25, "default": 8}, - }, - "required": ["query"], - "additionalProperties": False, - }, - }, - { - "name": "wiki_read", - "description": "Read one maintained wiki page by its repository-relative path or legacy slug.", - "parameters": { - "type": "object", - "properties": { - "path": { - "type": "string", - "description": "Wiki page path or slug, for example notes/hermes.md or notes/hermes.", - "maxLength": 4096, - } - }, - "required": ["path"], - "additionalProperties": False, - }, - }, - { - "name": "wiki_query", - "description": "Ask a question over the wiki and receive a synthesized, cited answer.", - "parameters": { - "type": "object", - "properties": { - "question": {"type": "string", "maxLength": 16384}, - "save_as_synthesis": {"type": "boolean", "default": False}, - }, - "required": ["question"], - "additionalProperties": False, - }, - }, - { - "name": "wiki_ingest", - "description": "Submit text or a public URL for asynchronous ingestion into the wiki.", - "parameters": { - "type": "object", - "properties": { - "content": { - "type": "string", - "description": "Text or public URL to ingest.", - "maxLength": 262144, - }, - "title": {"type": "string", "maxLength": 512}, - "source_type": {"type": "string", "enum": ["text", "url"], "default": "text"}, - }, - "required": ["content"], - "additionalProperties": False, - }, - }, - { - "name": "wiki_job_status", - "description": "Check the current state of a Substrate asynchronous job.", - "parameters": { - "type": "object", - "properties": {"job_id": {"type": "string", "maxLength": 512}}, - "required": ["job_id"], - "additionalProperties": False, - }, - }, -] - - -def _bounded_json(value: Any, *, limit: int = _MAX_TOOL_RESULT_BYTES) -> str: - """Serialize to a valid bounded JSON string without leaking arbitrary reprs.""" - try: - rendered = json.dumps(value, ensure_ascii=False, separators=(",", ":"), sort_keys=True) - except (TypeError, ValueError): - rendered = json.dumps({"error": "invalid_response"}, separators=(",", ":")) - encoded = rendered.encode("utf-8") - if len(encoded) <= limit: - return rendered - digest = hashlib.sha256(encoded).hexdigest() - return json.dumps( - {"error": "response_too_large", "sha256": digest, "original_bytes": len(encoded)}, - separators=(",", ":"), - sort_keys=True, - ) - - -def _context_is_primary(context: Any, *, default: bool = True) -> bool: - """Interpret Hermes string, mapping, and object runtime contexts consistently.""" - if context is None: - return default - if isinstance(context, str): - return context.strip().lower() not in _NON_PRIMARY_ROLES - if _context_value(context, "is_primary", True) is False: - return False - if _context_value(context, "primary", True) is False: - return False - role = ( - _context_value(context, "role") - or _context_value(context, "runtime_role") - or _context_value(context, "source") - ) - return not (isinstance(role, str) and role.lower() in _NON_PRIMARY_ROLES) - - -def _context_value(context: Any, key: str, default: Any = None) -> Any: - if isinstance(context, dict): - return context.get(key, default) - return getattr(context, key, default) - - -def _bounded_scope_value(value: Any) -> str: - if value is None: - return "" - if isinstance(value, (str, int, float, bool)): - return str(value)[:512] - return "" - - -class SubstrateWikiProvider(MemoryProvider): - """Hermes provider backed by the authenticated Substrate HTTP API.""" - - @property - def name(self) -> str: - return _PROVIDER_ID - - def __init__(self) -> None: - self._session_id = "" - self._home: Path | None = None - self._client: SubstrateClient | None = None - self._event_builder: CaptureEventBuilder | None = None - self._spool: DurableSpool | None = None - self._events: queue.Queue[dict[str, Any]] = queue.Queue(maxsize=256) - self._prefetch_jobs: queue.Queue[tuple[str, str]] = queue.Queue(maxsize=32) - self._stop = threading.Event() - self._wake = threading.Event() - self._worker: threading.Thread | None = None - self._prefetch_worker: threading.Thread | None = None - self._onboarding_worker: threading.Thread | None = None - self._prefetch_cache: dict[str, tuple[float, str]] = {} - self._latest_prefetch_cache: OrderedDict[str, tuple[float, str]] = OrderedDict() - self._cache_lock = threading.Lock() - self._capture_lock = threading.RLock() - self._delivery_lock = threading.Lock() - self._current_event: dict[str, Any] | None = None - self._current_persisted: Path | None = None - self._delivery_failure_streak = 0 - self._retry_random = random.Random() - self._message_offsets: OrderedDict[str, int] = OrderedDict() - self._secrets: tuple[str, ...] = () - self._scope: dict[str, str] = {"provider_id": _PROVIDER_ID} - self._initialized_primary = True - self._counters = { - "captured": 0, - "suppressed": 0, - "delivered": 0, - "delivery_failed": 0, - "spooled": 0, - "spool_evicted": 0, - "quarantined": 0, - "permanent_dropped": 0, - "dropped": 0, - "prefetch_queued": 0, - "prefetch_cached": 0, - "prefetch_failed": 0, - } - self._last_delivery_category = "none" - self._last_prefetch_category = "none" - self._settings: dict[str, Any] = { - "api_url": "https://app.trysubstrate.co", - "spool_max_items": 1000, - "spool_max_bytes": 10 * 1024 * 1024, - "prefetch_ttl_seconds": 60, - } - - def is_available(self) -> bool: - """The hosted provider activates before sign-in; still reject unsafe overrides.""" - override = os.environ.get("HERMES_API_URL", "") - return not override or override.rstrip("/") == "https://app.trysubstrate.co" - - def post_setup(self, hermes_home: str, config: dict[str, Any]) -> None: - """Hermes v0.20 setup hook: activate, sign in, then ask history consent.""" - from .onboarding import main as onboarding_main - - memory = config.setdefault("memory", {}) - if not isinstance(memory, dict): - memory = {} - config["memory"] = memory - memory["provider"] = _PROVIDER_ID - try: - from hermes_cli.config import save_config - - save_config(config) - except ImportError: - pass - mode = "auto" if sys.stdin.isatty() else "device" - onboarding_main([ - "--hermes-home", hermes_home, "--mode", mode, "--wait", "--history", "ask" - ]) - - def initialize(self, session_id: str, **kwargs: Any) -> None: - hermes_home = kwargs.get("hermes_home") - if not hermes_home: - raise ValueError("hermes_home is required") - self._session_id = str(session_id or "") - self._home = Path(hermes_home) / _PROVIDER_ID - if self._home.exists() and self._home.is_symlink(): - raise ValueError("plugin state directory must not be a symlink") - self._home.mkdir(parents=True, exist_ok=True, mode=0o700) - if os.name == "posix": - os.chmod(self._home, 0o700) - agent_context = kwargs.get("agent_context") - self._initialized_primary = _context_is_primary(agent_context) - self._scope = {"provider_id": _PROVIDER_ID} - for source, target in _SCOPE_KWARGS.items(): - value = _bounded_scope_value(kwargs.get(source)) - if value: - self._scope[target] = value - for field in _CONTEXT_SCOPE_FIELDS: - if field in self._scope: - continue - value = _bounded_scope_value(_context_value(agent_context, field)) - if value: - self._scope[field] = value - self._load_settings() - # Hosted onboarding is pinned; no local discovery/start or arbitrary origin. - self._settings["api_url"] = "https://app.trysubstrate.co" - self._client = SubstrateClient.from_env( - fallback_url="https://app.trysubstrate.co", - hermes_home=Path(hermes_home), - hosted_default=True, - ) - self._settings["spool_max_bytes"] = max( - _MIN_SPOOL_BYTES, int(self._settings["spool_max_bytes"]) - ) - self._spool = DurableSpool( - self._home / "spool", - max_items=int(self._settings["spool_max_items"]), - max_bytes=int(self._settings["spool_max_bytes"]), - ) - client_key = str(getattr(self._client, "api_key", "") or "") - self._secrets = ( - tuple(dict.fromkeys((*configured_secret_values(), client_key))) - if client_key - else configured_secret_values() - ) - self._event_builder = CaptureEventBuilder(self._scope, secrets=self._secrets) - self._stop.clear() - self._wake.clear() - self._worker = threading.Thread( - target=self._sender_loop, name="substrate_wiki_sender", daemon=True - ) - self._prefetch_worker = threading.Thread( - target=self._prefetch_loop, name="substrate_wiki_prefetch", daemon=True - ) - self._worker.start() - self._prefetch_worker.start() - if self._initialized_primary: - self._onboarding_worker = threading.Thread( - target=(self._repair_onboarding if client_key else self._begin_onboarding), - name=( - "substrate_wiki_onboarding_repair" - if client_key - else "substrate_wiki_onboarding" - ), - daemon=True, - ) - self._onboarding_worker.start() - - def get_tool_schemas(self) -> list[dict[str, Any]]: - return list(_TOOL_SCHEMAS) - - def handle_tool_call(self, tool_name: str, args: dict[str, Any], **kwargs: Any) -> str: - if not isinstance(args, dict): - return _bounded_json({"error": "invalid_arguments"}) - client = self._require_client() - try: - if tool_name == "wiki_search": - result = client.search( - self._input(args, "query", 4096), - limit=max(1, min(int(args.get("limit", 8)), 25)), - ) - elif tool_name == "wiki_read": - result = client.read_page(self._input(args, "path", 4096)) - elif tool_name == "wiki_query": - result = client.query_wiki( - self._input(args, "question", 16384), - save_as_synthesis=self._boolean(args, "save_as_synthesis", False), - ) - elif tool_name == "wiki_ingest": - result = client.ingest( - self._input(args, "content", 262144), - title=self._optional_input(args, "title", 512), - source_type=self._source_type(args.get("source_type", "text")), - ) - elif tool_name == "wiki_job_status": - result = client.job_status(self._input(args, "job_id", 512)) - else: - result = {"error": "unknown_tool"} - except (KeyError, TypeError, ValueError): - result = {"error": "invalid_arguments"} - except SubstrateAPIError as exc: - result = {"error": exc.category} - return _bounded_json(result) - - def get_config_schema(self) -> list[dict[str, Any]]: - """Hosted sign-in owns credentials; Hermes must not prompt for API keys.""" - return [] - - def save_config(self, values: dict[str, Any], hermes_home: str) -> None: - """Persist only the immutable hosted origin and non-secret tuning.""" - root = Path(hermes_home) / _PROVIDER_ID - if root.exists() and root.is_symlink(): - raise ValueError("plugin state directory must not be a symlink") - root.mkdir(parents=True, exist_ok=True, mode=0o700) - secure_atomic_json_write(root / "config.json", { - "api_url": "https://app.trysubstrate.co", - "hosted": True, - "spool_max_items": max(1, int(values.get("spool_max_items", self._settings["spool_max_items"]))), - "spool_max_bytes": max(_MIN_SPOOL_BYTES, int(values.get("spool_max_bytes", self._settings["spool_max_bytes"]))), - "prefetch_ttl_seconds": max(1, int(values.get("prefetch_ttl_seconds", self._settings["prefetch_ttl_seconds"]))), - }) - self._settings["api_url"] = "https://app.trysubstrate.co" - - def _repair_onboarding(self) -> None: - try: - from .onboarding import OnboardingManager - - assert self._home is not None - OnboardingManager(self._home.parent).repair(wait=False) - except Exception: # noqa: BLE001 - status remains content-free and resumable - return - - def _request_reconnect(self) -> None: - if self._home is None or not self._initialized_primary: - return - worker = self._onboarding_worker - if worker is not None and worker.is_alive(): - return - try: - from .credentials import credential_store - - credential_store(self._home.parent).delete() - except (OSError, ValueError): - return - self._onboarding_worker = threading.Thread( - target=self._begin_onboarding, - name="substrate_wiki_reconnect", - daemon=True, - ) - self._onboarding_worker.start() - - def _begin_onboarding(self) -> None: - """Complete hosted sign-in in the background, then wake durable delivery.""" - try: - from .onboarding import OnboardingManager - - assert self._home is not None - result = OnboardingManager(self._home.parent).run(mode="auto", wait=True) - if not result.get("authenticated"): - return - client = SubstrateClient.from_env( - fallback_url="https://app.trysubstrate.co", - hermes_home=self._home.parent, - hosted_default=True, - ) - if not client.api_key: - return - with self._capture_lock: - self._client = client - self._secrets = tuple( - dict.fromkeys((*configured_secret_values(), client.api_key)) - ) - self._event_builder = CaptureEventBuilder(self._scope, secrets=self._secrets) - self._wake.set() - except Exception: # noqa: BLE001 - surfaced by content-free onboarding status - return - - def system_prompt_block(self) -> str: - return ( - "Substrate Wiki is the single published memory. Automatic recall contains only canonical published entity " - "wiki pages, never raw conversation transcripts, private claims, or extraction state. Treat temporal " - "qualifiers and citations as evidence metadata, surface contradictions, and use wiki_ingest only when the " - "user asks to add source material." - ) - - def prefetch(self, query: str, *, session_id: str = "") -> str: - """Return already-cached cited context without performing network I/O.""" - if not isinstance(query, str) or not query.strip(): - return "" - sid = session_id or self._session_id - cache_key = self._cache_key(query[:4096], sid) - session_key = self._session_cache_key(sid) - with self._cache_lock: - self._evict_prefetch_locked() - cached = self._prefetch_cache.get(cache_key) - if cached: - return cached[1] - latest = self._latest_prefetch_cache.get(session_key) - return latest[1] if latest else "" - - def queue_prefetch(self, query: str, *, session_id: str = "") -> None: - """Queue cache warming on one bounded worker; never create per-turn threads.""" - if not isinstance(query, str) or not query.strip() or self._stop.is_set(): - return - job = (query[:4096], session_id or self._session_id) - try: - self._prefetch_jobs.put_nowait(job) - self._counters["prefetch_queued"] += 1 - except queue.Full: - return - - def sync_turn( - self, - user: Any, - assistant: Any, - *, - session_id: str = "", - runtime_context: Any = None, - ) -> None: - """Upload only the completed user/assistant pair. - - Deliberately use Hermes's legacy signature so the host does not pass the - full transcript, tool calls, or tool results into this provider. - """ - if not self._initialized_primary or not _context_is_primary(runtime_context): - self._counters["suppressed"] += 1 - return - sid = str(session_id or self._session_id) - messages: list[dict[str, Any]] = [] - if user is not None and user != "": - messages.append({"role": "user", "content": user}) - if assistant is not None and assistant != "": - messages.append({"role": "assistant", "content": assistant}) - if messages: - self._capture_turn(sid, messages) - - def on_pre_compress(self, messages: list[dict[str, Any]], **kwargs: Any) -> str: - del messages, kwargs - return "" - - def on_session_end(self, messages: list[dict[str, Any]], **kwargs: Any) -> None: - del messages, kwargs - - def on_memory_write( - self, - action: str, - target: str, - content: Any, - metadata: dict[str, Any] | None = None, - **kwargs: Any, - ) -> None: - if not self._initialized_primary: - self._counters["suppressed"] += 1 - del action, target, content, metadata, kwargs - - def on_session_switch( - self, - new_session_id: str, - *, - parent_session_id: str = "", - reset: bool = False, - rewound: bool = False, - **kwargs: Any, - ) -> None: - """Rebind future turn uploads without emitting a metadata event.""" - del parent_session_id, reset, rewound, kwargs - new_session = str(new_session_id or "") - with self._capture_lock: - self._session_id = new_session - - def shutdown(self) -> None: - """Stop workers within five seconds; each sender event is write-ahead spooled.""" - deadline = time.monotonic() + 5.0 - self._stop.set() - self._wake.set() - if self._prefetch_worker and self._prefetch_worker.is_alive(): - self._prefetch_worker.join(timeout=max(0.0, deadline - time.monotonic())) - while True: - try: - self._prefetch_jobs.get_nowait() - except queue.Empty: - break - else: - self._prefetch_jobs.task_done() - if self._worker and self._worker.is_alive(): - self._worker.join(timeout=max(0.0, deadline - time.monotonic())) - - def _load_settings(self) -> None: - assert self._home is not None - path = self._home / "config.json" - if path.is_symlink(): - return - try: - values = json.loads(path.read_text(encoding="utf-8")) - except (FileNotFoundError, OSError, UnicodeError, ValueError): - return - if not isinstance(values, dict): - return - api_url = values.get("api_url") - if isinstance(api_url, str) and ( - not api_url or SubstrateClient.is_allowed_base_url(api_url) - ): - self._settings["api_url"] = api_url.rstrip("/") - for key in ("spool_max_items", "spool_max_bytes", "prefetch_ttl_seconds"): - try: - if key in values: - minimum = _MIN_SPOOL_BYTES if key == "spool_max_bytes" else 1 - self._settings[key] = max(minimum, int(values[key])) - except (TypeError, ValueError): - continue - - def _capture_turn(self, session_id: str, messages: list[dict[str, Any]]) -> None: - """Spool one compact dialogue turn with monotonically increasing indexes.""" - with self._capture_lock: - start = self._message_offsets.get(session_id, 0) - events = self._require_event_builder().iter_message_events( - "turn", - session_id, - messages, - start_index=start, - ) - captured = 0 - for event in events: - if not self._enqueue(event): - return - captured += 1 - if not captured: - return - self._counters["captured"] += captured - self._message_offsets[session_id] = start + len(messages) - self._message_offsets.move_to_end(session_id) - while len(self._message_offsets) > _MAX_SESSIONS: - self._message_offsets.popitem(last=False) - - def _require_event_builder(self) -> CaptureEventBuilder: - if self._event_builder is None: - raise RuntimeError("event builder is not initialized") - return self._event_builder - - def _enqueue(self, event: dict[str, Any]) -> bool: - path = self._persist(event) - if path is None: - return False - self._wake.set() - return True - - def _sender_loop(self) -> None: - while True: - event, spool_path = self._next_event() - if event is None: - if self._stop.is_set(): - return - self._wake.wait(0.5) - self._wake.clear() - continue - with self._delivery_lock: - self._current_event = event - self._current_persisted = spool_path - try: - self._deliver(event) - except SubstrateAPIError as exc: - self._counters["delivery_failed"] += 1 - self._last_delivery_category = exc.category - if exc.category in {"http_401", "http_403"}: - self._request_reconnect() - if not self._is_transient_error(exc.category): - self._reset_delivery_backoff() - self._discard_spooled(spool_path, quarantine=True) - self._counters["permanent_dropped"] += 1 - else: - self._delivery_failure_streak += 1 - delay = self._retry_delay( - exc.category, - self._delivery_failure_streak, - retry_after=exc.retry_after, - ) - if spool_path is not None and self._spool is not None: - self._spool.release(spool_path) - if self._wait_for_retry(delay): - return - except (KeyError, TypeError, ValueError): - self._counters["delivery_failed"] += 1 - self._last_delivery_category = "invalid_spooled_event" - self._reset_delivery_backoff() - self._discard_spooled(spool_path, quarantine=True) - self._counters["permanent_dropped"] += 1 - except Exception: # A malformed custom client must not kill the sender. - self._counters["delivery_failed"] += 1 - self._last_delivery_category = "delivery_exception" - self._reset_delivery_backoff() - self._discard_spooled(spool_path, quarantine=True) - self._counters["permanent_dropped"] += 1 - else: - self._reset_delivery_backoff() - self._counters["delivered"] += 1 - self._last_delivery_category = "ok" - if spool_path is not None and self._spool is not None: - self._spool.remove(spool_path) - finally: - with self._delivery_lock: - self._current_event = None - self._current_persisted = None - if self._stop.is_set(): - return - - def _prefetch_loop(self) -> None: - while True: - try: - query, session_id = self._prefetch_jobs.get(timeout=0.25) - except queue.Empty: - if self._stop.is_set(): - return - continue - try: - if self._stop.is_set(): - continue - client = self._require_client() - memory_search = getattr(client, "memory_search", None) - if not callable(memory_search): - raise SubstrateAPIError("server_upgrade_required") - result = memory_search(query, limit=5, scope=self._scope) - cited = self._cited_prefetch(result) - if cited: - key = self._cache_key(query, session_id) - session_key = self._session_cache_key(session_id) - with self._cache_lock: - if session_id == self._session_id and not self._stop.is_set(): - self._evict_prefetch_locked() - while len(self._prefetch_cache) >= _MAX_PREFETCH_ENTRIES: - self._prefetch_cache.pop(next(iter(self._prefetch_cache))) - expires = time.monotonic() + int(self._settings["prefetch_ttl_seconds"]) - self._prefetch_cache[key] = (expires, cited) - self._latest_prefetch_cache[session_key] = (expires, cited) - self._latest_prefetch_cache.move_to_end(session_key) - while len(self._latest_prefetch_cache) > _MAX_SESSIONS: - self._latest_prefetch_cache.popitem(last=False) - self._counters["prefetch_cached"] += 1 - self._last_prefetch_category = "ok" - except SubstrateAPIError as exc: - self._counters["prefetch_failed"] += 1 - self._last_prefetch_category = exc.category - finally: - self._prefetch_jobs.task_done() - - @staticmethod - def _cited_prefetch(result: Any) -> str: - if not isinstance(result, dict): - return "" - raw_results = result.get("results") - if not isinstance(raw_results, list): - return "" - blocks: list[str] = [] - seen_entities: set[str] = set() - used = 0 - for item in raw_results[:5]: - if not isinstance(item, dict): - continue - citation = SubstrateWikiProvider._canonical_entity_path(item) - text = item.get("memory_card") - if not citation or not text: - continue - entity_id = str(item["entity_id"]).strip() - if entity_id in seen_entities: - continue - block = f"{text.strip()[:4096]}\nSource: {citation.strip()[:1024]}" - encoded = block.encode("utf-8") - if used + len(encoded) > _MAX_PREFETCH_BYTES: - break - blocks.append(block) - seen_entities.add(entity_id) - used += len(encoded) + 2 - return "\n\n".join(blocks) - - @staticmethod - def _canonical_entity_path(item: dict[str, Any]) -> str | None: - """Accept only immutable canonical entity wiki paths for automatic recall.""" - path = item.get("canonical_path") - entity_id = item.get("entity_id") - entity_type = item.get("entity_type") - if ( - item.get("page_type") != "entity" - or item.get("quality_version") != 2 - or not isinstance(item.get("memory_card"), str) - or not isinstance(path, str) - or not isinstance(entity_id, str) - or not entity_id.strip() - or len(entity_id) > 256 - or not isinstance(entity_type, str) - or entity_type not in _ENTITY_TYPES - or "\\" in path - ): - return None - parts = path.split("/") - if ( - len(parts) != 3 - or parts[0] != "entities" - or parts[1] not in _ENTITY_TYPES - or _ENTITY_FILENAME.fullmatch(parts[2]) is None - ): - return None - return path - - def _next_event(self) -> tuple[dict[str, Any] | None, Path | None]: - if self._spool is None: - return None, None - path = self._spool.claim_oldest() - if path is None: - return None, None - try: - return self._spool.load(path), path - except (OSError, UnicodeError, ValueError, TypeError): - self._spool.quarantine(path) - self._counters["quarantined"] += 1 - return None, None - - def _deliver(self, event: dict[str, Any]) -> None: - if not isinstance(event, dict): - raise ValueError("invalid event") - kind = event.get("kind") - event_id = event.get("event_id") - if not isinstance(kind, str) or not isinstance(event_id, str) or not event_id: - raise ValueError("invalid event") - path = { - "turn": "/api/v1/hermes/turns", - "pre_compress": "/api/v1/hermes/turns", - "session_boundary": "/api/v1/hermes/turns", - "session_end": "/api/v1/hermes/completed-sessions", - "memory_write": "/api/v1/hermes/memory-write-events", - }.get(kind) - if path is None: - raise ValueError("unknown event kind") - self._require_client().request("POST", path, body=event, idempotency_key=event_id) - - def _persist(self, event: dict[str, Any]) -> Path | None: - if self._spool is None: - self._counters["dropped"] += 1 - return None - before = self._spool.evicted_count - try: - path = self._spool.append(event) - self._counters["spooled"] += 1 - self._counters["spool_evicted"] += self._spool.evicted_count - before - return path - except (OSError, TypeError, ValueError): - self._counters["dropped"] += 1 - return None - - def _discard_spooled(self, path: Path | None, *, quarantine: bool) -> None: - if path is None or self._spool is None: - return - if quarantine: - self._spool.quarantine(path) - self._counters["quarantined"] += 1 - else: - self._spool.remove(path) - - def _retry_delay( - self, - category: str, - failure_streak: int, - *, - retry_after: float | None = None, - ) -> float: - base = ( - _AUTH_RETRY_BASE_SECONDS - if category in {"http_401", "http_403", "not_configured", "invalid_api_url"} - else _RETRY_BASE_SECONDS - ) - exponent = min(max(0, failure_streak - 1), _MAX_RETRY_EXPONENT) - delay = min(_MAX_RETRY_SECONDS, base * (2**exponent)) - jittered = delay * self._retry_random.uniform( - 1.0 - _RETRY_JITTER_FRACTION, - 1.0 + _RETRY_JITTER_FRACTION, - ) - if ( - isinstance(retry_after, (int, float)) - and not isinstance(retry_after, bool) - and math.isfinite(retry_after) - ): - retry_floor = max(0.0, min(float(retry_after), _MAX_RETRY_SECONDS)) - jittered = max(jittered, retry_floor) - return max(0.0, min(jittered, _MAX_RETRY_SECONDS)) - - def _reset_delivery_backoff(self) -> None: - self._delivery_failure_streak = 0 - - def _wait_for_retry(self, delay: float) -> bool: - return self._stop.wait(delay) - - @staticmethod - def _is_transient_error(category: str) -> bool: - if category in {"timeout", "transport_error", "not_configured", "invalid_api_url"}: - return True - if category in {"http_401", "http_403", "http_429"}: - return True - if category.startswith("http_"): - try: - return int(category[5:]) >= 500 - except ValueError: - return False - return False - - def status_snapshot(self) -> dict[str, Any]: - """Return bounded operational status without identifiers, content, URLs, or secrets.""" - return { - "provider_id": _PROVIDER_ID, - "initialized": self._home is not None, - "primary_runtime": self._initialized_primary, - "workers": { - "sender": bool(self._worker and self._worker.is_alive()), - "prefetch": bool(self._prefetch_worker and self._prefetch_worker.is_alive()), - }, - "queues": { - "events": min(self._events.qsize(), self._events.maxsize), - "events_capacity": self._events.maxsize, - "prefetch": min(self._prefetch_jobs.qsize(), self._prefetch_jobs.maxsize), - "prefetch_capacity": self._prefetch_jobs.maxsize, - "spool": min(len(self._spool), int(self._settings["spool_max_items"])) - if self._spool - else 0, - }, - "counters": {key: int(value) for key, value in self._counters.items()}, - "last_delivery_category": self._last_delivery_category, - "last_prefetch_category": self._last_prefetch_category, - } - - @staticmethod - def _read_persisted_api_url(root: Path) -> str: - if root.is_symlink(): - return "" - path = root / "config.json" - if path.is_symlink(): - return "" - try: - values = json.loads(path.read_text(encoding="utf-8")) - except (FileNotFoundError, OSError, UnicodeError, ValueError): - return "" - value = values.get("api_url") if isinstance(values, dict) else None - return ( - value.rstrip("/") - if isinstance(value, str) and SubstrateClient.is_allowed_base_url(value) - else "" - ) - - def _require_client(self) -> SubstrateClient: - if self._client is None: - raise SubstrateAPIError("not_initialized") - return self._client - - def _evict_prefetch_locked(self) -> None: - now = time.monotonic() - expired = [key for key, (expires, _) in self._prefetch_cache.items() if expires <= now] - for key in expired: - self._prefetch_cache.pop(key, None) - expired_sessions = [ - key for key, (expires, _) in self._latest_prefetch_cache.items() if expires <= now - ] - for key in expired_sessions: - self._latest_prefetch_cache.pop(key, None) - - @staticmethod - def _cache_key(query: str, session_id: str) -> str: - return hashlib.sha256(f"{session_id}\0{query}".encode()).hexdigest() - - @staticmethod - def _session_cache_key(session_id: str) -> str: - return hashlib.sha256(session_id.encode()).hexdigest() - - @staticmethod - def _boolean(args: dict[str, Any], key: str, default: bool) -> bool: - value = args.get(key, default) - if not isinstance(value, bool): - raise ValueError(key) - return value - - @staticmethod - def _input(args: dict[str, Any], key: str, maximum: int) -> str: - value = args[key] - if not isinstance(value, str) or not value.strip() or len(value) > maximum: - raise ValueError(key) - return value - - @classmethod - def _optional_input(cls, args: dict[str, Any], key: str, maximum: int) -> str | None: - if args.get(key) is None: - return None - return cls._input(args, key, maximum) - - @staticmethod - def _source_type(value: Any) -> str: - if value not in {"text", "url"}: - raise ValueError("source_type") - return str(value) - - -def register(ctx: Any) -> None: - ctx.register_memory_provider(SubstrateWikiProvider()) diff --git a/src/substrate_wiki/checkpoint.py b/src/substrate_wiki/checkpoint.py deleted file mode 100644 index 2e870c2..0000000 --- a/src/substrate_wiki/checkpoint.py +++ /dev/null @@ -1,508 +0,0 @@ -"""Content-free durable state for resumable Hermes history imports.""" - -from __future__ import annotations - -import hashlib -import json -import os -import re -import sqlite3 -import time -import uuid -from collections.abc import Iterator -from dataclasses import dataclass -from pathlib import Path -from typing import Any - -from .spool import secure_atomic_json_write - -PROTOCOL = "stream-v2" -CHECKPOINT_VERSION = 2 -TERMINAL_STATES = {"complete", "complete_with_failures", "cancelled"} -_REMOTE_COUNTER_FIELDS = ( - "failed", - "failed_windows", - "pending_resolution", - "pending_review", - "processed", - "processed_windows", - "projected_entities", - "projection_pending", - "published_claims", - "stub_count", -) -_MAX_COUNTER = (1 << 63) - 1 -_ERROR_CLASS_PATTERN = re.compile(r"[a-z][a-z0-9_]{0,127}") - - -def _now() -> float: - return time.time() - - -def _private_directory(path: Path) -> Path: - if path.exists() and path.is_symlink(): - raise OSError("import state directory must not be a symlink") - path.mkdir(parents=True, exist_ok=True, mode=0o700) - if os.name == "posix": - os.chmod(path, 0o700) - return path - - -@dataclass(frozen=True, slots=True) -class SessionDescriptor: - external_id: str - source: str - subject_id: str - user_hash: str - ordering: int - message_high_water: int - locator: str = "" - - -class ImportCheckpoint: - """Small SQLite job database containing identifiers and counters only.""" - - def __init__(self, path: Path) -> None: - if path.exists() and path.is_symlink(): - raise OSError("checkpoint must not be a symlink") - _private_directory(path.parent) - self.path = path - self.connection = sqlite3.connect(path, timeout=30.0) - self.connection.row_factory = sqlite3.Row - self.connection.execute("PRAGMA journal_mode=WAL") - self.connection.execute("PRAGMA synchronous=FULL") - self.connection.execute("PRAGMA foreign_keys=ON") - self._initialize() - if os.name == "posix": - os.chmod(path, 0o600) - - def close(self) -> None: - self.connection.close() - - def __enter__(self) -> ImportCheckpoint: - return self - - def __exit__(self, *_args: object) -> None: - self.close() - - def _initialize(self) -> None: - self.connection.executescript( - """ - CREATE TABLE IF NOT EXISTS job ( - singleton INTEGER PRIMARY KEY CHECK (singleton = 1), - checkpoint_version INTEGER NOT NULL, - job_id TEXT NOT NULL, - batch_id TEXT NOT NULL, - protocol TEXT NOT NULL, - source_kind TEXT NOT NULL, - source_locator TEXT NOT NULL, - source_locator_hash TEXT NOT NULL, - profile_hash TEXT NOT NULL, - agent_id TEXT NOT NULL, - state TEXT NOT NULL, - created_at REAL NOT NULL, - updated_at REAL NOT NULL, - last_progress_at REAL NOT NULL, - discovered INTEGER NOT NULL DEFAULT 0, - eligible INTEGER NOT NULL DEFAULT 0, - skipped INTEGER NOT NULL DEFAULT 0, - quarantined INTEGER NOT NULL DEFAULT 0, - delivered INTEGER NOT NULL DEFAULT 0, - deduplicated INTEGER NOT NULL DEFAULT 0, - checkpointed INTEGER NOT NULL DEFAULT 0, - processed_windows INTEGER NOT NULL DEFAULT 0, - failed_windows INTEGER NOT NULL DEFAULT 0, - processed INTEGER NOT NULL DEFAULT 0, - pending_review INTEGER NOT NULL DEFAULT 0, - failed INTEGER NOT NULL DEFAULT 0, - projected_entities INTEGER NOT NULL DEFAULT 0, - published_claims INTEGER NOT NULL DEFAULT 0, - stub_count INTEGER NOT NULL DEFAULT 0, - pending_resolution INTEGER NOT NULL DEFAULT 0, - projection_pending INTEGER NOT NULL DEFAULT 0, - peak_rss_bytes INTEGER NOT NULL DEFAULT 0, - error_class TEXT NOT NULL DEFAULT '' - ); - CREATE TABLE IF NOT EXISTS sessions ( - external_id TEXT PRIMARY KEY, - source TEXT NOT NULL, - subject_id TEXT NOT NULL, - user_hash TEXT NOT NULL, - source_order INTEGER NOT NULL, - message_high_water INTEGER NOT NULL, - locator TEXT NOT NULL DEFAULT '', - state TEXT NOT NULL DEFAULT 'pending', - next_message INTEGER NOT NULL DEFAULT 0, - session_digest TEXT NOT NULL DEFAULT '', - updated_at REAL NOT NULL - ); - CREATE INDEX IF NOT EXISTS sessions_order_idx - ON sessions(state, source_order, external_id); - CREATE TABLE IF NOT EXISTS acknowledgements ( - event_id TEXT PRIMARY KEY, - external_id TEXT NOT NULL, - message_boundary_start INTEGER NOT NULL, - message_boundary_end INTEGER NOT NULL, - phase TEXT NOT NULL, - duplicate INTEGER NOT NULL, - retry_count INTEGER NOT NULL, - acknowledged_at REAL NOT NULL - ); - CREATE TABLE IF NOT EXISTS legacy_batches ( - batch_id TEXT PRIMARY KEY, - selected INTEGER NOT NULL DEFAULT 0, - processed INTEGER NOT NULL DEFAULT 0, - delivered INTEGER NOT NULL DEFAULT 0, - checkpoint_time REAL NOT NULL DEFAULT 0 - ); - """ - ) - # v1.3 attaches to the existing v1.2 content-free checkpoint. Add only - # aggregate server counters in place; never replace or copy the database. - columns = { - str(row[1]) for row in self.connection.execute("PRAGMA table_info(job)") - } - for name in ( - "failed_windows", - "projected_entities", - "published_claims", - "stub_count", - "pending_resolution", - "projection_pending", - ): - if name not in columns: - self.connection.execute( - f"ALTER TABLE job ADD COLUMN {name} INTEGER NOT NULL DEFAULT 0" - ) - self.connection.commit() - - @classmethod - def create_or_attach( - cls, - hermes_home: Path, - *, - source_kind: str, - source_locator: str, - agent_id: str, - batch_id: str | None = None, - legacy_batches: list[dict[str, Any]] | None = None, - ) -> ImportCheckpoint: - imports = _private_directory(hermes_home / "substrate_wiki" / "imports") - jobs = _private_directory(imports / "jobs") - key = hashlib.sha256( - f"{hermes_home.resolve()}\0{source_kind}\0{source_locator}".encode() - ).hexdigest()[:24] - pointer = imports / f"active-{key}.json" - try: - value = json.loads(pointer.read_text(encoding="utf-8")) - active_id = str(value.get("job_id") or "") if isinstance(value, dict) else "" - except (FileNotFoundError, OSError, UnicodeError, json.JSONDecodeError): - active_id = "" - if active_id: - candidate = jobs / active_id / "checkpoint.db" - if candidate.is_file() and not candidate.is_symlink(): - checkpoint = cls(candidate) - if checkpoint.job().get("state") not in TERMINAL_STATES: - return checkpoint - checkpoint.close() - - job_id = uuid.uuid4().hex - selected_batch = batch_id or uuid.uuid4().hex - checkpoint = cls(_private_directory(jobs / job_id) / "checkpoint.db") - moment = _now() - checkpoint.connection.execute( - """INSERT INTO job ( - singleton, checkpoint_version, job_id, batch_id, protocol, - source_kind, source_locator, source_locator_hash, profile_hash, agent_id, - state, created_at, updated_at, last_progress_at - ) VALUES (1, ?, ?, ?, ?, ?, ?, ?, ?, ?, 'created', ?, ?, ?)""", - ( - CHECKPOINT_VERSION, - job_id, - selected_batch, - PROTOCOL, - source_kind, - source_locator, - hashlib.sha256(source_locator.encode()).hexdigest(), - hashlib.sha256(os.fspath(hermes_home.resolve()).encode()).hexdigest(), - agent_id or "default", - moment, - moment, - moment, - ), - ) - for item in legacy_batches or []: - legacy_id = str(item.get("batch_id") or "")[:128] - if not legacy_id: - continue - checkpoint.connection.execute( - """INSERT OR REPLACE INTO legacy_batches - (batch_id, selected, processed, delivered, checkpoint_time) - VALUES (?, ?, ?, ?, ?)""", - ( - legacy_id, - int(legacy_id == selected_batch), - max(0, int(item.get("processed", 0))), - max(0, int(item.get("delivered", 0))), - float(item.get("checkpoint_time", 0.0)), - ), - ) - checkpoint.connection.commit() - secure_atomic_json_write(pointer, {"job_id": job_id}) - return checkpoint - - def job(self) -> dict[str, Any]: - row = self.connection.execute("SELECT * FROM job WHERE singleton = 1").fetchone() - if row is None: - raise RuntimeError("checkpoint has no job") - return dict(row) - - def set_state(self, state: str, *, error_class: str = "") -> None: - moment = _now() - self.connection.execute( - "UPDATE job SET state=?, error_class=?, updated_at=?, last_progress_at=? WHERE singleton=1", - (state, error_class[:128], moment, moment), - ) - self.connection.commit() - - def set_inventory(self, *, discovered: int, eligible: int, skipped: int, quarantined: int) -> None: - moment = _now() - self.connection.execute( - """UPDATE job SET discovered=?, eligible=?, skipped=?, quarantined=?, - state='ready', updated_at=?, last_progress_at=? WHERE singleton=1""", - (discovered, eligible, skipped, quarantined, moment, moment), - ) - self.connection.commit() - - def add_session(self, session: SessionDescriptor) -> None: - self.connection.execute( - """INSERT OR IGNORE INTO sessions ( - external_id, source, subject_id, user_hash, source_order, - message_high_water, locator, state, next_message, updated_at - ) VALUES (?, ?, ?, ?, ?, ?, ?, 'pending', 0, ?)""", - ( - session.external_id, - session.source, - session.subject_id, - session.user_hash, - session.ordering, - session.message_high_water, - session.locator, - _now(), - ), - ) - - def finish_discovery(self) -> None: - self.connection.commit() - - def sessions(self, *, include_complete: bool = False) -> Iterator[SessionDescriptor]: - condition = "1=1" if include_complete else "state != 'complete'" - cursor = self.connection.execute( - f"SELECT * FROM sessions WHERE {condition} ORDER BY source_order, external_id" # noqa: S608 - ) - for row in cursor: - yield SessionDescriptor( - external_id=str(row["external_id"]), - source=str(row["source"]), - subject_id=str(row["subject_id"]), - user_hash=str(row["user_hash"]), - ordering=int(row["source_order"]), - message_high_water=int(row["message_high_water"]), - locator=str(row["locator"]), - ) - - def next_message(self, external_id: str) -> int: - row = self.connection.execute( - "SELECT next_message FROM sessions WHERE external_id=?", (external_id,) - ).fetchone() - return int(row[0]) if row is not None else 0 - - def session_progress(self, external_id: str) -> tuple[int, str]: - row = self.connection.execute( - "SELECT next_message, session_digest FROM sessions WHERE external_id=?", - (external_id,), - ).fetchone() - return (int(row[0]), str(row[1])) if row is not None else (0, "") - - def acknowledged(self, event_id: str) -> bool: - row = self.connection.execute( - "SELECT 1 FROM acknowledgements WHERE event_id=?", (event_id,) - ).fetchone() - return row is not None - - def acknowledge( - self, - *, - event_id: str, - external_id: str, - boundary_start: int, - boundary_end: int, - phase: str, - duplicate: bool, - retry_count: int = 0, - session_digest: str = "", - ) -> None: - moment = _now() - with self.connection: - inserted = self.connection.execute( - """INSERT OR IGNORE INTO acknowledgements ( - event_id, external_id, message_boundary_start, message_boundary_end, - phase, duplicate, retry_count, acknowledged_at - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?)""", - ( - event_id, - external_id, - boundary_start, - boundary_end, - phase, - int(duplicate), - retry_count, - moment, - ), - ).rowcount - if inserted: - self.connection.execute( - """UPDATE job SET delivered=delivered+1, - deduplicated=deduplicated+?, checkpointed=checkpointed+1, - state='running', updated_at=?, last_progress_at=? WHERE singleton=1""", - (int(duplicate), moment, moment), - ) - if phase == "message": - self.connection.execute( - """UPDATE sessions SET next_message=MAX(next_message, ?), - session_digest=CASE WHEN ? != '' THEN ? ELSE session_digest END, - state='running', updated_at=? WHERE external_id=?""", - (boundary_end, session_digest, session_digest, moment, external_id), - ) - elif phase == "session_end": - self.connection.execute( - "UPDATE sessions SET state='complete', updated_at=? WHERE external_id=?", - (moment, external_id), - ) - - def mark_completed_sessions(self, external_ids: set[str]) -> None: - with self.connection: - for external_id in external_ids: - self.connection.execute( - "UPDATE sessions SET state='complete', updated_at=? WHERE external_id=?", - (_now(), external_id), - ) - - def legacy_batch_ids(self) -> list[str]: - return [ - str(row[0]) - for row in self.connection.execute( - "SELECT batch_id FROM legacy_batches ORDER BY selected DESC, batch_id" - ) - ] - - def update_remote(self, status: dict[str, Any]) -> None: - """Persist only supplied, content-free aggregate server status fields. - - Status responses can be temporarily partial during a rolling server upgrade. - Missing fields therefore retain their last durable value instead of being - reset to zero. Error classes are restricted to the public sanitized-code - grammar before they can reach the local checkpoint or CLI output. - """ - assignments: list[str] = [] - values: list[int | str] = [] - for field in _REMOTE_COUNTER_FIELDS: - if field not in status or isinstance(status[field], bool): - continue - try: - value = min(_MAX_COUNTER, max(0, int(status[field]))) - except (OverflowError, TypeError, ValueError): - continue - assignments.append(f"{field}=?") - values.append(value) - if "error_class" in status: - raw_error = status.get("error_class") - error_class = ( - raw_error - if isinstance(raw_error, str) - and (not raw_error or _ERROR_CLASS_PATTERN.fullmatch(raw_error)) - else "server_error" - ) - assignments.append("error_class=?") - values.append(error_class) - if status.get("complete") is True: - current = self.job() - failed = status.get("failed", current["failed"]) - failed_windows = status.get("failed_windows", current["failed_windows"]) - try: - has_failures = int(failed) > 0 or int(failed_windows) > 0 - except (TypeError, ValueError): - has_failures = True - assignments.append("state=?") - values.append("complete_with_failures" if has_failures else "complete") - if assignments: - values.extend([int(_now()), int(_now())]) - self.connection.execute( - f"UPDATE job SET {', '.join(assignments)}, updated_at=?, last_progress_at=? WHERE singleton=1", # noqa: S608 - values, - ) - self.connection.commit() - - def update_peak_rss(self, value: int) -> None: - self.connection.execute( - "UPDATE job SET peak_rss_bytes=MAX(peak_rss_bytes, ?), updated_at=? WHERE singleton=1", - (max(0, int(value)), _now()), - ) - self.connection.commit() - - def status(self) -> dict[str, Any]: - row = self.job() - return { - "job_id": row["job_id"], - "batch_id": row["batch_id"], - "state": row["state"], - "legacy_batch_ids": self.legacy_batch_ids(), - "discovered": int(row["discovered"]), - "eligible": int(row["eligible"]), - "skipped": int(row["skipped"]), - "quarantined": int(row["quarantined"]), - "delivered": int(row["delivered"]), - "deduplicated": int(row["deduplicated"]), - "checkpointed": int(row["checkpointed"]), - "processed_windows": int(row["processed_windows"]), - "failed_windows": int(row["failed_windows"]), - "processed": int(row["processed"]), - "pending_review": int(row["pending_review"]), - "failed": int(row["failed"]), - "projected_entities": int(row["projected_entities"]), - "published_claims": int(row["published_claims"]), - "stub_count": int(row["stub_count"]), - "pending_resolution": int(row["pending_resolution"]), - "projection_pending": int(row["projection_pending"]), - "peak_rss_bytes": int(row["peak_rss_bytes"]), - "last_progress_at": float(row["last_progress_at"]), - "complete": str(row["state"]) in {"complete", "complete_with_failures"}, - "error_class": str(row["error_class"]), - } - - -def discover_legacy_checkpoints(imports_root: Path) -> list[dict[str, Any]]: - """Read v1.1 JSON checkpoints without changing them or exposing content.""" - found: list[dict[str, Any]] = [] - if not imports_root.is_dir() or imports_root.is_symlink(): - return found - for path in imports_root.glob("*.json"): - if path.name.startswith("active-") or path.is_symlink(): - continue - try: - value = json.loads(path.read_text(encoding="utf-8")) - batch_id = str(value.get("batch_id") or path.stem)[:128] - completed = value.get("completed_sessions") - completed_ids = { - str(item) for item in completed if isinstance(item, (str, int)) - } if isinstance(completed, list) else set() - found.append( - { - "batch_id": batch_id, - "completed_sessions": completed_ids, - "checkpoint_time": path.stat().st_mtime, - } - ) - except (OSError, UnicodeError, ValueError, TypeError): - continue - return found diff --git a/src/substrate_wiki/cli.py b/src/substrate_wiki/cli.py deleted file mode 100644 index 41037f0..0000000 --- a/src/substrate_wiki/cli.py +++ /dev/null @@ -1,279 +0,0 @@ -"""Managed OOM-safe history import commands for Substrate Wiki.""" - -from __future__ import annotations - -import argparse -import json -import os -import sys -import time -from pathlib import Path -from typing import Any - -from .checkpoint import ImportCheckpoint -from .client import SubstrateClient -from .onboarding import OnboardingError -from .history import ( - HermesHistoryImporter, - load_hermes_inventory, - remove_official_export_spill, - select_history_source, -) -from .supervisor import service_restart_count, start_service, stop_service -from .worker import checkpoint_path - - -def _hermes_home() -> Path: - try: - from hermes_constants import get_hermes_home - - return Path(get_hermes_home()) - except (ImportError, TypeError, ValueError): - return Path(os.environ.get("HERMES_HOME") or Path.home() / ".hermes") - - -def _fallback_url(home: Path) -> str: - path = home / "substrate_wiki" / "config.json" - try: - value = json.loads(path.read_text(encoding="utf-8")) - except (FileNotFoundError, OSError, UnicodeError, json.JSONDecodeError): - return "" - candidate = value.get("api_url") if isinstance(value, dict) else None - return str(candidate).rstrip("/") if isinstance(candidate, str) else "" - - -def _active_agent_id() -> str: - configured = os.environ.get("HERMES_PROFILE") or os.environ.get("HERMES_AGENT_ID") - if configured: - return configured - try: - from hermes_cli.profiles import get_active_profile_name - - return str(get_active_profile_name() or "default") - except (ImportError, OSError, TypeError, ValueError): - return "default" - - -def _job_paths(home: Path) -> list[Path]: - jobs = home / "substrate_wiki" / "imports" / "jobs" - if not jobs.is_dir() or jobs.is_symlink(): - return [] - return sorted(jobs.glob("*/checkpoint.db"), key=lambda item: item.stat().st_mtime, reverse=True) - - -def _open_job(home: Path, job_id: str | None) -> ImportCheckpoint: - if job_id: - return ImportCheckpoint(checkpoint_path(home, job_id)) - for path in _job_paths(home): - checkpoint = ImportCheckpoint(path) - if checkpoint.job()["state"] not in {"complete", "complete_with_failures", "cancelled"}: - return checkpoint - checkpoint.close() - paths = _job_paths(home) - if not paths: - raise FileNotFoundError("no import job") - return ImportCheckpoint(paths[0]) - - -def _wait_for_job(home: Path, job_id: str) -> dict[str, Any]: - while True: - with ImportCheckpoint(checkpoint_path(home, job_id)) as checkpoint: - status = checkpoint.status() - if status["complete"] or status["state"] in {"cancelled", "failed"}: - status["import_service_restart_count"] = service_restart_count(home, job_id) - return status - time.sleep(2.0) - - -def _import_history(args: argparse.Namespace, home: Path) -> dict[str, Any]: - export_path = Path(args.input).resolve() if args.input else None - if not args.yes: - inventory = load_hermes_inventory(home, export_path=export_path) - return { - "discovered": inventory.discovered, - "eligible": inventory.eligible, - "skipped": inventory.skipped, - "quarantined": inventory.quarantined, - "dry_run": True, - } - client = SubstrateClient.from_env(fallback_url=_fallback_url(home), timeout=30.0, hermes_home=home, hosted_default=True) - importer = HermesHistoryImporter( - hermes_home=home, - client=client, - source=select_history_source(home, export_path=export_path), - agent_id=_active_agent_id(), - ) - checkpoint = importer.prepare() - status = checkpoint.status() - job_id = str(status["job_id"]) - checkpoint.close() - status["import_service"] = start_service(home, job_id) - return _wait_for_job(home, job_id) if args.wait else status - - -def _status(args: argparse.Namespace, home: Path) -> dict[str, Any]: - with _open_job(home, args.job_id) as checkpoint: - status = checkpoint.status() - status["import_service_restart_count"] = service_restart_count( - home, str(status["job_id"]) - ) - return status - - -def _resume(args: argparse.Namespace, home: Path) -> dict[str, Any]: - if not args.yes: - raise ValueError("--yes is required") - with _open_job(home, args.job_id) as checkpoint: - status = checkpoint.status() - if status["state"] == "cancelled": - checkpoint.set_state("ready") - job_id = str(status["job_id"]) - start_service(home, job_id) - return _wait_for_job(home, job_id) if args.wait else _status(args, home) - - -def _cancel(args: argparse.Namespace, home: Path) -> dict[str, Any]: - if not args.yes: - raise ValueError("--yes is required") - with _open_job(home, args.job_id) as checkpoint: - job = checkpoint.job() - checkpoint.set_state("cancelled") - result = checkpoint.status() - stop_service(home, str(result["job_id"])) - remove_official_export_spill( - home, - source_kind=str(job["source_kind"]), - source_locator=str(job["source_locator"]), - ) - return result - - -def _onboarding(args: argparse.Namespace, home: Path) -> dict[str, Any]: - from .onboarding import OnboardingManager - - manager = OnboardingManager(home) - result = manager.begin(mode=args.mode, open_browser=not args.no_browser) - if result.get("phase") == "authorization_pending": - print( - f"Open {result.get('verification_uri_complete')} to sign in by email " - f"and connect Hermes. One-time code: {result.get('user_code')}", - file=sys.stderr, - flush=True, - ) - deadline = time.monotonic() + max(0.0, args.timeout) - while args.wait and result.get("phase") == "authorization_pending" and time.monotonic() < deadline: - time.sleep(5) - result = manager.advance() - if result.get("phase") == "awaiting_history_consent": - decision = args.history - if decision == "ask" and sys.stdin.isatty(): - print( - "Upload all eligible past Hermes conversations to Substrate? [y/n]: ", - end="", - file=sys.stderr, - flush=True, - ) - answer = sys.stdin.readline().strip().casefold() - if answer in {"y", "yes"}: - decision = "approve" - elif answer in {"n", "no"}: - decision = "decline" - if decision in {"approve", "decline"}: - result = manager.consent_history(decision == "approve") - return result - - -def _onboarding_status(_args: argparse.Namespace, home: Path) -> dict[str, Any]: - from .onboarding import OnboardingManager - - return OnboardingManager(home).status() - - -def _onboarding_repair(args: argparse.Namespace, home: Path) -> dict[str, Any]: - from .onboarding import OnboardingManager - - if not args.yes: - raise ValueError("--yes is required") - return OnboardingManager(home).repair(wait=args.wait) - - -def substrate_command(args: argparse.Namespace) -> None: - home = _hermes_home().resolve() - command = getattr(args, "substrate_command", None) - try: - if command == "onboard": - result = _onboarding(args, home) - elif command == "onboarding-status": - result = _onboarding_status(args, home) - elif command == "onboarding-repair": - result = _onboarding_repair(args, home) - elif command == "import-history": - result = _import_history(args, home) - elif command == "import-status": - result = _status(args, home) - elif command == "import-resume": - result = _resume(args, home) - elif command == "import-cancel": - result = _cancel(args, home) - else: - raise ValueError("unknown command") - except Exception as exc: # noqa: BLE001 - content-free CLI failure contract - result = {"error_class": type(exc).__name__, "complete": False} - if isinstance(exc, OnboardingError): - result["error_category"] = exc.category - print(json.dumps(result, sort_keys=True) if args.json else f"Substrate command failed: {type(exc).__name__}") - raise SystemExit(1) from None - if args.json: - print(json.dumps(result, sort_keys=True)) - else: - for key, value in result.items(): - print(f"{key}: {value}") - - -def register_cli(subparser: argparse.ArgumentParser) -> None: - commands = subparser.add_subparsers(dest="substrate_command") - - onboard = commands.add_parser("onboard", help="Sign in to the fixed hosted service") - onboard.add_argument("--mode", choices=("auto", "browser", "device"), default="auto") - onboard.add_argument("--wait", action="store_true") - onboard.add_argument("--timeout", type=float, default=900.0) - onboard.add_argument("--no-browser", action="store_true") - onboard.add_argument( - "--history", choices=("ask", "approve", "decline"), default="ask", - help="Historical-conversation consent; future capture remains enabled", - ) - onboard.add_argument("--json", action="store_true") - - onboarding_status = commands.add_parser( - "onboarding-status", help="Show redacted connection and automatic-import progress" - ) - onboarding_status.add_argument("--json", action="store_true") - - onboarding_repair = commands.add_parser( - "onboarding-repair", help="Repair authentication or resume the durable import" - ) - onboarding_repair.add_argument("--yes", action="store_true") - onboarding_repair.add_argument("--wait", action="store_true") - onboarding_repair.add_argument("--json", action="store_true") - - history = commands.add_parser("import-history", help="Start or attach to durable history replay") - history.add_argument("--yes", action="store_true") - history.add_argument("--wait", action="store_true") - history.add_argument("--json", action="store_true") - history.add_argument("--input", metavar="PATH") - - status = commands.add_parser("import-status", help="Show content-free import status") - status.add_argument("--job-id") - status.add_argument("--json", action="store_true") - - resume = commands.add_parser("import-resume", help="Resume the same durable import job") - resume.add_argument("--job-id") - resume.add_argument("--yes", action="store_true") - resume.add_argument("--wait", action="store_true") - resume.add_argument("--json", action="store_true") - - cancel = commands.add_parser("import-cancel", help="Stop a worker without deleting state") - cancel.add_argument("--job-id", required=True) - cancel.add_argument("--yes", action="store_true") - cancel.add_argument("--json", action="store_true") - subparser.set_defaults(func=substrate_command) diff --git a/src/substrate_wiki/client.py b/src/substrate_wiki/client.py deleted file mode 100644 index 9db744a..0000000 --- a/src/substrate_wiki/client.py +++ /dev/null @@ -1,668 +0,0 @@ -"""Bounded standard-library HTTP client for the Substrate wiki API.""" - -from __future__ import annotations - -import ipaddress -import json -import math -import os -from dataclasses import dataclass, field -from datetime import UTC, datetime -from email.utils import parsedate_to_datetime -from typing import Any -from urllib.error import HTTPError, URLError -from urllib.parse import unquote, urlencode, urlsplit -from urllib.request import HTTPRedirectHandler, Request, build_opener - -_MAX_RESPONSE_BYTES = 1024 * 1024 -_MAX_REQUEST_BYTES = 512 * 1024 -_MAX_QUERY_CHARS = 4096 - -# Endpoint-specific response shaping keeps a compromised or misbehaving origin from -# smuggling an arbitrary near-1MiB object through as a "search result" or "answer": -# every field is capped in count and length before it reaches the tool-call layer. -_MAX_RESULT_ITEMS = 25 -_MAX_CITATION_ITEMS = 50 -_MAX_SHORT_FIELD_CHARS = 2048 -_MAX_TEXT_FIELD_CHARS = 65536 -_MAX_MEMORY_CARD_CHARS = 8192 -_USER_AGENT = "substrate_wiki-hermes-plugin/2.0.5" -_PLUGIN_VERSION = (2, 0, 0) - - -class SubstrateAPIError(RuntimeError): - """A sanitized API failure with a stable category and optional retry hint.""" - - def __init__(self, category: str, *, retry_after: float | None = None) -> None: - self.category = category - self.retry_after = ( - float(retry_after) - if isinstance(retry_after, (int, float)) - and not isinstance(retry_after, bool) - and math.isfinite(retry_after) - and retry_after >= 0 - else None - ) - super().__init__(category) - - -def _strict_semver(value: Any) -> tuple[int, int, int] | None: - if not isinstance(value, str): - return None - parts = value.split(".") - if ( - len(parts) != 3 - or any(not part.isascii() or not part.isdigit() for part in parts) - or any(len(part) > 1 and part.startswith("0") for part in parts) - ): - return None - version = tuple(int(part) for part in parts) - if any(part > 1_000_000 for part in version): - return None - return version # type: ignore[return-value] - - -def validate_capabilities( - capabilities: dict[str, Any], *, require_replay: bool = True, require_entity: bool = True -) -> None: - """Validate the hosted connection without admitting partial/legacy contracts.""" - if capabilities.get("provider") != "substrate_wiki": - raise SubstrateAPIError("server_upgrade_required") - if require_replay: - replay = capabilities.get("history_replay") - minimum = _strict_semver(replay.get("min_plugin_version")) if isinstance(replay, dict) else None - valid_replay = ( - isinstance(replay, dict) - and 2 in capabilities.get("capture_schema_versions", []) - and replay.get("protocol") == "stream-v2" - and minimum is not None - and _PLUGIN_VERSION >= minimum - and replay.get("content_free_completion") is True - and replay.get("incremental_windows") is True - and int(replay.get("status_version", 0)) == 2 - and int(capabilities.get("max_event_bytes", 0)) == 262_144 - ) - if not valid_replay: - raise SubstrateAPIError("server_upgrade_required") - if require_entity: - entity = capabilities.get("entity_memory") - quality = capabilities.get("entity_quality") - entity_min = _strict_semver(entity.get("min_plugin_version")) if isinstance(entity, dict) else None - quality_min = _strict_semver(quality.get("min_plugin_version")) if isinstance(quality, dict) else None - valid_entity = ( - isinstance(entity, dict) and isinstance(quality, dict) - and entity.get("protocol") == "entity-wiki-v1" - and entity_min is not None and _PLUGIN_VERSION >= entity_min - and entity.get("search_endpoint") == "/api/v1/hermes/memory/search" - and entity.get("canonical_wiki_pages") is True - and entity.get("entity_page_type") == "entity" - and quality.get("protocol") == "entity-quality-v2" - and quality_min is not None and _PLUGIN_VERSION >= quality_min - and quality.get("memory_card") is True - and quality.get("quality_version") == 2 - and quality.get("canonical_redirects") is True - ) - if not valid_entity: - raise SubstrateAPIError("server_upgrade_required") - - -class _NoRedirectHandler(HTTPRedirectHandler): - """Reject redirects so bearer credentials never leave the configured origin.""" - - def redirect_request( - self, - req: Request, - fp: Any, - code: int, - msg: str, - headers: Any, - newurl: str, - ) -> None: - return None - - -def _content_type(headers: Any) -> str: - value = headers.get("Content-Type", "") if headers is not None else "" - return value.split(";", 1)[0].strip().lower() - - -def _retry_after_seconds(headers: Any, *, now: datetime | None = None) -> float | None: - value = headers.get("Retry-After", "") if headers is not None else "" - if not isinstance(value, str) or not value.strip(): - return None - text = value.strip() - try: - seconds = float(text) - except ValueError: - try: - target = parsedate_to_datetime(text) - except (TypeError, ValueError, OverflowError): - return None - if target.tzinfo is None: - target = target.replace(tzinfo=UTC) - moment = now or datetime.now(UTC) - seconds = (target - moment).total_seconds() - return seconds if math.isfinite(seconds) and seconds >= 0 else None - - -@dataclass(slots=True) -class SubstrateClient: - base_url: str - api_key: str - timeout: float = 10.0 - max_response_bytes: int = _MAX_RESPONSE_BYTES - _entity_wiki_capable: bool | None = field(default=None, init=False, repr=False) - - def __post_init__(self) -> None: - self.base_url = self.base_url.rstrip("/") - self.max_response_bytes = max(1024, min(int(self.max_response_bytes), 8 * 1024 * 1024)) - if self.base_url and not self.is_allowed_base_url(self.base_url): - raise SubstrateAPIError("invalid_api_url") - - @staticmethod - def is_allowed_base_url(base_url: str) -> bool: - """Return whether a base URL is safe for bearer-authenticated requests.""" - try: - parsed = urlsplit(base_url) - host = parsed.hostname - _port = parsed.port - except (TypeError, ValueError): - return False - raw_path = parsed.path or "" - decoded_path = unquote(raw_path) - path_parts = decoded_path.split("/") - if ( - not host - or parsed.username is not None - or parsed.password is not None - or parsed.query - or parsed.fragment - or ( - raw_path not in {"", "/"} - and (not raw_path.startswith("/") or raw_path.endswith("/")) - ) - or "//" in raw_path - or any(part in {".", ".."} for part in path_parts) - or "\\" in decoded_path - ): - return False - if parsed.scheme == "https": - return True - if parsed.scheme != "http": - return False - normalized_host = host.rstrip(".").lower() - if normalized_host == "localhost": - return True - try: - return ipaddress.ip_address(normalized_host).is_loopback - except ValueError: - return False - - @classmethod - def from_env( - cls, - *, - timeout: float = 10.0, - fallback_url: str = "", - hermes_home: Any = None, - hosted_default: bool = False, - ) -> SubstrateClient: - """Resolve explicit legacy env configuration, then hosted onboarding custody.""" - hosted_origin = os.environ.get( - "SUBSTRATE_WIKI_ORIGIN", "https://app.trysubstrate.co" - ).rstrip("/") - base_url = os.environ.get("HERMES_API_URL", "") or fallback_url - api_key = os.environ.get("HERMES_API_KEY", "") - if hosted_default: - if base_url and base_url.rstrip("/") != hosted_origin: - raise SubstrateAPIError("unsafe_hosted_origin_override") - base_url = hosted_origin - if not api_key and hermes_home is not None and base_url.rstrip("/") == hosted_origin: - from pathlib import Path - - from .credentials import credential_store - - api_key = credential_store(Path(hermes_home)).get() - return cls(base_url, api_key, timeout) - - def request( - self, - method: str, - path: str, - *, - query: dict[str, Any] | None = None, - body: dict[str, Any] | None = None, - idempotency_key: str | None = None, - ) -> Any: - if not self.base_url or not self.api_key: - raise SubstrateAPIError("not_configured") - if not isinstance(path, str) or not path.startswith("/") or path.startswith("//"): - raise SubstrateAPIError("invalid_request") - url = f"{self.base_url}{path}" - if query: - clean_query: dict[str, Any] = {} - for key, value in query.items(): - if value is None: - continue - text = str(value) - if len(text) > _MAX_QUERY_CHARS: - raise SubstrateAPIError("invalid_request") - clean_query[str(key)] = value - url += "?" + urlencode(clean_query) - headers = { - "Accept": "application/json", - "Authorization": f"Bearer {self.api_key}", - "User-Agent": _USER_AGENT, - } - data = None - if body is not None: - try: - data = json.dumps(body, ensure_ascii=False, separators=(",", ":")).encode("utf-8") - except (TypeError, ValueError): - raise SubstrateAPIError("invalid_request") from None - if len(data) > _MAX_REQUEST_BYTES: - raise SubstrateAPIError("request_too_large") - headers["Content-Type"] = "application/json" - if idempotency_key: - if len(idempotency_key) > 256: - raise SubstrateAPIError("invalid_request") - headers["Idempotency-Key"] = idempotency_key - request = Request(url, data=data, headers=headers, method=method.upper()) - try: - opener = build_opener(_NoRedirectHandler()) - with opener.open(request, timeout=self.timeout) as response: # noqa: S310 - if _content_type(getattr(response, "headers", None)) != "application/json": - raise SubstrateAPIError("invalid_content_type") - declared = response.headers.get("Content-Length") - if declared: - try: - if int(declared) > self.max_response_bytes: - raise SubstrateAPIError("response_too_large") - except ValueError: - raise SubstrateAPIError("invalid_response") from None - raw = response.read(self.max_response_bytes + 1) - if len(raw) > self.max_response_bytes: - raise SubstrateAPIError("response_too_large") - if not raw: - return {} - try: - decoded = raw.decode("utf-8", errors="strict") - value = json.loads(decoded) - except (UnicodeDecodeError, json.JSONDecodeError): - raise SubstrateAPIError("invalid_response") from None - return self._shape_response(path, value) - except SubstrateAPIError: - raise - except HTTPError as exc: - retry_after = _retry_after_seconds(exc.headers) if exc.code == 429 else None - raise SubstrateAPIError(f"http_{exc.code}", retry_after=retry_after) from None - except TimeoutError: - raise SubstrateAPIError("timeout") from None - except (URLError, OSError): - raise SubstrateAPIError("transport_error") from None - - @classmethod - def _shape_response(cls, path: str, value: Any) -> dict[str, Any] | list[Any]: - if path.endswith("/capabilities"): - if not isinstance(value, dict): - raise SubstrateAPIError("invalid_response") - replay = value.get("history_replay") - versions = value.get("capture_schema_versions") - if not isinstance(replay, dict) or not isinstance(versions, list): - raise SubstrateAPIError("invalid_response") - shaped: dict[str, Any] = { - "provider": cls._cap_scalar(value.get("provider"), _MAX_SHORT_FIELD_CHARS), - "server_commit": cls._cap_scalar( - value.get("server_commit"), _MAX_SHORT_FIELD_CHARS - ), - "capture_schema_versions": [ - item - for item in versions[:8] - if isinstance(item, int) and not isinstance(item, bool) - ], - "max_event_bytes": value.get("max_event_bytes"), - "history_replay": cls._select_fields( - replay, - ( - "protocol", - "min_plugin_version", - "content_free_completion", - "incremental_windows", - "status_version", - ), - ), - } - entity_memory = value.get("entity_memory") - if isinstance(entity_memory, dict): - shaped["entity_memory"] = cls._select_fields( - entity_memory, - ( - "protocol", - "min_plugin_version", - "search_endpoint", - "canonical_wiki_pages", - "entity_page_type", - ), - ) - entity_quality = value.get("entity_quality") - if isinstance(entity_quality, dict): - shaped["entity_quality"] = cls._select_fields( - entity_quality, - ( - "protocol", - "min_plugin_version", - "memory_card", - "quality_version", - "canonical_redirects", - ), - ) - return shaped - if path.endswith("/search") or path.endswith("/representation-context"): - items = value.get("results", []) if isinstance(value, dict) else value - if not isinstance(items, list): - raise SubstrateAPIError("invalid_response") - memory_only = path.endswith("/memory/search") or path.endswith( - "/representation-context" - ) - shaped_results = [ - cls._shape_memory_item(item) if memory_only else cls._shape_item(item) - for item in items[:_MAX_RESULT_ITEMS] - if isinstance(item, dict) - ] - return {"results": shaped_results} - if path.endswith("/read"): - if not isinstance(value, dict): - raise SubstrateAPIError("invalid_response") - return cls._select_fields( - value, ("path", "slug", "title", "summary", "body", "content", "updated_at") - ) - if path.endswith("/query"): - if not isinstance(value, dict): - raise SubstrateAPIError("invalid_response") - shaped = cls._select_fields( - value, ("answer", "insufficient_context", "saved", "synthesis_path") - ) - error = value.get("error") - if isinstance(error, dict): - shaped_error = cls._select_fields(error, ("code", "message", "retryable")) - if shaped_error: - shaped["error"] = shaped_error - citations = value.get("citations") - if isinstance(citations, list): - shaped["citations"] = [ - cls._cap_scalar(item, _MAX_SHORT_FIELD_CHARS) - for item in citations[:_MAX_CITATION_ITEMS] - if isinstance(item, (str, int, float, bool)) - ] - return shaped - if path.endswith("/ingest"): - if not isinstance(value, dict): - raise SubstrateAPIError("invalid_response") - return cls._select_fields(value, ("job_id", "status", "duplicate")) - if path.endswith("/job-status"): - if not isinstance(value, dict): - raise SubstrateAPIError("invalid_response") - shaped = cls._select_fields( - value, - ( - "id", - "job_id", - "status", - "kind", - "attempts", - "max_attempts", - "created_at", - "updated_at", - "error", - ), - ) - error_detail = value.get("error_detail") - if isinstance(error_detail, dict): - shaped_detail = cls._select_fields(error_detail, ("code", "retryable")) - if shaped_detail: - shaped["error_detail"] = shaped_detail - return shaped - if path.startswith("/api/v1/hermes/") and isinstance(value, dict): - shaped = cls._select_fields( - value, - ( - "event_id", - "accepted", - "stored", - "duplicate", - "status", - "action", - "batch_id", - "job_id", - "state", - "eligible", - "discovered", - "skipped", - "delivered", - "deduplicated", - "processed", - "processed_windows", - "checkpointed", - "peak_rss_bytes", - "last_progress_at", - "error_class", - "quarantined", - "pending_review", - "failed", - "failed_windows", - "projected_entities", - "published_claims", - "stub_count", - "pending_resolution", - "projection_pending", - "complete", - ), - ) - results = value.get("results") - if isinstance(results, list): - shaped["results"] = [ - cls._shape_item(item) - for item in results[:_MAX_RESULT_ITEMS] - if isinstance(item, dict) - ] - legacy = value.get("legacy_batch_ids") - if isinstance(legacy, list): - shaped["legacy_batch_ids"] = [ - str(item)[:128] for item in legacy[:64] if isinstance(item, (str, int)) - ] - return shaped - raise SubstrateAPIError("invalid_response") - - @classmethod - def _shape_memory_item(cls, item: dict[str, Any]) -> dict[str, Any]: - """Admit only the canonical v2 memory-card contract for automatic recall.""" - shaped = cls._select_fields( - item, - ( - "path", - "title", - "score", - "page_type", - "entity_id", - "entity_type", - "canonical_path", - "memory_card", - "quality_version", - ), - ) - roles = item.get("roles") - if isinstance(roles, list): - shaped["roles"] = [role[:64] for role in roles[:16] if isinstance(role, str)] - return shaped - - @classmethod - def _shape_item(cls, item: dict[str, Any]) -> dict[str, Any]: - shaped = cls._select_fields( - item, - ( - "path", - "slug", - "title", - "summary", - "text", - "snippet", - "content", - "citation", - "source", - "url", - "score", - "page_type", - "entity_id", - "entity_type", - "canonical_path", - "memory_card", - "quality_version", - ), - ) - roles = item.get("roles") - if isinstance(roles, list): - shaped["roles"] = [role[:64] for role in roles[:16] if isinstance(role, str)] - return shaped - - @classmethod - def _select_fields(cls, value: dict[str, Any], fields: tuple[str, ...]) -> dict[str, Any]: - shaped: dict[str, Any] = {} - for key in fields: - child = value.get(key) - if isinstance(child, (str, int, float, bool)) or child is None: - limit = ( - _MAX_MEMORY_CARD_CHARS - if key == "memory_card" - else ( - _MAX_TEXT_FIELD_CHARS - if key in {"body", "content", "answer", "text", "snippet"} - else _MAX_SHORT_FIELD_CHARS - ) - ) - shaped[key] = cls._cap_scalar(child, limit) - return shaped - - @staticmethod - def _cap_scalar(value: Any, maximum: int) -> Any: - return value[:maximum] if isinstance(value, str) else value - - def search(self, query: str, *, limit: int = 8) -> Any: - if not query or len(query) > _MAX_QUERY_CHARS: - raise SubstrateAPIError("invalid_request") - return self.request( - "POST", - "/api/v1/hermes/wiki/search", - body={"q": query, "limit": limit}, - ) - - def representation_context( - self, - query: str, - *, - limit: int = 8, - scope: dict[str, Any] | None = None, - ) -> Any: - """Compatibility method; v1.4 uses bounded canonical memory cards.""" - return self.memory_search(query, limit=limit, scope=scope) - - def memory_search( - self, - query: str, - *, - limit: int = 8, - scope: dict[str, Any] | None = None, - ) -> Any: - if not query or len(query) > _MAX_QUERY_CHARS: - raise SubstrateAPIError("invalid_request") - self.require_entity_wiki_capability() - selected = scope or {} - payload = { - "q": query, - "limit": limit, - "platform": selected.get("platform") or "cli", - } - for target, *candidates in ( - ("user_id", "user_id", "user"), - ("agent_id", "agent_id", "profile"), - ("agent_identity", "agent_identity"), - ("chat_type", "chat_type"), - ): - value = next((selected.get(key) for key in candidates if selected.get(key)), None) - if value is not None: - payload[target] = value - return self.request( - "POST", - "/api/v1/hermes/memory/search", - body=payload, - ) - - def require_entity_wiki_capability(self) -> None: - """Fail closed unless automatic recall resolves canonical entity pages.""" - if self._entity_wiki_capable is True: - return - capabilities = self.capabilities() - entity = capabilities.get("entity_memory") - quality = capabilities.get("entity_quality") - minimum = ( - _strict_semver(entity.get("min_plugin_version")) if isinstance(entity, dict) else None - ) - quality_minimum = ( - _strict_semver(quality.get("min_plugin_version")) if isinstance(quality, dict) else None - ) - valid = ( - isinstance(entity, dict) - and isinstance(quality, dict) - and capabilities.get("provider") == "substrate_wiki" - and entity.get("protocol") == "entity-wiki-v1" - and minimum is not None - and _PLUGIN_VERSION >= minimum - and entity.get("search_endpoint") == "/api/v1/hermes/memory/search" - and entity.get("canonical_wiki_pages") is True - and entity.get("entity_page_type") == "entity" - and quality.get("protocol") == "entity-quality-v2" - and quality_minimum is not None - and _PLUGIN_VERSION >= quality_minimum - and quality.get("memory_card") is True - and quality.get("quality_version") == 2 - and quality.get("canonical_redirects") is True - ) - if not valid: - raise SubstrateAPIError("server_upgrade_required") - self._entity_wiki_capable = True - - def import_status(self, batch_id: str) -> Any: - if not batch_id or len(batch_id) > 128: - raise SubstrateAPIError("invalid_request") - return self.request( - "GET", - "/api/v1/hermes/import-status", - query={"batch_id": batch_id}, - ) - - def capabilities(self) -> dict[str, Any]: - value = self.request("GET", "/api/v1/hermes/capabilities") - if not isinstance(value, dict): - raise SubstrateAPIError("invalid_response") - return value - - def read_page(self, path: str) -> Any: - if not path or len(path) > _MAX_QUERY_CHARS: - raise SubstrateAPIError("invalid_request") - return self.request("POST", "/api/v1/hermes/wiki/read", body={"path": path}) - - def query_wiki(self, question: str, *, save_as_synthesis: bool = False) -> Any: - return self.request( - "POST", - "/api/v1/hermes/wiki/query", - body={"question": question, "save_as_synthesis": save_as_synthesis}, - ) - - def ingest(self, content: str, *, title: str | None = None, source_type: str = "text") -> Any: - return self.request( - "POST", - "/api/v1/hermes/wiki/ingest", - body={"content": content, "title": title, "source_type": source_type}, - ) - - def job_status(self, job_id: str) -> Any: - return self.request("GET", "/api/v1/hermes/wiki/job-status", query={"job_id": job_id}) diff --git a/src/substrate_wiki/credentials.py b/src/substrate_wiki/credentials.py deleted file mode 100644 index 1363ad2..0000000 --- a/src/substrate_wiki/credentials.py +++ /dev/null @@ -1,241 +0,0 @@ -"""Profile-scoped credential custody for hosted Substrate onboarding. - -The public API never returns credential values. Native credential helpers are -preferred; an owner-private file is the deliberately small portability fallback. -""" -from __future__ import annotations - -import hashlib -import os -import shutil -import stat -import subprocess -from pathlib import Path - -_SERVICE = "co.trysubstrate.hermes" - - -def _profile_account(home: Path, slot: str) -> str: - digest = hashlib.sha256(os.fsencode(str(home.resolve()))).hexdigest()[:24] - return f"{digest}:{slot}" - - -class CredentialStore: - """Abstract secret slot storage.""" - backend = "unknown" - - def get(self, slot: str = "access-token") -> str: - raise NotImplementedError - - def put(self, value: str, slot: str = "access-token") -> None: - raise NotImplementedError - - def delete(self, slot: str = "access-token") -> None: - raise NotImplementedError - - -class SecretToolStore(CredentialStore): - backend = "secret-service" - - def __init__(self, home: Path) -> None: - self.account = _profile_account(home, "profile") - - def get(self, slot: str = "access-token") -> str: - result = subprocess.run( - ("secret-tool", "lookup", "service", _SERVICE, "account", self.account, "slot", slot), - stdin=subprocess.DEVNULL, capture_output=True, text=True, timeout=10, check=False, - ) - return result.stdout.rstrip("\n") if result.returncode == 0 else "" - - def put(self, value: str, slot: str = "access-token") -> None: - if not value or len(value) > 16384: - raise ValueError("invalid credential") - subprocess.run( - ("secret-tool", "store", "--label", "Substrate for Hermes", "service", _SERVICE, - "account", self.account, "slot", slot), input=value, text=True, - stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, timeout=10, check=True, - ) - - def delete(self, slot: str = "access-token") -> None: - subprocess.run( - ("secret-tool", "clear", "service", _SERVICE, "account", self.account, "slot", slot), - stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, - timeout=10, check=False, - ) - - -class MacOSKeychainStore(CredentialStore): - backend = "macos-keychain" - - def __init__(self, home: Path) -> None: - self.account = _profile_account(home, "profile") - - def get(self, slot: str = "access-token") -> str: - result = subprocess.run( - ("security", "find-generic-password", "-a", self.account, "-s", f"{_SERVICE}.{slot}", "-w"), - stdin=subprocess.DEVNULL, capture_output=True, text=True, timeout=10, check=False, - ) - return result.stdout.rstrip("\n") if result.returncode == 0 else "" - - def put(self, value: str, slot: str = "access-token") -> None: - if not value or len(value) > 16384: - raise ValueError("invalid credential") - # Apple's security tool has no stdin form. Avoid it when process - # inspection is not private by falling back to the protected file. - raise OSError("non-interactive keychain write unavailable") - - def delete(self, slot: str = "access-token") -> None: - subprocess.run( - ("security", "delete-generic-password", "-a", self.account, "-s", f"{_SERVICE}.{slot}"), - stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, - timeout=10, check=False, - ) - - -class PrivateFileStore(CredentialStore): - backend = "owner-private-file" - - def __init__(self, home: Path) -> None: - root = home / "substrate_wiki" / "credentials" - if root.exists() and root.is_symlink(): - raise OSError("credential directory must not be a symlink") - root.mkdir(parents=True, exist_ok=True, mode=0o700) - if os.name == "posix": - os.chmod(root, 0o700) - elif os.name == "nt": - icacls, username = shutil.which("icacls"), os.environ.get("USERNAME", "") - if not icacls or not username: - raise OSError("private Windows credential ACL unavailable") - subprocess.run( - (icacls, str(root), "/inheritance:r", "/grant:r", f"{username}:F"), - stdin=subprocess.DEVNULL, - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, - timeout=10, - check=True, - ) - self.root = root - - def _path(self, slot: str) -> Path: - if not slot or not slot.replace("-", "").isalnum(): - raise ValueError("invalid credential slot") - return self.root / slot - - def get(self, slot: str = "access-token") -> str: - path = self._path(slot) - if path.is_symlink(): - return "" - try: - info = path.stat(follow_symlinks=False) - if not stat.S_ISREG(info.st_mode): - return "" - if os.name == "posix": - getuid = getattr(os, "getuid", None) - if not callable(getuid) or info.st_uid != getuid() or stat.S_IMODE(info.st_mode) & 0o077: - return "" - value = path.read_text(encoding="utf-8") - except (FileNotFoundError, OSError, UnicodeError): - return "" - return value if 0 < len(value) <= 16384 else "" - - def put(self, value: str, slot: str = "access-token") -> None: - if not value or len(value) > 16384 or "\x00" in value: - raise ValueError("invalid credential") - path = self._path(slot) - if path.is_symlink(): - raise OSError("credential path must not be a symlink") - temporary = path.with_name(f".{path.name}.{os.getpid()}.tmp") - flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL - if hasattr(os, "O_NOFOLLOW"): - flags |= os.O_NOFOLLOW - descriptor = os.open(temporary, flags, 0o600) - try: - with os.fdopen(descriptor, "w", encoding="utf-8") as stream: - stream.write(value) - stream.flush() - os.fsync(stream.fileno()) - if os.name == "posix": - os.chmod(temporary, 0o600) - elif os.name == "nt": - icacls, username = shutil.which("icacls"), os.environ.get("USERNAME", "") - if not icacls or not username: - raise OSError("private Windows credential ACL unavailable") - subprocess.run( - (icacls, str(temporary), "/inheritance:r", "/grant:r", f"{username}:F"), - stdin=subprocess.DEVNULL, - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, - timeout=10, - check=True, - ) - os.replace(temporary, path) - finally: - try: - temporary.unlink() - except FileNotFoundError: - pass - - def delete(self, slot: str = "access-token") -> None: - path = self._path(slot) - if path.is_symlink(): - raise OSError("credential path must not be a symlink") - try: - path.unlink() - except FileNotFoundError: - pass - - -class PreferredCredentialStore(CredentialStore): - """Use a functioning native vault, otherwise the explicit private fallback.""" - def __init__(self, home: Path) -> None: - self.fallback = PrivateFileStore(home) - self.native: CredentialStore | None = None - self._active_backend = self.fallback.backend - if shutil.which("secret-tool"): - self.native = SecretToolStore(home) - elif sys_platform() == "darwin" and shutil.which("security"): - self.native = MacOSKeychainStore(home) - - @property - def backend(self) -> str: - return self._active_backend - - def get(self, slot: str = "access-token") -> str: - if self.native is not None: - try: - value = self.native.get(slot) - if value: - self._active_backend = self.native.backend - return value - except (OSError, subprocess.SubprocessError): - pass - return self.fallback.get(slot) - - def put(self, value: str, slot: str = "access-token") -> None: - if self.native is not None: - try: - self.native.put(value, slot) - self.fallback.delete(slot) - self._active_backend = self.native.backend - return - except (OSError, subprocess.SubprocessError): - pass - self.fallback.put(value, slot) - self._active_backend = self.fallback.backend - - def delete(self, slot: str = "access-token") -> None: - if self.native is not None: - try: - self.native.delete(slot) - except (OSError, subprocess.SubprocessError): - pass - self.fallback.delete(slot) - - -def sys_platform() -> str: - import sys - return sys.platform - - -def credential_store(home: Path) -> CredentialStore: - return PreferredCredentialStore(home.resolve()) diff --git a/src/substrate_wiki/events.py b/src/substrate_wiki/events.py deleted file mode 100644 index 14b157b..0000000 --- a/src/substrate_wiki/events.py +++ /dev/null @@ -1,585 +0,0 @@ -"""Canonical, bounded capture events shared by live streaming and replay.""" - -from __future__ import annotations - -import hashlib -import json -import re -import time -import uuid -from collections.abc import Iterable, Iterator, Sequence -from typing import Any, Protocol, runtime_checkable - -from .redaction import iter_redacted_text_chunks, redact - -SCHEMA_VERSION = 2 -MAX_CAPTURE_BYTES = 256 * 1024 -_EVENT_NAMESPACE = uuid.UUID("837bd8c2-df25-4a42-bdc1-d38f0c00a8bc") -_MESSAGE_FIELDS = ( - "role", - "content", - "timestamp", -) -_TEXT_BLOCK_TYPES = {"text", "input_text", "output_text"} -_BINARY_KEYS = { - "attachment", - "attachments", - "base64", - "binary", - "blob", - "bytes", - "file_content", - "image", - "images", -} -_DATA_URL = re.compile(r"^data:[^;,]+;base64,", re.IGNORECASE) - - -@runtime_checkable -class BoundedTextSource(Protocol): - """Repeatable source for text that must not be materialized as one string.""" - - def iter_text_chunks(self) -> Iterator[str]: ... - - -def canonical_bytes(value: Any) -> bytes: - return json.dumps( - value, - ensure_ascii=False, - separators=(",", ":"), - sort_keys=True, - ).encode("utf-8") - - -def content_digest(value: Any) -> str: - return hashlib.sha256(canonical_bytes(value)).hexdigest() - - -def _visible_json(value: Any, *, depth: int = 0) -> Any: - if depth > 12: - return "[NESTED_CONTENT_OMITTED]" - if value is None or isinstance(value, (bool, int, float)): - return value - if isinstance(value, str): - return "[BINARY_CONTENT_OMITTED]" if _DATA_URL.match(value) else value - if isinstance(value, list): - return [_visible_json(item, depth=depth + 1) for item in value] - if isinstance(value, dict): - return { - str(key): ( - "[BINARY_CONTENT_OMITTED]" - if str(key).casefold() in _BINARY_KEYS - else _visible_json(item, depth=depth + 1) - ) - for key, item in value.items() - } - return "[NON_JSON_CONTENT_OMITTED]" - - -def _visible_content(value: Any) -> Any: - """Keep textual message material while excluding binary/media payloads.""" - if value is None or isinstance(value, (str, bool, int, float)): - return value - if isinstance(value, list): - blocks: list[Any] = [] - for item in value: - if isinstance(item, str): - blocks.append(item) - continue - if not isinstance(item, dict): - continue - block_type = str(item.get("type") or "").lower() - if block_type in _TEXT_BLOCK_TYPES and isinstance(item.get("text"), str): - blocks.append({"type": block_type, "text": item["text"]}) - return blocks - if isinstance(value, dict): - block_type = str(value.get("type") or "").lower() - if block_type in _TEXT_BLOCK_TYPES and isinstance(value.get("text"), str): - return {"type": block_type, "text": value["text"]} - return _visible_json(value) - return "[NON_TEXT_CONTENT_OMITTED]" - - -def normalize_message( - message: dict[str, Any], - *, - index: int, - secrets: Sequence[str] = (), -) -> dict[str, Any] | None: - """Return a redacted, inference-safe message or ``None`` for system data.""" - role = str(message.get("role") or "").strip().lower() - if role not in {"user", "assistant"}: - return None - selected: dict[str, Any] = {"index": int(index), "role": role} - for field in _MESSAGE_FIELDS[1:]: - if field not in message or message[field] is None: - continue - selected[field] = ( - _visible_content(message[field]) - if field == "content" - else _visible_json(message[field]) - ) - # Reasoning fields, token/billing fields and arbitrary provider metadata are - # deliberately never copied into ``selected``. - return redact(selected, secrets) - - -def normalize_messages( - messages: Iterable[dict[str, Any]], - *, - start_index: int = 0, - secrets: Sequence[str] = (), -) -> list[dict[str, Any]]: - normalized: list[dict[str, Any]] = [] - for offset, message in enumerate(messages): - if not isinstance(message, dict): - continue - capture_index = message.get("_capture_index") - index = ( - int(capture_index) - if isinstance(capture_index, int) and not isinstance(capture_index, bool) - else start_index + offset - ) - item = normalize_message(message, index=index, secrets=secrets) - if item is not None: - normalized.append(item) - return normalized - - -def _utf8_boundaries(value: str, maximum: int) -> Iterator[tuple[int, int]]: - """Yield bounded character slices without ever encoding the remaining suffix.""" - start = 0 - while start < len(value): - # UTF-8 uses at least one byte per Python character, so no valid slice - # can contain more than ``maximum`` characters. Keeping ``high`` local - # prevents the binary search from creating multi-megabyte temporary - # strings for a large message. - low, high = start + 1, min(len(value), start + maximum) - while low < high: - middle = (low + high + 1) // 2 - if len(value[start:middle].encode("utf-8")) <= maximum: - low = middle - else: - high = middle - 1 - yield start, low - start = low - - -def _fragment_message(message: dict[str, Any], maximum: int) -> Iterator[dict[str, Any]]: - content = message.get("content") - if not isinstance(content, str): - if len(canonical_bytes(message)) <= maximum // 2: - yield message - return - # Non-text visible content is already bounded by the capture hooks in - # practice. Retain the legacy encoding for unusual structured records. - encoded = canonical_bytes(message).decode("utf-8") - encoded_digest = hashlib.sha256(encoded.encode("utf-8")).hexdigest() - count = sum(1 for _ in _utf8_boundaries(encoded, max(1024, maximum // 4))) - for index, (start, end) in enumerate( - _utf8_boundaries(encoded, max(1024, maximum // 4)) - ): - yield { - "role": message["role"], - "index": message["index"], - "content": encoded[start:end], - "fragment": { - "encoding": "canonical-json", - "index": index, - "count": count, - "sha256": encoded_digest, - }, - } - return - - fragment_bytes = max(1024, maximum // 4) - metadata = {key: value for key, value in message.items() if key != "content"} - metadata_bytes = canonical_bytes(metadata) - message_hasher = hashlib.sha256() - message_hasher.update(b"substrate-message-v2\0") - message_hasher.update(metadata_bytes) - message_hasher.update(b"\0content\0") - content_bytes = 0 - count = 0 - for start, end in _utf8_boundaries(content, fragment_bytes): - encoded_piece = content[start:end].encode("utf-8") - message_hasher.update(encoded_piece) - content_bytes += len(encoded_piece) - count += 1 - - # This size estimate is deliberately conservative. The exact event-size - # assertion still runs before delivery, but small messages avoid fragment - # metadata and retain the original envelope shape. - if content_bytes + len(metadata_bytes) + 256 <= maximum // 2: - yield message - return - - message_digest = message_hasher.hexdigest() - for index, (start, end) in enumerate(_utf8_boundaries(content, fragment_bytes)): - fragment = dict(metadata) - fragment["content"] = content[start:end] - fragment["fragment"] = { - "encoding": "utf8-content", - "index": index, - "count": count, - "sha256": message_digest, - } - yield fragment - - -def _fragment_streamed_message( - message: dict[str, Any], - source: BoundedTextSource, - *, - maximum: int, - secrets: Sequence[str], -) -> Iterator[dict[str, Any]]: - """Digest/count in one bounded pass, then emit fragments in a second pass.""" - fragment_bytes = max(1024, maximum // 4) - metadata_bytes = canonical_bytes(message) - digest = hashlib.sha256() - digest.update(b"substrate-message-v2\0") - digest.update(metadata_bytes) - digest.update(b"\0content\0") - count = 0 - for piece in _iter_streamed_utf8_fragments(source, secrets, fragment_bytes): - digest.update(piece.encode("utf-8")) - count += 1 - - message_digest = digest.hexdigest() - fragment_index = 0 - for piece in _iter_streamed_utf8_fragments(source, secrets, fragment_bytes): - fragment = dict(message) - fragment["content"] = piece - fragment["fragment"] = { - "encoding": "utf8-content", - "index": fragment_index, - "count": count, - "sha256": message_digest, - } - fragment_index += 1 - yield fragment - - -def _utf8_prefix_end(value: str, maximum: int) -> int: - """Return the longest leading character count within a UTF-8 byte budget.""" - - if not value or maximum <= 0: - return 0 - low, high = 1, min(len(value), maximum) - if len(value[0].encode("utf-8")) > maximum: - return 0 - while low < high: - middle = (low + high + 1) // 2 - if len(value[:middle].encode("utf-8")) <= maximum: - low = middle - else: - high = middle - 1 - return low - - -def _iter_streamed_utf8_fragments( - source: BoundedTextSource, - secrets: Sequence[str], - fragment_bytes: int, -) -> Iterator[str]: - """Pack sanitized text independently of source chunk boundaries.""" - - pending = "" - pending_bytes = 0 - for safe_chunk in iter_redacted_text_chunks(source.iter_text_chunks(), secrets): - remaining = safe_chunk - while remaining: - capacity = fragment_bytes - pending_bytes - end = _utf8_prefix_end(remaining, capacity) - if end == 0: - if not pending: - raise ValueError("fragment byte budget cannot encode one character") - yield pending - pending = "" - pending_bytes = 0 - continue - selected = remaining[:end] - pending += selected - selected_bytes = len(selected.encode("utf-8")) - pending_bytes += selected_bytes - remaining = remaining[end:] - if pending_bytes == fragment_bytes: - yield pending - pending = "" - pending_bytes = 0 - if pending: - yield pending - - -class CaptureEventBuilder: - """Build bounded events containing only session identity and raw dialogue.""" - - def __init__( - self, - scope: dict[str, Any], - *, - secrets: Sequence[str] = (), - max_capture_bytes: int = MAX_CAPTURE_BYTES, - ) -> None: - # ``scope`` remains accepted so older callers do not need an adapter, - # but none of it is duplicated into the upload envelope. - del scope - self.secrets = tuple(secrets) - self.max_capture_bytes = max(16 * 1024, min(max_capture_bytes, MAX_CAPTURE_BYTES)) - - def message_events( - self, - kind: str, - session_id: str, - messages: Sequence[dict[str, Any]], - *, - start_index: int = 0, - payload: dict[str, Any] | None = None, - capture_origin: str = "live", - batch_id: str = "", - deterministic: bool = False, - ) -> list[dict[str, Any]]: - """Compatibility list wrapper; history replay uses the iterator directly.""" - return list( - self.iter_message_events( - kind, - session_id, - messages, - start_index=start_index, - payload=payload, - capture_origin=capture_origin, - batch_id=batch_id, - deterministic=deterministic, - ) - ) - - def iter_message_events( - self, - kind: str, - session_id: str, - messages: Iterable[dict[str, Any]], - *, - start_index: int = 0, - payload: dict[str, Any] | None = None, - capture_origin: str = "live", - batch_id: str = "", - deterministic: bool = False, - ) -> Iterator[dict[str, Any]]: - """Yield bounded user/assistant events while retaining only one fragment.""" - - def fragments() -> Iterator[dict[str, Any]]: - for offset, raw in enumerate(messages): - if not isinstance(raw, dict): - continue - capture_index = raw.get("_capture_index") - index = ( - int(capture_index) - if isinstance(capture_index, int) and not isinstance(capture_index, bool) - else start_index + offset - ) - content = raw.get("content") - if isinstance(content, BoundedTextSource): - bounded = dict(raw) - bounded["content"] = "" - normalized = normalize_message(bounded, index=index, secrets=self.secrets) - if normalized is not None: - normalized.pop("content", None) - yield from _fragment_streamed_message( - normalized, - content, - maximum=self.max_capture_bytes, - secrets=self.secrets, - ) - continue - normalized = normalize_message(raw, index=index, secrets=self.secrets) - if normalized is not None: - yield from _fragment_message(normalized, self.max_capture_bytes) - - def groups() -> Iterator[list[dict[str, Any]]]: - current: list[dict[str, Any]] = [] - for fragment in fragments(): - candidate = [*current, fragment] - probe = self._group_event( - kind, - session_id, - candidate, - chunk_index=999_999_999, - final=False, - payload=payload, - capture_origin=capture_origin, - batch_id=batch_id, - deterministic=deterministic, - event_id="00000000-0000-0000-0000-000000000000", - validate=False, - ) - if current and len(canonical_bytes(probe)) > self.max_capture_bytes: - yield current - current = [fragment] - else: - current = candidate - if current: - yield current - - iterator = groups() - previous = next(iterator, None) - if previous is None: - yield self._event( - kind, - session_id, - { - **(payload or {}), - "messages": [], - }, - boundary={"start": start_index, "end": start_index}, - capture_origin=capture_origin, - batch_id=batch_id, - deterministic=deterministic, - ) - return - - chunk_index = 0 - for current in iterator: - yield self._group_event( - kind, - session_id, - previous, - chunk_index=chunk_index, - final=False, - payload=payload, - capture_origin=capture_origin, - batch_id=batch_id, - deterministic=deterministic, - ) - previous = current - chunk_index += 1 - yield self._group_event( - kind, - session_id, - previous, - chunk_index=chunk_index, - final=True, - payload=payload, - capture_origin=capture_origin, - batch_id=batch_id, - deterministic=deterministic, - ) - - def _group_event( - self, - kind: str, - session_id: str, - messages: list[dict[str, Any]], - *, - chunk_index: int, - final: bool, - payload: dict[str, Any] | None, - capture_origin: str, - batch_id: str, - deterministic: bool, - event_id: str | None = None, - validate: bool = True, - ) -> dict[str, Any]: - event = self._event( - kind, - session_id, - { - **(payload or {}), - "messages": messages, - }, - boundary={ - "start": min(int(message["index"]) for message in messages), - "end": max(int(message["index"]) for message in messages) + 1, - }, - capture_origin=capture_origin, - batch_id=batch_id, - deterministic=deterministic, - event_id=event_id, - ) - if validate and len(canonical_bytes(event)) > self.max_capture_bytes: - raise ValueError("capture event exceeds maximum size") - return event - - def payload_event( - self, - kind: str, - session_id: str, - payload: dict[str, Any], - *, - boundary: dict[str, int] | None = None, - capture_origin: str = "live", - batch_id: str = "", - deterministic: bool = False, - ) -> dict[str, Any]: - safe_payload = redact(payload, self.secrets) - event = self._event( - kind, - session_id, - safe_payload, - boundary=boundary or {"start": 0, "end": 0}, - capture_origin=capture_origin, - batch_id=batch_id, - deterministic=deterministic, - ) - if len(canonical_bytes(event)) <= self.max_capture_bytes: - return event - marker = { - "capture_truncated": True, - "original_bytes": len(canonical_bytes(safe_payload)), - "sha256": content_digest(safe_payload), - } - return self._event( - kind, - session_id, - marker, - boundary=boundary or {"start": 0, "end": 0}, - capture_origin=capture_origin, - batch_id=batch_id, - deterministic=deterministic, - ) - - @staticmethod - def _boundary(messages: Sequence[dict[str, Any]]) -> dict[str, int]: - indexes = [int(message["index"]) for message in messages] - return {"start": min(indexes), "end": max(indexes) + 1} - - def _event( - self, - kind: str, - session_id: str, - payload: dict[str, Any], - *, - boundary: dict[str, int], - capture_origin: str, - batch_id: str, - deterministic: bool, - event_id: str | None = None, - ) -> dict[str, Any]: - safe_session = str(session_id)[:512] - safe_payload = dict(payload) - identity = { - "kind": kind, - "session_id": safe_session, - "boundary": boundary, - "payload": safe_payload, - } - resolved_id = event_id - if resolved_id is None: - resolved_id = ( - str(uuid.uuid5(_EVENT_NAMESPACE, content_digest(identity))) - if deterministic - else str(uuid.uuid4()) - ) - event: dict[str, Any] = { - "schema_version": SCHEMA_VERSION, - "event_id": resolved_id, - "kind": kind, - "session_id": safe_session, - "created_at": 0 if deterministic else time.time(), - } - messages = safe_payload.pop("messages", None) - if isinstance(messages, list): - event["messages"] = messages - if safe_payload: - event["data"] = safe_payload - return event diff --git a/src/substrate_wiki/history.py b/src/substrate_wiki/history.py deleted file mode 100644 index 40a1e61..0000000 --- a/src/substrate_wiki/history.py +++ /dev/null @@ -1,1532 +0,0 @@ -"""OOM-safe Hermes history discovery and durable stream-v2 replay.""" - -from __future__ import annotations - -import codecs -import hashlib -import io -import json -import os -import sqlite3 -import stat -import sys -import tempfile -import time -from collections.abc import Callable, Iterator -from dataclasses import dataclass, field -from pathlib import Path -from typing import Any, BinaryIO, Protocol - -from .checkpoint import ( - TERMINAL_STATES, - ImportCheckpoint, - SessionDescriptor, - discover_legacy_checkpoints, -) -from .client import SubstrateAPIError, SubstrateClient, validate_capabilities -from .events import CaptureEventBuilder, content_digest -from .redaction import configured_secret_values - -_HUMAN_SOURCES = { - "acp", "api-server", "bluebubbles", "cli", "dingtalk", "discord", "email", - "feishu", "homeassistant", "matrix", "mattermost", "qqbot", "signal", "slack", - "sms", "teams", "telegram", "wecom", "weixin", "whatsapp", -} -_EXCLUDED_SOURCES = {"batch", "cron", "flush", "test", "webhook", "worker"} -_GROUP_CHAT_TYPES = {"channel", "group", "groupchat", "room", "supergroup"} -_TRANSIENT_CATEGORIES = {"http_429", "timeout", "transport_error"} -_JSON_CHUNK_BYTES = 64 * 1024 -_JSON_MEMORY_RECORD_BYTES = 1024 * 1024 -_JSON_SCALAR_BYTES = 16 * 1024 -_JSON_TEXT_OUTPUT_CHARS = 64 * 1024 -_SQLITE_TEXT_CHARS = 1024 * 1024 -_OFFICIAL_EXPORT_MAX_ORPHAN_AGE_SECONDS = 24 * 60 * 60 - - -class _ImportStopRequested(Exception): - """Internal control flow for a graceful stop between durable acknowledgements.""" - - -@dataclass(slots=True) -class HistorySession: - """Compatibility fixture and bounded in-memory source session.""" - - external_id: str - source: str - user_id: str - subject_id: str - messages: list[dict[str, Any]] - metadata: dict[str, Any] = field(default_factory=dict) - - -@dataclass(slots=True) -class HistoryInventory: - """Compatibility wrapper; production adapters never build this object.""" - - sessions: list[HistorySession] = field(default_factory=list) - discovered: int = 0 - eligible: int = 0 - skipped: int = 0 - quarantined: int = 0 - - -class ConversationReplaySource(Protocol): - source_kind: str - source_locator: str - - def discover(self, checkpoint: ImportCheckpoint) -> dict[str, int]: ... - - def iter_messages( - self, session: SessionDescriptor, *, start: int - ) -> Iterator[dict[str, Any]]: ... - - -@dataclass(frozen=True, slots=True) -class _ActiveOfficialExport: - locator: Path - state: str - updated_at: float - discovered_sessions: int - - -def _absolute_path(path: Path) -> Path: - """Return an absolute path without following a final symlink.""" - return Path(os.path.abspath(os.fspath(path))) - - -def _official_export_spill_root(hermes_home: Path) -> Path: - return _absolute_path( - hermes_home / "substrate_wiki" / "imports" / "spill" - ) - - -def _official_export_name(name: str) -> bool: - return name == "official-export.jsonl" or ( - name.startswith("official-export-") and name.endswith(".jsonl") - ) - - -def _official_export_root_is_safe(spill_root: Path) -> bool: - root = _absolute_path(spill_root) - # The final four components are spill/imports/substrate_wiki/HERMES_HOME. - # Refuse traversal through any symlink in storage managed by the plugin. - current = root - for _ in range(4): - if current.is_symlink(): - return False - current = current.parent - if not root.exists(): - return True - try: - details = root.lstat() - except FileNotFoundError: - return True - if not stat.S_ISDIR(details.st_mode): - return False - if os.name == "posix": - getuid = getattr(os, "getuid", None) - if ( - not callable(getuid) - or details.st_uid != int(getuid()) - or stat.S_IMODE(details.st_mode) & 0o077 - ): - return False - return True - - -def _validated_official_export_path( - spill_root: Path, locator: str | Path -) -> Path | None: - """Return a safe job-owned export path, refusing arbitrary checkpoint paths.""" - root = _absolute_path(spill_root) - candidate = _absolute_path(Path(locator)) - if not _official_export_root_is_safe(root) or candidate.is_symlink(): - return None - try: - if ( - os.path.normcase(os.fspath(candidate.parent)) - != os.path.normcase(os.fspath(root)) - or not _official_export_name(candidate.name) - ): - return None - except (OSError, ValueError): - return None - return candidate - - -def _private_official_export_file(path: Path) -> bool: - try: - details = path.lstat() - except FileNotFoundError: - return False - if not stat.S_ISREG(details.st_mode): - return False - if os.name == "posix": - getuid = getattr(os, "getuid", None) - if ( - not callable(getuid) - or details.st_uid != int(getuid()) - or stat.S_IMODE(details.st_mode) & 0o077 - ): - return False - return True - - -def remove_official_export_spill( - hermes_home: Path, *, source_kind: str, source_locator: str -) -> bool: - """Safely remove a terminal official export without trusting its checkpoint.""" - if source_kind != "hermes-official-export": - return False - candidate = _validated_official_export_path( - _official_export_spill_root(hermes_home), source_locator - ) - if candidate is None: - return False - try: - candidate.unlink() - except FileNotFoundError: - return False - return True - - -def _parse_json(value: Any) -> Any: - if not isinstance(value, str) or not value.strip(): - return value - try: - decoded, end = json.JSONDecoder().raw_decode(value) - except json.JSONDecodeError: - return value - return decoded if not value[end:].strip() else value - - -def _message_from_row(row: dict[str, Any], index: int) -> dict[str, Any] | None: - role = str(row.get("role") or "").lower() - if role not in {"user", "assistant", "tool"}: - return None - message: dict[str, Any] = { - "role": role, - "content": row.get("content") or "", - "_capture_index": index, - } - for key in ("tool_call_id", "tool_name", "timestamp", "platform_message_id", "name"): - if row.get(key) is not None: - message[key] = row[key] - if row.get("tool_calls") is not None: - message["tool_calls"] = _parse_json(row["tool_calls"]) - return message - - -def _disposition( - *, external_id: str, source: str, user_id: str, chat_type: str -) -> tuple[str, str, str]: - if not external_id: - return "quarantined", "", "" - if source in _EXCLUDED_SOURCES or source not in _HUMAN_SOURCES: - return "skipped", "", "" - if chat_type in _GROUP_CHAT_TYPES: - return "quarantined", "", "" - if source == "cli": - return "included", "owner", "" - if not user_id: - return "quarantined", "", "" - user_hash = hashlib.sha256(f"{source}\0{user_id}".encode()).hexdigest() - return "included", user_hash[:24], user_hash - - -class HermesSQLiteHistorySource: - """Read Hermes state.db through bounded cursors and a read-only transaction.""" - - source_kind = "hermes-sqlite" - - def __init__(self, database: Path) -> None: - self.database = database - self.source_locator = os.fspath(database.resolve()) - - def _connect(self) -> sqlite3.Connection: - if not self.database.is_file() or self.database.is_symlink(): - raise FileNotFoundError(self.database) - connection = sqlite3.connect( - f"{self.database.resolve().as_uri()}?mode=ro", uri=True, timeout=1.0 - ) - connection.row_factory = sqlite3.Row - connection.execute("PRAGMA query_only=ON") - connection.execute("BEGIN") - return connection - - @staticmethod - def _columns(connection: sqlite3.Connection, table: str) -> set[str]: - return {str(row[1]) for row in connection.execute(f"PRAGMA table_info({table})")} - - def discover(self, checkpoint: ImportCheckpoint) -> dict[str, int]: - connection = self._connect() - discovered = skipped = quarantined = eligible = 0 - try: - session_columns = self._columns(connection, "sessions") - message_columns = self._columns(connection, "messages") - if not {"id", "source"} <= session_columns or not { - "session_id", "role", "content" - } <= message_columns: - raise sqlite3.DatabaseError("unsupported Hermes session schema") - selected = [ - column - for column in ("id", "source", "user_id", "chat_type", "started_at") - if column in session_columns - ] - order = "started_at ASC, id ASC" if "started_at" in session_columns else "id ASC" - cursor = connection.execute( - f"SELECT {', '.join(selected)} FROM sessions ORDER BY {order}" # noqa: S608 - ) - for source_order, raw in enumerate(cursor): - discovered += 1 - row = dict(raw) - external_id = str(row.get("id") or "").strip() - source = str(row.get("source") or "").strip().casefold() - user_id = str(row.get("user_id") or "").strip() - chat_type = str(row.get("chat_type") or "").strip().casefold() - disposition, subject_id, user_hash = _disposition( - external_id=external_id, - source=source, - user_id=user_id, - chat_type=chat_type, - ) - if disposition != "included": - quarantined += int(disposition == "quarantined") - skipped += int(disposition == "skipped") - continue - condition = "session_id=?" - if "active" in message_columns: - condition += " AND active=1" - count = int( - connection.execute( - f"SELECT COUNT(*) FROM messages WHERE {condition}", # noqa: S608 - (external_id,), - ).fetchone()[0] - ) - if count <= 0: - skipped += 1 - continue - checkpoint.add_session( - SessionDescriptor( - external_id=external_id, - source=source, - subject_id=subject_id, - user_hash=user_hash, - ordering=source_order, - message_high_water=count, - ) - ) - eligible += 1 - checkpoint.finish_discovery() - connection.rollback() - finally: - connection.close() - result = { - "discovered": discovered, - "eligible": eligible, - "skipped": skipped, - "quarantined": quarantined, - } - checkpoint.set_inventory(**result) - return result - - def iter_messages( - self, session: SessionDescriptor, *, start: int - ) -> Iterator[dict[str, Any]]: - connection = self._connect() - try: - columns = self._columns(connection, "messages") - selected = [ - column - for column in ( - "role", "tool_call_id", "tool_name", "timestamp", - "platform_message_id", "tool_calls", "name", - ) - if column in columns - ] - locator = "id" if "id" in columns else "rowid" - selected.extend( - ( - f"{locator} AS _substrate_locator", - "length(content) AS _substrate_content_chars", - "typeof(content) AS _substrate_content_type", - f"CASE WHEN length(content) <= {_SQLITE_TEXT_CHARS} " - "THEN content ELSE NULL END AS content", - ) - ) - condition = "session_id=?" - if "active" in columns: - condition += " AND active=1" - order = ( - "timestamp ASC, id ASC" - if {"timestamp", "id"} <= columns - else f"{locator} ASC" - ) - cursor = connection.execute( - f"SELECT {', '.join(selected)} FROM messages WHERE {condition} " # noqa: S608 - f"ORDER BY {order} LIMIT ? OFFSET ?", - (session.external_id, session.message_high_water - start, start), - ) - for index, raw in enumerate(cursor, start=start): - row = dict(raw) - if ( - row.get("_substrate_content_type") == "text" - and int(row.get("_substrate_content_chars") or 0) > _SQLITE_TEXT_CHARS - ): - row["content"] = _SQLiteTextSource( - self.database, - locator, - row.get("_substrate_locator"), - ) - elif row.get("_substrate_content_type") != "text": - row["content"] = "[BINARY_CONTENT_OMITTED]" - message = _message_from_row(row, index) - if message is not None: - yield message - connection.rollback() - finally: - connection.close() - - -@dataclass(slots=True, frozen=True) -class _SQLiteTextSource: - """Repeatably stream one SQLite TEXT value through bounded substr queries.""" - - database: Path - locator_column: str - locator: Any - - def iter_text_chunks(self) -> Iterator[str]: - connection = sqlite3.connect( - f"{self.database.resolve().as_uri()}?mode=ro", uri=True, timeout=1.0 - ) - connection.execute("PRAGMA query_only=ON") - connection.execute("BEGIN") - try: - offset = 1 - while True: - row = connection.execute( - f"SELECT substr(content, ?, ?) FROM messages " # noqa: S608 - f"WHERE {self.locator_column}=?", - (offset, _SQLITE_TEXT_CHARS, self.locator), - ).fetchone() - value = row[0] if row else None - if not isinstance(value, str) or not value: - break - yield value - offset += len(value) - if len(value) < _SQLITE_TEXT_CHARS: - break - connection.rollback() - finally: - connection.close() - - -@dataclass(slots=True, frozen=True) -class _JSONSpan: - start: int - end: int - - @property - def size(self) -> int: - return self.end - self.start - - -class _JSONScanner: - """Forward-only bounded scanner for locating values in one JSON record.""" - - def __init__(self, raw: bytes | Path, *, start: int = 0) -> None: - self.stream: BinaryIO = io.BytesIO(raw) if isinstance(raw, bytes) else raw.open("rb", buffering=0) - self.stream.seek(start) - self.position = start - self.buffer = b"" - self.offset = 0 - - def close(self) -> None: - self.stream.close() - - def _fill(self) -> bool: - if self.offset < len(self.buffer): - return True - self.buffer = self.stream.read(_JSON_CHUNK_BYTES) - self.offset = 0 - return bool(self.buffer) - - def peek(self) -> int | None: - return self.buffer[self.offset] if self._fill() else None - - def take(self) -> int: - value = self.peek() - if value is None: - raise ValueError("unexpected end of JSON record") - self.offset += 1 - self.position += 1 - return value - - def skip_space(self) -> None: - while self.peek() in {9, 10, 13, 32}: - self.take() - - def string_span(self) -> _JSONSpan: - self.skip_space() - start = self.position - if self.take() != 34: - raise ValueError("expected JSON string") - escaped = False - while True: - value = self.take() - if escaped: - escaped = False - elif value == 92: - escaped = True - elif value == 34: - return _JSONSpan(start, self.position) - - def value_span(self) -> _JSONSpan: - self.skip_space() - start = self.position - first = self.peek() - if first is None: - raise ValueError("missing JSON value") - if first == 34: - return self.string_span() - if first in {91, 123}: - stack: list[int] = [] - in_string = False - escaped = False - while True: - value = self.take() - if in_string: - if escaped: - escaped = False - elif value == 92: - escaped = True - elif value == 34: - in_string = False - continue - if value == 34: - in_string = True - elif value in {91, 123}: - stack.append(value) - elif value in {93, 125}: - if not stack or (stack[-1], value) not in {(91, 93), (123, 125)}: - raise ValueError("mismatched JSON container") - stack.pop() - if not stack: - return _JSONSpan(start, self.position) - while self.peek() not in {None, 9, 10, 13, 32, 44, 93, 125}: - self.take() - if self.position == start: - raise ValueError("empty JSON value") - return _JSONSpan(start, self.position) - - -def _read_json_span(raw: bytes | Path, span: _JSONSpan, *, maximum: int) -> bytes: - if span.size < 0 or span.size > maximum: - raise ValueError("JSON value exceeds bounded decode limit") - if isinstance(raw, bytes): - return raw[span.start : span.end] - with raw.open("rb", buffering=0) as stream: - stream.seek(span.start) - value = stream.read(span.size) - if len(value) != span.size: - raise ValueError("truncated JSON value") - return value - - -def _decode_json_span(raw: bytes | Path, span: _JSONSpan, *, maximum: int) -> Any: - return json.loads(_read_json_span(raw, span, maximum=maximum).decode("utf-8")) - - -def _object_fields(raw: bytes | Path, span: _JSONSpan | None = None) -> Iterator[tuple[str, _JSONSpan]]: - scanner = _JSONScanner(raw, start=span.start if span else 0) - try: - scanner.skip_space() - if scanner.take() != 123: - raise ValueError("JSON value is not an object") - for _ in range(1024): - scanner.skip_space() - if scanner.peek() == 125: - scanner.take() - return - key_span = scanner.string_span() - key = _decode_json_span(raw, key_span, maximum=_JSON_SCALAR_BYTES) - if not isinstance(key, str): - raise ValueError("JSON object key is not text") - scanner.skip_space() - if scanner.take() != 58: - raise ValueError("missing JSON object colon") - yield key, scanner.value_span() - scanner.skip_space() - separator = scanner.take() - if separator == 125: - return - if separator != 44: - raise ValueError("invalid JSON object separator") - raise ValueError("JSON object has too many top-level fields") - finally: - scanner.close() - - -def _array_elements(raw: bytes | Path, span: _JSONSpan) -> Iterator[_JSONSpan]: - scanner = _JSONScanner(raw, start=span.start) - try: - scanner.skip_space() - if scanner.take() != 91: - raise ValueError("JSON value is not an array") - while True: - scanner.skip_space() - if scanner.peek() == 93: - scanner.take() - return - yield scanner.value_span() - scanner.skip_space() - separator = scanner.take() - if separator == 93: - return - if separator != 44: - raise ValueError("invalid JSON array separator") - finally: - scanner.close() - - -def _record_fields(raw: bytes | Path) -> dict[str, _JSONSpan]: - return {key: span for key, span in _object_fields(raw)} - - -def _bounded_scalar(raw: bytes | Path, span: _JSONSpan | None) -> Any: - if span is None: - return None - return _decode_json_span(raw, span, maximum=_JSON_SCALAR_BYTES) - - -def _first_span_byte(raw: bytes | Path, span: _JSONSpan) -> int | None: - value = _read_json_span(raw, _JSONSpan(span.start, min(span.end, span.start + 1)), maximum=1) - return value[0] if value else None - - -def _iter_json_range(path: Path, start: int, end: int) -> Iterator[bytes]: - with path.open("rb", buffering=0) as stream: - stream.seek(start) - remaining = end - start - while remaining > 0: - chunk = stream.read(min(_JSON_CHUNK_BYTES, remaining)) - if not chunk: - raise ValueError("truncated JSON string") - remaining -= len(chunk) - yield chunk - - -@dataclass(slots=True, frozen=True) -class _JSONLTextSource: - """Repeatably decode one oversized JSON string without reading its record.""" - - record: Path - span: _JSONSpan - - def iter_text_chunks(self) -> Iterator[str]: - if _first_span_byte(self.record, self.span) != 34: - raise ValueError("deferred JSON content is not a string") - decoder = codecs.getincrementaldecoder("utf-8")("strict") - pending = "" - escaped = False - unicode_digits: str | None = None - high_surrogate: int | None = None - - def append(value: str) -> list[str]: - nonlocal pending - if value: - pending += value - emitted: list[str] = [] - while len(pending) >= _JSON_TEXT_OUTPUT_CHARS: - emitted.append(pending[:_JSON_TEXT_OUTPUT_CHARS]) - pending = pending[_JSON_TEXT_OUTPUT_CHARS:] - return emitted - - def codepoint(value: int) -> str: - nonlocal high_surrogate - if 0xD800 <= value <= 0xDBFF: - replacement = "\ufffd" if high_surrogate is not None else "" - high_surrogate = value - return replacement - if 0xDC00 <= value <= 0xDFFF and high_surrogate is not None: - combined = 0x10000 + ((high_surrogate - 0xD800) << 10) + (value - 0xDC00) - high_surrogate = None - return chr(combined) - prefix = "\ufffd" if high_surrogate is not None else "" - high_surrogate = None - return prefix + ("\ufffd" if 0xDC00 <= value <= 0xDFFF else chr(value)) - - escapes = { - 34: '"', 47: "/", 92: "\\", 98: "\b", 102: "\f", - 110: "\n", 114: "\r", 116: "\t", - } - for raw_chunk in _iter_json_range(self.record, self.span.start + 1, self.span.end - 1): - cursor = 0 - while cursor < len(raw_chunk): - if unicode_digits is not None: - needed = 4 - len(unicode_digits) - taken = raw_chunk[cursor : cursor + needed] - unicode_digits += taken.decode("ascii") - cursor += len(taken) - if len(unicode_digits) == 4: - try: - resolved = codepoint(int(unicode_digits, 16)) - except ValueError as exc: - raise ValueError("invalid JSON unicode escape") from exc - unicode_digits = None - for emitted in append(resolved): - yield emitted - continue - if escaped: - marker = raw_chunk[cursor] - cursor += 1 - escaped = False - if marker == 117: - unicode_digits = "" - needed = min(4, len(raw_chunk) - cursor) - unicode_digits = raw_chunk[cursor : cursor + needed].decode("ascii") - cursor += needed - if len(unicode_digits) == 4: - try: - resolved = codepoint(int(unicode_digits, 16)) - except ValueError as exc: - raise ValueError("invalid JSON unicode escape") from exc - unicode_digits = None - for emitted in append(resolved): - yield emitted - elif marker in escapes: - prefix = "\ufffd" if high_surrogate is not None else "" - high_surrogate = None - for emitted in append(prefix + escapes[marker]): - yield emitted - else: - raise ValueError("invalid JSON escape") - continue - slash = raw_chunk.find(b"\\", cursor) - end = len(raw_chunk) if slash < 0 else slash - if end > cursor: - prefix = "\ufffd" if high_surrogate is not None else "" - high_surrogate = None - decoded = decoder.decode(raw_chunk[cursor:end], final=False) - for emitted in append(prefix + decoded): - yield emitted - cursor = end - if slash >= 0: - escaped = True - cursor += 1 - if escaped or unicode_digits is not None: - raise ValueError("truncated JSON escape") - suffix = decoder.decode(b"", final=True) - if high_surrogate is not None: - suffix = "\ufffd" + suffix - for emitted in append(suffix): - yield emitted - if pending: - yield pending - - -def _jsonl_message(raw: bytes | Path, span: _JSONSpan, index: int) -> dict[str, Any] | None: - if span.size <= _JSON_MEMORY_RECORD_BYTES: - value = _decode_json_span(raw, span, maximum=_JSON_MEMORY_RECORD_BYTES) - return _message_from_row(value, index) if isinstance(value, dict) else None - fields = {key: value_span for key, value_span in _object_fields(raw, span)} - role = _bounded_scalar(raw, fields.get("role")) - row: dict[str, Any] = {"role": role} - content_span = fields.get("content") - if content_span is not None: - if content_span.size <= _JSON_MEMORY_RECORD_BYTES: - row["content"] = _decode_json_span( - raw, content_span, maximum=_JSON_MEMORY_RECORD_BYTES - ) - elif isinstance(raw, Path) and _first_span_byte(raw, content_span) == 34: - row["content"] = _JSONLTextSource(raw, content_span) - else: - row["content"] = "[OVERSIZED_STRUCTURED_CONTENT_OMITTED]" - for key in ("tool_call_id", "tool_name", "timestamp", "platform_message_id", "name"): - value_span = fields.get(key) - if value_span is not None and value_span.size <= _JSON_SCALAR_BYTES: - row[key] = _decode_json_span(raw, value_span, maximum=_JSON_SCALAR_BYTES) - tool_calls = fields.get("tool_calls") - if tool_calls is not None: - row["tool_calls"] = ( - _decode_json_span(raw, tool_calls, maximum=_JSON_MEMORY_RECORD_BYTES) - if tool_calls.size <= _JSON_MEMORY_RECORD_BYTES - else "[OVERSIZED_TOOL_DATA_OMITTED]" - ) - return _message_from_row(row, index) - - -def _spill_root() -> Path: - identity = str(os.getuid()) if hasattr(os, "getuid") else str(os.getpid()) - root = Path(tempfile.gettempdir()) / f"substrate-wiki-import-{identity}" - if root.exists() and root.is_symlink(): - raise OSError("JSONL spill root must not be a symlink") - root.mkdir(parents=True, exist_ok=True, mode=0o700) - if os.name == "posix": - os.chmod(root, 0o700) - for stale in root.glob("active-*.record"): - try: - if not stale.is_symlink() and time.time() - stale.stat().st_mtime > 24 * 3600: - stale.unlink() - except OSError: - continue - return root - - -def _bounded_jsonl_records(path: Path) -> Iterator[tuple[int, bytes | Path]]: - """Yield one JSONL record using fixed reads; only the active record is retained.""" - spill_root = _spill_root() - with path.open("rb", buffering=0) as stream: - offset = 0 - record_start = 0 - pending = bytearray() - spill_path: Path | None = None - spill_stream: Any = None - - def append(value: bytes) -> None: - nonlocal pending, spill_path, spill_stream - if spill_stream is None and len(pending) + len(value) <= _JSON_MEMORY_RECORD_BYTES: - pending.extend(value) - return - if spill_stream is None: - spill_path = spill_root / f"active-{os.getpid()}-{time.time_ns()}.record" - flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL - if hasattr(os, "O_NOFOLLOW"): - flags |= os.O_NOFOLLOW - descriptor = os.open(spill_path, flags, 0o600) - spill_stream = os.fdopen(descriptor, "wb") - spill_stream.write(pending) - pending.clear() - spill_stream.write(value) - - def finish() -> bytes | Path | None: - nonlocal pending, spill_path, spill_stream - if spill_stream is not None: - spill_stream.flush() - os.fsync(spill_stream.fileno()) - spill_stream.close() - spill_stream = None - result: bytes | Path | None = spill_path - spill_path = None - return result - if pending.strip(): - result = bytes(pending) - pending.clear() - return result - pending.clear() - return None - - while True: - chunk = stream.read(_JSON_CHUNK_BYTES) - if not chunk: - record = finish() - if record is not None: - try: - yield record_start, record - finally: - if isinstance(record, Path): - record.unlink(missing_ok=True) - break - cursor = 0 - while cursor < len(chunk): - newline = chunk.find(b"\n", cursor) - if newline < 0: - append(chunk[cursor:]) - break - append(chunk[cursor:newline]) - record = finish() - if record is not None: - try: - yield record_start, record - finally: - if isinstance(record, Path): - record.unlink(missing_ok=True) - record_start = offset + newline + 1 - cursor = newline + 1 - offset += len(chunk) - - -class HermesJSONLHistorySource: - source_kind = "hermes-jsonl" - - def __init__(self, path: Path) -> None: - self.path = path - self.source_locator = os.fspath(path.resolve()) - - def discover(self, checkpoint: ImportCheckpoint) -> dict[str, int]: - discovered = skipped = quarantined = eligible = 0 - for source_order, (offset, raw) in enumerate(_bounded_jsonl_records(self.path)): - discovered += 1 - try: - fields = _record_fields(raw) - external_id = str( - _bounded_scalar(raw, fields.get("id")) - or _bounded_scalar(raw, fields.get("session_id")) - or "" - ).strip() - source = str(_bounded_scalar(raw, fields.get("source")) or "").strip().casefold() - user_id = str(_bounded_scalar(raw, fields.get("user_id")) or "").strip() - chat_type = str(_bounded_scalar(raw, fields.get("chat_type")) or "").strip().casefold() - messages_span = fields.get("messages") - message_count = ( - sum(1 for _ in _array_elements(raw, messages_span)) - if messages_span is not None - else 0 - ) - except (OSError, UnicodeError, ValueError, json.JSONDecodeError): - quarantined += 1 - continue - disposition, subject_id, user_hash = _disposition( - external_id=external_id, - source=source, - user_id=user_id, - chat_type=chat_type, - ) - if disposition != "included": - quarantined += int(disposition == "quarantined") - skipped += int(disposition == "skipped") - continue - if message_count <= 0: - skipped += 1 - continue - checkpoint.add_session( - SessionDescriptor( - external_id=external_id, - source=source, - subject_id=subject_id, - user_hash=user_hash, - ordering=source_order, - message_high_water=message_count, - locator=str(offset), - ) - ) - eligible += 1 - checkpoint.finish_discovery() - result = { - "discovered": discovered, - "eligible": eligible, - "skipped": skipped, - "quarantined": quarantined, - } - checkpoint.set_inventory(**result) - return result - - def iter_messages( - self, session: SessionDescriptor, *, start: int - ) -> Iterator[dict[str, Any]]: - wanted = int(session.locator or 0) - for offset, raw in _bounded_jsonl_records(self.path): - if offset != wanted: - continue - try: - messages = _record_fields(raw).get("messages") - except (OSError, UnicodeError, ValueError, json.JSONDecodeError): - return - if messages is None: - return - for index, message_span in enumerate(_array_elements(raw, messages)): - if index < start: - continue - if index >= session.message_high_water: - break - message = _jsonl_message(raw, message_span, index) - if message is not None: - yield message - return - - -class HermesOfficialExportSource(HermesJSONLHistorySource): - """Spill Hermes's supported export to a private JSONL file before replay.""" - - source_kind = "hermes-official-export" - - def __init__( - self, - database: Path, - imports_root: Path, - *, - existing_locator: Path | None = None, - ) -> None: - self.database = database - self.imports_root = _absolute_path(imports_root) - if existing_locator is None: - database_key = hashlib.sha256( - os.fspath(_absolute_path(database)).encode() - ).hexdigest()[:24] - target = self.imports_root / f"official-export-{database_key}.jsonl" - if target.exists() or target.is_symlink(): - # A final spill with no active checkpoint is an orphan. Keep - # it for age-based recovery, but never mistake it for this new - # job's immutable snapshot. - nonce = hashlib.sha256( - f"{os.getpid()}\0{time.time_ns()}".encode() - ).hexdigest()[:16] - target = self.imports_root / ( - f"official-export-{database_key}-{nonce}.jsonl" - ) - else: - selected = _validated_official_export_path( - self.imports_root, existing_locator - ) - if selected is None: - raise RuntimeError("unsafe official export checkpoint locator") - target = selected - # Establish the durable locator before invoking Hermes. ImportCheckpoint - # can therefore attach to an active job without refreshing its immutable - # export snapshot. - super().__init__(target) - - def _ensure_private_root(self) -> None: - if not _official_export_root_is_safe(self.imports_root): - raise OSError("official export directory is not private storage") - self.imports_root.mkdir(parents=True, exist_ok=True, mode=0o700) - if os.name == "posix": - os.chmod(self.imports_root, 0o700) - if not _official_export_root_is_safe(self.imports_root): - raise OSError("official export directory is not private storage") - - def _ensure_export(self) -> None: - self._ensure_private_root() - target = Path(self.source_locator) - if target.exists(): - if not _private_official_export_file(target): - raise OSError("official export spill is not a private regular file") - return - try: - from hermes_state import SessionDB - except ImportError as exc: # pragma: no cover - Hermes runtime only - raise RuntimeError("Hermes export API is unavailable") from exc - - temporary = self.imports_root / ( - f".{target.name}.{os.getpid()}.{time.time_ns()}.tmp" - ) - flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL - if hasattr(os, "O_NOFOLLOW"): - flags |= os.O_NOFOLLOW - db = SessionDB(db_path=self.database, read_only=True) - descriptor = -1 - try: - descriptor = os.open(temporary, flags, 0o600) - with os.fdopen(descriptor, "w", encoding="utf-8") as stream: - descriptor = -1 # fdopen owns the descriptor from this point. - exporter = getattr(db, "iter_export", None) - if not callable(exporter): - raise RuntimeError("Hermes streaming export API is unavailable") - for value in exporter(): - if isinstance(value, dict): - json.dump(value, stream, ensure_ascii=False, separators=(",", ":")) - stream.write("\n") - stream.flush() - os.fsync(stream.fileno()) - # Linking a completed private temporary file publishes it only if - # no concurrent invocation has already fixed this job's snapshot. - # Unlike os.replace(), this can never overwrite an active source. - try: - os.link(temporary, target) - except FileExistsError: - if not _private_official_export_file(target): - raise OSError( - "concurrent official export spill is not a private regular file" - ) from None - finally: - try: - if descriptor >= 0: - os.close(descriptor) - finally: - try: - db.close() - finally: - try: - temporary.unlink() - except FileNotFoundError: - pass - - def discover(self, checkpoint: ImportCheckpoint) -> dict[str, int]: - self._ensure_export() - return super().discover(checkpoint) - - -def _recover_official_export_spills( - hermes_home: Path, - *, - now: float | None = None, - max_orphan_age_seconds: float = _OFFICIAL_EXPORT_MAX_ORPHAN_AGE_SECONDS, -) -> list[_ActiveOfficialExport]: - """Preserve active immutable exports and clean terminal/stale spill files.""" - spill_root = _official_export_spill_root(hermes_home) - jobs_root = hermes_home / "substrate_wiki" / "imports" / "jobs" - active: list[_ActiveOfficialExport] = [] - referenced: set[str] = set() - checkpoint_scan_complete = True - - if jobs_root.is_dir() and not jobs_root.is_symlink(): - for job_directory in jobs_root.iterdir(): - if job_directory.is_symlink() or not job_directory.is_dir(): - continue - checkpoint_path = job_directory / "checkpoint.db" - if checkpoint_path.is_symlink() or not checkpoint_path.is_file(): - continue - connection: sqlite3.Connection | None = None - try: - uri = f"{checkpoint_path.resolve().as_uri()}?mode=ro" - connection = sqlite3.connect(uri, uri=True, timeout=1.0) - row = connection.execute( - """SELECT source_kind, source_locator, state, updated_at - FROM job WHERE singleton=1""" - ).fetchone() - if row is None or str(row[0]) != "hermes-official-export": - continue - discovered_sessions = int( - connection.execute("SELECT COUNT(*) FROM sessions").fetchone()[0] - ) - candidate = _validated_official_export_path(spill_root, str(row[1])) - state = str(row[2]) - if candidate is None: - if state not in TERMINAL_STATES: - raise RuntimeError( - "active official export checkpoint has an unsafe locator" - ) - continue - key = os.path.normcase(os.fspath(candidate)) - referenced.add(key) - if state in TERMINAL_STATES: - remove_official_export_spill( - hermes_home, - source_kind="hermes-official-export", - source_locator=os.fspath(candidate), - ) - continue - if candidate.exists() and not _private_official_export_file(candidate): - raise RuntimeError( - "active official export spill is not a private regular file" - ) - if ( - not candidate.exists() - and (state != "created" or discovered_sessions != 0) - ): - raise RuntimeError("active official export spill is missing") - active.append( - _ActiveOfficialExport( - locator=candidate, - state=state, - updated_at=float(row[3]), - discovered_sessions=discovered_sessions, - ) - ) - except sqlite3.Error: - checkpoint_scan_complete = False - continue - finally: - if connection is not None: - connection.close() - - if ( - checkpoint_scan_complete - and spill_root.is_dir() - and _official_export_root_is_safe(spill_root) - ): - cutoff = (time.time() if now is None else now) - max( - 0.0, max_orphan_age_seconds - ) - for candidate in spill_root.iterdir(): - if candidate.is_symlink(): - continue - try: - details = candidate.lstat() - except FileNotFoundError: - continue - if not stat.S_ISREG(details.st_mode): - continue - is_export = _official_export_name(candidate.name) - is_temporary = ( - candidate.name.startswith(".official-export-") - and candidate.name.endswith(".tmp") - ) or candidate.name == "official-export.tmp" - if not is_export and not is_temporary: - continue - if os.path.normcase(os.fspath(_absolute_path(candidate))) in referenced: - continue - if details.st_mtime <= cutoff: - try: - candidate.unlink() - except FileNotFoundError: - pass - - active.sort(key=lambda item: item.updated_at, reverse=True) - return active - - -class InMemoryHistorySource: - """Bounded compatibility adapter used only by unit tests.""" - - source_kind = "in-memory" - source_locator = "test" - - def __init__(self, inventory: HistoryInventory) -> None: - self.inventory = inventory - self.by_id = {session.external_id: session for session in inventory.sessions} - - def discover(self, checkpoint: ImportCheckpoint) -> dict[str, int]: - for ordering, session in enumerate(self.inventory.sessions): - checkpoint.add_session( - SessionDescriptor( - session.external_id, - session.source, - session.subject_id, - hashlib.sha256(session.user_id.encode()).hexdigest() if session.user_id else "", - ordering, - len(session.messages), - ) - ) - checkpoint.finish_discovery() - result = { - "discovered": self.inventory.discovered, - "eligible": len(self.inventory.sessions), - "skipped": self.inventory.skipped, - "quarantined": self.inventory.quarantined, - } - checkpoint.set_inventory(**result) - return result - - def iter_messages( - self, session: SessionDescriptor, *, start: int - ) -> Iterator[dict[str, Any]]: - for index, value in enumerate(self.by_id[session.external_id].messages[start:], start): - message = dict(value) - message["_capture_index"] = index - yield message - - -def select_history_source( - hermes_home: Path, *, export_path: Path | None = None -) -> ConversationReplaySource: - if export_path is not None: - return HermesJSONLHistorySource(export_path) - active_exports = _recover_official_export_spills(hermes_home) - database = hermes_home / "state.db" - if active_exports: - # Source selection must honor the locator already sealed into the - # checkpoint even if state.db has since changed or become readable. - return HermesOfficialExportSource( - database, - _official_export_spill_root(hermes_home), - existing_locator=active_exports[0].locator, - ) - if database.is_symlink(): - raise RuntimeError("Hermes state database must not be a symlink") - source = HermesSQLiteHistorySource(database) - try: - connection = source._connect() - session_columns = source._columns(connection, "sessions") - message_columns = source._columns(connection, "messages") - if not {"id", "source"} <= session_columns or not { - "session_id", "role", "content" - } <= message_columns: - raise sqlite3.DatabaseError("unsupported Hermes session schema") - connection.rollback() - connection.close() - return source - except (FileNotFoundError, OSError, sqlite3.DatabaseError): - return HermesOfficialExportSource( - database, _official_export_spill_root(hermes_home) - ) - - -def _endpoint(kind: str) -> str: - return "/api/v1/hermes/completed-sessions" if kind == "session_end" else "/api/v1/hermes/turns" - - -def _deliver_with_retry_meta( - client: SubstrateClient, - event: dict[str, Any], - *, - stop_requested: Callable[[], bool] | None = None, -) -> tuple[dict[str, Any], int]: - delay = 1.0 - retries = 0 - while True: - if stop_requested is not None and stop_requested(): - raise _ImportStopRequested - try: - result = client.request( - "POST", - _endpoint(str(event["kind"])), - body=event, - idempotency_key=str(event["event_id"]), - ) - return (result if isinstance(result, dict) else {}), retries - except SubstrateAPIError as exc: - transient = exc.category in _TRANSIENT_CATEGORIES or ( - exc.category.startswith("http_") - and exc.category[5:].isdigit() - and int(exc.category[5:]) >= 500 - ) - if not transient: - raise - retries += 1 - remaining = max(delay, exc.retry_after or 0.0) - while remaining > 0: - if stop_requested is not None and stop_requested(): - raise _ImportStopRequested from None - interval = min(0.25, remaining) - time.sleep(interval) - remaining -= interval - delay = min(300.0, delay * 2.0) - - -def _deliver_with_retry(client: SubstrateClient, event: dict[str, Any]) -> dict[str, Any]: - """Compatibility wrapper returning only the acknowledgement body.""" - return _deliver_with_retry_meta(client, event)[0] - - -def _peak_rss_bytes() -> int: - try: - import resource - - value = int( - resource.getrusage(resource.RUSAGE_SELF).ru_maxrss # type: ignore[attr-defined] - ) - return value if sys.platform == "darwin" else value * 1024 - except (AttributeError, ImportError, OSError, ValueError): - return 0 - - -def _legacy_candidates(hermes_home: Path, client: SubstrateClient) -> list[dict[str, Any]]: - candidates = discover_legacy_checkpoints(hermes_home / "substrate_wiki" / "imports") - for item in candidates: - try: - remote = client.import_status(str(item["batch_id"])) - except SubstrateAPIError: - remote = {} - if isinstance(remote, dict): - item["processed"] = max(0, int(remote.get("processed", 0))) - item["delivered"] = max(0, int(remote.get("delivered", 0))) - candidates.sort( - key=lambda item: ( - int(item.get("processed", 0)), - int(item.get("delivered", 0)), - float(item.get("checkpoint_time", 0)), - ), - reverse=True, - ) - return candidates - - -class HermesHistoryImporter: - """Stream one acknowledged event at a time from an immutable discovery snapshot.""" - - def __init__( - self, - *, - hermes_home: Path, - client: SubstrateClient, - source: ConversationReplaySource | None = None, - inventory: HistoryInventory | None = None, - agent_id: str = "default", - export_path: Path | None = None, - stop_requested: Callable[[], bool] | None = None, - ) -> None: - self.hermes_home = hermes_home - self.client = client - self.agent_id = agent_id or "default" - self.source = source or ( - InMemoryHistorySource(inventory) - if inventory is not None - else select_history_source(hermes_home, export_path=export_path) - ) - self.checkpoint: ImportCheckpoint | None = None - self.stop_requested = stop_requested or (lambda: False) - - def prepare(self) -> ImportCheckpoint: - capabilities = self.client.capabilities() - validate_capabilities(capabilities, require_replay=True, require_entity=False) - candidates = _legacy_candidates(self.hermes_home, self.client) - selected = str(candidates[0]["batch_id"]) if candidates else None - checkpoint = ImportCheckpoint.create_or_attach( - self.hermes_home, - source_kind=self.source.source_kind, - source_locator=self.source.source_locator, - agent_id=self.agent_id, - batch_id=selected, - legacy_batches=candidates, - ) - if checkpoint.job()["state"] == "created": - self.source.discover(checkpoint) - completed: set[str] = set() - for item in candidates: - completed.update(item.get("completed_sessions", set())) - checkpoint.mark_completed_sessions(completed) - self.checkpoint = checkpoint - return checkpoint - - @property - def batch_id(self) -> str: - if self.checkpoint is None: - raise RuntimeError("import has not been prepared") - return str(self.checkpoint.job()["batch_id"]) - - def run(self, *, wait: bool) -> dict[str, Any]: - checkpoint = self.checkpoint or self.prepare() - try: - if self.stop_requested(): - return checkpoint.status() - for session in checkpoint.sessions(): - if self.stop_requested() or checkpoint.job()["state"] == "cancelled": - return checkpoint.status() - if not self._deliver_session(checkpoint, session): - return checkpoint.status() - checkpoint.update_peak_rss(_peak_rss_bytes()) - if wait: - while True: - if self.stop_requested(): - return checkpoint.status() - remote = self.client.import_status(self.batch_id) - if isinstance(remote, dict): - checkpoint.update_remote(remote) - if remote.get("complete"): - break - for _ in range(20): - if self.stop_requested(): - return checkpoint.status() - time.sleep(0.1) - return checkpoint.status() - except _ImportStopRequested: - return checkpoint.status() - except Exception as exc: - checkpoint.set_state("failed", error_class=type(exc).__name__) - raise - - def _deliver_session( - self, checkpoint: ImportCheckpoint, session: SessionDescriptor - ) -> bool: - start, rolling = checkpoint.session_progress(session.external_id) - builder = CaptureEventBuilder( - { - "provider_id": "substrate_wiki", - "agent_id": self.agent_id, - "platform": session.source, - "subject_id": session.subject_id, - }, - secrets=tuple( - secret - for secret in dict.fromkeys( - (*configured_secret_values(), str(getattr(self.client, "api_key", "") or "")) - ) - if secret - ), - ) - messages = self.source.iter_messages(session, start=start) - for event in builder.iter_message_events( - "turn", - session.external_id, - messages, - start_index=start, - capture_origin="history_replay", - batch_id=self.batch_id, - deterministic=True, - ): - if self.stop_requested(): - return False - if checkpoint.job()["state"] == "cancelled": - return False - event_messages = event.get("messages", []) - if not isinstance(event_messages, list) or not event_messages: - continue - indexes = [ - int(message["index"]) - for message in event_messages - if isinstance(message, dict) and isinstance(message.get("index"), int) - ] - if not indexes: - continue - event_id = str(event["event_id"]) - if checkpoint.acknowledged(event_id): - continue - result, retries = _deliver_with_retry_meta( - self.client, event, stop_requested=self.stop_requested - ) - completed_indexes = [] - for message in event_messages: - if not isinstance(message, dict): - continue - fragment = message.get("fragment") - if not isinstance(fragment, dict) or int(fragment.get("index", 0)) + 1 >= int( - fragment.get("count", 1) - ): - completed_indexes.append(int(message["index"])) - boundary_start = min(indexes) - boundary_end = max(completed_indexes) + 1 if completed_indexes else boundary_start - rolling = content_digest({"previous": rolling, "event": event}) - checkpoint.acknowledge( - event_id=event_id, - external_id=session.external_id, - boundary_start=boundary_start, - boundary_end=boundary_end, - phase="message", - duplicate=bool(result.get("duplicate")), - retry_count=retries, - session_digest=rolling, - ) - completion = builder.payload_event( - "session_end", - session.external_id, - { - "session_complete": True, - "total_message_boundary": { - "start": 0, - "end": session.message_high_water, - }, - "session_digest": rolling, - "protocol": "stream-v2", - }, - boundary={"start": 0, "end": session.message_high_water}, - capture_origin="history_replay", - batch_id=self.batch_id, - deterministic=True, - ) - event_id = str(completion["event_id"]) - if not checkpoint.acknowledged(event_id): - result, retries = _deliver_with_retry_meta( - self.client, completion, stop_requested=self.stop_requested - ) - checkpoint.acknowledge( - event_id=event_id, - external_id=session.external_id, - boundary_start=0, - boundary_end=session.message_high_water, - phase="session_end", - duplicate=bool(result.get("duplicate")), - retry_count=retries, - session_digest=rolling, - ) - return True - - -def load_hermes_inventory( - hermes_home: Path, *, export_path: Path | None = None -) -> HistoryInventory: - """Compatibility preview implemented via a private temporary checkpoint.""" - source = select_history_source(hermes_home, export_path=export_path) - with tempfile.TemporaryDirectory(prefix="substrate-history-preview-") as directory: - checkpoint = ImportCheckpoint.create_or_attach( - Path(directory), - source_kind=source.source_kind, - source_locator=source.source_locator, - agent_id="preview", - ) - counts = source.discover(checkpoint) - inventory = HistoryInventory( - discovered=counts["discovered"], - eligible=counts["eligible"], - skipped=counts["skipped"], - quarantined=counts["quarantined"], - ) - checkpoint.close() - if source.source_kind == "hermes-official-export": - remove_official_export_spill( - hermes_home, - source_kind=source.source_kind, - source_locator=source.source_locator, - ) - return inventory diff --git a/src/substrate_wiki/onboarding.py b/src/substrate_wiki/onboarding.py deleted file mode 100644 index 245de4e..0000000 --- a/src/substrate_wiki/onboarding.py +++ /dev/null @@ -1,641 +0,0 @@ -"""Hosted-only, resumable Substrate onboarding for Hermes. - -Authentication uses the RFC 8628 device grant exposed by the fixed Azure -origin. Operational state is content-free; device/access credentials live only -in profile-scoped credential custody. -""" -from __future__ import annotations - -import argparse -import json -import os -import sys -import threading -import time -import webbrowser -from datetime import UTC, datetime -from pathlib import Path -from typing import Any, Callable -from urllib.error import HTTPError, URLError -from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit -from urllib.request import HTTPRedirectHandler, Request, build_opener - -from .client import SubstrateAPIError, SubstrateClient, validate_capabilities -from .credentials import CredentialStore, credential_store -from .spool import secure_atomic_json_write - -HOSTED_ORIGIN = os.environ.get( - "SUBSTRATE_WIKI_ORIGIN", "https://app.trysubstrate.co" -).rstrip("/") -CLIENT_ID = "substrate-hermes" -SCOPES = "capture retrieve" -DEVICE_GRANT = "urn:ietf:params:oauth:grant-type:device_code" -_STATE_VERSION = 1 -_PLUGIN_VERSION = "2.0.5" -_MAX_RESPONSE = 64 * 1024 -_CAPABILITY_TIMEOUT_SECONDS = 60.0 -_CAPABILITY_ATTEMPTS = 2 -_TRANSIENT_CAPABILITY_FAILURES = frozenset( - { - "timeout", - "transport_error", - "http_408", - "http_425", - "http_429", - "http_500", - "http_502", - "http_503", - "http_504", - } -) -_TRANSIENT_OAUTH_POLL_FAILURES = frozenset( - { - "transport_error", - "invalid_content_type", - "http_408", - "http_425", - "http_429", - "http_500", - "http_502", - "http_503", - "http_504", - } -) -_TRANSIENT_OAUTH_HTTP_STATUSES = frozenset({408, 425, 429, 500, 502, 503, 504}) -_SAFE_OAUTH_ERRORS = frozenset( - { - "access_denied", - "authorization_pending", - "expired_token", - "invalid_client", - "invalid_grant", - "invalid_request", - "invalid_scope", - "slow_down", - "unsupported_grant_type", - } -) -_TERMINAL = {"ready", "declined", "failed", "repair_required"} - - -class OnboardingError(RuntimeError): - def __init__(self, category: str) -> None: - self.category = category - super().__init__(category) - - -def _oauth_failure_category(status: int, value: dict[str, Any]) -> str: - if status in _TRANSIENT_OAUTH_HTTP_STATUSES: - return f"http_{status}" - error = value.get("error") - if isinstance(error, str) and error in _SAFE_OAUTH_ERRORS: - return error - return "invalid_response" - - -class _NoRedirect(HTTPRedirectHandler): - def redirect_request(self, req: Request, fp: Any, code: int, msg: str, - headers: Any, newurl: str) -> None: - return None - - -def _hosted_url(value: Any) -> str: - if not isinstance(value, str) or len(value) > 4096: - raise OnboardingError("invalid_response") - parsed = urlsplit(value) - if ( - f"{parsed.scheme}://{parsed.netloc}" != HOSTED_ORIGIN - or parsed.username or parsed.password or parsed.fragment - ): - raise OnboardingError("invalid_response") - return value - - -class HostedOAuthClient: - """Minimal no-redirect RFC 8628 client pinned to the hosted origin.""" - - def __init__(self, *, timeout: float = 60.0) -> None: - self.timeout = timeout - self._opener = build_opener(_NoRedirect()) - - def _post(self, path: str, values: dict[str, str]) -> tuple[int, dict[str, Any]]: - if path not in {"/oauth/device_authorization", "/oauth/token"}: - raise OnboardingError("invalid_request") - raw = urlencode(values).encode("ascii") - request = Request( - HOSTED_ORIGIN + path, - data=raw, - method="POST", - headers={ - "Accept": "application/json", - "Content-Type": "application/x-www-form-urlencoded", - "User-Agent": f"substrate_wiki-hermes-plugin/{_PLUGIN_VERSION}", - }, - ) - try: - response = self._opener.open(request, timeout=self.timeout) - except HTTPError as exc: - response = exc - except (URLError, OSError, TimeoutError): - raise OnboardingError("transport_error") from None - try: - status = int(response.status) - content_type = response.headers.get_content_type() - data = response.read(_MAX_RESPONSE + 1) - finally: - response.close() - if len(data) > _MAX_RESPONSE: - raise OnboardingError("response_too_large") - if content_type != "application/json": - raise OnboardingError("invalid_content_type") - try: - value = json.loads(data.decode("utf-8")) - except (UnicodeError, json.JSONDecodeError): - raise OnboardingError("invalid_response") from None - if not isinstance(value, dict): - raise OnboardingError("invalid_response") - return status, value - - def begin(self) -> dict[str, Any]: - status, value = self._post( - "/oauth/device_authorization", {"client_id": CLIENT_ID, "scope": SCOPES} - ) - if status != 200: - raise OnboardingError(_oauth_failure_category(status, value)) - required = ("device_code", "user_code", "verification_uri", "expires_in") - if not all(isinstance(value.get(key), (str, int)) for key in required): - raise OnboardingError("invalid_response") - device_code = value["device_code"] - user_code = value["user_code"] - if not isinstance(device_code, str) or not device_code or len(device_code) > 4096: - raise OnboardingError("invalid_response") - if not isinstance(user_code, str) or not user_code or len(user_code) > 64: - raise OnboardingError("invalid_response") - verification_uri = _hosted_url(value["verification_uri"]) - supplied_complete = value.get("verification_uri_complete") - if supplied_complete: - complete = _hosted_url(supplied_complete) - complete_query = dict(parse_qsl(urlsplit(complete).query, keep_blank_values=True)) - if complete_query.get("user_code") != user_code: - raise OnboardingError("invalid_response") - else: - parsed = urlsplit(verification_uri) - query = [ - (key, item) - for key, item in parse_qsl(parsed.query, keep_blank_values=True) - if key != "user_code" - ] - query.append(("user_code", user_code)) - complete = urlunsplit(parsed._replace(query=urlencode(query))) - return { - "device_code": device_code, - "user_code": user_code, - "verification_uri": verification_uri, - "verification_uri_complete": complete, - "expires_in": max(1, min(int(value["expires_in"]), 3600)), - "interval": max(1, min(int(value.get("interval", 5)), 60)), - } - - def poll(self, device_code: str) -> dict[str, Any]: - status, value = self._post( - "/oauth/token", - {"grant_type": DEVICE_GRANT, "device_code": device_code, "client_id": CLIENT_ID}, - ) - if status == 200: - token = value.get("access_token") - if ( - not isinstance(token, str) or not token or len(token) > 16384 - or str(value.get("token_type", "")).casefold() != "bearer" - or set(str(value.get("scope", SCOPES)).split()) != set(SCOPES.split()) - ): - raise OnboardingError("invalid_response") - return {"status": "approved", "access_token": token} - error = value.get("error") - if error in {"authorization_pending", "slow_down", "access_denied", "expired_token"}: - return {"status": str(error)} - raise OnboardingError(_oauth_failure_category(status, value)) - - -def _empty_state() -> dict[str, Any]: - return { - "state_version": _STATE_VERSION, - "phase": "new", - "hosted_origin": HOSTED_ORIGIN, - "updated_at": time.time(), - "attempt": 0, - } - - -def _receipt(decision: str) -> dict[str, Any]: - return { - "version": 1, - "scope": "hermes_history", - "decision": decision, - "recorded_at": datetime.now(UTC).isoformat().replace("+00:00", "Z"), - } - - -class OnboardingManager: - """One Hermes profile's idempotent hosted onboarding state machine.""" - - def __init__( - self, - home: Path, - *, - api: HostedOAuthClient | None = None, - store: CredentialStore | None = None, - capability_check: Callable[[str], dict[str, Any]] | None = None, - import_start: Callable[[Path], dict[str, Any]] | None = None, - opener: Callable[[str], bool] | None = None, - ) -> None: - self.home = home.resolve() - self.root = self.home / "substrate_wiki" / "onboarding" - if self.root.exists() and self.root.is_symlink(): - raise OSError("onboarding directory must not be a symlink") - self.root.mkdir(parents=True, exist_ok=True, mode=0o700) - if os.name == "posix": - os.chmod(self.root, 0o700) - self.path = self.root / "state.json" - self.api = api or HostedOAuthClient() - self.store = store or credential_store(self.home) - self.capability_check = capability_check or self._check_capabilities - self.import_start = import_start or _start_history_import - self.opener = opener or webbrowser.open - self._mutex = threading.RLock() - - def _load(self) -> dict[str, Any]: - if self.path.is_symlink(): - raise OSError("onboarding state must not be a symlink") - try: - value = json.loads(self.path.read_text(encoding="utf-8")) - except FileNotFoundError: - return _empty_state() - except (OSError, UnicodeError, json.JSONDecodeError): - return {**_empty_state(), "phase": "repair_required", "error_class": "state_corrupt"} - if ( - not isinstance(value, dict) - or value.get("state_version") != _STATE_VERSION - or value.get("hosted_origin") != HOSTED_ORIGIN - ): - return {**_empty_state(), "phase": "repair_required", "error_class": "state_incompatible"} - for forbidden in ("access_token", "api_key", "device_code", "token"): - value.pop(forbidden, None) - return value - - def _save(self, state: dict[str, Any]) -> None: - clean = dict(state) - for forbidden in ("access_token", "api_key", "device_code", "token"): - clean.pop(forbidden, None) - clean.update( - state_version=_STATE_VERSION, hosted_origin=HOSTED_ORIGIN, updated_at=time.time() - ) - secure_atomic_json_write(self.path, clean) - - def _refresh_import(self, state: dict[str, Any]) -> dict[str, Any]: - if state.get("phase") != "importing": - return state - job_id = state.get("import_job_id") - if not isinstance(job_id, str) or not job_id: - return state - try: - from .checkpoint import ImportCheckpoint - from .worker import checkpoint_path - - with ImportCheckpoint(checkpoint_path(self.home, job_id)) as checkpoint: - progress = checkpoint.status() - except (FileNotFoundError, OSError, ValueError): - return state - state["import"] = { - key: progress[key] - for key in ( - "job_id", "state", "discovered", "eligible", "delivered", "failed", - "skipped", "quarantined", "complete", - ) - if key in progress - } - if progress.get("complete"): - state["phase"] = "ready" - state["completed_at"] = time.time() - self._save(state) - return state - - def status(self) -> dict[str, Any]: - with self._mutex: - state = self._refresh_import(self._load()) - result = { - key: state[key] - for key in ( - "phase", "hosted_origin", "mode", "verification_uri", - "verification_uri_complete", "user_code", "expires_at", - "history_consent", "error_class", "capability_failure", - "oauth_poll_failure", "import", - "connected_at", "completed_at", - ) - if key in state - } - result["credential_backend"] = self.store.backend - result["authenticated"] = bool(self.store.get()) - result["complete"] = state.get("phase") in _TERMINAL - if state.get("phase") == "awaiting_history_consent": - result["action_required"] = "history_consent" - return result - - def _check_capabilities(self, token: str) -> dict[str, Any]: - """Validate the issued key, tolerating one hosted tenant cold start.""" - for attempt in range(_CAPABILITY_ATTEMPTS): - client = SubstrateClient( - HOSTED_ORIGIN, token, timeout=_CAPABILITY_TIMEOUT_SECONDS - ) - try: - capabilities = client.capabilities() - validate_capabilities(capabilities, require_replay=True, require_entity=False) - return {"provider": capabilities.get("provider"), "protocol": "stream-v2"} - except SubstrateAPIError as exc: - if ( - attempt + 1 >= _CAPABILITY_ATTEMPTS - or exc.category not in _TRANSIENT_CAPABILITY_FAILURES - ): - raise - time.sleep(1.0) - raise SubstrateAPIError("invalid_response") - - def begin(self, *, mode: str = "auto", open_browser: bool = True) -> dict[str, Any]: - with self._mutex: - state = self._load() - if self.store.get(): - if state.get("phase") in {"new", "authorization_pending", "failed"}: - state.update(phase="awaiting_history_consent", connected_at=time.time()) - state.pop("error_class", None) - self._save(state) - return self.status() - if ( - state.get("phase") == "authorization_pending" - and float(state.get("expires_at", 0)) > time.time() - and self.store.get("onboarding-device") - ): - return self.status() - grant = self.api.begin() - self.store.put(str(grant.pop("device_code")), "onboarding-device") - selected = _select_mode(mode) - expires_at = time.time() + int(grant.pop("expires_in")) - state = { - **_empty_state(), - **grant, - "phase": "authorization_pending", - "mode": selected, - "expires_at": expires_at, - "attempt": int(state.get("attempt", 0)) + 1, - } - self._save(state) - if open_browser and selected == "browser": - try: - self.opener(str(state["verification_uri_complete"])) - except (OSError, webbrowser.Error): - pass - return self.status() - - def advance(self) -> dict[str, Any]: - with self._mutex: - state = self._load() - if state.get("phase") != "authorization_pending": - return self.status() - if float(state.get("expires_at", 0)) <= time.time(): - self.store.delete("onboarding-device") - state.update(phase="failed", error_class="authorization_expired") - self._save(state) - return self.status() - device_code = self.store.get("onboarding-device") - if not device_code: - state.update(phase="repair_required", error_class="missing_device_credential") - self._save(state) - return self.status() - try: - response = self.api.poll(device_code) - except OnboardingError as exc: - if exc.category not in _TRANSIENT_OAUTH_POLL_FAILURES: - raise - state["oauth_poll_failure"] = exc.category - self._save(state) - return self.status() - state.pop("oauth_poll_failure", None) - poll_status = response["status"] - if poll_status in {"authorization_pending", "slow_down"}: - if poll_status == "slow_down": - state["interval"] = min(60, int(state.get("interval", 5)) + 5) - self._save(state) - return self.status() - if poll_status in {"access_denied", "expired_token"}: - self.store.delete("onboarding-device") - state.update( - phase="declined" if poll_status == "access_denied" else "failed", - error_class=poll_status, - ) - self._save(state) - return self.status() - token = response["access_token"] - try: - self.capability_check(token) - except (SubstrateAPIError, OnboardingError) as exc: - state.update( - phase="failed", - error_class="capability_check_failed", - capability_failure=exc.category, - ) - self._save(state) - return self.status() - self.store.put(token) - self.store.delete("onboarding-device") - state.update(phase="awaiting_history_consent", connected_at=time.time()) - state.pop("error_class", None) - state.pop("capability_failure", None) - state.pop("oauth_poll_failure", None) - self._save(state) - return self.status() - - def consent_history(self, approved: bool) -> dict[str, Any]: - with self._mutex: - state = self._load() - if not self.store.get(): - raise OnboardingError("authentication_required") - decision = "approved" if approved else "declined" - if state.get("phase") != "awaiting_history_consent": - receipt = state.get("history_consent") - if isinstance(receipt, dict) and receipt.get("decision") == decision: - return self.status() - raise OnboardingError("history_consent_not_pending") - state["history_consent"] = _receipt(decision) - if not approved: - state.update(phase="ready", completed_at=time.time()) - self._save(state) - return self.status() - # Consent must be durable before history discovery or worker launch. - state.update(phase="import_starting") - self._save(state) - try: - progress = self.import_start(self.home) - except (OSError, RuntimeError, ValueError, SubstrateAPIError) as exc: - state.update(phase="repair_required", error_class=type(exc).__name__) - self._save(state) - return self.status() - state["import"] = progress - job_id = progress.get("job_id") - if isinstance(job_id, str): - state["import_job_id"] = job_id - state["phase"] = "ready" if progress.get("complete") else "importing" - if state["phase"] == "ready": - state["completed_at"] = time.time() - self._save(state) - return self.status() - - def run( - self, *, mode: str = "auto", wait: bool = False, open_browser: bool = True, - timeout: float = 900.0, - ) -> dict[str, Any]: - result = self.begin(mode=mode, open_browser=open_browser) - if wait and result.get("phase") == "authorization_pending": - print( - f"Open {result.get('verification_uri_complete')} to sign in by email " - f"and connect Hermes. One-time code: {result.get('user_code')}", - file=sys.stderr, - flush=True, - ) - deadline = time.monotonic() + max(0.0, timeout) - while wait and result.get("phase") == "authorization_pending" and time.monotonic() < deadline: - interval = max(1, min(int(self._load().get("interval", 5)), 60)) - time.sleep(interval) - result = self.advance() - return result - - def repair(self, *, wait: bool = False) -> dict[str, Any]: - with self._mutex: - state = self._load() - token = self.store.get() - if token: - try: - self.capability_check(token) - except (SubstrateAPIError, OnboardingError): - self.store.delete() - else: - if state.get("phase") == "importing": - from .supervisor import start_service - job_id = state.get("import_job_id") - if isinstance(job_id, str): - start_service(self.home, job_id) - return self.status() - self.store.delete("onboarding-device") - self._save(_empty_state()) - return self.run(mode="auto", wait=wait) - - -def _select_mode(mode: str) -> str: - if mode in {"browser", "device"}: - return mode - if mode != "auto": - raise ValueError("mode must be auto, browser, or device") - graphical = ( - os.name == "nt" or sys.platform == "darwin" - or bool(os.environ.get("DISPLAY") or os.environ.get("WAYLAND_DISPLAY")) - ) - return "browser" if graphical else "device" - - -def _active_agent_id() -> str: - return os.environ.get("HERMES_PROFILE") or os.environ.get("HERMES_AGENT_ID") or "default" - - -def _start_history_import(home: Path) -> dict[str, Any]: - from .history import HermesHistoryImporter, select_history_source - from .supervisor import start_service - - try: - source = select_history_source(home) - client = SubstrateClient.from_env( - timeout=30.0, hermes_home=home, hosted_default=True - ) - importer = HermesHistoryImporter( - hermes_home=home, client=client, source=source, agent_id=_active_agent_id() - ) - checkpoint = importer.prepare() - status = checkpoint.status() - checkpoint.close() - if not status.get("complete"): - status["import_service"] = start_service(home, str(status["job_id"])) - return status - except (FileNotFoundError, NotADirectoryError): - return { - "state": "complete", "complete": True, "discovered": 0, "eligible": 0, - "delivered": 0, "skipped": 0, "quarantined": 0, - } - - -def _prompt_history(manager: OnboardingManager) -> dict[str, Any]: - status = manager.status() - if status.get("phase") != "awaiting_history_consent": - return status - if not sys.stdin.isatty(): - return status - print( - "Upload all eligible past Hermes conversations to your Substrate memory? [y/n]: ", - end="", file=sys.stderr, flush=True, - ) - try: - answer = sys.stdin.readline().strip().casefold() - except (EOFError, KeyboardInterrupt): - return manager.status() - if answer in {"y", "yes"}: - return manager.consent_history(True) - if answer in {"n", "no"}: - return manager.consent_history(False) - return manager.status() - - -def bootstrap_package() -> None: - parent = Path(__file__).resolve().parent.parent - if str(parent) not in sys.path: - sys.path.insert(0, str(parent)) - - -def main(argv: list[str] | None = None) -> int: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--hermes-home", type=Path, required=True) - parser.add_argument("--mode", choices=("auto", "browser", "device"), default="auto") - parser.add_argument("--wait", action="store_true") - parser.add_argument("--no-browser", action="store_true") - parser.add_argument("--status", action="store_true") - parser.add_argument("--repair", action="store_true") - parser.add_argument("--history", choices=("ask", "approve", "decline"), default="ask") - parser.add_argument("--json", action="store_true") - args = parser.parse_args(argv) - manager = OnboardingManager(args.hermes_home) - try: - if args.status: - result = manager.status() - elif args.repair: - result = manager.repair(wait=args.wait) - else: - result = manager.run( - mode=args.mode, wait=args.wait, open_browser=not args.no_browser - ) - if result.get("phase") == "awaiting_history_consent": - if args.history == "approve": - result = manager.consent_history(True) - elif args.history == "decline": - result = manager.consent_history(False) - else: - result = _prompt_history(manager) - except Exception as exc: # noqa: BLE001 - never render credential-bearing details - result = {"complete": False, "error_class": type(exc).__name__} - code = 1 - else: - code = 0 - if args.json: - print(json.dumps(result, sort_keys=True)) - else: - for key, value in result.items(): - print(f"{key}: {value}") - return code - - -if __name__ == "__main__": - bootstrap_package() - raise SystemExit(main()) diff --git a/src/substrate_wiki/plugin.yaml b/src/substrate_wiki/plugin.yaml deleted file mode 100644 index 60dfa4b..0000000 --- a/src/substrate_wiki/plugin.yaml +++ /dev/null @@ -1,5 +0,0 @@ -name: substrate_wiki -version: 2.0.5 -description: "Unified entity-centric Substrate Markdown wiki memory with durable asynchronous capture." -hooks: - - on_session_switch diff --git a/src/substrate_wiki/py.typed b/src/substrate_wiki/py.typed deleted file mode 100644 index e69de29..0000000 diff --git a/src/substrate_wiki/redaction.py b/src/substrate_wiki/redaction.py deleted file mode 100644 index f12b230..0000000 --- a/src/substrate_wiki/redaction.py +++ /dev/null @@ -1,431 +0,0 @@ -"""Best-effort credential redaction before session material reaches disk or network.""" - -from __future__ import annotations - -import math -import os -import re -from collections.abc import Iterable, Iterator, Mapping, Sequence, Set -from itertools import islice -from pathlib import Path -from typing import Any - -_REDACTED = "[REDACTED]" -# Live capture hashes only the latest 512 messages and history replay redacts -# one streamed message/event at a time. This ceiling still covers unusually -# wide bounded tool payloads without allowing a hook value to amplify into a -# gateway-sized second object graph. -_MAX_CAPTURE_ITEMS = 16_384 -_MAX_ENVIRONMENT_ITEMS = 4_096 -_MAX_CONFIGURED_SECRETS = 64 -_MAX_CONFIGURED_SECRET_BYTES = 64 * 1024 -_MAX_CONFIGURED_SECRET_CHARS = 16 * 1024 -# The longest bounded fixed-pattern match is the orphan/private-key form: up to -# 128 lines of 4,096 base64 characters plus indentation and delimiters. Keep a -# conservative overlap so a match cannot straddle released text. The overlap -# is independent of total message size and stays comfortably below the import -# worker's 256 MiB RSS ceiling even for wide Unicode strings. -_STREAM_REDACTION_OVERLAP_CHARS = 640 * 1024 -_STREAM_REDACTION_SCAN_CHARS = 256 * 1024 -_STREAM_CONTINUATION_TERMINATORS = frozenset("\r\n\t ,;}]&#@'\"") -_PRIORITY_SECRET_KEYS = ( - "HERMES_API_KEY", - "MINIMAX_API_KEY", - "NVIDIA_API_KEY", - "OPENAI_API_KEY", - "ANTHROPIC_API_KEY", - "AZURE_OPENAI_API_KEY", - "GOOGLE_API_KEY", - "GEMINI_API_KEY", - "MISTRAL_API_KEY", - "COHERE_API_KEY", -) -_CREDENTIAL_LABEL = ( - r"(?:api[-_ ]?key|access[-_ ]?key|secret[-_ ]?key|client[-_ ]?secret|" - r"access[-_ ]?token|refresh[-_ ]?token|id[-_ ]?token|" - r"auth(?:entication|orization)?[-_ ]?token|bearer[-_ ]?token|" - r"signing[-_ ]?key|password|passwd|passphrase|private[-_ ]?key|" - r"connection[-_ ]?string|credential|secret)" -) -_GENERIC_CREDENTIAL_LABEL = r"(?:[A-Z][A-Z0-9_]{0,255}(?:KEY|TOKEN|SECRET|PASSWORD))" -_SIGNED_CREDENTIAL_LABEL = ( - r"(?:x[-_ ]?amz[-_ ]?(?:signature|credential|security[-_ ]?token)|" - r"x[-_ ]?goog[-_ ]?(?:signature|credential)|google[-_ ]?access[-_ ]?id|" - r"aws[-_ ]?access[-_ ]?key[-_ ]?id|signature)" -) -_SENSITIVE_KEYS = re.compile( - rf"(?:^|[-_. ]){_CREDENTIAL_LABEL}(?:$|[-_. ])|" - r"^(?:authorization|proxy-authorization|cookie|set-cookie|" - rf"{_SIGNED_CREDENTIAL_LABEL})$", - re.IGNORECASE, -) -_PRIVATE_KEY_KIND = r"(?:OPENSSH|ENCRYPTED|RSA|EC|DSA)?(?:[ -]+)?PRIVATE KEY" -_PRIVATE_KEY_BLOCK = re.compile( - rf"-----BEGIN {_PRIVATE_KEY_KIND}-----" - rf"[\s\S]{{0,262144}}?-----END {_PRIVATE_KEY_KIND}-----", - re.IGNORECASE, -) -_PRIVATE_KEY_BEGIN = re.compile( - rf"-----BEGIN {_PRIVATE_KEY_KIND}-----[\s\S]{{0,262144}}", - re.IGNORECASE, -) -_PRIVATE_KEY_ORPHAN = re.compile( - rf"(?m)(?:^[ \t]{{0,32}}(?:[A-Za-z0-9+/]{{16,4096}}={{0,2}}|" - rf"(?:Proc-Type|DEK-Info):[^\r\n]{{1,4096}})[ \t]{{0,32}}\r?\n){{1,128}}" - rf"^-----END {_PRIVATE_KEY_KIND}-----", - re.IGNORECASE, -) -_PRIVATE_KEY_FOOTER = re.compile( - rf"-----END {_PRIVATE_KEY_KIND}-----", - re.IGNORECASE, -) -_PRIVATE_KEY_ADJACENT = re.compile( - rf"(?i)\b{_PRIVATE_KEY_KIND}\b\s*(?:(?::|=|\bis\b)\s*)?" - r"[A-Za-z0-9+/]{16,4096}={0,2}" - r"(?:\r?\n[ \t]{0,32}[A-Za-z0-9+/]{16,4096}={0,2}){0,127}" -) -_SAS_SIGNATURE = re.compile( - r"(?i)((?:[?&;]|&|^)sig=)[^&#;\s]{1,262144}" -) -_SAS_COMPANION = re.compile( - r"(?i)(? Iterator[re.Pattern[str]]: - """Skip regexes whose mandatory literal anchors are absent.""" - - folded = value.casefold() - for pattern, hints in zip(_PATTERNS, _PATTERN_HINTS, strict=True): - if any(hint in folded for hint in hints): - yield pattern - - -def configured_secret_values() -> tuple[str, ...]: - """Return a bounded, deterministic set of environment-backed secrets. - - Provider credentials are admitted first. An oversized or overpopulated - sensitive environment fails closed instead of silently omitting a value - that could subsequently reach the durable spool. - """ - values: set[str] = set() - total_bytes = 0 - - def admit(value: str) -> None: - nonlocal total_bytes - if not value or len(value) < 6 or value in values: - return - if len(value) > _MAX_CONFIGURED_SECRET_CHARS: - raise ValueError("configured secrets exceed redaction bounds") - encoded_bytes = len(value.encode("utf-8")) - if ( - len(values) >= _MAX_CONFIGURED_SECRETS - or total_bytes + encoded_bytes > _MAX_CONFIGURED_SECRET_BYTES - ): - raise ValueError("configured secrets exceed redaction bounds") - values.add(value) - total_bytes += encoded_bytes - - priority = set(_PRIORITY_SECRET_KEYS) - for key in _PRIORITY_SECRET_KEYS: - admit(os.environ.get(key, "")) - for index, (key, value) in enumerate(os.environ.items()): - if index >= _MAX_ENVIRONMENT_ITEMS: - raise ValueError("environment exceeds redaction scan bounds") - if key not in priority and value and _SENSITIVE_KEYS.search(key): - admit(value) - return tuple(sorted(values, key=lambda item: (-len(item), item))) - - -def redact_text(value: str, secrets: Sequence[str] | None = None) -> str: - redacted = value - for secret in secrets if secrets is not None else configured_secret_values(): - redacted = redacted.replace(secret, _REDACTED) - for pattern in _candidate_patterns(redacted): - redacted = pattern.sub(_REDACTED, redacted) - if _SAS_COMPANION.search(redacted): - redacted = _SAS_SIGNATURE.sub(r"\1[REDACTED]", redacted) - return redacted - - -def _redaction_spans(value: str, secrets: Sequence[str]) -> list[tuple[int, int]]: - """Return merged raw-input spans covered by the shared credential rules. - - ``redact_text`` applies substitutions sequentially, which is ideal for a - materialized hook value but does not retain a raw-to-output position map. - Streaming needs that map so it can release a stable prefix without losing - or duplicating adjacent safe text. Matching every rule against the same - raw bounded window is deliberately conservative: overlapping detections - collapse to one tombstone instead of exposing either match. - """ - - spans: list[tuple[int, int]] = [] - for secret in secrets: - if not secret: - continue - start = value.find(secret) - while start >= 0: - spans.append((start, start + len(secret))) - start = value.find(secret, start + max(1, len(secret))) - for pattern in _candidate_patterns(value): - spans.extend((match.start(), match.end()) for match in pattern.finditer(value)) - if _SAS_COMPANION.search(value): - spans.extend( - (match.start(), match.end()) for match in _SAS_SIGNATURE.finditer(value) - ) - if not spans: - return [] - spans.sort() - merged: list[tuple[int, int]] = [] - for start, end in spans: - if end <= start: - continue - if merged and start <= merged[-1][1]: - previous_start, previous_end = merged[-1] - merged[-1] = (previous_start, max(previous_end, end)) - else: - merged.append((start, end)) - return merged - - -class StreamingTextRedactor: - """Bounded deterministic redaction for a repeatable streamed text source. - - The class retains only a fixed overlap large enough for every configured - secret and bounded shared rule. A lexical continuation state handles the - few shared token rules with intentionally open-ended quantifiers. No raw - match is released, including when its opener/value is split at any source - chunk boundary. - """ - - def __init__(self, secrets: Sequence[str] = ()) -> None: - self._secrets = tuple(secrets) - self._buffer = "" - self._suppress_continuation = False - - @property - def buffered_chars(self) -> int: - """Content-free test/diagnostic measure of retained normalization state.""" - - return len(self._buffer) - - def _strip_continuation(self, value: str) -> str: - if not self._suppress_continuation: - return value - for index, character in enumerate(value): - if character.isspace() or character in _STREAM_CONTINUATION_TERMINATORS: - self._suppress_continuation = False - return value[index:] - return "" - - def _release(self, cutoff: int) -> str: - spans = _redaction_spans(self._buffer, self._secrets) - crossing = [ - (start, end) for start, end in spans if start < cutoff and end > cutoff - ] - if crossing: - safe_cutoff = min(start for start, _end in crossing) - # Bounded rules cannot span the conservative overlap. A match - # which nevertheless crosses it and reaches the buffer boundary - # is one of the open-ended bearer/provider token forms. Emit its - # single tombstone now and suppress lexical continuation so an - # arbitrarily large token cannot grow the retained buffer. - if any(end == len(self._buffer) for _start, end in crossing): - rendered = redact_text( - self._buffer[:safe_cutoff], self._secrets - ) + _REDACTED - self._buffer = "" - self._suppress_continuation = True - return rendered - cutoff = safe_cutoff - rendered = redact_text(self._buffer[:cutoff], self._secrets) - self._buffer = self._buffer[cutoff:] - return rendered - - def feed(self, value: str) -> Iterator[str]: - """Accept text and yield only prefixes safe against future input.""" - - if not isinstance(value, str): - raise TypeError("streamed message chunks must be strings") - offset = 0 - while offset < len(value): - end = min(len(value), offset + _STREAM_REDACTION_SCAN_CHARS) - piece = self._strip_continuation(value[offset:end]) - offset = end - if piece: - self._buffer += piece - if ( - len(self._buffer) - > _STREAM_REDACTION_OVERLAP_CHARS + _STREAM_REDACTION_SCAN_CHARS - ): - cutoff = len(self._buffer) - _STREAM_REDACTION_OVERLAP_CHARS - rendered = self._release(cutoff) - if rendered: - yield rendered - - def finish(self) -> Iterator[str]: - """Redact and release the final bounded suffix.""" - - if self._suppress_continuation: - self._suppress_continuation = False - if not self._buffer: - return - rendered = self._release(len(self._buffer)) - if rendered: - yield rendered - - -def iter_redacted_text_chunks( - chunks: Iterable[str], - secrets: Sequence[str] = (), -) -> Iterator[str]: - """Yield deterministic redacted text without materializing the source.""" - - redactor = StreamingTextRedactor(secrets) - for chunk in chunks: - yield from redactor.feed(chunk) - yield from redactor.finish() - - -def redact(value: Any, secrets: Sequence[str] | None = None) -> Any: - """Return a JSON-safe redacted copy without invoking arbitrary object reprs.""" - secret_values = tuple(secrets) if secrets is not None else configured_secret_values() - return _redact(value, secret_values, seen=set(), depth=0) - - -def _redact(value: Any, secrets: Sequence[str], *, seen: set[int], depth: int) -> Any: - if depth > 32: - return "[TRUNCATED]" - if value is None or isinstance(value, (bool, int)): - return value - if isinstance(value, float): - return value if math.isfinite(value) else "[NONFINITE]" - if isinstance(value, str): - return redact_text(value, secrets) - if isinstance(value, Path): - return redact_text(os.fspath(value), secrets) - if isinstance(value, (bytes, bytearray, memoryview)): - return "[BINARY]" - identity = id(value) - if identity in seen: - return "[CYCLE]" - if isinstance(value, Mapping): - seen.add(identity) - try: - result: dict[str, Any] = {} - for key, child in islice(value.items(), _MAX_CAPTURE_ITEMS): - raw_key = key if isinstance(key, str) else f"[{type(key).__name__}]" - text_key = redact_text(raw_key, secrets) - if _SENSITIVE_KEYS.search(raw_key): - text_key = _REDACTED - unique_key = text_key - suffix = 2 - while unique_key in result: - unique_key = f"{text_key}#{suffix}" - suffix += 1 - result[unique_key] = ( - _REDACTED - if _SENSITIVE_KEYS.search(raw_key) - else _redact(child, secrets, seen=seen, depth=depth + 1) - ) - return result - finally: - seen.discard(identity) - if isinstance(value, Set) and not isinstance(value, (str, bytes, bytearray)): - seen.add(identity) - try: - sanitized = [ - _redact(child, secrets, seen=seen, depth=depth + 1) - for child in islice(iter(value), _MAX_CAPTURE_ITEMS) - ] - return sorted(sanitized, key=lambda child: (type(child).__name__, str(child)[:256])) - finally: - seen.discard(identity) - if isinstance(value, Sequence) and not isinstance(value, (str, bytes, bytearray)): - seen.add(identity) - try: - return [ - _redact(child, secrets, seen=seen, depth=depth + 1) - for child in islice(iter(value), _MAX_CAPTURE_ITEMS) - ] - finally: - seen.discard(identity) - return f"[UNSUPPORTED:{type(value).__name__}]" diff --git a/src/substrate_wiki/spool.py b/src/substrate_wiki/spool.py deleted file mode 100644 index ba51b77..0000000 --- a/src/substrate_wiki/spool.py +++ /dev/null @@ -1,269 +0,0 @@ -"""Bounded, private, durable JSON spool for nonblocking Hermes delivery.""" - -from __future__ import annotations - -import json -import os -import stat -import threading -import time -import uuid -from pathlib import Path -from typing import Any - - -def _chmod_private(path: Path, mode: int) -> None: - if os.name == "posix": - os.chmod(path, mode) - - -def _fsync_directory(path: Path) -> None: - if os.name != "posix": - return - flags = os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) - descriptor = os.open(path, flags) - try: - os.fsync(descriptor) - finally: - os.close(descriptor) - - -def _safe_root(root: Path) -> Path: - absolute = root.absolute() - if absolute.exists() and absolute.is_symlink(): - raise OSError("spool root must not be a symlink") - absolute.mkdir(parents=True, exist_ok=True, mode=0o700) - if absolute.is_symlink() or not absolute.is_dir(): - raise OSError("invalid spool root") - _chmod_private(absolute, 0o700) - return absolute.resolve(strict=True) - - -def _safe_child(root: Path, path: Path) -> Path: - candidate = path.absolute() - try: - candidate.relative_to(root) - except ValueError: - raise ValueError("path escapes spool root") from None - if candidate.is_symlink(): - raise ValueError("spool files must not be symlinks") - return candidate - - -def secure_atomic_json_write(target: Path, value: Any) -> None: - """Write JSON with exclusive temp creation, fsync, replace, and directory fsync.""" - root = _safe_root(target.parent) - target = _safe_child(root, target) - payload = (json.dumps(value, ensure_ascii=False, indent=2, sort_keys=True) + "\n").encode("utf-8") - temporary = root / f".{target.name}.{os.getpid()}.{uuid.uuid4().hex}.tmp" - flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL - if hasattr(os, "O_NOFOLLOW"): - flags |= os.O_NOFOLLOW - descriptor = os.open(temporary, flags, 0o600) - try: - with os.fdopen(descriptor, "wb") as stream: - stream.write(payload) - stream.flush() - os.fsync(stream.fileno()) - _chmod_private(temporary, 0o600) - if target.exists() and target.is_symlink(): - raise OSError("target must not be a symlink") - os.replace(temporary, target) - _chmod_private(target, 0o600) - _fsync_directory(root) - finally: - try: - temporary.unlink() - except FileNotFoundError: - pass - - -class DurableSpool: - def __init__(self, root: Path, *, max_items: int = 1000, max_bytes: int = 10 * 1024 * 1024) -> None: - self.root = _safe_root(root) - self.max_items = max(1, max_items) - self.max_bytes = max(1024, max_bytes) - self._lock = threading.Lock() - self._sequence = 0 - self._claimed: set[Path] = set() - self.evicted_count = 0 - self.quarantined_count = 0 - self._quarantine = _safe_root(self.root / "corrupt") - - def append(self, event: dict[str, Any]) -> Path: - payload = json.dumps(event, ensure_ascii=False, separators=(",", ":"), sort_keys=True).encode("utf-8") - if len(payload) > self.max_bytes: - raise ValueError("event exceeds spool limit") - with self._lock: - self._sequence += 1 - target = self.root / ( - f"{time.time_ns():020d}-{os.getpid()}-{threading.get_ident()}-{self._sequence:08d}.json" - ) - self._write_payload_locked(target, payload) - self._trim_locked(protected=target) - files = self._files_locked() - total = sum(path.stat(follow_symlinks=False).st_size for path in files) - if len(files) > self.max_items or total > self.max_bytes: - try: - target.unlink() - _fsync_directory(self.root) - except FileNotFoundError: - pass - raise ValueError("spool capacity unavailable") - return target - - def oldest(self) -> Path | None: - with self._lock: - files = [path for path in self._files_locked() if path not in self._claimed] - return files[0] if files else None - - def claim_oldest(self) -> Path | None: - """Reserve the oldest event so capacity trimming cannot remove it in flight.""" - with self._lock: - files = [path for path in self._files_locked() if path not in self._claimed] - if not files: - return None - path = files[0] - self._claimed.add(path) - return path - - def release(self, path: Path) -> None: - with self._lock: - self._claimed.discard(_safe_child(self.root, path)) - - def load(self, path: Path) -> dict[str, Any]: - safe = _safe_child(self.root, path) - descriptor = self._open_readonly(safe) - with os.fdopen(descriptor, "rb") as stream: - raw = stream.read(self.max_bytes + 1) - if len(raw) > self.max_bytes: - raise ValueError("spooled event exceeds limit") - try: - value = json.loads(raw.decode("utf-8", errors="strict")) - except (UnicodeDecodeError, json.JSONDecodeError): - raise ValueError("corrupt spooled event") from None - if not isinstance(value, dict): - raise ValueError("invalid spooled event") - return value - - def remove(self, path: Path) -> None: - with self._lock: - safe = _safe_child(self.root, path) - self._claimed.discard(safe) - try: - safe.unlink() - _fsync_directory(self.root) - except FileNotFoundError: - pass - - def quarantine(self, path: Path) -> None: - """Move corrupt data aside without parsing or exposing its contents.""" - with self._lock: - safe = _safe_child(self.root, path) - self._claimed.discard(safe) - if not safe.exists() or safe.is_symlink(): - return - destination = self._quarantine / f"{safe.stem}-{uuid.uuid4().hex}.bad" - os.replace(safe, destination) - _chmod_private(destination, 0o600) - self.quarantined_count += 1 - self._trim_quarantine_locked() - _fsync_directory(self.root) - _fsync_directory(self._quarantine) - - def __len__(self) -> int: - with self._lock: - return len(self._files_locked()) - - def _write_payload_locked(self, target: Path, payload: bytes) -> None: - target = _safe_child(self.root, target) - temporary = self.root / f".{target.name}.{uuid.uuid4().hex}.tmp" - flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL - if hasattr(os, "O_NOFOLLOW"): - flags |= os.O_NOFOLLOW - descriptor = os.open(temporary, flags, 0o600) - try: - with os.fdopen(descriptor, "wb") as stream: - stream.write(payload) - stream.flush() - os.fsync(stream.fileno()) - _chmod_private(temporary, 0o600) - os.replace(temporary, target) - _chmod_private(target, 0o600) - _fsync_directory(self.root) - finally: - try: - temporary.unlink() - except FileNotFoundError: - pass - - @staticmethod - def _open_readonly(path: Path) -> int: - flags = os.O_RDONLY - if hasattr(os, "O_NOFOLLOW"): - flags |= os.O_NOFOLLOW - descriptor = os.open(path, flags) - info = os.fstat(descriptor) - if not stat.S_ISREG(info.st_mode): - os.close(descriptor) - raise ValueError("spool path is not a regular file") - return descriptor - - def _files_locked(self) -> list[Path]: - files: list[Path] = [] - for item in self.root.iterdir(): - if item.name.endswith(".json") and item.is_file() and not item.is_symlink(): - files.append(item) - return sorted(files, key=lambda item: item.name) - - def _trim_locked(self, *, protected: Path | None = None) -> None: - protected_paths = set(self._claimed) - if protected is not None: - protected_paths.add(protected) - files = self._files_locked() - sizes: dict[Path, int] = {} - for path in files: - try: - sizes[path] = path.stat(follow_symlinks=False).st_size - except FileNotFoundError: - sizes[path] = 0 - total = sum(sizes.values()) - changed = False - while files and (len(files) > self.max_items or total > self.max_bytes): - victim = next((path for path in files if path not in protected_paths), None) - if victim is None: - break - files.remove(victim) - try: - victim.unlink() - total -= sizes[victim] - self.evicted_count += 1 - changed = True - except FileNotFoundError: - continue - if changed: - _fsync_directory(self.root) - - def _trim_quarantine_locked(self) -> None: - files = sorted( - ( - item - for item in self._quarantine.iterdir() - if item.is_file() and not item.is_symlink() and item.name.endswith(".bad") - ), - key=lambda item: item.name, - ) - sizes: dict[Path, int] = {} - for item in files: - try: - sizes[item] = item.stat(follow_symlinks=False).st_size - except FileNotFoundError: - sizes[item] = 0 - total = sum(sizes.values()) - while files and (len(files) > self.max_items or total > self.max_bytes): - victim = files.pop(0) - try: - victim.unlink() - total -= sizes[victim] - except FileNotFoundError: - continue diff --git a/src/substrate_wiki/supervisor.py b/src/substrate_wiki/supervisor.py deleted file mode 100644 index b31f2b9..0000000 --- a/src/substrate_wiki/supervisor.py +++ /dev/null @@ -1,181 +0,0 @@ -"""Profile-scoped systemd service naming and control without secret arguments.""" - -from __future__ import annotations - -import argparse -import hashlib -import json -import os -import secrets -import time -import subprocess -import sys -from pathlib import Path - - -def home_hash(hermes_home: Path) -> str: - return hashlib.sha256(os.fspath(hermes_home.resolve()).encode()).hexdigest()[:12] - - -def unit_template_name(hermes_home: Path) -> str: - return f"substrate-wiki-import-{home_hash(hermes_home)}@.service" - - -def unit_instance_name(hermes_home: Path, job_id: str) -> str: - return f"substrate-wiki-import-{home_hash(hermes_home)}@{job_id}.service" - - -def _systemctl(*arguments: str, check: bool = True) -> subprocess.CompletedProcess[str]: - if os.name != "posix": - raise RuntimeError("systemd user services require Linux") - return subprocess.run( - ("systemctl", "--user", *arguments), - check=check, - stdin=subprocess.DEVNULL, - capture_output=True, - text=True, - timeout=30, - ) - - -def _runtime_path(hermes_home: Path, job_id: str) -> Path: - return hermes_home / "substrate_wiki" / "imports" / "jobs" / job_id / "worker.json" - - -def _systemd_unit_installed(hermes_home: Path) -> bool: - if os.name != "posix": - return False - unit = Path.home() / ".config" / "systemd" / "user" / unit_template_name(hermes_home) - return unit.is_file() and not unit.is_symlink() - - -def _pid_alive(pid: int) -> bool: - if pid <= 0: - return False - try: - os.kill(pid, 0) - except (OSError, ValueError): - return False - return True - - -def _portable_process_matches(pid: int, nonce: str) -> bool: - if not _pid_alive(pid) or not nonce: - return False - if sys.platform.startswith("linux"): - try: - command = Path(f"/proc/{pid}/cmdline").read_bytes().split(b"\0") - except OSError: - return False - return nonce.encode("ascii") in command - # Other platforms cannot safely prove PID ownership with the stdlib. - return False - - -def _portable_status(hermes_home: Path, job_id: str) -> dict[str, object]: - path = _runtime_path(hermes_home, job_id) - if path.is_symlink(): - return {} - try: - value = json.loads(path.read_text(encoding="utf-8")) - except (FileNotFoundError, OSError, UnicodeError, json.JSONDecodeError): - return {} - return value if isinstance(value, dict) else {} - - -def start_service(hermes_home: Path, job_id: str) -> str: - """Start once through systemd when installed, otherwise a detached worker. - - The portable path uses no shell and places no credential in argv or its - content-free runtime record. The durable checkpoint remains authoritative. - """ - if _systemd_unit_installed(hermes_home): - unit = unit_instance_name(hermes_home, job_id) - _systemctl("daemon-reload") - _systemctl("enable", "--now", unit) - return unit - previous = _portable_status(hermes_home, job_id) - try: - previous_pid = int(previous.get("pid", 0)) - except (TypeError, ValueError): - previous_pid = 0 - previous_nonce = str(previous.get("nonce", "")) - if _portable_process_matches(previous_pid, previous_nonce): - return f"process:{previous_pid}" - supervisor = Path(__file__).resolve() - nonce = secrets.token_hex(16) - command = (sys.executable, os.fspath(supervisor), "--hermes-home", - os.fspath(hermes_home.resolve()), "--job-id", job_id, - "--runtime-nonce", nonce) - kwargs: dict[str, object] = { - "stdin": subprocess.DEVNULL, "stdout": subprocess.DEVNULL, - "stderr": subprocess.DEVNULL, "close_fds": True, - } - if os.name == "nt": - kwargs["creationflags"] = int(getattr(subprocess, "CREATE_NEW_PROCESS_GROUP", 0)) | int(getattr(subprocess, "DETACHED_PROCESS", 0)) - else: - kwargs["start_new_session"] = True - process = subprocess.Popen(command, **kwargs) - path = _runtime_path(hermes_home, job_id) - path.parent.mkdir(parents=True, exist_ok=True, mode=0o700) - from .spool import secure_atomic_json_write - secure_atomic_json_write(path, {"pid": process.pid, "nonce": nonce, - "started_at": time.time(), - "restart_count": int(previous.get("restart_count", 0))}) - return f"process:{process.pid}" - - -def stop_service(hermes_home: Path, job_id: str) -> None: - if _systemd_unit_installed(hermes_home): - _systemctl("disable", "--now", unit_instance_name(hermes_home, job_id), check=False) - return - value = _portable_status(hermes_home, job_id) - try: - pid = int(value.get("pid", 0)) - except (TypeError, ValueError): - return - if _portable_process_matches(pid, str(value.get("nonce", ""))): - try: - os.kill(pid, 15) - except OSError: - pass - - -def service_restart_count(hermes_home: Path, job_id: str) -> int: - if _systemd_unit_installed(hermes_home): - result = _systemctl( - "show", unit_instance_name(hermes_home, job_id), "--property=NRestarts", "--value", - check=False, - ) - try: - return max(0, int(result.stdout.strip())) - except ValueError: - return 0 - try: - return max(0, int(_portable_status(hermes_home, job_id).get("restart_count", 0))) - except (TypeError, ValueError): - return 0 - - -def bootstrap_package() -> None: - """Allow direct execution from the installed package without PYTHONPATH changes.""" - package_parent = Path(__file__).resolve().parent.parent - if os.fspath(package_parent) not in sys.path: - sys.path.insert(0, os.fspath(package_parent)) - - -def main(argv: list[str] | None = None) -> int: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--hermes-home", required=True) - parser.add_argument("--job-id", required=True) - parser.add_argument("--runtime-nonce", required=False, default="") - args = parser.parse_args(argv) - bootstrap_package() - from substrate_wiki.worker import run_job - - run_job(Path(args.hermes_home).resolve(), args.job_id) - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/src/substrate_wiki/worker.py b/src/substrate_wiki/worker.py deleted file mode 100644 index 45e864b..0000000 --- a/src/substrate_wiki/worker.py +++ /dev/null @@ -1,138 +0,0 @@ -"""Dedicated history-import worker entry point used by the systemd user unit.""" - -from __future__ import annotations - -import argparse -import os -import signal -import sqlite3 -import subprocess -import threading -from pathlib import Path -from types import FrameType -from typing import Any - -from .checkpoint import TERMINAL_STATES, ImportCheckpoint -from .client import SubstrateClient, validate_capabilities -from .history import ( - ConversationReplaySource, - HermesHistoryImporter, - HermesJSONLHistorySource, - HermesSQLiteHistorySource, - remove_official_export_spill, -) - - -def checkpoint_path(hermes_home: Path, job_id: str) -> Path: - if not job_id or not job_id.isalnum() or len(job_id) > 128: - raise ValueError("invalid job id") - return hermes_home / "substrate_wiki" / "imports" / "jobs" / job_id / "checkpoint.db" - - -def _install_stop_handlers( - stop_event: threading.Event, -) -> dict[signal.Signals, Any]: - """Turn systemd stop into a durable acknowledgement-boundary pause.""" - if os.name != "posix" or threading.current_thread() is not threading.main_thread(): - return {} - previous: dict[signal.Signals, Any] = {} - - def request_stop(_signum: int, _frame: FrameType | None) -> None: - stop_event.set() - - for selected in (signal.SIGTERM, signal.SIGINT): - previous[selected] = signal.getsignal(selected) - signal.signal(selected, request_stop) - return previous - - -def _restore_stop_handlers(previous: dict[signal.Signals, Any]) -> None: - for selected, handler in previous.items(): - signal.signal(selected, handler) - - -def run_job(hermes_home: Path, job_id: str) -> dict[str, object]: - checkpoint = ImportCheckpoint(checkpoint_path(hermes_home, job_id)) - stop_event = threading.Event() - previous_handlers = _install_stop_handlers(stop_event) - source_kind = "" - locator = Path() - try: - job = checkpoint.job() - source_kind = str(job["source_kind"]) - locator = Path(str(job["source_locator"])) - source: ConversationReplaySource - if source_kind == "hermes-sqlite": - source = HermesSQLiteHistorySource(locator) - elif source_kind in {"hermes-jsonl", "hermes-official-export"}: - source = HermesJSONLHistorySource(locator) - else: - raise ValueError("unsupported durable source kind") - client = SubstrateClient.from_env(timeout=30.0, hermes_home=hermes_home, hosted_default=True) - importer = HermesHistoryImporter( - hermes_home=hermes_home, - client=client, - source=source, - agent_id=str(job["agent_id"]), - stop_requested=stop_event.is_set, - ) - importer.checkpoint = checkpoint - capabilities = client.capabilities() - try: - validate_capabilities(capabilities, require_replay=True, require_entity=False) - except Exception: - checkpoint.set_state("failed", error_class="server_upgrade_required") - raise RuntimeError("server_upgrade_required") from None - status = importer.run(wait=True) - if status.get("complete"): - from .supervisor import _systemd_unit_installed, unit_instance_name - - if not _systemd_unit_installed(hermes_home): - return status - subprocess.run( - ("systemctl", "--user", "disable", unit_instance_name(hermes_home, job_id)), - check=False, - stdin=subprocess.DEVNULL, - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, - timeout=30, - ) - return status - finally: - _restore_stop_handlers(previous_handlers) - # The official export is the durable replay source, not disposable - # scratch space while a job can still be resumed. In particular, a - # systemd SIGTERM is converted into a pause at an acknowledgement - # boundary; deleting the spill here would leave the unchanged - # checkpoint pointing at a missing source on restart. Remove it only - # after a genuinely terminal completion/cancellation. Failed jobs are - # intentionally resumable and therefore retain the source as well. - terminal = False - try: - terminal = str(checkpoint.job().get("state") or "") in TERMINAL_STATES - except (RuntimeError, sqlite3.Error): - terminal = False - if source_kind == "hermes-official-export" and terminal: - remove_official_export_spill( - hermes_home, - source_kind=source_kind, - source_locator=os.fspath(locator), - ) - checkpoint.close() - - -def main(argv: list[str] | None = None) -> int: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--hermes-home", type=Path, required=True) - parser.add_argument("--job-id", required=True) - args = parser.parse_args(argv) - try: - run_job(args.hermes_home.resolve(), args.job_id) - except Exception: # noqa: BLE001 - systemd receives only an exit code - return 1 - return 0 - - -if __name__ == "__main__": - os.umask(0o077) - raise SystemExit(main()) diff --git a/tests/fixtures/credential_redaction_vectors.json b/tests/fixtures/credential_redaction_vectors.json deleted file mode 100644 index 37fab6e..0000000 --- a/tests/fixtures/credential_redaction_vectors.json +++ /dev/null @@ -1,217 +0,0 @@ -[ - { - "name": "bearer_header", - "text": "Authorization: Bearer QWxwaGEuQmV0YS5HYW1tYQ==", - "secret_fragment": "QWxwaGEuQmV0YS5HYW1tYQ==" - }, - { - "name": "basic_header", - "text": "proxy-authorization=Basic dXNlcjpwYXNzd29yZA==", - "secret_fragment": "dXNlcjpwYXNzd29yZA==" - }, - { - "name": "environment_assignment", - "text": "HERMES_API_KEY=opaqueExampleValue_987654321", - "secret_fragment": "opaqueExampleValue_987654321" - }, - { - "name": "json_assignment", - "text": "{\"client_secret\":\"jsonExampleSecret_987654321\"}", - "secret_fragment": "jsonExampleSecret_987654321" - }, - { - "name": "natural_assignment", - "text": "api key is naturalExampleSecret_987654321", - "secret_fragment": "naturalExampleSecret_987654321" - }, - { - "name": "access_key_assignment", - "text": "access_key=accessKeyExampleValue_987654321", - "secret_fragment": "accessKeyExampleValue_987654321" - }, - { - "name": "passphrase_assignment", - "text": "passphrase is passphraseExampleValue_987654321", - "secret_fragment": "passphraseExampleValue_987654321" - }, - { - "name": "bearer_token_assignment", - "text": "bearer_token: bearerTokenExampleValue_987654321", - "secret_fragment": "bearerTokenExampleValue_987654321" - }, - { - "name": "authentication_token_assignment", - "text": "authentication token is authenticationTokenExampleValue_987654321", - "secret_fragment": "authenticationTokenExampleValue_987654321" - }, - { - "name": "connection_string_assignment", - "text": "connection_string=Endpoint=https://safe.invalid/;AccountKey=connectionKeyExampleValue_987654321", - "secret_fragments": [ - "Endpoint=https://safe.invalid/", - "connectionKeyExampleValue_987654321" - ] - }, - { - "name": "url_query", - "text": "https://example.invalid/callback?access_token=queryExample_987654321&mode=safe", - "secret_fragment": "queryExample_987654321" - }, - { - "name": "url_userinfo", - "text": "https://example-user:urlExamplePassword_987654321@example.invalid/", - "secret_fragments": [ - "example-user", - "urlExamplePassword_987654321" - ] - }, - { - "name": "postgres_userinfo", - "text": "postgresql+asyncpg://db-user:dbExamplePassword_987654321@database.invalid/app", - "secret_fragments": [ - "db-user", - "dbExamplePassword_987654321" - ] - }, - { - "name": "redis_userinfo", - "text": "rediss://:redisExamplePassword_987654321@cache.invalid/0", - "secret_fragment": "redisExamplePassword_987654321" - }, - { - "name": "amqp_userinfo", - "text": "amqps://queue-user:amqpExamplePassword_987654321@broker.invalid/vhost", - "secret_fragments": [ - "queue-user", - "amqpExamplePassword_987654321" - ] - }, - { - "name": "ftp_userinfo", - "text": "ftps://file-user:ftpExamplePassword_987654321@files.invalid/incoming", - "secret_fragments": [ - "file-user", - "ftpExamplePassword_987654321" - ] - }, - { - "name": "ssh_userinfo", - "text": "ssh://deploy-user:sshExamplePassword_987654321@host.invalid:22/", - "secret_fragments": [ - "deploy-user", - "sshExamplePassword_987654321" - ] - }, - { - "name": "azure_sas_signature", - "text": "https://blob.invalid/item?sv=2023-11-03&sp=r&se=2099-01-01T00%3A00%3A00Z&sig=azureExampleSignature%2B987654321%3D", - "secret_fragment": "azureExampleSignature%2B987654321%3D" - }, - { - "name": "azure_sas_signature_before_companion", - "text": "https://blob.invalid/item?sig=azureLeadingSignature%2B987654321%3D&sv=2023-11-03&sp=r", - "secret_fragment": "azureLeadingSignature%2B987654321%3D" - }, - { - "name": "azure_sas_connection_string", - "text": "SharedAccessSignature=sv=2023-11-03&sp=r&sig=azureConnectionSignature%2B987654321%3D", - "secret_fragment": "azureConnectionSignature%2B987654321%3D" - }, - { - "name": "aws_signed_url_components", - "text": "https://bucket.invalid/item?X-Amz-Credential=AKIAEXAMPLEONLY1234%2F20990101%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Security-Token=awsTemporaryToken_987654321&X-Amz-Signature=0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", - "secret_fragments": [ - "AKIAEXAMPLEONLY1234%2F20990101%2Fus-east-1%2Fs3%2Faws4_request", - "awsTemporaryToken_987654321", - "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef" - ] - }, - { - "name": "google_signed_url_components", - "text": "https://storage.invalid/item?X-Goog-Credential=service-account%40example.invalid%2F20990101%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Signature=abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789", - "secret_fragments": [ - "service-account%40example.invalid%2F20990101%2Fauto%2Fstorage%2Fgoog4_request", - "abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789" - ] - }, - { - "name": "google_v2_signed_url_components", - "text": "https://storage.invalid/item?GoogleAccessId=service-account%40example.invalid&Signature=googleV2ExampleSignature%2B987654321%3D&Expires=4070908800", - "secret_fragments": [ - "service-account%40example.invalid", - "googleV2ExampleSignature%2B987654321%3D" - ] - }, - { - "name": "aws_v2_signed_url_components", - "text": "https://download.invalid/item?AWSAccessKeyId=AKIAEXAMPLEONLY5678&Signature=awsV2ExampleSignature%2B987654321%3D&Expires=4070908800", - "secret_fragments": [ - "AKIAEXAMPLEONLY5678", - "awsV2ExampleSignature%2B987654321%3D" - ] - }, - { - "name": "html_encoded_signed_url_components", - "text": "https://bucket.invalid/item?X-Amz-Credential=AKIAHTMLONLY123456%2Fscope&X-Amz-Signature=htmlEncodedSignature987654321", - "secret_fragments": [ - "AKIAHTMLONLY123456%2Fscope", - "htmlEncodedSignature987654321" - ] - }, - { - "name": "generic_signed_url_signature", - "text": "https://download.invalid/item?expires=4070908800&signature=genericExampleSignature_987654321", - "secret_fragment": "genericExampleSignature_987654321" - }, - { - "name": "aws_signed_authorization", - "text": "Authorization: AWS4-HMAC-SHA256 Credential=AKIAHEADERONLY1234/20990101/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-date, Signature=abcdef0123456789abcdef0123456789", - "secret_fragments": [ - "AKIAHEADERONLY1234/20990101/us-east-1/s3/aws4_request", - "abcdef0123456789abcdef0123456789" - ] - }, - { - "name": "azure_signed_authorization", - "text": "Authorization: SharedKey example-account:YWJjZGVmZ2hpamtsbW5vcHFyc3R1dnd4eXo=", - "secret_fragments": [ - "example-account", - "YWJjZGVmZ2hpamtsbW5vcHFyc3R1dnd4eXo=" - ] - }, - { - "name": "provider_token", - "text": "sk-live-providerExampleToken987654321", - "secret_fragment": "providerExampleToken987654321" - }, - { - "name": "jwt", - "text": "eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiJleGFtcGxlIn0.c2lnbmF0dXJlRXhhbXBsZQ", - "secret_fragment": "eyJzdWIiOiJleGFtcGxlIn0" - }, - { - "name": "private_key", - "text": "-----BEGIN PRIVATE KEY-----\nZXhhbXBsZS1ub3QtYS1yZWFsLWtleQ==\n-----END PRIVATE KEY-----", - "secret_fragment": "ZXhhbXBsZS1ub3QtYS1yZWFsLWtleQ==" - }, - { - "name": "partial_openssh_private_key", - "text": "-----BEGIN OPENSSH PRIVATE KEY-----\nb3BlbnNzaC1rZXktdjEAAAAAexampleonly987654321", - "secret_fragment": "b3BlbnNzaC1rZXktdjEAAAAAexampleonly987654321" - }, - { - "name": "orphan_encrypted_private_key_footer", - "text": "TUlJRXhhbXBsZU9ubHlQcml2YXRlS2V5OTg3NjU0MzIx\n-----END ENCRYPTED PRIVATE KEY-----", - "secret_fragment": "TUlJRXhhbXBsZU9ubHlQcml2YXRlS2V5OTg3NjU0MzIx" - }, - { - "name": "standalone_private_key_footer", - "text": "orphan marker: -----END RSA PRIVATE KEY-----", - "secret_fragment": "-----END RSA PRIVATE KEY-----" - }, - { - "name": "adjacent_private_key_label", - "text": "encrypted private key: TUlJRXhhbXBsZU9ubHlBZGphY2VudEtleTk4NzY1NDMyMQ==", - "secret_fragment": "TUlJRXhhbXBsZU9ubHlBZGphY2VudEtleTk4NzY1NDMyMQ==" - } -] diff --git a/tests/fixtures/public-plugin-secret-sentinels.json b/tests/fixtures/public-plugin-secret-sentinels.json deleted file mode 100644 index 472e50f..0000000 --- a/tests/fixtures/public-plugin-secret-sentinels.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "purpose": "Synthetic adversarial redaction/publication-scanner fixture; never use as a credential", - "api_key": "SUBSTRATE_SYNTHETIC_SECRET_DO_NOT_USE_000000000000000000000001", - "authorization": "Bearer SUBSTRATE_SYNTHETIC_SECRET_DO_NOT_USE_000000000000000000000002", - "endpoint": "https://synthetic-memory.invalid" -} diff --git a/tests/test_entity_memory.py b/tests/test_entity_memory.py deleted file mode 100644 index 7d6de5f..0000000 --- a/tests/test_entity_memory.py +++ /dev/null @@ -1,256 +0,0 @@ -from __future__ import annotations - -import sys -from pathlib import Path -from typing import Any - -import pytest - -PLUGIN_ROOT = Path(__file__).parents[1] / "src" -sys.path.insert(0, str(PLUGIN_ROOT)) - -from substrate_wiki import SubstrateWikiProvider # noqa: E402 -from substrate_wiki.client import SubstrateAPIError, SubstrateClient # noqa: E402 - -CAPABILITIES = { - "provider": "substrate_wiki", - "server_commit": "b" * 40, - "capture_schema_versions": [2], - "max_event_bytes": 262_144, - "history_replay": { - "protocol": "stream-v2", - "min_plugin_version": "1.2.0", - "content_free_completion": True, - "incremental_windows": True, - "status_version": 2, - }, - "entity_memory": { - "protocol": "entity-wiki-v1", - "min_plugin_version": "1.3.0", - "search_endpoint": "/api/v1/hermes/memory/search", - "canonical_wiki_pages": True, - "entity_page_type": "entity", - }, - "entity_quality": { - "protocol": "entity-quality-v2", - "min_plugin_version": "1.4.0", - "memory_card": True, - "quality_version": 2, - "canonical_redirects": True, - }, -} - - -def test_client_requires_semantic_capability_then_uses_entity_memory_endpoint( - monkeypatch: pytest.MonkeyPatch, -) -> None: - calls: list[tuple[str, str, dict[str, Any]]] = [] - - def request(self: SubstrateClient, method: str, path: str, **kwargs: Any) -> dict[str, Any]: - calls.append((method, path, kwargs)) - if path.endswith("/capabilities"): - return CAPABILITIES - return { - "results": [ - { - "path": "entities/project/substrate--abc12345.md", - "canonical_path": "entities/project/substrate--abc12345.md", - "page_type": "entity", - "entity_id": "entity-substrate", - "entity_type": "project", - "quality_version": 2, - "memory_card": "Substrate project profile", - "content": "full page content must not enter automatic recall", - } - ] - } - - monkeypatch.setattr(SubstrateClient, "request", request) - client = SubstrateClient("https://wiki.example.test", "key") - - first = client.memory_search("substrate", scope={"platform": "cli"}) - second = client.memory_search("project", scope={"platform": "cli"}) - - assert first["results"][0]["path"].startswith("entities/") - assert first["results"][0]["memory_card"] == "Substrate project profile" - assert second["results"] - assert [path for _, path, _ in calls] == [ - "/api/v1/hermes/capabilities", - "/api/v1/hermes/memory/search", - "/api/v1/hermes/memory/search", - ] - assert calls[1][0] == "POST" - assert calls[1][2]["body"]["platform"] == "cli" - assert calls[1][2]["body"]["q"] == "substrate" - - -def test_capability_shaping_preserves_only_the_content_free_server_commit() -> None: - shaped = SubstrateClient._shape_response("/api/v1/hermes/capabilities", CAPABILITIES) - - assert shaped["server_commit"] == "b" * 40 - - -def test_memory_response_shaping_drops_full_page_content() -> None: - shaped = SubstrateClient._shape_response( - "/api/v1/hermes/memory/search", - { - "results": [ - { - "path": "entities/project/substrate--abc12345.md", - "canonical_path": "entities/project/substrate--abc12345.md", - "page_type": "entity", - "entity_id": "entity-substrate", - "entity_type": "project", - "quality_version": 2, - "memory_card": "Substrate project profile", - "content": "full page content must not enter automatic recall", - "snippet": "full body snippet must not enter automatic recall", - } - ] - }, - ) - - result = shaped["results"][0] - assert result["memory_card"] == "Substrate project profile" - assert "content" not in result - assert "snippet" not in result - - -def test_client_rejects_old_server_for_automatic_entity_recall( - monkeypatch: pytest.MonkeyPatch, -) -> None: - monkeypatch.setattr( - SubstrateClient, - "request", - lambda self, method, path, **kwargs: { - key: value for key, value in CAPABILITIES.items() if key != "entity_memory" - }, - ) - client = SubstrateClient("https://wiki.example.test", "key") - with pytest.raises(SubstrateAPIError, match="server_upgrade_required"): - client.memory_search("anything") - - -def test_client_rejects_server_without_entity_quality_v2( - monkeypatch: pytest.MonkeyPatch, -) -> None: - monkeypatch.setattr( - SubstrateClient, - "request", - lambda self, method, path, **kwargs: { - key: value for key, value in CAPABILITIES.items() if key != "entity_quality" - }, - ) - with pytest.raises(SubstrateAPIError, match="server_upgrade_required"): - SubstrateClient("https://wiki.example.test", "key").memory_search("anything") - - -@pytest.mark.parametrize( - ("field", "value"), - [ - ("protocol", "entity-quality-v1"), - ("min_plugin_version", "2.0.1"), - ("memory_card", False), - ("quality_version", 1), - ("canonical_redirects", False), - ], -) -def test_client_rejects_downgraded_entity_quality_contract( - field: str, - value: Any, - monkeypatch: pytest.MonkeyPatch, -) -> None: - capabilities = dict(CAPABILITIES) - capabilities["entity_quality"] = { - **CAPABILITIES["entity_quality"], - field: value, - } - monkeypatch.setattr( - SubstrateClient, - "request", - lambda self, method, path, **kwargs: capabilities, - ) - - with pytest.raises(SubstrateAPIError, match="server_upgrade_required"): - SubstrateClient("https://wiki.example.test", "key").memory_search("anything") - - -def test_client_accepts_exact_current_plugin_minimum(monkeypatch: pytest.MonkeyPatch) -> None: - capabilities = dict(CAPABILITIES) - capabilities["entity_quality"] = { - **CAPABILITIES["entity_quality"], - "min_plugin_version": "1.5.0", - } - monkeypatch.setattr( - SubstrateClient, - "request", - lambda self, method, path, **kwargs: capabilities, - ) - - SubstrateClient("https://wiki.example.test", "key").require_entity_wiki_capability() - - -def test_client_rejects_entity_capability_from_wrong_provider( - monkeypatch: pytest.MonkeyPatch, -) -> None: - monkeypatch.setattr( - SubstrateClient, - "request", - lambda self, method, path, **kwargs: {**CAPABILITIES, "provider": "other"}, - ) - with pytest.raises(SubstrateAPIError, match="server_upgrade_required"): - SubstrateClient("https://wiki.example.test", "key").memory_search("anything") - - -def test_semantic_gate_accepts_lower_minimum_and_retries_after_server_upgrade( - monkeypatch: pytest.MonkeyPatch, -) -> None: - calls = 0 - - def request(self: SubstrateClient, method: str, path: str, **kwargs: Any) -> dict[str, Any]: - nonlocal calls - if path.endswith("/capabilities"): - calls += 1 - if calls == 1: - return {key: value for key, value in CAPABILITIES.items() if key != "entity_memory"} - upgraded = dict(CAPABILITIES) - upgraded["entity_memory"] = { - **CAPABILITIES["entity_memory"], - "min_plugin_version": "1.2.9", - } - return upgraded - return {"results": []} - - monkeypatch.setattr(SubstrateClient, "request", request) - client = SubstrateClient("https://wiki.example.test", "key") - with pytest.raises(SubstrateAPIError, match="server_upgrade_required"): - client.memory_search("before") - assert client.memory_search("after") == {"results": []} - assert calls == 2 - - -@pytest.mark.parametrize("minimum", ["1.3", "1.3.0-beta", "01.3.0", "01.x.0", "1.3.0.0"]) -def test_semantic_gate_rejects_non_strict_versions( - minimum: str, - monkeypatch: pytest.MonkeyPatch, -) -> None: - capabilities = dict(CAPABILITIES) - capabilities["entity_memory"] = { - **CAPABILITIES["entity_memory"], - "min_plugin_version": minimum, - } - monkeypatch.setattr( - SubstrateClient, - "request", - lambda self, method, path, **kwargs: capabilities, - ) - with pytest.raises(SubstrateAPIError, match="server_upgrade_required"): - SubstrateClient("https://wiki.example.test", "key").memory_search("anything") - - -def test_v141_manifest_and_prompt_describe_one_published_memory() -> None: - manifest = (PLUGIN_ROOT / "substrate_wiki" / "plugin.yaml").read_text(encoding="utf-8") - prompt = SubstrateWikiProvider().system_prompt_block() - assert "version: 2.0.5" in manifest - assert "single published memory" in prompt - assert "canonical published entity" in prompt diff --git a/tests/test_hardening.py b/tests/test_hardening.py deleted file mode 100644 index f7753d2..0000000 --- a/tests/test_hardening.py +++ /dev/null @@ -1,792 +0,0 @@ -from __future__ import annotations - -import json -import os -import queue -import stat -import sys -import threading -import time -from collections import OrderedDict -from pathlib import Path -from typing import Any - -import pytest - -PLUGIN_ROOT = Path(__file__).parents[1] / "src" -sys.path.insert(0, str(PLUGIN_ROOT)) - -from substrate_wiki import SubstrateWikiProvider # noqa: E402 -from substrate_wiki.client import SubstrateAPIError, SubstrateClient # noqa: E402 -from substrate_wiki.spool import DurableSpool # noqa: E402 - - -class FakeClient: - def __init__(self) -> None: - self.searches: list[tuple[str, int]] = [] - self.memory_searches: list[tuple[str, int, dict[str, Any]]] = [] - self.queries: list[tuple[str, bool]] = [] - self.delivered: list[dict[str, Any]] = [] - self.fail = False - self.fail_category = "transport_error" - self.request_started = threading.Event() - self.block = threading.Event() - self.should_block = False - self.block_timeout = 2.0 - - def search(self, query: str, *, limit: int = 8) -> dict[str, Any]: - self.searches.append((query, limit)) - return {"results": [{"text": "cached", "citation": "topics/cached.md"}]} - - def memory_search( - self, query: str, *, limit: int = 8, scope: dict[str, Any] | None = None - ) -> dict[str, Any]: - self.memory_searches.append((query, limit, scope or {})) - return { - "results": [ - { - "memory_card": "cached", - "path": "entities/project/cached--a1b2c3d4.md", - "canonical_path": "entities/project/cached--a1b2c3d4.md", - "page_type": "entity", - "entity_id": "entity-cached", - "entity_type": "project", - "quality_version": 2, - } - ] - } - - def read_page(self, path: str) -> dict[str, Any]: - return {"path": path} - - def query_wiki(self, question: str, *, save_as_synthesis: bool = False) -> dict[str, Any]: - self.queries.append((question, save_as_synthesis)) - return {"answer": question, "citations": ["topics/a.md"]} - - def ingest(self, content: str, **kwargs: Any) -> dict[str, Any]: - return {"job_id": "job-1"} - - def job_status(self, job_id: str) -> dict[str, Any]: - return {"job_id": job_id, "status": "done"} - - def request(self, method: str, path: str, **kwargs: Any) -> dict[str, Any]: - self.request_started.set() - if self.should_block: - self.block.wait(self.block_timeout) - if self.fail: - raise SubstrateAPIError(self.fail_category) - self.delivered.append({"method": method, "path": path, **kwargs}) - return {} - - -def wait_until(predicate: Any, timeout: float = 3.0) -> None: - deadline = time.monotonic() + timeout - while time.monotonic() < deadline: - if predicate(): - return - time.sleep(0.01) - raise AssertionError("condition was not reached") - - -@pytest.fixture -def provider( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> tuple[SubstrateWikiProvider, FakeClient]: - monkeypatch.setenv("HERMES_API_URL", "https://wiki.example.test") - monkeypatch.setenv("HERMES_API_KEY", "secret-value") - fake = FakeClient() - monkeypatch.setattr("substrate_wiki.SubstrateClient.from_env", lambda **kwargs: fake) - instance = SubstrateWikiProvider() - instance.initialize("session-a", hermes_home=str(tmp_path)) - yield instance, fake - instance.shutdown() - - -def test_v0182_identity_direct_schemas_and_json_string_results( - provider: tuple[SubstrateWikiProvider, FakeClient], -) -> None: - instance, _ = provider - assert instance.name == "substrate_wiki" - schemas = instance.get_tool_schemas() - assert {schema["name"] for schema in schemas} == { - "wiki_search", - "wiki_read", - "wiki_query", - "wiki_ingest", - "wiki_job_status", - } - assert all("function" not in schema for schema in schemas) - result = instance.handle_tool_call("wiki_search", {"query": "x", "limit": 99}) - assert isinstance(result, str) - assert json.loads(result)["results"][0]["citation"] == "topics/cached.md" - assert json.loads(instance.handle_tool_call("wiki_read", {})) == {"error": "invalid_arguments"} - - -def test_config_persists_url_but_not_key_and_environment_url_wins( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - instance = SubstrateWikiProvider() - instance.save_config( - {"api_url": "https://config.example.test", "api_key": "must-not-persist"}, str(tmp_path) - ) - config = tmp_path / "substrate_wiki" / "config.json" - raw = config.read_text(encoding="utf-8") - assert "https://app.trysubstrate.co" in raw - assert "must-not-persist" not in raw - monkeypatch.setenv("HERMES_API_URL", "https://env.example.test") - monkeypatch.setenv("HERMES_API_KEY", "environment-only") - with pytest.raises(SubstrateAPIError, match="unsafe_hosted_origin_override"): - instance.initialize("s", hermes_home=str(tmp_path)) - - -def test_prefetch_is_cache_only_and_queue_uses_one_worker( - provider: tuple[SubstrateWikiProvider, FakeClient], -) -> None: - instance, fake = provider - worker = instance._prefetch_worker - assert instance.prefetch("topic", session_id="s") == "" - assert fake.memory_searches == [] - instance.queue_prefetch("topic", session_id="session-a") - wait_until(lambda: len(fake.memory_searches) == 1) - cached = instance.prefetch("topic", session_id="session-a") - assert "entities/project/cached--a1b2c3d4.md" in cached - assert instance.prefetch("different follow-up", session_id="session-a") == cached - assert instance.prefetch("different follow-up", session_id="other-session") == "" - for number in range(10): - instance.queue_prefetch(f"q-{number}", session_id="s") - assert instance._prefetch_worker is worker - - -def test_prefetch_exact_query_precedes_latest_session_fallback( - provider: tuple[SubstrateWikiProvider, FakeClient], -) -> None: - instance, _ = provider - now = time.monotonic() + 60 - exact_key = instance._cache_key("repeat", "session-a") - session_key = instance._session_cache_key("session-a") - with instance._cache_lock: - instance._prefetch_cache[exact_key] = (now, "exact") - instance._latest_prefetch_cache[session_key] = (now, "latest") - assert instance.prefetch("repeat", session_id="session-a") == "exact" - assert instance.prefetch("new", session_id="session-a") == "latest" - - -def test_only_completed_dialogue_turns_are_uploaded( - provider: tuple[SubstrateWikiProvider, FakeClient], -) -> None: - instance, fake = provider - messages = [{"role": "user", "content": "first"}] - assert instance.on_pre_compress(messages) == "" - instance.on_pre_compress(messages) - instance.on_memory_write( - "write", - "MEMORY.md", - "candidate", - metadata={"session_id": "memory-session", "provenance": "native", "secret": "omit"}, - ) - instance.sync_turn("ignored", "ignored", runtime_context={"role": "subagent"}) - instance.on_session_switch( - new_session_id="session-b", parent_session_id="parent", reset=True, rewound=False - ) - instance.sync_turn("new", "answer") - wait_until(lambda: len(fake.delivered) == 1) - bodies = [call["body"] for call in fake.delivered] - assert bodies[0]["session_id"] == "session-b" - assert bodies[0]["messages"] == [ - {"index": 0, "role": "user", "content": "new"}, - {"index": 1, "role": "assistant", "content": "answer"}, - ] - assert set(bodies[0]) == { - "schema_version", "event_id", "kind", "session_id", "created_at", "messages" - } - - -def test_sender_marks_each_in_memory_item_done_once_on_success_and_failure( - provider: tuple[SubstrateWikiProvider, FakeClient], -) -> None: - instance, fake = provider - fake.fail = True - instance.sync_turn("one", "answer") - wait_until(lambda: instance.status_snapshot()["counters"]["delivery_failed"] >= 1) - assert len(instance._spool) == 1 - fake.fail = False - instance._wake.set() - wait_until(lambda: len(fake.delivered) == 1) - assert len(instance._spool) == 0 - - -def test_spool_permissions_escape_symlink_and_corruption(tmp_path: Path) -> None: - spool = DurableSpool(tmp_path / "spool", max_items=4, max_bytes=4096) - path = spool.append({"secret": "redacted", "n": 1}) - if os.name == "posix": - assert stat.S_IMODE(spool.root.stat().st_mode) == 0o700 - assert stat.S_IMODE(path.stat().st_mode) == 0o600 - outside = tmp_path / "outside.json" - outside.write_text("{}", encoding="utf-8") - with pytest.raises(ValueError): - spool.load(outside) - path.write_bytes(b"\xffnot-json") - with pytest.raises(ValueError, match="corrupt"): - spool.load(path) - spool.quarantine(path) - assert spool.oldest() is None - assert len(list((spool.root / "corrupt").glob("*.bad"))) == 1 - if hasattr(os, "symlink"): - link = spool.root / "99999999999999999999-link.json" - try: - link.symlink_to(outside) - except OSError: - pass - else: - assert spool.oldest() is None - - -def test_client_rejects_wrong_content_type_oversize_and_non_json_shape( - monkeypatch: pytest.MonkeyPatch, -) -> None: - class Headers(dict[str, str]): - pass - - class Response: - def __init__(self, body: bytes, content_type: str) -> None: - self.body = body - self.headers = Headers({"Content-Type": content_type}) - - def __enter__(self) -> Response: - return self - - def __exit__(self, *args: Any) -> None: - return None - - def read(self, size: int = -1) -> bytes: - return self.body[:size] - - class Opener: - response: Response - - def open(self, request: Any, *, timeout: float) -> Response: - return self.response - - opener = Opener() - monkeypatch.setattr("substrate_wiki.client.build_opener", lambda *handlers: opener) - client = SubstrateClient("https://wiki.example.test", "secret", max_response_bytes=1024) - opener.response = Response(b"{}", "text/plain") - with pytest.raises(SubstrateAPIError, match="invalid_content_type"): - client.search("x") - opener.response = Response(b"{" + b"x" * 2000, "application/json") - with pytest.raises(SubstrateAPIError, match="response_too_large"): - client.search("x") - opener.response = Response(b'"scalar"', "application/json; charset=utf-8") - with pytest.raises(SubstrateAPIError, match="invalid_response"): - client.search("x") - oversized = { - "results": [ - {"title": "x" * 5000, "text": "y" * 70000, "citation": "c" * 5000, "extra": "omit"} - for _ in range(40) - ] - } - opener.response = Response(json.dumps(oversized).encode(), "application/json") - client.max_response_bytes = 8 * 1024 * 1024 - shaped = client.search("x") - assert len(shaped["results"]) == 25 - assert len(shaped["results"][0]["title"]) == 2048 - assert len(shaped["results"][0]["text"]) == 65536 - assert "extra" not in shaped["results"][0] - - -def test_saved_url_is_available_to_fresh_provider( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - provider = SubstrateWikiProvider() - provider.save_config({"api_url": "https://saved.example.test"}, str(tmp_path)) - monkeypatch.setenv("HERMES_HOME", str(tmp_path)) - monkeypatch.delenv("HERMES_API_URL", raising=False) - monkeypatch.setenv("HERMES_API_KEY", "env-only-key") - assert SubstrateWikiProvider().is_available() - - -def test_agent_context_scope_and_initialized_non_primary_suppression( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - monkeypatch.setenv("HERMES_API_URL", "https://wiki.example.test") - monkeypatch.setenv("HERMES_API_KEY", "key") - fake = FakeClient() - monkeypatch.setattr("substrate_wiki.SubstrateClient.from_env", lambda **kwargs: fake) - primary = SubstrateWikiProvider() - primary.initialize( - "s", - hermes_home=str(tmp_path / "primary"), - agent_context="primary", - agent_identity="Main", - agent_workspace="workspace", - user_id="user", - platform="cli", - agent_id="agent-1", - ) - primary.sync_turn("u", "a") - wait_until(lambda: len(fake.delivered) == 1) - body = fake.delivered[0]["body"] - assert body["session_id"] == "s" - assert not ({"scope", "agent_id", "agent_identity", "agent_workspace", "user_id", "platform"} & set(body)) - snapshot = primary.status_snapshot() - assert "session" not in json.dumps(snapshot).lower() - assert "https://" not in json.dumps(snapshot) - primary.shutdown() - - suppressed = SubstrateWikiProvider() - suppressed.initialize("s", hermes_home=str(tmp_path / "cron"), agent_context="cron") - suppressed.sync_turn("u", "a") - suppressed.on_memory_write("write", "x", "y") - assert suppressed.status_snapshot()["counters"]["suppressed"] == 2 - assert suppressed._events.empty() - suppressed.shutdown() - - -def test_sync_turn_uses_only_user_and_assistant_arguments( - provider: tuple[SubstrateWikiProvider, FakeClient], -) -> None: - instance, fake = provider - instance.sync_turn("repeat-user", "repeat-assistant") - wait_until(lambda: len(fake.delivered) == 1) - first = fake.delivered[0]["body"] - assert first["messages"] == [ - {"index": 0, "role": "user", "content": "repeat-user"}, - {"index": 1, "role": "assistant", "content": "repeat-assistant"}, - ] - instance.sync_turn("next-user", "next-assistant") - wait_until(lambda: len(fake.delivered) == 2) - assert [message["index"] for message in fake.delivered[1]["body"]["messages"]] == [2, 3] - - -def test_late_prefetch_from_old_session_is_not_published( - provider: tuple[SubstrateWikiProvider, FakeClient], -) -> None: - instance, fake = provider - gate = threading.Event() - - def delayed( - query: str, *, limit: int = 8, scope: dict[str, Any] | None = None - ) -> dict[str, Any]: - del query, limit, scope - gate.wait(2) - return { - "results": [ - { - "text": "old", - "path": "entities/person/old--deadbeef.md", - "page_type": "entity", - "entity_type": "person", - } - ] - } - - fake.memory_search = delayed # type: ignore[method-assign] - instance.queue_prefetch("old", session_id="session-a") - instance.on_session_switch(new_session_id="session-b") - gate.set() - wait_until(lambda: instance._prefetch_jobs.unfinished_tasks == 0) - assert instance.prefetch("old", session_id="session-a") == "" - - -def test_capture_is_json_safe_and_does_not_use_custom_repr( - provider: tuple[SubstrateWikiProvider, FakeClient], tmp_path: Path -) -> None: - instance, fake = provider - - class SecretObject: - def __repr__(self) -> str: - return "leaked-secret-repr" - - instance.sync_turn( - {"bytes": b"secret", "path": tmp_path / "page", "set": {"a", "b"}}, - SecretObject(), - ) - wait_until(lambda: len(fake.delivered) == 1) - rendered = json.dumps(fake.delivered[0]["body"]) - assert "leaked-secret-repr" not in rendered - assert "[NON_TEXT_CONTENT_OMITTED]" in rendered - - -def test_strict_boolean_and_safe_url_prefix( - provider: tuple[SubstrateWikiProvider, FakeClient], -) -> None: - instance, fake = provider - assert json.loads( - instance.handle_tool_call("wiki_query", {"question": "q", "save_as_synthesis": "false"}) - ) == {"error": "invalid_arguments"} - assert fake.queries == [] - assert SubstrateClient.is_allowed_base_url("https://wiki.example.test/substrate") - assert not SubstrateClient.is_allowed_base_url("https://wiki.example.test/a//b") - assert not SubstrateClient.is_allowed_base_url("https://wiki.example.test/a/../b") - assert not SubstrateClient.is_allowed_base_url("https://wiki.example.test/a/%2e%2e/b") - - -def test_capture_state_commits_only_after_durable_admission( - provider: tuple[SubstrateWikiProvider, FakeClient], monkeypatch: pytest.MonkeyPatch -) -> None: - instance, fake = provider - assert instance._spool is not None - fake.fail = True - fake.should_block = True - fake.block_timeout = 30.0 - original_append = instance._spool.append - monkeypatch.setattr( - instance._spool, "append", lambda event: (_ for _ in ()).throw(OSError("full")) - ) - instance.sync_turn("must retry", "answer") - assert instance.status_snapshot()["counters"]["dropped"] >= 1 - monkeypatch.setattr(instance._spool, "append", original_append) - instance.sync_turn("must retry", "answer") - try: - assert fake.request_started.wait(3.0) - paths = list(instance._spool.root.glob("*.json")) - assert len(paths) == 1 - event = instance._spool.load(paths[0]) - assert event["messages"] == [ - {"index": 0, "role": "user", "content": "must retry"}, - {"index": 1, "role": "assistant", "content": "answer"}, - ] - finally: - fake.block.set() - wait_until(lambda: instance.status_snapshot()["counters"]["delivery_failed"] >= 1) - - -def test_session_end_does_not_emit_a_second_payload( - provider: tuple[SubstrateWikiProvider, FakeClient], -) -> None: - instance, fake = provider - messages = [{"role": "user", "content": "one"}, {"role": "assistant", "content": "two"}] - instance.sync_turn("one", "two") - instance.on_session_end(messages) - wait_until(lambda: len(fake.delivered) == 1) - assert fake.delivered[0]["body"]["kind"] == "turn" - - -def test_spool_claim_protects_inflight_oldest_from_trim(tmp_path: Path) -> None: - spool = DurableSpool(tmp_path / "spool", max_items=2, max_bytes=4096) - first = spool.append({"event_id": "first"}) - spool.append({"event_id": "second"}) - assert spool.claim_oldest() == first - third = spool.append({"event_id": "third"}) - assert first.is_file() - assert third.is_file() - spool.release(first) - - -def test_auth_failure_remains_replayable( - provider: tuple[SubstrateWikiProvider, FakeClient], monkeypatch: pytest.MonkeyPatch -) -> None: - instance, fake = provider - fake.fail = True - fake.fail_category = "http_401" - monkeypatch.setattr(instance, "_wait_for_retry", lambda delay: instance._stop.wait(0.01)) - instance.sync_turn("u", "a") - wait_until(lambda: instance.status_snapshot()["counters"]["delivery_failed"] >= 1) - assert len(instance._spool) == 1 - assert instance.status_snapshot()["counters"]["quarantined"] == 0 - fake.fail = False - instance._wake.set() - wait_until(lambda: len(fake.delivered) == 1) - assert len(instance._spool) == 0 - - -def test_retry_delay_grows_caps_honors_rate_limit_and_auth_floor( - provider: tuple[SubstrateWikiProvider, FakeClient], monkeypatch: pytest.MonkeyPatch -) -> None: - instance, _ = provider - monkeypatch.setattr(instance._retry_random, "uniform", lambda low, high: 1.0) - assert instance._retry_delay("transport_error", 1) == 1 - assert instance._retry_delay("http_500", 4) == 8 - assert instance._retry_delay("http_500", 100) == 256 - assert instance._retry_delay("http_401", 1) == 30 - assert instance._retry_delay("http_403", 2) == 60 - assert instance._retry_delay("http_429", 1, retry_after=45) == 45 - assert instance._retry_delay("http_429", 1, retry_after=10000) == 300 - - -def test_delivery_backoff_grows_and_resets_after_success( - provider: tuple[SubstrateWikiProvider, FakeClient], monkeypatch: pytest.MonkeyPatch -) -> None: - instance, fake = provider - delays: list[float] = [] - - def wait_for_retry(delay: float) -> bool: - delays.append(delay) - if len(delays) == 2: - fake.fail = False - return instance._stop.is_set() - - monkeypatch.setattr(instance._retry_random, "uniform", lambda low, high: 1.0) - monkeypatch.setattr(instance, "_wait_for_retry", wait_for_retry) - fake.fail = True - fake.fail_category = "http_500" - instance.sync_turn("one", "answer") - wait_until(lambda: len(fake.delivered) == 1) - assert delays == [1, 2] - assert instance._delivery_failure_streak == 0 - - -def test_shutdown_interrupts_long_retry_cooldown( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - monkeypatch.setenv("HERMES_API_URL", "https://wiki.example.test") - monkeypatch.setenv("HERMES_API_KEY", "secret-value") - fake = FakeClient() - fake.fail = True - fake.fail_category = "http_401" - monkeypatch.setattr("substrate_wiki.SubstrateClient.from_env", lambda **kwargs: fake) - instance = SubstrateWikiProvider() - instance.initialize("session-a", hermes_home=str(tmp_path)) - instance.sync_turn("u", "a") - wait_until(lambda: instance.status_snapshot()["counters"]["delivery_failed"] >= 1) - started = time.monotonic() - instance.shutdown() - assert time.monotonic() - started < 1 - assert len(instance._spool) == 1 - - -def test_malformed_and_permanent_spool_head_is_quarantined( - provider: tuple[SubstrateWikiProvider, FakeClient], -) -> None: - instance, fake = provider - assert instance._spool is not None - instance._spool.append({"kind": "unknown", "event_id": "bad"}) - instance._wake.set() - wait_until(lambda: instance.status_snapshot()["counters"]["quarantined"] >= 1) - fake.fail = True - fake.fail_category = "http_400" - instance.sync_turn("u", "a") - wait_until(lambda: instance.status_snapshot()["counters"]["permanent_dropped"] >= 2) - assert instance._worker is not None and instance._worker.is_alive() - - -def test_prefetch_cache_is_bounded_and_expired_entries_are_evicted( - provider: tuple[SubstrateWikiProvider, FakeClient], -) -> None: - instance, _ = provider - now = time.monotonic() - with instance._cache_lock: - instance._prefetch_cache = {f"key-{index}": (now + 60, "value") for index in range(200)} - instance._prefetch_cache["expired"] = (now - 1, "old") - instance._latest_prefetch_cache = OrderedDict( - (f"session-{index}", (now + 60, "value")) for index in range(40) - ) - expired_session = instance._session_cache_key("expired-session") - instance._latest_prefetch_cache[expired_session] = (now - 1, "old") - instance._evict_prefetch_locked() - while len(instance._prefetch_cache) > 128: - instance._prefetch_cache.pop(next(iter(instance._prefetch_cache))) - while len(instance._latest_prefetch_cache) > 32: - instance._latest_prefetch_cache.popitem(last=False) - assert "expired" not in instance._prefetch_cache - assert expired_session not in instance._latest_prefetch_cache - assert len(instance._prefetch_cache) <= 128 - assert len(instance._latest_prefetch_cache) <= 32 - assert instance.prefetch("follow-up", session_id="expired-session") == "" - - -def test_prefetch_accepts_only_canonical_entity_path_and_job_status_keeps_id() -> None: - cited = SubstrateWikiProvider._cited_prefetch( - { - "results": [ - {"path": "topics/cached.md", "title": "Topic", "page_type": "topic"}, - { - "canonical_path": "entities/project/legacy--d4e5f6a7.md", - "snippet": "Legacy full-body evidence", - "page_type": "entity", - "entity_type": "project", - }, - { - "path": "entities/project/cached--a1b2c3d4.md", - "canonical_path": "entities/project/cached--a1b2c3d4.md", - "title": "Cached page", - "memory_card": "Cached project profile", - "page_type": "entity", - "entity_id": "entity-cached", - "entity_type": "project", - "quality_version": 2, - }, - { - "path": "entities/service/legacy--deadbeef.md", - "canonical_path": "entities/service/durable-worker--b2c3d4e5.md", - "title": "Durable worker", - "memory_card": "Stable service profile", - "page_type": "entity", - "entity_id": "entity-durable-worker", - "entity_type": "service", - "quality_version": 2, - }, - { - "canonical_path": "entities/product/retyped--c3d4e5f6.md", - "title": "Retyped project", - "memory_card": "Canonical path remains immutable after retyping.", - "page_type": "entity", - "entity_id": "entity-retyped", - "entity_type": "project", - "roles": ["project", "product"], - "quality_version": 2, - }, - ] - } - ) - assert "topics/cached" not in cited - assert "legacy--d4e5f6a7" not in cited - assert "Legacy full-body evidence" not in cited - assert "entities/project/cached--a1b2c3d4.md" in cited - assert "entities/service/durable-worker--b2c3d4e5.md" in cited - assert "entities/service/legacy--deadbeef.md" not in cited - assert "entities/product/retyped--c3d4e5f6.md" in cited - shaped = SubstrateClient._shape_response( - "/api/v1/hermes/wiki/job-status", - { - "id": "job-1", - "status": "failed", - "attempts": 1, - "max_attempts": 3, - "error": "could not connect to the public URL", - "error_detail": { - "code": "url_connect_failed", - "retryable": True, - "private": "omit", - }, - }, - ) - assert shaped["id"] == "job-1" - assert shaped["attempts"] == 1 - assert shaped["error_detail"] == {"code": "url_connect_failed", "retryable": True} - - -def test_prefetch_collapses_duplicate_canonical_entity_ids() -> None: - common = { - "canonical_path": "entities/project/cached--a1b2c3d4.md", - "page_type": "entity", - "entity_id": "entity-cached", - "entity_type": "project", - "quality_version": 2, - } - cited = SubstrateWikiProvider._cited_prefetch( - { - "results": [ - {**common, "memory_card": "Canonical card"}, - {**common, "memory_card": "Duplicate card must be collapsed"}, - ] - } - ) - - assert "Canonical card" in cited - assert "Duplicate card must be collapsed" not in cited - assert ( - SubstrateWikiProvider._cited_prefetch( - {"results": [{**common, "entity_id": "", "memory_card": "Unidentified"}]} - ) - == "" - ) - - -def test_query_response_shaping_keeps_bounded_persistence_contract() -> None: - shaped = SubstrateClient._shape_response( - "/api/v1/hermes/wiki/query", - { - "answer": "answer", - "insufficient_context": False, - "saved": False, - "synthesis_path": None, - "error": { - "code": "synthesis_persistence_failed", - "message": "The answer was generated but could not be saved.", - "retryable": True, - "traceback": "omit", - }, - }, - ) - - assert shaped["saved"] is False - assert shaped["synthesis_path"] is None - assert shaped["error"] == { - "code": "synthesis_persistence_failed", - "message": "The answer was generated but could not be saved.", - "retryable": True, - } - - -def test_wiki_read_schema_documents_path_or_slug(provider) -> None: - instance, _ = provider - schema = next(item for item in instance.get_tool_schemas() if item["name"] == "wiki_read") - - assert "path or legacy slug" in schema["description"] - assert ( - "notes/hermes.md or notes/hermes" - in schema["parameters"]["properties"]["path"]["description"] - ) - - -def test_redaction_sanitizes_secret_mapping_keys_without_collisions() -> None: - from substrate_wiki.redaction import redact - - secret = "very-secret-hermes-key" - sanitized = redact({secret: "first", "api_key": "second"}, (secret,)) - rendered = json.dumps(sanitized) - assert secret not in rendered - assert set(sanitized) == {"[REDACTED]", "[REDACTED]#2"} - - -def test_configured_secret_scan_prioritizes_provider_values_and_is_bounded( - monkeypatch: pytest.MonkeyPatch, -) -> None: - from substrate_wiki import redaction - - environment = { - "UNRELATED": "not-sensitive", - "HERMES_API_KEY": "hermes-required-value", - "MINIMAX_API_KEY": "minimax-required-value", - "NVIDIA_API_KEY": "nvidia-required-value", - **{ - f"EXTRA_{index:03d}_API_KEY": f"extra-sensitive-value-{index:03d}" - for index in range(redaction._MAX_CONFIGURED_SECRETS - 3) - }, - } - monkeypatch.setattr(redaction.os, "environ", environment) - - values = redaction.configured_secret_values() - - assert len(values) == redaction._MAX_CONFIGURED_SECRETS - assert environment["HERMES_API_KEY"] in values - assert environment["MINIMAX_API_KEY"] in values - assert environment["NVIDIA_API_KEY"] in values - assert values == tuple(sorted(values, key=lambda item: (-len(item), item))) - assert sum(len(value.encode("utf-8")) for value in values) <= ( - redaction._MAX_CONFIGURED_SECRET_BYTES - ) - - -def test_configured_secret_scan_fails_closed_without_logging_overflow_values( - monkeypatch: pytest.MonkeyPatch, -) -> None: - from substrate_wiki import redaction - - count_values = { - f"PROVIDER_{index:03d}_API_KEY": f"count-overflow-value-{index:03d}" - for index in range(redaction._MAX_CONFIGURED_SECRETS + 1) - } - monkeypatch.setattr(redaction.os, "environ", count_values) - with pytest.raises(ValueError) as count_error: - redaction.configured_secret_values() - assert str(count_error.value) == "configured secrets exceed redaction bounds" - assert not any(value in str(count_error.value) for value in count_values.values()) - - byte_values = { - f"PROVIDER_{index}_API_KEY": (chr(ord("a") + index) * 15_000) for index in range(5) - } - monkeypatch.setattr(redaction.os, "environ", byte_values) - with pytest.raises(ValueError) as byte_error: - redaction.configured_secret_values() - assert str(byte_error.value) == "configured secrets exceed redaction bounds" - assert not any(value in str(byte_error.value) for value in byte_values.values()) - - -def test_prefetch_queue_is_bounded(provider: tuple[SubstrateWikiProvider, FakeClient]) -> None: - instance, _ = provider - instance._prefetch_jobs = queue.Queue(maxsize=1) - instance._prefetch_jobs.put_nowait(("existing", "s")) - instance.queue_prefetch("discarded", session_id="s") - assert instance._prefetch_jobs.qsize() == 1 diff --git a/tests/test_history.py b/tests/test_history.py deleted file mode 100644 index 425b040..0000000 --- a/tests/test_history.py +++ /dev/null @@ -1,308 +0,0 @@ -from __future__ import annotations - -import sqlite3 -import sys -from pathlib import Path -from typing import Any - -import pytest - -PLUGIN_ROOT = Path(__file__).parents[1] / "src" -sys.path.insert(0, str(PLUGIN_ROOT)) - -from substrate_wiki.checkpoint import ImportCheckpoint # noqa: E402 -from substrate_wiki.client import SubstrateAPIError # noqa: E402 -from substrate_wiki.history import ( # noqa: E402 - HermesHistoryImporter, - HermesSQLiteHistorySource, -) - - -def _database(path: Path, messages: int = 3) -> None: - connection = sqlite3.connect(path) - try: - connection.executescript( - """ - CREATE TABLE sessions ( - id TEXT PRIMARY KEY, source TEXT NOT NULL, user_id TEXT, - chat_type TEXT, started_at REAL NOT NULL - ); - CREATE TABLE messages ( - id INTEGER PRIMARY KEY AUTOINCREMENT, session_id TEXT NOT NULL, - role TEXT NOT NULL, content TEXT NOT NULL, timestamp REAL NOT NULL, - active INTEGER NOT NULL DEFAULT 1 - ); - INSERT INTO sessions VALUES ('session-a', 'cli', NULL, NULL, 1.0); - """ - ) - connection.executemany( - "INSERT INTO messages(session_id, role, content, timestamp) VALUES (?, ?, ?, ?)", - ( - ("session-a", "user" if index % 2 == 0 else "assistant", f"message-{index}", index) - for index in range(messages) - ), - ) - connection.commit() - finally: - connection.close() - - -class Client: - def __init__(self, *, fail_after: int | None = None) -> None: - self.fail_after = fail_after - self.requests: list[dict[str, Any]] = [] - - def capabilities(self) -> dict[str, Any]: - return { - "provider": "substrate_wiki", - "capture_schema_versions": [2], - "max_event_bytes": 262_144, - "history_replay": { - "protocol": "stream-v2", - "min_plugin_version": "1.2.0", - "content_free_completion": True, - "incremental_windows": True, - "status_version": 2, - }, - } - - def request(self, method: str, path: str, **kwargs: Any) -> dict[str, Any]: - del method, path - if self.fail_after is not None and len(self.requests) >= self.fail_after: - raise SubstrateAPIError("http_400") - self.requests.append(kwargs["body"]) - return {"duplicate": False} - - def import_status(self, batch_id: str) -> dict[str, Any]: - return { - "batch_id": batch_id, - "processed_windows": 1, - "failed_windows": 0, - "processed": 1, - "pending_review": 0, - "failed": 0, - "projected_entities": 4, - "published_claims": 9, - "stub_count": 3, - "pending_resolution": 2, - "projection_pending": 0, - "complete": True, - } - - -def test_checkpoint_is_content_free_and_stable_when_state_db_changes(tmp_path: Path) -> None: - home = tmp_path / "home" - home.mkdir() - database = home / "state.db" - _database(database, messages=2) - client = Client() - importer = HermesHistoryImporter( - hermes_home=home, - client=client, # type: ignore[arg-type] - source=HermesSQLiteHistorySource(database), - ) - checkpoint = importer.prepare() - original_batch = importer.batch_id - connection = sqlite3.connect(database) - connection.execute("INSERT INTO sessions VALUES ('new-session', 'cli', NULL, NULL, 2.0)") - connection.execute( - "INSERT INTO messages(session_id, role, content, timestamp) VALUES ('new-session', 'user', 'new private content', 2.0)" - ) - connection.commit() - connection.close() - status = importer.run(wait=True) - checkpoint_path = checkpoint.path - checkpoint.close() - - assert status["batch_id"] == original_batch - assert status["eligible"] == 1 - assert status["failed_windows"] == 0 - assert status["projected_entities"] == 4 - assert status["published_claims"] == 9 - assert status["stub_count"] == 3 - assert status["pending_resolution"] == 2 - assert status["projection_pending"] == 0 - raw = checkpoint_path.read_bytes() - assert b"message-0" not in raw - assert b"new private content" not in raw - assert b"HERMES_API_KEY" not in raw - - -def test_resume_skips_every_acknowledged_event(tmp_path: Path) -> None: - home = tmp_path / "home" - home.mkdir() - database = home / "state.db" - _database(database, messages=3) - first_client = Client(fail_after=1) - first = HermesHistoryImporter( - hermes_home=home, - client=first_client, # type: ignore[arg-type] - source=HermesSQLiteHistorySource(database), - ) - with pytest.raises(SubstrateAPIError): - first.run(wait=True) - first_ids = {str(item["event_id"]) for item in first_client.requests} - assert len(first_ids) == 1 - if first.checkpoint is not None: - first.checkpoint.close() - - resumed_client = Client() - resumed = HermesHistoryImporter( - hermes_home=home, - client=resumed_client, # type: ignore[arg-type] - source=HermesSQLiteHistorySource(database), - ) - status = resumed.run(wait=True) - resumed_ids = {str(item["event_id"]) for item in resumed_client.requests} - assert status["complete"] is True - assert first_ids.isdisjoint(resumed_ids) - assert status["checkpointed"] == status["delivered"] - if resumed.checkpoint is not None: - resumed.checkpoint.close() - - -def test_graceful_stop_after_acknowledgement_resumes_same_job_and_batch( - tmp_path: Path, -) -> None: - home = tmp_path / "home" - home.mkdir() - database = home / "state.db" - _database(database, messages=2) - - class PausingClient(Client): - stop_requested = False - - def request(self, method: str, path: str, **kwargs: Any) -> dict[str, Any]: - result = super().request(method, path, **kwargs) - self.stop_requested = True - return result - - pausing_client = PausingClient() - paused = HermesHistoryImporter( - hermes_home=home, - client=pausing_client, # type: ignore[arg-type] - source=HermesSQLiteHistorySource(database), - stop_requested=lambda: pausing_client.stop_requested, - ) - paused_status = paused.run(wait=True) - original_job = paused_status["job_id"] - original_batch = paused_status["batch_id"] - assert paused_status["complete"] is False - assert paused_status["checkpointed"] == 1 - if paused.checkpoint is not None: - paused.checkpoint.close() - - resumed = HermesHistoryImporter( - hermes_home=home, - client=Client(), # type: ignore[arg-type] - source=HermesSQLiteHistorySource(database), - ) - resumed_status = resumed.run(wait=True) - assert resumed_status["job_id"] == original_job - assert resumed_status["batch_id"] == original_batch - assert resumed_status["complete"] is True - if resumed.checkpoint is not None: - resumed.checkpoint.close() - - -def test_v13_checkpoint_migration_preserves_job_and_partial_remote_status( - tmp_path: Path, -) -> None: - checkpoint = ImportCheckpoint.create_or_attach( - tmp_path, - source_kind="hermes-sqlite", - source_locator=str(tmp_path / "state.db"), - agent_id="default", - ) - original_job = checkpoint.status()["job_id"] - path = checkpoint.path - checkpoint.close() - with sqlite3.connect(path) as connection: - for column in ( - "failed_windows", - "projected_entities", - "published_claims", - "stub_count", - "pending_resolution", - "projection_pending", - ): - connection.execute(f"ALTER TABLE job DROP COLUMN {column}") - - with ImportCheckpoint(path) as migrated: - assert migrated.status()["job_id"] == original_job - assert migrated.status()["projected_entities"] == 0 - migrated.update_remote( - { - "processed_windows": 12, - "failed_windows": 2, - "processed": 4, - "pending_review": 7, - "failed": 0, - "projected_entities": 8, - "published_claims": 21, - "stub_count": 5, - "pending_resolution": 3, - "projection_pending": 6, - "error_class": "provider_exhausted", - } - ) - migrated.update_remote({"processed": 5}) - status = migrated.status() - assert status["processed"] == 5 - assert status["processed_windows"] == 12 - assert status["projected_entities"] == 8 - assert status["projection_pending"] == 6 - assert status["error_class"] == "provider_exhausted" - - migrated.update_remote({"error_class": "secret internal provider response"}) - assert migrated.status()["error_class"] == "server_error" - migrated.update_remote({"complete": True, "failed": 0}) - assert migrated.status()["state"] == "complete_with_failures" - - assert b"secret internal provider response" not in path.read_bytes() - - -def test_checkpoint_schema_contains_no_content_columns(tmp_path: Path) -> None: - checkpoint = ImportCheckpoint.create_or_attach( - tmp_path, - source_kind="hermes-sqlite", - source_locator=str(tmp_path / "state.db"), - agent_id="default", - ) - tables = { - str(row[0]) - for row in checkpoint.connection.execute( - "SELECT name FROM sqlite_master WHERE type='table'" - ) - } - columns = { - str(row[1]).casefold() - for table in tables - for row in checkpoint.connection.execute(f"PRAGMA table_info({table})") - } - assert not columns & { - "content", - "prompt", - "conclusion", - "tool_result", - "credential", - "secret", - } - checkpoint.close() - - -def test_systemd_installer_declares_separate_bounded_cgroup() -> None: - installer = (Path(__file__).parents[1] / "scripts" / "install_hermes_plugin.py").read_text( - encoding="utf-8" - ) - for setting in ( - "MemoryHigh=224M", - "MemoryMax=256M", - "OOMPolicy=stop", - "Restart=on-failure", - "NoNewPrivileges=true", - "PrivateTmp=true", - ): - assert setting in installer - assert "HERMES_API_KEY=" not in installer - assert "hermes-gateway" not in installer diff --git a/tests/test_history_replay.py b/tests/test_history_replay.py deleted file mode 100644 index f8a1a7c..0000000 --- a/tests/test_history_replay.py +++ /dev/null @@ -1,885 +0,0 @@ -from __future__ import annotations - -import hashlib -import json -import os -import sqlite3 -import sys -import tracemalloc -from pathlib import Path -from types import SimpleNamespace -from typing import Any - -import pytest - -PLUGIN_ROOT = Path(__file__).parents[1] / "src" -sys.path.insert(0, str(PLUGIN_ROOT)) - -from substrate_wiki import cli as import_cli # noqa: E402 -from substrate_wiki import worker as import_worker # noqa: E402 -from substrate_wiki.checkpoint import ImportCheckpoint # noqa: E402 -from substrate_wiki.client import SubstrateAPIError # noqa: E402 -from substrate_wiki.events import ( # noqa: E402 - BoundedTextSource, - CaptureEventBuilder, - canonical_bytes, -) -from substrate_wiki.history import ( # noqa: E402 - HermesHistoryImporter, - HermesJSONLHistorySource, - HermesSQLiteHistorySource, - HistoryInventory, - HistorySession, - _deliver_with_retry, - _recover_official_export_spills, - select_history_source, -) -from substrate_wiki.redaction import ( # noqa: E402 - StreamingTextRedactor, - redact_text, -) - - -class _ChunkedText: - def __init__(self, chunks: tuple[str, ...]) -> None: - self.chunks = chunks - - def iter_text_chunks(self): # type: ignore[no-untyped-def] - yield from self.chunks - - -def _streamed_capture( - text: str, - chunks: tuple[str, ...], - *, - secrets: tuple[str, ...] = (), - maximum: int = 16 * 1024, -) -> tuple[str, list[dict[str, Any]]]: - builder = CaptureEventBuilder( - {"platform": "cli"}, - secrets=secrets, - max_capture_bytes=maximum, - ) - events = list( - builder.iter_message_events( - "turn", - "stream-redaction", - ({"role": "user", "content": _ChunkedText(chunks)},), - capture_origin="history_replay", - deterministic=True, - ) - ) - rendered = "".join( - str(message["content"]) - for event in events - for message in event["messages"] - ) - assert sum(len(chunk) for chunk in chunks) == len(text) - return rendered, events - - -def _checkpoint(tmp_path: Path, source_kind: str, locator: str) -> ImportCheckpoint: - return ImportCheckpoint.create_or_attach( - tmp_path / "hermes", - source_kind=source_kind, - source_locator=locator, - agent_id="default", - ) - - -def _create_state_db(path: Path) -> None: - connection = sqlite3.connect(path) - try: - connection.executescript( - """ - CREATE TABLE sessions ( - id TEXT PRIMARY KEY, - source TEXT NOT NULL, - user_id TEXT, - chat_type TEXT, - parent_session_id TEXT, - started_at REAL NOT NULL, - ended_at REAL, - archived INTEGER NOT NULL DEFAULT 0, - profile_name TEXT - ); - CREATE TABLE messages ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - session_id TEXT NOT NULL, - role TEXT NOT NULL, - content TEXT, - tool_call_id TEXT, - tool_calls TEXT, - tool_name TEXT, - timestamp REAL NOT NULL, - reasoning TEXT, - platform_message_id TEXT, - active INTEGER NOT NULL DEFAULT 1 - ); - """ - ) - sessions = [ - ("cli-archived", "cli", None, None, None, 1.0, 2.0, 1, "default"), - ("cron-1", "cron", None, None, None, 3.0, 4.0, 0, None), - ("group-1", "telegram", "person-1", "group", None, 5.0, 6.0, 0, None), - ("missing-user", "telegram", None, "private", None, 7.0, 8.0, 0, None), - ("direct-1", "telegram", "person-2", "private", None, 9.0, 10.0, 0, None), - ] - connection.executemany( - "INSERT INTO sessions VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)", - sessions, - ) - rows = [ - ("cli-archived", "system", "hidden prompt", 1.0, "hidden reasoning"), - ("cli-archived", "user", "visible archived question", 1.1, None), - ("cli-archived", "assistant", "visible archived answer", 1.2, "private chain"), - ("cron-1", "user", "scheduled", 3.1, None), - ("group-1", "user", "group text", 5.1, None), - ("missing-user", "user", "ambiguous", 7.1, None), - ("direct-1", "user", "direct message", 9.1, None), - ] - connection.executemany( - "INSERT INTO messages (session_id, role, content, timestamp, reasoning) " - "VALUES (?, ?, ?, ?, ?)", - rows, - ) - connection.commit() - finally: - connection.close() - - -def test_shared_builder_is_deterministic_bounded_and_content_safe() -> None: - builder = CaptureEventBuilder( - {"platform": "cli"}, - secrets=("top-secret",), - max_capture_bytes=16 * 1024, - ) - messages = [ - {"role": "system", "content": "hidden"}, - { - "role": "user", - "content": "top-secret " + ("large visible text " * 3000), - "reasoning": "never copy this", - }, - { - "role": "assistant", - "content": {"text": "visible", "image": "data:image/png;base64,AAAA"}, - "tool_calls": [{"name": "inspect", "arguments": {"base64": "AAAA"}}], - }, - ] - - first = builder.message_events( - "session_end", - "session-1", - messages, - capture_origin="history_replay", - batch_id="batch-1", - deterministic=True, - ) - second = builder.message_events( - "session_end", - "session-1", - messages, - capture_origin="history_replay", - batch_id="batch-1", - deterministic=True, - ) - - assert first == second - assert len(first) > 1 - assert all(len(canonical_bytes(event)) <= 16 * 1024 for event in first) - encoded = json.dumps(first) - assert "top-secret" not in encoded - assert "never copy this" not in encoded - assert "hidden" not in encoded - assert "[BINARY_CONTENT_OMITTED]" in encoded - assert all( - set(event) == {"schema_version", "event_id", "kind", "session_id", "created_at", "messages"} - for event in first - ) - - -@pytest.mark.parametrize( - ("text", "secrets", "unsafe"), - ( - ( - "safe-before::known-secret-value::safe-after", - ("known-secret-value",), - ("known-secret-value",), - ), - ( - "safe-before Authorization: Bearer abcDEF0123456789._~- safe-after", - (), - ("abcDEF0123456789",), - ), - ( - 'safe-before api_key="assignment-secret-0123456789" safe-after', - (), - ("assignment-secret-0123456789",), - ), - ( - "safe-before github_pat_ABCdef0123456789 safe-after", - (), - ("github_pat_ABCdef0123456789",), - ), - ( - "safe-before -----BEGIN PRIVATE KEY-----\n" - "QUJDREVGR0hJSktMTU5PUFFSU1RVVldYWVo=\n" - "-----END PRIVATE KEY----- safe-after", - (), - ("QUJDREVGR0hJSktMTU5PUFFSU1RVVldYWVo",), - ), - ), -) -def test_streamed_redaction_is_safe_at_every_source_boundary( - text: str, - secrets: tuple[str, ...], - unsafe: tuple[str, ...], -) -> None: - expected = redact_text(text, secrets) - for boundary in range(len(text) + 1): - rendered, events = _streamed_capture( - text, - (text[:boundary], text[boundary:]), - secrets=secrets, - ) - assert rendered == expected - persisted_capture = canonical_bytes(events) - assert all(fragment not in rendered for fragment in unsafe) - assert all(fragment.encode() not in persisted_capture for fragment in unsafe) - assert all(len(canonical_bytes(event)) <= 16 * 1024 for event in events) - - -def test_streamed_redaction_is_chunk_independent_and_resume_deterministic() -> None: - secret = "resume-secret-0123456789" - text = ( - ("safe-caf\u00e9-\U0001f642/" * 2_000) - + f" api_key={secret} " - + ("durable-tail/" * 2_000) - ) - layouts = ( - (text,), - tuple(text[index : index + 1] for index in range(len(text))), - tuple(text[index : index + 997] for index in range(0, len(text), 997)), - ) - captures = [ - _streamed_capture(text, chunks, secrets=(secret,))[1] for chunks in layouts - ] - - assert captures[0] == captures[1] == captures[2] - assert secret.encode() not in canonical_bytes(captures[0]) - assert len(captures[0]) > 1 - - -def test_streamed_redactor_and_capture_memory_are_bounded_for_large_message() -> None: - chunk = "large-safe-history-\U0001f642/" * 2_048 - # More than 64 MiB of UTF-8 input, generated repeatably without holding the - # materialized message in the test process. - repetitions = 1_536 - - redactor = StreamingTextRedactor(()) - released_chars = 0 - tracemalloc.start() - try: - for _ in range(repetitions): - released_chars += sum(len(item) for item in redactor.feed(chunk)) - assert redactor.buffered_chars < 1_000_000 - released_chars += sum(len(item) for item in redactor.finish()) - _current, peak_bytes = tracemalloc.get_traced_memory() - finally: - tracemalloc.stop() - assert released_chars == len(chunk) * repetitions - assert peak_bytes < 32 * 1024 * 1024 - - capture_repetitions = 24 - - class _RepeatedText: - def iter_text_chunks(self): # type: ignore[no-untyped-def] - for _ in range(capture_repetitions): - yield chunk - - builder = CaptureEventBuilder({"platform": "cli"}) - captured_chars = 0 - event_count = 0 - for event in builder.iter_message_events( - "turn", - "bounded-large-redaction", - ({"role": "user", "content": _RepeatedText()},), - capture_origin="history_replay", - deterministic=True, - ): - event_count += 1 - captured_chars += sum( - len(str(message["content"])) - for message in event["messages"] - ) - - assert captured_chars == len(chunk) * capture_repetitions - assert event_count > 1 - - -def test_sqlite_adapter_reads_without_writes_and_filters_identity(tmp_path: Path) -> None: - database = tmp_path / "state.db" - _create_state_db(database) - before = hashlib.sha256(database.read_bytes()).hexdigest() - - source = HermesSQLiteHistorySource(database) - checkpoint = _checkpoint(tmp_path, source.source_kind, source.source_locator) - counts = source.discover(checkpoint) - sessions = list(checkpoint.sessions()) - - assert hashlib.sha256(database.read_bytes()).hexdigest() == before - assert counts["discovered"] == 5 - assert counts["skipped"] == 1 - assert counts["quarantined"] == 2 - assert [session.external_id for session in sessions] == [ - "cli-archived", - "direct-1", - ] - assert sessions[0].subject_id == "owner" - messages = list(source.iter_messages(sessions[0], start=0)) - assert [message["role"] for message in messages] == [ - "user", - "assistant", - ] - assert "reasoning" not in json.dumps(messages) - checkpoint.close() - - -def test_sqlite_large_message_is_read_and_emitted_in_bounded_slices(tmp_path: Path) -> None: - database = tmp_path / "state.db" - content = "bounded-history-" * 140_000 - connection = sqlite3.connect(database) - try: - connection.executescript( - """ - CREATE TABLE sessions ( - id TEXT PRIMARY KEY, source TEXT NOT NULL, started_at REAL NOT NULL - ); - CREATE TABLE messages ( - id INTEGER PRIMARY KEY AUTOINCREMENT, session_id TEXT NOT NULL, - role TEXT NOT NULL, content TEXT NOT NULL, timestamp REAL NOT NULL - ); - INSERT INTO sessions VALUES ('large', 'cli', 1.0); - """ - ) - connection.execute( - "INSERT INTO messages(session_id, role, content, timestamp) VALUES (?, ?, ?, ?)", - ("large", "user", content, 1.0), - ) - connection.commit() - finally: - connection.close() - - source = HermesSQLiteHistorySource(database) - checkpoint = _checkpoint(tmp_path, source.source_kind, source.source_locator) - source.discover(checkpoint) - message = next(source.iter_messages(next(checkpoint.sessions()), start=0)) - - deferred = message["content"] - assert isinstance(deferred, BoundedTextSource) - chunks = list(deferred.iter_text_chunks()) - assert chunks - assert max(len(chunk) for chunk in chunks) <= 1024 * 1024 - assert "".join(chunks) == content - - builder = CaptureEventBuilder({"platform": "cli"}) - events = list( - builder.iter_message_events( - "turn", "large", (message,), deterministic=True - ) - ) - assert all(len(canonical_bytes(event)) <= 256 * 1024 for event in events) - fragments = [ - item - for event in events - for item in event["messages"] - ] - assert "".join(str(item["content"]) for item in fragments) == content - assert {item["fragment"]["encoding"] for item in fragments} == {"utf8-content"} - checkpoint.close() - - -def test_adapters_do_not_use_fetchall_or_unbounded_file_reads() -> None: - source = (PLUGIN_ROOT / "substrate_wiki" / "history.py").read_text(encoding="utf-8") - assert "fetchall(" not in source - assert ".readline(" not in source - assert ".read()" not in source - assert "export_all(" not in source - assert "buffered.extend(" not in source - - -def test_official_jsonl_export_shape_is_supported(tmp_path: Path) -> None: - export = tmp_path / "sessions.jsonl" - export.write_text( - "\n".join( - ( - json.dumps( - { - "id": "cli-export", - "source": "cli", - "archived": 1, - "messages": [{"role": "user", "content": "exported"}], - } - ), - json.dumps( - { - "id": "group-export", - "source": "telegram", - "user_id": "person", - "chat_type": "group", - "messages": [{"role": "user", "content": "ambiguous"}], - } - ), - "not-json", - ) - ), - encoding="utf-8", - ) - - source = HermesJSONLHistorySource(export) - checkpoint = _checkpoint(tmp_path, source.source_kind, source.source_locator) - counts = source.discover(checkpoint) - sessions = list(checkpoint.sessions()) - - assert counts["discovered"] == 3 - assert [session.external_id for session in sessions] == ["cli-export"] - assert counts["quarantined"] == 2 - assert [item["content"] for item in source.iter_messages(sessions[0], start=0)] == [ - "exported" - ] - checkpoint.close() - - -def test_official_export_survives_acknowledged_pause_and_resumes_same_job( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, -) -> None: - hermes_home = tmp_path / "hermes" - export = ( - hermes_home - / "substrate_wiki" - / "imports" - / "spill" - / "official-export-test.jsonl" - ) - export.parent.mkdir(parents=True, mode=0o700) - export.write_text( - json.dumps( - { - "id": "official-session", - "source": "cli", - "messages": [{"role": "user", "content": "durable export fact"}], - } - ) - + "\n", - encoding="utf-8", - ) - export.chmod(0o600) - source = HermesJSONLHistorySource(export) - checkpoint = ImportCheckpoint.create_or_attach( - hermes_home, - source_kind="hermes-official-export", - source_locator=str(export.resolve()), - agent_id="default", - ) - source.discover(checkpoint) - initial = checkpoint.status() - job_id = str(initial["job_id"]) - batch_id = str(initial["batch_id"]) - checkpoint.close() - - stop_holder: dict[str, Any] = {} - - def install_stop_handlers(stop_event: Any) -> dict[Any, Any]: - stop_holder["event"] = stop_event - return {} - - class ReplayClient: - def __init__(self, *, stop_after: int | None, complete: bool) -> None: - self.stop_after = stop_after - self.complete = complete - self.event_ids: list[str] = [] - - @staticmethod - def capabilities() -> dict[str, Any]: - return { - "provider": "substrate_wiki", - "capture_schema_versions": [2], - "max_event_bytes": 262_144, - "history_replay": { - "protocol": "stream-v2", - "min_plugin_version": "1.2.0", - "content_free_completion": True, - "incremental_windows": True, - "status_version": 2, - }, - } - - def request(self, method: str, path: str, **kwargs: Any) -> dict[str, Any]: - del method, path - event_id = str(kwargs["body"]["event_id"]) - self.event_ids.append(event_id) - if self.stop_after is not None and len(self.event_ids) == self.stop_after: - stop_holder["event"].set() - return {"duplicate": False} - - def import_status(self, selected_batch: str) -> dict[str, Any]: - assert selected_batch == batch_id - return { - "batch_id": selected_batch, - "processed": 1, - "failed": 0, - "failed_windows": 0, - "complete": self.complete, - } - - first_client = ReplayClient(stop_after=2, complete=False) - monkeypatch.setattr(import_worker, "_install_stop_handlers", install_stop_handlers) - monkeypatch.setattr( - import_worker.SubstrateClient, - "from_env", - classmethod(lambda cls, **kwargs: first_client), - ) - monkeypatch.setattr(import_worker.subprocess, "run", lambda *args, **kwargs: None) - - paused = import_worker.run_job(hermes_home, job_id) - - assert paused["job_id"] == job_id - assert paused["batch_id"] == batch_id - assert paused["complete"] is False - assert paused["checkpointed"] == 2 - assert export.is_file() - - second_client = ReplayClient(stop_after=None, complete=True) - monkeypatch.setattr(import_worker, "_install_stop_handlers", lambda event: {}) - monkeypatch.setattr( - import_worker.SubstrateClient, - "from_env", - classmethod(lambda cls, **kwargs: second_client), - ) - - completed = import_worker.run_job(hermes_home, job_id) - - assert completed["job_id"] == job_id - assert completed["batch_id"] == batch_id - assert completed["complete"] is True - assert not set(first_client.event_ids) & set(second_client.event_ids) - assert not export.exists() - - -def test_official_export_attaches_before_refreshing_active_snapshot( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, -) -> None: - hermes_home = tmp_path / "hermes" - - class FakeSessionDB: - rows = [ - { - "id": "official-session", - "source": "cli", - "messages": [{"role": "user", "content": "immutable fact"}], - } - ] - opened = 0 - - def __init__(self, *, db_path: Path, read_only: bool) -> None: - assert db_path == hermes_home / "state.db" - assert read_only is True - jobs = hermes_home / "substrate_wiki" / "imports" / "jobs" - assert any(jobs.glob("*/checkpoint.db")) - type(self).opened += 1 - - def iter_export(self) -> Any: - yield from type(self).rows - - def close(self) -> None: - return None - - class CapabilityClient: - @staticmethod - def capabilities() -> dict[str, Any]: - return { - "provider": "substrate_wiki", - "capture_schema_versions": [2], - "max_event_bytes": 262_144, - "history_replay": { - "protocol": "stream-v2", - "min_plugin_version": "1.2.0", - "content_free_completion": True, - "incremental_windows": True, - "status_version": 2, - }, - } - - @staticmethod - def import_status(batch_id: str) -> dict[str, Any]: - del batch_id - return {} - - monkeypatch.setitem( - sys.modules, "hermes_state", SimpleNamespace(SessionDB=FakeSessionDB) - ) - first_source = select_history_source(hermes_home) - assert FakeSessionDB.opened == 0 - first_importer = HermesHistoryImporter( - hermes_home=hermes_home, - client=CapabilityClient(), # type: ignore[arg-type] - source=first_source, - ) - first_checkpoint = first_importer.prepare() - first_status = first_checkpoint.status() - export = Path(first_checkpoint.job()["source_locator"]) - first_digest = hashlib.sha256(export.read_bytes()).hexdigest() - first_checkpoint.close() - assert FakeSessionDB.opened == 1 - os.utime(export, (1.0, 1.0)) - - FakeSessionDB.rows = [ - { - "id": "new-session-that-must-wait-for-another-job", - "source": "cli", - "messages": [{"role": "user", "content": "new mutable history"}], - } - ] - second_source = select_history_source(hermes_home) - second_importer = HermesHistoryImporter( - hermes_home=hermes_home, - client=CapabilityClient(), # type: ignore[arg-type] - source=second_source, - ) - second_checkpoint = second_importer.prepare() - second_status = second_checkpoint.status() - - assert second_status["job_id"] == first_status["job_id"] - assert second_status["batch_id"] == first_status["batch_id"] - assert hashlib.sha256(export.read_bytes()).hexdigest() == first_digest - assert FakeSessionDB.opened == 1 - second_checkpoint.close() - - -def test_official_export_recovery_removes_only_expired_orphans( - tmp_path: Path, -) -> None: - hermes_home = tmp_path / "hermes" - spill = hermes_home / "substrate_wiki" / "imports" / "spill" - spill.mkdir(parents=True, mode=0o700) - expired = spill / "official-export-orphan.jsonl" - fresh = spill / "official-export-fresh.jsonl" - temporary = spill / ".official-export-orphan.jsonl.1.1.tmp" - unrelated = spill / "operator-note.txt" - for path in (expired, fresh, temporary, unrelated): - path.write_text("{}\n", encoding="utf-8") - path.chmod(0o600) - os.utime(expired, (1.0, 1.0)) - os.utime(temporary, (1.0, 1.0)) - os.utime(fresh, (999.0, 999.0)) - os.utime(unrelated, (1.0, 1.0)) - - active = _recover_official_export_spills( - hermes_home, now=1_000.0, max_orphan_age_seconds=100.0 - ) - - assert active == [] - assert not expired.exists() - assert not temporary.exists() - assert fresh.is_file() - assert unrelated.is_file() - - -def test_import_cancel_removes_job_owned_official_export( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, -) -> None: - hermes_home = tmp_path / "hermes" - export = ( - hermes_home - / "substrate_wiki" - / "imports" - / "spill" - / "official-export-cancel.jsonl" - ) - export.parent.mkdir(parents=True, mode=0o700) - export.write_text("{}\n", encoding="utf-8") - export.chmod(0o600) - checkpoint = ImportCheckpoint.create_or_attach( - hermes_home, - source_kind="hermes-official-export", - source_locator=str(export.resolve()), - agent_id="default", - ) - before = checkpoint.status() - checkpoint.close() - stopped: list[str] = [] - monkeypatch.setattr( - import_cli, - "stop_service", - lambda home, job_id: stopped.append(job_id), - ) - - result = import_cli._cancel( - SimpleNamespace(yes=True, job_id=before["job_id"]), hermes_home - ) - - assert result["job_id"] == before["job_id"] - assert result["batch_id"] == before["batch_id"] - assert result["state"] == "cancelled" - assert stopped == [before["job_id"]] - assert not export.exists() - - -def test_oversized_jsonl_record_streams_message_without_materializing_record( - tmp_path: Path, -) -> None: - export = tmp_path / "large-session.jsonl" - encoded_piece = "line\\nsmile\\u263a-" - decoded_piece = "line\nsmile☺-" - repetitions = 100_000 - with export.open("w", encoding="utf-8", newline="\n") as stream: - stream.write('{"id":"large-jsonl","source":"cli","messages":[') - stream.write('{"role":"user","content":"') - for _ in range(repetitions): - stream.write(encoded_piece) - stream.write('"}]}\n') - - source = HermesJSONLHistorySource(export) - checkpoint = _checkpoint(tmp_path, source.source_kind, source.source_locator) - counts = source.discover(checkpoint) - assert counts == {"discovered": 1, "eligible": 1, "skipped": 0, "quarantined": 0} - - iterator = source.iter_messages(next(checkpoint.sessions()), start=0) - message = next(iterator) - deferred = message["content"] - assert isinstance(deferred, BoundedTextSource) - chunks = list(deferred.iter_text_chunks()) - assert chunks - assert max(len(chunk) for chunk in chunks) <= 64 * 1024 - assert "".join(chunks) == decoded_piece * repetitions - - builder = CaptureEventBuilder({"platform": "cli"}) - events = list( - builder.iter_message_events( - "turn", "large-jsonl", (message,), deterministic=True - ) - ) - fragments = [item for event in events for item in event["messages"]] - assert "".join(str(item["content"]) for item in fragments) == decoded_piece * repetitions - assert all(len(canonical_bytes(event)) <= 256 * 1024 for event in events) - iterator.close() - checkpoint.close() - - -class _ReplayClient: - def __init__(self) -> None: - self.requests: list[tuple[str, dict[str, Any]]] = [] - - def request(self, method: str, path: str, **kwargs: Any) -> dict[str, Any]: - self.requests.append((path, kwargs)) - return {"duplicate": False} - - def capabilities(self) -> dict[str, Any]: - return { - "provider": "substrate_wiki", - "capture_schema_versions": [2], - "max_event_bytes": 262_144, - "history_replay": { - "protocol": "stream-v2", - "min_plugin_version": "1.2.0", - "content_free_completion": True, - "incremental_windows": True, - "status_version": 2, - }, - } - - def import_status(self, batch_id: str) -> dict[str, Any]: - return { - "batch_id": batch_id, - "processed": 1, - "pending_review": 0, - "failed": 0, - "complete": True, - } - - -def test_importer_resumes_from_checkpoint_and_uses_standard_endpoints(tmp_path: Path) -> None: - inventory = HistoryInventory( - sessions=[ - HistorySession( - external_id="session-a", - source="cli", - user_id="", - subject_id="owner", - messages=[ - {"role": "user", "content": "hello", "timestamp": 1.0}, - {"role": "assistant", "content": "hi", "timestamp": 2.0}, - ], - metadata={"archived": 1}, - ) - ], - discovered=1, - ) - first_client = _ReplayClient() - first = HermesHistoryImporter( - hermes_home=tmp_path, - client=first_client, # type: ignore[arg-type] - inventory=inventory, - ) - status = first.run(wait=True) - - assert status["complete"] is True - assert [path for path, _ in first_client.requests] == [ - "/api/v1/hermes/turns", - "/api/v1/hermes/completed-sessions", - ] - event_ids = [request["body"]["event_id"] for _, request in first_client.requests] - assert len(event_ids) == len(set(event_ids)) - assert all( - not request["body"].get("messages") - for path, request in first_client.requests - if path == "/api/v1/hermes/completed-sessions" - ) - - -def test_importer_refuses_server_without_stream_v2(tmp_path: Path) -> None: - class OldServer(_ReplayClient): - def capabilities(self) -> dict[str, Any]: - return {"max_event_bytes": 262_144, "history_replay": {"protocol": "legacy"}} - - importer = HermesHistoryImporter( - hermes_home=tmp_path, - client=OldServer(), # type: ignore[arg-type] - inventory=HistoryInventory(), - ) - with pytest.raises(SubstrateAPIError, match="server_upgrade_required"): - importer.prepare() - - -def test_delivery_retries_only_transient_failures(monkeypatch: pytest.MonkeyPatch) -> None: - class Client: - attempts = 0 - - def request(self, *args: Any, **kwargs: Any) -> dict[str, Any]: - self.attempts += 1 - if self.attempts == 1: - raise SubstrateAPIError("transport_error") - return {"stored": True} - - client = Client() - monkeypatch.setattr("substrate_wiki.history.time.sleep", lambda delay: None) - result = _deliver_with_retry( - client, # type: ignore[arg-type] - {"kind": "turn", "event_id": "event-1"}, - ) - assert result == {"stored": True} - assert client.attempts == 2 - - class Unauthorized: - def request(self, *args: Any, **kwargs: Any) -> dict[str, Any]: - raise SubstrateAPIError("http_401") - - with pytest.raises(SubstrateAPIError): - _deliver_with_retry( - Unauthorized(), # type: ignore[arg-type] - {"kind": "turn", "event_id": "event-2"}, - ) diff --git a/tests/test_import_memory.py b/tests/test_import_memory.py deleted file mode 100644 index bb061ab..0000000 --- a/tests/test_import_memory.py +++ /dev/null @@ -1,45 +0,0 @@ -from __future__ import annotations - -import json -import subprocess -import sys -from pathlib import Path - -import pytest - -ROOT = Path(__file__).parents[1] -BENCHMARK = ROOT / "scripts" / "benchmark_import_memory.py" - - -@pytest.mark.skipif(sys.platform == "win32", reason="ru_maxrss byte accounting is POSIX-only") -# Each parameter case has a 900-second child kill bound; keep pytest bounded just above it. -@pytest.mark.timeout(960) -@pytest.mark.parametrize( - ("arguments", "single", "source"), - [ - (["--size-mib", "256"], False, "sqlite"), - (["--size-mib", "1", "--single-message-mib", "64"], True, "sqlite"), - ( - ["--size-mib", "1", "--single-message-mib", "64", "--source", "jsonl"], - True, - "jsonl", - ), - ], -) -def test_importer_rss_stays_below_256_mib( - arguments: list[str], single: bool, source: str -) -> None: - result = subprocess.run( - [sys.executable, str(BENCHMARK), *arguments, "--limit-mib", "256"], - cwd=ROOT, - check=False, - capture_output=True, - text=True, - timeout=900, - ) - assert result.returncode == 0, result.stdout + result.stderr - status = json.loads(result.stdout) - assert status["complete"] is True - assert status["single_message"] is single - assert status["source"] == source - assert status["peak_rss_bytes"] <= status["limit_bytes"] diff --git a/tests/test_memory_provider.py b/tests/test_memory_provider.py deleted file mode 100644 index 0ec6af3..0000000 --- a/tests/test_memory_provider.py +++ /dev/null @@ -1,586 +0,0 @@ -from __future__ import annotations - -import json -import sys -import threading -import time -from collections.abc import Callable -from datetime import UTC, datetime, timedelta -from email.message import Message -from email.utils import format_datetime -from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer -from io import BytesIO -from pathlib import Path -from typing import Any -from urllib.error import HTTPError - -import pytest - -PLUGIN_ROOT = Path(__file__).parents[1] / "src" -sys.path.insert(0, str(PLUGIN_ROOT)) - -from substrate_wiki import SubstrateWikiProvider, register # noqa: E402 -from substrate_wiki.client import SubstrateAPIError, SubstrateClient # noqa: E402 -from substrate_wiki.redaction import redact, redact_text # noqa: E402 -from substrate_wiki.spool import DurableSpool # noqa: E402 - - -class FakeClient: - def __init__(self) -> None: - self.calls: list[tuple[str, tuple[Any, ...], dict[str, Any]]] = [] - self.delivered: list[dict[str, Any]] = [] - self.fail = False - self.block = threading.Event() - self.should_block = False - self.block_timeout = 2.0 - - def search(self, *args: Any, **kwargs: Any) -> dict[str, Any]: - self.calls.append(("search", args, kwargs)) - return {"results": ["hit"]} - - def memory_search(self, *args: Any, **kwargs: Any) -> dict[str, Any]: - self.calls.append(("memory_search", args, kwargs)) - return { - "results": [ - { - "memory_card": "hit", - "path": "entities/project/hit--a1b2c3d4.md", - "canonical_path": "entities/project/hit--a1b2c3d4.md", - "page_type": "entity", - "entity_id": "entity-hit", - "entity_type": "project", - "quality_version": 2, - } - ] - } - - def read_page(self, *args: Any, **kwargs: Any) -> dict[str, Any]: - self.calls.append(("read_page", args, kwargs)) - return {"path": args[0]} - - def query_wiki(self, *args: Any, **kwargs: Any) -> dict[str, Any]: - self.calls.append(("query_wiki", args, kwargs)) - return {"answer": "cited"} - - def ingest(self, *args: Any, **kwargs: Any) -> dict[str, Any]: - self.calls.append(("ingest", args, kwargs)) - return {"job_id": "job-1"} - - def job_status(self, *args: Any, **kwargs: Any) -> dict[str, Any]: - self.calls.append(("job_status", args, kwargs)) - return {"status": "succeeded"} - - def request(self, method: str, path: str, **kwargs: Any) -> dict[str, Any]: - if self.should_block: - self.block.wait(self.block_timeout) - if self.fail: - raise SubstrateAPIError("transport_error") - self.delivered.append({"method": method, "path": path, **kwargs}) - return {} - - -def wait_until(predicate: Any, timeout: float = 2.0) -> None: - deadline = time.monotonic() + timeout - while time.monotonic() < deadline: - if predicate(): - return - time.sleep(0.01) - raise AssertionError("condition was not reached") - - -def assert_returns_while_network_is_blocked( - callback: Callable[[], Any], release_network: threading.Event -) -> None: - """Prove callback completion without a scheduler-sensitive wall-clock budget.""" - completed = threading.Event() - errors: list[BaseException] = [] - - def invoke() -> None: - try: - callback() - except BaseException as exc: # pragma: no cover - reraised in the test thread - errors.append(exc) - finally: - completed.set() - - caller = threading.Thread(target=invoke, name="blocked-network-caller", daemon=True) - caller.start() - returned_before_network = completed.wait(5.0) - if not returned_before_network: - release_network.set() - caller.join(timeout=2.0) - assert returned_before_network, "callback waited for the blocked network sender" - if errors: - raise errors[0] - - -@pytest.fixture -def configured_env(monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setenv("HERMES_API_URL", "https://wiki.example.test") - monkeypatch.setenv("HERMES_API_KEY", "very-secret-hermes-key") - - -def make_provider(tmp_path: Path, configured_env: None, monkeypatch: pytest.MonkeyPatch) -> tuple[SubstrateWikiProvider, FakeClient]: - fake = FakeClient() - monkeypatch.setattr("substrate_wiki.SubstrateClient.from_env", lambda **kwargs: fake) - provider = SubstrateWikiProvider() - provider.initialize("session-1", hermes_home=str(tmp_path)) - return provider, fake - - -def test_registers_provider_and_availability_is_local(monkeypatch: pytest.MonkeyPatch) -> None: - captured: list[Any] = [] - - class Context: - def register_memory_provider(self, provider: Any) -> None: - captured.append(provider) - - register(Context()) - assert isinstance(captured[0], SubstrateWikiProvider) - assert captured[0].name == "substrate_wiki" - assert captured[0].is_available() - monkeypatch.setenv("HERMES_API_URL", "https://other.example.test") - assert not captured[0].is_available() - monkeypatch.setenv("HERMES_API_URL", "https://app.trysubstrate.co") - assert captured[0].is_available() - - -@pytest.mark.parametrize( - ("base_url", "allowed"), - [ - ("https://wiki.example.test", True), - ("http://wiki.example.test", False), - ("http://localhost:8000", True), - ("http://localhost.:8000", True), - ("http://127.0.0.1:8000", True), - ("http://[::1]:8000", True), - ("http://0.0.0.0:8000", False), - ("ftp://wiki.example.test", False), - ("https://user:password@wiki.example.test", False), - ], -) -def test_client_rejects_insecure_or_credentialed_base_urls(base_url: str, allowed: bool) -> None: - if allowed: - assert SubstrateClient(base_url, "key").base_url == base_url.rstrip("/") - else: - with pytest.raises(SubstrateAPIError) as caught: - SubstrateClient(base_url, "secret-bearer-value") - assert "secret-bearer-value" not in str(caught.value) - - -def test_availability_rejects_non_loopback_http(monkeypatch: pytest.MonkeyPatch) -> None: - provider = SubstrateWikiProvider() - monkeypatch.setenv("HERMES_API_KEY", "key") - monkeypatch.setenv("HERMES_API_URL", "http://wiki.example.test") - assert not provider.is_available() - monkeypatch.setenv("HERMES_API_URL", "http://127.0.0.1:8000") - assert not provider.is_available() - - -def test_contract_surface_and_tool_schemas() -> None: - provider = SubstrateWikiProvider() - required = { - "initialize", - "get_tool_schemas", - "handle_tool_call", - "get_config_schema", - "save_config", - "shutdown", - "prefetch", - "queue_prefetch", - "sync_turn", - "on_pre_compress", - "on_session_end", - } - assert all(callable(getattr(provider, name)) for name in required) - names = {schema["name"] for schema in provider.get_tool_schemas()} - assert names == {"wiki_search", "wiki_read", "wiki_query", "wiki_ingest", "wiki_job_status"} - - -def test_client_maps_tools_to_dedicated_machine_routes(monkeypatch: pytest.MonkeyPatch) -> None: - calls: list[Any] = [] - - class Response: - def __enter__(self) -> Response: - return self - - def __exit__(self, *args: Any) -> None: - return None - - headers = {"Content-Type": "application/json"} - - def read(self, size: int = -1) -> bytes: - return b"{}"[:size] - - class Opener: - def open(self, request: Any, *, timeout: float) -> Response: - calls.append((request, timeout)) - return Response() - - monkeypatch.setattr("substrate_wiki.client.build_opener", lambda *handlers: Opener()) - client = SubstrateClient("https://wiki.example.test", "key") - client.search("alpha", limit=3) - client.read_page("topics/a b.md") - client.query_wiki("Why?", save_as_synthesis=True) - client.ingest("source", title="Title") - client.job_status("job/1") - - assert [(call.full_url, call.get_method()) for call, _ in calls] == [ - ("https://wiki.example.test/api/v1/hermes/wiki/search", "POST"), - ("https://wiki.example.test/api/v1/hermes/wiki/read", "POST"), - ("https://wiki.example.test/api/v1/hermes/wiki/query", "POST"), - ("https://wiki.example.test/api/v1/hermes/wiki/ingest", "POST"), - ("https://wiki.example.test/api/v1/hermes/wiki/job-status?job_id=job%2F1", "GET"), - ] - assert json.loads(calls[0][0].data) == {"q": "alpha", "limit": 3} - assert json.loads(calls[1][0].data) == {"path": "topics/a b.md"} - - -def test_redirect_is_not_followed_and_authorization_stays_at_origin() -> None: - destination_headers: list[str | None] = [] - - class DestinationHandler(BaseHTTPRequestHandler): - def do_GET(self) -> None: - destination_headers.append(self.headers.get("Authorization")) - self.send_response(200) - self.end_headers() - self.wfile.write(b"{}") - - do_POST = do_GET - - def log_message(self, format: str, *args: Any) -> None: - return None - - destination = ThreadingHTTPServer(("127.0.0.1", 0), DestinationHandler) - destination_thread = threading.Thread(target=destination.serve_forever, daemon=True) - destination_thread.start() - redirect_url = f"http://127.0.0.1:{destination.server_port}/stolen" - - class RedirectHandler(BaseHTTPRequestHandler): - def do_GET(self) -> None: - self.send_response(302) - self.send_header("Location", redirect_url) - self.end_headers() - - do_POST = do_GET - - def log_message(self, format: str, *args: Any) -> None: - return None - - origin = ThreadingHTTPServer(("127.0.0.1", 0), RedirectHandler) - origin_thread = threading.Thread(target=origin.serve_forever, daemon=True) - origin_thread.start() - try: - client = SubstrateClient( - f"http://127.0.0.1:{origin.server_port}", "redirect-secret-bearer" - ) - with pytest.raises(SubstrateAPIError, match="http_302") as caught: - client.search("redirect") - assert "redirect-secret-bearer" not in str(caught.value) - assert destination_headers == [] - finally: - origin.shutdown() - destination.shutdown() - origin.server_close() - destination.server_close() - origin_thread.join(timeout=2) - destination_thread.join(timeout=2) - - -def test_http_errors_hide_response_body_and_bearer(monkeypatch: pytest.MonkeyPatch) -> None: - secret = "response-body-secret" - - class Opener: - def open(self, request: Any, *, timeout: float) -> Any: - raise HTTPError( - request.full_url, - 500, - "Bearer leaked-key", - Message(), - BytesIO(secret.encode()), - ) - - monkeypatch.setattr("substrate_wiki.client.build_opener", lambda *handlers: Opener()) - client = SubstrateClient("https://wiki.example.test", "request-bearer-secret") - with pytest.raises(SubstrateAPIError) as caught: - client.search("alpha") - rendered = str(caught.value) - assert rendered == "http_500" - assert caught.value.retry_after is None - assert secret not in rendered - assert "request-bearer-secret" not in rendered - assert "leaked-key" not in rendered - - -def test_http_429_exposes_only_sanitized_retry_after(monkeypatch: pytest.MonkeyPatch) -> None: - retry_after = "17" - - class Opener: - def open(self, request: Any, *, timeout: float) -> Any: - headers = Message() - headers["Retry-After"] = retry_after - headers["X-Secret"] = "must-not-surface" - raise HTTPError( - request.full_url, - 429, - "secret reason", - headers, - BytesIO(b"secret response body"), - ) - - monkeypatch.setattr("substrate_wiki.client.build_opener", lambda *handlers: Opener()) - client = SubstrateClient("https://wiki.example.test", "request-secret") - with pytest.raises(SubstrateAPIError) as caught: - client.search("alpha") - assert str(caught.value) == "http_429" - assert caught.value.retry_after == 17 - assert "must-not-surface" not in str(caught.value) - assert "request-secret" not in str(caught.value) - - -def test_retry_after_http_date_and_malformed_values() -> None: - from substrate_wiki.client import _retry_after_seconds - - now = datetime(2026, 7, 15, 12, 0, tzinfo=UTC) - future = format_datetime(now + timedelta(seconds=45), usegmt=True) - assert _retry_after_seconds({"Retry-After": future}, now=now) == 45 - assert _retry_after_seconds({"Retry-After": "not-a-delay"}, now=now) is None - assert _retry_after_seconds({"Retry-After": "-1"}, now=now) is None - - -def test_tools_route_to_client(tmp_path: Path, configured_env: None, monkeypatch: pytest.MonkeyPatch) -> None: - provider, fake = make_provider(tmp_path, configured_env, monkeypatch) - try: - assert json.loads(provider.handle_tool_call("wiki_search", {"query": "alpha", "limit": 99})) == {"results": ["hit"]} - assert json.loads(provider.handle_tool_call("wiki_read", {"path": "topics/a.md"})) == {"path": "topics/a.md"} - assert json.loads(provider.handle_tool_call("wiki_query", {"question": "Why?"})) == {"answer": "cited"} - assert json.loads(provider.handle_tool_call("wiki_ingest", {"content": "source"})) == {"job_id": "job-1"} - assert json.loads(provider.handle_tool_call("wiki_job_status", {"job_id": "job-1"})) == {"status": "succeeded"} - assert fake.calls[0] == ("search", ("alpha",), {"limit": 25}) - assert json.loads(provider.handle_tool_call("wiki_read", {})) == {"error": "invalid_arguments"} - assert "error" in json.loads(provider.handle_tool_call("not_a_tool", {})) - finally: - provider.shutdown() - - -def test_config_state_stays_under_hermes_home_and_excludes_secrets( - tmp_path: Path, configured_env: None -) -> None: - provider = SubstrateWikiProvider() - provider.save_config( - { - "api_url": "https://must-not-be-saved.example", - "api_key": "must-not-be-saved", - "spool_max_items": 12, - }, - str(tmp_path), - ) - path = tmp_path / "substrate_wiki" / "config.json" - values = json.loads(path.read_text(encoding="utf-8")) - assert values["spool_max_items"] == 12 - assert values["api_url"] == "https://app.trysubstrate.co" - assert "api_key" not in values - assert '"api_key"' not in path.read_text(encoding="utf-8") - assert provider.get_config_schema() == [] - - -def test_sync_turn_returns_without_waiting_for_network_and_redacts( - tmp_path: Path, configured_env: None, monkeypatch: pytest.MonkeyPatch -) -> None: - provider, fake = make_provider(tmp_path, configured_env, monkeypatch) - fake.should_block = True - fake.block_timeout = 30.0 - fake_bearer = "abcdefghijkl" + "mnop" - authorization_header = ": ".join(("Authorization", "Bearer " + fake_bearer)) - assert_returns_while_network_is_blocked( - lambda: provider.sync_turn( - authorization_header, - "api_key=plain-secret and very-secret-hermes-key", - ), - fake.block, - ) - fake.block.set() - wait_until(lambda: len(fake.delivered) == 1) - event = fake.delivered[0]["body"] - rendered = json.dumps(event) - assert fake_bearer not in rendered - assert "plain-secret" not in rendered - assert "very-secret-hermes-key" not in rendered - assert rendered.count("[REDACTED]") >= 1 - assert fake.delivered[0]["idempotency_key"] == event["event_id"] - provider.shutdown() - - -def test_lifecycle_hooks_return_without_waiting_for_network( - tmp_path: Path, configured_env: None, monkeypatch: pytest.MonkeyPatch -) -> None: - provider, fake = make_provider(tmp_path, configured_env, monkeypatch) - fake.should_block = True - fake.block_timeout = 30.0 - try: - assert_returns_while_network_is_blocked( - lambda: ( - provider.on_pre_compress([{"role": "user", "content": "x"}]), - provider.on_session_end([{"role": "assistant", "content": "y"}]), - provider.on_memory_write("write", "MEMORY.md", "candidate"), - ), - fake.block, - ) - fake.block.set() - assert fake.delivered == [] - finally: - fake.block.set() - provider.shutdown() - - -def test_offline_events_spool_and_replay_oldest_first( - tmp_path: Path, configured_env: None, monkeypatch: pytest.MonkeyPatch -) -> None: - provider, fake = make_provider(tmp_path, configured_env, monkeypatch) - fake.fail = True - provider.sync_turn("first", "answer one") - provider.sync_turn("second", "answer two") - spool_dir = tmp_path / "substrate_wiki" / "spool" - wait_until(lambda: len(list(spool_dir.glob("*.json"))) >= 1) - fake.fail = False - provider._wake.set() - wait_until(lambda: len(fake.delivered) == 2, timeout=4) - assert [call["body"]["messages"][0]["content"] for call in fake.delivered] == ["first", "second"] - assert len({call["idempotency_key"] for call in fake.delivered}) == 2 - wait_until(lambda: not list(spool_dir.glob("*.json"))) - provider.shutdown() - - -def test_spool_is_bounded_and_discards_oldest(tmp_path: Path) -> None: - spool = DurableSpool(tmp_path / "spool", max_items=2, max_bytes=4096) - spool.append({"n": 1}) - time.sleep(0.001) - spool.append({"n": 2}) - time.sleep(0.001) - spool.append({"n": 3}) - files = sorted((tmp_path / "spool").glob("*.json")) - assert len(files) == 2 - assert [json.loads(path.read_text(encoding="utf-8"))["n"] for path in files] == [2, 3] - - -def test_prefetch_is_cached_and_lifecycle_hooks_are_delivered( - tmp_path: Path, configured_env: None, monkeypatch: pytest.MonkeyPatch -) -> None: - provider, fake = make_provider(tmp_path, configured_env, monkeypatch) - try: - assert provider.prefetch("topic") == "" - provider.queue_prefetch("topic") - wait_until(lambda: provider.prefetch("topic") != "") - assert "entities/project/hit--a1b2c3d4.md" in provider.prefetch("topic") - provider.on_pre_compress([{"role": "user", "content": "x"}]) - provider.on_session_end([{"role": "assistant", "content": "y"}]) - provider.on_memory_write("write", "MEMORY.md", "candidate") - assert fake.delivered == [] - finally: - provider.shutdown() - - -def test_recursive_redaction_handles_headers_patterns_and_exact_secrets() -> None: - value = { - "Authorization": "Bearer top-secret", - "nested": ["password=hunter2", "exact-value", {"api_key": "xyz"}], - } - rendered = json.dumps(redact(value, ("exact-value",))) - assert "top-secret" not in rendered - assert "hunter2" not in rendered - assert "exact-value" not in rendered - assert "xyz" not in rendered - - -def test_redaction_covers_structured_partial_and_url_credentials() -> None: - stripe_marker = "«redacted:" + "sk" + "_live_…»" - samples = { - "json": '{"client_secret":"fake-high-entropy-value-123456"}', - "yaml": "refresh_token: fake-refresh-value-123456", - "shell": "export SIGNING_KEY='fake-signing-value-123456'", - "url": "https://demo-user:fake-pass-123456@example.test/path", - "provider": "«redacted:github_pat_…»", - "masked_provider": "sk-••••••••1234", - "stripe_provider": stripe_marker, - "jwt": "eyJfak...re12", - "partial": "api_key=«redacted:sk-…»", - } - - rendered = json.dumps(redact(samples, ())) - - for unsafe in ( - "fake-high-entropy-value-123456", - "fake-refresh-value-123456", - "fake-signing-value-123456", - "fake-pass-123456", - "«redacted:github_pat_…»", - "sk-••••••••1234", - stripe_marker, - - "sk-fakepartial123456", - ): - assert unsafe not in rendered - assert redact_text("skills remain useful", ()) == "skills remain useful" - - -def test_redaction_covers_signed_urls_non_http_userinfo_and_partial_private_keys() -> None: - samples = { - "database": "postgresql://db-user:databasePassword_987654321@db.invalid/app", - "sas": ( - "https://blob.invalid/item?sv=2023-11-03&sp=r" - "&sig=sasSignature_987654321" - ), - "aws": ( - "https://bucket.invalid/item?X-Amz-Credential=" - "AKIAEXAMPLEONLY1234%2Fscope&X-Amz-Security-Token=" - "temporarySession_987654321&X-Amz-Signature=" - "0123456789abcdef0123456789abcdef" - ), - "private_key": ( - "-----BEGIN OPENSSH PRIVATE KEY-----\n" - "b3BlbnNzaC1rZXktdjEAAAAAexampleonly987654321" - ), - } - - rendered = json.dumps(redact(samples, ())) - - for unsafe in ( - "db-user", - "databasePassword_987654321", - "sasSignature_987654321", - "AKIAEXAMPLEONLY1234%2Fscope", - "temporarySession_987654321", - "0123456789abcdef0123456789abcdef", - "b3BlbnNzaC1rZXktdjEAAAAAexampleonly987654321", - ): - assert unsafe not in rendered - - -def test_redaction_preserves_many_message_boundaries() -> None: - messages = [{"role": "user", "content": "x"} for _ in range(8_200)] - - sanitized = redact(messages, ()) - - assert len(sanitized) == len(messages) - assert sanitized[0] == {"role": "user", "content": "x"} - assert sanitized[-1] == {"role": "user", "content": "x"} - - -def test_redaction_bounds_pathological_container_amplification() -> None: - values = [{"value": index} for index in range(20_000)] - - sanitized = redact(values, ()) - - assert len(sanitized) == 16_384 - assert sanitized[-1] == {"value": 16_383} - - -def test_hosted_custody_refuses_arbitrary_legacy_origin(tmp_path: Path, monkeypatch) -> None: - from substrate_wiki.client import SubstrateAPIError, SubstrateClient - from substrate_wiki.credentials import credential_store - - credential_store(tmp_path).put("hosted-tenant-secret") - monkeypatch.setenv("HERMES_API_URL", "https://attacker.example") - monkeypatch.delenv("HERMES_API_KEY", raising=False) - with pytest.raises(SubstrateAPIError, match="unsafe_hosted_origin_override"): - SubstrateClient.from_env(hermes_home=tmp_path, hosted_default=True) diff --git a/tests/test_migration_baseline.py b/tests/test_migration_baseline.py deleted file mode 100644 index f948df1..0000000 --- a/tests/test_migration_baseline.py +++ /dev/null @@ -1,376 +0,0 @@ -from __future__ import annotations - -import hashlib -import json -import os -import subprocess -import sys -from pathlib import Path -from typing import Any - -import pytest - -from scripts.benchmark_migration import ( - PHASE_NAMES, - evaluate_budget, - load_manifest, - validate_receipt, -) - -ROOT = Path(__file__).parents[1] -BENCHMARK = ROOT / "scripts" / "benchmark_migration.py" -VERIFIER = ROOT / "scripts" / "verify_migration_baseline.py" -MANIFEST = ROOT / "benchmarks" / "hermes-migration-manifest.json" -SCHEMA = ROOT / "benchmarks" / "hermes-migration-receipt.schema.json" - - -def test_manifest_covers_required_sources_scales_and_stream_v2_concurrency() -> None: - manifest = load_manifest(MANIFEST) - - cases = {case["id"]: case for case in manifest["cases"]} - assert { - "small-sqlite", - "current-production-jsonl", - "large-sqlite", - "oversized-message-jsonl", - "sqlite-adapter", - "official-export-jsonl", - } <= set(cases) - assert {case["source"] for case in cases.values()} == {"sqlite", "jsonl"} - assert {case["fixture_kind"] for case in cases.values()} >= { - "sqlite", - "official_export_jsonl", - } - assert manifest["protocol"] == "stream-v2" - assert manifest["concurrency"] == [1, 2, 3, 4] - assert cases["current-production-jsonl"]["sessions"] == 1212 - assert cases["large-sqlite"]["sessions"] >= 2048 - assert cases["oversized-message-jsonl"]["oversized_message_bytes"] > 262_144 - - -def test_receipt_schema_requires_every_phase_and_lifecycle_boundary() -> None: - schema = json.loads(SCHEMA.read_text(encoding="utf-8")) - run = schema["$defs"]["run"] - assert set(PHASE_NAMES) == set(run["properties"]["phase_seconds"]["required"]) - assert run["properties"]["lifecycle_seconds"]["required"] == [ - "transferred", - "first_usable", - "fully_ready", - ] - assert schema["additionalProperties"] is False - assert run["additionalProperties"] is False - - -# The benchmark child has a 240-second kill bound; preserve 60 seconds for verification/cleanup. -@pytest.mark.timeout(300) -def test_schema_and_budget_verifier_accepts_a_fresh_contract_matrix(tmp_path: Path) -> None: - receipt_path = tmp_path / "receipt.json" - budget_path = tmp_path / "budget.json" - benchmark = subprocess.run( - [ - sys.executable, - str(BENCHMARK), - "--profile", - "contract", - "--output", - str(receipt_path), - ], - cwd=ROOT, - check=False, - capture_output=True, - text=True, - timeout=240, - ) - assert benchmark.returncode == 0, benchmark.stdout + benchmark.stderr - verification = subprocess.run( - [ - sys.executable, - str(VERIFIER), - "--receipt", - str(receipt_path), - "--budget", - str(budget_path), - "--write-budget", - ], - cwd=ROOT, - check=False, - capture_output=True, - text=True, - timeout=60, - ) - assert verification.returncode == 0, verification.stdout + verification.stderr - result = json.loads(verification.stdout) - assert result["runs"] == 4 - assert result["schema"] == "draft-2020-12" - assert result["status"] == "pass" - - -def test_default_verifier_selects_retained_canonical_receipt() -> None: - verification = subprocess.run( - [sys.executable, str(VERIFIER)], - cwd=ROOT, - check=False, - capture_output=True, - text=True, - timeout=60, - ) - assert verification.returncode == 0, verification.stdout + verification.stderr - assert json.loads(verification.stdout)["receipt"].endswith( - "benchmarks/evidence/hermes-migration-baseline.json" - ) - - -def test_single_case_worker_reports_process_local_rss(tmp_path: Path) -> None: - fixture_path = tmp_path / "state.db" - row_path = tmp_path / "row.json" - generated = subprocess.run( - [ - sys.executable, - str(BENCHMARK), - "--generate-case", - "small-sqlite", - "--output", - str(fixture_path), - ], - cwd=ROOT, - check=False, - capture_output=True, - text=True, - timeout=60, - ) - assert generated.returncode == 0, generated.stdout + generated.stderr - - worker = subprocess.run( - [ - sys.executable, - str(BENCHMARK), - "--run-case", - "small-sqlite", - "--concurrency", - "1", - "--fixture", - str(fixture_path), - "--output", - str(row_path), - ], - cwd=ROOT, - check=False, - capture_output=True, - text=True, - timeout=60, - ) - assert worker.returncode == 0, worker.stdout + worker.stderr - row = json.loads(row_path.read_text(encoding="utf-8")) - assert row["case_id"] == "small-sqlite" - assert row["concurrency"] == 1 - assert 0 < row["peak_rss_bytes"] < 256 * 1024 * 1024 - - -def test_idempotency_replay_covers_side_effecting_event_and_full_durable_state() -> None: - receipt = json.loads( - (ROOT / "benchmarks" / "evidence" / "hermes-migration-baseline.json").read_text( - encoding="utf-8" - ) - ) - expected_components = [ - "duplicate_acks", - "encoded_bytes", - "max_queue_depth", - "projected_digest", - "projected_rows", - "queue_items", - "received_digest", - "received_rows", - "redacted_events", - "redaction_failures", - "requests", - "work_queue_items", - ] - assert all( - run["integrity"]["idempotency_replay_kind"] == "session_end" - and run["integrity"]["idempotency_state_components"] == expected_components - and run["integrity"]["idempotency_replays"] == 1 - and run["integrity"]["duplicate_acks"] == 1 - and run["integrity"]["duplicate_side_effects"] == 0 - for run in receipt["runs"] - ) - - -def test_every_child_process_denies_socket_construction(tmp_path: Path) -> None: - env = dict(os.environ) - env["SUBSTRATE_BENCHMARK_NETWORK_DENIED"] = "1" - probe = subprocess.run( - [sys.executable, str(BENCHMARK), "--probe-network", "--output", str(tmp_path / "unused")], - cwd=ROOT, - env=env, - check=False, - capture_output=True, - text=True, - timeout=30, - ) - assert probe.returncode != 0 - assert "network access is disabled for this benchmark" in probe.stderr - - -def test_partial_startup_probe_terminates_without_barrier_hang(tmp_path: Path) -> None: - probe = subprocess.run( - [ - sys.executable, - str(BENCHMARK), - "--probe-partial-failure", - "--output", - str(tmp_path / "unused"), - ], - cwd=ROOT, - check=False, - capture_output=True, - text=True, - timeout=5, - ) - assert probe.returncode == 0, probe.stdout + probe.stderr - - -@pytest.mark.parametrize("case_id", ["../escape", "/absolute", "bad/slash", "UPPER"]) -def test_manifest_rejects_unsafe_case_ids(tmp_path: Path, case_id: str) -> None: - manifest = json.loads(MANIFEST.read_text(encoding="utf-8")) - manifest["cases"][0]["id"] = case_id - path = tmp_path / "manifest.json" - path.write_text(json.dumps(manifest), encoding="utf-8") - with pytest.raises(ValueError, match="case id"): - load_manifest(path) - - -# The benchmark child has a 240-second kill bound; preserve 60 seconds for receipt checks. -@pytest.mark.timeout(300) -def test_ci_profile_emits_content_free_aggregate_receipt(tmp_path: Path) -> None: - receipt_path = tmp_path / "receipt.json" - result = subprocess.run( - [ - sys.executable, - str(BENCHMARK), - "--manifest", - str(MANIFEST), - "--profile", - "contract", - "--output", - str(receipt_path), - ], - cwd=ROOT, - check=False, - capture_output=True, - text=True, - timeout=240, - ) - - assert result.returncode == 0, result.stdout + result.stderr - receipt = json.loads(receipt_path.read_text(encoding="utf-8")) - validate_receipt(receipt) - assert receipt["schema_version"] == 1 - assert receipt["protocol"] == "stream-v2" - assert receipt["harness_sha256"] == hashlib.sha256(BENCHMARK.read_bytes()).hexdigest() - assert receipt["hosted_calls"] == 0 - assert receipt["representative_provider"]["mode"] == "deterministic_simulation" - assert {run["concurrency"] for run in receipt["runs"]} == {1, 2, 3, 4} - assert all(set(run["phase_seconds"]) == set(PHASE_NAMES) for run in receipt["runs"]) - assert all( - run["lifecycle_seconds"]["transferred"] < run["lifecycle_seconds"]["fully_ready"] - and run["lifecycle_seconds"]["first_usable"] < run["lifecycle_seconds"]["fully_ready"] - and run["lifecycle_seconds"]["first_usable"] != run["lifecycle_seconds"]["transferred"] - for run in receipt["runs"] - ) - assert all( - 1 <= run["provider"]["max_in_flight"] <= run["concurrency"] - and 1 <= run["provider"]["worker_threads_used"] <= run["concurrency"] - for run in receipt["runs"] - ) - assert max(run["provider"]["max_in_flight"] for run in receipt["runs"]) == 4 - assert all( - run["integrity"]["idempotency_replays"] == 1 - and run["integrity"]["duplicate_acks"] == 1 - and run["integrity"]["duplicate_side_effects"] == 0 - for run in receipt["runs"] - ) - serialized = json.dumps(receipt, sort_keys=True).casefold() - for forbidden in ( - "prompt", - "transcript", - "api_key", - "secret", - "password", - "sk-benchmark-canary", - ): - assert forbidden not in serialized - - -def test_receipt_validator_rejects_content_bearing_trace_fields() -> None: - with pytest.raises(ValueError, match="content-free"): - validate_receipt({"runs": [{"prompt": "source content"}]}) - - -@pytest.mark.parametrize( - "receipt", - [ - {"runs": [{"source_locator": "/private/history.jsonl"}]}, - {"runs": [{"session_id": "private-session"}]}, - {"runs": [{"trace": "Authorization: Bearer private"}]}, - {"runs": [{"trace": "sk-private-looking-value"}]}, - ], -) -def test_receipt_validator_rejects_source_identifiers_and_secret_shapes( - receipt: dict[str, object], -) -> None: - with pytest.raises(ValueError, match="content-free"): - validate_receipt(receipt) - - -def test_budget_gate_covers_speed_resources_integrity_quality_quota_and_cost() -> None: - budget = { - "max_transferred_seconds": 2.0, - "max_first_usable_seconds": 3.0, - "max_fully_ready_seconds": 4.0, - "max_peak_rss_bytes": 256 * 1024 * 1024, - "max_cpu_seconds": 2.0, - "max_terminal_failures": 0, - "max_projection_failures": 0, - "max_redaction_failures": 0, - "min_redacted_events": 1, - "min_duplicate_acks": 1, - "max_duplicate_side_effects": 0, - "min_integrity_ratio": 1.0, - "max_provider_retries": 0, - "max_modeled_cost_usd": 0.01, - "max_modeled_quota_seconds": 60.0, - } - run: dict[str, Any] = { - "case_id": "synthetic", - "concurrency": 1, - "lifecycle_seconds": { - "transferred": 1.0, - "first_usable": 2.0, - "fully_ready": 3.0, - }, - "peak_rss_bytes": 32 * 1024 * 1024, - "cpu_seconds": 1.0, - "terminal_failures": 0, - "integrity": { - "expected_windows": 4, - "projected_windows": 4, - "redacted_events": 4, - "redaction_failures": 0, - "idempotency_replays": 1, - "duplicate_acks": 1, - "duplicate_side_effects": 0, - "complete": True, - }, - "quality": {"projection_failures": 0, "terminal_failures": 0}, - "provider": { - "retries": 0, - "modeled_cost_usd": 0.001, - "modeled_quota_seconds": 4.0, - }, - } - assert evaluate_budget(run, budget) == [] - - run["integrity"]["projected_windows"] = 3 - failures = evaluate_budget(run, budget) - assert any("integrity_ratio" in failure for failure in failures) diff --git a/tests/test_onboarding.py b/tests/test_onboarding.py deleted file mode 100644 index d91685f..0000000 --- a/tests/test_onboarding.py +++ /dev/null @@ -1,368 +0,0 @@ -from __future__ import annotations - -import json - -import pytest -from pathlib import Path - -from substrate_wiki.onboarding import ( - HOSTED_ORIGIN, - HostedOAuthClient, - OnboardingError, - OnboardingManager, - _prompt_history, -) - - -class Store: - backend = "test-vault" - def __init__(self): self.values = {} - def get(self, slot="access-token"): return self.values.get(slot, "") - def put(self, value, slot="access-token"): self.values[slot] = value - def delete(self, slot="access-token"): self.values.pop(slot, None) - - -class API: - def __init__(self): self.poll_count = 0 - def begin(self): - return {"device_code": "device-secret", "user_code": "ABCD-EFGH", - "verification_uri": HOSTED_ORIGIN + "/oauth/device", - "verification_uri_complete": HOSTED_ORIGIN + "/oauth/device?user_code=ABCD-EFGH", - "expires_in": 600, "interval": 1} - def poll(self, device_code): - assert device_code == "device-secret" - self.poll_count += 1 - return {"status": "approved", "access_token": "tenant-secret"} - - -def test_device_credentials_never_enter_state_and_consent_decline_keeps_connection(tmp_path: Path): - store = Store() - manager = OnboardingManager( - tmp_path, api=API(), store=store, capability_check=lambda token: {"ok": token}, - import_start=lambda home: (_ for _ in ()).throw(AssertionError("must not import")), - opener=lambda url: True, - ) - started = manager.begin(mode="device", open_browser=False) - assert started["user_code"] == "ABCD-EFGH" - raw = (tmp_path / "substrate_wiki/onboarding/state.json").read_text() - assert "device-secret" not in raw and "tenant-secret" not in raw - connected = manager.advance() - assert connected["phase"] == "awaiting_history_consent" - ready = manager.consent_history(False) - assert ready["phase"] == "ready" and ready["authenticated"] - state = json.loads((tmp_path / "substrate_wiki/onboarding/state.json").read_text()) - assert state["history_consent"]["decision"] == "declined" - - -def test_transient_oauth_poll_failure_preserves_grant_and_retries(tmp_path: Path): - class TransientPollAPI(API): - def poll(self, device_code): - assert device_code == "device-secret" - self.poll_count += 1 - if self.poll_count == 1: - raise OnboardingError("transport_error") - return {"status": "approved", "access_token": "tenant-secret"} - - store = Store() - api = TransientPollAPI() - manager = OnboardingManager( - tmp_path, api=api, store=store, capability_check=lambda token: {"ok": token} - ) - manager.begin(mode="device", open_browser=False) - - pending = manager.advance() - assert pending["phase"] == "authorization_pending" - assert pending["oauth_poll_failure"] == "transport_error" - assert store.get("onboarding-device") == "device-secret" - assert store.get() == "" - - connected = manager.advance() - assert api.poll_count == 2 - assert connected["phase"] == "awaiting_history_consent" - assert "oauth_poll_failure" not in connected - assert store.get() == "tenant-secret" - assert store.get("onboarding-device") == "" - - -def test_permanent_oauth_poll_failure_remains_fail_closed(tmp_path: Path): - class InvalidPollAPI(API): - def poll(self, device_code): - raise OnboardingError("invalid_response") - - store = Store() - manager = OnboardingManager(tmp_path, api=InvalidPollAPI(), store=store) - manager.begin(mode="device", open_browser=False) - - with pytest.raises(OnboardingError, match="invalid_response"): - manager.advance() - assert store.get("onboarding-device") == "device-secret" - assert store.get() == "" - - -def test_history_approval_starts_exactly_one_durable_job(tmp_path: Path): - store = Store() - store.put("tenant-secret") - calls = [] - manager = OnboardingManager( - tmp_path, store=store, capability_check=lambda token: {}, - import_start=lambda home: calls.append(home) or {"job_id": "job-1", "complete": False}, - ) - assert manager.begin()["phase"] == "awaiting_history_consent" - result = manager.consent_history(True) - assert result["phase"] == "importing" - assert calls == [tmp_path.resolve()] - assert manager.status()["phase"] == "importing" - - -def test_history_consent_is_durable_before_import_launch(tmp_path: Path): - store = Store() - store.put("tenant-secret") - - def launch(_home): - state = json.loads((tmp_path / "substrate_wiki/onboarding/state.json").read_text()) - assert state["history_consent"]["decision"] == "approved" - assert state["phase"] == "import_starting" - return {"job_id": "job-1", "complete": False} - - manager = OnboardingManager( - tmp_path, store=store, capability_check=lambda token: {}, import_start=launch - ) - manager.begin() - assert manager.consent_history(True)["phase"] == "importing" - - -def test_device_run_prints_complete_email_authorization_url(tmp_path: Path, capsys): - store = Store() - manager = OnboardingManager( - tmp_path, - api=API(), - store=store, - capability_check=lambda token: {"ok": token}, - opener=lambda url: True, - ) - result = manager.run(mode="device", wait=True, open_browser=False, timeout=5) - assert result["phase"] == "awaiting_history_consent" - output = capsys.readouterr().err - assert ( - f"Open {HOSTED_ORIGIN}/oauth/device?user_code=ABCD-EFGH " - "to sign in by email and connect Hermes" - ) in output - assert f"Open {HOSTED_ORIGIN}/oauth/device and enter" not in output - - -def test_device_response_constructs_complete_url_when_server_omits_it(monkeypatch): - client = HostedOAuthClient() - monkeypatch.setattr( - client, - "_post", - lambda path, values: ( - 200, - { - "device_code": "secret", - "user_code": "ABCD-EFGH", - "verification_uri": HOSTED_ORIGIN + "/oauth/device", - "expires_in": 600, - }, - ), - ) - assert client.begin()["verification_uri_complete"] == ( - HOSTED_ORIGIN + "/oauth/device?user_code=ABCD-EFGH" - ) - - -def test_device_response_rejects_complete_url_for_a_different_code(monkeypatch): - client = HostedOAuthClient() - monkeypatch.setattr( - client, - "_post", - lambda path, values: ( - 200, - { - "device_code": "secret", - "user_code": "ABCD-EFGH", - "verification_uri": HOSTED_ORIGIN + "/oauth/device", - "verification_uri_complete": ( - HOSTED_ORIGIN + "/oauth/device?user_code=DIFFERENT" - ), - "expires_in": 600, - }, - ), - ) - with pytest.raises(OnboardingError, match="invalid_response"): - client.begin() - - -def _valid_hosted_capabilities() -> dict[str, object]: - return { - "provider": "substrate_wiki", - "capture_schema_versions": [2], - "max_event_bytes": 262_144, - "history_replay": { - "protocol": "stream-v2", - "min_plugin_version": "1.2.0", - "content_free_completion": True, - "incremental_windows": True, - "status_version": 2, - }, - "entity_memory": { - "protocol": "entity-wiki-v1", - "min_plugin_version": "1.3.0", - "search_endpoint": "/api/v1/hermes/memory/search", - "canonical_wiki_pages": True, - "entity_page_type": "entity", - }, - "entity_quality": { - "protocol": "entity-quality-v2", - "min_plugin_version": "1.4.0", - "memory_card": True, - "quality_version": 2, - "canonical_redirects": True, - }, - } - - -def test_setup_handshake_requires_history_upload_but_not_entity_features( - tmp_path: Path, monkeypatch -): - from substrate_wiki.client import SubstrateClient - - capabilities = _valid_hosted_capabilities() - capabilities.pop("entity_memory") - capabilities.pop("entity_quality") - monkeypatch.setattr(SubstrateClient, "capabilities", lambda _client: capabilities) - - manager = OnboardingManager(tmp_path, store=Store()) - assert manager._check_capabilities("tenant-secret") == { - "provider": "substrate_wiki", - "protocol": "stream-v2", - } - - -@pytest.mark.parametrize( - ("status", "expected"), - [(503, "http_503"), (400, "invalid_response")], -) -def test_oauth_error_body_cannot_control_failure_category( - monkeypatch, status: int, expected: str -): - client = HostedOAuthClient() - hostile = "server-controlled credential sk_live_must_not_escape" - monkeypatch.setattr( - client, "_post", lambda path, values: (status, {"error": hostile}) - ) - - with pytest.raises(OnboardingError) as caught: - client.poll("device-secret") - assert caught.value.category == expected - assert hostile not in str(caught.value) - - -def test_capability_check_retries_one_tenant_cold_start(tmp_path: Path, monkeypatch): - from substrate_wiki.client import SubstrateAPIError, SubstrateClient - from substrate_wiki import onboarding - - calls = [] - - def capabilities(_client): - calls.append(True) - if len(calls) == 1: - raise SubstrateAPIError("timeout") - return _valid_hosted_capabilities() - - monkeypatch.setattr(SubstrateClient, "capabilities", capabilities) - monkeypatch.setattr(onboarding.time, "sleep", lambda _seconds: None) - store = Store() - manager = OnboardingManager(tmp_path, api=API(), store=store, opener=lambda _url: True) - - manager.begin(mode="device", open_browser=False) - result = manager.advance() - - assert len(calls) == 2 - assert result["phase"] == "awaiting_history_consent" - assert result["authenticated"] is True - assert store.get() == "tenant-secret" - assert store.get("onboarding-device") == "" - - -def test_permanent_capability_failure_is_not_retried_and_is_diagnostic( - tmp_path: Path, monkeypatch -): - from substrate_wiki.client import SubstrateAPIError, SubstrateClient - - calls = [] - - def capabilities(_client): - calls.append(True) - raise SubstrateAPIError("server_upgrade_required") - - monkeypatch.setattr(SubstrateClient, "capabilities", capabilities) - store = Store() - manager = OnboardingManager(tmp_path, api=API(), store=store, opener=lambda _url: True) - - manager.begin(mode="device", open_browser=False) - result = manager.advance() - - assert len(calls) == 1 - assert result["phase"] == "failed" - assert result["error_class"] == "capability_check_failed" - assert result["capability_failure"] == "server_upgrade_required" - assert result["authenticated"] is False - assert store.get() == "" - - -def test_pending_history_consent_is_explicit_and_idempotent(tmp_path: Path): - store = Store() - store.put("tenant-secret") - launches = [] - manager = OnboardingManager( - tmp_path, - store=store, - capability_check=lambda token: {}, - import_start=lambda home: launches.append(home) or {"job_id": "job-1", "complete": False}, - ) - pending = manager.begin() - assert pending["phase"] == "awaiting_history_consent" - assert pending["action_required"] == "history_consent" - - first = manager.consent_history(True) - second = manager.consent_history(True) - assert first["phase"] == second["phase"] == "importing" - assert launches == [tmp_path.resolve()] - with pytest.raises(OnboardingError, match="history_consent_not_pending"): - manager.consent_history(False) - - -def test_blank_history_answer_stays_pending(tmp_path: Path, monkeypatch): - class InteractiveBlank: - def isatty(self): return True - def readline(self): return "\n" - - store = Store() - store.put("tenant-secret") - manager = OnboardingManager(tmp_path, store=store, capability_check=lambda token: {}) - manager.begin() - monkeypatch.setattr("substrate_wiki.onboarding.sys.stdin", InteractiveBlank()) - result = _prompt_history(manager) - assert result["phase"] == "awaiting_history_consent" - assert result["action_required"] == "history_consent" - assert "history_consent" not in json.loads( - (tmp_path / "substrate_wiki/onboarding/state.json").read_text() - ) - - -def test_auto_device_run_always_prints_complete_authorization_url( - tmp_path: Path, monkeypatch, capsys -): - monkeypatch.delenv("DISPLAY", raising=False) - monkeypatch.delenv("WAYLAND_DISPLAY", raising=False) - manager = OnboardingManager( - tmp_path, - api=API(), - store=Store(), - capability_check=lambda token: {"ok": token}, - opener=lambda url: True, - ) - result = manager.run(mode="auto", wait=True, open_browser=True, timeout=5) - assert result["phase"] == "awaiting_history_consent" - assert result["action_required"] == "history_consent" - assert result["verification_uri_complete"] in capsys.readouterr().err diff --git a/tests/test_packaging.py b/tests/test_packaging.py index 6afdbdb..9180fda 100644 --- a/tests/test_packaging.py +++ b/tests/test_packaging.py @@ -1,98 +1,33 @@ +"""Release packaging tests for the Substrate retrieval plugin.""" + from __future__ import annotations -import hashlib -import importlib.util -import json -import os -import subprocess +import sys import zipfile from pathlib import Path -from types import ModuleType - -import pytest -REPOSITORY_ROOT = Path(__file__).parents[1] -BUILDER_PATH = REPOSITORY_ROOT / "scripts" / "build_plugin.py" -INSTALLER_PATH = REPOSITORY_ROOT / "scripts" / "install_hermes_plugin.py" +REPOSITORY_ROOT = Path(__file__).resolve().parents[1] RELEASE_WORKFLOW = REPOSITORY_ROOT / ".github" / "workflows" / "release.yml" -PLUGIN_SOURCE = REPOSITORY_ROOT / "src" / "substrate_wiki" -LEGACY_RELEASE_120 = REPOSITORY_ROOT / "legacy-assets" / "1.2.0" -LEGACY_RELEASE_130 = REPOSITORY_ROOT / "legacy-assets" / "1.3.0" -LEGACY_RELEASE_140 = REPOSITORY_ROOT / "legacy-assets" / "1.4.0" -LEGACY_RELEASE_141 = REPOSITORY_ROOT / "legacy-assets" / "1.4.1" -LEGACY_ARCHIVE_SHA256 = "2cbf504ec83352f23a1157777d24272b62e4b7300ad0ca991a0c4bc2e2df30b5" -LEGACY_INSTALLER_SHA256 = "bb9a8483d3d623528f573593eacebffa9483f52ecf84a452c01fa9c362b6879e" -LEGACY_130_ARCHIVE_SHA256 = "6827c00444c799c085ac7a3669721d672c0d7e1e703a8a491397bb16d0655c02" -LEGACY_130_INSTALLER_SHA256 = "59d4d0b8557a49ec18160f4245a533465f8c4c0eb344d235af155afb8845d1b1" -LEGACY_140_ARCHIVE_SHA256 = "df872d60dfc53668a0e6d30fd024e8d2f533306375980c3815ef1a483676c667" -LEGACY_140_INSTALLER_SHA256 = "13a05be49a83fab4c75171356d575dd85e00b27b8e09ce1602b87ae903741608" -LEGACY_141_ARCHIVE_SHA256 = "877ccf9b0212792b699d9c98912a26980675a6050df3bd319e927639e3d901f1" -LEGACY_141_INSTALLER_SHA256 = "7600b2681c3aebcb1b1492b0a04be38bbbec637089cbbcfb1cc26e8c10865b8d" -EXPECTED_MEMBERS = [ - 'substrate_wiki/', - 'substrate_wiki/PROVENANCE.json', - 'substrate_wiki/LICENSE', - 'substrate_wiki/README.md', - 'substrate_wiki/__init__.py', - 'substrate_wiki/checkpoint.py', - 'substrate_wiki/cli.py', - 'substrate_wiki/client.py', - 'substrate_wiki/credentials.py', - 'substrate_wiki/events.py', - 'substrate_wiki/history.py', - 'substrate_wiki/onboarding.py', - 'substrate_wiki/plugin.yaml', - 'substrate_wiki/py.typed', - 'substrate_wiki/redaction.py', - 'substrate_wiki/spool.py', - 'substrate_wiki/supervisor.py', - 'substrate_wiki/worker.py', -] - - -def load_builder() -> ModuleType: - spec = importlib.util.spec_from_file_location("build_plugin", BUILDER_PATH) - assert spec is not None and spec.loader is not None - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - return module - - -def load_installer() -> ModuleType: - spec = importlib.util.spec_from_file_location("install_hermes_plugin", INSTALLER_PATH) - assert spec is not None and spec.loader is not None - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - return module - - -def test_manifest_identity_matches_direct_install_package() -> None: - manifest = (PLUGIN_SOURCE / "plugin.yaml").read_text(encoding="utf-8") - assert "name: substrate_wiki\n" in manifest - assert "name: substrate-wiki" not in manifest +sys.path.insert(0, str(REPOSITORY_ROOT / "scripts")) +from build_release import ARCHIVE_NAME, FIXED_TIMESTAMP, PREFIX, build_archive_bytes # noqa: E402 -def test_packaged_readme_uses_the_standalone_release_installer() -> None: - readme = (PLUGIN_SOURCE / "README.md").read_text(encoding="utf-8") - assert "python3 install_hermes_plugin.py" in readme - assert "python scripts/install_hermes_plugin.py" not in readme +def plugin_version() -> str: + for line in (REPOSITORY_ROOT / "plugins" / "substrate" / "plugin.yaml").read_text().splitlines(): + if line.startswith("version:"): + return line.split(":", 1)[1].strip() + raise AssertionError("plugin.yaml has no version") -def test_root_readme_keeps_published_release_state_truthful() -> None: +def test_root_readme_describes_the_released_install() -> None: readme = (REPOSITORY_ROOT / "README.md").read_text(encoding="utf-8") - boundary = json.loads( - (REPOSITORY_ROOT / "docs" / "public-boundary.json").read_text(encoding="utf-8") - ) - - assert boundary["repository"]["candidate_source_of_truth"] is False - assert boundary["repository"]["source_of_truth"] is True - assert boundary["legal"]["status"] == "published" - assert "canonical editable source" in readme - assert "`v2.0.5` is not published yet" in readme - assert "releases/download/v2.0.5" in readme - assert "PLUGIN_SHA256_PENDING" in readme + assert "plugins/substrate" in readme + assert "--ref v0.3.0" in readme + assert "Hermes 0.21" in readme + assert "PLUGIN_SHA256_PENDING" not in readme + assert "is not published yet" not in readme def test_release_workflow_keeps_dependency_execution_out_of_privileged_publisher() -> None: @@ -119,666 +54,25 @@ def test_release_workflow_keeps_dependency_execution_out_of_privileged_publisher assert "setuptools" not in publish -def test_archive_is_canonical_and_release_clean(tmp_path: Path) -> None: - builder = load_builder() - archive_path = tmp_path / "substrate_wiki.zip" - archive_path.write_bytes(builder.build_archive_bytes()) +def test_archive_is_deterministic_and_release_clean(tmp_path: Path) -> None: + first = build_archive_bytes() + assert build_archive_bytes() == first + archive_path = tmp_path / ARCHIVE_NAME + archive_path.write_bytes(first) with zipfile.ZipFile(archive_path) as archive: - assert archive.namelist() == EXPECTED_MEMBERS + names = archive.namelist() + assert names == sorted(names) + assert f"{PREFIX}LICENSE" in names + assert f"{PREFIX}plugin.yaml" in names + assert f"{PREFIX}plugin.py" in names for info in archive.infolist(): - assert info.date_time == builder.FIXED_TIMESTAMP + assert info.date_time == FIXED_TIMESTAMP assert info.create_system == 3 assert "__pycache__" not in info.filename assert not info.filename.endswith((".pyc", ".pyo")) - assert archive.read("substrate_wiki/LICENSE") == (REPOSITORY_ROOT / "LICENSE").read_bytes() - for member in EXPECTED_MEMBERS[3:]: - source_path = PLUGIN_SOURCE / Path(member).relative_to("substrate_wiki") - assert archive.read(member) == source_path.read_bytes() - - -def test_archive_provenance_identifies_version_and_source_hashes(tmp_path: Path) -> None: - builder = load_builder() - archive_path = tmp_path / "substrate_wiki.zip" - archive_path.write_bytes(builder.build_archive_bytes()) - - with zipfile.ZipFile(archive_path) as archive: - provenance = json.loads(archive.read("substrate_wiki/PROVENANCE.json")) - - assert provenance == { - "build_format_version": 3, - "license_sha256": hashlib.sha256((REPOSITORY_ROOT / "LICENSE").read_bytes()).hexdigest(), - "plugin_version": "2.0.5", - "provider_id": "substrate_wiki", - "source_commit": "unknown", - "source_files": { - filename: hashlib.sha256((PLUGIN_SOURCE / filename).read_bytes()).hexdigest() - for filename in builder.REQUIRED_FILES - }, - "target_hermes_version": "0.20.0", - } - - -def test_builder_is_deterministic() -> None: - builder = load_builder() - - assert builder.build_archive_bytes() == builder.build_archive_bytes() - - -def test_v120_release_artifacts_remain_byte_pinned() -> None: - assert ( - hashlib.sha256((LEGACY_RELEASE_120 / "substrate_wiki.zip").read_bytes()).hexdigest() - == LEGACY_ARCHIVE_SHA256 - ) - assert ( - hashlib.sha256((LEGACY_RELEASE_120 / "install_hermes_plugin.py").read_bytes()).hexdigest() - == LEGACY_INSTALLER_SHA256 - ) - - -def test_v130_release_artifacts_remain_byte_pinned() -> None: - assert ( - hashlib.sha256((LEGACY_RELEASE_130 / "substrate_wiki.zip").read_bytes()).hexdigest() - == LEGACY_130_ARCHIVE_SHA256 - ) - assert ( - hashlib.sha256((LEGACY_RELEASE_130 / "install_hermes_plugin.py").read_bytes()).hexdigest() - == LEGACY_130_INSTALLER_SHA256 - ) - - -def test_v140_release_artifacts_remain_byte_pinned() -> None: - assert ( - hashlib.sha256((LEGACY_RELEASE_140 / "substrate_wiki.zip").read_bytes()).hexdigest() - == LEGACY_140_ARCHIVE_SHA256 - ) - assert ( - hashlib.sha256( - (LEGACY_RELEASE_140 / "install_hermes_plugin.py").read_bytes() - ).hexdigest() - == LEGACY_140_INSTALLER_SHA256 - ) - - -def test_v141_release_artifacts_remain_byte_pinned() -> None: - assert ( - hashlib.sha256((LEGACY_RELEASE_141 / "substrate_wiki.zip").read_bytes()).hexdigest() - == LEGACY_141_ARCHIVE_SHA256 - ) - assert ( - hashlib.sha256( - (LEGACY_RELEASE_141 / "install_hermes_plugin.py").read_bytes() - ).hexdigest() - == LEGACY_141_INSTALLER_SHA256 - ) - - -def test_compact_event_envelope_constants_are_locked() -> None: - source = (PLUGIN_SOURCE / "events.py").read_text(encoding="utf-8") - assert "SCHEMA_VERSION = 2" in source - assert "MAX_CAPTURE_BYTES = 256 * 1024" in source - assert '"tool_calls"' not in source - assert '"retention_days"' not in source - - -def test_publish_release_creates_exact_current_and_immutable_aliases(tmp_path: Path) -> None: - builder = load_builder() - archive_path = tmp_path / "current" / "substrate_wiki.zip" - releases_path = tmp_path / "releases" - archive = builder.build_archive_bytes(source_commit="a" * 40) - - release_archive, release_installer = builder.publish_release( - archive, - archive_path=archive_path, - releases_path=releases_path, - ) - - assert release_archive == releases_path / "2.0.5" / "substrate_wiki.zip" - assert release_installer == releases_path / "2.0.5" / "install_hermes_plugin.py" - assert archive_path.read_bytes() == release_archive.read_bytes() == archive - assert release_installer.read_bytes() == INSTALLER_PATH.read_bytes() - assert builder.check_release( - archive_path=archive_path, - releases_path=releases_path, - ) - - -def test_publish_release_refuses_to_replace_versioned_bytes(tmp_path: Path) -> None: - builder = load_builder() - releases_path = tmp_path / "releases" - versioned = releases_path / "2.0.5" / "substrate_wiki.zip" - versioned.parent.mkdir(parents=True) - versioned.write_bytes(b"different immutable bytes") - archive_path = tmp_path / "current" / "substrate_wiki.zip" - - with pytest.raises(ValueError, match="immutable release artifact differs"): - builder.publish_release( - builder.build_archive_bytes(source_commit="a" * 40), - archive_path=archive_path, - releases_path=releases_path, - ) - - assert versioned.read_bytes() == b"different immutable bytes" - assert not archive_path.exists() - - -def test_publish_release_preflights_both_immutable_artifacts(tmp_path: Path) -> None: - builder = load_builder() - releases_path = tmp_path / "releases" - versioned_installer = releases_path / "2.0.5" / "install_hermes_plugin.py" - versioned_installer.parent.mkdir(parents=True) - versioned_installer.write_bytes(b"conflicting immutable installer") - archive_path = tmp_path / "current" / "substrate_wiki.zip" - - with pytest.raises(ValueError, match="immutable release artifact differs"): - builder.publish_release( - builder.build_archive_bytes(source_commit="a" * 40), - archive_path=archive_path, - releases_path=releases_path, - ) - - assert not (releases_path / "2.0.5" / "substrate_wiki.zip").exists() - assert not archive_path.exists() - - -def test_publish_release_rejects_symlinked_immutable_artifact(tmp_path: Path) -> None: - builder = load_builder() - releases_path = tmp_path / "releases" - release_directory = releases_path / "2.0.5" - release_directory.mkdir(parents=True) - target = tmp_path / "elsewhere.zip" - archive = builder.build_archive_bytes(source_commit="a" * 40) - target.write_bytes(archive) - versioned = release_directory / "substrate_wiki.zip" - try: - versioned.symlink_to(target) - except OSError: - pytest.skip("symlink creation is unavailable") - - with pytest.raises(ValueError, match="artifact path is unsafe"): - builder.publish_release( - archive, - archive_path=tmp_path / "current" / "substrate_wiki.zip", - releases_path=releases_path, - ) - - assert versioned.is_symlink() - assert target.read_bytes() == archive - - -def test_installer_verifies_and_atomically_upgrades_with_rollback(tmp_path: Path) -> None: - builder = load_builder() - installer = load_installer() - archive = tmp_path / "substrate_wiki.zip" - archive.write_bytes(builder.build_archive_bytes(source_commit="a" * 40)) - digest = hashlib.sha256(archive.read_bytes()).hexdigest() - hermes_home = tmp_path / "hermes" - existing = hermes_home / "plugins" / "substrate_wiki" - existing.mkdir(parents=True) - (existing / "plugin.yaml").write_text("name: substrate_wiki\nversion: 1.0.0\n") - checkpoint = hermes_home / "substrate_wiki" / "imports" / "jobs" / "same-job" / "checkpoint.db" - checkpoint.parent.mkdir(parents=True) - checkpoint.write_bytes(b"content-free-checkpoint") - - result = installer.install(archive, hermes_home, expected_sha256=digest) - - assert result["action"] == "upgraded" - assert result["source_commit"] == "a" * 40 - assert "version: 2.0.5" in (existing / "plugin.yaml").read_text(encoding="utf-8") - rollback = Path(result["rollback"]) - assert "version: 1.0.0" in (rollback / "plugin.yaml").read_text(encoding="utf-8") - assert checkpoint.read_bytes() == b"content-free-checkpoint" - with pytest.raises(ValueError, match="SHA-256"): - installer.verify_archive(archive, "0" * 64) - - -def test_install_refuses_to_mutate_without_a_pinned_sha256(tmp_path: Path) -> None: - builder = load_builder() - installer = load_installer() - archive = tmp_path / "substrate_wiki.zip" - archive.write_bytes(builder.build_archive_bytes(source_commit="a" * 40)) - hermes_home = tmp_path / "hermes" - - with pytest.raises(ValueError, match="pinned archive SHA-256 is required"): - installer.install(archive, hermes_home) - - assert not (hermes_home / "plugins" / "substrate_wiki").exists() - - -def test_installer_cli_requires_the_pinned_sha256( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - installer = load_installer() - monkeypatch.setattr( - installer.sys, - "argv", - [ - "install_hermes_plugin.py", - "--archive", - os.fspath(tmp_path / "substrate_wiki.zip"), - "--yes", - ], - ) - - with pytest.raises(SystemExit) as error: - installer.main() - - assert error.value.code == 2 - - -def test_plugin_swap_restores_previous_version_when_hardening_fails( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - builder = load_builder() - installer = load_installer() - archive = tmp_path / "substrate_wiki.zip" - archive.write_bytes(builder.build_archive_bytes(source_commit="a" * 40)) - digest = hashlib.sha256(archive.read_bytes()).hexdigest() - hermes_home = tmp_path / "hermes" - existing = hermes_home / "plugins" / "substrate_wiki" - existing.mkdir(parents=True) - (existing / "plugin.yaml").write_text( - "name: substrate_wiki\nversion: 1.3.0\n", encoding="utf-8" - ) - - def fail_hardening(target: Path) -> None: - assert "version: 2.0.5" in (target / "plugin.yaml").read_text(encoding="utf-8") - raise OSError("permission hardening failed") - - monkeypatch.setattr(installer, "_harden_plugin_permissions", fail_hardening) - with pytest.raises(OSError, match="permission hardening failed"): - installer.install(archive, hermes_home, expected_sha256=digest) - - assert "version: 1.3.0" in (existing / "plugin.yaml").read_text(encoding="utf-8") - failed = list((hermes_home / "plugins").glob("substrate_wiki.failed-*")) - assert len(failed) == 1 - assert "version: 2.0.5" in (failed[0] / "plugin.yaml").read_text(encoding="utf-8") - - -def test_check_archive_preserves_sha_provenance_without_environment( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - builder = load_builder() - archive_path = tmp_path / "substrate_wiki.zip" - source_commit = "a" * 40 - - monkeypatch.setenv(builder.SOURCE_COMMIT_ENVIRONMENT_VARIABLE, source_commit) - archive_path.write_bytes(builder.build_archive_bytes()) - monkeypatch.delenv(builder.SOURCE_COMMIT_ENVIRONMENT_VARIABLE) - - assert builder.check_archive(archive_path) - - -def test_check_archive_rejects_malformed_provenance(tmp_path: Path) -> None: - builder = load_builder() - archive_path = tmp_path / "substrate_wiki.zip" - - with zipfile.ZipFile(archive_path, "w") as archive: - archive.writestr("substrate_wiki/PROVENANCE.json", '{"source_commit": "not-a-sha"}') - - with pytest.raises(ValueError, match="malformed provenance"): - builder.check_archive(archive_path) - - -def test_installer_rejects_unlisted_archive_members(tmp_path: Path) -> None: - builder = load_builder() - installer = load_installer() - archive_path = tmp_path / "substrate_wiki.zip" - archive_path.write_bytes(builder.build_archive_bytes(source_commit="a" * 40)) - with zipfile.ZipFile(archive_path, "a") as archive: - archive.writestr("substrate_wiki/unlisted.py", "raise RuntimeError('untrusted')\n") - - with pytest.raises(ValueError, match="unexpected file set"): - installer.verify_archive(archive_path) - - -def test_installer_rejects_license_bytes_that_do_not_match_provenance(tmp_path: Path) -> None: - builder = load_builder() - installer = load_installer() - canonical_path = tmp_path / "canonical.zip" - canonical_path.write_bytes(builder.build_archive_bytes(source_commit="a" * 40)) - archive_path = tmp_path / "tampered-license.zip" - with zipfile.ZipFile(canonical_path) as source, zipfile.ZipFile(archive_path, "w") as target: - for info in source.infolist(): - content = source.read(info.filename) - if info.filename == "substrate_wiki/LICENSE": - content = b"not the reviewed license\n" - target.writestr(info, content) - - with pytest.raises(ValueError, match="license digest mismatch"): - installer.verify_archive(archive_path) - - -def test_installer_rejects_non_regular_allowlisted_members(tmp_path: Path) -> None: - builder = load_builder() - installer = load_installer() - canonical_path = tmp_path / "canonical.zip" - canonical_path.write_bytes(builder.build_archive_bytes(source_commit="a" * 40)) - archive_path = tmp_path / "tampered.zip" - with zipfile.ZipFile(canonical_path) as source, zipfile.ZipFile(archive_path, "w") as target: - for info in source.infolist(): - content = source.read(info.filename) - if info.filename == "substrate_wiki/client.py": - info.external_attr = 0o40755 << 16 - target.writestr(info, content) - - with pytest.raises(ValueError, match="non-regular plugin file"): - installer.verify_archive(archive_path) - - -def test_installer_extracts_only_the_verified_archive_bytes( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - builder = load_builder() - installer = load_installer() - archive_path = tmp_path / "substrate_wiki.zip" - archive_path.write_bytes(builder.build_archive_bytes(source_commit="a" * 40)) - original_verify = installer.verify_archive - - def verify_then_swap(path: Path, expected_sha256: str = "") -> dict[str, object]: - result = original_verify(path, expected_sha256) - path.write_bytes(b"locally replaced after verification") - return result - - monkeypatch.setattr(installer, "verify_archive", verify_then_swap) - digest = hashlib.sha256(archive_path.read_bytes()).hexdigest() - - with pytest.raises(ValueError, match="changed after verification"): - installer.install( - archive_path, - tmp_path / "hermes", - expected_sha256=digest, - ) - - assert not (tmp_path / "hermes" / "plugins" / "substrate_wiki").exists() - - -def test_environment_path_rejects_a_symlink_before_resolving(tmp_path: Path) -> None: - installer = load_installer() - target = tmp_path / "profile.env" - target.write_text("HERMES_API_URL=x\nHERMES_API_KEY=x\n", encoding="utf-8") - link = tmp_path / "linked.env" - try: - link.symlink_to(target) - except OSError: - pytest.skip("symlink creation is unavailable") - - with pytest.raises(ValueError, match="non-symlink"): - installer._resolve_env_path(link) - - -def test_fresh_install_rolls_back_plugin_when_service_install_fails( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - builder = load_builder() - installer = load_installer() - archive_path = tmp_path / "substrate_wiki.zip" - archive_path.write_bytes(builder.build_archive_bytes(source_commit="a" * 40)) - hermes_home = tmp_path / "hermes" - - def fail_service(*args: object, **kwargs: object) -> dict[str, object]: - del args, kwargs - raise OSError("service setup failed") - - monkeypatch.setattr(installer, "install_import_service", fail_service) - digest = hashlib.sha256(archive_path.read_bytes()).hexdigest() - with pytest.raises(OSError, match="service setup failed"): - installer.install( - archive_path, - hermes_home, - expected_sha256=digest, - install_service=True, - ) - - assert not (hermes_home / "plugins" / "substrate_wiki").exists() - assert len(list((hermes_home / "plugins").glob("substrate_wiki.failed-*"))) == 1 - - -def test_upgrade_restores_plugin_and_unit_when_service_reload_fails( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - builder = load_builder() - installer = load_installer() - archive_path = tmp_path / "substrate_wiki.zip" - archive_path.write_bytes(builder.build_archive_bytes(source_commit="a" * 40)) - digest = hashlib.sha256(archive_path.read_bytes()).hexdigest() - - fake_home = tmp_path / "user" - hermes_home = fake_home / ".hermes" - existing = hermes_home / "plugins" / "substrate_wiki" - existing.mkdir(parents=True) - (existing / "plugin.yaml").write_text( - "name: substrate_wiki\nversion: 1.3.0\n", encoding="utf-8" - ) - (existing / "v13-sentinel.txt").write_text("prior plugin", encoding="utf-8") - - env_path = fake_home / "profile.env" - secret = "must-not-appear-in-unit-or-command" - env_path.write_text( - f"HERMES_API_URL=https://memory.example.invalid\nHERMES_API_KEY={secret}\n", - encoding="utf-8", - ) - units = fake_home / ".config" / "systemd" / "user" - units.mkdir(parents=True) - unit_name = ( - "substrate-wiki-import-" - + hashlib.sha256(os.fspath(hermes_home.resolve()).encode()).hexdigest()[:12] - + "@.service" - ) - unit_path = units / unit_name - old_unit = "[Service]\nExecStart=/safe/v13/import-worker\n" - unit_path.write_text(old_unit, encoding="utf-8") - - class PosixOSProxy: - """Exercise the POSIX-only installer without changing pathlib's host OS.""" - - name = "posix" - - def __getattr__(self, name: str) -> object: - return getattr(os, name) - - monkeypatch.setattr(installer, "os", PosixOSProxy()) - monkeypatch.setattr(installer.Path, "home", classmethod(lambda cls: fake_home)) - monkeypatch.setattr(installer, "_resolve_env_path", lambda explicit=None: env_path) - systemctl_calls: list[tuple[tuple[str, ...], dict[str, object]]] = [] - - def fail_checked_reload( - command: tuple[str, ...], **kwargs: object - ) -> subprocess.CompletedProcess[str]: - systemctl_calls.append((command, kwargs)) - if kwargs.get("check") is True: - assert command == ("systemctl", "--user", "daemon-reload") - assert "version: 2.0.5" in (existing / "plugin.yaml").read_text( - encoding="utf-8" - ) - plugin_rollbacks = list( - (hermes_home / "plugins").glob("substrate_wiki.rollback-*") - ) - assert len(plugin_rollbacks) == 1 - assert "version: 1.3.0" in ( - plugin_rollbacks[0] / "plugin.yaml" - ).read_text(encoding="utf-8") - assert "MemoryMax=256M" in unit_path.read_text(encoding="utf-8") - unit_rollbacks = list(units.glob(f"{unit_name}.rollback-*")) - assert len(unit_rollbacks) == 1 - assert unit_rollbacks[0].read_text(encoding="utf-8") == old_unit - raise subprocess.CalledProcessError(1, command) - return subprocess.CompletedProcess(command, 0, "", "") - - monkeypatch.setattr(installer.subprocess, "run", fail_checked_reload) - - with pytest.raises(subprocess.CalledProcessError): - installer.install( - archive_path, - hermes_home, - expected_sha256=digest, - install_service=True, - env_path=env_path, - ) - - assert "version: 1.3.0" in (existing / "plugin.yaml").read_text(encoding="utf-8") - assert (existing / "v13-sentinel.txt").read_text(encoding="utf-8") == "prior plugin" - failed_plugins = list((hermes_home / "plugins").glob("substrate_wiki.failed-*")) - assert len(failed_plugins) == 1 - assert "version: 2.0.5" in (failed_plugins[0] / "plugin.yaml").read_text( - encoding="utf-8" - ) - assert not list((hermes_home / "plugins").glob("substrate_wiki.rollback-*")) - assert unit_path.read_text(encoding="utf-8") == old_unit - assert not list(units.glob("*.rollback-*")) - assert [call[0] for call in systemctl_calls] == [ - ("systemctl", "--user", "daemon-reload"), - ("systemctl", "--user", "daemon-reload"), - ] - observable_install_state = "\n".join( - [ - unit_path.read_text(encoding="utf-8"), - *(" ".join(command) for command, _kwargs in systemctl_calls), - *(repr(kwargs) for _command, kwargs in systemctl_calls), - ] - ) - assert secret not in observable_install_state - assert "hermes-gateway" not in observable_install_state - - -@pytest.mark.skipif(os.name != "posix", reason="systemd user services are POSIX-only") -def test_import_unit_failure_restores_previous_unit( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - installer = load_installer() - fake_home = tmp_path / "user" - units = fake_home / ".config" / "systemd" / "user" - units.mkdir(parents=True) - hermes_home = fake_home / ".hermes" - plugin_target = hermes_home / "plugins" / "substrate_wiki" - plugin_target.mkdir(parents=True) - env_path = fake_home / ".env" - env_path.write_text("HERMES_API_URL=x\nHERMES_API_KEY=x\n", encoding="utf-8") - unit_name = ( - "substrate-wiki-import-" - + hashlib.sha256(os.fspath(hermes_home.resolve()).encode()).hexdigest()[:12] - + "@.service" - ) - unit_path = units / unit_name - unit_path.write_text("old-safe-unit\n", encoding="utf-8") - - monkeypatch.setattr(installer.Path, "home", classmethod(lambda cls: fake_home)) - monkeypatch.setattr(installer, "_resolve_env_path", lambda explicit=None: env_path) - - def fail_reload(command: tuple[str, ...], **kwargs: object) -> subprocess.CompletedProcess[str]: - if kwargs.get("check") is True: - raise subprocess.CalledProcessError(1, command) - return subprocess.CompletedProcess(command, 1, "", "") - - monkeypatch.setattr(installer.subprocess, "run", fail_reload) - with pytest.raises(subprocess.CalledProcessError): - installer.install_import_service(hermes_home, plugin_target, env_path=env_path) - - assert unit_path.read_text(encoding="utf-8") == "old-safe-unit\n" - assert not list(units.glob("*.rollback-*")) - - -@pytest.mark.skipif(os.name != "posix", reason="systemd user services are POSIX-only") -def test_import_unit_staging_failure_leaves_previous_unit_untouched( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - installer = load_installer() - fake_home = tmp_path / "user" - units = fake_home / ".config" / "systemd" / "user" - units.mkdir(parents=True) - hermes_home = fake_home / ".hermes" - plugin_target = hermes_home / "plugins" / "substrate_wiki" - plugin_target.mkdir(parents=True) - env_path = fake_home / ".env" - env_path.write_text("HERMES_API_URL=x\nHERMES_API_KEY=x\n", encoding="utf-8") - unit_name = ( - "substrate-wiki-import-" - + hashlib.sha256(os.fspath(hermes_home.resolve()).encode()).hexdigest()[:12] - + "@.service" - ) - unit_path = units / unit_name - unit_path.write_text("old-safe-unit\n", encoding="utf-8") - - monkeypatch.setattr(installer.Path, "home", classmethod(lambda cls: fake_home)) - monkeypatch.setattr(installer, "_resolve_env_path", lambda explicit=None: env_path) - monkeypatch.setattr( - installer.os, - "open", - lambda *args, **kwargs: (_ for _ in ()).throw(PermissionError("staging failed")), - ) - systemctl_calls: list[tuple[str, ...]] = [] - monkeypatch.setattr( - installer.subprocess, - "run", - lambda command, **kwargs: systemctl_calls.append(command), - ) - - with pytest.raises(PermissionError, match="staging failed"): - installer.install_import_service(hermes_home, plugin_target, env_path=env_path) - - assert unit_path.read_text(encoding="utf-8") == "old-safe-unit\n" - assert not list(units.glob("*.rollback-*")) - assert systemctl_calls == [] - - -def test_builder_excludes_generated_bytecode(tmp_path: Path) -> None: - builder = load_builder() - source = tmp_path / "substrate_wiki" - source.mkdir() - for filename in builder.REQUIRED_FILES: - (source / filename).write_bytes((PLUGIN_SOURCE / filename).read_bytes()) - expected = builder.build_archive_bytes(source) - cache = source / "__pycache__" - cache.mkdir() - (cache / "client.cpython-312.pyc").write_bytes(b"generated") - - assert builder.build_archive_bytes(source) == expected - - -def test_builder_rejects_unknown_release_files(tmp_path: Path) -> None: - builder = load_builder() - source = tmp_path / "substrate_wiki" - source.mkdir() - for filename in builder.REQUIRED_FILES: - (source / filename).write_bytes((PLUGIN_SOURCE / filename).read_bytes()) - (source / "notes.txt").write_text("not a release file", encoding="utf-8") - - with pytest.raises(ValueError, match="release-clean.*unexpected"): - builder.build_archive_bytes(source) - - -def test_builder_rejects_symlinked_source_members(tmp_path: Path) -> None: - builder = load_builder() - source = tmp_path / "substrate_wiki" - source.mkdir() - for filename in builder.REQUIRED_FILES: - if filename != "README.md": - (source / filename).write_bytes((PLUGIN_SOURCE / filename).read_bytes()) - try: - (source / "README.md").symlink_to(PLUGIN_SOURCE / "README.md") - except OSError: - pytest.skip("symlink creation is unavailable") - - with pytest.raises(ValueError, match="release-clean.*missing"): - builder.build_archive_bytes(source) - - -def test_check_command_verifies_sha_archive_without_environment( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] -) -> None: - builder = load_builder() - archive_path = tmp_path / "current" / "substrate_wiki.zip" - releases_path = tmp_path / "releases" - source_commit = "a" * 40 + assert archive.read(f"{PREFIX}LICENSE") == (REPOSITORY_ROOT / "LICENSE").read_bytes() - builder.publish_release( - builder.build_archive_bytes(source_commit=source_commit), - archive_path=archive_path, - releases_path=releases_path, - ) - monkeypatch.setattr(builder, "ARCHIVE_PATH", archive_path) - monkeypatch.setattr(builder, "RELEASES_PATH", releases_path) - monkeypatch.setattr(builder, "_require_commit_contains_source", lambda commit: None) - assert builder.main(["--check"]) == 0 - assert "release is current" in capsys.readouterr().out +def test_plugin_manifest_matches_release_version() -> None: + assert plugin_version() == "0.3.0" diff --git a/tests/test_publication_scanner.py b/tests/test_publication_scanner.py deleted file mode 100644 index 67b1e22..0000000 --- a/tests/test_publication_scanner.py +++ /dev/null @@ -1,408 +0,0 @@ -from __future__ import annotations - -import hashlib -import importlib.util -import json -import subprocess -from pathlib import Path -from types import ModuleType - -import pytest - -REPOSITORY_ROOT = Path(__file__).parents[1] -VERIFIER_PATH = REPOSITORY_ROOT / "scripts" / "verify_public_plugin_candidate.py" - - -def load_verifier() -> ModuleType: - spec = importlib.util.spec_from_file_location("verify_public_plugin_candidate", VERIFIER_PATH) - assert spec is not None and spec.loader is not None - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - return module - - -def git(root: Path, *args: str) -> None: - subprocess.run(("git", *args), cwd=root, check=True, capture_output=True) - - -def make_candidate(tmp_path: Path) -> tuple[ModuleType, Path, Path]: - verifier = load_verifier() - root = tmp_path / "candidate" - root.mkdir() - git(root, "init", "-q") - git(root, "config", "user.name", "Scanner Test") - git(root, "config", "user.email", "scanner@example.test") - - tracked = root / "README.md" - tracked.write_text("safe candidate\n", encoding="utf-8") - manifest = root / "docs" / "extraction-manifest.json" - manifest.parent.mkdir() - manifest.write_text( - json.dumps( - { - "entries": [ - { - "source": "host/README.md", - "destination": "README.md", - "source_sha256": "0" * 64, - "destination_sha256": __import__("hashlib").sha256( - tracked.read_bytes() - ).hexdigest(), - "class": "documentation", - } - ], - "destination_only": [], - "self_excluded_path": "docs/extraction-manifest.json", - } - ), - encoding="utf-8", - ) - value = json.loads(manifest.read_text(encoding="utf-8")) - policy = { - "entries": sorted( - ( - { - "source": item["source"], - "destination": item["destination"], - "class": item["class"], - } - for item in value["entries"] - ), - key=lambda item: item["destination"], - ), - "destination_only": [], - } - verifier.TRUSTED_INVENTORY_POLICY_SHA256 = hashlib.sha256( - json.dumps(policy, sort_keys=True, separators=(",", ":")).encode("utf-8") - ).hexdigest() - git(root, "add", ".") - git(root, "commit", "-qm", "safe base") - verifier.TRUSTED_HISTORICAL_BLOB_POLICY_SHA256 = ( - verifier._historical_blob_policy_sha256(root, value) - ) - return verifier, root, manifest - - -def test_destination_scanner_rejects_untracked_secret_file(tmp_path: Path) -> None: - verifier, root, manifest = make_candidate(tmp_path) - secret = "sk-" + "A" * 32 - extra = root / "private" / "production.env" - extra.parent.mkdir() - extra.write_text(secret, encoding="utf-8") - - findings = verifier.scan_candidate(root, manifest, layout="destination") - - assert any("unexpected untracked candidate file" in finding for finding in findings) - assert any("OpenAI-shaped credential" in finding for finding in findings) - - -def test_destination_scanner_invalidates_exact_file_exemption_on_mutation(tmp_path: Path) -> None: - verifier, root, manifest = make_candidate(tmp_path) - tracked = root / "README.md" - original_digest = verifier.hashlib.sha256(tracked.read_bytes()).hexdigest() - verifier.SYNTHETIC_FILE_SHA256_ALLOWLIST["README.md"] = frozenset({original_digest}) - assert verifier.scan_candidate(root, manifest, layout="destination") == [] - - tracked.write_text("Authorization: Bearer " + "B" * 32, encoding="utf-8") - - findings = verifier.scan_candidate(root, manifest, layout="destination") - assert any("Bearer credential" in finding for finding in findings) - - -def test_destination_scanner_rejects_secret_in_reachable_history(tmp_path: Path) -> None: - verifier, root, manifest = make_candidate(tmp_path) - historical = root / "historical.txt" - historical.write_text("sk-" + "C" * 32, encoding="utf-8") - git(root, "add", "historical.txt") - git(root, "commit", "-qm", "unsafe history") - historical.unlink() - git(root, "add", "-u") - git(root, "commit", "-qm", "remove working-tree secret") - - findings = verifier.scan_candidate(root, manifest, layout="destination") - - assert any(finding.startswith("git:") for finding in findings) - assert any("OpenAI-shaped credential" in finding for finding in findings) - - -def test_destination_scanner_rejects_ignored_secret_file(tmp_path: Path) -> None: - verifier, root, manifest = make_candidate(tmp_path) - secret_name = "ignored-production.env" - (root / ".git" / "info" / "exclude").write_text(secret_name + "\n", encoding="utf-8") - (root / secret_name).write_text("sk-" + "D" * 32, encoding="utf-8") - - findings = verifier.scan_candidate(root, manifest, layout="destination") - - assert any("unexpected untracked candidate file" in finding for finding in findings) - assert any("OpenAI-shaped credential" in finding for finding in findings) - - -def test_destination_scanner_rejects_duplicate_manifest_mapping(tmp_path: Path) -> None: - verifier, root, manifest = make_candidate(tmp_path) - value = json.loads(manifest.read_text(encoding="utf-8")) - value["entries"].append(dict(value["entries"][0])) - manifest.write_text(json.dumps(value), encoding="utf-8") - - findings = verifier.scan_candidate(root, manifest, layout="destination") - - assert any("duplicate source inventory entry" in finding for finding in findings) - assert any("duplicate destination inventory entry" in finding for finding in findings) - - -def test_destination_scanner_rejects_coordinated_held_code_inventory_rewrite( - tmp_path: Path, -) -> None: - verifier, root, manifest = make_candidate(tmp_path) - held = root / "infra" / "deploy.py" - held.parent.mkdir() - held.write_text("def deploy_private_broker():\n return 'held'\n", encoding="utf-8") - value = json.loads(manifest.read_text(encoding="utf-8")) - value["destination_only"].append( - { - "path": "infra/deploy.py", - "class": "standalone_repository_policy_or_test", - "sha256": hashlib.sha256(held.read_bytes()).hexdigest(), - } - ) - manifest.write_text(json.dumps(value), encoding="utf-8") - git(root, "add", ".") - git(root, "commit", "-qm", "attempt coordinated held-code admission") - - findings = verifier.scan_candidate(root, manifest, layout="destination") - - assert "candidate: closed inventory path/class policy mismatch" in findings - - -def test_destination_scanner_rejects_extraction_source_rewrite(tmp_path: Path) -> None: - verifier, root, manifest = make_candidate(tmp_path) - value = json.loads(manifest.read_text(encoding="utf-8")) - value["entries"][0]["source"] = "private-server/held-broker.py" - manifest.write_text(json.dumps(value), encoding="utf-8") - - findings = verifier.scan_candidate(root, manifest, layout="destination") - - assert "candidate: closed inventory path/class policy mismatch" in findings - - -def test_destination_scanner_rejects_secret_in_reachable_commit_message(tmp_path: Path) -> None: - verifier, root, manifest = make_candidate(tmp_path) - message = "unsafe commit " + "sk-" + "E" * 32 - git(root, "commit", "--allow-empty", "-qm", message) - - findings = verifier.scan_candidate(root, manifest, layout="destination") - - assert any(finding.startswith("git-object:") for finding in findings) - assert any("OpenAI-shaped credential" in finding for finding in findings) - - -def test_destination_scanner_rejects_secret_in_annotated_tag_message(tmp_path: Path) -> None: - verifier, root, manifest = make_candidate(tmp_path) - message = "unsafe tag " + "sk-" + "F" * 32 - git(root, "tag", "-a", "unsafe", "-m", message) - - findings = verifier.scan_candidate(root, manifest, layout="destination") - - assert any(finding.startswith("git-object:") for finding in findings) - assert any("OpenAI-shaped credential" in finding for finding in findings) - - -def test_destination_scanner_rejects_directly_referenced_secret_blob(tmp_path: Path) -> None: - verifier, root, manifest = make_candidate(tmp_path) - secret = ("sk-" + "G" * 32).encode("ascii") - object_id = subprocess.check_output( - ("git", "hash-object", "-w", "--stdin"), - cwd=root, - input=secret, - ).decode("ascii").strip() - git(root, "update-ref", "refs/tags/unsafe-direct-blob", object_id) - - findings = verifier.scan_candidate(root, manifest, layout="destination") - - assert any(finding.startswith(f"git-object:{object_id}:blob") for finding in findings) - assert any("OpenAI-shaped credential" in finding for finding in findings) - - -def test_destination_scanner_rejects_historical_hermes_api_key_assignment( - tmp_path: Path, -) -> None: - verifier, root, manifest = make_candidate(tmp_path) - historical = root / "production.env" - historical.write_text( - "HERMES_API_" + "KEY=" + "H" * 40 + "\n", - encoding="utf-8", - ) - git(root, "add", "production.env") - git(root, "commit", "-qm", "unsafe Hermes credential history") - historical.unlink() - git(root, "add", "-u") - git(root, "commit", "-qm", "remove working-tree credential") - - findings = verifier.scan_candidate(root, manifest, layout="destination") - - assert any(finding.startswith("git:") for finding in findings) - assert any("Hermes API credential assignment" in finding for finding in findings) - - -@pytest.mark.parametrize( - "shape", - ( - "quoted-json", - "bracket-assignment", - "f-string", - "bytes-map", - "raw-string", - "triple-quoted", - "shell-concatenated", - "implicit-concatenated-key", - "putenv-call", - "variable-concatenated-key", - "separate-variable-segments", - "unicode-escaped-key", - "your-prefix", - "example-prefix", - "replace-prefix", - ), -) -def test_destination_scanner_rejects_direct_hermes_key_assignment_shapes( - tmp_path: Path, - shape: str, -) -> None: - verifier, root, manifest = make_candidate(tmp_path) - key = "HERMES_API_" + "KEY" - suffix = "Q" * 40 - values = { - "quoted-json": f'"{key}": "{suffix}"', - "bracket-assignment": f'os.environ["{key}"] = "{suffix}"', - "f-string": f'{key} = f"{suffix}"', - "bytes-map": f'"{key}": b"{suffix}"', - "raw-string": f'{key} = r"{suffix}"', - "triple-quoted": f'{key} = """{suffix}"""', - "shell-concatenated": f'{key}="{suffix[:20]}""{suffix[20:]}"', - "implicit-concatenated-key": ( - f'os.environ["HERMES_" "API_KEY"] = "{suffix}"' - ), - "putenv-call": f'os.putenv("HERMES_API_KEY", "{suffix}")', - "variable-concatenated-key": ( - f'name = "HERMES_API_" + "KEY"; os.environ[name] = "{suffix}"' - ), - "separate-variable-segments": ( - 'prefix = "HERMES_"\nfamily = "API_"\nsuffix = "KEY"\n' - f'name = prefix + family + suffix\nos.environ[name] = "{suffix}"' - ), - "unicode-escaped-key": f'{{"HERMES_API_\\u004bEY": "{suffix}"}}', - "your-prefix": f"{key}=your-{suffix}", - "example-prefix": f"{key}=example{suffix}", - "replace-prefix": f"{key}=replace-me{suffix}", - } - payload = values[shape].encode("ascii") - object_id = subprocess.check_output( - ("git", "hash-object", "-w", "--stdin"), - cwd=root, - input=payload, - ).decode("ascii").strip() - git(root, "update-ref", f"refs/tags/unsafe-{shape}", object_id) - - findings = verifier.scan_candidate(root, manifest, layout="destination") - - assert any(finding.startswith(f"git-object:{object_id}:blob") for finding in findings) - assert any("Hermes API credential assignment" in finding for finding in findings) - - -def test_destination_scanner_rejects_add_then_delete_held_source(tmp_path: Path) -> None: - verifier, root, manifest = make_candidate(tmp_path) - held = root / "infra" / "deploy.py" - held.parent.mkdir() - held.write_text("def deploy_private_broker():\n return 'held'\n", encoding="utf-8") - git(root, "add", "infra/deploy.py") - git(root, "commit", "-qm", "unsafe held source history") - held.unlink() - git(root, "add", "-u") - git(root, "commit", "-qm", "remove held source from current tree") - - findings = verifier.scan_candidate(root, manifest, layout="destination") - - assert any( - finding.endswith("infra/deploy.py: historical path outside closed inventory") - for finding in findings - ) - - -def test_destination_scanner_rejects_historical_symlink_substitution(tmp_path: Path) -> None: - verifier, root, manifest = make_candidate(tmp_path) - tracked = root / "README.md" - tracked.unlink() - tracked.symlink_to("private-held-target") - git(root, "add", "README.md") - git(root, "commit", "-qm", "unsafe historical symlink") - tracked.unlink() - tracked.write_text("safe candidate\n", encoding="utf-8") - git(root, "add", "README.md") - git(root, "commit", "-qm", "restore regular candidate file") - - findings = verifier.scan_candidate(root, manifest, layout="destination") - - assert any("historical entry is not a regular file" in finding for finding in findings) - - -def test_destination_scanner_rejects_historical_allowed_path_substitution( - tmp_path: Path, -) -> None: - verifier, root, manifest = make_candidate(tmp_path) - tracked = root / "README.md" - tracked.write_text( - "def deploy_private_broker():\n return 'held implementation'\n", - encoding="utf-8", - ) - git(root, "add", "README.md") - git(root, "commit", "-qm", "hide held source at allowed path") - tracked.write_text("safe candidate\n", encoding="utf-8") - git(root, "add", "README.md") - git(root, "commit", "-qm", "restore reviewed allowed path") - - findings = verifier.scan_candidate(root, manifest, layout="destination") - - assert "candidate: historical blob policy mismatch" in findings - - -def test_destination_scanner_rejects_allowed_path_substitution_on_secondary_ref( - tmp_path: Path, -) -> None: - verifier, root, manifest = make_candidate(tmp_path) - reviewed_head = subprocess.check_output( - ("git", "rev-parse", "HEAD"), cwd=root, text=True - ).strip() - git(root, "switch", "-qc", "adversarial-secondary-ref") - tracked = root / "README.md" - tracked.write_text( - "def deploy_private_broker():\n return 'held implementation'\n", - encoding="utf-8", - ) - git(root, "add", "README.md") - git(root, "commit", "-qm", "publish held source on secondary ref") - git(root, "checkout", "-q", "--detach", reviewed_head) - - findings = verifier.scan_candidate(root, manifest, layout="destination") - - assert "candidate: historical blob policy mismatch" in findings - - -def test_destination_scanner_rejects_allowed_path_substitution_in_direct_tree_ref( - tmp_path: Path, -) -> None: - verifier, root, manifest = make_candidate(tmp_path) - tracked = root / "README.md" - tracked.write_text( - "def deploy_private_broker():\n return 'held implementation'\n", - encoding="utf-8", - ) - git(root, "add", "README.md") - tree_id = subprocess.check_output( - ("git", "write-tree"), cwd=root, text=True - ).strip() - git(root, "reset", "--hard", "-q", "HEAD") - git(root, "update-ref", "refs/tags/unsafe-direct-tree", tree_id) - - findings = verifier.scan_candidate(root, manifest, layout="destination") - - assert "candidate: historical blob policy mismatch" in findings - assert (f"tree:{tree_id}", tree_id) in verifier._publication_tree_roots(root) diff --git a/tests/test_redaction.py b/tests/test_redaction.py deleted file mode 100644 index 01ac479..0000000 --- a/tests/test_redaction.py +++ /dev/null @@ -1,31 +0,0 @@ -from __future__ import annotations - -import json -from pathlib import Path - -import pytest - -from substrate_wiki.redaction import redact_text - - -@pytest.mark.parametrize("vector", json.loads((Path(__file__).parent / "fixtures" / "credential_redaction_vectors.json").read_text(encoding="utf-8")), ids=lambda item: item["name"]) -def test_credential_vectors_are_redacted(vector: dict[str, object]) -> None: - rendered = redact_text(str(vector["text"]), ()) - fragments = vector.get("secret_fragments", [vector.get("secret_fragment")]) - assert isinstance(fragments, list) - for fragment in fragments: - assert isinstance(fragment, str) - assert fragment not in rendered - - -@pytest.mark.parametrize( - "value", - ( - "%79%61%32%39%2EFAKE0123456789abcdef", - "%2579%2561%2532%2539%252EFAKE0123456789", - "%41%4B%49%41FAKE0123456789AB", - "%2541%254B%2549%2541FAKE0123456789AB", - ), -) -def test_encoded_provider_credentials_are_redacted(value: str) -> None: - assert value not in redact_text(value, ()) diff --git a/uv.lock b/uv.lock index af1a1ca..d3b4305 100644 --- a/uv.lock +++ b/uv.lock @@ -11,28 +11,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, ] -[[package]] -name = "hermes-substrate-wiki" -version = "2.0.5" -source = { editable = "." } - -[package.optional-dependencies] -dev = [ - { name = "pytest" }, - { name = "pytest-timeout" }, - { name = "ruff" }, - { name = "setuptools" }, -] - -[package.metadata] -requires-dist = [ - { name = "pytest", marker = "extra == 'dev'", specifier = "==9.0.3" }, - { name = "pytest-timeout", marker = "extra == 'dev'", specifier = "==2.3.1" }, - { name = "ruff", marker = "extra == 'dev'", specifier = "==0.9.10" }, - { name = "setuptools", marker = "extra == 'dev'", specifier = "==83.0.0" }, -] -provides-extras = ["dev"] - [[package]] name = "iniconfig" version = "2.3.0" @@ -123,10 +101,21 @@ wheels = [ ] [[package]] -name = "setuptools" -version = "83.0.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/34/26/f5d29e25ffdb535afef2d35cdb55b325298f96debd670da4c325e08d70f4/setuptools-83.0.0.tar.gz", hash = "sha256:025bccbbf0fa05b6192bc64ae1e7b16e001fd6d6d4d5de03c97b1c1ade523bef", size = 1154254, upload-time = "2026-07-04T15:31:22.699Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/5d/40/e1e72872c6354b306daef1703549e8e83b4d43cfea356311bf722a043752/setuptools-83.0.0-py3-none-any.whl", hash = "sha256:29b23c360f22f414dc7336bb39178cc7bcbf6021ed2733cde173f09dba19abb3", size = 1008090, upload-time = "2026-07-04T15:31:20.885Z" }, +name = "substrate-retrieval" +version = "0.3.0" +source = { virtual = "." } + +[package.optional-dependencies] +dev = [ + { name = "pytest" }, + { name = "pytest-timeout" }, + { name = "ruff" }, ] + +[package.metadata] +requires-dist = [ + { name = "pytest", marker = "extra == 'dev'", specifier = "==9.0.3" }, + { name = "pytest-timeout", marker = "extra == 'dev'", specifier = "==2.3.1" }, + { name = "ruff", marker = "extra == 'dev'", specifier = "==0.9.10" }, +] +provides-extras = ["dev"]