From 3be2fcd3b3b76bf67494494e0c9787e310caf004 Mon Sep 17 00:00:00 2001 From: maphew Date: Fri, 10 Jul 2026 17:53:49 -0500 Subject: [PATCH 01/10] docs: add local-first multi-machine sync design (artifact ledger) Squashed follow-up changes: - fix(artifact): baseline curation of trashed sessions at opt-in - fix(artifact): roll back curation when metadata append fails - fix(artifact): delete corrupt bucket objects on pull so S3 self-heals - docs: design secure artifact S3 endpoints - docs: close S3 redirect downgrade gap - docs: plan secure artifact S3 endpoints - fix(artifact): secure S3 transport boundaries - test(artifact): tighten S3 security assertions - docs: explain S3 endpoint security policy - docs: clarify S3 corruption handling by scheme - docs: qualify S3 self-healing behavior - fix(artifact): pin insecure S3 override semantics - docs: design HTTP peer redirect rejection - docs: plan HTTP peer redirect rejection - fix(artifact): reject HTTP peer redirects - test(artifact): bound redirect source capture - docs: document peer redirect boundary and remove plans - fix(sync): harden artifact consistency boundaries - fix(sync): persist origin and heal invalid manifests - fix(sync): preserve provenance and retry state - fix(postgres): converge importer-first origin upgrades - fix(postgres): retain source state during consolidation - fix(sync): preserve remote history and purge durability - fix(server): serialize session lifecycle publication - fix(sync): harden artifact import and retention - fix(sync): cap artifact allocation amplification - fix(sync): bound nested artifact collections - fix(server): serialize curation publication - fix(artifact): harden sync boundaries - fix(sync): harden peer artifact convergence - fix(artifact): confine GC and recover deferred imports - fix(artifact): publish local sessions from HTTP peers Co-authored-by: Wes McKinney --- .github/workflows/ci.yml | 5 + Makefile | 13 +- README.md | 19 + SECURITY.md | 45 +- cmd/agentsview/cli.go | 80 +- cmd/agentsview/cli_test.go | 9 +- cmd/agentsview/main.go | 13 + cmd/agentsview/pg_watch_loop.go | 65 +- cmd/agentsview/sync.go | 202 +- cmd/agentsview/sync_gc.go | 131 + cmd/agentsview/sync_gc_auto_test.go | 142 + cmd/agentsview/sync_test.go | 290 +- cmd/agentsview/sync_watch.go | 198 ++ cmd/agentsview/sync_watch_test.go | 192 ++ docs/artifact-sync.md | 248 ++ docs/index.md | 6 +- docs/zensical.toml | 1 + frontend/messages/en.json | 96 +- frontend/messages/ko.json | 102 +- frontend/messages/zh-CN.json | 94 +- frontend/messages/zh-TW.json | 94 +- frontend/src/App.svelte | 5 + frontend/src/lib/api/generated/index.ts | 7 + .../models/ArtifactOriginsResponse.ts | 7 + .../lib/api/generated/models/ArtifactPeer.ts | 12 + .../generated/models/ArtifactPeersResponse.ts | 9 + .../generated/models/ArtifactPostResponse.ts | 12 + .../generated/models/DbMetadataConflict.ts | 18 + .../lib/api/generated/models/DbTopSession.ts | 1 - .../models/MetadataConflictsResponse.ts | 7 + .../generated/services/ArtifactsService.ts | 188 ++ .../api/generated/services/SessionsService.ts | 35 + .../lib/components/layout/AppHeader.svelte | 2 + .../layout/SessionBreadcrumb.svelte | 282 ++ .../layout/SessionBreadcrumb.test.ts | 97 + .../src/lib/components/peers/PeersPage.svelte | 343 ++ .../src/lib/components/trash/TrashPage.svelte | 116 +- .../lib/components/trash/TrashPage.test.ts | 152 + frontend/src/lib/i18n/i18n.test.ts | 2 +- frontend/src/lib/stores/router.svelte.ts | 2 + frontend/src/lib/stores/router.test.ts | 1 + internal/artifact/compression_test.go | 775 +++++ internal/artifact/export_usage_test.go | 92 + internal/artifact/format_test.go | 212 ++ internal/artifact/gc.go | 583 ++++ internal/artifact/gc_test.go | 630 ++++ internal/artifact/hlc.go | 261 ++ internal/artifact/hlc_test.go | 223 ++ internal/artifact/manifest_session.go | 227 ++ internal/artifact/manifest_session_test.go | 96 + internal/artifact/metadata.go | 481 +++ internal/artifact/metadata_test.go | 152 + internal/artifact/nested_limits_test.go | 475 +++ internal/artifact/peer.go | 726 ++++ internal/artifact/peer_test.go | 322 ++ internal/artifact/replay.go | 313 ++ internal/artifact/replay_test.go | 770 +++++ internal/artifact/sync.go | 2991 +++++++++++++++++ internal/artifact/sync_test.go | 2661 +++++++++++++++ internal/artifact/transport.go | 126 + internal/artifact/transport_http.go | 334 ++ internal/artifact/transport_http_test.go | 378 +++ internal/artifact/transport_s3.go | 661 ++++ .../artifact/transport_s3_miniotest_test.go | 151 + internal/artifact/transport_s3_test.go | 660 ++++ internal/artifact/twoinstance_test.go | 344 ++ internal/config/config.go | 187 ++ internal/config/config_test.go | 216 ++ internal/db/db.go | 67 + internal/db/db_test.go | 170 +- internal/db/messages.go | 34 + internal/db/metadata_baseline.go | 140 + internal/db/metadata_baseline_test.go | 52 + internal/db/metadata_replay.go | 830 +++++ internal/db/metadata_replay_test.go | 254 ++ internal/db/orphaned.go | 82 +- internal/db/read_only_test.go | 3 +- internal/db/schema.sql | 42 + internal/db/sessions.go | 161 +- internal/db/starred.go | 70 +- internal/db/store.go | 8 +- internal/db/store_contract_test.go | 41 +- internal/duckdb/curation.go | 8 +- internal/duckdb/metadata.go | 22 + internal/duckdb/store.go | 25 + internal/duckdb/store_contract_test.go | 15 +- internal/duckdb/store_test.go | 6 +- internal/duckdb/stubs.go | 9 +- internal/e2e/artifact_sync_test.go | 344 ++ internal/postgres/collision_pgtest_test.go | 1430 +++++++- internal/postgres/curation.go | 67 +- internal/postgres/curation_pgtest_test.go | 119 +- internal/postgres/integration_test.go | 2 +- internal/postgres/metadata.go | 22 + internal/postgres/name_source_pgtest_test.go | 10 +- internal/postgres/push.go | 1601 +++++++-- internal/postgres/push_pgtest_test.go | 305 +- internal/postgres/push_test.go | 481 ++- internal/postgres/schema.go | 12 +- internal/postgres/schema_pgtest_test.go | 51 + internal/postgres/schema_test.go | 98 +- internal/postgres/sessions.go | 25 + internal/postgres/store.go | 45 + internal/postgres/sync.go | 1 + .../server/artifact_http_transport_test.go | 192 ++ internal/server/artifact_peer_test.go | 522 +++ internal/server/bulk_star_metadata_test.go | 70 + internal/server/huma_route_groups.go | 1 + internal/server/huma_routes_artifacts.go | 392 +++ .../huma_routes_metadata_internal_test.go | 925 +++++ internal/server/huma_routes_pins.go | 105 +- internal/server/huma_routes_sessions.go | 277 +- internal/server/huma_routes_starred.go | 162 +- internal/server/metadata_events.go | 118 + internal/server/metadata_events_test.go | 992 ++++++ internal/server/server.go | 46 +- internal/server/session_mgmt_test.go | 42 + internal/sync/engine.go | 23 + internal/sync/engine_integration_test.go | 69 +- 119 files changed, 28366 insertions(+), 612 deletions(-) create mode 100644 cmd/agentsview/sync_gc.go create mode 100644 cmd/agentsview/sync_gc_auto_test.go create mode 100644 cmd/agentsview/sync_watch.go create mode 100644 cmd/agentsview/sync_watch_test.go create mode 100644 docs/artifact-sync.md create mode 100644 frontend/src/lib/api/generated/models/ArtifactOriginsResponse.ts create mode 100644 frontend/src/lib/api/generated/models/ArtifactPeer.ts create mode 100644 frontend/src/lib/api/generated/models/ArtifactPeersResponse.ts create mode 100644 frontend/src/lib/api/generated/models/ArtifactPostResponse.ts create mode 100644 frontend/src/lib/api/generated/models/DbMetadataConflict.ts create mode 100644 frontend/src/lib/api/generated/models/MetadataConflictsResponse.ts create mode 100644 frontend/src/lib/api/generated/services/ArtifactsService.ts create mode 100644 frontend/src/lib/components/peers/PeersPage.svelte create mode 100644 frontend/src/lib/components/trash/TrashPage.test.ts create mode 100644 internal/artifact/compression_test.go create mode 100644 internal/artifact/export_usage_test.go create mode 100644 internal/artifact/format_test.go create mode 100644 internal/artifact/gc.go create mode 100644 internal/artifact/gc_test.go create mode 100644 internal/artifact/hlc.go create mode 100644 internal/artifact/hlc_test.go create mode 100644 internal/artifact/manifest_session.go create mode 100644 internal/artifact/manifest_session_test.go create mode 100644 internal/artifact/metadata.go create mode 100644 internal/artifact/metadata_test.go create mode 100644 internal/artifact/nested_limits_test.go create mode 100644 internal/artifact/peer.go create mode 100644 internal/artifact/peer_test.go create mode 100644 internal/artifact/replay.go create mode 100644 internal/artifact/replay_test.go create mode 100644 internal/artifact/sync.go create mode 100644 internal/artifact/sync_test.go create mode 100644 internal/artifact/transport.go create mode 100644 internal/artifact/transport_http.go create mode 100644 internal/artifact/transport_http_test.go create mode 100644 internal/artifact/transport_s3.go create mode 100644 internal/artifact/transport_s3_miniotest_test.go create mode 100644 internal/artifact/transport_s3_test.go create mode 100644 internal/artifact/twoinstance_test.go create mode 100644 internal/db/metadata_baseline.go create mode 100644 internal/db/metadata_baseline_test.go create mode 100644 internal/db/metadata_replay.go create mode 100644 internal/db/metadata_replay_test.go create mode 100644 internal/duckdb/metadata.go create mode 100644 internal/e2e/artifact_sync_test.go create mode 100644 internal/postgres/metadata.go create mode 100644 internal/server/artifact_http_transport_test.go create mode 100644 internal/server/artifact_peer_test.go create mode 100644 internal/server/bulk_star_metadata_test.go create mode 100644 internal/server/huma_routes_artifacts.go create mode 100644 internal/server/metadata_events.go create mode 100644 internal/server/metadata_events_test.go diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 45574f66a..76f0a812a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -310,6 +310,11 @@ jobs: TEST_SSH_USER: testuser TEST_SSH_KEY: ${{ github.workspace }}/testdata/ssh/test_key + - name: Run MinIO object-store integration test + run: make test-minio + env: + CGO_ENABLED: "1" + e2e: runs-on: ubuntu-latest steps: diff --git a/Makefile b/Makefile index d44c7e33b..1df176f8b 100644 --- a/Makefile +++ b/Makefile @@ -33,7 +33,7 @@ AIR_BIN := $(shell if command -v air >/dev/null 2>&1; then command -v air; \ elif [ -x "$(GOPATH_FIRST)/bin/air" ]; then printf "%s" "$(GOPATH_FIRST)/bin/air"; \ fi) -.PHONY: build build-release install frontend frontend-dev dev check-air air-install desktop-dev desktop-build desktop-macos-app desktop-macos-dmg desktop-windows-installer desktop-linux-appimage desktop-app docs-install docs-build docs-serve docs-check docs-screenshots docs-assets-branch docs-generated-assets-branch docs-deploy-staging docs-deploy test test-short test-evalingest bench-backends bench-gate bench-gate-config test-postgres test-postgres-ci test-s3 postgres-up postgres-down test-ssh test-ssh-ci ssh-up ssh-down e2e e2e-duckdb vet lint lint-ci lint-golangci lint-golangci-ci nilaway nilaway-golangci-build lint-tools tidy clean release release-darwin-arm64 release-darwin-amd64 release-linux-amd64 install-hooks ensure-embed-dir pricing-snapshot sqlite-vec-header dev-snapshot help +.PHONY: build build-release install frontend frontend-dev dev check-air air-install desktop-dev desktop-build desktop-macos-app desktop-macos-dmg desktop-windows-installer desktop-linux-appimage desktop-app docs-install docs-build docs-serve docs-check docs-screenshots docs-assets-branch docs-generated-assets-branch docs-deploy-staging docs-deploy test test-short test-evalingest bench-backends bench-gate bench-gate-config test-postgres test-postgres-ci test-s3 test-minio postgres-up postgres-down test-ssh test-ssh-ci ssh-up ssh-down e2e e2e-duckdb vet lint lint-ci lint-golangci lint-golangci-ci nilaway nilaway-golangci-build lint-tools tidy clean release release-darwin-arm64 release-darwin-amd64 release-linux-amd64 install-hooks ensure-embed-dir pricing-snapshot sqlite-vec-header dev-snapshot help # Ensure go:embed has at least one file (no-op if frontend is built) ensure-embed-dir: @@ -348,6 +348,11 @@ test-postgres-ci: pricing-snapshot ensure-embed-dir test-s3: pricing-snapshot ensure-embed-dir CGO_ENABLED=1 go test -tags "fts5,s3test" -v ./internal/sync/... -run TestS3 -count=1 +# MinIO/S3 object-store integration test. testcontainers starts and tears down +# the MinIO container automatically, so this just needs a working Docker daemon. +test-minio: ensure-embed-dir + CGO_ENABLED=1 go test -tags "fts5,miniotest" -v ./internal/artifact/... -run MinIO -count=1 + # Start test SSH container ssh-up: docker compose -f docker-compose.test.yml up -d --build --wait sshd @@ -368,8 +373,9 @@ test-ssh: pricing-snapshot ensure-embed-dir ssh-up test-ssh-ci: pricing-snapshot ensure-embed-dir CGO_ENABLED=1 go test -tags "fts5,sshtest" -v ./internal/ssh/... -count=1 -# Run Playwright E2E tests -e2e: +# Run artifact sync and Playwright E2E tests +e2e: ensure-embed-dir + CGO_ENABLED=1 go test -tags "fts5,e2e" ./internal/e2e -v -count=1 cd frontend && npx playwright test # Run focused Playwright smoke tests against duckdb serve. @@ -544,6 +550,7 @@ help: @echo " bench-gate - Run the hot-path benchmarks CI gates PRs on" @echo " test-postgres - Run PostgreSQL integration tests" @echo " test-s3 - Run S3 discovery integration tests (Docker)" + @echo " test-minio - Run MinIO/S3 object-store integration test (needs Docker)" @echo " postgres-up - Start test PostgreSQL container" @echo " postgres-down - Stop test PostgreSQL container" @echo " test-ssh - Run SSH integration tests" diff --git a/README.md b/README.md index cbbfc033c..ddb6f4d08 100644 --- a/README.md +++ b/README.md @@ -103,6 +103,25 @@ Use `--public-origin` (repeatable or comma-separated) to trust additional browser origins. If you expose the UI beyond loopback, also enable `--require-auth`. +## Local-First Artifact Sync + +Artifact sync is for a fully trusted personal fleet. It exchanges immutable +artifacts through a dedicated folder, HTTP peer, or S3-compatible target and +imports them into each machine's local SQLite archive: + +```bash +agentsview sync --init /path/to/agentsview-artifacts +agentsview sync /path/to/agentsview-artifacts +agentsview sync --watch /path/to/agentsview-artifacts +``` + +Do not sync the live SQLite database, `AGENTSVIEW_DATA_DIR`, or whole raw agent +directories. Two intermittent machines only sync when both can reach the same +transport unless you provide a rendezvous such as a NAS folder, cloud-synced +folder, S3-compatible bucket, or always-on peer. See +[Trusted-Fleet Artifact Sync](docs/artifact-sync.md) for setup and safety +boundaries. + ## Docker The container image defaults to local `agentsview serve`. Set `PG_SERVE=1` to diff --git a/SECURITY.md b/SECURITY.md index 3dc153e7e..b46d4396a 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -30,11 +30,11 @@ risk via documented flags and config. attacker. - Parser crashes, excessive resource use, or active-content injection triggered by content inside supported session files. Session files often contain - web/tool output that agentsview did not author, and defensive parsing of that - content is a security-relevant concern. + web/tool output that agentsview did not author, and defensive parsing of + that content is a security-relevant concern. - Inadvertent exposure of secrets that appear in transcripts. agentsview ships a - best-effort secret detector and redacts findings in the UI and CLI by default - (see [Secrets subsystem](#secrets-subsystem)). + best-effort secret detector and redacts findings in the UI and CLI by + default (see [Secrets subsystem](#secrets-subsystem)). ### Explicitly out of scope (today) @@ -63,6 +63,7 @@ risk via documented flags and config. | HTTP server → caller | Loopback-trusted; bearer-gated for `/api/` when `--require-auth` | Static assets are not gated. | | Browser → HTTP server | Host-header allowlist + CORS + CSP + X-Frame-Options enforced | DNS-rebinding, framing, and cross-origin defenses. | | agentsview → PostgreSQL (pg push) | TLS required for non-loopback hosts | Plaintext rejected unless `allow_insecure = true` is set explicitly. | +| agentsview → artifact HTTP peer | HTTPS required for non-loopback peers | Plaintext rejected unless `sync --allow-insecure` is explicit. | | agentsview → update endpoint | One-way outbound, opt-out | Disable with `--no-update-check`. | | agentsview → LiteLLM pricing | One-way outbound, on-demand | Public JSON fetched from GitHub raw; no session data sent. | @@ -70,8 +71,8 @@ risk via documented flags and config. - The local archive (SQLite + FTS5 index) stores indexed session data in plaintext. This includes assistant responses, user prompts, tool arguments, - command output, file contents fetched by agents, and any secrets that may have - been pasted into an agent session. + command output, file contents fetched by agents, and any secrets that may + have been pasted into an agent session. - File permissions follow the user's umask. agentsview does not chmod the data directory and does not encrypt at rest. - Treat the agentsview data directory with the same care you would treat your @@ -89,10 +90,10 @@ explicitly because "data stays on your machine" is the default but is not a complete description of the system once optional features are in use. - **Local UI / API.** The HTTP server binds to `127.0.0.1` by default. When - exposed beyond loopback, `--require-auth` should be enabled. Authentication is - a bearer token applied to `/api/` routes only; static assets remain ungated. - Browser-facing defenses (Host-header allowlist, CORS restrictions, CSP, - `X-Frame-Options: DENY`) are always on. See the CLI reference for token + exposed beyond loopback, `--require-auth` should be enabled. Authentication + is a bearer token applied to `/api/` routes only; static assets remain + ungated. Browser-facing defenses (Host-header allowlist, CORS restrictions, + CSP, `X-Frame-Options: DENY`) are always on. See the CLI reference for token configuration. - **PostgreSQL sync.** `agentsview pg push` exports the local archive to a user-supplied PostgreSQL instance. Non-loopback DSNs are rejected unless TLS @@ -102,8 +103,14 @@ complete description of the system once optional features are in use. its access controls. - **SSH remote sync.** agentsview can pull session archives from another machine over SSH. Authentication is whatever the user's SSH configuration provides. - Pulled files are parsed locally as untrusted data and merged into the unified - archive. + Pulled files are parsed locally as untrusted data and merged into the + unified archive. +- **Artifact HTTP peer sync.** `agentsview sync http(s)://...` exchanges bearer + credentials and archive artifacts with a user-supplied peer. Non-loopback + peers require HTTPS by default; loopback HTTP is allowed, while remote + plaintext requires the deliberate `--allow-insecure` opt-in and emits a + warning. Redirects are rejected. Received artifacts remain untrusted + structured input even when the peer is part of a trusted personal fleet. - **Imports.** Imported archives from other agentsview instances or third-party exports are treated as untrusted structured data, no different from session files written by local agents. @@ -125,9 +132,9 @@ The HTTP server applies the following defenses unconditionally: - **CORS restrictions.** Cross-origin API requests must come from an allowed origin or carry the bearer token; preflight handling is explicit. - **Content-Security-Policy.** A policy pinning the server's exact origin for - script/style/image/font/default-src is set on non-API responses. `connect-src` - is intentionally widened to allow the remote-server feature in the SPA; this - is a documented tradeoff. + script/style/image/font/default-src is set on non-API responses. + `connect-src` is intentionally widened to allow the remote-server feature in + the SPA; this is a documented tradeoff. - **X-Frame-Options: DENY.** Framing is disallowed on non-API responses. ## Secrets subsystem @@ -181,11 +188,11 @@ its own proposal. 1. **Multi-user machine support.** Is agentsview ever meant to run on a shared host, and if so what are the minimum hardening steps? 1. **`allow_insecure` UX.** Should setting `[pg] allow_insecure = true` require - an additional confirmation (e.g., a `--yes-really` flag) on first use, given - that it disables the only protection against plaintext PG egress? + an additional confirmation (e.g., a `--yes-really` flag) on first use, + given that it disables the only protection against plaintext PG egress? 1. **Deletion guarantees.** Should "permanent delete" grow into a stronger - erasure path (VACUUM, WAL checkpoint + truncate, mirror propagation to PG/SSH - targets), or should the docs simply make the current limits clearer? + erasure path (VACUUM, WAL checkpoint + truncate, mirror propagation to + PG/SSH targets), or should the docs simply make the current limits clearer? 1. **Secret detection scope.** Should the detector expand (more patterns, structured-secret types), should redacted-by-default extend to exports, and should there be a "scrub-on-import" pass? diff --git a/cmd/agentsview/cli.go b/cmd/agentsview/cli.go index 8bf4431b9..591b46532 100644 --- a/cmd/agentsview/cli.go +++ b/cmd/agentsview/cli.go @@ -287,7 +287,7 @@ func newOpenAPICommand() *cobra.Command { func newSyncCommand() *cobra.Command { var cfg SyncConfig cmd := &cobra.Command{ - Use: "sync", + Use: "sync [artifact-folder]", Short: "Sync session data without serving", Long: "Sync session data into the local database without starting the\n" + "HTTP server.\n\n" + @@ -298,12 +298,33 @@ func newSyncCommand() *cobra.Command { "exits non-zero if any configured host failed.\n\n" + "With --host, syncs only that host. A running local daemon may use a\n" + "matching configured remote_hosts entry and transport; otherwise,\n" + - "ad hoc --host sync uses your existing SSH configuration and requires\n" + + "ad hoc --host sync falls back to SSH.\n\n" + + "With an artifact-folder argument or --artifact-folder, sync also\n" + + "exchanges local-first immutable artifacts with that folder target.\n" + + "Artifact sync v1 is for a fully trusted personal fleet. Use a\n" + + "dedicated artifact share folder; do not point this at the\n" + + "agentsview data directory, raw agent directories, or the live\n" + + "SQLite database file and its WAL/SHM files.\n\n" + + "Use --init with an artifact folder on first setup to generate and\n" + + "persist this machine's artifact origin, backfill existing local\n" + + "sessions into the artifact store, exchange with the folder target,\n" + + "and import any peer artifacts already present. Two intermittent\n" + + "machines only sync while both can reach the same transport; use a\n" + + "NAS, cloud folder, object store, or always-on peer as a rendezvous\n" + + "when asynchronous convergence matters.\n\n" + + "Use --watch with an artifact folder to keep syncing. Watch mode\n" + + "runs an initial local sync and artifact exchange, coalesces file\n" + + "changes with --debounce, retries failed exchanges on later\n" + + "changes or --interval ticks, and performs a final best-effort\n" + + "exchange on shutdown. Combining --init with --watch publishes\n" + + "the first-run baseline on the first successful exchange, then\n" + + "keeps watching.\n\n" + + "Remote sync uses your existing SSH configuration and requires\n" + "key-based (passwordless) auth; it never prompts for a password.", GroupID: groupCore, SilenceUsage: true, - Args: cobra.NoArgs, - PreRunE: func(cmd *cobra.Command, _ []string) error { + Args: cobra.MaximumNArgs(1), + PreRunE: func(cmd *cobra.Command, args []string) error { if cfg.Host == "" { if cmd.Flags().Changed("user") || cmd.Flags().Changed("port") { @@ -312,9 +333,23 @@ func newSyncCommand() *cobra.Command { ) } } + if err := applySyncArtifactTarget(&cfg, args, cmd.Flags().Changed("artifact-folder")); err != nil { + return err + } + if err := validateSyncConfig(cfg); err != nil { + return err + } return nil }, Run: func(cmd *cobra.Command, args []string) { + if cfg.Watch { + runSyncWatch(cfg) + return + } + if cmd.Flags().Changed("debounce") || cmd.Flags().Changed("interval") { + fmt.Fprintln(os.Stderr, + "warning: --debounce and --interval have no effect without --watch") + } runSync(cfg) }, } @@ -322,6 +357,10 @@ func newSyncCommand() *cobra.Command { &cfg.Full, "full", false, "Force a full resync regardless of data version", ) + cmd.Flags().BoolVar( + &cfg.Init, "init", false, + "Initialize artifact sync with the folder target", + ) cmd.Flags().StringVar( &cfg.Host, "host", "", "SSH hostname for remote sync", @@ -334,6 +373,38 @@ func newSyncCommand() *cobra.Command { &cfg.Port, "port", 0, "SSH port for remote sync (default: 22)", ) + cmd.Flags().StringVar( + &cfg.ArtifactFolder, "artifact-folder", "", + "Exchange local-first sync artifacts with a folder, http(s) peer, or s3:// target", + ) + cmd.Flags().StringVar( + &cfg.Token, "token", "", + "Bearer token for an http(s) artifact peer target", + ) + cmd.Flags().BoolVar( + &cfg.AllowInsecure, "allow-insecure", false, + "Allow plaintext HTTP to a non-loopback artifact peer", + ) + cmd.Flags().BoolVar( + &cfg.Watch, "watch", false, + "Run artifact folder sync continuously, syncing on change plus a periodic floor", + ) + cmd.Flags().DurationVar( + &cfg.Debounce, "debounce", defaultWatchDebounce, + "Coalesce window after a change before artifact sync (--watch only)", + ) + cmd.Flags().DurationVar( + &cfg.Interval, "interval", defaultWatchInterval, + "Periodic floor artifact sync interval (--watch only)", + ) + cmd.Flags().DurationVar( + &cfg.GCGrace, "gc-grace", defaultArtifactGCGrace, + "Minimum age before superseded artifacts are auto-collected after a folder sync", + ) + cmd.Flags().BoolVar( + &cfg.NoGC, "no-gc", false, + "Disable automatic garbage collection of superseded artifacts after a folder sync", + ) cmd.Flags().StringVar( &cfg.CPUProfile, "cpuprofile", "", "Write CPU profile to file (developer use)", @@ -351,6 +422,7 @@ func newSyncCommand() *cobra.Command { panic(err) } } + cmd.AddCommand(newSyncGCCommand()) return cmd } diff --git a/cmd/agentsview/cli_test.go b/cmd/agentsview/cli_test.go index c52d4056b..3fbfea091 100644 --- a/cmd/agentsview/cli_test.go +++ b/cmd/agentsview/cli_test.go @@ -103,6 +103,13 @@ func TestDuckDBPushHelpShowsProjectFlags(t *testing.T) { } } +func TestSyncHelpShowsArtifactTransportSafetyFlag(t *testing.T) { + help, err := executeCommand(newRootCommand(), "sync", "--help") + require.NoError(t, err, "Execute") + assert.Contains(t, help, "--allow-insecure") + assert.Contains(t, help, "non-loopback artifact peer") +} + func TestPGStatusHelpShowsProjectFlags(t *testing.T) { help, err := executeCommand(newRootCommand(), "pg", "status", "--help") require.NoError(t, err, "Execute") @@ -325,7 +332,7 @@ func TestRootHelpDocumentsRemoteHosts(t *testing.T) { func TestSyncHelpMentionsConfiguredHosts(t *testing.T) { help, err := executeCommand(newRootCommand(), "sync", "--help") require.NoError(t, err, "Execute") - for _, want := range []string{"remote_hosts", "--host", "passwordless"} { + for _, want := range []string{"remote_hosts", "--host", "passwordless", "trusted personal fleet", "rendezvous"} { assert.Contains(t, help, want, "sync help missing %q", want) } } diff --git a/cmd/agentsview/main.go b/cmd/agentsview/main.go index 043b26e79..aa1f51400 100644 --- a/cmd/agentsview/main.go +++ b/cmd/agentsview/main.go @@ -18,6 +18,7 @@ import ( _ "time/tzdata" "github.com/spf13/cobra" + "go.kenn.io/agentsview/internal/artifact" "go.kenn.io/agentsview/internal/config" "go.kenn.io/agentsview/internal/db" "go.kenn.io/agentsview/internal/parser" @@ -286,6 +287,18 @@ func runServe(cfg config.Config, opts serveOptions) { } cfg = preparedCfg + // Reconcile an already-adopted artifact origin so every origin lookup + // (recorder, peer import, folder sync) agrees: the config.toml origin is + // authoritative and overwrites a divergent DB sync-state value. Serve + // never creates an origin — a machine opts into artifact sync only via + // `sync --init`, a sync run, or an incoming peer exchange, and until then + // curation stays local with no metadata ledger writes. + if cfg.DataDir != "" && !database.ReadOnly() && cfg.ArtifactOriginID != "" { + if err := artifact.AdoptOrigin(database, cfg.ArtifactOriginID); err != nil { + fatal("reconcile artifact origin id: %v", err) + } + } + srvOpts := []server.Option{ server.WithVersion(server.VersionInfo{ Version: version, diff --git a/cmd/agentsview/pg_watch_loop.go b/cmd/agentsview/pg_watch_loop.go index 1e820ca74..fba90dad9 100644 --- a/cmd/agentsview/pg_watch_loop.go +++ b/cmd/agentsview/pg_watch_loop.go @@ -4,6 +4,9 @@ import ( "context" "log" "time" + + "go.kenn.io/agentsview/internal/config" + syncpkg "go.kenn.io/agentsview/internal/sync" ) // pushReason labels why a push was triggered, for logging. @@ -29,12 +32,12 @@ const defaultFlushTimeout = 30 * time.Second // under test. In production, after is time.After and floor is a // time.Ticker channel. type pushLoop struct { - debounce time.Duration - dirty chan struct{} - floor <-chan time.Time - after func(time.Duration) <-chan time.Time - push func(ctx context.Context, reason pushReason) error - label string + logPrefix string + debounce time.Duration + dirty chan struct{} + floor <-chan time.Time + after func(time.Duration) <-chan time.Time + push func(ctx context.Context, reason pushReason) error // flushTimeout bounds the final shutdown-flush push. Zero means // no bound (used in tests that inject a fake pusher). flushTimeout time.Duration @@ -46,22 +49,30 @@ func newPushLoop( debounce, interval time.Duration, push func(context.Context, pushReason) error, ) (*pushLoop, *time.Ticker) { - return newPushLoopWithLabel("pg watch", debounce, interval, push) + return newNamedPushLoop("pg watch", debounce, interval, push) } func newPushLoopWithLabel( label string, debounce, interval time.Duration, push func(context.Context, pushReason) error, +) (*pushLoop, *time.Ticker) { + return newNamedPushLoop(label, debounce, interval, push) +} + +func newNamedPushLoop( + logPrefix string, + debounce, interval time.Duration, + push func(context.Context, pushReason) error, ) (*pushLoop, *time.Ticker) { ticker := time.NewTicker(interval) return &pushLoop{ + logPrefix: logPrefix, debounce: debounce, dirty: make(chan struct{}, 1), floor: ticker.C, after: time.After, push: push, - label: label, flushTimeout: defaultFlushTimeout, }, ticker } @@ -112,6 +123,42 @@ func (l *pushLoop) Run(ctx context.Context) { func (l *pushLoop) doPush(ctx context.Context, reason pushReason) { if err := l.push(ctx, reason); err != nil { - log.Printf("%s: push (%s) failed: %v", l.label, reason, err) + prefix := l.logPrefix + if prefix == "" { + prefix = "watch" + } + log.Printf("%s: push (%s) failed: %v", prefix, reason, err) } } + +type watchedSinkConfig struct { + AppConfig config.Config + Engine *syncpkg.Engine + Debounce time.Duration + Interval time.Duration + LogPrefix string + Push func(context.Context, pushReason) error +} + +func runWatchedSink(ctx context.Context, cfg watchedSinkConfig) { + loop, ticker := newNamedPushLoop( + cfg.LogPrefix, cfg.Debounce, cfg.Interval, cfg.Push, + ) + defer ticker.Stop() + + stopWatcher, unwatchedDirs := startFileWatcher(cfg.AppConfig, cfg.Engine, + func(batch syncpkg.WatchBatch) { + syncWatchBatch(ctx, cfg.Engine, batch) + loop.NotifyDirty() + }, + ) + defer stopWatcher() + if len(unwatchedDirs) > 0 { + log.Printf( + "%s: %d root(s) not watched; relying on the %s floor for coverage", + cfg.LogPrefix, len(unwatchedDirs), cfg.Interval, + ) + } + + loop.Run(ctx) +} diff --git a/cmd/agentsview/sync.go b/cmd/agentsview/sync.go index cd42825ba..d9492bd6d 100644 --- a/cmd/agentsview/sync.go +++ b/cmd/agentsview/sync.go @@ -16,6 +16,7 @@ import ( "strings" "time" + "go.kenn.io/agentsview/internal/artifact" "go.kenn.io/agentsview/internal/config" "go.kenn.io/agentsview/internal/db" "go.kenn.io/agentsview/internal/parser" @@ -26,10 +27,23 @@ import ( // SyncConfig holds parsed CLI options for the sync command. type SyncConfig struct { - Full bool - Host string - User string - Port int + Full bool + Init bool + Watch bool + Debounce time.Duration + Interval time.Duration + Host string + User string + Port int + ArtifactFolder string + // Token is the bearer token used for an http(s):// artifact peer target. + Token string + // AllowInsecure permits plaintext HTTP to a non-loopback artifact peer. + AllowInsecure bool + // GCGrace is the minimum age before superseded artifacts are auto-collected + // after a folder sync. NoGC disables that automatic collection. + GCGrace time.Duration + NoGC bool // CPUProfile, MemProfile, and Trace are hidden flags that capture a // pprof CPU profile, allocation snapshot, and runtime trace for the // sync pass. Empty strings disable each independently. @@ -38,17 +52,54 @@ type SyncConfig struct { Trace string } +func applySyncArtifactTarget(cfg *SyncConfig, args []string, flagChanged bool) error { + if len(args) == 0 { + return nil + } + if flagChanged { + return errors.New("artifact folder target cannot be provided both as an argument and --artifact-folder") + } + cfg.ArtifactFolder = args[0] + return nil +} + +func validateSyncConfig(cfg SyncConfig) error { + if cfg.Init && cfg.Host != "" { + return errors.New("--init cannot be combined with --host") + } + if cfg.Init && cfg.ArtifactFolder == "" { + return errors.New( + "--init requires an artifact folder target", + ) + } + if cfg.Watch && cfg.Host != "" { + return errors.New("--watch cannot be combined with --host") + } + if cfg.Watch && cfg.ArtifactFolder == "" { + return errors.New("--watch requires an artifact folder target") + } + if cfg.Host != "" && cfg.ArtifactFolder != "" { + // SSH remote sync (--host) returns before artifact sync runs, so a + // combined invocation would silently ignore the artifact target. + return errors.New("--host cannot be combined with an artifact target") + } + return nil +} + func runSync(cfg SyncConfig) { - if doSync(cfg) { + hadRemoteFailures, err := doSync(cfg) + if err != nil { + fatal("sync: %v", err) + } + if hadRemoteFailures { os.Exit(1) } } -// doSync performs the sync run and reports whether any configured -// remote host failed. It owns the deferred cleanup (profile stop, -// db close) so runSync can translate the result into a non-zero -// exit code without skipping that cleanup. -func doSync(cfg SyncConfig) (hadRemoteFailures bool) { +// doSync performs the sync run and reports whether any configured remote host +// failed. It owns the deferred cleanup (profile stop, db close) so runSync can +// translate the result into a non-zero exit code without skipping that cleanup. +func doSync(cfg SyncConfig) (hadRemoteFailures bool, err error) { appCfg, err := config.LoadMinimal() if err != nil { log.Fatalf("loading config: %v", err) @@ -82,14 +133,18 @@ func doSync(cfg SyncConfig) (hadRemoteFailures bool) { } if includeLocal || len(remoteHosts) > 0 { - tr, err := ensureTransport( - &appCfg, transportIntentArchiveWrite, 0, - ) + tr, err := syncTransport(&appCfg, cfg) if err != nil { fatal("detecting daemon: %v", err) } if tr.Mode == transportHTTP { useDaemon := useDaemonForSync(tr) + if useDaemon && cfg.ArtifactFolder != "" { + return false, fmt.Errorf( + "artifact sync cannot run while a writable daemon owns the SQLite archive; " + + "run `agentsview serve stop` and retry", + ) + } if useDaemon && len(remoteHosts) > 0 { fmt.Println("Running sync with remotes via daemon...") progress := newRemoteProgressPrinter(os.Stdout, time.Now) @@ -102,7 +157,7 @@ func doSync(cfg SyncConfig) (hadRemoteFailures bool) { fatal("daemon remote sync: %v", err) } reportRemoteFailures(failures) - return len(failures) > 0 + return len(failures) > 0, nil } if useDaemon { start := time.Now() @@ -127,7 +182,7 @@ func doSync(cfg SyncConfig) (hadRemoteFailures bool) { fatal("daemon sync: %v", err) } printSyncSummary(stats, start) - return false + return false, nil } // Read-only mirror daemons do not own the local SQLite // archive. Remote sync can still proceed through the direct @@ -150,7 +205,7 @@ func doSync(cfg SyncConfig) (hadRemoteFailures bool) { if cfg.Host != "" { runRemoteSync(appCfg, database, cfg) - return false + return false, nil } failures := syncLocalAndRemotes( @@ -164,8 +219,18 @@ func doSync(cfg SyncConfig) (hadRemoteFailures bool) { return runRemoteSyncOnce(appCfg, database, rh, full) }, ) + if cfg.ArtifactFolder != "" { + runArtifactFolderSync(appCfg, database, cfg.ArtifactFolder, cfg) + } reportRemoteFailures(failures) - return len(failures) > 0 + return len(failures) > 0, nil +} + +func syncTransport(appCfg *config.Config, cfg SyncConfig) (transport, error) { + if cfg.ArtifactFolder != "" { + return detectTransport(appCfg.DataDir, appCfg.AuthToken, 0) + } + return ensureTransport(appCfg, transportIntentArchiveWrite, 0) } func useDaemonForSync(tr transport) bool { @@ -279,6 +344,109 @@ func (p *remoteProgressPrinter) finishCurrent() { p.inPlace = false } +// resolveArtifactOrigin returns this machine's artifact origin. A config +// origin wins; otherwise an origin already stored in database sync state -- +// for example one minted by an incoming peer exchange on serve before the +// config ever initialized an origin -- is promoted into the config; otherwise +// a new origin is generated and persisted. Without the promotion step, CLI +// sync would generate a second origin that serve later adopts as +// authoritative, stranding metadata events published under the DB origin. +// The resolved config authority is reconciled back into database sync state so +// DB-only consumers such as direct PG push use the same canonical identity. +func resolveArtifactOrigin( + appCfg config.Config, database *db.DB, +) (string, error) { + var origin string + var err error + if appCfg.ArtifactOriginID == "" { + stored, readErr := artifact.StoredOrigin(database) + if readErr != nil { + return "", readErr + } + if stored != "" { + origin, err = appCfg.AdoptArtifactOriginID(stored) + } else { + origin, err = appCfg.EnsureArtifactOriginID() + } + } else { + origin, err = appCfg.EnsureArtifactOriginID() + } + if err != nil { + return "", err + } + if err := artifact.AdoptOrigin(database, origin); err != nil { + return "", fmt.Errorf("reconciling artifact origin in database: %w", err) + } + return origin, nil +} + +func runArtifactFolderSync( + appCfg config.Config, database *db.DB, target string, cfg SyncConfig, +) { + origin, err := resolveArtifactOrigin(appCfg, database) + if err != nil { + fatal("artifact sync origin: %v", err) + } + ctx := context.Background() + res, err := syncArtifactFolder( + ctx, appCfg, database, target, origin, artifactPeerToken(cfg), + cfg.AllowInsecure, cfg.Init, nil, + ) + if err != nil { + fatal("artifact sync: %v", err) + } + printArtifactSyncSummary(res, cfg.Init) + if !cfg.NoGC { + autoGCAfterFolderSync(ctx, appCfg.DataDir, target, cfg.GCGrace) + } +} + +// artifactPeerToken resolves the bearer token for an HTTP peer target. Tokens +// are never inferred from local server auth because an explicit peer URL may +// point at an untrusted endpoint. +func artifactPeerToken(cfg SyncConfig) string { + return cfg.Token +} + +func syncArtifactFolder( + ctx context.Context, + appCfg config.Config, + database *db.DB, + target string, + origin string, + token string, + allowInsecure bool, + baselineMetadata bool, + onDataChanged func(), +) (artifact.SyncResult, error) { + if !artifact.IsFolderTarget(target) && !artifact.IsHTTPTarget(target) && !artifact.IsObjectTarget(target) { + return artifact.SyncResult{}, fmt.Errorf( + "artifact sync supports local folder, http(s) peer, or s3:// object-store targets: %s", + target, + ) + } + return artifact.Sync(ctx, database, artifact.SyncOptions{ + DataDir: appCfg.DataDir, + Target: target, + Origin: origin, + Token: token, + AllowInsecure: allowInsecure, + BaselineMetadata: baselineMetadata, + OnDataChanged: onDataChanged, + }) +} + +func printArtifactSyncSummary(res artifact.SyncResult, init bool) { + label := "Artifact sync" + if init { + label = "Artifact sync initialized" + } + fmt.Printf( + "%s (%s): exported %d sessions, imported %d sessions / %d messages / %d metadata events\n", + label, res.Origin, res.ExportedSessions, res.ImportedSessions, res.ImportedMessages, res.ImportedMetadata, + ) +} + // syncLocalAndRemotes runs the local sync, then the configured // remote hosts. A local resync (forced via --full or an automatic // data-version resync) forces every remote sync full as well, so diff --git a/cmd/agentsview/sync_gc.go b/cmd/agentsview/sync_gc.go new file mode 100644 index 000000000..5176458a6 --- /dev/null +++ b/cmd/agentsview/sync_gc.go @@ -0,0 +1,131 @@ +package main + +import ( + "context" + "fmt" + "io" + "log" + "path/filepath" + "time" + + "github.com/spf13/cobra" + "go.kenn.io/agentsview/internal/artifact" +) + +const defaultArtifactGCGrace = 7 * 24 * time.Hour + +// SyncGCConfig holds parsed options for artifact sync garbage collection. +type SyncGCConfig struct { + Target string + Grace time.Duration + DryRun bool +} + +func newSyncGCCommand() *cobra.Command { + var cfg SyncGCConfig + cfg.Grace = defaultArtifactGCGrace + cmd := &cobra.Command{ + Use: "gc ", + Short: "Garbage collect superseded sync artifacts", + Long: "Garbage collect superseded artifact sync files from a local\n" + + "artifact folder. GC keeps the latest checkpoint per origin,\n" + + "keeps every manifest, segment, and raw artifact reachable from\n" + + "that checkpoint, and only removes unreferenced files older than\n" + + "the grace window. Origins without checkpoints are skipped.", + SilenceUsage: true, + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + cfg.Target = args[0] + return runSyncGC(cmd, cfg) + }, + } + cmd.Flags().DurationVar( + &cfg.Grace, + "grace", + defaultArtifactGCGrace, + "Minimum age before unreferenced artifacts can be deleted", + ) + cmd.Flags().BoolVar( + &cfg.DryRun, + "dry-run", + false, + "Log artifacts that would be deleted without removing them", + ) + return cmd +} + +func runSyncGC(cmd *cobra.Command, cfg SyncGCConfig) error { + if !artifact.IsFolderTarget(cfg.Target) { + return fmt.Errorf( + "artifact gc currently supports local folder targets only: %s", + cfg.Target, + ) + } + res, err := artifact.GarbageCollect(cmd.Context(), artifact.GCOptions{ + Root: cfg.Target, + Grace: cfg.Grace, + DryRun: cfg.DryRun, + Logf: log.Printf, + }) + if err != nil { + return fmt.Errorf("artifact gc: %w", err) + } + printArtifactGCSummary(cmd.OutOrStdout(), res) + return nil +} + +// autoGCAfterFolderSync garbage-collects superseded artifacts from the local +// store and the shared folder target after a successful folder sync. Both are +// collected with the same synced checkpoint view so a later set-union exchange +// cannot re-propagate the deleted files. GC is opportunistic: failures are +// logged, not fatal. Non-folder targets are skipped because this operation +// cannot prune their remote history; collecting only the local store would make +// the next set-union exchange download every remote-only artifact again. +func autoGCAfterFolderSync(ctx context.Context, dataDir, target string, grace time.Duration) { + if !artifact.IsFolderTarget(target) { + return + } + if grace <= 0 { + grace = defaultArtifactGCGrace + } + roots := []string{filepath.Join(dataDir, "artifacts"), target} + for _, root := range roots { + res, err := artifact.GarbageCollect(ctx, artifact.GCOptions{ + Root: root, + Grace: grace, + Logf: log.Printf, + }) + if err != nil { + log.Printf("artifact gc (%s): %v", root, err) + continue + } + if res.Deleted > 0 { + log.Printf( + "artifact gc (%s): deleted %d artifact(s) (%s)", + root, res.Deleted, formatBytes(res.BytesDeleted), + ) + } + } +} + +func printArtifactGCSummary(w io.Writer, res artifact.GCResult) { + action := "deleted" + count := res.Deleted + bytes := res.BytesDeleted + if res.DryRun { + action = "would delete" + count = res.Eligible + bytes = res.BytesEligible + } + fmt.Fprintf( + w, + "Artifact GC: scanned %d origin(s), skipped %d without checkpoints, found %d unreferenced artifact(s), kept %d within grace, %s %d artifact(s) (%s)\n", + res.Origins, + res.SkippedOrigins, + res.Candidates, + res.KeptByGrace, + action, + count, + formatBytes(bytes), + ) +} diff --git a/cmd/agentsview/sync_gc_auto_test.go b/cmd/agentsview/sync_gc_auto_test.go new file mode 100644 index 000000000..d020c3050 --- /dev/null +++ b/cmd/agentsview/sync_gc_auto_test.go @@ -0,0 +1,142 @@ +package main + +import ( + "context" + "io/fs" + "os" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/require" + "go.kenn.io/agentsview/internal/artifact" + "go.kenn.io/agentsview/internal/db" +) + +// autoGCTestStore builds a data dir whose artifact store contains one superseded +// session manifest/segment/checkpoint (from a first export) plus the live set +// (from a second export after a content change), and mirrors it to a folder +// target. Every file is backdated past the grace window so the unreferenced +// ones are eligible for collection. +func autoGCTestStore(t *testing.T, target string) (dataDir, origin string) { + t.Helper() + ctx := context.Background() + dataDir = t.TempDir() + origin = "laptop-a1b2c3" + artifacts := filepath.Join(dataDir, "artifacts") + + database, err := db.Open(filepath.Join(t.TempDir(), "test.db")) + require.NoError(t, err) + t.Cleanup(func() { database.Close() }) + + require.NoError(t, database.UpsertSession(db.Session{ + ID: "sess-1", + Project: "alpha", + Machine: "local", + Agent: "claude", + MessageCount: 2, + UserMessageCount: 1, + CreatedAt: "2026-06-14T01:02:03Z", + })) + require.NoError(t, database.ReplaceSessionMessages("sess-1", []db.Message{ + {SessionID: "sess-1", Ordinal: 0, Role: "user", Content: "hello", ContentLength: 5}, + {SessionID: "sess-1", Ordinal: 1, Role: "assistant", Content: "world", ContentLength: 5}, + })) + _, err = artifact.Export(ctx, database, artifacts, origin) + require.NoError(t, err) + + // Change content so the next export supersedes the first manifest/segment. + require.NoError(t, database.ReplaceSessionMessages("sess-1", []db.Message{ + {SessionID: "sess-1", Ordinal: 0, Role: "user", Content: "hello", ContentLength: 5}, + {SessionID: "sess-1", Ordinal: 1, Role: "assistant", Content: "planet", ContentLength: 6}, + })) + _, err = artifact.Export(ctx, database, artifacts, origin) + require.NoError(t, err) + + require.NoError(t, artifact.CopyUnion(artifacts, target)) + + old := time.Now().Add(-72 * time.Hour) + backdateTree(t, artifacts, old) + backdateTree(t, target, old) + return dataDir, origin +} + +func backdateTree(t *testing.T, root string, ts time.Time) { + t.Helper() + require.NoError(t, filepath.Walk(root, func(p string, _ fs.FileInfo, err error) error { + if err != nil { + return err + } + return os.Chtimes(p, ts, ts) + })) +} + +func countFiles(t *testing.T, dir string) int { + t.Helper() + entries, err := os.ReadDir(dir) + if os.IsNotExist(err) { + return 0 + } + require.NoError(t, err) + n := 0 + for _, e := range entries { + if !e.IsDir() { + n++ + } + } + return n +} + +func TestAutoGCAfterFolderSyncCollectsBothLocalAndTarget(t *testing.T) { + target := t.TempDir() + dataDir, origin := autoGCTestStore(t, target) + localOriginRoot := filepath.Join(dataDir, "artifacts", origin) + targetOriginRoot := filepath.Join(target, origin) + + // Both stores start with the superseded set plus the live set. + for _, root := range []string{localOriginRoot, targetOriginRoot} { + require.Equal(t, 2, countFiles(t, filepath.Join(root, artifact.KindSegments))) + require.Equal(t, 2, countFiles(t, filepath.Join(root, artifact.KindManifests))) + require.Equal(t, 2, countFiles(t, filepath.Join(root, artifact.KindCheckpoints))) + } + + autoGCAfterFolderSync(context.Background(), dataDir, target, time.Hour) + + // Only the live set remains, in both the local store and the shared target, + // so a later set-union exchange cannot re-propagate the deleted files. + for _, root := range []string{localOriginRoot, targetOriginRoot} { + require.Equal(t, 1, countFiles(t, filepath.Join(root, artifact.KindSegments))) + require.Equal(t, 1, countFiles(t, filepath.Join(root, artifact.KindManifests))) + require.Equal(t, 1, countFiles(t, filepath.Join(root, artifact.KindCheckpoints))) + } +} + +func TestAutoGCAfterFolderSyncSkipsLocalGCForNonFolderTarget(t *testing.T) { + for _, tc := range []struct { + name string + target string + }{ + {name: "http", target: "https://peer.example.test:8080"}, + {name: "s3", target: "s3://bucket/artifacts"}, + } { + t.Run(tc.name, func(t *testing.T) { + // A non-folder target cannot be pruned by the same operation. Use a + // placeholder folder for its remote history: neither side may be + // collected, or the next set-union exchange would download the + // remote-only files again. + untouched := t.TempDir() + dataDir, origin := autoGCTestStore(t, untouched) + localOriginRoot := filepath.Join(dataDir, "artifacts", origin) + untouchedOriginRoot := filepath.Join(untouched, origin) + + autoGCAfterFolderSync(context.Background(), dataDir, tc.target, time.Hour) + + // Both stores retain the superseded set until a transport can prune + // them together. + require.Equal(t, 2, countFiles(t, + filepath.Join(localOriginRoot, artifact.KindSegments))) + require.Equal(t, 2, countFiles(t, + filepath.Join(untouchedOriginRoot, artifact.KindSegments))) + }) + } +} diff --git a/cmd/agentsview/sync_test.go b/cmd/agentsview/sync_test.go index d9535695a..1b65e6f4b 100644 --- a/cmd/agentsview/sync_test.go +++ b/cmd/agentsview/sync_test.go @@ -15,6 +15,7 @@ import ( "path/filepath" "strconv" "strings" + "sync/atomic" "testing" "time" @@ -527,12 +528,55 @@ func TestDoSyncUsesDaemonRouteWhenWritableDaemonRunning(t *testing.T) { registerSyncRouteTestRuntime(t, env.DataDir, ts.URL) - hadFailures := doSync(SyncConfig{}) + hadFailures, err := doSync(SyncConfig{}) + require.NoError(t, err) require.False(t, hadFailures) assert.True(t, syncCalled) env.assertNoLocalDB(t) } +func TestDoSyncRejectsArtifactTargetWhenWritableDaemonRunning(t *testing.T) { + env := newSyncCLIEnv(t) + target := t.TempDir() + + var syncCalled bool + ts := syncRouteTestServer(t, func(w http.ResponseWriter, r *http.Request) { + syncCalled = true + writeDoneSSE(t, w, agentsync.SyncStats{Synced: 7}) + }) + + registerSyncRouteTestRuntime(t, env.DataDir, ts.URL) + + hadFailures, err := doSync(SyncConfig{ArtifactFolder: target}) + require.Error(t, err) + assert.False(t, hadFailures) + assert.False(t, syncCalled, "daemon sync must not consume an artifact target") + assert.Contains(t, err.Error(), "artifact sync") + assert.Contains(t, err.Error(), "writable daemon") + env.assertNoLocalDB(t) +} + +func TestSyncTransportArtifactTargetUsesDirectDBWithoutAutoStartingDaemon(t *testing.T) { + env := newSyncCLIEnv(t) + var autostartCalled bool + stubStartBackgroundServeForTransport(t, func( + context.Context, *config.Config, time.Duration, + ) (*DaemonRuntime, error) { + autostartCalled = true + return nil, errors.New("unexpected daemon autostart") + }) + + appCfg := config.Config{ + DataDir: env.DataDir, + } + tr, err := syncTransport(&appCfg, SyncConfig{ + ArtifactFolder: t.TempDir(), + }) + require.NoError(t, err) + assert.Equal(t, transportDirect, tr.Mode) + assert.False(t, autostartCalled) +} + func TestDoSyncFullUsesDaemonResyncRoute(t *testing.T) { env := newSyncCLIEnv(t) @@ -554,7 +598,9 @@ func TestDoSyncFullUsesDaemonResyncRoute(t *testing.T) { var hadFailures bool out := captureStdout(t, func() { - hadFailures = doSync(SyncConfig{Full: true}) + var err error + hadFailures, err = doSync(SyncConfig{Full: true}) + require.NoError(t, err) }) require.False(t, hadFailures) assert.True(t, resyncCalled) @@ -590,13 +636,14 @@ func TestDoSyncRemoteHostUsesDaemonRouteWhenWritableDaemonRunning(t *testing.T) ts := remoteSyncRouteTestServer(t, handler) registerSyncRouteTestRuntime(t, env.DataDir, ts.URL) - hadFailures := doSync(SyncConfig{ + hadFailures, err := doSync(SyncConfig{ Host: "devbox", User: "alice", Port: 2222, Full: true, }) + require.NoError(t, err) require.False(t, hadFailures) assert.False(t, got.IncludeLocal) assert.True(t, got.Full) @@ -627,10 +674,12 @@ func TestDoSyncRemoteHostPrintsDaemonProgress(t *testing.T) { registerSyncRouteTestRuntime(t, env.DataDir, ts.URL) var hadFailures bool + var err error out := captureStdout(t, func() { - hadFailures = doSync(SyncConfig{Host: "devbox"}) + hadFailures, err = doSync(SyncConfig{Host: "devbox"}) }) + require.NoError(t, err) require.False(t, hadFailures) assert.Contains(t, out, "Running sync with remotes via daemon...") assert.Contains(t, out, "Resolving agent directories on devbox") @@ -709,8 +758,9 @@ user = "robot" ts := remoteSyncRouteTestServer(t, handler) registerSyncRouteTestRuntime(t, env.DataDir, ts.URL) - hadFailures := doSync(SyncConfig{}) + hadFailures, err := doSync(SyncConfig{}) + require.NoError(t, err) require.False(t, hadFailures) assert.True(t, got.IncludeLocal) require.Len(t, got.Hosts, 1) @@ -931,7 +981,6 @@ func TestRemoteFailureDisplaySanitizesHTTPErrors(t *testing.T) { }) } } - func TestRunHTTPRemoteSyncReachesMirrorPath(t *testing.T) { manifestRequests := 0 ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -966,3 +1015,232 @@ func TestRunHTTPRemoteSyncReachesMirrorPath(t *testing.T) { assert.Equal(t, 1, manifestRequests, "configured DataDir must route HTTP sync through the manifest/mirror path") } + +func TestNewSyncCommandRegistersArtifactFolderFlag(t *testing.T) { + cmd := newSyncCommand() + flag := cmd.Flags().Lookup("artifact-folder") + require.NotNil(t, flag) + assert.Equal(t, "", flag.DefValue) + assert.Contains(t, flag.Usage, "local-first sync artifacts") + initFlag := cmd.Flags().Lookup("init") + require.NotNil(t, initFlag) + assert.Equal(t, "false", initFlag.DefValue) + assert.Contains(t, initFlag.Usage, "Initialize artifact sync") + assert.Contains(t, cmd.Long, "do not point this at the") + assert.Contains(t, cmd.Long, "Use --init with an artifact folder") + watchFlag := cmd.Flags().Lookup("watch") + require.NotNil(t, watchFlag) + assert.Equal(t, "false", watchFlag.DefValue) + assert.Contains(t, watchFlag.Usage, "Run artifact folder sync continuously") + assert.Equal(t, defaultWatchDebounce.String(), cmd.Flags().Lookup("debounce").DefValue) + assert.Equal(t, defaultWatchInterval.String(), cmd.Flags().Lookup("interval").DefValue) + assert.Contains(t, cmd.Long, "Use --watch with an artifact folder") +} + +func TestApplySyncArtifactTargetUsesPositionalArgument(t *testing.T) { + cfg := SyncConfig{} + err := applySyncArtifactTarget(&cfg, []string{"/tmp/agentsview-share"}, false) + require.NoError(t, err) + assert.Equal(t, "/tmp/agentsview-share", cfg.ArtifactFolder) +} + +func TestApplySyncArtifactTargetRejectsArgumentAndFlag(t *testing.T) { + cfg := SyncConfig{ArtifactFolder: "/tmp/from-flag"} + err := applySyncArtifactTarget(&cfg, []string{"/tmp/from-arg"}, true) + require.Error(t, err) + assert.Contains(t, err.Error(), "both as an argument") + assert.Equal(t, "/tmp/from-flag", cfg.ArtifactFolder) +} + +func TestValidateSyncConfigInitRequiresArtifactFolder(t *testing.T) { + err := validateSyncConfig(SyncConfig{Init: true}) + require.Error(t, err) + assert.Contains(t, err.Error(), "--init requires") +} + +func TestValidateSyncConfigInitRejectsHost(t *testing.T) { + err := validateSyncConfig(SyncConfig{ + Init: true, + Host: "remote", + ArtifactFolder: "/tmp/agentsview-share", + }) + require.Error(t, err) + assert.Contains(t, err.Error(), "cannot be combined with --host") +} + +func TestValidateSyncConfigInitAllowsArtifactFolder(t *testing.T) { + err := validateSyncConfig(SyncConfig{ + Init: true, + ArtifactFolder: "/tmp/agentsview-share", + }) + require.NoError(t, err) +} + +func TestValidateSyncConfigWatchRequiresArtifactFolder(t *testing.T) { + err := validateSyncConfig(SyncConfig{Watch: true}) + require.Error(t, err) + assert.Contains(t, err.Error(), "--watch requires") +} + +func TestValidateSyncConfigWatchRejectsHost(t *testing.T) { + err := validateSyncConfig(SyncConfig{ + Watch: true, + Host: "remote", + ArtifactFolder: "/tmp/agentsview-share", + }) + require.Error(t, err) + assert.Contains(t, err.Error(), "cannot be combined with --host") +} + +func TestValidateSyncConfigWatchAllowsArtifactFolder(t *testing.T) { + err := validateSyncConfig(SyncConfig{ + Watch: true, + ArtifactFolder: "/tmp/agentsview-share", + Debounce: time.Second, + Interval: time.Minute, + }) + require.NoError(t, err) +} + +func TestValidateSyncConfigRejectsHostWithArtifactTarget(t *testing.T) { + err := validateSyncConfig(SyncConfig{ + Host: "remote", + ArtifactFolder: "/tmp/agentsview-share", + }) + require.Error(t, err) + assert.Contains(t, err.Error(), "--host cannot be combined with an artifact target") +} + +func TestValidateSyncConfigAllowsHostWithoutArtifactTarget(t *testing.T) { + require.NoError(t, validateSyncConfig(SyncConfig{Host: "remote"})) +} + +func TestArtifactPeerTokenDoesNotReuseLocalAuthToken(t *testing.T) { + got := artifactPeerToken(SyncConfig{ + ArtifactFolder: "https://peer.example.test", + }) + assert.Empty(t, got) + + got = artifactPeerToken(SyncConfig{ + ArtifactFolder: "https://peer.example.test", + Token: "peer-secret", + }) + assert.Equal(t, "peer-secret", got) +} + +func TestSyncArtifactFolderPlumbsInsecurePeerOptIn(t *testing.T) { + target, requests := insecureArtifactPeerTarget(t) + dataDir := t.TempDir() + database, err := db.Open(filepath.Join(dataDir, "sessions.db")) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, database.Close()) }) + + _, err = syncArtifactFolder( + context.Background(), config.Config{DataDir: dataDir}, database, + target, "desk-a1b2c3", "", true, false, nil, + ) + + require.NoError(t, err) + assert.Positive(t, requests.Load(), + "one-shot sync must reach an explicitly allowed plaintext peer") +} + +func insecureArtifactPeerTarget(t *testing.T) (string, *atomic.Int32) { + t.Helper() + var host net.IP + addrs, err := net.InterfaceAddrs() + require.NoError(t, err) + for _, addr := range addrs { + ip, _, parseErr := net.ParseCIDR(addr.String()) + if parseErr == nil && ip.To4() != nil && !ip.IsLoopback() && !ip.IsUnspecified() { + host = ip + break + } + } + if host == nil { + t.Skip("no non-loopback IPv4 interface available for plaintext peer plumbing test") + } + listener, err := net.Listen("tcp4", "0.0.0.0:0") + require.NoError(t, err) + var requests atomic.Int32 + peer := &httptest.Server{ + Listener: listener, + Config: &http.Server{Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests.Add(1) + w.Header().Set("Content-Type", "application/json") + switch { + case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/origins"): + _, _ = io.WriteString(w, `{"origins":[]}`) + case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/index"): + _, _ = io.WriteString(w, `{"origin":"desk-a1b2c3"}`) + case r.Method == http.MethodPost: + w.WriteHeader(http.StatusCreated) + default: + http.NotFound(w, r) + } + })}, + } + peer.Start() + t.Cleanup(peer.Close) + t.Setenv("HTTP_PROXY", "") + t.Setenv("HTTPS_PROXY", "") + t.Setenv("NO_PROXY", "*") + port := listener.Addr().(*net.TCPAddr).Port + return "http://" + net.JoinHostPort(host.String(), strconv.Itoa(port)), &requests +} + +func TestSyncGCHelpDocumentsMaintenanceFlags(t *testing.T) { + help, err := executeCommand(newRootCommand(), "sync", "gc", "--help") + require.NoError(t, err) + for _, want := range []string{"Garbage collect", "--grace", "--dry-run"} { + assert.Contains(t, help, want) + } +} + +func TestSyncGCRejectsNonFolderTarget(t *testing.T) { + _, err := executeCommand(newRootCommand(), "sync", "gc", "https://example.test/artifacts") + require.Error(t, err) + assert.Contains(t, err.Error(), "local folder targets only") +} + +func TestSyncGCDryRunPrintsSummary(t *testing.T) { + out, err := executeCommand(newRootCommand(), "sync", "gc", "--dry-run", t.TempDir()) + require.NoError(t, err) + assert.Contains(t, out, "Artifact GC:") + assert.Contains(t, out, "would delete 0 artifact(s)") +} + +func TestArtifactFolderPusherPushExportsToTarget(t *testing.T) { + dataDir := t.TempDir() + target := t.TempDir() + database, err := db.Open(filepath.Join(dataDir, "sessions.db")) + require.NoError(t, err) + t.Cleanup(func() { database.Close() }) + + dbtest.SeedSession(t, database, "sess-1", "alpha", func(s *db.Session) { + s.MessageCount = 1 + s.UserMessageCount = 1 + }) + require.NoError(t, database.ReplaceSessionMessages("sess-1", []db.Message{ + {SessionID: "sess-1", Ordinal: 0, Role: "user", Content: "hello", ContentLength: 5}, + })) + + pusher := &artifactFolderPusher{ + appCfg: config.Config{DataDir: dataDir}, + database: database, + target: target, + origin: "desk-a1b2c3", + } + require.NoError(t, pusher.push(context.Background(), reasonChange)) + + checkpoints, err := filepath.Glob( + filepath.Join(target, "desk-a1b2c3", "checkpoints", "*.json"), + ) + require.NoError(t, err) + require.Len(t, checkpoints, 1) + manifests, err := filepath.Glob( + filepath.Join(target, "desk-a1b2c3", "manifests", "*.json.zst"), + ) + require.NoError(t, err) + assert.Len(t, manifests, 1) +} diff --git a/cmd/agentsview/sync_watch.go b/cmd/agentsview/sync_watch.go new file mode 100644 index 000000000..fd085bd89 --- /dev/null +++ b/cmd/agentsview/sync_watch.go @@ -0,0 +1,198 @@ +package main + +import ( + "context" + "fmt" + "log" + "os" + "os/signal" + "syscall" + "time" + + "github.com/gofrs/flock" + "go.kenn.io/agentsview/internal/config" + "go.kenn.io/agentsview/internal/db" + "go.kenn.io/agentsview/internal/parser" + syncpkg "go.kenn.io/agentsview/internal/sync" + "go.kenn.io/kit/daemon" +) + +type artifactFolderPusher struct { + appCfg config.Config + database *db.DB + engine *syncpkg.Engine + target string + origin string + token string + allowInsecure bool + gcGrace time.Duration + gcEnabled bool + // baseline publishes first-run curation metadata (--init) on the next + // push. It stays set until a push succeeds so a failed initial exchange + // retries the baseline; AppendBaselineSnapshot skips already-covered + // fields, making the retry idempotent. Pushes run on a single loop + // goroutine, so no locking is needed. + baseline bool + onDataChanged func() +} + +func newArtifactWatchEngine( + database *db.DB, appCfg config.Config, +) *syncpkg.Engine { + return syncpkg.NewEngine(database, syncpkg.EngineConfig{ + AgentDirs: appCfg.AgentDirs, + IncludeCwdPrefixes: appCfg.SyncIncludeCwdPrefixes, + Machine: "local", + BlockedResultCategories: appCfg.ResultContentBlockedCategories, + }) +} + +func (p *artifactFolderPusher) push( + ctx context.Context, reason pushReason, +) error { + if p.engine != nil { + p.engine.SyncAll(ctx, nil) + // Export reads session rows outside a sync operation; flush + // debounced signal recomputes so manifests carry current signals. + p.engine.FlushSignals() + } + res, err := syncArtifactFolder( + ctx, p.appCfg, p.database, p.target, p.origin, p.token, + p.allowInsecure, p.baseline, p.onDataChanged, + ) + if err != nil { + return err + } + p.baseline = false + log.Printf( + "artifact watch: exported %d sessions, imported %d sessions, %d messages, %d metadata events (%s)", + res.ExportedSessions, res.ImportedSessions, res.ImportedMessages, + res.ImportedMetadata, reason, + ) + // Collect superseded artifacts on the periodic floor and at startup, not on + // every debounced change, to keep frequent edits cheap. + if p.gcEnabled && (reason == reasonStartup || reason == reasonInterval) { + autoGCAfterFolderSync(ctx, p.appCfg.DataDir, p.target, p.gcGrace) + } + return nil +} + +// runSyncWatch runs continuous artifact folder sync: an initial local sync and +// artifact exchange, then debounced file-change exchanges and a periodic floor. +func runSyncWatch(cfg SyncConfig) { + appCfg, err := config.LoadMinimal() + if err != nil { + log.Fatalf("loading config: %v", err) + } + if err := os.MkdirAll(appCfg.DataDir, 0o755); err != nil { + log.Fatalf("creating data dir: %v", err) + } + setupLogFileNamed(appCfg.DataDir, "artifact-watch.log") + + if cfg.ArtifactFolder == "" { + fatal("artifact watch: folder target is required") + } + + debounce := cfg.Debounce + if debounce <= 0 { + debounce = defaultWatchDebounce + } + interval := cfg.Interval + if interval <= 0 { + interval = defaultWatchInterval + } + + lockPath, err := (daemon.RuntimeStore{ + Dir: appCfg.DataDir, + Prefix: "artifact-watch", + }).LockPath() + if err != nil { + fatal("artifact watch: %v", err) + } + lock := flock.New(lockPath) + locked, err := lock.TryLock() + if err != nil { + fatal("artifact watch: locking %s: %v", lockPath, err) + } + if !locked { + fatal("artifact watch: already locked (%s)", lockPath) + } + defer func() { + if rerr := lock.Unlock(); rerr != nil { + log.Printf("artifact watch: releasing lock: %v", rerr) + } + }() + + applyClassifierConfig(appCfg) + database, writeLock := mustOpenWriteDB(context.Background(), appCfg) + defer closeWriteDB(database, writeLock) + + for _, def := range parser.Registry { + if !appCfg.IsUserConfigured(def.Type) { + continue + } + warnMissingDirs(appCfg.ResolveDirs(def.Type), string(def.Type)) + } + cleanResyncTemp(appCfg.DBPath) + + origin, err := resolveArtifactOrigin(appCfg, database) + if err != nil { + fatal("artifact watch origin: %v", err) + } + + ctx, stop := signal.NotifyContext( + context.Background(), os.Interrupt, syscall.SIGTERM, + ) + defer stop() + + engine := newArtifactWatchEngine(database, appCfg) + defer engine.Close() + + didResync := cfg.Full || database.NeedsResync() + if didResync { + engine.ResyncAll(ctx, nil) + } else { + engine.SyncAll(ctx, nil) + } + if ctx.Err() != nil { + return + } + + pusher := &artifactFolderPusher{ + appCfg: appCfg, + database: database, + engine: engine, + target: cfg.ArtifactFolder, + origin: origin, + token: artifactPeerToken(cfg), + allowInsecure: cfg.AllowInsecure, + gcGrace: cfg.GCGrace, + gcEnabled: !cfg.NoGC, + baseline: cfg.Init, + } + + log.Printf( + "artifact watch: starting (origin=%q target=%q debounce=%s interval=%s)", + origin, cfg.ArtifactFolder, debounce, interval, + ) + fmt.Printf( + "agentsview sync --watch: syncing artifacts as %q to %s "+ + "(debounce %s, floor %s)\n", + origin, cfg.ArtifactFolder, debounce, interval, + ) + + if err := pusher.push(ctx, reasonStartup); err != nil { + log.Printf("artifact watch: initial sync failed: %v", err) + } + + runWatchedSink(ctx, watchedSinkConfig{ + AppConfig: appCfg, + Engine: engine, + Debounce: debounce, + Interval: interval, + LogPrefix: "artifact watch", + Push: func(c context.Context, r pushReason) error { + return pusher.push(c, r) + }, + }) +} diff --git a/cmd/agentsview/sync_watch_test.go b/cmd/agentsview/sync_watch_test.go new file mode 100644 index 000000000..ad7fc0c2b --- /dev/null +++ b/cmd/agentsview/sync_watch_test.go @@ -0,0 +1,192 @@ +package main + +import ( + "context" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.kenn.io/agentsview/internal/artifact" + "go.kenn.io/agentsview/internal/config" + "go.kenn.io/agentsview/internal/db" + "go.kenn.io/agentsview/internal/parser" + "go.kenn.io/agentsview/internal/testjsonl" +) + +func openWatchTestDB(t *testing.T) *db.DB { + t.Helper() + database, err := db.Open(filepath.Join(t.TempDir(), "test.db")) + require.NoError(t, err) + t.Cleanup(func() { database.Close() }) + return database +} + +func seedStarredSession(t *testing.T, database *db.DB, id string) { + t.Helper() + require.NoError(t, database.UpsertSession(db.Session{ + ID: id, + Project: "alpha", + Machine: "local", + Agent: "claude", + MessageCount: 1, + UserMessageCount: 1, + CreatedAt: "2026-06-14T01:02:03Z", + })) + require.NoError(t, database.ReplaceSessionMessages(id, []db.Message{ + {SessionID: id, Ordinal: 0, Role: "user", Content: "hello", ContentLength: 5}, + })) + starred, err := database.StarSession(id) + require.NoError(t, err, "StarSession") + require.True(t, starred, "session should be newly starred") +} + +func TestArtifactFolderPusherPublishesBaselineOnFirstSuccessfulPush(t *testing.T) { + dataDir := t.TempDir() + target := t.TempDir() + database := openWatchTestDB(t) + seedStarredSession(t, database, "sess-1") + + pusher := &artifactFolderPusher{ + appCfg: config.Config{DataDir: dataDir}, + database: database, + target: target, + origin: "laptop-a1b2c3", + baseline: true, + } + require.NoError(t, pusher.push(context.Background(), reasonStartup)) + assert.False(t, pusher.baseline, + "baseline must be published at most once after a successful push") + + events, err := filepath.Glob( + filepath.Join(target, "laptop-a1b2c3", "meta", "*"), + ) + require.NoError(t, err) + assert.NotEmpty(t, events, + "--init --watch must publish baseline metadata events to the target") +} + +func TestArtifactFolderPusherRetainsBaselineAfterFailedPush(t *testing.T) { + dataDir := t.TempDir() + // A regular file as the folder target makes the exchange fail. + target := filepath.Join(t.TempDir(), "not-a-dir") + require.NoError(t, os.WriteFile(target, []byte("x"), 0o600)) + database := openWatchTestDB(t) + seedStarredSession(t, database, "sess-1") + + pusher := &artifactFolderPusher{ + appCfg: config.Config{DataDir: dataDir}, + database: database, + target: target, + origin: "laptop-a1b2c3", + baseline: true, + } + require.Error(t, pusher.push(context.Background(), reasonStartup)) + assert.True(t, pusher.baseline, + "a failed push must keep the baseline pending for the next retry") +} + +func TestArtifactFolderPusherPlumbsInsecurePeerOptIn(t *testing.T) { + target, requests := insecureArtifactPeerTarget(t) + dataDir := t.TempDir() + database := openWatchTestDB(t) + pusher := &artifactFolderPusher{ + appCfg: config.Config{DataDir: dataDir}, + database: database, + target: target, + origin: "desk-a1b2c3", + allowInsecure: true, + } + + require.NoError(t, pusher.push(context.Background(), reasonStartup)) + assert.Positive(t, requests.Load(), + "watch sync must reach an explicitly allowed plaintext peer") +} + +func TestArtifactWatchEngineHonorsConfiguredCwdPrefixes(t *testing.T) { + claudeDir := t.TempDir() + projectDir := filepath.Join(claudeDir, "-Users-alice-work") + require.NoError(t, os.MkdirAll(projectDir, 0o755)) + writeSession := func(name, cwd, prompt string) { + content := testjsonl.NewSessionBuilder(). + AddClaudeUser("2026-01-01T00:00:00Z", prompt, cwd). + AddClaudeAssistant("2026-01-01T00:00:01Z", "ok"). + String() + require.NoError(t, os.WriteFile( + filepath.Join(projectDir, name+".jsonl"), []byte(content), 0o644, + )) + } + writeSession("allowed-session", "/Users/alice/work/project", "allowed") + writeSession("blocked-session", "/Users/alice/personal/project", "blocked") + + database := openWatchTestDB(t) + engine := newArtifactWatchEngine(database, config.Config{ + AgentDirs: map[parser.AgentType][]string{ + parser.AgentClaude: {claudeDir}, + }, + SyncIncludeCwdPrefixes: []string{"/Users/alice/work"}, + }) + t.Cleanup(engine.Close) + stats := engine.SyncAll(context.Background(), nil) + require.Equal(t, 1, stats.Synced) + + allowed, err := database.GetSession(context.Background(), "allowed-session") + require.NoError(t, err) + require.NotNil(t, allowed) + assert.Equal(t, "allowed", *allowed.FirstMessage) + blocked, err := database.GetSession(context.Background(), "blocked-session") + require.NoError(t, err) + assert.Nil(t, blocked, + "watch mode must not ingest sessions outside sync_include_cwd_prefixes") +} + +func TestResolveArtifactOriginPromotesDBOriginIntoConfig(t *testing.T) { + dir := t.TempDir() + appCfg := config.Config{DataDir: dir} + database := openWatchTestDB(t) + require.NoError(t, artifact.AdoptOrigin(database, "laptop-a1b2c3"), + "seed DB-only origin") + + origin, err := resolveArtifactOrigin(appCfg, database) + require.NoError(t, err) + assert.Equal(t, "laptop-a1b2c3", origin, + "a DB-only origin must be reused, not replaced by a generated one") + + data, err := os.ReadFile(filepath.Join(dir, "config.toml")) + require.NoError(t, err) + assert.Contains(t, string(data), `artifact_origin_id = "laptop-a1b2c3"`, + "the DB origin must be promoted into config so serve adopts the same origin") +} + +func TestResolveArtifactOriginConfigWinsOverDB(t *testing.T) { + dir := t.TempDir() + appCfg := config.Config{DataDir: dir, ArtifactOriginID: "desktop-d4e5f6"} + database := openWatchTestDB(t) + require.NoError(t, artifact.AdoptOrigin(database, "laptop-a1b2c3"), + "seed DB origin") + + origin, err := resolveArtifactOrigin(appCfg, database) + require.NoError(t, err) + assert.Equal(t, "desktop-d4e5f6", origin) + + stored, err := artifact.StoredOrigin(database) + require.NoError(t, err) + assert.Equal(t, "desktop-d4e5f6", stored, + "the authoritative config origin must replace a divergent DB origin") +} + +func TestResolveArtifactOriginGeneratesWhenAbsentEverywhere(t *testing.T) { + dir := t.TempDir() + appCfg := config.Config{DataDir: dir} + database := openWatchTestDB(t) + + origin, err := resolveArtifactOrigin(appCfg, database) + require.NoError(t, err) + assert.Regexp(t, `^[a-z0-9]+(?:-[a-z0-9]+)*-[0-9a-f]{6}$`, origin) + + stored, err := artifact.StoredOrigin(database) + require.NoError(t, err) + assert.Equal(t, origin, stored, + "a directly generated CLI origin must be available to DB-only consumers") +} diff --git a/docs/artifact-sync.md b/docs/artifact-sync.md new file mode 100644 index 000000000..25de34197 --- /dev/null +++ b/docs/artifact-sync.md @@ -0,0 +1,248 @@ +--- +title: Trusted-Fleet Artifact Sync +description: Sync AgentsView archives between trusted personal machines without copying the live SQLite database +--- + +# Trusted-Fleet Artifact Sync + +Artifact sync exchanges immutable AgentsView artifacts between machines and +imports them into each machine's local SQLite archive. It is local-first: every +machine keeps its own complete database, and transports only move +content-addressed session artifacts plus metadata events. + +Use it for a fully trusted personal fleet: your laptop, desktop, home server, +NAS, or object-store bucket. Do not treat artifact sync as a team-sharing +security boundary. A peer that can write to the shared artifact target can +publish sessions and metadata for the fleet. + +## When To Use It + +Artifact sync makes sense when you want multiple machines to converge on the +same session archive without running PostgreSQL as the coordination point. + +It is a good fit for: + +- laptop plus desktop archives +- NAS, Syncthing, Dropbox, or rclone-backed rendezvous folders +- S3-compatible buckets such as MinIO, Backblaze B2, or AWS S3 +- trusted always-on AgentsView peers over HTTP + +Use [PostgreSQL Sync](/pg-sync/) or [DuckDB Mirror](/duckdb/) when you want a +read-only aggregation or analytics mirror. Those backends are still mirrors: +SQLite remains the local write/archive database, and artifact sync projects +foreign artifacts into ordinary SQLite rows before they can be pushed onward. + +## Quick Start + +Use a dedicated artifact share folder: + +```bash +agentsview sync --init /path/to/agentsview-artifacts +agentsview sync /path/to/agentsview-artifacts +``` + +Run `--init` once on each machine. It creates or adopts that machine's artifact +origin, backfills local sessions into the local artifact store, exchanges +artifacts with the target, and imports peer artifacts already present there. + +CLI artifact sync needs exclusive write access to the local archive. If a local +`agentsview` daemon is running, stop it first with `agentsview serve stop` and +retry. A running server still participates in a fleet as an +[HTTP peer](#http-peer) target for other machines. + +To keep a machine exchanging artifacts while it is online, run watch mode: + +```bash +agentsview sync --watch /path/to/agentsview-artifacts +``` + +Watch mode runs an initial local sync and artifact exchange, debounces local +session-file changes, retries failed exchanges on later changes or interval +ticks, and performs a final best-effort exchange on shutdown. + +## Targets + +### Folder + +```bash +agentsview sync /path/to/agentsview-artifacts +``` + +The folder may live on a local disk, NAS mount, Syncthing folder, Dropbox +folder, NFS share, or rclone-mounted bucket. The folder must be dedicated to +artifact sync. Do not point artifact sync at: + +- `AGENTSVIEW_DATA_DIR` +- the live SQLite database file or its WAL/SHM files +- a whole AgentsView data directory +- raw agent directories that contain live database files + +Copying the live SQLite database or the whole data directory with a general +file-sync tool is unsafe. Artifact sync exists specifically to avoid that. + +### HTTP Peer + +An AgentsView server can expose artifact exchange routes behind the existing +Bearer-token auth middleware: + +```bash +agentsview sync https://desktop:8080 --token +``` + +HTTP peer sync only sends an `Authorization` header when `--token` is provided. +It does not reuse the local server's `auth_token` for explicit peer URLs. HTTP +peer sync rejects redirects, so configure the final artifact API URL directly. +Credentials and artifact bodies are not forwarded to a redirect destination. + +Non-loopback peers require HTTPS by default. Plain `http://` remains available +for `localhost`, `127.0.0.0/8`, and `::1`. To connect to a remote plaintext peer +on a trusted test network, LAN, or VPN, opt in explicitly: + +```bash +agentsview sync http://desktop:8080 --token --allow-insecure +``` + +The override works for one-shot and `--watch` sync and logs a warning. It sends +the bearer token and full archive content without transport encryption, so use +HTTPS through a reverse proxy or VPN termination whenever possible. If you +expose a server beyond loopback, enable authentication and protect the token +like write access to the full archive. + +The HTTP client pulls every missing artifact from the peer and posts every local +artifact the peer is missing. Garbage collection of superseded artifacts on the +remote peer is the peer's own responsibility. + +### S3-Compatible Object Storage + +```bash +export AWS_ACCESS_KEY_ID=... +export AWS_SECRET_ACCESS_KEY=... +agentsview sync s3://my-bucket/agentsview +``` + +Credentials come from `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, and optional +`AWS_SESSION_TOKEN`. Region resolves from `AGENTSVIEW_S3_REGION`, then +`AWS_REGION`, defaulting to `us-east-1`. + +For MinIO, Backblaze B2, or another S3-compatible service, set +`AGENTSVIEW_S3_ENDPOINT`. A custom endpoint automatically uses path-style +addressing; `AGENTSVIEW_S3_PATH_STYLE=true` forces it otherwise. + +Custom endpoints default to HTTPS when no scheme is given. Plain `http://` is +accepted only for loopback hosts or when +`AGENTSVIEW_ALLOW_INSECURE_S3_ENDPOINT=true` is explicitly set. Use that +override only on a trusted test or private network because requests and artifact +content travel without TLS. Artifact sync rejects redirects. If a corrupt remote +object must be deleted so a later upload can heal the bucket, that deletion is +HTTPS-only, even when the insecure override permits other requests over HTTP. + +## How It Works + +Each install has a stable artifact origin ID. Locally owned sessions keep their +ordinary SQLite IDs and `machine='local'`. Foreign sessions are imported as +`~` with `machine=`. Source, parent, and +subagent relationship IDs are rewritten the same way before import, so SQLite, +PostgreSQL, and DuckDB see the same ordinary session graph after projection. + +The artifact store lives beside, not inside, the database: + +```text +$AGENTSVIEW_DATA_DIR/artifacts// + checkpoints/cp-.json + manifests/.json.zst + segments/.ndjson.zst + meta/-.json + raw/ +``` + +Artifact kinds: + +- **checkpoints** list the current manifest hash for each session published by + an origin +- **manifests** hold the canonical session header, usage events, and segment + references +- **segments** hold canonical message NDJSON +- **metadata events** record user edits such as rename, trash/restore, star, + pin/unpin, and delete-everywhere +- **raw artifacts** are optional source snapshots when a parser can provide a + safe regular-file snapshot + +All artifact files are immutable. Folder writes use no-replace semantics, and S3 +writes use conditional create, so repeated syncs are idempotent set-union +operations. + +## Metadata And Deletes + +A machine records metadata events only after it has an artifact origin, which it +gets by running `agentsview sync --init`, running any artifact sync, or +receiving a peer exchange. Until then curation stays local and writes nothing +under `artifacts/`; the `--init` baseline snapshot publishes the accumulated +curation state when the machine later joins a fleet. + +User curation converges through metadata events. Rename, trash/restore, star, +and pin changes are replayed deterministically with hybrid logical clocks. If +two peers edit the same metadata field close together, AgentsView records the +losing value in the local conflict log while still deriving one deterministic +current value. Conflicts and per-origin sync status are visible on the Peers +page in the UI, and a conflicted session shows a fork badge in its header. + +Emptying local trash is local-only. Fleet-wide permanent delete is explicit: +delete-everywhere writes a purge event and an exclusion tombstone so peers do +not resurrect the session from older artifacts. + +Checkpoint absence is never a deletion signal. A missing artifact, truncated +checkpoint, offline peer, or old target cannot remove local data. + +## Version And Failure Handling + +Artifact readers ignore unknown JSON fields. Unknown future metadata operations +are marked applied and skipped. Artifacts with a future format version are +deferred, not treated as successful imports, so older AgentsView versions keep +syncing the artifact kinds and versions they understand. + +Manifests that reference missing segments are also deferred. Import watermarks +advance only after all referenced content is hash-verified and applied. + +A corrupt artifact — a hash-mismatched segment or manifest, an undecodable +compressed file, or an unparseable checkpoint or metadata event — is quarantined +under a `.corrupt` suffix and skipped, so one damaged file never aborts a sync +or spreads to other machines. Read and export paths fall back to the previous +valid checkpoint, folder exchanges replace a quarantined copy when the share +still holds a valid one, and metadata events whose timestamps do not parse are +rejected at write time and skipped at replay. An object corrupt in place inside +an S3 bucket is eligible for remote deletion when a peer fetches it because it +is missing locally and validation fails. AgentsView attempts that deletion only +over HTTPS; if it succeeds, a later push from a valid holder can re-upload the +object. A corrupt remote checkpoint found while comparing a checkpoint already +present locally is retained for its owner to repair. Over permitted HTTP, +corrupt remote objects are retained and skipped; automatic deletion does not +occur. + +## Garbage Collection + +Folder syncs collect superseded artifacts automatically after each exchange, in +both the local artifact store and the folder target, once artifacts are older +than a seven-day grace window. Tune the window with `--gc-grace` or disable +automatic collection with `--no-gc`. + +The `agentsview sync gc` command runs the same conservative collection manually, +with `--dry-run` to preview: + +```bash +agentsview sync gc --dry-run /path/to/agentsview-artifacts +agentsview sync gc --grace 168h /path/to/agentsview-artifacts +``` + +GC keeps the latest checkpoint for each origin and every manifest, segment, and +raw artifact reachable from it. Origins without checkpoints are skipped rather +than interpreted as deleted. + +## Availability + +Two intermittent machines only sync directly while both are online and can reach +the same target. A NAS folder, always-on home server, S3-compatible bucket, +cloud-synced folder, or always-running AgentsView peer can act as a rendezvous. + +That rendezvous is a deployment convention, not a privileged architecture: +AgentsView still treats every participant as a peer and keeps the complete local +archive on each machine. diff --git a/docs/index.md b/docs/index.md index c9416c1f1..ca3a62f20 100644 --- a/docs/index.md +++ b/docs/index.md @@ -178,8 +178,10 @@ See [Activity](/activity/) for the full reference. AgentsView reads the session files that your [AI coding agents](/configuration/#session-discovery) leave on your machine and gives you a local-first desktop and web app to work with them. By default -everything stays on your machine. Optionally, [PostgreSQL sync](/pg-sync/) can -push session data to a shared database for team or multi-machine setups. +everything stays on your machine. Optionally, [artifact sync](/artifact-sync/) +can converge a trusted personal fleet without copying the live SQLite database, +and [PostgreSQL sync](/pg-sync/) can push session data to a shared database for +team or multi-machine dashboards.
diff --git a/docs/zensical.toml b/docs/zensical.toml index 3a8cf872e..5ed602cce 100644 --- a/docs/zensical.toml +++ b/docs/zensical.toml @@ -30,6 +30,7 @@ nav = [ {"Recall (Experimental)" = "recall.md"}, {"Configuration" = "configuration.md"}, {"Remote Access" = "remote-access.md"}, + {"Artifact Sync" = "artifact-sync.md"}, {"PostgreSQL Sync" = "pg-sync.md"}, {"DuckDB Mirror" = "duckdb.md"}, {"Changelog" = "changelog.md"}, diff --git a/frontend/messages/en.json b/frontend/messages/en.json index 8f4e8ad6f..de928b8f9 100644 --- a/frontend/messages/en.json +++ b/frontend/messages/en.json @@ -22,6 +22,7 @@ "nav_pinned": "Pinned", "nav_insights": "Insights", "nav_trash": "Trash", + "nav_peers": "Peers", "nav_search_sessions": "Search sessions...", "nav_search_sessions_shortcut": "Search sessions ({shortcut})", "header_transcript_normal_title": "Normal transcript - show all messages", @@ -187,6 +188,23 @@ "session_breadcrumb_find_in_session_shortcut": "Find in session (/)", "session_breadcrumb_rename": "Rename", "session_breadcrumb_delete": "Delete", + "session_breadcrumb_metadata_conflicts": "Metadata conflicts", + "session_breadcrumb_conflict_field_name": "Name", + "session_breadcrumb_conflict_field_trash": "Trash", + "session_breadcrumb_conflict_field_star": "Star", + "session_breadcrumb_conflict_field_delete_everywhere": "Delete everywhere", + "session_breadcrumb_conflict_field_pin": "Pin", + "session_breadcrumb_conflict_default_title": "Default title", + "session_breadcrumb_conflict_restored": "Restored", + "session_breadcrumb_conflict_in_trash": "In trash", + "session_breadcrumb_conflict_starred": "Starred", + "session_breadcrumb_conflict_unstarred": "Unstarred", + "session_breadcrumb_conflict_unpinned_target": "Unpinned {target}", + "session_breadcrumb_conflict_pinned_target": "Pinned {target}", + "session_breadcrumb_conflict_pinned_target_note": "Pinned {target}: {note}", + "session_breadcrumb_conflict_unknown_origin": "unknown origin", + "session_breadcrumb_conflict_current": "Current", + "session_breadcrumb_conflict_other": "Other", "session_breadcrumb_resumed_in": "Resumed in {target}", "session_breadcrumb_command_copied": "Command copied!", "session_breadcrumb_failed": "Failed", @@ -450,6 +468,57 @@ "sidebar_row_rename": "Rename", "sidebar_row_open_in_new_tab": "Open in new tab", "sidebar_row_delete": "Delete", + "peers_title": "Peers", + "peers_refresh_status": "Refresh peer status", + "peers_conflict_count_singular": "1 metadata conflict", + "peers_conflict_count_plural": "{count} metadata conflicts", + "peers_loading": "Loading peers...", + "peers_empty": "No peers yet", + "peers_unavailable": "Artifact sync is not available on this server.", + "peers_this_machine": "This machine", + "peers_in_sync": "In sync", + "peers_pending": "{count} pending", + "peers_local_sessions_title": "Sessions present in this database from this peer", + "peers_local_sessions": "{count} local", + "peers_published_sessions_title": "Sessions this peer has published", + "peers_published_sessions": "{count} published", + "peers_checkpoint_title": "Latest checkpoint sequence", + "peers_checkpoint": "checkpoint #{seq}", + "peers_last_published_title": "Last checkpoint published", + "peers_updated": "updated {time}", + "trash_title": "Trash", + "trash_empty_local_title": "Empty local trash", + "trash_emptying": "Emptying...", + "trash_empty_local": "Empty Local Trash", + "trash_loading": "Loading trash...", + "trash_empty": "Trash is empty", + "trash_empty_desc": "Deleted sessions will appear here.", + "trash_messages": [ + { + "declarations": [ + "input count", + "input countLabel", + "local countPlural = count: plural" + ], + "selectors": [ + "countPlural" + ], + "match": { + "countPlural=one": "{countLabel} msg", + "countPlural=other": "{countLabel} msgs" + } + } + ], + "trash_deleted": "deleted {time}", + "trash_restore_session": "Restore session", + "trash_restore": "Restore", + "trash_delete_everywhere_prompt": "Delete everywhere?", + "trash_confirm_delete_everywhere": "Confirm delete everywhere", + "trash_deleting": "Deleting...", + "trash_confirm": "Confirm", + "trash_cancel": "Cancel", + "trash_delete_everywhere": "Delete Everywhere", + "trash_delete_everywhere_title": "Delete everywhere", "shared_all_projects": "All Projects", "shared_project_filter_placeholder": "Filter projects...", "shared_select_project": "Select project", @@ -1621,33 +1690,6 @@ "system_boundary_stop_hook": "Stop hook feedback", "system_boundary_title": "System boundary: {subtype}", "system_boundary_show_content": "Show content", - "trash_loading": "Loading trash...", - "trash_empty": "Trash is empty", - "trash_empty_desc": "Deleted sessions will appear here.", - "trash_title": "Trash", - "trash_emptying": "Emptying...", - "trash_empty_trash": "Empty Trash", - "trash_msgs": [ - { - "declarations": [ - "input count", - "input countLabel", - "local countPlural = count: plural" - ], - "selectors": [ - "countPlural" - ], - "match": { - "countPlural=one": "{countLabel} msg", - "countPlural=other": "{countLabel} msgs" - } - } - ], - "trash_deleted_ago": "deleted {time}", - "trash_restore_session": "Restore session", - "trash_restore": "Restore", - "trash_permanently_delete": "Permanently delete", - "trash_delete_forever": "Delete Forever", "trends_term": "Term", "trends_per1k_messages": "Per 1k messages", "trends_count": "Count", diff --git a/frontend/messages/ko.json b/frontend/messages/ko.json index fbad5c3ab..f3fad279d 100644 --- a/frontend/messages/ko.json +++ b/frontend/messages/ko.json @@ -22,6 +22,7 @@ "nav_pinned": "고정됨", "nav_insights": "인사이트", "nav_trash": "휴지통", + "nav_peers": "피어", "nav_search_sessions": "세션 검색...", "nav_search_sessions_shortcut": "세션 검색 ({shortcut})", "header_transcript_normal_title": "일반 트랜스크립트 - 모든 메시지 표시", @@ -170,7 +171,9 @@ "input countLabel", "local countPlural = count: plural" ], - "selectors": ["countPlural"], + "selectors": [ + "countPlural" + ], "match": { "countPlural=other": "{countLabel}개 단계" } @@ -183,6 +186,23 @@ "session_breadcrumb_find_in_session_shortcut": "세션에서 찾기 (/)", "session_breadcrumb_rename": "이름 변경", "session_breadcrumb_delete": "삭제", + "session_breadcrumb_metadata_conflicts": "메타데이터 충돌", + "session_breadcrumb_conflict_field_name": "이름", + "session_breadcrumb_conflict_field_trash": "휴지통", + "session_breadcrumb_conflict_field_star": "고정", + "session_breadcrumb_conflict_field_delete_everywhere": "모든 기기에서 삭제", + "session_breadcrumb_conflict_field_pin": "메시지 고정", + "session_breadcrumb_conflict_default_title": "기본 제목", + "session_breadcrumb_conflict_restored": "복원됨", + "session_breadcrumb_conflict_in_trash": "휴지통에 있음", + "session_breadcrumb_conflict_starred": "고정됨", + "session_breadcrumb_conflict_unstarred": "고정 해제됨", + "session_breadcrumb_conflict_unpinned_target": "{target} 고정 해제됨", + "session_breadcrumb_conflict_pinned_target": "{target} 고정됨", + "session_breadcrumb_conflict_pinned_target_note": "{target} 고정됨: {note}", + "session_breadcrumb_conflict_unknown_origin": "알 수 없는 출처", + "session_breadcrumb_conflict_current": "현재", + "session_breadcrumb_conflict_other": "기타", "session_breadcrumb_resumed_in": "{target}에서 재개됨", "session_breadcrumb_command_copied": "명령어가 복사되었습니다!", "session_breadcrumb_failed": "실패", @@ -437,6 +457,56 @@ "sidebar_row_rename": "이름 변경", "sidebar_row_open_in_new_tab": "새 탭에서 열기", "sidebar_row_delete": "삭제", + "peers_title": "피어", + "peers_refresh_status": "피어 상태 새로 고침", + "peers_conflict_count_singular": "메타데이터 충돌 1건", + "peers_conflict_count_plural": "메타데이터 충돌 {count}건", + "peers_loading": "피어를 불러오는 중...", + "peers_empty": "피어가 없습니다", + "peers_unavailable": "이 서버에서는 아티팩트 동기화를 사용할 수 없습니다.", + "peers_this_machine": "이 컴퓨터", + "peers_in_sync": "동기화됨", + "peers_pending": "{count}개 대기 중", + "peers_local_sessions_title": "이 피어의 세션 중 이 데이터베이스에 있는 세션", + "peers_local_sessions": "로컬 {count}개", + "peers_published_sessions_title": "이 피어가 게시한 세션", + "peers_published_sessions": "게시됨 {count}개", + "peers_checkpoint_title": "최신 체크포인트 순번", + "peers_checkpoint": "체크포인트 #{seq}", + "peers_last_published_title": "마지막 체크포인트 게시 시각", + "peers_updated": "{time} 업데이트됨", + "trash_title": "휴지통", + "trash_empty_local_title": "로컬 휴지통 비우기", + "trash_emptying": "비우는 중...", + "trash_empty_local": "로컬 휴지통 비우기", + "trash_loading": "휴지통을 불러오는 중...", + "trash_empty": "휴지통이 비어 있습니다", + "trash_empty_desc": "삭제된 세션이 여기에 표시됩니다.", + "trash_messages": [ + { + "declarations": [ + "input count", + "input countLabel", + "local countPlural = count: plural" + ], + "selectors": [ + "countPlural" + ], + "match": { + "countPlural=other": "메시지 {countLabel}개" + } + } + ], + "trash_deleted": "{time} 삭제됨", + "trash_restore_session": "세션 복원", + "trash_restore": "복원", + "trash_delete_everywhere_prompt": "모든 기기에서 삭제할까요?", + "trash_confirm_delete_everywhere": "모든 기기에서 삭제 확인", + "trash_deleting": "삭제 중...", + "trash_confirm": "확인", + "trash_cancel": "취소", + "trash_delete_everywhere": "모든 기기에서 삭제", + "trash_delete_everywhere_title": "모든 기기에서 삭제", "shared_all_projects": "모든 프로젝트", "shared_project_filter_placeholder": "프로젝트 필터링...", "shared_select_project": "프로젝트 선택", @@ -576,7 +646,9 @@ "input countLabel", "local countPlural = count: plural" ], - "selectors": ["countPlural"], + "selectors": [ + "countPlural" + ], "match": { "countPlural=other": "세션 {countLabel}개" } @@ -1586,32 +1658,6 @@ "system_boundary_stop_hook": "중지 훅 피드백", "system_boundary_title": "시스템 경계: {subtype}", "system_boundary_show_content": "내용 표시", - "trash_loading": "휴지통을 불러오는 중...", - "trash_empty": "휴지통이 비어 있습니다", - "trash_empty_desc": "삭제된 세션이 여기에 표시됩니다.", - "trash_title": "휴지통", - "trash_emptying": "비우는 중...", - "trash_empty_trash": "휴지통 비우기", - "trash_msgs": [ - { - "declarations": [ - "input count", - "input countLabel", - "local countPlural = count: plural" - ], - "selectors": [ - "countPlural" - ], - "match": { - "countPlural=other": "메시지 {countLabel}개" - } - } - ], - "trash_deleted_ago": "{time} 삭제됨", - "trash_restore_session": "세션 복원", - "trash_restore": "복원", - "trash_permanently_delete": "영구 삭제", - "trash_delete_forever": "영구 삭제", "trends_term": "용어", "trends_per1k_messages": "메시지 1,000건당", "trends_count": "개수", diff --git a/frontend/messages/zh-CN.json b/frontend/messages/zh-CN.json index 6f0d623e3..9e778c49f 100644 --- a/frontend/messages/zh-CN.json +++ b/frontend/messages/zh-CN.json @@ -22,6 +22,7 @@ "nav_pinned": "已固定", "nav_insights": "洞察", "nav_trash": "回收站", + "nav_peers": "Peers", "nav_search_sessions": "搜索会话...", "nav_search_sessions_shortcut": "搜索会话 ({shortcut})", "header_transcript_normal_title": "普通 transcript - 显示所有消息", @@ -183,6 +184,23 @@ "session_breadcrumb_find_in_session_shortcut": "在会话中查找 (/)", "session_breadcrumb_rename": "重命名", "session_breadcrumb_delete": "删除", + "session_breadcrumb_metadata_conflicts": "元数据冲突", + "session_breadcrumb_conflict_field_name": "名称", + "session_breadcrumb_conflict_field_trash": "回收站", + "session_breadcrumb_conflict_field_star": "固定", + "session_breadcrumb_conflict_field_delete_everywhere": "全局删除", + "session_breadcrumb_conflict_field_pin": "固定", + "session_breadcrumb_conflict_default_title": "默认标题", + "session_breadcrumb_conflict_restored": "已恢复", + "session_breadcrumb_conflict_in_trash": "在回收站中", + "session_breadcrumb_conflict_starred": "已固定", + "session_breadcrumb_conflict_unstarred": "已取消固定", + "session_breadcrumb_conflict_unpinned_target": "已取消固定 {target}", + "session_breadcrumb_conflict_pinned_target": "已固定 {target}", + "session_breadcrumb_conflict_pinned_target_note": "已固定 {target}: {note}", + "session_breadcrumb_conflict_unknown_origin": "未知来源", + "session_breadcrumb_conflict_current": "当前", + "session_breadcrumb_conflict_other": "其他", "session_breadcrumb_resumed_in": "已在 {target} 中继续", "session_breadcrumb_command_copied": "命令已复制！", "session_breadcrumb_failed": "失败", @@ -437,6 +455,56 @@ "sidebar_row_rename": "重命名", "sidebar_row_open_in_new_tab": "在新标签页打开", "sidebar_row_delete": "删除", + "peers_title": "Peers", + "peers_refresh_status": "刷新 peer 状态", + "peers_conflict_count_singular": "1 个元数据冲突", + "peers_conflict_count_plural": "{count} 个元数据冲突", + "peers_loading": "正在加载 peers...", + "peers_empty": "暂无 peers", + "peers_unavailable": "此服务器不可用 artifact sync。", + "peers_this_machine": "此机器", + "peers_in_sync": "已同步", + "peers_pending": "{count} 个待处理", + "peers_local_sessions_title": "此数据库中来自该 peer 的会话", + "peers_local_sessions": "{count} 个本地", + "peers_published_sessions_title": "此 peer 已发布的会话", + "peers_published_sessions": "{count} 个已发布", + "peers_checkpoint_title": "最新 checkpoint 序号", + "peers_checkpoint": "checkpoint #{seq}", + "peers_last_published_title": "上次发布 checkpoint", + "peers_updated": "更新于 {time}", + "trash_title": "回收站", + "trash_empty_local_title": "清空本地回收站", + "trash_emptying": "正在清空...", + "trash_empty_local": "清空本地回收站", + "trash_loading": "正在加载回收站...", + "trash_empty": "回收站为空", + "trash_empty_desc": "已删除的会话会显示在这里。", + "trash_messages": [ + { + "declarations": [ + "input count", + "input countLabel", + "local countPlural = count: plural" + ], + "selectors": [ + "countPlural" + ], + "match": { + "countPlural=other": "{countLabel} 条消息" + } + } + ], + "trash_deleted": "删除于 {time}", + "trash_restore_session": "恢复会话", + "trash_restore": "恢复", + "trash_delete_everywhere_prompt": "全局删除？", + "trash_confirm_delete_everywhere": "确认全局删除", + "trash_deleting": "正在删除...", + "trash_confirm": "确认", + "trash_cancel": "取消", + "trash_delete_everywhere": "全局删除", + "trash_delete_everywhere_title": "全局删除", "shared_all_projects": "所有项目", "shared_project_filter_placeholder": "筛选项目...", "shared_select_project": "选择项目", @@ -1584,32 +1652,6 @@ "system_boundary_stop_hook": "停止钩子反馈", "system_boundary_title": "系统边界：{subtype}", "system_boundary_show_content": "显示内容", - "trash_loading": "正在加载回收站...", - "trash_empty": "回收站为空", - "trash_empty_desc": "已删除的会话会显示在这里。", - "trash_title": "回收站", - "trash_emptying": "正在清空...", - "trash_empty_trash": "清空回收站", - "trash_msgs": [ - { - "declarations": [ - "input count", - "input countLabel", - "local countPlural = count: plural" - ], - "selectors": [ - "countPlural" - ], - "match": { - "countPlural=other": "{countLabel} 条消息" - } - } - ], - "trash_deleted_ago": "{time} 删除", - "trash_restore_session": "恢复会话", - "trash_restore": "恢复", - "trash_permanently_delete": "永久删除", - "trash_delete_forever": "永久删除", "trends_term": "词项", "trends_per1k_messages": "每千条消息", "trends_count": "数量", diff --git a/frontend/messages/zh-TW.json b/frontend/messages/zh-TW.json index 513c6380c..9afe09ed4 100644 --- a/frontend/messages/zh-TW.json +++ b/frontend/messages/zh-TW.json @@ -22,6 +22,7 @@ "nav_pinned": "已固定", "nav_insights": "洞察", "nav_trash": "回收站", + "nav_peers": "Peers", "nav_search_sessions": "搜索會話...", "nav_search_sessions_shortcut": "搜索會話 ({shortcut})", "header_transcript_normal_title": "普通 transcript - 顯示所有消息", @@ -183,6 +184,23 @@ "session_breadcrumb_find_in_session_shortcut": "在會話中查找 (/)", "session_breadcrumb_rename": "重命名", "session_breadcrumb_delete": "刪除", + "session_breadcrumb_metadata_conflicts": "元資料衝突", + "session_breadcrumb_conflict_field_name": "名稱", + "session_breadcrumb_conflict_field_trash": "回收站", + "session_breadcrumb_conflict_field_star": "固定", + "session_breadcrumb_conflict_field_delete_everywhere": "全域刪除", + "session_breadcrumb_conflict_field_pin": "固定", + "session_breadcrumb_conflict_default_title": "預設標題", + "session_breadcrumb_conflict_restored": "已恢復", + "session_breadcrumb_conflict_in_trash": "在回收站中", + "session_breadcrumb_conflict_starred": "已固定", + "session_breadcrumb_conflict_unstarred": "已取消固定", + "session_breadcrumb_conflict_unpinned_target": "已取消固定 {target}", + "session_breadcrumb_conflict_pinned_target": "已固定 {target}", + "session_breadcrumb_conflict_pinned_target_note": "已固定 {target}: {note}", + "session_breadcrumb_conflict_unknown_origin": "未知來源", + "session_breadcrumb_conflict_current": "目前", + "session_breadcrumb_conflict_other": "其他", "session_breadcrumb_resumed_in": "已在 {target} 中繼續", "session_breadcrumb_command_copied": "命令已複製！", "session_breadcrumb_failed": "失敗", @@ -437,6 +455,56 @@ "sidebar_row_rename": "重命名", "sidebar_row_open_in_new_tab": "在新標籤頁打開", "sidebar_row_delete": "刪除", + "peers_title": "Peers", + "peers_refresh_status": "重新整理 peer 狀態", + "peers_conflict_count_singular": "1 個元資料衝突", + "peers_conflict_count_plural": "{count} 個元資料衝突", + "peers_loading": "正在加載 peers...", + "peers_empty": "暫無 peers", + "peers_unavailable": "此伺服器無法使用 artifact sync。", + "peers_this_machine": "此機器", + "peers_in_sync": "已同步", + "peers_pending": "{count} 個待處理", + "peers_local_sessions_title": "此資料庫中來自該 peer 的會話", + "peers_local_sessions": "{count} 個本地", + "peers_published_sessions_title": "此 peer 已發佈的會話", + "peers_published_sessions": "{count} 個已發佈", + "peers_checkpoint_title": "最新 checkpoint 序號", + "peers_checkpoint": "checkpoint #{seq}", + "peers_last_published_title": "上次發佈 checkpoint", + "peers_updated": "更新於 {time}", + "trash_title": "回收站", + "trash_empty_local_title": "清空本地回收站", + "trash_emptying": "正在清空...", + "trash_empty_local": "清空本地回收站", + "trash_loading": "正在加載回收站...", + "trash_empty": "回收站為空", + "trash_empty_desc": "已刪除的會話會顯示在這裡。", + "trash_messages": [ + { + "declarations": [ + "input count", + "input countLabel", + "local countPlural = count: plural" + ], + "selectors": [ + "countPlural" + ], + "match": { + "countPlural=other": "{countLabel} 條消息" + } + } + ], + "trash_deleted": "{time} 刪除", + "trash_restore_session": "恢復會話", + "trash_restore": "恢復", + "trash_delete_everywhere_prompt": "全域刪除？", + "trash_confirm_delete_everywhere": "確認全域刪除", + "trash_deleting": "正在刪除...", + "trash_confirm": "確認", + "trash_cancel": "取消", + "trash_delete_everywhere": "全域刪除", + "trash_delete_everywhere_title": "全域刪除", "shared_all_projects": "所有項目", "shared_project_filter_placeholder": "篩選項目...", "shared_select_project": "選擇項目", @@ -1584,32 +1652,6 @@ "system_boundary_stop_hook": "停止鉤子反饋", "system_boundary_title": "系統邊界：{subtype}", "system_boundary_show_content": "顯示內容", - "trash_loading": "正在加載回收站...", - "trash_empty": "回收站為空", - "trash_empty_desc": "已刪除的會話會顯示在這裡。", - "trash_title": "回收站", - "trash_emptying": "正在清空...", - "trash_empty_trash": "清空回收站", - "trash_msgs": [ - { - "declarations": [ - "input count", - "input countLabel", - "local countPlural = count: plural" - ], - "selectors": [ - "countPlural" - ], - "match": { - "countPlural=other": "{countLabel} 條消息" - } - } - ], - "trash_deleted_ago": "{time} 刪除", - "trash_restore_session": "恢復會話", - "trash_restore": "恢復", - "trash_permanently_delete": "永久刪除", - "trash_delete_forever": "永久刪除", "trends_term": "詞項", "trends_per1k_messages": "每千條消息", "trends_count": "數量", diff --git a/frontend/src/App.svelte b/frontend/src/App.svelte index 20999813d..0012501e2 100644 --- a/frontend/src/App.svelte +++ b/frontend/src/App.svelte @@ -64,6 +64,7 @@ import PinnedPage from "./lib/components/pinned/PinnedPage.svelte"; import TrashPage from "./lib/components/trash/TrashPage.svelte"; import RecentEditsPage from "./lib/components/recentedits/RecentEditsPage.svelte"; + import PeersPage from "./lib/components/peers/PeersPage.svelte"; import SettingsPage from "./lib/components/settings/SettingsPage.svelte"; import { sessions, filtersToParams } from "./lib/stores/sessions.svelte.js"; import { messages } from "./lib/stores/messages.svelte.js"; @@ -535,6 +536,10 @@
+{:else if router.route === "peers"} +
+ +
{:else if router.route === "settings"}
diff --git a/frontend/src/lib/api/generated/index.ts b/frontend/src/lib/api/generated/index.ts index 65131dd7e..e418c80c1 100644 --- a/frontend/src/lib/api/generated/index.ts +++ b/frontend/src/lib/api/generated/index.ts @@ -18,6 +18,10 @@ export type { AgentsResponse } from './models/AgentsResponse'; export type { AgentTotal } from './models/AgentTotal'; export type { ApiErrorResponse } from './models/ApiErrorResponse'; export type { ApplyWorktreeMappingsResponse } from './models/ApplyWorktreeMappingsResponse'; +export type { ArtifactOriginsResponse } from './models/ArtifactOriginsResponse'; +export type { ArtifactPeer } from './models/ArtifactPeer'; +export type { ArtifactPeersResponse } from './models/ArtifactPeersResponse'; +export type { ArtifactPostResponse } from './models/ArtifactPostResponse'; export type { BatchDeleteInputBody } from './models/BatchDeleteInputBody'; export type { BranchesResponse } from './models/BranchesResponse'; export type { BulkStarInputBody } from './models/BulkStarInputBody'; @@ -52,6 +56,7 @@ export type { DbHourOfWeekCell } from './models/DbHourOfWeekCell'; export type { DbHourOfWeekResponse } from './models/DbHourOfWeekResponse'; export type { DbInsight } from './models/DbInsight'; export type { DbMessage } from './models/DbMessage'; +export type { DbMetadataConflict } from './models/DbMetadataConflict'; export type { DbModelBreakdown } from './models/DbModelBreakdown'; export type { DbPeakContextDistribution } from './models/DbPeakContextDistribution'; export type { DbPercentiles } from './models/DbPercentiles'; @@ -149,6 +154,7 @@ export type { GithubConfigResponse } from './models/GithubConfigResponse'; export type { InsightCannedSessionFilters } from './models/InsightCannedSessionFilters'; export type { InsightsResponse } from './models/InsightsResponse'; export type { MachinesResponse } from './models/MachinesResponse'; +export type { MetadataConflictsResponse } from './models/MetadataConflictsResponse'; export type { ModelTotal } from './models/ModelTotal'; export type { Opener } from './models/Opener'; export type { OpenersResponse } from './models/OpenersResponse'; @@ -212,6 +218,7 @@ export type { WorktreeMappingsResponse } from './models/WorktreeMappingsResponse export { ActivityService } from './services/ActivityService'; export { AnalyticsService } from './services/AnalyticsService'; +export { ArtifactsService } from './services/ArtifactsService'; export { AssetsService } from './services/AssetsService'; export { ConfigService } from './services/ConfigService'; export { EmbeddingsService } from './services/EmbeddingsService'; diff --git a/frontend/src/lib/api/generated/models/ArtifactOriginsResponse.ts b/frontend/src/lib/api/generated/models/ArtifactOriginsResponse.ts new file mode 100644 index 000000000..076955347 --- /dev/null +++ b/frontend/src/lib/api/generated/models/ArtifactOriginsResponse.ts @@ -0,0 +1,7 @@ +/* generated using openapi-typescript-codegen -- do not edit */ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ +export type ArtifactOriginsResponse = { + origins: any[] | null; +}; diff --git a/frontend/src/lib/api/generated/models/ArtifactPeer.ts b/frontend/src/lib/api/generated/models/ArtifactPeer.ts new file mode 100644 index 000000000..bebe43eea --- /dev/null +++ b/frontend/src/lib/api/generated/models/ArtifactPeer.ts @@ -0,0 +1,12 @@ +/* generated using openapi-typescript-codegen -- do not edit */ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ +export type ArtifactPeer = { + checkpoint_seq: number; + is_local: boolean; + last_published?: string; + local_sessions: number; + origin: string; + published_sessions: number; +}; diff --git a/frontend/src/lib/api/generated/models/ArtifactPeersResponse.ts b/frontend/src/lib/api/generated/models/ArtifactPeersResponse.ts new file mode 100644 index 000000000..5e47fb005 --- /dev/null +++ b/frontend/src/lib/api/generated/models/ArtifactPeersResponse.ts @@ -0,0 +1,9 @@ +/* generated using openapi-typescript-codegen -- do not edit */ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ +export type ArtifactPeersResponse = { + conflict_count: number; + local_origin: string; + peers: any[] | null; +}; diff --git a/frontend/src/lib/api/generated/models/ArtifactPostResponse.ts b/frontend/src/lib/api/generated/models/ArtifactPostResponse.ts new file mode 100644 index 000000000..3caad6ae8 --- /dev/null +++ b/frontend/src/lib/api/generated/models/ArtifactPostResponse.ts @@ -0,0 +1,12 @@ +/* generated using openapi-typescript-codegen -- do not edit */ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ +export type ArtifactPostResponse = { + duplicate: boolean; + hash?: string; + kind: string; + name: string; + origin: string; + size: number; +}; diff --git a/frontend/src/lib/api/generated/models/DbMetadataConflict.ts b/frontend/src/lib/api/generated/models/DbMetadataConflict.ts new file mode 100644 index 000000000..66afad6d0 --- /dev/null +++ b/frontend/src/lib/api/generated/models/DbMetadataConflict.ts @@ -0,0 +1,18 @@ +/* generated using openapi-typescript-codegen -- do not edit */ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ +export type DbMetadataConflict = { + created_at: string; + field: string; + id: number; + losing_op: string; + losing_order_key: string; + losing_origin: string; + losing_value: string; + session_gid: string; + winning_op: string; + winning_order_key: string; + winning_origin: string; + winning_value: string; +}; diff --git a/frontend/src/lib/api/generated/models/DbTopSession.ts b/frontend/src/lib/api/generated/models/DbTopSession.ts index 11808f97d..ba2c1497a 100644 --- a/frontend/src/lib/api/generated/models/DbTopSession.ts +++ b/frontend/src/lib/api/generated/models/DbTopSession.ts @@ -15,4 +15,3 @@ export type DbTopSession = { started_at?: string; termination_status?: string; }; - diff --git a/frontend/src/lib/api/generated/models/MetadataConflictsResponse.ts b/frontend/src/lib/api/generated/models/MetadataConflictsResponse.ts new file mode 100644 index 000000000..1dc167d78 --- /dev/null +++ b/frontend/src/lib/api/generated/models/MetadataConflictsResponse.ts @@ -0,0 +1,7 @@ +/* generated using openapi-typescript-codegen -- do not edit */ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ +export type MetadataConflictsResponse = { + conflicts: any[] | null; +}; diff --git a/frontend/src/lib/api/generated/services/ArtifactsService.ts b/frontend/src/lib/api/generated/services/ArtifactsService.ts new file mode 100644 index 000000000..efa85b815 --- /dev/null +++ b/frontend/src/lib/api/generated/services/ArtifactsService.ts @@ -0,0 +1,188 @@ +/* generated using openapi-typescript-codegen -- do not edit */ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ +import type { ArtifactOriginsResponse } from '../models/ArtifactOriginsResponse'; +import type { ArtifactPeersResponse } from '../models/ArtifactPeersResponse'; +import type { ArtifactPostResponse } from '../models/ArtifactPostResponse'; +import type { CancelablePromise } from '../core/CancelablePromise'; +import { OpenAPI } from '../core/OpenAPI'; +import { request as __request } from '../core/request'; +export class ArtifactsService { + /** + * List artifact origins + * @returns ArtifactOriginsResponse OK + * @throws ApiError + */ + public static getApiV1ArtifactsOrigins(): CancelablePromise { + return __request(OpenAPI, { + method: 'GET', + url: '/api/v1/artifacts/origins', + errors: { + 400: `Bad Request`, + 401: `Unauthorized`, + 403: `Forbidden`, + 404: `Not Found`, + 409: `Conflict`, + 500: `Internal Server Error`, + 501: `Not Implemented`, + 502: `Bad Gateway`, + 503: `Service Unavailable`, + 504: `Gateway Timeout`, + }, + }); + } + /** + * List artifact peers + * @returns ArtifactPeersResponse OK + * @throws ApiError + */ + public static getApiV1ArtifactsPeers(): CancelablePromise { + return __request(OpenAPI, { + method: 'GET', + url: '/api/v1/artifacts/peers', + errors: { + 400: `Bad Request`, + 401: `Unauthorized`, + 403: `Forbidden`, + 404: `Not Found`, + 409: `Conflict`, + 500: `Internal Server Error`, + 501: `Not Implemented`, + 502: `Bad Gateway`, + 503: `Service Unavailable`, + 504: `Gateway Timeout`, + }, + }); + } + /** + * Get latest artifact checkpoint + * @returns string OK + * @throws ApiError + */ + public static getApiV1ArtifactsOriginCheckpoint({ + origin, + }: { + /** + * Artifact origin ID + */ + origin: string, + }): CancelablePromise { + return __request(OpenAPI, { + method: 'GET', + url: '/api/v1/artifacts/{origin}/checkpoint', + path: { + 'origin': origin, + }, + errors: { + 400: `Bad Request`, + 401: `Unauthorized`, + 403: `Forbidden`, + 404: `Not Found`, + 409: `Conflict`, + 422: `Unprocessable Entity`, + 500: `Internal Server Error`, + 501: `Not Implemented`, + 502: `Bad Gateway`, + 503: `Service Unavailable`, + 504: `Gateway Timeout`, + }, + }); + } + /** + * Get artifact + * @returns string OK + * @throws ApiError + */ + public static getApiV1ArtifactsOriginKindName({ + origin, + kind, + name, + }: { + /** + * Artifact origin ID + */ + origin: string, + /** + * Artifact kind + */ + kind: string, + /** + * Artifact filename or hash + */ + name: string, + }): CancelablePromise { + return __request(OpenAPI, { + method: 'GET', + url: '/api/v1/artifacts/{origin}/{kind}/{name}', + path: { + 'origin': origin, + 'kind': kind, + 'name': name, + }, + errors: { + 400: `Bad Request`, + 401: `Unauthorized`, + 403: `Forbidden`, + 404: `Not Found`, + 409: `Conflict`, + 422: `Unprocessable Entity`, + 500: `Internal Server Error`, + 501: `Not Implemented`, + 502: `Bad Gateway`, + 503: `Service Unavailable`, + 504: `Gateway Timeout`, + }, + }); + } + /** + * Post artifact + * @returns ArtifactPostResponse OK + * @throws ApiError + */ + public static postApiV1ArtifactsOriginKindName({ + origin, + kind, + name, + requestBody, + }: { + /** + * Artifact origin ID + */ + origin: string, + /** + * Artifact kind + */ + kind: string, + /** + * Artifact filename or hash + */ + name: string, + requestBody: Blob, + }): CancelablePromise { + return __request(OpenAPI, { + method: 'POST', + url: '/api/v1/artifacts/{origin}/{kind}/{name}', + path: { + 'origin': origin, + 'kind': kind, + 'name': name, + }, + body: requestBody, + mediaType: 'application/octet-stream', + errors: { + 400: `Bad Request`, + 401: `Unauthorized`, + 403: `Forbidden`, + 404: `Not Found`, + 409: `Conflict`, + 422: `Unprocessable Entity`, + 500: `Internal Server Error`, + 501: `Not Implemented`, + 502: `Bad Gateway`, + 503: `Service Unavailable`, + 504: `Gateway Timeout`, + }, + }); + } +} diff --git a/frontend/src/lib/api/generated/services/SessionsService.ts b/frontend/src/lib/api/generated/services/SessionsService.ts index 86730d5f3..14bca43ce 100644 --- a/frontend/src/lib/api/generated/services/SessionsService.ts +++ b/frontend/src/lib/api/generated/services/SessionsService.ts @@ -8,6 +8,7 @@ import type { DbSessionActivityResponse } from '../models/DbSessionActivityRespo import type { DbSessionTiming } from '../models/DbSessionTiming'; import type { DbSidebarSessionIndex } from '../models/DbSidebarSessionIndex'; import type { EmptyTrashResponse } from '../models/EmptyTrashResponse'; +import type { MetadataConflictsResponse } from '../models/MetadataConflictsResponse'; import type { OpenRequest } from '../models/OpenRequest'; import type { OpenSessionResponse } from '../models/OpenSessionResponse'; import type { OrdinalsResponse } from '../models/OrdinalsResponse'; @@ -847,6 +848,40 @@ export class SessionsService { }, }); } + /** + * List session metadata conflicts + * @returns MetadataConflictsResponse OK + * @throws ApiError + */ + public static getApiV1SessionsIdMetadataConflicts({ + id, + }: { + /** + * Session ID + */ + id: string, + }): CancelablePromise { + return __request(OpenAPI, { + method: 'GET', + url: '/api/v1/sessions/{id}/metadata-conflicts', + path: { + 'id': id, + }, + errors: { + 400: `Bad Request`, + 401: `Unauthorized`, + 403: `Forbidden`, + 404: `Not Found`, + 409: `Conflict`, + 422: `Unprocessable Entity`, + 500: `Internal Server Error`, + 501: `Not Implemented`, + 502: `Bad Gateway`, + 503: `Service Unavailable`, + 504: `Gateway Timeout`, + }, + }); + } /** * Open session directory * @returns OpenSessionResponse OK diff --git a/frontend/src/lib/components/layout/AppHeader.svelte b/frontend/src/lib/components/layout/AppHeader.svelte index 73a466a8a..e0568de7e 100644 --- a/frontend/src/lib/components/layout/AppHeader.svelte +++ b/frontend/src/lib/components/layout/AppHeader.svelte @@ -90,6 +90,7 @@ "insights", "trash", "recent-edits", + "peers", ] as const; const tabs: TopBarTab[] = $derived([ @@ -101,6 +102,7 @@ { id: "insights", label: m.nav_insights() }, { id: "trash", label: m.nav_trash() }, { id: "recent-edits", label: m.nav_recent_edits() }, + { id: "peers", label: m.nav_peers() }, ]); const activeTab = $derived( diff --git a/frontend/src/lib/components/layout/SessionBreadcrumb.svelte b/frontend/src/lib/components/layout/SessionBreadcrumb.svelte index b7beacac4..cc5498152 100644 --- a/frontend/src/lib/components/layout/SessionBreadcrumb.svelte +++ b/frontend/src/lib/components/layout/SessionBreadcrumb.svelte @@ -13,12 +13,14 @@ LinkIcon, SearchIcon, SquareTerminalIcon, + TriangleAlertIcon, } from "../../icons.js"; import { onMount } from "svelte"; import type { Session } from "../../api/types.js"; import { OpenersService, SessionsService, + type DbMetadataConflict, type ResumeRequest, type ResumeResponse, } from "../../api/generated/index"; @@ -65,6 +67,10 @@ let openFeedback = $state(""); let feedbackTimer: ReturnType | undefined; let sessionDir = $state(null); + let metadataConflicts = $state([]); + let conflictsOpen = $state(false); + let conflictFetchId: string | null = null; + let conflictRequestSeq = 0; interface Opener { id: string; @@ -129,6 +135,34 @@ }); }); + $effect(() => { + if (!session) { + metadataConflicts = []; + conflictsOpen = false; + conflictFetchId = null; + conflictRequestSeq++; + return; + } + const id = session.id; + if (id === conflictFetchId) return; + conflictFetchId = id; + metadataConflicts = []; + conflictsOpen = false; + const seq = ++conflictRequestSeq; + configureGeneratedClient(); + SessionsService.getApiV1SessionsIdMetadataConflicts({ id }) + .then((res) => { + if (seq !== conflictRequestSeq || session?.id !== id) return; + metadataConflicts = Array.isArray(res.conflicts) + ? (res.conflicts as DbMetadataConflict[]) + : []; + }) + .catch(() => { + if (seq !== conflictRequestSeq) return; + metadataConflicts = []; + }); + }); + let sessionCost = $state(null); let sessionUsageBreakdownCount = $state(0); let sessionUsageBreakdown = $state([]); @@ -480,6 +514,90 @@ : m.session_breadcrumb_failed()); } + type ConflictSide = "winning" | "losing"; + + function conflictFieldLabel(field: string): string { + if (field === "display_name") return m.session_breadcrumb_conflict_field_name(); + if (field === "deleted_at") return m.session_breadcrumb_conflict_field_trash(); + if (field === "starred") return m.session_breadcrumb_conflict_field_star(); + if (field === "purge") return m.session_breadcrumb_conflict_field_delete_everywhere(); + if (field.startsWith("pin:")) return m.session_breadcrumb_conflict_field_pin(); + return field.replaceAll("_", " "); + } + + function parseJSONValue(value: string): Record | null { + if (!value) return null; + try { + const parsed = JSON.parse(value) as unknown; + return parsed !== null && typeof parsed === "object" + ? parsed as Record + : null; + } catch { + return null; + } + } + + function conflictSideValue( + conflict: DbMetadataConflict, + side: ConflictSide, + ): string { + const op = side === "winning" + ? conflict.winning_op + : conflict.losing_op; + const value = side === "winning" + ? conflict.winning_value + : conflict.losing_value; + if (conflict.field === "display_name") { + const parsed = parseJSONValue(value); + const displayName = parsed?.display_name; + if (typeof displayName === "string" && displayName.trim()) { + return displayName; + } + return m.session_breadcrumb_conflict_default_title(); + } + if (conflict.field === "deleted_at") { + if (op === "restore") return m.session_breadcrumb_conflict_restored(); + if (op === "soft_delete") return m.session_breadcrumb_conflict_in_trash(); + } + if (conflict.field === "starred") { + if (op === "star") return m.session_breadcrumb_conflict_starred(); + if (op === "unstar") return m.session_breadcrumb_conflict_unstarred(); + } + if (conflict.field === "purge") { + return op === "purge" + ? m.session_breadcrumb_conflict_field_delete_everywhere() + : op; + } + if (conflict.field.startsWith("pin:")) { + const parsed = parseJSONValue(value); + const ordinal = parsed?.ordinal; + const sourceUUID = parsed?.source_uuid; + const note = parsed?.note; + const target = typeof sourceUUID === "string" && sourceUUID + ? sourceUUID.slice(0, 8) + : typeof ordinal === "number" + ? `#${ordinal}` + : "pin"; + if (op === "unpin") { + return m.session_breadcrumb_conflict_unpinned_target({ target }); + } + if (typeof note === "string" && note.trim()) { + return m.session_breadcrumb_conflict_pinned_target_note({ target, note }); + } + return m.session_breadcrumb_conflict_pinned_target({ target }); + } + return value || op; + } + + function conflictOrigin( + conflict: DbMetadataConflict, + side: ConflictSide, + ): string { + return side === "winning" + ? conflict.winning_origin || m.session_breadcrumb_conflict_unknown_origin() + : conflict.losing_origin || m.session_breadcrumb_conflict_unknown_origin(); + } + async function handleOpenIn(opener: Opener) { if (!session) return; showOpenMenu = false; @@ -584,6 +702,9 @@ } else if (showOpenMenu) { showOpenMenu = false; e.preventDefault(); + } else if (conflictsOpen) { + conflictsOpen = false; + e.preventDefault(); } return; } @@ -615,6 +736,9 @@ if (!(target as HTMLElement).closest?.(".open-group")) { showOpenMenu = false; } + if (!(target as HTMLElement).closest?.(".conflict-group")) { + conflictsOpen = false; + } } @@ -704,6 +828,63 @@ > {getGradeLabel(session.health_grade)} + {#if metadataConflicts.length > 0} + + + {#if conflictsOpen} +
+
{m.session_breadcrumb_metadata_conflicts()}
+ {#each metadataConflicts as conflict (conflict.id)} +
+
+ {conflictFieldLabel(conflict.field)} +
+
+ {m.session_breadcrumb_conflict_current()} + + {conflictSideValue(conflict, "winning")} + + + {conflictOrigin(conflict, "winning")} + +
+
+ {m.session_breadcrumb_conflict_other()} + + {conflictSideValue(conflict, "losing")} + + + {conflictOrigin(conflict, "losing")} + +
+
+ {/each} +
+ {/if} + + {/if} {#if showDropdown} +
+ + {#if conflictCount > 0} +
+
+ {/if} + + {#if loading} +
+ + {m.peers_loading()} +
+ {:else if peers.length === 0} + + {#snippet icon()} + + {:else} +
+ {#each peers as peer (peer.origin)} + {@const state = syncState(peer)} +
+
+ {#if peer.is_local} +
+
+
+ {peer.origin} + {#if peer.is_local} + {m.peers_this_machine()} + {/if} + {#if state === "synced"} + {m.peers_in_sync()} + {:else if state === "behind"} + + {m.peers_pending({ + count: formatNumber(peer.published_sessions - peer.local_sessions), + })} + + {/if} +
+
+ + {m.peers_local_sessions({ count: formatNumber(peer.local_sessions) })} + + / + + {m.peers_published_sessions({ count: formatNumber(peer.published_sessions) })} + + {#if peer.checkpoint_seq > 0} + · + + {m.peers_checkpoint({ seq: String(peer.checkpoint_seq) })} + + {/if} + {#if peer.last_published} + · + + {m.peers_updated({ time: formatRelativeTime(peer.last_published) })} + + {/if} +
+
+
+ {/each} +
+ {/if} +
+ + diff --git a/frontend/src/lib/components/trash/TrashPage.svelte b/frontend/src/lib/components/trash/TrashPage.svelte index 8c269a0f6..f42228556 100644 --- a/frontend/src/lib/components/trash/TrashPage.svelte +++ b/frontend/src/lib/components/trash/TrashPage.svelte @@ -12,6 +12,8 @@ let trashedSessions: Session[] = $state([]); let loading = $state(true); let emptying = $state(false); + let confirmingDeleteId = $state(null); + let deletingId = $state(null); interface TrashResponse { sessions: Session[]; @@ -40,6 +42,7 @@ configureGeneratedClient(); await SessionsService.postApiV1SessionsIdRestore({ id }); trashedSessions = trashedSessions.filter((s) => s.id !== id); + if (confirmingDeleteId === id) confirmingDeleteId = null; sessions.clearRecentlyDeleted(id); sessions.invalidateFilterCaches(); sessions.load(); @@ -48,15 +51,27 @@ } } + function requestPermanentDelete(id: string) { + confirmingDeleteId = id; + } + + function cancelPermanentDelete(id: string) { + if (confirmingDeleteId === id) confirmingDeleteId = null; + } + async function permanentDelete(id: string) { + deletingId = id; try { configureGeneratedClient(); await SessionsService.deleteApiV1SessionsIdPermanent({ id }); trashedSessions = trashedSessions.filter((s) => s.id !== id); + if (confirmingDeleteId === id) confirmingDeleteId = null; sessions.clearRecentlyDeleted(id); sessions.invalidateFilterCaches(); } catch { // silently fail + } finally { + if (deletingId === id) deletingId = null; } } @@ -82,6 +97,22 @@
+
+
+ {#if loading}
{m.trash_loading()}
{:else if trashedSessions.length === 0} @@ -91,19 +122,6 @@ {/snippet} {:else} -
-
-
{#each trashedSessions as session (session.id)}
@@ -112,12 +130,12 @@
{session.agent} {session.project} - {m.trash_msgs({ + {m.trash_messages({ count: session.user_message_count, countLabel: session.user_message_count.toLocaleString(), })} {#if session.deleted_at} - {m.trash_deleted_ago({ time: formatRelativeTime(session.deleted_at) })} + {m.trash_deleted({ time: formatRelativeTime(session.deleted_at) })} {/if}
@@ -129,13 +147,32 @@ > {m.trash_restore()} - + {#if confirmingDeleteId === session.id} + {m.trash_delete_everywhere_prompt()} + + + {:else} + + {/if}
{/each} @@ -269,6 +306,7 @@ .trash-card-actions { display: flex; + align-items: center; gap: 6px; flex-shrink: 0; } @@ -304,4 +342,38 @@ .perm-delete-btn:hover { background: color-mix(in srgb, var(--accent-red, #e55) 8%, transparent); } + + .perm-delete-btn--confirm { + border-color: var(--accent-red, #e55); + } + + .delete-confirm-label { + color: var(--text-muted); + font-size: 11px; + font-weight: 500; + white-space: nowrap; + } + + .cancel-delete-btn { + font-size: 11px; + font-weight: 500; + color: var(--text-muted); + background: none; + border: 1px solid var(--border-muted); + border-radius: var(--radius-sm); + padding: 4px 10px; + cursor: pointer; + transition: background 0.12s, color 0.12s; + } + + .cancel-delete-btn:hover:not(:disabled) { + color: var(--text-secondary); + background: var(--bg-surface-hover); + } + + .perm-delete-btn:disabled, + .cancel-delete-btn:disabled { + cursor: default; + opacity: 0.6; + } diff --git a/frontend/src/lib/components/trash/TrashPage.test.ts b/frontend/src/lib/components/trash/TrashPage.test.ts new file mode 100644 index 000000000..80843a33b --- /dev/null +++ b/frontend/src/lib/components/trash/TrashPage.test.ts @@ -0,0 +1,152 @@ +// @vitest-environment jsdom +import { + afterEach, + beforeEach, + describe, + expect, + it, + vi, +} from "vite-plus/test"; +import { mount, tick, unmount } from "svelte"; +// @ts-ignore +import TrashPage from "./TrashPage.svelte"; +import type { Session } from "../../api/types.js"; +import { SessionsService } from "../../api/generated/index"; + +vi.mock("../../api/runtime.js", async (importOriginal) => { + const orig = + await importOriginal(); + return { + ...orig, + configureGeneratedClient: vi.fn(), + }; +}); + +vi.mock("../../api/generated/index", async (importOriginal) => { + const orig = + await importOriginal(); + return { + ...orig, + SessionsService: { + deleteApiV1SessionsIdPermanent: vi.fn(), + deleteApiV1Trash: vi.fn(), + getApiV1Trash: vi.fn(), + postApiV1SessionsIdRestore: vi.fn(), + }, + }; +}); + +vi.mock("../../stores/sessions.svelte.js", () => ({ + sessions: { + clearRecentlyDeleted: vi.fn(), + invalidateFilterCaches: vi.fn(), + load: vi.fn(), + }, +})); + +const sessionsService = SessionsService as unknown as { + deleteApiV1SessionsIdPermanent: ReturnType; + deleteApiV1Trash: ReturnType; + getApiV1Trash: ReturnType; + postApiV1SessionsIdRestore: ReturnType; +}; + +function makeTrashedSession(overrides: Partial = {}): Session { + return { + id: "s1", + project: "alpha", + machine: "local", + agent: "claude", + first_message: "build this", + display_name: null, + started_at: "2026-06-14T01:02:03Z", + ended_at: "2026-06-14T01:03:03Z", + created_at: "2026-06-14T01:02:03Z", + deleted_at: "2026-06-14T01:04:03Z", + message_count: 2, + user_message_count: 1, + total_output_tokens: 0, + peak_context_tokens: 0, + is_automated: false, + ...overrides, + } as Session; +} + +function buttonByText(label: string): HTMLButtonElement { + const button = Array.from(document.querySelectorAll("button")) + .find((el) => el.textContent?.trim() === label); + expect(button).toBeTruthy(); + return button as HTMLButtonElement; +} + +beforeEach(() => { + sessionsService.getApiV1Trash + .mockReset() + .mockResolvedValue({ sessions: [makeTrashedSession()] }); + sessionsService.deleteApiV1SessionsIdPermanent + .mockReset() + .mockResolvedValue(undefined); + sessionsService.deleteApiV1Trash + .mockReset() + .mockResolvedValue({ deleted: 1 }); + sessionsService.postApiV1SessionsIdRestore + .mockReset() + .mockResolvedValue(undefined); +}); + +afterEach(() => { + document.body.innerHTML = ""; +}); + +describe("TrashPage", () => { + it("requires confirmation before deleting a session everywhere", async () => { + const component = mount(TrashPage, { target: document.body }); + + await vi.waitFor(() => { + expect(buttonByText("Delete Everywhere")).toBeTruthy(); + }); + + buttonByText("Delete Everywhere").dispatchEvent( + new MouseEvent("click", { bubbles: true }), + ); + await tick(); + + expect( + sessionsService.deleteApiV1SessionsIdPermanent, + ).not.toHaveBeenCalled(); + expect(document.body.textContent).toContain("Delete everywhere?"); + + buttonByText("Confirm").dispatchEvent( + new MouseEvent("click", { bubbles: true }), + ); + + await vi.waitFor(() => { + expect( + sessionsService.deleteApiV1SessionsIdPermanent, + ).toHaveBeenCalledWith({ id: "s1" }); + }); + + unmount(component); + }); + + it("keeps empty trash on the local endpoint", async () => { + const component = mount(TrashPage, { target: document.body }); + + await vi.waitFor(() => { + expect(buttonByText("Empty Local Trash")).toBeTruthy(); + }); + + buttonByText("Empty Local Trash").dispatchEvent( + new MouseEvent("click", { bubbles: true }), + ); + + await vi.waitFor(() => { + expect(sessionsService.deleteApiV1Trash).toHaveBeenCalled(); + }); + expect( + sessionsService.deleteApiV1SessionsIdPermanent, + ).not.toHaveBeenCalled(); + + unmount(component); + }); +}); diff --git a/frontend/src/lib/i18n/i18n.test.ts b/frontend/src/lib/i18n/i18n.test.ts index 4a6950b43..98a79b655 100644 --- a/frontend/src/lib/i18n/i18n.test.ts +++ b/frontend/src/lib/i18n/i18n.test.ts @@ -141,7 +141,7 @@ describe("i18n locale selection", () => { count: 2, countLabel: "2", })).toBe("2 sessions"); - expect(m.trash_msgs({ + expect(m.trash_messages({ count: 1, countLabel: "1", })).toBe("1 msg"); diff --git a/frontend/src/lib/stores/router.svelte.ts b/frontend/src/lib/stores/router.svelte.ts index e6bc5cd24..3f9cafe14 100644 --- a/frontend/src/lib/stores/router.svelte.ts +++ b/frontend/src/lib/stores/router.svelte.ts @@ -7,6 +7,7 @@ export type Route = | "pinned" | "trash" | "recent-edits" + | "peers" | "settings"; const VALID_ROUTES: ReadonlySet = new Set([ @@ -18,6 +19,7 @@ const VALID_ROUTES: ReadonlySet = new Set([ "pinned", "trash", "recent-edits", + "peers", "settings", ]); diff --git a/frontend/src/lib/stores/router.test.ts b/frontend/src/lib/stores/router.test.ts index f70881788..813c0c024 100644 --- a/frontend/src/lib/stores/router.test.ts +++ b/frontend/src/lib/stores/router.test.ts @@ -68,6 +68,7 @@ describe("parsePath", () => { "insights", "pinned", "trash", + "peers", "settings", ]) { setURL(`/${route}`); diff --git a/internal/artifact/compression_test.go b/internal/artifact/compression_test.go new file mode 100644 index 000000000..18e71cd8e --- /dev/null +++ b/internal/artifact/compression_test.go @@ -0,0 +1,775 @@ +package artifact + +import ( + "bytes" + "context" + "crypto/sha256" + "fmt" + "io" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/klauspost/compress/zstd" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.kenn.io/agentsview/internal/db" +) + +func TestReadCompressedRejectsDecodedOutputAboveKindLimit(t *testing.T) { + tests := []struct { + name string + extension string + total int64 + }{ + {name: "manifest", extension: manifestExtension, total: 16<<20 + 1}, + {name: "segment", extension: segmentExtension, total: 64<<20 + 1}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + compressed, _ := compressedArtifactWithPadding(t, []byte("{}\n"), tt.total) + path := filepath.Join(t.TempDir(), "artifact"+tt.extension) + require.NoError(t, os.WriteFile(path, compressed, 0o644)) + + _, err := readCompressed(path) + require.Error(t, err) + assert.ErrorIs(t, err, errCorruptArtifact) + assert.Contains(t, err.Error(), "decoded output exceeds") + }) + } +} + +func TestReadCompressedLimitsTotalAcrossConcatenatedFrames(t *testing.T) { + first, _ := compressedArtifactWithPadding(t, []byte("{}\n"), 9<<20) + second, _ := compressedArtifactWithPadding(t, nil, 9<<20) + path := filepath.Join(t.TempDir(), "artifact"+manifestExtension) + require.NoError(t, os.WriteFile(path, append(first, second...), 0o644)) + + _, err := readCompressed(path) + require.Error(t, err) + assert.ErrorIs(t, err, errCorruptArtifact) + assert.Contains(t, err.Error(), "decoded output exceeds") +} + +func TestReadCompressedRejectsFrameAboveWriterWindow(t *testing.T) { + compressed, _ := compressedArtifactWithPadding( + t, []byte("{}\n"), 9<<20, + zstd.WithWindowSize(16<<20), + zstd.WithEncoderConcurrency(1), + ) + path := filepath.Join(t.TempDir(), "artifact"+segmentExtension) + require.NoError(t, os.WriteFile(path, compressed, 0o644)) + + _, err := readCompressed(path) + require.Error(t, err) + assert.ErrorIs(t, err, errCorruptArtifact) + assert.Contains(t, err.Error(), "window size exceeded") +} + +func TestWriteArtifactRejectsOversizedDecodedOutputWithoutWriting(t *testing.T) { + root := t.TempDir() + origin := "peer-a1b2c3" + segmentData, err := canonicalJSON(segmentMessage{ + Version: formatVersion, + Ordinal: 0, + Role: "user", + Content: "hello", + }) + require.NoError(t, err) + manifestData, err := canonicalJSON(manifest{ + Version: formatVersion, + Origin: origin, + NativeSessionID: "sess-1", + Session: manifestSession{ + ID: "sess-1", + Machine: origin, + }, + Segments: []string{strings64("a")}, + }) + require.NoError(t, err) + + tests := []struct { + name string + kind string + extension string + prefix []byte + total int64 + }{ + { + name: "manifest", kind: KindManifests, extension: manifestExtension, + prefix: manifestData, total: 16<<20 + 1, + }, + { + name: "segment", kind: KindSegments, extension: segmentExtension, + prefix: segmentData, total: 64<<20 + 1, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + compressed, hash := compressedArtifactWithPadding(t, tt.prefix, tt.total) + + _, err := WriteArtifact(root, origin, tt.kind, hash, compressed) + require.Error(t, err) + assert.ErrorIs(t, err, ErrArtifactInvalid) + assert.Contains(t, err.Error(), "decoded output exceeds") + assert.NoFileExists(t, filepath.Join(root, origin, tt.kind, hash+tt.extension)) + }) + } +} + +func TestWriteArtifactRejectsFrameAboveWriterWindowWithoutWriting(t *testing.T) { + root := t.TempDir() + origin := "peer-a1b2c3" + prefix, err := canonicalJSON(segmentMessage{ + Version: formatVersion, + Ordinal: 0, + Role: "user", + Content: "hello", + }) + require.NoError(t, err) + compressed, hash := compressedArtifactWithPadding( + t, prefix, 9<<20, + zstd.WithWindowSize(16<<20), + zstd.WithEncoderConcurrency(1), + ) + + _, err = WriteArtifact(root, origin, KindSegments, hash, compressed) + require.Error(t, err) + assert.ErrorIs(t, err, ErrArtifactInvalid) + assert.Contains(t, err.Error(), "window size exceeded") + assert.NoFileExists(t, filepath.Join(root, origin, KindSegments, hash+segmentExtension)) +} + +func TestWriteArtifactRejectsManifestStructuralAmplification(t *testing.T) { + origin := "peer-a1b2c3" + validHash := strings64("a") + tests := []struct { + name string + manifest manifest + wantError string + }{ + { + name: "duplicate segment references", + manifest: manifest{ + Segments: []string{validHash, validHash}, + }, + wantError: "duplicate segment reference", + }, + { + name: "too many segment references", + manifest: manifest{ + Segments: syntheticSegmentHashes(17), + }, + wantError: "segment reference limit", + }, + { + name: "too many usage events", + manifest: manifest{ + Segments: []string{validHash}, + UsageEvents: make([]artifactUsageEvent, 32_769), + }, + wantError: "usage event limit", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + root := t.TempDir() + m := tt.manifest + m.Version = formatVersion + m.Origin = origin + m.NativeSessionID = "sess-1" + m.Session = manifestSession{ID: "sess-1", Machine: origin} + data, err := canonicalJSON(m) + require.NoError(t, err) + hash := hashHex(data) + + _, err = WriteArtifact( + root, origin, KindManifests, hash, compressPeerTestData(t, data), + ) + require.Error(t, err) + assert.ErrorIs(t, err, ErrArtifactInvalid) + assert.Contains(t, err.Error(), tt.wantError) + assert.NoFileExists(t, filepath.Join( + root, origin, KindManifests, hash+manifestExtension, + )) + }) + } +} + +func TestWriteArtifactRejectsSegmentRecordAmplification(t *testing.T) { + root := t.TempDir() + origin := "peer-a1b2c3" + data := syntheticSegmentRecords(t, 4_097) + hash := hashHex(data) + + _, err := WriteArtifact( + root, origin, KindSegments, hash, compressPeerTestData(t, data), + ) + require.Error(t, err) + assert.ErrorIs(t, err, ErrArtifactInvalid) + assert.Contains(t, err.Error(), "message record limit") + assert.NoFileExists(t, filepath.Join( + root, origin, KindSegments, hash+segmentExtension, + )) +} + +func TestWriteArtifactRejectsSegmentNestedAmplificationWithoutWriting(t *testing.T) { + origin := "peer-a1b2c3" + tests := []struct { + name string + record segmentMessage + wantError string + }{ + { + name: "too many tool calls in one message", + record: segmentMessage{ + ToolCalls: make([]segmentToolCall, 257), + }, + wantError: "tool call limit", + }, + { + name: "too many result events in one tool call", + record: segmentMessage{ + ToolCalls: []segmentToolCall{{ + ResultEvents: make([]segmentResultEvent, 1_025), + }}, + }, + wantError: "result event limit", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + root := t.TempDir() + record := tt.record + record.Version = formatVersion + record.Ordinal = 7 + record.Role = "assistant" + data, err := canonicalJSON(record) + require.NoError(t, err) + hash := hashHex(data) + + _, err = WriteArtifact( + root, origin, KindSegments, hash, compressPeerTestData(t, data), + ) + require.Error(t, err) + assert.ErrorIs(t, err, ErrArtifactInvalid) + assert.Contains(t, err.Error(), tt.wantError) + assert.NoFileExists(t, filepath.Join( + root, origin, KindSegments, hash+segmentExtension, + )) + }) + } +} + +func TestWriteArtifactRejectsBlankSegmentRecordsWithoutWriting(t *testing.T) { + root := t.TempDir() + origin := "peer-a1b2c3" + data := bytes.Repeat([]byte("\n"), 4_097) + hash := hashHex(data) + + _, err := WriteArtifact( + root, origin, KindSegments, hash, compressPeerTestData(t, data), + ) + require.Error(t, err) + assert.ErrorIs(t, err, ErrArtifactInvalid) + assert.Contains(t, err.Error(), "blank message record") + assert.NoFileExists(t, filepath.Join( + root, origin, KindSegments, hash+segmentExtension, + )) +} + +func TestReadManifestRejectsDuplicateSegmentReferences(t *testing.T) { + originRoot := t.TempDir() + hash := strings64("a") + data, err := canonicalJSON(manifest{ + Version: formatVersion, + Origin: "laptop-a1b2c3", + NativeSessionID: "sess-1", + Session: manifestSession{ + ID: "sess-1", + Machine: "laptop-a1b2c3", + }, + Segments: []string{hash, hash}, + }) + require.NoError(t, err) + manifestHash := hashHex(data) + path := filepath.Join(originRoot, KindManifests, manifestHash+manifestExtension) + require.NoError(t, writeCompressed(path, data)) + + _, err = readManifest(originRoot, manifestHash) + require.Error(t, err) + assert.ErrorIs(t, err, errCorruptArtifact) + assert.Contains(t, err.Error(), "duplicate segment reference") + assert.NoFileExists(t, path) + assert.FileExists(t, path+quarantineSuffix) +} + +func TestReadSegmentRejectsRecordAmplification(t *testing.T) { + originRoot := t.TempDir() + data := syntheticSegmentRecords(t, 4_097) + hash := hashHex(data) + path := filepath.Join(originRoot, KindSegments, hash+segmentExtension) + require.NoError(t, writeCompressed(path, data)) + + _, err := readSegmentMessages(originRoot, hash) + require.Error(t, err) + assert.ErrorIs(t, err, errCorruptArtifact) + assert.Contains(t, err.Error(), "message record limit") + assert.NoFileExists(t, path) + assert.FileExists(t, path+quarantineSuffix) +} + +func TestReadSegmentQuarantinesNestedAmplification(t *testing.T) { + tests := []struct { + name string + record segmentMessage + wantError string + }{ + { + name: "too many tool calls in one message", + record: segmentMessage{ + ToolCalls: make([]segmentToolCall, 257), + }, + wantError: "tool call limit", + }, + { + name: "too many result events in one tool call", + record: segmentMessage{ + ToolCalls: []segmentToolCall{{ + ResultEvents: make([]segmentResultEvent, 1_025), + }}, + }, + wantError: "result event limit", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + originRoot := t.TempDir() + record := tt.record + record.Version = formatVersion + record.Ordinal = 7 + record.Role = "assistant" + data, err := canonicalJSON(record) + require.NoError(t, err) + hash := hashHex(data) + path := filepath.Join(originRoot, KindSegments, hash+segmentExtension) + require.NoError(t, writeCompressed(path, data)) + + _, err = readSegmentMessages(originRoot, hash) + require.Error(t, err) + assert.ErrorIs(t, err, errCorruptArtifact) + assert.Contains(t, err.Error(), tt.wantError) + assert.NoFileExists(t, path) + assert.FileExists(t, path+quarantineSuffix) + }) + } +} + +func TestReadSegmentQuarantinesBlankRecords(t *testing.T) { + originRoot := t.TempDir() + data := bytes.Repeat([]byte("\n"), 4_097) + hash := hashHex(data) + path := filepath.Join(originRoot, KindSegments, hash+segmentExtension) + require.NoError(t, writeCompressed(path, data)) + + _, err := readSegmentMessages(originRoot, hash) + require.Error(t, err) + assert.ErrorIs(t, err, errCorruptArtifact) + assert.Contains(t, err.Error(), "blank message record") + assert.NoFileExists(t, path) + assert.FileExists(t, path+quarantineSuffix) +} + +func TestReadManifestMessagesRejectsSessionMessageAmplification(t *testing.T) { + originRoot := t.TempDir() + m := manifest{Segments: make([]string, 0, 9)} + ordinal := 0 + for segment := range 9 { + count := 4_096 + if segment == 8 { + count = 1 + } + data := syntheticSegmentRecordsFrom(t, ordinal, count) + ordinal += count + hash := hashHex(data) + require.NoError(t, writeCompressed( + filepath.Join(originRoot, KindSegments, hash+segmentExtension), data, + )) + m.Segments = append(m.Segments, hash) + } + + _, err := readManifestMessages(originRoot, m) + require.Error(t, err) + assert.ErrorIs(t, err, errCorruptArtifact) + assert.Contains(t, err.Error(), "session message limit") +} + +func TestReadManifestMessagesRejectsSessionDecodedByteBudgetWithSmallLimit(t *testing.T) { + originRoot := t.TempDir() + m := manifest{} + totalBytes := 0 + for ordinal := range 2 { + data := syntheticSegmentRecordsFrom(t, ordinal, 1) + totalBytes += len(data) + hash := hashHex(data) + require.NoError(t, writeCompressed( + filepath.Join(originRoot, KindSegments, hash+segmentExtension), data, + )) + m.Segments = append(m.Segments, hash) + } + limits := productionArtifactLimits() + limits.sessionDecodedBytes = int64(totalBytes - 1) + + _, err := readManifestMessagesWithLimits(originRoot, m, limits) + require.Error(t, err) + assert.ErrorIs(t, err, errCorruptArtifact) + assert.Contains(t, err.Error(), "session decoded byte limit") +} + +func TestExportChunksOnMessageRecordLimit(t *testing.T) { + ctx := context.Background() + database := testDB(t) + root := t.TempDir() + origin := "laptop-a1b2c3" + seedSession(t, database, "sess-1", "alpha") + msgs := make([]db.Message, 4_097) + for i := range msgs { + msgs[i] = db.Message{SessionID: "sess-1", Ordinal: i, Role: "user"} + } + require.NoError(t, database.ReplaceSessionMessages("sess-1", msgs)) + + _, err := Export(ctx, database, root, origin) + require.NoError(t, err) + cp, err := readLatestCheckpoint(filepath.Join(root, origin)) + require.NoError(t, err) + require.NotNil(t, cp) + m, err := readManifest(filepath.Join(root, origin), cp.Sessions[origin+"~sess-1"]) + require.NoError(t, err) + require.Len(t, m.Segments, 2) + got, err := readManifestMessages(filepath.Join(root, origin), m) + require.NoError(t, err) + assert.Len(t, got, 4_097) +} + +func TestExportRejectsOversizedGeneratedManifestBeforePublication(t *testing.T) { + ctx := context.Background() + database := testDB(t) + root := t.TempDir() + origin := "laptop-a1b2c3" + seedSession(t, database, "sess-1", "alpha", func(sess *db.Session) { + first := strings.Repeat("x", int(manifestDecodedLimit)) + sess.FirstMessage = &first + }) + + _, err := Export(ctx, database, root, origin) + require.Error(t, err) + assert.Contains(t, err.Error(), "generated manifest exceeds") + assertNoPublishedArtifactFiles(t, root, origin) + state, err := database.GetSyncState(exportStateKey(origin, "sess-1")) + require.NoError(t, err) + assert.Empty(t, state) +} + +func TestExportRejectsSessionMessageAmplificationBeforePublication(t *testing.T) { + ctx := context.Background() + database := testDB(t) + root := t.TempDir() + origin := "laptop-a1b2c3" + seedSession(t, database, "sess-1", "alpha") + msgs := make([]db.Message, 32_769) + for i := range msgs { + msgs[i] = db.Message{SessionID: "sess-1", Ordinal: i, Role: "user"} + } + require.NoError(t, database.ReplaceSessionMessages("sess-1", msgs)) + + _, err := Export(ctx, database, root, origin) + require.Error(t, err) + assert.Contains(t, err.Error(), "session message limit") + assertNoPublishedArtifactFiles(t, root, origin) + state, err := database.GetSyncState(exportStateKey(origin, "sess-1")) + require.NoError(t, err) + assert.Empty(t, state) +} + +func TestExportRejectsUsageEventAmplificationBeforePublication(t *testing.T) { + ctx := context.Background() + database := testDB(t) + root := t.TempDir() + origin := "laptop-a1b2c3" + seedSession(t, database, "sess-1", "alpha") + events := make([]db.UsageEvent, 32_769) + for i := range events { + events[i] = db.UsageEvent{ + SessionID: "sess-1", + Source: "fixture", + DedupKey: fmt.Sprintf("usage-%d", i), + } + } + require.NoError(t, database.ReplaceSessionUsageEvents("sess-1", events)) + + _, err := Export(ctx, database, root, origin) + require.Error(t, err) + assert.Contains(t, err.Error(), "usage event limit") + assertNoPublishedArtifactFiles(t, root, origin) + state, err := database.GetSyncState(exportStateKey(origin, "sess-1")) + require.NoError(t, err) + assert.Empty(t, state) +} + +func TestExportSessionRejectsAggregateLimitsBeforeWritingWithSmallLimits(t *testing.T) { + tests := []struct { + name string + configure func(*artifactLimits) + wantError string + }{ + { + name: "decoded bytes", + configure: func(limits *artifactLimits) { + limits.sessionDecodedBytes = 1 + }, + wantError: "session decoded byte limit", + }, + { + name: "segment references", + configure: func(limits *artifactLimits) { + limits.segmentMessages = 1 + limits.manifestSegments = 1 + }, + wantError: "segment reference limit", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctx := context.Background() + database := testDB(t) + root := t.TempDir() + origin := "laptop-a1b2c3" + seedSession(t, database, "sess-1", "alpha") + limits := productionArtifactLimits() + tt.configure(&limits) + + _, _, err := exportSessionWithLimits( + ctx, database, filepath.Join(root, origin), origin, "sess-1", "", limits, + ) + require.Error(t, err) + assert.Contains(t, err.Error(), tt.wantError) + assert.Empty(t, globArtifacts( + t, root, origin, KindSegments, "*"+segmentExtension, + )) + assert.Empty(t, globArtifacts( + t, root, origin, KindManifests, "*"+manifestExtension, + )) + }) + } +} + +func TestImportQuarantinesOversizedSegmentWithoutAdvancingState(t *testing.T) { + ctx := context.Background() + root := t.TempDir() + origin := "laptop-a1b2c3" + localOrigin := "desktop-d4e5f6" + importDB := testDB(t) + originRoot := filepath.Join(root, origin) + + prefix, err := canonicalJSON(segmentMessage{ + Version: formatVersion, + Ordinal: 0, + Role: "user", + Content: "hello", + ContentLength: 5, + }) + require.NoError(t, err) + compressed, segmentHash := compressedArtifactWithPadding(t, prefix, 64<<20+1) + segmentPath := filepath.Join(originRoot, KindSegments, segmentHash+segmentExtension) + require.NoError(t, os.MkdirAll(filepath.Dir(segmentPath), 0o755)) + require.NoError(t, os.WriteFile(segmentPath, compressed, 0o644)) + + gid := origin + "~sess-1" + m := manifest{ + Version: formatVersion, + Origin: origin, + NativeSessionID: "sess-1", + Session: manifestSession{ + ID: "sess-1", + Machine: origin, + }, + Segments: []string{segmentHash}, + } + manifestData, err := canonicalJSON(m) + require.NoError(t, err) + manifestHash := hashHex(manifestData) + require.NoError(t, writeCompressed( + filepath.Join(originRoot, KindManifests, manifestHash+manifestExtension), + manifestData, + )) + writeCheckpoint(t, originRoot, checkpoint{ + Version: formatVersion, + Origin: origin, + Sequence: 1, + Sessions: map[string]string{gid: manifestHash}, + }) + + res, err := ImportDetailed(ctx, importDB, root, localOrigin) + require.NoError(t, err) + assert.False(t, res.Changed()) + state, err := importDB.GetSyncState(importStateKey(origin, gid)) + require.NoError(t, err) + assert.Empty(t, state) + got, err := importDB.GetSessionFull(ctx, gid) + require.NoError(t, err) + assert.Nil(t, got) + assert.NoFileExists(t, segmentPath) + assert.FileExists(t, segmentPath+quarantineSuffix) +} + +func TestExportChunksLargeMultiMessageSessionInOrder(t *testing.T) { + ctx := context.Background() + database := testDB(t) + root := t.TempDir() + origin := "laptop-a1b2c3" + seedSession(t, database, "sess-1", "alpha") + content := strings.Repeat("x", 9<<20) + msgs := make([]db.Message, 4) + for i := range msgs { + msgs[i] = db.Message{ + SessionID: "sess-1", + Ordinal: i, + Role: "user", + Content: content, + ContentLength: len(content), + } + } + require.NoError(t, database.ReplaceSessionMessages("sess-1", msgs)) + + count, err := Export(ctx, database, root, origin) + require.NoError(t, err) + assert.Equal(t, 1, count) + cp, err := readLatestCheckpoint(filepath.Join(root, origin)) + require.NoError(t, err) + require.NotNil(t, cp) + m, err := readManifest(filepath.Join(root, origin), cp.Sessions[origin+"~sess-1"]) + require.NoError(t, err) + require.Len(t, m.Segments, 2) + + got, err := readManifestMessages(filepath.Join(root, origin), m) + require.NoError(t, err) + require.Len(t, got, 4) + for i := range got { + assert.Equal(t, i, got[i].Ordinal) + assert.Equal(t, content, got[i].Content) + } +} + +func TestExportRejectsSingleEncodedRecordAboveReadableLimit(t *testing.T) { + ctx := context.Background() + database := testDB(t) + root := t.TempDir() + origin := "laptop-a1b2c3" + seedSession(t, database, "sess-1", "alpha") + content := strings.Repeat("x", 64<<20) + require.NoError(t, database.ReplaceSessionMessages("sess-1", []db.Message{{ + SessionID: "sess-1", + Ordinal: 0, + Role: "user", + Content: content, + ContentLength: len(content), + }})) + + _, err := Export(ctx, database, root, origin) + require.Error(t, err) + assert.Contains(t, err.Error(), "encoded message record") + assert.Contains(t, err.Error(), "67108864-byte readable limit") + assert.Empty(t, globArtifacts(t, root, origin, KindCheckpoints, "cp-*.json")) +} + +func TestExportPreservesSmallSingleSegmentHash(t *testing.T) { + ctx := context.Background() + database := testDB(t) + root := t.TempDir() + origin := "laptop-a1b2c3" + seedSession(t, database, "sess-1", "alpha") + msgs, err := database.GetAllMessages(ctx, "sess-1") + require.NoError(t, err) + segmentData, err := encodeSegment(canonicalMessages(msgs)) + require.NoError(t, err) + wantHash := hashHex(segmentData) + + _, err = Export(ctx, database, root, origin) + require.NoError(t, err) + cp, err := readLatestCheckpoint(filepath.Join(root, origin)) + require.NoError(t, err) + require.NotNil(t, cp) + m, err := readManifest(filepath.Join(root, origin), cp.Sessions[origin+"~sess-1"]) + require.NoError(t, err) + assert.Equal(t, []string{wantHash}, m.Segments) +} + +type repeatedByteReader byte + +func (r repeatedByteReader) Read(p []byte) (int, error) { + for i := range p { + p[i] = byte(r) + } + return len(p), nil +} + +func compressedArtifactWithPadding( + t *testing.T, + prefix []byte, + total int64, + opts ...zstd.EOption, +) ([]byte, string) { + t.Helper() + require.LessOrEqual(t, int64(len(prefix)), total) + var compressed bytes.Buffer + enc, err := zstd.NewWriter(&compressed, opts...) + require.NoError(t, err) + h := sha256.New() + w := io.MultiWriter(enc, h) + _, err = w.Write(prefix) + require.NoError(t, err) + _, err = io.CopyN(w, repeatedByteReader('\n'), total-int64(len(prefix))) + require.NoError(t, err) + require.NoError(t, enc.Close()) + return compressed.Bytes(), fmt.Sprintf("%x", h.Sum(nil)) +} + +func syntheticSegmentHashes(count int) []string { + hashes := make([]string, count) + for i := range hashes { + hashes[i] = fmt.Sprintf("%064x", i+1) + } + return hashes +} + +func syntheticSegmentRecords(t *testing.T, count int) []byte { + return syntheticSegmentRecordsFrom(t, 0, count) +} + +func syntheticSegmentRecordsFrom(t *testing.T, start, count int) []byte { + t.Helper() + var data bytes.Buffer + for i := range count { + record, err := canonicalJSON(segmentMessage{ + Version: formatVersion, + Ordinal: start + i, + Role: "user", + }) + require.NoError(t, err) + _, err = data.Write(record) + require.NoError(t, err) + } + return data.Bytes() +} + +func assertNoPublishedArtifactFiles(t *testing.T, root, origin string) { + t.Helper() + for _, kind := range []string{ + KindCheckpoints, KindManifests, KindSegments, KindMeta, KindRaw, + } { + entries, err := os.ReadDir(filepath.Join(root, origin, kind)) + require.NoError(t, err) + assert.Empty(t, entries, "unexpected published %s artifact", kind) + } +} diff --git a/internal/artifact/export_usage_test.go b/internal/artifact/export_usage_test.go new file mode 100644 index 000000000..73876614d --- /dev/null +++ b/internal/artifact/export_usage_test.go @@ -0,0 +1,92 @@ +package artifact + +import ( + "context" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.kenn.io/agentsview/internal/db" +) + +func TestExportIncludesUsageOnlySession(t *testing.T) { + ctx := context.Background() + root := t.TempDir() + origin := "laptop-a1b2c3" + database := testDB(t) + + // A usage-only session: owned, zero messages, with a usage event. The + // sidebar list filter (message_count > 0) hides it, so export must not rely + // on that path. + require.NoError(t, database.UpsertSession(db.Session{ + ID: "usage-1", + Project: "alpha", + Machine: "local", + Agent: "claude", + CreatedAt: "2026-06-14T01:02:03Z", + })) + require.NoError(t, database.ReplaceSessionUsageEvents("usage-1", []db.UsageEvent{{ + SessionID: "usage-1", + Source: "assistant", + Model: "claude-opus-4-8", + InputTokens: 10, + OutputTokens: 20, + OccurredAt: "2026-06-14T01:02:30Z", + DedupKey: "usage-1:0", + }})) + + exported, err := Export(ctx, database, root, origin) + require.NoError(t, err) + assert.Equal(t, 1, exported) + + gid := origin + "~usage-1" + cp, err := readLatestCheckpoint(filepath.Join(root, origin)) + require.NoError(t, err) + require.NotNil(t, cp) + assert.Contains(t, cp.Sessions, gid, "usage-only session must be in the checkpoint") + + // It imports into a peer as a real session with its usage events intact. + importDB := testDB(t) + res, err := ImportDetailed(ctx, importDB, root, "desktop-d4e5f6") + require.NoError(t, err) + assert.Equal(t, 1, res.Sessions) + + got, err := importDB.GetSessionFull(ctx, gid) + require.NoError(t, err) + require.NotNil(t, got) + events, err := importDB.GetUsageEvents(ctx, gid) + require.NoError(t, err) + require.Len(t, events, 1) + assert.Equal(t, 10, events[0].InputTokens) + assert.Equal(t, 20, events[0].OutputTokens) +} + +func TestExportSkipsDeletedAndForeignSessions(t *testing.T) { + ctx := context.Background() + root := t.TempDir() + origin := "laptop-a1b2c3" + database := testDB(t) + + seedSession(t, database, "owned", "alpha") + // A soft-deleted owned session and a foreign-owned session are both excluded. + seedSession(t, database, "trashed", "alpha") + require.NoError(t, database.SoftDeleteSession("trashed")) + require.NoError(t, database.UpsertSession(db.Session{ + ID: "foreign", + Project: "alpha", + Machine: "desktop-d4e5f6", + Agent: "claude", + CreatedAt: "2026-06-14T01:02:03Z", + })) + + _, err := Export(ctx, database, root, origin) + require.NoError(t, err) + + cp, err := readLatestCheckpoint(filepath.Join(root, origin)) + require.NoError(t, err) + require.NotNil(t, cp) + assert.Contains(t, cp.Sessions, origin+"~owned") + assert.NotContains(t, cp.Sessions, origin+"~trashed") + assert.NotContains(t, cp.Sessions, origin+"~foreign") +} diff --git a/internal/artifact/format_test.go b/internal/artifact/format_test.go new file mode 100644 index 000000000..7c6b95d80 --- /dev/null +++ b/internal/artifact/format_test.go @@ -0,0 +1,212 @@ +package artifact + +import ( + "encoding/json" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.kenn.io/agentsview/internal/db" +) + +func TestCanonicalCheckpointGolden(t *testing.T) { + cp := checkpoint{ + Version: formatVersion, + Origin: "laptop-a1b2c3", + Sequence: 7, + Sessions: map[string]string{ + "laptop-a1b2c3~sess-b": "b222", + "laptop-a1b2c3~sess-a": "a111", + }, + } + + data, err := canonicalJSON(cp) + require.NoError(t, err) + + assert.Equal(t, + "{\"origin\":\"laptop-a1b2c3\",\"seq\":7,\"sessions\":{\"laptop-a1b2c3~sess-a\":\"a111\",\"laptop-a1b2c3~sess-b\":\"b222\"},\"v\":1}\n", + string(data), + ) + assert.Equal(t, "56fd64d35ebd700bfa2a50d41a97857b871e033e5c1e1d02dea55c25c7df7655", hashHex(data)) +} + +func TestCanonicalManifestGolden(t *testing.T) { + cost := 0.03125 + ordinal := 2 + parent := "parent-1" + name := "Fixture" + raw := rawSourceRef{ + Hash: "raw123", + Size: 4096, + MediaType: "application/jsonl", + Path: "claude/session.jsonl", + } + m := manifest{ + Version: formatVersion, + Origin: "laptop-a1b2c3", + NativeSessionID: "sess-1", + Session: manifestSession{ + ID: "sess-1", + Project: "alpha", + Machine: "laptop-a1b2c3", + Agent: "claude", + FirstMessage: new("hello"), + StartedAt: new("2026-06-14T01:02:03Z"), + EndedAt: new("2026-06-14T01:03:03Z"), + MessageCount: 2, + UserMessageCount: 1, + ParentSessionID: &parent, + RelationshipType: "subagent", + TotalOutputTokens: 42, + CreatedAt: "2026-06-14T01:02:03Z", + }, + SessionName: &name, + Segments: []string{"seg222", "seg111"}, + UsageEvents: []artifactUsageEvent{ + { + MessageOrdinal: &ordinal, + Source: "fixture", + Model: "claude-test", + InputTokens: 11, + OutputTokens: 7, + CostUSD: &cost, + CostStatus: "known", + CostSource: "fixture", + OccurredAt: "2026-06-14T01:02:04Z", + DedupKey: "usage-1", + }, + }, + RawSource: &raw, + DataVersion: 99, + Generation: 3, + SessionHasToolCalls: true, + SessionHasContextData: true, + SessionQualitySignals: &manifestQualitySignals{ + Version: 3, + ShortPromptCount: 2, + UnstructuredStart: true, + MissingSuccessCriteriaCount: 4, + MissingVerificationCount: 5, + DuplicatePromptCount: 6, + NoCodeContextCount: 7, + RunawayToolLoopCount: 1, + }, + } + + data, err := canonicalJSON(m) + require.NoError(t, err) + + assert.Equal(t, + "{\"data_version\":99,\"generation\":3,\"native_session_id\":\"sess-1\",\"origin\":\"laptop-a1b2c3\",\"raw_source\":{\"hash\":\"raw123\",\"media_type\":\"application/jsonl\",\"path\":\"claude/session.jsonl\",\"size\":4096},\"segments\":[\"seg222\",\"seg111\"],\"session\":{\"agent\":\"claude\",\"compaction_count\":0,\"consecutive_failure_max\":0,\"created_at\":\"2026-06-14T01:02:03Z\",\"edit_churn_count\":0,\"ended_at\":\"2026-06-14T01:03:03Z\",\"ended_with_role\":\"\",\"final_failure_streak\":0,\"first_message\":\"hello\",\"has_peak_context_tokens\":false,\"has_total_output_tokens\":false,\"id\":\"sess-1\",\"is_automated\":false,\"machine\":\"laptop-a1b2c3\",\"message_count\":2,\"mid_task_compaction_count\":0,\"outcome\":\"\",\"outcome_confidence\":\"\",\"parent_session_id\":\"parent-1\",\"peak_context_tokens\":0,\"project\":\"alpha\",\"relationship_type\":\"subagent\",\"secret_leak_count\":0,\"started_at\":\"2026-06-14T01:02:03Z\",\"tool_failure_signal_count\":0,\"tool_retry_count\":0,\"total_output_tokens\":42,\"user_message_count\":1},\"session_has_context_data\":true,\"session_has_tool_calls\":true,\"session_name\":\"Fixture\",\"session_quality_signals\":{\"duplicate_prompt_count\":6,\"missing_success_criteria_count\":4,\"missing_verification_count\":5,\"no_code_context_count\":7,\"runaway_tool_loop_count\":1,\"short_prompt_count\":2,\"unstructured_start\":true,\"version\":3},\"usage_events\":[{\"cost_source\":\"fixture\",\"cost_status\":\"known\",\"cost_usd\":0.03125,\"dedup_key\":\"usage-1\",\"input_tokens\":11,\"message_ordinal\":2,\"model\":\"claude-test\",\"occurred_at\":\"2026-06-14T01:02:04Z\",\"output_tokens\":7,\"source\":\"fixture\"}],\"v\":1}\n", + string(data), + ) + assert.Equal(t, "1a563d1b1642cf850bb2253643d8cb628a91499cbf48607a6c40b15af01b4a6f", hashHex(data)) +} + +func TestCanonicalMessageSegmentGolden(t *testing.T) { + msgs := []db.Message{ + { + ID: 99, + SessionID: "sess-1", + Ordinal: 2, + Role: "assistant", + Content: "world", + ContentLength: 5, + Timestamp: "2026-06-14T01:02:05Z", + HasToolUse: true, + Model: "claude-test", + TokenUsage: json.RawMessage(`{"output":2,"input":1}`), + OutputTokens: 2, + HasOutputTokens: true, + ClaudeMessageID: "msg-1", + ClaudeRequestID: "req-1", + SourceType: "jsonl", + SourceSubtype: "assistant", + SourceUUID: "uuid-msg-1", + SourceParentUUID: "uuid-parent", + ToolCalls: []db.ToolCall{ + { + MessageID: 99, + SessionID: "sess-1", + ToolName: "Read", + Category: "file", + ToolUseID: "tool-1", + InputJSON: "{\"file_path\":\"README.md\"}", + FilePath: "README.md", + ResultContentLength: 12, + ResultContent: "file content", + SubagentSessionID: "child-1", + ResultEvents: []db.ToolResultEvent{ + { + ToolUseID: "tool-1", + AgentID: "agent-1", + SubagentSessionID: "child-1", + Source: "tool_result", + Status: "success", + Content: "done", + ContentLength: 4, + Timestamp: "2026-06-14T01:02:06Z", + EventIndex: 0, + }, + }, + }, + }, + }, + } + + data, err := encodeSegment(msgs) + require.NoError(t, err) + + assert.Equal(t, + "{\"claude_message_id\":\"msg-1\",\"claude_request_id\":\"req-1\",\"content\":\"world\",\"content_length\":5,\"has_output_tokens\":true,\"has_tool_use\":true,\"model\":\"claude-test\",\"ordinal\":2,\"output_tokens\":2,\"role\":\"assistant\",\"source_parent_uuid\":\"uuid-parent\",\"source_subtype\":\"assistant\",\"source_type\":\"jsonl\",\"source_uuid\":\"uuid-msg-1\",\"timestamp\":\"2026-06-14T01:02:05Z\",\"token_usage\":{\"input\":1,\"output\":2},\"tool_calls\":[{\"call_index\":0,\"category\":\"file\",\"file_path\":\"README.md\",\"input_json\":\"{\\\"file_path\\\":\\\"README.md\\\"}\",\"result_content\":\"file content\",\"result_content_length\":12,\"result_events\":[{\"agent_id\":\"agent-1\",\"content\":\"done\",\"content_length\":4,\"event_index\":0,\"source\":\"tool_result\",\"status\":\"success\",\"subagent_session_id\":\"child-1\",\"timestamp\":\"2026-06-14T01:02:06Z\",\"tool_use_id\":\"tool-1\"}],\"subagent_session_id\":\"child-1\",\"tool_name\":\"Read\",\"tool_use_id\":\"tool-1\"}],\"v\":1}\n", + string(data), + ) + assert.NotContains(t, string(data), `"id"`) + assert.NotContains(t, string(data), `"session_id"`) + assert.NotContains(t, string(data), `"message_id"`) + assert.Equal(t, "f46c1edbc77dab4eb15f43bcb3ce196243c784445b07e9135c223b5d58c6dea5", hashHex(data)) +} + +func TestCanonicalMetadataEventGolden(t *testing.T) { + value := json.RawMessage(`{"display_name":"Renamed session"}`) + note := "remember this" + event := metadataEvent{ + Version: formatVersion, + HLC: "2026-06-14T010203.000000001Z-laptop-a1b2c3", + Origin: "laptop-a1b2c3", + SessionGID: "desktop-d4e5f6~sess-1", + Op: "rename", + Value: value, + Pin: &MetadataPin{ + SourceUUID: "uuid-msg-1", + Ordinal: 2, + Note: ¬e, + }, + } + + data, err := canonicalJSON(event) + require.NoError(t, err) + + assert.Equal(t, + "{\"hlc\":\"2026-06-14T010203.000000001Z-laptop-a1b2c3\",\"op\":\"rename\",\"origin\":\"laptop-a1b2c3\",\"pin\":{\"note\":\"remember this\",\"ordinal\":2,\"source_uuid\":\"uuid-msg-1\"},\"session_gid\":\"desktop-d4e5f6~sess-1\",\"v\":1,\"value\":{\"display_name\":\"Renamed session\"}}\n", + string(data), + ) + assert.Equal(t, "fcb36d602e56fe1616ba6e2f86e973adde4ef547e0ecf280b37eb534b60e4b71", hashHex(data)) +} + +func TestCompressedArtifactsUseUncompressedContentHash(t *testing.T) { + data := []byte("{\"v\":1}\n") + hash := hashHex(data) + path := filepath.Join(t.TempDir(), hash+manifestExtension) + + require.NoError(t, writeCompressed(path, data)) + read, err := readCompressed(path) + require.NoError(t, err) + + assert.Equal(t, data, read) + assert.Equal(t, "2b4248702881de2f5638efe96b233de1c0dd9be5dd24ec35ad030d6b06aede9a", hash) + _, err = os.Stat(path) + require.NoError(t, err) +} diff --git a/internal/artifact/gc.go b/internal/artifact/gc.go new file mode 100644 index 000000000..03b2626eb --- /dev/null +++ b/internal/artifact/gc.go @@ -0,0 +1,583 @@ +package artifact + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "io/fs" + "os" + "path/filepath" + "sort" + "time" +) + +// GCOptions configures conservative artifact garbage collection. +type GCOptions struct { + Root string + Grace time.Duration + DryRun bool + Now time.Time + Logf func(string, ...any) +} + +// GCResult summarizes one artifact garbage collection scan. +type GCResult struct { + DryRun bool + Origins int + SkippedOrigins int + Scanned int + Candidates int + Eligible int + KeptByGrace int + Deleted int + BytesEligible int64 + BytesDeleted int64 +} + +type gcRef struct { + kind string + name string +} + +type gcCandidate struct { + path string + origin string + kind string + name string + size int64 + modTime time.Time +} + +type gcOriginResult struct { + skipped bool + scanned int + candidates []gcCandidate + kindRoots map[string]*os.Root +} + +// GarbageCollect deletes or reports superseded artifacts that are no longer +// reachable from an origin's latest checkpoint and its live manifests. +func GarbageCollect(ctx context.Context, opts GCOptions) (GCResult, error) { + if opts.Root == "" { + return GCResult{}, fmt.Errorf("artifact gc root is required") + } + if opts.Grace < 0 { + return GCResult{}, fmt.Errorf("artifact gc grace must be >= 0") + } + now := opts.Now + if now.IsZero() { + now = time.Now() + } + cutoff := now.Add(-opts.Grace) + + storeRoot, err := openArtifactRoot(opts.Root, "gc") + if err != nil { + return GCResult{}, fmt.Errorf("opening artifact gc root: %w", err) + } + defer storeRoot.Close() + origins, err := listGCOrigins(storeRoot) + if err != nil { + return GCResult{}, fmt.Errorf("listing artifact origins: %w", err) + } + res := GCResult{DryRun: opts.DryRun} + for _, origin := range origins { + if err := ctx.Err(); err != nil { + return res, err + } + originRoot, err := openArtifactSubroot(storeRoot, origin, "gc origin") + if err != nil { + return res, fmt.Errorf("opening artifact gc origin %s: %w", origin, err) + } + originRes, err := collectGCCandidates(ctx, originRoot, origin) + if err != nil { + _ = originRoot.Close() + if errors.Is(err, errIncompleteArtifact) || + errors.Is(err, errCorruptArtifact) || + errors.Is(err, errFutureArtifactVersion) { + res.Origins++ + res.SkippedOrigins++ + logGC(opts, "artifact gc: skipping %s: %v", origin, err) + continue + } + return res, fmt.Errorf("scanning %s: %w", origin, err) + } + res.Origins++ + if originRes.skipped { + closeGCKindRoots(originRes.kindRoots) + _ = originRoot.Close() + res.SkippedOrigins++ + logGC(opts, "artifact gc: skipping %s with no checkpoints", origin) + continue + } + res.Scanned += originRes.scanned + for _, cand := range originRes.candidates { + res.Candidates++ + if cand.modTime.After(cutoff) { + res.KeptByGrace++ + continue + } + res.Eligible++ + res.BytesEligible += cand.size + if opts.DryRun { + logGC(opts, "artifact gc: would delete %s (%d bytes)", cand.path, cand.size) + continue + } + if err := originRes.kindRoots[cand.kind].Remove(cand.name); err != nil { + if errors.Is(err, fs.ErrNotExist) { + continue + } + closeGCKindRoots(originRes.kindRoots) + _ = originRoot.Close() + return res, fmt.Errorf("deleting %s: %w", cand.path, err) + } + res.Deleted++ + res.BytesDeleted += cand.size + logGC(opts, "artifact gc: deleted %s (%d bytes)", cand.path, cand.size) + } + closeGCKindRoots(originRes.kindRoots) + if err := originRoot.Close(); err != nil { + return res, fmt.Errorf("closing artifact gc origin %s: %w", origin, err) + } + } + return res, nil +} + +func listGCOrigins(root *os.Root) ([]string, error) { + entries, err := fs.ReadDir(root.FS(), ".") + if err != nil { + return nil, err + } + origins := make([]string, 0, len(entries)) + for _, ent := range entries { + if validateOriginID(ent.Name()) != nil { + continue + } + info, err := root.Lstat(ent.Name()) + if err != nil { + return nil, err + } + if !info.IsDir() { + return nil, fmt.Errorf("artifact origin %s is not a directory", ent.Name()) + } + origins = append(origins, ent.Name()) + } + sort.Strings(origins) + return origins, nil +} + +func collectGCCandidates(ctx context.Context, root *os.Root, origin string) (gcOriginResult, error) { + kindRoots, err := openGCKindRoots(root) + if err != nil { + return gcOriginResult{}, err + } + keepRoots := false + defer func() { + if !keepRoots { + closeGCKindRoots(kindRoots) + } + }() + checkpoints, err := validGCCheckpointNames(kindRoots[KindCheckpoints]) + if err != nil { + return gcOriginResult{}, err + } + if len(checkpoints) == 0 { + return gcOriginResult{skipped: true}, nil + } + + latestName := checkpoints[len(checkpoints)-1] + data, err := kindRoots[KindCheckpoints].ReadFile(latestName) + if err != nil { + return gcOriginResult{}, fmt.Errorf("reading live checkpoint: %w", err) + } + var cp checkpoint + if err := json.Unmarshal(data, &cp); err != nil { + return gcOriginResult{}, fmt.Errorf("%w: decoding live checkpoint: %v", errCorruptArtifact, err) + } + if err := validateCheckpoint(&cp, origin); err != nil { + if !errors.Is(err, errFutureArtifactVersion) { + err = fmt.Errorf("%w: validating live checkpoint: %v", errCorruptArtifact, err) + } + return gcOriginResult{}, err + } + if err := validateCheckpointSequenceIdentity(cp, latestName); err != nil { + return gcOriginResult{}, fmt.Errorf( + "%w: validating live checkpoint identity: %v", errCorruptArtifact, err, + ) + } + live := map[gcRef]struct{}{ + {kind: KindCheckpoints, name: latestName}: {}, + } + + gids := make([]string, 0, len(cp.Sessions)) + for gid := range cp.Sessions { + gids = append(gids, gid) + } + sort.Strings(gids) + for _, gid := range gids { + if err := ctx.Err(); err != nil { + return gcOriginResult{}, err + } + manifestHash := cp.Sessions[gid] + if err := validateHashHex(manifestHash); err != nil { + return gcOriginResult{}, fmt.Errorf("live manifest %s: %w", gid, err) + } + manifestName := manifestHash + manifestExtension + live[gcRef{kind: KindManifests, name: manifestName}] = struct{}{} + + m, err := readGCManifest(kindRoots[KindManifests], manifestHash) + if err != nil { + return gcOriginResult{}, fmt.Errorf("reading live manifest %s: %w", manifestHash, err) + } + if err := validateManifest(m, origin, gid); err != nil { + if !errors.Is(err, errFutureArtifactVersion) { + err = fmt.Errorf("%w: validating live manifest: %v", errCorruptArtifact, err) + } + return gcOriginResult{}, err + } + if err := validateGCManifestMessagesWithLimits( + kindRoots[KindSegments], m, productionArtifactLimits(), + ); err != nil { + return gcOriginResult{}, fmt.Errorf("reading live segments for %s: %w", gid, err) + } + for _, segmentHash := range m.Segments { + live[gcRef{kind: KindSegments, name: segmentHash + segmentExtension}] = struct{}{} + } + if m.RawSource != nil && m.RawSource.Hash != "" { + if err := validateLiveRawSource(kindRoots[KindRaw], *m.RawSource); err != nil { + return gcOriginResult{}, fmt.Errorf("reading live raw source for %s: %w", gid, err) + } + live[gcRef{kind: KindRaw, name: m.RawSource.Hash}] = struct{}{} + } + } + + candidates, scanned, err := scanGCCandidates(ctx, root, kindRoots, origin, live) + if err != nil { + return gcOriginResult{}, err + } + keepRoots = true + return gcOriginResult{ + scanned: scanned, candidates: candidates, kindRoots: kindRoots, + }, nil +} + +func openGCKindRoots(root *os.Root) (map[string]*os.Root, error) { + roots := make(map[string]*os.Root, 4) + for _, kind := range []string{KindCheckpoints, KindManifests, KindSegments, KindRaw} { + info, err := root.Lstat(kind) + if errors.Is(err, fs.ErrNotExist) { + continue + } + if err != nil { + closeGCKindRoots(roots) + return nil, err + } + if !info.IsDir() { + closeGCKindRoots(roots) + return nil, fmt.Errorf("artifact kind %s is not a directory", kind) + } + kindRoot, err := openArtifactSubroot(root, kind, "gc kind") + if err != nil { + closeGCKindRoots(roots) + return nil, err + } + roots[kind] = kindRoot + } + return roots, nil +} + +func closeGCKindRoots(roots map[string]*os.Root) { + for _, root := range roots { + _ = root.Close() + } +} + +func readGCManifest(root *os.Root, hash string) (manifest, error) { + if root == nil { + return manifest{}, fmt.Errorf("%w: manifest %s", errIncompleteArtifact, hash) + } + name := hash + manifestExtension + compressed, err := root.ReadFile(name) + if err != nil { + if errors.Is(err, fs.ErrNotExist) { + return manifest{}, fmt.Errorf("%w: manifest %s", errIncompleteArtifact, hash) + } + return manifest{}, err + } + data, err := readCompressedBytes(compressed, manifestDecodedLimit) + if err != nil { + quarantineArtifactRoot(root, name) + return manifest{}, fmt.Errorf("%w: decoding manifest %s: %v", errCorruptArtifact, hash, err) + } + if got := hashHex(data); got != hash { + quarantineArtifactRoot(root, name) + return manifest{}, fmt.Errorf( + "%w: manifest %s hash mismatch: got %s", errCorruptArtifact, hash, got, + ) + } + m, err := decodeManifestWithLimits(data, productionArtifactLimits()) + if err != nil { + quarantineArtifactRoot(root, name) + return manifest{}, fmt.Errorf("%w: decoding manifest %s: %v", errCorruptArtifact, hash, err) + } + return m, nil +} + +func validateGCManifestMessagesWithLimits( + root *os.Root, m manifest, limits artifactLimits, +) error { + if err := validateManifestReferencesWithLimits(m, limits); err != nil { + return fmt.Errorf("%w: %v", errCorruptArtifact, err) + } + if root == nil && len(m.Segments) > 0 { + return fmt.Errorf("%w: segment %s", errIncompleteArtifact, m.Segments[0]) + } + var decodedBytes int64 + totalMessages := 0 + totalNested := nestedCollectionCounts{} + for _, hash := range m.Segments { + name := hash + segmentExtension + compressed, err := root.ReadFile(name) + if err != nil { + if errors.Is(err, fs.ErrNotExist) { + return fmt.Errorf("%w: segment %s", errIncompleteArtifact, hash) + } + return err + } + data, err := readCompressedBytes(compressed, segmentDecodedLimit) + if err != nil { + quarantineArtifactRoot(root, name) + return fmt.Errorf("%w: segment %s: %v", errCorruptArtifact, hash, err) + } + if got := hashHex(data); got != hash { + quarantineArtifactRoot(root, name) + return fmt.Errorf( + "%w: segment %s hash mismatch: got %s", errCorruptArtifact, hash, got, + ) + } + preflight, err := preflightSegmentData(data, limits) + if err != nil { + if errors.Is(err, errFutureArtifactVersion) { + return fmt.Errorf("segment %s: %w", hash, err) + } + quarantineArtifactRoot(root, name) + return fmt.Errorf("%w: segment %s: %v", errCorruptArtifact, hash, err) + } + segmentBytes := int64(len(data)) + if segmentBytes > limits.sessionDecodedBytes-decodedBytes { + return fmt.Errorf( + "%w: session decoded byte limit exceeded: limit %d", + errCorruptArtifact, limits.sessionDecodedBytes, + ) + } + if len(preflight.records) > limits.sessionMessages-totalMessages { + return fmt.Errorf( + "%w: session message limit exceeded: limit %d", + errCorruptArtifact, limits.sessionMessages, + ) + } + if exceedsCollectionLimit( + totalNested.toolCalls, preflight.nested.toolCalls, limits.sessionToolCalls, + ) { + return fmt.Errorf( + "%w: session tool call limit exceeded: limit %d", + errCorruptArtifact, limits.sessionToolCalls, + ) + } + if exceedsCollectionLimit( + totalNested.resultEvents, + preflight.nested.resultEvents, + limits.sessionResultEvents, + ) { + return fmt.Errorf( + "%w: session result event limit exceeded: limit %d", + errCorruptArtifact, limits.sessionResultEvents, + ) + } + messages, err := decodePreflightedSegment(preflight) + if err != nil { + quarantineArtifactRoot(root, name) + return fmt.Errorf("%w: segment %s: %v", errCorruptArtifact, hash, err) + } + decodedBytes += segmentBytes + totalMessages += len(messages) + totalNested.toolCalls += preflight.nested.toolCalls + totalNested.resultEvents += preflight.nested.resultEvents + } + return nil +} + +func validateLiveRawSource(root *os.Root, ref rawSourceRef) error { + if err := validateHashHex(ref.Hash); err != nil { + return err + } + if root == nil { + return fmt.Errorf("%w: raw source %s", errIncompleteArtifact, ref.Hash) + } + info, err := root.Lstat(ref.Hash) + if err != nil { + if errors.Is(err, fs.ErrNotExist) { + return fmt.Errorf("%w: raw source %s", errIncompleteArtifact, ref.Hash) + } + return err + } + if !info.Mode().IsRegular() { + return fmt.Errorf("%w: raw source %s is not a regular file", errCorruptArtifact, ref.Hash) + } + if ref.Size != 0 && info.Size() != ref.Size { + return fmt.Errorf( + "%w: raw source %s size mismatch: got %d, want %d", + errCorruptArtifact, ref.Hash, info.Size(), ref.Size, + ) + } + + file, err := root.Open(ref.Hash) + if err != nil { + if errors.Is(err, fs.ErrNotExist) { + return fmt.Errorf("%w: raw source %s", errIncompleteArtifact, ref.Hash) + } + return err + } + defer file.Close() + openedInfo, err := file.Stat() + if err != nil { + return err + } + if !openedInfo.Mode().IsRegular() || !os.SameFile(info, openedInfo) { + return fmt.Errorf("%w: raw source %s changed while opening", errCorruptArtifact, ref.Hash) + } + h := sha256.New() + size, err := io.Copy(h, file) + if err != nil { + return err + } + if ref.Size != 0 && size != ref.Size { + return fmt.Errorf( + "%w: raw source %s size mismatch while reading: got %d, want %d", + errCorruptArtifact, ref.Hash, size, ref.Size, + ) + } + if got := hex.EncodeToString(h.Sum(nil)); got != ref.Hash { + return fmt.Errorf( + "%w: raw source %s hash mismatch: got %s", + errCorruptArtifact, ref.Hash, got, + ) + } + return nil +} + +func validGCCheckpointNames(root *os.Root) ([]string, error) { + if root == nil { + return nil, nil + } + entries, err := fs.ReadDir(root.FS(), ".") + if err != nil { + return nil, err + } + valid := make([]string, 0, len(entries)) + for _, ent := range entries { + if _, err := checkpointSequence(ent.Name()); err == nil { + valid = append(valid, ent.Name()) + } + } + sort.Strings(valid) + return valid, nil +} + +func scanGCCandidates( + ctx context.Context, + originRoot *os.Root, + kindRoots map[string]*os.Root, + origin string, + live map[gcRef]struct{}, +) ([]gcCandidate, int, error) { + var candidates []gcCandidate + scanned := 0 + for _, spec := range []struct { + kind string + valid func(string) bool + }{ + {kind: KindCheckpoints, valid: isGCCheckpointName}, + {kind: KindManifests, valid: isGCManifestName}, + {kind: KindSegments, valid: isGCSegmentName}, + {kind: KindRaw, valid: isGCRawName}, + } { + dir := filepath.Join(originRoot.Name(), spec.kind) + kindRoot := kindRoots[spec.kind] + if kindRoot == nil { + continue + } + entries, err := fs.ReadDir(kindRoot.FS(), ".") + if err != nil { + if errors.Is(err, fs.ErrNotExist) { + continue + } + return nil, scanned, fmt.Errorf("reading %s: %w", dir, err) + } + for _, ent := range entries { + if err := ctx.Err(); err != nil { + return nil, scanned, err + } + if ent.IsDir() || isTempArtifactEntry(ent.Name()) || !spec.valid(ent.Name()) { + continue + } + info, err := ent.Info() + if err != nil { + return nil, scanned, fmt.Errorf("stat %s: %w", filepath.Join(dir, ent.Name()), err) + } + if !info.Mode().IsRegular() { + continue + } + scanned++ + if _, ok := live[gcRef{kind: spec.kind, name: ent.Name()}]; ok { + continue + } + candidates = append(candidates, gcCandidate{ + path: filepath.Join(dir, ent.Name()), + origin: origin, + kind: spec.kind, + name: ent.Name(), + size: info.Size(), + modTime: info.ModTime(), + }) + } + } + sort.Slice(candidates, func(i, j int) bool { + if candidates[i].origin != candidates[j].origin { + return candidates[i].origin < candidates[j].origin + } + if candidates[i].kind != candidates[j].kind { + return candidates[i].kind < candidates[j].kind + } + return candidates[i].name < candidates[j].name + }) + return candidates, scanned, nil +} + +func isGCCheckpointName(name string) bool { + _, err := checkpointSequence(name) + return err == nil +} + +func isGCManifestName(name string) bool { + _, _, err := normalizeHashName(name, manifestExtension) + return err == nil +} + +func isGCSegmentName(name string) bool { + _, _, err := normalizeHashName(name, segmentExtension) + return err == nil +} + +func isGCRawName(name string) bool { + return validateHashHex(name) == nil +} + +func logGC(opts GCOptions, format string, args ...any) { + if opts.Logf != nil { + opts.Logf(format, args...) + } +} diff --git a/internal/artifact/gc_test.go b/internal/artifact/gc_test.go new file mode 100644 index 000000000..f18b4ba38 --- /dev/null +++ b/internal/artifact/gc_test.go @@ -0,0 +1,630 @@ +package artifact + +import ( + "context" + "fmt" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.kenn.io/agentsview/internal/db" +) + +func TestGarbageCollectDeletesSupersededArtifactsAfterGrace(t *testing.T) { + root, oldPaths, livePaths := supersededArtifactFixture(t) + now := time.Unix(1_800_000_000, 0) + touchPaths(t, now.Add(-2*time.Hour), oldPaths...) + + res, err := GarbageCollect(context.Background(), GCOptions{ + Root: root, + Grace: time.Hour, + Now: now, + }) + require.NoError(t, err) + + assert.Equal(t, 1, res.Origins) + assert.Equal(t, 3, res.Candidates) + assert.Equal(t, 3, res.Eligible) + assert.Equal(t, 3, res.Deleted) + assert.Zero(t, res.KeptByGrace) + for _, path := range oldPaths { + assertNoFile(t, path) + } + for _, path := range livePaths { + assertFileExists(t, path) + } +} + +func TestGarbageCollectDryRunLogsAndKeepsArtifacts(t *testing.T) { + root, oldPaths, _ := supersededArtifactFixture(t) + now := time.Unix(1_800_000_000, 0) + touchPaths(t, now.Add(-2*time.Hour), oldPaths...) + + var logs []string + res, err := GarbageCollect(context.Background(), GCOptions{ + Root: root, + Grace: time.Hour, + Now: now, + DryRun: true, + Logf: func(format string, args ...any) { + logs = append(logs, fmt.Sprintf(format, args...)) + }, + }) + require.NoError(t, err) + + assert.Equal(t, 3, res.Candidates) + assert.Equal(t, 3, res.Eligible) + assert.Zero(t, res.Deleted) + require.NotEmpty(t, logs) + assert.Contains(t, logs[0], "would delete") + for _, path := range oldPaths { + assertFileExists(t, path) + } +} + +func TestGarbageCollectKeepsUnreferencedArtifactsWithinGrace(t *testing.T) { + root, oldPaths, _ := supersededArtifactFixture(t) + now := time.Unix(1_800_000_000, 0) + touchPaths(t, now.Add(-30*time.Minute), oldPaths...) + + res, err := GarbageCollect(context.Background(), GCOptions{ + Root: root, + Grace: time.Hour, + Now: now, + }) + require.NoError(t, err) + + assert.Equal(t, 3, res.Candidates) + assert.Zero(t, res.Eligible) + assert.Equal(t, 3, res.KeptByGrace) + assert.Zero(t, res.Deleted) + for _, path := range oldPaths { + assertFileExists(t, path) + } +} + +func TestGarbageCollectRejectsSymlinkedArtifactKindWithoutDeletingTarget(t *testing.T) { + fixture := newGCClosureFixture(t) + external := t.TempDir() + liveSegment := filepath.Base(fixture.latest.segment) + liveData, err := os.ReadFile(fixture.latest.segment) + require.NoError(t, err) + require.NoError(t, os.WriteFile(filepath.Join(external, liveSegment), liveData, 0o644)) + staleName := strings.Repeat("a", 64) + segmentExtension + stalePath := filepath.Join(external, staleName) + require.NoError(t, os.WriteFile(stalePath, []byte("outside"), 0o644)) + require.NoError(t, os.RemoveAll(filepath.Join(fixture.originRoot, KindSegments))) + require.NoError(t, os.Symlink(external, filepath.Join(fixture.originRoot, KindSegments))) + + _, err = GarbageCollect(context.Background(), GCOptions{ + Root: fixture.root, + Now: time.Now().Add(time.Hour), + }) + require.Error(t, err) + assert.FileExists(t, stalePath) +} + +func TestGarbageCollectRejectsSymlinkedOrigin(t *testing.T) { + external := newGCClosureFixture(t) + root := t.TempDir() + require.NoError(t, os.Symlink( + external.originRoot, filepath.Join(root, external.origin), + )) + + _, err := GarbageCollect(context.Background(), GCOptions{Root: root}) + require.Error(t, err) + for _, path := range external.oldPaths { + assert.FileExists(t, path) + } +} + +func TestGarbageCollectSkipsOriginsWithoutCheckpoints(t *testing.T) { + root := t.TempDir() + origin := "laptop-a1b2c3" + hash := "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + path := filepath.Join(root, origin, KindManifests, hash+manifestExtension) + require.NoError(t, os.MkdirAll(filepath.Dir(path), 0o755)) + require.NoError(t, os.WriteFile(path, []byte("orphan"), 0o644)) + + res, err := GarbageCollect(context.Background(), GCOptions{ + Root: root, + Grace: 0, + Now: time.Unix(1_800_000_000, 0), + }) + require.NoError(t, err) + + assert.Equal(t, 1, res.Origins) + assert.Equal(t, 1, res.SkippedOrigins) + assert.Zero(t, res.Candidates) + assert.Zero(t, res.Deleted) + assertFileExists(t, path) +} + +func TestGarbageCollectSkipsOriginWithCorruptLiveManifest(t *testing.T) { + ctx := context.Background() + root := t.TempDir() + origin := "laptop-a1b2c3" + database := testDB(t) + seedSession(t, database, "sess-1", "alpha") + + _, err := Export(ctx, database, root, origin) + require.NoError(t, err) + + manifests := globArtifacts(t, root, origin, "manifests", "*"+manifestExtension) + require.Len(t, manifests, 1) + require.NoError(t, os.Remove(manifests[0])) + require.NoError(t, writeCompressed(manifests[0], []byte("tampered"))) + segments := globArtifacts(t, root, origin, "segments", "*"+segmentExtension) + require.Len(t, segments, 1) + + res, err := GarbageCollect(ctx, GCOptions{Root: root, Grace: 0}) + require.NoError(t, err) + assert.Equal(t, 1, res.SkippedOrigins) + assert.Zero(t, res.Deleted) + assertFileExists(t, segments[0]) +} + +func TestGarbageCollectKeepsRawReferencedByLiveManifest(t *testing.T) { + root := t.TempDir() + origin := "laptop-a1b2c3" + originRoot := filepath.Join(root, origin) + for _, dir := range []string{KindCheckpoints, KindManifests, KindSegments, KindRaw} { + require.NoError(t, os.MkdirAll(filepath.Join(originRoot, dir), 0o755)) + } + + segmentData := []byte("{\"content\":\"hello\",\"ordinal\":0,\"role\":\"user\",\"v\":1}\n") + segmentHash := hashHex(segmentData) + require.NoError(t, writeCompressed( + filepath.Join(originRoot, KindSegments, segmentHash+segmentExtension), + segmentData, + )) + + liveRaw := []byte("live raw") + liveRawHash := hashHex(liveRaw) + require.NoError(t, os.WriteFile( + filepath.Join(originRoot, KindRaw, liveRawHash), + liveRaw, + 0o644, + )) + staleRaw := []byte("stale raw") + staleRawHash := hashHex(staleRaw) + staleRawPath := filepath.Join(originRoot, KindRaw, staleRawHash) + require.NoError(t, os.WriteFile(staleRawPath, staleRaw, 0o644)) + + gid := origin + "~sess-1" + m := manifest{ + Version: formatVersion, + Origin: origin, + NativeSessionID: "sess-1", + Session: manifestSession{ID: "sess-1", Machine: origin}, + Segments: []string{segmentHash}, + RawSource: &rawSourceRef{ + Hash: liveRawHash, + Size: int64(len(liveRaw)), + }, + DataVersion: 1, + Generation: 1, + } + manifestData, err := canonicalJSON(m) + require.NoError(t, err) + manifestHash := hashHex(manifestData) + require.NoError(t, writeCompressed( + filepath.Join(originRoot, KindManifests, manifestHash+manifestExtension), + manifestData, + )) + writeCheckpoint(t, originRoot, checkpoint{ + Version: formatVersion, + Origin: origin, + Sequence: 1, + Sessions: map[string]string{gid: manifestHash}, + }) + + res, err := GarbageCollect(context.Background(), GCOptions{ + Root: root, + Grace: 0, + Now: time.Unix(1_800_000_000, 0), + }) + require.NoError(t, err) + + assert.Equal(t, 1, res.Candidates) + assert.Equal(t, 1, res.Deleted) + assertNoFile(t, staleRawPath) + assertFileExists(t, filepath.Join(originRoot, KindRaw, liveRawHash)) +} + +func TestGarbageCollectSkipsOriginWhenLatestClosureIsIncompleteOrCorrupt(t *testing.T) { + tests := []struct { + name string + mutate func(*testing.T, *gcClosureFixture) + }{ + { + name: "missing segment", + mutate: func(t *testing.T, fixture *gcClosureFixture) { + t.Helper() + require.NoError(t, os.Remove(fixture.latest.segment)) + }, + }, + { + name: "corrupt segment", + mutate: func(t *testing.T, fixture *gcClosureFixture) { + t.Helper() + require.NoError(t, os.Remove(fixture.latest.segment)) + require.NoError(t, writeCompressed(fixture.latest.segment, []byte("corrupt\n"))) + }, + }, + { + name: "future segment", + mutate: func(t *testing.T, fixture *gcClosureFixture) { + t.Helper() + data, err := canonicalJSON(segmentMessage{ + Version: formatVersion + 1, + Ordinal: 0, + Role: "user", + Content: "future", + }) + require.NoError(t, err) + hash := hashHex(data) + require.NoError(t, writeCompressed( + filepath.Join(fixture.originRoot, KindSegments, hash+segmentExtension), + data, + )) + m := fixture.latest.manifest + m.Segments = []string{hash} + manifestData, err := canonicalJSON(m) + require.NoError(t, err) + manifestHash := hashHex(manifestData) + require.NoError(t, writeCompressed( + filepath.Join(fixture.originRoot, KindManifests, manifestHash+manifestExtension), + manifestData, + )) + writeCheckpoint(t, fixture.originRoot, checkpoint{ + Version: formatVersion, + Origin: fixture.origin, + Sequence: 2, + Sessions: map[string]string{fixture.gid: manifestHash}, + }) + }, + }, + { + name: "invalid manifest reference", + mutate: func(t *testing.T, fixture *gcClosureFixture) { + t.Helper() + m := fixture.latest.manifest + m.Segments = []string{"not-a-segment-hash"} + manifestData, err := canonicalJSON(m) + require.NoError(t, err) + manifestHash := hashHex(manifestData) + require.NoError(t, writeCompressed( + filepath.Join(fixture.originRoot, KindManifests, manifestHash+manifestExtension), + manifestData, + )) + writeCheckpoint(t, fixture.originRoot, checkpoint{ + Version: formatVersion, + Origin: fixture.origin, + Sequence: 2, + Sessions: map[string]string{fixture.gid: manifestHash}, + }) + }, + }, + { + name: "tool call amplification", + mutate: func(t *testing.T, fixture *gcClosureFixture) { + t.Helper() + data := nestedSegmentData(t, segmentMessage{ + ToolCalls: make([]segmentToolCall, 257), + }) + hash := hashHex(data) + require.NoError(t, writeCompressed( + filepath.Join(fixture.originRoot, KindSegments, hash+segmentExtension), + data, + )) + m := fixture.latest.manifest + m.Segments = []string{hash} + replaceLatestGCManifest(t, fixture, m) + }, + }, + { + name: "result event amplification", + mutate: func(t *testing.T, fixture *gcClosureFixture) { + t.Helper() + data := nestedSegmentData(t, segmentMessage{ + ToolCalls: []segmentToolCall{{ + ResultEvents: make([]segmentResultEvent, 1_025), + }}, + }) + hash := hashHex(data) + require.NoError(t, writeCompressed( + filepath.Join(fixture.originRoot, KindSegments, hash+segmentExtension), + data, + )) + m := fixture.latest.manifest + m.Segments = []string{hash} + replaceLatestGCManifest(t, fixture, m) + }, + }, + { + name: "session message amplification", + mutate: func(t *testing.T, fixture *gcClosureFixture) { + t.Helper() + m := fixture.latest.manifest + m.Segments = nil + ordinal := 0 + for segment := range 9 { + count := 4_096 + if segment == 8 { + count = 1 + } + data := syntheticSegmentRecordsFrom(t, ordinal, count) + ordinal += count + hash := hashHex(data) + require.NoError(t, writeCompressed( + filepath.Join(fixture.originRoot, KindSegments, hash+segmentExtension), + data, + )) + m.Segments = append(m.Segments, hash) + } + replaceLatestGCManifest(t, fixture, m) + }, + }, + { + name: "checkpoint sequence mismatch", + mutate: func(t *testing.T, fixture *gcClosureFixture) { + t.Helper() + manifestData, err := canonicalJSON(fixture.latest.manifest) + require.NoError(t, err) + data, err := canonicalJSON(checkpoint{ + Version: formatVersion, + Origin: fixture.origin, + Sequence: 1, + Sessions: map[string]string{ + fixture.gid: hashHex(manifestData), + }, + }) + require.NoError(t, err) + path := filepath.Join( + fixture.originRoot, KindCheckpoints, "cp-0000000002.json", + ) + require.NoError(t, os.Remove(path)) + require.NoError(t, writeFileAtomic(path, data, 0o644)) + }, + }, + { + name: "missing raw", + mutate: func(t *testing.T, fixture *gcClosureFixture) { + t.Helper() + require.NoError(t, os.Remove(fixture.latest.raw)) + }, + }, + { + name: "raw size mismatch", + mutate: func(t *testing.T, fixture *gcClosureFixture) { + t.Helper() + require.NoError(t, os.WriteFile(fixture.latest.raw, []byte("wrong size"), 0o644)) + }, + }, + { + name: "raw hash mismatch", + mutate: func(t *testing.T, fixture *gcClosureFixture) { + t.Helper() + require.NoError(t, os.WriteFile(fixture.latest.raw, []byte("bad raw bytes"), 0o644)) + }, + }, + { + name: "raw is not regular", + mutate: func(t *testing.T, fixture *gcClosureFixture) { + t.Helper() + require.NoError(t, os.Remove(fixture.latest.raw)) + require.NoError(t, os.Mkdir(fixture.latest.raw, 0o755)) + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + fixture := newGCClosureFixture(t) + now := time.Unix(1_800_000_000, 0) + touchPaths(t, now.Add(-2*time.Hour), fixture.oldPaths...) + tt.mutate(t, fixture) + + res, err := GarbageCollect(context.Background(), GCOptions{ + Root: fixture.root, Grace: 0, Now: now, + }) + require.NoError(t, err) + assert.Equal(t, 1, res.Origins) + assert.Equal(t, 1, res.SkippedOrigins) + assert.Zero(t, res.Candidates) + assert.Zero(t, res.Deleted) + for _, path := range fixture.oldPaths { + assertFileExists(t, path) + } + }) + } +} + +type gcClosureFixture struct { + root string + origin string + originRoot string + gid string + oldPaths []string + latest gcGeneration +} + +type gcGeneration struct { + manifest manifest + segment string + raw string +} + +func newGCClosureFixture(t *testing.T) *gcClosureFixture { + t.Helper() + fixture := &gcClosureFixture{ + root: t.TempDir(), + origin: "laptop-a1b2c3", + } + fixture.originRoot = filepath.Join(fixture.root, fixture.origin) + fixture.gid = fixture.origin + "~sess-1" + for _, dir := range []string{KindCheckpoints, KindManifests, KindSegments, KindRaw} { + require.NoError(t, os.MkdirAll(filepath.Join(fixture.originRoot, dir), 0o755)) + } + + old := writeGCGeneration(t, fixture, 1, "old message", []byte("old raw bytes")) + fixture.latest = writeGCGeneration(t, fixture, 2, "new message", []byte("new raw bytes")) + oldManifestData, err := canonicalJSON(old.manifest) + require.NoError(t, err) + fixture.oldPaths = []string{ + filepath.Join(fixture.originRoot, KindCheckpoints, "cp-0000000001.json"), + filepath.Join(fixture.originRoot, KindManifests, hashHex(oldManifestData)+manifestExtension), + old.segment, + old.raw, + } + return fixture +} + +func writeGCGeneration( + t *testing.T, + fixture *gcClosureFixture, + sequence int, + content string, + raw []byte, +) gcGeneration { + t.Helper() + segmentData, err := canonicalJSON(segmentMessage{ + Version: formatVersion, + Ordinal: 0, + Role: "user", + Content: content, + ContentLength: len(content), + }) + require.NoError(t, err) + segmentHash := hashHex(segmentData) + segmentPath := filepath.Join( + fixture.originRoot, KindSegments, segmentHash+segmentExtension, + ) + require.NoError(t, writeCompressed(segmentPath, segmentData)) + + rawHash := hashHex(raw) + rawPath := filepath.Join(fixture.originRoot, KindRaw, rawHash) + require.NoError(t, os.WriteFile(rawPath, raw, 0o644)) + m := manifest{ + Version: formatVersion, + Origin: fixture.origin, + NativeSessionID: "sess-1", + Session: manifestSession{ + ID: "sess-1", + Machine: fixture.origin, + }, + Segments: []string{segmentHash}, + RawSource: &rawSourceRef{ + Hash: rawHash, + Size: int64(len(raw)), + }, + DataVersion: sequence, + Generation: sequence, + } + manifestData, err := canonicalJSON(m) + require.NoError(t, err) + manifestHash := hashHex(manifestData) + require.NoError(t, writeCompressed( + filepath.Join(fixture.originRoot, KindManifests, manifestHash+manifestExtension), + manifestData, + )) + writeCheckpoint(t, fixture.originRoot, checkpoint{ + Version: formatVersion, + Origin: fixture.origin, + Sequence: sequence, + Sessions: map[string]string{fixture.gid: manifestHash}, + }) + return gcGeneration{manifest: m, segment: segmentPath, raw: rawPath} +} + +func replaceLatestGCManifest(t *testing.T, fixture *gcClosureFixture, m manifest) { + t.Helper() + manifestData, err := canonicalJSON(m) + require.NoError(t, err) + manifestHash := hashHex(manifestData) + require.NoError(t, writeCompressed( + filepath.Join(fixture.originRoot, KindManifests, manifestHash+manifestExtension), + manifestData, + )) + writeCheckpoint(t, fixture.originRoot, checkpoint{ + Version: formatVersion, + Origin: fixture.origin, + Sequence: 2, + Sessions: map[string]string{fixture.gid: manifestHash}, + }) +} + +func supersededArtifactFixture(t *testing.T) (string, []string, []string) { + t.Helper() + ctx := context.Background() + database := testDB(t) + root := t.TempDir() + origin := "laptop-a1b2c3" + gid := origin + "~sess-1" + originRoot := filepath.Join(root, origin) + + seedSession(t, database, "sess-1", "alpha") + _, err := Export(ctx, database, root, origin) + require.NoError(t, err) + firstCP, err := readLatestCheckpoint(originRoot) + require.NoError(t, err) + require.NotNil(t, firstCP) + firstManifestHash := firstCP.Sessions[gid] + require.NotEmpty(t, firstManifestHash) + firstManifest, err := readManifest(originRoot, firstManifestHash) + require.NoError(t, err) + require.Len(t, firstManifest.Segments, 1) + + require.NoError(t, database.ReplaceSessionMessages("sess-1", []db.Message{ + {SessionID: "sess-1", Ordinal: 0, Role: "user", Content: "hello", ContentLength: 5}, + {SessionID: "sess-1", Ordinal: 1, Role: "assistant", Content: "planet", ContentLength: 6}, + })) + _, err = Export(ctx, database, root, origin) + require.NoError(t, err) + latestCP, err := readLatestCheckpoint(originRoot) + require.NoError(t, err) + require.NotNil(t, latestCP) + latestManifestHash := latestCP.Sessions[gid] + require.NotEmpty(t, latestManifestHash) + require.NotEqual(t, firstManifestHash, latestManifestHash) + latestManifest, err := readManifest(originRoot, latestManifestHash) + require.NoError(t, err) + require.Len(t, latestManifest.Segments, 1) + require.NotEqual(t, firstManifest.Segments[0], latestManifest.Segments[0]) + + oldPaths := []string{ + filepath.Join(originRoot, KindCheckpoints, "cp-0000000001.json"), + filepath.Join(originRoot, KindManifests, firstManifestHash+manifestExtension), + filepath.Join(originRoot, KindSegments, firstManifest.Segments[0]+segmentExtension), + } + livePaths := []string{ + filepath.Join(originRoot, KindCheckpoints, "cp-0000000002.json"), + filepath.Join(originRoot, KindManifests, latestManifestHash+manifestExtension), + filepath.Join(originRoot, KindSegments, latestManifest.Segments[0]+segmentExtension), + } + return root, oldPaths, livePaths +} + +func touchPaths(t *testing.T, ts time.Time, paths ...string) { + t.Helper() + for _, path := range paths { + require.NoError(t, os.Chtimes(path, ts, ts)) + } +} + +func assertFileExists(t *testing.T, path string) { + t.Helper() + info, err := os.Stat(path) + require.NoError(t, err) + assert.True(t, info.Mode().IsRegular()) +} + +func assertNoFile(t *testing.T, path string) { + t.Helper() + _, err := os.Stat(path) + require.ErrorIs(t, err, os.ErrNotExist) +} diff --git a/internal/artifact/hlc.go b/internal/artifact/hlc.go new file mode 100644 index 000000000..8b2b4fc7b --- /dev/null +++ b/internal/artifact/hlc.go @@ -0,0 +1,261 @@ +package artifact + +import ( + "errors" + "fmt" + "strconv" + "strings" + "sync" + "time" +) + +const ( + metadataHLCStateKey = "artifact_metadata_hlc" + defaultMetadataHLCMaxDrift = 5 * time.Minute + // hlcWallLayout deliberately omits the ":" separators of RFC3339 so the + // rendered timestamp is safe to embed directly in artifact filenames. + // Windows forbids ":" in path components, and metadata event files are + // named after the HLC. Fixed-width fields keep the result lexicographically + // sortable and round-trippable through ParseHLCTimestamp. + hlcWallLayout = "2006-01-02T150405.000000000Z" + hlcLogicalWidth = 20 +) + +type hlcStateStore interface { + GetSyncState(key string) (string, error) + SetSyncState(key, value string) error +} + +// HLCTimestamp is a hybrid logical clock value for metadata events. +type HLCTimestamp struct { + WallTime time.Time + Logical uint64 +} + +// String formats the timestamp in a lexicographically sortable form. +func (t HLCTimestamp) String() string { + return fmt.Sprintf( + "%s-%0*d", + normalizeHLCWallTime(t.WallTime).Format(hlcWallLayout), + hlcLogicalWidth, + t.Logical, + ) +} + +// ParseHLCTimestamp parses a timestamp produced by HLCTimestamp.String. +func ParseHLCTimestamp(s string) (HLCTimestamp, error) { + idx := strings.LastIndex(s, "-") + if idx < 0 { + return HLCTimestamp{}, fmt.Errorf("invalid HLC timestamp %q: missing logical counter", s) + } + wallPart := s[:idx] + logicalPart := s[idx+1:] + if len(logicalPart) != hlcLogicalWidth || !isDecimal(logicalPart) { + return HLCTimestamp{}, fmt.Errorf("invalid HLC timestamp %q: logical counter must be %d digits", s, hlcLogicalWidth) + } + wall, err := time.Parse(hlcWallLayout, wallPart) + if err != nil { + return HLCTimestamp{}, fmt.Errorf("invalid HLC timestamp %q: %w", s, err) + } + logical, err := strconv.ParseUint(logicalPart, 10, 64) + if err != nil { + return HLCTimestamp{}, fmt.Errorf("invalid HLC timestamp %q: %w", s, err) + } + return HLCTimestamp{ + WallTime: normalizeHLCWallTime(wall), + Logical: logical, + }, nil +} + +// Compare returns -1, 0, or 1 when t is ordered before, equal to, or after other. +func (t HLCTimestamp) Compare(other HLCTimestamp) int { + wall := normalizeHLCWallTime(t.WallTime) + otherWall := normalizeHLCWallTime(other.WallTime) + switch { + case wall.Before(otherWall): + return -1 + case wall.After(otherWall): + return 1 + case t.Logical < other.Logical: + return -1 + case t.Logical > other.Logical: + return 1 + default: + return 0 + } +} + +// OrderingKey appends a deterministic tie-breaker, usually the artifact hash. +func (t HLCTimestamp) OrderingKey(tieBreaker string) string { + return t.String() + "-" + tieBreaker +} + +// HLCClockOptions configures a persisted metadata HLC clock. +type HLCClockOptions struct { + StateKey string + Now func() time.Time + MaxDrift time.Duration +} + +// HLCClock persists a monotonic hybrid logical clock in the sync-state store. +type HLCClock struct { + mu sync.Mutex + store hlcStateStore + stateKey string + now func() time.Time + maxDrift time.Duration +} + +// NewHLCClock returns a persisted metadata HLC clock. +func NewHLCClock(store hlcStateStore, opts HLCClockOptions) *HLCClock { + stateKey := opts.StateKey + if stateKey == "" { + stateKey = metadataHLCStateKey + } + now := opts.Now + if now == nil { + now = time.Now + } + maxDrift := opts.MaxDrift + if maxDrift <= 0 { + maxDrift = defaultMetadataHLCMaxDrift + } + return &HLCClock{ + store: store, + stateKey: stateKey, + now: now, + maxDrift: maxDrift, + } +} + +// Next returns and persists the next local metadata-event timestamp. +func (c *HLCClock) Next() (HLCTimestamp, error) { + c.mu.Lock() + defer c.mu.Unlock() + + last, ok, err := c.load() + if err != nil { + return HLCTimestamp{}, err + } + now := c.currentWallTime() + if ok { + if err := c.checkPersistedDrift(last, now); err != nil { + return HLCTimestamp{}, err + } + if !now.After(last.WallTime) { + next := HLCTimestamp{WallTime: last.WallTime, Logical: last.Logical + 1} + return next, c.persist(next) + } + } + next := HLCTimestamp{WallTime: now} + return next, c.persist(next) +} + +// Observe returns and persists a timestamp that is after the local and remote HLCs. +func (c *HLCClock) Observe(remote HLCTimestamp) (HLCTimestamp, error) { + c.mu.Lock() + defer c.mu.Unlock() + + last, ok, err := c.load() + if err != nil { + return HLCTimestamp{}, err + } + now := c.currentWallTime() + remote = HLCTimestamp{ + WallTime: normalizeHLCWallTime(remote.WallTime), + Logical: remote.Logical, + } + if ok { + if err := c.checkPersistedDrift(last, now); err != nil { + return HLCTimestamp{}, err + } + } + if remote.WallTime.After(now.Add(c.maxDrift)) { + return HLCTimestamp{}, fmt.Errorf( + "remote HLC wall time %s is more than %s ahead of local time %s", + remote.WallTime.Format(hlcWallLayout), + c.maxDrift, + now.Format(hlcWallLayout), + ) + } + + next := mergeHLC(HLCTimestamp{WallTime: now}, last, ok, remote) + return next, c.persist(next) +} + +func (c *HLCClock) load() (HLCTimestamp, bool, error) { + if c.store == nil { + return HLCTimestamp{}, false, errors.New("HLC state store is required") + } + raw, err := c.store.GetSyncState(c.stateKey) + if err != nil { + return HLCTimestamp{}, false, fmt.Errorf("reading HLC state: %w", err) + } + if strings.TrimSpace(raw) == "" { + return HLCTimestamp{}, false, nil + } + stamp, err := ParseHLCTimestamp(raw) + if err != nil { + return HLCTimestamp{}, false, fmt.Errorf("reading HLC state: %w", err) + } + return stamp, true, nil +} + +func (c *HLCClock) persist(stamp HLCTimestamp) error { + if err := c.store.SetSyncState(c.stateKey, stamp.String()); err != nil { + return fmt.Errorf("persisting HLC state: %w", err) + } + return nil +} + +func (c *HLCClock) currentWallTime() time.Time { + return normalizeHLCWallTime(c.now()) +} + +func (c *HLCClock) checkPersistedDrift(last HLCTimestamp, now time.Time) error { + if last.WallTime.After(now.Add(c.maxDrift)) { + return fmt.Errorf( + "persisted HLC wall time %s is more than %s ahead of local time %s", + last.WallTime.Format(hlcWallLayout), + c.maxDrift, + now.Format(hlcWallLayout), + ) + } + return nil +} + +func mergeHLC(now, last HLCTimestamp, hasLast bool, remote HLCTimestamp) HLCTimestamp { + maxWall := now.WallTime + if hasLast && last.WallTime.After(maxWall) { + maxWall = last.WallTime + } + if remote.WallTime.After(maxWall) { + maxWall = remote.WallTime + } + + next := HLCTimestamp{WallTime: maxWall} + if hasLast && last.WallTime.Equal(maxWall) { + next.Logical = last.Logical + } + if remote.WallTime.Equal(maxWall) && remote.Logical > next.Logical { + next.Logical = remote.Logical + } + if now.WallTime.Equal(maxWall) && next.Logical == 0 { + return next + } + next.Logical++ + return next +} + +func normalizeHLCWallTime(t time.Time) time.Time { + return t.UTC().Round(0) +} + +func isDecimal(s string) bool { + for _, r := range s { + if r < '0' || r > '9' { + return false + } + } + return s != "" +} diff --git a/internal/artifact/hlc_test.go b/internal/artifact/hlc_test.go new file mode 100644 index 000000000..067776953 --- /dev/null +++ b/internal/artifact/hlc_test.go @@ -0,0 +1,223 @@ +package artifact + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestHLCClockNextPersistsAcrossRestarts(t *testing.T) { + database := testDB(t) + now := fixedHLCTime() + + firstClock := NewHLCClock(database, HLCClockOptions{ + Now: func() time.Time { return now }, + MaxDrift: 5 * time.Minute, + }) + first, err := firstClock.Next() + require.NoError(t, err) + assert.Equal(t, now, first.WallTime) + assert.Equal(t, uint64(0), first.Logical) + + persisted, err := database.GetSyncState(metadataHLCStateKey) + require.NoError(t, err) + assert.Equal(t, "2026-06-14T010203.000000001Z-00000000000000000000", persisted) + + secondClock := NewHLCClock(database, HLCClockOptions{ + Now: func() time.Time { return now }, + MaxDrift: 5 * time.Minute, + }) + second, err := secondClock.Next() + require.NoError(t, err) + assert.Equal(t, now, second.WallTime) + assert.Equal(t, uint64(1), second.Logical) +} + +func TestHLCClockNextMonotonicCases(t *testing.T) { + base := fixedHLCTime() + + tests := []struct { + name string + last HLCTimestamp + now time.Time + want HLCTimestamp + }{ + { + name: "same wall time increments logical counter", + last: HLCTimestamp{WallTime: base, Logical: 7}, + now: base, + want: HLCTimestamp{WallTime: base, Logical: 8}, + }, + { + name: "physical time ahead resets logical counter", + last: HLCTimestamp{WallTime: base, Logical: 7}, + now: base.Add(time.Nanosecond), + want: HLCTimestamp{WallTime: base.Add(time.Nanosecond), Logical: 0}, + }, + { + name: "backward skew within bound increments logical counter", + last: HLCTimestamp{WallTime: base, Logical: 7}, + now: base.Add(-2 * time.Minute), + want: HLCTimestamp{WallTime: base, Logical: 8}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + database := testDB(t) + require.NoError(t, database.SetSyncState(metadataHLCStateKey, tt.last.String())) + clock := NewHLCClock(database, HLCClockOptions{ + Now: func() time.Time { return tt.now }, + MaxDrift: 5 * time.Minute, + }) + + got, err := clock.Next() + require.NoError(t, err) + + assert.Equal(t, 0, tt.want.Compare(got)) + persisted, err := database.GetSyncState(metadataHLCStateKey) + require.NoError(t, err) + assert.Equal(t, tt.want.String(), persisted) + }) + } +} + +func TestHLCClockNextRejectsBackwardSkewBeyondBound(t *testing.T) { + database := testDB(t) + base := fixedHLCTime() + last := HLCTimestamp{WallTime: base, Logical: 7} + require.NoError(t, database.SetSyncState(metadataHLCStateKey, last.String())) + clock := NewHLCClock(database, HLCClockOptions{ + Now: func() time.Time { return base.Add(-10 * time.Minute) }, + MaxDrift: 5 * time.Minute, + }) + + got, err := clock.Next() + require.Error(t, err) + + assert.Equal(t, HLCTimestamp{}, got) + assert.Contains(t, err.Error(), "persisted HLC wall time") + persisted, err := database.GetSyncState(metadataHLCStateKey) + require.NoError(t, err) + assert.Equal(t, last.String(), persisted) +} + +func TestHLCClockObserveCases(t *testing.T) { + base := fixedHLCTime() + + tests := []struct { + name string + last *HLCTimestamp + now time.Time + remote HLCTimestamp + want HLCTimestamp + }{ + { + name: "remote future within bound is absorbed", + now: base, + remote: HLCTimestamp{WallTime: base.Add(time.Minute), Logical: 3}, + want: HLCTimestamp{WallTime: base.Add(time.Minute), Logical: 4}, + }, + { + name: "same wall uses max logical counter", + last: &HLCTimestamp{WallTime: base, Logical: 5}, + now: base, + remote: HLCTimestamp{WallTime: base, Logical: 7}, + want: HLCTimestamp{WallTime: base, Logical: 8}, + }, + { + name: "local physical time wins when ahead", + last: &HLCTimestamp{WallTime: base, Logical: 7}, + now: base.Add(time.Minute), + remote: HLCTimestamp{WallTime: base.Add(30 * time.Second), Logical: 9}, + want: HLCTimestamp{WallTime: base.Add(time.Minute), Logical: 0}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + database := testDB(t) + if tt.last != nil { + require.NoError(t, database.SetSyncState(metadataHLCStateKey, tt.last.String())) + } + clock := NewHLCClock(database, HLCClockOptions{ + Now: func() time.Time { return tt.now }, + MaxDrift: 5 * time.Minute, + }) + + got, err := clock.Observe(tt.remote) + require.NoError(t, err) + + assert.Equal(t, 0, tt.want.Compare(got)) + persisted, err := database.GetSyncState(metadataHLCStateKey) + require.NoError(t, err) + assert.Equal(t, tt.want.String(), persisted) + }) + } +} + +func TestHLCClockObserveRejectsRemoteFutureBeyondBound(t *testing.T) { + database := testDB(t) + base := fixedHLCTime() + last := HLCTimestamp{WallTime: base, Logical: 7} + require.NoError(t, database.SetSyncState(metadataHLCStateKey, last.String())) + clock := NewHLCClock(database, HLCClockOptions{ + Now: func() time.Time { return base }, + MaxDrift: 5 * time.Minute, + }) + + got, err := clock.Observe(HLCTimestamp{ + WallTime: base.Add(10 * time.Minute), + Logical: 3, + }) + require.Error(t, err) + + assert.Equal(t, HLCTimestamp{}, got) + assert.Contains(t, err.Error(), "remote HLC wall time") + persisted, err := database.GetSyncState(metadataHLCStateKey) + require.NoError(t, err) + assert.Equal(t, last.String(), persisted) +} + +func TestHLCTimestampOrderingKeyAndParse(t *testing.T) { + base := fixedHLCTime() + stamp := HLCTimestamp{WallTime: base, Logical: 42} + + text := stamp.String() + parsed, err := ParseHLCTimestamp(text) + require.NoError(t, err) + + assert.Equal(t, "2026-06-14T010203.000000001Z-00000000000000000042", text) + assert.Equal(t, 0, stamp.Compare(parsed)) + assert.Equal(t, -1, stamp.Compare(HLCTimestamp{WallTime: base, Logical: 43})) + assert.Equal(t, -1, stamp.Compare(HLCTimestamp{WallTime: base.Add(time.Nanosecond), Logical: 0})) + assert.Equal(t, 1, stamp.Compare(HLCTimestamp{WallTime: base.Add(-time.Nanosecond), Logical: 99})) + assert.Less(t, stamp.OrderingKey("a111"), stamp.OrderingKey("b222")) + assert.Less(t, + HLCTimestamp{WallTime: base, Logical: 41}.OrderingKey("ffff"), + stamp.OrderingKey("0000"), + ) +} + +func TestParseHLCTimestampRejectsMalformedValues(t *testing.T) { + tests := []string{ + "", + "2026-06-14T010203Z-00000000000000000000", + "2026-06-14T010203.000000001Z-42", + "2026-06-14T010203.000000001Z-0000000000000000000x", + "2026-06-14T01:02:03.000000001Z-00000000000000000000", + } + + for _, tt := range tests { + t.Run(tt, func(t *testing.T) { + _, err := ParseHLCTimestamp(tt) + require.Error(t, err) + }) + } +} + +func fixedHLCTime() time.Time { + return time.Date(2026, 6, 14, 1, 2, 3, 1, time.UTC) +} diff --git a/internal/artifact/manifest_session.go b/internal/artifact/manifest_session.go new file mode 100644 index 000000000..395bd4da8 --- /dev/null +++ b/internal/artifact/manifest_session.go @@ -0,0 +1,227 @@ +package artifact + +import "go.kenn.io/agentsview/internal/db" + +// manifestSession is the manifest wire representation of a session row. It is +// deliberately a separate type from db.Session: manifest bytes feed content +// hashes, so this struct is part of the pinned artifact format and its +// serialized output can never change for an existing value. Keeping it apart +// from the internal DB type means adding a db.Session field cannot silently +// re-hash every exported manifest; extending THIS struct is an explicit wire +// format decision (see TestManifestSessionMatchesDBSessionWireFormat). +type manifestSession struct { + ID string `json:"id"` + Project string `json:"project"` + Machine string `json:"machine"` + Agent string `json:"agent"` + FirstMessage *string `json:"first_message"` + DisplayName *string `json:"display_name,omitempty"` + StartedAt *string `json:"started_at"` + EndedAt *string `json:"ended_at"` + MessageCount int `json:"message_count"` + UserMessageCount int `json:"user_message_count"` + ParentSessionID *string `json:"parent_session_id,omitempty"` + RelationshipType string `json:"relationship_type,omitempty"` + TotalOutputTokens int `json:"total_output_tokens"` + PeakContextTokens int `json:"peak_context_tokens"` + HasTotalOutputTokens bool `json:"has_total_output_tokens"` + HasPeakContextTokens bool `json:"has_peak_context_tokens"` + IsAutomated bool `json:"is_automated"` + + ToolFailureSignalCount int `json:"tool_failure_signal_count"` + ToolRetryCount int `json:"tool_retry_count"` + EditChurnCount int `json:"edit_churn_count"` + ConsecutiveFailureMax int `json:"consecutive_failure_max"` + Outcome string `json:"outcome"` + OutcomeConfidence string `json:"outcome_confidence"` + EndedWithRole string `json:"ended_with_role"` + FinalFailureStreak int `json:"final_failure_streak"` + SignalsPendingSince *string `json:"signals_pending_since,omitempty"` + CompactionCount int `json:"compaction_count"` + MidTaskCompactionCount int `json:"mid_task_compaction_count"` + ContextPressureMax *float64 `json:"context_pressure_max,omitempty"` + HealthScore *int `json:"health_score,omitempty"` + HealthGrade *string `json:"health_grade,omitempty"` + QualitySignals *manifestQualitySignals `json:"quality_signals,omitempty"` + SecretLeakCount int `json:"secret_leak_count"` + + Cwd string `json:"cwd,omitempty"` + GitBranch string `json:"git_branch,omitempty"` + SourceSessionID string `json:"source_session_id,omitempty"` + SourceVersion string `json:"source_version,omitempty"` + TranscriptFidelity string `json:"transcript_fidelity,omitempty"` + ParserMalformedLines int `json:"parser_malformed_lines,omitempty"` + IsTruncated bool `json:"is_truncated,omitempty"` + + DeletedAt *string `json:"deleted_at,omitempty"` + TerminationStatus *string `json:"termination_status,omitempty"` + FilePath *string `json:"file_path,omitempty"` + FileSize *int64 `json:"file_size,omitempty"` + FileMtime *int64 `json:"file_mtime,omitempty"` + FileInode *int64 `json:"file_inode,omitempty"` + FileDevice *int64 `json:"file_device,omitempty"` + FileHash *string `json:"file_hash,omitempty"` + LocalModifiedAt *string `json:"local_modified_at,omitempty"` + CreatedAt string `json:"created_at"` +} + +// manifestQualitySignals mirrors db.QualitySignals for the same reason +// manifestSession mirrors db.Session: it appears in hashed manifest bytes. +type manifestQualitySignals struct { + Version int `json:"version"` + ShortPromptCount int `json:"short_prompt_count"` + UnstructuredStart bool `json:"unstructured_start"` + MissingSuccessCriteriaCount int `json:"missing_success_criteria_count"` + MissingVerificationCount int `json:"missing_verification_count"` + DuplicatePromptCount int `json:"duplicate_prompt_count"` + NoCodeContextCount int `json:"no_code_context_count"` + RunawayToolLoopCount int `json:"runaway_tool_loop_count"` +} + +func manifestSessionFromDB(s db.Session) manifestSession { + return manifestSession{ + ID: s.ID, + Project: s.Project, + Machine: s.Machine, + Agent: s.Agent, + FirstMessage: s.FirstMessage, + DisplayName: s.DisplayName, + StartedAt: s.StartedAt, + EndedAt: s.EndedAt, + MessageCount: s.MessageCount, + UserMessageCount: s.UserMessageCount, + ParentSessionID: s.ParentSessionID, + RelationshipType: s.RelationshipType, + TotalOutputTokens: s.TotalOutputTokens, + PeakContextTokens: s.PeakContextTokens, + HasTotalOutputTokens: s.HasTotalOutputTokens, + HasPeakContextTokens: s.HasPeakContextTokens, + IsAutomated: s.IsAutomated, + + ToolFailureSignalCount: s.ToolFailureSignalCount, + ToolRetryCount: s.ToolRetryCount, + EditChurnCount: s.EditChurnCount, + ConsecutiveFailureMax: s.ConsecutiveFailureMax, + Outcome: s.Outcome, + OutcomeConfidence: s.OutcomeConfidence, + EndedWithRole: s.EndedWithRole, + FinalFailureStreak: s.FinalFailureStreak, + SignalsPendingSince: s.SignalsPendingSince, + CompactionCount: s.CompactionCount, + MidTaskCompactionCount: s.MidTaskCompactionCount, + ContextPressureMax: s.ContextPressureMax, + HealthScore: s.HealthScore, + HealthGrade: s.HealthGrade, + QualitySignals: manifestQualitySignalsFromDB(s.QualitySignals), + SecretLeakCount: s.SecretLeakCount, + + Cwd: s.Cwd, + GitBranch: s.GitBranch, + SourceSessionID: s.SourceSessionID, + SourceVersion: s.SourceVersion, + TranscriptFidelity: s.TranscriptFidelity, + ParserMalformedLines: s.ParserMalformedLines, + IsTruncated: s.IsTruncated, + + DeletedAt: s.DeletedAt, + TerminationStatus: s.TerminationStatus, + FilePath: s.FilePath, + FileSize: s.FileSize, + FileMtime: s.FileMtime, + FileInode: s.FileInode, + FileDevice: s.FileDevice, + FileHash: s.FileHash, + LocalModifiedAt: s.LocalModifiedAt, + CreatedAt: s.CreatedAt, + } +} + +func (m manifestSession) dbSession() db.Session { + return db.Session{ + ID: m.ID, + Project: m.Project, + Machine: m.Machine, + Agent: m.Agent, + FirstMessage: m.FirstMessage, + DisplayName: m.DisplayName, + StartedAt: m.StartedAt, + EndedAt: m.EndedAt, + MessageCount: m.MessageCount, + UserMessageCount: m.UserMessageCount, + ParentSessionID: m.ParentSessionID, + RelationshipType: m.RelationshipType, + TotalOutputTokens: m.TotalOutputTokens, + PeakContextTokens: m.PeakContextTokens, + HasTotalOutputTokens: m.HasTotalOutputTokens, + HasPeakContextTokens: m.HasPeakContextTokens, + IsAutomated: m.IsAutomated, + + ToolFailureSignalCount: m.ToolFailureSignalCount, + ToolRetryCount: m.ToolRetryCount, + EditChurnCount: m.EditChurnCount, + ConsecutiveFailureMax: m.ConsecutiveFailureMax, + Outcome: m.Outcome, + OutcomeConfidence: m.OutcomeConfidence, + EndedWithRole: m.EndedWithRole, + FinalFailureStreak: m.FinalFailureStreak, + SignalsPendingSince: m.SignalsPendingSince, + CompactionCount: m.CompactionCount, + MidTaskCompactionCount: m.MidTaskCompactionCount, + ContextPressureMax: m.ContextPressureMax, + HealthScore: m.HealthScore, + HealthGrade: m.HealthGrade, + QualitySignals: m.QualitySignals.dbQualitySignals(), + SecretLeakCount: m.SecretLeakCount, + + Cwd: m.Cwd, + GitBranch: m.GitBranch, + SourceSessionID: m.SourceSessionID, + SourceVersion: m.SourceVersion, + TranscriptFidelity: m.TranscriptFidelity, + ParserMalformedLines: m.ParserMalformedLines, + IsTruncated: m.IsTruncated, + + DeletedAt: m.DeletedAt, + TerminationStatus: m.TerminationStatus, + FilePath: m.FilePath, + FileSize: m.FileSize, + FileMtime: m.FileMtime, + FileInode: m.FileInode, + FileDevice: m.FileDevice, + FileHash: m.FileHash, + LocalModifiedAt: m.LocalModifiedAt, + CreatedAt: m.CreatedAt, + } +} + +func manifestQualitySignalsFromDB(qs *db.QualitySignals) *manifestQualitySignals { + if qs == nil { + return nil + } + return &manifestQualitySignals{ + Version: qs.Version, + ShortPromptCount: qs.ShortPromptCount, + UnstructuredStart: qs.UnstructuredStart, + MissingSuccessCriteriaCount: qs.MissingSuccessCriteriaCount, + MissingVerificationCount: qs.MissingVerificationCount, + DuplicatePromptCount: qs.DuplicatePromptCount, + NoCodeContextCount: qs.NoCodeContextCount, + RunawayToolLoopCount: qs.RunawayToolLoopCount, + } +} + +func (m *manifestQualitySignals) dbQualitySignals() *db.QualitySignals { + if m == nil { + return nil + } + return &db.QualitySignals{ + Version: m.Version, + ShortPromptCount: m.ShortPromptCount, + UnstructuredStart: m.UnstructuredStart, + MissingSuccessCriteriaCount: m.MissingSuccessCriteriaCount, + MissingVerificationCount: m.MissingVerificationCount, + DuplicatePromptCount: m.DuplicatePromptCount, + NoCodeContextCount: m.NoCodeContextCount, + RunawayToolLoopCount: m.RunawayToolLoopCount, + } +} diff --git a/internal/artifact/manifest_session_test.go b/internal/artifact/manifest_session_test.go new file mode 100644 index 000000000..7bec4a8f4 --- /dev/null +++ b/internal/artifact/manifest_session_test.go @@ -0,0 +1,96 @@ +package artifact + +import ( + "fmt" + "reflect" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "go.kenn.io/agentsview/internal/db" +) + +// TestManifestSessionMatchesDBSessionWireFormat pins the manifest wire DTO to +// the JSON-visible fields of db.Session with a fully populated value: every +// exported field gets a distinct non-zero value, so a missing, extra, or +// transposed DTO field changes the canonical JSON and fails the comparison. +// +// If this test fails after adding a field to db.Session, that is the wire +// format asking for a decision: adding the field to manifestSession changes +// every manifest hash fleet-wide (a re-export/re-import of all sessions), so +// extend the DTO only when the field is genuinely part of the session content +// contract; otherwise leave the DTO alone and update populateWireFixture's +// expectations here. +func TestManifestSessionMatchesDBSessionWireFormat(t *testing.T) { + var sess db.Session + populateWireFixture(t, reflect.ValueOf(&sess).Elem(), 1) + + want, err := canonicalJSON(sess) + require.NoError(t, err) + got, err := canonicalJSON(manifestSessionFromDB(sess)) + require.NoError(t, err) + assert.Equal(t, string(want), string(got), + "manifestSession must serialize byte-identically to db.Session") + + roundTrip, err := canonicalJSON(manifestSessionFromDB(sess).dbSession()) + require.NoError(t, err) + assert.Equal(t, string(want), string(roundTrip), + "converting to the wire DTO and back must preserve every wire-visible field") +} + +func TestManifestQualitySignalsMatchesDBWireFormat(t *testing.T) { + var qs db.QualitySignals + populateWireFixture(t, reflect.ValueOf(&qs).Elem(), 100) + + want, err := canonicalJSON(qs) + require.NoError(t, err) + dto := manifestQualitySignalsFromDB(&qs) + require.NotNil(t, dto) + got, err := canonicalJSON(*dto) + require.NoError(t, err) + assert.Equal(t, string(want), string(got)) + + roundTrip, err := canonicalJSON(*dto.dbQualitySignals()) + require.NoError(t, err) + assert.Equal(t, string(want), string(roundTrip)) + + assert.Nil(t, manifestQualitySignalsFromDB(nil)) +} + +// populateWireFixture fills every exported field of a struct with a distinct +// deterministic non-zero value so field transpositions are detectable. +func populateWireFixture(t *testing.T, v reflect.Value, seed int) { + t.Helper() + for i := 0; i < v.NumField(); i++ { + field := v.Field(i) + if !field.CanSet() { + continue + } + setWireFixtureValue(t, field, seed+i) + } +} + +func setWireFixtureValue(t *testing.T, field reflect.Value, n int) { + t.Helper() + switch field.Kind() { + case reflect.String: + field.SetString(fmt.Sprintf("value-%d", n)) + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + field.SetInt(int64(n)) + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: + field.SetUint(uint64(n)) + case reflect.Bool: + field.SetBool(true) + case reflect.Float32, reflect.Float64: + field.SetFloat(float64(n) + 0.5) + case reflect.Pointer: + elem := reflect.New(field.Type().Elem()) + setWireFixtureValue(t, elem.Elem(), n) + field.Set(elem) + case reflect.Struct: + populateWireFixture(t, field, n*10) + default: + t.Fatalf("populateWireFixture: unhandled field kind %s; teach the fixture about it", field.Kind()) + } +} diff --git a/internal/artifact/metadata.go b/internal/artifact/metadata.go new file mode 100644 index 000000000..ca159cd20 --- /dev/null +++ b/internal/artifact/metadata.go @@ -0,0 +1,481 @@ +package artifact + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "path/filepath" + "strings" + "sync" + "time" + + "go.kenn.io/agentsview/internal/db" + "go.kenn.io/agentsview/internal/parser" +) + +const metadataEventExtension = ".json" + +// errOriginNotAdopted reports that this machine has no artifact origin, i.e. +// it never opted into artifact sync via `sync --init`, a sync run, or a peer +// exchange. Recording paths treat it as "stay local"; explicit sync paths +// treat it as a hard error. +var errOriginNotAdopted = errors.New("artifact origin not adopted") + +type metadataSuppressionKey struct{} + +// Metadata operation names written into the metadata event ledger. +const ( + MetadataOpRename = "rename" + MetadataOpSoftDelete = "soft_delete" + MetadataOpRestore = "restore" + MetadataOpStar = "star" + MetadataOpUnstar = "unstar" + MetadataOpPin = "pin" + MetadataOpUnpin = "unpin" + MetadataOpPurge = "purge" +) + +// MetadataPin identifies a pinned message with stable source coordinates. +type MetadataPin struct { + SourceUUID string `json:"source_uuid,omitempty"` + Ordinal int `json:"ordinal"` + Note *string `json:"note,omitempty"` +} + +// MetadataEventInput describes a local user metadata mutation to append. +type MetadataEventInput struct { + SessionID string + Op string + Value json.RawMessage + Pin *MetadataPin +} + +// MetadataRecord describes a metadata artifact written to disk. +type MetadataRecord struct { + HLC string + Origin string + SessionGID string + Op string + Hash string + Path string +} + +// MetadataPublishedError reports that the artifact file was durably written, +// but local replay bookkeeping failed afterward. +type MetadataPublishedError struct { + Record MetadataRecord + Err error +} + +func (e *MetadataPublishedError) Error() string { + return fmt.Sprintf("metadata event published but local replay state was not recorded: %v", e.Err) +} + +func (e *MetadataPublishedError) Unwrap() error { + return e.Err +} + +// MetadataRecorderOptions configures metadata event artifact writes. +type MetadataRecorderOptions struct { + DataDir string + Origin string + Now func() time.Time + MaxDrift time.Duration +} + +// MetadataRecorder appends canonical metadata event artifacts. +type MetadataRecorder struct { + mu sync.Mutex + database *db.DB + root string + origin string + clock *HLCClock +} + +// NewMetadataRecorder creates a metadata event recorder for the local artifact store. +func NewMetadataRecorder(database *db.DB, opts MetadataRecorderOptions) *MetadataRecorder { + root := "" + if strings.TrimSpace(opts.DataDir) != "" { + root = filepath.Join(opts.DataDir, "artifacts") + } + return &MetadataRecorder{ + database: database, + root: root, + origin: strings.TrimSpace(opts.Origin), + clock: NewHLCClock(database, HLCClockOptions{ + Now: opts.Now, + MaxDrift: opts.MaxDrift, + }), + } +} + +// WithMetadataEventSuppression marks a context as replaying metadata events. +func WithMetadataEventSuppression(ctx context.Context) context.Context { + return context.WithValue(ctx, metadataSuppressionKey{}, true) +} + +// MetadataEventsSuppressed reports whether local metadata event writes are disabled. +func MetadataEventsSuppressed(ctx context.Context) bool { + suppressed, _ := ctx.Value(metadataSuppressionKey{}).(bool) + return suppressed +} + +// Append writes one metadata event artifact unless ctx is replay-suppressed. +func (r *MetadataRecorder) Append(ctx context.Context, input MetadataEventInput) (MetadataRecord, error) { + if MetadataEventsSuppressed(ctx) { + return MetadataRecord{}, nil + } + if r == nil { + return MetadataRecord{}, nil + } + if r.database == nil { + return MetadataRecord{}, errors.New("metadata recorder database is required") + } + if r.root == "" { + return MetadataRecord{}, errors.New("metadata recorder data dir is required") + } + if input.SessionID == "" { + return MetadataRecord{}, errors.New("metadata event session id is required") + } + if err := validateMetadataOp(input.Op); err != nil { + return MetadataRecord{}, err + } + origin, err := r.resolveOrigin() + if errors.Is(err, errOriginNotAdopted) { + // The machine never opted into artifact sync: curation stays local. + // If it joins a fleet later, the `sync --init` baseline snapshot + // publishes the accumulated local curation state. + return MetadataRecord{}, nil + } + if err != nil { + return MetadataRecord{}, err + } + stamp, err := r.clock.Next() + if err != nil { + return MetadataRecord{}, err + } + event := metadataEvent{ + Version: formatVersion, + HLC: stamp.String(), + Origin: origin, + SessionGID: MetadataSessionGID(origin, input.SessionID), + Op: input.Op, + Value: input.Value, + Pin: input.Pin, + } + data, err := canonicalJSON(event) + if err != nil { + return MetadataRecord{}, err + } + hash := hashHex(data) + orderKey := stamp.OrderingKey(hash) + projection, err := metadataProjection(metadataArtifact{ + orderKey: orderKey, + hash: hash, + hlc: event.HLC, + event: event, + }, origin) + if err != nil { + return MetadataRecord{}, err + } + path := filepath.Join(r.root, origin, "meta", orderKey+metadataEventExtension) + record := MetadataRecord{ + HLC: event.HLC, + Origin: origin, + SessionGID: event.SessionGID, + Op: event.Op, + Hash: hash, + Path: path, + } + if err := writeFileAtomic(path, data, 0o644); err != nil { + return MetadataRecord{}, fmt.Errorf("writing metadata event: %w", err) + } + // Record the local event in the LWW replay register only after the artifact + // exists. Otherwise a failed publish can leave hidden local state that wins + // future LWW comparisons for an event no peer can import. + if _, err := r.database.RecordLocalMetadataProjection(ctx, projection); err != nil { + return record, &MetadataPublishedError{ + Record: record, + Err: fmt.Errorf("recording local metadata replay state: %w", err), + } + } + return record, nil +} + +// RepairLocalSessionMetadata rebuilds local replay bookkeeping for already +// published local metadata artifacts without re-applying their visible +// mutations. +func (r *MetadataRecorder) RepairLocalSessionMetadata( + ctx context.Context, + sessionID string, + ops ...string, +) (int, error) { + if r == nil { + return 0, nil + } + if r.database == nil { + return 0, errors.New("metadata recorder database is required") + } + if r.root == "" { + return 0, errors.New("metadata recorder data dir is required") + } + if sessionID == "" { + return 0, errors.New("metadata event session id is required") + } + opSet := make(map[string]struct{}, len(ops)) + for _, op := range ops { + if err := validateMetadataOp(op); err != nil { + return 0, err + } + opSet[op] = struct{}{} + } + origin, err := r.resolveOrigin() + if errors.Is(err, errOriginNotAdopted) { + // No origin means no published local artifacts to repair against. + return 0, nil + } + if err != nil { + return 0, err + } + events, err := readMetadataArtifacts(filepath.Join(r.root, origin), origin, nil) + if err != nil { + return 0, err + } + sessionGID := MetadataSessionGID(origin, sessionID) + repaired := 0 + for _, art := range events { + if err := ctx.Err(); err != nil { + return repaired, err + } + if art.event.SessionGID != sessionGID { + continue + } + if len(opSet) > 0 { + if _, ok := opSet[art.event.Op]; !ok { + continue + } + } + if err := validateMetadataArtifactEvent(art, origin); err != nil { + if errors.Is(err, errFutureArtifactVersion) { + continue + } + return repaired, err + } + if err := validateMetadataOp(art.event.Op); err != nil { + return repaired, err + } + projection, err := metadataProjection(art, origin) + if err != nil { + return repaired, err + } + if _, err := r.database.RecordLocalMetadataProjection(ctx, projection); err != nil { + return repaired, fmt.Errorf("repairing local metadata replay state: %w", err) + } + repaired++ + } + return repaired, nil +} + +// AppendBaseline writes metadata events for existing local curation that +// predates artifact metadata recording. +func (r *MetadataRecorder) AppendBaseline(ctx context.Context) (int, error) { + if r == nil { + return 0, nil + } + if r.database == nil { + return 0, errors.New("metadata recorder database is required") + } + snap, err := r.database.MetadataBaselineSnapshot(ctx) + if err != nil { + return 0, err + } + return r.AppendBaselineSnapshot(ctx, snap) +} + +// AppendBaselineSnapshot writes metadata events from a previously captured +// curation snapshot. Callers that import peer artifacts before initialization +// should capture the snapshot before that import so newly imported rows cannot +// be re-published as local baseline metadata. +func (r *MetadataRecorder) AppendBaselineSnapshot( + ctx context.Context, + snap db.MetadataBaselineSnapshot, +) (int, error) { + if r == nil { + return 0, nil + } + if r.database == nil { + return 0, errors.New("metadata recorder database is required") + } + origin, err := r.resolveOrigin() + if err != nil { + return 0, err + } + written := 0 + for _, rename := range snap.Renames { + covered, err := r.baselineFieldCovered(ctx, origin, rename.SessionID, "display_name") + if err != nil { + return written, err + } + if covered { + continue + } + value, err := metadataRenameValue(rename.DisplayName) + if err != nil { + return written, err + } + if _, err := r.Append(ctx, MetadataEventInput{ + SessionID: rename.SessionID, + Op: MetadataOpRename, + Value: value, + }); err != nil { + return written, fmt.Errorf("writing baseline rename metadata: %w", err) + } + written++ + } + for _, sessionID := range snap.StarredSessionIDs { + covered, err := r.baselineFieldCovered(ctx, origin, sessionID, "starred") + if err != nil { + return written, err + } + if covered { + continue + } + if _, err := r.Append(ctx, MetadataEventInput{ + SessionID: sessionID, + Op: MetadataOpStar, + }); err != nil { + return written, fmt.Errorf("writing baseline star metadata: %w", err) + } + written++ + } + for _, sessionID := range snap.SoftDeletedIDs { + covered, err := r.baselineFieldCovered(ctx, origin, sessionID, "deleted_at") + if err != nil { + return written, err + } + if covered { + continue + } + if _, err := r.Append(ctx, MetadataEventInput{ + SessionID: sessionID, + Op: MetadataOpSoftDelete, + }); err != nil { + return written, fmt.Errorf("writing baseline soft-delete metadata: %w", err) + } + written++ + } + for _, pin := range snap.Pins { + metadataPin := MetadataPin{ + SourceUUID: pin.SourceUUID, + Ordinal: pin.Ordinal, + Note: pin.Note, + } + covered, err := r.baselineFieldCovered( + ctx, origin, pin.SessionID, "pin:"+metadataPinAnchor(metadataPin), + ) + if err != nil { + return written, err + } + if covered { + continue + } + if _, err := r.Append(ctx, MetadataEventInput{ + SessionID: pin.SessionID, + Op: MetadataOpPin, + Pin: &metadataPin, + }); err != nil { + return written, fmt.Errorf("writing baseline pin metadata: %w", err) + } + written++ + } + return written, nil +} + +func (r *MetadataRecorder) baselineFieldCovered( + ctx context.Context, + origin string, + sessionID string, + field string, +) (bool, error) { + _, ok, err := r.database.MetadataReplayStateOp( + ctx, MetadataSessionGID(origin, sessionID), field, + ) + if err != nil { + return false, fmt.Errorf("checking baseline metadata field %s: %w", field, err) + } + return ok, nil +} + +// Import reads every foreign origin under root and imports referenced sessions +// plus metadata events, advancing this recorder's HLC clock past observed +// remote HLCs so later local edits stay causally ahead of imported peers. +func (r *MetadataRecorder) Import(ctx context.Context, root string) (ImportResult, error) { + if r == nil || r.database == nil { + return ImportResult{}, errors.New("metadata recorder database is required") + } + origin, err := r.resolveOrigin() + if err != nil { + return ImportResult{}, err + } + return importDetailed(ctx, r.database, r.clock, root, origin) +} + +// MetadataSessionGID returns the global metadata target ID for a session. +func MetadataSessionGID(origin, sessionID string) string { + if host, _ := parser.StripHostPrefix(sessionID); host != "" { + return sessionID + } + return origin + "~" + sessionID +} + +// resolveOrigin returns the recorder's origin without ever creating one: the +// explicit option wins, then the origin persisted in DB sync state. A machine +// with no origin anywhere has not opted into artifact sync and gets +// errOriginNotAdopted. The empty result is not cached, so a recorder built +// before opt-in starts resolving the origin as soon as it is adopted. +func (r *MetadataRecorder) resolveOrigin() (string, error) { + r.mu.Lock() + defer r.mu.Unlock() + if r.origin != "" { + if err := validateOriginID(r.origin); err != nil { + return "", fmt.Errorf("metadata recorder origin: %w", err) + } + return r.origin, nil + } + origin, err := StoredOrigin(r.database) + if err != nil { + return "", err + } + if origin == "" { + return "", errOriginNotAdopted + } + r.origin = origin + return origin, nil +} + +func metadataRenameValue(displayName *string) (json.RawMessage, error) { + data, err := json.Marshal(struct { + DisplayName *string `json:"display_name"` + }{DisplayName: displayName}) + if err != nil { + return nil, err + } + return json.RawMessage(data), nil +} + +func validateMetadataOp(op string) error { + switch op { + case MetadataOpRename, + MetadataOpSoftDelete, + MetadataOpRestore, + MetadataOpStar, + MetadataOpUnstar, + MetadataOpPin, + MetadataOpUnpin, + MetadataOpPurge: + return nil + default: + return fmt.Errorf("unsupported metadata event op %q", op) + } +} diff --git a/internal/artifact/metadata_test.go b/internal/artifact/metadata_test.go new file mode 100644 index 000000000..c720ca018 --- /dev/null +++ b/internal/artifact/metadata_test.go @@ -0,0 +1,152 @@ +package artifact + +import ( + "context" + "encoding/json" + "os" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestMetadataRecorderAppendWritesCanonicalEvent(t *testing.T) { + database := testDB(t) + dataDir := t.TempDir() + now := fixedHLCTime() + recorder := NewMetadataRecorder(database, MetadataRecorderOptions{ + DataDir: dataDir, + Origin: "laptop-a1b2c3", + Now: func() time.Time { return now }, + }) + + value := json.RawMessage(`{"display_name":"Renamed session"}`) + record, err := recorder.Append(context.Background(), MetadataEventInput{ + SessionID: "sess-1", + Op: MetadataOpRename, + Value: value, + }) + require.NoError(t, err) + + assert.Equal(t, "2026-06-14T010203.000000001Z-00000000000000000000", record.HLC) + assert.Equal(t, "laptop-a1b2c3", record.Origin) + assert.Equal(t, "laptop-a1b2c3~sess-1", record.SessionGID) + assert.Equal(t, MetadataOpRename, record.Op) + + data, err := os.ReadFile(record.Path) + require.NoError(t, err) + assert.Equal(t, + "{\"hlc\":\"2026-06-14T010203.000000001Z-00000000000000000000\",\"op\":\"rename\",\"origin\":\"laptop-a1b2c3\",\"session_gid\":\"laptop-a1b2c3~sess-1\",\"v\":1,\"value\":{\"display_name\":\"Renamed session\"}}\n", + string(data), + ) + assert.Equal(t, hashHex(data), record.Hash) + assert.Equal(t, + filepath.Join(dataDir, "artifacts", "laptop-a1b2c3", "meta", record.HLC+"-"+record.Hash+".json"), + record.Path, + ) + // The metadata event filename must be safe on every supported OS, + // including Windows, which forbids these characters in path components. + assert.NotContains(t, filepath.Base(record.Path), ":", + "metadata filename must not contain ':' (invalid on Windows)") + for _, c := range `<>:"/\|?*` { + assert.NotContainsf(t, record.HLC, string(c), + "HLC %q must not contain %q (invalid in Windows filenames)", record.HLC, string(c)) + } +} + +func TestImportObservesRemoteHLCForLaterLocalEdits(t *testing.T) { + ctx := context.Background() + root := t.TempDir() + dataDir := t.TempDir() + localOrigin := "desktop-d4e5f6" + peerOrigin := "laptop-a1b2c3" + database := testDB(t) + seedSession(t, database, "sess-1", "alpha") + peerGID := localOrigin + "~sess-1" + + recorderNow := fixedHLCTime() + // A peer event whose wall time is ahead of the recorder's local clock but + // within the drift bound. + remoteStamp := HLCTimestamp{WallTime: recorderNow.Add(2 * time.Minute), Logical: 5} + writeMetadataArtifact(t, filepath.Join(root, peerOrigin), + replayRenameEvent(t, peerOrigin, peerGID, remoteStamp.String(), "Peer name")) + + recorder := NewMetadataRecorder(database, MetadataRecorderOptions{ + DataDir: dataDir, + Origin: localOrigin, + Now: func() time.Time { return recorderNow }, + }) + + imported, err := recorder.Import(ctx, root) + require.NoError(t, err) + assert.Equal(t, 1, imported.Metadata) + + // The next local edit must receive an HLC strictly after the observed + // remote HLC even though the local wall clock is behind it. + rec, err := recorder.Append(ctx, MetadataEventInput{ + SessionID: "sess-1", + Op: MetadataOpStar, + }) + require.NoError(t, err) + localStamp, err := ParseHLCTimestamp(rec.HLC) + require.NoError(t, err) + assert.Equal(t, 1, localStamp.Compare(remoteStamp), + "local HLC %s must be after observed remote HLC %s", rec.HLC, remoteStamp.String()) +} + +func TestMetadataRecorderWithoutOriginIsNoOp(t *testing.T) { + database := testDB(t) + dataDir := t.TempDir() + recorder := NewMetadataRecorder(database, MetadataRecorderOptions{DataDir: dataDir}) + + record, err := recorder.Append(context.Background(), MetadataEventInput{ + SessionID: "sess-1", + Op: MetadataOpStar, + }) + require.NoError(t, err) + assert.Equal(t, MetadataRecord{}, record) + + origin, err := StoredOrigin(database) + require.NoError(t, err) + assert.Empty(t, origin, + "recorder must not mint an origin for a machine that never opted into artifact sync") + _, err = os.Stat(filepath.Join(dataDir, "artifacts")) + assert.True(t, os.IsNotExist(err), + "no artifact store should be created without an origin") + + repaired, err := recorder.RepairLocalSessionMetadata(context.Background(), "sess-1") + require.NoError(t, err) + assert.Zero(t, repaired) +} + +func TestMetadataRecorderRecordsAfterOriginAdopted(t *testing.T) { + database := testDB(t) + dataDir := t.TempDir() + recorder := NewMetadataRecorder(database, MetadataRecorderOptions{DataDir: dataDir}) + + record, err := recorder.Append(context.Background(), MetadataEventInput{ + SessionID: "sess-1", + Op: MetadataOpStar, + }) + require.NoError(t, err) + assert.Empty(t, record.Origin) + + require.NoError(t, AdoptOrigin(database, "desk-a1b2c3")) + + record, err = recorder.Append(context.Background(), MetadataEventInput{ + SessionID: "sess-1", + Op: MetadataOpStar, + }) + require.NoError(t, err) + assert.Equal(t, "desk-a1b2c3", record.Origin) + _, err = os.Stat(record.Path) + require.NoError(t, err, + "recorder must start writing events once the origin is adopted, without reconstruction") +} + +func TestMetadataSessionGID(t *testing.T) { + assert.Equal(t, "desk-a1b2c3~sess-1", MetadataSessionGID("desk-a1b2c3", "sess-1")) + assert.Equal(t, "laptop-d4e5f6~sess-1", MetadataSessionGID("desk-a1b2c3", "laptop-d4e5f6~sess-1")) +} diff --git a/internal/artifact/nested_limits_test.go b/internal/artifact/nested_limits_test.go new file mode 100644 index 000000000..a140b1de5 --- /dev/null +++ b/internal/artifact/nested_limits_test.go @@ -0,0 +1,475 @@ +package artifact + +import ( + "bytes" + "context" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.kenn.io/agentsview/internal/db" +) + +func TestDecodeSegmentRejectsAggregateNestedLimitsWithSmallLimits(t *testing.T) { + tests := []struct { + name string + records []segmentMessage + configure func(*artifactLimits) + wantError string + }{ + { + name: "tool calls per segment", + records: []segmentMessage{ + {ToolCalls: []segmentToolCall{{}}}, + {ToolCalls: []segmentToolCall{{}}}, + }, + configure: func(limits *artifactLimits) { + limits.segmentToolCalls = 1 + }, + wantError: "segment tool call limit", + }, + { + name: "result events per segment", + records: []segmentMessage{ + {ToolCalls: []segmentToolCall{{ResultEvents: []segmentResultEvent{{}}}}}, + {ToolCalls: []segmentToolCall{{ResultEvents: []segmentResultEvent{{}}}}}, + }, + configure: func(limits *artifactLimits) { + limits.segmentResultEvents = 1 + }, + wantError: "segment result event limit", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + limits := productionArtifactLimits() + tt.configure(&limits) + data := nestedSegmentData(t, tt.records...) + + _, err := decodeSegmentWithLimits(data, limits) + require.Error(t, err) + assert.Contains(t, err.Error(), tt.wantError) + }) + } +} + +func TestReadManifestMessagesRejectsSessionNestedLimitsWithSmallLimits(t *testing.T) { + tests := []struct { + name string + record segmentMessage + configure func(*artifactLimits) + wantError string + }{ + { + name: "tool calls per session", + record: segmentMessage{ToolCalls: []segmentToolCall{{}}}, + configure: func(limits *artifactLimits) { + limits.sessionToolCalls = 1 + }, + wantError: "session tool call limit", + }, + { + name: "result events per session", + record: segmentMessage{ToolCalls: []segmentToolCall{{ + ResultEvents: []segmentResultEvent{{}}, + }}}, + configure: func(limits *artifactLimits) { + limits.sessionResultEvents = 1 + }, + wantError: "session result event limit", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + originRoot := t.TempDir() + m := manifest{} + for ordinal := range 2 { + record := tt.record + record.Ordinal = ordinal + record.Content = string(rune('a' + ordinal)) + data := nestedSegmentData(t, record) + hash := hashHex(data) + require.NoError(t, writeCompressed( + filepath.Join(originRoot, KindSegments, hash+segmentExtension), data, + )) + m.Segments = append(m.Segments, hash) + } + limits := productionArtifactLimits() + tt.configure(&limits) + + _, err := readManifestMessagesWithLimits(originRoot, m, limits) + require.Error(t, err) + assert.ErrorIs(t, err, errCorruptArtifact) + assert.Contains(t, err.Error(), tt.wantError) + }) + } +} + +func TestPeerArtifactDefersFutureNestedSchema(t *testing.T) { + root := t.TempDir() + origin := "peer-a1b2c3" + data := []byte(`{"v":2,"ordinal":{"future_shape":true},"tool_calls":{"future_shape":true}}` + "\n") + hash := hashHex(data) + compressed := compressPeerTestData(t, data) + + res, err := WriteArtifact(root, origin, KindSegments, hash, compressed) + require.NoError(t, err) + assert.Equal(t, hash+segmentExtension, res.Name) + stored, err := ReadArtifact(root, origin, KindSegments, hash) + require.NoError(t, err) + assert.Equal(t, compressed, stored.Data) +} + +func TestDecodeSegmentAcceptsCanonicalTrailingNewlineAndEmptySession(t *testing.T) { + record := nestedSegmentData(t, segmentMessage{}) + tests := []struct { + name string + data []byte + want int + }{ + {name: "canonical trailing newline", data: record, want: 1}, + {name: "zero byte empty segment", data: nil, want: 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + msgs, err := decodeSegment(tt.data) + require.NoError(t, err) + assert.Len(t, msgs, tt.want) + }) + } +} + +func TestImportQuarantinesNestedAmplificationWithoutAdvancingState(t *testing.T) { + tests := []struct { + name string + record segmentMessage + }{ + { + name: "too many tool calls", + record: segmentMessage{ + ToolCalls: make([]segmentToolCall, 257), + }, + }, + { + name: "too many result events", + record: segmentMessage{ + ToolCalls: []segmentToolCall{{ + ResultEvents: make([]segmentResultEvent, 1_025), + }}, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctx := context.Background() + root := t.TempDir() + origin := "laptop-a1b2c3" + localOrigin := "desktop-d4e5f6" + importDB := testDB(t) + gid, segmentPath := writeNestedImportFixture( + t, root, origin, tt.record, + ) + + res, err := ImportDetailed(ctx, importDB, root, localOrigin) + require.NoError(t, err) + assert.False(t, res.Changed()) + state, err := importDB.GetSyncState(importStateKey(origin, gid)) + require.NoError(t, err) + assert.Empty(t, state) + got, err := importDB.GetSessionFull(ctx, gid) + require.NoError(t, err) + assert.Nil(t, got) + assert.NoFileExists(t, segmentPath) + assert.FileExists(t, segmentPath+quarantineSuffix) + }) + } +} + +func TestExportRejectsNestedAmplificationBeforePublication(t *testing.T) { + tests := []struct { + name string + message db.Message + wantError string + }{ + { + name: "too many tool calls in one message", + message: db.Message{ + ToolCalls: make([]db.ToolCall, 257), + }, + wantError: "tool call limit exceeded for message ordinal 0", + }, + { + name: "too many result events in one tool call", + message: db.Message{ + ToolCalls: []db.ToolCall{{ + ResultEvents: make([]db.ToolResultEvent, 1_025), + }}, + }, + wantError: "result event limit exceeded for tool call 0 in message ordinal 0", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctx := context.Background() + database := testDB(t) + root := t.TempDir() + origin := "laptop-a1b2c3" + seedSession(t, database, "sess-1", "alpha") + message := tt.message + message.SessionID = "sess-1" + message.Ordinal = 0 + message.Role = "assistant" + require.NoError(t, database.ReplaceSessionMessages("sess-1", []db.Message{message})) + + _, err := Export(ctx, database, root, origin) + require.Error(t, err) + assert.Contains(t, err.Error(), tt.wantError) + assertNoPublishedArtifactFiles(t, root, origin) + state, err := database.GetSyncState(exportStateKey(origin, "sess-1")) + require.NoError(t, err) + assert.Empty(t, state) + }) + } +} + +func TestExportChunksOnAggregateNestedLimitsWithSmallLimits(t *testing.T) { + tests := []struct { + name string + resultEvents []db.ToolResultEvent + configure func(*artifactLimits) + }{ + { + name: "tool calls per segment", + configure: func(limits *artifactLimits) { + limits.segmentToolCalls = 2 + }, + }, + { + name: "result events per segment", + resultEvents: []db.ToolResultEvent{{EventIndex: 0}}, + configure: func(limits *artifactLimits) { + limits.segmentResultEvents = 2 + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctx := context.Background() + database := testDB(t) + root := t.TempDir() + origin := "laptop-a1b2c3" + originRoot := filepath.Join(root, origin) + for _, kind := range []string{KindManifests, KindSegments} { + require.NoError(t, os.MkdirAll(filepath.Join(originRoot, kind), 0o755)) + } + seedSession(t, database, "sess-1", "alpha") + msgs := make([]db.Message, 3) + for ordinal := range msgs { + msgs[ordinal] = db.Message{ + SessionID: "sess-1", + Ordinal: ordinal, + Role: "assistant", + ToolCalls: []db.ToolCall{{ + ResultEvents: tt.resultEvents, + }}, + } + } + require.NoError(t, database.ReplaceSessionMessages("sess-1", msgs)) + limits := productionArtifactLimits() + tt.configure(&limits) + + manifestHash, changed, err := exportSessionWithLimits( + ctx, database, originRoot, origin, "sess-1", "", limits, + ) + require.NoError(t, err) + assert.True(t, changed) + m, err := readManifest(originRoot, manifestHash) + require.NoError(t, err) + require.Len(t, m.Segments, 2) + got, err := readManifestMessages(originRoot, m) + require.NoError(t, err) + require.Len(t, got, 3) + for ordinal := range got { + assert.Equal(t, ordinal, got[ordinal].Ordinal) + require.Len(t, got[ordinal].ToolCalls, 1) + assert.Len(t, got[ordinal].ToolCalls[0].ResultEvents, len(tt.resultEvents)) + } + }) + } +} + +func TestExportRejectsMessageThatCannotFitNestedSegmentLimits(t *testing.T) { + tests := []struct { + name string + message db.Message + configure func(*artifactLimits) + wantError string + }{ + { + name: "tool calls cannot split across segments", + message: db.Message{ + ToolCalls: []db.ToolCall{{}, {}}, + }, + configure: func(limits *artifactLimits) { + limits.segmentToolCalls = 1 + }, + wantError: "2 tool calls", + }, + { + name: "one tool result history cannot split across segments", + message: db.Message{ + ToolCalls: []db.ToolCall{{ + ResultEvents: []db.ToolResultEvent{{EventIndex: 0}, {EventIndex: 1}}, + }}, + }, + configure: func(limits *artifactLimits) { + limits.segmentResultEvents = 1 + }, + wantError: "2 result events", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctx := context.Background() + database := testDB(t) + root := t.TempDir() + origin := "laptop-a1b2c3" + seedSession(t, database, "sess-1", "alpha") + message := tt.message + message.SessionID = "sess-1" + message.Ordinal = 0 + message.Role = "assistant" + require.NoError(t, database.ReplaceSessionMessages("sess-1", []db.Message{message})) + limits := productionArtifactLimits() + tt.configure(&limits) + + _, _, err := exportSessionWithLimits( + ctx, database, filepath.Join(root, origin), origin, "sess-1", "", limits, + ) + require.Error(t, err) + assert.Contains(t, err.Error(), "cannot fit in one segment") + assert.Contains(t, err.Error(), tt.wantError) + assert.Empty(t, globArtifacts( + t, root, origin, KindSegments, "*"+segmentExtension, + )) + assert.Empty(t, globArtifacts( + t, root, origin, KindManifests, "*"+manifestExtension, + )) + }) + } +} + +func TestExportRejectsSessionNestedLimitsBeforeWritingWithSmallLimits(t *testing.T) { + tests := []struct { + name string + configure func(*artifactLimits) + wantError string + }{ + { + name: "tool calls per session", + configure: func(limits *artifactLimits) { + limits.sessionToolCalls = 1 + }, + wantError: "session tool call limit", + }, + { + name: "result events per session", + configure: func(limits *artifactLimits) { + limits.sessionResultEvents = 1 + }, + wantError: "session result event limit", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctx := context.Background() + database := testDB(t) + root := t.TempDir() + origin := "laptop-a1b2c3" + originRoot := filepath.Join(root, origin) + seedSession(t, database, "sess-1", "alpha") + msgs := make([]db.Message, 2) + for ordinal := range msgs { + msgs[ordinal] = db.Message{ + SessionID: "sess-1", + Ordinal: ordinal, + Role: "assistant", + ToolCalls: []db.ToolCall{{ + ResultEvents: []db.ToolResultEvent{{EventIndex: 0}}, + }}, + } + } + require.NoError(t, database.ReplaceSessionMessages("sess-1", msgs)) + limits := productionArtifactLimits() + tt.configure(&limits) + + _, _, err := exportSessionWithLimits( + ctx, database, originRoot, origin, "sess-1", "", limits, + ) + require.Error(t, err) + assert.Contains(t, err.Error(), tt.wantError) + assert.Empty(t, globArtifacts( + t, root, origin, KindSegments, "*"+segmentExtension, + )) + assert.Empty(t, globArtifacts( + t, root, origin, KindManifests, "*"+manifestExtension, + )) + }) + } +} + +func nestedSegmentData(t *testing.T, records ...segmentMessage) []byte { + t.Helper() + var data bytes.Buffer + for ordinal := range records { + records[ordinal].Version = formatVersion + records[ordinal].Ordinal = ordinal + records[ordinal].Role = "assistant" + encoded, err := canonicalJSON(records[ordinal]) + require.NoError(t, err) + _, err = data.Write(encoded) + require.NoError(t, err) + } + return data.Bytes() +} + +func writeNestedImportFixture( + t *testing.T, + root, origin string, + record segmentMessage, +) (string, string) { + t.Helper() + originRoot := filepath.Join(root, origin) + data := nestedSegmentData(t, record) + segmentHash := hashHex(data) + segmentPath := filepath.Join(originRoot, KindSegments, segmentHash+segmentExtension) + require.NoError(t, writeCompressed(segmentPath, data)) + + gid := origin + "~sess-1" + m := manifest{ + Version: formatVersion, + Origin: origin, + NativeSessionID: "sess-1", + Session: manifestSession{ + ID: "sess-1", + Machine: origin, + }, + Segments: []string{segmentHash}, + } + manifestData, err := canonicalJSON(m) + require.NoError(t, err) + manifestHash := hashHex(manifestData) + require.NoError(t, writeCompressed( + filepath.Join(originRoot, KindManifests, manifestHash+manifestExtension), + manifestData, + )) + writeCheckpoint(t, originRoot, checkpoint{ + Version: formatVersion, + Origin: origin, + Sequence: 1, + Sessions: map[string]string{gid: manifestHash}, + }) + return gid, segmentPath +} diff --git a/internal/artifact/peer.go b/internal/artifact/peer.go new file mode 100644 index 000000000..654500ae3 --- /dev/null +++ b/internal/artifact/peer.go @@ -0,0 +1,726 @@ +package artifact + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "io" + "io/fs" + "log" + "os" + "path/filepath" + "slices" + "sort" + "strings" + "time" + + "github.com/klauspost/compress/zstd" +) + +const ( + KindCheckpoints = "checkpoints" + KindManifests = "manifests" + KindSegments = "segments" + KindMeta = "meta" + KindRaw = "raw" +) + +var ( + ErrArtifactInvalid = errors.New("invalid artifact") + ErrArtifactNotFound = errors.New("artifact not found") + ErrArtifactConflict = errors.New("artifact conflict") +) + +// PeerArtifact is one immutable artifact file served through the peer API. +type PeerArtifact struct { + Origin string + Kind string + Name string + Hash string + ContentType string + Data []byte +} + +// PeerArtifactWrite describes the result of a peer artifact write. +type PeerArtifactWrite struct { + Origin string + Kind string + Name string + Hash string + Size int64 + Duplicate bool +} + +type peerArtifactSpec struct { + origin string + kind string + dir string + name string + hash string + path string + contentType string +} + +// ListOrigins returns valid origin directories in the artifact store. +func ListOrigins(root string) ([]string, error) { + if strings.TrimSpace(root) == "" { + return nil, fmt.Errorf("%w: artifact root is required", ErrArtifactInvalid) + } + entries, err := os.ReadDir(root) + if err != nil { + if errors.Is(err, fs.ErrNotExist) { + return []string{}, nil + } + return nil, err + } + origins := make([]string, 0, len(entries)) + for _, ent := range entries { + if !ent.IsDir() { + continue + } + origin := ent.Name() + if validateOriginID(origin) == nil { + origins = append(origins, origin) + } + } + sort.Strings(origins) + return origins, nil +} + +// OriginArtifactIndex lists the artifact filenames an origin currently holds, +// grouped by kind. It is the enumeration the HTTP peer transport needs for a +// set-union pull, since metadata events are not referenced by the checkpoint +// and therefore cannot be discovered from it. +type OriginArtifactIndex struct { + Origin string `json:"origin"` + Checkpoints []string `json:"checkpoints"` + Manifests []string `json:"manifests"` + Segments []string `json:"segments"` + Meta []string `json:"meta"` + Raw []string `json:"raw"` +} + +// ListArtifacts enumerates the valid artifact filenames an origin holds, grouped +// by kind, skipping temp files and entries that do not match a kind's naming +// rules. Missing kind directories yield empty lists, not errors. +func ListArtifacts(root, origin string) (OriginArtifactIndex, error) { + if strings.TrimSpace(root) == "" { + return OriginArtifactIndex{}, fmt.Errorf("%w: artifact root is required", ErrArtifactInvalid) + } + if err := validateOriginID(origin); err != nil { + return OriginArtifactIndex{}, fmt.Errorf("%w: %v", ErrArtifactInvalid, err) + } + idx := OriginArtifactIndex{Origin: origin} + var err error + if idx.Checkpoints, err = listArtifactKind(root, origin, KindCheckpoints); err != nil { + return OriginArtifactIndex{}, err + } + if idx.Manifests, err = listArtifactKind(root, origin, KindManifests); err != nil { + return OriginArtifactIndex{}, err + } + if idx.Segments, err = listArtifactKind(root, origin, KindSegments); err != nil { + return OriginArtifactIndex{}, err + } + if idx.Meta, err = listArtifactKind(root, origin, KindMeta); err != nil { + return OriginArtifactIndex{}, err + } + if idx.Raw, err = listArtifactKind(root, origin, KindRaw); err != nil { + return OriginArtifactIndex{}, err + } + return idx, nil +} + +func listArtifactKind(root, origin, kind string) ([]string, error) { + dir := filepath.Join(root, origin, kind) + entries, err := os.ReadDir(dir) + if err != nil { + if errors.Is(err, fs.ErrNotExist) { + return []string{}, nil + } + return nil, err + } + names := make([]string, 0, len(entries)) + for _, ent := range entries { + if ent.IsDir() || isTempArtifactEntry(ent.Name()) { + continue + } + if !validArtifactKindName(kind, ent.Name()) { + continue + } + names = append(names, ent.Name()) + } + sort.Strings(names) + return names, nil +} + +func validArtifactKindName(kind, name string) bool { + switch kind { + case KindCheckpoints: + return isGCCheckpointName(name) + case KindManifests: + return isGCManifestName(name) + case KindSegments: + return isGCSegmentName(name) + case KindRaw: + return isGCRawName(name) + case KindMeta: + _, _, err := normalizeMetadataName(name) + return err == nil + default: + return false + } +} + +// ReadLatestCheckpoint returns the newest checkpoint file for an origin. +func ReadLatestCheckpoint(root, origin string) (PeerArtifact, error) { + if strings.TrimSpace(root) == "" { + return PeerArtifact{}, fmt.Errorf("%w: artifact root is required", ErrArtifactInvalid) + } + if err := validateOriginID(origin); err != nil { + return PeerArtifact{}, fmt.Errorf("%w: %v", ErrArtifactInvalid, err) + } + originRoot := filepath.Join(root, origin) + path, err := latestCheckpointPath(originRoot) + if err != nil { + return PeerArtifact{}, err + } + data, err := os.ReadFile(path) + if err != nil { + if errors.Is(err, fs.ErrNotExist) { + return PeerArtifact{}, ErrArtifactNotFound + } + return PeerArtifact{}, err + } + if err := validateCheckpointData(data, origin, filepath.Base(path)); err != nil { + return PeerArtifact{}, err + } + return PeerArtifact{ + Origin: origin, + Kind: KindCheckpoints, + Name: filepath.Base(path), + ContentType: "application/json", + Data: data, + }, nil +} + +// OriginCheckpointSummary describes the latest checkpoint published by one +// origin. Found is false when the origin has no checkpoint yet. +type OriginCheckpointSummary struct { + Sequence int + SessionCount int + ModTime time.Time + Found bool +} + +// CheckpointSummary returns summary information about an origin's latest +// checkpoint without decoding any session bundles. It is the read side of the +// peers status view. A checkpoint that no longer decodes is quarantined and +// the summary falls back to the newest one that does, so one corrupt file +// cannot make the peers view unusable. +func CheckpointSummary(root, origin string) (OriginCheckpointSummary, error) { + if strings.TrimSpace(root) == "" { + return OriginCheckpointSummary{}, fmt.Errorf("%w: artifact root is required", ErrArtifactInvalid) + } + if err := validateOriginID(origin); err != nil { + return OriginCheckpointSummary{}, fmt.Errorf("%w: %v", ErrArtifactInvalid, err) + } + paths, err := filepath.Glob(filepath.Join(root, origin, "checkpoints", "cp-*.json")) + if err != nil { + return OriginCheckpointSummary{}, err + } + sort.Strings(paths) + for _, path := range slices.Backward(paths) { + info, err := os.Stat(path) + if err != nil { + if errors.Is(err, fs.ErrNotExist) { + continue + } + return OriginCheckpointSummary{}, err + } + data, err := os.ReadFile(path) + if err != nil { + if errors.Is(err, fs.ErrNotExist) { + continue + } + return OriginCheckpointSummary{}, err + } + var cp checkpoint + if err := json.Unmarshal(data, &cp); err != nil { + log.Printf("artifact: skipping corrupt checkpoint %s in peer summary: %v", path, err) + quarantineArtifact(path) + continue + } + return OriginCheckpointSummary{ + Sequence: cp.Sequence, + SessionCount: len(cp.Sessions), + ModTime: info.ModTime(), + Found: true, + }, nil + } + return OriginCheckpointSummary{}, nil +} + +// ReadArtifact returns one artifact file by kind and name. +func ReadArtifact(root, origin, kind, name string) (PeerArtifact, error) { + spec, err := makePeerArtifactSpec(root, origin, kind, name) + if err != nil { + return PeerArtifact{}, err + } + return readArtifactSpec(spec) +} + +// ReadArtifactForServe returns one artifact for peer download. A stored file +// that fails content validation is quarantined and reported as not found: +// dropping it from the served set removes it from the origin index, so a peer +// that still holds a valid copy re-uploads it on a later push and the store +// heals, instead of every pull failing against the same bytes forever. A +// request whose origin, kind, or name is itself invalid is still rejected as +// invalid without touching any file. +func ReadArtifactForServe(root, origin, kind, name string) (PeerArtifact, error) { + spec, err := makePeerArtifactSpec(root, origin, kind, name) + if err != nil { + return PeerArtifact{}, err + } + art, err := readArtifactSpec(spec) + if err != nil && errors.Is(err, ErrArtifactInvalid) { + log.Printf("artifact: quarantining corrupt artifact %s/%s/%s requested by peer: %v", + origin, kind, name, err) + quarantineArtifact(spec.path) + return PeerArtifact{}, ErrArtifactNotFound + } + return art, err +} + +func readArtifactSpec(spec peerArtifactSpec) (PeerArtifact, error) { + data, err := os.ReadFile(spec.path) + if err != nil { + if errors.Is(err, fs.ErrNotExist) { + return PeerArtifact{}, ErrArtifactNotFound + } + return PeerArtifact{}, err + } + if err := validateArtifactData(spec, data); err != nil { + return PeerArtifact{}, err + } + return PeerArtifact{ + Origin: spec.origin, + Kind: spec.kind, + Name: spec.name, + Hash: spec.hash, + ContentType: spec.contentType, + Data: data, + }, nil +} + +// WriteArtifact verifies and stores one peer artifact. Existing identical +// content is accepted as a duplicate; existing different content is rejected. +func WriteArtifact(root, origin, kind, name string, data []byte) (PeerArtifactWrite, error) { + spec, err := makePeerArtifactSpec(root, origin, kind, name) + if err != nil { + return PeerArtifactWrite{}, err + } + if len(data) == 0 { + return PeerArtifactWrite{}, fmt.Errorf("%w: artifact body is empty", ErrArtifactInvalid) + } + if err := validateArtifactData(spec, data); err != nil { + return PeerArtifactWrite{}, err + } + if duplicate, err := existingArtifactMatches(spec.path, data); err != nil { + if isArtifactPathConflict(err) { + return PeerArtifactWrite{}, fmt.Errorf("%w: %v", ErrArtifactConflict, err) + } + return PeerArtifactWrite{}, err + } else if duplicate { + return PeerArtifactWrite{ + Origin: spec.origin, + Kind: spec.kind, + Name: spec.name, + Hash: spec.hash, + Size: int64(len(data)), + Duplicate: true, + }, nil + } + if err := writeFileAtomic(spec.path, data, 0o644); err != nil { + if isArtifactPathConflict(err) { + return PeerArtifactWrite{}, fmt.Errorf("%w: %v", ErrArtifactConflict, err) + } + return PeerArtifactWrite{}, err + } + return PeerArtifactWrite{ + Origin: spec.origin, + Kind: spec.kind, + Name: spec.name, + Hash: spec.hash, + Size: int64(len(data)), + }, nil +} + +func latestCheckpointPath(originRoot string) (string, error) { + paths, err := filepath.Glob(filepath.Join(originRoot, "checkpoints", "cp-*.json")) + if err != nil { + return "", err + } + if len(paths) == 0 { + return "", ErrArtifactNotFound + } + sort.Strings(paths) + return paths[len(paths)-1], nil +} + +func makePeerArtifactSpec(root, origin, kind, name string) (peerArtifactSpec, error) { + if strings.TrimSpace(root) == "" { + return peerArtifactSpec{}, fmt.Errorf("%w: artifact root is required", ErrArtifactInvalid) + } + if err := validateOriginID(origin); err != nil { + return peerArtifactSpec{}, fmt.Errorf("%w: %v", ErrArtifactInvalid, err) + } + if err := validateArtifactName(name); err != nil { + return peerArtifactSpec{}, err + } + spec, err := artifactSpecForKind(origin, kind, name) + if err != nil { + return peerArtifactSpec{}, err + } + spec.path = filepath.Join(root, origin, spec.dir, spec.name) + return spec, nil +} + +// artifactSpecForKind resolves an artifact's kind, canonical name, and +// content hash from its path identity alone, without requiring a valid origin +// id or store root, so on-disk files can be validated in place. +func artifactSpecForKind(origin, kind, name string) (peerArtifactSpec, error) { + kind = normalizeArtifactKind(kind) + spec := peerArtifactSpec{ + origin: origin, + kind: kind, + } + switch kind { + case KindCheckpoints: + filename, err := normalizeCheckpointName(name) + if err != nil { + return peerArtifactSpec{}, err + } + spec.dir = "checkpoints" + spec.name = filename + spec.contentType = "application/json" + case KindManifests: + filename, hash, err := normalizeHashName(name, manifestExtension) + if err != nil { + return peerArtifactSpec{}, err + } + spec.dir = "manifests" + spec.name = filename + spec.hash = hash + spec.contentType = "application/zstd" + case KindSegments: + filename, hash, err := normalizeHashName(name, segmentExtension) + if err != nil { + return peerArtifactSpec{}, err + } + spec.dir = "segments" + spec.name = filename + spec.hash = hash + spec.contentType = "application/zstd" + case KindMeta: + filename, hash, err := normalizeMetadataName(name) + if err != nil { + return peerArtifactSpec{}, err + } + spec.dir = "meta" + spec.name = filename + spec.hash = hash + spec.contentType = "application/json" + case KindRaw: + if err := validateHashHex(name); err != nil { + return peerArtifactSpec{}, err + } + spec.dir = "raw" + spec.name = name + spec.hash = name + spec.contentType = "application/octet-stream" + default: + return peerArtifactSpec{}, fmt.Errorf("%w: unsupported artifact kind %q", ErrArtifactInvalid, kind) + } + return spec, nil +} + +func normalizeArtifactKind(kind string) string { + switch strings.TrimSpace(strings.ToLower(kind)) { + case "checkpoint", "checkpoints": + return KindCheckpoints + case "manifest", "manifests": + return KindManifests + case "segment", "segments": + return KindSegments + case "meta", "metadata": + return KindMeta + case "raw": + return KindRaw + default: + return strings.TrimSpace(strings.ToLower(kind)) + } +} + +func validateArtifactName(name string) error { + if strings.TrimSpace(name) == "" { + return fmt.Errorf("%w: artifact name is required", ErrArtifactInvalid) + } + if strings.Contains(name, "/") || strings.Contains(name, "\\") || strings.Contains(name, "..") { + return fmt.Errorf("%w: invalid artifact name", ErrArtifactInvalid) + } + return nil +} + +func normalizeHashName(name, extension string) (filename, hash string, err error) { + hash = strings.TrimSuffix(name, extension) + if err := validateHashHex(hash); err != nil { + return "", "", err + } + return hash + extension, hash, nil +} + +func normalizeMetadataName(name string) (filename, hash string, err error) { + base := strings.TrimSuffix(name, metadataEventExtension) + idx := strings.LastIndex(base, "-") + if idx < 0 { + return "", "", fmt.Errorf("%w: metadata artifact missing hash suffix", ErrArtifactInvalid) + } + hash = base[idx+1:] + if err := validateHashHex(hash); err != nil { + return "", "", err + } + return base + metadataEventExtension, hash, nil +} + +func normalizeCheckpointName(name string) (string, error) { + base := strings.TrimSuffix(name, ".json") + if _, err := checkpointSequence(base + ".json"); err != nil { + return "", err + } + return base + ".json", nil +} + +func checkpointSequence(filename string) (int, error) { + base := strings.TrimSuffix(filename, ".json") + if len(base) != len("cp-0000000000") || !strings.HasPrefix(base, "cp-") { + return 0, fmt.Errorf("%w: invalid checkpoint name", ErrArtifactInvalid) + } + seq := 0 + for _, r := range base[len("cp-"):] { + if r < '0' || r > '9' { + return 0, fmt.Errorf("%w: invalid checkpoint name", ErrArtifactInvalid) + } + seq = seq*10 + int(r-'0') + } + if seq <= 0 { + return 0, fmt.Errorf("%w: invalid checkpoint sequence", ErrArtifactInvalid) + } + return seq, nil +} + +func validateHashHex(hash string) error { + if len(hash) != 64 { + return fmt.Errorf("%w: invalid artifact hash", ErrArtifactInvalid) + } + for _, r := range hash { + if (r < '0' || r > '9') && (r < 'a' || r > 'f') { + return fmt.Errorf("%w: invalid artifact hash", ErrArtifactInvalid) + } + } + return nil +} + +func validateArtifactData(spec peerArtifactSpec, data []byte) error { + switch spec.kind { + case KindCheckpoints: + return validateCheckpointData(data, spec.origin, spec.name) + case KindManifests: + return validateManifestArtifactData(data, spec.origin, spec.hash) + case KindSegments: + return validateSegmentArtifactData(data, spec.hash) + case KindMeta: + return validateMetadataArtifactData(data, spec.origin, spec.name, spec.hash) + case KindRaw: + if got := hashHex(data); got != spec.hash { + return fmt.Errorf("%w: raw artifact hash mismatch: got %s", ErrArtifactInvalid, got) + } + return nil + default: + return fmt.Errorf("%w: unsupported artifact kind %q", ErrArtifactInvalid, spec.kind) + } +} + +func validateCheckpointData(data []byte, origin, filename string) error { + var cp checkpoint + if err := json.Unmarshal(data, &cp); err != nil { + return fmt.Errorf("%w: decoding checkpoint: %v", ErrArtifactInvalid, err) + } + if cp.Version > formatVersion { + if cp.Origin != origin { + return fmt.Errorf( + "%w: checkpoint origin mismatch for %s: got %q", + ErrArtifactInvalid, origin, cp.Origin, + ) + } + if err := validateCheckpointSequenceIdentity(cp, filename); err != nil { + return fmt.Errorf("%w: %v", ErrArtifactInvalid, err) + } + if err := validateCheckpointReferences(&cp, origin); err != nil { + return fmt.Errorf("%w: %v", ErrArtifactInvalid, err) + } + return nil + } + if err := validateCheckpoint(&cp, origin); err != nil { + return fmt.Errorf("%w: %v", ErrArtifactInvalid, err) + } + if err := validateCheckpointSequenceIdentity(cp, filename); err != nil { + return fmt.Errorf("%w: %v", ErrArtifactInvalid, err) + } + return nil +} + +func validateCheckpointSequenceIdentity(cp checkpoint, filename string) error { + seq, err := checkpointSequence(filename) + if err != nil { + return err + } + if cp.Sequence != seq { + return fmt.Errorf( + "checkpoint sequence mismatch: name has %d, body has %d", + seq, cp.Sequence, + ) + } + return nil +} + +func validateManifestArtifactData(data []byte, origin, hash string) error { + decoded, err := readCompressedBytes(data, manifestDecodedLimit) + if err != nil { + return fmt.Errorf("%w: decoding manifest compression: %v", ErrArtifactInvalid, err) + } + if got := hashHex(decoded); got != hash { + return fmt.Errorf("%w: manifest hash mismatch: got %s", ErrArtifactInvalid, got) + } + m, err := decodeManifestWithLimits(decoded, productionArtifactLimits()) + if err != nil { + return fmt.Errorf("%w: decoding manifest: %v", ErrArtifactInvalid, err) + } + if m.Version > formatVersion { + if m.Origin != origin { + return fmt.Errorf("%w: manifest origin mismatch for %s: got %q", ErrArtifactInvalid, origin, m.Origin) + } + return nil + } + if m.Origin != origin { + return fmt.Errorf("%w: manifest origin mismatch for %s: got %q", ErrArtifactInvalid, origin, m.Origin) + } + if m.NativeSessionID == "" || m.Session.ID != m.NativeSessionID || m.Session.Machine != origin { + return fmt.Errorf("%w: manifest session identity mismatch", ErrArtifactInvalid) + } + if len(m.Segments) == 0 { + return fmt.Errorf("%w: manifest has no message segments", ErrArtifactInvalid) + } + if err := validateManifestReferences(m); err != nil { + return fmt.Errorf("%w: %v", ErrArtifactInvalid, err) + } + return nil +} + +func validateSegmentArtifactData(data []byte, hash string) error { + decoded, err := readCompressedBytes(data, segmentDecodedLimit) + if err != nil { + return fmt.Errorf("%w: decoding segment compression: %v", ErrArtifactInvalid, err) + } + if got := hashHex(decoded); got != hash { + return fmt.Errorf("%w: segment hash mismatch: got %s", ErrArtifactInvalid, got) + } + if _, err := decodeSegment(decoded); err != nil { + if errors.Is(err, errFutureArtifactVersion) { + if ferr := validateFutureSegmentData(decoded); ferr != nil { + return fmt.Errorf("%w: decoding segment: %v", ErrArtifactInvalid, ferr) + } + return nil + } + return fmt.Errorf("%w: decoding segment: %v", ErrArtifactInvalid, err) + } + return nil +} + +func validateFutureSegmentData(data []byte) error { + records, err := segmentRecords(data, maxSegmentMessages) + if err != nil { + return err + } + for _, line := range records { + var record struct { + Version int `json:"v"` + } + if err := json.Unmarshal(line, &record); err != nil { + return err + } + if record.Version <= formatVersion { + return fmt.Errorf("message segment has unsupported artifact version %d", record.Version) + } + } + return nil +} + +func validateMetadataArtifactData(data []byte, origin, filename, hash string) error { + if got := hashHex(data); got != hash { + return fmt.Errorf("%w: metadata artifact hash mismatch: got %s", ErrArtifactInvalid, got) + } + base := strings.TrimSuffix(filename, metadataEventExtension) + hlc := strings.TrimSuffix(base, "-"+hash) + var event metadataEvent + if err := json.Unmarshal(data, &event); err != nil { + return fmt.Errorf("%w: decoding metadata event: %v", ErrArtifactInvalid, err) + } + art := metadataArtifact{ + path: filename, + hlc: hlc, + hash: hash, + event: event, + } + if err := validateMetadataArtifactEvent(art, origin); err != nil { + if errors.Is(err, errFutureArtifactVersion) { + return nil + } + return fmt.Errorf("%w: %v", ErrArtifactInvalid, err) + } + // Unknown ops stay accepted for forward compatibility (replay marks + // them applied and skips them), but a known op must carry a payload + // that projects cleanly or replay could never apply it. + if err := validateMetadataOp(event.Op); err == nil { + if _, _, _, _, err := metadataProjectionFields(event); err != nil { + return fmt.Errorf("%w: metadata event payload: %v", ErrArtifactInvalid, err) + } + } + return nil +} + +func readCompressedBytes(data []byte, maxDecoded int64) ([]byte, error) { + dec, err := zstd.NewReader( + bytes.NewReader(data), + zstd.WithDecoderConcurrency(1), + zstd.WithDecoderLowmem(true), + zstd.WithDecoderMaxMemory(uint64(maxDecoded)), + zstd.WithDecoderMaxWindow(zstdMaxWindowSize), + ) + if err != nil { + return nil, err + } + defer dec.Close() + out, err := io.ReadAll(io.LimitReader(dec, maxDecoded+1)) + if err != nil { + return nil, err + } + if int64(len(out)) > maxDecoded { + return nil, fmt.Errorf("decoded output exceeds %d-byte limit", maxDecoded) + } + return out, nil +} + +func isArtifactPathConflict(err error) bool { + return errors.Is(err, errArtifactPathConflict) +} diff --git a/internal/artifact/peer_test.go b/internal/artifact/peer_test.go new file mode 100644 index 000000000..d9a4245c8 --- /dev/null +++ b/internal/artifact/peer_test.go @@ -0,0 +1,322 @@ +package artifact + +import ( + "bytes" + "context" + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/klauspost/compress/zstd" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "go.kenn.io/agentsview/internal/db" +) + +func TestPeerArtifactWriteReadManifestDuplicate(t *testing.T) { + root := t.TempDir() + origin := "peer-a1b2c3" + startedAt := "2026-06-14T01:02:03Z" + segmentData, err := encodeSegment([]db.Message{{ + Ordinal: 0, + Role: "user", + Content: "hello", + }}) + require.NoError(t, err) + segmentHash := hashHex(segmentData) + manifestData, err := canonicalJSON(manifest{ + Version: formatVersion, + Origin: origin, + NativeSessionID: "sess-1", + Session: manifestSession{ + ID: "sess-1", + Machine: origin, + Agent: "claude", + Project: "alpha", + StartedAt: &startedAt, + CreatedAt: "2026-06-14T01:02:03Z", + MessageCount: 1, + UserMessageCount: 1, + TotalOutputTokens: 0, + PeakContextTokens: 0, + }, + Segments: []string{segmentHash}, + DataVersion: 1, + Generation: 1, + }) + require.NoError(t, err) + manifestHash := hashHex(manifestData) + compressed := compressPeerTestData(t, manifestData) + + res, err := WriteArtifact(root, origin, "manifest", manifestHash, compressed) + require.NoError(t, err) + assert.False(t, res.Duplicate) + assert.Equal(t, KindManifests, res.Kind) + assert.Equal(t, manifestHash+manifestExtension, res.Name) + + res, err = WriteArtifact(root, origin, "manifests", manifestHash+manifestExtension, compressed) + require.NoError(t, err) + assert.True(t, res.Duplicate) + + got, err := ReadArtifact(root, origin, "manifests", manifestHash) + require.NoError(t, err) + assert.Equal(t, "application/zstd", got.ContentType) + assert.Equal(t, compressed, got.Data) +} + +func TestPeerArtifactRejectsHashMismatch(t *testing.T) { + root := t.TempDir() + origin := "peer-a1b2c3" + data := compressPeerTestData(t, []byte("not the named hash\n")) + + _, err := WriteArtifact(root, origin, "segments", strings64("0"), data) + require.Error(t, err) + assert.ErrorIs(t, err, ErrArtifactInvalid) +} + +func TestPeerArtifactCheckpointDuplicateAndConflict(t *testing.T) { + root := t.TempDir() + origin := "peer-a1b2c3" + first := []byte(`{"origin":"peer-a1b2c3","seq":1,"sessions":{},"v":1}` + "\n") + second := []byte(`{"origin":"peer-a1b2c3","seq":1,"sessions":{"peer-a1b2c3~sess-1":"` + strings64("a") + `"},"v":1}` + "\n") + + res, err := WriteArtifact(root, origin, "checkpoints", "cp-0000000001", first) + require.NoError(t, err) + assert.False(t, res.Duplicate) + + res, err = WriteArtifact(root, origin, "checkpoints", "cp-0000000001.json", first) + require.NoError(t, err) + assert.True(t, res.Duplicate) + + _, err = WriteArtifact(root, origin, "checkpoints", "cp-0000000001", second) + require.Error(t, err) + assert.ErrorIs(t, err, ErrArtifactConflict) +} + +func TestPeerArtifactRejectsInvalidReferences(t *testing.T) { + root := t.TempDir() + origin := "peer-a1b2c3" + + checkpointData := []byte(`{"origin":"peer-a1b2c3","seq":1,"sessions":{"peer-a1b2c3~sess-1":"../outside"},"v":1}` + "\n") + _, err := WriteArtifact(root, origin, KindCheckpoints, "cp-0000000001", checkpointData) + require.Error(t, err) + assert.ErrorIs(t, err, ErrArtifactInvalid) + + manifestData, err := canonicalJSON(manifest{ + Version: formatVersion, + Origin: origin, + NativeSessionID: "sess-1", + Session: manifestSession{ + ID: "sess-1", + Machine: origin, + Agent: "claude", + Project: "alpha", + CreatedAt: "2026-06-14T01:02:03Z", + }, + Segments: []string{"../outside"}, + }) + require.NoError(t, err) + manifestHash := hashHex(manifestData) + _, err = WriteArtifact( + root, + origin, + KindManifests, + manifestHash, + compressPeerTestData(t, manifestData), + ) + require.Error(t, err) + assert.ErrorIs(t, err, ErrArtifactInvalid) +} + +func TestPeerArtifactMetadataMustMatchOrigin(t *testing.T) { + root := t.TempDir() + origin := "peer-a1b2c3" + body := []byte(`{"hlc":"2026-06-14T010203.000000001Z-other-b2c3d4","op":"rename","origin":"other-b2c3d4","session_gid":"other-b2c3d4~sess-1","v":1,"value":{"display_name":"Remote"}}` + "\n") + hash := hashHex(body) + name := "2026-06-14T010203.000000001Z-other-b2c3d4-" + hash + + _, err := WriteArtifact(root, origin, "meta", name, body) + require.Error(t, err) + assert.ErrorIs(t, err, ErrArtifactInvalid) +} + +func TestPeerArtifactMetadataRejectsMalformedKnownOpPayload(t *testing.T) { + root := t.TempDir() + origin := "peer-a1b2c3" + hlc := "2026-06-14T010203.000000001Z-peer-a1b2c3" + + tests := []struct { + name string + op string + value json.RawMessage + }{ + {name: "pin missing payload", op: MetadataOpPin}, + {name: "unpin missing payload", op: MetadataOpUnpin}, + {name: "rename non-object value", op: MetadataOpRename, value: json.RawMessage(`[1,2]`)}, + {name: "rename missing value", op: MetadataOpRename}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + data, err := canonicalJSON(metadataEvent{ + Version: formatVersion, + HLC: hlc, + Origin: origin, + SessionGID: origin + "~sess-1", + Op: tt.op, + Value: tt.value, + }) + require.NoError(t, err) + name := hlc + "-" + hashHex(data) + _, err = WriteArtifact(root, origin, KindMeta, name, data) + require.Error(t, err) + assert.ErrorIs(t, err, ErrArtifactInvalid) + }) + } +} + +func TestPeerArtifactStoresFutureVersionArtifacts(t *testing.T) { + root := t.TempDir() + origin := "peer-a1b2c3" + + checkpointData, err := canonicalJSON(checkpoint{ + Version: formatVersion + 1, + Origin: origin, + Sequence: 1, + Sessions: map[string]string{ + origin + "~sess-1": strings64("a"), + }, + }) + require.NoError(t, err) + res, err := WriteArtifact(root, origin, KindCheckpoints, "cp-0000000001", checkpointData) + require.NoError(t, err) + assert.False(t, res.Duplicate) + + got, err := ReadArtifact(root, origin, KindCheckpoints, "cp-0000000001") + require.NoError(t, err) + assert.Equal(t, checkpointData, got.Data) + + segmentData, err := canonicalJSON(segmentMessage{ + Version: formatVersion + 1, + Ordinal: 0, + Role: "user", + Content: "future segment", + }) + require.NoError(t, err) + compressedSegment := compressPeerTestData(t, segmentData) + segmentHash := hashHex(segmentData) + res, err = WriteArtifact(root, origin, KindSegments, segmentHash, compressedSegment) + require.NoError(t, err) + assert.Equal(t, segmentHash+segmentExtension, res.Name) + got, err = ReadArtifact(root, origin, KindSegments, segmentHash) + require.NoError(t, err) + assert.Equal(t, compressedSegment, got.Data) + + futureManifestData, err := canonicalJSON(struct { + Version int `json:"v"` + Origin string `json:"origin"` + Future string `json:"future"` + }{ + Version: formatVersion + 1, + Origin: origin, + Future: "schema-owned-by-newer-peer", + }) + require.NoError(t, err) + compressedManifest := compressPeerTestData(t, futureManifestData) + manifestHash := hashHex(futureManifestData) + res, err = WriteArtifact(root, origin, KindManifests, manifestHash, compressedManifest) + require.NoError(t, err) + assert.Equal(t, manifestHash+manifestExtension, res.Name) + got, err = ReadArtifact(root, origin, KindManifests, manifestHash) + require.NoError(t, err) + assert.Equal(t, compressedManifest, got.Data) + + hlc := "2026-06-14T010203.000000001Z-peer-a1b2c3" + metadataData, err := canonicalJSON(metadataEvent{ + Version: formatVersion + 1, + HLC: hlc, + Origin: origin, + SessionGID: origin + "~sess-1", + Op: "future_op", + Value: json.RawMessage(`{"future":true}`), + }) + require.NoError(t, err) + metadataHash := hashHex(metadataData) + metadataName := hlc + "-" + metadataHash + res, err = WriteArtifact(root, origin, KindMeta, metadataName, metadataData) + require.NoError(t, err) + assert.Equal(t, metadataName+metadataEventExtension, res.Name) + got, err = ReadArtifact(root, origin, KindMeta, metadataName) + require.NoError(t, err) + assert.Equal(t, metadataData, got.Data) +} + +func TestPeerArtifactRejectsMixedFutureSegmentVersions(t *testing.T) { + root := t.TempDir() + origin := "peer-a1b2c3" + futureLine, err := canonicalJSON(segmentMessage{ + Version: formatVersion + 1, + Ordinal: 0, + Role: "user", + Content: "future", + }) + require.NoError(t, err) + currentLine, err := canonicalJSON(segmentMessage{ + Version: formatVersion, + Ordinal: 1, + Role: "assistant", + Content: "current", + }) + require.NoError(t, err) + segmentData := append(futureLine, currentLine...) + compressed := compressPeerTestData(t, segmentData) + hash := hashHex(segmentData) + + _, err = WriteArtifact(root, origin, KindSegments, hash, compressed) + require.Error(t, err) + assert.ErrorIs(t, err, ErrArtifactInvalid) +} + +func compressPeerTestData(t *testing.T, data []byte) []byte { + t.Helper() + var buf bytes.Buffer + enc, err := zstd.NewWriter(&buf) + require.NoError(t, err) + _, err = enc.Write(data) + require.NoError(t, err) + require.NoError(t, enc.Close()) + return buf.Bytes() +} + +func strings64(ch string) string { + return strings.Repeat(ch, 64) +} + +func TestCheckpointSummarySkipsAndQuarantinesCorruptLatestCheckpoint(t *testing.T) { + ctx := context.Background() + root := t.TempDir() + origin := "laptop-a1b2c3" + database := testDB(t) + seedSession(t, database, "sess-1", "alpha") + + _, err := Export(ctx, database, root, origin) + require.NoError(t, err) + seedSession(t, database, "sess-2", "beta") + _, err = Export(ctx, database, root, origin) + require.NoError(t, err) + + latest := filepath.Join(root, origin, "checkpoints", "cp-0000000002.json") + require.NoError(t, os.WriteFile(latest, []byte("not json"), 0o644)) + + // The summary falls back to the newest checkpoint that still decodes and + // quarantines the corrupt one, instead of failing the peers status view. + summary, err := CheckpointSummary(root, origin) + require.NoError(t, err) + require.True(t, summary.Found) + assert.Equal(t, 1, summary.Sequence) + assert.Equal(t, 1, summary.SessionCount) + assert.NoFileExists(t, latest) + assert.FileExists(t, latest+quarantineSuffix) +} diff --git a/internal/artifact/replay.go b/internal/artifact/replay.go new file mode 100644 index 000000000..5f007be63 --- /dev/null +++ b/internal/artifact/replay.go @@ -0,0 +1,313 @@ +package artifact + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io/fs" + "log" + "os" + "path/filepath" + "sort" + "strings" + + "go.kenn.io/agentsview/internal/db" +) + +type metadataArtifact struct { + path string + orderKey string + hash string + hlc string + event metadataEvent +} + +func replayMetadata( + ctx context.Context, + database *db.DB, + clock *HLCClock, + originRoot, origin, localOrigin string, + appliedEvents map[db.MetadataEventIdentity]struct{}, +) (int, error) { + events, err := readMetadataArtifacts(originRoot, origin, appliedEvents) + if err != nil { + return 0, err + } + changed := 0 + for _, art := range events { + if err := ctx.Err(); err != nil { + return changed, err + } + stamp, perr := ParseHLCTimestamp(art.hlc) + if perr != nil { + // A future format version may change the HLC shape; defer such + // events for an upgraded app instead of discarding them. + if art.event.Version > formatVersion { + continue + } + // LWW ordering is a raw order-key comparison, so an event whose + // HLC does not parse could sort above every real timestamp and + // win permanently. The artifact is immutable; mark it applied so + // it can never enter replay ordering. + log.Printf("artifact: skipping metadata event %s with invalid HLC: %v", art.path, perr) + if err := database.MarkMetadataEventApplied(ctx, origin, art.orderKey, art.hash); err != nil { + return changed, err + } + continue + } + // Advance the local HLC past this remote event before applying it, so a + // later local edit is causally ahead of the peer. If the clock cannot be + // advanced (the remote wall time is beyond the drift bound), defer the + // event rather than dragging local state to a value a future local edit + // could not out-order; a later run retries once wall time advances. + if clock != nil { + if _, oerr := clock.Observe(stamp); oerr != nil { + continue + } + } + if err := validateMetadataArtifactEvent(art, origin); err != nil { + if errors.Is(err, errFutureArtifactVersion) { + continue + } + return changed, err + } + if err := validateMetadataOp(art.event.Op); err != nil { + if err := database.MarkMetadataEventApplied(ctx, origin, art.orderKey, art.hash); err != nil { + return changed, err + } + continue + } + projection, err := metadataProjection(art, localOrigin) + if err != nil { + // A known op with a malformed payload can never become + // applicable: the artifact is immutable. Mark it applied so + // one bad event cannot permanently block the origin's replay. + log.Printf( + "artifact: skipping malformed metadata event %s: %v", + art.path, err, + ) + if err := database.MarkMetadataEventApplied(ctx, origin, art.orderKey, art.hash); err != nil { + return changed, err + } + continue + } + res, err := database.ApplyMetadataProjection(ctx, projection) + if err != nil { + // The event targets a session or message that is not durable + // locally yet. Skip only this event, leaving it unapplied so a + // later run retries it, and keep replaying the rest of the origin. + if errors.Is(err, db.ErrMetadataTargetUnavailable) { + continue + } + return changed, fmt.Errorf("replaying metadata event %s: %w", art.path, err) + } + if res.Applied || res.Conflict { + changed++ + } + } + return changed, nil +} + +func readMetadataArtifacts( + originRoot, origin string, + appliedEvents map[db.MetadataEventIdentity]struct{}, +) ([]metadataArtifact, error) { + paths, err := filepath.Glob(filepath.Join(originRoot, "meta", "*"+metadataEventExtension)) + if err != nil { + return nil, err + } + events := make([]metadataArtifact, 0, len(paths)) + for _, path := range paths { + orderKey, keyErr := metadataArtifactOrderKey(path) + if keyErr == nil { + identity := db.MetadataEventIdentity{Origin: origin, OrderKey: orderKey} + if _, ok := appliedEvents[identity]; ok { + continue + } + } + art, err := readMetadataArtifact(path) + if err != nil { + if errors.Is(err, fs.ErrNotExist) { + continue + } + if errors.Is(err, errCorruptArtifact) { + quarantineArtifact(path) + } + log.Printf("artifact: skipping unreadable metadata event %s for %s: %v", path, origin, err) + continue + } + events = append(events, art) + } + sort.Slice(events, func(i, j int) bool { + return events[i].orderKey < events[j].orderKey + }) + return events, nil +} + +func readMetadataArtifact(path string) (metadataArtifact, error) { + name, err := metadataArtifactOrderKey(path) + if err != nil { + return metadataArtifact{}, err + } + idx := strings.LastIndex(name, "-") + if idx < 0 { + return metadataArtifact{}, fmt.Errorf("metadata artifact %s missing hash suffix", filepath.Base(path)) + } + hlc := name[:idx] + hash := name[idx+1:] + data, err := os.ReadFile(path) + if err != nil { + return metadataArtifact{}, err + } + if got := hashHex(data); got != hash { + return metadataArtifact{}, fmt.Errorf("%w: metadata artifact %s hash mismatch: got %s", errCorruptArtifact, filepath.Base(path), got) + } + var event metadataEvent + if err := json.Unmarshal(data, &event); err != nil { + return metadataArtifact{}, fmt.Errorf("decoding metadata artifact %s: %w", filepath.Base(path), err) + } + return metadataArtifact{ + path: path, + orderKey: name, + hash: hash, + hlc: hlc, + event: event, + }, nil +} + +func metadataArtifactOrderKey(path string) (string, error) { + base := filepath.Base(path) + name := strings.TrimSuffix(base, metadataEventExtension) + if name == base { + return "", fmt.Errorf("metadata artifact %s missing %s extension", base, metadataEventExtension) + } + return name, nil +} + +func validateMetadataArtifactEvent(art metadataArtifact, origin string) error { + if art.event.HLC != art.hlc { + return fmt.Errorf("metadata event %s HLC mismatch: got %q", art.path, art.event.HLC) + } + if art.event.Origin != origin { + return fmt.Errorf( + "metadata event %s origin mismatch for %s: got %q", + art.path, origin, art.event.Origin, + ) + } + if art.event.SessionGID == "" { + return fmt.Errorf("metadata event %s has empty session GID", art.path) + } + if art.event.Version > formatVersion { + return fmt.Errorf( + "%w: metadata event %s has artifact version %d", + errFutureArtifactVersion, art.path, art.event.Version, + ) + } + if art.event.Version != formatVersion { + return fmt.Errorf( + "metadata event %s has unsupported artifact version %d", + art.path, art.event.Version, + ) + } + // Checked after the version gate: a future format may change the HLC + // shape, but a current-version event with an unparseable HLC would + // poison raw order-key LWW comparison and must never be accepted. + if _, err := ParseHLCTimestamp(art.hlc); err != nil { + return fmt.Errorf("metadata event %s has invalid HLC: %v", art.path, err) + } + return nil +} + +func metadataProjection(art metadataArtifact, localOrigin string) (db.MetadataProjection, error) { + event := art.event + field, value, displayName, pin, err := metadataProjectionFields(event) + if err != nil { + return db.MetadataProjection{}, err + } + return db.MetadataProjection{ + EventOrigin: event.Origin, + OrderKey: art.orderKey, + HLC: event.HLC, + ArtifactHash: art.hash, + SessionGID: event.SessionGID, + LocalSessionID: metadataLocalSessionID(localOrigin, event.SessionGID), + Field: field, + Op: event.Op, + Value: value, + DisplayName: displayName, + Pin: pin, + }, nil +} + +func metadataProjectionFields( + event metadataEvent, +) (field string, value string, displayName *string, pin *db.MetadataPinProjection, err error) { + switch event.Op { + case MetadataOpRename: + var payload struct { + DisplayName *string `json:"display_name"` + } + if err := json.Unmarshal(event.Value, &payload); err != nil { + return "", "", nil, nil, fmt.Errorf("decoding rename metadata value: %w", err) + } + value, err := metadataCanonicalValue(event.Value) + return "display_name", value, payload.DisplayName, nil, err + case MetadataOpSoftDelete, MetadataOpRestore: + return "deleted_at", event.Op, nil, nil, nil + case MetadataOpStar, MetadataOpUnstar: + return "starred", event.Op, nil, nil, nil + case MetadataOpPin, MetadataOpUnpin: + if event.Pin == nil { + return "", "", nil, nil, fmt.Errorf("%s metadata event missing pin payload", event.Op) + } + value, err := metadataCanonicalPin(*event.Pin) + if err != nil { + return "", "", nil, nil, err + } + return "pin:" + metadataPinAnchor(*event.Pin), value, nil, &db.MetadataPinProjection{ + SourceUUID: event.Pin.SourceUUID, + Ordinal: event.Pin.Ordinal, + Note: event.Pin.Note, + }, nil + case MetadataOpPurge: + return "purge", event.Op, nil, nil, nil + default: + return "", "", nil, nil, fmt.Errorf("unsupported metadata event op %q", event.Op) + } +} + +func metadataLocalSessionID(localOrigin, gid string) string { + prefix := localOrigin + "~" + if after, ok := strings.CutPrefix(gid, prefix); ok { + return after + } + return gid +} + +func metadataPinAnchor(pin MetadataPin) string { + if pin.SourceUUID != "" { + return "source_uuid:" + pin.SourceUUID + } + return fmt.Sprintf("ordinal:%d", pin.Ordinal) +} + +func metadataCanonicalValue(raw json.RawMessage) (string, error) { + if len(bytes.TrimSpace(raw)) == 0 { + return "", nil + } + data, err := canonicalJSON(raw) + if err != nil { + return "", err + } + return string(bytes.TrimSpace(data)), nil +} + +func metadataCanonicalPin(pin MetadataPin) (string, error) { + data, err := canonicalJSON(pin) + if err != nil { + return "", err + } + return string(bytes.TrimSpace(data)), nil +} diff --git a/internal/artifact/replay_test.go b/internal/artifact/replay_test.go new file mode 100644 index 000000000..cbbc1f926 --- /dev/null +++ b/internal/artifact/replay_test.go @@ -0,0 +1,770 @@ +package artifact + +import ( + "context" + "database/sql" + "encoding/json" + "os" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.kenn.io/agentsview/internal/db" +) + +func TestImportReplaysMetadataRenameDeterministically(t *testing.T) { + ctx := context.Background() + origin := "laptop-a1b2c3" + localOrigin := "desktop-d4e5f6" + gid := origin + "~sess-1" + stamp := replayTestHLC(0, 7) + events := []metadataEvent{ + replayRenameEvent(t, origin, gid, stamp, "Alpha"), + replayRenameEvent(t, origin, gid, stamp, "Beta"), + } + + for _, tc := range []struct { + name string + writeOrder []int + }{ + {name: "forward write order", writeOrder: []int{0, 1}}, + {name: "reverse write order", writeOrder: []int{1, 0}}, + } { + t.Run(tc.name, func(t *testing.T) { + root := t.TempDir() + exportDB := testDB(t) + importDB := testDB(t) + seedSession(t, exportDB, "sess-1", "alpha") + _, err := Export(ctx, exportDB, root, origin) + require.NoError(t, err) + + arts := make([]metadataArtifact, len(events)) + for _, idx := range tc.writeOrder { + arts[idx] = writeMetadataArtifact(t, filepath.Join(root, origin), events[idx]) + } + wantName := "Alpha" + if arts[1].orderKey > arts[0].orderKey { + wantName = "Beta" + } + + imported, messages, err := Import(ctx, importDB, root, localOrigin) + require.NoError(t, err) + assert.Equal(t, 1, imported) + assert.Equal(t, 2, messages) + + got, err := importDB.GetSession(ctx, gid) + require.NoError(t, err) + require.NotNil(t, got) + require.NotNil(t, got.DisplayName) + assert.Equal(t, wantName, *got.DisplayName) + assertMetadataConflictCount(t, importDB, gid, "display_name", 0) + assert.Equal(t, 2, metadataAppliedCount(t, importDB, origin)) + + imported, messages, err = Import(ctx, importDB, root, localOrigin) + require.NoError(t, err) + assert.Zero(t, imported) + assert.Zero(t, messages) + assertMetadataConflictCount(t, importDB, gid, "display_name", 0) + assert.Equal(t, 2, metadataAppliedCount(t, importDB, origin)) + }) + } +} + +func TestImportAppliesEarlierOriginMetadataToLaterOriginContentInOnePass(t *testing.T) { + ctx := context.Background() + metadataOrigin := "alpha-a1b2c3" + contentOrigin := "zulu-d4e5f6" + localOrigin := "desktop-c7d8e9" + root := t.TempDir() + exportDB := testDB(t) + importDB := testDB(t) + seedSession(t, exportDB, "sess-1", "alpha") + _, err := Export(ctx, exportDB, root, contentOrigin) + require.NoError(t, err) + + gid := contentOrigin + "~sess-1" + writeMetadataArtifact(t, filepath.Join(root, metadataOrigin), replayRenameEvent( + t, metadataOrigin, gid, replayTestHLC(0, 0), "Renamed before import", + )) + writeMetadataArtifact(t, filepath.Join(root, metadataOrigin), metadataEvent{ + Version: formatVersion, + HLC: replayTestHLC(time.Nanosecond, 0), + Origin: metadataOrigin, + SessionGID: gid, + Op: MetadataOpSoftDelete, + }) + + res, err := ImportDetailed(ctx, importDB, root, localOrigin) + require.NoError(t, err) + assert.Equal(t, 1, res.Sessions) + assert.Equal(t, 2, res.Messages) + assert.Equal(t, 2, res.Metadata) + + var displayName, deletedAt sql.NullString + err = importDB.Reader().QueryRowContext(ctx, + `SELECT display_name, deleted_at FROM sessions WHERE id = ?`, gid, + ).Scan(&displayName, &deletedAt) + require.NoError(t, err) + require.True(t, displayName.Valid) + assert.Equal(t, "Renamed before import", displayName.String) + require.True(t, deletedAt.Valid) + assert.Equal(t, "2026-06-14T01:02:03.000Z", deletedAt.String) + assert.Equal(t, 2, metadataAppliedCount(t, importDB, metadataOrigin)) +} + +func TestImportSkipsUnknownMetadataOpAndContinues(t *testing.T) { + ctx := context.Background() + origin := "laptop-a1b2c3" + localOrigin := "desktop-d4e5f6" + root := t.TempDir() + gid := origin + "~sess-1" + exportDB := testDB(t) + importDB := testDB(t) + seedSession(t, exportDB, "sess-1", "alpha") + _, err := Export(ctx, exportDB, root, origin) + require.NoError(t, err) + + unknown := metadataEvent{ + Version: formatVersion, + HLC: replayTestHLC(0, 0), + Origin: origin, + SessionGID: gid, + Op: "future_tag", + Value: json.RawMessage(`{"tag":"later"}`), + } + writeMetadataArtifact(t, filepath.Join(root, origin), unknown) + writeMetadataArtifact(t, filepath.Join(root, origin), + replayRenameEvent(t, origin, gid, replayTestHLC(time.Nanosecond, 0), "Known winner")) + + _, _, err = Import(ctx, importDB, root, localOrigin) + require.NoError(t, err) + + got, err := importDB.GetSession(ctx, gid) + require.NoError(t, err) + require.NotNil(t, got) + require.NotNil(t, got.DisplayName) + assert.Equal(t, "Known winner", *got.DisplayName) + assert.Equal(t, 2, metadataAppliedCount(t, importDB, origin)) +} + +// A known op with a malformed payload (here: pin without its pin +// payload) is immutable and can never become applicable, so replay +// must quarantine it as applied instead of aborting the origin. +func TestImportQuarantinesMalformedKnownOpAndContinues(t *testing.T) { + ctx := context.Background() + origin := "laptop-a1b2c3" + localOrigin := "desktop-d4e5f6" + root := t.TempDir() + gid := origin + "~sess-1" + exportDB := testDB(t) + importDB := testDB(t) + seedSession(t, exportDB, "sess-1", "alpha") + _, err := Export(ctx, exportDB, root, origin) + require.NoError(t, err) + + malformed := metadataEvent{ + Version: formatVersion, + HLC: replayTestHLC(0, 0), + Origin: origin, + SessionGID: gid, + Op: MetadataOpPin, + } + writeMetadataArtifact(t, filepath.Join(root, origin), malformed) + writeMetadataArtifact(t, filepath.Join(root, origin), + replayRenameEvent(t, origin, gid, replayTestHLC(time.Nanosecond, 0), "Known winner")) + + _, _, err = Import(ctx, importDB, root, localOrigin) + require.NoError(t, err) + + got, err := importDB.GetSession(ctx, gid) + require.NoError(t, err) + require.NotNil(t, got) + require.NotNil(t, got.DisplayName) + assert.Equal(t, "Known winner", *got.DisplayName) + // Both events are marked applied so replay never retries the + // malformed one. + assert.Equal(t, 2, metadataAppliedCount(t, importDB, origin)) + + // A later sync neither errors nor reprocesses the malformed event. + _, _, err = Import(ctx, importDB, root, localOrigin) + require.NoError(t, err) + assert.Equal(t, 2, metadataAppliedCount(t, importDB, origin)) +} + +func TestImportSkipsAndQuarantinesCorruptMetadataEvent(t *testing.T) { + ctx := context.Background() + origin := "laptop-a1b2c3" + localOrigin := "desktop-d4e5f6" + root := t.TempDir() + gid := origin + "~sess-1" + exportDB := testDB(t) + importDB := testDB(t) + seedSession(t, exportDB, "sess-1", "alpha") + _, err := Export(ctx, exportDB, root, origin) + require.NoError(t, err) + + valid := writeMetadataArtifact(t, filepath.Join(root, origin), + replayRenameEvent(t, origin, gid, replayTestHLC(0, 0), "Winner")) + // A corrupt event: well-formed name whose hash no longer matches the + // file bytes, as bit rot or an interrupted file-sync write produces. + corrupt := replayRenameEvent(t, origin, gid, replayTestHLC(time.Nanosecond, 0), "PWNED") + data, err := canonicalJSON(corrupt) + require.NoError(t, err) + stamp, err := ParseHLCTimestamp(corrupt.HLC) + require.NoError(t, err) + corruptPath := filepath.Join(root, origin, "meta", + stamp.OrderingKey(hashHex([]byte("other")))+metadataEventExtension) + require.NoError(t, writeFileAtomic(corruptPath, data, 0o644)) + + imported, messages, err := Import(ctx, importDB, root, localOrigin) + require.NoError(t, err) + assert.Equal(t, 1, imported) + assert.Equal(t, 2, messages) + + got, err := importDB.GetSession(ctx, gid) + require.NoError(t, err) + require.NotNil(t, got) + require.NotNil(t, got.DisplayName) + assert.Equal(t, "Winner", *got.DisplayName) + assert.Equal(t, 1, metadataAppliedCount(t, importDB, origin)) + + assert.NoFileExists(t, corruptPath) + assert.FileExists(t, corruptPath+quarantineSuffix) + assert.FileExists(t, valid.path) + + // A later sync neither errors nor reprocesses the corrupt event. + _, _, err = Import(ctx, importDB, root, localOrigin) + require.NoError(t, err) + assert.Equal(t, 1, metadataAppliedCount(t, importDB, origin)) +} + +func TestImportSkipsUnparseableHLCMetadataEvent(t *testing.T) { + ctx := context.Background() + origin := "laptop-a1b2c3" + localOrigin := "desktop-d4e5f6" + root := t.TempDir() + gid := origin + "~sess-1" + exportDB := testDB(t) + importDB := testDB(t) + seedSession(t, exportDB, "sess-1", "alpha") + _, err := Export(ctx, exportDB, root, origin) + require.NoError(t, err) + + // A forged event whose HLC does not parse but whose filename hash is + // valid. Its order key sorts lexicographically above every real + // timestamp, so if accepted it would win LWW permanently (#1034). + forgedHLC := "9999-99-99T999999.000000000Z-00000000000000000000" + forged := replayRenameEvent(t, origin, gid, forgedHLC, "PWNED") + data, err := canonicalJSON(forged) + require.NoError(t, err) + forgedPath := filepath.Join(root, origin, "meta", + forgedHLC+"-"+hashHex(data)+metadataEventExtension) + require.NoError(t, writeFileAtomic(forgedPath, data, 0o644)) + writeMetadataArtifact(t, filepath.Join(root, origin), + replayRenameEvent(t, origin, gid, replayTestHLC(0, 0), "Legit")) + + imported, messages, err := Import(ctx, importDB, root, localOrigin) + require.NoError(t, err) + assert.Equal(t, 1, imported) + assert.Equal(t, 2, messages) + + got, err := importDB.GetSession(ctx, gid) + require.NoError(t, err) + require.NotNil(t, got) + require.NotNil(t, got.DisplayName) + assert.Equal(t, "Legit", *got.DisplayName) + // Both events are marked applied so replay never retries the forged one. + assert.Equal(t, 2, metadataAppliedCount(t, importDB, origin)) + + // Repeat imports stay clean and never resurrect the forged value. + _, _, err = Import(ctx, importDB, root, localOrigin) + require.NoError(t, err) + got, err = importDB.GetSession(ctx, gid) + require.NoError(t, err) + require.NotNil(t, got) + require.NotNil(t, got.DisplayName) + assert.Equal(t, "Legit", *got.DisplayName) + assert.Equal(t, 2, metadataAppliedCount(t, importDB, origin)) +} + +func TestImportSoftDeleteUsesEventHLCForDeletedAt(t *testing.T) { + ctx := context.Background() + origin := "laptop-a1b2c3" + localOrigin := "desktop-d4e5f6" + root := t.TempDir() + gid := origin + "~sess-1" + exportDB := testDB(t) + importDB := testDB(t) + seedSession(t, exportDB, "sess-1", "alpha") + _, err := Export(ctx, exportDB, root, origin) + require.NoError(t, err) + + writeMetadataArtifact(t, filepath.Join(root, origin), metadataEvent{ + Version: formatVersion, + HLC: replayTestHLC(0, 0), + Origin: origin, + SessionGID: gid, + Op: MetadataOpSoftDelete, + }) + + _, _, err = Import(ctx, importDB, root, localOrigin) + require.NoError(t, err) + + var deletedAt sql.NullString + err = importDB.Reader().QueryRowContext(ctx, + `SELECT deleted_at FROM sessions WHERE id = ?`, gid).Scan(&deletedAt) + require.NoError(t, err) + require.True(t, deletedAt.Valid) + // Trash retention must be anchored to the author-side event time, not the + // moment each machine happened to import the event, or retention windows + // drift across the fleet. + assert.Equal(t, "2026-06-14T01:02:03.000Z", deletedAt.String) +} + +func TestWriteArtifactRejectsUnparseableHLCMetadataEvent(t *testing.T) { + root := t.TempDir() + origin := "laptop-a1b2c3" + gid := origin + "~sess-1" + forgedHLC := "9999-99-99T999999.000000000Z-00000000000000000000" + forged := replayRenameEvent(t, origin, gid, forgedHLC, "PWNED") + data, err := canonicalJSON(forged) + require.NoError(t, err) + name := forgedHLC + "-" + hashHex(data) + metadataEventExtension + + _, err = WriteArtifact(root, origin, KindMeta, name, data) + require.Error(t, err) + assert.ErrorIs(t, err, ErrArtifactInvalid) +} + +func TestImportDefersFutureVersionMetadataEventAndContinues(t *testing.T) { + ctx := context.Background() + origin := "laptop-a1b2c3" + localOrigin := "desktop-d4e5f6" + root := t.TempDir() + gid := origin + "~sess-1" + exportDB := testDB(t) + importDB := testDB(t) + seedSession(t, exportDB, "sess-1", "alpha") + _, err := Export(ctx, exportDB, root, origin) + require.NoError(t, err) + + future := metadataEvent{ + Version: formatVersion + 1, + HLC: replayTestHLC(0, 0), + Origin: origin, + SessionGID: gid, + Op: "future_tag", + Value: json.RawMessage(`{"tag":"later"}`), + } + writeMetadataArtifact(t, filepath.Join(root, origin), future) + writeMetadataArtifact(t, filepath.Join(root, origin), + replayRenameEvent(t, origin, gid, replayTestHLC(time.Nanosecond, 0), "Known winner")) + + _, _, err = Import(ctx, importDB, root, localOrigin) + require.NoError(t, err) + + got, err := importDB.GetSession(ctx, gid) + require.NoError(t, err) + require.NotNil(t, got) + require.NotNil(t, got.DisplayName) + assert.Equal(t, "Known winner", *got.DisplayName) + assert.Equal(t, 1, metadataAppliedCount(t, importDB, origin)) +} + +func TestImportObservesFutureVersionMetadataHLCBeforeDeferring(t *testing.T) { + ctx := context.Background() + origin := "laptop-a1b2c3" + localOrigin := "desktop-d4e5f6" + root := t.TempDir() + importDB := testDB(t) + now := fixedHLCTime() + remote := HLCTimestamp{WallTime: now.Add(time.Minute)} + event := metadataEvent{ + Version: formatVersion + 1, + HLC: remote.String(), + Origin: origin, + SessionGID: origin + "~sess-1", + Op: "future_tag", + Value: json.RawMessage(`{"tag":"later"}`), + } + writeMetadataArtifact(t, filepath.Join(root, origin), event) + + clock := NewHLCClock(importDB, HLCClockOptions{ + Now: func() time.Time { return now }, + }) + _, err := importDetailed(ctx, importDB, clock, root, localOrigin) + require.NoError(t, err) + + next, err := clock.Next() + require.NoError(t, err) + assert.Positive(t, next.Compare(remote), + "a local edit after deferring a future-version event must sort after that peer event") + assert.Equal(t, 0, metadataAppliedCount(t, importDB, origin)) +} + +func TestImportAlreadyAppliedMetadataDoesNotAdvanceResetHLC(t *testing.T) { + ctx := context.Background() + origin := "laptop-a1b2c3" + localOrigin := "desktop-d4e5f6" + root := t.TempDir() + exportDB := testDB(t) + importDB := testDB(t) + seedSession(t, exportDB, "sess-1", "alpha") + _, err := Export(ctx, exportDB, root, origin) + require.NoError(t, err) + + now := fixedHLCTime() + remote := HLCTimestamp{WallTime: now.Add(time.Minute)} + writeMetadataArtifact(t, filepath.Join(root, origin), + replayRenameEvent(t, origin, origin+"~sess-1", remote.String(), "Renamed")) + clock := NewHLCClock(importDB, HLCClockOptions{Now: func() time.Time { return now }}) + _, err = importDetailed(ctx, importDB, clock, root, localOrigin) + require.NoError(t, err) + assert.Equal(t, 1, metadataAppliedCount(t, importDB, origin)) + + older := HLCTimestamp{WallTime: now.Add(-time.Minute)}.String() + require.NoError(t, importDB.SetSyncState(metadataHLCStateKey, older)) + clock = NewHLCClock(importDB, HLCClockOptions{Now: func() time.Time { return now }}) + _, err = importDetailed(ctx, importDB, clock, root, localOrigin) + require.NoError(t, err) + + persisted, err := importDB.GetSyncState(metadataHLCStateKey) + require.NoError(t, err) + assert.Equal(t, older, persisted, + "an already-applied event must be skipped before its HLC is observed") +} + +func TestImportSkipsAlreadyAppliedMetadataBeforeReadingArtifact(t *testing.T) { + ctx := context.Background() + origin := "laptop-a1b2c3" + localOrigin := "desktop-d4e5f6" + root := t.TempDir() + exportDB := testDB(t) + importDB := testDB(t) + seedSession(t, exportDB, "sess-1", "alpha") + _, err := Export(ctx, exportDB, root, origin) + require.NoError(t, err) + art := writeMetadataArtifact(t, filepath.Join(root, origin), + replayRenameEvent(t, origin, origin+"~sess-1", replayTestHLC(0, 0), "Renamed")) + _, _, err = Import(ctx, importDB, root, localOrigin) + require.NoError(t, err) + assert.Equal(t, 1, metadataAppliedCount(t, importDB, origin)) + + require.NoError(t, os.WriteFile(art.path, []byte("corrupt after apply"), 0o644)) + _, _, err = Import(ctx, importDB, root, localOrigin) + require.NoError(t, err) + + assert.FileExists(t, art.path, + "applied identity should be skipped before reading the artifact bytes") + assert.NoFileExists(t, art.path+quarantineSuffix) +} + +func TestImportDoesNotAdvanceMetadataWatermarkWhenTargetMissing(t *testing.T) { + ctx := context.Background() + origin := "laptop-a1b2c3" + localOrigin := "desktop-d4e5f6" + root := t.TempDir() + originRoot := filepath.Join(root, origin) + importDB := testDB(t) + writeMetadataArtifact(t, originRoot, + replayRenameEvent(t, origin, localOrigin+"~sess-1", replayTestHLC(0, 0), "Remote rename")) + + imported, messages, err := Import(ctx, importDB, root, localOrigin) + require.NoError(t, err) + assert.Zero(t, imported) + assert.Zero(t, messages) + assert.Equal(t, 0, metadataAppliedCount(t, importDB, origin)) + + seedSession(t, importDB, "sess-1", "alpha") + imported, messages, err = Import(ctx, importDB, root, localOrigin) + require.NoError(t, err) + assert.Zero(t, imported) + assert.Zero(t, messages) + + got, err := importDB.GetSession(ctx, "sess-1") + require.NoError(t, err) + require.NotNil(t, got) + require.NotNil(t, got.DisplayName) + assert.Equal(t, "Remote rename", *got.DisplayName) + assert.Equal(t, 1, metadataAppliedCount(t, importDB, origin)) +} + +func TestImportReplaysMetadataPinAndUnpin(t *testing.T) { + ctx := context.Background() + origin := "laptop-a1b2c3" + localOrigin := "desktop-d4e5f6" + root := t.TempDir() + gid := origin + "~sess-1" + exportDB := testDB(t) + importDB := testDB(t) + seedSession(t, exportDB, "sess-1", "alpha") + require.NoError(t, exportDB.ReplaceSessionMessages("sess-1", []db.Message{ + { + SessionID: "sess-1", + Ordinal: 0, + Role: "user", + Content: "hello", + ContentLength: 5, + SourceUUID: "uuid-question", + }, + { + SessionID: "sess-1", + Ordinal: 1, + Role: "assistant", + Content: "world", + ContentLength: 5, + SourceUUID: "uuid-answer", + }, + })) + _, err := Export(ctx, exportDB, root, origin) + require.NoError(t, err) + + note := "remember" + writeMetadataArtifact(t, filepath.Join(root, origin), metadataEvent{ + Version: formatVersion, + HLC: replayTestHLC(0, 0), + Origin: origin, + SessionGID: gid, + Op: MetadataOpPin, + Pin: &MetadataPin{ + SourceUUID: "uuid-answer", + Ordinal: 1, + Note: ¬e, + }, + }) + _, _, err = Import(ctx, importDB, root, localOrigin) + require.NoError(t, err) + pins, err := importDB.ListPinnedMessages(ctx, gid, "") + require.NoError(t, err) + require.Len(t, pins, 1) + assert.Equal(t, 1, pins[0].Ordinal) + require.NotNil(t, pins[0].Note) + assert.Equal(t, note, *pins[0].Note) + + writeMetadataArtifact(t, filepath.Join(root, origin), metadataEvent{ + Version: formatVersion, + HLC: replayTestHLC(time.Nanosecond, 0), + Origin: origin, + SessionGID: gid, + Op: MetadataOpUnpin, + Pin: &MetadataPin{ + SourceUUID: "uuid-answer", + Ordinal: 1, + }, + }) + _, _, err = Import(ctx, importDB, root, localOrigin) + require.NoError(t, err) + pins, err = importDB.ListPinnedMessages(ctx, gid, "") + require.NoError(t, err) + assert.Empty(t, pins) + assert.Equal(t, 2, metadataAppliedCount(t, importDB, origin)) +} + +func TestLocalMetadataEditBeatsLowerHLCPeerEvent(t *testing.T) { + ctx := context.Background() + root := t.TempDir() + dataDir := t.TempDir() + localOrigin := "desktop-d4e5f6" + peerOrigin := "laptop-a1b2c3" + gid := localOrigin + "~sess-1" + + database := testDB(t) + seedSession(t, database, "sess-1", "alpha") + // Simulate the local rename handler: mutate the session, then record the + // metadata event artifact and replay register entry. + require.NoError(t, database.RenameSession("sess-1", new("Local name"))) + + now := fixedHLCTime().Add(time.Hour) + recorder := NewMetadataRecorder(database, MetadataRecorderOptions{ + DataDir: dataDir, + Origin: localOrigin, + Now: func() time.Time { return now }, + }) + value, err := json.Marshal(struct { + DisplayName string `json:"display_name"` + }{DisplayName: "Local name"}) + require.NoError(t, err) + localRec, err := recorder.Append(ctx, MetadataEventInput{ + SessionID: "sess-1", + Op: MetadataOpRename, + Value: value, + }) + require.NoError(t, err) + localOrderKey := localRec.HLC + "-" + localRec.Hash + + // A peer renames the same desktop-originated session at an earlier HLC. + peerArt := writeMetadataArtifact(t, filepath.Join(root, peerOrigin), + replayRenameEvent(t, peerOrigin, gid, replayTestHLC(0, 0), "Peer name")) + require.Less(t, peerArt.orderKey, localOrderKey) + + res, err := ImportDetailed(ctx, database, root, localOrigin) + require.NoError(t, err) + assert.Equal(t, 1, res.Metadata) + + got, err := database.GetSession(ctx, "sess-1") + require.NoError(t, err) + require.NotNil(t, got.DisplayName) + assert.Equal(t, "Local name", *got.DisplayName) + assertMetadataConflict(t, database, gid, "display_name", localOrderKey, peerArt.orderKey) +} + +func TestImportContinuesPastUnavailableMetadataTarget(t *testing.T) { + ctx := context.Background() + root := t.TempDir() + localOrigin := "desktop-d4e5f6" + peerOrigin := "laptop-a1b2c3" + database := testDB(t) + seedSession(t, database, "sess-present", "alpha") + missingGID := localOrigin + "~sess-missing" + presentGID := localOrigin + "~sess-present" + + // The earlier event targets a session that is not durable locally; the + // later event targets an existing session and must still apply. + writeMetadataArtifact(t, filepath.Join(root, peerOrigin), + replayRenameEvent(t, peerOrigin, missingGID, replayTestHLC(0, 0), "Missing target")) + writeMetadataArtifact(t, filepath.Join(root, peerOrigin), + replayRenameEvent(t, peerOrigin, presentGID, replayTestHLC(time.Nanosecond, 0), "Present target")) + + res, err := ImportDetailed(ctx, database, root, localOrigin) + require.NoError(t, err) + assert.Equal(t, 1, res.Metadata) + + got, err := database.GetSession(ctx, "sess-present") + require.NoError(t, err) + require.NotNil(t, got.DisplayName) + assert.Equal(t, "Present target", *got.DisplayName) + // Only the present-target event is marked applied; the unavailable one is + // left for a later run to retry. + assert.Equal(t, 1, metadataAppliedCount(t, database, peerOrigin)) + + // Once the missing session exists, a later import applies its event too. + seedSession(t, database, "sess-missing", "beta") + res, err = ImportDetailed(ctx, database, root, localOrigin) + require.NoError(t, err) + assert.Equal(t, 1, res.Metadata) + got, err = database.GetSession(ctx, "sess-missing") + require.NoError(t, err) + require.NotNil(t, got.DisplayName) + assert.Equal(t, "Missing target", *got.DisplayName) + assert.Equal(t, 2, metadataAppliedCount(t, database, peerOrigin)) +} + +func TestReplayDefersRemoteEventBeyondClockDrift(t *testing.T) { + ctx := context.Background() + root := t.TempDir() + dataDir := t.TempDir() + localOrigin := "desktop-d4e5f6" + peerOrigin := "laptop-a1b2c3" + gid := localOrigin + "~sess-1" + + database := testDB(t) + seedSession(t, database, "sess-1", "alpha") + + now := fixedHLCTime() + // A peer event whose wall time is an hour ahead — well beyond the default + // 5-minute drift bound, so the local clock cannot be advanced past it. + future := HLCTimestamp{WallTime: now.Add(time.Hour)} + writeMetadataArtifact(t, filepath.Join(root, peerOrigin), + replayRenameEvent(t, peerOrigin, gid, future.String(), "From the future")) + + recorder := NewMetadataRecorder(database, MetadataRecorderOptions{ + DataDir: dataDir, + Origin: localOrigin, + Now: func() time.Time { return now }, + }) + res, err := recorder.Import(ctx, root) + require.NoError(t, err) + assert.Zero(t, res.Metadata, "event beyond the drift bound must be deferred, not applied") + + // The local session is untouched and the event is left unapplied for a later + // run to retry once wall time catches up. + got, err := database.GetSession(ctx, "sess-1") + require.NoError(t, err) + require.NotNil(t, got) + require.NotNil(t, got.DisplayName) + assert.NotEqual(t, "From the future", *got.DisplayName, "deferred rename must not be applied") + assert.Equal(t, 0, metadataAppliedCount(t, database, peerOrigin)) +} + +func replayRenameEvent( + t *testing.T, + origin, gid, hlc, displayName string, +) metadataEvent { + t.Helper() + value, err := json.Marshal(struct { + DisplayName string `json:"display_name"` + }{ + DisplayName: displayName, + }) + require.NoError(t, err) + return metadataEvent{ + Version: formatVersion, + HLC: hlc, + Origin: origin, + SessionGID: gid, + Op: MetadataOpRename, + Value: value, + } +} + +func replayTestHLC(offset time.Duration, logical uint64) string { + return HLCTimestamp{WallTime: fixedHLCTime().Add(offset), Logical: logical}.String() +} + +func writeMetadataArtifact(t *testing.T, originRoot string, event metadataEvent) metadataArtifact { + t.Helper() + stamp, err := ParseHLCTimestamp(event.HLC) + require.NoError(t, err) + data, err := canonicalJSON(event) + require.NoError(t, err) + hash := hashHex(data) + path := filepath.Join(originRoot, "meta", stamp.OrderingKey(hash)+metadataEventExtension) + require.NoError(t, writeFileAtomic(path, data, 0o644)) + art, err := readMetadataArtifact(path) + require.NoError(t, err) + return art +} + +func assertMetadataConflict( + t *testing.T, + database *db.DB, + gid, field, wantWinning, wantLosing string, +) { + t.Helper() + var winning, losing string + err := database.Reader().QueryRowContext(context.Background(), + `SELECT winning_order_key, losing_order_key + FROM metadata_conflicts + WHERE session_gid = ? AND field = ?`, + gid, field, + ).Scan(&winning, &losing) + require.NoError(t, err) + assert.Equal(t, wantWinning, winning) + assert.Equal(t, wantLosing, losing) +} + +func assertMetadataConflictCount(t *testing.T, database *db.DB, gid, field string, want int) { + t.Helper() + var got int + err := database.Reader().QueryRowContext(context.Background(), + `SELECT COUNT(*) + FROM metadata_conflicts + WHERE session_gid = ? AND field = ?`, + gid, field, + ).Scan(&got) + require.NoError(t, err) + assert.Equal(t, want, got) +} + +func metadataAppliedCount(t *testing.T, database *db.DB, origin string) int { + t.Helper() + var count int + err := database.Reader().QueryRowContext(context.Background(), + `SELECT COUNT(*) FROM metadata_applied_events WHERE origin = ?`, + origin, + ).Scan(&count) + require.NoError(t, err) + return count +} diff --git a/internal/artifact/sync.go b/internal/artifact/sync.go new file mode 100644 index 000000000..98c76919b --- /dev/null +++ b/internal/artifact/sync.go @@ -0,0 +1,2991 @@ +package artifact + +import ( + "bytes" + "context" + "crypto/rand" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "io/fs" + "log" + "maps" + "net" + "os" + "path/filepath" + "reflect" + "slices" + "sort" + "strconv" + "strings" + "syscall" + "time" + + "github.com/klauspost/compress/zstd" + "go.kenn.io/agentsview/internal/config" + "go.kenn.io/agentsview/internal/db" +) + +const ( + formatVersion = 1 + originStateKey = "artifact_origin_id" + importStatePrefix = "artifact_import:" + exportStatePrefix = "artifact_export:" + tempFilePrefix = ".tmp-" + manifestExtension = ".json.zst" + segmentExtension = ".ndjson.zst" + + manifestDecodedLimit = int64(16 << 20) + segmentDecodedLimit = int64(64 << 20) + segmentTargetSize = int64(32 << 20) + // zstd.NewWriter documents an 8 MiB maximum default window. Matching it + // keeps existing package-written artifacts readable without accepting + // attacker-selected large decoder windows. + zstdMaxWindowSize = uint64(8 << 20) + + // Cardinality caps complement the byte caps: 4,096 records keeps one + // segment's decoded object graph bounded, while 32,768 records and 256 MiB + // leave ample room for unusually long sessions without letting many valid + // chunks amplify during aggregation. Sixteen references accommodate uneven + // 32 MiB chunks; the aggregate byte cap remains the final session bound. + maxManifestSegments = 16 + maxManifestUsageEvents = 32_768 + maxSegmentMessages = 4_096 + maxSessionMessages = 32_768 + maxSessionDecodedBytes = int64(256 << 20) + + // Nested collections need independent caps because compact empty objects can + // amplify far beyond the decoded byte budget when unmarshaled. A message may + // still describe unusually wide tool fan-out, and one tool may retain a long + // result history. Segment totals keep one decoded chunk modest; session totals + // allow eight full nested-budget segments, matching the message-count ratio. + maxMessageToolCalls = 256 + maxToolResultEvents = 1_024 + maxSegmentToolCalls = 8_192 + maxSegmentResultEvents = 32_768 + maxSessionToolCalls = 65_536 + maxSessionResultEvents = 262_144 +) + +// artifactLimits bounds decoded collection cardinality in addition to raw +// bytes. The production values are intentionally generous for real sessions +// while preventing small JSON records from amplifying into unbounded Go +// object graphs. +type artifactLimits struct { + manifestSegments int + manifestUsageEvents int + segmentMessages int + sessionMessages int + sessionDecodedBytes int64 + messageToolCalls int + toolResultEvents int + segmentToolCalls int + segmentResultEvents int + sessionToolCalls int + sessionResultEvents int +} + +func productionArtifactLimits() artifactLimits { + return artifactLimits{ + manifestSegments: maxManifestSegments, + manifestUsageEvents: maxManifestUsageEvents, + segmentMessages: maxSegmentMessages, + sessionMessages: maxSessionMessages, + sessionDecodedBytes: maxSessionDecodedBytes, + messageToolCalls: maxMessageToolCalls, + toolResultEvents: maxToolResultEvents, + segmentToolCalls: maxSegmentToolCalls, + segmentResultEvents: maxSegmentResultEvents, + sessionToolCalls: maxSessionToolCalls, + sessionResultEvents: maxSessionResultEvents, + } +} + +type nestedCollectionCounts struct { + toolCalls int + resultEvents int +} + +type segmentPreflight struct { + records [][]byte + nested nestedCollectionCounts +} + +func exceedsCollectionLimit(current, additional, limit int) bool { + return current > limit || additional > limit-current +} + +var errIncompleteArtifact = errors.New("incomplete artifact") + +// errCorruptArtifact reports an artifact whose bytes fail content validation: +// a hash mismatch or an undecodable compressed stream. Corrupt artifacts are +// quarantined and skipped so one bad file cannot abort sync permanently. +var errCorruptArtifact = errors.New("corrupt artifact") + +// quarantineSuffix is appended to a corrupt artifact's filename when it is +// quarantined. Quarantined files are ignored by every read path and are never +// mirrored by CopyUnion, so a re-fetched valid copy can take the original name. +const quarantineSuffix = ".corrupt" + +var errFutureArtifactVersion = errors.New("future artifact version") + +var writeFileAtomicBeforeCommit func(path string) +var writeFileAtomicLink = os.Link + +// SyncOptions configures a local-first artifact folder sync. +type SyncOptions struct { + DataDir string + Target string + Origin string + // Now is the wall-clock source for advancing the metadata HLC past + // observed remote events. When nil, time.Now is used. Sharing it with the + // local metadata recorder keeps import and local edits on one time base. + Now func() time.Time + // Token is the Bearer token for an HTTP peer target. It is ignored by + // folder and object-store targets. + Token string + // AllowInsecure permits plaintext HTTP to a non-loopback peer. Loopback + // HTTP remains allowed without this override. + AllowInsecure bool + // BaselineMetadata writes metadata events for existing local curation before + // exchanging artifacts. It is intended for first-time initialization. + BaselineMetadata bool + // OnDataChanged is called after a foreign import writes local rows. + OnDataChanged func() +} + +// Sync runs one artifact sync, selecting the transport from the target shape: +// an http(s):// URL uses the HTTP peer transport, anything else is treated as a +// local folder target. +func Sync(ctx context.Context, database *db.DB, opts SyncOptions) (SyncResult, error) { + if opts.Target == "" { + return SyncResult{}, errors.New("artifact sync target is required") + } + if IsHTTPTarget(opts.Target) { + tr, err := newHTTPTransport(opts.Target, opts.Token, opts.AllowInsecure) + if err != nil { + return SyncResult{}, err + } + return syncWithTransport(ctx, database, opts, tr) + } + if IsObjectTarget(opts.Target) { + tr, err := newObjectTransport(opts.Target, ObjectStoreOptionsFromEnv()) + if err != nil { + return SyncResult{}, err + } + return syncWithTransport(ctx, database, opts, tr) + } + return syncWithTransport(ctx, database, opts, &folderTransport{target: opts.Target}) +} + +// SyncResult summarizes a folder artifact sync run. +type SyncResult struct { + Origin string + ExportedSessions int + ImportedSessions int + ImportedMessages int + ImportedMetadata int +} + +// ImportResult summarizes local rows changed by artifact import. +type ImportResult struct { + Sessions int + Messages int + Metadata int + Deferred int +} + +// Changed reports whether the import wrote user-visible local data. +func (r ImportResult) Changed() bool { + return r.Sessions > 0 || r.Messages > 0 || r.Metadata > 0 +} + +// SyncFolder exports local sessions to the local artifact store, exchanges the +// store with target, and imports foreign origins from the exchanged artifacts. +func SyncFolder(ctx context.Context, database *db.DB, opts SyncOptions) (SyncResult, error) { + if opts.Target == "" { + return SyncResult{}, errors.New("artifact sync target is required") + } + return syncWithTransport(ctx, database, opts, &folderTransport{target: opts.Target}) +} + +// syncWithTransport runs one artifact sync over any transport: export local +// sessions, exchange the store with the remote via set-union, then import +// foreign origins. Folder, HTTP peer, and object-store targets differ only in +// the transport's Prepare and Exchange. +func syncWithTransport( + ctx context.Context, + database *db.DB, + opts SyncOptions, + tr Transport, +) (SyncResult, error) { + if opts.DataDir == "" { + return SyncResult{}, errors.New("artifact sync data dir is required") + } + localRoot := filepath.Join(opts.DataDir, "artifacts") + if err := tr.Prepare(ctx, localRoot); err != nil { + return SyncResult{}, err + } + if err := os.MkdirAll(localRoot, 0o755); err != nil { + return SyncResult{}, fmt.Errorf("creating local artifact store: %w", err) + } + origin := opts.Origin + if origin == "" { + var err error + origin, err = EnsureOrigin(database) + if err != nil { + return SyncResult{}, err + } + } else if err := validateOriginID(origin); err != nil { + return SyncResult{}, err + } + + clock := NewHLCClock(database, HLCClockOptions{Now: opts.Now}) + var imported ImportResult + var baselineSnapshot db.MetadataBaselineSnapshot + if opts.BaselineMetadata { + var err error + baselineSnapshot, err = database.MetadataBaselineSnapshot(ctx) + if err != nil { + return SyncResult{}, err + } + if err := tr.Exchange(ctx, localRoot); err != nil { + return SyncResult{}, err + } + preBaselineImported, err := importDetailed(ctx, database, clock, localRoot, origin) + if err != nil { + return SyncResult{}, err + } + imported.Sessions += preBaselineImported.Sessions + imported.Messages += preBaselineImported.Messages + imported.Metadata += preBaselineImported.Metadata + + recorder := NewMetadataRecorder(database, MetadataRecorderOptions{ + DataDir: opts.DataDir, + Origin: origin, + Now: opts.Now, + }) + if _, err := recorder.AppendBaselineSnapshot(ctx, baselineSnapshot); err != nil { + return SyncResult{}, err + } + } + exported, err := Export(ctx, database, localRoot, origin) + if err != nil { + return SyncResult{}, err + } + if err := tr.Exchange(ctx, localRoot); err != nil { + return SyncResult{}, err + } + postExportImported, err := importDetailed(ctx, database, clock, localRoot, origin) + if err != nil { + return SyncResult{}, err + } + imported.Sessions += postExportImported.Sessions + imported.Messages += postExportImported.Messages + imported.Metadata += postExportImported.Metadata + if imported.Changed() && opts.OnDataChanged != nil { + opts.OnDataChanged() + } + return SyncResult{ + Origin: origin, + ExportedSessions: exported, + ImportedSessions: imported.Sessions, + ImportedMessages: imported.Messages, + ImportedMetadata: imported.Metadata, + }, nil +} + +// EnsureOrigin returns the persisted origin ID, creating one when absent. +func EnsureOrigin(database *db.DB) (string, error) { + origin, err := StoredOrigin(database) + if err != nil { + return "", err + } + if origin != "" { + return origin, nil + } + origin, err = newOriginID() + if err != nil { + return "", err + } + if err := validateOriginID(origin); err != nil { + return "", fmt.Errorf("generated artifact origin: %w", err) + } + if err := database.SetSyncState(originStateKey, origin); err != nil { + return "", fmt.Errorf("persisting artifact origin: %w", err) + } + return origin, nil +} + +// AdoptOrigin persists origin as this machine's artifact origin in the database +// sync state so DB-derived lookups (EnsureOrigin and its callers) agree with the +// authoritative config origin. It validates the input and is idempotent: it only +// writes when the stored value differs. The config origin always wins, so a +// previously stored value is overwritten to converge on a single origin. +func AdoptOrigin(database *db.DB, origin string) error { + if err := validateOriginID(origin); err != nil { + return fmt.Errorf("adopting artifact origin: %w", err) + } + existing, err := StoredOrigin(database) + if err != nil { + return err + } + if existing == origin { + return nil + } + if err := database.SetSyncState(originStateKey, origin); err != nil { + return fmt.Errorf("persisting artifact origin: %w", err) + } + return nil +} + +// StoredOrigin returns the persisted origin ID without creating one. +func StoredOrigin(database *db.DB) (string, error) { + origin, err := database.GetSyncState(originStateKey) + if err != nil { + return "", fmt.Errorf("reading artifact origin: %w", err) + } + if origin != "" { + if err := validateOriginID(origin); err != nil { + return "", fmt.Errorf("stored artifact origin: %w", err) + } + return origin, nil + } + return "", nil +} + +// ImportedSessionIDs returns the durable session IDs written by artifact +// import. A foreign machine~id shape is shared by other import mechanisms, so +// callers must use this provenance instead of inferring artifact ownership from +// the session row alone. +func ImportedSessionIDs(database *db.DB) (map[string]struct{}, error) { + states, err := database.SyncStatesWithPrefix(importStatePrefix) + if err != nil { + return nil, fmt.Errorf("reading artifact import provenance: %w", err) + } + ids := make(map[string]struct{}, len(states)) + for key := range states { + rest := strings.TrimPrefix(key, importStatePrefix) + origin, gid, ok := strings.Cut(rest, ":") + if !ok || origin == "" || !strings.HasPrefix(gid, origin+"~") { + continue + } + ids[gid] = struct{}{} + } + return ids, nil +} + +func newOriginID() (string, error) { + host, err := os.Hostname() + if err != nil || strings.TrimSpace(host) == "" { + host = "machine" + } + host = sanitizeOriginPart(host) + if host == "" || host == "local" { + host = "machine" + } + var suffix [3]byte + if _, err := rand.Read(suffix[:]); err != nil { + return "", fmt.Errorf("generating artifact origin suffix: %w", err) + } + return fmt.Sprintf("%s-%s", host, hex.EncodeToString(suffix[:])), nil +} + +func validateOriginID(origin string) error { + return config.ValidateArtifactOriginID(origin) +} + +func validateDisjointRoots(localRoot, target string) error { + localAbs, err := filepath.Abs(localRoot) + if err != nil { + return fmt.Errorf("resolving local artifact store: %w", err) + } + targetAbs, err := filepath.Abs(target) + if err != nil { + return fmt.Errorf("resolving artifact sync target: %w", err) + } + localAbs = filepath.Clean(localAbs) + targetAbs = filepath.Clean(targetAbs) + localCanonical, err := canonicalArtifactPath(localAbs) + if err != nil { + return fmt.Errorf("resolving local artifact store symlinks: %w", err) + } + targetCanonical, err := canonicalArtifactPath(targetAbs) + if err != nil { + return fmt.Errorf("resolving artifact sync target symlinks: %w", err) + } + if rootsOverlap(localAbs, targetAbs) || rootsOverlap(localCanonical, targetCanonical) { + return fmt.Errorf( + "artifact sync target %s must not overlap local artifact store %s", + targetCanonical, localCanonical, + ) + } + return nil +} + +func canonicalArtifactPath(path string) (string, error) { + missing := make([]string, 0, 2) + current := path + for { + resolved, err := filepath.EvalSymlinks(current) + if err == nil { + for _, part := range slices.Backward(missing) { + resolved = filepath.Join(resolved, part) + } + return filepath.Clean(resolved), nil + } + if !errors.Is(err, fs.ErrNotExist) { + return "", err + } + parent := filepath.Dir(current) + if parent == current { + return "", err + } + missing = append(missing, filepath.Base(current)) + current = parent + } +} + +func rootsOverlap(a, b string) bool { + return a == b || pathContains(a, b) || pathContains(b, a) +} + +func pathContains(parent, child string) bool { + rel, err := filepath.Rel(parent, child) + if err != nil { + return false + } + return rel != "." && rel != ".." && !strings.HasPrefix(rel, ".."+string(filepath.Separator)) && !filepath.IsAbs(rel) +} + +func sanitizeOriginPart(s string) string { + s = strings.ToLower(strings.TrimSpace(s)) + var b strings.Builder + lastDash := false + for _, r := range s { + ok := r >= 'a' && r <= 'z' || r >= '0' && r <= '9' + if ok { + b.WriteRune(r) + lastDash = false + continue + } + if !lastDash { + b.WriteByte('-') + lastDash = true + } + } + return strings.Trim(b.String(), "-") +} + +type checkpoint struct { + Version int `json:"v"` + Origin string `json:"origin"` + Sequence int `json:"seq"` + Sessions map[string]string `json:"sessions"` +} + +type manifest struct { + Version int `json:"v"` + Origin string `json:"origin"` + NativeSessionID string `json:"native_session_id"` + Session manifestSession `json:"session"` + SessionName *string `json:"session_name,omitempty"` + Segments []string `json:"segments"` + UsageEvents []artifactUsageEvent `json:"usage_events,omitempty"` + RawSource *rawSourceRef `json:"raw_source,omitempty"` + DataVersion int `json:"data_version"` + Generation int `json:"generation"` + // Signal state persisted on the session row but absent from the wire + // Session above, which mirrors only db.Session's JSON-visible fields. + // Carried explicitly so an imported session keeps its tool-call, context, + // and quality signal state instead of resetting to false/zero. Secret-scan + // state is deliberately not carried: findings live outside the manifest, + // so imported sessions are treated as unscanned (see rewriteForImport). + SessionHasToolCalls bool `json:"session_has_tool_calls,omitempty"` + SessionHasContextData bool `json:"session_has_context_data,omitempty"` + SessionQualitySignals *manifestQualitySignals `json:"session_quality_signals,omitempty"` +} + +type artifactUsageEvent struct { + MessageOrdinal *int `json:"message_ordinal,omitempty"` + Source string `json:"source"` + Model string `json:"model"` + InputTokens int `json:"input_tokens,omitempty"` + OutputTokens int `json:"output_tokens,omitempty"` + CacheCreationInputTokens int `json:"cache_creation_input_tokens,omitempty"` + CacheReadInputTokens int `json:"cache_read_input_tokens,omitempty"` + ReasoningTokens int `json:"reasoning_tokens,omitempty"` + CostUSD *float64 `json:"cost_usd,omitempty"` + CostStatus string `json:"cost_status,omitempty"` + CostSource string `json:"cost_source,omitempty"` + OccurredAt string `json:"occurred_at,omitempty"` + DedupKey string `json:"dedup_key,omitempty"` +} + +type rawSourceRef struct { + Hash string `json:"hash"` + Size int64 `json:"size"` + MediaType string `json:"media_type,omitempty"` + Path string `json:"path,omitempty"` +} + +type metadataEvent struct { + Version int `json:"v"` + HLC string `json:"hlc"` + Origin string `json:"origin"` + SessionGID string `json:"session_gid"` + Op string `json:"op"` + Value json.RawMessage `json:"value,omitempty"` + Pin *MetadataPin `json:"pin,omitempty"` +} + +type segmentMessage struct { + Version int `json:"v"` + Ordinal int `json:"ordinal"` + Role string `json:"role"` + Content string `json:"content"` + ThinkingText string `json:"thinking_text,omitempty"` + Timestamp string `json:"timestamp,omitempty"` + HasThinking bool `json:"has_thinking,omitempty"` + HasToolUse bool `json:"has_tool_use,omitempty"` + ContentLength int `json:"content_length,omitempty"` + Model string `json:"model,omitempty"` + TokenUsage json.RawMessage `json:"token_usage,omitempty"` + ContextTokens int `json:"context_tokens,omitempty"` + OutputTokens int `json:"output_tokens,omitempty"` + HasContextTokens bool `json:"has_context_tokens,omitempty"` + HasOutputTokens bool `json:"has_output_tokens,omitempty"` + ClaudeMessageID string `json:"claude_message_id,omitempty"` + ClaudeRequestID string `json:"claude_request_id,omitempty"` + ToolCalls []segmentToolCall `json:"tool_calls,omitempty"` + IsSystem bool `json:"is_system,omitempty"` + SourceType string `json:"source_type,omitempty"` + SourceSubtype string `json:"source_subtype,omitempty"` + SourceUUID string `json:"source_uuid,omitempty"` + SourceParentUUID string `json:"source_parent_uuid,omitempty"` + IsSidechain bool `json:"is_sidechain,omitempty"` + IsCompactBoundary bool `json:"is_compact_boundary,omitempty"` +} + +type segmentToolCall struct { + CallIndex int `json:"call_index"` + ToolName string `json:"tool_name"` + Category string `json:"category,omitempty"` + ToolUseID string `json:"tool_use_id,omitempty"` + InputJSON string `json:"input_json,omitempty"` + FilePath string `json:"file_path,omitempty"` + SkillName string `json:"skill_name,omitempty"` + ResultContentLength int `json:"result_content_length,omitempty"` + ResultContent string `json:"result_content,omitempty"` + SubagentSessionID string `json:"subagent_session_id,omitempty"` + ResultEvents []segmentResultEvent `json:"result_events,omitempty"` +} + +type segmentResultEvent struct { + ToolUseID string `json:"tool_use_id,omitempty"` + AgentID string `json:"agent_id,omitempty"` + SubagentSessionID string `json:"subagent_session_id,omitempty"` + Source string `json:"source"` + Status string `json:"status"` + Content string `json:"content"` + ContentLength int `json:"content_length,omitempty"` + Timestamp string `json:"timestamp,omitempty"` + EventIndex int `json:"event_index"` +} + +// Export writes current machine-owned sessions into root/origin. +func Export(ctx context.Context, database *db.DB, root, origin string) (int, error) { + if err := validateOriginID(origin); err != nil { + return 0, err + } + originRoot := filepath.Join(root, origin) + for _, dir := range []string{"checkpoints", "manifests", "segments", "meta", "raw"} { + if err := os.MkdirAll(filepath.Join(originRoot, dir), 0o755); err != nil { + return 0, fmt.Errorf("creating %s: %w", dir, err) + } + } + + // Enumerate owned sessions from a raw query rather than the sidebar list API + // so usage-only (zero-message) sessions are still exported. + ids, err := database.ListOwnedSessionIDsForExport(ctx) + if err != nil { + return 0, err + } + var exported int + sessions := map[string]string{} + for _, id := range ids { + stateKey := exportStateKey(origin, id) + prevHash, err := database.GetSyncState(stateKey) + if err != nil { + return 0, fmt.Errorf("reading export state for %s: %w", id, err) + } + hash, changed, err := exportSession(ctx, database, originRoot, origin, id, prevHash) + if err != nil { + return 0, err + } + sessions[origin+"~"+id] = hash + if changed { + if err := database.SetSyncState(stateKey, hash); err != nil { + return 0, fmt.Errorf("writing export state for %s: %w", id, err) + } + exported++ + } + } + if latestCheckpointMatches(originRoot, origin, sessions) { + return exported, nil + } + seq, err := nextCheckpointSequence(originRoot) + if err != nil { + return 0, err + } + cp := checkpoint{Version: formatVersion, Origin: origin, Sequence: seq, Sessions: sessions} + data, err := canonicalJSON(cp) + if err != nil { + return 0, err + } + checkpointPath := filepath.Join(originRoot, "checkpoints", fmt.Sprintf("cp-%010d.json", seq)) + if err := writeFileAtomic(checkpointPath, data, 0o644); err != nil { + return 0, fmt.Errorf("writing checkpoint: %w", err) + } + return exported, nil +} + +func latestCheckpointMatches(originRoot, origin string, sessions map[string]string) bool { + paths, err := filepath.Glob(filepath.Join(originRoot, "checkpoints", "cp-*.json")) + if err != nil || len(paths) == 0 { + return false + } + path := paths[len(paths)-1] + data, err := os.ReadFile(path) + if err != nil { + return false + } + var cp checkpoint + if err := json.Unmarshal(data, &cp); err != nil { + return false + } + if err := validateCheckpoint(&cp, origin); err != nil { + return false + } + if err := validateCheckpointSequenceIdentity(cp, filepath.Base(path)); err != nil { + return false + } + return maps.Equal(cp.Sessions, sessions) +} + +// nextCheckpointSequence derives the next checkpoint sequence from the +// highest existing checkpoint filename, quarantined ones included, so a +// corrupt latest checkpoint cannot block export and a sequence number that +// may already have been published to peers is never reused for different +// content. A latest live checkpoint whose body no longer parses is +// quarantined so read paths fall back to the previous valid checkpoint and a +// valid peer copy can heal it. +func nextCheckpointSequence(originRoot string) (int, error) { + dir := filepath.Join(originRoot, "checkpoints") + live, err := filepath.Glob(filepath.Join(dir, "cp-*.json")) + if err != nil { + return 0, err + } + quarantined, err := filepath.Glob(filepath.Join(dir, "cp-*.json"+quarantineSuffix)) + if err != nil { + return 0, err + } + sort.Strings(live) + if len(live) > 0 { + latest := live[len(live)-1] + data, err := os.ReadFile(latest) + if err != nil && !errors.Is(err, fs.ErrNotExist) { + return 0, err + } + var cp checkpoint + if err == nil { + if uerr := json.Unmarshal(data, &cp); uerr != nil { + log.Printf("artifact: skipping corrupt checkpoint %s: %v", latest, uerr) + quarantineArtifact(latest) + } + } + } + maxSeq := 0 + for _, p := range append(live, quarantined...) { + name := strings.TrimSuffix(filepath.Base(p), quarantineSuffix) + var seq int + if _, serr := fmt.Sscanf(name, "cp-%d.json", &seq); serr != nil { + continue + } + if seq > maxSeq { + maxSeq = seq + } + } + return maxSeq + 1, nil +} + +func exportStateKey(origin, sessionID string) string { + return exportStatePrefix + origin + ":" + sessionID +} + +func exportSession( + ctx context.Context, + database *db.DB, + originRoot, origin, sessionID, prevManifestHash string, +) (string, bool, error) { + return exportSessionWithLimits( + ctx, database, originRoot, origin, sessionID, prevManifestHash, + productionArtifactLimits(), + ) +} + +func exportSessionWithLimits( + ctx context.Context, + database *db.DB, + originRoot, origin, sessionID, prevManifestHash string, + limits artifactLimits, +) (string, bool, error) { + sess, err := database.GetSessionFull(ctx, sessionID) + if err != nil { + return "", false, fmt.Errorf("loading session %s for artifact export: %w", sessionID, err) + } + if sess == nil { + return "", false, fmt.Errorf("session %s disappeared during artifact export", sessionID) + } + msgs, err := database.GetAllMessages(ctx, sessionID) + if err != nil { + return "", false, fmt.Errorf("loading messages for artifact export %s: %w", sessionID, err) + } + usageEvents, err := database.GetUsageEvents(ctx, sessionID) + if err != nil { + return "", false, fmt.Errorf("loading usage events for artifact export %s: %w", sessionID, err) + } + if len(msgs) > limits.sessionMessages { + return "", false, fmt.Errorf( + "session message limit exceeded for %s: got %d, limit %d", + sessionID, len(msgs), limits.sessionMessages, + ) + } + if len(usageEvents) > limits.manifestUsageEvents { + return "", false, fmt.Errorf( + "manifest usage event limit exceeded for %s: got %d, limit %d", + sessionID, len(usageEvents), limits.manifestUsageEvents, + ) + } + if err := validateExportNestedCollections(msgs, limits); err != nil { + return "", false, fmt.Errorf("validating nested collections for %s: %w", sessionID, err) + } + + wireMessages := canonicalMessages(msgs) + segmentHashes := make([]string, 0, 1) + seenSegmentHashes := make(map[string]struct{}) + var sessionDecodedBytes int64 + if err := forEachEncodedSegmentWithLimits(wireMessages, limits, func(data []byte) error { + if len(segmentHashes) >= limits.manifestSegments { + return fmt.Errorf( + "manifest segment reference limit exceeded for %s: limit %d", + sessionID, limits.manifestSegments, + ) + } + segmentBytes := int64(len(data)) + if segmentBytes > limits.sessionDecodedBytes-sessionDecodedBytes { + return fmt.Errorf( + "session decoded byte limit exceeded for %s: limit %d", + sessionID, limits.sessionDecodedBytes, + ) + } + segmentHash := hashHex(data) + if _, ok := seenSegmentHashes[segmentHash]; ok { + return fmt.Errorf("generated duplicate segment reference %s", segmentHash) + } + seenSegmentHashes[segmentHash] = struct{}{} + segmentHashes = append(segmentHashes, segmentHash) + sessionDecodedBytes += segmentBytes + return nil + }); err != nil { + return "", false, err + } + + wireSession := manifestSessionFromDB(*sess) + wireSession.Machine = origin + normalizeManifestSessionLocalState(&wireSession) + m := manifest{ + Version: formatVersion, + Origin: origin, + NativeSessionID: sessionID, + Session: wireSession, + SessionName: sess.SessionName, + Segments: segmentHashes, + UsageEvents: canonicalUsageEvents(usageEvents), + DataVersion: sess.DataVersion, + Generation: 1, + SessionHasToolCalls: sess.HasToolCalls, + SessionHasContextData: sess.HasContextData, + SessionQualitySignals: manifestQualitySignalsFromDB(sess.StoredQualitySignals()), + } + manifestData, err := canonicalJSON(m) + if err != nil { + return "", false, err + } + if int64(len(manifestData)) > manifestDecodedLimit { + return "", false, fmt.Errorf( + "generated manifest exceeds %d-byte readable limit: got %d bytes", + manifestDecodedLimit, len(manifestData), + ) + } + manifestHash := hashHex(manifestData) + if compatible, err := previousManifestMatchesAfterLocalStateNormalization( + originRoot, origin, sessionID, prevManifestHash, manifestHash, + ); err != nil { + return "", false, err + } else if compatible { + return prevManifestHash, false, nil + } + manifestPath := filepath.Join(originRoot, "manifests", manifestHash+manifestExtension) + if prevManifestHash == manifestHash { + _, valid, err := readValidExportArtifacts( + originRoot, origin, sessionID, manifestHash, + ) + if err != nil { + return "", false, err + } + if valid { + return manifestHash, false, nil + } + } + segmentIndex := 0 + if err := forEachEncodedSegmentWithLimits(wireMessages, limits, func(data []byte) error { + segmentHash := hashHex(data) + if segmentIndex >= len(segmentHashes) || segmentHashes[segmentIndex] != segmentHash { + return errors.New("message segment encoding changed between export passes") + } + segmentIndex++ + if err := healComputedExportSegment(originRoot, segmentHash); err != nil { + return fmt.Errorf("validating computed segment: %w", err) + } + segmentPath := filepath.Join(originRoot, "segments", segmentHash+segmentExtension) + if err := writeCompressed(segmentPath, data); err != nil { + return fmt.Errorf("writing segment: %w", err) + } + return nil + }); err != nil { + return "", false, fmt.Errorf("exporting segments for %s: %w", sessionID, err) + } + if err := writeCompressed(manifestPath, manifestData); err != nil { + return "", false, fmt.Errorf("writing manifest for %s: %w", sessionID, err) + } + return manifestHash, true, nil +} + +// healComputedExportSegment validates only the segment path derived from the +// current local messages. Corrupt bytes are quarantined by the shared reader so +// the subsequent immutable write can recreate them. Missing artifacts are +// expected; semantic errors such as a future segment version remain untouched +// and surface to the caller. +func healComputedExportSegment(originRoot, segmentHash string) error { + _, err := readManifestMessages(originRoot, manifest{ + Segments: []string{segmentHash}, + }) + if err == nil || errors.Is(err, errIncompleteArtifact) || + errors.Is(err, errCorruptArtifact) { + return nil + } + return err +} + +func normalizeManifestSessionLocalState(sess *manifestSession) { + // Keep non-content, machine-local state out of the canonical manifest so a + // source-only change to it does not alter the content hash and trigger a + // re-import that clears the importer's local findings. secret_leak_count is + // import-discarded secret state (see rewriteForImport); local_modified_at is + // the local sync watermark, which import ignores (the importer stamps its + // own) -- and a secret rescan bumps both even when no exported message + // content changed. The file_* fields are source-file bookkeeping that + // import clears (see clearImportedSessionSourceState); a touch, move, or + // re-download of the source file changes them without changing any + // exported content. + sess.SecretLeakCount = 0 + sess.LocalModifiedAt = nil + sess.FilePath = nil + sess.FileSize = nil + sess.FileMtime = nil + sess.FileInode = nil + sess.FileDevice = nil + sess.FileHash = nil +} + +func previousManifestMatchesAfterLocalStateNormalization( + originRoot, origin, sessionID, prevManifestHash, currentManifestHash string, +) (bool, error) { + if prevManifestHash == "" || prevManifestHash == currentManifestHash { + return false, nil + } + prev, valid, err := readValidExportArtifacts( + originRoot, origin, sessionID, prevManifestHash, + ) + if err != nil { + return false, err + } + if !valid { + return false, nil + } + normalizeManifestSessionLocalState(&prev.Session) + data, err := canonicalJSON(prev) + if err != nil { + return false, err + } + return hashHex(data) == currentManifestHash, nil +} + +func readValidExportArtifacts( + originRoot, origin, sessionID, manifestHash string, +) (manifest, bool, error) { + m, err := readManifest(originRoot, manifestHash) + if err != nil { + if errors.Is(err, errIncompleteArtifact) || errors.Is(err, errCorruptArtifact) { + return manifest{}, false, nil + } + return manifest{}, false, err + } + if err := validateManifest(m, origin, origin+"~"+sessionID); err != nil { + if errors.Is(err, errFutureArtifactVersion) { + return manifest{}, false, nil + } + quarantineArtifact(filepath.Join( + originRoot, KindManifests, manifestHash+manifestExtension, + )) + return manifest{}, false, nil + } + if _, err := readManifestMessages(originRoot, m); err != nil { + if errors.Is(err, errIncompleteArtifact) || + errors.Is(err, errCorruptArtifact) || + errors.Is(err, errFutureArtifactVersion) { + return manifest{}, false, nil + } + return manifest{}, false, err + } + return m, true, nil +} + +// Import reads every foreign origin under root and imports referenced sessions. +func Import(ctx context.Context, database *db.DB, root, localOrigin string) (int, int, error) { + res, err := ImportDetailed(ctx, database, root, localOrigin) + return res.Sessions, res.Messages, err +} + +// ImportDetailed reads every foreign origin under root and imports referenced +// sessions plus metadata events. +func ImportDetailed(ctx context.Context, database *db.DB, root, localOrigin string) (ImportResult, error) { + return importDetailed(ctx, database, nil, root, localOrigin) +} + +// importDetailed imports foreign origins, advancing clock past observed remote +// metadata HLCs. When clock is nil a default clock backed by database is used so +// the persisted metadata clock is still advanced. +func importDetailed( + ctx context.Context, + database *db.DB, + clock *HLCClock, + root, localOrigin string, +) (ImportResult, error) { + if clock == nil { + clock = NewHLCClock(database, HLCClockOptions{}) + } + entries, err := os.ReadDir(root) + if err != nil { + if errors.Is(err, fs.ErrNotExist) { + return ImportResult{}, nil + } + return ImportResult{}, fmt.Errorf("reading artifact roots: %w", err) + } + appliedEvents, err := database.MetadataAppliedEventIdentities(ctx) + if err != nil { + return ImportResult{}, err + } + foreignOrigins := make([]string, 0, len(entries)) + for _, ent := range entries { + if ent.IsDir() && ent.Name() != localOrigin { + foreignOrigins = append(foreignOrigins, ent.Name()) + } + } + var res ImportResult + for _, origin := range foreignOrigins { + originRes, err := importOriginContent( + ctx, database, filepath.Join(root, origin), origin, + ) + if err != nil { + return res, err + } + res.Sessions += originRes.Sessions + res.Messages += originRes.Messages + res.Deferred += originRes.Deferred + } + for _, origin := range foreignOrigins { + metadata, err := replayMetadata( + ctx, database, clock, filepath.Join(root, origin), origin, + localOrigin, appliedEvents, + ) + if err != nil { + return res, err + } + res.Metadata += metadata + } + return res, nil +} + +func importOriginContent( + ctx context.Context, + database *db.DB, + originRoot, origin string, +) (ImportResult, error) { + cp, err := readLatestCompatibleCheckpoint(originRoot, origin) + if err != nil { + return ImportResult{}, fmt.Errorf("reading checkpoint for %s: %w", origin, err) + } + var res ImportResult + if cp != nil { + keys := make([]string, 0, len(cp.Sessions)) + for k := range cp.Sessions { + keys = append(keys, k) + } + sort.Strings(keys) + + for _, gid := range keys { + if err := ctx.Err(); err != nil { + return res, err + } + manifestHash := cp.Sessions[gid] + stateKey := importStateKey(origin, gid) + prevHash, err := database.GetSyncState(stateKey) + if err != nil { + return res, fmt.Errorf("reading import state for %s: %w", gid, err) + } + if prevHash == manifestHash { + continue + } + m, err := readManifest(originRoot, manifestHash) + if err != nil { + if errors.Is(err, errIncompleteArtifact) { + res.Deferred++ + continue + } + if errors.Is(err, errCorruptArtifact) { + log.Printf("artifact: skipping session %s from %s: %v", gid, origin, err) + res.Deferred++ + continue + } + return res, err + } + if err := validateManifest(m, origin, gid); err != nil { + if errors.Is(err, errFutureArtifactVersion) { + continue + } + return res, err + } + msgs, err := readManifestMessages(originRoot, m) + if err != nil { + if errors.Is(err, errIncompleteArtifact) || errors.Is(err, errFutureArtifactVersion) { + if errors.Is(err, errIncompleteArtifact) { + res.Deferred++ + } + continue + } + if errors.Is(err, errCorruptArtifact) { + log.Printf("artifact: skipping session %s from %s: %v", gid, origin, err) + res.Deferred++ + continue + } + return res, err + } + write := rewriteForImport(m, msgs) + writeRes, err := database.WriteSessionBatchAtomic([]db.SessionBatchWrite{write}) + if err != nil { + if errors.Is(err, db.ErrSessionExcluded) || errors.Is(err, db.ErrSessionTrashed) { + continue + } + return res, fmt.Errorf("importing artifact session %s: %w", gid, err) + } + res.Sessions += writeRes.WrittenSessions + res.Messages += writeRes.WrittenMessages + if _, err := database.ReapplyMetadataReplayState(ctx, gid, write.Session.ID); err != nil { + return res, fmt.Errorf("reapplying metadata after importing artifact session %s: %w", gid, err) + } + if err := database.SetSyncState(stateKey, manifestHash); err != nil { + return res, fmt.Errorf("writing import state for %s: %w", gid, err) + } + } + } + return res, nil +} + +func importStateKey(origin, gid string) string { + return importStatePrefix + origin + ":" + gid +} + +func validateCheckpoint(cp *checkpoint, origin string) error { + if cp.Version > formatVersion { + return fmt.Errorf( + "%w: checkpoint for %s has artifact version %d", + errFutureArtifactVersion, origin, cp.Version, + ) + } + if cp.Version != formatVersion { + return fmt.Errorf( + "checkpoint for %s has unsupported artifact version %d", + origin, cp.Version, + ) + } + if cp.Origin != origin { + return fmt.Errorf( + "checkpoint origin mismatch for %s: got %q", + origin, cp.Origin, + ) + } + return validateCheckpointReferences(cp, origin) +} + +func validateCheckpointReferences(cp *checkpoint, origin string) error { + for gid, manifestHash := range cp.Sessions { + if gid == "" { + return fmt.Errorf("checkpoint for %s contains empty session id", origin) + } + if !strings.HasPrefix(gid, origin+"~") { + return fmt.Errorf( + "checkpoint session %s does not belong to origin %s", + gid, origin, + ) + } + if strings.TrimSpace(manifestHash) == "" { + return fmt.Errorf("checkpoint session %s has empty manifest hash", gid) + } + if err := validateHashHex(manifestHash); err != nil { + return fmt.Errorf("checkpoint session %s has invalid manifest hash: %w", gid, err) + } + } + return nil +} + +func validateManifest(m manifest, origin, gid string) error { + if m.Version > formatVersion { + return fmt.Errorf( + "%w: manifest %s has artifact version %d", + errFutureArtifactVersion, gid, m.Version, + ) + } + if m.Version != formatVersion { + return fmt.Errorf( + "manifest %s has unsupported artifact version %d", + gid, m.Version, + ) + } + if m.Origin != origin { + return fmt.Errorf( + "manifest origin mismatch for %s: got %q", + gid, m.Origin, + ) + } + if m.NativeSessionID == "" { + return fmt.Errorf("manifest %s has empty native session id", gid) + } + expectedGID := origin + "~" + m.NativeSessionID + if gid != expectedGID { + return fmt.Errorf( + "manifest session id mismatch: checkpoint has %s, manifest has %s", + gid, expectedGID, + ) + } + if m.Session.ID != m.NativeSessionID { + return fmt.Errorf( + "manifest %s session row id mismatch: got %q", + gid, m.Session.ID, + ) + } + if m.Session.Machine != origin { + return fmt.Errorf( + "manifest %s session row machine mismatch: got %q", + gid, m.Session.Machine, + ) + } + if len(m.Segments) == 0 { + return fmt.Errorf("manifest %s has no message segments", gid) + } + if err := validateManifestReferences(m); err != nil { + return err + } + return nil +} + +func validateManifestReferences(m manifest) error { + return validateManifestReferencesWithLimits(m, productionArtifactLimits()) +} + +func validateManifestReferencesWithLimits(m manifest, limits artifactLimits) error { + if len(m.Segments) > limits.manifestSegments { + return fmt.Errorf( + "manifest segment reference limit exceeded: got %d, limit %d", + len(m.Segments), limits.manifestSegments, + ) + } + seen := make(map[string]struct{}, len(m.Segments)) + for _, segmentHash := range m.Segments { + if err := validateHashHex(segmentHash); err != nil { + return fmt.Errorf("manifest segment has invalid hash: %w", err) + } + if _, ok := seen[segmentHash]; ok { + return fmt.Errorf("manifest has duplicate segment reference %s", segmentHash) + } + seen[segmentHash] = struct{}{} + } + if len(m.UsageEvents) > limits.manifestUsageEvents { + return fmt.Errorf( + "manifest usage event limit exceeded: got %d, limit %d", + len(m.UsageEvents), limits.manifestUsageEvents, + ) + } + if m.RawSource != nil && m.RawSource.Hash != "" { + if err := validateHashHex(m.RawSource.Hash); err != nil { + return fmt.Errorf("manifest raw source has invalid hash: %w", err) + } + } + return nil +} + +func rewriteForImport(m manifest, msgs []db.Message) db.SessionBatchWrite { + importedID := m.Origin + "~" + m.NativeSessionID + sess := m.Session.dbSession() + sess.ID = importedID + sess.Machine = m.Origin + sess.SessionName = m.SessionName + clearImportedSessionSourceState(&sess) + // Restore signal state dropped from the Session JSON; signalsFromSession + // reads these fields below to persist the imported session's signal columns. + sess.HasToolCalls = m.SessionHasToolCalls + sess.HasContextData = m.SessionHasContextData + sess.ApplyQualitySignals(m.SessionQualitySignals.dbQualitySignals()) + // Secret findings are not carried in the manifest, so an imported session has + // no finding rows. Treat it as unscanned rather than trusting the source scan: + // clear the rules version (json:"-", so already absent) and the leak count + // (carried in the Session JSON) so the count stays consistent with the zero + // findings and `secrets scan --backfill` rescans it with local rules. Stamping + // it scanned-at-source-version would make backfill (secrets_rules_version != + // current) skip a secret-bearing session, leaving no revealable findings. + sess.SecretsRulesVersion = "" + sess.SecretLeakCount = 0 + sess.SourceSessionID = prefixImportedSessionID(m.Origin, sess.SourceSessionID) + if sess.ParentSessionID != nil { + prefixed := prefixImportedSessionID(m.Origin, *sess.ParentSessionID) + sess.ParentSessionID = &prefixed + } + for i := range msgs { + msgs[i].ID = 0 + msgs[i].SessionID = importedID + for j := range msgs[i].ToolCalls { + msgs[i].ToolCalls[j].MessageID = 0 + msgs[i].ToolCalls[j].SessionID = importedID + msgs[i].ToolCalls[j].SubagentSessionID = prefixImportedSessionID( + m.Origin, + msgs[i].ToolCalls[j].SubagentSessionID, + ) + for k := range msgs[i].ToolCalls[j].ResultEvents { + ev := &msgs[i].ToolCalls[j].ResultEvents[k] + ev.SubagentSessionID = prefixImportedSessionID(m.Origin, ev.SubagentSessionID) + } + } + } + usageEvents := dbUsageEvents(m.UsageEvents, importedID) + return db.SessionBatchWrite{ + Session: sess, + Messages: msgs, + UsageEvents: usageEvents, + Signals: signalsFromSession(sess), + DataVersion: m.DataVersion, + ReplaceMessages: true, + } +} + +func clearImportedSessionSourceState(sess *db.Session) { + sess.FilePath = nil + sess.FileSize = nil + sess.FileMtime = nil + sess.NextOrdinal = 0 + sess.LastEntryUUID = nil + sess.FileInode = nil + sess.FileDevice = nil + sess.FileHash = nil +} + +func prefixImportedSessionID(origin, id string) string { + if id == "" || strings.Contains(id, "~") { + return id + } + return origin + "~" + id +} + +func signalsFromSession(s db.Session) db.SessionSignalUpdate { + update := db.SessionSignalUpdate{ + ToolFailureSignalCount: s.ToolFailureSignalCount, + ToolRetryCount: s.ToolRetryCount, + EditChurnCount: s.EditChurnCount, + ConsecutiveFailureMax: s.ConsecutiveFailureMax, + Outcome: s.Outcome, + OutcomeConfidence: s.OutcomeConfidence, + EndedWithRole: s.EndedWithRole, + FinalFailureStreak: s.FinalFailureStreak, + SignalsPendingSince: s.SignalsPendingSince, + CompactionCount: s.CompactionCount, + MidTaskCompactionCount: s.MidTaskCompactionCount, + ContextPressureMax: s.ContextPressureMax, + HealthScore: s.HealthScore, + HealthGrade: s.HealthGrade, + HasToolCalls: s.HasToolCalls, + HasContextData: s.HasContextData, + SecretLeakCount: s.SecretLeakCount, + SecretsRulesVersion: s.SecretsRulesVersion, + } + if qs := s.StoredQualitySignals(); qs != nil { + update.QualitySignals = *qs + } + return update +} + +func readLatestCheckpoint(originRoot string) (*checkpoint, error) { + paths, err := filepath.Glob(filepath.Join(originRoot, "checkpoints", "cp-*.json")) + if err != nil { + return nil, err + } + if len(paths) == 0 { + return nil, nil + } + sort.Strings(paths) + data, err := os.ReadFile(paths[len(paths)-1]) + if err != nil { + return nil, err + } + var cp checkpoint + if err := json.Unmarshal(data, &cp); err != nil { + return nil, err + } + return &cp, nil +} + +func readLatestCompatibleCheckpoint(originRoot, origin string) (*checkpoint, error) { + paths, err := filepath.Glob(filepath.Join(originRoot, "checkpoints", "cp-*.json")) + if err != nil { + return nil, err + } + sort.Strings(paths) + for _, v := range slices.Backward(paths) { + data, err := os.ReadFile(v) + if err != nil { + if errors.Is(err, fs.ErrNotExist) { + continue + } + return nil, err + } + var cp checkpoint + if err := json.Unmarshal(data, &cp); err != nil { + log.Printf("artifact: skipping corrupt checkpoint %s: %v", v, err) + quarantineArtifact(v) + continue + } + if err := validateCheckpoint(&cp, origin); err != nil { + if errors.Is(err, errFutureArtifactVersion) { + continue + } + log.Printf("artifact: skipping invalid checkpoint %s: %v", v, err) + quarantineArtifact(v) + continue + } + if err := validateCheckpointSequenceIdentity(cp, filepath.Base(v)); err != nil { + log.Printf("artifact: skipping invalid checkpoint %s: %v", v, err) + quarantineArtifact(v) + continue + } + return &cp, nil + } + return nil, nil +} + +func readManifest(originRoot, hash string) (manifest, error) { + if err := validateHashHex(hash); err != nil { + return manifest{}, err + } + path := filepath.Join(originRoot, "manifests", hash+manifestExtension) + data, err := readCompressed(path) + if err != nil { + if errors.Is(err, fs.ErrNotExist) { + return manifest{}, fmt.Errorf("%w: manifest %s", errIncompleteArtifact, hash) + } + if errors.Is(err, errCorruptArtifact) { + quarantineArtifact(path) + return manifest{}, fmt.Errorf("manifest %s: %w", hash, err) + } + return manifest{}, fmt.Errorf("reading manifest %s: %w", hash, err) + } + if got := hashHex(data); got != hash { + quarantineArtifact(path) + return manifest{}, fmt.Errorf("%w: manifest %s hash mismatch: got %s", errCorruptArtifact, hash, got) + } + m, err := decodeManifestWithLimits(data, productionArtifactLimits()) + if err != nil { + quarantineArtifact(path) + return manifest{}, fmt.Errorf("%w: decoding manifest %s: %v", errCorruptArtifact, hash, err) + } + return m, nil +} + +func decodeManifestWithLimits(data []byte, limits artifactLimits) (manifest, error) { + var envelope struct { + Version int `json:"v"` + Origin string `json:"origin"` + Segments json.RawMessage `json:"segments"` + UsageEvents json.RawMessage `json:"usage_events"` + } + if err := json.Unmarshal(data, &envelope); err != nil { + return manifest{}, err + } + // Future manifests are retained for forward compatibility. Reading only + // their scalar header avoids allocating collections whose schema this + // version does not understand. + if envelope.Version > formatVersion { + return manifest{Version: envelope.Version, Origin: envelope.Origin}, nil + } + if err := preflightManifestCollections( + envelope.Segments, envelope.UsageEvents, limits, + ); err != nil { + return manifest{}, err + } + var m manifest + if err := json.Unmarshal(data, &m); err != nil { + return manifest{}, err + } + return m, nil +} + +func preflightManifestCollections( + segments, usageEvents json.RawMessage, + limits artifactLimits, +) error { + if err := preflightSegmentReferences(segments, limits.manifestSegments); err != nil { + return err + } + return preflightJSONArrayCount( + usageEvents, "manifest usage event", limits.manifestUsageEvents, + ) +} + +func preflightSegmentReferences(data json.RawMessage, limit int) error { + trimmed := bytes.TrimSpace(data) + if len(trimmed) == 0 || bytes.Equal(trimmed, []byte("null")) { + return nil + } + dec := json.NewDecoder(bytes.NewReader(trimmed)) + token, err := dec.Token() + if err != nil { + return err + } + if token != json.Delim('[') { + return errors.New("manifest segments must be an array") + } + seen := make(map[string]struct{}, min(limit, 16)) + count := 0 + for dec.More() { + if count >= limit { + return fmt.Errorf("manifest segment reference limit exceeded: limit %d", limit) + } + var hash string + if err := dec.Decode(&hash); err != nil { + return fmt.Errorf("decoding manifest segment reference: %w", err) + } + if _, ok := seen[hash]; ok { + return fmt.Errorf("manifest has duplicate segment reference %s", hash) + } + seen[hash] = struct{}{} + count++ + } + _, err = dec.Token() + return err +} + +func preflightJSONArrayCount(data json.RawMessage, name string, limit int) error { + _, err := countJSONArrayElements(data, name, limit) + return err +} + +func countJSONArrayElements( + data json.RawMessage, + name string, + limit int, +) (int, error) { + trimmed := bytes.TrimSpace(data) + if len(trimmed) == 0 || bytes.Equal(trimmed, []byte("null")) { + return 0, nil + } + dec := json.NewDecoder(bytes.NewReader(trimmed)) + token, err := dec.Token() + if err != nil { + return 0, err + } + if token != json.Delim('[') { + return 0, fmt.Errorf("%ss must be an array", name) + } + count := 0 + for dec.More() { + if count >= limit { + return 0, fmt.Errorf("%s limit exceeded: limit %d", name, limit) + } + var value json.RawMessage + if err := dec.Decode(&value); err != nil { + return 0, fmt.Errorf("decoding %s: %w", name, err) + } + count++ + } + if _, err := dec.Token(); err != nil { + return 0, err + } + return count, nil +} + +func readManifestMessages(originRoot string, m manifest) ([]db.Message, error) { + return readManifestMessagesWithLimits(originRoot, m, productionArtifactLimits()) +} + +func readManifestMessagesWithLimits( + originRoot string, + m manifest, + limits artifactLimits, +) ([]db.Message, error) { + var msgs []db.Message + err := walkManifestMessagesWithLimits( + originRoot, m, limits, + func(segmentMsgs []db.Message) { + msgs = append(msgs, segmentMsgs...) + }, + ) + return msgs, err +} + +func walkManifestMessagesWithLimits( + originRoot string, + m manifest, + limits artifactLimits, + visit func([]db.Message), +) error { + if err := validateManifestReferencesWithLimits(m, limits); err != nil { + return fmt.Errorf("%w: %v", errCorruptArtifact, err) + } + var decodedBytes int64 + totalMessages := 0 + totalNested := nestedCollectionCounts{} + for _, segmentHash := range m.Segments { + segment, err := readSegmentPreflightWithLimits( + originRoot, segmentHash, limits, + ) + if err != nil { + return err + } + segmentBytes := int64(len(segment.data)) + if segmentBytes > limits.sessionDecodedBytes-decodedBytes { + return fmt.Errorf( + "%w: session decoded byte limit exceeded: limit %d", + errCorruptArtifact, limits.sessionDecodedBytes, + ) + } + if len(segment.preflight.records) > limits.sessionMessages-totalMessages { + return fmt.Errorf( + "%w: session message limit exceeded: limit %d", + errCorruptArtifact, limits.sessionMessages, + ) + } + if exceedsCollectionLimit( + totalNested.toolCalls, + segment.preflight.nested.toolCalls, + limits.sessionToolCalls, + ) { + return fmt.Errorf( + "%w: session tool call limit exceeded: limit %d", + errCorruptArtifact, limits.sessionToolCalls, + ) + } + if exceedsCollectionLimit( + totalNested.resultEvents, + segment.preflight.nested.resultEvents, + limits.sessionResultEvents, + ) { + return fmt.Errorf( + "%w: session result event limit exceeded: limit %d", + errCorruptArtifact, limits.sessionResultEvents, + ) + } + segmentMsgs, err := decodeStoredSegment(segment) + if err != nil { + return err + } + decodedBytes += segmentBytes + totalMessages += len(segmentMsgs) + totalNested.toolCalls += segment.preflight.nested.toolCalls + totalNested.resultEvents += segment.preflight.nested.resultEvents + if visit != nil { + visit(segmentMsgs) + } + } + return nil +} + +func readSegmentMessages(originRoot, segmentHash string) ([]db.Message, error) { + msgs, _, err := readSegmentMessagesWithLimits( + originRoot, segmentHash, productionArtifactLimits(), + ) + return msgs, err +} + +func readSegmentMessagesWithLimits( + originRoot, segmentHash string, + limits artifactLimits, +) ([]db.Message, int64, error) { + segment, err := readSegmentPreflightWithLimits(originRoot, segmentHash, limits) + if err != nil { + return nil, 0, err + } + segmentMsgs, err := decodeStoredSegment(segment) + if err != nil { + return nil, 0, err + } + return segmentMsgs, int64(len(segment.data)), nil +} + +type storedSegmentPreflight struct { + path string + hash string + data []byte + preflight segmentPreflight +} + +func readSegmentPreflightWithLimits( + originRoot, segmentHash string, + limits artifactLimits, +) (storedSegmentPreflight, error) { + if err := validateHashHex(segmentHash); err != nil { + return storedSegmentPreflight{}, fmt.Errorf("manifest segment has invalid hash: %w", err) + } + path := filepath.Join(originRoot, "segments", segmentHash+segmentExtension) + data, err := readCompressed(path) + if err != nil { + if errors.Is(err, fs.ErrNotExist) { + return storedSegmentPreflight{}, fmt.Errorf( + "%w: segment %s", errIncompleteArtifact, segmentHash, + ) + } + if errors.Is(err, errCorruptArtifact) { + quarantineArtifact(path) + return storedSegmentPreflight{}, fmt.Errorf("segment %s: %w", segmentHash, err) + } + return storedSegmentPreflight{}, fmt.Errorf("reading segment %s: %w", segmentHash, err) + } + if got := hashHex(data); got != segmentHash { + quarantineArtifact(path) + return storedSegmentPreflight{}, fmt.Errorf( + "%w: segment %s hash mismatch: got %s", + errCorruptArtifact, segmentHash, got, + ) + } + preflight, err := preflightSegmentData(data, limits) + if err != nil { + if errors.Is(err, errFutureArtifactVersion) { + return storedSegmentPreflight{}, fmt.Errorf("segment %s: %w", segmentHash, err) + } + quarantineArtifact(path) + return storedSegmentPreflight{}, fmt.Errorf( + "%w: segment %s: %v", errCorruptArtifact, segmentHash, err, + ) + } + return storedSegmentPreflight{ + path: path, hash: segmentHash, data: data, preflight: preflight, + }, nil +} + +func decodeStoredSegment(segment storedSegmentPreflight) ([]db.Message, error) { + segmentMsgs, err := decodePreflightedSegment(segment.preflight) + if err == nil { + return segmentMsgs, nil + } + quarantineArtifact(segment.path) + return nil, fmt.Errorf("%w: segment %s: %v", errCorruptArtifact, segment.hash, err) +} + +func canonicalMessages(msgs []db.Message) []db.Message { + out := make([]db.Message, len(msgs)) + for i, msg := range msgs { + msg.ID = 0 + msg.SessionID = "" + if len(msg.ToolCalls) > 0 { + calls := make([]db.ToolCall, len(msg.ToolCalls)) + copy(calls, msg.ToolCalls) + for j := range calls { + calls[j].MessageID = 0 + calls[j].SessionID = "" + } + msg.ToolCalls = calls + } + out[i] = msg + } + return out +} + +func canonicalUsageEvents(events []db.UsageEvent) []artifactUsageEvent { + out := make([]artifactUsageEvent, len(events)) + for i, ev := range events { + out[i] = artifactUsageEvent{ + MessageOrdinal: ev.MessageOrdinal, + Source: ev.Source, + Model: ev.Model, + InputTokens: ev.InputTokens, + OutputTokens: ev.OutputTokens, + CacheCreationInputTokens: ev.CacheCreationInputTokens, + CacheReadInputTokens: ev.CacheReadInputTokens, + ReasoningTokens: ev.ReasoningTokens, + CostUSD: ev.CostUSD, + CostStatus: ev.CostStatus, + CostSource: ev.CostSource, + OccurredAt: ev.OccurredAt, + DedupKey: ev.DedupKey, + } + } + return out +} + +func validateExportNestedCollections(msgs []db.Message, limits artifactLimits) error { + total := nestedCollectionCounts{} + for _, msg := range msgs { + messageNested, err := dbMessageNestedCounts(msg, limits) + if err != nil { + return err + } + if err := validateMessageFitsSegment(msg.Ordinal, messageNested, limits); err != nil { + return err + } + if exceedsCollectionLimit( + total.toolCalls, messageNested.toolCalls, limits.sessionToolCalls, + ) { + return fmt.Errorf( + "session tool call limit exceeded at message ordinal %d: limit %d", + msg.Ordinal, limits.sessionToolCalls, + ) + } + if exceedsCollectionLimit( + total.resultEvents, messageNested.resultEvents, limits.sessionResultEvents, + ) { + return fmt.Errorf( + "session result event limit exceeded at message ordinal %d: limit %d", + msg.Ordinal, limits.sessionResultEvents, + ) + } + total.toolCalls += messageNested.toolCalls + total.resultEvents += messageNested.resultEvents + } + return nil +} + +func dbMessageNestedCounts( + msg db.Message, + limits artifactLimits, +) (nestedCollectionCounts, error) { + if len(msg.ToolCalls) > limits.messageToolCalls { + return nestedCollectionCounts{}, fmt.Errorf( + "tool call limit exceeded for message ordinal %d: got %d, limit %d", + msg.Ordinal, len(msg.ToolCalls), limits.messageToolCalls, + ) + } + counts := nestedCollectionCounts{toolCalls: len(msg.ToolCalls)} + for toolIndex, call := range msg.ToolCalls { + if len(call.ResultEvents) > limits.toolResultEvents { + return nestedCollectionCounts{}, fmt.Errorf( + "result event limit exceeded for tool call %d in message ordinal %d: got %d, limit %d", + toolIndex, msg.Ordinal, len(call.ResultEvents), limits.toolResultEvents, + ) + } + counts.resultEvents += len(call.ResultEvents) + } + return counts, nil +} + +func validateMessageFitsSegment( + ordinal int, + counts nestedCollectionCounts, + limits artifactLimits, +) error { + if counts.toolCalls > limits.segmentToolCalls { + return fmt.Errorf( + "message ordinal %d cannot fit in one segment: got %d tool calls, segment limit %d", + ordinal, counts.toolCalls, limits.segmentToolCalls, + ) + } + if counts.resultEvents > limits.segmentResultEvents { + return fmt.Errorf( + "message ordinal %d cannot fit in one segment: got %d result events, segment limit %d", + ordinal, counts.resultEvents, limits.segmentResultEvents, + ) + } + return nil +} + +func encodeSegment(msgs []db.Message) ([]byte, error) { + var buf bytes.Buffer + for _, msg := range msgs { + data, err := canonicalJSON(segmentMessageFromDB(msg)) + if err != nil { + return nil, fmt.Errorf("encoding message segment: %w", err) + } + buf.Write(data) + } + return buf.Bytes(), nil +} + +// forEachEncodedSegment emits deterministic NDJSON chunks while bounding each +// chunk to the size accepted by readers. Chunks normally stop near +// segmentTargetSize; a single larger record remains intact up to the hard +// readable limit so message records are never split. +func forEachEncodedSegmentWithLimits( + msgs []db.Message, + limits artifactLimits, + emit func([]byte) error, +) error { + var buf bytes.Buffer + segmentMessages := 0 + segmentNested := nestedCollectionCounts{} + flush := func() error { + if err := emit(buf.Bytes()); err != nil { + return err + } + buf.Reset() + segmentMessages = 0 + segmentNested = nestedCollectionCounts{} + return nil + } + for _, msg := range msgs { + messageNested, err := dbMessageNestedCounts(msg, limits) + if err != nil { + return err + } + if err := validateMessageFitsSegment(msg.Ordinal, messageNested, limits); err != nil { + return err + } + data, err := canonicalJSON(segmentMessageFromDB(msg)) + if err != nil { + return fmt.Errorf("encoding message segment: %w", err) + } + if int64(len(data)) > segmentDecodedLimit { + return fmt.Errorf( + "encoded message record at ordinal %d exceeds %d-byte readable limit", + msg.Ordinal, segmentDecodedLimit, + ) + } + if buf.Len() > 0 && (int64(buf.Len()+len(data)) > segmentTargetSize || + segmentMessages >= limits.segmentMessages || + exceedsCollectionLimit( + segmentNested.toolCalls, + messageNested.toolCalls, + limits.segmentToolCalls, + ) || exceedsCollectionLimit( + segmentNested.resultEvents, + messageNested.resultEvents, + limits.segmentResultEvents, + )) { + if err := flush(); err != nil { + return err + } + } + _, _ = buf.Write(data) + segmentMessages++ + segmentNested.toolCalls += messageNested.toolCalls + segmentNested.resultEvents += messageNested.resultEvents + } + if buf.Len() > 0 || len(msgs) == 0 { + return flush() + } + return nil +} + +func decodeSegment(data []byte) ([]db.Message, error) { + return decodeSegmentWithLimits(data, productionArtifactLimits()) +} + +func decodeSegmentWithLimits(data []byte, limits artifactLimits) ([]db.Message, error) { + preflight, err := preflightSegmentData(data, limits) + if err != nil { + return nil, err + } + return decodePreflightedSegment(preflight) +} + +func decodePreflightedSegment(preflight segmentPreflight) ([]db.Message, error) { + msgs := make([]db.Message, 0, len(preflight.records)) + for _, line := range preflight.records { + var record segmentMessage + if err := json.Unmarshal(line, &record); err != nil { + return nil, fmt.Errorf("decoding message segment: %w", err) + } + msgs = append(msgs, record.dbMessage()) + } + return msgs, nil +} + +func segmentRecords(data []byte, limit int) ([][]byte, error) { + capacity := min(max(limit, 0), 64) + records := make([][]byte, 0, capacity) + remaining := data + lineNumber := 0 + for len(remaining) > 0 { + lineNumber++ + newline := bytes.IndexByte(remaining, '\n') + line := remaining + if newline >= 0 { + line = remaining[:newline] + remaining = remaining[newline+1:] + } else { + remaining = nil + } + if len(records) >= limit { + return nil, fmt.Errorf( + "message record limit exceeded: limit %d per segment", limit, + ) + } + if len(bytes.TrimSpace(line)) == 0 { + return nil, fmt.Errorf("blank message record at line %d", lineNumber) + } + records = append(records, line) + } + return records, nil +} + +func preflightSegmentData(data []byte, limits artifactLimits) (segmentPreflight, error) { + records, err := segmentRecords(data, limits.segmentMessages) + if err != nil { + return segmentPreflight{}, err + } + preflight := segmentPreflight{records: records} + for _, line := range records { + var header struct { + Version int `json:"v"` + } + if err := json.Unmarshal(line, &header); err != nil { + return segmentPreflight{}, fmt.Errorf("decoding message segment header: %w", err) + } + if header.Version > formatVersion { + return segmentPreflight{}, fmt.Errorf( + "%w: message segment has artifact version %d", + errFutureArtifactVersion, header.Version, + ) + } + if header.Version != formatVersion { + return segmentPreflight{}, fmt.Errorf( + "message segment has unsupported artifact version %d", + header.Version, + ) + } + messageNested, err := preflightMessageNestedCollections(line, limits) + if err != nil { + return segmentPreflight{}, err + } + if exceedsCollectionLimit( + preflight.nested.toolCalls, + messageNested.toolCalls, + limits.segmentToolCalls, + ) { + return segmentPreflight{}, fmt.Errorf( + "segment tool call limit exceeded: limit %d", limits.segmentToolCalls, + ) + } + if exceedsCollectionLimit( + preflight.nested.resultEvents, + messageNested.resultEvents, + limits.segmentResultEvents, + ) { + return segmentPreflight{}, fmt.Errorf( + "segment result event limit exceeded: limit %d", + limits.segmentResultEvents, + ) + } + preflight.nested.toolCalls += messageNested.toolCalls + preflight.nested.resultEvents += messageNested.resultEvents + } + return preflight, nil +} + +func preflightMessageNestedCollections( + line []byte, + limits artifactLimits, +) (nestedCollectionCounts, error) { + var envelope struct { + Ordinal int `json:"ordinal"` + ToolCalls json.RawMessage `json:"tool_calls"` + } + if err := json.Unmarshal(line, &envelope); err != nil { + return nestedCollectionCounts{}, fmt.Errorf( + "decoding message segment collections: %w", err, + ) + } + return preflightToolCallCollections(envelope.ToolCalls, envelope.Ordinal, limits) +} + +func preflightToolCallCollections( + data json.RawMessage, + ordinal int, + limits artifactLimits, +) (nestedCollectionCounts, error) { + trimmed := bytes.TrimSpace(data) + if len(trimmed) == 0 || bytes.Equal(trimmed, []byte("null")) { + return nestedCollectionCounts{}, nil + } + dec := json.NewDecoder(bytes.NewReader(trimmed)) + token, err := dec.Token() + if err != nil { + return nestedCollectionCounts{}, err + } + if token != json.Delim('[') { + return nestedCollectionCounts{}, errors.New("message tool_calls must be an array") + } + counts := nestedCollectionCounts{} + for dec.More() { + if counts.toolCalls >= limits.messageToolCalls { + return nestedCollectionCounts{}, fmt.Errorf( + "tool call limit exceeded for message ordinal %d: limit %d per message", + ordinal, limits.messageToolCalls, + ) + } + var toolEnvelope struct { + ResultEvents json.RawMessage `json:"result_events"` + } + if err := dec.Decode(&toolEnvelope); err != nil { + return nestedCollectionCounts{}, fmt.Errorf( + "decoding tool call %d in message ordinal %d: %w", + counts.toolCalls, ordinal, err, + ) + } + resultEvents, err := countJSONArrayElements( + toolEnvelope.ResultEvents, "result event", limits.toolResultEvents, + ) + if err != nil { + return nestedCollectionCounts{}, fmt.Errorf( + "preflighting tool call %d in message ordinal %d: %w", + counts.toolCalls, ordinal, err, + ) + } + counts.toolCalls++ + counts.resultEvents += resultEvents + } + if _, err := dec.Token(); err != nil { + return nestedCollectionCounts{}, err + } + return counts, nil +} + +func dbUsageEvents(events []artifactUsageEvent, sessionID string) []db.UsageEvent { + out := make([]db.UsageEvent, len(events)) + for i, ev := range events { + out[i] = db.UsageEvent{ + SessionID: sessionID, + MessageOrdinal: ev.MessageOrdinal, + Source: ev.Source, + Model: ev.Model, + InputTokens: ev.InputTokens, + OutputTokens: ev.OutputTokens, + CacheCreationInputTokens: ev.CacheCreationInputTokens, + CacheReadInputTokens: ev.CacheReadInputTokens, + ReasoningTokens: ev.ReasoningTokens, + CostUSD: ev.CostUSD, + CostStatus: ev.CostStatus, + CostSource: ev.CostSource, + OccurredAt: ev.OccurredAt, + DedupKey: ev.DedupKey, + } + } + return out +} + +func segmentMessageFromDB(msg db.Message) segmentMessage { + record := segmentMessage{ + Version: formatVersion, + Ordinal: msg.Ordinal, + Role: msg.Role, + Content: msg.Content, + ThinkingText: msg.ThinkingText, + Timestamp: msg.Timestamp, + HasThinking: msg.HasThinking, + HasToolUse: msg.HasToolUse, + ContentLength: msg.ContentLength, + Model: msg.Model, + TokenUsage: msg.TokenUsage, + ContextTokens: msg.ContextTokens, + OutputTokens: msg.OutputTokens, + HasContextTokens: msg.HasContextTokens, + HasOutputTokens: msg.HasOutputTokens, + ClaudeMessageID: msg.ClaudeMessageID, + ClaudeRequestID: msg.ClaudeRequestID, + IsSystem: msg.IsSystem, + SourceType: msg.SourceType, + SourceSubtype: msg.SourceSubtype, + SourceUUID: msg.SourceUUID, + SourceParentUUID: msg.SourceParentUUID, + IsSidechain: msg.IsSidechain, + IsCompactBoundary: msg.IsCompactBoundary, + } + if len(msg.ToolCalls) > 0 { + record.ToolCalls = make([]segmentToolCall, len(msg.ToolCalls)) + for i, call := range msg.ToolCalls { + record.ToolCalls[i] = segmentToolCall{ + CallIndex: i, + ToolName: call.ToolName, + Category: call.Category, + ToolUseID: call.ToolUseID, + InputJSON: call.InputJSON, + FilePath: call.FilePath, + SkillName: call.SkillName, + ResultContentLength: call.ResultContentLength, + ResultContent: call.ResultContent, + SubagentSessionID: call.SubagentSessionID, + } + if len(call.ResultEvents) > 0 { + record.ToolCalls[i].ResultEvents = make([]segmentResultEvent, len(call.ResultEvents)) + for j, ev := range call.ResultEvents { + record.ToolCalls[i].ResultEvents[j] = segmentResultEvent{ + ToolUseID: ev.ToolUseID, + AgentID: ev.AgentID, + SubagentSessionID: ev.SubagentSessionID, + Source: ev.Source, + Status: ev.Status, + Content: ev.Content, + ContentLength: ev.ContentLength, + Timestamp: ev.Timestamp, + EventIndex: ev.EventIndex, + } + } + } + } + } + return record +} + +func (m segmentMessage) dbMessage() db.Message { + msg := db.Message{ + Ordinal: m.Ordinal, + Role: m.Role, + Content: m.Content, + ThinkingText: m.ThinkingText, + Timestamp: m.Timestamp, + HasThinking: m.HasThinking, + HasToolUse: m.HasToolUse, + ContentLength: m.ContentLength, + Model: m.Model, + TokenUsage: m.TokenUsage, + ContextTokens: m.ContextTokens, + OutputTokens: m.OutputTokens, + HasContextTokens: m.HasContextTokens, + HasOutputTokens: m.HasOutputTokens, + ClaudeMessageID: m.ClaudeMessageID, + ClaudeRequestID: m.ClaudeRequestID, + IsSystem: m.IsSystem, + SourceType: m.SourceType, + SourceSubtype: m.SourceSubtype, + SourceUUID: m.SourceUUID, + SourceParentUUID: m.SourceParentUUID, + IsSidechain: m.IsSidechain, + IsCompactBoundary: m.IsCompactBoundary, + } + if len(m.ToolCalls) > 0 { + msg.ToolCalls = make([]db.ToolCall, len(m.ToolCalls)) + for i, call := range m.ToolCalls { + msg.ToolCalls[i] = db.ToolCall{ + ToolName: call.ToolName, + Category: call.Category, + ToolUseID: call.ToolUseID, + InputJSON: call.InputJSON, + FilePath: call.FilePath, + SkillName: call.SkillName, + ResultContentLength: call.ResultContentLength, + ResultContent: call.ResultContent, + SubagentSessionID: call.SubagentSessionID, + } + if len(call.ResultEvents) > 0 { + msg.ToolCalls[i].ResultEvents = make([]db.ToolResultEvent, len(call.ResultEvents)) + for j, ev := range call.ResultEvents { + msg.ToolCalls[i].ResultEvents[j] = db.ToolResultEvent{ + ToolUseID: ev.ToolUseID, + AgentID: ev.AgentID, + SubagentSessionID: ev.SubagentSessionID, + Source: ev.Source, + Status: ev.Status, + Content: ev.Content, + ContentLength: ev.ContentLength, + Timestamp: ev.Timestamp, + EventIndex: ev.EventIndex, + } + } + } + } + } + return msg +} + +func canonicalJSON(v any) ([]byte, error) { + var buf bytes.Buffer + if err := writeCanonicalJSON(&buf, reflect.ValueOf(v)); err != nil { + return nil, fmt.Errorf("encoding canonical artifact JSON: %w", err) + } + buf.WriteByte('\n') + return buf.Bytes(), nil +} + +func writeCanonicalJSON(buf *bytes.Buffer, v reflect.Value) error { + if !v.IsValid() { + buf.WriteString("null") + return nil + } + if v.Kind() == reflect.Interface { + if v.IsNil() { + buf.WriteString("null") + return nil + } + return writeCanonicalJSON(buf, v.Elem()) + } + if v.Kind() == reflect.Pointer { + if v.IsNil() { + buf.WriteString("null") + return nil + } + return writeCanonicalJSON(buf, v.Elem()) + } + if v.Type() == reflect.TypeFor[json.RawMessage]() { + raw := v.Interface().(json.RawMessage) + if len(raw) == 0 { + buf.WriteString("null") + return nil + } + dec := json.NewDecoder(bytes.NewReader(raw)) + dec.UseNumber() + var decoded any + if err := dec.Decode(&decoded); err != nil { + return err + } + return writeCanonicalJSON(buf, reflect.ValueOf(decoded)) + } + if v.Type() == reflect.TypeFor[json.Number]() { + buf.WriteString(v.Interface().(json.Number).String()) + return nil + } + switch v.Kind() { + case reflect.Bool: + buf.WriteString(strconv.FormatBool(v.Bool())) + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + buf.WriteString(strconv.FormatInt(v.Int(), 10)) + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: + buf.WriteString(strconv.FormatUint(v.Uint(), 10)) + case reflect.Float32, reflect.Float64: + data, err := json.Marshal(v.Interface()) + if err != nil { + return err + } + buf.Write(data) + case reflect.String: + data, err := json.Marshal(v.String()) + if err != nil { + return err + } + buf.Write(data) + case reflect.Slice, reflect.Array: + buf.WriteByte('[') + for i := 0; i < v.Len(); i++ { + if i > 0 { + buf.WriteByte(',') + } + if err := writeCanonicalJSON(buf, v.Index(i)); err != nil { + return err + } + } + buf.WriteByte(']') + case reflect.Map: + return writeCanonicalMap(buf, v) + case reflect.Struct: + return writeCanonicalStruct(buf, v) + default: + return fmt.Errorf("unsupported canonical JSON kind %s", v.Kind()) + } + return nil +} + +func writeCanonicalMap(buf *bytes.Buffer, v reflect.Value) error { + if v.IsNil() { + buf.WriteString("null") + return nil + } + if v.Type().Key().Kind() != reflect.String { + return fmt.Errorf("unsupported canonical map key type %s", v.Type().Key()) + } + keys := make([]string, 0, v.Len()) + for _, key := range v.MapKeys() { + keys = append(keys, key.String()) + } + sort.Strings(keys) + buf.WriteByte('{') + for i, key := range keys { + if i > 0 { + buf.WriteByte(',') + } + keyData, err := json.Marshal(key) + if err != nil { + return err + } + buf.Write(keyData) + buf.WriteByte(':') + if err := writeCanonicalJSON(buf, v.MapIndex(reflect.ValueOf(key))); err != nil { + return err + } + } + buf.WriteByte('}') + return nil +} + +type canonicalField struct { + name string + value reflect.Value +} + +func writeCanonicalStruct(buf *bytes.Buffer, v reflect.Value) error { + fields := make([]canonicalField, 0, v.NumField()) + t := v.Type() + for i := 0; i < v.NumField(); i++ { + field := t.Field(i) + if field.PkgPath != "" { + continue + } + name, omitEmpty, skip := jsonField(field) + if skip { + continue + } + value := v.Field(i) + if omitEmpty && isCanonicalEmpty(value) { + continue + } + fields = append(fields, canonicalField{name: name, value: value}) + } + sort.Slice(fields, func(i, j int) bool { + return fields[i].name < fields[j].name + }) + + buf.WriteByte('{') + for i, field := range fields { + if i > 0 { + buf.WriteByte(',') + } + name, err := json.Marshal(field.name) + if err != nil { + return err + } + buf.Write(name) + buf.WriteByte(':') + if err := writeCanonicalJSON(buf, field.value); err != nil { + return err + } + } + buf.WriteByte('}') + return nil +} + +func jsonField(field reflect.StructField) (name string, omitEmpty bool, skip bool) { + name = field.Name + tag := field.Tag.Get("json") + if tag == "-" { + return "", false, true + } + if tag == "" { + return name, false, false + } + parts := strings.Split(tag, ",") + if parts[0] != "" { + name = parts[0] + } + for _, opt := range parts[1:] { + if opt == "omitempty" { + omitEmpty = true + } + } + return name, omitEmpty, false +} + +func isCanonicalEmpty(v reflect.Value) bool { + if !v.IsValid() { + return true + } + switch v.Kind() { + case reflect.Array: + return v.Len() == 0 + case reflect.Map, reflect.Slice, reflect.String: + return v.Len() == 0 + case reflect.Bool: + return !v.Bool() + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + return v.Int() == 0 + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: + return v.Uint() == 0 + case reflect.Float32, reflect.Float64: + return v.Float() == 0 + case reflect.Interface, reflect.Pointer: + return v.IsNil() + } + return false +} + +func hashHex(data []byte) string { + sum := sha256.Sum256(data) + return hex.EncodeToString(sum[:]) +} + +func writeCompressed(path string, data []byte) error { + var buf bytes.Buffer + enc, err := zstd.NewWriter(&buf) + if err != nil { + return err + } + if _, err := enc.Write(data); err != nil { + enc.Close() + return err + } + if err := enc.Close(); err != nil { + return err + } + return writeFileAtomic(path, buf.Bytes(), 0o644) +} + +func readCompressed(path string) ([]byte, error) { + data, err := os.ReadFile(path) + if err != nil { + return nil, err + } + limit, err := compressedArtifactDecodedLimit(path) + if err != nil { + return nil, fmt.Errorf("%w: %v", errCorruptArtifact, err) + } + out, err := readCompressedBytes(data, limit) + if err != nil { + return nil, fmt.Errorf("%w: %v", errCorruptArtifact, err) + } + return out, nil +} + +func compressedArtifactDecodedLimit(path string) (int64, error) { + switch { + case strings.HasSuffix(path, manifestExtension): + return manifestDecodedLimit, nil + case strings.HasSuffix(path, segmentExtension): + return segmentDecodedLimit, nil + default: + return 0, fmt.Errorf("unknown compressed artifact extension for %s", filepath.Base(path)) + } +} + +// errArtifactPathConflict reports that an immutable artifact path already +// holds different content. Callers detect it with errors.Is. +var errArtifactPathConflict = errors.New("artifact path conflict") + +// quarantineArtifact renames a corrupt artifact aside so read paths treat it +// as missing and a valid copy can be re-fetched under the original name. The +// rename is best-effort: on failure the file stays in place and is rescanned. +func quarantineArtifact(path string) { + dst := path + quarantineSuffix + _ = os.Remove(dst) + err := os.Rename(path, dst) + switch { + case err == nil: + log.Printf("artifact: quarantined corrupt artifact %s", path) + case !errors.Is(err, fs.ErrNotExist): + log.Printf("artifact: quarantining %s: %v", path, err) + } +} + +func writeFileAtomic(path string, data []byte, perm fs.FileMode) error { + if done, err := existingArtifactMatches(path, data); err != nil || done { + return err + } + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + return err + } + tmp, err := os.CreateTemp(filepath.Dir(path), tempFilePrefix+"*") + if err != nil { + return err + } + tmpName := tmp.Name() + defer os.Remove(tmpName) + if _, err := tmp.Write(data); err != nil { + tmp.Close() + return err + } + if err := tmp.Chmod(perm); err != nil { + tmp.Close() + return err + } + if err := tmp.Close(); err != nil { + return err + } + if done, err := existingArtifactMatches(path, data); err != nil || done { + return err + } + if writeFileAtomicBeforeCommit != nil { + writeFileAtomicBeforeCommit(path) + } + if err := writeFileAtomicLink(tmpName, path); err != nil { + if errors.Is(err, fs.ErrExist) { + if done, matchErr := existingArtifactMatches(path, data); matchErr != nil || done { + return matchErr + } + return fmt.Errorf("%w at %s", errArtifactPathConflict, path) + } + if isHardLinkUnsupported(err) { + return writeFileNoReplace(path, data, perm) + } + return err + } + return nil +} + +func isHardLinkUnsupported(err error) bool { + return errors.Is(err, syscall.ENOTSUP) || + errors.Is(err, syscall.EOPNOTSUPP) || + errors.Is(err, syscall.EXDEV) || + errors.Is(err, syscall.EPERM) +} + +func writeFileNoReplace(path string, data []byte, perm fs.FileMode) error { + file, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_EXCL, perm) + if err != nil { + if errors.Is(err, fs.ErrExist) { + if done, matchErr := existingArtifactMatches(path, data); matchErr != nil || done { + return matchErr + } + return fmt.Errorf("%w at %s", errArtifactPathConflict, path) + } + return err + } + created := true + defer func() { + if created { + _ = os.Remove(path) + } + }() + if _, err := file.Write(data); err != nil { + _ = file.Close() + return err + } + if err := file.Chmod(perm); err != nil { + _ = file.Close() + return err + } + if err := file.Close(); err != nil { + return err + } + created = false + return nil +} + +func existingArtifactMatches(path string, data []byte) (bool, error) { + info, err := os.Lstat(path) + if err != nil { + if errors.Is(err, fs.ErrNotExist) { + return false, nil + } + return false, err + } + if info.IsDir() { + return false, fmt.Errorf("artifact destination %s is a directory", path) + } + if !info.Mode().IsRegular() { + return false, fmt.Errorf("artifact destination %s is not a regular file", path) + } + existing, err := os.ReadFile(path) + if err != nil { + return false, err + } + if bytes.Equal(existing, data) { + return true, nil + } + return false, fmt.Errorf("%w at %s", errArtifactPathConflict, path) +} + +// CopyUnion copies files from src into dst without deleting files that only +// exist in dst. Existing identical files are left in place. +func CopyUnion(src, dst string) error { + srcRoot, err := openArtifactRoot(src, "source") + if err != nil { + return err + } + defer srcRoot.Close() + dstRoot, err := openArtifactRoot(dst, "destination") + if err != nil { + return err + } + defer dstRoot.Close() + if err := validateDisjointRoots(srcRoot.Name(), dstRoot.Name()); err != nil { + return err + } + + return fs.WalkDir(srcRoot.FS(), ".", func(path string, ent fs.DirEntry, walkErr error) error { + if walkErr != nil { + return walkErr + } + if isTempArtifactEntry(ent.Name()) { + if ent.IsDir() { + return filepath.SkipDir + } + return nil + } + if !ent.IsDir() && strings.HasSuffix(ent.Name(), quarantineSuffix) { + return nil + } + if path == "." { + return nil + } + rel := filepath.FromSlash(path) + if ent.IsDir() { + return dstRoot.MkdirAll(rel, 0o755) + } + info, err := ent.Info() + if err != nil { + return err + } + if !info.Mode().IsRegular() { + return fmt.Errorf( + "artifact source %s is not a regular file", + filepath.Join(srcRoot.Name(), rel), + ) + } + return copyUnionFile(srcRoot, dstRoot, rel) + }) +} + +func openArtifactRoot(path, role string) (*os.Root, error) { + info, err := os.Lstat(path) + if err != nil { + return nil, err + } + if !info.IsDir() { + return nil, fmt.Errorf("artifact %s root %s is not a directory", role, path) + } + root, err := os.OpenRoot(path) + if err != nil { + return nil, fmt.Errorf("opening artifact %s root: %w", role, err) + } + openedInfo, err := root.Stat(".") + if err != nil { + _ = root.Close() + return nil, fmt.Errorf("stating artifact %s root: %w", role, err) + } + currentInfo, err := os.Lstat(path) + if err != nil { + _ = root.Close() + return nil, err + } + if !currentInfo.IsDir() || !os.SameFile(openedInfo, currentInfo) { + _ = root.Close() + return nil, fmt.Errorf("artifact %s root %s changed while opening", role, path) + } + return root, nil +} + +func openArtifactSubroot(parent *os.Root, rel, role string) (*os.Root, error) { + info, err := parent.Lstat(rel) + if err != nil { + return nil, err + } + if !info.IsDir() { + return nil, fmt.Errorf("artifact %s %s is not a directory", role, rel) + } + root, err := parent.OpenRoot(rel) + if err != nil { + return nil, fmt.Errorf("opening artifact %s: %w", role, err) + } + openedInfo, err := root.Stat(".") + if err != nil { + _ = root.Close() + return nil, fmt.Errorf("stating artifact %s: %w", role, err) + } + currentInfo, err := parent.Lstat(rel) + if err != nil { + _ = root.Close() + return nil, err + } + if !currentInfo.IsDir() || !os.SameFile(openedInfo, currentInfo) { + _ = root.Close() + return nil, fmt.Errorf("artifact %s %s changed while opening", role, rel) + } + return root, nil +} + +// copyUnionFile copies one artifact file into the destination store. Corrupt +// content-addressed artifacts are never mirrored: an invalid source is skipped +// and an invalid destination under a valid source's name is repaired, so one +// bad file cannot spread between stores or wedge the exchange. +func copyUnionFile(srcRoot, dstRoot *os.Root, rel string) error { + path := filepath.Join(srcRoot.Name(), rel) + srcData, info, err := readRootRegularFile(srcRoot, rel) + if err != nil { + return err + } + to := filepath.Join(dstRoot.Name(), rel) + existing, err := dstRoot.Lstat(rel) + if err != nil && !errors.Is(err, fs.ErrNotExist) { + return err + } + if err == nil { + if existing.IsDir() { + return fmt.Errorf("artifact destination %s is a directory", to) + } + if !existing.Mode().IsRegular() { + return fmt.Errorf("artifact destination %s is not a regular file", to) + } + if existing.Size() == info.Size() { + dstData, err := dstRoot.ReadFile(rel) + if err != nil { + return err + } + if bytes.Equal(srcData, dstData) { + return nil + } + } + return reconcileArtifactConflict(path, srcData, dstRoot, rel, info.Mode().Perm()) + } + if known, verr := validateKnownArtifact(rel, srcData); known && verr != nil { + // Quarantine at detection instead of leaving the corrupt file in + // place: the export "unchanged" fast path only checks that the + // content-addressed files exist, so an owned artifact left corrupt + // would never be regenerated, and an imported one would never be + // re-fetched. Renaming it away lets both recover on the next round. + log.Printf("artifact: not mirroring corrupt artifact %s: %v", path, verr) + quarantineArtifactRoot(srcRoot, rel) + return nil + } + return writeFileAtomicRoot(dstRoot, rel, srcData, info.Mode().Perm()) +} + +func readRootRegularFile(root *os.Root, rel string) ([]byte, fs.FileInfo, error) { + file, err := root.Open(rel) + if err != nil { + return nil, nil, err + } + defer file.Close() + info, err := file.Stat() + if err != nil { + return nil, nil, err + } + if !info.Mode().IsRegular() { + return nil, nil, fmt.Errorf( + "artifact source %s is not a regular file", filepath.Join(root.Name(), rel), + ) + } + data, err := io.ReadAll(file) + if err != nil { + return nil, nil, err + } + return data, info, nil +} + +func quarantineArtifactRoot(root *os.Root, rel string) { + dst := rel + quarantineSuffix + _ = root.Remove(dst) + err := root.Rename(rel, dst) + path := filepath.Join(root.Name(), rel) + switch { + case err == nil: + log.Printf("artifact: quarantined corrupt artifact %s", path) + case !errors.Is(err, fs.ErrNotExist): + log.Printf("artifact: quarantining %s: %v", path, err) + } +} + +// reconcileArtifactConflict handles a same-name, different-content pair. For a +// recognized artifact whose source validates and destination does not, the +// destination is repaired in place; a corrupt source is skipped instead of +// mirrored. Everything else keeps the write-once conflict error. +func reconcileArtifactConflict( + path string, + srcData []byte, + dstRoot *os.Root, + rel string, + perm fs.FileMode, +) error { + to := filepath.Join(dstRoot.Name(), rel) + known, srcErr := validateKnownArtifact(rel, srcData) + if !known { + return fmt.Errorf("%w at %s", errArtifactPathConflict, to) + } + if srcErr != nil { + log.Printf("artifact: not mirroring corrupt artifact %s: %v", path, srcErr) + return nil + } + dstData, err := dstRoot.ReadFile(rel) + if err != nil { + return err + } + if _, dstErr := validateKnownArtifact(rel, dstData); dstErr != nil { + log.Printf("artifact: repairing corrupt artifact %s from %s", to, path) + return replaceFileAtomicRoot(dstRoot, rel, srcData, perm) + } + return fmt.Errorf("%w at %s", errArtifactPathConflict, to) +} + +func writeFileAtomicRoot(root *os.Root, rel string, data []byte, perm fs.FileMode) error { + if done, err := existingArtifactMatchesRoot(root, rel, data); err != nil || done { + return err + } + if err := root.MkdirAll(filepath.Dir(rel), 0o755); err != nil { + return err + } + tmp, tmpRel, err := createRootTemp(root, filepath.Dir(rel)) + if err != nil { + return err + } + defer func() { _ = root.Remove(tmpRel) }() + if err := writeAndCloseArtifact(tmp, data, perm); err != nil { + return err + } + if done, err := existingArtifactMatchesRoot(root, rel, data); err != nil || done { + return err + } + if err := root.Link(tmpRel, rel); err != nil { + if errors.Is(err, fs.ErrExist) { + if done, matchErr := existingArtifactMatchesRoot(root, rel, data); matchErr != nil || done { + return matchErr + } + return fmt.Errorf("%w at %s", errArtifactPathConflict, filepath.Join(root.Name(), rel)) + } + if isHardLinkUnsupported(err) { + return writeFileNoReplaceRoot(root, rel, data, perm) + } + return err + } + return nil +} + +func replaceFileAtomicRoot(root *os.Root, rel string, data []byte, perm fs.FileMode) error { + if err := root.MkdirAll(filepath.Dir(rel), 0o755); err != nil { + return err + } + tmp, tmpRel, err := createRootTemp(root, filepath.Dir(rel)) + if err != nil { + return err + } + defer func() { _ = root.Remove(tmpRel) }() + if err := writeAndCloseArtifact(tmp, data, perm); err != nil { + return err + } + return root.Rename(tmpRel, rel) +} + +func createRootTemp(root *os.Root, dir string) (*os.File, string, error) { + for range 100 { + var suffix [8]byte + if _, err := rand.Read(suffix[:]); err != nil { + return nil, "", err + } + rel := filepath.Join(dir, tempFilePrefix+hex.EncodeToString(suffix[:])) + file, err := root.OpenFile(rel, os.O_RDWR|os.O_CREATE|os.O_EXCL, 0o600) + if err == nil { + return file, rel, nil + } + if !errors.Is(err, fs.ErrExist) { + return nil, "", err + } + } + return nil, "", errors.New("creating temporary artifact file: too many collisions") +} + +func writeAndCloseArtifact(file *os.File, data []byte, perm fs.FileMode) error { + if _, err := file.Write(data); err != nil { + _ = file.Close() + return err + } + if err := file.Chmod(perm); err != nil { + _ = file.Close() + return err + } + return file.Close() +} + +func writeFileNoReplaceRoot(root *os.Root, rel string, data []byte, perm fs.FileMode) error { + file, err := root.OpenFile(rel, os.O_WRONLY|os.O_CREATE|os.O_EXCL, perm) + if err != nil { + if errors.Is(err, fs.ErrExist) { + if done, matchErr := existingArtifactMatchesRoot(root, rel, data); matchErr != nil || done { + return matchErr + } + return fmt.Errorf("%w at %s", errArtifactPathConflict, filepath.Join(root.Name(), rel)) + } + return err + } + return writeAndCloseArtifact(file, data, perm) +} + +func existingArtifactMatchesRoot(root *os.Root, rel string, data []byte) (bool, error) { + info, err := root.Lstat(rel) + if err != nil { + if errors.Is(err, fs.ErrNotExist) { + return false, nil + } + return false, err + } + path := filepath.Join(root.Name(), rel) + if info.IsDir() { + return false, fmt.Errorf("artifact destination %s is a directory", path) + } + if !info.Mode().IsRegular() { + return false, fmt.Errorf("artifact destination %s is not a regular file", path) + } + existing, err := root.ReadFile(rel) + if err != nil { + return false, err + } + if bytes.Equal(existing, data) { + return true, nil + } + return false, fmt.Errorf("%w at %s", errArtifactPathConflict, path) +} + +// validateKnownArtifact validates one store file's bytes against its +// path-derived identity. known is false when rel does not name a recognized +// origin/kind/name artifact; such files are not validatable and are mirrored +// as-is for forward compatibility. +func validateKnownArtifact(rel string, data []byte) (known bool, err error) { + parts := strings.Split(filepath.ToSlash(rel), "/") + if len(parts) != 3 { + return false, nil + } + spec, err := artifactSpecForKind(parts[0], parts[1], parts[2]) + if err != nil { + return false, nil + } + return true, validateArtifactData(spec, data) +} + +func isTempArtifactEntry(name string) bool { + return strings.HasPrefix(name, tempFilePrefix) +} + +// IsFolderTarget reports whether target is a local filesystem target rather +// than a future HTTP or object-store target. +func IsFolderTarget(target string) bool { + if target == "" || strings.Contains(target, "://") { + return false + } + if isWindowsDrivePath(target) { + return true + } + _, _, err := net.SplitHostPort(target) + return err != nil +} + +func isWindowsDrivePath(target string) bool { + if len(target) < 3 || target[1] != ':' { + return false + } + c := target[0] + if (c < 'A' || c > 'Z') && (c < 'a' || c > 'z') { + return false + } + return target[2] == '\\' || target[2] == '/' +} diff --git a/internal/artifact/sync_test.go b/internal/artifact/sync_test.go new file mode 100644 index 000000000..36fb1cac6 --- /dev/null +++ b/internal/artifact/sync_test.go @@ -0,0 +1,2661 @@ +package artifact + +import ( + "context" + "errors" + "fmt" + "os" + "path/filepath" + "strings" + "syscall" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.kenn.io/agentsview/internal/db" +) + +func TestEnsureOriginPersists(t *testing.T) { + database := testDB(t) + + first, err := EnsureOrigin(database) + require.NoError(t, err) + require.NotEmpty(t, first) + require.NotEqual(t, "local", first) + + second, err := EnsureOrigin(database) + require.NoError(t, err) + assert.Equal(t, first, second) +} + +func TestIsFolderTargetAcceptsWindowsDrivePaths(t *testing.T) { + tests := []struct { + name string + target string + want bool + }{ + {"windows backslash path", `C:\Users\runner\artifacts`, true}, + {"windows slash path", `C:/Users/runner/artifacts`, true}, + {"posix path", "/tmp/agentsview-artifacts", true}, + {"relative path", "artifacts", true}, + {"http peer", "https://peer.example.test/artifacts", false}, + {"s3 target", "s3://bucket/artifacts", false}, + {"host port", "localhost:8080", false}, + {"empty", "", false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.want, IsFolderTarget(tt.target)) + }) + } +} + +func TestAdoptOriginPersistsConfigOrigin(t *testing.T) { + database := testDB(t) + + require.NoError(t, AdoptOrigin(database, "desk-a1b2c3")) + + stored, err := StoredOrigin(database) + require.NoError(t, err) + assert.Equal(t, "desk-a1b2c3", stored) + + // EnsureOrigin and its callers now agree with the adopted origin instead + // of generating a divergent DB-only value. + ensured, err := EnsureOrigin(database) + require.NoError(t, err) + assert.Equal(t, "desk-a1b2c3", ensured) +} + +func TestAdoptOriginIsIdempotent(t *testing.T) { + database := testDB(t) + + require.NoError(t, AdoptOrigin(database, "desk-a1b2c3")) + require.NoError(t, AdoptOrigin(database, "desk-a1b2c3")) + + stored, err := StoredOrigin(database) + require.NoError(t, err) + assert.Equal(t, "desk-a1b2c3", stored) +} + +func TestAdoptOriginOverwritesDivergentDBOrigin(t *testing.T) { + database := testDB(t) + + // Simulate the pre-fix state: the recorder generated a DB-only origin + // before the authoritative config origin existed. + stale, err := EnsureOrigin(database) + require.NoError(t, err) + require.NotEqual(t, "desk-a1b2c3", stale) + + require.NoError(t, AdoptOrigin(database, "desk-a1b2c3")) + + stored, err := StoredOrigin(database) + require.NoError(t, err) + assert.Equal(t, "desk-a1b2c3", stored) +} + +func TestAdoptOriginRejectsInvalidOrigin(t *testing.T) { + database := testDB(t) + + err := AdoptOrigin(database, "../outside") + require.Error(t, err) + assert.Contains(t, err.Error(), "adopting artifact origin") + + stored, err := StoredOrigin(database) + require.NoError(t, err) + assert.Empty(t, stored) +} + +func TestEnsureOriginRejectsInvalidPersistedOrigin(t *testing.T) { + database := testDB(t) + require.NoError(t, database.SetSyncState(originStateKey, "../outside")) + + origin, err := EnsureOrigin(database) + require.Error(t, err) + assert.Empty(t, origin) + assert.Contains(t, err.Error(), "stored artifact origin") + assert.Contains(t, err.Error(), "invalid artifact origin") +} + +func TestSyncFolderRoundTripImportsForeignSession(t *testing.T) { + ctx := context.Background() + share := t.TempDir() + aData := t.TempDir() + bData := t.TempDir() + aDB := testDB(t) + bDB := testDB(t) + + require.NoError(t, aDB.SetSyncState(originStateKey, "laptop-a1b2c3")) + require.NoError(t, bDB.SetSyncState(originStateKey, "desktop-d4e5f6")) + seedSession(t, aDB, "sess-1", "alpha") + + aRes, err := SyncFolder(ctx, aDB, SyncOptions{DataDir: aData, Target: share}) + require.NoError(t, err) + assert.Equal(t, "laptop-a1b2c3", aRes.Origin) + assert.Equal(t, 1, aRes.ExportedSessions) + + bRes, err := SyncFolder(ctx, bDB, SyncOptions{DataDir: bData, Target: share}) + require.NoError(t, err) + assert.Equal(t, 1, bRes.ImportedSessions) + assert.Equal(t, 2, bRes.ImportedMessages) + + bRes, err = SyncFolder(ctx, bDB, SyncOptions{DataDir: bData, Target: share}) + require.NoError(t, err) + assert.Zero(t, bRes.ImportedSessions) + assert.Zero(t, bRes.ImportedMessages) + + got, err := bDB.GetSessionFull(ctx, "laptop-a1b2c3~sess-1") + require.NoError(t, err) + require.NotNil(t, got) + assert.Equal(t, "laptop-a1b2c3", got.Machine) + assert.Equal(t, "alpha", got.Project) + + msgs, err := bDB.GetAllMessages(ctx, got.ID) + require.NoError(t, err) + require.Len(t, msgs, 2) + assert.Equal(t, "user", msgs[0].Role) + assert.Equal(t, "hello", msgs[0].Content) + assert.Equal(t, "assistant", msgs[1].Role) + assert.Equal(t, "world", msgs[1].Content) + assert.Equal(t, filepath.Join(bData, "artifacts", "laptop-a1b2c3"), filepath.Join(bData, "artifacts", got.Machine)) +} + +func TestSyncFolderRoundTripPreservesSessionName(t *testing.T) { + ctx := context.Background() + share := t.TempDir() + aData := t.TempDir() + bData := t.TempDir() + aDB := testDB(t) + bDB := testDB(t) + + require.NoError(t, aDB.SetSyncState(originStateKey, "laptop-a1b2c3")) + require.NoError(t, bDB.SetSyncState(originStateKey, "desktop-d4e5f6")) + sessionName := "Parser Provided Title" + seedSession(t, aDB, "sess-1", "alpha", func(s *db.Session) { + s.SessionName = &sessionName + }) + + _, err := SyncFolder(ctx, aDB, SyncOptions{DataDir: aData, Target: share}) + require.NoError(t, err) + + bRes, err := SyncFolder(ctx, bDB, SyncOptions{DataDir: bData, Target: share}) + require.NoError(t, err) + require.Equal(t, 1, bRes.ImportedSessions) + + got, err := bDB.GetSessionFull(ctx, "laptop-a1b2c3~sess-1") + require.NoError(t, err) + require.NotNil(t, got) + require.NotNil(t, got.SessionName) + assert.Equal(t, sessionName, *got.SessionName) +} + +func TestSyncFolderInitBaselineMetadataConvergesCuration(t *testing.T) { + ctx := context.Background() + share := t.TempDir() + aData := t.TempDir() + bData := t.TempDir() + aDB := testDB(t) + bDB := testDB(t) + + require.NoError(t, AdoptOrigin(aDB, "laptop-a1b2c3")) + require.NoError(t, AdoptOrigin(bDB, "desktop-d4e5f6")) + seedSession(t, aDB, "sess-1", "alpha") + require.NoError(t, aDB.ReplaceSessionMessages("sess-1", []db.Message{ + { + SessionID: "sess-1", + Ordinal: 0, + Role: "user", + Content: "hello", + ContentLength: 5, + SourceUUID: "uuid-question", + }, + { + SessionID: "sess-1", + Ordinal: 1, + Role: "assistant", + Content: "world", + ContentLength: 5, + SourceUUID: "uuid-answer", + }, + })) + displayName := "Already renamed" + require.NoError(t, aDB.RenameSession("sess-1", &displayName)) + starred, err := aDB.StarSession("sess-1") + require.NoError(t, err) + require.True(t, starred) + msgs, err := aDB.GetAllMessages(ctx, "sess-1") + require.NoError(t, err) + require.Len(t, msgs, 2) + note := "already pinned" + _, err = aDB.PinMessage("sess-1", msgs[1].ID, ¬e) + require.NoError(t, err) + + _, err = SyncFolder(ctx, aDB, SyncOptions{ + DataDir: aData, + Target: share, + BaselineMetadata: true, + }) + require.NoError(t, err) + res, err := SyncFolder(ctx, bDB, SyncOptions{ + DataDir: bData, + Target: share, + }) + require.NoError(t, err) + assert.Equal(t, 1, res.ImportedSessions) + assert.Equal(t, 2, res.ImportedMessages) + assert.Equal(t, 3, res.ImportedMetadata) + + gid := "laptop-a1b2c3~sess-1" + got, err := bDB.GetSessionFull(ctx, gid) + require.NoError(t, err) + require.NotNil(t, got) + require.NotNil(t, got.DisplayName) + assert.Equal(t, displayName, *got.DisplayName) + stars, err := bDB.ListStarredSessionIDs(ctx) + require.NoError(t, err) + assert.Equal(t, []string{gid}, stars) + pins, err := bDB.ListPinnedMessages(ctx, gid, "") + require.NoError(t, err) + require.Len(t, pins, 1) + assert.Equal(t, 1, pins[0].Ordinal) + require.NotNil(t, pins[0].Note) + assert.Equal(t, note, *pins[0].Note) + + op, ok, err := bDB.MetadataReplayStateOp(ctx, gid, "display_name") + require.NoError(t, err) + assert.True(t, ok) + assert.Equal(t, MetadataOpRename, op) + op, ok, err = bDB.MetadataReplayStateOp(ctx, gid, "starred") + require.NoError(t, err) + assert.True(t, ok) + assert.Equal(t, MetadataOpStar, op) + op, ok, err = bDB.MetadataReplayStateOp(ctx, gid, "pin:source_uuid:uuid-answer") + require.NoError(t, err) + assert.True(t, ok) + assert.Equal(t, MetadataOpPin, op) +} + +func TestSyncFolderInitBaselineMetadataConvergesTrash(t *testing.T) { + ctx := context.Background() + share := t.TempDir() + aData := t.TempDir() + bData := t.TempDir() + aDB := testDB(t) + bDB := testDB(t) + + require.NoError(t, AdoptOrigin(aDB, "laptop-a1b2c3")) + require.NoError(t, AdoptOrigin(bDB, "desktop-d4e5f6")) + seedSession(t, aDB, "sess-1", "alpha") + + _, err := SyncFolder(ctx, aDB, SyncOptions{ + DataDir: aData, + Target: share, + }) + require.NoError(t, err) + _, err = SyncFolder(ctx, bDB, SyncOptions{ + DataDir: bData, + Target: share, + }) + require.NoError(t, err) + + require.NoError(t, aDB.SoftDeleteSession("sess-1")) + + _, err = SyncFolder(ctx, aDB, SyncOptions{ + DataDir: aData, + Target: share, + BaselineMetadata: true, + }) + require.NoError(t, err) + res, err := SyncFolder(ctx, bDB, SyncOptions{ + DataDir: bData, + Target: share, + }) + require.NoError(t, err) + assert.Equal(t, 1, res.ImportedMetadata) + + gid := "laptop-a1b2c3~sess-1" + got, err := bDB.GetSessionFull(ctx, gid) + require.NoError(t, err) + require.NotNil(t, got) + require.NotNil(t, got.DeletedAt) + op, ok, err := bDB.MetadataReplayStateOp(ctx, gid, "deleted_at") + require.NoError(t, err) + assert.True(t, ok) + assert.Equal(t, MetadataOpSoftDelete, op) +} + +func TestSyncFolderInitBaselinesCurationOfTrashedSession(t *testing.T) { + ctx := context.Background() + share := t.TempDir() + aData := t.TempDir() + bData := t.TempDir() + aDB := testDB(t) + bDB := testDB(t) + + require.NoError(t, AdoptOrigin(aDB, "laptop-a1b2c3")) + require.NoError(t, AdoptOrigin(bDB, "desktop-d4e5f6")) + seedSession(t, aDB, "sess-1", "alpha") + require.NoError(t, aDB.ReplaceSessionMessages("sess-1", []db.Message{{ + SessionID: "sess-1", Ordinal: 0, Role: "user", Content: "hello", + ContentLength: 5, SourceUUID: "uuid-question", + }})) + displayName := "Renamed before trash" + require.NoError(t, aDB.RenameSession("sess-1", &displayName)) + starred, err := aDB.StarSession("sess-1") + require.NoError(t, err) + require.True(t, starred) + msgs, err := aDB.GetAllMessages(ctx, "sess-1") + require.NoError(t, err) + require.Len(t, msgs, 1) + note := "pinned before trash" + _, err = aDB.PinMessage("sess-1", msgs[0].ID, ¬e) + require.NoError(t, err) + require.NoError(t, aDB.SoftDeleteSession("sess-1")) + + // The session sits in trash when the machine first opts in: its curation + // must still baseline, or a later restore reaches peers without it. + _, err = SyncFolder(ctx, aDB, SyncOptions{ + DataDir: aData, + Target: share, + BaselineMetadata: true, + }) + require.NoError(t, err) + _, err = SyncFolder(ctx, bDB, SyncOptions{DataDir: bData, Target: share}) + require.NoError(t, err) + + // Restoring on A publishes the session content; B then converges on a + // visible session that kept its pre-init name, star, and pin. + _, err = aDB.RestoreSession("sess-1") + require.NoError(t, err) + recorder := NewMetadataRecorder(aDB, MetadataRecorderOptions{ + DataDir: aData, + Origin: "laptop-a1b2c3", + }) + _, err = recorder.Append(ctx, MetadataEventInput{ + SessionID: "sess-1", + Op: MetadataOpRestore, + }) + require.NoError(t, err) + _, err = SyncFolder(ctx, aDB, SyncOptions{DataDir: aData, Target: share}) + require.NoError(t, err) + _, err = SyncFolder(ctx, bDB, SyncOptions{DataDir: bData, Target: share}) + require.NoError(t, err) + + gid := "laptop-a1b2c3~sess-1" + got, err := bDB.GetSessionFull(ctx, gid) + require.NoError(t, err) + require.NotNil(t, got) + assert.Nil(t, got.DeletedAt) + require.NotNil(t, got.DisplayName) + assert.Equal(t, displayName, *got.DisplayName) + stars, err := bDB.ListStarredSessionIDs(ctx) + require.NoError(t, err) + assert.Equal(t, []string{gid}, stars) + pins, err := bDB.ListPinnedMessages(ctx, gid, "") + require.NoError(t, err) + require.Len(t, pins, 1) + require.NotNil(t, pins[0].Note) + assert.Equal(t, note, *pins[0].Note) +} + +func TestSyncFolderImportClearsSourceFileState(t *testing.T) { + ctx := context.Background() + share := t.TempDir() + aData := t.TempDir() + bData := t.TempDir() + aDB := testDB(t) + bDB := testDB(t) + sourcePath := filepath.Join(t.TempDir(), "shared-session.jsonl") + peerHash := strings64("a") + localHash := strings64("b") + lastEntryUUID := "entry-99" + + require.NoError(t, aDB.SetSyncState(originStateKey, "laptop-a1b2c3")) + require.NoError(t, bDB.SetSyncState(originStateKey, "desktop-d4e5f6")) + seedSession(t, aDB, "sess-1", "alpha", func(s *db.Session) { + s.FilePath = &sourcePath + s.FileSize = new(int64) + *s.FileSize = 4096 + s.FileMtime = new(int64) + *s.FileMtime = 200 + s.NextOrdinal = 99 + s.LastEntryUUID = &lastEntryUUID + s.FileInode = new(int64) + *s.FileInode = 12345 + s.FileDevice = new(int64) + *s.FileDevice = 67890 + s.FileHash = &peerHash + }) + seedSession(t, bDB, "local-sess", "alpha", func(s *db.Session) { + s.FilePath = &sourcePath + s.FileMtime = new(int64) + *s.FileMtime = 100 + s.FileHash = &localHash + }) + + _, err := SyncFolder(ctx, aDB, SyncOptions{DataDir: aData, Target: share}) + require.NoError(t, err) + res, err := SyncFolder(ctx, bDB, SyncOptions{DataDir: bData, Target: share}) + require.NoError(t, err) + require.Equal(t, 1, res.ImportedSessions) + + imported, err := bDB.GetSessionFull(ctx, "laptop-a1b2c3~sess-1") + require.NoError(t, err) + require.NotNil(t, imported) + assert.Nil(t, imported.FilePath) + assert.Nil(t, imported.FileSize) + assert.Nil(t, imported.FileMtime) + assert.Zero(t, imported.NextOrdinal) + assert.Nil(t, imported.LastEntryUUID) + assert.Nil(t, imported.FileInode) + assert.Nil(t, imported.FileDevice) + assert.Nil(t, imported.FileHash) + + ids, err := bDB.ListSessionIDsByFilePath(sourcePath, "claude") + require.NoError(t, err) + assert.Equal(t, []string{"local-sess"}, ids) + gotHash, ok := bDB.GetFileHashByPath(sourcePath) + require.True(t, ok) + assert.Equal(t, localHash, gotHash) +} + +func TestSyncFolderRoundTripPreservesSessionSignals(t *testing.T) { + ctx := context.Background() + share := t.TempDir() + aData := t.TempDir() + bData := t.TempDir() + aDB := testDB(t) + bDB := testDB(t) + + require.NoError(t, aDB.SetSyncState(originStateKey, "laptop-a1b2c3")) + require.NoError(t, bDB.SetSyncState(originStateKey, "desktop-d4e5f6")) + seedSession(t, aDB, "sess-1", "alpha") + + // Signal columns are written outside UpsertSession, so seed them through + // the same writer paths the live app uses. These include fields the Session + // JSON drops (json:"-"): has_tool_calls, has_context_data, and the quality + // scalars. Secret-scan state is seeded too, but unlike the others it is + // deliberately not carried across import (asserted below). + require.NoError(t, aDB.UpdateSessionSignals("sess-1", db.SessionSignalUpdate{ + HasToolCalls: true, + HasContextData: true, + Outcome: "success", + QualitySignals: db.QualitySignals{ + Version: 3, + ShortPromptCount: 2, + UnstructuredStart: true, + RunawayToolLoopCount: 1, + }, + })) + require.NoError(t, aDB.ReplaceSessionSecretFindings("sess-1", nil, 0, "rules-v7")) + + _, err := SyncFolder(ctx, aDB, SyncOptions{DataDir: aData, Target: share}) + require.NoError(t, err) + + bRes, err := SyncFolder(ctx, bDB, SyncOptions{DataDir: bData, Target: share}) + require.NoError(t, err) + require.Equal(t, 1, bRes.ImportedSessions) + + got, err := bDB.GetSessionFull(ctx, "laptop-a1b2c3~sess-1") + require.NoError(t, err) + require.NotNil(t, got) + assert.True(t, got.HasToolCalls, "has_tool_calls should survive the round trip") + assert.True(t, got.HasContextData, "has_context_data should survive the round trip") + assert.Equal(t, "success", got.Outcome) + // Secret findings are not carried in the manifest, so the imported session + // is treated as unscanned: the source rules version is dropped so + // `secrets scan --backfill` rescans it with local rules. + assert.Empty(t, got.SecretsRulesVersion, + "secret-scan state must not be restored without findings") + + qs := got.StoredQualitySignals() + require.NotNil(t, qs, "quality signals should survive the round trip") + assert.Equal(t, 3, qs.Version) + assert.Equal(t, 2, qs.ShortPromptCount) + assert.True(t, qs.UnstructuredStart) + assert.Equal(t, 1, qs.RunawayToolLoopCount) +} + +func TestSyncFolderRoundTripRewritesForeignRelationshipIDs(t *testing.T) { + ctx := context.Background() + share := t.TempDir() + aData := t.TempDir() + bData := t.TempDir() + aDB := testDB(t) + bDB := testDB(t) + + require.NoError(t, aDB.SetSyncState(originStateKey, "laptop-a1b2c3")) + require.NoError(t, bDB.SetSyncState(originStateKey, "desktop-d4e5f6")) + seedSession(t, aDB, "source-1", "alpha") + seedSession(t, aDB, "parent-1", "alpha") + seedSession(t, aDB, "child-1", "alpha") + parentID := "parent-1" + seedSession(t, aDB, "sess-1", "alpha", func(s *db.Session) { + s.SourceSessionID = "source-1" + s.ParentSessionID = &parentID + }) + require.NoError(t, aDB.ReplaceSessionMessages("sess-1", []db.Message{ + { + SessionID: "sess-1", + Ordinal: 0, + Role: "assistant", + Content: "delegating", + ContentLength: 10, + ToolCalls: []db.ToolCall{{ + ToolName: "Task", + Category: "Task", + ToolUseID: "toolu_1", + SubagentSessionID: "child-1", + ResultEvents: []db.ToolResultEvent{{ + ToolUseID: "toolu_1", + AgentID: "agent-1", + SubagentSessionID: "child-1", + Source: "tool_result", + Status: "success", + Content: "done", + ContentLength: 4, + EventIndex: 0, + }}, + }}, + }, + })) + + _, err := SyncFolder(ctx, aDB, SyncOptions{DataDir: aData, Target: share}) + require.NoError(t, err) + _, err = SyncFolder(ctx, bDB, SyncOptions{DataDir: bData, Target: share}) + require.NoError(t, err) + + got, err := bDB.GetSessionFull(ctx, "laptop-a1b2c3~sess-1") + require.NoError(t, err) + require.NotNil(t, got) + assert.Equal(t, "laptop-a1b2c3~source-1", got.SourceSessionID) + require.NotNil(t, got.ParentSessionID) + assert.Equal(t, "laptop-a1b2c3~parent-1", *got.ParentSessionID) + + msgs, err := bDB.GetAllMessages(ctx, got.ID) + require.NoError(t, err) + require.Len(t, msgs, 1) + require.Len(t, msgs[0].ToolCalls, 1) + assert.Equal(t, "laptop-a1b2c3~child-1", msgs[0].ToolCalls[0].SubagentSessionID) + require.Len(t, msgs[0].ToolCalls[0].ResultEvents, 1) + assert.Equal(t, "laptop-a1b2c3~child-1", + msgs[0].ToolCalls[0].ResultEvents[0].SubagentSessionID) +} + +// Recent Edits and edit file grouping read the persisted +// tool_calls.file_path column, so the segment format must carry it: +// imported foreign sessions get no parse-time re-derivation and the +// one-time file_path backfill has already run on existing databases. +func TestSyncFolderRoundTripPreservesToolCallFilePath(t *testing.T) { + ctx := context.Background() + share := t.TempDir() + aData := t.TempDir() + bData := t.TempDir() + aDB := testDB(t) + bDB := testDB(t) + + require.NoError(t, aDB.SetSyncState(originStateKey, "laptop-a1b2c3")) + require.NoError(t, bDB.SetSyncState(originStateKey, "desktop-d4e5f6")) + seedSession(t, aDB, "sess-1", "alpha") + require.NoError(t, aDB.ReplaceSessionMessages("sess-1", []db.Message{ + { + SessionID: "sess-1", + Ordinal: 0, + Role: "assistant", + Content: "editing", + ContentLength: 7, + ToolCalls: []db.ToolCall{{ + ToolName: "Edit", + Category: "Edit", + ToolUseID: "toolu_1", + InputJSON: `{"file_path":"src/app.go"}`, + FilePath: "src/app.go", + }}, + }, + })) + + _, err := SyncFolder(ctx, aDB, SyncOptions{DataDir: aData, Target: share}) + require.NoError(t, err) + _, err = SyncFolder(ctx, bDB, SyncOptions{DataDir: bData, Target: share}) + require.NoError(t, err) + + msgs, err := bDB.GetAllMessages(ctx, "laptop-a1b2c3~sess-1") + require.NoError(t, err) + require.Len(t, msgs, 1) + require.Len(t, msgs[0].ToolCalls, 1) + assert.Equal(t, "src/app.go", msgs[0].ToolCalls[0].FilePath) +} + +// TestSyncFolderImportLeavesScannedSessionBackfillable verifies that a session +// scanned for secrets at the source (rules version, a finding row, a nonzero +// leak count) is imported as unscanned, because the manifest carries no finding +// rows. The imported session must have no leak count and no rules version, so it +// stays a `secrets scan --backfill` candidate even when the source rules version +// is current on the importing machine. Stamping it scanned-at-source-version +// would skip a secret-bearing session, leaving a leak count with no revealable +// findings. +func TestSyncFolderImportLeavesScannedSessionBackfillable(t *testing.T) { + ctx := context.Background() + share := t.TempDir() + aData := t.TempDir() + bData := t.TempDir() + aDB := testDB(t) + bDB := testDB(t) + + require.NoError(t, aDB.SetSyncState(originStateKey, "laptop-a1b2c3")) + require.NoError(t, bDB.SetSyncState(originStateKey, "desktop-d4e5f6")) + seedSession(t, aDB, "sess-1", "alpha") + + // The source session is fully scanned: a finding row, a nonzero leak count, + // and a rules version that is also current on the importing machine below. + const rulesVersion = "rules-current" + findings := []db.SecretFinding{{ + SessionID: "sess-1", + RuleName: "aws-access-key", + Confidence: "definite", + LocationKind: "message", + MessageOrdinal: 1, + MatchStart: 4, + MatchEnd: 24, + RedactedMatch: "AKIA…MPLE", + RulesVersion: rulesVersion, + }} + require.NoError(t, aDB.ReplaceSessionSecretFindings("sess-1", findings, 1, rulesVersion)) + + _, err := SyncFolder(ctx, aDB, SyncOptions{DataDir: aData, Target: share}) + require.NoError(t, err) + + bRes, err := SyncFolder(ctx, bDB, SyncOptions{DataDir: bData, Target: share}) + require.NoError(t, err) + require.Equal(t, 1, bRes.ImportedSessions) + + const importedID = "laptop-a1b2c3~sess-1" + got, err := bDB.GetSessionFull(ctx, importedID) + require.NoError(t, err) + require.NotNil(t, got) + // Imported session must be unscanned so its state is consistent with the zero + // findings carried in the manifest. + assert.Empty(t, got.SecretsRulesVersion, "imported session must not be stamped scanned") + assert.Zero(t, got.SecretLeakCount, "imported session must not claim leaks without findings") + + // With the source rules version current on the importing machine, backfill + // must still treat the imported session as a candidate (secrets_rules_version + // "" != current) instead of skipping it. + cands, err := bDB.SecretScanCandidates(ctx, db.SecretScanCandidateFilter{ + CurrentVersion: rulesVersion, + OnlyStale: true, + }) + require.NoError(t, err) + assert.Contains(t, cands, importedID, + "secret-bearing imported session must be a backfill candidate, not skipped") +} + +// TestSyncFolderSourceLeakCountChangeKeepsLocalFindings verifies that a +// source-side secret rescan that changes only secret_leak_count (not message +// content) does not alter the artifact manifest hash, so the importer neither +// re-imports the session nor clears the findings it scanned locally. +// secret_leak_count is the only secret field carried in the Session JSON, and +// import discards secret-scan state, so it must not influence the +// content-addressed manifest. +func TestSyncFolderSourceLeakCountChangeKeepsLocalFindings(t *testing.T) { + ctx := context.Background() + share := t.TempDir() + aData := t.TempDir() + bData := t.TempDir() + aDB := testDB(t) + bDB := testDB(t) + + require.NoError(t, aDB.SetSyncState(originStateKey, "laptop-a1b2c3")) + require.NoError(t, bDB.SetSyncState(originStateKey, "desktop-d4e5f6")) + seedSession(t, aDB, "sess-1", "alpha") + + // First round trip: A exports sess-1 (no secrets yet), B imports it. + _, err := SyncFolder(ctx, aDB, SyncOptions{DataDir: aData, Target: share}) + require.NoError(t, err) + bRes, err := SyncFolder(ctx, bDB, SyncOptions{DataDir: bData, Target: share}) + require.NoError(t, err) + require.Equal(t, 1, bRes.ImportedSessions) + + const importedID = "laptop-a1b2c3~sess-1" + + // B scans the imported session locally and records a finding. + bFinding := []db.SecretFinding{{ + SessionID: importedID, RuleName: "aws-access-key", Confidence: "definite", + LocationKind: "message", MessageOrdinal: 0, MatchStart: 4, MatchEnd: 24, + RedactedMatch: "AKIA…MPLE", RulesVersion: "rules-b", + }} + require.NoError(t, bDB.ReplaceSessionSecretFindings(importedID, bFinding, 1, "rules-b")) + + // A rescans sess-1: only secret_leak_count changes (0 -> 1); the message + // content A exports is untouched. + aFinding := []db.SecretFinding{{ + SessionID: "sess-1", RuleName: "aws-access-key", Confidence: "definite", + LocationKind: "message", MessageOrdinal: 0, MatchStart: 4, MatchEnd: 24, + RedactedMatch: "AKIA…MPLE", RulesVersion: "rules-a", + }} + require.NoError(t, aDB.ReplaceSessionSecretFindings("sess-1", aFinding, 1, "rules-a")) + + // Second round trip after the source-only rescan. + _, err = SyncFolder(ctx, aDB, SyncOptions{DataDir: aData, Target: share}) + require.NoError(t, err) + bRes, err = SyncFolder(ctx, bDB, SyncOptions{DataDir: bData, Target: share}) + require.NoError(t, err) + assert.Zero(t, bRes.ImportedSessions, + "a source-only leak-count change must not re-import the session") + + // B's locally scanned findings and scan state survive. + got, err := bDB.SessionSecretFindings(ctx, importedID) + require.NoError(t, err) + assert.Len(t, got, 1, "importer's local secret findings must not be cleared") + + sess, err := bDB.GetSessionFull(ctx, importedID) + require.NoError(t, err) + require.NotNil(t, sess) + assert.Equal(t, 1, sess.SecretLeakCount, "importer's leak count preserved") + assert.Equal(t, "rules-b", sess.SecretsRulesVersion, "importer's scan version preserved") +} + +func TestExportSessionReusesPreNormalizationManifestHashForIgnoredLocalFields(t *testing.T) { + ctx := context.Background() + database := testDB(t) + root := t.TempDir() + origin := "laptop-a1b2c3" + originRoot := filepath.Join(root, origin) + seedSession(t, database, "sess-1", "alpha") + + finding := []db.SecretFinding{{ + SessionID: "sess-1", RuleName: "aws-access-key", Confidence: "definite", + LocationKind: "message", MessageOrdinal: 0, MatchStart: 0, MatchEnd: 5, + RedactedMatch: "hello", RulesVersion: "rules-a", + }} + require.NoError(t, database.ReplaceSessionSecretFindings("sess-1", finding, 1, "rules-a")) + + prevManifestHash := writePreNormalizationManifest(t, ctx, database, originRoot, origin, "sess-1") + require.NoError(t, database.SetSyncState(exportStateKey(origin, "sess-1"), prevManifestHash)) + + gotHash, changed, err := exportSession(ctx, database, originRoot, origin, "sess-1", prevManifestHash) + require.NoError(t, err) + assert.Equal(t, prevManifestHash, gotHash) + assert.False(t, changed, + "an old manifest that only differs by ignored local fields should remain the export watermark") + + manifests := globArtifacts(t, root, origin, "manifests", "*"+manifestExtension) + require.Len(t, manifests, 1) + assert.Equal(t, prevManifestHash+manifestExtension, filepath.Base(manifests[0])) +} + +func TestExportSessionRejectsCorruptPreNormalizationFastPath(t *testing.T) { + ctx := context.Background() + database := testDB(t) + root := t.TempDir() + origin := "laptop-a1b2c3" + originRoot := filepath.Join(root, origin) + seedSession(t, database, "sess-1", "alpha") + require.NoError(t, database.ReplaceSessionSecretFindings("sess-1", []db.SecretFinding{{ + SessionID: "sess-1", RuleName: "test", Confidence: "definite", + LocationKind: "message", MessageOrdinal: 0, + MatchStart: 0, MatchEnd: 5, RedactedMatch: "hello", RulesVersion: "rules-a", + }}, 1, "rules-a")) + + prevHash := writePreNormalizationManifest( + t, ctx, database, originRoot, origin, "sess-1", + ) + prev, err := readManifest(originRoot, prevHash) + require.NoError(t, err) + require.Len(t, prev.Segments, 1) + segmentPath := filepath.Join( + originRoot, KindSegments, prev.Segments[0]+segmentExtension, + ) + require.NoError(t, os.WriteFile(segmentPath, []byte("corrupt"), 0o644)) + + gotHash, changed, err := exportSession( + ctx, database, originRoot, origin, "sess-1", prevHash, + ) + require.NoError(t, err) + assert.NotEqual(t, prevHash, gotHash) + assert.True(t, changed) + assert.FileExists(t, segmentPath+quarantineSuffix) + manifest, err := readManifest(originRoot, gotHash) + require.NoError(t, err) + _, err = readManifestMessages(originRoot, manifest) + require.NoError(t, err) +} + +func TestExportSessionQuarantinesSemanticInvalidPreNormalizationManifest(t *testing.T) { + ctx := context.Background() + database := testDB(t) + root := t.TempDir() + origin := "laptop-a1b2c3" + originRoot := filepath.Join(root, origin) + seedSession(t, database, "sess-1", "alpha") + require.NoError(t, database.ReplaceSessionSecretFindings("sess-1", []db.SecretFinding{{ + SessionID: "sess-1", RuleName: "test", Confidence: "definite", + LocationKind: "message", MessageOrdinal: 0, + MatchStart: 0, MatchEnd: 5, RedactedMatch: "hello", RulesVersion: "rules-a", + }}, 1, "rules-a")) + + previousHash := writePreNormalizationManifest( + t, ctx, database, originRoot, origin, "sess-1", + ) + previous, err := readManifest(originRoot, previousHash) + require.NoError(t, err) + previous.Session.Machine = "wrong-origin" + invalidData, err := canonicalJSON(previous) + require.NoError(t, err) + invalidHash := hashHex(invalidData) + invalidPath := filepath.Join( + originRoot, KindManifests, invalidHash+manifestExtension, + ) + require.NoError(t, writeCompressed(invalidPath, invalidData)) + + decoded, err := readManifest(originRoot, invalidHash) + require.NoError(t, err, "precondition: manifest must decode and match its content hash") + assert.Equal(t, "wrong-origin", decoded.Session.Machine) + + gotHash, changed, err := exportSession( + ctx, database, originRoot, origin, "sess-1", invalidHash, + ) + require.NoError(t, err) + assert.NotEqual(t, invalidHash, gotHash) + assert.True(t, changed) + assert.NoFileExists(t, invalidPath) + assert.FileExists(t, invalidPath+quarantineSuffix) + + regenerated, err := readManifest(originRoot, gotHash) + require.NoError(t, err) + require.NoError(t, validateManifest( + regenerated, origin, origin+"~sess-1", + )) + _, err = readManifestMessages(originRoot, regenerated) + require.NoError(t, err) +} + +func TestExportSessionHealsInvalidManifestAndComputedSegmentTogether(t *testing.T) { + ctx := context.Background() + database := testDB(t) + root := t.TempDir() + origin := "laptop-a1b2c3" + originRoot := filepath.Join(root, origin) + seedSession(t, database, "sess-1", "alpha") + require.NoError(t, database.ReplaceSessionSecretFindings("sess-1", []db.SecretFinding{{ + SessionID: "sess-1", RuleName: "test", Confidence: "definite", + LocationKind: "message", MessageOrdinal: 0, + MatchStart: 0, MatchEnd: 5, RedactedMatch: "hello", RulesVersion: "rules-a", + }}, 1, "rules-a")) + + previousHash := writePreNormalizationManifest( + t, ctx, database, originRoot, origin, "sess-1", + ) + previous, err := readManifest(originRoot, previousHash) + require.NoError(t, err) + require.Len(t, previous.Segments, 1) + computedSegmentPath := filepath.Join( + originRoot, KindSegments, previous.Segments[0]+segmentExtension, + ) + + decoyData, err := encodeSegment([]db.Message{{ + Ordinal: 0, Role: "user", Content: "decoy", ContentLength: 5, + }}) + require.NoError(t, err) + decoyHash := hashHex(decoyData) + decoyPath := filepath.Join( + originRoot, KindSegments, decoyHash+segmentExtension, + ) + require.NoError(t, writeCompressed(decoyPath, decoyData)) + previous.Session.Machine = "wrong-origin" + previous.Segments = []string{decoyHash} + invalidData, err := canonicalJSON(previous) + require.NoError(t, err) + invalidHash := hashHex(invalidData) + invalidPath := filepath.Join( + originRoot, KindManifests, invalidHash+manifestExtension, + ) + require.NoError(t, writeCompressed(invalidPath, invalidData)) + + require.NoError(t, os.Remove(computedSegmentPath)) + require.NoError(t, os.WriteFile(computedSegmentPath, []byte("corrupt"), 0o644)) + + gotHash, changed, err := exportSession( + ctx, database, originRoot, origin, "sess-1", invalidHash, + ) + require.NoError(t, err) + assert.True(t, changed) + assert.NoFileExists(t, invalidPath) + assert.FileExists(t, invalidPath+quarantineSuffix) + assert.FileExists(t, computedSegmentPath) + assert.FileExists(t, computedSegmentPath+quarantineSuffix) + assert.FileExists(t, decoyPath, + "an invalid manifest must not make export follow its untrusted segment hash") + assert.NoFileExists(t, decoyPath+quarantineSuffix) + + regenerated, err := readManifest(originRoot, gotHash) + require.NoError(t, err) + _, err = readManifestMessages(originRoot, regenerated) + require.NoError(t, err) +} + +func TestReadValidExportArtifactsDefersFutureManifestWithoutQuarantine(t *testing.T) { + originRoot := t.TempDir() + origin := "laptop-a1b2c3" + segmentData, err := encodeSegment([]db.Message{{ + Ordinal: 0, Role: "user", Content: "hello", ContentLength: 5, + }}) + require.NoError(t, err) + segmentHash := hashHex(segmentData) + require.NoError(t, writeCompressed( + filepath.Join(originRoot, KindSegments, segmentHash+segmentExtension), + segmentData, + )) + m := manifest{ + Version: formatVersion + 1, + Origin: origin, + NativeSessionID: "sess-1", + Session: manifestSession{ + ID: "sess-1", + Machine: origin, + }, + Segments: []string{segmentHash}, + } + manifestData, err := canonicalJSON(m) + require.NoError(t, err) + manifestHash := hashHex(manifestData) + manifestPath := filepath.Join( + originRoot, KindManifests, manifestHash+manifestExtension, + ) + require.NoError(t, writeCompressed(manifestPath, manifestData)) + + _, valid, err := readValidExportArtifacts( + originRoot, origin, "sess-1", manifestHash, + ) + require.NoError(t, err) + assert.False(t, valid) + assert.FileExists(t, manifestPath) + assert.NoFileExists(t, manifestPath+quarantineSuffix) +} + +func TestReadValidExportArtifactsKeepsManifestForUnavailableSegment(t *testing.T) { + tests := []struct { + name string + writeSegment func(t *testing.T, originRoot string) (string, string) + segmentPresent bool + segmentQuarantined bool + }{ + { + name: "missing", + writeSegment: func(t *testing.T, originRoot string) (string, string) { + t.Helper() + hash := strings.Repeat("a", 64) + return hash, filepath.Join( + originRoot, KindSegments, hash+segmentExtension, + ) + }, + }, + { + name: "corrupt", + writeSegment: func(t *testing.T, originRoot string) (string, string) { + t.Helper() + hash := strings.Repeat("b", 64) + path := filepath.Join( + originRoot, KindSegments, hash+segmentExtension, + ) + require.NoError(t, writeCompressed(path, []byte("tampered"))) + return hash, path + }, + segmentQuarantined: true, + }, + { + name: "future version", + writeSegment: func(t *testing.T, originRoot string) (string, string) { + t.Helper() + data, err := canonicalJSON(segmentMessage{ + Version: formatVersion + 1, + Ordinal: 0, + Role: "user", + Content: "hello", + ContentLength: 5, + }) + require.NoError(t, err) + hash := hashHex(data) + path := filepath.Join( + originRoot, KindSegments, hash+segmentExtension, + ) + require.NoError(t, writeCompressed(path, data)) + return hash, path + }, + segmentPresent: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + originRoot := t.TempDir() + origin := "laptop-a1b2c3" + segmentHash, segmentPath := tt.writeSegment(t, originRoot) + m := manifest{ + Version: formatVersion, + Origin: origin, + NativeSessionID: "sess-1", + Session: manifestSession{ + ID: "sess-1", + Machine: origin, + }, + Segments: []string{segmentHash}, + } + manifestData, err := canonicalJSON(m) + require.NoError(t, err) + manifestHash := hashHex(manifestData) + manifestPath := filepath.Join( + originRoot, KindManifests, manifestHash+manifestExtension, + ) + require.NoError(t, writeCompressed(manifestPath, manifestData)) + + _, valid, err := readValidExportArtifacts( + originRoot, origin, "sess-1", manifestHash, + ) + require.NoError(t, err) + assert.False(t, valid) + assert.FileExists(t, manifestPath) + assert.NoFileExists(t, manifestPath+quarantineSuffix) + if tt.segmentPresent { + assert.FileExists(t, segmentPath) + } else { + assert.NoFileExists(t, segmentPath) + } + if tt.segmentQuarantined { + assert.FileExists(t, segmentPath+quarantineSuffix) + } else { + assert.NoFileExists(t, segmentPath+quarantineSuffix) + } + }) + } +} + +func TestSyncFolderNotifiesWhenImportWritesData(t *testing.T) { + ctx := context.Background() + share := t.TempDir() + aData := t.TempDir() + bData := t.TempDir() + aDB := testDB(t) + bDB := testDB(t) + + require.NoError(t, aDB.SetSyncState(originStateKey, "laptop-a1b2c3")) + require.NoError(t, bDB.SetSyncState(originStateKey, "desktop-d4e5f6")) + seedSession(t, aDB, "sess-1", "alpha") + + _, err := SyncFolder(ctx, aDB, SyncOptions{DataDir: aData, Target: share}) + require.NoError(t, err) + + changes := 0 + _, err = SyncFolder(ctx, bDB, SyncOptions{ + DataDir: bData, + Target: share, + OnDataChanged: func() { changes++ }, + }) + require.NoError(t, err) + assert.Equal(t, 1, changes) + + _, err = SyncFolder(ctx, bDB, SyncOptions{ + DataDir: bData, + Target: share, + OnDataChanged: func() { changes++ }, + }) + require.NoError(t, err) + assert.Equal(t, 1, changes) +} + +func TestSyncFolderUsesProvidedOrigin(t *testing.T) { + ctx := context.Background() + share := t.TempDir() + dataDir := t.TempDir() + database := testDB(t) + seedSession(t, database, "sess-1", "alpha") + + res, err := SyncFolder(ctx, database, SyncOptions{ + DataDir: dataDir, + Target: share, + Origin: "configured-a1b2c3", + }) + require.NoError(t, err) + + assert.Equal(t, "configured-a1b2c3", res.Origin) + persisted, err := database.GetSyncState(originStateKey) + require.NoError(t, err) + assert.Empty(t, persisted) + manifests := globArtifacts(t, share, "configured-a1b2c3", "manifests", "*"+manifestExtension) + assert.Len(t, manifests, 1) +} + +func TestImportMaintainsFTS(t *testing.T) { + ctx := context.Background() + root := t.TempDir() + origin := "laptop-a1b2c3" + exportDB := testDB(t) + importDB := testDB(t) + if !importDB.HasFTS() { + t.Skip("FTS unavailable") + } + seedSession(t, exportDB, "sess-1", "alpha") + + _, err := Export(ctx, exportDB, root, origin) + require.NoError(t, err) + imported, messages, err := Import(ctx, importDB, root, "desktop-d4e5f6") + require.NoError(t, err) + require.Equal(t, 1, imported) + require.Equal(t, 2, messages) + + page, err := importDB.Search(ctx, db.SearchFilter{Query: "world", Limit: 10}) + require.NoError(t, err) + require.Len(t, page.Results, 1) + assert.Equal(t, origin+"~sess-1", page.Results[0].SessionID) +} + +func TestImportPreservesPinsAndStatsOnRewrite(t *testing.T) { + ctx := context.Background() + root := t.TempDir() + origin := "laptop-a1b2c3" + exportDB := testDB(t) + importDB := testDB(t) + seedSession(t, exportDB, "sess-1", "alpha") + + _, err := Export(ctx, exportDB, root, origin) + require.NoError(t, err) + imported, messages, err := Import(ctx, importDB, root, "desktop-d4e5f6") + require.NoError(t, err) + require.Equal(t, 1, imported) + require.Equal(t, 2, messages) + + gid := origin + "~sess-1" + importedMsgs, err := importDB.GetAllMessages(ctx, gid) + require.NoError(t, err) + require.Len(t, importedMsgs, 2) + note := "keep this pin" + _, err = importDB.PinMessage(gid, importedMsgs[1].ID, ¬e) + require.NoError(t, err) + + require.NoError(t, exportDB.ReplaceSessionMessages("sess-1", []db.Message{ + {SessionID: "sess-1", Ordinal: 0, Role: "user", Content: "hello", ContentLength: 5}, + {SessionID: "sess-1", Ordinal: 1, Role: "assistant", Content: "planet", ContentLength: 6}, + })) + _, err = Export(ctx, exportDB, root, origin) + require.NoError(t, err) + imported, messages, err = Import(ctx, importDB, root, "desktop-d4e5f6") + require.NoError(t, err) + require.Equal(t, 1, imported) + require.Equal(t, 2, messages) + + pins, err := importDB.ListPinnedMessages(ctx, gid, "") + require.NoError(t, err) + require.Len(t, pins, 1) + assert.Equal(t, 1, pins[0].Ordinal) + require.NotNil(t, pins[0].Note) + assert.Equal(t, note, *pins[0].Note) + + allPins, err := importDB.ListPinnedMessages(ctx, "", "") + require.NoError(t, err) + require.Len(t, allPins, 1) + assert.Equal(t, gid, allPins[0].SessionID) + require.NotNil(t, allPins[0].Content) + assert.Equal(t, "planet", *allPins[0].Content) + + stats, err := importDB.GetStats(ctx, false, false) + require.NoError(t, err) + assert.Equal(t, 1, stats.SessionCount) + assert.Equal(t, 2, stats.MessageCount) + assert.Equal(t, 1, stats.ProjectCount) + assert.Equal(t, 1, stats.MachineCount) +} + +func TestImportDoesNotAdvanceStateForExcludedOrTrashedSessions(t *testing.T) { + ctx := context.Background() + root := t.TempDir() + origin := "laptop-a1b2c3" + exportDB := testDB(t) + seedSession(t, exportDB, "sess-1", "alpha") + _, err := Export(ctx, exportDB, root, origin) + require.NoError(t, err) + + gid := origin + "~sess-1" + tests := []struct { + name string + seed func(*testing.T, *db.DB) + }{ + { + name: "excluded", + seed: func(t *testing.T, database *db.DB) { + t.Helper() + seedSession(t, database, gid, "alpha", func(s *db.Session) { + s.Machine = origin + }) + require.NoError(t, database.DeleteSession(gid)) + }, + }, + { + name: "trashed", + seed: func(t *testing.T, database *db.DB) { + t.Helper() + seedSession(t, database, gid, "alpha", func(s *db.Session) { + s.Machine = origin + }) + require.NoError(t, database.SoftDeleteSession(gid)) + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + importDB := testDB(t) + tt.seed(t, importDB) + + imported, messages, err := Import(ctx, importDB, root, "desktop-d4e5f6") + require.NoError(t, err) + assert.Zero(t, imported) + assert.Zero(t, messages) + + state, err := importDB.GetSyncState(importStateKey(origin, gid)) + require.NoError(t, err) + assert.Empty(t, state) + }) + } +} + +func TestSyncFolderRetriesIncompleteForeignArtifacts(t *testing.T) { + ctx := context.Background() + share := t.TempDir() + aData := t.TempDir() + bData := t.TempDir() + aDB := testDB(t) + bDB := testDB(t) + + require.NoError(t, aDB.SetSyncState(originStateKey, "laptop-a1b2c3")) + require.NoError(t, bDB.SetSyncState(originStateKey, "desktop-d4e5f6")) + seedSession(t, aDB, "sess-1", "alpha") + + _, err := SyncFolder(ctx, aDB, SyncOptions{DataDir: aData, Target: share}) + require.NoError(t, err) + segments := globArtifacts(t, share, "laptop-a1b2c3", "segments", "*"+segmentExtension) + require.Len(t, segments, 1) + require.NoError(t, os.Remove(segments[0])) + + res, err := SyncFolder(ctx, bDB, SyncOptions{DataDir: bData, Target: share}) + require.NoError(t, err) + assert.Zero(t, res.ImportedSessions) + assert.Zero(t, res.ImportedMessages) + + got, err := bDB.GetSessionFull(ctx, "laptop-a1b2c3~sess-1") + require.NoError(t, err) + assert.Nil(t, got) + + require.NoError(t, CopyUnion(filepath.Join(aData, "artifacts"), share)) + res, err = SyncFolder(ctx, bDB, SyncOptions{DataDir: bData, Target: share}) + require.NoError(t, err) + assert.Equal(t, 1, res.ImportedSessions) + assert.Equal(t, 2, res.ImportedMessages) +} + +func TestSyncFolderSkipsCheckpointWithMissingManifest(t *testing.T) { + ctx := context.Background() + share := t.TempDir() + aData := t.TempDir() + bData := t.TempDir() + aDB := testDB(t) + bDB := testDB(t) + + require.NoError(t, aDB.SetSyncState(originStateKey, "laptop-a1b2c3")) + require.NoError(t, bDB.SetSyncState(originStateKey, "desktop-d4e5f6")) + seedSession(t, aDB, "sess-1", "alpha") + + _, err := SyncFolder(ctx, aDB, SyncOptions{DataDir: aData, Target: share}) + require.NoError(t, err) + manifests := globArtifacts(t, share, "laptop-a1b2c3", "manifests", "*"+manifestExtension) + require.Len(t, manifests, 1) + require.NoError(t, os.Remove(manifests[0])) + + res, err := SyncFolder(ctx, bDB, SyncOptions{DataDir: bData, Target: share}) + require.NoError(t, err) + assert.Zero(t, res.ImportedSessions) + assert.Zero(t, res.ImportedMessages) +} + +func TestImportQuarantinesMismatchedCheckpointOrigin(t *testing.T) { + ctx := context.Background() + root := t.TempDir() + origin := "laptop-a1b2c3" + database := testDB(t) + seedSession(t, database, "sess-1", "alpha") + + _, err := Export(ctx, database, root, origin) + require.NoError(t, err) + + originRoot := filepath.Join(root, origin) + cp, err := readLatestCheckpoint(originRoot) + require.NoError(t, err) + require.NotNil(t, cp) + cp.Origin = "spoofed-origin" + writeCheckpoint(t, originRoot, *cp) + checkpointPath := filepath.Join( + originRoot, KindCheckpoints, "cp-0000000001.json", + ) + + importDB := testDB(t) + imported, messages, err := Import(ctx, importDB, root, "desktop-d4e5f6") + require.NoError(t, err) + assert.Zero(t, imported) + assert.Zero(t, messages) + assert.NoFileExists(t, checkpointPath) + assert.FileExists(t, checkpointPath+quarantineSuffix) + + got, err := importDB.GetSessionFull(ctx, origin+"~sess-1") + require.NoError(t, err) + assert.Nil(t, got) +} + +func TestImportRejectsMismatchedManifestOrigin(t *testing.T) { + ctx := context.Background() + root := t.TempDir() + origin := "laptop-a1b2c3" + database := testDB(t) + seedSession(t, database, "sess-1", "alpha") + + _, err := Export(ctx, database, root, origin) + require.NoError(t, err) + + originRoot := filepath.Join(root, origin) + cp, err := readLatestCheckpoint(originRoot) + require.NoError(t, err) + require.NotNil(t, cp) + gid := origin + "~sess-1" + manifestHash := cp.Sessions[gid] + require.NotEmpty(t, manifestHash) + + m, err := readManifest(originRoot, manifestHash) + require.NoError(t, err) + m.Origin = "spoofed-origin" + manifestData, err := canonicalJSON(m) + require.NoError(t, err) + spoofedHash := hashHex(manifestData) + require.NoError(t, writeCompressed(filepath.Join(originRoot, "manifests", spoofedHash+manifestExtension), manifestData)) + cp.Sessions[gid] = spoofedHash + writeCheckpoint(t, originRoot, *cp) + + importDB := testDB(t) + imported, messages, err := Import(ctx, importDB, root, "desktop-d4e5f6") + require.Error(t, err) + assert.Contains(t, err.Error(), "manifest origin mismatch") + assert.Zero(t, imported) + assert.Zero(t, messages) + + got, err := importDB.GetSessionFull(ctx, gid) + require.NoError(t, err) + assert.Nil(t, got) +} + +func TestImportSkipsAndQuarantinesCorruptManifest(t *testing.T) { + ctx := context.Background() + root := t.TempDir() + origin := "laptop-a1b2c3" + database := testDB(t) + seedSession(t, database, "sess-1", "alpha") + seedSession(t, database, "sess-2", "beta") + + _, err := Export(ctx, database, root, origin) + require.NoError(t, err) + + originRoot := filepath.Join(root, origin) + cp, err := readLatestCheckpoint(originRoot) + require.NoError(t, err) + require.NotNil(t, cp) + manifestHash := cp.Sessions[origin+"~sess-1"] + require.NotEmpty(t, manifestHash) + m, err := readManifest(originRoot, manifestHash) + require.NoError(t, err) + m.Session.Project = "tampered" + data, err := canonicalJSON(m) + require.NoError(t, err) + path := filepath.Join(originRoot, "manifests", manifestHash+manifestExtension) + require.NoError(t, os.Remove(path)) + require.NoError(t, writeCompressed(path, data)) + + importDB := testDB(t) + imported, messages, err := Import(ctx, importDB, root, "desktop-d4e5f6") + require.NoError(t, err) + assert.Equal(t, 1, imported) + assert.Equal(t, 2, messages) + + got, err := importDB.GetSessionFull(ctx, origin+"~sess-1") + require.NoError(t, err) + assert.Nil(t, got) + got, err = importDB.GetSessionFull(ctx, origin+"~sess-2") + require.NoError(t, err) + assert.NotNil(t, got) + + assert.NoFileExists(t, path) + assert.FileExists(t, path+quarantineSuffix) + + // A later sync neither errors nor resurrects the corrupt manifest. + imported, messages, err = Import(ctx, importDB, root, "desktop-d4e5f6") + require.NoError(t, err) + assert.Zero(t, imported) + assert.Zero(t, messages) +} + +func TestImportQuarantinesInvalidCheckpointManifestReference(t *testing.T) { + ctx := context.Background() + root := t.TempDir() + origin := "laptop-a1b2c3" + importDB := testDB(t) + originRoot := filepath.Join(root, origin) + writeCheckpoint(t, originRoot, checkpoint{ + Version: formatVersion, + Origin: origin, + Sequence: 1, + Sessions: map[string]string{ + origin + "~sess-1": "../outside", + }, + }) + checkpointPath := filepath.Join( + originRoot, KindCheckpoints, "cp-0000000001.json", + ) + + imported, messages, err := Import(ctx, importDB, root, "desktop-d4e5f6") + require.NoError(t, err) + assert.Zero(t, imported) + assert.Zero(t, messages) + assert.NoFileExists(t, checkpointPath) + assert.FileExists(t, checkpointPath+quarantineSuffix) +} + +func TestImportSkipsAndQuarantinesCorruptSegment(t *testing.T) { + ctx := context.Background() + root := t.TempDir() + origin := "laptop-a1b2c3" + database := testDB(t) + seedSession(t, database, "sess-1", "alpha") + seedSession(t, database, "sess-2", "beta") + // Distinct messages so the two sessions do not share one + // content-addressed segment. + require.NoError(t, database.ReplaceSessionMessages("sess-2", []db.Message{ + {SessionID: "sess-2", Ordinal: 0, Role: "user", Content: "howdy", ContentLength: 5}, + {SessionID: "sess-2", Ordinal: 1, Role: "assistant", Content: "hi there", ContentLength: 8}, + })) + + _, err := Export(ctx, database, root, origin) + require.NoError(t, err) + + originRoot := filepath.Join(root, origin) + cp, err := readLatestCheckpoint(originRoot) + require.NoError(t, err) + require.NotNil(t, cp) + manifestHash := cp.Sessions[origin+"~sess-1"] + require.NotEmpty(t, manifestHash) + m, err := readManifest(originRoot, manifestHash) + require.NoError(t, err) + require.Len(t, m.Segments, 1) + segmentPath := filepath.Join(originRoot, "segments", m.Segments[0]+segmentExtension) + require.NoError(t, os.Remove(segmentPath)) + require.NoError(t, writeCompressed(segmentPath, []byte("{\"v\":1,\"ordinal\":0,\"role\":\"user\",\"content\":\"tampered\"}\n"))) + + importDB := testDB(t) + imported, messages, err := Import(ctx, importDB, root, "desktop-d4e5f6") + require.NoError(t, err) + assert.Equal(t, 1, imported) + assert.Equal(t, 2, messages) + + got, err := importDB.GetSessionFull(ctx, origin+"~sess-1") + require.NoError(t, err) + assert.Nil(t, got) + got, err = importDB.GetSessionFull(ctx, origin+"~sess-2") + require.NoError(t, err) + assert.NotNil(t, got) + + assert.NoFileExists(t, segmentPath) + assert.FileExists(t, segmentPath+quarantineSuffix) +} + +func TestImportFallsBackPastCorruptLatestCheckpoint(t *testing.T) { + ctx := context.Background() + root := t.TempDir() + origin := "laptop-a1b2c3" + database := testDB(t) + seedSession(t, database, "sess-1", "alpha") + + _, err := Export(ctx, database, root, origin) + require.NoError(t, err) + + originRoot := filepath.Join(root, origin) + corruptPath := filepath.Join(originRoot, "checkpoints", "cp-0000000002.json") + require.NoError(t, os.WriteFile(corruptPath, []byte("not json"), 0o644)) + + importDB := testDB(t) + imported, messages, err := Import(ctx, importDB, root, "desktop-d4e5f6") + require.NoError(t, err) + assert.Equal(t, 1, imported) + assert.Equal(t, 2, messages) + + got, err := importDB.GetSessionFull(ctx, origin+"~sess-1") + require.NoError(t, err) + assert.NotNil(t, got) + + assert.NoFileExists(t, corruptPath) + assert.FileExists(t, corruptPath+quarantineSuffix) +} + +func TestImportSkipsAndQuarantinesUndecodableManifest(t *testing.T) { + ctx := context.Background() + root := t.TempDir() + origin := "laptop-a1b2c3" + database := testDB(t) + seedSession(t, database, "sess-1", "alpha") + seedSession(t, database, "sess-2", "beta") + + _, err := Export(ctx, database, root, origin) + require.NoError(t, err) + + // A manifest whose bytes hash to the checkpoint reference but do not + // decode as JSON: hash verification passes, only the decode fails. + originRoot := filepath.Join(root, origin) + junk := []byte("not json") + junkHash := hashHex(junk) + path := filepath.Join(originRoot, "manifests", junkHash+manifestExtension) + require.NoError(t, writeCompressed(path, junk)) + cp, err := readLatestCheckpoint(originRoot) + require.NoError(t, err) + require.NotNil(t, cp) + cp.Sessions[origin+"~sess-1"] = junkHash + writeCheckpoint(t, originRoot, *cp) + + importDB := testDB(t) + imported, messages, err := Import(ctx, importDB, root, "desktop-d4e5f6") + require.NoError(t, err) + assert.Equal(t, 1, imported) + assert.Equal(t, 2, messages) + + got, err := importDB.GetSessionFull(ctx, origin+"~sess-1") + require.NoError(t, err) + assert.Nil(t, got) + got, err = importDB.GetSessionFull(ctx, origin+"~sess-2") + require.NoError(t, err) + assert.NotNil(t, got) + + assert.NoFileExists(t, path) + assert.FileExists(t, path+quarantineSuffix) +} + +func TestImportSkipsAndQuarantinesUndecodableSegment(t *testing.T) { + ctx := context.Background() + root := t.TempDir() + origin := "laptop-a1b2c3" + database := testDB(t) + seedSession(t, database, "sess-1", "alpha") + seedSession(t, database, "sess-2", "beta") + + _, err := Export(ctx, database, root, origin) + require.NoError(t, err) + + // A segment whose bytes hash to the manifest reference but do not decode + // as NDJSON: hash verification passes, only the decode fails. + originRoot := filepath.Join(root, origin) + junk := []byte("not ndjson\n") + junkHash := hashHex(junk) + segPath := filepath.Join(originRoot, "segments", junkHash+segmentExtension) + require.NoError(t, writeCompressed(segPath, junk)) + + cp, err := readLatestCheckpoint(originRoot) + require.NoError(t, err) + require.NotNil(t, cp) + m, err := readManifest(originRoot, cp.Sessions[origin+"~sess-1"]) + require.NoError(t, err) + m.Segments = []string{junkHash} + data, err := canonicalJSON(m) + require.NoError(t, err) + manifestHash := hashHex(data) + require.NoError(t, writeCompressed( + filepath.Join(originRoot, "manifests", manifestHash+manifestExtension), data)) + cp.Sessions[origin+"~sess-1"] = manifestHash + writeCheckpoint(t, originRoot, *cp) + + importDB := testDB(t) + imported, messages, err := Import(ctx, importDB, root, "desktop-d4e5f6") + require.NoError(t, err) + assert.Equal(t, 1, imported) + assert.Equal(t, 2, messages) + + got, err := importDB.GetSessionFull(ctx, origin+"~sess-1") + require.NoError(t, err) + assert.Nil(t, got) + got, err = importDB.GetSessionFull(ctx, origin+"~sess-2") + require.NoError(t, err) + assert.NotNil(t, got) + + assert.NoFileExists(t, segPath) + assert.FileExists(t, segPath+quarantineSuffix) +} + +func TestExportAdvancesPastCorruptLatestCheckpoint(t *testing.T) { + ctx := context.Background() + root := t.TempDir() + origin := "laptop-a1b2c3" + database := testDB(t) + seedSession(t, database, "sess-1", "alpha") + + _, err := Export(ctx, database, root, origin) + require.NoError(t, err) + + originRoot := filepath.Join(root, origin) + corruptPath := filepath.Join(originRoot, "checkpoints", "cp-0000000001.json") + require.NoError(t, os.WriteFile(corruptPath, []byte("not json"), 0o644)) + + seedSession(t, database, "sess-2", "beta") + _, err = Export(ctx, database, root, origin) + require.NoError(t, err, "corrupt latest checkpoint must not block export") + + // The corrupt body is quarantined so read paths fall back and a valid + // peer copy can heal it, and its sequence number is not reused: the name + // may already have been published to peers with the original content. + assert.NoFileExists(t, corruptPath) + assert.FileExists(t, corruptPath+quarantineSuffix) + cp, err := readLatestCheckpoint(originRoot) + require.NoError(t, err) + require.NotNil(t, cp) + assert.Equal(t, 2, cp.Sequence) + assert.Contains(t, cp.Sessions, origin+"~sess-1") + assert.Contains(t, cp.Sessions, origin+"~sess-2") +} + +func TestImportRejectsInvalidManifestSegmentReference(t *testing.T) { + ctx := context.Background() + root := t.TempDir() + origin := "laptop-a1b2c3" + importDB := testDB(t) + originRoot := filepath.Join(root, origin) + m := manifest{ + Version: formatVersion, + Origin: origin, + NativeSessionID: "sess-1", + Session: manifestSession{ + ID: "sess-1", + Machine: origin, + Agent: "claude", + Project: "alpha", + CreatedAt: "2026-06-14T01:02:03Z", + }, + Segments: []string{"../outside"}, + } + data, err := canonicalJSON(m) + require.NoError(t, err) + manifestHash := hashHex(data) + require.NoError(t, writeCompressed(filepath.Join(originRoot, "manifests", manifestHash+manifestExtension), data)) + writeCheckpoint(t, originRoot, checkpoint{ + Version: formatVersion, + Origin: origin, + Sequence: 1, + Sessions: map[string]string{ + origin + "~sess-1": manifestHash, + }, + }) + + imported, messages, err := Import(ctx, importDB, root, "desktop-d4e5f6") + require.Error(t, err) + assert.ErrorIs(t, err, ErrArtifactInvalid) + assert.Contains(t, err.Error(), "invalid artifact hash") + assert.Zero(t, imported) + assert.Zero(t, messages) +} + +func TestSyncFolderRejectsOverlappingRoots(t *testing.T) { + ctx := context.Background() + + tests := []struct { + name string + target func(string) string + }{ + { + name: "target is data dir", + target: func(dataDir string) string { + return dataDir + }, + }, + { + name: "target is artifact store", + target: func(dataDir string) string { + return filepath.Join(dataDir, "artifacts") + }, + }, + { + name: "target inside artifact store", + target: func(dataDir string) string { + return filepath.Join(dataDir, "artifacts", "share") + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + database := testDB(t) + dataDir := t.TempDir() + + _, err := SyncFolder(ctx, database, SyncOptions{ + DataDir: dataDir, + Target: tt.target(dataDir), + }) + require.Error(t, err) + assert.Contains(t, err.Error(), "must not overlap") + }) + } +} + +func TestSyncFolderRejectsSymlinkedAncestorResolvingInsideArtifactStore(t *testing.T) { + dataDir := t.TempDir() + localRoot := filepath.Join(dataDir, "artifacts") + targetDir := filepath.Join(localRoot, "shared") + require.NoError(t, os.MkdirAll(targetDir, 0o755)) + alias := filepath.Join(t.TempDir(), "alias") + require.NoError(t, os.Symlink(localRoot, alias)) + + err := validateDisjointRoots(localRoot, filepath.Join(alias, "shared")) + require.Error(t, err) + assert.Contains(t, err.Error(), "must not overlap") +} + +func TestExportSkipsUnchangedCheckpoint(t *testing.T) { + ctx := context.Background() + database := testDB(t) + root := t.TempDir() + origin := "laptop-a1b2c3" + seedSession(t, database, "sess-1", "alpha") + + count, err := Export(ctx, database, root, origin) + require.NoError(t, err) + assert.Equal(t, 1, count) + count, err = Export(ctx, database, root, origin) + require.NoError(t, err) + assert.Zero(t, count) + + checkpoints, err := filepath.Glob(filepath.Join(root, origin, "checkpoints", "cp-*.json")) + require.NoError(t, err) + assert.Equal(t, []string{ + filepath.Join(root, origin, "checkpoints", "cp-0000000001.json"), + }, checkpoints) + + manifests := globArtifacts(t, root, origin, "manifests", "*"+manifestExtension) + assert.Len(t, manifests, 1) +} + +func TestExportUnchangedWhenOnlySourceFileBookkeepingChanges(t *testing.T) { + ctx := context.Background() + database := testDB(t) + root := t.TempDir() + origin := "laptop-a1b2c3" + fileFields := func(path string, size, mtime, inode, device int64, hash string) func(*db.Session) { + return func(s *db.Session) { + s.FilePath = &path + s.FileSize = &size + s.FileMtime = &mtime + s.FileInode = &inode + s.FileDevice = &device + s.FileHash = &hash + } + } + seedSession(t, database, "sess-1", "alpha", + fileFields("/old/sess.jsonl", 4096, 100, 11, 22, strings64("a"))) + + count, err := Export(ctx, database, root, origin) + require.NoError(t, err) + require.Equal(t, 1, count) + cp, err := readLatestCheckpoint(filepath.Join(root, origin)) + require.NoError(t, err) + require.NotNil(t, cp) + firstHash := cp.Sessions[origin+"~sess-1"] + require.NotEmpty(t, firstHash) + + // Touch/move the source file: every file_* bookkeeping column changes + // while the exported session content stays identical. + seedSession(t, database, "sess-1", "alpha", + fileFields("/new/sess.jsonl", 8192, 200, 33, 44, strings64("b"))) + + count, err = Export(ctx, database, root, origin) + require.NoError(t, err) + assert.Zero(t, count, + "source-file bookkeeping changes must not re-export the session") + cp, err = readLatestCheckpoint(filepath.Join(root, origin)) + require.NoError(t, err) + require.NotNil(t, cp) + assert.Equal(t, firstHash, cp.Sessions[origin+"~sess-1"], + "manifest hash must not change when only file bookkeeping changes") +} + +func TestExportEmitsNewManifestAfterDataVersionChange(t *testing.T) { + ctx := context.Background() + database := testDB(t) + root := t.TempDir() + origin := "laptop-a1b2c3" + seedSession(t, database, "sess-1", "alpha") + + count, err := Export(ctx, database, root, origin) + require.NoError(t, err) + require.Equal(t, 1, count) + cp, err := readLatestCheckpoint(filepath.Join(root, origin)) + require.NoError(t, err) + require.NotNil(t, cp) + firstHash := cp.Sessions[origin+"~sess-1"] + require.NotEmpty(t, firstHash) + + require.NoError(t, database.SetSessionDataVersion("sess-1", 42)) + count, err = Export(ctx, database, root, origin) + require.NoError(t, err) + assert.Equal(t, 1, count) + cp, err = readLatestCheckpoint(filepath.Join(root, origin)) + require.NoError(t, err) + require.NotNil(t, cp) + nextHash := cp.Sessions[origin+"~sess-1"] + require.NotEmpty(t, nextHash) + assert.NotEqual(t, firstHash, nextHash) + + m, err := readManifest(filepath.Join(root, origin), nextHash) + require.NoError(t, err) + assert.Equal(t, 42, m.DataVersion) +} + +func TestExportIncludesLocalOwnedSessionClasses(t *testing.T) { + ctx := context.Background() + database := testDB(t) + root := t.TempDir() + origin := "laptop-a1b2c3" + seedSession(t, database, "file-sess", "alpha") + seedSession(t, database, "claude-ai-sess", "bravo", func(s *db.Session) { + s.Agent = "claude-ai" + }) + seedSession(t, database, "upload-sess", "charlie", func(s *db.Session) { + s.Agent = "upload" + }) + seedSession(t, database, "orphan-sess", "delta", func(s *db.Session) { + s.SourceSessionID = "missing-source" + }) + + count, err := Export(ctx, database, root, origin) + require.NoError(t, err) + assert.Equal(t, 4, count) + + cp, err := readLatestCheckpoint(filepath.Join(root, origin)) + require.NoError(t, err) + require.NotNil(t, cp) + assert.Contains(t, cp.Sessions, origin+"~file-sess") + assert.Contains(t, cp.Sessions, origin+"~claude-ai-sess") + assert.Contains(t, cp.Sessions, origin+"~upload-sess") + assert.Contains(t, cp.Sessions, origin+"~orphan-sess") +} + +func TestImportDefersFutureVersionCheckpoint(t *testing.T) { + ctx := context.Background() + root := t.TempDir() + origin := "laptop-a1b2c3" + importDB := testDB(t) + cp := checkpoint{ + Version: formatVersion + 1, + Origin: origin, + Sequence: 1, + Sessions: map[string]string{ + origin + "~sess-1": "future-manifest", + }, + } + originRoot := filepath.Join(root, origin) + require.NoError(t, os.MkdirAll(filepath.Join(originRoot, "checkpoints"), 0o755)) + data, err := canonicalJSON(cp) + require.NoError(t, err) + require.NoError(t, writeFileAtomic( + filepath.Join(originRoot, "checkpoints", "cp-0000000001.json"), + data, + 0o644, + )) + + res, err := ImportDetailed(ctx, importDB, root, "desktop-d4e5f6") + require.NoError(t, err) + assert.False(t, res.Changed()) +} + +func TestImportUsesLatestCompatibleCheckpointBeforeFutureCheckpoint(t *testing.T) { + ctx := context.Background() + root := t.TempDir() + origin := "laptop-a1b2c3" + localOrigin := "desktop-d4e5f6" + exportDB := testDB(t) + importDB := testDB(t) + seedSession(t, exportDB, "sess-1", "alpha") + + _, err := Export(ctx, exportDB, root, origin) + require.NoError(t, err) + originRoot := filepath.Join(root, origin) + writeCheckpoint(t, originRoot, checkpoint{ + Version: formatVersion + 1, + Origin: origin, + Sequence: 2, + Sessions: map[string]string{ + origin + "~future": "future-manifest", + }, + }) + + res, err := ImportDetailed(ctx, importDB, root, localOrigin) + require.NoError(t, err) + assert.Equal(t, 1, res.Sessions) + assert.Equal(t, 2, res.Messages) +} + +func TestImportQuarantinesInvalidCheckpointAndFallsBack(t *testing.T) { + tests := []struct { + name string + mutate func(checkpoint) checkpoint + }{ + { + name: "origin mismatch", + mutate: func(cp checkpoint) checkpoint { + cp.Origin = "wrong-origin" + return cp + }, + }, + { + name: "filename sequence mismatch", + mutate: func(cp checkpoint) checkpoint { + cp.Sequence = 99 + return cp + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctx := context.Background() + root := t.TempDir() + origin := "laptop-a1b2c3" + exportDB := testDB(t) + importDB := testDB(t) + seedSession(t, exportDB, "sess-1", "alpha") + + _, err := Export(ctx, exportDB, root, origin) + require.NoError(t, err) + originRoot := filepath.Join(root, origin) + first, err := readLatestCheckpoint(originRoot) + require.NoError(t, err) + require.NotNil(t, first) + invalid := tt.mutate(checkpoint{ + Version: formatVersion, + Origin: origin, + Sequence: 2, + Sessions: first.Sessions, + }) + invalidPath := filepath.Join( + originRoot, KindCheckpoints, "cp-0000000002.json", + ) + data, err := canonicalJSON(invalid) + require.NoError(t, err) + require.NoError(t, writeFileAtomic(invalidPath, data, 0o644)) + + res, err := ImportDetailed(ctx, importDB, root, "desktop-d4e5f6") + require.NoError(t, err) + assert.Equal(t, 1, res.Sessions) + assert.FileExists(t, invalidPath+quarantineSuffix) + assert.NoFileExists(t, invalidPath) + }) + } +} + +func TestImportDefersFutureVersionManifest(t *testing.T) { + ctx := context.Background() + root := t.TempDir() + origin := "laptop-a1b2c3" + importDB := testDB(t) + originRoot := filepath.Join(root, origin) + m := manifest{ + Version: formatVersion + 1, + Origin: origin, + NativeSessionID: "sess-1", + Session: manifestSession{ + ID: "sess-1", + Machine: origin, + }, + Segments: []string{"future-segment"}, + } + data, err := canonicalJSON(m) + require.NoError(t, err) + hash := hashHex(data) + require.NoError(t, writeCompressed(filepath.Join(originRoot, "manifests", hash+manifestExtension), data)) + writeCheckpoint(t, originRoot, checkpoint{ + Version: formatVersion, + Origin: origin, + Sequence: 1, + Sessions: map[string]string{ + origin + "~sess-1": hash, + }, + }) + + res, err := ImportDetailed(ctx, importDB, root, "desktop-d4e5f6") + require.NoError(t, err) + assert.False(t, res.Changed()) + state, err := importDB.GetSyncState(importStateKey(origin, origin+"~sess-1")) + require.NoError(t, err) + assert.Empty(t, state) +} + +func TestImportDefersFutureVersionSegment(t *testing.T) { + ctx := context.Background() + root := t.TempDir() + origin := "laptop-a1b2c3" + importDB := testDB(t) + originRoot := filepath.Join(root, origin) + segmentData, err := canonicalJSON(segmentMessage{ + Version: formatVersion + 1, + Ordinal: 0, + Role: "user", + Content: "hello", + ContentLength: 5, + }) + require.NoError(t, err) + segmentHash := hashHex(segmentData) + require.NoError(t, writeCompressed(filepath.Join(originRoot, "segments", segmentHash+segmentExtension), segmentData)) + m := manifest{ + Version: formatVersion, + Origin: origin, + NativeSessionID: "sess-1", + Session: manifestSession{ + ID: "sess-1", + Machine: origin, + }, + Segments: []string{segmentHash}, + } + manifestData, err := canonicalJSON(m) + require.NoError(t, err) + manifestHash := hashHex(manifestData) + require.NoError(t, writeCompressed(filepath.Join(originRoot, "manifests", manifestHash+manifestExtension), manifestData)) + writeCheckpoint(t, originRoot, checkpoint{ + Version: formatVersion, + Origin: origin, + Sequence: 1, + Sessions: map[string]string{ + origin + "~sess-1": manifestHash, + }, + }) + + res, err := ImportDetailed(ctx, importDB, root, "desktop-d4e5f6") + require.NoError(t, err) + assert.False(t, res.Changed()) + state, err := importDB.GetSyncState(importStateKey(origin, origin+"~sess-1")) + require.NoError(t, err) + assert.Empty(t, state) +} + +func TestExportScrubsUnstableArtifactIDs(t *testing.T) { + ctx := context.Background() + database := testDB(t) + root := t.TempDir() + origin := "laptop-a1b2c3" + seedSession(t, database, "sess-1", "alpha") + require.NoError(t, database.ReplaceSessionUsageEvents("sess-1", []db.UsageEvent{ + { + SessionID: "sess-1", + Source: "fixture", + Model: "claude-test", + InputTokens: 1, + OccurredAt: "2026-06-14T01:02:04Z", + DedupKey: "usage-1", + }, + })) + + _, err := Export(ctx, database, root, origin) + require.NoError(t, err) + cp, err := readLatestCheckpoint(filepath.Join(root, origin)) + require.NoError(t, err) + require.NotNil(t, cp) + manifestHash := cp.Sessions[origin+"~sess-1"] + require.NotEmpty(t, manifestHash) + + m, err := readManifest(filepath.Join(root, origin), manifestHash) + require.NoError(t, err) + require.Len(t, m.UsageEvents, 1) + manifestData, err := canonicalJSON(m) + require.NoError(t, err) + assert.NotContains(t, string(manifestData), `"ID"`) + assert.NotContains(t, string(manifestData), `"SessionID"`) + + msgs, err := readManifestMessages(filepath.Join(root, origin), m) + require.NoError(t, err) + require.Len(t, msgs, 2) + for _, msg := range msgs { + assert.Zero(t, msg.ID) + assert.Empty(t, msg.SessionID) + } +} + +func TestExportRejectsInvalidOriginBeforeCreatingPaths(t *testing.T) { + ctx := context.Background() + database := testDB(t) + parent := t.TempDir() + root := filepath.Join(parent, "artifacts") + seedSession(t, database, "sess-1", "alpha") + + count, err := Export(ctx, database, root, "../outside") + require.Error(t, err) + assert.Zero(t, count) + assert.Contains(t, err.Error(), "invalid artifact origin") + + _, statErr := os.Stat(filepath.Join(parent, "outside")) + assert.ErrorIs(t, statErr, os.ErrNotExist) +} + +func TestExportSkipsForeignSessions(t *testing.T) { + ctx := context.Background() + database := testDB(t) + dataDir := t.TempDir() + share := t.TempDir() + seedSession(t, database, "other~sess-1", "alpha", func(s *db.Session) { + s.Machine = "other" + }) + seedSession(t, database, "legacy-remote~sess-2", "bravo", func(s *db.Session) { + s.Machine = "remote" + }) + + res, err := SyncFolder(ctx, database, SyncOptions{DataDir: dataDir, Target: share}) + require.NoError(t, err) + assert.Zero(t, res.ExportedSessions) + + origin := res.Origin + manifests := globArtifacts(t, filepath.Join(dataDir, "artifacts"), origin, "manifests", "*"+manifestExtension) + assert.Empty(t, manifests) +} + +func TestCopyUnionRejectsPathConflicts(t *testing.T) { + src := t.TempDir() + dst := t.TempDir() + // An unrecognized artifact shape cannot be content-validated, so the + // write-once conflict error is preserved as-is. + rel := filepath.Join("origin", "blobs", "data.bin") + require.NoError(t, os.MkdirAll(filepath.Join(src, filepath.Dir(rel)), 0o755)) + require.NoError(t, os.MkdirAll(filepath.Join(dst, filepath.Dir(rel)), 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(src, rel), []byte("alpha"), 0o644)) + require.NoError(t, os.WriteFile(filepath.Join(dst, rel), []byte("bravo"), 0o644)) + + err := CopyUnion(src, dst) + require.Error(t, err) + assert.ErrorIs(t, err, errArtifactPathConflict) +} + +func TestCopyUnionDetectsCheckpointSequenceConflict(t *testing.T) { + src := t.TempDir() + dst := t.TempDir() + rel := filepath.Join("origin", "checkpoints", "cp-0000000001.json") + require.NoError(t, os.MkdirAll(filepath.Join(src, filepath.Dir(rel)), 0o755)) + require.NoError(t, os.MkdirAll(filepath.Join(dst, filepath.Dir(rel)), 0o755)) + hashA := hashHex([]byte("a")) + hashB := hashHex([]byte("b")) + srcData, err := canonicalJSON(checkpoint{ + Version: formatVersion, Origin: "origin", Sequence: 1, + Sessions: map[string]string{"origin~a": hashA}, + }) + require.NoError(t, err) + dstData, err := canonicalJSON(checkpoint{ + Version: formatVersion, Origin: "origin", Sequence: 1, + Sessions: map[string]string{"origin~b": hashB}, + }) + require.NoError(t, err) + require.NoError(t, os.WriteFile(filepath.Join(src, rel), srcData, 0o644)) + require.NoError(t, os.WriteFile(filepath.Join(dst, rel), dstData, 0o644)) + + // Two valid checkpoints diverging under the same name is real divergence, + // not corruption, and must keep failing loudly. + err = CopyUnion(src, dst) + require.Error(t, err) + assert.ErrorIs(t, err, errArtifactPathConflict) +} + +func TestWriteFileAtomicIsWriteOnce(t *testing.T) { + path := filepath.Join(t.TempDir(), "origin", "segments", "artifact.ndjson.zst") + first := []byte("first") + second := []byte("second") + + require.NoError(t, writeFileAtomic(path, first, 0o644)) + require.NoError(t, writeFileAtomic(path, first, 0o644)) + err := writeFileAtomic(path, second, 0o644) + require.Error(t, err) + assert.ErrorIs(t, err, errArtifactPathConflict) + got, readErr := os.ReadFile(path) + require.NoError(t, readErr) + assert.Equal(t, first, got) +} + +func TestWriteFileAtomicDoesNotReplaceFileCreatedDuringCommit(t *testing.T) { + path := filepath.Join(t.TempDir(), "origin", "segments", "artifact.ndjson.zst") + first := []byte("first") + second := []byte("second") + + writeFileAtomicBeforeCommit = func(commitPath string) { + require.Equal(t, path, commitPath) + require.NoError(t, os.WriteFile(path, first, 0o644)) + } + t.Cleanup(func() { writeFileAtomicBeforeCommit = nil }) + + err := writeFileAtomic(path, second, 0o644) + require.Error(t, err) + assert.ErrorIs(t, err, errArtifactPathConflict) + got, readErr := os.ReadFile(path) + require.NoError(t, readErr) + assert.Equal(t, first, got) +} + +func TestWriteFileAtomicFallsBackWhenHardLinksUnsupported(t *testing.T) { + path := filepath.Join(t.TempDir(), "origin", "segments", "artifact.ndjson.zst") + data := []byte("payload") + + origLink := writeFileAtomicLink + writeFileAtomicLink = func(_, _ string) error { + return &os.LinkError{Op: "link", Err: syscall.ENOTSUP} + } + t.Cleanup(func() { writeFileAtomicLink = origLink }) + + require.NoError(t, writeFileAtomic(path, data, 0o644)) + got, err := os.ReadFile(path) + require.NoError(t, err) + assert.Equal(t, data, got) +} + +func TestCopyUnionRejectsNonRegularSources(t *testing.T) { + src := t.TempDir() + dst := t.TempDir() + target := filepath.Join(t.TempDir(), "target.txt") + require.NoError(t, os.WriteFile(target, []byte("secret"), 0o644)) + link := filepath.Join(src, "link.txt") + if err := os.Symlink(target, link); err != nil { + t.Skipf("symlink creation unavailable: %v", err) + } + + err := CopyUnion(src, dst) + require.Error(t, err) + assert.Contains(t, err.Error(), "not a regular file") +} + +func TestCopyUnionRejectsSourceReplacedByEscapingSymlink(t *testing.T) { + src := t.TempDir() + dst := t.TempDir() + rel := filepath.Join("origin", "blobs", "data.bin") + path := filepath.Join(src, rel) + require.NoError(t, os.MkdirAll(filepath.Dir(path), 0o755)) + require.NoError(t, os.WriteFile(path, []byte("artifact"), 0o644)) + secret := filepath.Join(t.TempDir(), "secret.txt") + require.NoError(t, os.WriteFile(secret, []byte("private"), 0o600)) + require.NoError(t, os.Remove(path)) + if err := os.Symlink(secret, path); err != nil { + t.Skipf("symlink creation unavailable: %v", err) + } + srcRoot, err := os.OpenRoot(src) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, srcRoot.Close()) }) + dstRoot, err := os.OpenRoot(dst) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, dstRoot.Close()) }) + + err = copyUnionFile(srcRoot, dstRoot, rel) + + require.Error(t, err) + assert.NoFileExists(t, filepath.Join(dst, rel)) +} + +func TestCopyUnionRejectsNonRegularDestinations(t *testing.T) { + src := t.TempDir() + dst := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(src, "artifact.txt"), []byte("secret"), 0o644)) + target := filepath.Join(t.TempDir(), "target.txt") + require.NoError(t, os.WriteFile(target, []byte("secret"), 0o644)) + link := filepath.Join(dst, "artifact.txt") + if err := os.Symlink(target, link); err != nil { + t.Skipf("symlink creation unavailable: %v", err) + } + + err := CopyUnion(src, dst) + require.Error(t, err) + assert.Contains(t, err.Error(), "not a regular file") +} + +func TestCopyUnionConfinesDestinationSymlinksToRoot(t *testing.T) { + src := t.TempDir() + dst := t.TempDir() + outside := t.TempDir() + rel := filepath.Join("origin", "blobs", "data.bin") + require.NoError(t, os.MkdirAll(filepath.Join(src, filepath.Dir(rel)), 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(src, rel), []byte("artifact"), 0o644)) + if err := os.Symlink(outside, filepath.Join(dst, "origin")); err != nil { + t.Skipf("symlink creation unavailable: %v", err) + } + + err := CopyUnion(src, dst) + + require.Error(t, err) + assert.NoFileExists(t, filepath.Join(outside, "blobs", "data.bin")) +} + +func TestCopyUnionSkipsAtomicTempFiles(t *testing.T) { + src := t.TempDir() + dst := t.TempDir() + rel := filepath.Join("origin", "segments", "artifact.ndjson.zst") + tmpRel := filepath.Join("origin", "segments", tempFilePrefix+"leftover") + require.NoError(t, os.MkdirAll(filepath.Join(src, "origin", "segments"), 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(src, rel), []byte("artifact"), 0o644)) + require.NoError(t, os.WriteFile(filepath.Join(src, tmpRel), []byte("partial"), 0o644)) + + require.NoError(t, CopyUnion(src, dst)) + + got, err := os.ReadFile(filepath.Join(dst, rel)) + require.NoError(t, err) + assert.Equal(t, []byte("artifact"), got) + _, err = os.Stat(filepath.Join(dst, tmpRel)) + assert.True(t, os.IsNotExist(err), "temp artifact should not be copied") +} + +func TestCopyUnionRepeatedExchangeLeavesExistingArtifactsAndCopiesMissing(t *testing.T) { + src := t.TempDir() + dst := t.TempDir() + existingRel := filepath.Join("origin", "manifests", "existing.json.zst") + missingRel := filepath.Join("origin", "checkpoints", "cp-0000000002.json") + require.NoError(t, os.MkdirAll(filepath.Join(src, filepath.Dir(existingRel)), 0o755)) + require.NoError(t, os.MkdirAll(filepath.Join(src, filepath.Dir(missingRel)), 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(src, existingRel), []byte("manifest"), 0o644)) + + require.NoError(t, CopyUnion(src, dst)) + dstExisting := filepath.Join(dst, existingRel) + pinnedMtime := time.Unix(1_700_000_000, 0) + require.NoError(t, os.Chtimes(dstExisting, pinnedMtime, pinnedMtime)) + checkpointData, err := canonicalJSON(checkpoint{Version: formatVersion, Origin: "origin", Sequence: 2}) + require.NoError(t, err) + require.NoError(t, os.WriteFile(filepath.Join(src, missingRel), checkpointData, 0o644)) + require.NoError(t, os.WriteFile(filepath.Join(src, "origin", tempFilePrefix+"leftover"), []byte("partial"), 0o644)) + + require.NoError(t, CopyUnion(src, dst)) + + info, err := os.Stat(dstExisting) + require.NoError(t, err) + assert.True(t, info.ModTime().Equal(pinnedMtime), "existing artifact should not be rewritten") + got, err := os.ReadFile(filepath.Join(dst, missingRel)) + require.NoError(t, err) + assert.Equal(t, checkpointData, got) + _, err = os.Stat(filepath.Join(dst, "origin", tempFilePrefix+"leftover")) + assert.True(t, os.IsNotExist(err), "stale temp file should not be copied while resuming") +} + +func TestCopyUnionSkipsCorruptAndQuarantinedArtifacts(t *testing.T) { + ctx := context.Background() + src := t.TempDir() + dst := t.TempDir() + origin := "laptop-a1b2c3" + database := testDB(t) + seedSession(t, database, "sess-1", "alpha") + + _, err := Export(ctx, database, src, origin) + require.NoError(t, err) + + originRoot := filepath.Join(src, origin) + segments := globArtifacts(t, src, origin, "segments", "*"+segmentExtension) + require.Len(t, segments, 1) + require.NoError(t, os.Remove(segments[0])) + require.NoError(t, writeCompressed(segments[0], []byte("tampered"))) + quarantined := filepath.Join(originRoot, "manifests", "leftover.json.zst"+quarantineSuffix) + require.NoError(t, os.WriteFile(quarantined, []byte("junk"), 0o644)) + // A forged metadata event with a valid hash but an unparseable HLC must + // not propagate between stores (#1034). + forgedHLC := "9999-99-99T999999.000000000Z-00000000000000000000" + forged := replayRenameEvent(t, origin, origin+"~sess-1", forgedHLC, "PWNED") + forgedData, err := canonicalJSON(forged) + require.NoError(t, err) + forgedName := forgedHLC + "-" + hashHex(forgedData) + metadataEventExtension + require.NoError(t, writeFileAtomic( + filepath.Join(originRoot, "meta", forgedName), forgedData, 0o644)) + + require.NoError(t, CopyUnion(src, dst)) + + assert.NoFileExists(t, filepath.Join(dst, origin, "segments", filepath.Base(segments[0]))) + assert.NoFileExists(t, filepath.Join(dst, origin, "manifests", "leftover.json.zst"+quarantineSuffix)) + assert.NoFileExists(t, filepath.Join(dst, origin, "meta", forgedName)) + assert.Len(t, globArtifacts(t, dst, origin, "manifests", "*"+manifestExtension), 1) + assert.Len(t, globArtifacts(t, dst, origin, "checkpoints", "cp-*.json"), 1) +} + +func TestCopyUnionQuarantinesCorruptSourceAndExportRegenerates(t *testing.T) { + ctx := context.Background() + src := t.TempDir() + dst := t.TempDir() + origin := "laptop-a1b2c3" + database := testDB(t) + seedSession(t, database, "sess-1", "alpha") + + _, err := Export(ctx, database, src, origin) + require.NoError(t, err) + + segments := globArtifacts(t, src, origin, "segments", "*"+segmentExtension) + require.Len(t, segments, 1) + require.NoError(t, os.Remove(segments[0])) + require.NoError(t, writeCompressed(segments[0], []byte("tampered"))) + + require.NoError(t, CopyUnion(src, dst)) + + // The corrupt source is quarantined at detection, not left in place, so + // the export "unchanged" fast path stops seeing it as present. + assert.NoFileExists(t, segments[0]) + assert.FileExists(t, segments[0]+quarantineSuffix) + assert.NoFileExists(t, filepath.Join(dst, origin, "segments", filepath.Base(segments[0]))) + + // The next export regenerates the segment instead of treating the session + // as unchanged, and the following exchange publishes it. + _, err = Export(ctx, database, src, origin) + require.NoError(t, err) + assert.FileExists(t, segments[0]) + require.NoError(t, CopyUnion(src, dst)) + assert.FileExists(t, filepath.Join(dst, origin, "segments", filepath.Base(segments[0]))) +} + +func TestExportQuarantinesAndRegeneratesCorruptUnchangedArtifacts(t *testing.T) { + ctx := context.Background() + for _, kind := range []string{KindManifests, KindSegments} { + t.Run(kind, func(t *testing.T) { + root := t.TempDir() + origin := "laptop-a1b2c3" + database := testDB(t) + seedSession(t, database, "sess-1", "alpha") + _, err := Export(ctx, database, root, origin) + require.NoError(t, err) + + pattern := "*" + manifestExtension + if kind == KindSegments { + pattern = "*" + segmentExtension + } + paths := globArtifacts(t, root, origin, kind, pattern) + require.Len(t, paths, 1) + path := paths[0] + require.NoError(t, os.WriteFile(path, []byte("corrupt in place"), 0o644)) + + exported, err := Export(ctx, database, root, origin) + require.NoError(t, err) + assert.Equal(t, 1, exported, + "regeneration must count the session as changed") + assert.FileExists(t, path) + assert.FileExists(t, path+quarantineSuffix) + + manifestPaths := globArtifacts( + t, root, origin, KindManifests, "*"+manifestExtension, + ) + require.Len(t, manifestPaths, 1) + manifestHash := strings.TrimSuffix( + filepath.Base(manifestPaths[0]), manifestExtension, + ) + manifest, err := readManifest(filepath.Join(root, origin), manifestHash) + require.NoError(t, err) + messages, err := readManifestMessages(filepath.Join(root, origin), manifest) + require.NoError(t, err) + require.Len(t, messages, 2) + assert.Equal(t, "hello", messages[0].Content) + }) + } +} + +func TestCopyUnionRepairsCorruptDestinationArtifact(t *testing.T) { + ctx := context.Background() + src := t.TempDir() + dst := t.TempDir() + origin := "laptop-a1b2c3" + database := testDB(t) + seedSession(t, database, "sess-1", "alpha") + + _, err := Export(ctx, database, src, origin) + require.NoError(t, err) + require.NoError(t, CopyUnion(src, dst)) + + segments := globArtifacts(t, dst, origin, "segments", "*"+segmentExtension) + require.Len(t, segments, 1) + require.NoError(t, os.Remove(segments[0])) + require.NoError(t, writeCompressed(segments[0], []byte("tampered"))) + + require.NoError(t, CopyUnion(src, dst)) + + want, err := os.ReadFile(filepath.Join(src, origin, "segments", filepath.Base(segments[0]))) + require.NoError(t, err) + got, err := os.ReadFile(segments[0]) + require.NoError(t, err) + assert.Equal(t, want, got, "valid source should repair corrupt destination") +} + +func TestSyncFolderHealsCorruptSegmentInShare(t *testing.T) { + ctx := context.Background() + share := t.TempDir() + aData := t.TempDir() + bData := t.TempDir() + aDB := testDB(t) + bDB := testDB(t) + + require.NoError(t, aDB.SetSyncState(originStateKey, "laptop-a1b2c3")) + require.NoError(t, bDB.SetSyncState(originStateKey, "desktop-d4e5f6")) + seedSession(t, aDB, "sess-1", "alpha") + + _, err := SyncFolder(ctx, aDB, SyncOptions{DataDir: aData, Target: share}) + require.NoError(t, err) + + // Corrupt A's segment in the share, as an interrupted file-sync write would. + segments := globArtifacts(t, share, "laptop-a1b2c3", "segments", "*"+segmentExtension) + require.Len(t, segments, 1) + require.NoError(t, os.Remove(segments[0])) + require.NoError(t, writeCompressed(segments[0], []byte("tampered"))) + + // B tolerates the corrupt share: sync succeeds, the poison is not + // mirrored into B's local store, and repeat runs stay healthy. + res, err := SyncFolder(ctx, bDB, SyncOptions{DataDir: bData, Target: share}) + require.NoError(t, err) + assert.Zero(t, res.ImportedSessions) + assert.NoFileExists(t, filepath.Join(bData, "artifacts", "laptop-a1b2c3", "segments", filepath.Base(segments[0]))) + _, err = SyncFolder(ctx, bDB, SyncOptions{DataDir: bData, Target: share}) + require.NoError(t, err) + + // A still holds the valid copy and repairs the share on its next sync. + _, err = SyncFolder(ctx, aDB, SyncOptions{DataDir: aData, Target: share}) + require.NoError(t, err) + want, err := os.ReadFile(filepath.Join(aData, "artifacts", "laptop-a1b2c3", "segments", filepath.Base(segments[0]))) + require.NoError(t, err) + got, err := os.ReadFile(segments[0]) + require.NoError(t, err) + require.Equal(t, want, got, "publisher should repair the corrupt share copy") + + // With the share healed, B converges. + res, err = SyncFolder(ctx, bDB, SyncOptions{DataDir: bData, Target: share}) + require.NoError(t, err) + assert.Equal(t, 1, res.ImportedSessions) + assert.Equal(t, 2, res.ImportedMessages) +} + +func globArtifacts(t *testing.T, root, origin, kind, pattern string) []string { + t.Helper() + paths, err := filepath.Glob(filepath.Join(root, origin, kind, pattern)) + require.NoError(t, err) + return paths +} + +func writeCheckpoint(t *testing.T, originRoot string, cp checkpoint) { + t.Helper() + data, err := canonicalJSON(cp) + require.NoError(t, err) + path := filepath.Join(originRoot, "checkpoints", fmt.Sprintf("cp-%010d.json", cp.Sequence)) + if err := os.Remove(path); err != nil && !errors.Is(err, os.ErrNotExist) { + require.NoError(t, err) + } + require.NoError(t, writeFileAtomic(path, data, 0o644)) +} + +func writePreNormalizationManifest( + t *testing.T, + ctx context.Context, + database *db.DB, + originRoot, origin, sessionID string, +) string { + t.Helper() + sess, err := database.GetSessionFull(ctx, sessionID) + require.NoError(t, err) + require.NotNil(t, sess) + msgs, err := database.GetAllMessages(ctx, sessionID) + require.NoError(t, err) + usageEvents, err := database.GetUsageEvents(ctx, sessionID) + require.NoError(t, err) + segmentData, err := encodeSegment(canonicalMessages(msgs)) + require.NoError(t, err) + segmentHash := hashHex(segmentData) + require.NoError(t, writeCompressed( + filepath.Join(originRoot, "segments", segmentHash+segmentExtension), + segmentData, + )) + + wireSession := manifestSessionFromDB(*sess) + wireSession.Machine = origin + require.NotZero(t, wireSession.SecretLeakCount, "precondition: old manifest carries source scan count") + require.NotNil(t, wireSession.LocalModifiedAt, "precondition: old manifest carries local watermark") + m := manifest{ + Version: formatVersion, + Origin: origin, + NativeSessionID: sessionID, + Session: wireSession, + SessionName: sess.SessionName, + Segments: []string{segmentHash}, + UsageEvents: canonicalUsageEvents(usageEvents), + DataVersion: sess.DataVersion, + Generation: 1, + SessionHasToolCalls: sess.HasToolCalls, + SessionHasContextData: sess.HasContextData, + SessionQualitySignals: manifestQualitySignalsFromDB(sess.StoredQualitySignals()), + } + manifestData, err := canonicalJSON(m) + require.NoError(t, err) + manifestHash := hashHex(manifestData) + require.NoError(t, writeCompressed( + filepath.Join(originRoot, "manifests", manifestHash+manifestExtension), + manifestData, + )) + return manifestHash +} + +func testDB(t *testing.T) *db.DB { + t.Helper() + database, err := db.Open(filepath.Join(t.TempDir(), "test.db")) + require.NoError(t, err) + t.Cleanup(func() { database.Close() }) + return database +} + +func seedSession(t *testing.T, database *db.DB, id, project string, opts ...func(*db.Session)) { + t.Helper() + sess := db.Session{ + ID: id, + Project: project, + Machine: "local", + Agent: "claude", + MessageCount: 2, + UserMessageCount: 1, + FirstMessage: new("hello"), + StartedAt: new("2026-06-14T01:02:03Z"), + EndedAt: new("2026-06-14T01:03:03Z"), + SessionName: new("Test Session"), + CreatedAt: "2026-06-14T01:02:03Z", + } + for _, opt := range opts { + opt(&sess) + } + require.NoError(t, database.UpsertSession(sess)) + require.NoError(t, database.ReplaceSessionMessages(id, []db.Message{ + {SessionID: id, Ordinal: 0, Role: "user", Content: "hello", ContentLength: 5}, + {SessionID: id, Ordinal: 1, Role: "assistant", Content: "world", ContentLength: 5}, + })) +} diff --git a/internal/artifact/transport.go b/internal/artifact/transport.go new file mode 100644 index 000000000..922d6e20e --- /dev/null +++ b/internal/artifact/transport.go @@ -0,0 +1,126 @@ +package artifact + +import ( + "bytes" + "context" + "errors" + "fmt" + "log" + "os" + "path/filepath" +) + +// Transport exchanges immutable, content-addressed artifacts between the local +// store and a remote target using set-union semantics: publish local-only +// artifacts to the remote and fetch remote-only artifacts into the local store. +// Because every artifact is write-once and content-addressed, exchange is +// order-independent and idempotent, so folder, HTTP peer, and object-store +// targets are interchangeable behind this interface. +type Transport interface { + // Prepare validates the target against the local store and creates any + // remote-side structure required before exchange. It runs before the local + // export so a misconfigured target fails fast. + Prepare(ctx context.Context, localRoot string) error + // Exchange performs the set-union publish (local-only -> remote) and fetch + // (remote-only -> local). + Exchange(ctx context.Context, localRoot string) error +} + +// folderTransport exchanges artifacts with a local filesystem folder: a synced +// share such as Syncthing, Dropbox, NFS, or an rclone mount. +type folderTransport struct { + target string +} + +func (t *folderTransport) Prepare(_ context.Context, localRoot string) error { + if err := validateDisjointRoots(localRoot, t.target); err != nil { + return err + } + if err := os.MkdirAll(t.target, 0o755); err != nil { + return fmt.Errorf("creating artifact sync target: %w", err) + } + return nil +} + +func (t *folderTransport) Exchange(_ context.Context, localRoot string) error { + if err := CopyUnion(localRoot, t.target); err != nil { + return fmt.Errorf("publishing artifacts: %w", err) + } + if err := CopyUnion(t.target, localRoot); err != nil { + return fmt.Errorf("fetching artifacts: %w", err) + } + return nil +} + +// reconcileCommonCheckpoint guards the HTTP and S3 exchanges against silent +// checkpoint divergence: checkpoints are sequence-named rather than +// content-addressed, so a rebuilt store or an accidentally shared origin id +// can hold different bytes under the same name, which name-set comparison +// alone would skip forever. The highest common checkpoint name is compared by +// content each exchange: a corrupt local copy is quarantined (the caller +// refreshes its index so the remote copy is re-fetched), a corrupt remote +// copy is left for its owner to repair, and a valid-but-divergent pair fails +// loudly with the same conflict error the folder transport raises. Divergence +// buried below the highest common sequence is not scanned; import only trusts +// the latest compatible checkpoint, so it cannot change the imported state. +func reconcileCommonCheckpoint( + localRoot, origin string, + local, remote OriginArtifactIndex, + fetch func(name string) ([]byte, error), +) (quarantined bool, err error) { + name := highestCommonCheckpoint(local, remote) + if name == "" { + return false, nil + } + remoteData, err := fetch(name) + if err != nil { + if errors.Is(err, ErrArtifactNotFound) { + // The remote no longer serves its copy, typically after + // quarantining a corrupt one; nothing to compare this round. + return false, nil + } + return false, err + } + art, err := ReadArtifact(localRoot, origin, KindCheckpoints, name) + if err != nil { + if errors.Is(err, ErrArtifactInvalid) { + log.Printf("artifact: quarantining corrupt local checkpoint %s/%s: %v", origin, name, err) + quarantineArtifact(filepath.Join(localRoot, origin, KindCheckpoints, name)) + return true, nil + } + if errors.Is(err, ErrArtifactNotFound) { + return false, nil + } + return false, err + } + if bytes.Equal(art.Data, remoteData) { + return false, nil + } + spec, err := artifactSpecForKind(origin, KindCheckpoints, name) + if err != nil { + return false, err + } + if verr := validateArtifactData(spec, remoteData); verr != nil { + log.Printf("artifact: remote holds corrupt checkpoint %s/%s: %v", origin, name, verr) + return false, nil + } + return false, fmt.Errorf( + "%w: checkpoint %s/%s differs between the local store and the remote; "+ + "was this origin's artifact store rebuilt or its origin id reused?", + errArtifactPathConflict, origin, name, + ) +} + +func highestCommonCheckpoint(local, remote OriginArtifactIndex) string { + names := make(map[string]struct{}, len(local.Checkpoints)) + for _, name := range local.Checkpoints { + names[name] = struct{}{} + } + best := "" + for _, name := range remote.Checkpoints { + if _, ok := names[name]; ok && name > best { + best = name + } + } + return best +} diff --git a/internal/artifact/transport_http.go b/internal/artifact/transport_http.go new file mode 100644 index 000000000..deb03fd99 --- /dev/null +++ b/internal/artifact/transport_http.go @@ -0,0 +1,334 @@ +package artifact + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "log" + "net/http" + "net/url" + "path/filepath" + "strings" + "time" +) + +const ( + artifactAPIPath = "/api/v1/artifacts" + httpTransportTimeout = 120 * time.Second + httpTransportMaxErrLen = 512 +) + +// IsHTTPTarget reports whether target is an HTTP(S) peer URL rather than a local +// folder target. +func IsHTTPTarget(target string) bool { + return strings.HasPrefix(target, "http://") || strings.HasPrefix(target, "https://") +} + +// httpTransport exchanges artifacts with a remote agentsview peer over its +// authenticated HTTP artifact API: list origins, list an origin's artifact +// index, get an artifact by name, and post an artifact. Content addressing makes +// "does the peer have X" a name-set comparison, so exchange is a stateless, +// idempotent set-union in both directions. +type httpTransport struct { + base string + origin string + token string + client *http.Client +} + +func newHTTPTransport(target, token string, allowInsecure bool) (*httpTransport, error) { + u, err := url.Parse(strings.TrimRight(target, "/")) + if err != nil { + return nil, fmt.Errorf("parsing peer URL %q: %w", target, err) + } + if u.Scheme != "http" && u.Scheme != "https" { + return nil, fmt.Errorf("peer target must be http:// or https://: %q", target) + } + if u.Host == "" { + return nil, fmt.Errorf("peer target is missing a host: %q", target) + } + if u.Scheme == "http" && !allowInsecure && !isLoopbackEndpointHost(u.Hostname()) { + return nil, fmt.Errorf( + "insecure artifact peer %q requires HTTPS", + target, + ) + } + if u.Scheme == "http" && allowInsecure && !isLoopbackEndpointHost(u.Hostname()) { + log.Printf( + "warning: artifact sync to %q uses plaintext HTTP; credentials and archive content are not encrypted in transit", + target, + ) + } + base := strings.TrimRight(u.String(), "/") + if !strings.HasSuffix(base, artifactAPIPath) { + base += artifactAPIPath + } + peerOrigin := (&url.URL{Scheme: u.Scheme, Host: u.Host}).String() + return &httpTransport{ + base: base, + origin: peerOrigin, + token: token, + client: &http.Client{ + Timeout: httpTransportTimeout, + CheckRedirect: func(_ *http.Request, _ []*http.Request) error { + return http.ErrUseLastResponse + }, + }, + }, nil +} + +// Prepare validates that the peer is reachable and authenticated before the +// local export runs, so a wrong URL or token fails fast. +func (t *httpTransport) Prepare(ctx context.Context, _ string) error { + if _, err := t.listOrigins(ctx); err != nil { + return fmt.Errorf("connecting to artifact peer: %w", err) + } + return nil +} + +func (t *httpTransport) Exchange(ctx context.Context, localRoot string) error { + if err := t.pull(ctx, localRoot); err != nil { + return fmt.Errorf("fetching artifacts from peer: %w", err) + } + if err := t.push(ctx, localRoot); err != nil { + return fmt.Errorf("publishing artifacts to peer: %w", err) + } + return nil +} + +// pull fetches every artifact the peer holds that is missing locally. +func (t *httpTransport) pull(ctx context.Context, localRoot string) error { + origins, err := t.listOrigins(ctx) + if err != nil { + return err + } + for _, origin := range origins { + remote, err := t.getIndex(ctx, origin) + if err != nil { + return err + } + local, err := ListArtifacts(localRoot, origin) + if err != nil { + return err + } + quarantined, err := reconcileCommonCheckpoint(localRoot, origin, local, remote, + func(name string) ([]byte, error) { + return t.getArtifact(ctx, origin, KindCheckpoints, name) + }) + if err != nil { + return err + } + if quarantined { + if local, err = ListArtifacts(localRoot, origin); err != nil { + return err + } + } + for _, item := range missingItems(remote, local) { + data, err := t.getArtifact(ctx, origin, item.kind, item.name) + if err != nil { + if errors.Is(err, ErrArtifactNotFound) { + log.Printf("artifact: peer no longer serves %s/%s/%s; skipping", + origin, item.kind, item.name) + continue + } + return err + } + if _, err := WriteArtifact(localRoot, origin, item.kind, item.name, data); err != nil { + if errors.Is(err, ErrArtifactInvalid) { + log.Printf("artifact: skipping corrupt artifact %s/%s/%s from peer: %v", + origin, item.kind, item.name, err) + continue + } + return err + } + } + } + return nil +} + +// push uploads every local artifact the peer is missing. +func (t *httpTransport) push(ctx context.Context, localRoot string) error { + origins, err := ListOrigins(localRoot) + if err != nil { + return err + } + for _, origin := range origins { + local, err := ListArtifacts(localRoot, origin) + if err != nil { + return err + } + remote, err := t.getIndex(ctx, origin) + if err != nil { + return err + } + for _, item := range missingItems(local, remote) { + art, err := ReadArtifact(localRoot, origin, item.kind, item.name) + if err != nil { + if skipUnpublishableArtifact(localRoot, origin, item, err) { + continue + } + return err + } + if err := t.postArtifact(ctx, origin, item.kind, item.name, art.Data); err != nil { + return err + } + } + } + return nil +} + +type artifactItem struct { + kind string + name string +} + +// skipUnpublishableArtifact reports whether a local artifact that failed to +// read before publishing should be skipped: a corrupt file is quarantined so +// a valid copy can be re-fetched, and a file already removed (quarantined or +// collected by a concurrent run) is simply gone. Anything else aborts. +func skipUnpublishableArtifact(localRoot, origin string, item artifactItem, err error) bool { + if errors.Is(err, ErrArtifactInvalid) { + log.Printf("artifact: not publishing corrupt artifact %s/%s/%s: %v", + origin, item.kind, item.name, err) + quarantineArtifact(filepath.Join(localRoot, origin, item.kind, item.name)) + return true + } + return errors.Is(err, ErrArtifactNotFound) +} + +func indexItems(idx OriginArtifactIndex) []artifactItem { + items := make([]artifactItem, 0, + len(idx.Checkpoints)+len(idx.Manifests)+len(idx.Segments)+len(idx.Meta)+len(idx.Raw)) + for _, group := range []struct { + kind string + names []string + }{ + {KindSegments, idx.Segments}, + {KindRaw, idx.Raw}, + {KindManifests, idx.Manifests}, + {KindMeta, idx.Meta}, + {KindCheckpoints, idx.Checkpoints}, + } { + for _, name := range group.names { + items = append(items, artifactItem{kind: group.kind, name: name}) + } + } + return items +} + +// missingItems returns artifacts present in have but absent from other. +func missingItems(have, other OriginArtifactIndex) []artifactItem { + present := make(map[artifactItem]struct{}) + for _, it := range indexItems(other) { + present[it] = struct{}{} + } + var out []artifactItem + for _, it := range indexItems(have) { + if _, ok := present[it]; !ok { + out = append(out, it) + } + } + return out +} + +func (t *httpTransport) listOrigins(ctx context.Context) ([]string, error) { + var resp struct { + Origins []string `json:"origins"` + } + if err := t.getJSON(ctx, t.base+"/origins", &resp); err != nil { + return nil, err + } + return resp.Origins, nil +} + +func (t *httpTransport) getIndex(ctx context.Context, origin string) (OriginArtifactIndex, error) { + var idx OriginArtifactIndex + if err := t.getJSON(ctx, t.base+"/"+url.PathEscape(origin)+"/index", &idx); err != nil { + return OriginArtifactIndex{}, err + } + return idx, nil +} + +func (t *httpTransport) getArtifact(ctx context.Context, origin, kind, name string) ([]byte, error) { + u := t.base + "/" + url.PathEscape(origin) + "/" + url.PathEscape(kind) + "/" + url.PathEscape(name) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, u, nil) + if err != nil { + return nil, err + } + t.authorize(req) + resp, err := t.client.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + if resp.StatusCode == http.StatusNotFound { + // The peer listed this artifact but no longer serves it, typically + // because it quarantined a corrupt copy between index and fetch. + return nil, fmt.Errorf("%w: %s", ErrArtifactNotFound, resp.Status) + } + if resp.StatusCode != http.StatusOK { + return nil, httpStatusError(resp) + } + return io.ReadAll(resp.Body) +} + +func (t *httpTransport) postArtifact(ctx context.Context, origin, kind, name string, data []byte) error { + u := t.base + "/" + url.PathEscape(origin) + "/" + url.PathEscape(kind) + "/" + url.PathEscape(name) + req, err := http.NewRequestWithContext(ctx, http.MethodPost, u, bytes.NewReader(data)) + if err != nil { + return err + } + req.Header.Set("Content-Type", "application/octet-stream") + req.Header.Set("Origin", t.origin) + t.authorize(req) + resp, err := t.client.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusCreated { + return httpStatusError(resp) + } + _, _ = io.Copy(io.Discard, resp.Body) + return nil +} + +func (t *httpTransport) getJSON(ctx context.Context, u string, out any) error { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, u, nil) + if err != nil { + return err + } + t.authorize(req) + resp, err := t.client.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return httpStatusError(resp) + } + return json.NewDecoder(resp.Body).Decode(out) +} + +func (t *httpTransport) authorize(req *http.Request) { + if t.token != "" { + req.Header.Set("Authorization", "Bearer "+t.token) + } +} + +func httpStatusError(resp *http.Response) error { + body, _ := io.ReadAll(io.LimitReader(resp.Body, httpTransportMaxErrLen)) + msg := strings.TrimSpace(string(body)) + if resp.StatusCode == http.StatusUnauthorized { + return fmt.Errorf("%w: peer rejected the bearer token (401)", errHTTPPeer) + } + if msg == "" { + return fmt.Errorf("%w: %s", errHTTPPeer, resp.Status) + } + return fmt.Errorf("%w: %s: %s", errHTTPPeer, resp.Status, msg) +} + +var errHTTPPeer = errors.New("artifact peer request failed") diff --git a/internal/artifact/transport_http_test.go b/internal/artifact/transport_http_test.go new file mode 100644 index 000000000..d76a58cf6 --- /dev/null +++ b/internal/artifact/transport_http_test.go @@ -0,0 +1,378 @@ +package artifact + +import ( + "bytes" + "context" + "encoding/json" + "io" + "log" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "sort" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.kenn.io/agentsview/internal/db" +) + +// fakeArtifactPeer is an in-memory peer implementing the artifact API surface +// the HTTP transport exchanges against: origin listing, per-origin index, and +// artifact get/post. +type fakeArtifactPeer struct { + mu sync.Mutex + arts map[string][]byte // "origin/kind/name" -> bytes + posts []string +} + +func newFakeArtifactPeer() *fakeArtifactPeer { + return &fakeArtifactPeer{arts: map[string][]byte{}} +} + +func (p *fakeArtifactPeer) put(origin, kind, name string, data []byte) { + p.mu.Lock() + defer p.mu.Unlock() + p.arts[origin+"/"+kind+"/"+name] = append([]byte(nil), data...) +} + +func (p *fakeArtifactPeer) has(origin, kind, name string) bool { + p.mu.Lock() + defer p.mu.Unlock() + _, ok := p.arts[origin+"/"+kind+"/"+name] + return ok +} + +func (p *fakeArtifactPeer) postedKinds() []string { + p.mu.Lock() + defer p.mu.Unlock() + kinds := make([]string, 0, len(p.posts)) + for _, key := range p.posts { + parts := strings.SplitN(key, "/", 3) + if len(parts) == 3 { + kinds = append(kinds, parts[1]) + } + } + return kinds +} + +func (p *fakeArtifactPeer) ServeHTTP(w http.ResponseWriter, r *http.Request) { + rest := strings.TrimPrefix(r.URL.Path, artifactAPIPath+"/") + p.mu.Lock() + defer p.mu.Unlock() + if rest == "origins" { + seen := map[string]bool{} + for key := range p.arts { + seen[strings.SplitN(key, "/", 2)[0]] = true + } + origins := make([]string, 0, len(seen)) + for origin := range seen { + origins = append(origins, origin) + } + sort.Strings(origins) + _ = json.NewEncoder(w).Encode(map[string]any{"origins": origins}) + return + } + parts := strings.Split(rest, "/") + if len(parts) == 2 && parts[1] == "index" { + idx := OriginArtifactIndex{Origin: parts[0]} + for key := range p.arts { + kp := strings.SplitN(key, "/", 3) + if kp[0] != parts[0] { + continue + } + switch kp[1] { + case KindCheckpoints: + idx.Checkpoints = append(idx.Checkpoints, kp[2]) + case KindManifests: + idx.Manifests = append(idx.Manifests, kp[2]) + case KindSegments: + idx.Segments = append(idx.Segments, kp[2]) + case KindMeta: + idx.Meta = append(idx.Meta, kp[2]) + case KindRaw: + idx.Raw = append(idx.Raw, kp[2]) + } + } + _ = json.NewEncoder(w).Encode(idx) + return + } + if len(parts) == 3 { + switch r.Method { + case http.MethodGet: + data, ok := p.arts[rest] + if !ok { + http.NotFound(w, r) + return + } + _, _ = w.Write(data) + case http.MethodPost: + data, _ := io.ReadAll(r.Body) + p.arts[rest] = data + p.posts = append(p.posts, rest) + w.WriteHeader(http.StatusCreated) + } + return + } + http.NotFound(w, r) +} + +func TestHTTPTransportRequiresTLSForNonLoopbackPeers(t *testing.T) { + tests := []struct { + name string + target string + wantErr bool + }{ + {name: "public HTTP", target: "http://203.0.113.10:8080", wantErr: true}, + {name: "hostname HTTP", target: "http://peer.example.test:8080", wantErr: true}, + {name: "public HTTPS", target: "https://peer.example.test:8443"}, + {name: "localhost HTTP", target: "http://localhost:8080"}, + {name: "IPv4 loopback HTTP", target: "http://127.0.0.1:8080"}, + {name: "IPv6 loopback HTTP", target: "http://[::1]:8080"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tr, err := newHTTPTransport(tt.target, "", false) + if tt.wantErr { + require.Error(t, err) + assert.Contains(t, err.Error(), "requires HTTPS") + assert.Nil(t, tr) + return + } + require.NoError(t, err) + assert.NotNil(t, tr) + }) + } +} + +func TestHTTPTransportAllowsExplicitRemotePlaintextOptIn(t *testing.T) { + var logs bytes.Buffer + previousOutput := log.Writer() + log.SetOutput(&logs) + t.Cleanup(func() { log.SetOutput(previousOutput) }) + + tr, err := newHTTPTransport("http://peer.example.test:8080", "", true) + + require.NoError(t, err) + require.NotNil(t, tr) + assert.Equal(t, "http://peer.example.test:8080"+artifactAPIPath, tr.base) + assert.Contains(t, logs.String(), "warning") + assert.Contains(t, logs.String(), "plaintext HTTP") +} + +func TestHTTPTransportPrepareHonorsCanceledSync(t *testing.T) { + var requests atomic.Int32 + peer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + requests.Add(1) + _ = json.NewEncoder(w).Encode(map[string]any{"origins": []string{}}) + })) + t.Cleanup(peer.Close) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + _, err := Sync(ctx, testDB(t), SyncOptions{ + DataDir: t.TempDir(), + Target: peer.URL, + Origin: "laptop-a1b2c3", + }) + + require.ErrorIs(t, err, context.Canceled) + assert.Zero(t, requests.Load(), "canceled preparation must not contact the peer") +} + +func TestHTTPTransportPullSkipsCorruptRemoteArtifact(t *testing.T) { + origin := "desktop-d4e5f6" + remoteRoot := exportStore(t, origin, func(database *db.DB) { + seedSession(t, database, "sess-7", "beta") + }) + remoteIdx, err := ListArtifacts(remoteRoot, origin) + require.NoError(t, err) + peer := newFakeArtifactPeer() + for _, item := range indexItems(remoteIdx) { + art, err := ReadArtifact(remoteRoot, origin, item.kind, item.name) + require.NoError(t, err) + peer.put(origin, item.kind, item.name, art.Data) + } + corruptName := hashHex([]byte("corrupt")) + segmentExtension + peer.put(origin, KindSegments, corruptName, []byte("garbage")) + + srv := httptest.NewServer(peer) + t.Cleanup(srv.Close) + tr, err := newHTTPTransport(srv.URL, "", false) + require.NoError(t, err) + localRoot := filepath.Join(t.TempDir(), "artifacts") + require.NoError(t, tr.Exchange(context.Background(), localRoot)) + + gotIdx, err := ListArtifacts(localRoot, origin) + require.NoError(t, err) + assert.ElementsMatch(t, indexItems(remoteIdx), indexItems(gotIdx)) + assert.NoFileExists(t, filepath.Join(localRoot, origin, KindSegments, corruptName)) +} + +func TestHTTPTransportPushSkipsAndQuarantinesCorruptLocalArtifact(t *testing.T) { + origin := "laptop-a1b2c3" + localRoot := exportStore(t, origin, func(database *db.DB) { + seedSession(t, database, "sess-1", "alpha") + }) + validIdx, err := ListArtifacts(localRoot, origin) + require.NoError(t, err) + corruptName := hashHex([]byte("junk")) + segmentExtension + corruptPath := filepath.Join(localRoot, origin, KindSegments, corruptName) + require.NoError(t, os.WriteFile(corruptPath, []byte("garbage"), 0o644)) + + peer := newFakeArtifactPeer() + srv := httptest.NewServer(peer) + t.Cleanup(srv.Close) + tr, err := newHTTPTransport(srv.URL, "", false) + require.NoError(t, err) + require.NoError(t, tr.Exchange(context.Background(), localRoot)) + + for _, item := range indexItems(validIdx) { + assert.True(t, peer.has(origin, item.kind, item.name), + "expected %s/%s on the peer", item.kind, item.name) + } + assert.False(t, peer.has(origin, KindSegments, corruptName)) + assert.NoFileExists(t, corruptPath) + assert.FileExists(t, corruptPath+quarantineSuffix) +} + +func TestHTTPTransportPushPublishesDependenciesBeforeCheckpoint(t *testing.T) { + localRoot := exportStore(t, "laptop-a1b2c3", func(database *db.DB) { + seedSession(t, database, "sess-1", "alpha") + }) + peer := newFakeArtifactPeer() + server := httptest.NewServer(peer) + t.Cleanup(server.Close) + transport, err := newHTTPTransport(server.URL, "", false) + require.NoError(t, err) + + require.NoError(t, transport.Exchange(context.Background(), localRoot)) + + assert.Equal(t, + []string{KindSegments, KindManifests, KindCheckpoints}, + peer.postedKinds(), + ) +} + +func TestHTTPTransportExchangeDetectsDivergentCheckpoint(t *testing.T) { + origin := "laptop-a1b2c3" + localRoot := exportStore(t, origin, func(database *db.DB) { + seedSession(t, database, "sess-1", "alpha") + }) + checkpoints := globArtifacts(t, localRoot, origin, KindCheckpoints, "cp-*.json") + require.Len(t, checkpoints, 1) + name := filepath.Base(checkpoints[0]) + + // The peer holds a different, equally valid checkpoint under the same + // sequence name, as a rebuilt store under a reused origin id would. + divergent, err := canonicalJSON(checkpoint{ + Version: formatVersion, Origin: origin, Sequence: 1, + Sessions: map[string]string{origin + "~other": hashHex([]byte("other"))}, + }) + require.NoError(t, err) + peer := newFakeArtifactPeer() + peer.put(origin, KindCheckpoints, name, divergent) + + srv := httptest.NewServer(peer) + t.Cleanup(srv.Close) + tr, err := newHTTPTransport(srv.URL, "", false) + require.NoError(t, err) + + err = tr.Exchange(context.Background(), localRoot) + require.Error(t, err) + assert.ErrorIs(t, err, errArtifactPathConflict) +} + +func TestHTTPTransportExchangeRepairsCorruptLocalCheckpoint(t *testing.T) { + origin := "laptop-a1b2c3" + localRoot := exportStore(t, origin, func(database *db.DB) { + seedSession(t, database, "sess-1", "alpha") + }) + checkpoints := globArtifacts(t, localRoot, origin, KindCheckpoints, "cp-*.json") + require.Len(t, checkpoints, 1) + name := filepath.Base(checkpoints[0]) + valid, err := os.ReadFile(checkpoints[0]) + require.NoError(t, err) + + peer := newFakeArtifactPeer() + peer.put(origin, KindCheckpoints, name, valid) + require.NoError(t, os.WriteFile(checkpoints[0], []byte("not json"), 0o644)) + + srv := httptest.NewServer(peer) + t.Cleanup(srv.Close) + tr, err := newHTTPTransport(srv.URL, "", false) + require.NoError(t, err) + require.NoError(t, tr.Exchange(context.Background(), localRoot)) + + got, err := os.ReadFile(checkpoints[0]) + require.NoError(t, err) + assert.Equal(t, valid, got, "corrupt local checkpoint should be re-fetched from the peer") +} + +func TestHTTPTransportPostArtifactSetsPeerOrigin(t *testing.T) { + var gotOrigin string + peer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotOrigin = r.Header.Get("Origin") + w.WriteHeader(http.StatusCreated) + })) + defer peer.Close() + + tr, err := newHTTPTransport(peer.URL+"/api/v1/artifacts", "", false) + require.NoError(t, err) + + err = tr.postArtifact(context.Background(), "peer-a1b2c3", KindSegments, strings64("a"), []byte("artifact")) + require.NoError(t, err) + + assert.Equal(t, peer.URL, gotOrigin) +} + +func TestHTTPTransportRejectsRedirectedArtifactPost(t *testing.T) { + type requestCapture struct { + authorization string + body []byte + readErr error + } + + var destinationReached atomic.Bool + destination := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + destinationReached.Store(true) + w.WriteHeader(http.StatusCreated) + })) + t.Cleanup(destination.Close) + + captured := make(chan requestCapture, 1) + source := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body, err := io.ReadAll(r.Body) + captured <- requestCapture{ + authorization: r.Header.Get("Authorization"), + body: body, + readErr: err, + } + http.Redirect(w, r, destination.URL, http.StatusTemporaryRedirect) + })) + t.Cleanup(source.Close) + + tr, err := newHTTPTransport(source.URL, "peer-secret", false) + require.NoError(t, err) + tr.client.Transport = source.Client().Transport + + err = tr.postArtifact(context.Background(), "peer-a1b2c3", KindSegments, strings64("a"), []byte("artifact-secret")) + require.Error(t, err) + assert.ErrorIs(t, err, errHTTPPeer) + var got requestCapture + select { + case got = <-captured: + case <-time.After(time.Second): + require.FailNow(t, "redirect source was not reached", "timed out waiting for the artifact POST") + } + require.NoError(t, got.readErr) + assert.Equal(t, "Bearer peer-secret", got.authorization) + assert.Equal(t, []byte("artifact-secret"), got.body) + assert.False(t, destinationReached.Load()) +} diff --git a/internal/artifact/transport_s3.go b/internal/artifact/transport_s3.go new file mode 100644 index 000000000..dc7f3ae41 --- /dev/null +++ b/internal/artifact/transport_s3.go @@ -0,0 +1,661 @@ +package artifact + +import ( + "bytes" + "context" + "crypto/hmac" + "crypto/sha256" + "encoding/hex" + "encoding/xml" + "errors" + "fmt" + "io" + "log" + "net" + "net/http" + "net/url" + "os" + "sort" + "strconv" + "strings" + "time" +) + +// emptyPayloadSHA256 is the hex SHA-256 of the empty string, the payload hash +// SigV4 uses for requests without a body (GET and ListObjectsV2). +const emptyPayloadSHA256 = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + +var errObjectStore = errors.New("artifact object store request failed") + +// IsObjectTarget reports whether target is an S3-compatible object-store URL +// rather than a local folder or HTTP peer target. +func IsObjectTarget(target string) bool { + return strings.HasPrefix(target, "s3://") +} + +// ObjectStoreOptions configures the S3-compatible object-store transport. It +// carries static credentials plus the addressing details needed to talk to AWS +// S3 or a compatible service such as MinIO or Backblaze B2. +type ObjectStoreOptions struct { + // Endpoint is the service base URL. Empty means real AWS S3, addressed in + // virtual-host style at https://s3..amazonaws.com. A non-empty value + // (host or full URL) forces path-style addressing. + Endpoint string + Region string + AccessKeyID string + SecretAccessKey string + SessionToken string + // AllowInsecureEndpoint permits a non-loopback custom HTTP endpoint. + AllowInsecureEndpoint bool + // PathStyle forces path-style addressing (//) instead of + // virtual-host addressing (.host/). It is implied by a custom + // Endpoint. + PathStyle bool +} + +// ObjectStoreOptionsFromEnv reads object-store credentials and addressing from +// the environment, preferring agentsview-specific variables over the standard +// AWS ones for region and endpoint. Region defaults to us-east-1, and a custom +// endpoint forces path-style addressing because MinIO, B2, and local test +// servers do not support virtual-host buckets. +func ObjectStoreOptionsFromEnv() ObjectStoreOptions { + region := os.Getenv("AGENTSVIEW_S3_REGION") + if region == "" { + region = os.Getenv("AWS_REGION") + } + if region == "" { + region = "us-east-1" + } + endpoint := os.Getenv("AGENTSVIEW_S3_ENDPOINT") + pathStyle := os.Getenv("AGENTSVIEW_S3_PATH_STYLE") == "true" + allowInsecureEndpoint := false + switch strings.ToLower(strings.TrimSpace(os.Getenv("AGENTSVIEW_ALLOW_INSECURE_S3_ENDPOINT"))) { + case "1", "true", "yes": + allowInsecureEndpoint = true + } + if endpoint != "" { + pathStyle = true + } + return ObjectStoreOptions{ + Endpoint: endpoint, + Region: region, + AccessKeyID: os.Getenv("AWS_ACCESS_KEY_ID"), + SecretAccessKey: os.Getenv("AWS_SECRET_ACCESS_KEY"), + SessionToken: os.Getenv("AWS_SESSION_TOKEN"), + AllowInsecureEndpoint: allowInsecureEndpoint, + PathStyle: pathStyle, + } +} + +// s3Transport exchanges artifacts with an S3-compatible bucket using the same +// set-union semantics as the folder and HTTP peer transports: list every object +// under the prefix once, fetch remote-only artifacts into the local store, and +// upload local-only artifacts to the bucket. Content addressing makes "does the +// bucket have X" a name-set comparison, so exchange is stateless and idempotent. +type s3Transport struct { + bucket string + prefix string // no leading or trailing slash; may be empty + endpoint *url.URL + pathStyle bool + opts ObjectStoreOptions + client *http.Client +} + +func newObjectTransport(target string, opts ObjectStoreOptions) (*s3Transport, error) { + if !IsObjectTarget(target) { + return nil, fmt.Errorf("object store target must be s3://: %q", target) + } + rest := strings.TrimPrefix(target, "s3://") + bucket, prefix, _ := strings.Cut(rest, "/") + prefix = strings.Trim(prefix, "/") + if bucket == "" { + return nil, fmt.Errorf("object store target is missing a bucket: %q", target) + } + if opts.AccessKeyID == "" || opts.SecretAccessKey == "" { + return nil, errors.New("object store target requires AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY") + } + region := opts.Region + if region == "" { + region = "us-east-1" + opts.Region = region + } + var endpoint *url.URL + if opts.Endpoint == "" { + endpoint = &url.URL{Scheme: "https", Host: "s3." + region + ".amazonaws.com"} + } else { + raw := opts.Endpoint + if !strings.Contains(raw, "://") { + raw = "https://" + raw + } + u, err := url.Parse(raw) + if err != nil { + return nil, fmt.Errorf("parsing object store endpoint %q: %w", opts.Endpoint, err) + } + if u.Host == "" { + return nil, fmt.Errorf("object store endpoint is missing a host: %q", opts.Endpoint) + } + scheme := strings.ToLower(u.Scheme) + switch scheme { + case "https": + case "http": + if !opts.AllowInsecureEndpoint && !isLoopbackEndpointHost(u.Hostname()) { + return nil, fmt.Errorf("insecure S3 endpoint %q requires HTTPS or AGENTSVIEW_ALLOW_INSECURE_S3_ENDPOINT", opts.Endpoint) + } + default: + return nil, fmt.Errorf("object store endpoint %q uses unsupported scheme %q; only http and https are allowed", opts.Endpoint, u.Scheme) + } + endpoint = &url.URL{Scheme: scheme, Host: u.Host} + opts.PathStyle = true + } + return &s3Transport{ + bucket: bucket, + prefix: prefix, + endpoint: endpoint, + pathStyle: opts.PathStyle, + opts: opts, + client: &http.Client{ + Timeout: httpTransportTimeout, + CheckRedirect: func(_ *http.Request, _ []*http.Request) error { + return http.ErrUseLastResponse + }, + }, + }, nil +} + +func isLoopbackEndpointHost(host string) bool { + if strings.EqualFold(host, "localhost") { + return true + } + if address, _, ok := strings.Cut(host, "%"); ok { + host = address + } + ip := net.ParseIP(host) + return ip != nil && ip.IsLoopback() +} + +// Prepare verifies the bucket is reachable and the credentials are accepted +// before the local export runs, so a wrong endpoint or key fails fast. +func (t *s3Transport) Prepare(ctx context.Context, _ string) error { + if _, err := t.listPage(ctx, "", 1); err != nil { + return fmt.Errorf("connecting to object store: %w", err) + } + return nil +} + +func (t *s3Transport) Exchange(ctx context.Context, localRoot string) error { + remote, err := t.listRemote(ctx) + if err != nil { + return fmt.Errorf("listing object store artifacts: %w", err) + } + if err := t.pull(ctx, localRoot, remote); err != nil { + return fmt.Errorf("fetching artifacts from object store: %w", err) + } + if err := t.push(ctx, localRoot, remote); err != nil { + return fmt.Errorf("publishing artifacts to object store: %w", err) + } + return nil +} + +// pull fetches every artifact the bucket holds that is missing locally. +func (t *s3Transport) pull(ctx context.Context, localRoot string, remote map[string]OriginArtifactIndex) error { + origins := make([]string, 0, len(remote)) + for origin := range remote { + origins = append(origins, origin) + } + sort.Strings(origins) + for _, origin := range origins { + local, err := ListArtifacts(localRoot, origin) + if err != nil { + return err + } + quarantined, err := reconcileCommonCheckpoint(localRoot, origin, local, remote[origin], + func(name string) ([]byte, error) { + return t.getObject(ctx, t.objectKey(origin, KindCheckpoints, name)) + }) + if err != nil { + return err + } + if quarantined { + if local, err = ListArtifacts(localRoot, origin); err != nil { + return err + } + } + for _, item := range missingItems(remote[origin], local) { + data, err := t.getObject(ctx, t.objectKey(origin, item.kind, item.name)) + if err != nil { + return err + } + if _, err := WriteArtifact(localRoot, origin, item.kind, item.name, data); err != nil { + if errors.Is(err, ErrArtifactInvalid) { + // Delete the corrupt object so its name stops masking a + // valid copy: push only uploads names the bucket is + // missing, so a corrupt object left in place would never + // be re-uploaded by a peer that still holds good bytes. + // Content validation tolerates future format versions, so + // only deterministic corruption reaches this branch. A + // failed delete keeps the plain skip. + log.Printf("artifact: detected corrupt artifact %s/%s/%s in object store: %v", + origin, item.kind, item.name, err) + if derr := t.deleteObject(ctx, t.objectKey(origin, item.kind, item.name)); derr != nil { + log.Printf("artifact: deleting corrupt object %s/%s/%s: %v", + origin, item.kind, item.name, derr) + } + continue + } + return err + } + } + } + return nil +} + +// push uploads every local artifact the bucket is missing. +func (t *s3Transport) push(ctx context.Context, localRoot string, remote map[string]OriginArtifactIndex) error { + origins, err := ListOrigins(localRoot) + if err != nil { + return err + } + for _, origin := range origins { + local, err := ListArtifacts(localRoot, origin) + if err != nil { + return err + } + for _, item := range missingItems(local, remote[origin]) { + art, err := ReadArtifact(localRoot, origin, item.kind, item.name) + if err != nil { + if skipUnpublishableArtifact(localRoot, origin, item, err) { + continue + } + return err + } + if err := t.putObject(ctx, t.objectKey(origin, item.kind, item.name), art.Data); err != nil { + return err + } + } + } + return nil +} + +// listRemote walks the whole prefix once via paginated ListObjectsV2 and groups +// the valid artifact keys into a per-origin index, the enumeration both pull and +// push compare against. +func (t *s3Transport) listRemote(ctx context.Context) (map[string]OriginArtifactIndex, error) { + indexes := map[string]*OriginArtifactIndex{} + token := "" + for { + result, err := t.listPage(ctx, token, 0) + if err != nil { + return nil, err + } + for _, c := range result.Contents { + t.indexKey(indexes, c.Key) + } + if !result.IsTruncated || result.NextContinuationToken == "" { + break + } + token = result.NextContinuationToken + } + out := make(map[string]OriginArtifactIndex, len(indexes)) + for origin, idx := range indexes { + out[origin] = *idx + } + return out, nil +} + +// indexKey parses one object key into origin/kind/name and records it in the +// per-origin index, skipping keys that do not have exactly three path segments +// under the prefix or whose origin, kind, and name fail the same validation +// ListArtifacts applies locally. +func (t *s3Transport) indexKey(indexes map[string]*OriginArtifactIndex, key string) { + rel := key + if t.prefix != "" { + p := t.prefix + "/" + if !strings.HasPrefix(key, p) { + return + } + rel = key[len(p):] + } + parts := strings.Split(rel, "/") + if len(parts) != 3 { + return + } + origin, kind, name := parts[0], parts[1], parts[2] + if validateOriginID(origin) != nil { + return + } + if !validArtifactKindName(kind, name) { + return + } + idx := indexes[origin] + if idx == nil { + idx = &OriginArtifactIndex{Origin: origin} + indexes[origin] = idx + } + switch kind { + case KindCheckpoints: + idx.Checkpoints = append(idx.Checkpoints, name) + case KindManifests: + idx.Manifests = append(idx.Manifests, name) + case KindSegments: + idx.Segments = append(idx.Segments, name) + case KindMeta: + idx.Meta = append(idx.Meta, name) + case KindRaw: + idx.Raw = append(idx.Raw, name) + } +} + +// objectKey builds the bucket key for one artifact, joining the optional prefix +// with origin/kind/name. +func (t *s3Transport) objectKey(origin, kind, name string) string { + parts := make([]string, 0, 4) + if t.prefix != "" { + parts = append(parts, t.prefix) + } + parts = append(parts, origin, kind, name) + return strings.Join(parts, "/") +} + +// listBucketResult is the subset of the ListObjectsV2 response we consume. +type listBucketResult struct { + XMLName xml.Name `xml:"ListBucketResult"` + IsTruncated bool `xml:"IsTruncated"` + NextContinuationToken string `xml:"NextContinuationToken"` + Contents []struct { + Key string `xml:"Key"` + } `xml:"Contents"` +} + +// listPage performs one ListObjectsV2 request. A maxKeys of 0 leaves the limit +// to the server default; the reachability check in Prepare uses 1. +func (t *s3Transport) listPage(ctx context.Context, token string, maxKeys int) (listBucketResult, error) { + q := url.Values{} + q.Set("list-type", "2") + if t.prefix != "" { + q.Set("prefix", t.prefix+"/") + } + if token != "" { + q.Set("continuation-token", token) + } + if maxKeys > 0 { + q.Set("max-keys", strconv.Itoa(maxKeys)) + } + req, err := t.newRequest(ctx, http.MethodGet, "", q, nil) + if err != nil { + return listBucketResult{}, err + } + resp, err := t.client.Do(req) + if err != nil { + return listBucketResult{}, err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return listBucketResult{}, t.statusError(resp) + } + body, err := io.ReadAll(resp.Body) + if err != nil { + return listBucketResult{}, err + } + var result listBucketResult + if err := xml.Unmarshal(body, &result); err != nil { + return listBucketResult{}, fmt.Errorf("decoding object store listing: %w", err) + } + return result, nil +} + +func (t *s3Transport) getObject(ctx context.Context, key string) ([]byte, error) { + req, err := t.newRequest(ctx, http.MethodGet, key, nil, nil) + if err != nil { + return nil, err + } + resp, err := t.client.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return nil, t.statusError(resp) + } + return io.ReadAll(resp.Body) +} + +func (t *s3Transport) putObject(ctx context.Context, key string, data []byte) error { + req, err := t.newRequest(ctx, http.MethodPut, key, nil, data) + if err != nil { + return err + } + // Write-once: refuse to overwrite an existing object. Artifacts are + // immutable and content-addressed, so a collision means either a harmless + // re-upload of identical bytes or a genuine origin-ID conflict that must be + // surfaced, not silently merged. + req.Header.Set("If-None-Match", "*") + resp, err := t.client.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + switch resp.StatusCode { + case http.StatusOK, http.StatusCreated: + _, _ = io.Copy(io.Discard, resp.Body) + return nil + case http.StatusPreconditionFailed: + _, _ = io.Copy(io.Discard, resp.Body) + return t.reconcileExistingObject(ctx, key, data) + default: + return t.statusError(resp) + } +} + +// deleteObject removes one object; a missing key counts as success. +func (t *s3Transport) deleteObject(ctx context.Context, key string) error { + if t.endpoint.Scheme != "https" { + return fmt.Errorf("refusing to delete object through insecure S3 endpoint %q", t.endpoint.String()) + } + req, err := t.newRequest(ctx, http.MethodDelete, key, nil, nil) + if err != nil { + return err + } + resp, err := t.client.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + switch resp.StatusCode { + case http.StatusOK, http.StatusNoContent, http.StatusNotFound: + _, _ = io.Copy(io.Discard, resp.Body) + return nil + default: + return t.statusError(resp) + } +} + +// reconcileExistingObject handles a write-once collision: identical content is +// an accepted duplicate, differing content is a conflict. +func (t *s3Transport) reconcileExistingObject(ctx context.Context, key string, data []byte) error { + existing, err := t.getObject(ctx, key) + if err != nil { + return fmt.Errorf("comparing conflicting object %s: %w", key, err) + } + if bytes.Equal(existing, data) { + return nil + } + return fmt.Errorf("%w: object %s already exists with different content", errObjectStore, key) +} + +// newRequest builds and signs one object-store request. An empty key targets the +// bucket itself (used for listing). A nil body sends no payload and signs the +// empty-payload hash; a non-nil body signs its real SHA-256. +func (t *s3Transport) newRequest(ctx context.Context, method, key string, query url.Values, body []byte) (*http.Request, error) { + host := t.endpoint.Host + var rawPath string + if t.pathStyle { + rawPath = "/" + t.bucket + if key != "" { + rawPath += "/" + key + } + } else { + host = t.bucket + "." + t.endpoint.Host + rawPath = "/" + key + } + u := &url.URL{ + Scheme: t.endpoint.Scheme, + Host: host, + Path: rawPath, + RawPath: s3EncodePath(rawPath), + RawQuery: canonicalQueryString(query), + } + var reader io.Reader + payloadHash := emptyPayloadSHA256 + if body != nil { + reader = bytes.NewReader(body) + payloadHash = hashHex(body) + } + req, err := http.NewRequestWithContext(ctx, method, u.String(), reader) + if err != nil { + return nil, err + } + if body != nil { + req.Header.Set("Content-Type", "application/octet-stream") + } + signRequest(req, payloadHash, t.opts, time.Now()) + return req, nil +} + +func (t *s3Transport) statusError(resp *http.Response) error { + body, _ := io.ReadAll(io.LimitReader(resp.Body, httpTransportMaxErrLen)) + msg := strings.TrimSpace(string(body)) + if msg == "" { + return fmt.Errorf("%w: %s", errObjectStore, resp.Status) + } + return fmt.Errorf("%w: %s: %s", errObjectStore, resp.Status, msg) +} + +// signRequest applies AWS Signature Version 4 for the "s3" service to req, +// setting the x-amz-date, x-amz-content-sha256, optional x-amz-security-token, +// and Authorization headers. payloadSHA256Hex must be the hex SHA-256 of the +// request body (the empty-payload hash for bodyless requests). +func signRequest(req *http.Request, payloadSHA256Hex string, opts ObjectStoreOptions, now time.Time) { + now = now.UTC() + amzDate := now.Format("20060102T150405Z") + dateStamp := now.Format("20060102") + + req.Header.Set("X-Amz-Date", amzDate) + req.Header.Set("X-Amz-Content-Sha256", payloadSHA256Hex) + if opts.SessionToken != "" { + req.Header.Set("X-Amz-Security-Token", opts.SessionToken) + } + + // Sign the host header plus every x-amz-* header we set. They are already in + // lowercase canonical form, so we sort by name to build the canonical block. + type header struct{ name, value string } + headers := []header{ + {"host", req.URL.Host}, + {"x-amz-content-sha256", payloadSHA256Hex}, + {"x-amz-date", amzDate}, + } + if opts.SessionToken != "" { + headers = append(headers, header{"x-amz-security-token", opts.SessionToken}) + } + sort.Slice(headers, func(i, j int) bool { return headers[i].name < headers[j].name }) + + var canonicalHeaders strings.Builder + signedNames := make([]string, 0, len(headers)) + for _, h := range headers { + canonicalHeaders.WriteString(h.name) + canonicalHeaders.WriteByte(':') + canonicalHeaders.WriteString(strings.TrimSpace(h.value)) + canonicalHeaders.WriteByte('\n') + signedNames = append(signedNames, h.name) + } + signedHeaders := strings.Join(signedNames, ";") + + canonicalRequest := strings.Join([]string{ + req.Method, + req.URL.EscapedPath(), + req.URL.RawQuery, + canonicalHeaders.String(), + signedHeaders, + payloadSHA256Hex, + }, "\n") + + scope := dateStamp + "/" + opts.Region + "/s3/aws4_request" + stringToSign := strings.Join([]string{ + "AWS4-HMAC-SHA256", + amzDate, + scope, + hashHex([]byte(canonicalRequest)), + }, "\n") + + signingKey := sigV4SigningKey(opts.SecretAccessKey, dateStamp, opts.Region, "s3") + signature := hex.EncodeToString(hmacSHA256(signingKey, stringToSign)) + + req.Header.Set("Authorization", fmt.Sprintf( + "AWS4-HMAC-SHA256 Credential=%s/%s, SignedHeaders=%s, Signature=%s", + opts.AccessKeyID, scope, signedHeaders, signature, + )) +} + +func sigV4SigningKey(secret, dateStamp, region, service string) []byte { + kDate := hmacSHA256([]byte("AWS4"+secret), dateStamp) + kRegion := hmacSHA256(kDate, region) + kService := hmacSHA256(kRegion, service) + return hmacSHA256(kService, "aws4_request") +} + +func hmacSHA256(key []byte, data string) []byte { + h := hmac.New(sha256.New, key) + h.Write([]byte(data)) + return h.Sum(nil) +} + +// canonicalQueryString renders query parameters in the sorted, S3-encoded form +// the SigV4 canonical request requires. +func canonicalQueryString(q url.Values) string { + if len(q) == 0 { + return "" + } + keys := make([]string, 0, len(q)) + for k := range q { + keys = append(keys, k) + } + sort.Strings(keys) + parts := make([]string, 0, len(q)) + for _, k := range keys { + values := append([]string(nil), q[k]...) + sort.Strings(values) + for _, v := range values { + parts = append(parts, s3URIEncode(k)+"="+s3URIEncode(v)) + } + } + return strings.Join(parts, "&") +} + +// s3EncodePath URI-encodes a path, encoding each segment per S3 rules while +// preserving the '/' separators. +func s3EncodePath(path string) string { + segments := strings.Split(path, "/") + for i, seg := range segments { + segments[i] = s3URIEncode(seg) + } + return strings.Join(segments, "/") +} + +// s3URIEncode percent-encodes s following the AWS SigV4 rules: only the +// unreserved characters A-Z a-z 0-9 - _ . ~ pass through unencoded, and every +// other byte is encoded as uppercase %XX (notably space becomes %20, not '+'). +func s3URIEncode(s string) string { + var b strings.Builder + for i := 0; i < len(s); i++ { + c := s[i] + if (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || (c >= '0' && c <= '9') || + c == '-' || c == '_' || c == '.' || c == '~' { + b.WriteByte(c) + continue + } + b.WriteByte('%') + const hexDigits = "0123456789ABCDEF" + b.WriteByte(hexDigits[c>>4]) + b.WriteByte(hexDigits[c&0xf]) + } + return b.String() +} diff --git a/internal/artifact/transport_s3_miniotest_test.go b/internal/artifact/transport_s3_miniotest_test.go new file mode 100644 index 000000000..f21376f76 --- /dev/null +++ b/internal/artifact/transport_s3_miniotest_test.go @@ -0,0 +1,151 @@ +//go:build miniotest + +package artifact + +import ( + "context" + "fmt" + "net/http" + "os" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/testcontainers/testcontainers-go" + "github.com/testcontainers/testcontainers-go/wait" +) + +// TestS3TransportMinIORoundTrip exercises the object-store transport against a +// real MinIO server in a container, validating that the hand-rolled SigV4 +// signing is accepted by a genuine S3 implementation end to end: create bucket, +// push a producer's artifacts, list them, and pull them into a fresh store. +func TestS3TransportMinIORoundTrip(t *testing.T) { + ctx := context.Background() + endpoint, accessKey, secretKey := startMinIO(t, ctx) + + tr, err := newObjectTransport("s3://agentsview/sync", ObjectStoreOptions{ + Endpoint: endpoint, + Region: "us-east-1", + AccessKeyID: accessKey, + SecretAccessKey: secretKey, + AllowInsecureEndpoint: true, + PathStyle: true, + }) + require.NoError(t, err) + requireCreateBucket(t, ctx, tr) + + // Producer store: one exported session plus a star metadata event. + origin := "laptop-a1b2c3" + prod := testDB(t) + seedSession(t, prod, "sess-1", "alpha") + prodDir := t.TempDir() + prodRoot := filepath.Join(prodDir, "artifacts") + _, err = Export(ctx, prod, prodRoot, origin) + require.NoError(t, err) + recorder := NewMetadataRecorder(prod, MetadataRecorderOptions{DataDir: prodDir, Origin: origin}) + _, err = recorder.Append(ctx, MetadataEventInput{SessionID: "sess-1", Op: MetadataOpStar}) + require.NoError(t, err) + + want, err := ListArtifacts(prodRoot, origin) + require.NoError(t, err) + require.NotEmpty(t, want.Manifests) + require.NotEmpty(t, want.Meta) + + // Push to MinIO, then confirm the bucket lists exactly the producer's set. + require.NoError(t, tr.Prepare(context.Background(), prodRoot)) + require.NoError(t, tr.Exchange(ctx, prodRoot)) + + remote, err := tr.listRemote(ctx) + require.NoError(t, err) + require.Contains(t, remote, origin) + assertIndexEqual(t, want, remote[origin]) + + // A fresh consumer store pulls every artifact back. + consDir := t.TempDir() + consRoot := filepath.Join(consDir, "artifacts") + require.NoError(t, os.MkdirAll(consRoot, 0o755)) + require.NoError(t, tr.Exchange(ctx, consRoot)) + + got, err := ListArtifacts(consRoot, origin) + require.NoError(t, err) + assertIndexEqual(t, want, got) + + // Re-running is a no-op set-union: nothing new to fetch or upload. + require.NoError(t, tr.Exchange(ctx, consRoot)) + got, err = ListArtifacts(consRoot, origin) + require.NoError(t, err) + assertIndexEqual(t, want, got) +} + +func assertIndexEqual(t *testing.T, want, got OriginArtifactIndex) { + t.Helper() + assert.ElementsMatch(t, want.Checkpoints, got.Checkpoints, "checkpoints") + assert.ElementsMatch(t, want.Manifests, got.Manifests, "manifests") + assert.ElementsMatch(t, want.Segments, got.Segments, "segments") + assert.ElementsMatch(t, want.Meta, got.Meta, "meta") + assert.ElementsMatch(t, want.Raw, got.Raw, "raw") +} + +// requireCreateBucket issues a signed CreateBucket request through the transport, +// reusing the same SigV4 path under test. An already-owned bucket is fine. +func requireCreateBucket(t *testing.T, ctx context.Context, tr *s3Transport) { + t.Helper() + var lastStatus string + for attempt := 0; attempt < 10; attempt++ { + req, err := tr.newRequest(ctx, http.MethodPut, "", nil, nil) + require.NoError(t, err) + resp, err := tr.client.Do(req) + require.NoError(t, err) + lastStatus = resp.Status + status := resp.StatusCode + resp.Body.Close() + // 200 = created, 409 = already owned. 503 can still occur briefly while + // MinIO finishes initializing; retry those. + if status == http.StatusOK || status == http.StatusConflict { + return + } + if status != http.StatusServiceUnavailable { + break + } + time.Sleep(500 * time.Millisecond) + } + require.FailNowf(t, "create bucket failed", "last status: %s", lastStatus) +} + +func startMinIO(t *testing.T, ctx context.Context) (endpoint, accessKey, secretKey string) { + t.Helper() + const user, pass = "minioadmin", "minioadmin" + container, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{ + ContainerRequest: testcontainers.ContainerRequest{ + Image: "minio/minio:RELEASE.2025-07-23T15-54-02Z", + ExposedPorts: []string{"9000/tcp"}, + Env: map[string]string{ + "MINIO_ROOT_USER": user, + "MINIO_ROOT_PASSWORD": pass, + }, + Cmd: []string{"server", "/data"}, + // "ready" (not "live") signals MinIO can actually serve S3 requests; + // "live" only means the process is up and races CreateBucket to 503. + WaitingFor: wait.ForHTTP("/minio/health/ready"). + WithPort("9000/tcp"). + WithStartupTimeout(2 * time.Minute), + }, + Started: true, + }) + if err != nil { + t.Skipf("could not start MinIO container (is Docker available?): %v", err) + } + t.Cleanup(func() { + stopCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + _ = container.Terminate(stopCtx) + }) + + host, err := container.Host(ctx) + require.NoError(t, err) + port, err := container.MappedPort(ctx, "9000/tcp") + require.NoError(t, err) + return fmt.Sprintf("http://%s:%s", host, port.Port()), user, pass +} diff --git a/internal/artifact/transport_s3_test.go b/internal/artifact/transport_s3_test.go new file mode 100644 index 000000000..474cacaa0 --- /dev/null +++ b/internal/artifact/transport_s3_test.go @@ -0,0 +1,660 @@ +package artifact + +import ( + "context" + "encoding/xml" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "sort" + "strconv" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.kenn.io/agentsview/internal/db" +) + +// mockS3 is an in-memory, path-style S3-compatible server backing a single +// bucket. It implements just enough of ListObjectsV2, GetObject, PutObject, and +// DeleteObject to exercise the object-store transport, and verifies that +// requests arrive signed (Authorization plus x-amz-date) without re-validating +// the signature. +type mockS3 struct { + t *testing.T + bucket string + pageSize int + + mu sync.Mutex + objects map[string][]byte + deletes int +} + +func newMockS3(t *testing.T, bucket string, pageSize int) *mockS3 { + return &mockS3{ + t: t, + bucket: bucket, + pageSize: pageSize, + objects: map[string][]byte{}, + } +} + +func (m *mockS3) put(key string, data []byte) { + m.mu.Lock() + defer m.mu.Unlock() + m.objects[key] = append([]byte(nil), data...) +} + +func (m *mockS3) has(key string) bool { + m.mu.Lock() + defer m.mu.Unlock() + _, ok := m.objects[key] + return ok +} + +func (m *mockS3) deleteCount() int { + m.mu.Lock() + defer m.mu.Unlock() + return m.deletes +} + +func (m *mockS3) ServeHTTP(w http.ResponseWriter, r *http.Request) { + assert.True(m.t, strings.HasPrefix(r.Header.Get("Authorization"), "AWS4-HMAC-SHA256"), + "request must carry a SigV4 Authorization header") + assert.NotEmpty(m.t, r.Header.Get("X-Amz-Date"), "request must carry an x-amz-date header") + + bucketPath := "/" + m.bucket + if r.Method == http.MethodGet && r.URL.Path == bucketPath && r.URL.Query().Get("list-type") == "2" { + m.list(w, r) + return + } + if !strings.HasPrefix(r.URL.Path, bucketPath+"/") { + http.Error(w, "not found", http.StatusNotFound) + return + } + key := strings.TrimPrefix(r.URL.Path, bucketPath+"/") + switch r.Method { + case http.MethodGet: + m.mu.Lock() + data, ok := m.objects[key] + m.mu.Unlock() + if !ok { + http.Error(w, "no such key", http.StatusNotFound) + return + } + _, _ = w.Write(data) + case http.MethodPut: + body := make([]byte, 0) + buf := make([]byte, 4096) + for { + n, err := r.Body.Read(buf) + body = append(body, buf[:n]...) + if err != nil { + break + } + } + // Honor the write-once conditional: reject when the key already exists. + if r.Header.Get("If-None-Match") == "*" && m.has(key) { + http.Error(w, "precondition failed", http.StatusPreconditionFailed) + return + } + m.put(key, body) + w.WriteHeader(http.StatusOK) + case http.MethodDelete: + m.mu.Lock() + m.deletes++ + delete(m.objects, key) + m.mu.Unlock() + w.WriteHeader(http.StatusNoContent) + default: + http.Error(w, "method not allowed", http.StatusMethodNotAllowed) + } +} + +func (m *mockS3) list(w http.ResponseWriter, r *http.Request) { + prefix := r.URL.Query().Get("prefix") + token := r.URL.Query().Get("continuation-token") + + m.mu.Lock() + keys := make([]string, 0, len(m.objects)) + for k := range m.objects { + if strings.HasPrefix(k, prefix) { + keys = append(keys, k) + } + } + m.mu.Unlock() + sort.Strings(keys) + + start := 0 + if token != "" { + start, _ = strconv.Atoi(token) + } + pageSize := m.pageSize + if pageSize <= 0 { + pageSize = 1000 + } + end := start + pageSize + truncated := end < len(keys) + if end > len(keys) { + end = len(keys) + } + + type contentsXML struct { + Key string `xml:"Key"` + } + type resultXML struct { + XMLName xml.Name `xml:"ListBucketResult"` + IsTruncated bool `xml:"IsTruncated"` + Contents []contentsXML `xml:"Contents"` + NextContinuationToken string `xml:"NextContinuationToken,omitempty"` + } + out := resultXML{IsTruncated: truncated} + for _, k := range keys[start:end] { + out.Contents = append(out.Contents, contentsXML{Key: k}) + } + if truncated { + out.NextContinuationToken = strconv.Itoa(end) + } + w.Header().Set("Content-Type", "application/xml") + require.NoError(m.t, xml.NewEncoder(w).Encode(out)) +} + +func testObjectOptions(endpoint string) ObjectStoreOptions { + return ObjectStoreOptions{ + Endpoint: endpoint, + Region: "us-east-1", + AccessKeyID: "AKIDEXAMPLE", + SecretAccessKey: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", + PathStyle: true, + } +} + +func TestS3TransportPrepareHonorsCanceledSync(t *testing.T) { + var requests atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + requests.Add(1) + _, _ = w.Write([]byte("")) + })) + t.Cleanup(server.Close) + transport, err := newObjectTransport("s3://bucket/arts", testObjectOptions(server.URL)) + require.NoError(t, err) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + _, err = syncWithTransport(ctx, testDB(t), SyncOptions{ + DataDir: t.TempDir(), + Target: "s3://bucket/arts", + Origin: "laptop-a1b2c3", + }, transport) + + require.ErrorIs(t, err, context.Canceled) + assert.Zero(t, requests.Load(), "canceled preparation must not contact the object store") +} + +// exportStore exports one origin's sessions into a fresh artifact store and +// returns the artifact root. +func exportStore(t *testing.T, origin string, seed func(*db.DB)) string { + t.Helper() + database := testDB(t) + seed(database) + root := filepath.Join(t.TempDir(), "artifacts") + _, err := Export(context.Background(), database, root, origin) + require.NoError(t, err) + return root +} + +func TestS3TransportPushRoundTrip(t *testing.T) { + origin := "laptop-a1b2c3" + database := testDB(t) + seedSession(t, database, "sess-1", "alpha") + + dataDir := t.TempDir() + localRoot := filepath.Join(dataDir, "artifacts") + _, err := Export(context.Background(), database, localRoot, origin) + require.NoError(t, err) + + // Append a metadata event so a meta artifact is part of the push. + rec := NewMetadataRecorder(database, MetadataRecorderOptions{ + DataDir: dataDir, + Origin: origin, + Now: func() time.Time { return fixedHLCTime() }, + }) + _, err = database.StarSession("sess-1") + require.NoError(t, err) + _, err = rec.Append(context.Background(), MetadataEventInput{ + SessionID: "sess-1", + Op: MetadataOpStar, + }) + require.NoError(t, err) + + mock := newMockS3(t, "bucket", 0) + srv := httptest.NewServer(mock) + t.Cleanup(srv.Close) + + tr, err := newObjectTransport("s3://bucket/arts", testObjectOptions(srv.URL)) + require.NoError(t, err) + require.NoError(t, tr.Prepare(context.Background(), localRoot)) + require.NoError(t, tr.Exchange(context.Background(), localRoot)) + + idx, err := ListArtifacts(localRoot, origin) + require.NoError(t, err) + items := indexItems(idx) + require.NotEmpty(t, items) + assert.NotEmpty(t, idx.Meta, "the star event should have produced a meta artifact") + for _, item := range items { + key := "arts/" + origin + "/" + item.kind + "/" + item.name + assert.True(t, mock.has(key), "expected object %q in bucket", key) + } +} + +func TestS3TransportPullRoundTrip(t *testing.T) { + origin := "desktop-d4e5f6" + + // Produce a populated store for the origin and upload it into the bucket so + // the transport must pull it down into an empty local store. + remoteRoot := exportStore(t, origin, func(database *db.DB) { + seedSession(t, database, "sess-7", "beta") + seedSession(t, database, "sess-8", "beta") + }) + remoteIdx, err := ListArtifacts(remoteRoot, origin) + require.NoError(t, err) + uploaded := indexItems(remoteIdx) + require.NotEmpty(t, uploaded) + + // Use a small page size and more than one object to exercise the + // continuation-token pagination path. + mock := newMockS3(t, "bucket", 2) + for _, item := range uploaded { + art, err := ReadArtifact(remoteRoot, origin, item.kind, item.name) + require.NoError(t, err) + mock.put("arts/"+origin+"/"+item.kind+"/"+item.name, art.Data) + } + require.Greater(t, len(uploaded), 2, "need multiple pages to test pagination") + + srv := httptest.NewServer(mock) + t.Cleanup(srv.Close) + + localRoot := filepath.Join(t.TempDir(), "artifacts") + tr, err := newObjectTransport("s3://bucket/arts", testObjectOptions(srv.URL)) + require.NoError(t, err) + require.NoError(t, tr.Exchange(context.Background(), localRoot)) + + gotIdx, err := ListArtifacts(localRoot, origin) + require.NoError(t, err) + assert.ElementsMatch(t, indexItems(remoteIdx), indexItems(gotIdx)) +} + +func TestS3TransportPullRetainsCorruptRemoteArtifactOverHTTP(t *testing.T) { + origin := "desktop-d4e5f6" + remoteRoot := exportStore(t, origin, func(database *db.DB) { + seedSession(t, database, "sess-7", "beta") + }) + remoteIdx, err := ListArtifacts(remoteRoot, origin) + require.NoError(t, err) + uploaded := indexItems(remoteIdx) + mock := newMockS3(t, "bucket", 0) + for _, item := range uploaded { + art, err := ReadArtifact(remoteRoot, origin, item.kind, item.name) + require.NoError(t, err) + mock.put("arts/"+origin+"/"+item.kind+"/"+item.name, art.Data) + } + corruptName := hashHex([]byte("corrupt")) + segmentExtension + mock.put("arts/"+origin+"/segments/"+corruptName, []byte("garbage")) + + srv := httptest.NewServer(mock) + t.Cleanup(srv.Close) + localRoot := filepath.Join(t.TempDir(), "artifacts") + tr, err := newObjectTransport("s3://bucket/arts", testObjectOptions(srv.URL)) + require.NoError(t, err) + require.NoError(t, tr.Exchange(context.Background(), localRoot)) + + gotIdx, err := ListArtifacts(localRoot, origin) + require.NoError(t, err) + assert.ElementsMatch(t, uploaded, indexItems(gotIdx)) + assert.NoFileExists(t, filepath.Join(localRoot, origin, KindSegments, corruptName)) + assert.True(t, mock.has("arts/"+origin+"/segments/"+corruptName)) + assert.Zero(t, mock.deleteCount()) +} + +func TestS3TransportPullDeletesCorruptRemoteObjectSoPushHeals(t *testing.T) { + origin := "desktop-d4e5f6" + ownerRoot := exportStore(t, origin, func(database *db.DB) { + seedSession(t, database, "sess-7", "beta") + }) + ownerIdx, err := ListArtifacts(ownerRoot, origin) + require.NoError(t, err) + require.Len(t, ownerIdx.Segments, 1) + segKey := "arts/" + origin + "/segments/" + ownerIdx.Segments[0] + + mock := newMockS3(t, "bucket", 0) + for _, item := range indexItems(ownerIdx) { + art, err := ReadArtifact(ownerRoot, origin, item.kind, item.name) + require.NoError(t, err) + mock.put("arts/"+origin+"/"+item.kind+"/"+item.name, art.Data) + } + // The bucket copy is corrupted in place: its name still lists, so pushes + // from valid holders would otherwise skip it forever. + mock.put(segKey, []byte("garbage")) + + srv := httptest.NewTLSServer(mock) + t.Cleanup(srv.Close) + + // An empty peer's pull fails to validate the object and deletes it, so + // the name stops masking the valid copy. + emptyRoot := filepath.Join(t.TempDir(), "artifacts") + tr, err := newObjectTransport("s3://bucket/arts", testObjectOptions(srv.URL)) + require.NoError(t, err) + tr.client.Transport = srv.Client().Transport + require.NoError(t, tr.Exchange(context.Background(), emptyRoot)) + assert.False(t, mock.has(segKey), "corrupt object deleted from the bucket") + assert.Equal(t, 1, mock.deleteCount(), "expected one DELETE for the corrupt object") + + // The owner's next exchange re-uploads its valid copy, and the empty + // peer's next pull completes its store. + ownerTr, err := newObjectTransport("s3://bucket/arts", testObjectOptions(srv.URL)) + require.NoError(t, err) + ownerTr.client.Transport = srv.Client().Transport + require.NoError(t, ownerTr.Exchange(context.Background(), ownerRoot)) + assert.True(t, mock.has(segKey), "valid copy re-uploaded") + + require.NoError(t, tr.Exchange(context.Background(), emptyRoot)) + gotIdx, err := ListArtifacts(emptyRoot, origin) + require.NoError(t, err) + assert.ElementsMatch(t, indexItems(ownerIdx), indexItems(gotIdx)) +} + +func TestS3TransportRejectsRedirect(t *testing.T) { + var sourceReached atomic.Bool + var destinationReached atomic.Bool + destination := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + destinationReached.Store(true) + w.Header().Set("Content-Type", "application/xml") + _, _ = w.Write([]byte("false")) + })) + t.Cleanup(destination.Close) + + source := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + sourceReached.Store(true) + http.Redirect(w, r, destination.URL, http.StatusTemporaryRedirect) + })) + t.Cleanup(source.Close) + + tr, err := newObjectTransport("s3://bucket/arts", testObjectOptions(source.URL)) + require.NoError(t, err) + tr.client.Transport = source.Client().Transport + + _, err = tr.listPage(context.Background(), "", 0) + require.Error(t, err) + assert.True(t, sourceReached.Load()) + assert.False(t, destinationReached.Load()) +} + +func TestS3TransportPushSkipsAndQuarantinesCorruptLocalArtifact(t *testing.T) { + origin := "laptop-a1b2c3" + localRoot := exportStore(t, origin, func(database *db.DB) { + seedSession(t, database, "sess-1", "alpha") + }) + validIdx, err := ListArtifacts(localRoot, origin) + require.NoError(t, err) + corruptName := hashHex([]byte("junk")) + segmentExtension + corruptPath := filepath.Join(localRoot, origin, KindSegments, corruptName) + require.NoError(t, os.WriteFile(corruptPath, []byte("garbage"), 0o644)) + + mock := newMockS3(t, "bucket", 0) + srv := httptest.NewServer(mock) + t.Cleanup(srv.Close) + tr, err := newObjectTransport("s3://bucket/arts", testObjectOptions(srv.URL)) + require.NoError(t, err) + require.NoError(t, tr.Exchange(context.Background(), localRoot)) + + for _, item := range indexItems(validIdx) { + assert.True(t, mock.has("arts/"+origin+"/"+item.kind+"/"+item.name), + "expected object %s/%s in bucket", item.kind, item.name) + } + assert.False(t, mock.has("arts/"+origin+"/segments/"+corruptName)) + assert.NoFileExists(t, corruptPath) + assert.FileExists(t, corruptPath+quarantineSuffix) +} + +func TestS3TransportExchangeDetectsDivergentCheckpoint(t *testing.T) { + origin := "laptop-a1b2c3" + localRoot := exportStore(t, origin, func(database *db.DB) { + seedSession(t, database, "sess-1", "alpha") + }) + checkpoints := globArtifacts(t, localRoot, origin, KindCheckpoints, "cp-*.json") + require.Len(t, checkpoints, 1) + name := filepath.Base(checkpoints[0]) + + divergent, err := canonicalJSON(checkpoint{ + Version: formatVersion, Origin: origin, Sequence: 1, + Sessions: map[string]string{origin + "~other": hashHex([]byte("other"))}, + }) + require.NoError(t, err) + mock := newMockS3(t, "bucket", 0) + mock.put("arts/"+origin+"/"+KindCheckpoints+"/"+name, divergent) + + srv := httptest.NewServer(mock) + t.Cleanup(srv.Close) + tr, err := newObjectTransport("s3://bucket/arts", testObjectOptions(srv.URL)) + require.NoError(t, err) + + err = tr.Exchange(context.Background(), localRoot) + require.Error(t, err) + assert.ErrorIs(t, err, errArtifactPathConflict) +} + +func TestS3TransportExchangeRepairsCorruptLocalCheckpoint(t *testing.T) { + origin := "laptop-a1b2c3" + localRoot := exportStore(t, origin, func(database *db.DB) { + seedSession(t, database, "sess-1", "alpha") + }) + checkpoints := globArtifacts(t, localRoot, origin, KindCheckpoints, "cp-*.json") + require.Len(t, checkpoints, 1) + name := filepath.Base(checkpoints[0]) + valid, err := os.ReadFile(checkpoints[0]) + require.NoError(t, err) + + mock := newMockS3(t, "bucket", 0) + mock.put("arts/"+origin+"/"+KindCheckpoints+"/"+name, valid) + require.NoError(t, os.WriteFile(checkpoints[0], []byte("not json"), 0o644)) + + srv := httptest.NewServer(mock) + t.Cleanup(srv.Close) + tr, err := newObjectTransport("s3://bucket/arts", testObjectOptions(srv.URL)) + require.NoError(t, err) + require.NoError(t, tr.Exchange(context.Background(), localRoot)) + + got, err := os.ReadFile(checkpoints[0]) + require.NoError(t, err) + assert.Equal(t, valid, got, "corrupt local checkpoint should be re-fetched from the bucket") +} + +func TestS3TransportWriteOnceRejectsDivergentContent(t *testing.T) { + mock := newMockS3(t, "bucket", 0) + srv := httptest.NewServer(mock) + t.Cleanup(srv.Close) + tr, err := newObjectTransport("s3://bucket/arts", testObjectOptions(srv.URL)) + require.NoError(t, err) + ctx := context.Background() + key := "arts/laptop-a1b2c3/raw/deadbeef" + + // First write creates the object. + require.NoError(t, tr.putObject(ctx, key, []byte("one"))) + // An identical re-write is an accepted duplicate, not an error. + require.NoError(t, tr.putObject(ctx, key, []byte("one"))) + // Divergent content at the same key is a conflict, never a silent overwrite. + err = tr.putObject(ctx, key, []byte("two")) + require.Error(t, err) + assert.ErrorIs(t, err, errObjectStore) + // The original content is preserved. + got, err := tr.getObject(ctx, key) + require.NoError(t, err) + assert.Equal(t, []byte("one"), got) +} + +func TestIsObjectTarget(t *testing.T) { + tests := []struct { + name string + target string + want bool + }{ + {"s3 url", "s3://bucket/prefix", true}, + {"s3 bucket only", "s3://bucket", true}, + {"http peer", "http://example.com", false}, + {"https peer", "https://example.com", false}, + {"folder path", "/var/data/share", false}, + {"empty", "", false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.want, IsObjectTarget(tt.target)) + }) + } +} + +func TestNewObjectTransport(t *testing.T) { + creds := ObjectStoreOptions{ + Region: "us-east-1", + AccessKeyID: "AK", + SecretAccessKey: "SK", + } + + t.Run("missing bucket", func(t *testing.T) { + _, err := newObjectTransport("s3://", creds) + require.Error(t, err) + assert.Contains(t, err.Error(), "bucket") + }) + + t.Run("missing credentials", func(t *testing.T) { + _, err := newObjectTransport("s3://bucket/prefix", ObjectStoreOptions{Region: "us-east-1"}) + require.Error(t, err) + assert.Contains(t, err.Error(), "AWS_ACCESS_KEY_ID") + }) + + t.Run("not an object target", func(t *testing.T) { + _, err := newObjectTransport("https://example.com", creds) + require.Error(t, err) + }) + + t.Run("parses bucket and prefix", func(t *testing.T) { + tr, err := newObjectTransport("s3://bucket/some/prefix/", creds) + require.NoError(t, err) + assert.Equal(t, "bucket", tr.bucket) + assert.Equal(t, "some/prefix", tr.prefix) + assert.Equal(t, "s3.us-east-1.amazonaws.com", tr.endpoint.Host) + assert.False(t, tr.pathStyle, "real AWS defaults to virtual-host addressing") + }) + + t.Run("custom endpoint forces path style", func(t *testing.T) { + tr, err := newObjectTransport("s3://bucket", ObjectStoreOptions{ + Endpoint: "http://localhost:9000", + Region: "us-east-1", + AccessKeyID: "AK", + SecretAccessKey: "SK", + }) + require.NoError(t, err) + assert.True(t, tr.pathStyle) + assert.Equal(t, "localhost:9000", tr.endpoint.Host) + assert.Empty(t, tr.prefix) + }) + + t.Run("rejects insecure remote endpoint by default", func(t *testing.T) { + options := creds + options.Endpoint = "http://minio.lan:9000" + + _, err := newObjectTransport("s3://bucket", options) + require.Error(t, err) + assert.Contains(t, err.Error(), "insecure S3 endpoint") + assert.Contains(t, err.Error(), "AGENTSVIEW_ALLOW_INSECURE_S3_ENDPOINT") + }) + + t.Run("allows opted-in insecure remote endpoint", func(t *testing.T) { + options := creds + options.Endpoint = "http://minio.lan:9000" + options.AllowInsecureEndpoint = true + + tr, err := newObjectTransport("s3://bucket", options) + require.NoError(t, err) + assert.Equal(t, "http", tr.endpoint.Scheme) + }) + + for _, endpoint := range []string{ + "http://localhost:9000", + "http://LOCALHOST:9000", + "http://127.0.0.1:9000", + "http://[::1]:9000", + } { + t.Run("allows loopback endpoint "+endpoint, func(t *testing.T) { + options := creds + options.Endpoint = endpoint + + tr, err := newObjectTransport("s3://bucket", options) + require.NoError(t, err) + assert.Equal(t, "http", tr.endpoint.Scheme) + }) + } + + t.Run("bare host defaults to HTTPS", func(t *testing.T) { + options := creds + options.Endpoint = "minio.lan:9000" + + tr, err := newObjectTransport("s3://bucket", options) + require.NoError(t, err) + assert.Equal(t, "https", tr.endpoint.Scheme) + }) + + t.Run("rejects unsupported endpoint scheme", func(t *testing.T) { + options := creds + options.Endpoint = "ftp://minio.lan" + + _, err := newObjectTransport("s3://bucket", options) + require.Error(t, err) + assert.Contains(t, err.Error(), "ftp") + }) +} + +func TestObjectStoreOptionsFromEnvAllowsInsecureEndpoint(t *testing.T) { + for _, value := range []string{"1", "true", "yes", "YES"} { + t.Run(value, func(t *testing.T) { + t.Setenv("AWS_ACCESS_KEY_ID", "AK") + t.Setenv("AWS_SECRET_ACCESS_KEY", "SK") + t.Setenv("AGENTSVIEW_S3_ENDPOINT", "http://minio.lan:9000") + t.Setenv("AGENTSVIEW_ALLOW_INSECURE_S3_ENDPOINT", value) + + tr, err := newObjectTransport("s3://bucket", ObjectStoreOptionsFromEnv()) + require.NoError(t, err) + assert.Equal(t, "http", tr.endpoint.Scheme) + }) + } +} + +func TestObjectStoreOptionsFromEnvRejectsInvalidInsecureEndpointOverride(t *testing.T) { + tests := []struct { + name string + value string + }{ + {name: "empty", value: ""}, + {name: "zero", value: "0"}, + {name: "false", value: "false"}, + {name: "no", value: "no"}, + {name: "typo", value: "treu"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Setenv("AWS_ACCESS_KEY_ID", "AK") + t.Setenv("AWS_SECRET_ACCESS_KEY", "SK") + t.Setenv("AGENTSVIEW_S3_ENDPOINT", "http://minio.lan:9000") + t.Setenv("AGENTSVIEW_ALLOW_INSECURE_S3_ENDPOINT", tt.value) + + _, err := newObjectTransport("s3://bucket", ObjectStoreOptionsFromEnv()) + require.Error(t, err) + assert.Contains(t, err.Error(), "insecure S3 endpoint") + assert.Contains(t, err.Error(), "AGENTSVIEW_ALLOW_INSECURE_S3_ENDPOINT") + }) + } +} diff --git a/internal/artifact/twoinstance_test.go b/internal/artifact/twoinstance_test.go new file mode 100644 index 000000000..7b361fbcd --- /dev/null +++ b/internal/artifact/twoinstance_test.go @@ -0,0 +1,344 @@ +package artifact + +import ( + "context" + "database/sql" + "encoding/json" + "slices" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.kenn.io/agentsview/internal/db" +) + +// syncInstance models one machine in a two-instance artifact-sync harness: its +// own database, data dir, origin, and metadata recorder, all driven through the +// public folder-sync API against a shared target folder. +type syncInstance struct { + t *testing.T + db *db.DB + dataDir string + origin string + now time.Time + rec *MetadataRecorder +} + +func newSyncInstance(t *testing.T, origin string) *syncInstance { + t.Helper() + in := &syncInstance{ + t: t, + db: testDB(t), + dataDir: t.TempDir(), + origin: origin, + now: fixedHLCTime(), + } + in.rec = NewMetadataRecorder(in.db, MetadataRecorderOptions{ + DataDir: in.dataDir, + Origin: origin, + Now: func() time.Time { return in.now }, + }) + return in +} + +// at sets the wall clock used for the instance's next local metadata edits. +func (in *syncInstance) at(ts time.Time) *syncInstance { + in.now = ts + return in +} + +// sync exports local sessions to the shared target, exchanges the union both +// ways, and imports foreign origins, mirroring `agentsview sync `. +func (in *syncInstance) sync(target string) SyncResult { + in.t.Helper() + res, err := SyncFolder(context.Background(), in.db, SyncOptions{ + DataDir: in.dataDir, + Target: target, + Origin: in.origin, + Now: func() time.Time { return in.now }, + }) + require.NoError(in.t, err) + return res +} + +// syncInit mirrors `agentsview sync --init ` for baseline metadata +// initialization. +func (in *syncInstance) syncInit(target string) SyncResult { + in.t.Helper() + res, err := SyncFolder(context.Background(), in.db, SyncOptions{ + DataDir: in.dataDir, + Target: target, + Origin: in.origin, + Now: func() time.Time { return in.now }, + BaselineMetadata: true, + }) + require.NoError(in.t, err) + return res +} + +// rename mirrors the rename handler: mutate the local row, then append the +// metadata event (which also records the local LWW register entry). +func (in *syncInstance) rename(localID, name string) { + in.t.Helper() + require.NoError(in.t, in.db.RenameSession(localID, &name)) + value, err := json.Marshal(struct { + DisplayName string `json:"display_name"` + }{DisplayName: name}) + require.NoError(in.t, err) + _, err = in.rec.Append(context.Background(), MetadataEventInput{ + SessionID: localID, + Op: MetadataOpRename, + Value: value, + }) + require.NoError(in.t, err) +} + +// star mirrors the star handler: star the local row, then append the event. +func (in *syncInstance) star(localID string) { + in.t.Helper() + _, err := in.db.StarSession(localID) + require.NoError(in.t, err) + _, err = in.rec.Append(context.Background(), MetadataEventInput{ + SessionID: localID, + Op: MetadataOpStar, + }) + require.NoError(in.t, err) +} + +// purge mirrors the permanent-delete handler: soft delete, permanently delete +// from trash, then append the purge event. +func (in *syncInstance) purge(localID string) { + in.t.Helper() + require.NoError(in.t, in.db.SoftDeleteSession(localID)) + _, err := in.db.DeleteSessionIfTrashed(localID) + require.NoError(in.t, err) + _, err = in.rec.Append(context.Background(), MetadataEventInput{ + SessionID: localID, + Op: MetadataOpPurge, + }) + require.NoError(in.t, err) +} + +func (in *syncInstance) displayName(t *testing.T, id string) *string { + t.Helper() + got, err := in.db.GetSession(context.Background(), id) + require.NoError(t, err) + require.NotNil(t, got) + return got.DisplayName +} + +func (in *syncInstance) requireSession(t *testing.T, id string) *db.Session { + t.Helper() + got, err := in.db.GetSession(context.Background(), id) + require.NoError(t, err) + require.NotNil(t, got, "session %s should exist", id) + return got +} + +func (in *syncInstance) isStarred(t *testing.T, id string) bool { + t.Helper() + ids, err := in.db.ListStarredSessionIDs(context.Background()) + require.NoError(t, err) + return slices.Contains(ids, id) +} + +func TestTwoInstanceSessionAndRenamePropagate(t *testing.T) { + target := t.TempDir() + a := newSyncInstance(t, "laptop-a1b2c3") + b := newSyncInstance(t, "desktop-d4e5f6") + gid := a.origin + "~sess-1" + + seedSession(t, a.db, "sess-1", "alpha") + a.at(fixedHLCTime()).rename("sess-1", "Renamed on A") + + a.sync(target) + res := b.sync(target) + assert.Equal(t, 1, res.ImportedSessions) + assert.GreaterOrEqual(t, res.ImportedMessages, 1) + assert.Equal(t, 1, res.ImportedMetadata) + + got, err := b.db.GetSession(context.Background(), gid) + require.NoError(t, err) + require.NotNil(t, got) + require.NotNil(t, got.DisplayName) + assert.Equal(t, "Renamed on A", *got.DisplayName) + assert.Equal(t, a.origin, got.Machine) + + // Re-syncing is idempotent: no new rows, no new conflicts. + res = b.sync(target) + assert.Zero(t, res.ImportedSessions) + assert.Zero(t, res.ImportedMetadata) +} + +func TestTwoInstanceConcurrentRenameConverges(t *testing.T) { + target := t.TempDir() + a := newSyncInstance(t, "laptop-a1b2c3") + b := newSyncInstance(t, "desktop-d4e5f6") + gid := a.origin + "~sess-1" + bLocalID := gid // the session is foreign on B, keyed by its global id + + // Share the session from A to B first so both can edit it. + seedSession(t, a.db, "sess-1", "alpha") + a.sync(target) + b.sync(target) + b.requireSession(t, bLocalID) + + // Both rename the same session concurrently. A uses the later HLC (within the + // clock drift bound so the receiver can observe it), so A wins deterministically + // on both machines. + b.at(fixedHLCTime()).rename(bLocalID, "Renamed on B") + a.at(fixedHLCTime().Add(time.Minute)).rename("sess-1", "Renamed on A") + + // Exchange until quiescent: A publishes its edit, B pulls it (A wins on B), + // then A pulls B's losing edit and records the conflict. + a.sync(target) + b.sync(target) + a.sync(target) + + require.NotNil(t, a.displayName(t, "sess-1")) + require.NotNil(t, b.displayName(t, bLocalID)) + assert.Equal(t, "Renamed on A", *a.displayName(t, "sess-1")) + assert.Equal(t, "Renamed on A", *b.displayName(t, bLocalID)) + + // Both machines independently record exactly one losing edit for the field. + assertMetadataConflictCount(t, a.db, gid, "display_name", 1) + assertMetadataConflictCount(t, b.db, gid, "display_name", 1) +} + +func TestSyncInitDoesNotLetBaselineOutrankExistingPeerMetadata(t *testing.T) { + target := t.TempDir() + a := newSyncInstance(t, "laptop-a1b2c3") + b := newSyncInstance(t, "desktop-d4e5f6") + gid := a.origin + "~sess-1" + + seedSession(t, a.db, "sess-1", "alpha") + a.sync(target) + b.sync(target) + b.requireSession(t, gid) + + peerName := "Peer newer title" + b.at(fixedHLCTime().Add(time.Minute)).rename(gid, peerName) + b.sync(target) + + staleLocalName := "Stale local title" + require.NoError(t, a.db.RenameSession("sess-1", &staleLocalName)) + + res := a.at(fixedHLCTime().Add(2 * time.Minute)).syncInit(target) + assert.Equal(t, 1, res.ImportedMetadata) + require.NotNil(t, a.displayName(t, "sess-1")) + assert.Equal(t, peerName, *a.displayName(t, "sess-1")) + + b.sync(target) + require.NotNil(t, b.displayName(t, gid)) + assert.Equal(t, peerName, *b.displayName(t, gid)) + assertMetadataConflictCount(t, a.db, gid, "display_name", 0) + assertMetadataConflictCount(t, b.db, gid, "display_name", 0) +} + +func TestSyncInitDoesNotBaselineRowsCreatedByPreBaselineImport(t *testing.T) { + target := t.TempDir() + a := newSyncInstance(t, "laptop-a1b2c3") + b := newSyncInstance(t, "desktop-d4e5f6") + gid := b.origin + "~sess-1" + + seedSession(t, b.db, "sess-1", "alpha") + b.sync(target) + + require.NoError(t, a.db.Update(func(tx *sql.Tx) error { + _, err := tx.Exec(` + CREATE TRIGGER test_imported_display_name + AFTER INSERT ON sessions + WHEN NEW.id = 'desktop-d4e5f6~sess-1' + BEGIN + UPDATE sessions + SET display_name = 'Imported stale title' + WHERE id = NEW.id; + END`) + return err + })) + + res := a.syncInit(target) + assert.Equal(t, 1, res.ImportedSessions) + assert.Equal(t, 0, res.ImportedMetadata) + require.NotNil(t, a.displayName(t, gid)) + assert.Equal(t, "Imported stale title", *a.displayName(t, gid)) + + _, ok, err := a.db.MetadataReplayStateOp(context.Background(), gid, "display_name") + require.NoError(t, err) + assert.False(t, ok, "import-created curation must not become local baseline metadata") + assert.Empty(t, globArtifacts(t, target, a.origin, "meta", "*"+metadataEventExtension)) +} + +func TestSyncReappliesMetadataReplayStateAfterManifestRefresh(t *testing.T) { + target := t.TempDir() + a := newSyncInstance(t, "laptop-a1b2c3") + b := newSyncInstance(t, "desktop-d4e5f6") + gid := b.origin + "~sess-1" + + sourceName := "Source title" + seedSession(t, b.db, "sess-1", "alpha", func(s *db.Session) { + s.SessionName = &sourceName + }) + b.sync(target) + a.sync(target) + a.requireSession(t, gid) + + localName := "Local winning title" + a.at(fixedHLCTime().Add(time.Minute)).rename(gid, localName) + require.NotNil(t, a.displayName(t, gid)) + assert.Equal(t, localName, *a.displayName(t, gid)) + + require.NoError(t, a.db.Update(func(tx *sql.Tx) error { + _, err := tx.Exec(`UPDATE sessions SET display_name = NULL WHERE id = ?`, gid) + return err + })) + refreshedSourceName := "Refreshed source title" + seedSession(t, b.db, "sess-1", "alpha", func(s *db.Session) { + s.SessionName = &refreshedSourceName + }) + b.sync(target) + + a.sync(target) + require.NotNil(t, a.displayName(t, gid)) + assert.Equal(t, localName, *a.displayName(t, gid)) +} + +func TestTwoInstanceStarAndPurgePropagate(t *testing.T) { + target := t.TempDir() + a := newSyncInstance(t, "laptop-a1b2c3") + b := newSyncInstance(t, "desktop-d4e5f6") + gid := a.origin + "~sess-1" + + seedSession(t, a.db, "sess-1", "alpha") + a.sync(target) + b.sync(target) + b.requireSession(t, gid) + + // B stars the shared session; the star converges back to A. + b.at(fixedHLCTime()).star(gid) + b.sync(target) + a.sync(target) + assert.True(t, a.isStarred(t, "sess-1")) + assert.True(t, b.isStarred(t, gid)) + + // A purges the session; the purge tombstone propagates to B and blocks + // re-import of the now-superseded manifest. The HLC stays within the drift + // bound so B can observe it on import. + a.at(fixedHLCTime().Add(time.Minute)).purge("sess-1") + a.sync(target) + b.sync(target) + + gotA, err := a.db.GetSession(context.Background(), "sess-1") + require.NoError(t, err) + assert.Nil(t, gotA) + gotB, err := b.db.GetSession(context.Background(), gid) + require.NoError(t, err) + assert.Nil(t, gotB) + + // A later sync must not resurrect the purged session on B. + b.sync(target) + gotB, err = b.db.GetSession(context.Background(), gid) + require.NoError(t, err) + assert.Nil(t, gotB) +} diff --git a/internal/config/config.go b/internal/config/config.go index 6017767fa..54500d756 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -4,7 +4,9 @@ import ( "bytes" "crypto/rand" "encoding/base64" + "encoding/hex" "encoding/json" + "errors" "flag" "fmt" "log" @@ -437,6 +439,7 @@ type Config struct { GithubToken string `json:"github_token,omitempty" toml:"github_token"` Terminal TerminalConfig `json:"terminal,omitempty" toml:"terminal"` AuthToken string `json:"auth_token,omitempty" toml:"auth_token"` + ArtifactOriginID string `json:"artifact_origin_id,omitempty" toml:"artifact_origin_id"` RequireAuth bool `json:"require_auth" toml:"require_auth"` NoBrowser bool `json:"no_browser" toml:"no_browser"` DisableUpdateCheck bool `json:"disable_update_check" toml:"disable_update_check"` @@ -966,6 +969,7 @@ func (c *Config) applyConfigTOML(data string) error { ResultContentBlockedCategories []string `toml:"result_content_blocked_categories"` Terminal TerminalConfig `toml:"terminal"` AuthToken string `toml:"auth_token"` + ArtifactOriginID string `toml:"artifact_origin_id"` RequireAuth bool `toml:"require_auth"` RemoteAccess bool `toml:"remote_access"` DisableUpdateCheck bool `toml:"disable_update_check"` @@ -1036,6 +1040,9 @@ func (c *Config) applyConfigTOML(data string) error { if file.AuthToken != "" && c.AuthToken == "" { c.AuthToken = file.AuthToken } + if file.ArtifactOriginID != "" { + c.ArtifactOriginID = file.ArtifactOriginID + } c.RequireAuth = file.RequireAuth || file.RemoteAccess c.DisableUpdateCheck = file.DisableUpdateCheck if meta.IsDefined("default_pg") { @@ -1627,6 +1634,11 @@ func finalize(cfg *Config) error { if err := cfg.Vector.Validate(); err != nil { return err } + if cfg.ArtifactOriginID != "" { + if err := ValidateArtifactOriginID(cfg.ArtifactOriginID); err != nil { + return fmt.Errorf("invalid artifact origin id: %w", err) + } + } return nil } @@ -2408,6 +2420,11 @@ func (c *Config) SaveSettings(patch map[string]any) error { c.AuthToken = s } } + if v, ok := patch["artifact_origin_id"]; ok { + if s, ok := v.(string); ok { + c.ArtifactOriginID = s + } + } if v, ok := patch["require_auth"]; ok { if b, ok := v.(bool); ok { c.RequireAuth = b @@ -2447,6 +2464,176 @@ func (c *Config) EnsureAuthToken() error { }) } +// EnsureArtifactOriginID generates and persists a stable artifact sync origin +// ID if one does not already exist. +func (c *Config) EnsureArtifactOriginID() (string, error) { + if c.ArtifactOriginID != "" { + if err := ValidateArtifactOriginID(c.ArtifactOriginID); err != nil { + return "", fmt.Errorf("stored artifact origin: %w", err) + } + return c.ArtifactOriginID, nil + } + + var origin string + if err := c.withConfigLock(func() error { + existing, err := c.readConfigMap() + if err != nil { + return err + } + if stored, ok := existing["artifact_origin_id"].(string); ok && stored != "" { + if err := ValidateArtifactOriginID(stored); err != nil { + return fmt.Errorf("stored artifact origin: %w", err) + } + c.ArtifactOriginID = stored + origin = stored + return nil + } + + machineName, err := c.artifactOriginMachineName() + if err != nil { + return err + } + generated, err := newArtifactOriginID(machineName) + if err != nil { + return err + } + if err := ValidateArtifactOriginID(generated); err != nil { + return fmt.Errorf("generated artifact origin: %w", err) + } + + existing["artifact_origin_id"] = generated + if err := c.writeConfigMap(existing); err != nil { + return err + } + c.ArtifactOriginID = generated + origin = generated + return nil + }); err != nil { + return "", err + } + return origin, nil +} + +// AdoptArtifactOriginID persists origin as this machine's artifact sync +// origin unless the config file already records one, in which case the +// recorded origin wins and is returned. Artifact sync uses this to promote an +// origin that exists only in database sync state -- for example one minted by +// an incoming peer exchange before the config ever initialized an origin -- +// so the machine keeps publishing under a single origin instead of generating +// a competing config origin that would strand earlier metadata events. +func (c *Config) AdoptArtifactOriginID(origin string) (string, error) { + if err := ValidateArtifactOriginID(origin); err != nil { + return "", fmt.Errorf("adopting artifact origin: %w", err) + } + if c.ArtifactOriginID != "" { + if err := ValidateArtifactOriginID(c.ArtifactOriginID); err != nil { + return "", fmt.Errorf("stored artifact origin: %w", err) + } + return c.ArtifactOriginID, nil + } + + adopted := origin + if err := c.withConfigLock(func() error { + existing, err := c.readConfigMap() + if err != nil { + return err + } + if stored, ok := existing["artifact_origin_id"].(string); ok && stored != "" { + if err := ValidateArtifactOriginID(stored); err != nil { + return fmt.Errorf("stored artifact origin: %w", err) + } + c.ArtifactOriginID = stored + adopted = stored + return nil + } + + existing["artifact_origin_id"] = origin + if err := c.writeConfigMap(existing); err != nil { + return err + } + c.ArtifactOriginID = origin + return nil + }); err != nil { + return "", err + } + return adopted, nil +} + +func (c *Config) artifactOriginMachineName() (string, error) { + pgMachine := strings.TrimSpace(c.PG.MachineName) + if pgMachine != "" { + if pgMachine == "local" { + return "", fmt.Errorf( + "machine name %q is reserved; choose a different pg.machine_name", + pgMachine, + ) + } + return c.PG.MachineName, nil + } + host, err := os.Hostname() + if err != nil || strings.TrimSpace(host) == "" { + return "machine", nil + } + return host, nil +} + +func newArtifactOriginID(machine string) (string, error) { + base := sanitizeArtifactOriginPart(machine) + if base == "" || base == "local" { + base = "machine" + } + var suffix [3]byte + if _, err := rand.Read(suffix[:]); err != nil { + return "", fmt.Errorf("generating artifact origin suffix: %w", err) + } + return fmt.Sprintf("%s-%s", base, hex.EncodeToString(suffix[:])), nil +} + +func sanitizeArtifactOriginPart(s string) string { + s = strings.ToLower(strings.TrimSpace(s)) + var b strings.Builder + lastDash := false + for _, r := range s { + ok := r >= 'a' && r <= 'z' || r >= '0' && r <= '9' + if ok { + b.WriteRune(r) + lastDash = false + continue + } + if !lastDash { + b.WriteByte('-') + lastDash = true + } + } + return strings.Trim(b.String(), "-") +} + +// ValidateArtifactOriginID checks the persisted single-writer origin prefix. +func ValidateArtifactOriginID(origin string) error { + if origin == "" { + return errors.New("artifact origin is required") + } + if origin != strings.TrimSpace(origin) { + return fmt.Errorf("invalid artifact origin %q", origin) + } + if origin == "local" { + return fmt.Errorf("invalid artifact origin %q", origin) + } + if strings.ContainsAny(origin, `/\`) || filepath.Base(origin) != origin { + return fmt.Errorf("invalid artifact origin %q", origin) + } + if strings.HasPrefix(origin, "-") || strings.HasSuffix(origin, "-") { + return fmt.Errorf("invalid artifact origin %q", origin) + } + for _, r := range origin { + if r >= 'a' && r <= 'z' || r >= '0' && r <= '9' || r == '-' { + continue + } + return fmt.Errorf("invalid artifact origin %q", origin) + } + return nil +} + // SaveGithubToken persists the GitHub token to the config file. func (c *Config) SaveGithubToken(token string) error { return c.withConfigLock(func() error { diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 885986600..1d0d3395a 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -13,6 +13,7 @@ import ( "time" "github.com/BurntSushi/toml" + "github.com/gofrs/flock" "github.com/spf13/pflag" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -474,6 +475,221 @@ func TestLoad_PublicURLMergedIntoOrigins(t *testing.T) { assert.Equal(t, "https://viewer.example.test", strings.Join(cfg.PublicOrigins, ",")) } +func TestLoad_ArtifactOriginIDFromConfigFile(t *testing.T) { + tmp := setupTestEnv(t) + writeConfig(t, tmp, map[string]any{ + "artifact_origin_id": "desk-abcdef", + }) + + cfg, err := LoadMinimal() + require.NoError(t, err) + + assert.Equal(t, "desk-abcdef", cfg.ArtifactOriginID) +} + +func TestLoad_ArtifactOriginIDRejectsInvalid(t *testing.T) { + tmp := setupTestEnv(t) + writeConfig(t, tmp, map[string]any{ + "artifact_origin_id": "local", + }) + + _, err := LoadMinimal() + require.Error(t, err) + assert.Contains(t, err.Error(), "invalid artifact origin id") +} + +func TestEnsureArtifactOriginIDPersists(t *testing.T) { + tmp := setupTestEnv(t) + cfg, err := LoadMinimal() + require.NoError(t, err) + + origin, err := cfg.EnsureArtifactOriginID() + require.NoError(t, err) + assert.Regexp(t, `^[a-z0-9]+(?:-[a-z0-9]+)*-[0-9a-f]{6}$`, origin) + + data, err := os.ReadFile(filepath.Join(tmp, configFileName)) + require.NoError(t, err) + assert.Contains(t, string(data), `artifact_origin_id = "`+origin+`"`) + + reloaded, err := LoadMinimal() + require.NoError(t, err) + assert.Equal(t, origin, reloaded.ArtifactOriginID) + again, err := reloaded.EnsureArtifactOriginID() + require.NoError(t, err) + assert.Equal(t, origin, again) +} + +func TestEnsureArtifactOriginIDUsesConfiguredMachineName(t *testing.T) { + tmp := setupTestEnv(t) + writeConfig(t, tmp, map[string]any{ + "pg": map[string]any{ + "machine_name": "Desk Box", + }, + }) + cfg, err := LoadMinimal() + require.NoError(t, err) + + origin, err := cfg.EnsureArtifactOriginID() + require.NoError(t, err) + + assert.Regexp(t, `^desk-box-[0-9a-f]{6}$`, origin) +} + +func TestEnsureArtifactOriginIDRejectsReservedMachineName(t *testing.T) { + tmp := setupTestEnv(t) + writeConfig(t, tmp, map[string]any{ + "pg": map[string]any{ + "machine_name": "local", + }, + }) + cfg, err := LoadMinimal() + require.NoError(t, err) + + origin, err := cfg.EnsureArtifactOriginID() + require.Error(t, err) + assert.Empty(t, origin) + assert.Contains(t, err.Error(), "reserved") +} + +func TestEnsureArtifactOriginIDDoesNotRewriteExistingOrigin(t *testing.T) { + tmp := setupTestEnv(t) + writeConfig(t, tmp, map[string]any{ + "artifact_origin_id": "original-a1b2c3", + "pg": map[string]any{ + "machine_name": "new-machine", + }, + }) + cfg, err := LoadMinimal() + require.NoError(t, err) + + origin, err := cfg.EnsureArtifactOriginID() + require.NoError(t, err) + + assert.Equal(t, "original-a1b2c3", origin) +} + +func TestAdoptArtifactOriginIDPersists(t *testing.T) { + tmp := setupTestEnv(t) + cfg, err := LoadMinimal() + require.NoError(t, err) + + adopted, err := cfg.AdoptArtifactOriginID("laptop-a1b2c3") + require.NoError(t, err) + assert.Equal(t, "laptop-a1b2c3", adopted) + + data, err := os.ReadFile(filepath.Join(tmp, configFileName)) + require.NoError(t, err) + assert.Contains(t, string(data), `artifact_origin_id = "laptop-a1b2c3"`) + + reloaded, err := LoadMinimal() + require.NoError(t, err) + origin, err := reloaded.EnsureArtifactOriginID() + require.NoError(t, err) + assert.Equal(t, "laptop-a1b2c3", origin, + "later ensure must reuse the adopted origin instead of generating") +} + +func TestAdoptArtifactOriginIDKeepsExistingFileOrigin(t *testing.T) { + tmp := setupTestEnv(t) + cfg, err := LoadMinimal() + require.NoError(t, err) + + // The config file gains an origin after load but before adoption; the + // recorded origin wins. + writeConfig(t, tmp, map[string]any{ + "artifact_origin_id": "desktop-d4e5f6", + }) + + adopted, err := cfg.AdoptArtifactOriginID("laptop-a1b2c3") + require.NoError(t, err) + assert.Equal(t, "desktop-d4e5f6", adopted) + assert.Equal(t, "desktop-d4e5f6", cfg.ArtifactOriginID) +} + +func TestAdoptArtifactOriginIDKeepsInMemoryOrigin(t *testing.T) { + setupTestEnv(t) + cfg, err := LoadMinimal() + require.NoError(t, err) + cfg.ArtifactOriginID = "desktop-d4e5f6" + + adopted, err := cfg.AdoptArtifactOriginID("laptop-a1b2c3") + require.NoError(t, err) + assert.Equal(t, "desktop-d4e5f6", adopted) +} + +func TestAdoptArtifactOriginIDRejectsInvalidOrigin(t *testing.T) { + setupTestEnv(t) + cfg, err := LoadMinimal() + require.NoError(t, err) + + _, err = cfg.AdoptArtifactOriginID("local") + require.Error(t, err) + assert.Empty(t, cfg.ArtifactOriginID) +} + +func TestEnsureArtifactOriginIDReusesOriginWrittenBeforeLock(t *testing.T) { + tmp := setupTestEnv(t) + cfg, err := LoadMinimal() + require.NoError(t, err) + + lock := flock.New(cfg.configPath() + ".lock") + require.NoError(t, lock.Lock()) + t.Cleanup(func() { + _ = lock.Unlock() + }) + + type ensureResult struct { + origin string + err error + } + done := make(chan ensureResult, 1) + go func() { + origin, err := cfg.EnsureArtifactOriginID() + done <- ensureResult{origin: origin, err: err} + }() + + select { + case res := <-done: + require.Failf(t, "EnsureArtifactOriginID ignored config lock", + "origin=%q err=%v", res.origin, res.err) + case <-time.After(250 * time.Millisecond): + } + + writeConfig(t, tmp, map[string]any{ + "artifact_origin_id": "winner-a1b2c3", + }) + require.NoError(t, lock.Unlock()) + + select { + case res := <-done: + require.NoError(t, res.err) + assert.Equal(t, "winner-a1b2c3", res.origin) + assert.Equal(t, "winner-a1b2c3", cfg.ArtifactOriginID) + case <-time.After(2 * time.Second): + require.Fail(t, "EnsureArtifactOriginID did not finish after lock release") + } +} + +func TestMigrateJSONToTOMLPreservesArtifactOriginID(t *testing.T) { + tmp := setupTestEnv(t) + jsonPath := filepath.Join(tmp, "config.json") + require.NoError(t, os.WriteFile( + jsonPath, + []byte(`{"artifact_origin_id":"desk-abcdef"}`), + 0o600, + )) + + cfg, err := LoadMinimal() + require.NoError(t, err) + + assert.Equal(t, "desk-abcdef", cfg.ArtifactOriginID) + _, err = os.Stat(jsonPath + ".bak") + require.NoError(t, err) + data, err := os.ReadFile(filepath.Join(tmp, configFileName)) + require.NoError(t, err) + assert.Contains(t, string(data), `artifact_origin_id = "desk-abcdef"`) +} + func TestLoad_ProxyConfigFromFile(t *testing.T) { cfg := loadMinimalWithConfig(t, map[string]any{ "public_url": "https://viewer.example.test", diff --git a/internal/db/db.go b/internal/db/db.go index 62aca4b03..13b94d5e1 100644 --- a/internal/db/db.go +++ b/internal/db/db.go @@ -1639,6 +1639,47 @@ func (db *DB) migrateColumns() error { return err } + if _, err := w.Exec(` + CREATE TABLE IF NOT EXISTS metadata_applied_events ( + origin TEXT NOT NULL, + order_key TEXT NOT NULL, + artifact_hash TEXT NOT NULL, + applied_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')), + PRIMARY KEY (origin, order_key) + ); + CREATE TABLE IF NOT EXISTS metadata_replay_state ( + session_gid TEXT NOT NULL, + field TEXT NOT NULL, + order_key TEXT NOT NULL, + hlc TEXT NOT NULL, + artifact_hash TEXT NOT NULL, + origin TEXT NOT NULL, + op TEXT NOT NULL, + value TEXT NOT NULL DEFAULT '', + updated_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')), + PRIMARY KEY (session_gid, field) + ); + CREATE TABLE IF NOT EXISTS metadata_conflicts ( + id INTEGER PRIMARY KEY, + session_gid TEXT NOT NULL, + field TEXT NOT NULL, + winning_order_key TEXT NOT NULL, + losing_order_key TEXT NOT NULL, + winning_origin TEXT NOT NULL, + losing_origin TEXT NOT NULL, + winning_op TEXT NOT NULL, + losing_op TEXT NOT NULL, + winning_value TEXT NOT NULL DEFAULT '', + losing_value TEXT NOT NULL DEFAULT '', + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')), + UNIQUE(session_gid, field, winning_order_key, losing_order_key) + ); + `); err != nil { + return fmt.Errorf( + "creating metadata replay tables: %w", err, + ) + } + if err := db.ensureUsageEventsSchemaLocked(w); err != nil { return err } @@ -2885,6 +2926,32 @@ func (db *DB) GetSyncState(key string) (string, error) { return value, err } +// SyncStatesWithPrefix reads all non-empty sync-state entries whose keys start +// with prefix in one query. +func (db *DB) SyncStatesWithPrefix(prefix string) (map[string]string, error) { + rows, err := db.getReader().Query( + `SELECT key, value FROM pg_sync_state + WHERE substr(key, 1, length(?)) = ? AND value <> ''`, + prefix, prefix, + ) + if err != nil { + return nil, err + } + defer rows.Close() + states := map[string]string{} + for rows.Next() { + var key, value string + if err := rows.Scan(&key, &value); err != nil { + return nil, err + } + states[key] = value + } + if err := rows.Err(); err != nil { + return nil, err + } + return states, nil +} + // SetSyncState writes a value to the pg_sync_state table. func (db *DB) SetSyncState(key, value string) error { db.mu.Lock() diff --git a/internal/db/db_test.go b/internal/db/db_test.go index be473bb31..59502aa8c 100644 --- a/internal/db/db_test.go +++ b/internal/db/db_test.go @@ -892,6 +892,47 @@ func TestInsertMessages_PreservesToolResultEvents(t *testing.T) { assert.Equal(t, "subagent_notification", tc.ResultEvents[1].Source, "result event 1 source") } +func TestSessionSubagentSessionIDs(t *testing.T) { + d := testDB(t) + insertSession(t, d, "s-sub", "proj") + require.NoError(t, d.InsertMessages([]Message{ + { + SessionID: "s-sub", Ordinal: 0, Role: "assistant", + Content: "spawn", HasToolUse: true, + ToolCalls: []ToolCall{ + { + SessionID: "s-sub", ToolName: "Task", Category: "Task", + ToolUseID: "call-1", SubagentSessionID: "sub-a", + ResultEvents: []ToolResultEvent{ + { + ToolUseID: "call-1", SubagentSessionID: "sub-a", + Source: "subagent", Status: "ok", EventIndex: 0, + }, + { + ToolUseID: "call-1", SubagentSessionID: "sub-b", + Source: "subagent", Status: "ok", EventIndex: 1, + }, + }, + }, + { + SessionID: "s-sub", ToolName: "Read", Category: "file", + ToolUseID: "call-2", + }, + }, + }, + }), "InsertMessages") + + ids, err := d.SessionSubagentSessionIDs("s-sub") + require.NoError(t, err, "SessionSubagentSessionIDs") + // "sub-a" appears on both the tool call and a result event (deduped); + // "sub-b" only on a result event; the empty subagent id is excluded. + assert.ElementsMatch(t, []string{"sub-a", "sub-b"}, ids) + + none, err := d.SessionSubagentSessionIDs("missing") + require.NoError(t, err, "SessionSubagentSessionIDs missing") + assert.Empty(t, none) +} + func TestOpenPreservesDataAtCurrentVersion(t *testing.T) { dir := t.TempDir() path := filepath.Join(dir, "test.db") @@ -4384,12 +4425,17 @@ func TestStarSession(t *testing.T) { assert.Equal(t, []string{"s1"}, ids, "listed = %v, want [s1]", ids) // Unstar. - err = d.UnstarSession("s1") + removed, err := d.UnstarSession("s1") require.NoError(t, err, "UnstarSession") + assert.True(t, removed, "UnstarSession should report removed star") ids, err = d.ListStarredSessionIDs(ctx) require.NoError(t, err, "ListStarredSessionIDs after unstar") assert.Empty(t, ids, "listed after unstar = %v, want []", ids) + removed, err = d.UnstarSession("s1") + require.NoError(t, err, "UnstarSession no-op") + assert.False(t, removed, "UnstarSession should report no-op") + // Star non-existent session returns false (no FK error). ok, err = d.StarSession("nonexistent") require.NoError(t, err, "StarSession nonexistent") @@ -4403,8 +4449,13 @@ func TestBulkStarSessions(t *testing.T) { insertSession(t, d, "s2", "proj") // Bulk star with mix of valid and invalid IDs. - err := d.BulkStarSessions([]string{"s1", "s2", "nonexistent"}) + starred, err := d.BulkStarSessions([]string{"s1", "s2", "nonexistent"}) require.NoError(t, err, "BulkStarSessions") + assert.ElementsMatch(t, []string{"s1", "s2"}, starred, "starred ids returned") + + starred, err = d.BulkStarSessions([]string{"s1", "s2"}) + require.NoError(t, err, "BulkStarSessions already starred") + assert.Empty(t, starred, "already-starred ids should not be returned") ids, err := d.ListStarredSessionIDs(ctx) require.NoError(t, err, "ListStarredSessionIDs") @@ -4679,6 +4730,16 @@ func TestCopySyncStateFrom_OnlyCopiesDurablePGKeys(t *testing.T) { srcDB := testDBAtPath(t, srcPath, "src") require.NoError(t, srcDB.SetSyncState("pg_push_marker_id", "marker-123"), "seed source marker") + require.NoError(t, srcDB.SetSyncState("artifact_origin_id", "laptop-a1b2c3"), + "seed source artifact origin") + require.NoError(t, srcDB.SetSyncState("artifact_metadata_hlc", "hlc-42"), + "seed source artifact hlc") + require.NoError(t, + srcDB.SetSyncState("artifact_import:peer-b4c5d6:peer-b4c5d6~sess-1", "hash-imp"), + "seed source import watermark") + require.NoError(t, + srcDB.SetSyncState("artifact_export:laptop-a1b2c3:sess-2", "hash-exp"), + "seed source export watermark") require.NoError(t, srcDB.SetSyncState("last_sync_started_at", "old-start"), "seed source started") require.NoError(t, srcDB.SetSyncState("last_sync_finished_at", "old-finish"), @@ -4700,6 +4761,18 @@ func TestCopySyncStateFrom_OnlyCopiesDurablePGKeys(t *testing.T) { require.NoError(t, err, "GetSyncState pg_push_marker_id") assert.Equal(t, "marker-123", gotMarker) + durableArtifactKeys := map[string]string{ + "artifact_origin_id": "laptop-a1b2c3", + "artifact_metadata_hlc": "hlc-42", + "artifact_import:peer-b4c5d6:peer-b4c5d6~sess-1": "hash-imp", + "artifact_export:laptop-a1b2c3:sess-2": "hash-exp", + } + for key, want := range durableArtifactKeys { + got, err := dstDB.GetSyncState(key) + require.NoError(t, err, "GetSyncState %s", key) + assert.Equal(t, want, got, "artifact key %s must survive the copy", key) + } + gotStarted, err := dstDB.GetSyncState("last_sync_started_at") require.NoError(t, err, "GetSyncState last_sync_started_at") assert.Equal(t, "new-start", gotStarted) @@ -4732,6 +4805,99 @@ func TestCopySyncStateFrom_PropagatesErrors(t *testing.T) { assert.Equal(t, "safe", got) } +func TestCopyMetadataReplayFrom(t *testing.T) { + dir := t.TempDir() + ctx := context.Background() + + srcPath := filepath.Join(dir, "src.db") + srcDB := testDBAtPath(t, srcPath, "src") + + // Seed the LWW register and applied-event markers without touching + // session rows, exactly as local curation bookkeeping does. + winner := MetadataProjection{ + EventOrigin: "laptop-a1b2c3", + OrderKey: "0000000002", + HLC: "hlc-2", + ArtifactHash: "hash-2", + SessionGID: "laptop-a1b2c3~sess-1", + LocalSessionID: "sess-1", + Field: "display_name", + Op: "rename", + Value: `{"display_name":"kept"}`, + } + res, err := srcDB.RecordLocalMetadataProjection(ctx, winner) + require.NoError(t, err, "record winning projection") + require.True(t, res.Applied, "winning projection applied") + + // A stale peer event loses and records a conflict row. + loser := winner + loser.EventOrigin = "peer-b4c5d6" + loser.OrderKey = "0000000001" + loser.HLC = "hlc-1" + loser.ArtifactHash = "hash-1" + loser.Value = `{"display_name":"stale"}` + res, err = srcDB.RecordLocalMetadataProjection(ctx, loser) + require.NoError(t, err, "record losing projection") + require.True(t, res.Conflict, "losing projection records a conflict") + require.NoError(t, srcDB.Close(), "Close src") + + dstPath := filepath.Join(dir, "dst.db") + dstDB := testDBAtPath(t, dstPath, "dst") + defer dstDB.Close() + + require.NoError(t, dstDB.CopyMetadataReplayFrom(srcPath), + "CopyMetadataReplayFrom") + + for _, ev := range []MetadataProjection{winner, loser} { + applied, err := dstDB.MetadataEventApplied(ctx, ev.EventOrigin, ev.OrderKey) + require.NoError(t, err, "MetadataEventApplied %s/%s", ev.EventOrigin, ev.OrderKey) + assert.True(t, applied, + "applied-event marker %s/%s must survive the copy", + ev.EventOrigin, ev.OrderKey) + } + + op, ok, err := dstDB.MetadataReplayStateOp(ctx, winner.SessionGID, winner.Field) + require.NoError(t, err, "MetadataReplayStateOp") + require.True(t, ok, "replay state must survive the copy") + assert.Equal(t, "rename", op) + + conflicts, err := dstDB.ListMetadataConflicts(ctx, []string{winner.SessionGID}) + require.NoError(t, err, "ListMetadataConflicts") + require.Len(t, conflicts, 1, "conflict row must survive the copy") + assert.Equal(t, "laptop-a1b2c3", conflicts[0].WinningOrigin) + assert.Equal(t, "peer-b4c5d6", conflicts[0].LosingOrigin) +} + +func TestCopyMetadataReplayFrom_NoSourceTables(t *testing.T) { + dir := t.TempDir() + ctx := context.Background() + + // Simulate an older source database that predates the metadata + // replay tables. + srcPath := filepath.Join(dir, "src.db") + srcDB := testDBAtPath(t, srcPath, "src") + for _, table := range []string{ + "metadata_applied_events", + "metadata_replay_state", + "metadata_conflicts", + } { + _, err := srcDB.getWriter().Exec("DROP TABLE " + table) + require.NoError(t, err, "drop %s", table) + } + require.NoError(t, srcDB.Close(), "Close src") + + dstPath := filepath.Join(dir, "dst.db") + dstDB := testDBAtPath(t, dstPath, "dst") + defer dstDB.Close() + + require.NoError(t, dstDB.CopyMetadataReplayFrom(srcPath), + "CopyMetadataReplayFrom with missing source tables") + + applied, err := dstDB.MetadataEventApplied(ctx, "laptop-a1b2c3", "0000000001") + require.NoError(t, err, "MetadataEventApplied") + assert.False(t, applied) +} + func TestCopySessionMetadataFrom(t *testing.T) { dir := t.TempDir() ctx := context.Background() diff --git a/internal/db/messages.go b/internal/db/messages.go index f4ca876f0..4aa8acdb6 100644 --- a/internal/db/messages.go +++ b/internal/db/messages.go @@ -2132,6 +2132,40 @@ func (db *DB) ToolResultEventFingerprintWithTimestampNormalizer( return b.String(), rows.Err() } +// SessionSubagentSessionIDs returns the distinct non-empty subagent_session_id +// values referenced by a session's tool calls and tool result events. Used by +// PG push to decide whether a session whose content otherwise matches PG still +// needs its message rows replaced so subagent links can be re-resolved. +func (db *DB) SessionSubagentSessionIDs(sessionID string) ([]string, error) { + rows, err := db.getReader().Query(` + SELECT DISTINCT subagent_session_id FROM ( + SELECT subagent_session_id FROM tool_calls + WHERE session_id = ? + UNION + SELECT subagent_session_id FROM tool_result_events + WHERE session_id = ? + ) + WHERE subagent_session_id IS NOT NULL AND subagent_session_id != ''`, + sessionID, sessionID, + ) + if err != nil { + return nil, fmt.Errorf( + "querying subagent session ids for %s: %w", sessionID, err, + ) + } + defer rows.Close() + + var ids []string + for rows.Next() { + var id string + if err := rows.Scan(&id); err != nil { + return nil, err + } + ids = append(ids, id) + } + return ids, rows.Err() +} + // GetMessageByOrdinal returns a single message by session ID and ordinal. func (db *DB) GetMessageByOrdinal( sessionID string, ordinal int, diff --git a/internal/db/metadata_baseline.go b/internal/db/metadata_baseline.go new file mode 100644 index 000000000..c57b74804 --- /dev/null +++ b/internal/db/metadata_baseline.go @@ -0,0 +1,140 @@ +package db + +import ( + "context" + "database/sql" + "fmt" +) + +// MetadataBaselineSnapshot captures existing local user curation that predates +// artifact metadata event recording. +type MetadataBaselineSnapshot struct { + Renames []MetadataBaselineRename + StarredSessionIDs []string + SoftDeletedIDs []string + Pins []MetadataBaselinePin +} + +// MetadataBaselineRename is one session display-name override. +type MetadataBaselineRename struct { + SessionID string + DisplayName *string +} + +// MetadataBaselinePin is one pinned message represented in metadata-event +// coordinates. +type MetadataBaselinePin struct { + SessionID string + SourceUUID string + Ordinal int + Note *string +} + +// MetadataBaselineSnapshot returns the current curation rows that need baseline +// metadata events during artifact sync initialization. +func (db *DB) MetadataBaselineSnapshot(ctx context.Context) (MetadataBaselineSnapshot, error) { + var snap MetadataBaselineSnapshot + + // Curation queries do not filter on deleted_at: a session sitting in + // trash at opt-in still baselines its name, star, and pins, so a later + // restore reaches peers with them instead of only the soft delete. + renameRows, err := db.getReader().QueryContext(ctx, ` + SELECT id, display_name + FROM sessions + WHERE display_name IS NOT NULL + ORDER BY id`) + if err != nil { + return snap, fmt.Errorf("listing baseline renames: %w", err) + } + defer renameRows.Close() + for renameRows.Next() { + var sessionID string + var displayName string + if err := renameRows.Scan(&sessionID, &displayName); err != nil { + return snap, fmt.Errorf("scanning baseline rename: %w", err) + } + displayNameCopy := displayName + snap.Renames = append(snap.Renames, MetadataBaselineRename{ + SessionID: sessionID, + DisplayName: &displayNameCopy, + }) + } + if err := renameRows.Err(); err != nil { + return snap, fmt.Errorf("iterating baseline renames: %w", err) + } + + starRows, err := db.getReader().QueryContext(ctx, ` + SELECT ss.session_id + FROM starred_sessions ss + JOIN sessions s + ON s.id = ss.session_id + ORDER BY ss.session_id`) + if err != nil { + return snap, fmt.Errorf("listing baseline stars: %w", err) + } + defer starRows.Close() + for starRows.Next() { + var sessionID string + if err := starRows.Scan(&sessionID); err != nil { + return snap, fmt.Errorf("scanning baseline star: %w", err) + } + snap.StarredSessionIDs = append(snap.StarredSessionIDs, sessionID) + } + if err := starRows.Err(); err != nil { + return snap, fmt.Errorf("iterating baseline stars: %w", err) + } + + deletedRows, err := db.getReader().QueryContext(ctx, ` + SELECT id + FROM sessions + WHERE deleted_at IS NOT NULL + ORDER BY id`) + if err != nil { + return snap, fmt.Errorf("listing baseline soft deletes: %w", err) + } + defer deletedRows.Close() + for deletedRows.Next() { + var sessionID string + if err := deletedRows.Scan(&sessionID); err != nil { + return snap, fmt.Errorf("scanning baseline soft delete: %w", err) + } + snap.SoftDeletedIDs = append(snap.SoftDeletedIDs, sessionID) + } + if err := deletedRows.Err(); err != nil { + return snap, fmt.Errorf("iterating baseline soft deletes: %w", err) + } + + pinRows, err := db.getReader().QueryContext(ctx, ` + SELECT p.session_id, COALESCE(m.source_uuid, ''), + m.ordinal, p.note + FROM pinned_messages p + JOIN sessions s + ON s.id = p.session_id + JOIN messages m + ON m.id = p.message_id + AND m.session_id = p.session_id + ORDER BY p.session_id, m.ordinal, p.id`) + if err != nil { + return snap, fmt.Errorf("listing baseline pins: %w", err) + } + defer pinRows.Close() + for pinRows.Next() { + var pin MetadataBaselinePin + var note sql.NullString + if err := pinRows.Scan( + &pin.SessionID, &pin.SourceUUID, &pin.Ordinal, ¬e, + ); err != nil { + return snap, fmt.Errorf("scanning baseline pin: %w", err) + } + if note.Valid { + noteCopy := note.String + pin.Note = ¬eCopy + } + snap.Pins = append(snap.Pins, pin) + } + if err := pinRows.Err(); err != nil { + return snap, fmt.Errorf("iterating baseline pins: %w", err) + } + + return snap, nil +} diff --git a/internal/db/metadata_baseline_test.go b/internal/db/metadata_baseline_test.go new file mode 100644 index 000000000..85857f75b --- /dev/null +++ b/internal/db/metadata_baseline_test.go @@ -0,0 +1,52 @@ +package db + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// A session that was renamed, starred, and pinned before the machine first +// opted into artifact sync must baseline that curation even while it sits in +// trash: only the soft delete would otherwise publish, and a later restore +// would reach peers without the name, star, or pin. +func TestMetadataBaselineSnapshotIncludesTrashedSessions(t *testing.T) { + d := testDB(t) + ctx := context.Background() + + require.NoError(t, d.UpsertSession(Session{ + ID: "s1", Project: "proj", Machine: "local", Agent: "claude", + MessageCount: 1, CreatedAt: "2026-01-01T00:00:00Z", + })) + require.NoError(t, d.InsertMessages([]Message{{ + SessionID: "s1", Ordinal: 0, Role: "user", Content: "hi", + ContentLength: 2, SourceUUID: "uuid-1", + }})) + name := "Kept name" + require.NoError(t, d.RenameSession("s1", &name)) + starred, err := d.StarSession("s1") + require.NoError(t, err) + require.True(t, starred) + msgs, err := d.GetAllMessages(ctx, "s1") + require.NoError(t, err) + require.Len(t, msgs, 1) + note := "kept pin" + _, err = d.PinMessage("s1", msgs[0].ID, ¬e) + require.NoError(t, err) + require.NoError(t, d.SoftDeleteSession("s1")) + + snap, err := d.MetadataBaselineSnapshot(ctx) + require.NoError(t, err) + + require.Len(t, snap.Renames, 1) + assert.Equal(t, "s1", snap.Renames[0].SessionID) + require.NotNil(t, snap.Renames[0].DisplayName) + assert.Equal(t, name, *snap.Renames[0].DisplayName) + assert.Equal(t, []string{"s1"}, snap.StarredSessionIDs) + assert.Equal(t, []string{"s1"}, snap.SoftDeletedIDs) + require.Len(t, snap.Pins, 1) + assert.Equal(t, "s1", snap.Pins[0].SessionID) + assert.Equal(t, "uuid-1", snap.Pins[0].SourceUUID) +} diff --git a/internal/db/metadata_replay.go b/internal/db/metadata_replay.go new file mode 100644 index 000000000..d01f9a15f --- /dev/null +++ b/internal/db/metadata_replay.go @@ -0,0 +1,830 @@ +package db + +import ( + "context" + "database/sql" + "encoding/json" + "errors" + "fmt" + "strings" + "time" +) + +// ErrMetadataTargetUnavailable means a metadata event depends on session or +// message content that is not durable locally yet. +var ErrMetadataTargetUnavailable = errors.New("metadata target unavailable") + +// MetadataPinProjection identifies a pinned message during metadata replay. +type MetadataPinProjection struct { + SourceUUID string `json:"source_uuid,omitempty"` + Ordinal int `json:"ordinal"` + Note *string `json:"note,omitempty"` +} + +// MetadataProjection is one decoded artifact metadata event ready for replay. +type MetadataProjection struct { + EventOrigin string + OrderKey string + HLC string + ArtifactHash string + SessionGID string + LocalSessionID string + Field string + Op string + Value string + DisplayName *string + Pin *MetadataPinProjection +} + +// MetadataApplyResult summarizes how replay handled an event. +type MetadataApplyResult struct { + Applied bool + Skipped bool + Conflict bool + Duplicate bool +} + +// MetadataConflict is a losing metadata value recorded during deterministic +// replay. +type MetadataConflict struct { + ID int64 `json:"id"` + SessionGID string `json:"session_gid"` + Field string `json:"field"` + WinningOrderKey string `json:"winning_order_key"` + LosingOrderKey string `json:"losing_order_key"` + WinningOrigin string `json:"winning_origin"` + LosingOrigin string `json:"losing_origin"` + WinningOp string `json:"winning_op"` + LosingOp string `json:"losing_op"` + WinningValue string `json:"winning_value"` + LosingValue string `json:"losing_value"` + CreatedAt string `json:"created_at"` +} + +type metadataReplayState struct { + OrderKey string + HLC string + ArtifactHash string + Origin string + Op string + Value string +} + +// MetadataEventIdentity identifies one immutable artifact metadata event. +type MetadataEventIdentity struct { + Origin string + OrderKey string +} + +// MetadataAppliedEventIdentities bulk-loads the events already durably handled +// by metadata replay. +func (db *DB) MetadataAppliedEventIdentities( + ctx context.Context, +) (map[MetadataEventIdentity]struct{}, error) { + rows, err := db.getReader().QueryContext(ctx, + `SELECT origin, order_key FROM metadata_applied_events`, + ) + if err != nil { + return nil, fmt.Errorf("listing applied metadata events: %w", err) + } + defer rows.Close() + + identities := make(map[MetadataEventIdentity]struct{}) + for rows.Next() { + var identity MetadataEventIdentity + if err := rows.Scan(&identity.Origin, &identity.OrderKey); err != nil { + return nil, fmt.Errorf("scanning applied metadata event: %w", err) + } + identities[identity] = struct{}{} + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("iterating applied metadata events: %w", err) + } + return identities, nil +} + +// MetadataEventApplied reports whether an artifact metadata event was already +// durably handled. +func (db *DB) MetadataEventApplied(ctx context.Context, origin, orderKey string) (bool, error) { + var exists int + err := db.getReader().QueryRowContext(ctx, + `SELECT 1 FROM metadata_applied_events + WHERE origin = ? AND order_key = ?`, + origin, orderKey, + ).Scan(&exists) + if err == sql.ErrNoRows { + return false, nil + } + if err != nil { + return false, fmt.Errorf("checking metadata event %s/%s: %w", origin, orderKey, err) + } + return true, nil +} + +// ListMetadataConflicts returns conflict rows for one or more global session +// identifiers. +func (db *DB) ListMetadataConflicts( + ctx context.Context, + sessionGIDs []string, +) ([]MetadataConflict, error) { + ids := uniqueNonEmptyStrings(sessionGIDs) + if len(ids) == 0 { + return []MetadataConflict{}, nil + } + placeholders := strings.TrimRight(strings.Repeat("?,", len(ids)), ",") + args := make([]any, len(ids)) + for i, id := range ids { + args[i] = id + } + rows, err := db.getReader().QueryContext(ctx, + `SELECT id, session_gid, field, winning_order_key, losing_order_key, + winning_origin, losing_origin, winning_op, losing_op, + winning_value, losing_value, created_at + FROM metadata_conflicts + WHERE session_gid IN (`+placeholders+`) + AND winning_origin <> losing_origin + ORDER BY created_at DESC, id DESC`, + args..., + ) + if err != nil { + return nil, fmt.Errorf("listing metadata conflicts: %w", err) + } + defer rows.Close() + + conflicts := []MetadataConflict{} + for rows.Next() { + var c MetadataConflict + if err := rows.Scan( + &c.ID, &c.SessionGID, &c.Field, + &c.WinningOrderKey, &c.LosingOrderKey, + &c.WinningOrigin, &c.LosingOrigin, + &c.WinningOp, &c.LosingOp, + &c.WinningValue, &c.LosingValue, + &c.CreatedAt, + ); err != nil { + return nil, fmt.Errorf("scanning metadata conflict: %w", err) + } + conflicts = append(conflicts, c) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("iterating metadata conflicts: %w", err) + } + return conflicts, nil +} + +// CountMetadataConflicts returns the total number of recorded metadata +// conflicts across all sessions. +func (db *DB) CountMetadataConflicts(ctx context.Context) (int, error) { + var count int + err := db.getReader().QueryRowContext(ctx, + `SELECT COUNT(*) FROM metadata_conflicts + WHERE winning_origin <> losing_origin`, + ).Scan(&count) + if err != nil { + return 0, fmt.Errorf("counting metadata conflicts: %w", err) + } + return count, nil +} + +// MarkMetadataEventApplied records a metadata event that was intentionally +// skipped, such as an unknown future op. +func (db *DB) MarkMetadataEventApplied(ctx context.Context, origin, orderKey, hash string) error { + db.mu.Lock() + defer db.mu.Unlock() + _, err := db.getWriter().ExecContext(ctx, + `INSERT OR IGNORE INTO metadata_applied_events + (origin, order_key, artifact_hash) + VALUES (?, ?, ?)`, + origin, orderKey, hash, + ) + if err != nil { + return fmt.Errorf("marking metadata event %s/%s applied: %w", origin, orderKey, err) + } + return nil +} + +// ApplyMetadataProjection applies one known metadata event if it wins the +// per-field LWW register, recording conflicts and the applied-event marker in +// the same transaction. +func (db *DB) ApplyMetadataProjection( + ctx context.Context, + ev MetadataProjection, +) (MetadataApplyResult, error) { + return db.applyMetadataProjection(ctx, ev, true) +} + +// RecordLocalMetadataProjection records the LWW register, conflict rows, and +// applied-event marker for a locally-originated metadata event whose session +// mutation the caller has already applied. It runs the same per-field LWW +// bookkeeping as replay but does not re-apply the mutation, so a later peer +// event with a lower order key cannot silently overwrite a newer local edit. +func (db *DB) RecordLocalMetadataProjection( + ctx context.Context, + ev MetadataProjection, +) (MetadataApplyResult, error) { + return db.applyMetadataProjection(ctx, ev, false) +} + +// MetadataReplayStateOp returns the current LWW operation recorded for a +// metadata field. +func (db *DB) MetadataReplayStateOp( + ctx context.Context, + sessionGID string, + field string, +) (string, bool, error) { + var op string + err := db.getReader().QueryRowContext(ctx, + `SELECT op FROM metadata_replay_state + WHERE session_gid = ? AND field = ?`, + sessionGID, field, + ).Scan(&op) + if err == sql.ErrNoRows { + return "", false, nil + } + if err != nil { + return "", false, fmt.Errorf("reading metadata replay state: %w", err) + } + return op, true, nil +} + +// ReapplyMetadataReplayState reapplies the current visible metadata projection +// for a session from the durable replay register. It does not alter LWW state or +// applied-event markers; it only repairs fields that content import may have +// overwritten or invalidated while replacing session/message rows. +func (db *DB) ReapplyMetadataReplayState( + ctx context.Context, + sessionGID string, + localSessionID string, +) (int, error) { + if err := db.requireWritable(); err != nil { + return 0, err + } + if strings.TrimSpace(sessionGID) == "" || strings.TrimSpace(localSessionID) == "" { + return 0, errors.New("metadata replay session id is required") + } + + db.mu.Lock() + defer db.mu.Unlock() + tx, err := db.getWriter().BeginTx(ctx, nil) + if err != nil { + return 0, fmt.Errorf("begin metadata reapply tx: %w", err) + } + defer func() { _ = tx.Rollback() }() + + rows, err := tx.QueryContext(ctx, + `SELECT field, order_key, hlc, artifact_hash, origin, op, value + FROM metadata_replay_state + WHERE session_gid = ? + ORDER BY field`, + sessionGID, + ) + if err != nil { + return 0, fmt.Errorf("reading metadata replay state: %w", err) + } + projections := []MetadataProjection{} + for rows.Next() { + var field string + var state metadataReplayState + if err := rows.Scan( + &field, &state.OrderKey, &state.HLC, &state.ArtifactHash, + &state.Origin, &state.Op, &state.Value, + ); err != nil { + rows.Close() + return 0, fmt.Errorf("scanning metadata replay state: %w", err) + } + ev, err := metadataReplayStateProjection(sessionGID, localSessionID, field, state) + if err != nil { + rows.Close() + return 0, err + } + projections = append(projections, ev) + } + if err := rows.Close(); err != nil { + return 0, fmt.Errorf("closing metadata replay state rows: %w", err) + } + if err := rows.Err(); err != nil { + return 0, fmt.Errorf("iterating metadata replay state: %w", err) + } + + applied := 0 + for _, ev := range projections { + if err := ctx.Err(); err != nil { + return applied, err + } + if err := applyMetadataProjectionTx(ctx, tx, ev); err != nil { + if errors.Is(err, ErrMetadataTargetUnavailable) { + continue + } + return applied, fmt.Errorf("reapplying metadata replay state: %w", err) + } + applied++ + } + if err := tx.Commit(); err != nil { + return applied, fmt.Errorf("commit metadata reapply tx: %w", err) + } + return applied, nil +} + +func (db *DB) applyMetadataProjection( + ctx context.Context, + ev MetadataProjection, + applyMutation bool, +) (MetadataApplyResult, error) { + if ev.EventOrigin == "" || ev.OrderKey == "" || ev.ArtifactHash == "" { + return MetadataApplyResult{}, errors.New("metadata projection event identity is required") + } + if ev.SessionGID == "" || ev.LocalSessionID == "" || ev.Field == "" || ev.Op == "" { + return MetadataApplyResult{}, errors.New("metadata projection target is required") + } + + db.mu.Lock() + defer db.mu.Unlock() + tx, err := db.getWriter().BeginTx(ctx, nil) + if err != nil { + return MetadataApplyResult{}, fmt.Errorf("begin metadata replay tx: %w", err) + } + defer func() { _ = tx.Rollback() }() + + already, err := metadataEventAppliedTx(ctx, tx, ev.EventOrigin, ev.OrderKey) + if err != nil { + return MetadataApplyResult{}, err + } + if already { + if err := tx.Commit(); err != nil { + return MetadataApplyResult{}, fmt.Errorf("commit metadata replay duplicate: %w", err) + } + return MetadataApplyResult{Skipped: true, Duplicate: true}, nil + } + + current, hasCurrent, err := metadataReplayStateTx(ctx, tx, ev.SessionGID, ev.Field) + if err != nil { + return MetadataApplyResult{}, err + } + result := MetadataApplyResult{} + if hasCurrent && ev.OrderKey <= current.OrderKey { + if metadataStateDiffers(current.Op, current.Value, ev.Op, ev.Value) && + metadataConflictOriginsDiffer(current.Origin, ev.EventOrigin) { + if err := insertMetadataConflictTx(ctx, tx, metadataConflict{ + sessionGID: ev.SessionGID, + field: ev.Field, + winningOrderKey: current.OrderKey, + losingOrderKey: ev.OrderKey, + winningOrigin: current.Origin, + losingOrigin: ev.EventOrigin, + winningOp: current.Op, + losingOp: ev.Op, + winningValue: current.Value, + losingValue: ev.Value, + }); err != nil { + return MetadataApplyResult{}, err + } + result.Conflict = true + } + if err := markMetadataEventAppliedTx(ctx, tx, ev.EventOrigin, ev.OrderKey, ev.ArtifactHash); err != nil { + return MetadataApplyResult{}, err + } + if err := tx.Commit(); err != nil { + return MetadataApplyResult{}, fmt.Errorf("commit metadata replay loser: %w", err) + } + result.Skipped = true + return result, nil + } + + if hasCurrent && metadataStateDiffers(current.Op, current.Value, ev.Op, ev.Value) && + metadataConflictOriginsDiffer(ev.EventOrigin, current.Origin) { + if err := insertMetadataConflictTx(ctx, tx, metadataConflict{ + sessionGID: ev.SessionGID, + field: ev.Field, + winningOrderKey: ev.OrderKey, + losingOrderKey: current.OrderKey, + winningOrigin: ev.EventOrigin, + losingOrigin: current.Origin, + winningOp: ev.Op, + losingOp: current.Op, + winningValue: ev.Value, + losingValue: current.Value, + }); err != nil { + return MetadataApplyResult{}, err + } + result.Conflict = true + } + if applyMutation { + if err := applyMetadataProjectionTx(ctx, tx, ev); err != nil { + return MetadataApplyResult{}, err + } + } + if err := upsertMetadataReplayStateTx(ctx, tx, ev); err != nil { + return MetadataApplyResult{}, err + } + if err := markMetadataEventAppliedTx(ctx, tx, ev.EventOrigin, ev.OrderKey, ev.ArtifactHash); err != nil { + return MetadataApplyResult{}, err + } + if err := tx.Commit(); err != nil { + return MetadataApplyResult{}, fmt.Errorf("commit metadata replay: %w", err) + } + result.Applied = true + return result, nil +} + +func metadataEventAppliedTx(ctx context.Context, tx *sql.Tx, origin, orderKey string) (bool, error) { + var exists int + err := tx.QueryRowContext(ctx, + `SELECT 1 FROM metadata_applied_events + WHERE origin = ? AND order_key = ?`, + origin, orderKey, + ).Scan(&exists) + if err == sql.ErrNoRows { + return false, nil + } + if err != nil { + return false, fmt.Errorf("checking metadata event %s/%s: %w", origin, orderKey, err) + } + return true, nil +} + +func metadataReplayStateTx( + ctx context.Context, + tx *sql.Tx, + sessionGID, field string, +) (metadataReplayState, bool, error) { + var state metadataReplayState + err := tx.QueryRowContext(ctx, + `SELECT order_key, hlc, artifact_hash, origin, op, value + FROM metadata_replay_state + WHERE session_gid = ? AND field = ?`, + sessionGID, field, + ).Scan( + &state.OrderKey, &state.HLC, &state.ArtifactHash, + &state.Origin, &state.Op, &state.Value, + ) + if err == sql.ErrNoRows { + return metadataReplayState{}, false, nil + } + if err != nil { + return metadataReplayState{}, false, fmt.Errorf("reading metadata replay state: %w", err) + } + return state, true, nil +} + +func metadataReplayStateProjection( + sessionGID string, + localSessionID string, + field string, + state metadataReplayState, +) (MetadataProjection, error) { + ev := MetadataProjection{ + EventOrigin: state.Origin, + OrderKey: state.OrderKey, + HLC: state.HLC, + ArtifactHash: state.ArtifactHash, + SessionGID: sessionGID, + LocalSessionID: localSessionID, + Field: field, + Op: state.Op, + Value: state.Value, + } + switch state.Op { + case "rename": + var payload struct { + DisplayName *string `json:"display_name"` + } + if err := json.Unmarshal([]byte(state.Value), &payload); err != nil { + return MetadataProjection{}, fmt.Errorf("decoding rename metadata replay state: %w", err) + } + ev.DisplayName = payload.DisplayName + case "pin", "unpin": + var pin MetadataPinProjection + if err := json.Unmarshal([]byte(state.Value), &pin); err != nil { + return MetadataProjection{}, fmt.Errorf("decoding pin metadata replay state: %w", err) + } + ev.Pin = &pin + } + return ev, nil +} + +// metadataEventWallTime renders a replayed event's HLC wall-clock portion in +// the sessions.deleted_at column format, so trash retention is anchored to +// when the deletion happened on the authoring machine rather than when each +// peer imported the event. The HLC shape ("-<20-digit logical>", wall +// layout without ":" separators) is pinned by the artifact format contract in +// internal/artifact. Returns "" when the HLC has no parseable wall portion. +func metadataEventWallTime(hlc string) string { + idx := strings.LastIndex(hlc, "-") + if idx <= 0 { + return "" + } + wall, err := time.Parse("2006-01-02T150405.000000000Z", hlc[:idx]) + if err != nil { + return "" + } + return wall.UTC().Format("2006-01-02T15:04:05.000Z") +} + +func applyMetadataProjectionTx(ctx context.Context, tx *sql.Tx, ev MetadataProjection) error { + switch ev.Op { + case "rename": + if err := requireMetadataSessionTx(ctx, tx, ev.LocalSessionID); err != nil { + return err + } + _, err := tx.ExecContext(ctx, + `UPDATE sessions + SET display_name = ?, + local_modified_at = strftime('%Y-%m-%dT%H:%M:%fZ','now') + WHERE id = ?`, + ev.DisplayName, ev.LocalSessionID, + ) + return err + case "soft_delete": + if err := requireMetadataSessionTx(ctx, tx, ev.LocalSessionID); err != nil { + return err + } + _, err := tx.ExecContext(ctx, + `UPDATE sessions + SET deleted_at = COALESCE(deleted_at, NULLIF(?, ''), strftime('%Y-%m-%dT%H:%M:%fZ','now')), + local_modified_at = strftime('%Y-%m-%dT%H:%M:%fZ','now') + WHERE id = ?`, + metadataEventWallTime(ev.HLC), ev.LocalSessionID, + ) + return err + case "restore": + if err := requireMetadataSessionTx(ctx, tx, ev.LocalSessionID); err != nil { + return err + } + _, err := tx.ExecContext(ctx, + `UPDATE sessions + SET deleted_at = NULL, + local_modified_at = strftime('%Y-%m-%dT%H:%M:%fZ','now') + WHERE id = ?`, + ev.LocalSessionID, + ) + return err + case "star": + if err := requireMetadataSessionTx(ctx, tx, ev.LocalSessionID); err != nil { + return err + } + _, err := tx.ExecContext(ctx, + `INSERT OR IGNORE INTO starred_sessions (session_id) + VALUES (?)`, + ev.LocalSessionID, + ) + return err + case "unstar": + if err := requireMetadataSessionTx(ctx, tx, ev.LocalSessionID); err != nil { + return err + } + _, err := tx.ExecContext(ctx, + `DELETE FROM starred_sessions WHERE session_id = ?`, + ev.LocalSessionID, + ) + return err + case "pin": + if err := requireMetadataSessionTx(ctx, tx, ev.LocalSessionID); err != nil { + return err + } + if ev.Pin == nil { + return errors.New("pin metadata event missing pin payload") + } + return applyMetadataPinTx(ctx, tx, ev.LocalSessionID, *ev.Pin) + case "unpin": + if err := requireMetadataSessionTx(ctx, tx, ev.LocalSessionID); err != nil { + return err + } + if ev.Pin == nil { + return errors.New("unpin metadata event missing pin payload") + } + return unpinMetadataTx(ctx, tx, ev.LocalSessionID, *ev.Pin) + case "purge": + return applyMetadataPurgeTx(ctx, tx, ev.LocalSessionID) + default: + return fmt.Errorf("unsupported metadata op %q", ev.Op) + } +} + +func applyMetadataPinTx(ctx context.Context, tx *sql.Tx, sessionID string, pin MetadataPinProjection) error { + msg, ok, err := metadataPinTargetTx(ctx, tx, sessionID, pin) + if err != nil { + return err + } + if !ok { + return fmt.Errorf("%w: pin target %s ordinal %d", + ErrMetadataTargetUnavailable, sessionID, pin.Ordinal) + } + _, err = tx.ExecContext(ctx, + `INSERT INTO pinned_messages (session_id, message_id, ordinal, note) + VALUES (?, ?, ?, ?) + ON CONFLICT(session_id, message_id) DO UPDATE SET note = excluded.note`, + sessionID, msg.id, msg.ordinal, pin.Note, + ) + return err +} + +func applyMetadataPurgeTx(ctx context.Context, tx *sql.Tx, sessionID string) error { + aliasIDs, err := sessionAliasIDsTx(tx, "id = ?", sessionID) + if err != nil { + return err + } + if _, err := tx.ExecContext(ctx, + `INSERT OR IGNORE INTO excluded_sessions (id) VALUES (?)`, + sessionID, + ); err != nil { + return err + } + for _, aliasID := range aliasIDs { + if err := excludeSessionIDTx(tx, aliasID); err != nil { + return fmt.Errorf("excluding metadata purge alias %s: %w", aliasID, err) + } + } + _, err = tx.ExecContext(ctx, + `DELETE FROM sessions WHERE id = ?`, + sessionID, + ) + return err +} + +func requireMetadataSessionTx(ctx context.Context, tx *sql.Tx, id string) error { + var exists int + err := tx.QueryRowContext(ctx, + `SELECT 1 FROM sessions WHERE id = ?`, + id, + ).Scan(&exists) + if err == sql.ErrNoRows { + return fmt.Errorf("%w: session %s", ErrMetadataTargetUnavailable, id) + } + if err != nil { + return fmt.Errorf("checking metadata session %s: %w", id, err) + } + return nil +} + +type metadataPinTarget struct { + id int64 + ordinal int +} + +func metadataPinTargetTx( + ctx context.Context, + tx *sql.Tx, + sessionID string, + pin MetadataPinProjection, +) (metadataPinTarget, bool, error) { + if pin.SourceUUID != "" { + target, ok, err := metadataPinTargetByQueryTx(ctx, tx, + `SELECT id, ordinal FROM messages + WHERE session_id = ? AND source_uuid = ? + ORDER BY ordinal LIMIT 1`, + sessionID, pin.SourceUUID, + ) + if err != nil || ok { + return target, ok, err + } + } + return metadataPinTargetByQueryTx(ctx, tx, + `SELECT id, ordinal FROM messages + WHERE session_id = ? AND ordinal = ? + ORDER BY id LIMIT 1`, + sessionID, pin.Ordinal, + ) +} + +func metadataPinTargetByQueryTx( + ctx context.Context, + tx *sql.Tx, + query string, + args ...any, +) (metadataPinTarget, bool, error) { + var target metadataPinTarget + err := tx.QueryRowContext(ctx, query, args...).Scan(&target.id, &target.ordinal) + if err == sql.ErrNoRows { + return metadataPinTarget{}, false, nil + } + if err != nil { + return metadataPinTarget{}, false, fmt.Errorf("finding metadata pin target: %w", err) + } + return target, true, nil +} + +func unpinMetadataTx( + ctx context.Context, + tx *sql.Tx, + sessionID string, + pin MetadataPinProjection, +) error { + if pin.SourceUUID != "" { + res, err := tx.ExecContext(ctx, + `DELETE FROM pinned_messages + WHERE session_id = ? + AND message_id IN ( + SELECT id FROM messages + WHERE session_id = ? AND source_uuid = ? + )`, + sessionID, sessionID, pin.SourceUUID, + ) + if err != nil { + return err + } + if n, _ := res.RowsAffected(); n > 0 { + return nil + } + } + _, err := tx.ExecContext(ctx, + `DELETE FROM pinned_messages + WHERE session_id = ? + AND message_id IN ( + SELECT id FROM messages + WHERE session_id = ? AND ordinal = ? + )`, + sessionID, sessionID, pin.Ordinal, + ) + return err +} + +func upsertMetadataReplayStateTx(ctx context.Context, tx *sql.Tx, ev MetadataProjection) error { + _, err := tx.ExecContext(ctx, + `INSERT INTO metadata_replay_state + (session_gid, field, order_key, hlc, artifact_hash, origin, op, value, updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, strftime('%Y-%m-%dT%H:%M:%fZ','now')) + ON CONFLICT(session_gid, field) DO UPDATE SET + order_key = excluded.order_key, + hlc = excluded.hlc, + artifact_hash = excluded.artifact_hash, + origin = excluded.origin, + op = excluded.op, + value = excluded.value, + updated_at = excluded.updated_at`, + ev.SessionGID, ev.Field, ev.OrderKey, ev.HLC, ev.ArtifactHash, + ev.EventOrigin, ev.Op, ev.Value, + ) + if err != nil { + return fmt.Errorf("upserting metadata replay state: %w", err) + } + return nil +} + +func markMetadataEventAppliedTx( + ctx context.Context, + tx *sql.Tx, + origin, orderKey, hash string, +) error { + _, err := tx.ExecContext(ctx, + `INSERT OR IGNORE INTO metadata_applied_events + (origin, order_key, artifact_hash) + VALUES (?, ?, ?)`, + origin, orderKey, hash, + ) + if err != nil { + return fmt.Errorf("marking metadata event %s/%s applied: %w", origin, orderKey, err) + } + return nil +} + +type metadataConflict struct { + sessionGID string + field string + winningOrderKey string + losingOrderKey string + winningOrigin string + losingOrigin string + winningOp string + losingOp string + winningValue string + losingValue string +} + +func insertMetadataConflictTx(ctx context.Context, tx *sql.Tx, c metadataConflict) error { + _, err := tx.ExecContext(ctx, + `INSERT OR IGNORE INTO metadata_conflicts + (session_gid, field, winning_order_key, losing_order_key, + winning_origin, losing_origin, winning_op, losing_op, + winning_value, losing_value) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, + c.sessionGID, c.field, c.winningOrderKey, c.losingOrderKey, + c.winningOrigin, c.losingOrigin, c.winningOp, c.losingOp, + c.winningValue, c.losingValue, + ) + if err != nil { + return fmt.Errorf("inserting metadata conflict: %w", err) + } + return nil +} + +func metadataStateDiffers(aOp, aValue, bOp, bValue string) bool { + return aOp != bOp || aValue != bValue +} + +func metadataConflictOriginsDiffer(winningOrigin, losingOrigin string) bool { + return winningOrigin != losingOrigin +} + +func uniqueNonEmptyStrings(values []string) []string { + seen := make(map[string]bool, len(values)) + unique := make([]string, 0, len(values)) + for _, value := range values { + value = strings.TrimSpace(value) + if value == "" || seen[value] { + continue + } + seen[value] = true + unique = append(unique, value) + } + return unique +} diff --git a/internal/db/metadata_replay_test.go b/internal/db/metadata_replay_test.go new file mode 100644 index 000000000..44494e1fa --- /dev/null +++ b/internal/db/metadata_replay_test.go @@ -0,0 +1,254 @@ +package db + +import ( + "context" + "database/sql" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestApplyMetadataProjectionSessionOps(t *testing.T) { + ctx := context.Background() + d := testDB(t) + insertSession(t, d, "s1", "alpha") + + applyMetadataProjectionForTest(t, d, MetadataProjection{ + EventOrigin: "laptop-a1b2c3", + OrderKey: "0001-a", + HLC: "0001", + ArtifactHash: "a", + SessionGID: "desktop-d4e5f6~s1", + LocalSessionID: "s1", + Field: "starred", + Op: "star", + Value: "star", + }) + assert.Equal(t, 1, metadataTableCount(t, d, "starred_sessions", "session_id = 's1'")) + + applyMetadataProjectionForTest(t, d, MetadataProjection{ + EventOrigin: "laptop-a1b2c3", + OrderKey: "0002-a", + HLC: "0002", + ArtifactHash: "a", + SessionGID: "desktop-d4e5f6~s1", + LocalSessionID: "s1", + Field: "starred", + Op: "unstar", + Value: "unstar", + }) + assert.Equal(t, 0, metadataTableCount(t, d, "starred_sessions", "session_id = 's1'")) + + applyMetadataProjectionForTest(t, d, MetadataProjection{ + EventOrigin: "laptop-a1b2c3", + OrderKey: "0003-a", + HLC: "0003", + ArtifactHash: "a", + SessionGID: "desktop-d4e5f6~s1", + LocalSessionID: "s1", + Field: "deleted_at", + Op: "soft_delete", + Value: "soft_delete", + }) + got, err := d.GetSession(ctx, "s1") + require.NoError(t, err) + assert.Nil(t, got) + + applyMetadataProjectionForTest(t, d, MetadataProjection{ + EventOrigin: "laptop-a1b2c3", + OrderKey: "0004-a", + HLC: "0004", + ArtifactHash: "a", + SessionGID: "desktop-d4e5f6~s1", + LocalSessionID: "s1", + Field: "deleted_at", + Op: "restore", + Value: "restore", + }) + got, err = d.GetSession(ctx, "s1") + require.NoError(t, err) + require.NotNil(t, got) + assert.Equal(t, "s1", got.ID) + + applyMetadataProjectionForTest(t, d, MetadataProjection{ + EventOrigin: "laptop-a1b2c3", + OrderKey: "0005-a", + HLC: "0005", + ArtifactHash: "a", + SessionGID: "desktop-d4e5f6~s1", + LocalSessionID: "s1", + Field: "purge", + Op: "purge", + Value: "purge", + }) + got, err = d.GetSessionFull(ctx, "s1") + require.NoError(t, err) + assert.Nil(t, got) + assert.Equal(t, 1, metadataTableCount(t, d, "excluded_sessions", "id = 's1'")) +} + +func TestApplyMetadataProjectionRequiresPinTargetBeforeMarkingApplied(t *testing.T) { + d := testDB(t) + insertSession(t, d, "s1", "alpha") + + result, err := d.ApplyMetadataProjection(context.Background(), MetadataProjection{ + EventOrigin: "laptop-a1b2c3", + OrderKey: "0001-a", + HLC: "0001", + ArtifactHash: "a", + SessionGID: "desktop-d4e5f6~s1", + LocalSessionID: "s1", + Field: "pin:source_uuid:missing", + Op: "pin", + Value: `{"ordinal":1,"source_uuid":"missing"}`, + Pin: &MetadataPinProjection{ + SourceUUID: "missing", + Ordinal: 1, + }, + }) + + require.ErrorIs(t, err, ErrMetadataTargetUnavailable) + assert.False(t, result.Applied) + applied, checkErr := d.MetadataEventApplied(context.Background(), "laptop-a1b2c3", "0001-a") + require.NoError(t, checkErr) + assert.False(t, applied) + assert.Equal(t, 0, metadataTableCount(t, d, "metadata_replay_state", "session_gid = 'desktop-d4e5f6~s1'")) +} + +func TestApplyMetadataProjectionDoesNotConflictSameOriginSequentialEdits(t *testing.T) { + d := testDB(t) + insertSession(t, d, "s1", "alpha") + firstName := "one" + secondName := "two" + + events := []MetadataProjection{ + { + EventOrigin: "laptop-a1b2c3", + OrderKey: "0001-a", + HLC: "0001", + ArtifactHash: "a", + SessionGID: "desktop-d4e5f6~s1", + LocalSessionID: "s1", + Field: "starred", + Op: "star", + Value: "star", + }, + { + EventOrigin: "laptop-a1b2c3", + OrderKey: "0002-a", + HLC: "0002", + ArtifactHash: "b", + SessionGID: "desktop-d4e5f6~s1", + LocalSessionID: "s1", + Field: "starred", + Op: "unstar", + Value: "unstar", + }, + { + EventOrigin: "laptop-a1b2c3", + OrderKey: "0003-a", + HLC: "0003", + ArtifactHash: "c", + SessionGID: "desktop-d4e5f6~s1", + LocalSessionID: "s1", + Field: "display_name", + Op: "rename", + Value: `{"display_name":"one"}`, + DisplayName: &firstName, + }, + { + EventOrigin: "laptop-a1b2c3", + OrderKey: "0004-a", + HLC: "0004", + ArtifactHash: "d", + SessionGID: "desktop-d4e5f6~s1", + LocalSessionID: "s1", + Field: "display_name", + Op: "rename", + Value: `{"display_name":"two"}`, + DisplayName: &secondName, + }, + } + + for _, ev := range events { + result, err := d.ApplyMetadataProjection(context.Background(), ev) + require.NoError(t, err) + assert.True(t, result.Applied) + assert.False(t, result.Conflict) + } + + assert.Equal(t, 0, metadataTableCount(t, d, "metadata_conflicts", "1 = 1")) +} + +func TestMetadataConflictQueriesIgnoreSameOriginRows(t *testing.T) { + ctx := context.Background() + d := testDB(t) + _, err := d.getWriter().ExecContext(ctx, + `INSERT INTO metadata_conflicts + (session_gid, field, winning_order_key, losing_order_key, + winning_origin, losing_origin, winning_op, losing_op, + winning_value, losing_value) + VALUES + ('desktop-d4e5f6~s1', 'display_name', '0002-a', '0001-a', + 'desktop-d4e5f6', 'desktop-d4e5f6', 'rename', 'rename', + '{"display_name":"two"}', '{"display_name":"one"}'), + ('desktop-d4e5f6~s1', 'display_name', '0003-b', '0002-a', + 'laptop-a1b2c3', 'desktop-d4e5f6', 'rename', 'rename', + '{"display_name":"peer"}', '{"display_name":"two"}')`, + ) + require.NoError(t, err) + + conflicts, err := d.ListMetadataConflicts(ctx, []string{"desktop-d4e5f6~s1"}) + require.NoError(t, err) + require.Len(t, conflicts, 1) + assert.Equal(t, "laptop-a1b2c3", conflicts[0].WinningOrigin) + assert.Equal(t, "desktop-d4e5f6", conflicts[0].LosingOrigin) + + count, err := d.CountMetadataConflicts(ctx) + require.NoError(t, err) + assert.Equal(t, 1, count) +} + +func TestApplyMetadataProjectionPurgeExcludesFallbackAlias(t *testing.T) { + d := testDB(t) + filePath := "/tmp/vibe/session_20260616_083518_abc123/messages.jsonl" + insertSession(t, d, "vibe:canonical-1", "alpha", func(s *Session) { + s.Agent = "vibe" + s.FilePath = &filePath + }) + + applyMetadataProjectionForTest(t, d, MetadataProjection{ + EventOrigin: "laptop-a1b2c3", + OrderKey: "0001-a", + HLC: "0001", + ArtifactHash: "a", + SessionGID: "laptop-a1b2c3~vibe:canonical-1", + LocalSessionID: "vibe:canonical-1", + Field: "purge", + Op: "purge", + Value: "purge", + }) + + assert.Equal(t, 1, metadataTableCount(t, d, "excluded_sessions", "id = 'vibe:canonical-1'")) + assert.Equal(t, 1, metadataTableCount(t, d, "excluded_sessions", "id = 'vibe:session_20260616_083518_abc123'")) +} + +func applyMetadataProjectionForTest(t *testing.T, d *DB, ev MetadataProjection) { + t.Helper() + result, err := d.ApplyMetadataProjection(context.Background(), ev) + require.NoError(t, err) + assert.True(t, result.Applied) + assert.False(t, result.Duplicate) +} + +func metadataTableCount(t *testing.T, d *DB, table, where string) int { + t.Helper() + var count int + err := d.Reader().QueryRow("SELECT COUNT(*) FROM " + table + " WHERE " + where).Scan(&count) + if err == sql.ErrNoRows { + return 0 + } + require.NoError(t, err) + return count +} diff --git a/internal/db/orphaned.go b/internal/db/orphaned.go index 935db50fd..33428e4d3 100644 --- a/internal/db/orphaned.go +++ b/internal/db/orphaned.go @@ -309,9 +309,13 @@ func (d *DB) CopyTrashedDataFrom(sourcePath string) (int, error) { return count, nil } -// CopySyncStateFrom copies pg_sync_state rows from the source database into the -// current database. ResyncAll uses this to preserve durable local sync metadata -// such as the PG push owner marker across the temp-DB swap. +// CopySyncStateFrom copies durable pg_sync_state rows from the source database +// into the current database. ResyncAll uses this to preserve durable local sync +// metadata across the temp-DB swap: the PG push owner marker plus the artifact +// ledger state written by internal/artifact (origin identity, metadata HLC, +// and per-session import/export watermarks, all keyed with an "artifact_" +// prefix). Transient bookkeeping such as last_sync_* timestamps is +// deliberately left behind so the rebuilt DB reports its own sync times. func (d *DB) CopySyncStateFrom(sourcePath string) error { d.mu.Lock() defer d.mu.Unlock() @@ -347,13 +351,83 @@ func (d *DB) CopySyncStateFrom(sourcePath string) error { _, err = conn.ExecContext(ctx, ` INSERT OR REPLACE INTO main.pg_sync_state (key, value) SELECT key, value FROM old_db.pg_sync_state - WHERE key = 'pg_push_marker_id'`) + WHERE key = 'pg_push_marker_id' + OR key LIKE 'artifact\_%' ESCAPE '\'`) if err != nil { return fmt.Errorf("copying sync state: %w", err) } return nil } +// CopyMetadataReplayFrom copies the durable artifact metadata replay tables +// (metadata_applied_events, metadata_replay_state, metadata_conflicts) from +// the source database. ResyncAll uses this so previously applied peer +// metadata events are not replayed against an empty LWW register after a +// full rebuild, which would let old events overwrite newer local state. +func (d *DB) CopyMetadataReplayFrom(sourcePath string) error { + d.mu.Lock() + defer d.mu.Unlock() + + ctx := context.Background() + conn, err := d.getWriter().Conn(ctx) + if err != nil { + return fmt.Errorf("acquiring connection: %w", err) + } + defer conn.Close() + + if _, err := conn.ExecContext( + ctx, "ATTACH DATABASE ? AS old_db", sourcePath, + ); err != nil { + return fmt.Errorf("attaching source db: %w", err) + } + defer func() { + _, _ = execWithoutCancel(ctx, conn, "DETACH DATABASE old_db") + }() + + copies := []struct { + table string + columns string + }{ + { + "metadata_applied_events", + "origin, order_key, artifact_hash, applied_at", + }, + { + "metadata_replay_state", + "session_gid, field, order_key, hlc, artifact_hash, " + + "origin, op, value, updated_at", + }, + { + "metadata_conflicts", + "session_gid, field, winning_order_key, losing_order_key, " + + "winning_origin, losing_origin, winning_op, losing_op, " + + "winning_value, losing_value, created_at", + }, + } + for _, c := range copies { + // Older databases may predate the metadata replay tables. + var tableExists int + err := conn.QueryRowContext(ctx, + "SELECT 1 FROM old_db.sqlite_master WHERE type='table' AND name=?", + c.table, + ).Scan(&tableExists) + if err != nil { + if errors.Is(err, sql.ErrNoRows) { + continue + } + return fmt.Errorf("probing %s table: %w", c.table, err) + } + if _, err := conn.ExecContext(ctx, fmt.Sprintf( + `INSERT OR IGNORE INTO main.%s (%s) + SELECT %s FROM old_db.%s`, + c.table, c.columns, c.columns, c.table, + )); err != nil { + return fmt.Errorf("copying %s: %w", c.table, err) + } + } + return nil +} + // CopyExcludedSessionsFrom copies the excluded_sessions table // from the source DB so permanently deleted sessions survive // full DB rebuilds. The source must not have active connections. diff --git a/internal/db/read_only_test.go b/internal/db/read_only_test.go index 03c9dd1cc..d84028939 100644 --- a/internal/db/read_only_test.go +++ b/internal/db/read_only_test.go @@ -234,7 +234,8 @@ func TestOpenReadOnlyWriteMethodsReturnErrReadOnly(t *testing.T) { return readonly.InsertMessages(nil) }) requireReadOnlyOp(t, "BulkStarSessions", func() error { - return readonly.BulkStarSessions(nil) + _, err := readonly.BulkStarSessions(nil) + return err }) requireReadOnlyOp(t, "DeleteParserExcludedSessions", func() error { _, err := readonly.DeleteParserExcludedSessions(nil) diff --git a/internal/db/schema.sql b/internal/db/schema.sql index 2978024aa..bc723a072 100644 --- a/internal/db/schema.sql +++ b/internal/db/schema.sql @@ -516,6 +516,48 @@ CREATE TABLE IF NOT EXISTS pg_sync_state ( value TEXT NOT NULL ); +-- Metadata replay: durable record of handled artifact events. This lets +-- import scan the small append-only feed repeatedly without replaying +-- duplicates or relying on an unsafe max-HLC watermark. +CREATE TABLE IF NOT EXISTS metadata_applied_events ( + origin TEXT NOT NULL, + order_key TEXT NOT NULL, + artifact_hash TEXT NOT NULL, + applied_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')), + PRIMARY KEY (origin, order_key) +); + +-- Per-field LWW winners for metadata replay. +CREATE TABLE IF NOT EXISTS metadata_replay_state ( + session_gid TEXT NOT NULL, + field TEXT NOT NULL, + order_key TEXT NOT NULL, + hlc TEXT NOT NULL, + artifact_hash TEXT NOT NULL, + origin TEXT NOT NULL, + op TEXT NOT NULL, + value TEXT NOT NULL DEFAULT '', + updated_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')), + PRIMARY KEY (session_gid, field) +); + +-- Losing metadata values that were overridden by deterministic LWW replay. +CREATE TABLE IF NOT EXISTS metadata_conflicts ( + id INTEGER PRIMARY KEY, + session_gid TEXT NOT NULL, + field TEXT NOT NULL, + winning_order_key TEXT NOT NULL, + losing_order_key TEXT NOT NULL, + winning_origin TEXT NOT NULL, + losing_origin TEXT NOT NULL, + winning_op TEXT NOT NULL, + losing_op TEXT NOT NULL, + winning_value TEXT NOT NULL DEFAULT '', + losing_value TEXT NOT NULL DEFAULT '', + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')), + UNIQUE(session_gid, field, winning_order_key, losing_order_key) +); + -- Model pricing for cost calculation CREATE TABLE IF NOT EXISTS model_pricing ( model_pattern TEXT PRIMARY KEY, diff --git a/internal/db/sessions.go b/internal/db/sessions.go index 55e1b57a4..c5ccf8fc2 100644 --- a/internal/db/sessions.go +++ b/internal/db/sessions.go @@ -2205,6 +2205,35 @@ func sqliteLikeEscape(value string) string { return value } +// ListOwnedSessionIDsForExport returns the IDs of locally-owned, non-deleted +// sessions for artifact export, ordered by id. Unlike ListSessions it does not +// apply the sidebar visibility filter (message_count > 0), so zero-message +// usage-only sessions are still published. +func (db *DB) ListOwnedSessionIDsForExport(ctx context.Context) ([]string, error) { + rows, err := db.getReader().QueryContext(ctx, + `SELECT id FROM sessions + WHERE machine = 'local' AND deleted_at IS NULL + ORDER BY id`, + ) + if err != nil { + return nil, fmt.Errorf("listing sessions for artifact export: %w", err) + } + defer rows.Close() + + var ids []string + for rows.Next() { + var id string + if err := rows.Scan(&id); err != nil { + return nil, fmt.Errorf("scanning export session ID: %w", err) + } + ids = append(ids, id) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("iterating export session IDs: %w", err) + } + return ids, nil +} + // GetDataVersionByPath returns the minimum data_version for // sessions matching a file_path. Returns 0 when no session // exists for the path. @@ -2604,6 +2633,32 @@ func (db *DB) GetBranches( return branches, rows.Err() } +// MachineSessionCounts returns the number of non-deleted sessions per machine, +// keyed by machine name. Child sessions are included so the count reflects the +// full corpus owned by or imported from each origin. +func (db *DB) MachineSessionCounts(ctx context.Context) (map[string]int, error) { + rows, err := db.getReader().QueryContext(ctx, + `SELECT machine, COUNT(*) FROM sessions + WHERE deleted_at IS NULL + GROUP BY machine`, + ) + if err != nil { + return nil, fmt.Errorf("counting sessions per machine: %w", err) + } + defer rows.Close() + + counts := map[string]int{} + for rows.Next() { + var machine string + var count int + if err := rows.Scan(&machine, &count); err != nil { + return nil, fmt.Errorf("scanning machine session count: %w", err) + } + counts[machine] = count + } + return counts, rows.Err() +} + // scanSessionRows iterates rows and scans each using // scanSessionRow. func scanSessionRows(rows *sql.Rows) ([]Session, error) { @@ -2750,8 +2805,16 @@ func (db *DB) SoftDeleteSession(id string) error { // deleted_at. Sessions that are already soft-deleted are skipped. // Returns the count of newly deleted rows. func (db *DB) SoftDeleteSessions(ids []string) (int, error) { + deleted, err := db.SoftDeleteSessionsReturningIDs(ids) + return len(deleted), err +} + +// SoftDeleteSessionsReturningIDs marks multiple sessions as deleted by setting +// deleted_at and returns the IDs that were newly deleted. Sessions that are +// already soft-deleted are skipped. +func (db *DB) SoftDeleteSessionsReturningIDs(ids []string) ([]string, error) { if len(ids) == 0 { - return 0, nil + return []string{}, nil } db.mu.Lock() @@ -2759,11 +2822,11 @@ func (db *DB) SoftDeleteSessions(ids []string) (int, error) { tx, err := db.getWriter().Begin() if err != nil { - return 0, fmt.Errorf("beginning soft-delete tx: %w", err) + return nil, fmt.Errorf("beginning soft-delete tx: %w", err) } defer func() { _ = tx.Rollback() }() - total := 0 + deleted := make([]string, 0, len(ids)) const batchSize = 500 for i := 0; i < len(ids); i += batchSize { end := min(i+batchSize, len(ids)) @@ -2775,24 +2838,100 @@ func (db *DB) SoftDeleteSessions(ids []string) (int, error) { } placeholders := strings.Repeat(",?", len(batch))[1:] - res, err := tx.Exec( + rows, err := tx.Query( + `SELECT id FROM sessions + WHERE id IN (`+placeholders+`) AND deleted_at IS NULL`, + args..., + ) + if err != nil { + return nil, fmt.Errorf("loading soft-delete batch ids: %w", err) + } + batchIDs := map[string]struct{}{} + for rows.Next() { + var id string + if err := rows.Scan(&id); err != nil { + _ = rows.Close() + return nil, fmt.Errorf("scanning soft-delete batch id: %w", err) + } + batchIDs[id] = struct{}{} + } + if err := rows.Close(); err != nil { + return nil, fmt.Errorf("closing soft-delete batch ids: %w", err) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("iterating soft-delete batch ids: %w", err) + } + for _, id := range batch { + if _, ok := batchIDs[id]; ok { + deleted = append(deleted, id) + delete(batchIDs, id) + } + } + + if _, err := tx.Exec( `UPDATE sessions SET deleted_at = strftime('%Y-%m-%dT%H:%M:%fZ','now'), local_modified_at = strftime('%Y-%m-%dT%H:%M:%fZ','now') WHERE id IN (`+placeholders+`) AND deleted_at IS NULL`, args..., - ) - if err != nil { - return 0, fmt.Errorf("soft-deleting batch: %w", err) + ); err != nil { + return nil, fmt.Errorf("soft-deleting batch: %w", err) } - n, _ := res.RowsAffected() - total += int(n) } if err := tx.Commit(); err != nil { - return 0, fmt.Errorf("committing soft-delete tx: %w", err) + return nil, fmt.Errorf("committing soft-delete tx: %w", err) } - return total, nil + return deleted, nil +} + +// TrashedSessionIDs returns requested sessions that currently exist in the +// trash. The result preserves first-seen input order and omits duplicates. +func (db *DB) TrashedSessionIDs(ids []string) ([]string, error) { + if len(ids) == 0 { + return []string{}, nil + } + trashed := make(map[string]struct{}, len(ids)) + const batchSize = 500 + for i := 0; i < len(ids); i += batchSize { + end := min(i+batchSize, len(ids)) + batch := ids[i:end] + args := make([]any, len(batch)) + for j, id := range batch { + args[j] = id + } + placeholders := strings.Repeat(",?", len(batch))[1:] + rows, err := db.getReader().Query( + `SELECT id FROM sessions + WHERE id IN (`+placeholders+`) AND deleted_at IS NOT NULL`, + args..., + ) + if err != nil { + return nil, fmt.Errorf("loading trashed session ids: %w", err) + } + for rows.Next() { + var id string + if err := rows.Scan(&id); err != nil { + _ = rows.Close() + return nil, fmt.Errorf("scanning trashed session id: %w", err) + } + trashed[id] = struct{}{} + } + if err := rows.Close(); err != nil { + return nil, fmt.Errorf("closing trashed session ids: %w", err) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("iterating trashed session ids: %w", err) + } + } + out := make([]string, 0, len(trashed)) + for _, id := range ids { + if _, ok := trashed[id]; ok { + out = append(out, id) + delete(trashed, id) + } + } + return out, nil } // RestoreSession clears deleted_at, making the session visible again. diff --git a/internal/db/starred.go b/internal/db/starred.go index 926ec3412..0b04faf70 100644 --- a/internal/db/starred.go +++ b/internal/db/starred.go @@ -3,6 +3,7 @@ package db import ( "context" "database/sql" + "errors" "fmt" ) @@ -39,18 +40,22 @@ func (db *DB) StarSession(sessionID string) (bool, error) { return true, nil // already starred } -// UnstarSession removes a session's star. -func (db *DB) UnstarSession(sessionID string) error { +// UnstarSession removes a session's star and reports whether a row was removed. +func (db *DB) UnstarSession(sessionID string) (bool, error) { db.mu.Lock() defer db.mu.Unlock() - _, err := db.getWriter().Exec( + res, err := db.getWriter().Exec( "DELETE FROM starred_sessions WHERE session_id = ?", sessionID, ) if err != nil { - return fmt.Errorf("unstarring session %s: %w", sessionID, err) + return false, fmt.Errorf("unstarring session %s: %w", sessionID, err) } - return nil + n, err := res.RowsAffected() + if err != nil { + return false, fmt.Errorf("checking unstar result for %s: %w", sessionID, err) + } + return n > 0, nil } // ListStarredSessionIDs returns all starred session IDs. @@ -78,12 +83,12 @@ func (db *DB) ListStarredSessionIDs( // BulkStarSessions stars multiple sessions in a single transaction. // Used for migrating localStorage stars to the database. -func (db *DB) BulkStarSessions(sessionIDs []string) error { +func (db *DB) BulkStarSessions(sessionIDs []string) ([]string, error) { if err := db.requireWritable(); err != nil { - return err + return nil, err } if len(sessionIDs) == 0 { - return nil + return nil, nil } db.mu.Lock() @@ -91,27 +96,50 @@ func (db *DB) BulkStarSessions(sessionIDs []string) error { tx, err := db.getWriter().Begin() if err != nil { - return fmt.Errorf("beginning transaction: %w", err) + return nil, fmt.Errorf("beginning transaction: %w", err) } defer func() { _ = tx.Rollback() }() - // Use INSERT ... SELECT ... WHERE EXISTS so that stale IDs - // (sessions pruned or deleted from disk) are silently skipped - // instead of causing a foreign key violation that aborts the - // entire migration transaction. - stmt, err := tx.Prepare(` - INSERT OR IGNORE INTO starred_sessions (session_id) - SELECT ? WHERE EXISTS (SELECT 1 FROM sessions WHERE id = ?)`) + // Check existence separately from the insert so stale IDs (sessions pruned + // or deleted from disk) are silently skipped instead of aborting the + // migration transaction, and so the caller learns which sessions were + // actually starred and need a converging metadata event. + exists, err := tx.Prepare(`SELECT 1 FROM sessions WHERE id = ?`) + if err != nil { + return nil, fmt.Errorf("preparing existence statement: %w", err) + } + defer exists.Close() + insert, err := tx.Prepare( + `INSERT OR IGNORE INTO starred_sessions (session_id) VALUES (?)`) if err != nil { - return fmt.Errorf("preparing statement: %w", err) + return nil, fmt.Errorf("preparing statement: %w", err) } - defer stmt.Close() + defer insert.Close() + starred := make([]string, 0, len(sessionIDs)) for _, id := range sessionIDs { - if _, err := stmt.Exec(id, id); err != nil { - return fmt.Errorf("starring session %s: %w", id, err) + var one int + switch err := exists.QueryRow(id).Scan(&one); { + case errors.Is(err, sql.ErrNoRows): + continue + case err != nil: + return nil, fmt.Errorf("checking session %s: %w", id, err) + } + res, err := insert.Exec(id) + if err != nil { + return nil, fmt.Errorf("starring session %s: %w", id, err) + } + rowsAffected, err := res.RowsAffected() + if err != nil { + return nil, fmt.Errorf("checking star insert result for %s: %w", id, err) + } + if rowsAffected > 0 { + starred = append(starred, id) } } - return tx.Commit() + if err := tx.Commit(); err != nil { + return nil, fmt.Errorf("committing star transaction: %w", err) + } + return starred, nil } diff --git a/internal/db/store.go b/internal/db/store.go index 7b3d27f7b..ad24f0d3f 100644 --- a/internal/db/store.go +++ b/internal/db/store.go @@ -58,6 +58,9 @@ type Store interface { GetSessionVersion(id string) (count int, version int64, ok bool) // Metadata. + ListMetadataConflicts(ctx context.Context, sessionGIDs []string) ([]MetadataConflict, error) + CountMetadataConflicts(ctx context.Context) (int, error) + MachineSessionCounts(ctx context.Context) (map[string]int, error) GetStats(ctx context.Context, excludeOneShot, excludeAutomated bool) (Stats, error) GetProjects(ctx context.Context, excludeOneShot, excludeAutomated bool) ([]ProjectInfo, error) GetAgents(ctx context.Context, excludeOneShot, excludeAutomated bool) ([]AgentInfo, error) @@ -92,9 +95,9 @@ type Store interface { // Stars. StarSession(sessionID string) (bool, error) - UnstarSession(sessionID string) error + UnstarSession(sessionID string) (bool, error) ListStarredSessionIDs(ctx context.Context) ([]string, error) - BulkStarSessions(sessionIDs []string) error + BulkStarSessions(sessionIDs []string) ([]string, error) // Pins. PinMessage(sessionID string, messageID int64, note *string) (int64, error) @@ -128,6 +131,7 @@ type Store interface { RenameSession(id string, displayName *string) error SoftDeleteSession(id string) error SoftDeleteSessions(ids []string) (int, error) + SoftDeleteSessionsReturningIDs(ids []string) ([]string, error) RestoreSession(id string) (int64, error) DeleteSessionIfTrashed(id string) (int64, error) ListTrashedSessions(ctx context.Context) ([]Session, error) diff --git a/internal/db/store_contract_test.go b/internal/db/store_contract_test.go index 8eee23dd6..6ad31bba0 100644 --- a/internal/db/store_contract_test.go +++ b/internal/db/store_contract_test.go @@ -129,6 +129,7 @@ func TestStoreContract(t *testing.T) { {"stars_and_pins", contractStarsAndPins}, {"analytics_trends_and_usage", contractAnalyticsTrendsAndUsage}, {"local_only_methods", contractLocalOnlyMethods}, + {"machine_counts_and_conflicts", contractMachineCountsAndConflicts}, } for _, backend := range storeContractBackends() { @@ -243,6 +244,32 @@ func contractSessionsCursorFiltersAndDates( require.Equal(t, []string{"linux", "mac"}, machines) } +func contractMachineCountsAndConflicts( + t *testing.T, + store Store, + _ storeContractFixture, + _ storeContractBackend, +) { + t.Helper() + ctx := context.Background() + + counts, err := store.MachineSessionCounts(ctx) + require.NoError(t, err) + // The seed places non-deleted sessions on both machines. + require.Positive(t, counts["linux"]) + require.Positive(t, counts["mac"]) + for machine := range counts { + require.Contains(t, []string{"linux", "mac"}, machine, + "unexpected machine in session counts: %q", machine) + } + + // No conflicts are seeded; every backend reports zero (read-only + // mirrors do not carry the local metadata ledger at all). + conflicts, err := store.CountMetadataConflicts(ctx) + require.NoError(t, err) + require.Equal(t, 0, conflicts) +} + func contractMessagesOrderingAndToolResults( t *testing.T, store Store, @@ -385,13 +412,23 @@ func contractStarsAndPins( ok, err := store.StarSession(fixture.alphaID) require.NoError(t, err) require.True(t, ok) - require.NoError(t, store.BulkStarSessions([]string{fixture.gammaID, "missing-session"})) + bulkStarred, err := store.BulkStarSessions([]string{fixture.gammaID, "missing-session"}) + require.NoError(t, err) + require.Equal(t, []string{fixture.gammaID}, bulkStarred) + bulkStarred, err = store.BulkStarSessions([]string{fixture.alphaID, fixture.gammaID}) + require.NoError(t, err) + require.Empty(t, bulkStarred) stars, err := store.ListStarredSessionIDs(ctx) require.NoError(t, err) require.ElementsMatch(t, []string{fixture.alphaID, fixture.gammaID}, stars) - require.NoError(t, store.UnstarSession(fixture.gammaID)) + removed, err := store.UnstarSession(fixture.gammaID) + require.NoError(t, err) + require.True(t, removed) + removed, err = store.UnstarSession(fixture.gammaID) + require.NoError(t, err) + require.False(t, removed) stars, err = store.ListStarredSessionIDs(ctx) require.NoError(t, err) require.Equal(t, []string{fixture.alphaID}, stars) diff --git a/internal/duckdb/curation.go b/internal/duckdb/curation.go index d0bfd5ab3..c51fa9330 100644 --- a/internal/duckdb/curation.go +++ b/internal/duckdb/curation.go @@ -12,8 +12,8 @@ func (s *Store) StarSession(sessionID string) (bool, error) { return false, db.ErrReadOnly } -func (s *Store) UnstarSession(sessionID string) error { - return db.ErrReadOnly +func (s *Store) UnstarSession(sessionID string) (bool, error) { + return false, db.ErrReadOnly } func (s *Store) ListStarredSessionIDs(ctx context.Context) ([]string, error) { @@ -35,8 +35,8 @@ func (s *Store) ListStarredSessionIDs(ctx context.Context) ([]string, error) { return ids, rows.Err() } -func (s *Store) BulkStarSessions(sessionIDs []string) error { - return db.ErrReadOnly +func (s *Store) BulkStarSessions(sessionIDs []string) ([]string, error) { + return nil, db.ErrReadOnly } func (s *Store) PinMessage(sessionID string, messageID int64, note *string) (int64, error) { diff --git a/internal/duckdb/metadata.go b/internal/duckdb/metadata.go new file mode 100644 index 000000000..dd2736052 --- /dev/null +++ b/internal/duckdb/metadata.go @@ -0,0 +1,22 @@ +package duckdb + +import ( + "context" + + "go.kenn.io/agentsview/internal/db" +) + +// ListMetadataConflicts returns no rows for DuckDB read mode because the local +// artifact metadata ledger is not part of the analytical mirror. +func (s *Store) ListMetadataConflicts( + context.Context, + []string, +) ([]db.MetadataConflict, error) { + return []db.MetadataConflict{}, nil +} + +// CountMetadataConflicts returns zero for DuckDB read mode because the local +// artifact metadata ledger is not part of the analytical mirror. +func (s *Store) CountMetadataConflicts(context.Context) (int, error) { + return 0, nil +} diff --git a/internal/duckdb/store.go b/internal/duckdb/store.go index c5f61b37f..4e2cad097 100644 --- a/internal/duckdb/store.go +++ b/internal/duckdb/store.go @@ -654,6 +654,31 @@ func (s *Store) GetAgents(ctx context.Context, excludeOneShot, excludeAutomated return out, rows.Err() } +// MachineSessionCounts returns the number of non-deleted sessions per machine, +// keyed by machine name. +func (s *Store) MachineSessionCounts(ctx context.Context) (map[string]int, error) { + rows, err := s.duck.QueryContext(ctx, + `SELECT machine, COUNT(*) FROM sessions + WHERE deleted_at IS NULL + GROUP BY machine`, + ) + if err != nil { + return nil, err + } + defer rows.Close() + + counts := map[string]int{} + for rows.Next() { + var machine string + var count int + if err := rows.Scan(&machine, &count); err != nil { + return nil, err + } + counts[machine] = count + } + return counts, rows.Err() +} + func (s *Store) GetMachines(ctx context.Context, excludeOneShot, excludeAutomated bool) ([]string, error) { rows, err := s.queryContext(ctx, `SELECT DISTINCT machine FROM sessions WHERE `+ diff --git a/internal/duckdb/store_contract_test.go b/internal/duckdb/store_contract_test.go index bc506d5ad..5e1a87fdd 100644 --- a/internal/duckdb/store_contract_test.go +++ b/internal/duckdb/store_contract_test.go @@ -233,6 +233,15 @@ func duckContractSessionsCursorsAndMetadata( machines, err := store.GetMachines(ctx, false, false) require.NoError(t, err) require.Equal(t, []string{"test-machine"}, machines) + + counts, err := store.MachineSessionCounts(ctx) + require.NoError(t, err) + require.Len(t, counts, 1) + require.Positive(t, counts["test-machine"]) + + conflicts, err := store.CountMetadataConflicts(ctx) + require.NoError(t, err) + require.Equal(t, 0, conflicts) } func duckContractMessagesSearchAndSecrets( @@ -310,8 +319,10 @@ func duckContractReadOnlyCuration( ok, err := store.StarSession(fixture.betaID) require.ErrorIs(t, err, db.ErrReadOnly) require.False(t, ok) - require.ErrorIs(t, store.UnstarSession(fixture.alphaID), db.ErrReadOnly) - require.ErrorIs(t, store.BulkStarSessions([]string{fixture.betaID}), db.ErrReadOnly) + _, err = store.UnstarSession(fixture.alphaID) + require.ErrorIs(t, err, db.ErrReadOnly) + _, bulkErr := store.BulkStarSessions([]string{fixture.betaID}) + require.ErrorIs(t, bulkErr, db.ErrReadOnly) pinID, err := store.PinMessage(fixture.alphaID, 1, nil) require.ErrorIs(t, err, db.ErrReadOnly) diff --git a/internal/duckdb/store_test.go b/internal/duckdb/store_test.go index 0d27bbfa6..f9d42c79b 100644 --- a/internal/duckdb/store_test.go +++ b/internal/duckdb/store_test.go @@ -688,8 +688,10 @@ func TestStoreCurationMethods(t *testing.T) { ok, err := store.StarSession(fixture.betaID) require.ErrorIs(t, err, db.ErrReadOnly) assert.False(t, ok) - require.ErrorIs(t, store.BulkStarSessions([]string{fixture.betaID}), db.ErrReadOnly) - require.ErrorIs(t, store.UnstarSession(fixture.alphaID), db.ErrReadOnly) + _, bulkErr := store.BulkStarSessions([]string{fixture.betaID}) + require.ErrorIs(t, bulkErr, db.ErrReadOnly) + _, err = store.UnstarSession(fixture.alphaID) + require.ErrorIs(t, err, db.ErrReadOnly) starred, err = store.ListStarredSessionIDs(ctx) require.NoError(t, err) assert.Equal(t, []string{fixture.alphaID}, starred) diff --git a/internal/duckdb/stubs.go b/internal/duckdb/stubs.go index 91ea32e52..c8c445345 100644 --- a/internal/duckdb/stubs.go +++ b/internal/duckdb/stubs.go @@ -15,9 +15,12 @@ func (s *Store) GetInsight(_ context.Context, _ int64) (*db.Insight, error) { re func (s *Store) GetCachedInsight(_ context.Context, _ string) (*db.Insight, error) { return nil, nil } -func (s *Store) RenameSession(_ string, _ *string) error { return db.ErrReadOnly } -func (s *Store) SoftDeleteSession(_ string) error { return db.ErrReadOnly } -func (s *Store) SoftDeleteSessions(_ []string) (int, error) { return 0, db.ErrReadOnly } +func (s *Store) RenameSession(_ string, _ *string) error { return db.ErrReadOnly } +func (s *Store) SoftDeleteSession(_ string) error { return db.ErrReadOnly } +func (s *Store) SoftDeleteSessions(_ []string) (int, error) { return 0, db.ErrReadOnly } +func (s *Store) SoftDeleteSessionsReturningIDs(_ []string) ([]string, error) { + return nil, db.ErrReadOnly +} func (s *Store) RestoreSession(_ string) (int64, error) { return 0, db.ErrReadOnly } func (s *Store) DeleteSessionIfTrashed(_ string) (int64, error) { return 0, db.ErrReadOnly } func (s *Store) ListTrashedSessions(_ context.Context) ([]db.Session, error) { diff --git a/internal/e2e/artifact_sync_test.go b/internal/e2e/artifact_sync_test.go new file mode 100644 index 000000000..5d406b3b1 --- /dev/null +++ b/internal/e2e/artifact_sync_test.go @@ -0,0 +1,344 @@ +//go:build e2e + +package e2e + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "net/url" + "os" + "path/filepath" + "sort" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "go.kenn.io/agentsview/internal/artifact" + "go.kenn.io/agentsview/internal/config" + "go.kenn.io/agentsview/internal/db" + "go.kenn.io/agentsview/internal/dbtest" + "go.kenn.io/agentsview/internal/parser" + "go.kenn.io/agentsview/internal/server" + syncpkg "go.kenn.io/agentsview/internal/sync" +) + +const e2eToken = "artifact-e2e-token" + +func TestArtifactSyncTwoInstanceFolderAndHTTP(t *testing.T) { + ctx := context.Background() + root := preservedWorkspace(t) + share := filepath.Join(root, "share") + require.NoError(t, os.MkdirAll(share, 0o755)) + + a := openE2ENode(t, filepath.Join(root, "node-a"), "laptop-a1b2c3") + defer a.Close() + b := openE2ENode(t, filepath.Join(root, "node-b"), "desktop-d4e5f6") + defer b.Close() + + seedE2ESession(t, a.DB, "sess-1", "alpha", "world") + + folderSync(t, a, share) + folderSync(t, b, share) + + importedID := a.Origin + "~sess-1" + assertSessionProject(t, b.DB, importedID, "alpha") + assertSearchFinds(t, b.DB, "world", importedID) + + renameWithMetadata(t, a, "sess-1", "Alpha from laptop") + folderSync(t, a, share) + folderSync(t, b, share) + assertSessionDisplayName(t, b.DB, importedID, "Alpha from laptop") + + renameWithMetadata(t, b, importedID, "Bravo from desktop") + folderSync(t, b, share) + folderSync(t, a, share) + assertSessionDisplayName(t, a.DB, "sess-1", "Bravo from desktop") + + b.Close() + b = openE2ENode(t, filepath.Join(root, "node-b"), "desktop-d4e5f6") + defer b.Close() + + require.NoError(t, a.DB.ReplaceSessionMessages("sess-1", []db.Message{ + {SessionID: "sess-1", Ordinal: 0, Role: "user", Content: "hello", ContentLength: 5}, + {SessionID: "sess-1", Ordinal: 1, Role: "assistant", Content: "planet", ContentLength: 6}, + })) + _, err := artifact.Export(ctx, a.DB, filepath.Join(a.DataDir, "artifacts"), a.Origin) + require.NoError(t, err) + postOriginArtifacts(t, a, b, a.Origin) + + assertMessagesContain(t, b.DB, importedID, "planet") + assertSearchFinds(t, b.DB, "planet", importedID) + + writeForeignRename(t, b, "writer-a1b2c3", importedID, "Fork one") + writeForeignRename(t, b, "writer-b1b2c3", importedID, "Fork two") + imported, err := artifact.ImportDetailed( + ctx, + b.DB, + filepath.Join(b.DataDir, "artifacts"), + b.Origin, + ) + require.NoError(t, err) + assert.NotZero(t, imported.Metadata) + + conflicts, err := b.DB.ListMetadataConflicts(ctx, []string{importedID}) + require.NoError(t, err) + require.NotEmpty(t, conflicts) + assert.Equal(t, "display_name", conflicts[0].Field) + + apiConflicts := getMetadataConflicts(t, b, importedID) + require.NotEmpty(t, apiConflicts.Conflicts) + assert.Equal(t, importedID, apiConflicts.Conflicts[0].SessionGID) +} + +type e2eNode struct { + DataDir string + DBPath string + Origin string + DB *db.DB + Server *httptest.Server +} + +func openE2ENode(t *testing.T, dataDir, origin string) *e2eNode { + t.Helper() + require.NoError(t, os.MkdirAll(dataDir, 0o755)) + dbPath := filepath.Join(dataDir, "sessions.db") + database, err := db.Open(dbPath) + require.NoError(t, err) + + emptyAgentDir := filepath.Join(dataDir, "empty-agent-dir") + require.NoError(t, os.MkdirAll(emptyAgentDir, 0o755)) + broadcaster := server.NewBroadcaster(0) + engine := syncpkg.NewEngine(database, syncpkg.EngineConfig{ + AgentDirs: map[parser.AgentType][]string{ + parser.AgentClaude: {emptyAgentDir}, + }, + Machine: origin, + Emitter: broadcaster, + }) + cfg := config.Config{ + Host: "127.0.0.1", + Port: 0, + DataDir: dataDir, + DBPath: dbPath, + WriteTimeout: 30 * time.Second, + RequireAuth: true, + AuthToken: e2eToken, + ArtifactOriginID: origin, + } + srv := server.New(cfg, database, engine, server.WithBroadcaster(broadcaster)) + return &e2eNode{ + DataDir: dataDir, + DBPath: dbPath, + Origin: origin, + DB: database, + Server: httptest.NewServer(srv.Handler()), + } +} + +func (n *e2eNode) Close() { + if n == nil { + return + } + if n.Server != nil { + n.Server.Close() + n.Server = nil + } + if n.DB != nil { + n.DB.Close() + n.DB = nil + } +} + +func preservedWorkspace(t *testing.T) string { + t.Helper() + root, err := os.MkdirTemp("", "agentsview-artifact-e2e-*") + require.NoError(t, err) + t.Cleanup(func() { + if t.Failed() { + t.Logf("preserved artifact sync e2e workspace: %s", root) + return + } + require.NoError(t, os.RemoveAll(root)) + }) + return root +} + +func seedE2ESession(t *testing.T, database *db.DB, id, project, assistantText string) { + t.Helper() + started := "2026-06-14T01:02:03Z" + ended := "2026-06-14T01:03:03Z" + first := "hello" + dbtest.SeedSession(t, database, id, project, func(s *db.Session) { + s.MessageCount = 2 + s.UserMessageCount = 1 + s.FirstMessage = &first + s.StartedAt = &started + s.EndedAt = &ended + }) + require.NoError(t, database.ReplaceSessionMessages(id, []db.Message{ + {SessionID: id, Ordinal: 0, Role: "user", Content: "hello", ContentLength: 5}, + {SessionID: id, Ordinal: 1, Role: "assistant", Content: assistantText, ContentLength: len(assistantText)}, + })) +} + +func folderSync(t *testing.T, n *e2eNode, share string) artifact.SyncResult { + t.Helper() + res, err := artifact.SyncFolder(context.Background(), n.DB, artifact.SyncOptions{ + DataDir: n.DataDir, + Target: share, + Origin: n.Origin, + }) + require.NoError(t, err) + return res +} + +func renameWithMetadata(t *testing.T, n *e2eNode, sessionID, displayName string) { + t.Helper() + require.NoError(t, n.DB.RenameSession(sessionID, &displayName)) + // A local edit records its own replay register in this node's db, exactly as + // the rename handler does. + appendRenameArtifact(t, n.DB, n.DataDir, n.Origin, sessionID, displayName) +} + +func writeForeignRename(t *testing.T, n *e2eNode, origin, sessionID, displayName string) { + t.Helper() + // A foreign origin's event arrives as an artifact file written by another + // machine. Record it through a throwaway db so it is not pre-marked applied + // in this node's db, leaving the real import to replay it. + scratch, err := db.Open(filepath.Join(t.TempDir(), "scratch.db")) + require.NoError(t, err) + t.Cleanup(func() { scratch.Close() }) + appendRenameArtifact(t, scratch, n.DataDir, origin, sessionID, displayName) +} + +func appendRenameArtifact(t *testing.T, database *db.DB, dataDir, origin, sessionID, displayName string) { + t.Helper() + value, err := json.Marshal(struct { + DisplayName *string `json:"display_name"` + }{DisplayName: &displayName}) + require.NoError(t, err) + recorder := artifact.NewMetadataRecorder(database, artifact.MetadataRecorderOptions{ + DataDir: dataDir, + Origin: origin, + }) + _, err = recorder.Append(context.Background(), artifact.MetadataEventInput{ + SessionID: sessionID, + Op: artifact.MetadataOpRename, + Value: json.RawMessage(value), + }) + require.NoError(t, err) +} + +func postOriginArtifacts(t *testing.T, from, to *e2eNode, origin string) { + t.Helper() + for _, kind := range []string{ + artifact.KindSegments, + artifact.KindRaw, + artifact.KindManifests, + artifact.KindMeta, + artifact.KindCheckpoints, + } { + pattern := filepath.Join(from.DataDir, "artifacts", origin, kind, "*") + paths, err := filepath.Glob(pattern) + require.NoError(t, err) + sort.Strings(paths) + for _, path := range paths { + info, err := os.Stat(path) + require.NoError(t, err) + if !info.Mode().IsRegular() { + continue + } + postArtifact(t, to, origin, kind, filepath.Base(path), path) + } + } +} + +func postArtifact(t *testing.T, to *e2eNode, origin, kind, name, path string) { + t.Helper() + body, err := os.ReadFile(path) + require.NoError(t, err) + req, err := http.NewRequest( + http.MethodPost, + to.Server.URL+"/api/v1/artifacts/"+origin+"/"+kind+"/"+url.PathEscape(name), + bytes.NewReader(body), + ) + require.NoError(t, err) + req.Header.Set("Authorization", "Bearer "+e2eToken) + req.Header.Set("Content-Type", "application/octet-stream") + resp, err := http.DefaultClient.Do(req) + require.NoError(t, err) + defer resp.Body.Close() + require.Equal(t, http.StatusOK, resp.StatusCode) +} + +type metadataConflictsResponse struct { + Conflicts []db.MetadataConflict `json:"conflicts"` +} + +func getMetadataConflicts(t *testing.T, n *e2eNode, sessionID string) metadataConflictsResponse { + t.Helper() + req, err := http.NewRequest( + http.MethodGet, + n.Server.URL+"/api/v1/sessions/"+url.PathEscape(sessionID)+"/metadata-conflicts", + nil, + ) + require.NoError(t, err) + req.Header.Set("Authorization", "Bearer "+e2eToken) + resp, err := http.DefaultClient.Do(req) + require.NoError(t, err) + defer resp.Body.Close() + require.Equal(t, http.StatusOK, resp.StatusCode) + var out metadataConflictsResponse + require.NoError(t, json.NewDecoder(resp.Body).Decode(&out)) + return out +} + +func assertSessionProject(t *testing.T, database *db.DB, sessionID, project string) { + t.Helper() + sess, err := database.GetSessionFull(context.Background(), sessionID) + require.NoError(t, err) + require.NotNil(t, sess) + assert.Equal(t, project, sess.Project) +} + +func assertSessionDisplayName(t *testing.T, database *db.DB, sessionID, displayName string) { + t.Helper() + sess, err := database.GetSessionFull(context.Background(), sessionID) + require.NoError(t, err) + require.NotNil(t, sess) + require.NotNil(t, sess.DisplayName) + assert.Equal(t, displayName, *sess.DisplayName) +} + +func assertMessagesContain(t *testing.T, database *db.DB, sessionID, text string) { + t.Helper() + msgs, err := database.GetAllMessages(context.Background(), sessionID) + require.NoError(t, err) + for _, msg := range msgs { + if msg.Content == text { + return + } + } + require.Fail(t, fmt.Sprintf("session %s messages did not contain %q", sessionID, text)) +} + +func assertSearchFinds(t *testing.T, database *db.DB, query, sessionID string) { + t.Helper() + page, err := database.Search(context.Background(), db.SearchFilter{ + Query: query, + Limit: 10, + }) + require.NoError(t, err) + for _, result := range page.Results { + if result.SessionID == sessionID { + return + } + } + require.Fail(t, fmt.Sprintf("search %q did not find %s", query, sessionID)) +} diff --git a/internal/postgres/collision_pgtest_test.go b/internal/postgres/collision_pgtest_test.go index e1ea596d8..cc46da747 100644 --- a/internal/postgres/collision_pgtest_test.go +++ b/internal/postgres/collision_pgtest_test.go @@ -4,15 +4,873 @@ package postgres import ( "context" + "database/sql" "path/filepath" "testing" + "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "go.kenn.io/agentsview/internal/artifact" "go.kenn.io/agentsview/internal/db" ) +func TestPushArtifactNativeAndImportedCopiesShareOriginIdentity(t *testing.T) { + pgURL := testPGURL(t) + ctx := context.Background() + const originA = "origin-a1b2c3" + const originB = "origin-b4c5d6" + const nativeID = "native-id" + const canonicalID = originA + "~" + nativeID + const childID = "child-id" + const canonicalChildID = originA + "~" + childID + + for _, tc := range []struct { + name string + schema string + importerFirst bool + }{ + { + name: "origin pushes first", + schema: "agentsview_artifact_identity_origin_first_test", + }, + { + name: "importer pushes first", + schema: "agentsview_artifact_identity_importer_first_test", + importerFirst: true, + }, + } { + t.Run(tc.name, func(t *testing.T) { + pg, err := Open(pgURL, tc.schema, true) + require.NoError(t, err, "Open") + t.Cleanup(func() { require.NoError(t, pg.Close()) }) + _, err = pg.Exec(`DROP SCHEMA IF EXISTS ` + tc.schema + ` CASCADE`) + require.NoError(t, err, "drop schema") + require.NoError(t, EnsureSchema(ctx, pg, tc.schema), "EnsureSchema") + + originDB, err := db.Open(filepath.Join(t.TempDir(), "origin.db")) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, originDB.Close()) }) + require.NoError(t, artifact.AdoptOrigin(originDB, originA)) + require.NoError(t, originDB.UpsertSession(db.Session{ + ID: nativeID, Project: "alpha", Machine: "local", Agent: "claude", + MessageCount: 1, UserMessageCount: 1, + CreatedAt: "2026-01-01T00:00:00Z", + })) + require.NoError(t, originDB.ReplaceSessionMessages(nativeID, []db.Message{{ + SessionID: nativeID, Ordinal: 0, Role: "user", + Content: "hello", ContentLength: 5, + }})) + parentID := nativeID + require.NoError(t, originDB.UpsertSession(db.Session{ + ID: childID, Project: "alpha", Machine: "local", Agent: "claude", + MessageCount: 1, UserMessageCount: 1, + CreatedAt: "2026-01-01T00:00:01Z", ParentSessionID: &parentID, + })) + require.NoError(t, originDB.ReplaceSessionMessages(childID, []db.Message{{ + SessionID: childID, Ordinal: 0, Role: "user", + Content: "child", ContentLength: 5, + }})) + + artifactRoot := t.TempDir() + _, err = artifact.Export(ctx, originDB, artifactRoot, originA) + require.NoError(t, err) + importerDB, err := db.Open(filepath.Join(t.TempDir(), "importer.db")) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, importerDB.Close()) }) + require.NoError(t, artifact.AdoptOrigin(importerDB, originB)) + imported, messages, err := artifact.Import(ctx, importerDB, artifactRoot, originB) + require.NoError(t, err) + require.Equal(t, 2, imported) + require.Equal(t, 2, messages) + + // Advance the origin after the importer captured its artifact snapshot. + // An origin-first push must not be rolled back when that stale replica + // subsequently pushes the same canonical owner marker. + require.NoError(t, originDB.UpsertSession(db.Session{ + ID: nativeID, Project: "fresh-origin", Machine: "local", Agent: "claude", + MessageCount: 1, UserMessageCount: 1, + CreatedAt: "2026-01-01T00:00:00Z", + })) + require.NoError(t, originDB.ReplaceSessionMessages(nativeID, []db.Message{{ + SessionID: nativeID, Ordinal: 0, Role: "user", + Content: "fresh", ContentLength: 5, + }})) + + originSync := &Sync{ + pg: pg, local: originDB, machine: "host-a", + schema: tc.schema, schemaDone: true, + } + importerSync := &Sync{ + pg: pg, local: importerDB, machine: "host-b", + schema: tc.schema, schemaDone: true, + } + pushers := []*Sync{originSync, importerSync} + if tc.importerFirst { + pushers[0], pushers[1] = pushers[1], pushers[0] + } + for _, pusher := range pushers { + result, pushErr := pusher.Push(ctx, false, nil) + require.NoError(t, pushErr) + assert.Zero(t, result.Errors) + assert.Zero(t, result.SkippedConflicts) + } + + var id, machine, ownerMarker string + err = pg.QueryRowContext(ctx, ` + SELECT id, machine, owner_marker + FROM sessions + WHERE id IN ($1, $2) + `, nativeID, canonicalID).Scan(&id, &machine, &ownerMarker) + require.NoError(t, err) + assert.Equal(t, canonicalID, id) + assert.Equal(t, originA, machine) + assert.Equal(t, artifactOwnerMarkerPrefix+originA, ownerMarker) + var project, content string + require.NoError(t, pg.QueryRowContext(ctx, ` + SELECT s.project, m.content + FROM sessions s + JOIN messages m ON m.session_id = s.id + WHERE s.id = $1 AND m.ordinal = 0 + `, canonicalID).Scan(&project, &content)) + assert.Equal(t, "fresh-origin", project) + assert.Equal(t, "fresh", content) + var parent string + require.NoError(t, pg.QueryRowContext(ctx, ` + SELECT parent_session_id FROM sessions WHERE id = $1 + `, canonicalChildID).Scan(&parent)) + assert.Equal(t, canonicalID, parent, + "artifact relationships must resolve through the canonical origin identity") + + for _, pusher := range []*Sync{importerSync, originSync} { + result, pushErr := pusher.Push(ctx, true, nil) + require.NoError(t, pushErr) + assert.Zero(t, result.Errors) + assert.Zero(t, result.SkippedConflicts) + } + var count int + require.NoError(t, pg.QueryRowContext(ctx, ` + SELECT COUNT(*) FROM sessions WHERE id IN ($1, $2) + `, nativeID, canonicalID).Scan(&count)) + assert.Equal(t, 1, count, + "native and imported copies must keep one PG row across repeated pushes") + }) + } +} + +func TestPushSSHShapedSessionDoesNotAdoptArtifactOwnership(t *testing.T) { + pgURL := testPGURL(t) + ctx := context.Background() + const schema = "agentsview_artifact_provenance_guard_test" + const remoteOrigin = "origin-a1b2c3" + const localOrigin = "origin-b4c5d6" + const sessionID = remoteOrigin + "~native-id" + + pg, err := Open(pgURL, schema, true) + require.NoError(t, err, "Open") + t.Cleanup(func() { require.NoError(t, pg.Close()) }) + _, err = pg.Exec(`DROP SCHEMA IF EXISTS ` + schema + ` CASCADE`) + require.NoError(t, err, "drop schema") + require.NoError(t, EnsureSchema(ctx, pg, schema), "EnsureSchema") + + localDB, err := db.Open(filepath.Join(t.TempDir(), "local.db")) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, localDB.Close()) }) + require.NoError(t, artifact.AdoptOrigin(localDB, localOrigin)) + require.NoError(t, localDB.UpsertSession(db.Session{ + ID: sessionID, Project: "ssh-copy", Machine: remoteOrigin, Agent: "claude", + MessageCount: 1, UserMessageCount: 1, + CreatedAt: "2026-01-01T00:00:00Z", + })) + require.NoError(t, localDB.ReplaceSessionMessages(sessionID, []db.Message{{ + SessionID: sessionID, Ordinal: 0, Role: "user", + Content: "ssh copy", ContentLength: 8, + }})) + + _, err = pg.ExecContext(ctx, ` + INSERT INTO sessions ( + id, machine, owner_marker, project, agent, created_at + ) VALUES ($1, $2, $3, $4, $5, NOW()) + `, sessionID, remoteOrigin, artifactOwnerMarkerPrefix+remoteOrigin, + "genuine-artifact", "claude") + require.NoError(t, err, "seed genuine artifact-owned row") + + syncer := &Sync{ + pg: pg, local: localDB, machine: "ssh-importer", + schema: schema, schemaDone: true, + } + result, err := syncer.Push(ctx, false, nil) + require.NoError(t, err) + assert.Equal(t, 1, result.SkippedConflicts, + "an SSH-shaped row without artifact import provenance must retain legacy collision protection") + assert.Zero(t, result.SessionsPushed) + + var project, ownerMarker string + require.NoError(t, pg.QueryRowContext(ctx, ` + SELECT project, owner_marker FROM sessions WHERE id = $1 + `, sessionID).Scan(&project, &ownerMarker)) + assert.Equal(t, "genuine-artifact", project) + assert.Equal(t, artifactOwnerMarkerPrefix+remoteOrigin, ownerMarker) +} + +func TestPushArtifactOriginAdoptionReusesLegacyBareRows(t *testing.T) { + pgURL := testPGURL(t) + ctx := context.Background() + const schema = "agentsview_artifact_origin_upgrade_test" + const originA = "origin-a1b2c3" + const originB = "origin-b4c5d6" + const parentID = "native-parent" + const childID = "native-child" + const canonicalParentID = originA + "~" + parentID + const canonicalChildID = originA + "~" + childID + const stateScope = "work" + + pg, err := Open(pgURL, schema, true) + require.NoError(t, err, "Open") + t.Cleanup(func() { require.NoError(t, pg.Close()) }) + _, err = pg.Exec(`DROP SCHEMA IF EXISTS ` + schema + ` CASCADE`) + require.NoError(t, err, "drop schema") + require.NoError(t, EnsureSchema(ctx, pg, schema), "EnsureSchema") + + originDB, err := db.Open(filepath.Join(t.TempDir(), "origin.db")) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, originDB.Close()) }) + require.NoError(t, originDB.UpsertSession(db.Session{ + ID: parentID, Project: "alpha", Machine: "local", Agent: "claude", + MessageCount: 1, UserMessageCount: 1, + CreatedAt: "2026-01-01T00:00:00Z", + })) + require.NoError(t, originDB.ReplaceSessionMessages(parentID, []db.Message{{ + SessionID: parentID, Ordinal: 0, Role: "user", + Content: "parent", ContentLength: 6, SourceUUID: "parent-source", + }})) + parentRef := parentID + require.NoError(t, originDB.UpsertSession(db.Session{ + ID: childID, Project: "alpha", Machine: "local", Agent: "claude", + MessageCount: 1, UserMessageCount: 1, + CreatedAt: "2026-01-01T00:00:01Z", ParentSessionID: &parentRef, + })) + require.NoError(t, originDB.ReplaceSessionMessages(childID, []db.Message{{ + SessionID: childID, Ordinal: 0, Role: "user", + Content: "child", ContentLength: 5, SourceUUID: "child-source", + }})) + + originSync := &Sync{ + pg: pg, local: originDB, machine: "host-a", + schema: schema, schemaDone: true, + syncState: newScopedSyncStateStore(originDB, stateScope, false), + syncStateTarget: stateScope, + } + first, err := originSync.Push(ctx, false, nil) + require.NoError(t, err) + require.Equal(t, 2, first.SessionsPushed) + watermark, err := originDB.GetSyncState("last_push_at:" + stateScope) + require.NoError(t, err) + require.NotEmpty(t, watermark, "precondition: initial push established incremental state") + // Simulate a pusher upgraded from a build predating identity-mode state. + require.NoError(t, originDB.SetSyncState( + "pg_artifact_identity_v1:"+stateScope, "", + )) + + _, err = pg.ExecContext(ctx, + `UPDATE sessions SET display_name = 'PG title' WHERE id = $1`, + parentID, + ) + require.NoError(t, err, "seed PG-local display name") + _, err = pg.ExecContext(ctx, + `INSERT INTO starred_sessions (session_id) VALUES ($1)`, + parentID, + ) + require.NoError(t, err, "seed PG-local star") + _, err = pg.ExecContext(ctx, ` + INSERT INTO pinned_messages ( + session_id, message_id, ordinal, source_uuid, note + ) + SELECT $1, ordinal, ordinal, COALESCE(source_uuid, ''), 'PG pin' + FROM messages WHERE session_id = $1 AND ordinal = 0 + `, parentID) + require.NoError(t, err, "seed PG-local pin") + + before, err := originDB.GetSessionFull(ctx, parentID) + require.NoError(t, err) + require.NotNil(t, before) + require.NoError(t, artifact.AdoptOrigin(originDB, originA)) + afterAdopt, err := originDB.GetSessionFull(ctx, parentID) + require.NoError(t, err) + require.NotNil(t, afterAdopt) + assert.Equal(t, before.LocalModifiedAt, afterAdopt.LocalModifiedAt, + "origin adoption must not need to mutate session timestamps") + + upgraded, err := originSync.Push(ctx, false, nil) + require.NoError(t, err) + assert.Equal(t, 2, upgraded.SessionsPushed, + "identity-mode change must revisit unchanged sessions") + assert.Zero(t, upgraded.Errors) + assert.Zero(t, upgraded.SkippedConflicts) + + var stableBareRows int + require.NoError(t, pg.QueryRowContext(ctx, ` + SELECT COUNT(*) FROM sessions + WHERE id IN ($1, $2) AND machine = $3 AND owner_marker = $4 + `, parentID, childID, originA, artifactOwnerMarkerPrefix+originA).Scan(&stableBareRows)) + assert.Equal(t, 2, stableBareRows, + "same-owner legacy bare rows must upgrade in place to stable artifact ownership") + var canonicalRows int + require.NoError(t, pg.QueryRowContext(ctx, ` + SELECT COUNT(*) FROM sessions WHERE id IN ($1, $2) + `, canonicalParentID, canonicalChildID).Scan(&canonicalRows)) + assert.Zero(t, canonicalRows, "upgrade must not migrate primary keys or duplicate rows") + assert.Equal(t, parentID, pgParentSessionID(t, ctx, pg, childID)) + assertPGArtifactUpgradeCuration(t, ctx, pg, parentID) + mode, err := originDB.GetSyncState("pg_artifact_identity_v1:" + stateScope) + require.NoError(t, err) + assert.Equal(t, artifactOwnerMarkerPrefix+originA, mode, + "successful push must persist the target-scoped identity mode") + + artifactRoot := t.TempDir() + _, err = artifact.Export(ctx, originDB, artifactRoot, originA) + require.NoError(t, err) + importerDB, err := db.Open(filepath.Join(t.TempDir(), "importer.db")) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, importerDB.Close()) }) + require.NoError(t, artifact.AdoptOrigin(importerDB, originB)) + imported, messages, err := artifact.Import(ctx, importerDB, artifactRoot, originB) + require.NoError(t, err) + require.Equal(t, 2, imported) + require.Equal(t, 2, messages) + + importerSync := &Sync{ + pg: pg, local: importerDB, machine: "host-b", + schema: schema, schemaDone: true, + syncState: newScopedSyncStateStore(importerDB, stateScope, false), + syncStateTarget: stateScope, + } + importResult, err := importerSync.Push(ctx, false, nil) + require.NoError(t, err) + assert.Zero(t, importResult.Errors) + assert.Zero(t, importResult.SkippedConflicts) + require.NoError(t, pg.QueryRowContext(ctx, ` + SELECT COUNT(*) FROM sessions + WHERE id IN ($1, $2, $3, $4) + `, parentID, childID, canonicalParentID, canonicalChildID).Scan(&stableBareRows)) + assert.Equal(t, 2, stableBareRows, + "an imported copy must reuse the stable legacy bare aliases") + assert.Equal(t, parentID, pgParentSessionID(t, ctx, pg, childID)) + assertPGArtifactUpgradeCuration(t, ctx, pg, parentID) + + _, err = pg.ExecContext(ctx, ` + UPDATE sessions SET parent_session_id = $1 WHERE id = $2 + `, canonicalParentID, childID) + require.NoError(t, err, "seed stale canonical relationship") + time.Sleep(5 * time.Millisecond) + name := "Imported child" + require.NoError(t, importerDB.RenameSession(canonicalChildID, &name)) + replicaResult, err := importerSync.Push(ctx, false, nil) + require.NoError(t, err) + assert.Zero(t, replicaResult.SessionsPushed, + "an imported replica must not update an existing canonical alias") + assert.Zero(t, replicaResult.SkippedConflicts) + assert.Equal(t, canonicalParentID, pgParentSessionID(t, ctx, pg, childID), + "a stale replica must leave the canonical row unchanged") + + originName := "Origin child" + require.NoError(t, originDB.RenameSession(childID, &originName)) + fallbackResult, err := originSync.Push(ctx, false, nil) + require.NoError(t, err) + assert.Equal(t, 1, fallbackResult.SessionsPushed, + "the modified origin child should use committed-PG relationship fallback") + assert.Zero(t, fallbackResult.SkippedConflicts) + assert.Equal(t, parentID, pgParentSessionID(t, ctx, pg, childID), + "origin relationship fallback must resolve the stable bare parent alias") + assertPGArtifactUpgradeCuration(t, ctx, pg, parentID) +} + +func TestPushArtifactOriginAdoptionConvergesImporterFirst(t *testing.T) { + pgURL := testPGURL(t) + ctx := context.Background() + const schema = "agentsview_artifact_origin_upgrade_importer_first_test" + const originA = "origin-a1b2c3" + const originB = "origin-b4c5d6" + const parentID = "native-parent" + const childID = "native-child" + const canonicalParentID = originA + "~" + parentID + const canonicalChildID = originA + "~" + childID + const referenceID = "pg-reference-holder" + const stateScope = "work" + const sourceDisplayName = "Source title" + const localDisplayName = "PG title" + const pinCreatedAt = "2026-01-04T00:00:00Z" + const updatedSourceDisplay = "Origin renamed child" + const canonicalChildDeletedAt = "2026-01-05T00:00:00Z" + + pg, err := Open(pgURL, schema, true) + require.NoError(t, err, "Open") + t.Cleanup(func() { require.NoError(t, pg.Close()) }) + _, err = pg.Exec(`DROP SCHEMA IF EXISTS ` + schema + ` CASCADE`) + require.NoError(t, err, "drop schema") + require.NoError(t, EnsureSchema(ctx, pg, schema), "EnsureSchema") + + originDB, err := db.Open(filepath.Join(t.TempDir(), "origin.db")) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, originDB.Close()) }) + require.NoError(t, originDB.UpsertSession(db.Session{ + ID: parentID, Project: "alpha", Machine: "local", Agent: "claude", + MessageCount: 1, UserMessageCount: 1, + CreatedAt: "2026-01-01T00:00:00Z", + })) + require.NoError(t, originDB.ReplaceSessionMessages(parentID, []db.Message{{ + SessionID: parentID, Ordinal: 0, Role: "user", + Content: "parent", ContentLength: 6, SourceUUID: "parent-source", + }})) + parentRef := parentID + require.NoError(t, originDB.UpsertSession(db.Session{ + ID: childID, Project: "alpha", Machine: "local", Agent: "claude", + MessageCount: 1, UserMessageCount: 0, HasToolCalls: true, + CreatedAt: "2026-01-01T00:00:01Z", + ParentSessionID: &parentRef, + SourceSessionID: parentID, + })) + require.NoError(t, originDB.ReplaceSessionMessages(childID, []db.Message{{ + SessionID: childID, Ordinal: 0, Role: "assistant", + Content: "child", ContentLength: 5, SourceUUID: "child-source", + HasToolUse: true, + ToolCalls: []db.ToolCall{{ + ToolName: "subagent", Category: "Task", ToolUseID: "call-parent", + SubagentSessionID: parentID, + ResultEvents: []db.ToolResultEvent{{ + ToolUseID: "call-parent", SubagentSessionID: parentID, + Source: "tool_result", Status: "completed", Content: "done", + ContentLength: 4, + }}, + }}, + }})) + + originSync := &Sync{ + pg: pg, local: originDB, machine: "host-a", + schema: schema, schemaDone: true, + syncState: newScopedSyncStateStore(originDB, stateScope, false), + syncStateTarget: stateScope, + } + legacyResult, err := originSync.Push(ctx, false, nil) + require.NoError(t, err) + require.Equal(t, 2, legacyResult.SessionsPushed) + require.NoError(t, originDB.SetSyncState( + "pg_artifact_identity_v1:"+stateScope, "", + )) + + _, err = pg.ExecContext(ctx, ` + UPDATE sessions + SET display_name = $1, + source_display_name = $2 + WHERE id = $3 + `, localDisplayName, sourceDisplayName, parentID) + require.NoError(t, err, "seed PG-local session curation") + _, err = pg.ExecContext(ctx, ` + INSERT INTO starred_sessions (session_id, created_at) + VALUES ($1, '2026-01-04T00:00:00Z'::timestamptz) + `, parentID) + require.NoError(t, err, "seed PG-local star") + _, err = pg.ExecContext(ctx, ` + INSERT INTO pinned_messages ( + session_id, message_id, ordinal, source_uuid, note, created_at + ) + VALUES ($1, 0, 0, 'parent-source', 'PG pin', $2::timestamptz) + `, parentID, pinCreatedAt) + require.NoError(t, err, "seed PG-local pin") + + require.NoError(t, artifact.AdoptOrigin(originDB, originA)) + artifactRoot := t.TempDir() + _, err = artifact.Export(ctx, originDB, artifactRoot, originA) + require.NoError(t, err) + importerDB, err := db.Open(filepath.Join(t.TempDir(), "importer.db")) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, importerDB.Close()) }) + require.NoError(t, artifact.AdoptOrigin(importerDB, originB)) + imported, messages, err := artifact.Import( + ctx, importerDB, artifactRoot, originB, + ) + require.NoError(t, err) + require.Equal(t, 2, imported) + require.Equal(t, 2, messages) + + importerSync := &Sync{ + pg: pg, local: importerDB, machine: "host-b", + schema: schema, schemaDone: true, + syncState: newScopedSyncStateStore(importerDB, stateScope, false), + syncStateTarget: stateScope, + } + importerResult, err := importerSync.Push(ctx, false, nil) + require.NoError(t, err) + assert.Zero(t, importerResult.Errors) + assert.Zero(t, importerResult.SkippedConflicts) + assertPGArtifactNativeRows(t, ctx, pg, parentID, canonicalParentID, 2) + assertPGArtifactNativeRows(t, ctx, pg, childID, canonicalChildID, 2) + _, err = pg.ExecContext(ctx, ` + UPDATE sessions + SET deleted_at = $1::timestamptz, + source_deleted_at = NULL + WHERE id = $2 + `, canonicalChildDeletedAt, canonicalChildID) + require.NoError(t, err, "seed newer canonical PG curation") + require.NoError(t, originDB.RenameSession( + childID, new(updatedSourceDisplay), + )) + require.NoError(t, originDB.SoftDeleteSession(parentID)) + updatedParent, err := originDB.GetSessionFull(ctx, parentID) + require.NoError(t, err) + require.NotNil(t, updatedParent) + require.NotNil(t, updatedParent.DeletedAt) + updatedSourceDeletedAt, ok := ParseSQLiteTimestamp(*updatedParent.DeletedAt) + require.True(t, ok, "parse source soft-delete timestamp") + wantSourceDeletedAt := updatedSourceDeletedAt.UTC().Format(time.RFC3339) + + _, err = pg.ExecContext(ctx, ` + INSERT INTO sessions ( + id, machine, owner_marker, project, agent, created_at, + parent_session_id, source_session_id + ) VALUES ($1, 'pg-only', 'pg-only-owner', 'alpha', 'claude', NOW(), $2, $2) + `, referenceID, parentID) + require.NoError(t, err, "seed PG-only session relationships") + _, err = pg.ExecContext(ctx, ` + INSERT INTO messages (session_id, ordinal, role, content) + VALUES ($1, 0, 'assistant', 'reference') + `, referenceID) + require.NoError(t, err, "seed PG-only message") + _, err = pg.ExecContext(ctx, ` + INSERT INTO tool_calls ( + session_id, tool_name, category, call_index, tool_use_id, + subagent_session_id, message_ordinal + ) VALUES ($1, 'subagent', 'Task', 0, 'pg-call', $2, 0) + `, referenceID, parentID) + require.NoError(t, err, "seed PG-only tool call relationship") + _, err = pg.ExecContext(ctx, ` + INSERT INTO tool_result_events ( + session_id, tool_call_message_ordinal, call_index, + tool_use_id, subagent_session_id, source, status, + content, content_length, event_index + ) VALUES ($1, 0, 0, 'pg-call', $2, 'tool_result', + 'completed', 'done', 4, 0) + `, referenceID, parentID) + require.NoError(t, err, "seed PG-only tool result relationship") + + upgradeResult, err := originSync.Push(ctx, false, nil) + require.NoError(t, err) + assert.Equal(t, 2, upgradeResult.SessionsPushed, + "identity-mode change must revisit unchanged origin sessions") + assert.Zero(t, upgradeResult.Errors) + assert.Zero(t, upgradeResult.SkippedConflicts) + assertPGArtifactNativeRows(t, ctx, pg, parentID, canonicalParentID, 1) + assertPGArtifactNativeRows(t, ctx, pg, childID, canonicalChildID, 1) + assertPGArtifactStableOwner(t, ctx, pg, canonicalParentID, originA) + assertPGArtifactStableOwner(t, ctx, pg, canonicalChildID, originA) + assertPGArtifactUpgradeCurationDetails( + t, ctx, pg, canonicalParentID, + localDisplayName, sourceDisplayName, + wantSourceDeletedAt, wantSourceDeletedAt, pinCreatedAt, + ) + assertPGArtifactUpdatedSourceDisplay( + t, ctx, pg, canonicalChildID, + updatedSourceDisplay, canonicalChildDeletedAt, + ) + assertPGArtifactUpgradeRelationships( + t, ctx, pg, canonicalParentID, canonicalChildID, referenceID, + ) + + for _, pusher := range []*Sync{importerSync, originSync} { + repeated, pushErr := pusher.Push(ctx, true, nil) + require.NoError(t, pushErr) + assert.Zero(t, repeated.Errors) + assert.Zero(t, repeated.SkippedConflicts) + } + assertPGArtifactNativeRows(t, ctx, pg, parentID, canonicalParentID, 1) + assertPGArtifactNativeRows(t, ctx, pg, childID, canonicalChildID, 1) + assertPGArtifactStableOwner(t, ctx, pg, canonicalParentID, originA) + assertPGArtifactStableOwner(t, ctx, pg, canonicalChildID, originA) + assertPGArtifactUpgradeCurationAfterRepeatedPushes( + t, ctx, pg, canonicalParentID, + localDisplayName, wantSourceDeletedAt, pinCreatedAt, + ) + assertPGArtifactUpgradeRelationships( + t, ctx, pg, canonicalParentID, canonicalChildID, referenceID, + ) +} + +func TestPushArtifactOriginAdoptionIgnoresForeignBareAlias(t *testing.T) { + pgURL := testPGURL(t) + ctx := context.Background() + const schema = "agentsview_artifact_origin_upgrade_proof_test" + const origin = "origin-a1b2c3" + const nativeID = "native-id" + const canonicalID = origin + "~" + nativeID + + pg, err := Open(pgURL, schema, true) + require.NoError(t, err, "Open") + t.Cleanup(func() { require.NoError(t, pg.Close()) }) + _, err = pg.Exec(`DROP SCHEMA IF EXISTS ` + schema + ` CASCADE`) + require.NoError(t, err, "drop schema") + require.NoError(t, EnsureSchema(ctx, pg, schema), "EnsureSchema") + + localDB, err := db.Open(filepath.Join(t.TempDir(), "local.db")) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, localDB.Close()) }) + require.NoError(t, artifact.AdoptOrigin(localDB, origin)) + require.NoError(t, localDB.UpsertSession(db.Session{ + ID: nativeID, Project: "local-project", Machine: "local", Agent: "claude", + MessageCount: 1, UserMessageCount: 1, + CreatedAt: "2026-01-01T00:00:00Z", + })) + require.NoError(t, localDB.ReplaceSessionMessages(nativeID, []db.Message{{ + SessionID: nativeID, Ordinal: 0, Role: "user", + Content: "local", ContentLength: 5, + }})) + + _, err = pg.ExecContext(ctx, ` + INSERT INTO sessions ( + id, machine, owner_marker, project, agent, created_at + ) VALUES + ($1, $2, $3, 'canonical-before', 'claude', NOW()), + ($4, 'legacy-host', 'different-random-marker', + 'legacy-before', 'claude', NOW()) + `, canonicalID, origin, artifactOwnerMarkerPrefix+origin, nativeID) + require.NoError(t, err, "seed unproven duplicate pair") + + syncer := &Sync{ + pg: pg, local: localDB, machine: "host-a", + schema: schema, schemaDone: true, + } + result, err := syncer.Push(ctx, true, nil) + require.NoError(t, err) + assert.Zero(t, result.Errors) + assert.Zero(t, result.SkippedConflicts) + assert.Equal(t, 1, result.SessionsPushed) + + rows, err := pg.QueryContext(ctx, ` + SELECT id, project FROM sessions + WHERE id IN ($1, $2) ORDER BY id + `, nativeID, canonicalID) + require.NoError(t, err) + defer rows.Close() + projects := map[string]string{} + for rows.Next() { + var id, project string + require.NoError(t, rows.Scan(&id, &project)) + projects[id] = project + } + require.NoError(t, rows.Err()) + assert.Equal(t, map[string]string{ + nativeID: "legacy-before", + canonicalID: "local-project", + }, projects, "a foreign bare collision must not block the stable canonical row") +} + +func assertPGArtifactNativeRows( + t *testing.T, ctx context.Context, pg *sql.DB, + bareID, canonicalID string, want int, +) { + t.Helper() + var count int + require.NoError(t, pg.QueryRowContext(ctx, ` + SELECT COUNT(*) FROM sessions WHERE id IN ($1, $2) + `, bareID, canonicalID).Scan(&count)) + assert.Equal(t, want, count) +} + +func assertPGArtifactStableOwner( + t *testing.T, ctx context.Context, pg *sql.DB, + sessionID, origin string, +) { + t.Helper() + var machine, ownerMarker string + require.NoError(t, pg.QueryRowContext(ctx, ` + SELECT machine, owner_marker FROM sessions WHERE id = $1 + `, sessionID).Scan(&machine, &ownerMarker)) + assert.Equal(t, origin, machine) + assert.Equal(t, artifactOwnerMarkerPrefix+origin, ownerMarker) +} + +func assertPGArtifactUpgradeCurationDetails( + t *testing.T, ctx context.Context, pg *sql.DB, sessionID, + wantDisplay, wantSourceDisplay, wantDeleted, wantSourceDeleted, + wantPinCreated string, +) { + t.Helper() + assertPGArtifactSessionCuration( + t, ctx, pg, sessionID, + wantDisplay, wantSourceDisplay, wantDeleted, wantSourceDeleted, + ) + + var stars int + require.NoError(t, pg.QueryRowContext(ctx, ` + SELECT COUNT(*) FROM starred_sessions WHERE session_id = $1 + `, sessionID).Scan(&stars)) + assert.Equal(t, 1, stars) + + var ordinal, messageID int + var sourceUUID, note string + var createdAt time.Time + require.NoError(t, pg.QueryRowContext(ctx, ` + SELECT message_id, ordinal, source_uuid, note, created_at + FROM pinned_messages WHERE session_id = $1 + `, sessionID).Scan( + &messageID, &ordinal, &sourceUUID, ¬e, &createdAt, + )) + assert.Equal(t, 0, messageID) + assert.Equal(t, 0, ordinal) + assert.Equal(t, "parent-source", sourceUUID) + assert.Equal(t, "PG pin", note) + assert.Equal(t, wantPinCreated, createdAt.UTC().Format(time.RFC3339)) +} + +func assertPGArtifactSessionCuration( + t *testing.T, ctx context.Context, pg *sql.DB, sessionID, + wantDisplay, wantSourceDisplay, wantDeleted, wantSourceDeleted string, +) { + t.Helper() + var displayName, sourceDisplayName string + var deletedAt, sourceDeletedAt sql.NullTime + require.NoError(t, pg.QueryRowContext(ctx, ` + SELECT display_name, source_display_name, deleted_at, source_deleted_at + FROM sessions WHERE id = $1 + `, sessionID).Scan( + &displayName, &sourceDisplayName, &deletedAt, &sourceDeletedAt, + )) + assert.Equal(t, wantDisplay, displayName) + assert.Equal(t, wantSourceDisplay, sourceDisplayName) + if assert.True(t, deletedAt.Valid, "deleted_at") { + assert.Equal(t, wantDeleted, deletedAt.Time.UTC().Format(time.RFC3339)) + } + if assert.True(t, sourceDeletedAt.Valid, "source_deleted_at") { + assert.Equal(t, wantSourceDeleted, + sourceDeletedAt.Time.UTC().Format(time.RFC3339)) + } +} + +func assertPGArtifactUpdatedSourceDisplay( + t *testing.T, ctx context.Context, pg *sql.DB, sessionID, + wantDisplay, wantDeleted string, +) { + t.Helper() + var displayName, sourceDisplayName sql.NullString + var deletedAt time.Time + var sourceDeletedAt sql.NullTime + require.NoError(t, pg.QueryRowContext(ctx, ` + SELECT display_name, source_display_name, deleted_at, source_deleted_at + FROM sessions WHERE id = $1 + `, sessionID).Scan( + &displayName, &sourceDisplayName, &deletedAt, &sourceDeletedAt, + )) + if assert.True(t, displayName.Valid, "display_name") { + assert.Equal(t, wantDisplay, displayName.String) + } + if assert.True(t, sourceDisplayName.Valid, "source_display_name") { + assert.Equal(t, wantDisplay, sourceDisplayName.String) + } + assert.Equal(t, wantDeleted, deletedAt.UTC().Format(time.RFC3339)) + assert.False(t, sourceDeletedAt.Valid, + "canonical delete override must retain the current source baseline") +} + +func assertPGArtifactUpgradeCurationAfterRepeatedPushes( + t *testing.T, ctx context.Context, pg *sql.DB, sessionID, + wantDisplay, wantDeleted, wantPinCreated string, +) { + t.Helper() + var displayName string + var deletedAt time.Time + require.NoError(t, pg.QueryRowContext(ctx, ` + SELECT display_name, deleted_at + FROM sessions WHERE id = $1 + `, sessionID).Scan(&displayName, &deletedAt)) + assert.Equal(t, wantDisplay, displayName) + assert.Equal(t, wantDeleted, deletedAt.UTC().Format(time.RFC3339)) + + var stars int + require.NoError(t, pg.QueryRowContext(ctx, ` + SELECT COUNT(*) FROM starred_sessions WHERE session_id = $1 + `, sessionID).Scan(&stars)) + assert.Equal(t, 1, stars) + + var ordinal, messageID int + var sourceUUID, note string + var createdAt time.Time + require.NoError(t, pg.QueryRowContext(ctx, ` + SELECT message_id, ordinal, source_uuid, note, created_at + FROM pinned_messages WHERE session_id = $1 + `, sessionID).Scan( + &messageID, &ordinal, &sourceUUID, ¬e, &createdAt, + )) + assert.Equal(t, 0, messageID) + assert.Equal(t, 0, ordinal) + assert.Equal(t, "parent-source", sourceUUID) + assert.Equal(t, "PG pin", note) + assert.Equal(t, wantPinCreated, createdAt.UTC().Format(time.RFC3339)) +} + +func assertPGArtifactUpgradeRelationships( + t *testing.T, ctx context.Context, pg *sql.DB, + canonicalParentID, canonicalChildID, referenceID string, +) { + t.Helper() + var parentID, sourceID string + require.NoError(t, pg.QueryRowContext(ctx, ` + SELECT parent_session_id, source_session_id + FROM sessions WHERE id = $1 + `, canonicalChildID).Scan(&parentID, &sourceID)) + assert.Equal(t, canonicalParentID, parentID) + assert.Equal(t, canonicalParentID, sourceID) + require.NoError(t, pg.QueryRowContext(ctx, ` + SELECT parent_session_id, source_session_id + FROM sessions WHERE id = $1 + `, referenceID).Scan(&parentID, &sourceID)) + assert.Equal(t, canonicalParentID, parentID) + assert.Equal(t, canonicalParentID, sourceID) + + var toolCallSubagentID string + require.NoError(t, pg.QueryRowContext(ctx, ` + SELECT subagent_session_id FROM tool_calls WHERE session_id = $1 + `, referenceID).Scan(&toolCallSubagentID)) + assert.Equal(t, canonicalParentID, toolCallSubagentID, "tool_calls") + var toolResultSubagentID string + require.NoError(t, pg.QueryRowContext(ctx, ` + SELECT subagent_session_id + FROM tool_result_events WHERE session_id = $1 + `, referenceID).Scan(&toolResultSubagentID)) + assert.Equal(t, canonicalParentID, toolResultSubagentID, "tool_result_events") +} + +func pgParentSessionID( + t *testing.T, ctx context.Context, pg *sql.DB, sessionID string, +) string { + t.Helper() + var parentID string + require.NoError(t, pg.QueryRowContext(ctx, ` + SELECT COALESCE(parent_session_id, '') FROM sessions WHERE id = $1 + `, sessionID).Scan(&parentID)) + return parentID +} + +func assertPGArtifactUpgradeCuration( + t *testing.T, ctx context.Context, pg *sql.DB, sessionID string, +) { + t.Helper() + var displayName string + require.NoError(t, pg.QueryRowContext(ctx, ` + SELECT COALESCE(display_name, '') FROM sessions WHERE id = $1 + `, sessionID).Scan(&displayName)) + assert.Equal(t, "PG title", displayName) + var stars, pins int + require.NoError(t, pg.QueryRowContext(ctx, ` + SELECT COUNT(*) FROM starred_sessions WHERE session_id = $1 + `, sessionID).Scan(&stars)) + require.NoError(t, pg.QueryRowContext(ctx, ` + SELECT COUNT(*) FROM pinned_messages + WHERE session_id = $1 AND note = 'PG pin' + `, sessionID).Scan(&pins)) + assert.Equal(t, 1, stars) + assert.Equal(t, 1, pins) +} + // TestPushSessionGuardsAgainstCrossMachineCollision verifies that when two // machines share the same session ID (from dotfile sync, directory restore, etc.), // the second machine's push is skipped if the session is already owned by a @@ -84,7 +942,10 @@ func TestPushSessionGuardsAgainstCrossMachineCollision(t *testing.T) { // Execute pushSession. tx, err := pg.BeginTx(ctx, nil) require.NoError(t, err, "BeginTx") - err = sync.pushSession(ctx, tx, sess, markerID, nil) + err = sync.pushSession(ctx, tx, sess, pushedSessionIdentity{ + ID: sess.ID, + Machine: sess.Machine, + }, markerID, nil) require.ErrorIs(t, err, errSessionOwnershipConflict, "pushSession should return ownership conflict sentinel") require.NoError(t, tx.Commit(), "Commit") @@ -156,7 +1017,10 @@ func TestPushSessionAllowsMachineRenameForSameOwnerMarker(t *testing.T) { tx, err := pg.BeginTx(ctx, nil) require.NoError(t, err, "BeginTx") - require.NoError(t, sync.pushSession(ctx, tx, sess, markerID, nil), "pushSession") + require.NoError(t, sync.pushSession(ctx, tx, sess, pushedSessionIdentity{ + ID: sess.ID, + Machine: "renamed-host", + }, markerID, nil), "pushSession") require.NoError(t, tx.Commit(), "Commit") var machine, ownerMarker string @@ -168,6 +1032,214 @@ func TestPushSessionAllowsMachineRenameForSameOwnerMarker(t *testing.T) { assert.Equal(t, markerID, ownerMarker) } +// TestPushResolvesRelationshipIDsToPrefixedTargets verifies that when a +// referenced session is pushed under a collision-avoidance prefix, the +// relationship ids pointing at it -- source_session_id, parent_session_id, and +// a tool-call subagent_session_id -- are rewritten to the prefixed id so child +// and subagent rows link to the right PG session instead of a foreign machine's +// row or a dangling id. A non-colliding parent keeps its bare id. +func TestPushResolvesRelationshipIDsToPrefixedTargets(t *testing.T) { + pgURL := testPGURL(t) + + const schema = "agentsview_relationship_resolution_test" + pg, err := Open(pgURL, schema, true) + require.NoError(t, err, "Open") + defer pg.Close() + + ctx := context.Background() + _, err = pg.Exec(`DROP SCHEMA IF EXISTS ` + schema + ` CASCADE`) + require.NoError(t, err, "drop schema") + require.NoError(t, EnsureSchema(ctx, pg, schema), "EnsureSchema") + + localDB, err := db.Open(filepath.Join(t.TempDir(), "local.db")) + require.NoError(t, err, "db.Open") + defer localDB.Close() + + sync := &Sync{ + pg: pg, + local: localDB, + machine: "machine-b", + schema: schema, + schemaDone: true, + } + + // A foreign machine already owns the bare "shared" id, so machine-b's + // "shared" session must be pushed under the "machine-b~shared" prefix. + _, err = pg.ExecContext(ctx, ` + INSERT INTO sessions ( + id, machine, owner_marker, project, agent, created_at + ) VALUES ($1, $2, $3, $4, $5, NOW()) + `, "shared", "machine-a", "foreign-owner", "test-proj", "claude") + require.NoError(t, err, "insert foreign-owned shared session") + + sharedParent := "shared" + plainParent := "plain" + sessions := []db.Session{ + {ID: "shared", Project: "test-proj", Machine: "machine-b", + Agent: "claude", MessageCount: 1, CreatedAt: "2026-01-01T00:00:00Z"}, + {ID: "plain", Project: "test-proj", Machine: "machine-b", + Agent: "claude", MessageCount: 1, CreatedAt: "2026-01-01T00:00:00Z"}, + {ID: "child-shared", Project: "test-proj", Machine: "machine-b", + Agent: "claude", MessageCount: 1, CreatedAt: "2026-01-01T00:00:00Z", + SourceSessionID: "shared", ParentSessionID: &sharedParent}, + {ID: "child-plain", Project: "test-proj", Machine: "machine-b", + Agent: "claude", MessageCount: 1, CreatedAt: "2026-01-01T00:00:00Z", + ParentSessionID: &plainParent}, + } + for _, s := range sessions { + require.NoError(t, localDB.UpsertSession(s), "UpsertSession "+s.ID) + } + + // child-shared references the colliding "shared" id from a tool call too. + require.NoError(t, localDB.InsertMessages([]db.Message{{ + SessionID: "child-shared", Ordinal: 0, Role: "assistant", + Content: "spawning", HasToolUse: true, + ToolCalls: []db.ToolCall{{ + ToolName: "subagent", Category: "Task", + SubagentSessionID: "shared", + }}, + }}), "InsertMessages child-shared") + for _, id := range []string{"shared", "plain", "child-plain"} { + require.NoError(t, localDB.InsertMessages([]db.Message{{ + SessionID: id, Ordinal: 0, Role: "user", + Content: "hi", ContentLength: 2, + }}), "InsertMessages "+id) + } + + _, err = sync.Push(ctx, false, nil) + require.NoError(t, err, "Push") + + const prefixedShared = "machine-b~shared" + var n int + require.NoError(t, pg.QueryRowContext(ctx, + `SELECT COUNT(*) FROM sessions WHERE id = $1 AND machine = $2`, + prefixedShared, "machine-b").Scan(&n), "count prefixed shared") + assert.Equal(t, 1, n, "machine-b's shared session stored under prefixed id") + + var source, parent string + require.NoError(t, pg.QueryRowContext(ctx, + `SELECT source_session_id, parent_session_id + FROM sessions WHERE id = $1`, + "child-shared").Scan(&source, &parent), "read child-shared relations") + assert.Equal(t, prefixedShared, source, "source_session_id resolved") + assert.Equal(t, prefixedShared, parent, "parent_session_id resolved") + + var subagent string + require.NoError(t, pg.QueryRowContext(ctx, + `SELECT subagent_session_id FROM tool_calls WHERE session_id = $1`, + "child-shared").Scan(&subagent), "read child-shared subagent link") + assert.Equal(t, prefixedShared, subagent, "subagent_session_id resolved") + + var plainParentGot string + require.NoError(t, pg.QueryRowContext(ctx, + `SELECT parent_session_id FROM sessions WHERE id = $1`, + "child-plain").Scan(&plainParentGot), "read child-plain parent") + assert.Equal(t, "plain", plainParentGot, "non-colliding parent stays bare") +} + +// TestPushRepairsStaleSubagentLinkOnIncrementalPush verifies that an +// incremental push repairs a PG tool-call subagent link left at its unprefixed +// local id by a push that predated the collision: the parent's tool call was +// pushed while "sub-1" was still unclaimed, and the subagent session only +// later collided with a foreign owner and moved under "machine-b~sub-1". The +// local rows never change, so both the session candidacy fingerprint and the +// message fast path would otherwise skip the parent and never rewrite the +// link; only the resolved subagent id distinguishes it from PG. +func TestPushRepairsStaleSubagentLinkOnIncrementalPush(t *testing.T) { + pgURL := testPGURL(t) + + const schema = "agentsview_stale_subagent_repair_test" + pg, err := Open(pgURL, schema, true) + require.NoError(t, err, "Open") + defer pg.Close() + + ctx := context.Background() + _, err = pg.Exec(`DROP SCHEMA IF EXISTS ` + schema + ` CASCADE`) + require.NoError(t, err, "drop schema") + require.NoError(t, EnsureSchema(ctx, pg, schema), "EnsureSchema") + + localDB, err := db.Open(filepath.Join(t.TempDir(), "local.db")) + require.NoError(t, err, "db.Open") + defer localDB.Close() + + sync := &Sync{ + pg: pg, + local: localDB, + machine: "machine-b", + schema: schema, + schemaDone: true, + } + + // The parent references "sub-1" before any session by that id exists in + // PG or locally, so the first push writes the tool-call link at its bare + // local id. + require.NoError(t, localDB.UpsertSession(db.Session{ + ID: "parent-1", Project: "proj", Machine: "machine-b", Agent: "claude", + MessageCount: 1, CreatedAt: "2026-01-01T00:00:00Z", + }), "UpsertSession parent-1") + require.NoError(t, localDB.InsertMessages([]db.Message{{ + SessionID: "parent-1", Ordinal: 0, Role: "assistant", + Content: "spawning", HasToolUse: true, + ToolCalls: []db.ToolCall{{ + ToolName: "subagent", Category: "Task", SubagentSessionID: "sub-1", + }}, + }}), "InsertMessages parent-1") + + _, err = sync.Push(ctx, false, nil) + require.NoError(t, err, "first Push") + + subagent := func() string { + var s string + require.NoError(t, pg.QueryRowContext(ctx, + `SELECT subagent_session_id FROM tool_calls WHERE session_id = $1`, + "parent-1").Scan(&s), "read parent-1 subagent link") + return s + } + require.Equal(t, "sub-1", subagent(), + "first push keeps the bare link while the id is unclaimed") + + // A foreign machine claims the bare "sub-1" id, then machine-b's subagent + // session appears locally and is pushed under the "machine-b~sub-1" + // prefix. The parent's local row is unchanged, so its PG link now points + // at the foreign machine's session. + _, err = pg.ExecContext(ctx, ` + INSERT INTO sessions ( + id, machine, owner_marker, project, agent, created_at + ) VALUES ($1, $2, $3, $4, $5, NOW()) + `, "sub-1", "machine-a", "foreign-owner", "proj", "claude") + require.NoError(t, err, "insert foreign-owned subagent session") + + require.NoError(t, localDB.UpsertSession(db.Session{ + ID: "sub-1", Project: "proj", Machine: "machine-b", Agent: "claude", + MessageCount: 1, CreatedAt: "2026-01-01T00:00:00Z", + }), "UpsertSession sub-1") + require.NoError(t, localDB.InsertMessages([]db.Message{{ + SessionID: "sub-1", Ordinal: 0, Role: "user", Content: "hi", ContentLength: 2, + }}), "InsertMessages sub-1") + + const prefixedSub = "machine-b~sub-1" + _, err = sync.Push(ctx, false, nil) + require.NoError(t, err, "second Push") + var n int + require.NoError(t, pg.QueryRowContext(ctx, + `SELECT COUNT(*) FROM sessions WHERE id = $1 AND machine = $2`, + prefixedSub, "machine-b").Scan(&n), "count prefixed sub-1") + require.Equal(t, 1, n, "machine-b's subagent stored under prefixed id") + require.Equal(t, "sub-1", subagent(), + "precondition: parent was not a candidate, so its link is stale") + + // Re-list the parent without touching its message content, so only the + // resolved subagent id distinguishes it from what was last pushed. + require.NoError(t, localDB.BumpLocalModifiedAt("parent-1"), + "mark parent-1 modified") + + res, err := sync.Push(ctx, false, nil) + require.NoError(t, err, "third Push") + assert.Zero(t, res.Errors, "third push should report no failures") + assert.Equal(t, prefixedSub, subagent(), + "incremental push repairs the stale subagent link") +} + func TestPushSessionAdoptsLegacyLocalSentinelRow(t *testing.T) { pgURL := testPGURL(t) @@ -215,7 +1287,10 @@ func TestPushSessionAdoptsLegacyLocalSentinelRow(t *testing.T) { require.NoError(t, err, "BeginTx") markerID, err := sync.pushMarkerID() require.NoError(t, err, "pushMarkerID") - require.NoError(t, sync.pushSession(ctx, tx, sess, markerID, nil), "pushSession") + require.NoError(t, sync.pushSession(ctx, tx, sess, pushedSessionIdentity{ + ID: sess.ID, + Machine: "host-a", + }, markerID, nil), "pushSession") require.NoError(t, tx.Commit(), "Commit") var machine, ownerMarker string @@ -226,3 +1301,352 @@ func TestPushSessionAdoptsLegacyLocalSentinelRow(t *testing.T) { assert.Equal(t, "host-a", machine) assert.Equal(t, markerID, ownerMarker) } + +// TestResolveOwnedPushIDReusesLegacyPrefixAfterRename verifies that the shared +// id resolver (used by both resolvePushedSessionIdentity and +// relationshipResolver.lookup) reuses a row owned under a prior machine prefix. +// A pusher that once stored a colliding session under "old-host~id" and later +// renamed to "new-host" must resolve back to "old-host~id" instead of minting a +// duplicate "new-host~id". +func TestResolveOwnedPushIDReusesLegacyPrefixAfterRename(t *testing.T) { + pgURL := testPGURL(t) + + const schema = "agentsview_legacy_prefix_resolve_test" + pg, err := Open(pgURL, schema, true) + require.NoError(t, err, "Open") + defer pg.Close() + + ctx := context.Background() + _, err = pg.Exec(`DROP SCHEMA IF EXISTS ` + schema + ` CASCADE`) + require.NoError(t, err, "drop schema") + require.NoError(t, EnsureSchema(ctx, pg, schema), "EnsureSchema") + + localDB, err := db.Open(filepath.Join(t.TempDir(), "local.db")) + require.NoError(t, err, "db.Open") + defer localDB.Close() + + sync := &Sync{ + pg: pg, + local: localDB, + machine: "new-host", + schema: schema, + schemaDone: true, + } + markerID, err := sync.pushMarkerID() + require.NoError(t, err, "pushMarkerID") + + // A foreign machine owns the bare id, which is why this pusher stored its + // session under the old machine prefix before the rename. + _, err = pg.ExecContext(ctx, ` + INSERT INTO sessions (id, machine, owner_marker, project, agent, created_at) + VALUES ($1, $2, $3, $4, $5, NOW())`, + "sess-x", "foreign", "foreign-owner", "proj", "claude") + require.NoError(t, err, "insert foreign bare row") + _, err = pg.ExecContext(ctx, ` + INSERT INTO sessions (id, machine, owner_marker, project, agent, created_at) + VALUES ($1, $2, $3, $4, $5, NOW())`, + "old-host~sess-x", "old-host", markerID, "proj", "claude") + require.NoError(t, err, "insert owned legacy-prefixed row") + + identity := pushedSessionIdentity{Machine: "new-host"} + got, err := sync.resolveOwnedPushIdentityID( + ctx, "sess-x", identity, markerID, []string{"old-host"}, + ) + require.NoError(t, err) + assert.Equal(t, "old-host~sess-x", got, + "a renamed pusher must reuse the row it owns under the old machine prefix") + + // Without the old machine name, the resolver cannot find the owned row and + // would mint a duplicate under the new prefix -- the blind spot being fixed. + got, err = sync.resolveOwnedPushIdentityID(ctx, "sess-x", identity, markerID, nil) + require.NoError(t, err) + assert.Equal(t, "new-host~sess-x", got, + "precondition: without the legacy machine the resolver duplicates the row") +} + +// TestPushReusesLegacyPrefixedRowAfterRename verifies the end-to-end push: +// after a machine rename, a session previously stored under the old machine +// prefix is updated in place rather than duplicated under the new prefix. +func TestPushReusesLegacyPrefixedRowAfterRename(t *testing.T) { + pgURL := testPGURL(t) + + const schema = "agentsview_legacy_prefix_push_test" + pg, err := Open(pgURL, schema, true) + require.NoError(t, err, "Open") + defer pg.Close() + + ctx := context.Background() + _, err = pg.Exec(`DROP SCHEMA IF EXISTS ` + schema + ` CASCADE`) + require.NoError(t, err, "drop schema") + require.NoError(t, EnsureSchema(ctx, pg, schema), "EnsureSchema") + + localDB, err := db.Open(filepath.Join(t.TempDir(), "local.db")) + require.NoError(t, err, "db.Open") + defer localDB.Close() + + sync := &Sync{ + pg: pg, + local: localDB, + machine: "new-host", + schema: schema, + schemaDone: true, + } + markerID, err := sync.pushMarkerID() + require.NoError(t, err, "pushMarkerID") + + // Record that this marker last pushed as "old-host", so the rename push + // treats "old-host" as a legacy machine prefix. + _, err = pg.ExecContext(ctx, ` + INSERT INTO sync_metadata (key, value) VALUES ($1, $2) + ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value`, + pushMarkerKeyPrefix+markerID, "old-host") + require.NoError(t, err, "seed push marker machine") + + // A foreign machine owns the bare id; this pusher's session lives under the + // old machine prefix from before the rename. + _, err = pg.ExecContext(ctx, ` + INSERT INTO sessions (id, machine, owner_marker, project, agent, created_at) + VALUES ($1, $2, $3, $4, $5, NOW())`, + "sess-x", "foreign", "foreign-owner", "proj", "claude") + require.NoError(t, err, "insert foreign bare row") + _, err = pg.ExecContext(ctx, ` + INSERT INTO sessions (id, machine, owner_marker, project, agent, created_at) + VALUES ($1, $2, $3, $4, $5, NOW())`, + "old-host~sess-x", "old-host", markerID, "proj", "claude") + require.NoError(t, err, "insert owned legacy-prefixed row") + + sess := db.Session{ + ID: "sess-x", Project: "proj", Machine: "local", Agent: "claude", + MessageCount: 1, CreatedAt: "2026-01-01T00:00:00Z", + } + require.NoError(t, localDB.UpsertSession(sess), "UpsertSession") + require.NoError(t, localDB.InsertMessages([]db.Message{{ + SessionID: "sess-x", Ordinal: 0, Role: "user", Content: "hi", ContentLength: 2, + }}), "InsertMessages") + + _, err = sync.Push(ctx, false, nil) + require.NoError(t, err, "Push") + + var dup int + require.NoError(t, pg.QueryRowContext(ctx, + `SELECT COUNT(*) FROM sessions WHERE id = $1`, "new-host~sess-x").Scan(&dup), + "count new-prefix duplicate") + assert.Equal(t, 0, dup, "rename must not create a duplicate under the new prefix") + + // The legacy-prefixed row was updated in place: the machine column reflects + // the rename and the pushed message landed there. + var machine string + require.NoError(t, pg.QueryRowContext(ctx, + `SELECT machine FROM sessions WHERE id = $1`, "old-host~sess-x").Scan(&machine), + "read reused row machine") + assert.Equal(t, "new-host", machine, "owned legacy row updated in place") + var msgs int + require.NoError(t, pg.QueryRowContext(ctx, + `SELECT COUNT(*) FROM messages WHERE session_id = $1`, "old-host~sess-x").Scan(&msgs), + "count reused row messages") + assert.Equal(t, 1, msgs, "message pushed to the reused legacy row") + + var foreignOwner string + require.NoError(t, pg.QueryRowContext(ctx, + `SELECT owner_marker FROM sessions WHERE id = $1`, "sess-x").Scan(&foreignOwner), + "read foreign bare row") + assert.Equal(t, "foreign-owner", foreignOwner, "foreign bare row not adopted") +} + +// TestPushSkipsPrefixedConflictWithoutAbortingBatch verifies that a single +// per-session ownership conflict on the current-machine prefixed id is skipped +// and reported, while unrelated sessions in the same push still go through. The +// conflict must not fail the whole push from identity pre-resolution. +func TestPushSkipsPrefixedConflictWithoutAbortingBatch(t *testing.T) { + pgURL := testPGURL(t) + + const schema = "agentsview_prefixed_conflict_skip_test" + pg, err := Open(pgURL, schema, true) + require.NoError(t, err, "Open") + defer pg.Close() + + ctx := context.Background() + _, err = pg.Exec(`DROP SCHEMA IF EXISTS ` + schema + ` CASCADE`) + require.NoError(t, err, "drop schema") + require.NoError(t, EnsureSchema(ctx, pg, schema), "EnsureSchema") + + localDB, err := db.Open(filepath.Join(t.TempDir(), "local.db")) + require.NoError(t, err, "db.Open") + defer localDB.Close() + + sync := &Sync{ + pg: pg, + local: localDB, + machine: "new-host", + schema: schema, + schemaDone: true, + } + markerID, err := sync.pushMarkerID() + require.NoError(t, err, "pushMarkerID") + + // A different owner holds both the bare id and the current-machine prefixed + // id, so "conf-1" has nowhere to land and must be skipped as a conflict. + for _, id := range []string{"conf-1", "new-host~conf-1"} { + _, err = pg.ExecContext(ctx, ` + INSERT INTO sessions (id, machine, owner_marker, project, agent, created_at) + VALUES ($1, $2, $3, $4, $5, NOW())`, + id, "other-host", "other-owner", "proj", "claude") + require.NoError(t, err, "insert foreign "+id) + } + + for _, s := range []db.Session{ + {ID: "conf-1", Project: "proj", Machine: "new-host", Agent: "claude", + MessageCount: 1, CreatedAt: "2026-01-01T00:00:00Z"}, + {ID: "clean-1", Project: "proj", Machine: "new-host", Agent: "claude", + MessageCount: 1, CreatedAt: "2026-01-01T00:00:00Z"}, + } { + require.NoError(t, localDB.UpsertSession(s), "UpsertSession "+s.ID) + require.NoError(t, localDB.InsertMessages([]db.Message{{ + SessionID: s.ID, Ordinal: 0, Role: "user", Content: "hi", ContentLength: 2, + }}), "InsertMessages "+s.ID) + } + + res, err := sync.Push(ctx, false, nil) + require.NoError(t, err, "a per-session conflict must not fail the whole push") + assert.Equal(t, 1, res.SkippedConflicts, "the conflicting session is reported as skipped") + + var clean int + require.NoError(t, pg.QueryRowContext(ctx, + `SELECT COUNT(*) FROM sessions WHERE id = $1 AND owner_marker = $2`, + "clean-1", markerID).Scan(&clean), "count clean session") + assert.Equal(t, 1, clean, "unrelated session pushed despite the conflict") + + var owner string + require.NoError(t, pg.QueryRowContext(ctx, + `SELECT owner_marker FROM sessions WHERE id = $1`, "new-host~conf-1").Scan(&owner), + "read foreign prefixed row") + assert.Equal(t, "other-owner", owner, "foreign prefixed row not overwritten") +} + +func TestPushSkipsRelationshipsToPrefixedOwnershipConflict(t *testing.T) { + pgURL := testPGURL(t) + + const schema = "agentsview_prefixed_conflict_relationship_test" + pg, err := Open(pgURL, schema, true) + require.NoError(t, err, "Open") + defer pg.Close() + + ctx := context.Background() + _, err = pg.Exec(`DROP SCHEMA IF EXISTS ` + schema + ` CASCADE`) + require.NoError(t, err, "drop schema") + require.NoError(t, EnsureSchema(ctx, pg, schema), "EnsureSchema") + + localDB, err := db.Open(filepath.Join(t.TempDir(), "local.db")) + require.NoError(t, err, "db.Open") + defer localDB.Close() + + sync := &Sync{ + pg: pg, + local: localDB, + machine: "new-host", + schema: schema, + schemaDone: true, + } + + for _, id := range []string{"conf-1", "new-host~conf-1"} { + _, err = pg.ExecContext(ctx, ` + INSERT INTO sessions (id, machine, owner_marker, project, agent, created_at) + VALUES ($1, $2, $3, $4, $5, NOW())`, + id, "other-host", "other-owner", "proj", "claude") + require.NoError(t, err, "insert foreign "+id) + } + + sourceID := "conf-1" + for _, s := range []db.Session{ + {ID: "conf-1", Project: "proj", Machine: "new-host", Agent: "claude", + MessageCount: 1, CreatedAt: "2026-01-01T00:00:00Z"}, + {ID: "child-1", Project: "proj", Machine: "new-host", Agent: "claude", + SourceSessionID: sourceID, MessageCount: 1, CreatedAt: "2026-01-01T00:00:00Z"}, + } { + require.NoError(t, localDB.UpsertSession(s), "UpsertSession "+s.ID) + require.NoError(t, localDB.InsertMessages([]db.Message{{ + SessionID: s.ID, Ordinal: 0, Role: "user", Content: "hi", ContentLength: 2, + }}), "InsertMessages "+s.ID) + } + + res, err := sync.Push(ctx, false, nil) + require.NoError(t, err, "relationship to a per-session conflict must not fail the whole push") + assert.Equal(t, 2, res.SkippedConflicts, + "the conflicted session and the dependent relationship session are skipped") + assert.Zero(t, res.Errors) + + var childRows int + require.NoError(t, pg.QueryRowContext(ctx, + `SELECT COUNT(*) FROM sessions WHERE id = $1`, "child-1").Scan(&childRows), + "count child session") + assert.Equal(t, 0, childRows, "dependent session must not be pushed with a foreign source link") + + var foreignRefs int + require.NoError(t, pg.QueryRowContext(ctx, + `SELECT COUNT(*) FROM sessions WHERE source_session_id = $1`, + "new-host~conf-1").Scan(&foreignRefs), "count references to foreign prefixed row") + assert.Equal(t, 0, foreignRefs, "no pushed row may point at the foreign prefixed session") +} + +func TestPushSkipsRelationshipsToAlreadyPrefixedOwnershipConflict(t *testing.T) { + pgURL := testPGURL(t) + + const schema = "agentsview_already_prefixed_conflict_relationship_test" + pg, err := Open(pgURL, schema, true) + require.NoError(t, err, "Open") + defer pg.Close() + + ctx := context.Background() + _, err = pg.Exec(`DROP SCHEMA IF EXISTS ` + schema + ` CASCADE`) + require.NoError(t, err, "drop schema") + require.NoError(t, EnsureSchema(ctx, pg, schema), "EnsureSchema") + + localDB, err := db.Open(filepath.Join(t.TempDir(), "local.db")) + require.NoError(t, err, "db.Open") + defer localDB.Close() + + sync := &Sync{ + pg: pg, + local: localDB, + machine: "new-host", + schema: schema, + schemaDone: true, + } + + const conflictedID = "new-host~conf-1" + _, err = pg.ExecContext(ctx, ` + INSERT INTO sessions (id, machine, owner_marker, project, agent, created_at) + VALUES ($1, $2, $3, $4, $5, NOW())`, + conflictedID, "other-host", "other-owner", "proj", "claude") + require.NoError(t, err, "insert foreign already-prefixed row") + + for _, s := range []db.Session{ + {ID: conflictedID, Project: "proj", Machine: "new-host", Agent: "claude", + MessageCount: 1, CreatedAt: "2026-01-01T00:00:00Z"}, + {ID: "child-1", Project: "proj", Machine: "new-host", Agent: "claude", + SourceSessionID: conflictedID, MessageCount: 1, CreatedAt: "2026-01-01T00:00:00Z"}, + } { + require.NoError(t, localDB.UpsertSession(s), "UpsertSession "+s.ID) + require.NoError(t, localDB.InsertMessages([]db.Message{{ + SessionID: s.ID, Ordinal: 0, Role: "user", Content: "hi", ContentLength: 2, + }}), "InsertMessages "+s.ID) + } + + res, err := sync.Push(ctx, false, nil) + require.NoError(t, err, "already-prefixed relationship conflict must not fail the whole push") + assert.Equal(t, 2, res.SkippedConflicts, + "the conflicted already-prefixed session and its dependent are skipped") + assert.Zero(t, res.Errors) + + var childRows int + require.NoError(t, pg.QueryRowContext(ctx, + `SELECT COUNT(*) FROM sessions WHERE id = $1`, "child-1").Scan(&childRows), + "count child session") + assert.Equal(t, 0, childRows, "dependent session must not be pushed with a foreign source link") + + var foreignRefs int + require.NoError(t, pg.QueryRowContext(ctx, + `SELECT COUNT(*) FROM sessions WHERE source_session_id = $1`, + conflictedID).Scan(&foreignRefs), "count references to foreign already-prefixed row") + assert.Equal(t, 0, foreignRefs, "no pushed row may point at the foreign already-prefixed session") +} diff --git a/internal/postgres/curation.go b/internal/postgres/curation.go index 9d1b96ebf..1bb59fddd 100644 --- a/internal/postgres/curation.go +++ b/internal/postgres/curation.go @@ -3,6 +3,7 @@ package postgres import ( "context" "database/sql" + "errors" "fmt" "time" @@ -43,15 +44,20 @@ func (s *Store) StarSession(sessionID string) (bool, error) { } // UnstarSession removes a session star from the shared PG dashboard -// metadata. -func (s *Store) UnstarSession(sessionID string) error { - if _, err := s.pg.Exec( +// metadata and reports whether a row was removed. +func (s *Store) UnstarSession(sessionID string) (bool, error) { + res, err := s.pg.Exec( `DELETE FROM starred_sessions WHERE session_id = $1`, sessionID, - ); err != nil { - return fmt.Errorf("unstarring session %s: %w", sessionID, err) + ) + if err != nil { + return false, fmt.Errorf("unstarring session %s: %w", sessionID, err) } - return nil + n, err := res.RowsAffected() + if err != nil { + return false, fmt.Errorf("checking unstar result for %s: %w", sessionID, err) + } + return n > 0, nil } // ListStarredSessionIDs returns shared PG-starred session IDs. @@ -80,35 +86,60 @@ func (s *Store) ListStarredSessionIDs( // BulkStarSessions stars multiple existing sessions in one transaction. // Unknown session IDs are skipped. -func (s *Store) BulkStarSessions(sessionIDs []string) error { +func (s *Store) BulkStarSessions(sessionIDs []string) ([]string, error) { if len(sessionIDs) == 0 { - return nil + return nil, nil } tx, err := s.pg.Begin() if err != nil { - return fmt.Errorf("beginning star transaction: %w", err) + return nil, fmt.Errorf("beginning star transaction: %w", err) } defer func() { _ = tx.Rollback() }() - stmt, err := tx.Prepare(` + // Check existence separately from the insert so stale IDs are skipped and + // the caller learns which sessions were actually starred, mirroring the + // SQLite backend. + exists, err := tx.Prepare(`SELECT 1 FROM sessions WHERE id = $1`) + if err != nil { + return nil, fmt.Errorf("preparing existence statement: %w", err) + } + defer exists.Close() + insert, err := tx.Prepare(` INSERT INTO starred_sessions (session_id) - SELECT $1 WHERE EXISTS ( - SELECT 1 FROM sessions WHERE id = $1 - ) + VALUES ($1) ON CONFLICT (session_id) DO NOTHING`) if err != nil { - return fmt.Errorf("preparing star statement: %w", err) + return nil, fmt.Errorf("preparing star statement: %w", err) } - defer stmt.Close() + defer insert.Close() + starred := make([]string, 0, len(sessionIDs)) for _, id := range sessionIDs { - if _, err := stmt.Exec(id); err != nil { - return fmt.Errorf("starring session %s: %w", id, err) + var one int + switch err := exists.QueryRow(id).Scan(&one); { + case errors.Is(err, sql.ErrNoRows): + continue + case err != nil: + return nil, fmt.Errorf("checking session %s: %w", id, err) + } + res, err := insert.Exec(id) + if err != nil { + return nil, fmt.Errorf("starring session %s: %w", id, err) + } + rowsAffected, err := res.RowsAffected() + if err != nil { + return nil, fmt.Errorf("checking star insert result for %s: %w", id, err) + } + if rowsAffected > 0 { + starred = append(starred, id) } } - return tx.Commit() + if err := tx.Commit(); err != nil { + return nil, fmt.Errorf("committing star transaction: %w", err) + } + return starred, nil } // PinMessage creates or updates a shared PG pin for a message. PG diff --git a/internal/postgres/curation_pgtest_test.go b/internal/postgres/curation_pgtest_test.go index 784924c3a..c424903d5 100644 --- a/internal/postgres/curation_pgtest_test.go +++ b/internal/postgres/curation_pgtest_test.go @@ -69,9 +69,12 @@ func TestStoreStarsAndPins(t *testing.T) { ok, err = store.StarSession("missing") require.NoError(t, err, "StarSession missing") assert.False(t, ok, "StarSession missing") - require.NoError(t, store.BulkStarSessions( - []string{"cur-star-2", "missing"}, - ), "BulkStarSessions") + bulkStarred, err := store.BulkStarSessions([]string{"cur-star-2", "missing"}) + require.NoError(t, err, "BulkStarSessions") + assert.Equal(t, []string{"cur-star-2"}, bulkStarred, "starred ids returned") + bulkStarred, err = store.BulkStarSessions([]string{"cur-star-1", "cur-star-2"}) + require.NoError(t, err, "BulkStarSessions already starred") + assert.Empty(t, bulkStarred, "already-starred ids should not be returned") ids, err := store.ListStarredSessionIDs(ctx) require.NoError(t, err, "ListStarredSessionIDs") @@ -83,7 +86,12 @@ func TestStoreStarsAndPins(t *testing.T) { for _, id := range ids { assert.True(t, wantStars[id], "unexpected starred id %q in %v", id, ids) } - require.NoError(t, store.UnstarSession("cur-star-1"), "UnstarSession") + removed, err := store.UnstarSession("cur-star-1") + require.NoError(t, err, "UnstarSession") + require.True(t, removed, "UnstarSession") + removed, err = store.UnstarSession("cur-star-1") + require.NoError(t, err, "UnstarSession no-op") + require.False(t, removed, "UnstarSession no-op") ids, err = store.ListStarredSessionIDs(ctx) require.NoError(t, err, "ListStarredSessionIDs after unstar") require.Len(t, ids, 1) @@ -247,6 +255,109 @@ func TestPushPreservesMultiplePGPinsBySourceUUID(t *testing.T) { assert.Equal(t, 3, pin.Ordinal) } +func TestPushBackfillsLegacyPinSourceUUIDBeforeOrdinalShift(t *testing.T) { + pgURL := testPGURL(t) + cleanPGSchema(t, pgURL) + t.Cleanup(func() { cleanPGSchema(t, pgURL) }) + + local := testDB(t) + ps, err := New( + pgURL, "agentsview", local, + "curation-machine", true, + SyncOptions{}, + ) + require.NoError(t, err, "New sync") + defer ps.Close() + + ctx := context.Background() + require.NoError(t, ps.EnsureSchema(ctx), "EnsureSchema") + + sess := db.Session{ + ID: "pg-legacy-pin-shift", + Project: "proj-curation", + Machine: "local", + Agent: "codex", + MessageCount: 2, + UserMessageCount: 1, + CreatedAt: "2026-05-01T00:00:00Z", + } + require.NoError(t, local.UpsertSession(sess), "UpsertSession first") + require.NoError(t, local.InsertMessages([]db.Message{ + { + SessionID: "pg-legacy-pin-shift", + Ordinal: 0, + Role: "user", + Content: "question", + SourceUUID: "uuid-question", + }, + { + SessionID: "pg-legacy-pin-shift", + Ordinal: 1, + Role: "assistant", + Content: "answer", + SourceUUID: "uuid-answer", + }, + }), "InsertMessages first") + _, err = ps.Push(ctx, false, nil) + require.NoError(t, err, "Push first") + + store, err := NewStore(pgURL, "agentsview", true) + require.NoError(t, err, "NewStore") + defer store.Close() + + note := "legacy pin" + _, err = store.PinMessage("pg-legacy-pin-shift", 1, ¬e) + require.NoError(t, err, "PinMessage") + _, err = ps.pg.ExecContext(ctx, ` + UPDATE pinned_messages + SET source_uuid = '' + WHERE session_id = $1 AND message_id = $2`, + "pg-legacy-pin-shift", 1, + ) + require.NoError(t, err, "clear source_uuid to simulate legacy pin") + + sess.MessageCount = 3 + require.NoError(t, local.UpsertSession(sess), "UpsertSession second") + require.NoError(t, local.ReplaceSessionMessages( + "pg-legacy-pin-shift", + []db.Message{ + { + SessionID: "pg-legacy-pin-shift", + Ordinal: 0, + Role: "user", + Content: "question", + SourceUUID: "uuid-question", + }, + { + SessionID: "pg-legacy-pin-shift", + Ordinal: 1, + Role: "user", + Content: "[compact]", + SourceUUID: "uuid-boundary", + IsCompactBoundary: true, + }, + { + SessionID: "pg-legacy-pin-shift", + Ordinal: 2, + Role: "assistant", + Content: "answer", + SourceUUID: "uuid-answer", + }, + }, + ), "ReplaceSessionMessages") + + _, err = ps.Push(ctx, true, nil) + require.NoError(t, err, "Push rewrite") + + pins, err := store.ListPinnedMessages(ctx, "pg-legacy-pin-shift", "") + require.NoError(t, err, "ListPinnedMessages") + require.Len(t, pins, 1) + assert.Equal(t, int64(2), pins[0].MessageID) + assert.Equal(t, 2, pins[0].Ordinal) + require.NotNil(t, pins[0].Note) + assert.Equal(t, note, *pins[0].Note) +} + func TestReconcilePinnedMessagesPrefersCurrentTargetPin(t *testing.T) { pgURL := testPGURL(t) diff --git a/internal/postgres/integration_test.go b/internal/postgres/integration_test.go index 84e6f7b63..c905ca0c7 100644 --- a/internal/postgres/integration_test.go +++ b/internal/postgres/integration_test.go @@ -163,7 +163,7 @@ func TestPushSecretFindingsReportsChange(t *testing.T) { pushOnce := func() bool { tx, err := ps.pg.BeginTx(ctx, nil) require.NoError(t, err, "begin tx") - changed, err := ps.pushSecretFindings(ctx, tx, sessID) + changed, err := ps.pushSecretFindings(ctx, tx, sessID, sessID) if err != nil { _ = tx.Rollback() t.Fatalf("pushSecretFindings: %v", err) diff --git a/internal/postgres/metadata.go b/internal/postgres/metadata.go new file mode 100644 index 000000000..6d052f618 --- /dev/null +++ b/internal/postgres/metadata.go @@ -0,0 +1,22 @@ +package postgres + +import ( + "context" + + "go.kenn.io/agentsview/internal/db" +) + +// ListMetadataConflicts returns no rows for PostgreSQL read mode because the +// local artifact metadata ledger is not part of the shared SQL mirror. +func (s *Store) ListMetadataConflicts( + context.Context, + []string, +) ([]db.MetadataConflict, error) { + return []db.MetadataConflict{}, nil +} + +// CountMetadataConflicts returns zero for PostgreSQL read mode because the local +// artifact metadata ledger is not part of the shared SQL mirror. +func (s *Store) CountMetadataConflicts(context.Context) (int, error) { + return 0, nil +} diff --git a/internal/postgres/name_source_pgtest_test.go b/internal/postgres/name_source_pgtest_test.go index dad3987a1..c8701dd74 100644 --- a/internal/postgres/name_source_pgtest_test.go +++ b/internal/postgres/name_source_pgtest_test.go @@ -62,7 +62,10 @@ func TestPushSessionNameRoundTrip(t *testing.T) { // Push via pushSession directly. tx, err := pg.BeginTx(ctx, nil) require.NoError(t, err, "BeginTx") - if err := sync.pushSession(ctx, tx, sess, markerID, nil); err != nil { + if err := sync.pushSession(ctx, tx, sess, pushedSessionIdentity{ + ID: sess.ID, + Machine: sess.Machine, + }, markerID, nil); err != nil { _ = tx.Rollback() t.Fatalf("pushSession: %v", err) } @@ -105,7 +108,10 @@ func TestPushSessionNameRoundTrip(t *testing.T) { tx2, err := pg.BeginTx(ctx, nil) require.NoError(t, err, "BeginTx (second)") - if err := sync.pushSession(ctx, tx2, sess, markerID, nil); err != nil { + if err := sync.pushSession(ctx, tx2, sess, pushedSessionIdentity{ + ID: sess.ID, + Machine: sess.Machine, + }, markerID, nil); err != nil { _ = tx2.Rollback() t.Fatalf("pushSession (second): %v", err) } diff --git a/internal/postgres/push.go b/internal/postgres/push.go index 7e6bcec85..c532bec44 100644 --- a/internal/postgres/push.go +++ b/internal/postgres/push.go @@ -16,6 +16,7 @@ import ( "strings" "time" + "go.kenn.io/agentsview/internal/artifact" "go.kenn.io/agentsview/internal/db" "go.kenn.io/agentsview/internal/export" ) @@ -24,6 +25,9 @@ const ( lastPushBoundaryStateKey = "last_push_boundary_state" lastPushTargetFingerprintKey = "pg_target_fingerprint_v1" sessionAliasBackfillStateKey = "pg_session_alias_backfill_v1" + artifactIdentityModeStateKey = "pg_artifact_identity_v1" + artifactOwnerMarkerPrefix = "artifact-origin:" + legacyArtifactIdentityMode = "legacy" ) // pushMarkerIDStateKey names the local sync-state entry holding this DB's @@ -37,6 +41,7 @@ const ( var errSessionOwnershipConflict = errors.New("session ownership conflict") var errSessionExcluded = errors.New("session excluded") +var errArtifactReplicaExists = errors.New("artifact replica already exists") type pushBoundaryState struct { Cutoff string `json:"cutoff"` @@ -167,6 +172,30 @@ func (s *Sync) Push( if err != nil { return result, err } + localArtifactOrigin, err := artifact.StoredOrigin(s.local) + if err != nil { + return result, err + } + artifactImportedSessions, err := artifact.ImportedSessionIDs(s.local) + if err != nil { + return result, err + } + artifactIdentityMode := currentArtifactIdentityMode(localArtifactOrigin) + storedArtifactIdentityMode, err := state.GetSyncState( + artifactIdentityModeStateKey, + ) + if err != nil { + return result, fmt.Errorf( + "reading %s: %w", artifactIdentityModeStateKey, err, + ) + } + if lastPush != "" && storedArtifactIdentityMode != artifactIdentityMode { + log.Printf( + "pgsync: artifact identity mode changed; forcing full push", + ) + lastPush = "" + full = true + } markerMachine, markerMachineAliases, markerExists, err := s.pgPushMarkerMachineState(ctx, markerID) if err != nil { return result, err @@ -281,6 +310,7 @@ func (s *Sync) Push( var priorFingerprints map[string]string sessionFingerprints := make(map[string]string, len(sessionByID)) + sessionIdentities := make(map[string]pushedSessionIdentity, len(sessionByID)) if !full { var bErr error priorFingerprints, _, _, bErr = readBoundaryAndFingerprints( @@ -322,8 +352,19 @@ func (s *Sync) Push( } } + for id, sess := range sessionByID { + _, artifactImported := artifactImportedSessions[sess.ID] + identity, err := s.resolvePushedSessionIdentity( + ctx, sess, localArtifactOrigin, artifactImported, + markerID, legacyMarkerMachines, + ) + if err != nil { + return result, err + } + sessionIdentities[id] = identity + } if err := purgePGExcludedPushSessions( - ctx, s.pg, sessionByID, + ctx, s.pg, sessionByID, sessionIdentities, ); err != nil { return result, err } @@ -336,10 +377,14 @@ func (s *Sync) Push( "computing local usage event fingerprints: %w", err, ) } - // The fingerprint loop issues several local queries per candidate - // session; on a full push that covers every session and runs for - // minutes, so it reports its own progress phase rather than sitting - // silent until the first batch lands. + if err := s.markRelationshipConflicts( + ctx, sessionByID, sessionIdentities, markerID, legacyMarkerMachines, + ); err != nil { + return result, err + } + // Fingerprint preparation resolves relationship targets and reads local + // dependency state in bounded batches. Full pushes can spend minutes here, + // so report this phase rather than staying silent until writes begin. log.Printf("pgsync: computing push fingerprints for %d candidate session(s)", len(sessionByID)) reportPrepare := func(done int) { @@ -363,6 +408,36 @@ func (s *Sync) Push( return result, err } for _, id := range chunk { + sess := sessionByID[id] + identity := sessionIdentities[id] + prepared++ + if identity.Conflict { + if prepared%pushPrepareProgressStride == 0 { + reportPrepare(prepared) + } + continue + } + // Resolve relationship ids (source/parent) to the ids their target + // sessions are stored under in PG once every identity is known, so the + // fingerprint and the written row agree and child rows link correctly + // even when a target was pushed under a collision-avoidance prefix. + resolver := s.newRelationshipResolver( + sessionIdentities, identity, + markerID, legacyMarkerMachines, + ) + if sess.SourceSessionID != "" || sess.ParentSessionID != nil { + resolvedSource, err := resolver.resolve(ctx, sess.SourceSessionID) + if err != nil { + return result, err + } + sess.SourceSessionID = resolvedSource + resolvedParent, err := resolver.resolvePtr(ctx, sess.ParentSessionID) + if err != nil { + return result, err + } + sess.ParentSessionID = resolvedParent + sessionByID[id] = sess + } usageFP, usageKnown := usageFingerprints[id] dependencyFP, err := depState.dependencyFingerprint( s.local, id, usageFP, usageKnown, @@ -373,12 +448,14 @@ func (s *Sync) Push( id, err, ) } - sess := sessionByID[id] + subagentFP, err := resolver.resolvedSubagentLinkFingerprint(ctx, id) + if err != nil { + return result, err + } sessionFingerprints[id] = sessionPushFingerprint( - sess, pushedSessionMachine(sess, s.machine), - usageFP, markerID, dependencyFP, + sess, identity.ID, identity.Machine, + usageFP, identity.effectiveOwnerMarker(markerID), dependencyFP+subagentFP, ) - prepared++ if prepared%pushPrepareProgressStride == 0 { reportPrepare(prepared) } @@ -388,6 +465,9 @@ func (s *Sync) Push( if len(priorFingerprints) > 0 { for id := range sessionByID { + if sessionIdentities[id].Conflict { + continue + } if priorFingerprints[id] == sessionFingerprints[id] { delete(sessionByID, id) } @@ -438,6 +518,11 @@ func (s *Sync) Push( ); err != nil { return result, err } + if err := completeArtifactIdentityMode( + state, artifactIdentityMode, result, + ); err != nil { + return result, err + } result.Vectors, err = s.runVectorPushPhase(ctx, full, nil, onProgress) if err != nil { return result, err @@ -446,61 +531,32 @@ func (s *Sync) Push( return result, nil } - var pushed []db.Session - // Sessions whose individual retry also failed: their PG sessions/messages - // rows are stale or absent, so the vector phase must not push their newer - // local vectors ahead of them. - var failedSessions map[string]struct{} - const batchSize = 50 - for i := 0; i < len(sessions); i += batchSize { - end := min(i+batchSize, len(sessions)) - batch := sessions[i:end] - - batchResult, err := s.pushBatch( - ctx, batch, full, markerID, legacyMarkerMachines, - usageFingerprints, &pushed, - ) - if err != nil { - return result, err - } - if batchResult.ok { - result.SessionsPushed += batchResult.sessions - result.MessagesPushed += batchResult.messages - result.SkippedConflicts += batchResult.skippedConflicts - } else { - // Batch failed — retry each session individually - // so one bad session doesn't block the rest. - for _, sess := range batch { - sr, retryErr := s.pushBatch( - ctx, []db.Session{sess}, - full, markerID, legacyMarkerMachines, - usageFingerprints, &pushed, - ) - if retryErr != nil { - return result, retryErr - } - if sr.ok { - result.SessionsPushed += sr.sessions - result.MessagesPushed += sr.messages - result.SkippedConflicts += sr.skippedConflicts - } else { - result.Errors++ - if failedSessions == nil { - failedSessions = make(map[string]struct{}) - } - failedSessions[sess.ID] = struct{}{} - } - } - } - if onProgress != nil { - onProgress(PushProgress{ - SessionsDone: end, - SessionsTotal: len(sessions), - MessagesDone: result.MessagesPushed, - SkippedConflicts: result.SkippedConflicts, - Errors: result.Errors, - }) - } + sink := pgSessionSink{ + sync: s, + full: full, + markerID: markerID, + legacyMarkerMachines: legacyMarkerMachines, + sessionUsageFingerprints: usageFingerprints, + identities: sessionIdentities, + } + pushed, err := drainSessionBatches( + ctx, sessions, sink, &result, onProgress, + ) + if err != nil { + return result, err + } + // Sessions not written by the session phase include failed retries and + // ownership conflicts. Their vectors must not advance ahead of the + // sessions/messages rows they depend on. + failedSessions := make(map[string]struct{}, len(sessions)-len(pushed)) + for _, sess := range sessions { + failedSessions[sess.ID] = struct{}{} + } + for _, sess := range pushed { + delete(failedSessions, sess.ID) + } + if len(failedSessions) == 0 { + failedSessions = nil } if s.isFiltered() { @@ -543,6 +599,11 @@ func (s *Sync) Push( ); err != nil { return result, err } + if err := completeArtifactIdentityMode( + state, artifactIdentityMode, result, + ); err != nil { + return result, err + } result.Vectors, err = s.runVectorPushPhase(ctx, full, failedSessions, onProgress) if err != nil { return result, err @@ -628,6 +689,99 @@ func filterProjectIdentityObservations( return out } +// sessionBatchSink persists batches of sessions to a target during a push. +// Extracting this seam keeps the batch/retry/progress orchestration +// (drainSessionBatches) free of any SQL: PostgreSQL is one sink today, and the +// artifact exporter can become another without duplicating the loop. +type sessionBatchSink interface { + // writeBatch persists batch atomically and appends successfully written + // sessions to *pushed. It returns ok=false (and no error) when the batch + // failed for a reason the caller should recover from by retrying each + // session individually. A non-nil error is fatal and aborts the push. + writeBatch( + ctx context.Context, batch []db.Session, pushed *[]db.Session, + ) (batchResult, error) +} + +// pgSessionSink writes batches to PostgreSQL via Sync.pushBatch. It binds the +// per-push parameters (full mode, push marker identity) so the orchestration +// does not need to thread them through. +type pgSessionSink struct { + sync *Sync + full bool + markerID string + legacyMarkerMachines []string + sessionUsageFingerprints map[string]string + identities map[string]pushedSessionIdentity +} + +func (p pgSessionSink) writeBatch( + ctx context.Context, batch []db.Session, pushed *[]db.Session, +) (batchResult, error) { + return p.sync.pushBatch( + ctx, batch, p.full, p.markerID, p.legacyMarkerMachines, + p.sessionUsageFingerprints, pushed, p.identities, + ) +} + +// drainSessionBatches pushes sessions through sink in fixed-size batches, +// accumulating counts into result and reporting progress after each batch. +// When a batch fails (ok=false) it retries each session individually so one +// bad session does not block the rest; sessions that still fail are counted as +// errors. It returns the sessions successfully written, in push order. +func drainSessionBatches( + ctx context.Context, + sessions []db.Session, + sink sessionBatchSink, + result *PushResult, + onProgress func(PushProgress), +) ([]db.Session, error) { + var pushed []db.Session + const batchSize = 50 + for i := 0; i < len(sessions); i += batchSize { + end := min(i+batchSize, len(sessions)) + batch := sessions[i:end] + + batchResult, err := sink.writeBatch(ctx, batch, &pushed) + if err != nil { + return pushed, err + } + if batchResult.ok { + result.SessionsPushed += batchResult.sessions + result.MessagesPushed += batchResult.messages + result.SkippedConflicts += batchResult.skippedConflicts + } else { + // Batch failed — retry each session individually + // so one bad session doesn't block the rest. + for _, sess := range batch { + sr, retryErr := sink.writeBatch( + ctx, []db.Session{sess}, &pushed, + ) + if retryErr != nil { + return pushed, retryErr + } + if sr.ok { + result.SessionsPushed += sr.sessions + result.MessagesPushed += sr.messages + result.SkippedConflicts += sr.skippedConflicts + } else { + result.Errors++ + } + } + } + if onProgress != nil { + onProgress(PushProgress{ + SessionsDone: end, + SessionsTotal: len(sessions), + MessagesDone: result.MessagesPushed, + SkippedConflicts: result.SkippedConflicts, + Errors: result.Errors, + }) + } + } + return pushed, nil +} + // pgPushMarkerMachineState reports whether this host's push marker is present // in PG and returns the current machine plus legacy machine aliases stored with // the marker. @@ -886,11 +1040,12 @@ func (s *Sync) pushBatch( legacyMarkerMachines []string, sessionUsageFingerprints map[string]string, pushed *[]db.Session, + identities map[string]pushedSessionIdentity, ) (batchResult, error) { preloadComparisons := len(batch) > 0 && !full result, err := s.pushBatchAttempt( ctx, batch, full, markerID, legacyMarkerMachines, - sessionUsageFingerprints, pushed, preloadComparisons, + sessionUsageFingerprints, pushed, identities, preloadComparisons, ) if err == nil || !errors.Is(err, errPushComparisonPreload) { return result, err @@ -902,7 +1057,7 @@ func (s *Sync) pushBatch( ) return s.pushBatchAttempt( ctx, batch, full, markerID, legacyMarkerMachines, - sessionUsageFingerprints, pushed, false, + sessionUsageFingerprints, pushed, identities, false, ) } @@ -914,6 +1069,7 @@ func (s *Sync) pushBatchAttempt( legacyMarkerMachines []string, sessionUsageFingerprints map[string]string, pushed *[]db.Session, + identities map[string]pushedSessionIdentity, preloadComparisons bool, ) (batchResult, error) { tx, err := s.pg.BeginTx(ctx, nil) @@ -928,7 +1084,17 @@ func (s *Sync) pushBatchAttempt( skippedConflicts := 0 sessionIDs := make([]string, 0, len(batch)) for _, sess := range batch { - sessionIDs = append(sessionIDs, sess.ID) + identity := identities[sess.ID] + if identity.Conflict { + continue + } + if identity.ID == "" { + identity = pushedSessionIdentity{ + ID: sess.ID, + Machine: pushedSessionMachine(sess, s.machine), + } + } + sessionIDs = append(sessionIDs, identity.ID) } comparisons := (*pushMessageComparison)(nil) if preloadComparisons && len(sessionIDs) > 0 { @@ -945,9 +1111,23 @@ func (s *Sync) pushBatchAttempt( } for _, sess := range batch { + identity := identities[sess.ID] + if identity.Conflict { + skippedConflicts++ + continue + } + if identity.ID == "" { + identity = pushedSessionIdentity{ + ID: sess.ID, + Machine: pushedSessionMachine(sess, s.machine), + } + } if err := s.pushSession( - ctx, tx, sess, markerID, legacyMarkerMachines, + ctx, tx, sess, identity, markerID, legacyMarkerMachines, ); err != nil { + if errors.Is(err, errArtifactReplicaExists) { + continue + } if errors.Is(err, errSessionOwnershipConflict) { skippedConflicts++ continue @@ -964,8 +1144,11 @@ func (s *Sync) pushBatchAttempt( return batchResult{}, nil } + resolver := s.newRelationshipResolver( + identities, identity, markerID, legacyMarkerMachines, + ) msgCount, err := s.pushMessages( - ctx, tx, sess.ID, full, + ctx, tx, sess.ID, identity.ID, resolver, full, sessionUsageFingerprints, comparisons, ) if err != nil { @@ -978,7 +1161,9 @@ func (s *Sync) pushBatchAttempt( return batchResult{}, nil } - findingsChanged, err := s.pushSecretFindings(ctx, tx, sess.ID) + findingsChanged, err := s.pushSecretFindings( + ctx, tx, sess.ID, identity.ID, + ) if err != nil { log.Printf( "pgsync: secret findings %s: %v", @@ -998,11 +1183,11 @@ func (s *Sync) pushBatchAttempt( UPDATE sessions SET updated_at = NOW() WHERE id = $1`, - sess.ID, + identity.ID, ); err != nil { log.Printf( "pgsync: bumping updated_at %s: %v", - sess.ID, err, + identity.ID, err, ) _ = tx.Rollback() *pushed = (*pushed)[:len(*pushed)-n] @@ -1154,6 +1339,25 @@ func completeSessionAliasBackfill( return markSessionAliasBackfillDone(local) } +func currentArtifactIdentityMode(origin string) string { + if origin == "" { + return legacyArtifactIdentityMode + } + return artifactOwnerMarkerPrefix + origin +} + +func completeArtifactIdentityMode( + local syncStateStore, mode string, result PushResult, +) error { + if result.Errors > 0 { + return nil + } + if err := local.SetSyncState(artifactIdentityModeStateKey, mode); err != nil { + return fmt.Errorf("updating %s: %w", artifactIdentityModeStateKey, err) + } + return nil +} + func persistPushTargetFingerprint( local syncStateStore, fingerprint string, @@ -1367,12 +1571,19 @@ func pgExcludedSessionIDsQuery(ids []string) (string, []any) { } func purgePGExcludedPushSessions( - ctx context.Context, pg *sql.DB, sessionByID map[string]db.Session, + ctx context.Context, + pg *sql.DB, + sessionByID map[string]db.Session, + identities map[string]pushedSessionIdentity, ) error { tombstoneIDsBySession := make(map[string][]string, len(sessionByID)) candidateIDs := []string{} for id, sess := range sessionByID { - tombstoneIDs := pgSessionTombstoneIDs(sess) + identity := identities[id] + if identity.Conflict || identity.ID == "" { + continue + } + tombstoneIDs := pgSessionTombstoneIDsForPushedID(sess, identity.ID) tombstoneIDsBySession[id] = tombstoneIDs candidateIDs = append(candidateIDs, tombstoneIDs...) } @@ -1437,9 +1648,9 @@ func deletePGExcludedSessionRows( } func deletePGSessionIfExcluded( - ctx context.Context, tx *sql.Tx, sess db.Session, + ctx context.Context, tx *sql.Tx, sess db.Session, pushedID string, ) (bool, error) { - ids := pgSessionTombstoneIDs(sess) + ids := pgSessionTombstoneIDsForPushedID(sess, pushedID) excluded, err := readPGExcludedSessionIDs(ctx, tx, ids) if err != nil { return false, err @@ -1456,17 +1667,25 @@ func deletePGSessionIfExcluded( return true, nil } +func pgSessionTombstoneIDsForPushedID(sess db.Session, pushedID string) []string { + if pushedID != "" { + sess.ID = pushedID + } + return pgSessionTombstoneIDs(sess) +} + // sessionPushFingerprint builds the change-detection fingerprint for a -// session. pushedMachine is the value pushSession actually writes to PG -// (pushedSessionMachine), not the raw sess.Machine: a "local"/empty sentinel -// row is written under the fallback machine, so the fingerprint must track the -// fallback to force a re-push when s.machine changes. +// session. pushedID and pushedMachine are the values pushSession actually +// writes to PG. They may differ from sess.ID/sess.Machine when a native +// session ID collides across machines or a "local"/empty sentinel row is +// written under the fallback machine, so the fingerprint must track them to +// force a re-push when the resolved PG identity changes. func sessionPushFingerprint( - sess db.Session, pushedMachine, + sess db.Session, pushedID, pushedMachine, usageEventFingerprint, ownerMarker, dependencyFingerprint string, ) string { fields := []string{ - sess.ID, + pushedID, sess.Project, pushedMachine, ownerMarker, @@ -1527,21 +1746,553 @@ func sessionPushFingerprint( sess.SecretsRulesVersion, usageEventFingerprint, } - var b strings.Builder - for _, f := range fields { - fmt.Fprintf(&b, "%d:%s", len(f), f) + var b strings.Builder + for _, f := range fields { + fmt.Fprintf(&b, "%d:%s", len(f), f) + } + return b.String() +} + +// pushedSessionMachine resolves the machine field for a PG row. Old rows +// pushed before this fix with machine="local" will be repaired gradually as +// each session is modified (message count change, etc.) and re-fingerprinted. +func pushedSessionMachine(sess db.Session, fallbackMachine string) string { + if sess.Machine != "" && sess.Machine != "local" { + return sess.Machine + } + return fallbackMachine +} + +func artifactPushIdentity( + sess db.Session, localOrigin string, artifactImported bool, +) (id, machine, ownerMarker string, ok bool) { + origin := "" + switch { + case localOrigin != "" && (sess.Machine == "" || sess.Machine == "local"): + origin = localOrigin + case localOrigin != "" && artifactImported && + sess.Machine != "" && sess.Machine != "local" && + strings.HasPrefix(sess.ID, sess.Machine+"~"): + origin = sess.Machine + default: + return "", "", "", false + } + return prefixedSessionID(origin, sess.ID), origin, + artifactOwnerMarkerPrefix + origin, true +} + +type pushedSessionIdentity struct { + ID string + Machine string + OwnerMarker string + LegacyOwnerMarkers []string + ArtifactReplica bool + AliasIDs []string + LegacyDuplicateID string + Conflict bool +} + +func (i pushedSessionIdentity) effectiveOwnerMarker(fallback string) string { + if i.OwnerMarker != "" { + return i.OwnerMarker + } + return fallback +} + +func (s *Sync) markRelationshipConflicts( + ctx context.Context, + sessionByID map[string]db.Session, + identities map[string]pushedSessionIdentity, + markerID string, + legacyMarkerMachines []string, +) error { + changed := true + for changed { + changed = false + for id, sess := range sessionByID { + identity := identities[id] + if identity.Conflict { + continue + } + resolver := s.newRelationshipResolver( + identities, identity, markerID, legacyMarkerMachines, + ) + if sess.SourceSessionID != "" { + if _, err := resolver.resolve(ctx, sess.SourceSessionID); err != nil { + if errors.Is(err, errSessionOwnershipConflict) { + identity.Conflict = true + identities[id] = identity + changed = true + continue + } + return err + } + } + if sess.ParentSessionID != nil && *sess.ParentSessionID != "" { + if _, err := resolver.resolve(ctx, *sess.ParentSessionID); err != nil { + if errors.Is(err, errSessionOwnershipConflict) { + identity.Conflict = true + identities[id] = identity + changed = true + continue + } + return err + } + } + if _, err := resolver.sessionNeedsSubagentRewrite(ctx, id); err != nil { + if errors.Is(err, errSessionOwnershipConflict) { + identity.Conflict = true + identities[id] = identity + changed = true + continue + } + return err + } + } + } + return nil +} + +// resolvePushedSessionIdentity decides the PG id a local session is stored +// under. A session this sync owns -- by matching push marker, or an adoptable +// legacy/ownerless row (see sameSessionOwner) -- is updated in place: an +// existing row under the current or any prior machine prefix is reused, so +// machine renames and marker adoption keep updating the same row instead of +// creating a duplicate. Only a bare id already held by a different owner +// collides; that session is stored under the current machine prefix so both +// rows coexist instead of ping-ponging (issue 655). A collision is not rejected +// here: pushSession skips the conflicting row, so one conflicting session never +// fails the whole push. +func (s *Sync) resolvePushedSessionIdentity( + ctx context.Context, + sess db.Session, + localArtifactOrigin string, + artifactImported bool, + markerID string, + legacyMarkerMachines []string, +) (pushedSessionIdentity, error) { + identity := pushedSessionIdentity{ + ID: sess.ID, + Machine: pushedSessionMachine(sess, s.machine), + } + artifactIdentity := false + if id, machine, ownerMarker, ok := artifactPushIdentity( + sess, localArtifactOrigin, artifactImported, + ); ok { + artifactIdentity = true + identity.ID = id + identity.Machine = machine + identity.OwnerMarker = ownerMarker + identity.LegacyOwnerMarkers = []string{markerID} + identity.ArtifactReplica = artifactImported + identity.AliasIDs = artifactPushAliasIDs(sess, id, machine) + } + canonicalID := identity.ID + id, err := s.resolveOwnedPushIdentityID( + ctx, identity.ID, identity, markerID, legacyMarkerMachines, + ) + if err != nil { + if errors.Is(err, errSessionOwnershipConflict) { + identity.ID = "" + identity.Conflict = true + return identity, nil + } + return pushedSessionIdentity{}, err + } + identity.ID = id + if artifactIdentity && id == canonicalID { + legacyDuplicateID, conflict, resolveErr := + s.artifactLegacyDuplicateCandidate( + ctx, canonicalID, identity, markerID, + ) + if resolveErr != nil { + return pushedSessionIdentity{}, resolveErr + } + if conflict { + identity.ID = "" + identity.Conflict = true + return identity, nil + } + identity.LegacyDuplicateID = legacyDuplicateID + } + return identity, nil +} + +// artifactLegacyDuplicateCandidate recognizes the narrow upgrade state where +// an importer already created the stable artifact id while this origin still +// owns its pre-artifact bare row. Both rows must already have the exact owners +// expected for that history. A foreign-owned bare alias is an unrelated id +// collision and is ignored; only a current-marker alias with invalid canonical +// ownership is surfaced as a conflict rather than guessed safe to merge. +func (s *Sync) artifactLegacyDuplicateCandidate( + ctx context.Context, + canonicalID string, + identity pushedSessionIdentity, + markerID string, +) (legacyDuplicateID string, conflict bool, err error) { + if canonicalID == "" || identity.OwnerMarker == "" || markerID == "" { + return "", false, nil + } + canonicalMachine, canonicalOwnerMarker, canonicalExists, canonicalErr := + s.pgSessionOwner(ctx, canonicalID) + if canonicalErr != nil { + return "", false, canonicalErr + } + if !canonicalExists { + return "", false, nil + } + for _, aliasID := range uniqueNonEmptyStrings(identity.AliasIDs) { + if aliasID == canonicalID { + continue + } + _, aliasOwnerMarker, aliasExists, aliasErr := s.pgSessionOwner( + ctx, aliasID, + ) + if aliasErr != nil { + return "", false, aliasErr + } + if !aliasExists { + continue + } + if aliasOwnerMarker != markerID { + continue + } + if canonicalMachine != identity.Machine || + canonicalOwnerMarker != identity.OwnerMarker { + return "", true, nil + } + return aliasID, false, nil + } + return "", false, nil +} + +func (s *Sync) resolveOwnedPushIdentityID( + ctx context.Context, + localID string, + identity pushedSessionIdentity, + markerID string, + legacyMarkerMachines []string, +) (string, error) { + machine := identity.Machine + currentPrefixedID := prefixedSessionID(machine, localID) + currentPrefixConflict := false + for _, candidate := range pushIDMachinePrefixes(machine, legacyMarkerMachines) { + prefixedID := prefixedSessionID(candidate, localID) + if prefixedID == localID { + continue + } + existingMachine, ownerMarker, ok, err := s.pgSessionOwner(ctx, prefixedID) + if err != nil { + return "", err + } + if ok && samePushedSessionOwner( + ownerMarker, existingMachine, identity, markerID, legacyMarkerMachines, + ) { + return prefixedID, nil + } + if ok && prefixedID == currentPrefixedID { + currentPrefixConflict = true + } + } + existingMachine, ownerMarker, ok, err := s.pgSessionOwner(ctx, localID) + if err != nil { + return "", err + } + if ok && samePushedSessionOwner( + ownerMarker, existingMachine, identity, markerID, legacyMarkerMachines, + ) { + return localID, nil + } + for _, aliasID := range uniqueNonEmptyStrings(identity.AliasIDs) { + if aliasID == localID { + continue + } + aliasMachine, aliasOwnerMarker, aliasExists, aliasErr := s.pgSessionOwner( + ctx, aliasID, + ) + if aliasErr != nil { + return "", aliasErr + } + if aliasExists && samePushedSessionOwner( + aliasOwnerMarker, aliasMachine, identity, + markerID, legacyMarkerMachines, + ) { + return aliasID, nil + } + } + if ok && !samePushedSessionOwner( + ownerMarker, existingMachine, identity, markerID, legacyMarkerMachines, + ) { + if localID == currentPrefixedID || currentPrefixConflict { + return "", errSessionOwnershipConflict + } + return prefixedSessionID(machine, localID), nil + } + return localID, nil +} + +func artifactPushAliasIDs( + sess db.Session, canonicalID, origin string, +) []string { + aliasID := sess.ID + if sess.Machine != "" && sess.Machine != "local" { + aliasID = strings.TrimPrefix(sess.ID, origin+"~") + } + if aliasID == "" || aliasID == canonicalID { + return nil + } + return []string{aliasID} +} + +func artifactCanonicalAliasIDs(origin, canonicalID string) []string { + prefix := origin + "~" + if origin == "" || !strings.HasPrefix(canonicalID, prefix) { + return nil + } + aliasID := strings.TrimPrefix(canonicalID, prefix) + if aliasID == "" || aliasID == canonicalID { + return nil + } + return []string{aliasID} +} + +// pushIDMachinePrefixes lists the machine names whose id prefixes identify rows +// this owner may already hold: the current machine first, then prior machine +// names (legacy marker machines) the same marker pushed under before a rename. +// The current machine is not repeated when it also appears in the legacy set. +func pushIDMachinePrefixes(machine string, legacyMarkerMachines []string) []string { + prefixes := make([]string, 0, len(legacyMarkerMachines)+1) + if machine != "" { + prefixes = append(prefixes, machine) + } + for _, m := range legacyMarkerMachines { + if m == machine || m == "" { + continue + } + prefixes = append(prefixes, m) + } + return prefixes +} + +// pgSessionOwner returns the machine and owner_marker of a PG session row, and +// whether it exists. owner_marker is empty for legacy rows pushed before the +// marker model. +func (s *Sync) pgSessionOwner( + ctx context.Context, + id string, +) (string, string, bool, error) { + var machine string + var ownerMarker sql.NullString + err := s.pg.QueryRowContext(ctx, + `SELECT machine, owner_marker FROM sessions WHERE id = $1`, + id, + ).Scan(&machine, &ownerMarker) + if err != nil { + if errors.Is(err, sql.ErrNoRows) { + return "", "", false, nil + } + return "", "", false, fmt.Errorf( + "reading pg session owner for %s: %w", + id, err, + ) + } + return machine, ownerMarker.String, true, nil +} + +func prefixedSessionID(machine, id string) string { + if machine == "" || id == "" { + return id + } + prefix := machine + "~" + if strings.HasPrefix(id, prefix) { + return id + } + return prefix + id +} + +// relationshipResolver maps the local session ids that appear in a session's +// relationship fields (source/parent) and in tool-call subagent links to the +// ids those target sessions are stored under in PG. When a target session was +// pushed under a collision-avoidance prefix (machine~id), the unprefixed local +// id would dangle or point at a different machine's session; resolving keeps +// child and subagent rows linked to the correct PG row. +// +// A child session and the sessions it references (parent, source, subagents) +// originate on the same machine, so the resolver is scoped to one machine and +// mirrors resolvePushedSessionIdentity's ownership rule. +type relationshipResolver struct { + sync *Sync + identities map[string]pushedSessionIdentity + identity pushedSessionIdentity + markerID string + legacyMarkerMachines []string + cache map[string]string +} + +func (s *Sync) newRelationshipResolver( + identities map[string]pushedSessionIdentity, + identity pushedSessionIdentity, + markerID string, + legacyMarkerMachines []string, +) relationshipResolver { + return relationshipResolver{ + sync: s, + identities: identities, + identity: identity, + markerID: markerID, + legacyMarkerMachines: legacyMarkerMachines, + cache: make(map[string]string), + } +} + +// resolve maps a local session id to the id it is stored under in PG. It +// prefers the in-run identity map (which already reflects collision prefixing +// for every session in this push) and falls back to committed PG state for +// targets outside the push window. +func (r relationshipResolver) resolve( + ctx context.Context, localID string, +) (string, error) { + if localID == "" { + return "", nil + } + if identity, ok := r.identities[localID]; ok { + if identity.Conflict { + return "", errSessionOwnershipConflict + } + if identity.ID != "" { + return identity.ID, nil + } + } + if resolved, ok := r.cache[localID]; ok { + return resolved, nil + } + resolved, err := r.lookup(ctx, localID) + if err != nil { + return "", err + } + r.cache[localID] = resolved + return resolved, nil +} + +// resolvePtr resolves a relationship id held behind a pointer, preserving nil +// and returning the original pointer when the value is unchanged. +func (r relationshipResolver) resolvePtr( + ctx context.Context, localID *string, +) (*string, error) { + if localID == nil || *localID == "" { + return localID, nil + } + resolved, err := r.resolve(ctx, *localID) + if err != nil { + return nil, err + } + if resolved == *localID { + return localID, nil + } + return &resolved, nil +} + +// lookup resolves an id absent from the in-run identity map by consulting +// committed PG state, sharing resolveOwnedPushIdentityID with identity +// resolution: it reuses a row owned under the current or any legacy machine +// prefix (so a target pushed before a rename still resolves), returns the +// current prefix when the bare id is held by another owner, and otherwise keeps +// the bare id. +func (r relationshipResolver) lookup( + ctx context.Context, localID string, +) (string, error) { + identity := r.identity + if r.identity.OwnerMarker != "" { + localID = prefixedSessionID(r.identity.Machine, localID) + identity.AliasIDs = artifactCanonicalAliasIDs( + r.identity.Machine, localID, + ) + } + return r.sync.resolveOwnedPushIdentityID( + ctx, localID, identity, r.markerID, r.legacyMarkerMachines, + ) +} + +// rewriteSubagentIDs resolves the subagent session link on each tool call and +// result event in msgs in place so they reference the PG ids of the subagent +// sessions rather than their unprefixed local ids. +func (r relationshipResolver) rewriteSubagentIDs( + ctx context.Context, msgs []db.Message, +) error { + for i := range msgs { + for j := range msgs[i].ToolCalls { + tc := &msgs[i].ToolCalls[j] + resolved, err := r.resolve(ctx, tc.SubagentSessionID) + if err != nil { + return err + } + tc.SubagentSessionID = resolved + for k := range tc.ResultEvents { + ev := &tc.ResultEvents[k] + resolvedEv, err := r.resolve(ctx, ev.SubagentSessionID) + if err != nil { + return err + } + ev.SubagentSessionID = resolvedEv + } + } + } + return nil +} + +// sessionNeedsSubagentRewrite reports whether any subagent link in the local +// session resolves to a different PG id than its stored local value. The push +// fast path compares local and PG fingerprints built from the unprefixed local +// ids, so a row already in PG with a stale unprefixed subagent id would match +// and skip the rewrite; this check forces message replacement in that case. +func (r relationshipResolver) sessionNeedsSubagentRewrite( + ctx context.Context, localSessionID string, +) (bool, error) { + ids, err := r.sync.local.SessionSubagentSessionIDs(localSessionID) + if err != nil { + return false, fmt.Errorf( + "reading subagent session ids for %s: %w", localSessionID, err, + ) + } + for _, id := range ids { + resolved, err := r.resolve(ctx, id) + if err != nil { + return false, err + } + if resolved != id { + return true, nil + } } - return b.String() + return false, nil } -// pushedSessionMachine resolves the machine field for a PG row. Old rows -// pushed before this fix with machine="local" will be repaired gradually as -// each session is modified (message count change, etc.) and re-fingerprinted. -func pushedSessionMachine(sess db.Session, fallbackMachine string) string { - if sess.Machine != "" && sess.Machine != "local" { - return sess.Machine +// resolvedSubagentLinkFingerprint fingerprints the PG ids the session's +// subagent links resolve to. The dependency fingerprint is built from the +// local rows, which keep their unprefixed ids, so a collision alias that +// appears after the session was last pushed would leave the stored fingerprint +// unchanged and the fast path would skip the session with its PG tool-call +// links stale; folding the resolved ids in forces that re-push. Sessions +// without subagent links contribute an empty string so their fingerprints are +// unaffected. +func (r relationshipResolver) resolvedSubagentLinkFingerprint( + ctx context.Context, localSessionID string, +) (string, error) { + ids, err := r.sync.local.SessionSubagentSessionIDs(localSessionID) + if err != nil { + return "", fmt.Errorf( + "reading subagent session ids for %s: %w", localSessionID, err, + ) } - return fallbackMachine + var b strings.Builder + for _, id := range ids { + resolved, err := r.resolve(ctx, id) + if err != nil { + return "", err + } + b.WriteString("\x1f") + b.WriteString(resolved) + } + return b.String(), nil } func sameSessionOwner( @@ -1563,6 +2314,29 @@ func sameSessionOwner( return existingMachine == pushedMachine } +func samePushedSessionOwner( + existingOwnerMarker, existingMachine string, + identity pushedSessionIdentity, + markerID string, + legacyMarkerMachines []string, +) bool { + if identity.OwnerMarker == "" { + return sameSessionOwner( + existingOwnerMarker, existingMachine, markerID, + identity.Machine, legacyMarkerMachines, + ) + } + if existingOwnerMarker != "" { + return existingOwnerMarker == identity.OwnerMarker || + slices.Contains(identity.LegacyOwnerMarkers, existingOwnerMarker) + } + // Ownerless artifact rows are legacy-compatible only when PG's marker + // history proves this pusher previously wrote under that machine name. + // Matching the artifact origin alone would let an importer adopt a row it + // did not create. + return slices.Contains(legacyMarkerMachines, existingMachine) +} + func stringValue(value *string) string { if value == nil { return "" @@ -1617,34 +2391,48 @@ func nilStrTS(s *string) any { // local-only and used solely by the sync engine to detect // re-parsed sessions. func (s *Sync) pushSession( - ctx context.Context, tx *sql.Tx, sess db.Session, markerID string, + ctx context.Context, tx *sql.Tx, sess db.Session, + identity pushedSessionIdentity, markerID string, legacyMarkerMachines []string, ) error { + if identity.LegacyDuplicateID != "" { + if err := verifyArtifactLegacyDuplicateOwnership( + ctx, tx, identity, markerID, + ); err != nil { + return err + } + } createdAt, _ := ParseSQLiteTimestamp(sess.CreatedAt) isAutomated := sess.IsAutomated - pushedMachine := pushedSessionMachine(sess, s.machine) + ownerMarker := identity.effectiveOwnerMarker(markerID) var existingMachine sql.NullString var existingOwnerMarker sql.NullString checkErr := tx.QueryRowContext(ctx, - `SELECT machine, owner_marker FROM sessions WHERE id = $1`, sess.ID, + `SELECT machine, owner_marker FROM sessions WHERE id = $1`, + identity.ID, ).Scan(&existingMachine, &existingOwnerMarker) if checkErr != nil && !errors.Is(checkErr, sql.ErrNoRows) { - return fmt.Errorf("checking session ownership %s: %w", sess.ID, checkErr) + return fmt.Errorf( + "checking session ownership %s: %w", identity.ID, checkErr, + ) } - if checkErr == nil && !sameSessionOwner( + if checkErr == nil && !samePushedSessionOwner( existingOwnerMarker.String, existingMachine.String, + identity, markerID, - pushedMachine, legacyMarkerMachines, ) { log.Printf( "pgsync: session %s: skipping — already owned by machine %q, "+ "this pusher is %q; sync from the origin machine to update", - sess.ID, existingMachine.String, pushedMachine, + identity.ID, existingMachine.String, identity.Machine, ) return errSessionOwnershipConflict } + if checkErr == nil && identity.ArtifactReplica { + return errArtifactReplicaExists + } if legacyMarkerMachines == nil { legacyMarkerMachines = []string{} } @@ -1652,6 +2440,10 @@ func (s *Sync) pushSession( if err != nil { return fmt.Errorf("encoding legacy marker machines: %w", err) } + legacyOwnerMarkersJSON, err := json.Marshal(identity.LegacyOwnerMarkers) + if err != nil { + return fmt.Errorf("encoding legacy owner markers: %w", err) + } result, err := tx.ExecContext(ctx, ` INSERT INTO sessions ( id, machine, owner_marker, project, agent, @@ -1779,7 +2571,10 @@ func (s *Sync) pushSession( SELECT jsonb_array_elements_text($59::jsonb) )) ) - OR sessions.owner_marker = EXCLUDED.owner_marker) + OR sessions.owner_marker = EXCLUDED.owner_marker + OR sessions.owner_marker IN ( + SELECT jsonb_array_elements_text($60::jsonb) + )) AND NOT EXISTS ( SELECT 1 FROM excluded_sessions WHERE id = EXCLUDED.id @@ -1840,7 +2635,7 @@ func (s *Sync) pushSession( OR sessions.duplicate_prompt_count IS DISTINCT FROM EXCLUDED.duplicate_prompt_count OR sessions.no_code_context_count IS DISTINCT FROM EXCLUDED.no_code_context_count OR sessions.runaway_tool_loop_count IS DISTINCT FROM EXCLUDED.runaway_tool_loop_count)`, - sess.ID, pushedMachine, markerID, + identity.ID, identity.Machine, ownerMarker, sanitizePG(sess.Project), sess.Agent, nilStr(sess.FirstMessage), @@ -1880,12 +2675,13 @@ func (s *Sync) pushSession( sess.NoCodeContextCount, sess.RunawayToolLoopCount, sanitizePG(sess.TranscriptFidelity), string(legacyMarkerMachinesJSON), + string(legacyOwnerMarkersJSON), ) if err != nil { return err } if rowsAffected, rowsErr := result.RowsAffected(); rowsErr == nil && rowsAffected == 0 { - excluded, excludedErr := deletePGSessionIfExcluded(ctx, tx, sess) + excluded, excludedErr := deletePGSessionIfExcluded(ctx, tx, sess, identity.ID) if excludedErr != nil { return excludedErr } @@ -1893,7 +2689,8 @@ func (s *Sync) pushSession( return errSessionExcluded } refreshErr := tx.QueryRowContext(ctx, - `SELECT machine, owner_marker FROM sessions WHERE id = $1`, sess.ID, + `SELECT machine, owner_marker FROM sessions WHERE id = $1`, + identity.ID, ).Scan(&existingMachine, &existingOwnerMarker) if refreshErr != nil { // The guarded upsert changed no rows and we cannot @@ -1907,30 +2704,234 @@ func (s *Sync) pushSession( sess.ID, refreshErr, ) } - if !sameSessionOwner( + if !samePushedSessionOwner( existingOwnerMarker.String, existingMachine.String, - markerID, pushedMachine, legacyMarkerMachines, + identity, markerID, legacyMarkerMachines, ) { log.Printf( "pgsync: session %s: skipping — already owned by machine %q, this pusher is %q; sync from the origin machine to update", - sess.ID, existingMachine.String, pushedMachine, + identity.ID, existingMachine.String, identity.Machine, ) return errSessionOwnershipConflict } } - excluded, excludedErr := deletePGSessionIfExcluded(ctx, tx, sess) + excluded, excludedErr := deletePGSessionIfExcluded(ctx, tx, sess, identity.ID) if excludedErr != nil { return excludedErr } if excluded { return errSessionExcluded } - if err := replacePGSessionAliases(ctx, tx, sess); err != nil { + if identity.LegacyDuplicateID != "" { + if err := consolidateArtifactLegacyDuplicate( + ctx, tx, identity, + ); err != nil { + return err + } + } + aliasSession := sess + aliasSession.ID = identity.ID + if err := replacePGSessionAliases(ctx, tx, aliasSession); err != nil { return err } return nil } +type pgSessionOwnership struct { + machine string + ownerMarker string +} + +// verifyArtifactLegacyDuplicateOwnership locks both sides of an artifact +// upgrade merge before pushSession changes either row. The canonical row must +// have the stable artifact owner and machine, while the bare row must still be +// owned by this pusher's exact pre-artifact random marker. +func verifyArtifactLegacyDuplicateOwnership( + ctx context.Context, + tx *sql.Tx, + identity pushedSessionIdentity, + markerID string, +) error { + if identity.ID == "" || identity.LegacyDuplicateID == "" || + identity.ID == identity.LegacyDuplicateID || + identity.OwnerMarker == "" || markerID == "" { + return fmt.Errorf( + "%w: invalid artifact duplicate consolidation identity", + errSessionOwnershipConflict, + ) + } + + rows, err := tx.QueryContext(ctx, ` + SELECT id, machine, owner_marker + FROM sessions + WHERE id IN ($1, $2) + ORDER BY id + FOR UPDATE + `, identity.ID, identity.LegacyDuplicateID) + if err != nil { + return fmt.Errorf( + "locking artifact duplicate sessions: %w", err, + ) + } + owners := make(map[string]pgSessionOwnership, 2) + for rows.Next() { + var id string + var owner pgSessionOwnership + if err := rows.Scan(&id, &owner.machine, &owner.ownerMarker); err != nil { + _ = rows.Close() + return fmt.Errorf( + "reading artifact duplicate ownership: %w", err, + ) + } + owners[id] = owner + } + if err := rows.Close(); err != nil { + return fmt.Errorf( + "closing artifact duplicate ownership rows: %w", err, + ) + } + if err := rows.Err(); err != nil { + return fmt.Errorf( + "reading artifact duplicate ownership rows: %w", err, + ) + } + + canonical, canonicalOK := owners[identity.ID] + legacy, legacyOK := owners[identity.LegacyDuplicateID] + if !canonicalOK || !legacyOK || + canonical.machine != identity.Machine || + canonical.ownerMarker != identity.OwnerMarker || + legacy.ownerMarker != markerID { + return fmt.Errorf( + "%w: artifact duplicate ownership changed before consolidation", + errSessionOwnershipConflict, + ) + } + return nil +} + +// consolidateArtifactLegacyDuplicate transfers PG-local state to the stable +// canonical artifact row, rewrites references that would otherwise dangle, +// and removes the proven legacy duplicate. The caller holds row locks from +// verifyArtifactLegacyDuplicateOwnership for the duration of this transaction. +func consolidateArtifactLegacyDuplicate( + ctx context.Context, + tx *sql.Tx, + identity pushedSessionIdentity, +) error { + result, err := tx.ExecContext(ctx, ` + UPDATE sessions AS canonical + SET display_name = CASE + WHEN legacy.display_name IS DISTINCT FROM + legacy.source_display_name + THEN legacy.display_name + ELSE canonical.display_name + END, + source_display_name = CASE + WHEN legacy.display_name IS DISTINCT FROM + legacy.source_display_name + THEN legacy.source_display_name + ELSE canonical.source_display_name + END, + deleted_at = CASE + WHEN legacy.deleted_at IS DISTINCT FROM + legacy.source_deleted_at + THEN legacy.deleted_at + ELSE canonical.deleted_at + END, + source_deleted_at = CASE + WHEN legacy.deleted_at IS DISTINCT FROM + legacy.source_deleted_at + THEN legacy.source_deleted_at + ELSE canonical.source_deleted_at + END + FROM sessions AS legacy + WHERE canonical.id = $1 AND legacy.id = $2 + `, identity.ID, identity.LegacyDuplicateID) + if err != nil { + return fmt.Errorf("copying artifact duplicate session curation: %w", err) + } + if rowsAffected, rowsErr := result.RowsAffected(); rowsErr != nil { + return fmt.Errorf("counting artifact duplicate curation rows: %w", rowsErr) + } else if rowsAffected != 1 { + return fmt.Errorf( + "copying artifact duplicate session curation affected %d rows", + rowsAffected, + ) + } + + if _, err := tx.ExecContext(ctx, ` + INSERT INTO starred_sessions (session_id, created_at) + SELECT $1, created_at + FROM starred_sessions + WHERE session_id = $2 + ON CONFLICT (session_id) DO UPDATE SET + created_at = EXCLUDED.created_at + `, identity.ID, identity.LegacyDuplicateID); err != nil { + return fmt.Errorf("copying artifact duplicate star: %w", err) + } + if _, err := tx.ExecContext(ctx, ` + INSERT INTO pinned_messages ( + session_id, message_id, ordinal, source_uuid, note, created_at + ) + SELECT $1, message_id, ordinal, source_uuid, note, created_at + FROM pinned_messages + WHERE session_id = $2 + ON CONFLICT (session_id, message_id) DO UPDATE SET + ordinal = EXCLUDED.ordinal, + source_uuid = EXCLUDED.source_uuid, + note = EXCLUDED.note, + created_at = EXCLUDED.created_at + `, identity.ID, identity.LegacyDuplicateID); err != nil { + return fmt.Errorf("copying artifact duplicate pins: %w", err) + } + + if _, err := tx.ExecContext(ctx, ` + UPDATE sessions + SET parent_session_id = $1 + WHERE parent_session_id = $2 + `, identity.ID, identity.LegacyDuplicateID); err != nil { + return fmt.Errorf("rewriting artifact duplicate parent links: %w", err) + } + if _, err := tx.ExecContext(ctx, ` + UPDATE sessions + SET source_session_id = $1 + WHERE source_session_id = $2 + `, identity.ID, identity.LegacyDuplicateID); err != nil { + return fmt.Errorf("rewriting artifact duplicate source links: %w", err) + } + if _, err := tx.ExecContext(ctx, ` + UPDATE tool_calls + SET subagent_session_id = $1 + WHERE subagent_session_id = $2 + `, identity.ID, identity.LegacyDuplicateID); err != nil { + return fmt.Errorf("rewriting artifact duplicate tool-call links: %w", err) + } + if _, err := tx.ExecContext(ctx, ` + UPDATE tool_result_events + SET subagent_session_id = $1 + WHERE subagent_session_id = $2 + `, identity.ID, identity.LegacyDuplicateID); err != nil { + return fmt.Errorf("rewriting artifact duplicate tool-result links: %w", err) + } + + result, err = tx.ExecContext(ctx, ` + DELETE FROM sessions WHERE id = $1 + `, identity.LegacyDuplicateID) + if err != nil { + return fmt.Errorf("deleting artifact legacy duplicate: %w", err) + } + if rowsAffected, rowsErr := result.RowsAffected(); rowsErr != nil { + return fmt.Errorf("counting deleted artifact duplicate rows: %w", rowsErr) + } else if rowsAffected != 1 { + return fmt.Errorf( + "deleting artifact legacy duplicate affected %d rows", + rowsAffected, + ) + } + return nil +} + // pushMessages replaces a session's messages and tool calls // in PG. It skips the replacement when the PG message count // already matches the local count, avoiding redundant work @@ -1938,12 +2939,14 @@ func (s *Sync) pushSession( func (s *Sync) pushMessages( ctx context.Context, tx *sql.Tx, - sessionID string, + localSessionID string, + pgSessionID string, + resolver relationshipResolver, full bool, sessionUsageFingerprints map[string]string, comparisons *pushMessageComparison, ) (int, error) { - localCount, err := s.local.MessageCount(sessionID) + localCount, err := s.local.MessageCount(localSessionID) if err != nil { return 0, fmt.Errorf( "counting local messages: %w", err, @@ -1952,7 +2955,7 @@ func (s *Sync) pushMessages( if localCount == 0 { if _, err := tx.ExecContext(ctx, `DELETE FROM tool_result_events WHERE session_id = $1`, - sessionID, + pgSessionID, ); err != nil { return 0, fmt.Errorf( "deleting stale pg tool_result_events: %w", err, @@ -1960,7 +2963,7 @@ func (s *Sync) pushMessages( } if _, err := tx.ExecContext(ctx, `DELETE FROM tool_calls WHERE session_id = $1`, - sessionID, + pgSessionID, ); err != nil { return 0, fmt.Errorf( "deleting stale pg tool_calls: %w", err, @@ -1968,7 +2971,7 @@ func (s *Sync) pushMessages( } if _, err := tx.ExecContext(ctx, `DELETE FROM messages WHERE session_id = $1`, - sessionID, + pgSessionID, ); err != nil { return 0, fmt.Errorf( "deleting stale pg messages: %w", err, @@ -1979,11 +2982,13 @@ func (s *Sync) pushMessages( // state.db-only session) with zero messages. Sync them here // too so their cost reaches PG instead of being dropped with // the rest of the message-replace path below. - if err := s.replaceUsageEvents(ctx, tx, sessionID); err != nil { + if err := s.replaceUsageEvents( + ctx, tx, localSessionID, pgSessionID, + ); err != nil { return 0, err } if err := reconcilePinnedMessages( - ctx, tx, sessionID, + ctx, tx, pgSessionID, ); err != nil { return 0, err } @@ -1991,7 +2996,7 @@ func (s *Sync) pushMessages( } pgAgg, pgToolAgg, hasPreloadedComparisons := comparisonAggregates( - sessionID, comparisons, + pgSessionID, comparisons, ) if !hasPreloadedComparisons { if err := tx.QueryRowContext(ctx, @@ -2006,7 +3011,7 @@ func (s *Sync) pushMessages( ) FROM messages WHERE session_id = $1`, - sessionID, + pgSessionID, ).Scan( &pgAgg.Count, &pgAgg.Sum, &pgAgg.Max, &pgAgg.Min, @@ -2021,7 +3026,7 @@ func (s *Sync) pushMessages( COALESCE(SUM(result_content_length), 0) FROM tool_calls WHERE session_id = $1`, - sessionID, + pgSessionID, ).Scan(&pgToolAgg.Count, &pgToolAgg.Sum); err != nil { return 0, fmt.Errorf( "counting pg tool_calls: %w", err, @@ -2030,182 +3035,198 @@ func (s *Sync) pushMessages( } if !full && pgAgg.Count == localCount && pgAgg.Count > 0 { - localFP := pushLocalMessageFingerprint{} - - localFP.Sum, localFP.Max, localFP.Min, err = s.local.MessageContentFingerprint( - sessionID, - ) - if err != nil { - return 0, fmt.Errorf( - "computing local content fingerprint: %w", - err, - ) - } - localFP.ContentHashFP, err = s.local.MessageContentHashFingerprint( - sessionID, - ) - if err != nil { - return 0, fmt.Errorf( - "computing local content hash fingerprint: %w", - err, - ) - } - localFP.RoleTimeFP, err = localMessageRoleTimePGFingerprint( - s.local, sessionID, - ) - if err != nil { - return 0, fmt.Errorf( - "computing local role/time fingerprint: %w", - err, - ) - } - localFP.FlagsFP, err = s.local.MessageFlagsFingerprint(sessionID) - if err != nil { - return 0, fmt.Errorf( - "computing local message flags fingerprint: %w", - err, - ) - } - localFP.SystemFP, err = s.local.SystemMessageFingerprint(sessionID) - if err != nil { - return 0, fmt.Errorf( - "computing local system message fingerprint: %w", err, - ) - } - localFP.ToolCallCount, err = s.local.ToolCallCount(sessionID) - if err != nil { - return 0, fmt.Errorf( - "counting local tool_calls: %w", err, - ) - } - localFP.ToolCallSum, err = s.local.ToolCallContentFingerprint( - sessionID, - ) - if err != nil { - return 0, fmt.Errorf( - "computing local tool_call content fingerprint: %w", - err, - ) - } - localFP.ToolCallFP, err = s.local.ToolCallFingerprint(sessionID) - if err != nil { - return 0, fmt.Errorf( - "computing local tool_call fingerprint: %w", err, - ) - } - localFP.ToolResultFP, err = localToolResultEventPGFingerprint( - s.local, sessionID, + // A row already in PG with a stale unprefixed subagent id matches the + // local fingerprint (both unprefixed), so the skip below would never + // repair it. Force replacement when any subagent link now resolves to a + // different pushed id. + subagentRewrite, err := resolver.sessionNeedsSubagentRewrite( + ctx, localSessionID, ) if err != nil { - return 0, fmt.Errorf( - "computing local tool_result_event fingerprint: %w", err, - ) - } - localFP.TokenFP, err = s.local.MessageTokenFingerprint(sessionID) - if err != nil { - return 0, fmt.Errorf( - "computing local token fingerprint: %w", - err, - ) + return 0, err } + if !subagentRewrite { + localFP := pushLocalMessageFingerprint{} - usageFromMap := false - if sessionUsageFingerprints != nil { - var ok bool - localFP.UsageEventFP, ok = sessionUsageFingerprints[sessionID] - usageFromMap = ok - } - if !usageFromMap { - localFP.UsageEventFP, err = s.local.UsageEventFingerprint(sessionID) + localFP.Sum, localFP.Max, localFP.Min, err = s.local.MessageContentFingerprint( + localSessionID, + ) if err != nil { return 0, fmt.Errorf( - "computing local usage event fingerprint: %w", + "computing local content fingerprint: %w", err, ) } - } - - if comparisons == nil { - pgContentHashFP, err := pgMessageContentHashFingerprint( - ctx, tx, sessionID, + localFP.ContentHashFP, err = s.local.MessageContentHashFingerprint( + localSessionID, ) if err != nil { return 0, fmt.Errorf( - "computing pg content hash fingerprint: %w", + "computing local content hash fingerprint: %w", err, ) } - pgRoleTimeFP, err := pgMessageRoleTimeFingerprint( - ctx, tx, sessionID, + localFP.RoleTimeFP, err = localMessageRoleTimePGFingerprint( + s.local, localSessionID, ) if err != nil { return 0, fmt.Errorf( - "computing pg role/time fingerprint: %w", + "computing local role/time fingerprint: %w", err, ) } - pgFlagsFP, err := pgMessageFlagsFingerprint(ctx, tx, sessionID) + localFP.FlagsFP, err = s.local.MessageFlagsFingerprint(localSessionID) if err != nil { return 0, fmt.Errorf( - "computing pg message flags fingerprint: %w", + "computing local message flags fingerprint: %w", err, ) } - pgTokenFP, err := pgMessageTokenFingerprint(ctx, tx, sessionID) + localFP.SystemFP, err = s.local.SystemMessageFingerprint(localSessionID) if err != nil { return 0, fmt.Errorf( - "computing pg token fingerprint: %w", - err, + "computing local system message fingerprint: %w", err, ) } - pgTCFP, err := pgToolCallFingerprint(ctx, tx, sessionID) + localFP.ToolCallCount, err = s.local.ToolCallCount(localSessionID) if err != nil { return 0, fmt.Errorf( - "computing pg tool_call fingerprint: %w", - err, + "counting local tool_calls: %w", err, ) } - pgResultFP, err := pgToolResultEventFingerprint(ctx, tx, sessionID) + localFP.ToolCallSum, err = s.local.ToolCallContentFingerprint( + localSessionID, + ) if err != nil { return 0, fmt.Errorf( - "computing pg tool_result_event fingerprint: %w", + "computing local tool_call content fingerprint: %w", err, ) } - pgUsageFP, err := pgUsageEventFingerprint(ctx, tx, sessionID) + localFP.ToolCallFP, err = s.local.ToolCallFingerprint(localSessionID) + if err != nil { + return 0, fmt.Errorf( + "computing local tool_call fingerprint: %w", err, + ) + } + localFP.ToolResultFP, err = localToolResultEventPGFingerprint( + s.local, localSessionID, + ) + if err != nil { + return 0, fmt.Errorf( + "computing local tool_result_event fingerprint: %w", err, + ) + } + localFP.TokenFP, err = s.local.MessageTokenFingerprint(localSessionID) if err != nil { return 0, fmt.Errorf( - "computing pg usage event fingerprint: %w", + "computing local token fingerprint: %w", err, ) } - if localFP.Sum == pgAgg.Sum && - localFP.Max == pgAgg.Max && - localFP.Min == pgAgg.Min && - localFP.ContentHashFP == pgContentHashFP && - localFP.RoleTimeFP == pgRoleTimeFP && - localFP.FlagsFP == pgFlagsFP && - localFP.SystemFP == pgAgg.SysFP && - localFP.ToolCallCount == pgToolAgg.Count && - localFP.ToolCallSum == pgToolAgg.Sum && - localFP.ToolCallFP == pgTCFP && - localFP.ToolResultFP == pgResultFP && - localFP.TokenFP == pgTokenFP && - localFP.UsageEventFP == pgUsageFP { + usageFromMap := false + if sessionUsageFingerprints != nil { + var ok bool + localFP.UsageEventFP, ok = sessionUsageFingerprints[localSessionID] + usageFromMap = ok + } + if !usageFromMap { + localFP.UsageEventFP, err = s.local.UsageEventFingerprint(localSessionID) + if err != nil { + return 0, fmt.Errorf( + "computing local usage event fingerprint: %w", + err, + ) + } + } + + if comparisons == nil { + pgContentHashFP, err := pgMessageContentHashFingerprint( + ctx, tx, pgSessionID, + ) + if err != nil { + return 0, fmt.Errorf( + "computing pg content hash fingerprint: %w", + err, + ) + } + pgRoleTimeFP, err := pgMessageRoleTimeFingerprint( + ctx, tx, pgSessionID, + ) + if err != nil { + return 0, fmt.Errorf( + "computing pg role/time fingerprint: %w", + err, + ) + } + pgFlagsFP, err := pgMessageFlagsFingerprint(ctx, tx, pgSessionID) + if err != nil { + return 0, fmt.Errorf( + "computing pg message flags fingerprint: %w", + err, + ) + } + pgTokenFP, err := pgMessageTokenFingerprint(ctx, tx, pgSessionID) + if err != nil { + return 0, fmt.Errorf( + "computing pg token fingerprint: %w", + err, + ) + } + pgTCFP, err := pgToolCallFingerprint(ctx, tx, pgSessionID) + if err != nil { + return 0, fmt.Errorf( + "computing pg tool_call fingerprint: %w", + err, + ) + } + pgResultFP, err := pgToolResultEventFingerprint(ctx, tx, pgSessionID) + if err != nil { + return 0, fmt.Errorf( + "computing pg tool_result_event fingerprint: %w", + err, + ) + } + pgUsageFP, err := pgUsageEventFingerprint(ctx, tx, pgSessionID) + if err != nil { + return 0, fmt.Errorf( + "computing pg usage event fingerprint: %w", + err, + ) + } + + if localFP.Sum == pgAgg.Sum && + localFP.Max == pgAgg.Max && + localFP.Min == pgAgg.Min && + localFP.ContentHashFP == pgContentHashFP && + localFP.RoleTimeFP == pgRoleTimeFP && + localFP.FlagsFP == pgFlagsFP && + localFP.SystemFP == pgAgg.SysFP && + localFP.ToolCallCount == pgToolAgg.Count && + localFP.ToolCallSum == pgToolAgg.Sum && + localFP.ToolCallFP == pgTCFP && + localFP.ToolResultFP == pgResultFP && + localFP.TokenFP == pgTokenFP && + localFP.UsageEventFP == pgUsageFP { + return 0, nil + } + } else if shouldSkipSessionMessages( + pgSessionID, localCount, localFP, full, comparisons, + ) { return 0, nil } - } else if shouldSkipSessionMessages( - sessionID, localCount, localFP, full, comparisons, - ) { - return 0, nil } } + if err := backfillLegacyPinnedMessageSourceUUIDs(ctx, tx, pgSessionID); err != nil { + return 0, err + } + if _, err := tx.ExecContext(ctx, ` DELETE FROM tool_result_events WHERE session_id = $1 - `, sessionID); err != nil { + `, pgSessionID); err != nil { return 0, fmt.Errorf( "deleting pg tool_result_events: %w", err, ) @@ -2213,7 +3234,7 @@ func (s *Sync) pushMessages( if _, err := tx.ExecContext(ctx, ` DELETE FROM tool_calls WHERE session_id = $1 - `, sessionID); err != nil { + `, pgSessionID); err != nil { return 0, fmt.Errorf( "deleting pg tool_calls: %w", err, ) @@ -2221,12 +3242,14 @@ func (s *Sync) pushMessages( if _, err := tx.ExecContext(ctx, ` DELETE FROM messages WHERE session_id = $1 - `, sessionID); err != nil { + `, pgSessionID); err != nil { return 0, fmt.Errorf( "deleting pg messages: %w", err, ) } - if err := s.replaceUsageEvents(ctx, tx, sessionID); err != nil { + if err := s.replaceUsageEvents( + ctx, tx, localSessionID, pgSessionID, + ); err != nil { return 0, err } @@ -2234,7 +3257,7 @@ func (s *Sync) pushMessages( startOrdinal := 0 for { msgs, err := s.local.GetMessages( - ctx, sessionID, startOrdinal, + ctx, localSessionID, startOrdinal, db.MaxMessageLimit, true, ) if err != nil { @@ -2251,23 +3274,29 @@ func (s *Sync) pushMessages( return count, fmt.Errorf( "pushMessages %s: ordinal did not "+ "advance (start=%d, last=%d)", - sessionID, startOrdinal, + localSessionID, startOrdinal, msgs[len(msgs)-1].Ordinal, ) } + if err := resolver.rewriteSubagentIDs(ctx, msgs); err != nil { + return count, fmt.Errorf( + "resolving subagent session ids: %w", err, + ) + } + if err := bulkInsertMessages( - ctx, tx, sessionID, msgs, + ctx, tx, pgSessionID, msgs, ); err != nil { return count, err } if err := bulkInsertToolCalls( - ctx, tx, sessionID, msgs, + ctx, tx, pgSessionID, msgs, ); err != nil { return count, err } if err := bulkInsertToolResultEvents( - ctx, tx, sessionID, msgs, + ctx, tx, pgSessionID, msgs, ); err != nil { return count, err } @@ -2275,7 +3304,7 @@ func (s *Sync) pushMessages( startOrdinal = nextOrdinal } - if err := reconcilePinnedMessages(ctx, tx, sessionID); err != nil { + if err := reconcilePinnedMessages(ctx, tx, pgSessionID); err != nil { return count, err } @@ -2289,19 +3318,22 @@ func (s *Sync) pushMessages( // zero-message and the normal message-replace paths in pushMessages call // this so a session's cost always reaches PG. func (s *Sync) replaceUsageEvents( - ctx context.Context, tx *sql.Tx, sessionID string, + ctx context.Context, tx *sql.Tx, + localSessionID string, pgSessionID string, ) error { if _, err := tx.ExecContext(ctx, ` DELETE FROM usage_events WHERE session_id = $1 - `, sessionID); err != nil { + `, pgSessionID); err != nil { return fmt.Errorf("deleting pg usage_events: %w", err) } - usageEvents, err := s.local.GetUsageEvents(ctx, sessionID) + usageEvents, err := s.local.GetUsageEvents(ctx, localSessionID) if err != nil { return fmt.Errorf("reading local usage events: %w", err) } - if err := bulkInsertUsageEvents(ctx, tx, usageEvents); err != nil { + if err := bulkInsertUsageEvents( + ctx, tx, pgSessionID, usageEvents, + ); err != nil { return err } return nil @@ -2310,20 +3342,8 @@ func (s *Sync) replaceUsageEvents( func reconcilePinnedMessages( ctx context.Context, tx *sql.Tx, sessionID string, ) error { - if _, err := tx.ExecContext(ctx, ` - UPDATE pinned_messages p - SET source_uuid = m.source_uuid - FROM messages m - WHERE p.session_id = $1 - AND m.session_id = p.session_id - AND m.ordinal = p.message_id - AND p.source_uuid = '' - AND m.source_uuid <> ''`, - sessionID, - ); err != nil { - return fmt.Errorf( - "backfilling pg pin source_uuid: %w", err, - ) + if err := backfillLegacyPinnedMessageSourceUUIDs(ctx, tx, sessionID); err != nil { + return err } // Move shifted source-backed pins out of the real ordinal range @@ -2480,6 +3500,27 @@ func reconcilePinnedMessages( return nil } +func backfillLegacyPinnedMessageSourceUUIDs( + ctx context.Context, tx *sql.Tx, sessionID string, +) error { + if _, err := tx.ExecContext(ctx, ` + UPDATE pinned_messages p + SET source_uuid = m.source_uuid + FROM messages m + WHERE p.session_id = $1 + AND m.session_id = p.session_id + AND m.ordinal = p.message_id + AND p.source_uuid = '' + AND m.source_uuid <> ''`, + sessionID, + ); err != nil { + return fmt.Errorf( + "backfilling pg pin source_uuid: %w", err, + ) + } + return nil +} + func pgMessageTokenFingerprint( ctx context.Context, tx *sql.Tx, sessionID string, ) (string, error) { @@ -2848,7 +3889,8 @@ func bulkInsertMessages( } func bulkInsertUsageEvents( - ctx context.Context, tx *sql.Tx, events []db.UsageEvent, + ctx context.Context, tx *sql.Tx, + sessionID string, events []db.UsageEvent, ) error { if len(events) == 0 { return nil @@ -2891,7 +3933,7 @@ func bulkInsertUsageEvents( cost = *ev.CostUSD } args = append(args, - ev.SessionID, + sessionID, ordinal, sanitizePG(ev.Source), sanitizePG(ev.Model), @@ -3127,31 +4169,32 @@ func bulkInsertToolResultEvents( // sessions.secrets_rules_version is pushed by pushSession alongside // the rest of the session columns. func (s *Sync) pushSecretFindings( - ctx context.Context, tx *sql.Tx, sessionID string, + ctx context.Context, tx *sql.Tx, + localSessionID string, pgSessionID string, ) (bool, error) { res, err := tx.ExecContext(ctx, `DELETE FROM secret_findings WHERE session_id = $1`, - sessionID, + pgSessionID, ) if err != nil { return false, fmt.Errorf( "deleting pg secret_findings for %s: %w", - sessionID, err, + pgSessionID, err, ) } deleted, err := res.RowsAffected() if err != nil { return false, fmt.Errorf( "counting deleted secret_findings for %s: %w", - sessionID, err, + pgSessionID, err, ) } - findings, err := s.local.SessionSecretFindings(ctx, sessionID) + findings, err := s.local.SessionSecretFindings(ctx, localSessionID) if err != nil { return false, fmt.Errorf( "reading local secret_findings for %s: %w", - sessionID, err, + localSessionID, err, ) } if len(findings) == 0 { @@ -3184,7 +4227,7 @@ func (s *Sync) pushSecretFindings( p+5, p+6, p+7, p+8, p+9, p+10, p+11, ) args = append(args, - f.SessionID, f.RuleName, f.Confidence, + pgSessionID, f.RuleName, f.Confidence, f.LocationKind, f.MessageOrdinal, f.CallIndex, f.EventIndex, f.MatchStart, f.MatchEnd, f.MatchIndex, @@ -3196,7 +4239,7 @@ func (s *Sync) pushSecretFindings( ); err != nil { return false, fmt.Errorf( "bulk inserting secret_findings for %s: %w", - sessionID, err, + pgSessionID, err, ) } } diff --git a/internal/postgres/push_pgtest_test.go b/internal/postgres/push_pgtest_test.go index 4c03b7f5f..017bf5821 100644 --- a/internal/postgres/push_pgtest_test.go +++ b/internal/postgres/push_pgtest_test.go @@ -373,7 +373,10 @@ func TestPushSessionTerminationStatus(t *testing.T) { t.Helper() tx, err := pg.BeginTx(ctx, nil) require.NoError(t, err, "BeginTx") - if err := sync.pushSession(ctx, tx, s, markerID, nil); err != nil { + if err := sync.pushSession(ctx, tx, s, pushedSessionIdentity{ + ID: s.ID, + Machine: s.Machine, + }, markerID, nil); err != nil { _ = tx.Rollback() t.Fatalf("pushSession: %v", err) } @@ -439,7 +442,12 @@ func TestPushSessionPreservesSourceMachine(t *testing.T) { require.NoError(t, err, "BeginTx") markerID, err := sync.pushMarkerID() require.NoError(t, err, "pushMarkerID") - require.NoError(t, sync.pushSession(ctx, tx, remoteSession, markerID, nil), "pushSession") + require.NoError(t, sync.pushSession( + ctx, tx, remoteSession, pushedSessionIdentity{ + ID: remoteSession.ID, + Machine: remoteSession.Machine, + }, markerID, nil, + ), "pushSession") require.NoError(t, tx.Commit(), "Commit") var got string @@ -788,7 +796,7 @@ func TestPushSessionSkipsPGExcludedSession(t *testing.T) { Machine: "test-machine", Agent: "claude", CreatedAt: "2026-01-01T00:00:00Z", - }, markerID, nil) + }, pushedSessionIdentity{ID: sessionID, Machine: "test-machine"}, markerID, nil) require.ErrorIs(t, err, errSessionExcluded) require.NoError(t, tx.Rollback(), "Rollback") @@ -1549,6 +1557,272 @@ func TestPushIncrementalWithOnlyForeignMachineSessions(t *testing.T) { "second incremental push must not rewrite the session") } +func TestPushPreservesMixedLocalAndImportedMachinesForPGStore(t *testing.T) { + pgURL := testPGURL(t) + + const schema = "agentsview_push_mixed_machine_attribution_test" + pg, err := Open(pgURL, schema, true) + require.NoError(t, err, "Open") + defer pg.Close() + + ctx := context.Background() + _, err = pg.Exec(`DROP SCHEMA IF EXISTS ` + schema + ` CASCADE`) + require.NoError(t, err, "drop schema") + require.NoError(t, EnsureSchema(ctx, pg, schema), "EnsureSchema") + + localDB := testDB(t) + sync := &Sync{ + pg: pg, + local: localDB, + machine: "laptop-a1b2c3", + schema: schema, + schemaDone: true, + } + + localSess := db.Session{ + ID: "local-session-001", + Project: "local-proj", + Machine: "local", + Agent: "claude", + MessageCount: 1, + CreatedAt: "2026-01-01T00:00:00Z", + } + importedSess := db.Session{ + ID: "desktop-d4e5f6~foreign-session-001", + Project: "foreign-proj", + Machine: "desktop-d4e5f6", + Agent: "codex", + MessageCount: 1, + CreatedAt: "2026-01-01T00:01:00Z", + } + for _, sess := range []db.Session{localSess, importedSess} { + require.NoError(t, localDB.UpsertSession(sess), "UpsertSession %s", sess.ID) + require.NoError(t, localDB.InsertMessages([]db.Message{{ + SessionID: sess.ID, + Ordinal: 0, + Role: "assistant", + Content: "hello from " + sess.Machine, + ContentLength: len("hello from " + sess.Machine), + }}), "InsertMessages %s", sess.ID) + } + + res, err := sync.Push(ctx, false, nil) + require.NoError(t, err, "first Push") + assert.Zero(t, res.Errors, "first push should report no failures") + assert.Equal(t, 2, res.SessionsPushed) + + rows, err := pg.Query(`SELECT id, machine FROM sessions`) + require.NoError(t, err, "querying pushed machines") + defer rows.Close() + gotMachines := map[string]string{} + for rows.Next() { + var id, machine string + require.NoError(t, rows.Scan(&id, &machine), "scanning pushed machine") + gotMachines[id] = machine + } + require.NoError(t, rows.Err(), "iterating pushed machines") + assert.Equal(t, "laptop-a1b2c3", gotMachines[localSess.ID]) + assert.Equal(t, "desktop-d4e5f6", gotMachines[importedSess.ID]) + + store, err := NewStore(pgURL, schema, true) + require.NoError(t, err, "NewStore") + defer store.Close() + machines, err := store.GetMachines(ctx, false, false) + require.NoError(t, err, "GetMachines") + assert.ElementsMatch(t, []string{"desktop-d4e5f6", "laptop-a1b2c3"}, machines) + + importedPage, err := store.ListSessions(ctx, db.SessionFilter{ + Machine: "desktop-d4e5f6", + Limit: 10, + }) + require.NoError(t, err, "ListSessions imported machine") + require.Equal(t, 1, importedPage.Total) + require.Len(t, importedPage.Sessions, 1) + assert.Equal(t, importedSess.ID, importedPage.Sessions[0].ID) + assert.Equal(t, "desktop-d4e5f6", importedPage.Sessions[0].Machine) + + localPage, err := store.ListSessions(ctx, db.SessionFilter{ + Machine: "laptop-a1b2c3", + Limit: 10, + }) + require.NoError(t, err, "ListSessions local machine") + require.Equal(t, 1, localPage.Total) + require.Len(t, localPage.Sessions, 1) + assert.Equal(t, localSess.ID, localPage.Sessions[0].ID) + assert.Equal(t, "laptop-a1b2c3", localPage.Sessions[0].Machine) + + res, err = sync.Push(ctx, false, nil) + require.NoError(t, err, "second Push") + assert.Zero(t, res.Errors, "second push should report no failures") + assert.Zero(t, res.SessionsPushed, "unchanged mixed-machine sessions should not be re-pushed") +} + +func TestPushKeepsSameNativeIDDistinctAcrossMachines(t *testing.T) { + pgURL := testPGURL(t) + + const schema = "agentsview_push_native_id_collision_test" + pg, err := Open(pgURL, schema, true) + require.NoError(t, err, "Open") + defer pg.Close() + + ctx := context.Background() + _, err = pg.Exec(`DROP SCHEMA IF EXISTS ` + schema + ` CASCADE`) + require.NoError(t, err, "drop schema") + require.NoError(t, EnsureSchema(ctx, pg, schema), "EnsureSchema") + + const nativeID = "shared-native-session-001" + const machineA = "laptop-a1b2c3" + const machineB = "desktop-d4e5f6" + const projectA = "laptop-proj" + const projectB = "desktop-proj" + pgIDA := nativeID + pgIDB := prefixedSessionID(machineB, nativeID) + + localA := testDB(t) + localB := testDB(t) + syncA := &Sync{ + pg: pg, + local: localA, + machine: machineA, + schema: schema, + schemaDone: true, + } + syncB := &Sync{ + pg: pg, + local: localB, + machine: machineB, + schema: schema, + schemaDone: true, + } + + seed := func(local *db.DB, project, content string) db.Session { + t.Helper() + sess := db.Session{ + ID: nativeID, + Project: project, + Machine: "local", + Agent: "claude", + MessageCount: 1, + CreatedAt: "2026-01-01T00:00:00Z", + } + require.NoError(t, local.UpsertSession(sess), "UpsertSession") + require.NoError(t, local.InsertMessages([]db.Message{{ + SessionID: nativeID, + Ordinal: 0, + Role: "assistant", + Content: content, + ContentLength: len(content), + }}), "InsertMessages") + return sess + } + sessA := seed(localA, projectA, "from laptop") + sessB := seed(localB, projectB, "from desktop") + + res, err := syncA.Push(ctx, false, nil) + require.NoError(t, err, "Push A") + assert.Zero(t, res.Errors, "first A push should report no failures") + assert.Equal(t, 1, res.SessionsPushed) + + res, err = syncB.Push(ctx, false, nil) + require.NoError(t, err, "Push B") + assert.Zero(t, res.Errors, "first B push should report no failures") + assert.Equal(t, 1, res.SessionsPushed) + + rows, err := pg.Query( + `SELECT id, machine, project FROM sessions ORDER BY id`, + ) + require.NoError(t, err, "querying pushed collision rows") + defer rows.Close() + got := map[string]db.Session{} + for rows.Next() { + var row db.Session + require.NoError(t, rows.Scan( + &row.ID, &row.Machine, &row.Project, + ), "scanning collision row") + got[row.ID] = row + } + require.NoError(t, rows.Err(), "iterating collision rows") + require.Len(t, got, 2) + assert.Equal(t, machineA, got[pgIDA].Machine) + assert.Equal(t, projectA, got[pgIDA].Project) + assert.Equal(t, machineB, got[pgIDB].Machine) + assert.Equal(t, projectB, got[pgIDB].Project) + + var contentA, contentB string + require.NoError(t, pg.QueryRow( + `SELECT content FROM messages WHERE session_id = $1 AND ordinal = 0`, + pgIDA, + ).Scan(&contentA), "reading A message") + require.NoError(t, pg.QueryRow( + `SELECT content FROM messages WHERE session_id = $1 AND ordinal = 0`, + pgIDB, + ).Scan(&contentB), "reading B message") + assert.Equal(t, "from laptop", contentA) + assert.Equal(t, "from desktop", contentB) + + store, err := NewStore(pgURL, schema, true) + require.NoError(t, err, "NewStore") + defer store.Close() + pageA, err := store.ListSessions(ctx, db.SessionFilter{ + Machine: machineA, + Limit: 10, + }) + require.NoError(t, err, "ListSessions A") + require.Equal(t, 1, pageA.Total) + require.Len(t, pageA.Sessions, 1) + assert.Equal(t, pgIDA, pageA.Sessions[0].ID) + assert.Equal(t, projectA, pageA.Sessions[0].Project) + pageB, err := store.ListSessions(ctx, db.SessionFilter{ + Machine: machineB, + Limit: 10, + }) + require.NoError(t, err, "ListSessions B") + require.Equal(t, 1, pageB.Total) + require.Len(t, pageB.Sessions, 1) + assert.Equal(t, pgIDB, pageB.Sessions[0].ID) + assert.Equal(t, projectB, pageB.Sessions[0].Project) + + res, err = syncA.Push(ctx, false, nil) + require.NoError(t, err, "second Push A") + assert.Zero(t, res.Errors, "second A push should report no failures") + assert.Zero(t, res.SessionsPushed) + res, err = syncB.Push(ctx, false, nil) + require.NoError(t, err, "second Push B") + assert.Zero(t, res.Errors, "second B push should report no failures") + assert.Zero(t, res.SessionsPushed) + + var rowCount int + require.NoError(t, pg.QueryRow( + `SELECT COUNT(*) FROM sessions WHERE id IN ($1, $2)`, + pgIDA, pgIDB, + ).Scan(&rowCount), "counting collision rows") + assert.Equal(t, 2, rowCount) + + updatedB := sessB + updatedB.Project = "desktop-proj-updated" + require.NoError(t, localB.UpsertSession(updatedB), "update B session") + // UpsertSession does not write local_modified_at, so mark the session + // modified explicitly; otherwise the incremental push will not re-list it. + require.NoError(t, localB.BumpLocalModifiedAt(nativeID), + "mark updated B session modified") + res, err = syncB.Push(ctx, false, nil) + require.NoError(t, err, "updated Push B") + assert.Zero(t, res.Errors, "updated B push should report no failures") + assert.Equal(t, 1, res.SessionsPushed) + + var projectAfterA, projectAfterB string + require.NoError(t, pg.QueryRow( + `SELECT project FROM sessions WHERE id = $1`, + pgIDA, + ).Scan(&projectAfterA), "reading A project after B update") + require.NoError(t, pg.QueryRow( + `SELECT project FROM sessions WHERE id = $1`, + pgIDB, + ).Scan(&projectAfterB), "reading B project after update") + assert.Equal(t, sessA.Project, projectAfterA) + assert.Equal(t, updatedB.Project, projectAfterB) +} + // TestPushDetectsResetWhenCompetingMachineRowsExist verifies that a PG reset is // detected even when another pusher has repopulated rows under a machine value // this host also writes. The local session carries Machine "remote-host" (as a @@ -2058,11 +2332,18 @@ func TestPushReportsSkippedConflicts(t *testing.T) { schemaDone: true, } - const sessID = "conflict-001" + // A same-id collision against a different live machine is resolved by + // storing the session under a machine-prefixed id, not skipped. A skipped + // conflict now arises only when the id cannot be re-prefixed: an imported + // foreign-origin session already carries its origin's prefix, so when PG + // holds that id under a different owner marker -- e.g. a third machine + // imported and pushed the same session first -- this push must leave the + // row to its owner and report it as a skipped conflict. + const sessID = "machine-a~conflict-001" require.NoError(t, localDB.UpsertSession(db.Session{ ID: sessID, Project: "proj", - Machine: "machine-b", + Machine: "machine-a", Agent: "claude", MessageCount: 1, CreatedAt: "2026-01-01T00:00:00Z", @@ -2087,4 +2368,18 @@ func TestPushReportsSkippedConflicts(t *testing.T) { assert.Zero(t, res.Errors, "push should not report failed sessions") assert.Zero(t, res.SessionsPushed, "conflicting session should not be counted as pushed") assert.Equal(t, 1, res.SkippedConflicts, "skipped conflicts should be observable in PushResult") + + // The conflicting row is left untouched and no doubly-prefixed row is + // created in this pusher's namespace. + var ownerMarker string + require.NoError(t, pg.QueryRow( + `SELECT owner_marker FROM sessions WHERE id = $1`, sessID, + ).Scan(&ownerMarker), "reading conflicting row owner") + assert.Equal(t, "other-owner", ownerMarker) + var doublyPrefixed int + require.NoError(t, pg.QueryRow( + `SELECT COUNT(*) FROM sessions WHERE id = $1`, + prefixedSessionID("machine-b", sessID), + ).Scan(&doublyPrefixed), "counting doubly-prefixed rows") + assert.Zero(t, doublyPrefixed) } diff --git a/internal/postgres/push_test.go b/internal/postgres/push_test.go index 153552d3f..f88d12836 100644 --- a/internal/postgres/push_test.go +++ b/internal/postgres/push_test.go @@ -6,6 +6,7 @@ import ( "database/sql/driver" "encoding/json" "errors" + "fmt" "io" "path/filepath" "strings" @@ -301,6 +302,23 @@ func TestSessionAliasBackfillForcesOneFullPush(t *testing.T) { assert.False(t, needed) } +func TestArtifactIdentityModePersistsOnlyAfterSuccessfulPush(t *testing.T) { + store := &syncStateStoreStub{values: map[string]string{ + artifactIdentityModeStateKey: legacyArtifactIdentityMode, + }} + mode := artifactOwnerMarkerPrefix + "origin-a1b2c3" + + require.NoError(t, completeArtifactIdentityMode( + store, mode, PushResult{Errors: 1}, + )) + assert.Equal(t, legacyArtifactIdentityMode, + store.values[artifactIdentityModeStateKey], + "a partial push must leave the prior mode so the next run retries the transition") + + require.NoError(t, completeArtifactIdentityMode(store, mode, PushResult{})) + assert.Equal(t, mode, store.values[artifactIdentityModeStateKey]) +} + func TestCompleteSessionAliasBackfillMarksDoneUnlessErrors(t *testing.T) { for _, tc := range []struct { name string @@ -503,6 +521,125 @@ func TestSessionAliasBackfillKeysStayFilteredForPushState(t *testing.T) { assert.Empty(t, store.values["last_push_at:work"]) } +// scriptedSink is a sessionBatchSink that records calls and writes every +// session except those named in fail. It mirrors pushBatch semantics: a +// multi-session batch containing a failing session rolls back as a unit +// (ok=false) so the driver retries each session individually, and a failing +// single-session batch returns ok=false. A session named in fatal makes the +// batch containing it return a fatal error. +type scriptedSink struct { + fail map[string]bool + fatal string + calls [][]string +} + +func (s *scriptedSink) writeBatch( + _ context.Context, batch []db.Session, pushed *[]db.Session, +) (batchResult, error) { + ids := make([]string, len(batch)) + for i, sess := range batch { + ids[i] = sess.ID + } + s.calls = append(s.calls, ids) + for _, sess := range batch { + if sess.ID == s.fatal { + return batchResult{}, errors.New("fatal sink error") + } + if s.fail[sess.ID] { + // Whole batch rolls back without writing. + return batchResult{ok: false}, nil + } + } + msgs := 0 + for _, sess := range batch { + *pushed = append(*pushed, sess) + msgs += 2 + } + return batchResult{ok: true, sessions: len(batch), messages: msgs}, nil +} + +func sessionsWithIDs(ids ...string) []db.Session { + out := make([]db.Session, len(ids)) + for i, id := range ids { + out[i] = db.Session{ID: id} + } + return out +} + +func TestDrainSessionBatchesChunksAndReportsProgress(t *testing.T) { + var ids []string + for i := range 120 { + ids = append(ids, fmt.Sprintf("s%03d", i)) + } + sessions := sessionsWithIDs(ids...) + sink := &scriptedSink{} + + var result PushResult + var progress []PushProgress + pushed, err := drainSessionBatches( + context.Background(), sessions, sink, &result, + func(p PushProgress) { progress = append(progress, p) }, + ) + require.NoError(t, err) + + // Batched in chunks of 50: 50 + 50 + 20. + require.Len(t, sink.calls, 3) + assert.Len(t, sink.calls[0], 50) + assert.Len(t, sink.calls[1], 50) + assert.Len(t, sink.calls[2], 20) + + assert.Equal(t, 120, result.SessionsPushed) + assert.Equal(t, 240, result.MessagesPushed) + assert.Equal(t, 0, result.Errors) + assert.Len(t, pushed, 120) + + require.Len(t, progress, 3) + assert.Equal(t, PushProgress{SessionsDone: 50, SessionsTotal: 120, MessagesDone: 100}, progress[0]) + assert.Equal(t, PushProgress{SessionsDone: 100, SessionsTotal: 120, MessagesDone: 200}, progress[1]) + assert.Equal(t, PushProgress{SessionsDone: 120, SessionsTotal: 120, MessagesDone: 240}, progress[2]) +} + +func TestDrainSessionBatchesRetriesFailedBatchIndividually(t *testing.T) { + sessions := sessionsWithIDs("a", "b", "c") + sink := &scriptedSink{fail: map[string]bool{"b": true}} + + var result PushResult + pushed, err := drainSessionBatches( + context.Background(), sessions, sink, &result, nil, + ) + require.NoError(t, err) + + // First the whole batch (rolls back), then each session individually. + require.Len(t, sink.calls, 4) + assert.Equal(t, []string{"a", "b", "c"}, sink.calls[0]) + assert.Equal(t, []string{"a"}, sink.calls[1]) + assert.Equal(t, []string{"b"}, sink.calls[2]) + assert.Equal(t, []string{"c"}, sink.calls[3]) + + assert.Equal(t, 2, result.SessionsPushed) + assert.Equal(t, 4, result.MessagesPushed) + assert.Equal(t, 1, result.Errors) + pushedIDs := make([]string, len(pushed)) + for i, sess := range pushed { + pushedIDs[i] = sess.ID + } + assert.Equal(t, []string{"a", "c"}, pushedIDs) +} + +func TestDrainSessionBatchesFatalErrorAborts(t *testing.T) { + sessions := sessionsWithIDs("a", "b", "c") + sink := &scriptedSink{fatal: "b"} + + var result PushResult + pushed, err := drainSessionBatches( + context.Background(), sessions, sink, &result, nil, + ) + require.Error(t, err) + assert.Equal(t, 0, result.SessionsPushed) + assert.Equal(t, 0, result.Errors) + assert.Empty(t, pushed) +} + func TestReadPushBoundaryStateValidity(t *testing.T) { const cutoff = "2026-03-11T12:34:56.123Z" @@ -586,6 +723,97 @@ func TestPGExcludedSessionIDsQueryUsesSingleArrayParameter(t *testing.T) { ) } +func TestArtifactPushIdentityCanonicalizesNativeAndImportedCopies(t *testing.T) { + tests := []struct { + name string + session db.Session + localOrigin string + imported bool + wantID string + wantMachine string + wantOwner string + wantOK bool + }{ + { + name: "locally owned artifact session", + session: db.Session{ID: "native-id", Machine: "local"}, + localOrigin: "origin-a1b2c3", + wantID: "origin-a1b2c3~native-id", + wantMachine: "origin-a1b2c3", + wantOwner: "artifact-origin:origin-a1b2c3", + wantOK: true, + }, + { + name: "imported artifact session", + session: db.Session{ID: "origin-a1b2c3~native-id", Machine: "origin-a1b2c3"}, + localOrigin: "origin-b4c5d6", + imported: true, + wantID: "origin-a1b2c3~native-id", + wantMachine: "origin-a1b2c3", + wantOwner: "artifact-origin:origin-a1b2c3", + wantOK: true, + }, + { + name: "ssh-shaped session without artifact provenance stays legacy", + session: db.Session{ID: "origin-a1b2c3~native-id", Machine: "origin-a1b2c3"}, + localOrigin: "origin-b4c5d6", + wantOK: false, + }, + { + name: "no artifact origin preserves legacy collision behavior", + session: db.Session{ID: "native-id", Machine: "local"}, + wantOK: false, + }, + { + name: "prefixed foreign session without local artifact opt in stays legacy", + session: db.Session{ID: "remote-host~native-id", Machine: "remote-host"}, + wantOK: false, + }, + { + name: "unrelated foreign machine id is not treated as imported", + session: db.Session{ID: "native-id", Machine: "remote-host"}, + localOrigin: "origin-b4c5d6", + wantOK: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + id, machine, owner, ok := artifactPushIdentity( + tt.session, tt.localOrigin, tt.imported, + ) + assert.Equal(t, tt.wantOK, ok) + assert.Equal(t, tt.wantID, id) + assert.Equal(t, tt.wantMachine, machine) + assert.Equal(t, tt.wantOwner, owner) + }) + } +} + +func TestArtifactPushOwnershipAdoptsOnlyCurrentPushersLegacyMarker(t *testing.T) { + identity := pushedSessionIdentity{ + Machine: "origin-a1b2c3", + OwnerMarker: "artifact-origin:origin-a1b2c3", + LegacyOwnerMarkers: []string{"this-pusher-marker"}, + } + + assert.True(t, samePushedSessionOwner( + "artifact-origin:origin-a1b2c3", "origin-a1b2c3", + identity, "this-pusher-marker", nil, + )) + assert.True(t, samePushedSessionOwner( + "this-pusher-marker", "origin-a1b2c3", + identity, "this-pusher-marker", nil, + )) + assert.False(t, samePushedSessionOwner( + "another-pusher-marker", "origin-a1b2c3", + identity, "this-pusher-marker", nil, + )) + assert.False(t, samePushedSessionOwner( + "", "origin-a1b2c3", identity, "this-pusher-marker", nil, + ), "matching an artifact origin must not let an importer seize an ownerless row") +} + func TestDeletePGExcludedSessionRowsUsesSingleArrayParameter(t *testing.T) { execer := &capturePGExec{} @@ -636,6 +864,7 @@ func TestPushSessionRechecksExclusionAfterSuccessfulUpsert(t *testing.T) { Agent: "claude", CreatedAt: "2026-01-01T00:00:00Z", }, + pushedSessionIdentity{ID: "sess-race", Machine: "push-machine"}, "marker", nil, ) @@ -669,6 +898,7 @@ func TestPushSessionStoresVibeFallbackAlias(t *testing.T) { CreatedAt: "2026-01-01T00:00:00Z", FilePath: &filePath, }, + pushedSessionIdentity{ID: "vibe:canonical-uuid", Machine: "push-machine"}, "marker", nil, ) @@ -680,6 +910,95 @@ func TestPushSessionStoresVibeFallbackAlias(t *testing.T) { require.NoError(t, tx.Rollback(), "Rollback") } +func TestPushSessionStoresAliasUnderResolvedPGID(t *testing.T) { + state := &pushSessionProbeState{aliases: map[string]string{}} + pg := newPushSessionProbeDB(t, state) + tx, err := pg.BeginTx(context.Background(), nil) + require.NoError(t, err, "BeginTx") + + sessionDir := filepath.Join( + t.TempDir(), + "session_20260616_083518_alias1", + ) + filePath := filepath.Join(sessionDir, "messages.jsonl") + syncer := &Sync{machine: "push-machine"} + err = syncer.pushSession( + context.Background(), tx, + db.Session{ + ID: "vibe:canonical-uuid", + Project: "proj", + Machine: "push-machine", + Agent: "vibe", + CreatedAt: "2026-01-01T00:00:00Z", + FilePath: &filePath, + }, + pushedSessionIdentity{ + ID: "push-machine~vibe:canonical-uuid", + Machine: "push-machine", + }, + "marker", nil, + ) + + require.NoError(t, err, "pushSession") + assert.Empty(t, state.aliases["vibe:canonical-uuid"]) + assert.Equal(t, + "push-machine~vibe:session_20260616_083518_alias1", + state.aliases["push-machine~vibe:canonical-uuid"], + ) + require.NoError(t, tx.Rollback(), "Rollback") +} + +func TestPushSessionIgnoresBareTombstoneForResolvedPGID(t *testing.T) { + state := &pushSessionProbeState{ + aliases: map[string]string{}, + existingExcluded: map[string]bool{ + "vibe:canonical-deleted": true, + }, + excludedIDs: map[string]bool{}, + } + pg := newPushSessionProbeDB(t, state) + tx, err := pg.BeginTx(context.Background(), nil) + require.NoError(t, err, "BeginTx") + + sessionDir := filepath.Join( + t.TempDir(), + "session_20260616_083518_bare01", + ) + filePath := filepath.Join(sessionDir, "messages.jsonl") + syncer := &Sync{machine: "push-machine"} + err = syncer.pushSession( + context.Background(), tx, + db.Session{ + ID: "vibe:canonical-deleted", + Project: "proj", + Machine: "push-machine", + Agent: "vibe", + CreatedAt: "2026-01-01T00:00:00Z", + FilePath: &filePath, + }, + pushedSessionIdentity{ + ID: "push-machine~vibe:canonical-deleted", + Machine: "push-machine", + }, + "marker", nil, + ) + + require.NoError(t, err, "pushSession") + assert.False(t, + state.excludedIDs["vibe:canonical-deleted"], + "resolved pushes must not adopt another owner's bare tombstone", + ) + assert.False(t, + state.deletedExcluded, + "resolved pushes must not delete another owner's bare row", + ) + assert.Equal(t, + "push-machine~vibe:session_20260616_083518_bare01", + state.aliases["push-machine~vibe:canonical-deleted"], + ) + require.NoError(t, tx.Rollback(), "Rollback") +} + func TestPushSessionExcludesVibeFallbackAliasWhenCanonicalExcluded(t *testing.T) { state := &pushSessionProbeState{ existingExcluded: map[string]bool{ @@ -707,6 +1026,7 @@ func TestPushSessionExcludesVibeFallbackAliasWhenCanonicalExcluded(t *testing.T) CreatedAt: "2026-01-01T00:00:00Z", FilePath: &filePath, }, + pushedSessionIdentity{ID: "vibe:canonical-deleted", Machine: "push-machine"}, "marker", nil, ) @@ -745,6 +1065,7 @@ func TestPushSessionSkipsVibeCanonicalWhenFallbackAliasExcluded(t *testing.T) { CreatedAt: "2026-01-01T00:00:00Z", FilePath: &filePath, }, + pushedSessionIdentity{ID: "vibe:canonical-active", Machine: "push-machine"}, "marker", nil, ) @@ -782,9 +1103,15 @@ func TestPurgePGExcludedPushSessionsChecksDerivedAliases(t *testing.T) { FilePath: &filePath, }, } + identities := map[string]pushedSessionIdentity{ + "vibe:canonical-unchanged": { + ID: "vibe:canonical-unchanged", + Machine: "push-machine", + }, + } err := purgePGExcludedPushSessions( - context.Background(), pg, sessionByID, + context.Background(), pg, sessionByID, identities, ) require.NoError(t, err, "purgePGExcludedPushSessions") @@ -799,6 +1126,51 @@ func TestPurgePGExcludedPushSessionsChecksDerivedAliases(t *testing.T) { assert.Equal(t, 0, state.upserts) } +func TestPurgePGExcludedPushSessionsUsesResolvedPGID(t *testing.T) { + state := &pushSessionProbeState{ + existingExcluded: map[string]bool{ + "vibe:canonical-foreign": true, + }, + excludedIDs: map[string]bool{}, + } + pg := newPushSessionProbeDB(t, state) + + sessionDir := filepath.Join( + t.TempDir(), + "session_20260616_083518_resolved", + ) + filePath := filepath.Join(sessionDir, "messages.jsonl") + sessionByID := map[string]db.Session{ + "vibe:canonical-foreign": { + ID: "vibe:canonical-foreign", + Project: "proj", + Machine: "push-machine", + Agent: "vibe", + CreatedAt: "2026-01-01T00:00:00Z", + FilePath: &filePath, + }, + } + identities := map[string]pushedSessionIdentity{ + "vibe:canonical-foreign": { + ID: "push-machine~vibe:canonical-foreign", + Machine: "push-machine", + }, + } + + err := purgePGExcludedPushSessions( + context.Background(), pg, sessionByID, identities, + ) + + require.NoError(t, err, "purgePGExcludedPushSessions") + assert.Contains(t, sessionByID, "vibe:canonical-foreign") + assert.Empty(t, state.excludedIDs) + assert.False(t, + state.deletedExcluded, + "resolved purge must not delete another owner's bare row", + ) + assert.Equal(t, 1, state.exclusionChecks) +} + type pushSessionProbeDriver struct{} type pushSessionProbeConn struct { @@ -1003,7 +1375,7 @@ func TestSessionPushFingerprintDiffers(t *testing.T) { CreatedAt: "2026-03-11T12:00:00Z", } - fp1 := sessionPushFingerprint(base, base.Machine, "", "", "") + fp1 := sessionPushFingerprint(base, base.ID, base.Machine, "", "", "") tests := []struct { name string @@ -1099,13 +1471,14 @@ func TestSessionPushFingerprintDiffers(t *testing.T) { for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { modified := tc.modify(base) - fp2 := sessionPushFingerprint(modified, modified.Machine, "", "", "") + fp2 := sessionPushFingerprint( + modified, modified.ID, modified.Machine, "", "", "") require.NotEqual(t, fp1, fp2, "fingerprint should differ after %s", tc.name) }) } - assert.Equal(t, fp1, sessionPushFingerprint(base, base.Machine, "", "", ""), + assert.Equal(t, fp1, sessionPushFingerprint(base, base.ID, base.Machine, "", "", ""), "identical sessions should produce identical fingerprints") } @@ -1125,24 +1498,24 @@ func TestSessionPushFingerprintIgnoresVolatileStatFields(t *testing.T) { LocalModifiedAt: &localModifiedAt, CreatedAt: "2026-03-11T12:00:00Z", } - baseFP := sessionPushFingerprint(base, base.Machine, "", "", "deps") + baseFP := sessionPushFingerprint(base, base.ID, base.Machine, "", "", "deps") statOnlyMtime := int64(1700000001000000000) statOnlyModifiedAt := "2026-03-11T12:00:01.000Z" statOnly := base statOnly.FileMtime = &statOnlyMtime statOnly.LocalModifiedAt = &statOnlyModifiedAt - assert.Equal(t, baseFP, sessionPushFingerprint(statOnly, statOnly.Machine, "", "", "deps"), + assert.Equal(t, baseFP, sessionPushFingerprint(statOnly, statOnly.ID, statOnly.Machine, "", "", "deps"), "file stat churn should not change push candidacy") contentChanged := statOnly contentChanged.MessageCount++ assert.NotEqual(t, baseFP, - sessionPushFingerprint(contentChanged, contentChanged.Machine, "", "", "deps"), + sessionPushFingerprint(contentChanged, contentChanged.ID, contentChanged.Machine, "", "", "deps"), "content changes should still change push candidacy") assert.NotEqual(t, baseFP, - sessionPushFingerprint(statOnly, statOnly.Machine, "", "", "changed-deps"), + sessionPushFingerprint(statOnly, statOnly.ID, statOnly.Machine, "", "", "changed-deps"), "dependent row changes should still change push candidacy") } @@ -1181,7 +1554,7 @@ func TestLocalSessionDependencyPushFingerprintTracksMessageEditsWithoutFileHash( CreatedAt: "2026-03-11T12:00:00Z", } fpBefore := sessionPushFingerprint( - session, session.Machine, "", "", depsBefore, + session, session.ID, session.Machine, "", "", depsBefore, ) require.NoError(t, localDB.ReplaceSessionMessages(sessID, []db.Message{{ @@ -1195,7 +1568,7 @@ func TestLocalSessionDependencyPushFingerprintTracksMessageEditsWithoutFileHash( ) require.NoError(t, err) fpAfter := sessionPushFingerprint( - session, session.Machine, "", "", depsAfter, + session, session.ID, session.Machine, "", "", depsAfter, ) assert.NotEqual(t, depsBefore, depsAfter) @@ -1216,12 +1589,29 @@ func TestSessionPushFingerprintIncludesUsageEventFingerprint( CreatedAt: "2026-03-11T12:00:00Z", } - withoutUsage := sessionPushFingerprint(base, base.Machine, "", "", "") - withUsage := sessionPushFingerprint(base, base.Machine, "usage-fp", "", "") + withoutUsage := sessionPushFingerprint(base, base.ID, base.Machine, "", "", "") + withUsage := sessionPushFingerprint(base, base.ID, base.Machine, "usage-fp", "", "") assert.NotEqual(t, withoutUsage, withUsage, "usage event fingerprint should affect session fingerprint") } +func TestSessionPushFingerprintTracksResolvedID(t *testing.T) { + base := db.Session{ + ID: "sess-001", + Project: "proj", + Machine: "laptop", + Agent: "claude", + CreatedAt: "2026-03-11T12:00:00Z", + } + + native := sessionPushFingerprint(base, base.ID, base.Machine, "", "", "") + prefixed := sessionPushFingerprint( + base, prefixedSessionID(base.Machine, base.ID), base.Machine, "", "", "", + ) + assert.NotEqual(t, native, prefixed, + "resolved PG id must affect session fingerprint") +} + func TestSessionPushFingerprintTracksResolvedMachine(t *testing.T) { sentinel := db.Session{ ID: "sess-001", @@ -1231,9 +1621,11 @@ func TestSessionPushFingerprintTracksResolvedMachine(t *testing.T) { CreatedAt: "2026-03-11T12:00:00Z", } fpA := sessionPushFingerprint( - sentinel, pushedSessionMachine(sentinel, "host-a"), "", "", "") + sentinel, sentinel.ID, + pushedSessionMachine(sentinel, "host-a"), "", "", "") fpB := sessionPushFingerprint( - sentinel, pushedSessionMachine(sentinel, "host-b"), "", "", "") + sentinel, sentinel.ID, + pushedSessionMachine(sentinel, "host-b"), "", "", "") assert.NotEqual(t, fpA, fpB, "sentinel session fingerprint must change with the fallback machine") @@ -1245,9 +1637,11 @@ func TestSessionPushFingerprintTracksResolvedMachine(t *testing.T) { CreatedAt: "2026-03-11T12:00:00Z", } fp1 := sessionPushFingerprint( - real, pushedSessionMachine(real, "host-a"), "", "", "") + real, real.ID, + pushedSessionMachine(real, "host-a"), "", "", "") fp2 := sessionPushFingerprint( - real, pushedSessionMachine(real, "host-b"), "", "", "") + real, real.ID, + pushedSessionMachine(real, "host-b"), "", "", "") assert.Equal(t, fp1, fp2, "a session with a real machine ignores the fallback") } @@ -1291,6 +1685,46 @@ func TestPushedSessionMachine(t *testing.T) { } } +func TestPrefixedSessionID(t *testing.T) { + tests := []struct { + name string + machine string + id string + want string + }{ + { + name: "prefixes native id", + machine: "host-a", + id: "sess-001", + want: "host-a~sess-001", + }, + { + name: "keeps already prefixed id", + machine: "host-a", + id: "host-a~sess-001", + want: "host-a~sess-001", + }, + { + name: "keeps empty machine", + machine: "", + id: "sess-001", + want: "sess-001", + }, + { + name: "keeps empty id", + machine: "host-a", + id: "", + want: "", + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + assert.Equal(t, tc.want, prefixedSessionID(tc.machine, tc.id)) + }) + } +} + func TestSessionPushFingerprintNoFieldCollisions( t *testing.T, ) { @@ -1305,8 +1739,8 @@ func TestSessionPushFingerprintNoFieldCollisions( CreatedAt: "2026-03-11T12:00:00Z", } assert.NotEqual(t, - sessionPushFingerprint(s1, s1.Machine, "", "", ""), - sessionPushFingerprint(s2, s2.Machine, "", "", ""), + sessionPushFingerprint(s1, s1.ID, s1.Machine, "", "", ""), + sessionPushFingerprint(s2, s2.ID, s2.Machine, "", "", ""), "length-prefixed fingerprints should not collide") } @@ -1591,7 +2025,16 @@ func TestFinalizePushStateMergesPriorFingerprints( require.NoError(t, finalizePushState( store, cutoff, cycle2Sessions, priorFingerprints, - map[string]string{"sess-002": sessionPushFingerprint(cycle2Sessions[0], cycle2Sessions[0].Machine, "", "", "")}, + map[string]string{ + "sess-002": sessionPushFingerprint( + cycle2Sessions[0], + cycle2Sessions[0].ID, + cycle2Sessions[0].Machine, + "", + "", + "", + ), + }, )) raw := store.values[lastPushBoundaryStateKey] diff --git a/internal/postgres/schema.go b/internal/postgres/schema.go index af09dc0d4..a30cfc428 100644 --- a/internal/postgres/schema.go +++ b/internal/postgres/schema.go @@ -219,10 +219,6 @@ CREATE INDEX IF NOT EXISTS idx_pinned_session CREATE INDEX IF NOT EXISTS idx_pinned_created ON pinned_messages (created_at DESC); -CREATE INDEX IF NOT EXISTS idx_pinned_source_uuid - ON pinned_messages (session_id, source_uuid) - WHERE source_uuid <> ''; - CREATE TABLE IF NOT EXISTS model_pricing ( model_pattern TEXT PRIMARY KEY, input_per_mtok DOUBLE PRECISION NOT NULL DEFAULT 0, @@ -693,6 +689,11 @@ func EnsureSchema( `thinking_text TEXT NOT NULL DEFAULT ''`, "adding messages.thinking_text", }, + { + "pinned_messages", "source_uuid", + `source_uuid TEXT NOT NULL DEFAULT ''`, + "adding pinned_messages.source_uuid", + }, { "sessions", "termination_status", `termination_status TEXT`, @@ -876,6 +877,9 @@ func createPartialIndexesPG(ctx context.Context, db *sql.DB) error { ON messages(session_id) WHERE is_sidechain = TRUE`, `CREATE INDEX IF NOT EXISTS idx_messages_source_uuid ON messages(source_uuid) WHERE source_uuid != ''`, + `CREATE INDEX IF NOT EXISTS idx_pinned_source_uuid + ON pinned_messages(session_id, source_uuid) + WHERE source_uuid <> ''`, `CREATE INDEX IF NOT EXISTS idx_messages_usage_covering ON messages(timestamp, session_id, ordinal, model, claude_message_id, claude_request_id) diff --git a/internal/postgres/schema_pgtest_test.go b/internal/postgres/schema_pgtest_test.go index 491cc9af2..103138155 100644 --- a/internal/postgres/schema_pgtest_test.go +++ b/internal/postgres/schema_pgtest_test.go @@ -123,3 +123,54 @@ func TestToolCallsFilePathIndex(t *testing.T) { require.NoError(t, err, "checking idx_tool_calls_file_path") assert.True(t, exists, "idx_tool_calls_file_path index missing") } + +func TestEnsureSchemaMigratesPinnedMessageSourceUUIDBeforeIndex(t *testing.T) { + pgURL := testPGURL(t) + cleanSchemaTestPG(t, pgURL) + t.Cleanup(func() { cleanSchemaTestPG(t, pgURL) }) + + pg, err := Open(pgURL, schemaTestSchema, true) + require.NoError(t, err, "connecting to pg") + defer pg.Close() + + ctx := context.Background() + _, err = pg.ExecContext(ctx, + `CREATE SCHEMA IF NOT EXISTS `+schemaTestSchema) + require.NoError(t, err, "creating test schema") + _, err = pg.ExecContext(ctx, ` + CREATE TABLE pinned_messages ( + id BIGSERIAL PRIMARY KEY, + session_id TEXT NOT NULL, + message_id INT NOT NULL, + ordinal INT NOT NULL, + note TEXT, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + UNIQUE (session_id, message_id) + )`) + require.NoError(t, err, "creating legacy pinned_messages table") + + require.NoError(t, EnsureSchema(ctx, pg, schemaTestSchema), + "EnsureSchema should add source_uuid before creating its index") + + var columnExists bool + err = pg.QueryRowContext(ctx, ` + SELECT EXISTS ( + SELECT 1 FROM information_schema.columns + WHERE table_schema = $1 + AND table_name = 'pinned_messages' + AND column_name = 'source_uuid' + )`, schemaTestSchema).Scan(&columnExists) + require.NoError(t, err, "checking pinned_messages.source_uuid") + assert.True(t, columnExists, "pinned_messages.source_uuid missing") + + var indexExists bool + err = pg.QueryRowContext(ctx, ` + SELECT EXISTS ( + SELECT 1 FROM pg_indexes + WHERE schemaname = $1 + AND tablename = 'pinned_messages' + AND indexname = 'idx_pinned_source_uuid' + )`, schemaTestSchema).Scan(&indexExists) + require.NoError(t, err, "checking idx_pinned_source_uuid") + assert.True(t, indexExists, "idx_pinned_source_uuid index missing") +} diff --git a/internal/postgres/schema_test.go b/internal/postgres/schema_test.go index 7b0f6ab98..2bbe54789 100644 --- a/internal/postgres/schema_test.go +++ b/internal/postgres/schema_test.go @@ -6,6 +6,7 @@ import ( "database/sql/driver" "errors" "io" + "slices" "strings" "sync" "testing" @@ -109,19 +110,20 @@ func (c *schemaProbeConn) Begin() (driver.Tx, error) { func (c *schemaProbeConn) ExecContext( _ context.Context, query string, args []driver.NamedValue, ) (driver.Result, error) { + normalized := strings.ToLower(query) + if strings.Contains(normalized, "idx_pinned_source_uuid") && + c.state.hasColumn("pinned_messages", "session_id") && + !c.state.hasColumn("pinned_messages", "source_uuid") { + return nil, errors.New(`ERROR: column "source_uuid" does not exist (SQLSTATE 42703)`) + } c.state.mu.Lock() c.state.execs = append(c.state.execs, query) c.state.execArgs = append( c.state.execArgs, append([]driver.NamedValue(nil), args...), ) c.state.mu.Unlock() - normalized := strings.ToLower(query) if strings.Contains(normalized, "alter table") { - c.state.mu.Lock() - c.state.alterTableExecs = append( - c.state.alterTableExecs, query, - ) - c.state.mu.Unlock() + c.state.recordAlterTable(query) } if strings.Contains(normalized, "insert into sync_metadata") && len(args) > 0 { @@ -136,6 +138,65 @@ func (c *schemaProbeConn) ExecContext( return driver.RowsAffected(0), nil } +func (s *schemaProbeState) hasColumn(table, column string) bool { + s.mu.Lock() + defer s.mu.Unlock() + return slices.Contains(s.existingColumnNames[table], column) +} + +func (s *schemaProbeState) recordAlterTable(query string) { + s.mu.Lock() + defer s.mu.Unlock() + s.alterTableExecs = append(s.alterTableExecs, query) + + table, ok := alterTableName(query) + if !ok { + return + } + if s.existingColumnNames == nil { + s.existingColumnNames = map[string][]string{} + } + for _, column := range alterTableColumns(query) { + exists := slices.Contains(s.existingColumnNames[table], column) + if !exists { + s.existingColumnNames[table] = append( + s.existingColumnNames[table], column, + ) + } + } +} + +func alterTableName(query string) (string, bool) { + const prefix = `ALTER TABLE "` + _, after, ok := strings.Cut(query, prefix) + if !ok { + return "", false + } + rest := after + before0, _, ok0 := strings.Cut(rest, `"`) + if !ok0 { + return "", false + } + return before0, true +} + +func alterTableColumns(query string) []string { + const marker = "ADD COLUMN IF NOT EXISTS " + parts := strings.Split(query, marker) + if len(parts) < 2 { + return nil + } + columns := make([]string, 0, len(parts)-1) + for _, part := range parts[1:] { + fields := strings.Fields(part) + if len(fields) == 0 { + continue + } + columns = append(columns, strings.Trim(fields[0], `",`)) + } + return columns +} + func (c *schemaProbeConn) QueryContext( _ context.Context, query string, args []driver.NamedValue, ) (driver.Rows, error) { @@ -808,6 +869,9 @@ func TestEnsureSchemaGroupsMissingColumnMigrationsByTable(t *testing.T) { "tool_calls": { "call_index", "file_path", }, + "pinned_messages": { + "source_uuid", + }, }) require.NoError(t, EnsureSchema(context.Background(), db, "agentsview")) @@ -819,3 +883,25 @@ func TestEnsureSchemaGroupsMissingColumnMigrationsByTable(t *testing.T) { // it contributes no ALTER. assert.Equal(t, 2, state.alterTableExecCount(), "ALTER TABLE execs") } + +func TestEnsureSchemaMigratesPinnedMessageSourceUUID(t *testing.T) { + db, state := newSchemaProbeDB(t, map[string][]string{ + "sessions": { + "has_total_output_tokens", + "has_peak_context_tokens", + }, + "messages": { + "has_context_tokens", + "has_output_tokens", + }, + "pinned_messages": { + "id", "session_id", "message_id", "ordinal", + "note", "created_at", + }, + }) + + require.NoError(t, EnsureSchema(context.Background(), db, "agentsview")) + + assert.Contains(t, state.executedSQL(), + "ALTER TABLE \"pinned_messages\" ADD COLUMN IF NOT EXISTS source_uuid TEXT NOT NULL DEFAULT ''") +} diff --git a/internal/postgres/sessions.go b/internal/postgres/sessions.go index 7af2e9fb5..adf162d51 100644 --- a/internal/postgres/sessions.go +++ b/internal/postgres/sessions.go @@ -1088,6 +1088,31 @@ func (s *Store) GetAgents( } // GetMachines returns distinct machine names. +// MachineSessionCounts returns the number of non-deleted sessions per machine, +// keyed by machine name. +func (s *Store) MachineSessionCounts(ctx context.Context) (map[string]int, error) { + rows, err := s.pg.QueryContext(ctx, + `SELECT machine, COUNT(*) FROM sessions + WHERE deleted_at IS NULL + GROUP BY machine`, + ) + if err != nil { + return nil, err + } + defer rows.Close() + + counts := map[string]int{} + for rows.Next() { + var machine string + var count int + if err := rows.Scan(&machine, &count); err != nil { + return nil, err + } + counts[machine] = count + } + return counts, rows.Err() +} + func (s *Store) GetMachines( ctx context.Context, excludeOneShot, excludeAutomated bool, diff --git a/internal/postgres/store.go b/internal/postgres/store.go index b214188d5..072e43a88 100644 --- a/internal/postgres/store.go +++ b/internal/postgres/store.go @@ -451,6 +451,51 @@ func (s *Store) SoftDeleteSessions(ids []string) (int, error) { return total, nil } +// SoftDeleteSessionsReturningIDs moves multiple sessions to the trash and +// returns the IDs that changed state. +func (s *Store) SoftDeleteSessionsReturningIDs(ids []string) ([]string, error) { + if len(ids) == 0 { + return nil, nil + } + deleted := make([]string, 0, len(ids)) + const batchSize = 500 + for start := 0; start < len(ids); start += batchSize { + end := min(start+batchSize, len(ids)) + pb := ¶mBuilder{} + placeholders := make([]string, 0, end-start) + for _, id := range ids[start:end] { + placeholders = append(placeholders, pb.add(id)) + } + rows, err := s.pg.Query( + `UPDATE sessions + SET deleted_at = NOW(), + updated_at = NOW() + WHERE id IN (`+strings.Join(placeholders, ",")+ + `) AND deleted_at IS NULL + RETURNING id`, + pb.args..., + ) + if err != nil { + return deleted, mapPGWriteError("soft deleting sessions", err) + } + for rows.Next() { + var id string + if err := rows.Scan(&id); err != nil { + _ = rows.Close() + return deleted, fmt.Errorf("scanning soft deleted session: %w", err) + } + deleted = append(deleted, id) + } + if err := rows.Close(); err != nil { + return deleted, fmt.Errorf("closing soft deleted session rows: %w", err) + } + if err := rows.Err(); err != nil { + return deleted, fmt.Errorf("iterating soft deleted sessions: %w", err) + } + } + return deleted, nil +} + // RestoreSession restores a trashed session. func (s *Store) RestoreSession(id string) (int64, error) { res, err := s.pg.Exec( diff --git a/internal/postgres/sync.go b/internal/postgres/sync.go index d70f122dc..1322ac09b 100644 --- a/internal/postgres/sync.go +++ b/internal/postgres/sync.go @@ -52,6 +52,7 @@ func (s *scopedSyncStateStore) ensureMigration() error { "last_push_at", lastPushBoundaryStateKey, lastPushTargetFingerprintKey, + artifactIdentityModeStateKey, } { scopedKey := s.scopedKey(key) scopedValue, err := s.base.GetSyncState(scopedKey) diff --git a/internal/server/artifact_http_transport_test.go b/internal/server/artifact_http_transport_test.go new file mode 100644 index 000000000..53b8e2d38 --- /dev/null +++ b/internal/server/artifact_http_transport_test.go @@ -0,0 +1,192 @@ +package server_test + +import ( + "context" + "encoding/json" + "net/http/httptest" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "go.kenn.io/agentsview/internal/artifact" + "go.kenn.io/agentsview/internal/db" + "go.kenn.io/agentsview/internal/dbtest" +) + +// newClientNode opens a client-only artifact store (a db plus data dir) that +// drives artifact.Sync against an HTTP peer. +func newClientNode(t *testing.T, sessionID, project string) (*db.DB, string) { + t.Helper() + dir := t.TempDir() + database, err := db.Open(filepath.Join(dir, "client.db")) + require.NoError(t, err) + t.Cleanup(func() { database.Close() }) + dbtest.SeedSession(t, database, sessionID, project, func(s *db.Session) { + s.MessageCount = 2 + s.UserMessageCount = 1 + }) + require.NoError(t, database.ReplaceSessionMessages(sessionID, []db.Message{ + {SessionID: sessionID, Ordinal: 0, Role: "user", Content: "hello", ContentLength: 5}, + {SessionID: sessionID, Ordinal: 1, Role: "assistant", Content: "world", ContentLength: 5}, + })) + return database, dir +} + +func TestArtifactHTTPTransportSyncsSessionsAndMetadata(t *testing.T) { + ctx := context.Background() + const token = "secret" + const aOrigin = "laptop-a1b2c3" + + // Node B: a real server exposing the artifact peer API behind auth. + te := setup(t, withAuth(token), withArtifactOrigin("desktop-d4e5f6")) + peer := httptest.NewServer(te.srv.Handler()) + defer peer.Close() + + aDB, aDir := newClientNode(t, "sess-1", "alpha") + + syncToPeer := func() { + _, err := artifact.Sync(ctx, aDB, artifact.SyncOptions{ + DataDir: aDir, + Target: peer.URL, + Origin: aOrigin, + Token: token, + }) + require.NoError(t, err) + } + + // A pushes its session over HTTP; B imports it on receipt. + syncToPeer() + importedID := aOrigin + "~sess-1" + gotB, err := te.db.GetSession(ctx, importedID) + require.NoError(t, err) + require.NotNil(t, gotB, "peer should import the pushed session") + assert.Equal(t, "alpha", gotB.Project) + + // A renames the session and syncs again; the metadata event is enumerated + // via the index route, posted, and replayed on B. + display := "Renamed on A" + require.NoError(t, aDB.RenameSession("sess-1", &display)) + recorder := artifact.NewMetadataRecorder(aDB, artifact.MetadataRecorderOptions{ + DataDir: aDir, + Origin: aOrigin, + }) + value, err := json.Marshal(struct { + DisplayName *string `json:"display_name"` + }{DisplayName: &display}) + require.NoError(t, err) + _, err = recorder.Append(ctx, artifact.MetadataEventInput{ + SessionID: "sess-1", + Op: artifact.MetadataOpRename, + Value: value, + }) + require.NoError(t, err) + + syncToPeer() + gotB, err = te.db.GetSession(ctx, importedID) + require.NoError(t, err) + require.NotNil(t, gotB) + require.NotNil(t, gotB.DisplayName) + assert.Equal(t, display, *gotB.DisplayName) +} + +func TestArtifactHTTPTransportPullsRemoteSessions(t *testing.T) { + ctx := context.Background() + const token = "secret" + const bOrigin = "desktop-d4e5f6" + + // Node B owns a session but has not run a separate artifact publisher. + te := setup(t, withAuth(token), withArtifactOrigin(bOrigin)) + peer := httptest.NewServer(te.srv.Handler()) + defer peer.Close() + dbtest.SeedSession(t, te.db, "remote-1", "bravo", func(s *db.Session) { + s.MessageCount = 2 + s.UserMessageCount = 1 + }) + require.NoError(t, te.db.ReplaceSessionMessages("remote-1", []db.Message{ + {SessionID: "remote-1", Ordinal: 0, Role: "user", Content: "ping", ContentLength: 4}, + {SessionID: "remote-1", Ordinal: 1, Role: "assistant", Content: "pong", ContentLength: 4}, + })) + displayName := "Renamed before HTTP publishing" + require.NoError(t, te.db.RenameSession("remote-1", &displayName)) + + aDB, aDir := newClientNode(t, "sess-1", "alpha") + _, err := artifact.Sync(ctx, aDB, artifact.SyncOptions{ + DataDir: aDir, + Target: peer.URL, + Origin: "laptop-a1b2c3", + Token: token, + }) + require.NoError(t, err) + + // A pulled B's session. + gotA, err := aDB.GetSession(ctx, bOrigin+"~remote-1") + require.NoError(t, err) + require.NotNil(t, gotA, "client should pull the remote session") + assert.Equal(t, "bravo", gotA.Project) + require.NotNil(t, gotA.DisplayName) + assert.Equal(t, displayName, *gotA.DisplayName) +} + +func TestArtifactHTTPTransportRejectsBadToken(t *testing.T) { + te := setup(t, withAuth("secret"), withArtifactOrigin("desktop-d4e5f6")) + peer := httptest.NewServer(te.srv.Handler()) + defer peer.Close() + aDB, aDir := newClientNode(t, "sess-1", "alpha") + + _, err := artifact.Sync(context.Background(), aDB, artifact.SyncOptions{ + DataDir: aDir, + Target: peer.URL, + Origin: "laptop-a1b2c3", + Token: "wrong", + }) + require.Error(t, err) + assert.Contains(t, err.Error(), "peer") +} + +func TestArtifactHTTPTransportPullRepairsCorruptOwnedArtifact(t *testing.T) { + ctx := context.Background() + const token = "secret" + const bOrigin = "desktop-d4e5f6" + + te := setup(t, withAuth(token), withArtifactOrigin(bOrigin)) + peer := httptest.NewServer(te.srv.Handler()) + defer peer.Close() + dbtest.SeedSession(t, te.db, "remote-1", "bravo", func(s *db.Session) { + s.MessageCount = 2 + s.UserMessageCount = 1 + }) + require.NoError(t, te.db.ReplaceSessionMessages("remote-1", []db.Message{ + {SessionID: "remote-1", Ordinal: 0, Role: "user", Content: "ping", ContentLength: 4}, + {SessionID: "remote-1", Ordinal: 1, Role: "assistant", Content: "pong", ContentLength: 4}, + })) + _, err := artifact.Export(ctx, te.db, filepath.Join(te.dataDir, "artifacts"), bOrigin) + require.NoError(t, err) + + // Corrupt the stored segment so the real server's GET-side content + // validation fails for it. + segments, err := filepath.Glob(filepath.Join( + te.dataDir, "artifacts", bOrigin, "segments", "*")) + require.NoError(t, err) + require.Len(t, segments, 1) + require.NoError(t, os.WriteFile(segments[0], []byte("corrupt"), 0o644)) + + // Peer discovery refreshes owned artifacts, so the corrupt segment is + // quarantined and regenerated before the client enumerates the store. + aDB, aDir := newClientNode(t, "sess-1", "alpha") + _, err = artifact.Sync(ctx, aDB, artifact.SyncOptions{ + DataDir: aDir, + Target: peer.URL, + Origin: "laptop-a1b2c3", + Token: token, + }) + require.NoError(t, err) + got, err := aDB.GetSession(ctx, bOrigin+"~remote-1") + require.NoError(t, err) + require.NotNil(t, got) + assert.Equal(t, "bravo", got.Project) + assert.FileExists(t, segments[0]) + assert.FileExists(t, segments[0]+".corrupt") +} diff --git a/internal/server/artifact_peer_test.go b/internal/server/artifact_peer_test.go new file mode 100644 index 000000000..4a685f0c1 --- /dev/null +++ b/internal/server/artifact_peer_test.go @@ -0,0 +1,522 @@ +package server_test + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "net/http" + "net/http/httptest" + "net/url" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "go.kenn.io/agentsview/internal/artifact" + "go.kenn.io/agentsview/internal/config" + "go.kenn.io/agentsview/internal/db" + "go.kenn.io/agentsview/internal/dbtest" + "go.kenn.io/agentsview/internal/server" +) + +type artifactOriginsBody struct { + Origins []string `json:"origins"` +} + +type artifactPostBody struct { + Origin string `json:"origin"` + Kind string `json:"kind"` + Name string `json:"name"` + Hash string `json:"hash,omitempty"` + Size int64 `json:"size"` + Duplicate bool `json:"duplicate"` +} + +type artifactRemoteStore struct { + db.Store +} + +func (artifactRemoteStore) ReadOnly() bool { return true } + +func (artifactRemoteStore) MachineSessionCounts( + context.Context, +) (map[string]int, error) { + return map[string]int{}, nil +} + +func (artifactRemoteStore) CountMetadataConflicts(context.Context) (int, error) { + return 0, nil +} + +func TestArtifactPeerRoutesRequireBearerAuthWhenConfigured(t *testing.T) { + te := setup(t, withAuth("secret")) + + w := artifactPeerRequest(t, te, http.MethodGet, "/api/v1/artifacts/origins", nil, "") + assertStatus(t, w, http.StatusUnauthorized) + + w = artifactPeerRequest(t, te, http.MethodGet, "/api/v1/artifacts/origins", nil, "secret") + assertStatus(t, w, http.StatusOK) +} + +func TestArtifactPeerRoutesPostDuplicateAndFetch(t *testing.T) { + te := setup(t, withAuth("secret")) + origin := "peer-a1b2c3" + metadataBody, metadataName := peerMetadataArtifact( + origin, + "2026-06-14T010203.000000001Z-00000000000000000000", + ) + + w := artifactPeerRequest( + t, te, http.MethodPost, + "/api/v1/artifacts/"+origin+"/meta/"+url.PathEscape(metadataName), + metadataBody, "secret", + ) + assertStatus(t, w, http.StatusOK) + posted := decode[artifactPostBody](t, w) + assert.False(t, posted.Duplicate) + assert.Equal(t, "meta", posted.Kind) + assert.Equal(t, metadataName, posted.Name) + + w = artifactPeerRequest( + t, te, http.MethodPost, + "/api/v1/artifacts/"+origin+"/meta/"+url.PathEscape(metadataName), + metadataBody, "secret", + ) + assertStatus(t, w, http.StatusOK) + posted = decode[artifactPostBody](t, w) + assert.True(t, posted.Duplicate) + + w = artifactPeerRequest( + t, te, http.MethodGet, + "/api/v1/artifacts/"+origin+"/meta/"+url.PathEscape(metadataName), + nil, "secret", + ) + assertStatus(t, w, http.StatusOK) + assert.Equal(t, "application/json", w.Header().Get("Content-Type")) + assert.Equal(t, metadataBody, w.Body.Bytes()) + + checkpoint := []byte(`{"origin":"peer-a1b2c3","seq":1,"sessions":{},"v":1}` + "\n") + w = artifactPeerRequest( + t, te, http.MethodPost, + "/api/v1/artifacts/"+origin+"/checkpoints/cp-0000000001", + checkpoint, "secret", + ) + assertStatus(t, w, http.StatusOK) + + w = artifactPeerRequest( + t, te, http.MethodGet, + "/api/v1/artifacts/"+origin+"/checkpoint", + nil, "secret", + ) + assertStatus(t, w, http.StatusOK) + assert.Equal(t, checkpoint, w.Body.Bytes()) + + w = artifactPeerRequest(t, te, http.MethodGet, "/api/v1/artifacts/origins", nil, "secret") + assertStatus(t, w, http.StatusOK) + origins := decode[artifactOriginsBody](t, w) + assert.Contains(t, origins.Origins, origin) +} + +func TestArtifactPeerPostRejectsRemoteStoreBeforeWrite(t *testing.T) { + dir := tempDirWithRetryCleanup(t) + cfg := config.Config{ + Host: "127.0.0.1", + Port: 0, + DataDir: dir, + WriteTimeout: 30 * time.Second, + } + srv := server.New(cfg, artifactRemoteStore{}, nil) + te := &testEnv{ + srv: srv, + handler: wrapTestHandler(cfg, srv.Handler()), + dataDir: dir, + } + origin := "peer-a1b2c3" + metadataBody, metadataName := peerMetadataArtifact( + origin, + "2026-06-14T010203.000000001Z-00000000000000000000", + ) + + w := artifactPeerRequest( + t, te, http.MethodPost, + "/api/v1/artifacts/"+origin+"/meta/"+url.PathEscape(metadataName), + metadataBody, "", + ) + + assertStatus(t, w, http.StatusNotImplemented) + assertArtifactMissing(t, dir, origin, "meta", metadataName) +} + +func TestArtifactPeerReadRoutesRejectRemoteStoreBeforeReadingFiles(t *testing.T) { + dir := tempDirWithRetryCleanup(t) + cfg := config.Config{ + Host: "127.0.0.1", + Port: 0, + DataDir: dir, + WriteTimeout: 30 * time.Second, + } + origin := "peer-a1b2c3" + checkpoint := []byte(`{"origin":"peer-a1b2c3","seq":1,"sessions":{},"v":1}` + "\n") + _, err := artifact.WriteArtifact( + filepath.Join(dir, "artifacts"), + origin, + "checkpoints", + "cp-0000000001", + checkpoint, + ) + require.NoError(t, err) + + srv := server.New(cfg, artifactRemoteStore{}, nil) + te := &testEnv{ + srv: srv, + handler: wrapTestHandler(cfg, srv.Handler()), + dataDir: dir, + } + + for _, path := range []string{ + "/api/v1/artifacts/origins", + "/api/v1/artifacts/peers", + "/api/v1/artifacts/" + origin + "/index", + "/api/v1/artifacts/" + origin + "/checkpoint", + "/api/v1/artifacts/" + origin + "/checkpoints/cp-0000000001", + } { + w := artifactPeerRequest(t, te, http.MethodGet, path, nil, "") + assertStatus(t, w, http.StatusNotImplemented) + } +} + +func TestArtifactPeerPostRejectsReadOnlySQLiteBeforeWrite(t *testing.T) { + dir := tempDirWithRetryCleanup(t) + dbPath := filepath.Join(dir, "test.db") + writable, err := db.Open(dbPath) + require.NoError(t, err) + require.NoError(t, writable.Close()) + readonly, err := db.OpenReadOnly(dbPath) + require.NoError(t, err) + t.Cleanup(func() { readonly.Close() }) + + cfg := config.Config{ + Host: "127.0.0.1", + Port: 0, + DataDir: dir, + DBPath: dbPath, + ArtifactOriginID: "desktop-d4e5f6", + WriteTimeout: 30 * time.Second, + } + srv := server.New(cfg, readonly, nil) + te := &testEnv{ + srv: srv, + handler: wrapTestHandler(cfg, srv.Handler()), + db: readonly, + dataDir: dir, + } + origin := "peer-a1b2c3" + metadataBody, metadataName := peerMetadataArtifact( + origin, + "2026-06-14T010203.000000001Z-00000000000000000000", + ) + + w := artifactPeerRequest( + t, te, http.MethodPost, + "/api/v1/artifacts/"+origin+"/meta/"+url.PathEscape(metadataName), + metadataBody, "", + ) + + assertStatus(t, w, http.StatusNotImplemented) + assertArtifactMissing(t, dir, origin, "meta", metadataName) +} + +type artifactPeerBody struct { + Origin string `json:"origin"` + IsLocal bool `json:"is_local"` + CheckpointSeq int `json:"checkpoint_seq"` + PublishedSessions int `json:"published_sessions"` + LocalSessions int `json:"local_sessions"` + LastPublished string `json:"last_published"` +} + +type artifactPeersBody struct { + LocalOrigin string `json:"local_origin"` + Peers []artifactPeerBody `json:"peers"` + ConflictCount int `json:"conflict_count"` +} + +func TestArtifactPeersStatus(t *testing.T) { + local := "desktop-d4e5f6" + te := setup(t, withArtifactOrigin(local)) + ctx := context.Background() + artifactRoot := filepath.Join(te.dataDir, "artifacts") + first := "hi" + + // Two owned sessions, exported so the local origin gets a checkpoint. + dbtest.SeedSession(t, te.db, "local-1", "proj", func(s *db.Session) { s.FirstMessage = &first }) + dbtest.SeedSession(t, te.db, "local-2", "proj", func(s *db.Session) { s.FirstMessage = &first }) + exported, err := artifact.Export(ctx, te.db, artifactRoot, local) + require.NoError(t, err) + require.Equal(t, 2, exported) + + // A foreign peer publishes one session that the server imports. + origin := "peer-a1b2c3" + peerRoot := t.TempDir() + peerDB, err := db.Open(filepath.Join(t.TempDir(), "peer.db")) + require.NoError(t, err) + t.Cleanup(func() { peerDB.Close() }) + dbtest.SeedSession(t, peerDB, "sess-1", "alpha", func(s *db.Session) { s.FirstMessage = &first }) + require.NoError(t, peerDB.ReplaceSessionMessages("sess-1", []db.Message{ + {SessionID: "sess-1", Ordinal: 0, Role: "user", Content: "hello", ContentLength: 5}, + })) + _, err = artifact.Export(ctx, peerDB, peerRoot, origin) + require.NoError(t, err) + postArtifactFile(t, te, origin, "segments", oneArtifactPath(t, peerRoot, origin, "segments", "*")) + postArtifactFile(t, te, origin, "manifests", oneArtifactPath(t, peerRoot, origin, "manifests", "*")) + postArtifactFile(t, te, origin, "checkpoints", oneArtifactPath(t, peerRoot, origin, "checkpoints", "*")) + + w := artifactPeerRequest(t, te, http.MethodGet, "/api/v1/artifacts/peers", nil, "") + assertStatus(t, w, http.StatusOK) + body := decode[artifactPeersBody](t, w) + + assert.Equal(t, local, body.LocalOrigin) + assert.Equal(t, 0, body.ConflictCount) + require.Len(t, body.Peers, 2) + + byOrigin := map[string]artifactPeerBody{} + for _, p := range body.Peers { + byOrigin[p.Origin] = p + } + + localPeer, ok := byOrigin[local] + require.True(t, ok, "local origin present in peers") + assert.True(t, localPeer.IsLocal) + assert.Equal(t, 2, localPeer.PublishedSessions) + assert.Equal(t, 2, localPeer.LocalSessions) + assert.NotEmpty(t, localPeer.LastPublished) + + peer, ok := byOrigin[origin] + require.True(t, ok, "foreign origin present in peers") + assert.False(t, peer.IsLocal) + assert.Equal(t, 1, peer.PublishedSessions) + assert.Equal(t, 1, peer.LocalSessions) + assert.Equal(t, 1, peer.CheckpointSeq) +} + +func TestArtifactPeersStatusPublishesEmptyLocalOrigin(t *testing.T) { + te := setup(t, withArtifactOrigin("desktop-d4e5f6")) + // Discovery publishes an explicit empty checkpoint for a configured origin. + w := artifactPeerRequest(t, te, http.MethodGet, "/api/v1/artifacts/peers", nil, "") + assertStatus(t, w, http.StatusOK) + body := decode[artifactPeersBody](t, w) + assert.Equal(t, "desktop-d4e5f6", body.LocalOrigin) + require.Len(t, body.Peers, 1) + assert.True(t, body.Peers[0].IsLocal) + assert.Equal(t, 0, body.Peers[0].PublishedSessions) + assert.Equal(t, 1, body.Peers[0].CheckpointSeq) + assert.NotEmpty(t, body.Peers[0].LastPublished) +} + +func TestArtifactPeerPostRejectsHashMismatch(t *testing.T) { + te := setup(t, withAuth("secret")) + origin := "peer-a1b2c3" + metadataBody, _ := peerMetadataArtifact( + origin, + "2026-06-14T010203.000000001Z-00000000000000000000", + ) + badName := "2026-06-14T010203.000000001Z-peer-a1b2c3-" + strings.Repeat("0", 64) + + w := artifactPeerRequest( + t, te, http.MethodPost, + "/api/v1/artifacts/"+origin+"/meta/"+url.PathEscape(badName), + metadataBody, "secret", + ) + assertStatus(t, w, http.StatusBadRequest) +} + +func TestArtifactPeerPostImportsAndEmitsDataChanged(t *testing.T) { + te := setup(t, withArtifactOrigin("desktop-d4e5f6")) + origin := "peer-a1b2c3" + artifactRoot := t.TempDir() + peerDB, err := db.Open(filepath.Join(t.TempDir(), "peer.db")) + require.NoError(t, err) + t.Cleanup(func() { peerDB.Close() }) + + first := "hello" + started := "2026-06-14T01:02:03Z" + ended := "2026-06-14T01:03:03Z" + dbtest.SeedSession(t, peerDB, "sess-1", "alpha", func(s *db.Session) { + s.MessageCount = 2 + s.UserMessageCount = 1 + s.FirstMessage = &first + s.StartedAt = &started + s.EndedAt = &ended + }) + require.NoError(t, peerDB.ReplaceSessionMessages("sess-1", []db.Message{ + {SessionID: "sess-1", Ordinal: 0, Role: "user", Content: "hello", ContentLength: 5}, + {SessionID: "sess-1", Ordinal: 1, Role: "assistant", Content: "world", ContentLength: 5}, + })) + _, err = artifact.Export(context.Background(), peerDB, artifactRoot, origin) + require.NoError(t, err) + + postArtifactFile(t, te, origin, "segments", + oneArtifactPath(t, artifactRoot, origin, "segments", "*")) + postArtifactFile(t, te, origin, "manifests", + oneArtifactPath(t, artifactRoot, origin, "manifests", "*")) + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + req := httptest.NewRequest(http.MethodGet, "/api/v1/events", nil).WithContext(ctx) + stream := &flushRecorder{ResponseRecorder: httptest.NewRecorder()} + done := make(chan struct{}) + go func() { + te.handler.ServeHTTP(stream, req) + close(done) + }() + time.Sleep(100 * time.Millisecond) + + postArtifactFile(t, te, origin, "checkpoints", + oneArtifactPath(t, artifactRoot, origin, "checkpoints", "*")) + te.waitForSSEEvent(t, stream, "data_changed", 3*time.Second) + + // Live clients only refresh the session index on the "sessions" + // scope and only invalidate hydrated session details on the + // "messages" scope; an import needs both. + assert.Eventually(t, func() bool { + scopes := dataChangedScopes(stream) + return scopes["messages"] && scopes["sessions"] + }, 3*time.Second, 10*time.Millisecond, + "import must emit data_changed with both messages and sessions scopes") + + got, err := te.db.GetSession(context.Background(), origin+"~sess-1") + require.NoError(t, err) + require.NotNil(t, got) + assert.Equal(t, origin, got.Machine) + assert.Equal(t, "alpha", got.Project) + + cancel() + <-done +} + +// dataChangedScopes collects the scope payloads of every data_changed +// event written to the SSE stream so far. +func dataChangedScopes(w *flushRecorder) map[string]bool { + scopes := make(map[string]bool) + for _, e := range parseSSE(w.BodyString()) { + if e.Event != "data_changed" { + continue + } + var payload struct { + Scope string `json:"scope"` + } + if json.Unmarshal([]byte(e.Data), &payload) == nil { + scopes[payload.Scope] = true + } + } + return scopes +} + +func artifactPeerRequest( + t *testing.T, + te *testEnv, + method string, + path string, + body []byte, + token string, +) *httptest.ResponseRecorder { + t.Helper() + req := httptest.NewRequest(method, path, bytes.NewReader(body)) + req.Header.Set("Content-Type", "application/octet-stream") + if token != "" { + req.Header.Set("Authorization", "Bearer "+token) + } + w := httptest.NewRecorder() + te.handler.ServeHTTP(w, req) + return w +} + +func postArtifactFile( + t *testing.T, + te *testEnv, + origin string, + kind string, + path string, +) { + t.Helper() + body, err := os.ReadFile(path) + require.NoError(t, err) + w := artifactPeerRequest( + t, te, http.MethodPost, + "/api/v1/artifacts/"+origin+"/"+kind+"/"+url.PathEscape(filepath.Base(path)), + body, "", + ) + assertStatus(t, w, http.StatusOK) +} + +func assertArtifactMissing( + t *testing.T, + dataDir string, + origin string, + kind string, + name string, +) { + t.Helper() + path := filepath.Join(dataDir, "artifacts", origin, kind, name) + _, err := os.Stat(path) + assert.True(t, errors.Is(err, os.ErrNotExist), + "artifact should not have been written at %s", path) +} + +func oneArtifactPath( + t *testing.T, + root string, + origin string, + kind string, + pattern string, +) string { + t.Helper() + paths, err := filepath.Glob(filepath.Join(root, origin, kind, pattern)) + require.NoError(t, err) + require.Len(t, paths, 1) + return paths[0] +} + +func peerMetadataArtifact(origin, hlc string) ([]byte, string) { + body := []byte(`{"hlc":"` + hlc + `","op":"rename","origin":"` + origin + `","session_gid":"` + origin + `~sess-1","v":1,"value":{"display_name":"Remote"}}` + "\n") + sum := sha256.Sum256(body) + hash := hex.EncodeToString(sum[:]) + return body, hlc + "-" + hash + ".json" +} + +func TestArtifactPeersStatusToleratesCorruptLatestCheckpoint(t *testing.T) { + local := "desktop-d4e5f6" + te := setup(t, withArtifactOrigin(local)) + ctx := context.Background() + artifactRoot := filepath.Join(te.dataDir, "artifacts") + first := "hi" + + dbtest.SeedSession(t, te.db, "local-1", "proj", func(s *db.Session) { s.FirstMessage = &first }) + _, err := artifact.Export(ctx, te.db, artifactRoot, local) + require.NoError(t, err) + dbtest.SeedSession(t, te.db, "local-2", "proj", func(s *db.Session) { s.FirstMessage = &first }) + _, err = artifact.Export(ctx, te.db, artifactRoot, local) + require.NoError(t, err) + + latest := filepath.Join(artifactRoot, local, "checkpoints", "cp-0000000002.json") + require.NoError(t, os.WriteFile(latest, []byte("not json"), 0o644)) + + // One corrupt newest checkpoint must not make the peers page unusable: + // discovery quarantines it and publishes the current two-session state at + // the next unused sequence. + w := artifactPeerRequest(t, te, http.MethodGet, "/api/v1/artifacts/peers", nil, "") + assertStatus(t, w, http.StatusOK) + body := decode[artifactPeersBody](t, w) + require.Len(t, body.Peers, 1) + assert.Equal(t, 3, body.Peers[0].CheckpointSeq) + assert.Equal(t, 2, body.Peers[0].PublishedSessions) + assert.NoFileExists(t, latest) + assert.FileExists(t, latest+".corrupt") +} diff --git a/internal/server/bulk_star_metadata_test.go b/internal/server/bulk_star_metadata_test.go new file mode 100644 index 000000000..b1e26e100 --- /dev/null +++ b/internal/server/bulk_star_metadata_test.go @@ -0,0 +1,70 @@ +package server_test + +import ( + "context" + "net/http" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "go.kenn.io/agentsview/internal/artifact" +) + +func TestBulkStarAppendsMetadataEvents(t *testing.T) { + te := setup(t, withArtifactOrigin("desktop-d4e5f6")) + te.seedSession(t, "s1", "alpha", 2) + te.seedSession(t, "s2", "beta", 2) + + w := te.requestJSON(t, http.MethodPost, "/api/v1/starred/bulk", + `{"session_ids":["s1","s2","missing"]}`) + require.Equal(t, http.StatusNoContent, w.Code, "body: %s", w.Body.String()) + + // Both existing sessions are starred; the missing one is skipped. + w = te.get(t, "/api/v1/starred") + require.Equal(t, http.StatusOK, w.Code, "body: %s", w.Body.String()) + list := decode[starredHandlerResponse](t, w) + assert.ElementsMatch(t, []string{"s1", "s2"}, list.SessionIDs) + + // A star metadata event artifact was written for each session actually + // starred, so the migrated stars converge through artifact sync. The missing + // session produces no event. + metaDir := filepath.Join(te.dataDir, "artifacts", "desktop-d4e5f6", "meta") + entries, err := os.ReadDir(metaDir) + require.NoError(t, err) + assert.Len(t, entries, 2, "one star event per existing session") +} + +func TestBulkStarRetriesRepairPublishedMetadataState(t *testing.T) { + te := setup(t, withArtifactOrigin("desktop-d4e5f6")) + te.seedSession(t, "s1", "alpha", 2) + + execTestDDL(t, te, ` +CREATE TRIGGER fail_metadata_replay_state_insert +BEFORE INSERT ON metadata_replay_state +BEGIN + SELECT RAISE(FAIL, 'forced metadata replay failure'); +END; +`) + + w := te.requestJSON(t, http.MethodPost, "/api/v1/starred/bulk", + `{"session_ids":["s1"]}`) + require.Equal(t, http.StatusInternalServerError, w.Code, "body: %s", w.Body.String()) + ids, err := te.db.ListStarredSessionIDs(context.Background()) + require.NoError(t, err) + assert.Equal(t, []string{"s1"}, ids) + assert.Equal(t, 0, serverMetadataTableCount(t, te, "metadata_replay_state", "session_gid = 'desktop-d4e5f6~s1'")) + events := readMetadataEvents(t, te) + require.Len(t, events, 1) + assert.Equal(t, artifact.MetadataOpStar, events[0].Op) + + execTestDDL(t, te, `DROP TRIGGER fail_metadata_replay_state_insert`) + w = te.requestJSON(t, http.MethodPost, "/api/v1/starred/bulk", + `{"session_ids":["s1"]}`) + require.Equal(t, http.StatusNoContent, w.Code, "body: %s", w.Body.String()) + assert.Equal(t, artifact.MetadataOpStar, + serverMetadataReplayOp(t, te, "desktop-d4e5f6~s1", "starred")) + assert.Len(t, readMetadataEvents(t, te), 1) +} diff --git a/internal/server/huma_route_groups.go b/internal/server/huma_route_groups.go index 1f9c99657..2056776b9 100644 --- a/internal/server/huma_route_groups.go +++ b/internal/server/huma_route_groups.go @@ -27,6 +27,7 @@ func (s *Server) registerTypedAPIRoutes() { s.registerImportRoutes() s.registerAssetRoutes() s.registerEmbeddingsRoutes() + s.registerArtifactRoutes() } type routeGroup struct { diff --git a/internal/server/huma_routes_artifacts.go b/internal/server/huma_routes_artifacts.go new file mode 100644 index 000000000..58fcf8555 --- /dev/null +++ b/internal/server/huma_routes_artifacts.go @@ -0,0 +1,392 @@ +package server + +import ( + "context" + "errors" + "net/http" + "path/filepath" + "time" + + "go.kenn.io/agentsview/internal/artifact" + "go.kenn.io/agentsview/internal/db" +) + +func (s *Server) registerArtifactRoutes() { + group := newRouteGroup(s.api, "/api/v1/artifacts", "Artifacts") + + get(s, group, "/origins", "List artifact origins", s.humaListArtifactOrigins) + get(s, group, "/peers", "List artifact peers", s.humaListArtifactPeers) + get(s, group, "/{origin}/index", "List artifact index for an origin", s.humaGetArtifactIndex) + raw(s, group, http.MethodGet, "/{origin}/checkpoint", "Get latest artifact checkpoint", s.humaGetArtifactCheckpoint) + raw(s, group, http.MethodGet, "/{origin}/{kind}/{name}", "Get artifact", s.humaGetArtifact) + post(s, group, "/{origin}/{kind}/{name}", "Post artifact", s.humaPostArtifact) +} + +type artifactOriginInput struct { + Origin string `path:"origin" required:"true" doc:"Artifact origin ID"` +} + +type artifactPathInput struct { + Origin string `path:"origin" required:"true" doc:"Artifact origin ID"` + Kind string `path:"kind" required:"true" doc:"Artifact kind"` + Name string `path:"name" required:"true" doc:"Artifact filename or hash"` +} + +type artifactPostInput struct { + Origin string `path:"origin" required:"true" doc:"Artifact origin ID"` + Kind string `path:"kind" required:"true" doc:"Artifact kind"` + Name string `path:"name" required:"true" doc:"Artifact filename or hash"` + RawBody []byte `contentType:"application/octet-stream"` +} + +type artifactOriginsResponse struct { + Origins []string `json:"origins"` +} + +// artifactPeer is one origin's status in the peers view: what it has published +// (from its latest checkpoint) and how much of it has landed locally. +type artifactPeer struct { + Origin string `json:"origin"` + IsLocal bool `json:"is_local"` + CheckpointSeq int `json:"checkpoint_seq"` + PublishedSessions int `json:"published_sessions"` + LocalSessions int `json:"local_sessions"` + LastPublished string `json:"last_published,omitempty"` +} + +type artifactPeersResponse struct { + LocalOrigin string `json:"local_origin"` + Peers []artifactPeer `json:"peers"` + ConflictCount int `json:"conflict_count"` +} + +type artifactPostResponse struct { + Origin string `json:"origin"` + Kind string `json:"kind"` + Name string `json:"name"` + Hash string `json:"hash,omitempty"` + Size int64 `json:"size"` + Duplicate bool `json:"duplicate"` +} + +func (s *Server) artifactRoot() (string, error) { + if s.cfg.DataDir == "" { + return "", apiError(http.StatusServiceUnavailable, "artifact store not configured") + } + return filepath.Join(s.cfg.DataDir, "artifacts"), nil +} + +func (s *Server) artifactCapableRoot() (string, error) { + if err := s.requireLocalArtifactStore(); err != nil { + return "", err + } + return s.artifactRoot() +} + +func (s *Server) humaListArtifactOrigins( + ctx context.Context, + _ *emptyInput, +) (*jsonOutput[artifactOriginsResponse], error) { + root, err := s.artifactCapableRoot() + if err != nil { + return nil, err + } + if err := s.publishLocalArtifacts(ctx, root); err != nil { + return nil, err + } + origins, err := artifact.ListOrigins(root) + if err != nil { + return nil, artifactRouteError("list artifact origins", err) + } + return &jsonOutput[artifactOriginsResponse]{ + Body: artifactOriginsResponse{Origins: origins}, + }, nil +} + +func (s *Server) humaGetArtifactIndex( + _ context.Context, + in *artifactOriginInput, +) (*jsonOutput[artifact.OriginArtifactIndex], error) { + root, err := s.artifactCapableRoot() + if err != nil { + return nil, err + } + index, err := artifact.ListArtifacts(root, in.Origin) + if err != nil { + return nil, artifactRouteError("list artifact index", err) + } + return &jsonOutput[artifact.OriginArtifactIndex]{Body: index}, nil +} + +// localArtifactOrigin returns this machine's artifact origin without creating +// one. It prefers the configured origin and falls back to the persisted DB +// value so read-only callers never mint a new identity. +func (s *Server) localArtifactOrigin() string { + if s.cfg.ArtifactOriginID != "" { + return s.cfg.ArtifactOriginID + } + if local, ok := s.db.(*db.DB); ok { + if origin, err := artifact.StoredOrigin(local); err == nil { + return origin + } + } + return "" +} + +func (s *Server) humaListArtifactPeers( + ctx context.Context, + _ *emptyInput, +) (*jsonOutput[artifactPeersResponse], error) { + root, err := s.artifactCapableRoot() + if err != nil { + return nil, err + } + if err := s.publishLocalArtifacts(ctx, root); err != nil { + return nil, err + } + origins, err := artifact.ListOrigins(root) + if err != nil { + return nil, artifactRouteError("list artifact origins", err) + } + counts, err := s.db.MachineSessionCounts(ctx) + if err != nil { + return nil, internalError("machine session counts", err) + } + conflicts, err := s.db.CountMetadataConflicts(ctx) + if err != nil { + return nil, internalError("count metadata conflicts", err) + } + + localOrigin := s.localArtifactOrigin() + // Always surface this machine even before its first export. + seen := map[string]bool{} + ordered := make([]string, 0, len(origins)+1) + if localOrigin != "" { + ordered = append(ordered, localOrigin) + seen[localOrigin] = true + } + for _, origin := range origins { + if seen[origin] { + continue + } + seen[origin] = true + ordered = append(ordered, origin) + } + + peers := make([]artifactPeer, 0, len(ordered)) + for _, origin := range ordered { + summary, err := artifact.CheckpointSummary(root, origin) + if err != nil { + return nil, artifactRouteError("read artifact checkpoint", err) + } + isLocal := origin == localOrigin + machineKey := origin + if isLocal { + // Owned sessions keep machine "local" in the local DB. + machineKey = "local" + } + last := "" + if summary.Found { + last = summary.ModTime.UTC().Format(time.RFC3339) + } + peers = append(peers, artifactPeer{ + Origin: origin, + IsLocal: isLocal, + CheckpointSeq: summary.Sequence, + PublishedSessions: summary.SessionCount, + LocalSessions: counts[machineKey], + LastPublished: last, + }) + } + + return &jsonOutput[artifactPeersResponse]{ + Body: artifactPeersResponse{ + LocalOrigin: localOrigin, + Peers: peers, + ConflictCount: conflicts, + }, + }, nil +} + +// publishLocalArtifacts refreshes the server's owned origin immediately before +// peer discovery. HTTP transports begin every exchange with origin discovery, +// so this makes the server a publisher without requiring a separate folder +// sync process while keeping individual artifact reads side-effect free. +func (s *Server) publishLocalArtifacts(ctx context.Context, root string) error { + local, err := s.writableArtifactImportDB() + if err != nil { + return err + } + origin := s.localArtifactOrigin() + if origin == "" { + return nil + } + + s.lockSessionLifecycle() + defer s.sessionLifecycleMu.Unlock() + if !s.artifactBaselineDone && s.metadata != nil { + if _, err := s.metadata.AppendBaseline(ctx); err != nil { + return artifactRouteError("baseline local artifact metadata", err) + } + s.artifactBaselineDone = true + } + if s.engine != nil { + s.engine.FlushSignals() + } + if _, err := artifact.Export(ctx, local, root, origin); err != nil { + return artifactRouteError("export local artifacts", err) + } + return nil +} + +func (s *Server) humaGetArtifactCheckpoint( + _ context.Context, + in *artifactOriginInput, +) (*bytesOutput, error) { + root, err := s.artifactCapableRoot() + if err != nil { + return nil, err + } + art, err := artifact.ReadLatestCheckpoint(root, in.Origin) + if err != nil { + return nil, artifactRouteError("get artifact checkpoint", err) + } + return artifactBytesOutput(art), nil +} + +func (s *Server) humaGetArtifact( + _ context.Context, + in *artifactPathInput, +) (*bytesOutput, error) { + root, err := s.artifactCapableRoot() + if err != nil { + return nil, err + } + art, err := artifact.ReadArtifactForServe(root, in.Origin, in.Kind, in.Name) + if err != nil { + return nil, artifactRouteError("get artifact", err) + } + return artifactBytesOutput(art), nil +} + +func (s *Server) humaPostArtifact( + ctx context.Context, + in *artifactPostInput, +) (*jsonOutput[artifactPostResponse], error) { + local, err := s.writableArtifactImportDB() + if err != nil { + return nil, err + } + root, err := s.artifactRoot() + if err != nil { + return nil, err + } + res, err := artifact.WriteArtifact(root, in.Origin, in.Kind, in.Name, in.RawBody) + if err != nil { + return nil, artifactRouteError("post artifact", err) + } + s.lockSessionLifecycle() + defer s.sessionLifecycleMu.Unlock() + if res.Kind == artifact.KindCheckpoints || res.Kind == artifact.KindMeta || s.artifactImportPending { + importRes, err := s.importPeerArtifacts(ctx, local, root) + if err != nil { + return nil, err + } + s.artifactImportPending = importRes.Deferred > 0 + } + return &jsonOutput[artifactPostResponse]{ + Body: artifactPostResponse{ + Origin: res.Origin, + Kind: res.Kind, + Name: res.Name, + Hash: res.Hash, + Size: res.Size, + Duplicate: res.Duplicate, + }, + }, nil +} + +func (s *Server) writableArtifactImportDB() (*db.DB, error) { + if err := s.requireLocalArtifactStore(); err != nil { + return nil, err + } + return s.db.(*db.DB), nil +} + +func (s *Server) requireLocalArtifactStore() error { + local, ok := s.db.(*db.DB) + if !ok { + return apiError(http.StatusNotImplemented, + "artifact routes are not available in remote mode") + } + if local.ReadOnly() { + return apiError(http.StatusNotImplemented, + "artifact routes are not available in read-only mode") + } + return nil +} + +func (s *Server) importPeerArtifacts( + ctx context.Context, local *db.DB, root string, +) (artifact.ImportResult, error) { + var res artifact.ImportResult + var err error + if s.metadata != nil { + // An incoming peer exchange is this machine's opt-in to artifact + // sync, so mint the origin here. The recorder itself never creates + // one: ordinary curation stays ledger-free until a machine opts in. + if s.cfg.ArtifactOriginID == "" { + if _, err := artifact.EnsureOrigin(local); err != nil { + return artifact.ImportResult{}, internalError("artifact import origin", err) + } + } + // Route through the recorder so import and local appends share one HLC + // clock, keeping later local edits causally ahead of imported peers. + res, err = s.metadata.Import(ctx, root) + } else { + localOrigin := s.cfg.ArtifactOriginID + if localOrigin == "" { + localOrigin, err = artifact.EnsureOrigin(local) + if err != nil { + return artifact.ImportResult{}, internalError("artifact import origin", err) + } + } + res, err = artifact.ImportDetailed(ctx, local, root, localOrigin) + } + if err != nil { + return artifact.ImportResult{}, artifactRouteError("import peer artifacts", err) + } + if res.Changed() && s.broadcaster != nil { + // Imports add sessions and apply curation metadata, so live + // clients need the session-index refresh that only the + // "sessions" scope triggers; "messages" additionally + // invalidates hydrated session details and cached signal + // detail. Emit "sessions" last so a coalesced burst resolves + // to the index refresh. + s.broadcaster.Emit("messages") + s.broadcaster.Emit("sessions") + } + return res, nil +} + +func artifactBytesOutput(art artifact.PeerArtifact) *bytesOutput { + return &bytesOutput{ + ContentType: art.ContentType, + NoSniff: "nosniff", + CacheControl: "no-store", + Body: art.Data, + } +} + +func artifactRouteError(logPrefix string, err error) error { + switch { + case errors.Is(err, artifact.ErrArtifactInvalid): + return apiError(http.StatusBadRequest, err.Error()) + case errors.Is(err, artifact.ErrArtifactNotFound): + return apiError(http.StatusNotFound, "artifact not found") + case errors.Is(err, artifact.ErrArtifactConflict): + return apiError(http.StatusConflict, "artifact conflict") + default: + return internalError(logPrefix, err) + } +} diff --git a/internal/server/huma_routes_metadata_internal_test.go b/internal/server/huma_routes_metadata_internal_test.go index a658370a6..9dc7419ee 100644 --- a/internal/server/huma_routes_metadata_internal_test.go +++ b/internal/server/huma_routes_metadata_internal_test.go @@ -2,11 +2,21 @@ package server import ( "context" + "crypto/sha256" + "encoding/hex" + "errors" + "slices" + "sync" + "sync/atomic" "testing" + "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "go.kenn.io/agentsview/internal/artifact" "go.kenn.io/agentsview/internal/config" + "go.kenn.io/agentsview/internal/db" + "go.kenn.io/agentsview/internal/dbtest" "go.kenn.io/agentsview/internal/service" ) @@ -36,3 +46,918 @@ func TestHumaGetSessionStatsUsesServerGitHubToken(t *testing.T) { require.NoError(t, err) assert.Equal(t, "server-token", spy.got.GHToken) } + +func TestHumaBatchDeleteRestoresOnlyUnpublishedNewDeletions(t *testing.T) { + tests := []struct { + name string + failure error + wantDeleted []string + }{ + { + name: "ordinary append failure restores current and later", + failure: errors.New("artifact write failed"), + wantDeleted: []string{"already-trashed", "s1"}, + }, + { + name: "published failure keeps current and restores later", + failure: &artifact.MetadataPublishedError{ + Err: errors.New("replay bookkeeping failed"), + }, + wantDeleted: []string{"already-trashed", "s1", "s2"}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + database := dbtest.OpenTestDB(t) + for _, id := range []string{"already-trashed", "s1", "s2", "s3"} { + dbtest.SeedSession(t, database, id, "alpha") + } + require.NoError(t, database.SoftDeleteSession("already-trashed")) + + var appended []string + srv := &Server{ + db: database, + metadataAppend: func( + _ context.Context, input artifact.MetadataEventInput, + ) error { + appended = append(appended, input.SessionID) + if input.SessionID == "s2" { + return tt.failure + } + return nil + }, + } + in := &batchDeleteInput{} + in.Body.SessionIDs = []string{"already-trashed", "s1", "s2", "s3"} + + _, err := srv.humaBatchDeleteSessions(context.Background(), in) + + require.Error(t, err) + assert.Equal(t, []string{"s1", "s2"}, appended, + "publication must stop at the first failed event") + for _, id := range []string{"already-trashed", "s1", "s2", "s3"} { + session, getErr := database.GetSessionFull(context.Background(), id) + require.NoError(t, getErr) + require.NotNil(t, session) + assert.Equal(t, containsString(tt.wantDeleted, id), session.DeletedAt != nil, + "unexpected trash state for %s", id) + } + }) + } +} + +func containsString(values []string, want string) bool { + return slices.Contains(values, want) +} + +type batchRestoreStore struct { + db.Store + restored []string + failID string + failErr error +} + +func (s *batchRestoreStore) RestoreSession(id string) (int64, error) { + s.restored = append(s.restored, id) + if id == s.failID { + return 0, s.failErr + } + return 1, nil +} + +func TestRestoreBatchDeletedSessionsJoinsFailuresAndContinues(t *testing.T) { + cause := errors.New("metadata append failed") + restoreErr := errors.New("restore failed") + store := &batchRestoreStore{failID: "s2", failErr: restoreErr} + srv := &Server{db: store} + + err := srv.restoreBatchDeletedSessions([]string{"s2", "s3"}, cause) + + assert.ErrorIs(t, err, cause) + assert.ErrorIs(t, err, restoreErr) + assert.Equal(t, []string{"s2", "s3"}, store.restored, + "a restoration failure must not prevent later sessions from being restored") +} + +func TestRestoreLifecycleWaitsForFailedRestoreCompensation(t *testing.T) { + const origin = "desk-a1b2c3" + database := dbtest.OpenTestDB(t) + dbtest.SeedSession(t, database, "s1", "alpha") + require.NoError(t, database.SoftDeleteSession("s1")) + recordSoftDeleteReplayState(t, database, origin, "s1") + + firstAppendStarted := make(chan struct{}) + secondAppendStarted := make(chan struct{}) + secondLockAttempted := make(chan struct{}) + releaseFirstAppend := make(chan struct{}) + var appendCalls atomic.Int32 + var lockAttempts atomic.Int32 + srv := &Server{ + db: database, + cfg: config.Config{ArtifactOriginID: origin}, + beforeSessionLifecycleLock: func() { + if lockAttempts.Add(1) == 2 { + close(secondLockAttempted) + } + }, + metadataAppend: func( + _ context.Context, input artifact.MetadataEventInput, + ) error { + if input.Op != artifact.MetadataOpRestore { + return errors.New("unexpected metadata op") + } + switch appendCalls.Add(1) { + case 1: + close(firstAppendStarted) + <-releaseFirstAppend + return errors.New("artifact write failed") + case 2: + close(secondAppendStarted) + return nil + default: + return errors.New("unexpected extra metadata append") + } + }, + } + + firstResult := make(chan error, 1) + go func() { + _, err := srv.humaRestoreSession(context.Background(), &idPathInput{ID: "s1"}) + firstResult <- err + }() + <-firstAppendStarted + + secondResult := make(chan error, 1) + go func() { + _, err := srv.humaRestoreSession(context.Background(), &idPathInput{ID: "s1"}) + secondResult <- err + }() + <-secondLockAttempted + + interleaved := false + select { + case <-secondAppendStarted: + interleaved = true + case <-time.After(500 * time.Millisecond): + } + close(releaseFirstAppend) + firstErr := <-firstResult + secondErr := <-secondResult + + assert.False(t, interleaved, + "a second restore must not publish before the first restore compensates") + require.Error(t, firstErr) + require.NoError(t, secondErr) + assert.Equal(t, int32(2), appendCalls.Load()) + session, err := database.GetSessionFull(context.Background(), "s1") + require.NoError(t, err) + require.NotNil(t, session) + assert.Nil(t, session.DeletedAt, + "the later successful restore must remain visible locally") +} + +func TestPermanentDeleteLifecycleExcludesConcurrentRestore(t *testing.T) { + const origin = "desk-a1b2c3" + database := dbtest.OpenTestDB(t) + dbtest.SeedSession(t, database, "s1", "alpha") + require.NoError(t, database.SoftDeleteSession("s1")) + recordSoftDeleteReplayState(t, database, origin, "s1") + + firstAppendStarted := make(chan struct{}) + secondAppendStarted := make(chan struct{}) + secondLockAttempted := make(chan struct{}) + releaseFirstAppend := make(chan struct{}) + var appendCalls atomic.Int32 + var lockAttempts atomic.Int32 + srv := &Server{ + db: database, + cfg: config.Config{ArtifactOriginID: origin}, + beforeSessionLifecycleLock: func() { + if lockAttempts.Add(1) == 2 { + close(secondLockAttempted) + } + }, + metadataAppend: func( + _ context.Context, input artifact.MetadataEventInput, + ) error { + switch appendCalls.Add(1) { + case 1: + if input.Op != artifact.MetadataOpPurge { + return errors.New("unexpected first metadata op") + } + close(firstAppendStarted) + <-releaseFirstAppend + return nil + case 2: + if input.Op != artifact.MetadataOpRestore { + return errors.New("unexpected second metadata op") + } + close(secondAppendStarted) + return nil + default: + return errors.New("unexpected extra metadata append") + } + }, + } + + purgeResult := make(chan error, 1) + go func() { + _, err := srv.humaPermanentDeleteSession( + context.Background(), &idPathInput{ID: "s1"}, + ) + purgeResult <- err + }() + <-firstAppendStarted + + restoreResult := make(chan error, 1) + go func() { + _, err := srv.humaRestoreSession(context.Background(), &idPathInput{ID: "s1"}) + restoreResult <- err + }() + <-secondLockAttempted + + interleaved := false + select { + case <-secondAppendStarted: + interleaved = true + case <-time.After(500 * time.Millisecond): + } + close(releaseFirstAppend) + purgeErr := <-purgeResult + restoreErr := <-restoreResult + + assert.False(t, interleaved, + "restore must not publish while a purge has reserved the trashed session") + require.NoError(t, purgeErr) + require.Error(t, restoreErr) + assert.Equal(t, int32(1), appendCalls.Load()) + session, err := database.GetSessionFull(context.Background(), "s1") + require.NoError(t, err) + assert.Nil(t, session, + "a durable purge must delete the local session before restore can run") +} + +func TestPermanentDeleteLifecycleExcludesConcurrentPeerRestore(t *testing.T) { + const ( + localOrigin = "desk-a1b2c3" + peerOrigin = "peer-b2c3d4" + ) + database := dbtest.OpenTestDB(t) + dbtest.SeedSession(t, database, "s1", "alpha") + require.NoError(t, database.SoftDeleteSession("s1")) + recordSoftDeleteReplayState(t, database, localOrigin, "s1") + + firstAppendStarted := make(chan struct{}) + secondLockAttempted := make(chan struct{}) + releaseFirstAppend := make(chan struct{}) + var appendCalls atomic.Int32 + var lockAttempts atomic.Int32 + srv := &Server{ + db: database, + cfg: config.Config{ + ArtifactOriginID: localOrigin, + DataDir: t.TempDir(), + }, + beforeSessionLifecycleLock: func() { + if lockAttempts.Add(1) == 2 { + close(secondLockAttempted) + } + }, + metadataAppend: func( + _ context.Context, input artifact.MetadataEventInput, + ) error { + if appendCalls.Add(1) != 1 || input.Op != artifact.MetadataOpPurge { + return errors.New("unexpected metadata append") + } + close(firstAppendStarted) + <-releaseFirstAppend + return nil + }, + } + + purgeResult := make(chan error, 1) + go func() { + _, err := srv.humaPermanentDeleteSession( + context.Background(), &idPathInput{ID: "s1"}, + ) + purgeResult <- err + }() + <-firstAppendStarted + + body, name := peerRestoreArtifact( + peerOrigin, artifact.MetadataSessionGID(localOrigin, "s1"), + ) + peerResult := make(chan error, 1) + go func() { + _, err := srv.humaPostArtifact(context.Background(), &artifactPostInput{ + Origin: peerOrigin, + Kind: artifact.KindMeta, + Name: name, + RawBody: body, + }) + peerResult <- err + }() + + peerCompletedBeforeRelease := false + var peerErr error + select { + case <-secondLockAttempted: + select { + case peerErr = <-peerResult: + peerCompletedBeforeRelease = true + case <-time.After(500 * time.Millisecond): + } + case peerErr = <-peerResult: + peerCompletedBeforeRelease = true + case <-time.After(2 * time.Second): + require.FailNow(t, "peer restore reached neither lifecycle lock nor completion") + } + close(releaseFirstAppend) + purgeErr := <-purgeResult + if !peerCompletedBeforeRelease { + peerErr = <-peerResult + } + + assert.False(t, peerCompletedBeforeRelease, + "peer restore import must wait for the durable purge to delete locally") + require.NoError(t, purgeErr) + require.NoError(t, peerErr) + assert.Equal(t, int32(1), appendCalls.Load()) + session, err := database.GetSessionFull(context.Background(), "s1") + require.NoError(t, err) + assert.Nil(t, session, + "peer restore must not interleave between purge publication and deletion") +} + +func TestArtifactContentPostDefersDatabaseImport(t *testing.T) { + ctx := context.Background() + const peerOrigin = "peer-b2c3d4" + peerDB := dbtest.OpenTestDB(t) + dbtest.SeedSession(t, peerDB, "s1", "alpha") + peerRoot := t.TempDir() + _, err := artifact.Export(ctx, peerDB, peerRoot, peerOrigin) + require.NoError(t, err) + index, err := artifact.ListArtifacts(peerRoot, peerOrigin) + require.NoError(t, err) + require.Len(t, index.Segments, 1) + segment, err := artifact.ReadArtifact( + peerRoot, peerOrigin, artifact.KindSegments, index.Segments[0], + ) + require.NoError(t, err) + + local := dbtest.OpenTestDB(t) + server := &Server{ + db: local, + cfg: config.Config{DataDir: t.TempDir()}, + } + _, err = server.humaPostArtifact(ctx, &artifactPostInput{ + Origin: peerOrigin, + Kind: artifact.KindSegments, + Name: segment.Name, + RawBody: segment.Data, + }) + require.NoError(t, err) + + localOrigin, err := artifact.StoredOrigin(local) + require.NoError(t, err) + assert.Empty(t, localOrigin, + "content-only upload must not trigger a full import or enroll the receiver") +} + +func TestArtifactDependencyPostRetriesDeferredCheckpointImport(t *testing.T) { + ctx := context.Background() + const peerOrigin = "peer-b2c3d4" + peerDB := dbtest.OpenTestDB(t) + dbtest.SeedSession(t, peerDB, "s1", "alpha") + peerRoot := t.TempDir() + _, err := artifact.Export(ctx, peerDB, peerRoot, peerOrigin) + require.NoError(t, err) + index, err := artifact.ListArtifacts(peerRoot, peerOrigin) + require.NoError(t, err) + require.Len(t, index.Checkpoints, 1) + require.Len(t, index.Manifests, 1) + require.Len(t, index.Segments, 1) + + local := dbtest.OpenTestDB(t) + server := &Server{db: local, cfg: config.Config{DataDir: t.TempDir()}} + post := func(kind, name string) { + t.Helper() + art, readErr := artifact.ReadArtifact(peerRoot, peerOrigin, kind, name) + require.NoError(t, readErr) + _, postErr := server.humaPostArtifact(ctx, &artifactPostInput{ + Origin: peerOrigin, Kind: kind, Name: name, RawBody: art.Data, + }) + require.NoError(t, postErr) + } + + post(artifact.KindCheckpoints, index.Checkpoints[0]) + post(artifact.KindManifests, index.Manifests[0]) + got, err := local.GetSession(ctx, peerOrigin+"~s1") + require.NoError(t, err) + assert.Nil(t, got) + + post(artifact.KindSegments, index.Segments[0]) + got, err = local.GetSession(ctx, peerOrigin+"~s1") + require.NoError(t, err) + require.NotNil(t, got) + assert.Equal(t, "alpha", got.Project) +} + +func peerRestoreArtifact(origin, sessionGID string) ([]byte, string) { + const hlc = "2026-07-10T010203.000000001Z-00000000000000000000" + body := []byte(`{"hlc":"` + hlc + `","op":"restore","origin":"` + origin + + `","session_gid":"` + sessionGID + `","v":1}` + "\n") + sum := sha256.Sum256(body) + hash := hex.EncodeToString(sum[:]) + return body, hlc + "-" + hash + ".json" +} + +func recordSoftDeleteReplayState(t *testing.T, database *db.DB, origin, sessionID string) { + t.Helper() + _, err := database.RecordLocalMetadataProjection(context.Background(), db.MetadataProjection{ + EventOrigin: origin, + OrderKey: "0001-soft-delete", + HLC: "0001", + ArtifactHash: "soft-delete", + SessionGID: artifact.MetadataSessionGID(origin, sessionID), + LocalSessionID: sessionID, + Field: "deleted_at", + Op: artifact.MetadataOpSoftDelete, + Value: artifact.MetadataOpSoftDelete, + }) + require.NoError(t, err) +} + +type curationAppendGate struct { + firstStarted chan struct{} + secondStarted chan struct{} + releaseFirst chan struct{} + releaseOnce sync.Once + calls atomic.Int32 + inputs [2]artifact.MetadataEventInput +} + +func newCurationAppendGate() *curationAppendGate { + return &curationAppendGate{ + firstStarted: make(chan struct{}), + secondStarted: make(chan struct{}), + releaseFirst: make(chan struct{}), + } +} + +func (g *curationAppendGate) append( + _ context.Context, input artifact.MetadataEventInput, +) error { + call := g.calls.Add(1) + if call <= int32(len(g.inputs)) { + g.inputs[call-1] = input + } + switch call { + case 1: + close(g.firstStarted) + <-g.releaseFirst + return errors.New("artifact write failed") + case 2: + close(g.secondStarted) + return nil + default: + return errors.New("unexpected extra metadata append") + } +} + +func (g *curationAppendGate) release() { + g.releaseOnce.Do(func() { close(g.releaseFirst) }) +} + +func curationMetadataRecorder(t *testing.T, database *db.DB) *artifact.MetadataRecorder { + t.Helper() + return artifact.NewMetadataRecorder(database, artifact.MetadataRecorderOptions{ + DataDir: t.TempDir(), + Origin: "desk-a1b2c3", + }) +} + +func waitForSecondCurationRequest( + t *testing.T, + secondLockAttempted <-chan struct{}, + secondAppendStarted <-chan struct{}, +) bool { + t.Helper() + select { + case <-secondLockAttempted: + select { + case <-secondAppendStarted: + return true + case <-time.After(500 * time.Millisecond): + return false + } + case <-secondAppendStarted: + return true + case <-time.After(2 * time.Second): + require.FailNow(t, "second curation request reached neither lifecycle lock nor metadata append") + return false + } +} + +func TestRenameLifecycleWaitsForFailedAppendCompensation(t *testing.T) { + database := dbtest.OpenTestDB(t) + original := "original" + dbtest.SeedSession(t, database, "s1", "alpha", func(session *db.Session) { + session.DisplayName = &original + }) + + gate := newCurationAppendGate() + t.Cleanup(gate.release) + secondLockAttempted := make(chan struct{}) + var lockAttempts atomic.Int32 + srv := &Server{ + db: database, + beforeSessionLifecycleLock: func() { + if lockAttempts.Add(1) == 2 { + close(secondLockAttempted) + } + }, + metadataAppend: gate.append, + } + + firstName := "first" + firstResult := make(chan error, 1) + go func() { + _, err := srv.humaRenameSession(context.Background(), &renameSessionInput{ + ID: "s1", + Body: renameRequest{DisplayName: &firstName}, + }) + firstResult <- err + }() + <-gate.firstStarted + + secondName := "second" + secondResult := make(chan error, 1) + go func() { + _, err := srv.humaRenameSession(context.Background(), &renameSessionInput{ + ID: "s1", + Body: renameRequest{DisplayName: &secondName}, + }) + secondResult <- err + }() + + interleaved := waitForSecondCurationRequest( + t, secondLockAttempted, gate.secondStarted, + ) + gate.release() + firstErr := <-firstResult + secondErr := <-secondResult + + assert.False(t, interleaved, + "a later rename must not publish before the failed rename compensates") + require.Error(t, firstErr) + require.NoError(t, secondErr) + assert.Equal(t, int32(2), gate.calls.Load()) + assert.Equal(t, artifact.MetadataOpRename, gate.inputs[0].Op) + assert.Equal(t, artifact.MetadataOpRename, gate.inputs[1].Op) + assert.JSONEq(t, `{"display_name":"second"}`, string(gate.inputs[1].Value)) + session, err := database.GetSession(context.Background(), "s1") + require.NoError(t, err) + require.NotNil(t, session) + require.NotNil(t, session.DisplayName) + assert.Equal(t, "second", *session.DisplayName, + "SQLite must retain the value represented by the later durable rename") +} + +func TestStarLifecycleWaitsForFailedAppendCompensation(t *testing.T) { + database := dbtest.OpenTestDB(t) + dbtest.SeedSession(t, database, "s1", "alpha") + + gate := newCurationAppendGate() + t.Cleanup(gate.release) + secondLockAttempted := make(chan struct{}) + var lockAttempts atomic.Int32 + srv := &Server{ + db: database, + metadata: curationMetadataRecorder(t, database), + beforeSessionLifecycleLock: func() { + if lockAttempts.Add(1) == 2 { + close(secondLockAttempted) + } + }, + metadataAppend: gate.append, + } + + firstResult := make(chan error, 1) + go func() { + _, err := srv.humaStarSession(context.Background(), &idPathInput{ID: "s1"}) + firstResult <- err + }() + <-gate.firstStarted + + secondResult := make(chan error, 1) + go func() { + _, err := srv.humaStarSession(context.Background(), &idPathInput{ID: "s1"}) + secondResult <- err + }() + + interleaved := waitForSecondCurationRequest( + t, secondLockAttempted, gate.secondStarted, + ) + gate.release() + firstErr := <-firstResult + secondErr := <-secondResult + + assert.False(t, interleaved, + "a later star must not publish before the failed star compensates") + require.Error(t, firstErr) + require.NoError(t, secondErr) + assert.Equal(t, int32(2), gate.calls.Load()) + assert.Equal(t, artifact.MetadataOpStar, gate.inputs[0].Op) + assert.Equal(t, artifact.MetadataOpStar, gate.inputs[1].Op) + starred, err := database.ListStarredSessionIDs(context.Background()) + require.NoError(t, err) + assert.Equal(t, []string{"s1"}, starred, + "SQLite must retain the star represented by the later durable event") +} + +func seedCurationPinMessage(t *testing.T, database *db.DB) int64 { + t.Helper() + dbtest.SeedSession(t, database, "s1", "alpha") + message := dbtest.UserMsg("s1", 0, "investigate") + message.SourceUUID = "message-a1b2c3" + dbtest.SeedMessages(t, database, message) + messages, err := database.GetAllMessages(context.Background(), "s1") + require.NoError(t, err) + require.Len(t, messages, 1) + return messages[0].ID +} + +func TestPinLifecycleWaitsForFailedAppendCompensation(t *testing.T) { + database := dbtest.OpenTestDB(t) + messageID := seedCurationPinMessage(t, database) + + gate := newCurationAppendGate() + t.Cleanup(gate.release) + secondLockAttempted := make(chan struct{}) + var lockAttempts atomic.Int32 + srv := &Server{ + db: database, + metadata: curationMetadataRecorder(t, database), + beforeSessionLifecycleLock: func() { + if lockAttempts.Add(1) == 2 { + close(secondLockAttempted) + } + }, + metadataAppend: gate.append, + } + + firstNote := "first" + firstResult := make(chan error, 1) + go func() { + _, err := srv.humaPinMessage(context.Background(), &pinMessageInput{ + ID: "s1", + MessageID: messageID, + Body: pinRequest{Note: &firstNote}, + }) + firstResult <- err + }() + <-gate.firstStarted + + secondNote := "second" + secondResult := make(chan error, 1) + go func() { + _, err := srv.humaPinMessage(context.Background(), &pinMessageInput{ + ID: "s1", + MessageID: messageID, + Body: pinRequest{Note: &secondNote}, + }) + secondResult <- err + }() + + interleaved := waitForSecondCurationRequest( + t, secondLockAttempted, gate.secondStarted, + ) + gate.release() + firstErr := <-firstResult + secondErr := <-secondResult + + assert.False(t, interleaved, + "a later pin must not publish before the failed pin compensates") + require.Error(t, firstErr) + require.NoError(t, secondErr) + assert.Equal(t, int32(2), gate.calls.Load()) + assert.Equal(t, artifact.MetadataOpPin, gate.inputs[0].Op) + assert.Equal(t, artifact.MetadataOpPin, gate.inputs[1].Op) + require.NotNil(t, gate.inputs[1].Pin) + require.NotNil(t, gate.inputs[1].Pin.Note) + assert.Equal(t, "second", *gate.inputs[1].Pin.Note) + pins, err := database.ListPinnedMessages(context.Background(), "s1", "") + require.NoError(t, err) + require.Len(t, pins, 1, + "SQLite must retain the pin represented by the later durable event") + require.NotNil(t, pins[0].Note) + assert.Equal(t, "second", *pins[0].Note) +} + +func assertCurationMutationWaitsForLifecycle( + t *testing.T, + srv *Server, + wantOp string, + mutate func() error, + assertBefore func(), + assertAfter func(), +) { + t.Helper() + lockAttempted := make(chan struct{}) + appended := make(chan artifact.MetadataEventInput, 2) + var lockAttempts atomic.Int32 + srv.beforeSessionLifecycleLock = func() { + if lockAttempts.Add(1) == 1 { + close(lockAttempted) + } + } + srv.metadataAppend = func( + _ context.Context, input artifact.MetadataEventInput, + ) error { + appended <- input + return nil + } + + srv.sessionLifecycleMu.Lock() + locked := true + defer func() { + if locked { + srv.sessionLifecycleMu.Unlock() + } + }() + + result := make(chan error, 1) + go func() { result <- mutate() }() + + completedBeforeLock := false + var mutationErr error + select { + case <-lockAttempted: + case mutationErr = <-result: + completedBeforeLock = true + case <-time.After(2 * time.Second): + require.FailNow(t, "curation mutation reached neither lifecycle lock nor completion") + } + assert.False(t, completedBeforeLock, + "curation mutation must wait for the shared lifecycle boundary") + assertBefore() + + srv.sessionLifecycleMu.Unlock() + locked = false + if !completedBeforeLock { + mutationErr = <-result + } + require.NoError(t, mutationErr) + assertAfter() + assert.Equal(t, int32(1), lockAttempts.Load()) + select { + case input := <-appended: + assert.Equal(t, wantOp, input.Op) + case <-time.After(2 * time.Second): + require.FailNow(t, "curation mutation did not append metadata") + } + select { + case input := <-appended: + assert.Fail(t, "curation mutation appended extra metadata", "op: %s", input.Op) + default: + } +} + +func TestUnstarMutationWaitsForSessionLifecycle(t *testing.T) { + database := dbtest.OpenTestDB(t) + dbtest.SeedSession(t, database, "s1", "alpha") + starred, err := database.StarSession("s1") + require.NoError(t, err) + require.True(t, starred) + srv := &Server{db: database} + + assertStarred := func(want []string) func() { + return func() { + ids, listErr := database.ListStarredSessionIDs(context.Background()) + require.NoError(t, listErr) + assert.ElementsMatch(t, want, ids) + } + } + assertCurationMutationWaitsForLifecycle( + t, + srv, + artifact.MetadataOpUnstar, + func() error { + _, handlerErr := srv.humaUnstarSession( + context.Background(), &idPathInput{ID: "s1"}, + ) + return handlerErr + }, + assertStarred([]string{"s1"}), + assertStarred([]string{}), + ) +} + +func TestBulkStarMutationWaitsForSessionLifecycle(t *testing.T) { + database := dbtest.OpenTestDB(t) + dbtest.SeedSession(t, database, "s1", "alpha") + srv := &Server{db: database} + in := &bulkStarInput{} + in.Body.SessionIDs = []string{"s1"} + + assertStarred := func(want []string) func() { + return func() { + ids, err := database.ListStarredSessionIDs(context.Background()) + require.NoError(t, err) + assert.ElementsMatch(t, want, ids) + } + } + assertCurationMutationWaitsForLifecycle( + t, + srv, + artifact.MetadataOpStar, + func() error { + _, err := srv.humaBulkStar(context.Background(), in) + return err + }, + assertStarred([]string{}), + assertStarred([]string{"s1"}), + ) +} + +func TestBulkStarEmptyInputDoesNotWaitForSessionLifecycle(t *testing.T) { + lockAttempted := make(chan struct{}) + var appendCalls atomic.Int32 + srv := &Server{ + beforeSessionLifecycleLock: func() { close(lockAttempted) }, + metadataAppend: func( + _ context.Context, _ artifact.MetadataEventInput, + ) error { + appendCalls.Add(1) + return nil + }, + } + srv.sessionLifecycleMu.Lock() + locked := true + defer func() { + if locked { + srv.sessionLifecycleMu.Unlock() + } + }() + + result := make(chan error, 1) + go func() { + _, err := srv.humaBulkStar(context.Background(), &bulkStarInput{}) + result <- err + }() + + select { + case err := <-result: + require.NoError(t, err) + case <-lockAttempted: + srv.sessionLifecycleMu.Unlock() + locked = false + <-result + require.FailNow(t, "empty bulk star waited for the lifecycle boundary") + case <-time.After(2 * time.Second): + require.FailNow(t, "empty bulk star did not complete") + } + assert.Equal(t, int32(0), appendCalls.Load()) +} + +func TestUnpinMutationWaitsForSessionLifecycle(t *testing.T) { + database := dbtest.OpenTestDB(t) + messageID := seedCurationPinMessage(t, database) + note := "keep" + _, err := database.PinMessage("s1", messageID, ¬e) + require.NoError(t, err) + srv := &Server{ + db: database, + metadata: curationMetadataRecorder(t, database), + } + + assertPinned := func(want bool) func() { + return func() { + pins, listErr := database.ListPinnedMessages(context.Background(), "s1", "") + require.NoError(t, listErr) + if !want { + assert.Empty(t, pins) + return + } + require.Len(t, pins, 1) + require.NotNil(t, pins[0].Note) + assert.Equal(t, "keep", *pins[0].Note) + } + } + assertCurationMutationWaitsForLifecycle( + t, + srv, + artifact.MetadataOpUnpin, + func() error { + _, handlerErr := srv.humaUnpinMessage(context.Background(), &messagePathInput{ + ID: "s1", + MessageID: messageID, + }) + return handlerErr + }, + assertPinned(true), + assertPinned(false), + ) +} diff --git a/internal/server/huma_routes_pins.go b/internal/server/huma_routes_pins.go index 13eb81ce2..f0cdfec40 100644 --- a/internal/server/huma_routes_pins.go +++ b/internal/server/huma_routes_pins.go @@ -2,8 +2,11 @@ package server import ( "context" + "errors" + "fmt" "net/http" + "go.kenn.io/agentsview/internal/artifact" "go.kenn.io/agentsview/internal/db" ) @@ -63,9 +66,25 @@ func (s *Server) humaListSessionPins( } func (s *Server) humaPinMessage( - _ context.Context, + ctx context.Context, in *pinMessageInput, ) (*createdOutput[pinMessageResponse], error) { + s.lockSessionLifecycle() + defer s.sessionLifecycleMu.Unlock() + + var prior *db.PinnedMessage + var pin *artifact.MetadataPin + if s.metadata != nil { + var err error + prior, err = s.findPinnedMessage(ctx, in.ID, in.MessageID) + if err != nil { + return nil, internalError("pin message prior state", err) + } + pin, err = s.metadataPinForMessage(ctx, in.ID, in.MessageID, in.Body.Note) + if err != nil { + return nil, internalError("pin message metadata lookup", err) + } + } id, err := s.db.PinMessage(in.ID, in.MessageID, in.Body.Note) if err != nil { if handled := handleHumaReadOnly(err); handled != nil { @@ -77,6 +96,20 @@ func (s *Server) humaPinMessage( return nil, apiError(http.StatusBadRequest, "message does not belong to this session") } + if pin != nil { + if err := s.appendMetadataEvent(ctx, artifact.MetadataEventInput{ + SessionID: in.ID, + Op: artifact.MetadataOpPin, + Pin: pin, + }); err != nil { + var publishedErr *artifact.MetadataPublishedError + if errors.As(err, &publishedErr) { + return nil, internalError("pin message metadata event", err) + } + return nil, internalError("pin message metadata event", + s.restorePinState(in.ID, in.MessageID, prior, err)) + } + } return &createdOutput[pinMessageResponse]{ Status: http.StatusCreated, Body: pinMessageResponse{ID: id}, @@ -84,14 +117,82 @@ func (s *Server) humaPinMessage( } func (s *Server) humaUnpinMessage( - _ context.Context, + ctx context.Context, in *messagePathInput, ) (*noContentOutput, error) { + s.lockSessionLifecycle() + defer s.sessionLifecycleMu.Unlock() + + var prior *db.PinnedMessage + var pin *artifact.MetadataPin + if s.metadata != nil { + var err error + prior, err = s.findPinnedMessage(ctx, in.ID, in.MessageID) + if err != nil { + return nil, internalError("unpin message prior state", err) + } + pin, err = s.metadataPinForMessage(ctx, in.ID, in.MessageID, nil) + if err != nil { + return nil, internalError("unpin message metadata lookup", err) + } + } if err := s.db.UnpinMessage(in.ID, in.MessageID); err != nil { if handled := handleHumaReadOnly(err); handled != nil { return nil, handled } return nil, internalError("unpin message", err) } + if pin != nil { + if err := s.appendMetadataEvent(ctx, artifact.MetadataEventInput{ + SessionID: in.ID, + Op: artifact.MetadataOpUnpin, + Pin: pin, + }); err != nil { + var publishedErr *artifact.MetadataPublishedError + if errors.As(err, &publishedErr) { + return nil, internalError("unpin message metadata event", err) + } + return nil, internalError("unpin message metadata event", + s.restorePinState(in.ID, in.MessageID, prior, err)) + } + } return &noContentOutput{Status: http.StatusNoContent}, nil } + +// findPinnedMessage returns the current pinned_messages row for the +// message, or nil when the message is not pinned. +func (s *Server) findPinnedMessage( + ctx context.Context, sessionID string, messageID int64, +) (*db.PinnedMessage, error) { + pins, err := s.db.ListPinnedMessages(ctx, sessionID, "") + if err != nil { + return nil, err + } + for i := range pins { + if pins[i].MessageID == messageID { + return &pins[i], nil + } + } + return nil, nil +} + +// restorePinState puts the pinned_messages row for the message back to +// prior after a pre-publish metadata failure, so local pin state never +// diverges from the durable ledger. It returns baseErr joined with any +// restore failure. +func (s *Server) restorePinState( + sessionID string, messageID int64, prior *db.PinnedMessage, baseErr error, +) error { + if prior != nil { + if _, err := s.db.PinMessage(sessionID, messageID, prior.Note); err != nil { + return errors.Join(baseErr, + fmt.Errorf("restore pin after metadata failure: %w", err)) + } + return baseErr + } + if err := s.db.UnpinMessage(sessionID, messageID); err != nil { + return errors.Join(baseErr, + fmt.Errorf("remove pin after metadata failure: %w", err)) + } + return baseErr +} diff --git a/internal/server/huma_routes_sessions.go b/internal/server/huma_routes_sessions.go index 45240f2ec..500a96803 100644 --- a/internal/server/huma_routes_sessions.go +++ b/internal/server/huma_routes_sessions.go @@ -15,6 +15,7 @@ import ( "time" "github.com/danielgtaylor/huma/v2" + "go.kenn.io/agentsview/internal/artifact" "go.kenn.io/agentsview/internal/db" "go.kenn.io/agentsview/internal/parser" "go.kenn.io/agentsview/internal/service" @@ -34,6 +35,7 @@ func (s *Server) registerSessionRoutes() { get(s, group, "/sessions/{id}/activity", "Get session activity", s.humaGetSessionActivity) get(s, group, "/sessions/{id}/timing", "Get session timing", s.humaSessionTiming) get(s, group, "/sessions/{id}/usage", "Get session usage", s.humaSessionUsage) + get(s, group, "/sessions/{id}/metadata-conflicts", "List session metadata conflicts", s.humaListMetadataConflicts) stream(s, group, http.MethodGet, "/sessions/{id}/watch", "Watch session events", s.humaWatchSession) stream(s, group, http.MethodGet, "/events", "Watch server events", s.humaEvents) raw(s, group, http.MethodGet, "/sessions/{id}/export", "Export session as HTML", s.humaExportSession) @@ -547,6 +549,43 @@ type emptyTrashResponse struct { Deleted int `json:"deleted"` } +type metadataConflictsResponse struct { + Conflicts []db.MetadataConflict `json:"conflicts"` +} + +func (s *Server) humaListMetadataConflicts( + ctx context.Context, + in *idPathInput, +) (*jsonOutput[metadataConflictsResponse], error) { + session, err := s.db.GetSessionFull(ctx, in.ID) + if err != nil { + return nil, internalError("metadata conflict session lookup", err) + } + if session == nil { + return nil, apiError(http.StatusNotFound, "session not found") + } + gids := []string{in.ID} + if localDB, ok := s.db.(*db.DB); ok && !strings.Contains(in.ID, "~") { + origin, err := artifact.StoredOrigin(localDB) + if err != nil { + return nil, internalError("read artifact origin", err) + } + if origin != "" { + gids = append(gids, artifact.MetadataSessionGID(origin, in.ID)) + } + } + conflicts, err := s.db.ListMetadataConflicts(ctx, gids) + if err != nil { + return nil, internalError("list metadata conflicts", err) + } + if conflicts == nil { + conflicts = []db.MetadataConflict{} + } + return &jsonOutput[metadataConflictsResponse]{ + Body: metadataConflictsResponse{Conflicts: conflicts}, + }, nil +} + func (s *Server) humaGetSessionDir( ctx context.Context, in *idPathInput, @@ -657,6 +696,9 @@ func (s *Server) humaRenameSession( ctx context.Context, in *renameSessionInput, ) (*jsonOutput[*db.Session], error) { + s.lockSessionLifecycle() + defer s.sessionLifecycleMu.Unlock() + session, err := s.db.GetSession(ctx, in.ID) if err != nil { return nil, internalError("rename session lookup", err) @@ -674,6 +716,27 @@ func (s *Server) humaRenameSession( } return nil, internalError("rename session", err) } + value, err := renameMetadataValue(displayName) + if err != nil { + return nil, internalError("rename session metadata value", err) + } + if err := s.appendMetadataEvent(ctx, artifact.MetadataEventInput{ + SessionID: in.ID, + Op: artifact.MetadataOpRename, + Value: value, + }); err != nil { + var publishedErr *artifact.MetadataPublishedError + if errors.As(err, &publishedErr) { + return nil, internalError("rename session metadata event", err) + } + if restoreErr := s.db.RenameSession(in.ID, session.DisplayName); restoreErr != nil { + return nil, internalError( + "rename session metadata event", + errors.Join(err, fmt.Errorf("restore display name after metadata failure: %w", restoreErr)), + ) + } + return nil, internalError("rename session metadata event", err) + } updated, err := s.db.GetSession(ctx, in.ID) if err != nil { @@ -689,6 +752,9 @@ func (s *Server) humaDeleteSession( ctx context.Context, in *idPathInput, ) (*noContentOutput, error) { + s.lockSessionLifecycle() + defer s.sessionLifecycleMu.Unlock() + session, err := s.db.GetSessionFull(ctx, in.ID) if err != nil { return nil, internalError("delete session lookup", err) @@ -702,6 +768,32 @@ func (s *Server) humaDeleteSession( } return nil, internalError("soft delete session", err) } + if err := s.appendMetadataEvent(ctx, artifact.MetadataEventInput{ + SessionID: in.ID, + Op: artifact.MetadataOpSoftDelete, + }); err != nil { + var publishedErr *artifact.MetadataPublishedError + if errors.As(err, &publishedErr) { + return nil, internalError("soft delete session metadata event", err) + } + // Only undo a trashing this request performed: SoftDeleteSession + // is a no-op on an already-trashed session, and restoring one of + // those would revert an earlier, ledger-backed deletion. + if session.DeletedAt == nil { + if n, restoreErr := s.db.RestoreSession(in.ID); restoreErr != nil { + return nil, internalError( + "soft delete session metadata event", + errors.Join(err, fmt.Errorf("restore session after metadata failure: %w", restoreErr)), + ) + } else if n == 0 { + return nil, internalError( + "soft delete session metadata event", + errors.Join(err, fmt.Errorf("restore session after metadata failure: session %q not in trash", in.ID)), + ) + } + } + return nil, internalError("soft delete session metadata event", err) + } return &noContentOutput{Status: http.StatusNoContent}, nil } @@ -711,26 +803,107 @@ type batchDeleteInput struct { } } +type trashedSessionIDStore interface { + TrashedSessionIDs(ids []string) ([]string, error) +} + +type excludedSessionStore interface { + IsSessionExcluded(id string) bool +} + func (s *Server) humaBatchDeleteSessions( - _ context.Context, + ctx context.Context, in *batchDeleteInput, ) (*noContentOutput, error) { if len(in.Body.SessionIDs) == 0 { return &noContentOutput{Status: http.StatusNoContent}, nil } - if _, err := s.db.SoftDeleteSessions(in.Body.SessionIDs); err != nil { + s.lockSessionLifecycle() + defer s.sessionLifecycleMu.Unlock() + + deletedIDs, err := s.db.SoftDeleteSessionsReturningIDs(in.Body.SessionIDs) + if err != nil { if handled := handleHumaReadOnly(err); handled != nil { return nil, handled } return nil, internalError("batch delete sessions", err) } + newlyDeleted := make(map[string]struct{}, len(deletedIDs)) + for _, id := range deletedIDs { + newlyDeleted[id] = struct{}{} + } + for i, id := range deletedIDs { + if err := s.appendMetadataEvent(ctx, artifact.MetadataEventInput{ + SessionID: id, + Op: artifact.MetadataOpSoftDelete, + }); err != nil { + rollbackStart := i + var publishedErr *artifact.MetadataPublishedError + if errors.As(err, &publishedErr) { + rollbackStart++ + } + rollback := make([]string, 0, len(deletedIDs)-rollbackStart) + for _, rollbackID := range deletedIDs[rollbackStart:] { + if _, ok := newlyDeleted[rollbackID]; ok { + rollback = append(rollback, rollbackID) + } + } + return nil, internalError( + "batch delete session metadata event", + s.restoreBatchDeletedSessions(rollback, err), + ) + } + } + store, ok := s.db.(trashedSessionIDStore) + if !ok { + return &noContentOutput{Status: http.StatusNoContent}, nil + } + trashedIDs, err := store.TrashedSessionIDs(in.Body.SessionIDs) + if err != nil { + return nil, internalError("batch delete retry lookup", err) + } + for _, id := range trashedIDs { + if _, ok := newlyDeleted[id]; ok { + continue + } + if err := s.ensureLocalMetadataEvent(ctx, artifact.MetadataEventInput{ + SessionID: id, + Op: artifact.MetadataOpSoftDelete, + }, "deleted_at", artifact.MetadataOpSoftDelete); err != nil { + return nil, internalError("batch delete metadata repair", err) + } + } return &noContentOutput{Status: http.StatusNoContent}, nil } +func (s *Server) restoreBatchDeletedSessions(ids []string, cause error) error { + errs := []error{cause} + for _, id := range ids { + n, err := s.db.RestoreSession(id) + if err != nil { + errs = append(errs, + fmt.Errorf("restore session %s after metadata failure: %w", id, err)) + continue + } + if n == 0 { + errs = append(errs, + fmt.Errorf("restore session %s after metadata failure: session not in trash", id)) + } + } + return errors.Join(errs...) +} + func (s *Server) humaRestoreSession( - _ context.Context, + ctx context.Context, in *idPathInput, ) (*noContentOutput, error) { + s.lockSessionLifecycle() + defer s.sessionLifecycleMu.Unlock() + + metadataInput := artifact.MetadataEventInput{ + SessionID: in.ID, + Op: artifact.MetadataOpRestore, + } n, err := s.db.RestoreSession(in.ID) if err != nil { if handled := handleHumaReadOnly(err); handled != nil { @@ -739,25 +912,116 @@ func (s *Server) humaRestoreSession( return nil, internalError("restore session", err) } if n == 0 { + session, err := s.db.GetSessionFull(ctx, in.ID) + if err != nil { + return nil, internalError("restore session retry lookup", err) + } + if session != nil && session.DeletedAt == nil { + op, ok, err := s.metadataReplayStateOp(ctx, in.ID, "deleted_at") + if err != nil { + return nil, internalError("restore session metadata retry lookup", err) + } + if !ok || op != artifact.MetadataOpSoftDelete { + return nil, apiError(http.StatusNotFound, "session not found or not in trash") + } + _, err = s.repairLocalMetadataEvent(ctx, metadataInput) + if err != nil { + return nil, internalError("restore session metadata repair", err) + } + op, ok, err = s.metadataReplayStateOp(ctx, in.ID, "deleted_at") + if err != nil { + return nil, internalError("restore session metadata retry lookup", err) + } + if !ok || op != artifact.MetadataOpRestore { + if err := s.appendMetadataEvent(ctx, metadataInput); err != nil { + return nil, internalError("restore session metadata event", err) + } + } + return &noContentOutput{Status: http.StatusNoContent}, nil + } return nil, apiError(http.StatusNotFound, "session not found or not in trash") } + if err := s.appendMetadataEvent(ctx, metadataInput); err != nil { + var publishedErr *artifact.MetadataPublishedError + if errors.As(err, &publishedErr) { + return nil, internalError("restore session metadata event", err) + } + deletedIDs, rollbackErr := s.db.SoftDeleteSessionsReturningIDs([]string{in.ID}) + if rollbackErr != nil { + return nil, internalError( + "restore session metadata event", + errors.Join(err, fmt.Errorf("return session to trash after metadata failure: %w", rollbackErr)), + ) + } + if len(deletedIDs) != 1 || deletedIDs[0] != in.ID { + return nil, internalError( + "restore session metadata event", + errors.Join(err, fmt.Errorf("return session %q to trash after metadata failure: session no longer visible", in.ID)), + ) + } + return nil, internalError("restore session metadata event", err) + } return &noContentOutput{Status: http.StatusNoContent}, nil } func (s *Server) humaPermanentDeleteSession( - _ context.Context, + ctx context.Context, in *idPathInput, ) (*noContentOutput, error) { + s.lockSessionLifecycle() + defer s.sessionLifecycleMu.Unlock() + + metadataInput := artifact.MetadataEventInput{ + SessionID: in.ID, + Op: artifact.MetadataOpPurge, + } + session, err := s.db.GetSessionFull(ctx, in.ID) + if err != nil { + return nil, internalError("permanent delete session lookup", err) + } + if session == nil { + if store, ok := s.db.(excludedSessionStore); ok && store.IsSessionExcluded(in.ID) { + repaired, err := s.repairLocalMetadataEvent(ctx, metadataInput) + if err != nil { + return nil, internalError("permanent delete session metadata repair", err) + } + if repaired == 0 { + if err := s.appendMetadataEvent(ctx, metadataInput); err != nil { + return nil, internalError("permanent delete session metadata event", err) + } + } + return &noContentOutput{Status: http.StatusNoContent}, nil + } + return nil, apiError(http.StatusConflict, "session not found or not in trash") + } + if session.DeletedAt == nil { + return nil, apiError(http.StatusConflict, "session not found or not in trash") + } + metadataErr := s.ensureLocalMetadataEvent( + ctx, metadataInput, "purge", artifact.MetadataOpPurge, + ) + if metadataErr != nil { + var publishedErr *artifact.MetadataPublishedError + if !errors.As(metadataErr, &publishedErr) { + return nil, internalError("permanent delete session metadata event", metadataErr) + } + } n, err := s.db.DeleteSessionIfTrashed(in.ID) if err != nil { if handled := handleHumaReadOnly(err); handled != nil { return nil, handled } - return nil, internalError("permanent delete session", err) + return nil, internalError( + "permanent delete session", + errors.Join(metadataErr, err), + ) } if n == 0 { return nil, apiError(http.StatusConflict, "session not found or not in trash") } + if metadataErr != nil { + return nil, internalError("permanent delete session metadata event", metadataErr) + } return &noContentOutput{Status: http.StatusNoContent}, nil } @@ -776,6 +1040,9 @@ func (s *Server) humaEmptyTrash( _ context.Context, _ *emptyInput, ) (*jsonOutput[emptyTrashResponse], error) { + s.lockSessionLifecycle() + defer s.sessionLifecycleMu.Unlock() + count, err := s.db.EmptyTrash() if err != nil { if handled := handleHumaReadOnly(err); handled != nil { diff --git a/internal/server/huma_routes_starred.go b/internal/server/huma_routes_starred.go index 1235123b4..3b8284705 100644 --- a/internal/server/huma_routes_starred.go +++ b/internal/server/huma_routes_starred.go @@ -2,7 +2,12 @@ package server import ( "context" + "errors" + "fmt" "net/http" + "slices" + + "go.kenn.io/agentsview/internal/artifact" ) func (s *Server) registerStarredRoutes() { @@ -39,9 +44,23 @@ func (s *Server) humaListStarred( } func (s *Server) humaStarSession( - _ context.Context, + ctx context.Context, in *idPathInput, ) (*noContentOutput, error) { + s.lockSessionLifecycle() + defer s.sessionLifecycleMu.Unlock() + + // Prior state decides rollback: StarSession reports success for an + // already-starred session too, and that star must survive a failed + // metadata append. + wasStarred := false + if s.metadata != nil { + var err error + wasStarred, err = s.sessionStarred(ctx, in.ID) + if err != nil { + return nil, internalError("star session prior state", err) + } + } ok, err := s.db.StarSession(in.ID) if err != nil { if handled := handleHumaReadOnly(err); handled != nil { @@ -52,34 +71,167 @@ func (s *Server) humaStarSession( if !ok { return nil, apiError(http.StatusNotFound, "session not found") } + if err := s.appendMetadataEvent(ctx, artifact.MetadataEventInput{ + SessionID: in.ID, + Op: artifact.MetadataOpStar, + }); err != nil { + var publishedErr *artifact.MetadataPublishedError + if errors.As(err, &publishedErr) { + return nil, internalError("star session metadata event", err) + } + if !wasStarred { + if _, removeErr := s.db.UnstarSession(in.ID); removeErr != nil { + return nil, internalError( + "star session metadata event", + errors.Join(err, fmt.Errorf("remove star after metadata failure: %w", removeErr)), + ) + } + } + return nil, internalError("star session metadata event", err) + } return &noContentOutput{Status: http.StatusNoContent}, nil } +// sessionStarred reports whether the session is currently starred. +func (s *Server) sessionStarred(ctx context.Context, id string) (bool, error) { + ids, err := s.db.ListStarredSessionIDs(ctx) + if err != nil { + return false, err + } + if slices.Contains(ids, id) { + return true, nil + } + return false, nil +} + func (s *Server) humaUnstarSession( - _ context.Context, + ctx context.Context, in *idPathInput, ) (*noContentOutput, error) { - if err := s.db.UnstarSession(in.ID); err != nil { + s.lockSessionLifecycle() + defer s.sessionLifecycleMu.Unlock() + + removed, err := s.db.UnstarSession(in.ID) + if err != nil { if handled := handleHumaReadOnly(err); handled != nil { return nil, handled } return nil, internalError("unstar session", err) } + if !removed { + if _, err := s.repairLocalMetadataEvent(ctx, artifact.MetadataEventInput{ + SessionID: in.ID, + Op: artifact.MetadataOpUnstar, + }); err != nil { + return nil, internalError("unstar session metadata repair", err) + } + return &noContentOutput{Status: http.StatusNoContent}, nil + } + if err := s.appendMetadataEvent(ctx, artifact.MetadataEventInput{ + SessionID: in.ID, + Op: artifact.MetadataOpUnstar, + }); err != nil { + var publishedErr *artifact.MetadataPublishedError + if errors.As(err, &publishedErr) { + return nil, internalError("unstar session metadata event", err) + } + if restored, restoreErr := s.db.StarSession(in.ID); restoreErr != nil { + return nil, internalError( + "unstar session metadata event", + errors.Join(err, fmt.Errorf("restore star after metadata failure: %w", restoreErr)), + ) + } else if !restored { + return nil, internalError( + "unstar session metadata event", + errors.Join(err, fmt.Errorf("restore star after metadata failure: session %q not found", in.ID)), + ) + } + return nil, internalError("unstar session metadata event", err) + } return &noContentOutput{Status: http.StatusNoContent}, nil } func (s *Server) humaBulkStar( - _ context.Context, + ctx context.Context, in *bulkStarInput, ) (*noContentOutput, error) { if len(in.Body.SessionIDs) == 0 { return &noContentOutput{Status: http.StatusNoContent}, nil } - if err := s.db.BulkStarSessions(in.Body.SessionIDs); err != nil { + s.lockSessionLifecycle() + defer s.sessionLifecycleMu.Unlock() + + starred, err := s.db.BulkStarSessions(in.Body.SessionIDs) + if err != nil { if handled := handleHumaReadOnly(err); handled != nil { return nil, handled } return nil, internalError("bulk star", err) } + newlyStarred := make(map[string]struct{}, len(starred)) + // Emit one star event per session actually starred so localStorage star + // migration converges through artifact sync, matching single-session star. + for i, id := range starred { + newlyStarred[id] = struct{}{} + if err := s.appendMetadataEvent(ctx, artifact.MetadataEventInput{ + SessionID: id, + Op: artifact.MetadataOpStar, + }); err != nil { + // Stars whose events are already in the ledger stay; the + // rest were created by this request without a ledger event + // to sync them, so they are removed. A published error + // means the failed event itself is durably recorded, so + // its star stays too. + rollback := starred[i:] + var publishedErr *artifact.MetadataPublishedError + if errors.As(err, &publishedErr) { + rollback = starred[i+1:] + } + return nil, internalError("bulk star metadata event", + s.rollbackBulkStar(rollback, err)) + } + } + starredIDs, err := s.db.ListStarredSessionIDs(ctx) + if err != nil { + return nil, internalError("bulk star metadata repair", err) + } + starredNow := make(map[string]struct{}, len(starredIDs)) + for _, id := range starredIDs { + starredNow[id] = struct{}{} + } + seenRetry := map[string]struct{}{} + for _, id := range in.Body.SessionIDs { + if _, ok := newlyStarred[id]; ok { + continue + } + if _, ok := starredNow[id]; !ok { + continue + } + if _, ok := seenRetry[id]; ok { + continue + } + seenRetry[id] = struct{}{} + if err := s.ensureLocalMetadataEvent(ctx, artifact.MetadataEventInput{ + SessionID: id, + Op: artifact.MetadataOpStar, + }, "starred", artifact.MetadataOpStar); err != nil { + return nil, internalError("bulk star metadata repair", err) + } + } return &noContentOutput{Status: http.StatusNoContent}, nil } + +// rollbackBulkStar removes stars this request created but never +// recorded in the ledger, so local state does not run ahead of the +// artifact log. The returned error joins the append failure with any +// rollback failures. +func (s *Server) rollbackBulkStar(ids []string, cause error) error { + errs := []error{cause} + for _, id := range ids { + if _, err := s.db.UnstarSession(id); err != nil { + errs = append(errs, + fmt.Errorf("remove star %s after metadata failure: %w", id, err)) + } + } + return errors.Join(errs...) +} diff --git a/internal/server/metadata_events.go b/internal/server/metadata_events.go new file mode 100644 index 000000000..e1ff538d6 --- /dev/null +++ b/internal/server/metadata_events.go @@ -0,0 +1,118 @@ +package server + +import ( + "context" + "encoding/json" + "fmt" + + "go.kenn.io/agentsview/internal/artifact" +) + +func (s *Server) appendMetadataEvent( + ctx context.Context, + input artifact.MetadataEventInput, +) error { + if artifact.MetadataEventsSuppressed(ctx) { + return nil + } + if s.metadataAppend != nil { + return s.metadataAppend(ctx, input) + } + if s.metadata == nil { + return nil + } + _, err := s.metadata.Append(ctx, input) + return err +} + +func (s *Server) repairLocalMetadataEvent( + ctx context.Context, + input artifact.MetadataEventInput, +) (int, error) { + if s.metadata == nil { + return 0, nil + } + return s.metadata.RepairLocalSessionMetadata(ctx, input.SessionID, input.Op) +} + +func (s *Server) ensureLocalMetadataEvent( + ctx context.Context, + input artifact.MetadataEventInput, + field string, + wantOp string, +) error { + if s.metadata == nil { + if s.metadataAppend != nil { + return s.appendMetadataEvent(ctx, input) + } + return nil + } + if _, err := s.repairLocalMetadataEvent(ctx, input); err != nil { + return err + } + op, ok, err := s.metadataReplayStateOp(ctx, input.SessionID, field) + if err != nil { + return err + } + if ok && op == wantOp { + return nil + } + return s.appendMetadataEvent(ctx, input) +} + +type metadataReplayStateStore interface { + MetadataReplayStateOp(ctx context.Context, sessionGID string, field string) (string, bool, error) +} + +func (s *Server) metadataReplayStateOp( + ctx context.Context, + sessionID string, + field string, +) (string, bool, error) { + store, ok := s.db.(metadataReplayStateStore) + if !ok { + return "", false, nil + } + origin := s.localArtifactOrigin() + if origin == "" { + return "", false, nil + } + return store.MetadataReplayStateOp(ctx, artifact.MetadataSessionGID(origin, sessionID), field) +} + +func renameMetadataValue(displayName *string) (json.RawMessage, error) { + data, err := json.Marshal(struct { + DisplayName *string `json:"display_name"` + }{DisplayName: displayName}) + if err != nil { + return nil, err + } + return json.RawMessage(data), nil +} + +func (s *Server) metadataPinForMessage( + ctx context.Context, + sessionID string, + messageID int64, + note *string, +) (*artifact.MetadataPin, error) { + msgs, err := s.db.GetAllMessages(ctx, sessionID) + if err != nil { + return nil, fmt.Errorf("loading message for metadata pin: %w", err) + } + for _, msg := range msgs { + if msg.ID != messageID { + continue + } + pin := &artifact.MetadataPin{ + SourceUUID: msg.SourceUUID, + Ordinal: msg.Ordinal, + } + if note != nil { + noteCopy := *note + pin.Note = ¬eCopy + } + return pin, nil + } + return nil, nil +} diff --git a/internal/server/metadata_events_test.go b/internal/server/metadata_events_test.go new file mode 100644 index 000000000..e50aa7268 --- /dev/null +++ b/internal/server/metadata_events_test.go @@ -0,0 +1,992 @@ +package server_test + +import ( + "context" + "database/sql" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "sort" + "strings" + "testing" + + _ "github.com/mattn/go-sqlite3" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "go.kenn.io/agentsview/internal/artifact" + "go.kenn.io/agentsview/internal/config" + "go.kenn.io/agentsview/internal/db" +) + +type recordedMetadataEvent struct { + Version int `json:"v"` + HLC string `json:"hlc"` + Origin string `json:"origin"` + SessionGID string `json:"session_gid"` + Op string `json:"op"` + Value map[string]any `json:"value,omitempty"` + Pin *recordedMetadataPin `json:"pin,omitempty"` +} + +type recordedMetadataPin struct { + SourceUUID string `json:"source_uuid,omitempty"` + Ordinal int `json:"ordinal,omitempty"` + Note *string `json:"note,omitempty"` +} + +func withArtifactOrigin(origin string) setupOption { + return func(c *config.Config) { c.ArtifactOriginID = origin } +} + +func TestMetadataEventsAppendForUserMutations(t *testing.T) { + te := setup(t, withArtifactOrigin("desk-a1b2c3")) + te.seedSession(t, "s1", "alpha", 2) + te.seedMessages(t, "s1", 2, func(i int, m *db.Message) { + if i == 1 { + m.SourceUUID = "uuid-answer" + } + }) + msgs, err := te.db.GetAllMessages(context.Background(), "s1") + require.NoError(t, err) + require.Len(t, msgs, 2) + messageID := msgs[1].ID + + w := te.patch(t, "/api/v1/sessions/s1/rename", `{"display_name":"Pinned investigation"}`) + require.Equal(t, http.StatusOK, w.Code, "body: %s", w.Body.String()) + + w = te.put(t, "/api/v1/sessions/s1/star", `{}`) + require.Equal(t, http.StatusNoContent, w.Code, "body: %s", w.Body.String()) + + w = te.del(t, "/api/v1/sessions/s1/star") + require.Equal(t, http.StatusNoContent, w.Code, "body: %s", w.Body.String()) + + w = te.post(t, fmt.Sprintf("/api/v1/sessions/s1/messages/%d/pin", messageID), `{"note":"remember"}`) + require.Equal(t, http.StatusCreated, w.Code, "body: %s", w.Body.String()) + + w = te.del(t, fmt.Sprintf("/api/v1/sessions/s1/messages/%d/pin", messageID)) + require.Equal(t, http.StatusNoContent, w.Code, "body: %s", w.Body.String()) + + w = te.del(t, "/api/v1/sessions/s1") + require.Equal(t, http.StatusNoContent, w.Code, "body: %s", w.Body.String()) + + w = te.post(t, "/api/v1/sessions/s1/restore", `{}`) + require.Equal(t, http.StatusNoContent, w.Code, "body: %s", w.Body.String()) + + w = te.del(t, "/api/v1/sessions/s1") + require.Equal(t, http.StatusNoContent, w.Code, "body: %s", w.Body.String()) + + w = te.del(t, "/api/v1/sessions/s1/permanent") + require.Equal(t, http.StatusNoContent, w.Code, "body: %s", w.Body.String()) + + events := readMetadataEvents(t, te) + require.Len(t, events, 9) + assert.Equal(t, []string{ + artifact.MetadataOpRename, + artifact.MetadataOpStar, + artifact.MetadataOpUnstar, + artifact.MetadataOpPin, + artifact.MetadataOpUnpin, + artifact.MetadataOpSoftDelete, + artifact.MetadataOpRestore, + artifact.MetadataOpSoftDelete, + artifact.MetadataOpPurge, + }, metadataOps(events)) + for _, event := range events { + assert.Equal(t, 1, event.Version) + assert.NotEmpty(t, event.HLC) + assert.Equal(t, "desk-a1b2c3", event.Origin) + assert.Equal(t, "desk-a1b2c3~s1", event.SessionGID) + } + assert.Equal(t, "Pinned investigation", events[0].Value["display_name"]) + require.NotNil(t, events[3].Pin) + assert.Equal(t, "uuid-answer", events[3].Pin.SourceUUID) + assert.Equal(t, 1, events[3].Pin.Ordinal) + require.NotNil(t, events[3].Pin.Note) + assert.Equal(t, "remember", *events[3].Pin.Note) + require.NotNil(t, events[4].Pin) + assert.Equal(t, "uuid-answer", events[4].Pin.SourceUUID) + assert.Equal(t, 1, events[4].Pin.Ordinal) + assert.Nil(t, events[4].Pin.Note) +} + +func TestMetadataEventsAppendWithoutSyncEngine(t *testing.T) { + te := setupNoSyncMode(t) + require.NoError(t, artifact.AdoptOrigin(te.db, "nosync-a1b2c3")) + te.seedSession(t, "s1", "alpha", 2) + + w := te.patch(t, "/api/v1/sessions/s1/rename", `{"display_name":"No sync title"}`) + require.Equal(t, http.StatusOK, w.Code, "body: %s", w.Body.String()) + + events := readMetadataEvents(t, te) + require.Len(t, events, 1) + assert.Equal(t, artifact.MetadataOpRename, events[0].Op) + assert.Equal(t, "nosync-a1b2c3", events[0].Origin) + assert.Equal(t, "nosync-a1b2c3~s1", events[0].SessionGID) +} + +func TestMetadataEventsNotRecordedWithoutOptIn(t *testing.T) { + te := setup(t) + te.seedSession(t, "s1", "alpha", 2) + + w := te.patch(t, "/api/v1/sessions/s1/rename", `{"display_name":"Local only"}`) + require.Equal(t, http.StatusOK, w.Code, "body: %s", w.Body.String()) + w = te.put(t, "/api/v1/sessions/s1/star", `{}`) + require.Equal(t, http.StatusNoContent, w.Code, "body: %s", w.Body.String()) + + assert.Empty(t, readMetadataEvents(t, te), + "curation must not write ledger events before the machine opts into artifact sync") + origin, err := artifact.StoredOrigin(te.db) + require.NoError(t, err) + assert.Empty(t, origin, + "curation must not mint an artifact origin before opt-in") +} + +func TestMetadataEventsEmptyTrashStaysLocalOnly(t *testing.T) { + te := setup(t, withArtifactOrigin("desk-a1b2c3")) + for _, id := range []string{"s1", "s2"} { + te.seedSession(t, id, "alpha", 2) + w := te.del(t, "/api/v1/sessions/"+id) + require.Equal(t, http.StatusNoContent, w.Code, "delete %s body: %s", id, w.Body.String()) + } + + w := te.del(t, "/api/v1/trash") + require.Equal(t, http.StatusOK, w.Code, "body: %s", w.Body.String()) + + events := readMetadataEvents(t, te) + require.Len(t, events, 2) + assert.Equal(t, []string{ + artifact.MetadataOpSoftDelete, + artifact.MetadataOpSoftDelete, + }, metadataOps(events)) +} + +func TestMetadataEventsBatchDeleteRecordsNewAndUnrecordedTrashedSessions(t *testing.T) { + te := setup(t, withArtifactOrigin("desk-a1b2c3")) + for _, id := range []string{"s1", "s2", "s3"} { + te.seedSession(t, id, "alpha", 2) + } + require.NoError(t, te.db.SoftDeleteSession("s3")) + + w := te.requestJSON(t, http.MethodPost, "/api/v1/sessions/batch-delete", + `{"session_ids":["s1","s2","s3","missing"]}`) + require.Equal(t, http.StatusNoContent, w.Code, "body: %s", w.Body.String()) + + events := readMetadataEvents(t, te) + require.Len(t, events, 3) + assert.Equal(t, []string{ + artifact.MetadataOpSoftDelete, + artifact.MetadataOpSoftDelete, + artifact.MetadataOpSoftDelete, + }, metadataOps(events)) + assert.ElementsMatch(t, []string{ + "desk-a1b2c3~s1", + "desk-a1b2c3~s2", + "desk-a1b2c3~s3", + }, []string{events[0].SessionGID, events[1].SessionGID, events[2].SessionGID}) +} + +func TestMetadataEventsBatchDeleteRetriesAlreadyDeletedSessions(t *testing.T) { + te := setup(t, withArtifactOrigin("desk-a1b2c3")) + for _, id := range []string{"s1", "s2"} { + te.seedSession(t, id, "alpha", 2) + require.NoError(t, te.db.SoftDeleteSession(id)) + } + + w := te.requestJSON(t, http.MethodPost, "/api/v1/sessions/batch-delete", + `{"session_ids":["s1","s2","missing"]}`) + require.Equal(t, http.StatusNoContent, w.Code, "body: %s", w.Body.String()) + + events := readMetadataEvents(t, te) + require.Len(t, events, 2) + assert.Equal(t, []string{ + artifact.MetadataOpSoftDelete, + artifact.MetadataOpSoftDelete, + }, metadataOps(events)) + assert.ElementsMatch(t, []string{ + "desk-a1b2c3~s1", + "desk-a1b2c3~s2", + }, []string{events[0].SessionGID, events[1].SessionGID}) +} + +func TestMetadataEventsBatchDeleteRepairsPublishedFailureOnRetry(t *testing.T) { + te := setup(t, withArtifactOrigin("desk-a1b2c3")) + for _, id := range []string{"s1", "s2"} { + te.seedSession(t, id, "alpha", 2) + } + execTestDDL(t, te, ` +CREATE TRIGGER fail_s1_metadata_replay_state_insert +BEFORE INSERT ON metadata_replay_state +WHEN NEW.session_gid = 'desk-a1b2c3~s1' +BEGIN + SELECT RAISE(FAIL, 'forced s1 metadata replay failure'); +END; +`) + + w := te.requestJSON(t, http.MethodPost, "/api/v1/sessions/batch-delete", + `{"session_ids":["s1","s2"]}`) + require.Equal(t, http.StatusInternalServerError, w.Code, "body: %s", w.Body.String()) + + s1, err := te.db.GetSessionFull(context.Background(), "s1") + require.NoError(t, err) + require.NotNil(t, s1) + assert.NotNil(t, s1.DeletedAt, + "published failure keeps the session whose artifact is durable in trash") + s2, err := te.db.GetSessionFull(context.Background(), "s2") + require.NoError(t, err) + require.NotNil(t, s2) + assert.Nil(t, s2.DeletedAt, + "later unpublished sessions are restored after the failure") + assert.Equal(t, 0, serverMetadataTableCount( + t, te, "metadata_replay_state", + "session_gid = 'desk-a1b2c3~s1' AND field = 'deleted_at'", + )) + firstEvents := readMetadataEvents(t, te) + require.Len(t, firstEvents, 1) + assert.Equal(t, "desk-a1b2c3~s1", firstEvents[0].SessionGID) + + execTestDDL(t, te, `DROP TRIGGER fail_s1_metadata_replay_state_insert`) + w = te.requestJSON(t, http.MethodPost, "/api/v1/sessions/batch-delete", + `{"session_ids":["s1","s2"]}`) + require.Equal(t, http.StatusNoContent, w.Code, "body: %s", w.Body.String()) + + for _, id := range []string{"s1", "s2"} { + sess, getErr := te.db.GetSessionFull(context.Background(), id) + require.NoError(t, getErr) + require.NotNil(t, sess) + assert.NotNil(t, sess.DeletedAt, "retry must trash %s", id) + assert.Equal(t, artifact.MetadataOpSoftDelete, + serverMetadataReplayOp(t, te, "desk-a1b2c3~"+id, "deleted_at")) + } + events := readMetadataEvents(t, te) + require.Len(t, events, 2, + "retry must repair the first artifact instead of publishing a duplicate") + counts := map[string]int{} + for _, event := range events { + counts[event.SessionGID]++ + } + assert.Equal(t, map[string]int{ + "desk-a1b2c3~s1": 1, + "desk-a1b2c3~s2": 1, + }, counts) +} + +func TestMetadataEventsUnstarOnlyRecordsRemovedStars(t *testing.T) { + te := setup(t, withArtifactOrigin("desk-a1b2c3")) + te.seedSession(t, "s1", "alpha", 2) + + w := te.del(t, "/api/v1/sessions/missing/star") + require.Equal(t, http.StatusNoContent, w.Code, "body: %s", w.Body.String()) + w = te.del(t, "/api/v1/sessions/s1/star") + require.Equal(t, http.StatusNoContent, w.Code, "body: %s", w.Body.String()) + assert.Empty(t, readMetadataEvents(t, te)) + + ok, err := te.db.StarSession("s1") + require.NoError(t, err) + require.True(t, ok) + w = te.del(t, "/api/v1/sessions/s1/star") + require.Equal(t, http.StatusNoContent, w.Code, "body: %s", w.Body.String()) + + events := readMetadataEvents(t, te) + require.Len(t, events, 1) + assert.Equal(t, artifact.MetadataOpUnstar, events[0].Op) + assert.Equal(t, "desk-a1b2c3~s1", events[0].SessionGID) +} + +func TestMetadataEventsUnstarRestoresStarWhenArtifactWriteFails(t *testing.T) { + te := setup(t, withArtifactOrigin("desk-a1b2c3")) + te.seedSession(t, "s1", "alpha", 2) + ok, err := te.db.StarSession("s1") + require.NoError(t, err) + require.True(t, ok) + + metaDir := filepath.Join(te.dataDir, "artifacts", "desk-a1b2c3", "meta") + require.NoError(t, os.MkdirAll(filepath.Dir(metaDir), 0o755)) + require.NoError(t, os.WriteFile(metaDir, []byte("not a directory"), 0o644)) + + w := te.del(t, "/api/v1/sessions/s1/star") + require.Equal(t, http.StatusInternalServerError, w.Code, "body: %s", w.Body.String()) + ids, err := te.db.ListStarredSessionIDs(context.Background()) + require.NoError(t, err) + assert.Equal(t, []string{"s1"}, ids) + assert.Equal(t, 0, serverMetadataTableCount(t, te, "metadata_replay_state", "session_gid = 'desk-a1b2c3~s1'")) + assert.Equal(t, 0, serverMetadataTableCount(t, te, "metadata_applied_events", "origin = 'desk-a1b2c3'")) + + require.NoError(t, os.Remove(metaDir)) + w = te.del(t, "/api/v1/sessions/s1/star") + require.Equal(t, http.StatusNoContent, w.Code, "body: %s", w.Body.String()) + ids, err = te.db.ListStarredSessionIDs(context.Background()) + require.NoError(t, err) + assert.Empty(t, ids) + + events := readMetadataEvents(t, te) + require.Len(t, events, 1) + assert.Equal(t, artifact.MetadataOpUnstar, events[0].Op) + assert.Equal(t, "desk-a1b2c3~s1", events[0].SessionGID) +} + +func TestMetadataEventsUnstarDoesNotRestoreStarWhenArtifactPublished(t *testing.T) { + te := setup(t, withArtifactOrigin("desk-a1b2c3")) + te.seedSession(t, "s1", "alpha", 2) + ok, err := te.db.StarSession("s1") + require.NoError(t, err) + require.True(t, ok) + + execTestDDL(t, te, ` +CREATE TRIGGER fail_metadata_replay_state_insert +BEFORE INSERT ON metadata_replay_state +BEGIN + SELECT RAISE(FAIL, 'forced metadata replay failure'); +END; +`) + + w := te.del(t, "/api/v1/sessions/s1/star") + require.Equal(t, http.StatusInternalServerError, w.Code, "body: %s", w.Body.String()) + ids, err := te.db.ListStarredSessionIDs(context.Background()) + require.NoError(t, err) + assert.Empty(t, ids) + assert.Equal(t, 0, serverMetadataTableCount(t, te, "metadata_replay_state", "session_gid = 'desk-a1b2c3~s1'")) + assert.Equal(t, 0, serverMetadataTableCount(t, te, "metadata_applied_events", "origin = 'desk-a1b2c3'")) + + events := readMetadataEvents(t, te) + require.Len(t, events, 1) + assert.Equal(t, artifact.MetadataOpUnstar, events[0].Op) + assert.Equal(t, "desk-a1b2c3~s1", events[0].SessionGID) +} + +func TestMetadataEventsNoopUnstarRepairsPublishedArtifactState(t *testing.T) { + te := setup(t, withArtifactOrigin("desk-a1b2c3")) + te.seedSession(t, "s1", "alpha", 2) + + w := te.put(t, "/api/v1/sessions/s1/star", `{}`) + require.Equal(t, http.StatusNoContent, w.Code, "body: %s", w.Body.String()) + assert.Equal(t, artifact.MetadataOpStar, + serverMetadataReplayOp(t, te, "desk-a1b2c3~s1", "starred")) + + execTestDDL(t, te, ` +CREATE TRIGGER fail_metadata_replay_state_insert +BEFORE INSERT ON metadata_replay_state +BEGIN + SELECT RAISE(FAIL, 'forced metadata replay failure'); +END; +`) + + w = te.del(t, "/api/v1/sessions/s1/star") + require.Equal(t, http.StatusInternalServerError, w.Code, "body: %s", w.Body.String()) + assert.Equal(t, artifact.MetadataOpStar, + serverMetadataReplayOp(t, te, "desk-a1b2c3~s1", "starred")) + unstarOrderKey := metadataEventOrderKey(t, te, artifact.MetadataOpUnstar) + + execTestDDL(t, te, `DROP TRIGGER fail_metadata_replay_state_insert`) + w = te.del(t, "/api/v1/sessions/s1/star") + require.Equal(t, http.StatusNoContent, w.Code, "body: %s", w.Body.String()) + assert.Equal(t, artifact.MetadataOpUnstar, + serverMetadataReplayOp(t, te, "desk-a1b2c3~s1", "starred")) + + remoteHLC, remoteHash := splitMetadataOrderKey(t, unstarOrderKey) + _, err := te.db.ApplyMetadataProjection(context.Background(), db.MetadataProjection{ + EventOrigin: "peer-b2c3d4", + OrderKey: unstarOrderKey, + HLC: remoteHLC, + ArtifactHash: remoteHash, + SessionGID: "desk-a1b2c3~s1", + LocalSessionID: "s1", + Field: "starred", + Op: artifact.MetadataOpStar, + Value: artifact.MetadataOpStar, + }) + require.NoError(t, err) + ids, err := te.db.ListStarredSessionIDs(context.Background()) + require.NoError(t, err) + assert.Empty(t, ids) +} + +func TestMetadataEventsStarRollsBackWhenArtifactWriteFails(t *testing.T) { + te := setup(t, withArtifactOrigin("desk-a1b2c3")) + te.seedSession(t, "s1", "alpha", 2) + metaDir := breakMetadataArtifactDir(t, te) + + w := te.put(t, "/api/v1/sessions/s1/star", `{}`) + require.Equal(t, http.StatusInternalServerError, w.Code, "body: %s", w.Body.String()) + ids, err := te.db.ListStarredSessionIDs(context.Background()) + require.NoError(t, err) + assert.Empty(t, ids, "failed metadata publish must roll back the star") + + require.NoError(t, os.Remove(metaDir)) + w = te.put(t, "/api/v1/sessions/s1/star", `{}`) + require.Equal(t, http.StatusNoContent, w.Code, "body: %s", w.Body.String()) + events := readMetadataEvents(t, te) + require.Len(t, events, 1) + assert.Equal(t, artifact.MetadataOpStar, events[0].Op) + + // A pre-existing star survives a later failed re-star: the failed + // append changed nothing this request needs to undo. + // (breakMetadataArtifactDir also wipes recorded events.) + breakMetadataArtifactDir(t, te) + w = te.put(t, "/api/v1/sessions/s1/star", `{}`) + require.Equal(t, http.StatusInternalServerError, w.Code, "body: %s", w.Body.String()) + ids, err = te.db.ListStarredSessionIDs(context.Background()) + require.NoError(t, err) + assert.Equal(t, []string{"s1"}, ids, + "failed re-star must not remove the pre-existing star") +} + +func TestMetadataEventsStarKeepsStarWhenArtifactPublished(t *testing.T) { + te := setup(t, withArtifactOrigin("desk-a1b2c3")) + te.seedSession(t, "s1", "alpha", 2) + + execTestDDL(t, te, ` +CREATE TRIGGER fail_metadata_replay_state_insert +BEFORE INSERT ON metadata_replay_state +BEGIN + SELECT RAISE(FAIL, 'forced metadata replay failure'); +END; +`) + + w := te.put(t, "/api/v1/sessions/s1/star", `{}`) + require.Equal(t, http.StatusInternalServerError, w.Code, "body: %s", w.Body.String()) + ids, err := te.db.ListStarredSessionIDs(context.Background()) + require.NoError(t, err) + assert.Equal(t, []string{"s1"}, ids, + "published metadata event must keep the local star") + + events := readMetadataEvents(t, te) + require.Len(t, events, 1) + assert.Equal(t, artifact.MetadataOpStar, events[0].Op) +} + +func TestMetadataEventsBulkStarRollsBackWhenArtifactWriteFails(t *testing.T) { + te := setup(t, withArtifactOrigin("desk-a1b2c3")) + te.seedSession(t, "s1", "alpha", 2) + te.seedSession(t, "s2", "alpha", 2) + ok, err := te.db.StarSession("s1") + require.NoError(t, err) + require.True(t, ok) + + breakMetadataArtifactDir(t, te) + w := te.requestJSON(t, http.MethodPost, "/api/v1/starred/bulk", + `{"session_ids":["s1","s2"]}`) + require.Equal(t, http.StatusInternalServerError, w.Code, "body: %s", w.Body.String()) + + ids, err := te.db.ListStarredSessionIDs(context.Background()) + require.NoError(t, err) + assert.Equal(t, []string{"s1"}, ids, + "failed metadata publish must roll back only the stars this request created") + assert.Empty(t, readMetadataEvents(t, te)) +} + +func TestMetadataEventsRenameRestoresNameWhenArtifactWriteFails(t *testing.T) { + te := setup(t, withArtifactOrigin("desk-a1b2c3")) + te.seedSession(t, "s1", "alpha", 2) + w := te.patch(t, "/api/v1/sessions/s1/rename", `{"display_name":"keep"}`) + require.Equal(t, http.StatusOK, w.Code, "body: %s", w.Body.String()) + + breakMetadataArtifactDir(t, te) + w = te.patch(t, "/api/v1/sessions/s1/rename", `{"display_name":"replace"}`) + require.Equal(t, http.StatusInternalServerError, w.Code, "body: %s", w.Body.String()) + + session, err := te.db.GetSession(context.Background(), "s1") + require.NoError(t, err) + require.NotNil(t, session) + require.NotNil(t, session.DisplayName) + assert.Equal(t, "keep", *session.DisplayName, + "failed metadata publish must restore the prior display name") + // breakMetadataArtifactDir also wiped the first rename's event, so + // no events remain after the rolled-back rename. + assert.Empty(t, readMetadataEvents(t, te)) +} + +func TestMetadataEventsDeleteRestoresSessionWhenArtifactWriteFails(t *testing.T) { + te := setup(t, withArtifactOrigin("desk-a1b2c3")) + te.seedSession(t, "s1", "alpha", 2) + metaDir := breakMetadataArtifactDir(t, te) + + w := te.del(t, "/api/v1/sessions/s1") + require.Equal(t, http.StatusInternalServerError, w.Code, "body: %s", w.Body.String()) + session, err := te.db.GetSessionFull(context.Background(), "s1") + require.NoError(t, err) + require.NotNil(t, session) + assert.Nil(t, session.DeletedAt, + "failed metadata publish must restore the soft-deleted session") + + require.NoError(t, os.Remove(metaDir)) + w = te.del(t, "/api/v1/sessions/s1") + require.Equal(t, http.StatusNoContent, w.Code, "body: %s", w.Body.String()) + + events := readMetadataEvents(t, te) + require.Len(t, events, 1) + assert.Equal(t, artifact.MetadataOpSoftDelete, events[0].Op) +} + +func seedPinTestMessage(t *testing.T, te *testEnv) int64 { + t.Helper() + te.seedSession(t, "s1", "alpha", 2) + te.seedMessages(t, "s1", 2) + msgs, err := te.db.GetAllMessages(context.Background(), "s1") + require.NoError(t, err) + require.Len(t, msgs, 2) + return msgs[1].ID +} + +func breakMetadataArtifactDir(t *testing.T, te *testEnv) string { + t.Helper() + metaDir := filepath.Join(te.dataDir, "artifacts", "desk-a1b2c3", "meta") + require.NoError(t, os.RemoveAll(metaDir)) + require.NoError(t, os.MkdirAll(filepath.Dir(metaDir), 0o755)) + require.NoError(t, os.WriteFile(metaDir, []byte("not a directory"), 0o644)) + return metaDir +} + +func TestMetadataEventsPinRollsBackWhenArtifactWriteFails(t *testing.T) { + te := setup(t, withArtifactOrigin("desk-a1b2c3")) + messageID := seedPinTestMessage(t, te) + metaDir := breakMetadataArtifactDir(t, te) + pinPath := fmt.Sprintf("/api/v1/sessions/s1/messages/%d/pin", messageID) + + w := te.post(t, pinPath, `{"note":"remember"}`) + require.Equal(t, http.StatusInternalServerError, w.Code, "body: %s", w.Body.String()) + pins, err := te.db.ListPinnedMessages(context.Background(), "s1", "") + require.NoError(t, err) + assert.Empty(t, pins, "failed metadata publish must roll back the pin") + assert.Equal(t, 0, serverMetadataTableCount(t, te, "metadata_replay_state", "session_gid = 'desk-a1b2c3~s1'")) + assert.Equal(t, 0, serverMetadataTableCount(t, te, "metadata_applied_events", "origin = 'desk-a1b2c3'")) + + require.NoError(t, os.Remove(metaDir)) + w = te.post(t, pinPath, `{"note":"remember"}`) + require.Equal(t, http.StatusCreated, w.Code, "body: %s", w.Body.String()) + pins, err = te.db.ListPinnedMessages(context.Background(), "s1", "") + require.NoError(t, err) + require.Len(t, pins, 1) + + events := readMetadataEvents(t, te) + require.Len(t, events, 1) + assert.Equal(t, artifact.MetadataOpPin, events[0].Op) +} + +func TestMetadataEventsRepinRestoresPriorNoteWhenArtifactWriteFails(t *testing.T) { + te := setup(t, withArtifactOrigin("desk-a1b2c3")) + messageID := seedPinTestMessage(t, te) + pinPath := fmt.Sprintf("/api/v1/sessions/s1/messages/%d/pin", messageID) + + w := te.post(t, pinPath, `{"note":"keep"}`) + require.Equal(t, http.StatusCreated, w.Code, "body: %s", w.Body.String()) + + breakMetadataArtifactDir(t, te) + w = te.post(t, pinPath, `{"note":"replace"}`) + require.Equal(t, http.StatusInternalServerError, w.Code, "body: %s", w.Body.String()) + + pins, err := te.db.ListPinnedMessages(context.Background(), "s1", "") + require.NoError(t, err) + require.Len(t, pins, 1) + require.NotNil(t, pins[0].Note) + assert.Equal(t, "keep", *pins[0].Note, + "failed re-pin must restore the prior note") +} + +func TestMetadataEventsUnpinRestoresPinWhenArtifactWriteFails(t *testing.T) { + te := setup(t, withArtifactOrigin("desk-a1b2c3")) + messageID := seedPinTestMessage(t, te) + pinPath := fmt.Sprintf("/api/v1/sessions/s1/messages/%d/pin", messageID) + + w := te.post(t, pinPath, `{"note":"remember"}`) + require.Equal(t, http.StatusCreated, w.Code, "body: %s", w.Body.String()) + + metaDir := breakMetadataArtifactDir(t, te) + w = te.del(t, pinPath) + require.Equal(t, http.StatusInternalServerError, w.Code, "body: %s", w.Body.String()) + pins, err := te.db.ListPinnedMessages(context.Background(), "s1", "") + require.NoError(t, err) + require.Len(t, pins, 1, "failed metadata publish must restore the pin") + require.NotNil(t, pins[0].Note) + assert.Equal(t, "remember", *pins[0].Note) + + require.NoError(t, os.Remove(metaDir)) + w = te.del(t, pinPath) + require.Equal(t, http.StatusNoContent, w.Code, "body: %s", w.Body.String()) + pins, err = te.db.ListPinnedMessages(context.Background(), "s1", "") + require.NoError(t, err) + assert.Empty(t, pins) + + events := readMetadataEvents(t, te) + require.Len(t, events, 1) + assert.Equal(t, artifact.MetadataOpUnpin, events[0].Op) +} + +func TestMetadataEventsPinKeepsPinWhenArtifactPublished(t *testing.T) { + te := setup(t, withArtifactOrigin("desk-a1b2c3")) + messageID := seedPinTestMessage(t, te) + pinPath := fmt.Sprintf("/api/v1/sessions/s1/messages/%d/pin", messageID) + + execTestDDL(t, te, ` +CREATE TRIGGER fail_metadata_replay_state_insert +BEFORE INSERT ON metadata_replay_state +BEGIN + SELECT RAISE(FAIL, 'forced metadata replay failure'); +END; +`) + + w := te.post(t, pinPath, `{"note":"remember"}`) + require.Equal(t, http.StatusInternalServerError, w.Code, "body: %s", w.Body.String()) + + pins, err := te.db.ListPinnedMessages(context.Background(), "s1", "") + require.NoError(t, err) + require.Len(t, pins, 1, + "published metadata event must keep the local pin") + + events := readMetadataEvents(t, te) + require.Len(t, events, 1) + assert.Equal(t, artifact.MetadataOpPin, events[0].Op) +} + +func TestMetadataEventsPermanentDeleteRetriesExcludedSession(t *testing.T) { + te := setup(t, withArtifactOrigin("desk-a1b2c3")) + te.seedSession(t, "s1", "alpha", 2) + + w := te.del(t, "/api/v1/sessions/s1") + require.Equal(t, http.StatusNoContent, w.Code, "body: %s", w.Body.String()) + + execTestDDL(t, te, ` +CREATE TRIGGER fail_metadata_replay_state_insert +BEFORE INSERT ON metadata_replay_state +BEGIN + SELECT RAISE(FAIL, 'forced metadata replay failure'); +END; +`) + + w = te.del(t, "/api/v1/sessions/s1/permanent") + require.Equal(t, http.StatusInternalServerError, w.Code, "body: %s", w.Body.String()) + got, err := te.db.GetSessionFull(context.Background(), "s1") + require.NoError(t, err) + assert.Nil(t, got) + assert.True(t, te.db.IsSessionExcluded("s1")) + deleteMetadataEventsByOp(t, te, artifact.MetadataOpPurge) + + execTestDDL(t, te, `DROP TRIGGER fail_metadata_replay_state_insert`) + w = te.del(t, "/api/v1/sessions/s1/permanent") + require.Equal(t, http.StatusNoContent, w.Code, "body: %s", w.Body.String()) + + events := readMetadataEvents(t, te) + assert.Equal(t, []string{ + artifact.MetadataOpSoftDelete, + artifact.MetadataOpPurge, + }, metadataOps(events)) + assert.Equal(t, artifact.MetadataOpPurge, + serverMetadataReplayOp(t, te, "desk-a1b2c3~s1", "purge")) +} + +func TestMetadataEventsPermanentDeleteRetainsSessionWhenArtifactWriteFails(t *testing.T) { + te := setup(t, withArtifactOrigin("desk-a1b2c3")) + te.seedSession(t, "s1", "alpha", 2) + + w := te.del(t, "/api/v1/sessions/s1") + require.Equal(t, http.StatusNoContent, w.Code, "body: %s", w.Body.String()) + metaDir := breakMetadataArtifactDir(t, te) + + w = te.del(t, "/api/v1/sessions/s1/permanent") + require.Equal(t, http.StatusInternalServerError, w.Code, "body: %s", w.Body.String()) + session, err := te.db.GetSessionFull(context.Background(), "s1") + require.NoError(t, err) + require.NotNil(t, session, "purge must not remove the only local copy before publication") + assert.NotNil(t, session.DeletedAt) + assert.False(t, te.db.IsSessionExcluded("s1")) + assert.Empty(t, readMetadataEvents(t, te)) + + require.NoError(t, os.Remove(metaDir)) + w = te.del(t, "/api/v1/sessions/s1/permanent") + require.Equal(t, http.StatusNoContent, w.Code, "body: %s", w.Body.String()) + session, err = te.db.GetSessionFull(context.Background(), "s1") + require.NoError(t, err) + assert.Nil(t, session) + assert.True(t, te.db.IsSessionExcluded("s1")) + events := readMetadataEvents(t, te) + require.Len(t, events, 1) + assert.Equal(t, artifact.MetadataOpPurge, events[0].Op) +} + +func TestMetadataEventsRestoreRepairsPublishedArtifactState(t *testing.T) { + te := setup(t, withArtifactOrigin("desk-a1b2c3")) + te.seedSession(t, "s1", "alpha", 2) + + w := te.del(t, "/api/v1/sessions/s1") + require.Equal(t, http.StatusNoContent, w.Code, "body: %s", w.Body.String()) + assert.Equal(t, artifact.MetadataOpSoftDelete, + serverMetadataReplayOp(t, te, "desk-a1b2c3~s1", "deleted_at")) + + execTestDDL(t, te, ` +CREATE TRIGGER fail_metadata_replay_state_insert +BEFORE INSERT ON metadata_replay_state +BEGIN + SELECT RAISE(FAIL, 'forced metadata replay failure'); +END; +`) + + w = te.post(t, "/api/v1/sessions/s1/restore", `{}`) + require.Equal(t, http.StatusInternalServerError, w.Code, "body: %s", w.Body.String()) + restored, err := te.db.GetSessionFull(context.Background(), "s1") + require.NoError(t, err) + require.NotNil(t, restored) + assert.Nil(t, restored.DeletedAt) + assert.Equal(t, artifact.MetadataOpSoftDelete, + serverMetadataReplayOp(t, te, "desk-a1b2c3~s1", "deleted_at")) + restoreOrderKey := metadataEventOrderKey(t, te, artifact.MetadataOpRestore) + + execTestDDL(t, te, `DROP TRIGGER fail_metadata_replay_state_insert`) + w = te.post(t, "/api/v1/sessions/s1/restore", `{}`) + require.Equal(t, http.StatusNoContent, w.Code, "body: %s", w.Body.String()) + assert.Equal(t, artifact.MetadataOpRestore, + serverMetadataReplayOp(t, te, "desk-a1b2c3~s1", "deleted_at")) + + restoreHLC, _ := splitMetadataOrderKey(t, restoreOrderKey) + olderHash := strings.Repeat("0", 64) + _, err = te.db.ApplyMetadataProjection(context.Background(), db.MetadataProjection{ + EventOrigin: "peer-b2c3d4", + OrderKey: restoreHLC + "-" + olderHash, + HLC: restoreHLC, + ArtifactHash: olderHash, + SessionGID: "desk-a1b2c3~s1", + LocalSessionID: "s1", + Field: "deleted_at", + Op: artifact.MetadataOpSoftDelete, + Value: artifact.MetadataOpSoftDelete, + }) + require.NoError(t, err) + restored, err = te.db.GetSessionFull(context.Background(), "s1") + require.NoError(t, err) + require.NotNil(t, restored) + assert.Nil(t, restored.DeletedAt) +} + +func TestMetadataEventsRestoreRetrashesSessionWhenArtifactWriteFails(t *testing.T) { + te := setup(t, withArtifactOrigin("desk-a1b2c3")) + te.seedSession(t, "s1", "alpha", 2) + + w := te.del(t, "/api/v1/sessions/s1") + require.Equal(t, http.StatusNoContent, w.Code, "body: %s", w.Body.String()) + breakMetadataArtifactDir(t, te) + + w = te.post(t, "/api/v1/sessions/s1/restore", `{}`) + require.Equal(t, http.StatusInternalServerError, w.Code, "body: %s", w.Body.String()) + session, err := te.db.GetSessionFull(context.Background(), "s1") + require.NoError(t, err) + require.NotNil(t, session) + assert.NotNil(t, session.DeletedAt, + "failed pre-publication restore must return the session to trash") + assert.Equal(t, artifact.MetadataOpSoftDelete, + serverMetadataReplayOp(t, te, "desk-a1b2c3~s1", "deleted_at")) + assert.Empty(t, readMetadataEvents(t, te)) +} + +func TestMetadataEventsRestoreRetriesWithoutPublishedArtifact(t *testing.T) { + te := setup(t, withArtifactOrigin("desk-a1b2c3")) + te.seedSession(t, "s1", "alpha", 2) + + w := te.del(t, "/api/v1/sessions/s1") + require.Equal(t, http.StatusNoContent, w.Code, "body: %s", w.Body.String()) + + execTestDDL(t, te, ` +CREATE TRIGGER fail_metadata_replay_state_insert +BEFORE INSERT ON metadata_replay_state +BEGIN + SELECT RAISE(FAIL, 'forced metadata replay failure'); +END; +`) + + w = te.post(t, "/api/v1/sessions/s1/restore", `{}`) + require.Equal(t, http.StatusInternalServerError, w.Code, "body: %s", w.Body.String()) + restored, err := te.db.GetSessionFull(context.Background(), "s1") + require.NoError(t, err) + require.NotNil(t, restored) + assert.Nil(t, restored.DeletedAt) + deleteMetadataEventsByOp(t, te, artifact.MetadataOpRestore) + + execTestDDL(t, te, `DROP TRIGGER fail_metadata_replay_state_insert`) + w = te.post(t, "/api/v1/sessions/s1/restore", `{}`) + require.Equal(t, http.StatusNoContent, w.Code, "body: %s", w.Body.String()) + + events := readMetadataEvents(t, te) + assert.Equal(t, []string{ + artifact.MetadataOpSoftDelete, + artifact.MetadataOpRestore, + }, metadataOps(events)) + assert.Equal(t, artifact.MetadataOpRestore, + serverMetadataReplayOp(t, te, "desk-a1b2c3~s1", "deleted_at")) +} + +func TestMetadataEventsRestoreRetryPublishesWhenOlderRestoreLoses(t *testing.T) { + te := setup(t, withArtifactOrigin("desk-a1b2c3")) + te.seedSession(t, "s1", "alpha", 2) + + w := te.del(t, "/api/v1/sessions/s1") + require.Equal(t, http.StatusNoContent, w.Code, "body: %s", w.Body.String()) + w = te.post(t, "/api/v1/sessions/s1/restore", `{}`) + require.Equal(t, http.StatusNoContent, w.Code, "body: %s", w.Body.String()) + olderRestore := metadataEventOrderKey(t, te, artifact.MetadataOpRestore) + + w = te.del(t, "/api/v1/sessions/s1") + require.Equal(t, http.StatusNoContent, w.Code, "body: %s", w.Body.String()) + assert.Equal(t, artifact.MetadataOpSoftDelete, + serverMetadataReplayOp(t, te, "desk-a1b2c3~s1", "deleted_at")) + + execTestDDL(t, te, ` +CREATE TRIGGER fail_metadata_replay_state_insert +BEFORE INSERT ON metadata_replay_state +BEGIN + SELECT RAISE(FAIL, 'forced metadata replay failure'); +END; +`) + + w = te.post(t, "/api/v1/sessions/s1/restore", `{}`) + require.Equal(t, http.StatusInternalServerError, w.Code, "body: %s", w.Body.String()) + assert.Equal(t, artifact.MetadataOpSoftDelete, + serverMetadataReplayOp(t, te, "desk-a1b2c3~s1", "deleted_at")) + for _, key := range metadataEventOrderKeys(t, te, artifact.MetadataOpRestore) { + if key != olderRestore { + deleteMetadataEventOrderKey(t, te, key) + } + } + + execTestDDL(t, te, `DROP TRIGGER fail_metadata_replay_state_insert`) + w = te.post(t, "/api/v1/sessions/s1/restore", `{}`) + require.Equal(t, http.StatusNoContent, w.Code, "body: %s", w.Body.String()) + + assert.Equal(t, artifact.MetadataOpRestore, + serverMetadataReplayOp(t, te, "desk-a1b2c3~s1", "deleted_at")) + assert.Len(t, metadataEventOrderKeys(t, te, artifact.MetadataOpRestore), 2) +} + +func TestMetadataEventsSuppressedDuringReplay(t *testing.T) { + te := setup(t, withArtifactOrigin("desk-a1b2c3")) + te.seedSession(t, "s1", "alpha", 2) + + ctx := artifact.WithMetadataEventSuppression(context.Background()) + req := httptest.NewRequest( + http.MethodPatch, + "/api/v1/sessions/s1/rename", + strings.NewReader(`{"display_name":"Replay name"}`), + ).WithContext(ctx) + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Origin", "http://127.0.0.1:0") + w := httptest.NewRecorder() + te.handler.ServeHTTP(w, req) + require.Equal(t, http.StatusOK, w.Code, "body: %s", w.Body.String()) + + renamed, err := te.db.GetSession(context.Background(), "s1") + require.NoError(t, err) + require.NotNil(t, renamed) + require.NotNil(t, renamed.DisplayName) + assert.Equal(t, "Replay name", *renamed.DisplayName) + assert.Empty(t, readMetadataEvents(t, te)) +} + +// execTestDDL runs schema DDL (failure-injection triggers) on the test +// database through a short-lived write connection. The server's Reader() pool +// opens with mode=ro, so tests cannot install triggers through it. +func execTestDDL(t *testing.T, te *testEnv, stmt string) { + t.Helper() + raw, err := sql.Open("sqlite3", "file:"+te.db.Path()+"?_busy_timeout=5000") + require.NoError(t, err) + defer func() { require.NoError(t, raw.Close()) }() + _, err = raw.Exec(stmt) + require.NoError(t, err) +} + +func readMetadataEvents(t *testing.T, te *testEnv) []recordedMetadataEvent { + t.Helper() + paths, err := filepath.Glob(filepath.Join(te.dataDir, "artifacts", "*", "meta", "*.json")) + require.NoError(t, err) + sort.Strings(paths) + events := make([]recordedMetadataEvent, 0, len(paths)) + for _, path := range paths { + data, err := os.ReadFile(path) + require.NoError(t, err) + var event recordedMetadataEvent + require.NoError(t, json.Unmarshal(data, &event)) + events = append(events, event) + } + return events +} + +func metadataEventOrderKey(t *testing.T, te *testEnv, op string) string { + t.Helper() + keys := metadataEventOrderKeys(t, te, op) + require.NotEmpty(t, keys, "metadata event op %s not found", op) + return keys[0] +} + +func metadataEventOrderKeys(t *testing.T, te *testEnv, op string) []string { + t.Helper() + paths, err := filepath.Glob(filepath.Join(te.dataDir, "artifacts", "*", "meta", "*.json")) + require.NoError(t, err) + sort.Strings(paths) + keys := make([]string, 0) + for _, path := range paths { + data, err := os.ReadFile(path) + require.NoError(t, err) + var event recordedMetadataEvent + require.NoError(t, json.Unmarshal(data, &event)) + if event.Op == op { + keys = append(keys, strings.TrimSuffix(filepath.Base(path), ".json")) + } + } + return keys +} + +func splitMetadataOrderKey(t *testing.T, orderKey string) (string, string) { + t.Helper() + idx := strings.LastIndex(orderKey, "-") + require.NotEqual(t, -1, idx, "order key %q missing hash suffix", orderKey) + return orderKey[:idx], orderKey[idx+1:] +} + +func deleteMetadataEventOrderKey(t *testing.T, te *testEnv, orderKey string) { + t.Helper() + paths, err := filepath.Glob(filepath.Join(te.dataDir, "artifacts", "*", "meta", orderKey+".json")) + require.NoError(t, err) + require.Len(t, paths, 1) + require.NoError(t, os.Remove(paths[0])) +} + +func deleteMetadataEventsByOp(t *testing.T, te *testEnv, op string) { + t.Helper() + paths, err := filepath.Glob(filepath.Join(te.dataDir, "artifacts", "*", "meta", "*.json")) + require.NoError(t, err) + for _, path := range paths { + data, err := os.ReadFile(path) + require.NoError(t, err) + var event recordedMetadataEvent + require.NoError(t, json.Unmarshal(data, &event)) + if event.Op == op { + require.NoError(t, os.Remove(path)) + } + } +} + +func metadataOps(events []recordedMetadataEvent) []string { + ops := make([]string, len(events)) + for i, event := range events { + ops[i] = event.Op + } + return ops +} + +func serverMetadataReplayOp(t *testing.T, te *testEnv, sessionGID, field string) string { + t.Helper() + var op string + err := te.db.Reader().QueryRow( + `SELECT op FROM metadata_replay_state WHERE session_gid = ? AND field = ?`, + sessionGID, field, + ).Scan(&op) + require.NoError(t, err) + return op +} + +func serverMetadataTableCount(t *testing.T, te *testEnv, table, where string) int { + t.Helper() + var count int + err := te.db.Reader().QueryRow("SELECT COUNT(*) FROM " + table + " WHERE " + where).Scan(&count) + require.NoError(t, err) + return count +} diff --git a/internal/server/server.go b/internal/server/server.go index 1a3e2f2ed..3f91bcf2c 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -19,6 +19,7 @@ import ( "github.com/danielgtaylor/huma/v2" "github.com/danielgtaylor/huma/v2/adapters/humago" + "go.kenn.io/agentsview/internal/artifact" "go.kenn.io/agentsview/internal/config" "go.kenn.io/agentsview/internal/db" "go.kenn.io/agentsview/internal/insight" @@ -49,18 +50,23 @@ const ( // Server is the HTTP server that serves the SPA and REST API. type Server struct { - mu gosync.RWMutex - cfg config.Config - db db.Store - engine *sync.Engine - onDemandEngine *sync.Engine - sessions service.SessionService - broadcaster *Broadcaster - mux *http.ServeMux - api huma.API - httpSrv *http.Server - version VersionInfo - dataDir string + mu gosync.RWMutex + sessionLifecycleMu gosync.Mutex + artifactImportPending bool + artifactBaselineDone bool + cfg config.Config + db db.Store + engine *sync.Engine + onDemandEngine *sync.Engine + sessions service.SessionService + broadcaster *Broadcaster + metadata *artifact.MetadataRecorder + metadataAppend func(context.Context, artifact.MetadataEventInput) error + mux *http.ServeMux + api huma.API + httpSrv *http.Server + version VersionInfo + dataDir string // baseCtx, when set, is used as the base context for all // incoming requests. Cancelling it causes SSE handlers to @@ -78,6 +84,9 @@ type Server struct { // handler, used only by tests to guarantee handlers // exceed a short timeout. Zero in production. handlerDelay time.Duration + // beforeSessionLifecycleLock observes lock attempts in concurrency tests. + // Production servers leave it nil. + beforeSessionLifecycleLock func() // updateCheckFn is the function called to check for // updates. Defaults to update.CheckForUpdate; tests @@ -113,6 +122,13 @@ type Server struct { vectorPushSource postgres.VectorPushSource } +func (s *Server) lockSessionLifecycle() { + if s.beforeSessionLifecycleLock != nil { + s.beforeSessionLifecycleLock() + } + s.sessionLifecycleMu.Lock() +} + // New creates a new Server. func New( cfg config.Config, database db.Store, engine *sync.Engine, @@ -157,6 +173,12 @@ func New( spaFS: dist, spaHandler: http.FileServerFS(dist), } + if local, ok := database.(*db.DB); ok && !local.ReadOnly() && cfg.DataDir != "" { + s.metadata = artifact.NewMetadataRecorder(local, artifact.MetadataRecorderOptions{ + DataDir: cfg.DataDir, + Origin: cfg.ArtifactOriginID, + }) + } for _, opt := range opts { opt(s) } diff --git a/internal/server/session_mgmt_test.go b/internal/server/session_mgmt_test.go index ea4512d82..df27bda53 100644 --- a/internal/server/session_mgmt_test.go +++ b/internal/server/session_mgmt_test.go @@ -2,6 +2,7 @@ package server_test import ( "context" + "database/sql" "net/http" "testing" @@ -19,6 +20,10 @@ type emptyTrashHandlerResponse struct { Deleted int `json:"deleted"` } +type metadataConflictsHandlerResponse struct { + Conflicts []db.MetadataConflict `json:"conflicts"` +} + func TestSessionManagementRenameHandler(t *testing.T) { te := setup(t) te.seedSession(t, "s1", "alpha", 2) @@ -94,3 +99,40 @@ func TestSessionManagementEmptyTrashHandler(t *testing.T) { trash := decode[trashHandlerResponse](t, w) assert.Empty(t, trash.Sessions) } + +func TestSessionManagementMetadataConflictsHandler(t *testing.T) { + te := setup(t) + te.seedSession(t, "s1", "alpha", 2) + require.NoError(t, te.db.SetSyncState("artifact_origin_id", "desktop-d4e5f6")) + require.NoError(t, te.db.Update(func(tx *sql.Tx) error { + _, err := tx.Exec( + `INSERT INTO metadata_conflicts + (session_gid, field, winning_order_key, losing_order_key, + winning_origin, losing_origin, winning_op, losing_op, + winning_value, losing_value) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, + "desktop-d4e5f6~s1", + "display_name", + "2026-06-14T01:02:03.000000002Z-00000000000000000000-bbb", + "2026-06-14T01:02:03.000000002Z-00000000000000000000-aaa", + "desktop-d4e5f6", + "laptop-a1b2c3", + "rename", + "rename", + `{"display_name":"Winner"}`, + `{"display_name":"Other"}`, + ) + return err + })) + + w := te.get(t, "/api/v1/sessions/s1/metadata-conflicts") + require.Equal(t, http.StatusOK, w.Code, "body: %s", w.Body.String()) + resp := decode[metadataConflictsHandlerResponse](t, w) + require.Len(t, resp.Conflicts, 1) + assert.Equal(t, "display_name", resp.Conflicts[0].Field) + assert.Equal(t, `{"display_name":"Winner"}`, resp.Conflicts[0].WinningValue) + + w = te.get(t, "/api/v1/sessions/missing/metadata-conflicts") + require.Equal(t, http.StatusNotFound, w.Code, "body: %s", w.Body.String()) + assertErrorResponse(t, w, "session not found") +} diff --git a/internal/sync/engine.go b/internal/sync/engine.go index a5f84df5c..584907df4 100644 --- a/internal/sync/engine.go +++ b/internal/sync/engine.go @@ -1459,6 +1459,29 @@ func (e *Engine) resyncAllLocked( return stats } + // Copy the artifact metadata replay tables so peer metadata + // events keep their durable LWW state across the swap. Losing + // this state would let already-applied peer events replay and + // overwrite newer local curation, so failure aborts the swap + // just like the sync state copy above. + if err := newDB.CopyMetadataReplayFrom(origPath); err != nil { + log.Printf("resync: copy metadata replay state: %v", err) + stats.Aborted = true + stats.Warnings = append(stats.Warnings, + "metadata replay copy failed, aborting swap: "+err.Error(), + ) + newDB.Close() + removeTempDB(tempPath) + restoreSkipCache() + if rerr := origDB.Reopen(); rerr != nil { + log.Printf("resync: recovery reopen: %v", rerr) + } + e.mu.Lock() + e.lastSyncStats = stats + e.mu.Unlock() + return stats + } + // Copy insights into newDB from the quiesced old DB file. tInsights := time.Now() reportResyncPhase( diff --git a/internal/sync/engine_integration_test.go b/internal/sync/engine_integration_test.go index ff26305f0..dbc2515a2 100644 --- a/internal/sync/engine_integration_test.go +++ b/internal/sync/engine_integration_test.go @@ -2245,7 +2245,9 @@ func TestSyncEngineHashSkip(t *testing.T) { different := testjsonl.NewSessionBuilder(). AddClaudeUser(tsZero, "msg2"). String() - os.WriteFile(path, []byte(different), 0o644) + require.NoError(t, os.WriteFile(path, []byte(different), 0o644), "rewrite changed session") + future := time.Unix(0, mtime).Add(time.Second) + require.NoError(t, os.Chtimes(path, future, future), "advance changed file mtime") // Third sync — mtime changed → re-synced runSyncAndAssert(t, env.engine, sync.SyncStats{TotalSessions: 1 + 0, Synced: 1, Skipped: 0}) @@ -4968,6 +4970,8 @@ func TestSyncPathsOpenCodeStorageChildUpdateAdvancesSessionMtime( `{"id":"part-a1","sessionID":"oc-storage-mtime","messageID":"msg-a1","type":"text","text":"updated reply","time":{"created":1704067201000}}`, ), 0o644) require.NoError(t, err, "rewrite part") + childMtime := time.Unix(0, initialMtime).Add(time.Second) + require.NoError(t, os.Chtimes(partPath, childMtime, childMtime), "advance part mtime") err = os.Chtimes( sessionPath, time.Unix(0, sessionMtime), @@ -5742,6 +5746,8 @@ func TestSyncPathsMiMoCodeStorageIgnoresStaleSessionSkipCache(t *testing.T) { t, sessionID, "msg-a1", "part-a1", "rewritten mimo reply", 1704067201000, ) + childMtime := sessionMtime.Add(time.Second) + require.NoError(t, os.Chtimes(partPath, childMtime, childMtime), "advance part mtime") require.NoError(t, os.Chtimes(sessionPath, sessionMtime, sessionMtime), "restore session mtime") @@ -11125,6 +11131,67 @@ func TestResyncAllPreservesPGPushMarkerID(t *testing.T) { assert.Equal(t, "marker-123", got) } +func TestResyncAllPreservesArtifactMetadataState(t *testing.T) { + env := setupTestEnv(t) + ctx := context.Background() + + content := testjsonl.NewSessionBuilder(). + AddClaudeUser(tsEarly, "hello"). + AddClaudeAssistant(tsZeroS5, "hi"). + String() + env.writeClaudeSession(t, "proj", "sess.jsonl", content) + env.engine.SyncAll(ctx, nil) + + artifactState := map[string]string{ + "artifact_origin_id": "laptop-a1b2c3", + "artifact_metadata_hlc": "hlc-42", + "artifact_import:peer-b4c5d6:peer-b4c5d6~sess-9": "hash-imp", + "artifact_export:laptop-a1b2c3:sess.jsonl": "hash-exp", + } + for key, value := range artifactState { + require.NoError(t, env.db.SetSyncState(key, value), + "SetSyncState %s", key) + } + + projection := db.MetadataProjection{ + EventOrigin: "peer-b4c5d6", + OrderKey: "0000000001", + HLC: "hlc-1", + ArtifactHash: "hash-1", + SessionGID: "peer-b4c5d6~sess-9", + LocalSessionID: "peer-b4c5d6~sess-9", + Field: "display_name", + Op: "rename", + Value: `{"display_name":"peer name"}`, + } + _, err := env.db.RecordLocalMetadataProjection(ctx, projection) + require.NoError(t, err, "RecordLocalMetadataProjection") + + stats := env.engine.ResyncAll(ctx, nil) + require.False(t, stats.Aborted, "ResyncAll aborted: %+v", stats) + + for key, want := range artifactState { + got, err := env.db.GetSyncState(key) + require.NoError(t, err, "GetSyncState %s after resync", key) + assert.Equal(t, want, got, + "artifact sync state %s must survive resync", key) + } + + applied, err := env.db.MetadataEventApplied( + ctx, projection.EventOrigin, projection.OrderKey, + ) + require.NoError(t, err, "MetadataEventApplied after resync") + assert.True(t, applied, + "applied peer metadata event must survive resync") + + op, ok, err := env.db.MetadataReplayStateOp( + ctx, projection.SessionGID, projection.Field, + ) + require.NoError(t, err, "MetadataReplayStateOp after resync") + require.True(t, ok, "metadata replay state must survive resync") + assert.Equal(t, "rename", op) +} + func TestOpenCodeExcludedSessionsAreSkipped(t *testing.T) { env := setupSingleAgentTestEnv(t, parser.AgentOpenCode) From 34062c61306f7cea69a0b0972febe50780a205dc Mon Sep 17 00:00:00 2001 From: Wes McKinney Date: Fri, 10 Jul 2026 18:13:40 -0500 Subject: [PATCH 02/10] fix(frontend): refresh session conflict badges Artifact imports can add metadata conflicts while a reader keeps the same session open. Caching conflict results only by session ID prevented live session refreshes from exposing those conflicts until navigation or reload. Keep asynchronous stale-response protection while treating each refreshed session snapshot as an opportunity to reload conflict state. VALID (fixed): #1 -- same-ID session refreshes now refetch metadata conflicts. INVALID (dismissed): none. PEDANTIC (skipped): none. --- .../layout/SessionBreadcrumb.svelte | 4 -- .../layout/SessionBreadcrumb.test.ts | 38 +++++++++++++------ 2 files changed, 26 insertions(+), 16 deletions(-) diff --git a/frontend/src/lib/components/layout/SessionBreadcrumb.svelte b/frontend/src/lib/components/layout/SessionBreadcrumb.svelte index cc5498152..8eff03a3d 100644 --- a/frontend/src/lib/components/layout/SessionBreadcrumb.svelte +++ b/frontend/src/lib/components/layout/SessionBreadcrumb.svelte @@ -69,7 +69,6 @@ let sessionDir = $state(null); let metadataConflicts = $state([]); let conflictsOpen = $state(false); - let conflictFetchId: string | null = null; let conflictRequestSeq = 0; interface Opener { @@ -139,13 +138,10 @@ if (!session) { metadataConflicts = []; conflictsOpen = false; - conflictFetchId = null; conflictRequestSeq++; return; } const id = session.id; - if (id === conflictFetchId) return; - conflictFetchId = id; metadataConflicts = []; conflictsOpen = false; const seq = ++conflictRequestSeq; diff --git a/frontend/src/lib/components/layout/SessionBreadcrumb.test.ts b/frontend/src/lib/components/layout/SessionBreadcrumb.test.ts index feb7c98ef..c3a67566a 100644 --- a/frontend/src/lib/components/layout/SessionBreadcrumb.test.ts +++ b/frontend/src/lib/components/layout/SessionBreadcrumb.test.ts @@ -810,10 +810,27 @@ describe("SessionBreadcrumb", () => { unmount(component); }); - it("does not retry failed metadata conflict fetches until the session changes", async () => { - sessionsService.getApiV1SessionsIdMetadataConflicts.mockRejectedValue( - new Error("boom"), - ); + it("refreshes metadata conflicts when the open session refreshes", async () => { + sessionsService.getApiV1SessionsIdMetadataConflicts + .mockResolvedValueOnce({ conflicts: [] }) + .mockResolvedValueOnce({ + conflicts: [ + { + id: 42, + session_gid: "desk-a1b2c3~run:aaa", + field: "display_name", + winning_order_key: "2026-06-14T01:02:04Z-desk-a1b2c3", + losing_order_key: "2026-06-14T01:02:03Z-lap-b2c3d4", + winning_origin: "desk-a1b2c3", + losing_origin: "lap-b2c3d4", + winning_op: "rename", + losing_op: "rename", + winning_value: '{"display_name":"Current title"}', + losing_value: '{"display_name":"Other title"}', + created_at: "2026-06-14T01:02:05Z", + }, + ], + }); const component = createClassComponent({ component: SessionBreadcrumb, @@ -835,18 +852,15 @@ describe("SessionBreadcrumb", () => { message_count: 3, }), }); - await flushPromises(); - expect( - sessionsService.getApiV1SessionsIdMetadataConflicts, - ).toHaveBeenCalledTimes(1); - - component.$set({ - session: makeSession("claude", { id: "run:bbb" }), + await vi.waitFor(() => { + expect(document.querySelector(".conflict-badge")).toBeTruthy(); }); - await flushPromises(); expect( sessionsService.getApiV1SessionsIdMetadataConflicts, ).toHaveBeenCalledTimes(2); + expect( + sessionsService.getApiV1SessionsIdMetadataConflicts, + ).toHaveBeenLastCalledWith({ id: "run:aaa" }); component.$destroy(); }); From 71e6ebbd7fb864014d377f391e0fa5d677012ef7 Mon Sep 17 00:00:00 2001 From: Wes McKinney Date: Fri, 10 Jul 2026 22:12:02 -0500 Subject: [PATCH 03/10] perf(sync): avoid redundant watch and pin reads Watcher change bursts and startup already synchronize local session data, so repeating full discovery before artifact publication made incremental updates archive-sized. Keep the periodic floor as the coverage path for unwatched roots while preserving signal flushing on every export.\n\nPin metadata publication only needs a message ordinal and source UUID. A backend-parity point lookup prevents one pin or unpin from materializing the full transcript and its tool data while the lifecycle lock is held. --- cmd/agentsview/sync_watch.go | 14 +++++- cmd/agentsview/sync_watch_test.go | 44 ++++++++++++++++ internal/db/messages.go | 20 ++++++++ internal/db/store.go | 1 + internal/duckdb/messages.go | 20 ++++++++ internal/postgres/messages.go | 24 +++++++++ .../huma_routes_metadata_internal_test.go | 50 +++++++++++++++++++ internal/server/metadata_events.go | 27 +++++----- 8 files changed, 183 insertions(+), 17 deletions(-) diff --git a/cmd/agentsview/sync_watch.go b/cmd/agentsview/sync_watch.go index fd085bd89..f70c0d1cf 100644 --- a/cmd/agentsview/sync_watch.go +++ b/cmd/agentsview/sync_watch.go @@ -20,7 +20,7 @@ import ( type artifactFolderPusher struct { appCfg config.Config database *db.DB - engine *syncpkg.Engine + engine artifactWatchSyncer target string origin string token string @@ -36,6 +36,11 @@ type artifactFolderPusher struct { onDataChanged func() } +type artifactWatchSyncer interface { + SyncAll(context.Context, syncpkg.ProgressFunc) syncpkg.SyncStats + FlushSignals() +} + func newArtifactWatchEngine( database *db.DB, appCfg config.Config, ) *syncpkg.Engine { @@ -51,7 +56,12 @@ func (p *artifactFolderPusher) push( ctx context.Context, reason pushReason, ) error { if p.engine != nil { - p.engine.SyncAll(ctx, nil) + // Startup already performed a full sync, and watcher change bursts have + // already applied their targeted paths. Only the periodic floor needs a + // full discovery pass to cover roots that could not be watched. + if reason == reasonInterval { + p.engine.SyncAll(ctx, nil) + } // Export reads session rows outside a sync operation; flush // debounced signal recomputes so manifests carry current signals. p.engine.FlushSignals() diff --git a/cmd/agentsview/sync_watch_test.go b/cmd/agentsview/sync_watch_test.go index ad7fc0c2b..68976a0f8 100644 --- a/cmd/agentsview/sync_watch_test.go +++ b/cmd/agentsview/sync_watch_test.go @@ -12,9 +12,26 @@ import ( "go.kenn.io/agentsview/internal/config" "go.kenn.io/agentsview/internal/db" "go.kenn.io/agentsview/internal/parser" + syncpkg "go.kenn.io/agentsview/internal/sync" "go.kenn.io/agentsview/internal/testjsonl" ) +type countingArtifactWatchSyncer struct { + syncAllCalls int + flushCalls int +} + +func (s *countingArtifactWatchSyncer) SyncAll( + _ context.Context, _ syncpkg.ProgressFunc, +) syncpkg.SyncStats { + s.syncAllCalls++ + return syncpkg.SyncStats{} +} + +func (s *countingArtifactWatchSyncer) FlushSignals() { + s.flushCalls++ +} + func openWatchTestDB(t *testing.T) *db.DB { t.Helper() database, err := db.Open(filepath.Join(t.TempDir(), "test.db")) @@ -104,6 +121,33 @@ func TestArtifactFolderPusherPlumbsInsecurePeerOptIn(t *testing.T) { "watch sync must reach an explicitly allowed plaintext peer") } +func TestArtifactFolderPusherOnlyRunsFullDiscoveryForInterval(t *testing.T) { + dataDir := t.TempDir() + target := t.TempDir() + database := openWatchTestDB(t) + syncer := &countingArtifactWatchSyncer{} + pusher := &artifactFolderPusher{ + appCfg: config.Config{DataDir: dataDir}, + database: database, + engine: syncer, + target: target, + origin: "desk-a1b2c3", + } + + for _, reason := range []pushReason{reasonStartup, reasonChange, reasonShutdown} { + require.NoError(t, pusher.push(context.Background(), reason)) + } + assert.Zero(t, syncer.syncAllCalls, + "startup and watcher-driven pushes already synchronized local files") + assert.Equal(t, 3, syncer.flushCalls, + "every export must flush pending signal recomputes") + + require.NoError(t, pusher.push(context.Background(), reasonInterval)) + assert.Equal(t, 1, syncer.syncAllCalls, + "the periodic floor must discover changes from unwatched roots") + assert.Equal(t, 4, syncer.flushCalls) +} + func TestArtifactWatchEngineHonorsConfiguredCwdPrefixes(t *testing.T) { claudeDir := t.TempDir() projectDir := filepath.Join(claudeDir, "-Users-alice-work") diff --git a/internal/db/messages.go b/internal/db/messages.go index 4aa8acdb6..1b681cf2e 100644 --- a/internal/db/messages.go +++ b/internal/db/messages.go @@ -363,6 +363,26 @@ func (db *DB) GetAllMessages( return msgs, nil } +// GetMessageForMetadataPin returns only the stable fields needed to publish a +// pin metadata event. It deliberately avoids loading message content, tool +// calls, and tool-result events for an otherwise single-row lookup. +func (db *DB) GetMessageForMetadataPin( + ctx context.Context, sessionID string, messageID int64, +) (*Message, error) { + row := db.getReader().QueryRowContext(ctx, ` + SELECT id, session_id, ordinal, COALESCE(source_uuid, '') + FROM messages + WHERE session_id = ? AND id = ?`, sessionID, messageID) + var msg Message + if err := row.Scan(&msg.ID, &msg.SessionID, &msg.Ordinal, &msg.SourceUUID); err != nil { + if errors.Is(err, sql.ErrNoRows) { + return nil, nil + } + return nil, fmt.Errorf("querying message metadata for pin: %w", err) + } + return &msg, nil +} + // EmbeddableUnit is one embedding document: a single embeddable user // message, or a run of contiguous embeddable assistant messages. type EmbeddableUnit struct { diff --git a/internal/db/store.go b/internal/db/store.go index ad24f0d3f..97b36056f 100644 --- a/internal/db/store.go +++ b/internal/db/store.go @@ -40,6 +40,7 @@ type Store interface { GetMessages(ctx context.Context, sessionID string, from, limit int, asc bool) ([]Message, error) GetMessagesWindow(ctx context.Context, sessionID string, w MessageWindow) ([]Message, error) GetAllMessages(ctx context.Context, sessionID string) ([]Message, error) + GetMessageForMetadataPin(ctx context.Context, sessionID string, messageID int64) (*Message, error) GetSessionActivity(ctx context.Context, sessionID string) (*SessionActivityResponse, error) // Timing. diff --git a/internal/duckdb/messages.go b/internal/duckdb/messages.go index dadaf0ea2..f4e700278 100644 --- a/internal/duckdb/messages.go +++ b/internal/duckdb/messages.go @@ -3,6 +3,7 @@ package duckdb import ( "context" "database/sql" + "errors" "fmt" "slices" "strings" @@ -236,6 +237,25 @@ func (s *Store) GetAllMessages(ctx context.Context, sessionID string) ([]db.Mess return msgs, nil } +// GetMessageForMetadataPin returns only the stable message identity fields +// needed for metadata pin events. +func (s *Store) GetMessageForMetadataPin( + ctx context.Context, sessionID string, messageID int64, +) (*db.Message, error) { + row := s.queryRowContext(ctx, ` + SELECT id, session_id, ordinal, COALESCE(source_uuid, '') + FROM messages + WHERE session_id = ? AND id = ?`, sessionID, messageID) + var msg db.Message + if err := row.Scan(&msg.ID, &msg.SessionID, &msg.Ordinal, &msg.SourceUUID); err != nil { + if errors.Is(err, sql.ErrNoRows) { + return nil, nil + } + return nil, fmt.Errorf("querying message metadata for pin: %w", err) + } + return &msg, nil +} + func scanMessages(rows *sql.Rows) ([]db.Message, error) { var msgs []db.Message for rows.Next() { diff --git a/internal/postgres/messages.go b/internal/postgres/messages.go index 481bb8ed9..451500155 100644 --- a/internal/postgres/messages.go +++ b/internal/postgres/messages.go @@ -2,6 +2,8 @@ package postgres import ( "context" + "database/sql" + "errors" "fmt" "slices" "strings" @@ -249,6 +251,28 @@ func (s *Store) GetAllMessages( return msgs, nil } +// GetMessageForMetadataPin returns the stable message identity fields used by +// metadata pin events. PostgreSQL message IDs exposed through the API are +// ordinals, matching PinMessage. +func (s *Store) GetMessageForMetadataPin( + ctx context.Context, sessionID string, messageID int64, +) (*db.Message, error) { + var msg db.Message + err := s.pg.QueryRowContext(ctx, ` + SELECT ordinal, session_id, ordinal, COALESCE(source_uuid, '') + FROM messages + WHERE session_id = $1 AND ordinal = $2`, sessionID, messageID).Scan( + &msg.ID, &msg.SessionID, &msg.Ordinal, &msg.SourceUUID, + ) + if errors.Is(err, sql.ErrNoRows) { + return nil, nil + } + if err != nil { + return nil, fmt.Errorf("querying message metadata for pin: %w", err) + } + return &msg, nil +} + // SearchSession performs ILIKE substring search within a single // session's messages, returning matching ordinals. func (s *Store) SearchSession( diff --git a/internal/server/huma_routes_metadata_internal_test.go b/internal/server/huma_routes_metadata_internal_test.go index 9dc7419ee..e2509c89a 100644 --- a/internal/server/huma_routes_metadata_internal_test.go +++ b/internal/server/huma_routes_metadata_internal_test.go @@ -689,6 +689,56 @@ func seedCurationPinMessage(t *testing.T, database *db.DB) int64 { return messages[0].ID } +type metadataPinPointLookupStore struct { + db.Store + message *db.Message + fullReads int + pointReads int + lookupSessionID string + lookupMessageID int64 +} + +func (s *metadataPinPointLookupStore) GetAllMessages( + _ context.Context, _ string, +) ([]db.Message, error) { + s.fullReads++ + return nil, errors.New("full transcript read must not be used for pin metadata") +} + +func (s *metadataPinPointLookupStore) GetMessageForMetadataPin( + _ context.Context, sessionID string, messageID int64, +) (*db.Message, error) { + s.pointReads++ + s.lookupSessionID = sessionID + s.lookupMessageID = messageID + return s.message, nil +} + +func TestMetadataPinUsesPointMessageLookup(t *testing.T) { + store := &metadataPinPointLookupStore{message: &db.Message{ + ID: 42, + SessionID: "s1", + Ordinal: 7, + SourceUUID: "message-a1b2c3", + }} + srv := &Server{db: store} + note := "remember" + + pin, err := srv.metadataPinForMessage( + context.Background(), "s1", 42, ¬e, + ) + require.NoError(t, err) + require.NotNil(t, pin) + assert.Equal(t, "message-a1b2c3", pin.SourceUUID) + assert.Equal(t, 7, pin.Ordinal) + require.NotNil(t, pin.Note) + assert.Equal(t, "remember", *pin.Note) + assert.Equal(t, 1, store.pointReads) + assert.Zero(t, store.fullReads) + assert.Equal(t, "s1", store.lookupSessionID) + assert.Equal(t, int64(42), store.lookupMessageID) +} + func TestPinLifecycleWaitsForFailedAppendCompensation(t *testing.T) { database := dbtest.OpenTestDB(t) messageID := seedCurationPinMessage(t, database) diff --git a/internal/server/metadata_events.go b/internal/server/metadata_events.go index e1ff538d6..d220e553e 100644 --- a/internal/server/metadata_events.go +++ b/internal/server/metadata_events.go @@ -96,23 +96,20 @@ func (s *Server) metadataPinForMessage( messageID int64, note *string, ) (*artifact.MetadataPin, error) { - msgs, err := s.db.GetAllMessages(ctx, sessionID) + msg, err := s.db.GetMessageForMetadataPin(ctx, sessionID, messageID) if err != nil { return nil, fmt.Errorf("loading message for metadata pin: %w", err) } - for _, msg := range msgs { - if msg.ID != messageID { - continue - } - pin := &artifact.MetadataPin{ - SourceUUID: msg.SourceUUID, - Ordinal: msg.Ordinal, - } - if note != nil { - noteCopy := *note - pin.Note = ¬eCopy - } - return pin, nil + if msg == nil { + return nil, nil + } + pin := &artifact.MetadataPin{ + SourceUUID: msg.SourceUUID, + Ordinal: msg.Ordinal, + } + if note != nil { + noteCopy := *note + pin.Note = ¬eCopy } - return nil, nil + return pin, nil } From c7df24f2fa5877dae5c361ecb0f8d32bb6972954 Mon Sep 17 00:00:00 2001 From: Wes McKinney Date: Fri, 10 Jul 2026 22:16:19 -0500 Subject: [PATCH 04/10] perf(postgres): scope artifact provenance to push candidates Periodic PG pushes previously scanned and allocated every historical artifact-import state before determining whether any sessions needed consideration. Large replicated archives therefore imposed work even on no-change watch ticks.\n\nDerive exact provenance keys only after the candidate window is assembled and resolve them through bounded primary-key batches. Empty candidate windows now avoid provenance I/O entirely. --- internal/artifact/sync.go | 44 ++++++++++++++++++++-------- internal/artifact/sync_test.go | 46 +++++++++++++++++++++++++++++ internal/db/db.go | 53 ++++++++++++++++++++++------------ internal/db/sync_state_test.go | 31 ++++++++++++++++++++ internal/postgres/push.go | 10 ++++--- 5 files changed, 149 insertions(+), 35 deletions(-) create mode 100644 internal/db/sync_state_test.go diff --git a/internal/artifact/sync.go b/internal/artifact/sync.go index 98c76919b..cfd121406 100644 --- a/internal/artifact/sync.go +++ b/internal/artifact/sync.go @@ -357,23 +357,43 @@ func StoredOrigin(database *db.DB) (string, error) { return "", nil } -// ImportedSessionIDs returns the durable session IDs written by artifact -// import. A foreign machine~id shape is shared by other import mechanisms, so -// callers must use this provenance instead of inferring artifact ownership from -// the session row alone. -func ImportedSessionIDs(database *db.DB) (map[string]struct{}, error) { - states, err := database.SyncStatesWithPrefix(importStatePrefix) +type syncStateValueReader interface { + SyncStateValues(keys []string) (map[string]string, error) +} + +// ImportedSessionIDs returns the candidate session IDs with durable artifact +// import provenance. A foreign machine~id shape is shared by other import +// mechanisms, so callers must query the exact provenance keys rather than +// infer artifact ownership from the session row or scan all historical imports. +func ImportedSessionIDs( + database syncStateValueReader, candidateIDs []string, +) (map[string]struct{}, error) { + ids := make(map[string]struct{}) + if len(candidateIDs) == 0 { + return ids, nil + } + keys := make([]string, 0, len(candidateIDs)) + keyToID := make(map[string]string, len(candidateIDs)) + for _, gid := range candidateIDs { + origin, nativeID, ok := strings.Cut(gid, "~") + if !ok || origin == "" || nativeID == "" { + continue + } + key := importStateKey(origin, gid) + keys = append(keys, key) + keyToID[key] = gid + } + if len(keys) == 0 { + return ids, nil + } + states, err := database.SyncStateValues(keys) if err != nil { return nil, fmt.Errorf("reading artifact import provenance: %w", err) } - ids := make(map[string]struct{}, len(states)) for key := range states { - rest := strings.TrimPrefix(key, importStatePrefix) - origin, gid, ok := strings.Cut(rest, ":") - if !ok || origin == "" || !strings.HasPrefix(gid, origin+"~") { - continue + if gid, ok := keyToID[key]; ok { + ids[gid] = struct{}{} } - ids[gid] = struct{}{} } return ids, nil } diff --git a/internal/artifact/sync_test.go b/internal/artifact/sync_test.go index 36fb1cac6..4c02f03a0 100644 --- a/internal/artifact/sync_test.go +++ b/internal/artifact/sync_test.go @@ -29,6 +29,52 @@ func TestEnsureOriginPersists(t *testing.T) { assert.Equal(t, first, second) } +type recordingSyncStateValueReader struct { + states map[string]string + keys []string + calls int +} + +func (r *recordingSyncStateValueReader) SyncStateValues( + keys []string, +) (map[string]string, error) { + r.calls++ + r.keys = append([]string(nil), keys...) + result := make(map[string]string) + for _, key := range keys { + if value := r.states[key]; value != "" { + result[key] = value + } + } + return result, nil +} + +func TestImportedSessionIDsReadsOnlyCandidateProvenance(t *testing.T) { + reader := &recordingSyncStateValueReader{states: map[string]string{ + "artifact_import:desk-a1b2c3:desk-a1b2c3~one": "manifest-one", + "artifact_import:laptop-d4e5f6:laptop-d4e5f6~two": "manifest-two", + }} + + got, err := ImportedSessionIDs(reader, []string{ + "desk-a1b2c3~one", + "local-session", + "phone-112233~missing", + }) + require.NoError(t, err) + assert.Equal(t, map[string]struct{}{"desk-a1b2c3~one": {}}, got) + assert.Equal(t, []string{ + "artifact_import:desk-a1b2c3:desk-a1b2c3~one", + "artifact_import:phone-112233:phone-112233~missing", + }, reader.keys) + assert.Equal(t, 1, reader.calls) + + empty, err := ImportedSessionIDs(reader, nil) + require.NoError(t, err) + assert.Empty(t, empty) + assert.Equal(t, 1, reader.calls, + "a no-candidate push must not query artifact provenance") +} + func TestIsFolderTargetAcceptsWindowsDrivePaths(t *testing.T) { tests := []struct { name string diff --git a/internal/db/db.go b/internal/db/db.go index 13b94d5e1..466628c3d 100644 --- a/internal/db/db.go +++ b/internal/db/db.go @@ -2926,28 +2926,43 @@ func (db *DB) GetSyncState(key string) (string, error) { return value, err } -// SyncStatesWithPrefix reads all non-empty sync-state entries whose keys start -// with prefix in one query. -func (db *DB) SyncStatesWithPrefix(prefix string) (map[string]string, error) { - rows, err := db.getReader().Query( - `SELECT key, value FROM pg_sync_state - WHERE substr(key, 1, length(?)) = ? AND value <> ''`, - prefix, prefix, - ) - if err != nil { - return nil, err - } - defer rows.Close() +// SyncStateValues reads the non-empty values for exact sync-state keys. Queries +// are bounded so callers can bulk-resolve state without exceeding SQLite's +// historical variable limit. +func (db *DB) SyncStateValues(keys []string) (map[string]string, error) { states := map[string]string{} - for rows.Next() { - var key, value string - if err := rows.Scan(&key, &value); err != nil { + const batchSize = 900 + for start := 0; start < len(keys); start += batchSize { + end := min(start+batchSize, len(keys)) + batch := keys[start:end] + placeholders := strings.TrimSuffix(strings.Repeat("?,", len(batch)), ",") + args := make([]any, len(batch)) + for i, key := range batch { + args[i] = key + } + rows, err := db.getReader().Query( + `SELECT key, value FROM pg_sync_state + WHERE key IN (`+placeholders+`) AND value <> ''`, + args..., + ) + if err != nil { + return nil, err + } + for rows.Next() { + var key, value string + if err := rows.Scan(&key, &value); err != nil { + rows.Close() + return nil, err + } + states[key] = value + } + if err := rows.Err(); err != nil { + rows.Close() + return nil, err + } + if err := rows.Close(); err != nil { return nil, err } - states[key] = value - } - if err := rows.Err(); err != nil { - return nil, err } return states, nil } diff --git a/internal/db/sync_state_test.go b/internal/db/sync_state_test.go new file mode 100644 index 000000000..8e04ebc75 --- /dev/null +++ b/internal/db/sync_state_test.go @@ -0,0 +1,31 @@ +package db + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestSyncStateValuesReadsOnlyExactKeysAcrossBatches(t *testing.T) { + database := testDB(t) + for i := range 905 { + key := fmt.Sprintf("artifact_import:desk:desk~%04d", i) + require.NoError(t, database.SetSyncState(key, fmt.Sprintf("hash-%04d", i))) + } + require.NoError(t, database.SetSyncState("unrelated", "keep-out")) + + got, err := database.SyncStateValues([]string{ + "artifact_import:desk:desk~0000", + "artifact_import:desk:desk~0899", + "artifact_import:desk:desk~0904", + "missing", + }) + require.NoError(t, err) + assert.Equal(t, map[string]string{ + "artifact_import:desk:desk~0000": "hash-0000", + "artifact_import:desk:desk~0899": "hash-0899", + "artifact_import:desk:desk~0904": "hash-0904", + }, got) +} diff --git a/internal/postgres/push.go b/internal/postgres/push.go index c532bec44..ec6761742 100644 --- a/internal/postgres/push.go +++ b/internal/postgres/push.go @@ -176,10 +176,6 @@ func (s *Sync) Push( if err != nil { return result, err } - artifactImportedSessions, err := artifact.ImportedSessionIDs(s.local) - if err != nil { - return result, err - } artifactIdentityMode := currentArtifactIdentityMode(localArtifactOrigin) storedArtifactIdentityMode, err := state.GetSyncState( artifactIdentityModeStateKey, @@ -352,6 +348,12 @@ func (s *Sync) Push( } } + artifactImportedSessions, err := artifact.ImportedSessionIDs( + s.local, mapKeys(sessionByID), + ) + if err != nil { + return result, err + } for id, sess := range sessionByID { _, artifactImported := artifactImportedSessions[sess.ID] identity, err := s.resolvePushedSessionIdentity( From a200adc36e4d8ef90643f06a60f91f2b1f2a0f76 Mon Sep 17 00:00:00 2001 From: Wes McKinney Date: Fri, 10 Jul 2026 22:17:36 -0500 Subject: [PATCH 05/10] perf(postgres): index artifact relationship rewrites Importer-first artifact convergence rewrites reverse references before deleting a legacy duplicate. Without reverse indexes, every duplicate forced full scans of sessions, tool calls, and tool-result events inside the push transaction.\n\nCreate the indexes after column migrations so both new and legacy schemas can accelerate consolidation safely. --- internal/postgres/schema.go | 8 ++++++++ internal/postgres/schema_test.go | 6 ++++++ 2 files changed, 14 insertions(+) diff --git a/internal/postgres/schema.go b/internal/postgres/schema.go index a30cfc428..16ba6d60c 100644 --- a/internal/postgres/schema.go +++ b/internal/postgres/schema.go @@ -871,6 +871,8 @@ func createPartialIndexesPG(ctx context.Context, db *sql.DB) error { ON sessions(cwd) WHERE cwd != ''`, `CREATE INDEX IF NOT EXISTS idx_sessions_project_git_branch ON sessions(project, git_branch) WHERE git_branch != ''`, + `CREATE INDEX IF NOT EXISTS idx_sessions_source_session + ON sessions(source_session_id) WHERE source_session_id != ''`, `CREATE INDEX IF NOT EXISTS idx_messages_compact_boundary ON messages(session_id, ordinal) WHERE is_compact_boundary = TRUE`, `CREATE INDEX IF NOT EXISTS idx_messages_sidechain @@ -893,6 +895,12 @@ func createPartialIndexesPG(ctx context.Context, db *sql.DB) error { // SQLite partial index so legacy schemas migrate cleanly. `CREATE INDEX IF NOT EXISTS idx_tool_calls_file_path ON tool_calls(file_path) WHERE file_path IS NOT NULL`, + `CREATE INDEX IF NOT EXISTS idx_tool_calls_subagent_session + ON tool_calls(subagent_session_id) + WHERE subagent_session_id IS NOT NULL`, + `CREATE INDEX IF NOT EXISTS idx_tool_result_events_subagent_session + ON tool_result_events(subagent_session_id) + WHERE subagent_session_id IS NOT NULL`, // idx_messages_session_role backs the dense-flow unit-range boundary // fetch (user ordinals by session), mirroring the SQLite index. `CREATE INDEX IF NOT EXISTS idx_messages_session_role diff --git a/internal/postgres/schema_test.go b/internal/postgres/schema_test.go index 2bbe54789..89193cc75 100644 --- a/internal/postgres/schema_test.go +++ b/internal/postgres/schema_test.go @@ -830,6 +830,12 @@ func TestEnsureSchemaCreatesSessionTraversalIndex(t *testing.T) { assert.Contains(t, state.executedSQL(), "CREATE INDEX IF NOT EXISTS idx_sessions_parent") + assert.Contains(t, state.executedSQL(), + "CREATE INDEX IF NOT EXISTS idx_sessions_source_session") + assert.Contains(t, state.executedSQL(), + "CREATE INDEX IF NOT EXISTS idx_tool_calls_subagent_session") + assert.Contains(t, state.executedSQL(), + "CREATE INDEX IF NOT EXISTS idx_tool_result_events_subagent_session") } func TestEnsureSchemaGroupsMissingColumnMigrationsByTable(t *testing.T) { From 7fe9180f20e4e333be74ca025d86a799a224f791 Mon Sep 17 00:00:00 2001 From: Wes McKinney Date: Fri, 10 Jul 2026 22:20:49 -0500 Subject: [PATCH 06/10] perf(postgres): bulk cache session ownership Identity preparation previously issued serialized owner lookups for each candidate and repeated many of them during alias and relationship resolution. The latency dominated full pushes to remote PostgreSQL even when later fingerprint comparison skipped the session.\n\nPreload every candidate, legacy-prefix, canonical, and alias owner in one array query. Cache both rows and absences for the full push so out-of-window relationship lookups are queried at most once. --- internal/postgres/push.go | 168 +++++++++++++++++++++++++++++---- internal/postgres/push_test.go | 94 ++++++++++++++++++ 2 files changed, 246 insertions(+), 16 deletions(-) diff --git a/internal/postgres/push.go b/internal/postgres/push.go index ec6761742..11f7075fe 100644 --- a/internal/postgres/push.go +++ b/internal/postgres/push.go @@ -14,6 +14,7 @@ import ( "slices" "sort" "strings" + "sync" "time" "go.kenn.io/agentsview/internal/artifact" @@ -354,6 +355,13 @@ func (s *Sync) Push( if err != nil { return result, err } + ctx, err = s.preloadPGSessionOwners(ctx, s.pushIdentityOwnerCandidateIDs( + sessionByID, artifactImportedSessions, localArtifactOrigin, + markerID, legacyMarkerMachines, + )) + if err != nil { + return result, err + } for id, sess := range sessionByID { _, artifactImported := artifactImportedSessions[sess.ID] identity, err := s.resolvePushedSessionIdentity( @@ -1855,6 +1863,31 @@ func (s *Sync) markRelationshipConflicts( return nil } +func initialPushedSessionIdentity( + sess db.Session, + fallbackMachine string, + localArtifactOrigin string, + artifactImported bool, + markerID string, +) (pushedSessionIdentity, bool) { + identity := pushedSessionIdentity{ + ID: sess.ID, + Machine: pushedSessionMachine(sess, fallbackMachine), + } + if id, machine, ownerMarker, ok := artifactPushIdentity( + sess, localArtifactOrigin, artifactImported, + ); ok { + identity.ID = id + identity.Machine = machine + identity.OwnerMarker = ownerMarker + identity.LegacyOwnerMarkers = []string{markerID} + identity.ArtifactReplica = artifactImported + identity.AliasIDs = artifactPushAliasIDs(sess, id, machine) + return identity, true + } + return identity, false +} + // resolvePushedSessionIdentity decides the PG id a local session is stored // under. A session this sync owns -- by matching push marker, or an adoptable // legacy/ownerless row (see sameSessionOwner) -- is updated in place: an @@ -1873,22 +1906,9 @@ func (s *Sync) resolvePushedSessionIdentity( markerID string, legacyMarkerMachines []string, ) (pushedSessionIdentity, error) { - identity := pushedSessionIdentity{ - ID: sess.ID, - Machine: pushedSessionMachine(sess, s.machine), - } - artifactIdentity := false - if id, machine, ownerMarker, ok := artifactPushIdentity( - sess, localArtifactOrigin, artifactImported, - ); ok { - artifactIdentity = true - identity.ID = id - identity.Machine = machine - identity.OwnerMarker = ownerMarker - identity.LegacyOwnerMarkers = []string{markerID} - identity.ArtifactReplica = artifactImported - identity.AliasIDs = artifactPushAliasIDs(sess, id, machine) - } + identity, artifactIdentity := initialPushedSessionIdentity( + sess, s.machine, localArtifactOrigin, artifactImported, markerID, + ) canonicalID := identity.ID id, err := s.resolveOwnedPushIdentityID( ctx, identity.ID, identity, markerID, legacyMarkerMachines, @@ -1920,6 +1940,42 @@ func (s *Sync) resolvePushedSessionIdentity( return identity, nil } +func (s *Sync) pushIdentityOwnerCandidateIDs( + sessionByID map[string]db.Session, + artifactImportedSessions map[string]struct{}, + localArtifactOrigin string, + markerID string, + legacyMarkerMachines []string, +) []string { + ids := make(map[string]struct{}, len(sessionByID)*3) + for _, sess := range sessionByID { + _, artifactImported := artifactImportedSessions[sess.ID] + identity, _ := initialPushedSessionIdentity( + sess, s.machine, localArtifactOrigin, artifactImported, markerID, + ) + ids[identity.ID] = struct{}{} + for _, machine := range pushIDMachinePrefixes( + identity.Machine, legacyMarkerMachines, + ) { + candidateID := prefixedSessionID(machine, identity.ID) + if candidateID != identity.ID { + ids[candidateID] = struct{}{} + } + } + for _, aliasID := range uniqueNonEmptyStrings(identity.AliasIDs) { + ids[aliasID] = struct{}{} + } + } + result := make([]string, 0, len(ids)) + for id := range ids { + if id != "" { + result = append(result, id) + } + } + sort.Strings(result) + return result +} + // artifactLegacyDuplicateCandidate recognizes the narrow upgrade state where // an importer already created the stable artifact id while this origin still // owns its pre-artifact bare row. Both rows must already have the exact owners @@ -2076,6 +2132,72 @@ func pushIDMachinePrefixes(machine string, legacyMarkerMachines []string) []stri return prefixes } +type pgSessionOwnerRecord struct { + machine string + ownerMarker string + exists bool +} + +type pgSessionOwnerCache struct { + mu sync.Mutex + entries map[string]pgSessionOwnerRecord +} + +type pgSessionOwnerCacheContextKey struct{} + +func (c *pgSessionOwnerCache) get(id string) (pgSessionOwnerRecord, bool) { + c.mu.Lock() + defer c.mu.Unlock() + record, ok := c.entries[id] + return record, ok +} + +func (c *pgSessionOwnerCache) put(id string, record pgSessionOwnerRecord) { + c.mu.Lock() + defer c.mu.Unlock() + c.entries[id] = record +} + +// preloadPGSessionOwners resolves a candidate set in one PG round trip and +// records both hits and misses. pgSessionOwner reuses this cache and memoizes +// any relationship targets discovered later in the same push. +func (s *Sync) preloadPGSessionOwners( + ctx context.Context, ids []string, +) (context.Context, error) { + unique := uniqueNonEmptyStrings(ids) + cache := &pgSessionOwnerCache{ + entries: make(map[string]pgSessionOwnerRecord, len(unique)), + } + for _, id := range unique { + cache.entries[id] = pgSessionOwnerRecord{} + } + if len(unique) == 0 { + return context.WithValue(ctx, pgSessionOwnerCacheContextKey{}, cache), nil + } + rows, err := s.pg.QueryContext(ctx, ` + SELECT id, machine, owner_marker + FROM sessions + WHERE id = ANY($1)`, unique) + if err != nil { + return ctx, fmt.Errorf("preloading pg session owners: %w", err) + } + defer rows.Close() + for rows.Next() { + var id, machine string + var ownerMarker sql.NullString + if err := rows.Scan(&id, &machine, &ownerMarker); err != nil { + return ctx, fmt.Errorf("scanning pg session owner: %w", err) + } + cache.entries[id] = pgSessionOwnerRecord{ + machine: machine, ownerMarker: ownerMarker.String, exists: true, + } + } + if err := rows.Err(); err != nil { + return ctx, fmt.Errorf("iterating pg session owners: %w", err) + } + return context.WithValue(ctx, pgSessionOwnerCacheContextKey{}, cache), nil +} + // pgSessionOwner returns the machine and owner_marker of a PG session row, and // whether it exists. owner_marker is empty for legacy rows pushed before the // marker model. @@ -2083,6 +2205,12 @@ func (s *Sync) pgSessionOwner( ctx context.Context, id string, ) (string, string, bool, error) { + cache, _ := ctx.Value(pgSessionOwnerCacheContextKey{}).(*pgSessionOwnerCache) + if cache != nil { + if record, ok := cache.get(id); ok { + return record.machine, record.ownerMarker, record.exists, nil + } + } var machine string var ownerMarker sql.NullString err := s.pg.QueryRowContext(ctx, @@ -2091,6 +2219,9 @@ func (s *Sync) pgSessionOwner( ).Scan(&machine, &ownerMarker) if err != nil { if errors.Is(err, sql.ErrNoRows) { + if cache != nil { + cache.put(id, pgSessionOwnerRecord{}) + } return "", "", false, nil } return "", "", false, fmt.Errorf( @@ -2098,6 +2229,11 @@ func (s *Sync) pgSessionOwner( id, err, ) } + if cache != nil { + cache.put(id, pgSessionOwnerRecord{ + machine: machine, ownerMarker: ownerMarker.String, exists: true, + }) + } return machine, ownerMarker.String, true, nil } diff --git a/internal/postgres/push_test.go b/internal/postgres/push_test.go index f88d12836..459faad81 100644 --- a/internal/postgres/push_test.go +++ b/internal/postgres/push_test.go @@ -1194,6 +1194,13 @@ type pushSessionProbeState struct { aliases map[string]string excludedIDs map[string]bool existingExcluded map[string]bool + ownerQueries int + owners map[string]pushSessionProbeOwner +} + +type pushSessionProbeOwner struct { + machine string + marker string } var ( @@ -1310,7 +1317,29 @@ func (c *pushSessionProbeConn) QueryContext( defer c.state.mu.Unlock() switch { + case strings.Contains(normalized, "select id, machine, owner_marker"): + c.state.ownerQueries++ + values := [][]driver.Value{} + for _, id := range namedValueStrings(args) { + if owner, ok := c.state.owners[id]; ok { + values = append(values, []driver.Value{id, owner.machine, owner.marker}) + } + } + return &pushSessionProbeRows{ + columns: []string{"id", "machine", "owner_marker"}, + values: values, + }, nil case strings.Contains(normalized, "select machine, owner_marker"): + c.state.ownerQueries++ + if len(args) > 0 { + id, _ := args[0].Value.(string) + if owner, ok := c.state.owners[id]; ok { + return &pushSessionProbeRows{ + columns: []string{"machine", "owner_marker"}, + values: [][]driver.Value{{owner.machine, owner.marker}}, + }, nil + } + } return &pushSessionProbeRows{ columns: []string{"machine", "owner_marker"}, }, nil @@ -1347,6 +1376,71 @@ func (c *pushSessionProbeConn) QueryContext( } } +func TestPreloadPGSessionOwnersUsesOneQueryAndCachesMisses(t *testing.T) { + state := &pushSessionProbeState{owners: map[string]pushSessionProbeOwner{ + "owned-a": {machine: "desk", marker: "marker-a"}, + "owned-b": {machine: "laptop", marker: "marker-b"}, + }} + sync := &Sync{pg: newPushSessionProbeDB(t, state)} + + ctx, err := sync.preloadPGSessionOwners( + context.Background(), []string{"owned-a", "owned-b", "missing"}, + ) + require.NoError(t, err) + for _, tc := range []struct { + id string + machine string + marker string + exists bool + }{ + {id: "owned-a", machine: "desk", marker: "marker-a", exists: true}, + {id: "owned-b", machine: "laptop", marker: "marker-b", exists: true}, + {id: "missing"}, + } { + machine, marker, exists, lookupErr := sync.pgSessionOwner(ctx, tc.id) + require.NoError(t, lookupErr) + assert.Equal(t, tc.machine, machine) + assert.Equal(t, tc.marker, marker) + assert.Equal(t, tc.exists, exists) + } + assert.Equal(t, 1, state.ownerQueries, + "preloaded hits and misses must use one owner query") + + _, _, exists, err := sync.pgSessionOwner(ctx, "late-miss") + require.NoError(t, err) + assert.False(t, exists) + _, _, exists, err = sync.pgSessionOwner(ctx, "late-miss") + require.NoError(t, err) + assert.False(t, exists) + assert.Equal(t, 2, state.ownerQueries, + "an owner first discovered after preload must be memoized") +} + +func TestPushIdentityOwnerCandidateIDsCoverLegacyAndArtifactAliases(t *testing.T) { + sync := &Sync{machine: "desk"} + sessions := map[string]db.Session{ + "plain": { + ID: "plain", Machine: "local", + }, + "remote-a1b2c3~imported": { + ID: "remote-a1b2c3~imported", Machine: "remote-a1b2c3", + }, + } + imported := map[string]struct{}{"remote-a1b2c3~imported": {}} + + got := sync.pushIdentityOwnerCandidateIDs( + sessions, imported, "desk-origin", "marker", []string{"old-desk"}, + ) + assert.ElementsMatch(t, []string{ + "desk-origin~plain", + "old-desk~desk-origin~plain", + "plain", + "remote-a1b2c3~imported", + "old-desk~remote-a1b2c3~imported", + "imported", + }, got) +} + func (pushSessionProbeTx) Commit() error { return nil } func (pushSessionProbeTx) Rollback() error { return nil } From d75804f64f38cf8b8c93df980d05b029e8e29011 Mon Sep 17 00:00:00 2001 From: matt wilkie Date: Wed, 15 Jul 2026 10:04:24 -0700 Subject: [PATCH 07/10] fix(sync): discover pending files during shutdown --- cmd/agentsview/sync_watch.go | 7 ++-- cmd/agentsview/sync_watch_test.go | 56 ++++++++++++++++++++++++++++--- 2 files changed, 56 insertions(+), 7 deletions(-) diff --git a/cmd/agentsview/sync_watch.go b/cmd/agentsview/sync_watch.go index f70c0d1cf..a85a049ea 100644 --- a/cmd/agentsview/sync_watch.go +++ b/cmd/agentsview/sync_watch.go @@ -57,9 +57,10 @@ func (p *artifactFolderPusher) push( ) error { if p.engine != nil { // Startup already performed a full sync, and watcher change bursts have - // already applied their targeted paths. Only the periodic floor needs a - // full discovery pass to cover roots that could not be watched. - if reason == reasonInterval { + // already applied their targeted paths. The periodic floor covers roots + // that could not be watched, while shutdown discovery recovers changes + // still waiting in the watcher's batching window before the final export. + if reason == reasonInterval || reason == reasonShutdown { p.engine.SyncAll(ctx, nil) } // Export reads session rows outside a sync operation; flush diff --git a/cmd/agentsview/sync_watch_test.go b/cmd/agentsview/sync_watch_test.go index 68976a0f8..7ae1009ae 100644 --- a/cmd/agentsview/sync_watch_test.go +++ b/cmd/agentsview/sync_watch_test.go @@ -121,7 +121,7 @@ func TestArtifactFolderPusherPlumbsInsecurePeerOptIn(t *testing.T) { "watch sync must reach an explicitly allowed plaintext peer") } -func TestArtifactFolderPusherOnlyRunsFullDiscoveryForInterval(t *testing.T) { +func TestArtifactFolderPusherRunsFullDiscoveryForIntervalAndShutdown(t *testing.T) { dataDir := t.TempDir() target := t.TempDir() database := openWatchTestDB(t) @@ -134,20 +134,68 @@ func TestArtifactFolderPusherOnlyRunsFullDiscoveryForInterval(t *testing.T) { origin: "desk-a1b2c3", } - for _, reason := range []pushReason{reasonStartup, reasonChange, reasonShutdown} { + for _, reason := range []pushReason{reasonStartup, reasonChange} { require.NoError(t, pusher.push(context.Background(), reason)) } assert.Zero(t, syncer.syncAllCalls, "startup and watcher-driven pushes already synchronized local files") - assert.Equal(t, 3, syncer.flushCalls, + assert.Equal(t, 2, syncer.flushCalls, "every export must flush pending signal recomputes") - require.NoError(t, pusher.push(context.Background(), reasonInterval)) + require.NoError(t, pusher.push(context.Background(), reasonShutdown)) assert.Equal(t, 1, syncer.syncAllCalls, + "shutdown must discover changes still pending in the watcher batch") + require.NoError(t, pusher.push(context.Background(), reasonInterval)) + assert.Equal(t, 2, syncer.syncAllCalls, "the periodic floor must discover changes from unwatched roots") assert.Equal(t, 4, syncer.flushCalls) } +func TestArtifactFolderPusherShutdownDiscoversPendingFilesystemChange(t *testing.T) { + claudeDir := t.TempDir() + projectDir := filepath.Join(claudeDir, "-Users-alice-work") + require.NoError(t, os.MkdirAll(projectDir, 0o755)) + content := testjsonl.NewSessionBuilder(). + AddClaudeUser("2026-01-01T00:00:00Z", "pending change", "/Users/alice/work/project"). + AddClaudeAssistant("2026-01-01T00:00:01Z", "saved"). + String() + require.NoError(t, os.WriteFile( + filepath.Join(projectDir, "pending-session.jsonl"), []byte(content), 0o644, + )) + + dataDir := t.TempDir() + target := t.TempDir() + database := openWatchTestDB(t) + appCfg := config.Config{ + DataDir: dataDir, + AgentDirs: map[parser.AgentType][]string{ + parser.AgentClaude: {claudeDir}, + }, + } + engine := newArtifactWatchEngine(database, appCfg) + t.Cleanup(engine.Close) + pusher := &artifactFolderPusher{ + appCfg: appCfg, + database: database, + engine: engine, + target: target, + origin: "desk-a1b2c3", + } + + require.NoError(t, pusher.push(context.Background(), reasonShutdown)) + + session, err := database.GetSession(context.Background(), "pending-session") + require.NoError(t, err) + require.NotNil(t, session, + "the shutdown exchange must ingest a change still pending in the watcher batch") + assert.Equal(t, "pending change", *session.FirstMessage) + summary, err := artifact.CheckpointSummary(target, "desk-a1b2c3") + require.NoError(t, err) + require.True(t, summary.Found) + assert.Equal(t, 1, summary.SessionCount, + "the shutdown exchange must publish the newly ingested session") +} + func TestArtifactWatchEngineHonorsConfiguredCwdPrefixes(t *testing.T) { claudeDir := t.TempDir() projectDir := filepath.Join(claudeDir, "-Users-alice-work") From f3dd25680c79ef3356aaddb9289f69a6cc8b500a Mon Sep 17 00:00:00 2001 From: matt wilkie Date: Wed, 15 Jul 2026 10:04:25 -0700 Subject: [PATCH 08/10] fix(sync): derive peer status from checkpoint provenance --- internal/artifact/peer.go | 45 +++++++++----- internal/artifact/sync.go | 30 +++++++++ internal/server/artifact_peer_test.go | 79 ++++++++++++++++++++++++ internal/server/huma_routes_artifacts.go | 20 ++++-- 4 files changed, 154 insertions(+), 20 deletions(-) diff --git a/internal/artifact/peer.go b/internal/artifact/peer.go index 654500ae3..76636762a 100644 --- a/internal/artifact/peer.go +++ b/internal/artifact/peer.go @@ -8,6 +8,7 @@ import ( "io" "io/fs" "log" + "maps" "os" "path/filepath" "slices" @@ -204,20 +205,22 @@ func ReadLatestCheckpoint(root, origin string) (PeerArtifact, error) { }, nil } -// OriginCheckpointSummary describes the latest checkpoint published by one -// origin. Found is false when the origin has no checkpoint yet. +// OriginCheckpointSummary describes the latest valid checkpoint published by +// one origin, including its exact global-session-ID to manifest-hash mapping. +// Found is false when the origin has no compatible checkpoint yet. type OriginCheckpointSummary struct { - Sequence int - SessionCount int - ModTime time.Time - Found bool + Sequence int + SessionCount int + SessionManifests map[string]string + ModTime time.Time + Found bool } -// CheckpointSummary returns summary information about an origin's latest +// CheckpointSummary returns summary information about an origin's latest valid // checkpoint without decoding any session bundles. It is the read side of the -// peers status view. A checkpoint that no longer decodes is quarantined and -// the summary falls back to the newest one that does, so one corrupt file -// cannot make the peers view unusable. +// peers status view. A checkpoint that no longer validates is quarantined and +// the summary falls back to the newest valid one, so one corrupt file cannot +// make the peers view unusable. func CheckpointSummary(root, origin string) (OriginCheckpointSummary, error) { if strings.TrimSpace(root) == "" { return OriginCheckpointSummary{}, fmt.Errorf("%w: artifact root is required", ErrArtifactInvalid) @@ -251,11 +254,25 @@ func CheckpointSummary(root, origin string) (OriginCheckpointSummary, error) { quarantineArtifact(path) continue } + if err := validateCheckpoint(&cp, origin); err != nil { + if errors.Is(err, errFutureArtifactVersion) { + continue + } + log.Printf("artifact: skipping invalid checkpoint %s in peer summary: %v", path, err) + quarantineArtifact(path) + continue + } + if err := validateCheckpointSequenceIdentity(cp, filepath.Base(path)); err != nil { + log.Printf("artifact: skipping invalid checkpoint %s in peer summary: %v", path, err) + quarantineArtifact(path) + continue + } return OriginCheckpointSummary{ - Sequence: cp.Sequence, - SessionCount: len(cp.Sessions), - ModTime: info.ModTime(), - Found: true, + Sequence: cp.Sequence, + SessionCount: len(cp.Sessions), + SessionManifests: maps.Clone(cp.Sessions), + ModTime: info.ModTime(), + Found: true, }, nil } return OriginCheckpointSummary{}, nil diff --git a/internal/artifact/sync.go b/internal/artifact/sync.go index cfd121406..ddaa7b6bf 100644 --- a/internal/artifact/sync.go +++ b/internal/artifact/sync.go @@ -398,6 +398,36 @@ func ImportedSessionIDs( return ids, nil } +// CountImportedCheckpointSessions returns how many sessions in one checkpoint +// have landed at the exact manifest version it publishes. Import provenance is +// independent of the session row's current lifecycle state, so a locally +// trashed session remains landed while a stale active row does not. +func CountImportedCheckpointSessions( + database syncStateValueReader, origin string, sessionManifests map[string]string, +) (int, error) { + if len(sessionManifests) == 0 { + return 0, nil + } + keys := make([]string, 0, len(sessionManifests)) + expectedByKey := make(map[string]string, len(sessionManifests)) + for gid, manifestHash := range sessionManifests { + key := importStateKey(origin, gid) + keys = append(keys, key) + expectedByKey[key] = manifestHash + } + states, err := database.SyncStateValues(keys) + if err != nil { + return 0, fmt.Errorf("reading checkpoint import provenance: %w", err) + } + landed := 0 + for key, expectedHash := range expectedByKey { + if states[key] == expectedHash { + landed++ + } + } + return landed, nil +} + func newOriginID() (string, error) { host, err := os.Hostname() if err != nil || strings.TrimSpace(host) == "" { diff --git a/internal/server/artifact_peer_test.go b/internal/server/artifact_peer_test.go index 4a685f0c1..bdb39a631 100644 --- a/internal/server/artifact_peer_test.go +++ b/internal/server/artifact_peer_test.go @@ -306,6 +306,85 @@ func TestArtifactPeersStatus(t *testing.T) { assert.Equal(t, 1, peer.CheckpointSeq) } +func TestArtifactPeersStatusUsesLatestCheckpointImportProvenance(t *testing.T) { + te := setup(t, withArtifactOrigin("desktop-d4e5f6")) + ctx := context.Background() + first := "hello" + + // This peer is fully imported, then its local row is trashed. The import + // provenance still proves the published manifest landed successfully. + trashedOrigin := "trashed-a1b2c3" + trashedRoot := t.TempDir() + trashedDB, err := db.Open(filepath.Join(t.TempDir(), "trashed-peer.db")) + require.NoError(t, err) + t.Cleanup(func() { trashedDB.Close() }) + dbtest.SeedSession(t, trashedDB, "sess-1", "alpha", func(s *db.Session) { + s.FirstMessage = &first + }) + require.NoError(t, trashedDB.ReplaceSessionMessages("sess-1", []db.Message{ + {SessionID: "sess-1", Ordinal: 0, Role: "user", Content: "hello", ContentLength: 5}, + })) + _, err = artifact.Export(ctx, trashedDB, trashedRoot, trashedOrigin) + require.NoError(t, err) + postArtifactFile(t, te, trashedOrigin, "segments", + oneArtifactPath(t, trashedRoot, trashedOrigin, "segments", "*")) + postArtifactFile(t, te, trashedOrigin, "manifests", + oneArtifactPath(t, trashedRoot, trashedOrigin, "manifests", "*")) + postArtifactFile(t, te, trashedOrigin, "checkpoints", + oneArtifactPath(t, trashedRoot, trashedOrigin, "checkpoints", "*")) + require.NoError(t, te.db.SoftDeleteSession(trashedOrigin+"~sess-1")) + + // This peer's first manifest landed, but its latest checkpoint references a + // newer manifest that has not arrived. The active stale row is not current. + staleOrigin := "stale-d4e5f6" + staleRoot := t.TempDir() + staleDB, err := db.Open(filepath.Join(t.TempDir(), "stale-peer.db")) + require.NoError(t, err) + t.Cleanup(func() { staleDB.Close() }) + dbtest.SeedSession(t, staleDB, "sess-1", "before", func(s *db.Session) { + s.FirstMessage = &first + }) + require.NoError(t, staleDB.ReplaceSessionMessages("sess-1", []db.Message{ + {SessionID: "sess-1", Ordinal: 0, Role: "user", Content: "hello", ContentLength: 5}, + })) + _, err = artifact.Export(ctx, staleDB, staleRoot, staleOrigin) + require.NoError(t, err) + postArtifactFile(t, te, staleOrigin, "segments", + oneArtifactPath(t, staleRoot, staleOrigin, "segments", "*")) + postArtifactFile(t, te, staleOrigin, "manifests", + oneArtifactPath(t, staleRoot, staleOrigin, "manifests", "*")) + postArtifactFile(t, te, staleOrigin, "checkpoints", + filepath.Join(staleRoot, staleOrigin, "checkpoints", "cp-0000000001.json")) + + dbtest.SeedSession(t, staleDB, "sess-1", "after", func(s *db.Session) { + s.FirstMessage = &first + }) + exported, err := artifact.Export(ctx, staleDB, staleRoot, staleOrigin) + require.NoError(t, err) + require.Equal(t, 1, exported) + postArtifactFile(t, te, staleOrigin, "checkpoints", + filepath.Join(staleRoot, staleOrigin, "checkpoints", "cp-0000000002.json")) + + w := artifactPeerRequest(t, te, http.MethodGet, "/api/v1/artifacts/peers", nil, "") + assertStatus(t, w, http.StatusOK) + body := decode[artifactPeersBody](t, w) + byOrigin := make(map[string]artifactPeerBody, len(body.Peers)) + for _, peer := range body.Peers { + byOrigin[peer.Origin] = peer + } + + trashedPeer, ok := byOrigin[trashedOrigin] + require.True(t, ok) + assert.Equal(t, 1, trashedPeer.PublishedSessions) + assert.Equal(t, 1, trashedPeer.LocalSessions, + "a locally trashed row remains landed when its manifest provenance matches") + stalePeer, ok := byOrigin[staleOrigin] + require.True(t, ok) + assert.Equal(t, 1, stalePeer.PublishedSessions) + assert.Equal(t, 0, stalePeer.LocalSessions, + "an active row is pending when its imported manifest is older than the checkpoint") +} + func TestArtifactPeersStatusPublishesEmptyLocalOrigin(t *testing.T) { te := setup(t, withArtifactOrigin("desktop-d4e5f6")) // Discovery publishes an explicit empty checkpoint for a configured origin. diff --git a/internal/server/huma_routes_artifacts.go b/internal/server/huma_routes_artifacts.go index 58fcf8555..01fa6f706 100644 --- a/internal/server/huma_routes_artifacts.go +++ b/internal/server/huma_routes_artifacts.go @@ -148,9 +148,9 @@ func (s *Server) humaListArtifactPeers( if err != nil { return nil, artifactRouteError("list artifact origins", err) } - counts, err := s.db.MachineSessionCounts(ctx) + localDB, err := s.writableArtifactImportDB() if err != nil { - return nil, internalError("machine session counts", err) + return nil, err } conflicts, err := s.db.CountMetadataConflicts(ctx) if err != nil { @@ -180,10 +180,18 @@ func (s *Server) humaListArtifactPeers( return nil, artifactRouteError("read artifact checkpoint", err) } isLocal := origin == localOrigin - machineKey := origin + var landedSessions int if isLocal { - // Owned sessions keep machine "local" in the local DB. - machineKey = "local" + // publishLocalArtifacts just refreshed this origin, so every session + // in its latest checkpoint is present locally at that manifest. + landedSessions = summary.SessionCount + } else { + landedSessions, err = artifact.CountImportedCheckpointSessions( + localDB, origin, summary.SessionManifests, + ) + if err != nil { + return nil, internalError("count landed artifact sessions", err) + } } last := "" if summary.Found { @@ -194,7 +202,7 @@ func (s *Server) humaListArtifactPeers( IsLocal: isLocal, CheckpointSeq: summary.Sequence, PublishedSessions: summary.SessionCount, - LocalSessions: counts[machineKey], + LocalSessions: landedSessions, LastPublished: last, }) } From 324a2c70dd62e6081a0308bb179eda5472d654be Mon Sep 17 00:00:00 2001 From: matt wilkie Date: Wed, 15 Jul 2026 10:13:09 -0700 Subject: [PATCH 09/10] test(sync): cover shutdown and peer status edges --- cmd/agentsview/sync_watch_test.go | 55 +++++++++++++++++++++++++-- internal/server/artifact_peer_test.go | 11 +++++- 2 files changed, 62 insertions(+), 4 deletions(-) diff --git a/cmd/agentsview/sync_watch_test.go b/cmd/agentsview/sync_watch_test.go index 7ae1009ae..1d38bc2f4 100644 --- a/cmd/agentsview/sync_watch_test.go +++ b/cmd/agentsview/sync_watch_test.go @@ -5,6 +5,7 @@ import ( "os" "path/filepath" "testing" + "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -17,14 +18,16 @@ import ( ) type countingArtifactWatchSyncer struct { - syncAllCalls int - flushCalls int + syncAllCalls int + flushCalls int + syncAllHasDeadline bool } func (s *countingArtifactWatchSyncer) SyncAll( - _ context.Context, _ syncpkg.ProgressFunc, + ctx context.Context, _ syncpkg.ProgressFunc, ) syncpkg.SyncStats { s.syncAllCalls++ + _, s.syncAllHasDeadline = ctx.Deadline() return syncpkg.SyncStats{} } @@ -151,6 +154,51 @@ func TestArtifactFolderPusherRunsFullDiscoveryForIntervalAndShutdown(t *testing. assert.Equal(t, 4, syncer.flushCalls) } +func TestArtifactFolderPusherFlushesPendingWatchBatchOnShutdown(t *testing.T) { + database := openWatchTestDB(t) + syncer := &countingArtifactWatchSyncer{} + pusher := &artifactFolderPusher{ + appCfg: config.Config{DataDir: t.TempDir()}, + database: database, + engine: syncer, + target: t.TempDir(), + origin: "desk-a1b2c3", + } + debounceArmed := make(chan struct{}) + neverFire := make(chan time.Time) + loop := &pushLoop{ + debounce: time.Hour, + dirty: make(chan struct{}, 1), + floor: make(chan time.Time), + after: func(time.Duration) <-chan time.Time { + close(debounceArmed) + return neverFire + }, + push: pusher.push, + flushTimeout: time.Second, + } + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan struct{}) + go func() { + loop.Run(ctx) + close(done) + }() + + loop.NotifyDirty() + <-debounceArmed + cancel() + select { + case <-done: + case <-time.After(5 * time.Second): + require.FailNow(t, "shutdown flush did not complete") + } + + assert.Equal(t, 1, syncer.syncAllCalls, + "shutdown must replace the pending watcher batch with full discovery") + assert.True(t, syncer.syncAllHasDeadline, + "shutdown discovery must use the bounded flush context") +} + func TestArtifactFolderPusherShutdownDiscoversPendingFilesystemChange(t *testing.T) { claudeDir := t.TempDir() projectDir := filepath.Join(claudeDir, "-Users-alice-work") @@ -188,6 +236,7 @@ func TestArtifactFolderPusherShutdownDiscoversPendingFilesystemChange(t *testing require.NoError(t, err) require.NotNil(t, session, "the shutdown exchange must ingest a change still pending in the watcher batch") + require.NotNil(t, session.FirstMessage) assert.Equal(t, "pending change", *session.FirstMessage) summary, err := artifact.CheckpointSummary(target, "desk-a1b2c3") require.NoError(t, err) diff --git a/internal/server/artifact_peer_test.go b/internal/server/artifact_peer_test.go index bdb39a631..9f691cd74 100644 --- a/internal/server/artifact_peer_test.go +++ b/internal/server/artifact_peer_test.go @@ -364,6 +364,15 @@ func TestArtifactPeersStatusUsesLatestCheckpointImportProvenance(t *testing.T) { require.Equal(t, 1, exported) postArtifactFile(t, te, staleOrigin, "checkpoints", filepath.Join(staleRoot, staleOrigin, "checkpoints", "cp-0000000002.json")) + require.NoError(t, te.db.UpsertSession(db.Session{ + ID: staleOrigin + "~unrelated", + Project: "unrelated", + Machine: staleOrigin, + Agent: "claude", + MessageCount: 1, + UserMessageCount: 1, + CreatedAt: "2026-07-15T12:00:00Z", + }), "seed an active same-origin row absent from the checkpoint") w := artifactPeerRequest(t, te, http.MethodGet, "/api/v1/artifacts/peers", nil, "") assertStatus(t, w, http.StatusOK) @@ -382,7 +391,7 @@ func TestArtifactPeersStatusUsesLatestCheckpointImportProvenance(t *testing.T) { require.True(t, ok) assert.Equal(t, 1, stalePeer.PublishedSessions) assert.Equal(t, 0, stalePeer.LocalSessions, - "an active row is pending when its imported manifest is older than the checkpoint") + "stale and checkpoint-unrelated active rows must not count as landed") } func TestArtifactPeersStatusPublishesEmptyLocalOrigin(t *testing.T) { From ad3bf000973a4570f1d72c5b46e5014cc63e2ddf Mon Sep 17 00:00:00 2001 From: matt wilkie Date: Wed, 15 Jul 2026 10:15:03 -0700 Subject: [PATCH 10/10] test(sync): bound shutdown regression wait --- cmd/agentsview/sync_watch_test.go | 6 +++++- internal/server/artifact_peer_test.go | 8 ++++---- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/cmd/agentsview/sync_watch_test.go b/cmd/agentsview/sync_watch_test.go index 1d38bc2f4..1e69efd26 100644 --- a/cmd/agentsview/sync_watch_test.go +++ b/cmd/agentsview/sync_watch_test.go @@ -185,7 +185,11 @@ func TestArtifactFolderPusherFlushesPendingWatchBatchOnShutdown(t *testing.T) { }() loop.NotifyDirty() - <-debounceArmed + select { + case <-debounceArmed: + case <-time.After(5 * time.Second): + require.FailNow(t, "pending watch batch did not arm the debounce") + } cancel() select { case <-done: diff --git a/internal/server/artifact_peer_test.go b/internal/server/artifact_peer_test.go index 9f691cd74..b6a8b99cd 100644 --- a/internal/server/artifact_peer_test.go +++ b/internal/server/artifact_peer_test.go @@ -309,7 +309,7 @@ func TestArtifactPeersStatus(t *testing.T) { func TestArtifactPeersStatusUsesLatestCheckpointImportProvenance(t *testing.T) { te := setup(t, withArtifactOrigin("desktop-d4e5f6")) ctx := context.Background() - first := "hello" + firstMessage := "hello" // This peer is fully imported, then its local row is trashed. The import // provenance still proves the published manifest landed successfully. @@ -319,7 +319,7 @@ func TestArtifactPeersStatusUsesLatestCheckpointImportProvenance(t *testing.T) { require.NoError(t, err) t.Cleanup(func() { trashedDB.Close() }) dbtest.SeedSession(t, trashedDB, "sess-1", "alpha", func(s *db.Session) { - s.FirstMessage = &first + s.FirstMessage = &firstMessage }) require.NoError(t, trashedDB.ReplaceSessionMessages("sess-1", []db.Message{ {SessionID: "sess-1", Ordinal: 0, Role: "user", Content: "hello", ContentLength: 5}, @@ -342,7 +342,7 @@ func TestArtifactPeersStatusUsesLatestCheckpointImportProvenance(t *testing.T) { require.NoError(t, err) t.Cleanup(func() { staleDB.Close() }) dbtest.SeedSession(t, staleDB, "sess-1", "before", func(s *db.Session) { - s.FirstMessage = &first + s.FirstMessage = &firstMessage }) require.NoError(t, staleDB.ReplaceSessionMessages("sess-1", []db.Message{ {SessionID: "sess-1", Ordinal: 0, Role: "user", Content: "hello", ContentLength: 5}, @@ -357,7 +357,7 @@ func TestArtifactPeersStatusUsesLatestCheckpointImportProvenance(t *testing.T) { filepath.Join(staleRoot, staleOrigin, "checkpoints", "cp-0000000001.json")) dbtest.SeedSession(t, staleDB, "sess-1", "after", func(s *db.Session) { - s.FirstMessage = &first + s.FirstMessage = &firstMessage }) exported, err := artifact.Export(ctx, staleDB, staleRoot, staleOrigin) require.NoError(t, err)