diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 45574f66a..76f0a812a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -310,6 +310,11 @@ jobs: TEST_SSH_USER: testuser TEST_SSH_KEY: ${{ github.workspace }}/testdata/ssh/test_key + - name: Run MinIO object-store integration test + run: make test-minio + env: + CGO_ENABLED: "1" + e2e: runs-on: ubuntu-latest steps: diff --git a/Makefile b/Makefile index d44c7e33b..1df176f8b 100644 --- a/Makefile +++ b/Makefile @@ -33,7 +33,7 @@ AIR_BIN := $(shell if command -v air >/dev/null 2>&1; then command -v air; \ elif [ -x "$(GOPATH_FIRST)/bin/air" ]; then printf "%s" "$(GOPATH_FIRST)/bin/air"; \ fi) -.PHONY: build build-release install frontend frontend-dev dev check-air air-install desktop-dev desktop-build desktop-macos-app desktop-macos-dmg desktop-windows-installer desktop-linux-appimage desktop-app docs-install docs-build docs-serve docs-check docs-screenshots docs-assets-branch docs-generated-assets-branch docs-deploy-staging docs-deploy test test-short test-evalingest bench-backends bench-gate bench-gate-config test-postgres test-postgres-ci test-s3 postgres-up postgres-down test-ssh test-ssh-ci ssh-up ssh-down e2e e2e-duckdb vet lint lint-ci lint-golangci lint-golangci-ci nilaway nilaway-golangci-build lint-tools tidy clean release release-darwin-arm64 release-darwin-amd64 release-linux-amd64 install-hooks ensure-embed-dir pricing-snapshot sqlite-vec-header dev-snapshot help +.PHONY: build build-release install frontend frontend-dev dev check-air air-install desktop-dev desktop-build desktop-macos-app desktop-macos-dmg desktop-windows-installer desktop-linux-appimage desktop-app docs-install docs-build docs-serve docs-check docs-screenshots docs-assets-branch docs-generated-assets-branch docs-deploy-staging docs-deploy test test-short test-evalingest bench-backends bench-gate bench-gate-config test-postgres test-postgres-ci test-s3 test-minio postgres-up postgres-down test-ssh test-ssh-ci ssh-up ssh-down e2e e2e-duckdb vet lint lint-ci lint-golangci lint-golangci-ci nilaway nilaway-golangci-build lint-tools tidy clean release release-darwin-arm64 release-darwin-amd64 release-linux-amd64 install-hooks ensure-embed-dir pricing-snapshot sqlite-vec-header dev-snapshot help # Ensure go:embed has at least one file (no-op if frontend is built) ensure-embed-dir: @@ -348,6 +348,11 @@ test-postgres-ci: pricing-snapshot ensure-embed-dir test-s3: pricing-snapshot ensure-embed-dir CGO_ENABLED=1 go test -tags "fts5,s3test" -v ./internal/sync/... -run TestS3 -count=1 +# MinIO/S3 object-store integration test. testcontainers starts and tears down +# the MinIO container automatically, so this just needs a working Docker daemon. +test-minio: ensure-embed-dir + CGO_ENABLED=1 go test -tags "fts5,miniotest" -v ./internal/artifact/... -run MinIO -count=1 + # Start test SSH container ssh-up: docker compose -f docker-compose.test.yml up -d --build --wait sshd @@ -368,8 +373,9 @@ test-ssh: pricing-snapshot ensure-embed-dir ssh-up test-ssh-ci: pricing-snapshot ensure-embed-dir CGO_ENABLED=1 go test -tags "fts5,sshtest" -v ./internal/ssh/... -count=1 -# Run Playwright E2E tests -e2e: +# Run artifact sync and Playwright E2E tests +e2e: ensure-embed-dir + CGO_ENABLED=1 go test -tags "fts5,e2e" ./internal/e2e -v -count=1 cd frontend && npx playwright test # Run focused Playwright smoke tests against duckdb serve. @@ -544,6 +550,7 @@ help: @echo " bench-gate - Run the hot-path benchmarks CI gates PRs on" @echo " test-postgres - Run PostgreSQL integration tests" @echo " test-s3 - Run S3 discovery integration tests (Docker)" + @echo " test-minio - Run MinIO/S3 object-store integration test (needs Docker)" @echo " postgres-up - Start test PostgreSQL container" @echo " postgres-down - Stop test PostgreSQL container" @echo " test-ssh - Run SSH integration tests" diff --git a/README.md b/README.md index cbbfc033c..ddb6f4d08 100644 --- a/README.md +++ b/README.md @@ -103,6 +103,25 @@ Use `--public-origin` (repeatable or comma-separated) to trust additional browser origins. If you expose the UI beyond loopback, also enable `--require-auth`. +## Local-First Artifact Sync + +Artifact sync is for a fully trusted personal fleet. It exchanges immutable +artifacts through a dedicated folder, HTTP peer, or S3-compatible target and +imports them into each machine's local SQLite archive: + +```bash +agentsview sync --init /path/to/agentsview-artifacts +agentsview sync /path/to/agentsview-artifacts +agentsview sync --watch /path/to/agentsview-artifacts +``` + +Do not sync the live SQLite database, `AGENTSVIEW_DATA_DIR`, or whole raw agent +directories. Two intermittent machines only sync when both can reach the same +transport unless you provide a rendezvous such as a NAS folder, cloud-synced +folder, S3-compatible bucket, or always-on peer. See +[Trusted-Fleet Artifact Sync](docs/artifact-sync.md) for setup and safety +boundaries. + ## Docker The container image defaults to local `agentsview serve`. Set `PG_SERVE=1` to diff --git a/SECURITY.md b/SECURITY.md index 3dc153e7e..b46d4396a 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -30,11 +30,11 @@ risk via documented flags and config. attacker. - Parser crashes, excessive resource use, or active-content injection triggered by content inside supported session files. Session files often contain - web/tool output that agentsview did not author, and defensive parsing of that - content is a security-relevant concern. + web/tool output that agentsview did not author, and defensive parsing of + that content is a security-relevant concern. - Inadvertent exposure of secrets that appear in transcripts. agentsview ships a - best-effort secret detector and redacts findings in the UI and CLI by default - (see [Secrets subsystem](#secrets-subsystem)). + best-effort secret detector and redacts findings in the UI and CLI by + default (see [Secrets subsystem](#secrets-subsystem)). ### Explicitly out of scope (today) @@ -63,6 +63,7 @@ risk via documented flags and config. | HTTP server → caller | Loopback-trusted; bearer-gated for `/api/` when `--require-auth` | Static assets are not gated. | | Browser → HTTP server | Host-header allowlist + CORS + CSP + X-Frame-Options enforced | DNS-rebinding, framing, and cross-origin defenses. | | agentsview → PostgreSQL (pg push) | TLS required for non-loopback hosts | Plaintext rejected unless `allow_insecure = true` is set explicitly. | +| agentsview → artifact HTTP peer | HTTPS required for non-loopback peers | Plaintext rejected unless `sync --allow-insecure` is explicit. | | agentsview → update endpoint | One-way outbound, opt-out | Disable with `--no-update-check`. | | agentsview → LiteLLM pricing | One-way outbound, on-demand | Public JSON fetched from GitHub raw; no session data sent. | @@ -70,8 +71,8 @@ risk via documented flags and config. - The local archive (SQLite + FTS5 index) stores indexed session data in plaintext. This includes assistant responses, user prompts, tool arguments, - command output, file contents fetched by agents, and any secrets that may have - been pasted into an agent session. + command output, file contents fetched by agents, and any secrets that may + have been pasted into an agent session. - File permissions follow the user's umask. agentsview does not chmod the data directory and does not encrypt at rest. - Treat the agentsview data directory with the same care you would treat your @@ -89,10 +90,10 @@ explicitly because "data stays on your machine" is the default but is not a complete description of the system once optional features are in use. - **Local UI / API.** The HTTP server binds to `127.0.0.1` by default. When - exposed beyond loopback, `--require-auth` should be enabled. Authentication is - a bearer token applied to `/api/` routes only; static assets remain ungated. - Browser-facing defenses (Host-header allowlist, CORS restrictions, CSP, - `X-Frame-Options: DENY`) are always on. See the CLI reference for token + exposed beyond loopback, `--require-auth` should be enabled. Authentication + is a bearer token applied to `/api/` routes only; static assets remain + ungated. Browser-facing defenses (Host-header allowlist, CORS restrictions, + CSP, `X-Frame-Options: DENY`) are always on. See the CLI reference for token configuration. - **PostgreSQL sync.** `agentsview pg push` exports the local archive to a user-supplied PostgreSQL instance. Non-loopback DSNs are rejected unless TLS @@ -102,8 +103,14 @@ complete description of the system once optional features are in use. its access controls. - **SSH remote sync.** agentsview can pull session archives from another machine over SSH. Authentication is whatever the user's SSH configuration provides. - Pulled files are parsed locally as untrusted data and merged into the unified - archive. + Pulled files are parsed locally as untrusted data and merged into the + unified archive. +- **Artifact HTTP peer sync.** `agentsview sync http(s)://...` exchanges bearer + credentials and archive artifacts with a user-supplied peer. Non-loopback + peers require HTTPS by default; loopback HTTP is allowed, while remote + plaintext requires the deliberate `--allow-insecure` opt-in and emits a + warning. Redirects are rejected. Received artifacts remain untrusted + structured input even when the peer is part of a trusted personal fleet. - **Imports.** Imported archives from other agentsview instances or third-party exports are treated as untrusted structured data, no different from session files written by local agents. @@ -125,9 +132,9 @@ The HTTP server applies the following defenses unconditionally: - **CORS restrictions.** Cross-origin API requests must come from an allowed origin or carry the bearer token; preflight handling is explicit. - **Content-Security-Policy.** A policy pinning the server's exact origin for - script/style/image/font/default-src is set on non-API responses. `connect-src` - is intentionally widened to allow the remote-server feature in the SPA; this - is a documented tradeoff. + script/style/image/font/default-src is set on non-API responses. + `connect-src` is intentionally widened to allow the remote-server feature in + the SPA; this is a documented tradeoff. - **X-Frame-Options: DENY.** Framing is disallowed on non-API responses. ## Secrets subsystem @@ -181,11 +188,11 @@ its own proposal. 1. **Multi-user machine support.** Is agentsview ever meant to run on a shared host, and if so what are the minimum hardening steps? 1. **`allow_insecure` UX.** Should setting `[pg] allow_insecure = true` require - an additional confirmation (e.g., a `--yes-really` flag) on first use, given - that it disables the only protection against plaintext PG egress? + an additional confirmation (e.g., a `--yes-really` flag) on first use, + given that it disables the only protection against plaintext PG egress? 1. **Deletion guarantees.** Should "permanent delete" grow into a stronger - erasure path (VACUUM, WAL checkpoint + truncate, mirror propagation to PG/SSH - targets), or should the docs simply make the current limits clearer? + erasure path (VACUUM, WAL checkpoint + truncate, mirror propagation to + PG/SSH targets), or should the docs simply make the current limits clearer? 1. **Secret detection scope.** Should the detector expand (more patterns, structured-secret types), should redacted-by-default extend to exports, and should there be a "scrub-on-import" pass? diff --git a/cmd/agentsview/cli.go b/cmd/agentsview/cli.go index 8bf4431b9..591b46532 100644 --- a/cmd/agentsview/cli.go +++ b/cmd/agentsview/cli.go @@ -287,7 +287,7 @@ func newOpenAPICommand() *cobra.Command { func newSyncCommand() *cobra.Command { var cfg SyncConfig cmd := &cobra.Command{ - Use: "sync", + Use: "sync [artifact-folder]", Short: "Sync session data without serving", Long: "Sync session data into the local database without starting the\n" + "HTTP server.\n\n" + @@ -298,12 +298,33 @@ func newSyncCommand() *cobra.Command { "exits non-zero if any configured host failed.\n\n" + "With --host, syncs only that host. A running local daemon may use a\n" + "matching configured remote_hosts entry and transport; otherwise,\n" + - "ad hoc --host sync uses your existing SSH configuration and requires\n" + + "ad hoc --host sync falls back to SSH.\n\n" + + "With an artifact-folder argument or --artifact-folder, sync also\n" + + "exchanges local-first immutable artifacts with that folder target.\n" + + "Artifact sync v1 is for a fully trusted personal fleet. Use a\n" + + "dedicated artifact share folder; do not point this at the\n" + + "agentsview data directory, raw agent directories, or the live\n" + + "SQLite database file and its WAL/SHM files.\n\n" + + "Use --init with an artifact folder on first setup to generate and\n" + + "persist this machine's artifact origin, backfill existing local\n" + + "sessions into the artifact store, exchange with the folder target,\n" + + "and import any peer artifacts already present. Two intermittent\n" + + "machines only sync while both can reach the same transport; use a\n" + + "NAS, cloud folder, object store, or always-on peer as a rendezvous\n" + + "when asynchronous convergence matters.\n\n" + + "Use --watch with an artifact folder to keep syncing. Watch mode\n" + + "runs an initial local sync and artifact exchange, coalesces file\n" + + "changes with --debounce, retries failed exchanges on later\n" + + "changes or --interval ticks, and performs a final best-effort\n" + + "exchange on shutdown. Combining --init with --watch publishes\n" + + "the first-run baseline on the first successful exchange, then\n" + + "keeps watching.\n\n" + + "Remote sync uses your existing SSH configuration and requires\n" + "key-based (passwordless) auth; it never prompts for a password.", GroupID: groupCore, SilenceUsage: true, - Args: cobra.NoArgs, - PreRunE: func(cmd *cobra.Command, _ []string) error { + Args: cobra.MaximumNArgs(1), + PreRunE: func(cmd *cobra.Command, args []string) error { if cfg.Host == "" { if cmd.Flags().Changed("user") || cmd.Flags().Changed("port") { @@ -312,9 +333,23 @@ func newSyncCommand() *cobra.Command { ) } } + if err := applySyncArtifactTarget(&cfg, args, cmd.Flags().Changed("artifact-folder")); err != nil { + return err + } + if err := validateSyncConfig(cfg); err != nil { + return err + } return nil }, Run: func(cmd *cobra.Command, args []string) { + if cfg.Watch { + runSyncWatch(cfg) + return + } + if cmd.Flags().Changed("debounce") || cmd.Flags().Changed("interval") { + fmt.Fprintln(os.Stderr, + "warning: --debounce and --interval have no effect without --watch") + } runSync(cfg) }, } @@ -322,6 +357,10 @@ func newSyncCommand() *cobra.Command { &cfg.Full, "full", false, "Force a full resync regardless of data version", ) + cmd.Flags().BoolVar( + &cfg.Init, "init", false, + "Initialize artifact sync with the folder target", + ) cmd.Flags().StringVar( &cfg.Host, "host", "", "SSH hostname for remote sync", @@ -334,6 +373,38 @@ func newSyncCommand() *cobra.Command { &cfg.Port, "port", 0, "SSH port for remote sync (default: 22)", ) + cmd.Flags().StringVar( + &cfg.ArtifactFolder, "artifact-folder", "", + "Exchange local-first sync artifacts with a folder, http(s) peer, or s3:// target", + ) + cmd.Flags().StringVar( + &cfg.Token, "token", "", + "Bearer token for an http(s) artifact peer target", + ) + cmd.Flags().BoolVar( + &cfg.AllowInsecure, "allow-insecure", false, + "Allow plaintext HTTP to a non-loopback artifact peer", + ) + cmd.Flags().BoolVar( + &cfg.Watch, "watch", false, + "Run artifact folder sync continuously, syncing on change plus a periodic floor", + ) + cmd.Flags().DurationVar( + &cfg.Debounce, "debounce", defaultWatchDebounce, + "Coalesce window after a change before artifact sync (--watch only)", + ) + cmd.Flags().DurationVar( + &cfg.Interval, "interval", defaultWatchInterval, + "Periodic floor artifact sync interval (--watch only)", + ) + cmd.Flags().DurationVar( + &cfg.GCGrace, "gc-grace", defaultArtifactGCGrace, + "Minimum age before superseded artifacts are auto-collected after a folder sync", + ) + cmd.Flags().BoolVar( + &cfg.NoGC, "no-gc", false, + "Disable automatic garbage collection of superseded artifacts after a folder sync", + ) cmd.Flags().StringVar( &cfg.CPUProfile, "cpuprofile", "", "Write CPU profile to file (developer use)", @@ -351,6 +422,7 @@ func newSyncCommand() *cobra.Command { panic(err) } } + cmd.AddCommand(newSyncGCCommand()) return cmd } diff --git a/cmd/agentsview/cli_test.go b/cmd/agentsview/cli_test.go index c52d4056b..3fbfea091 100644 --- a/cmd/agentsview/cli_test.go +++ b/cmd/agentsview/cli_test.go @@ -103,6 +103,13 @@ func TestDuckDBPushHelpShowsProjectFlags(t *testing.T) { } } +func TestSyncHelpShowsArtifactTransportSafetyFlag(t *testing.T) { + help, err := executeCommand(newRootCommand(), "sync", "--help") + require.NoError(t, err, "Execute") + assert.Contains(t, help, "--allow-insecure") + assert.Contains(t, help, "non-loopback artifact peer") +} + func TestPGStatusHelpShowsProjectFlags(t *testing.T) { help, err := executeCommand(newRootCommand(), "pg", "status", "--help") require.NoError(t, err, "Execute") @@ -325,7 +332,7 @@ func TestRootHelpDocumentsRemoteHosts(t *testing.T) { func TestSyncHelpMentionsConfiguredHosts(t *testing.T) { help, err := executeCommand(newRootCommand(), "sync", "--help") require.NoError(t, err, "Execute") - for _, want := range []string{"remote_hosts", "--host", "passwordless"} { + for _, want := range []string{"remote_hosts", "--host", "passwordless", "trusted personal fleet", "rendezvous"} { assert.Contains(t, help, want, "sync help missing %q", want) } } diff --git a/cmd/agentsview/main.go b/cmd/agentsview/main.go index 043b26e79..aa1f51400 100644 --- a/cmd/agentsview/main.go +++ b/cmd/agentsview/main.go @@ -18,6 +18,7 @@ import ( _ "time/tzdata" "github.com/spf13/cobra" + "go.kenn.io/agentsview/internal/artifact" "go.kenn.io/agentsview/internal/config" "go.kenn.io/agentsview/internal/db" "go.kenn.io/agentsview/internal/parser" @@ -286,6 +287,18 @@ func runServe(cfg config.Config, opts serveOptions) { } cfg = preparedCfg + // Reconcile an already-adopted artifact origin so every origin lookup + // (recorder, peer import, folder sync) agrees: the config.toml origin is + // authoritative and overwrites a divergent DB sync-state value. Serve + // never creates an origin — a machine opts into artifact sync only via + // `sync --init`, a sync run, or an incoming peer exchange, and until then + // curation stays local with no metadata ledger writes. + if cfg.DataDir != "" && !database.ReadOnly() && cfg.ArtifactOriginID != "" { + if err := artifact.AdoptOrigin(database, cfg.ArtifactOriginID); err != nil { + fatal("reconcile artifact origin id: %v", err) + } + } + srvOpts := []server.Option{ server.WithVersion(server.VersionInfo{ Version: version, diff --git a/cmd/agentsview/pg_watch_loop.go b/cmd/agentsview/pg_watch_loop.go index 1e820ca74..fba90dad9 100644 --- a/cmd/agentsview/pg_watch_loop.go +++ b/cmd/agentsview/pg_watch_loop.go @@ -4,6 +4,9 @@ import ( "context" "log" "time" + + "go.kenn.io/agentsview/internal/config" + syncpkg "go.kenn.io/agentsview/internal/sync" ) // pushReason labels why a push was triggered, for logging. @@ -29,12 +32,12 @@ const defaultFlushTimeout = 30 * time.Second // under test. In production, after is time.After and floor is a // time.Ticker channel. type pushLoop struct { - debounce time.Duration - dirty chan struct{} - floor <-chan time.Time - after func(time.Duration) <-chan time.Time - push func(ctx context.Context, reason pushReason) error - label string + logPrefix string + debounce time.Duration + dirty chan struct{} + floor <-chan time.Time + after func(time.Duration) <-chan time.Time + push func(ctx context.Context, reason pushReason) error // flushTimeout bounds the final shutdown-flush push. Zero means // no bound (used in tests that inject a fake pusher). flushTimeout time.Duration @@ -46,22 +49,30 @@ func newPushLoop( debounce, interval time.Duration, push func(context.Context, pushReason) error, ) (*pushLoop, *time.Ticker) { - return newPushLoopWithLabel("pg watch", debounce, interval, push) + return newNamedPushLoop("pg watch", debounce, interval, push) } func newPushLoopWithLabel( label string, debounce, interval time.Duration, push func(context.Context, pushReason) error, +) (*pushLoop, *time.Ticker) { + return newNamedPushLoop(label, debounce, interval, push) +} + +func newNamedPushLoop( + logPrefix string, + debounce, interval time.Duration, + push func(context.Context, pushReason) error, ) (*pushLoop, *time.Ticker) { ticker := time.NewTicker(interval) return &pushLoop{ + logPrefix: logPrefix, debounce: debounce, dirty: make(chan struct{}, 1), floor: ticker.C, after: time.After, push: push, - label: label, flushTimeout: defaultFlushTimeout, }, ticker } @@ -112,6 +123,42 @@ func (l *pushLoop) Run(ctx context.Context) { func (l *pushLoop) doPush(ctx context.Context, reason pushReason) { if err := l.push(ctx, reason); err != nil { - log.Printf("%s: push (%s) failed: %v", l.label, reason, err) + prefix := l.logPrefix + if prefix == "" { + prefix = "watch" + } + log.Printf("%s: push (%s) failed: %v", prefix, reason, err) } } + +type watchedSinkConfig struct { + AppConfig config.Config + Engine *syncpkg.Engine + Debounce time.Duration + Interval time.Duration + LogPrefix string + Push func(context.Context, pushReason) error +} + +func runWatchedSink(ctx context.Context, cfg watchedSinkConfig) { + loop, ticker := newNamedPushLoop( + cfg.LogPrefix, cfg.Debounce, cfg.Interval, cfg.Push, + ) + defer ticker.Stop() + + stopWatcher, unwatchedDirs := startFileWatcher(cfg.AppConfig, cfg.Engine, + func(batch syncpkg.WatchBatch) { + syncWatchBatch(ctx, cfg.Engine, batch) + loop.NotifyDirty() + }, + ) + defer stopWatcher() + if len(unwatchedDirs) > 0 { + log.Printf( + "%s: %d root(s) not watched; relying on the %s floor for coverage", + cfg.LogPrefix, len(unwatchedDirs), cfg.Interval, + ) + } + + loop.Run(ctx) +} diff --git a/cmd/agentsview/sync.go b/cmd/agentsview/sync.go index cd42825ba..d9492bd6d 100644 --- a/cmd/agentsview/sync.go +++ b/cmd/agentsview/sync.go @@ -16,6 +16,7 @@ import ( "strings" "time" + "go.kenn.io/agentsview/internal/artifact" "go.kenn.io/agentsview/internal/config" "go.kenn.io/agentsview/internal/db" "go.kenn.io/agentsview/internal/parser" @@ -26,10 +27,23 @@ import ( // SyncConfig holds parsed CLI options for the sync command. type SyncConfig struct { - Full bool - Host string - User string - Port int + Full bool + Init bool + Watch bool + Debounce time.Duration + Interval time.Duration + Host string + User string + Port int + ArtifactFolder string + // Token is the bearer token used for an http(s):// artifact peer target. + Token string + // AllowInsecure permits plaintext HTTP to a non-loopback artifact peer. + AllowInsecure bool + // GCGrace is the minimum age before superseded artifacts are auto-collected + // after a folder sync. NoGC disables that automatic collection. + GCGrace time.Duration + NoGC bool // CPUProfile, MemProfile, and Trace are hidden flags that capture a // pprof CPU profile, allocation snapshot, and runtime trace for the // sync pass. Empty strings disable each independently. @@ -38,17 +52,54 @@ type SyncConfig struct { Trace string } +func applySyncArtifactTarget(cfg *SyncConfig, args []string, flagChanged bool) error { + if len(args) == 0 { + return nil + } + if flagChanged { + return errors.New("artifact folder target cannot be provided both as an argument and --artifact-folder") + } + cfg.ArtifactFolder = args[0] + return nil +} + +func validateSyncConfig(cfg SyncConfig) error { + if cfg.Init && cfg.Host != "" { + return errors.New("--init cannot be combined with --host") + } + if cfg.Init && cfg.ArtifactFolder == "" { + return errors.New( + "--init requires an artifact folder target", + ) + } + if cfg.Watch && cfg.Host != "" { + return errors.New("--watch cannot be combined with --host") + } + if cfg.Watch && cfg.ArtifactFolder == "" { + return errors.New("--watch requires an artifact folder target") + } + if cfg.Host != "" && cfg.ArtifactFolder != "" { + // SSH remote sync (--host) returns before artifact sync runs, so a + // combined invocation would silently ignore the artifact target. + return errors.New("--host cannot be combined with an artifact target") + } + return nil +} + func runSync(cfg SyncConfig) { - if doSync(cfg) { + hadRemoteFailures, err := doSync(cfg) + if err != nil { + fatal("sync: %v", err) + } + if hadRemoteFailures { os.Exit(1) } } -// doSync performs the sync run and reports whether any configured -// remote host failed. It owns the deferred cleanup (profile stop, -// db close) so runSync can translate the result into a non-zero -// exit code without skipping that cleanup. -func doSync(cfg SyncConfig) (hadRemoteFailures bool) { +// doSync performs the sync run and reports whether any configured remote host +// failed. It owns the deferred cleanup (profile stop, db close) so runSync can +// translate the result into a non-zero exit code without skipping that cleanup. +func doSync(cfg SyncConfig) (hadRemoteFailures bool, err error) { appCfg, err := config.LoadMinimal() if err != nil { log.Fatalf("loading config: %v", err) @@ -82,14 +133,18 @@ func doSync(cfg SyncConfig) (hadRemoteFailures bool) { } if includeLocal || len(remoteHosts) > 0 { - tr, err := ensureTransport( - &appCfg, transportIntentArchiveWrite, 0, - ) + tr, err := syncTransport(&appCfg, cfg) if err != nil { fatal("detecting daemon: %v", err) } if tr.Mode == transportHTTP { useDaemon := useDaemonForSync(tr) + if useDaemon && cfg.ArtifactFolder != "" { + return false, fmt.Errorf( + "artifact sync cannot run while a writable daemon owns the SQLite archive; " + + "run `agentsview serve stop` and retry", + ) + } if useDaemon && len(remoteHosts) > 0 { fmt.Println("Running sync with remotes via daemon...") progress := newRemoteProgressPrinter(os.Stdout, time.Now) @@ -102,7 +157,7 @@ func doSync(cfg SyncConfig) (hadRemoteFailures bool) { fatal("daemon remote sync: %v", err) } reportRemoteFailures(failures) - return len(failures) > 0 + return len(failures) > 0, nil } if useDaemon { start := time.Now() @@ -127,7 +182,7 @@ func doSync(cfg SyncConfig) (hadRemoteFailures bool) { fatal("daemon sync: %v", err) } printSyncSummary(stats, start) - return false + return false, nil } // Read-only mirror daemons do not own the local SQLite // archive. Remote sync can still proceed through the direct @@ -150,7 +205,7 @@ func doSync(cfg SyncConfig) (hadRemoteFailures bool) { if cfg.Host != "" { runRemoteSync(appCfg, database, cfg) - return false + return false, nil } failures := syncLocalAndRemotes( @@ -164,8 +219,18 @@ func doSync(cfg SyncConfig) (hadRemoteFailures bool) { return runRemoteSyncOnce(appCfg, database, rh, full) }, ) + if cfg.ArtifactFolder != "" { + runArtifactFolderSync(appCfg, database, cfg.ArtifactFolder, cfg) + } reportRemoteFailures(failures) - return len(failures) > 0 + return len(failures) > 0, nil +} + +func syncTransport(appCfg *config.Config, cfg SyncConfig) (transport, error) { + if cfg.ArtifactFolder != "" { + return detectTransport(appCfg.DataDir, appCfg.AuthToken, 0) + } + return ensureTransport(appCfg, transportIntentArchiveWrite, 0) } func useDaemonForSync(tr transport) bool { @@ -279,6 +344,109 @@ func (p *remoteProgressPrinter) finishCurrent() { p.inPlace = false } +// resolveArtifactOrigin returns this machine's artifact origin. A config +// origin wins; otherwise an origin already stored in database sync state -- +// for example one minted by an incoming peer exchange on serve before the +// config ever initialized an origin -- is promoted into the config; otherwise +// a new origin is generated and persisted. Without the promotion step, CLI +// sync would generate a second origin that serve later adopts as +// authoritative, stranding metadata events published under the DB origin. +// The resolved config authority is reconciled back into database sync state so +// DB-only consumers such as direct PG push use the same canonical identity. +func resolveArtifactOrigin( + appCfg config.Config, database *db.DB, +) (string, error) { + var origin string + var err error + if appCfg.ArtifactOriginID == "" { + stored, readErr := artifact.StoredOrigin(database) + if readErr != nil { + return "", readErr + } + if stored != "" { + origin, err = appCfg.AdoptArtifactOriginID(stored) + } else { + origin, err = appCfg.EnsureArtifactOriginID() + } + } else { + origin, err = appCfg.EnsureArtifactOriginID() + } + if err != nil { + return "", err + } + if err := artifact.AdoptOrigin(database, origin); err != nil { + return "", fmt.Errorf("reconciling artifact origin in database: %w", err) + } + return origin, nil +} + +func runArtifactFolderSync( + appCfg config.Config, database *db.DB, target string, cfg SyncConfig, +) { + origin, err := resolveArtifactOrigin(appCfg, database) + if err != nil { + fatal("artifact sync origin: %v", err) + } + ctx := context.Background() + res, err := syncArtifactFolder( + ctx, appCfg, database, target, origin, artifactPeerToken(cfg), + cfg.AllowInsecure, cfg.Init, nil, + ) + if err != nil { + fatal("artifact sync: %v", err) + } + printArtifactSyncSummary(res, cfg.Init) + if !cfg.NoGC { + autoGCAfterFolderSync(ctx, appCfg.DataDir, target, cfg.GCGrace) + } +} + +// artifactPeerToken resolves the bearer token for an HTTP peer target. Tokens +// are never inferred from local server auth because an explicit peer URL may +// point at an untrusted endpoint. +func artifactPeerToken(cfg SyncConfig) string { + return cfg.Token +} + +func syncArtifactFolder( + ctx context.Context, + appCfg config.Config, + database *db.DB, + target string, + origin string, + token string, + allowInsecure bool, + baselineMetadata bool, + onDataChanged func(), +) (artifact.SyncResult, error) { + if !artifact.IsFolderTarget(target) && !artifact.IsHTTPTarget(target) && !artifact.IsObjectTarget(target) { + return artifact.SyncResult{}, fmt.Errorf( + "artifact sync supports local folder, http(s) peer, or s3:// object-store targets: %s", + target, + ) + } + return artifact.Sync(ctx, database, artifact.SyncOptions{ + DataDir: appCfg.DataDir, + Target: target, + Origin: origin, + Token: token, + AllowInsecure: allowInsecure, + BaselineMetadata: baselineMetadata, + OnDataChanged: onDataChanged, + }) +} + +func printArtifactSyncSummary(res artifact.SyncResult, init bool) { + label := "Artifact sync" + if init { + label = "Artifact sync initialized" + } + fmt.Printf( + "%s (%s): exported %d sessions, imported %d sessions / %d messages / %d metadata events\n", + label, res.Origin, res.ExportedSessions, res.ImportedSessions, res.ImportedMessages, res.ImportedMetadata, + ) +} + // syncLocalAndRemotes runs the local sync, then the configured // remote hosts. A local resync (forced via --full or an automatic // data-version resync) forces every remote sync full as well, so diff --git a/cmd/agentsview/sync_gc.go b/cmd/agentsview/sync_gc.go new file mode 100644 index 000000000..5176458a6 --- /dev/null +++ b/cmd/agentsview/sync_gc.go @@ -0,0 +1,131 @@ +package main + +import ( + "context" + "fmt" + "io" + "log" + "path/filepath" + "time" + + "github.com/spf13/cobra" + "go.kenn.io/agentsview/internal/artifact" +) + +const defaultArtifactGCGrace = 7 * 24 * time.Hour + +// SyncGCConfig holds parsed options for artifact sync garbage collection. +type SyncGCConfig struct { + Target string + Grace time.Duration + DryRun bool +} + +func newSyncGCCommand() *cobra.Command { + var cfg SyncGCConfig + cfg.Grace = defaultArtifactGCGrace + cmd := &cobra.Command{ + Use: "gc ", + Short: "Garbage collect superseded sync artifacts", + Long: "Garbage collect superseded artifact sync files from a local\n" + + "artifact folder. GC keeps the latest checkpoint per origin,\n" + + "keeps every manifest, segment, and raw artifact reachable from\n" + + "that checkpoint, and only removes unreferenced files older than\n" + + "the grace window. Origins without checkpoints are skipped.", + SilenceUsage: true, + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + cfg.Target = args[0] + return runSyncGC(cmd, cfg) + }, + } + cmd.Flags().DurationVar( + &cfg.Grace, + "grace", + defaultArtifactGCGrace, + "Minimum age before unreferenced artifacts can be deleted", + ) + cmd.Flags().BoolVar( + &cfg.DryRun, + "dry-run", + false, + "Log artifacts that would be deleted without removing them", + ) + return cmd +} + +func runSyncGC(cmd *cobra.Command, cfg SyncGCConfig) error { + if !artifact.IsFolderTarget(cfg.Target) { + return fmt.Errorf( + "artifact gc currently supports local folder targets only: %s", + cfg.Target, + ) + } + res, err := artifact.GarbageCollect(cmd.Context(), artifact.GCOptions{ + Root: cfg.Target, + Grace: cfg.Grace, + DryRun: cfg.DryRun, + Logf: log.Printf, + }) + if err != nil { + return fmt.Errorf("artifact gc: %w", err) + } + printArtifactGCSummary(cmd.OutOrStdout(), res) + return nil +} + +// autoGCAfterFolderSync garbage-collects superseded artifacts from the local +// store and the shared folder target after a successful folder sync. Both are +// collected with the same synced checkpoint view so a later set-union exchange +// cannot re-propagate the deleted files. GC is opportunistic: failures are +// logged, not fatal. Non-folder targets are skipped because this operation +// cannot prune their remote history; collecting only the local store would make +// the next set-union exchange download every remote-only artifact again. +func autoGCAfterFolderSync(ctx context.Context, dataDir, target string, grace time.Duration) { + if !artifact.IsFolderTarget(target) { + return + } + if grace <= 0 { + grace = defaultArtifactGCGrace + } + roots := []string{filepath.Join(dataDir, "artifacts"), target} + for _, root := range roots { + res, err := artifact.GarbageCollect(ctx, artifact.GCOptions{ + Root: root, + Grace: grace, + Logf: log.Printf, + }) + if err != nil { + log.Printf("artifact gc (%s): %v", root, err) + continue + } + if res.Deleted > 0 { + log.Printf( + "artifact gc (%s): deleted %d artifact(s) (%s)", + root, res.Deleted, formatBytes(res.BytesDeleted), + ) + } + } +} + +func printArtifactGCSummary(w io.Writer, res artifact.GCResult) { + action := "deleted" + count := res.Deleted + bytes := res.BytesDeleted + if res.DryRun { + action = "would delete" + count = res.Eligible + bytes = res.BytesEligible + } + fmt.Fprintf( + w, + "Artifact GC: scanned %d origin(s), skipped %d without checkpoints, found %d unreferenced artifact(s), kept %d within grace, %s %d artifact(s) (%s)\n", + res.Origins, + res.SkippedOrigins, + res.Candidates, + res.KeptByGrace, + action, + count, + formatBytes(bytes), + ) +} diff --git a/cmd/agentsview/sync_gc_auto_test.go b/cmd/agentsview/sync_gc_auto_test.go new file mode 100644 index 000000000..d020c3050 --- /dev/null +++ b/cmd/agentsview/sync_gc_auto_test.go @@ -0,0 +1,142 @@ +package main + +import ( + "context" + "io/fs" + "os" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/require" + "go.kenn.io/agentsview/internal/artifact" + "go.kenn.io/agentsview/internal/db" +) + +// autoGCTestStore builds a data dir whose artifact store contains one superseded +// session manifest/segment/checkpoint (from a first export) plus the live set +// (from a second export after a content change), and mirrors it to a folder +// target. Every file is backdated past the grace window so the unreferenced +// ones are eligible for collection. +func autoGCTestStore(t *testing.T, target string) (dataDir, origin string) { + t.Helper() + ctx := context.Background() + dataDir = t.TempDir() + origin = "laptop-a1b2c3" + artifacts := filepath.Join(dataDir, "artifacts") + + database, err := db.Open(filepath.Join(t.TempDir(), "test.db")) + require.NoError(t, err) + t.Cleanup(func() { database.Close() }) + + require.NoError(t, database.UpsertSession(db.Session{ + ID: "sess-1", + Project: "alpha", + Machine: "local", + Agent: "claude", + MessageCount: 2, + UserMessageCount: 1, + CreatedAt: "2026-06-14T01:02:03Z", + })) + require.NoError(t, database.ReplaceSessionMessages("sess-1", []db.Message{ + {SessionID: "sess-1", Ordinal: 0, Role: "user", Content: "hello", ContentLength: 5}, + {SessionID: "sess-1", Ordinal: 1, Role: "assistant", Content: "world", ContentLength: 5}, + })) + _, err = artifact.Export(ctx, database, artifacts, origin) + require.NoError(t, err) + + // Change content so the next export supersedes the first manifest/segment. + require.NoError(t, database.ReplaceSessionMessages("sess-1", []db.Message{ + {SessionID: "sess-1", Ordinal: 0, Role: "user", Content: "hello", ContentLength: 5}, + {SessionID: "sess-1", Ordinal: 1, Role: "assistant", Content: "planet", ContentLength: 6}, + })) + _, err = artifact.Export(ctx, database, artifacts, origin) + require.NoError(t, err) + + require.NoError(t, artifact.CopyUnion(artifacts, target)) + + old := time.Now().Add(-72 * time.Hour) + backdateTree(t, artifacts, old) + backdateTree(t, target, old) + return dataDir, origin +} + +func backdateTree(t *testing.T, root string, ts time.Time) { + t.Helper() + require.NoError(t, filepath.Walk(root, func(p string, _ fs.FileInfo, err error) error { + if err != nil { + return err + } + return os.Chtimes(p, ts, ts) + })) +} + +func countFiles(t *testing.T, dir string) int { + t.Helper() + entries, err := os.ReadDir(dir) + if os.IsNotExist(err) { + return 0 + } + require.NoError(t, err) + n := 0 + for _, e := range entries { + if !e.IsDir() { + n++ + } + } + return n +} + +func TestAutoGCAfterFolderSyncCollectsBothLocalAndTarget(t *testing.T) { + target := t.TempDir() + dataDir, origin := autoGCTestStore(t, target) + localOriginRoot := filepath.Join(dataDir, "artifacts", origin) + targetOriginRoot := filepath.Join(target, origin) + + // Both stores start with the superseded set plus the live set. + for _, root := range []string{localOriginRoot, targetOriginRoot} { + require.Equal(t, 2, countFiles(t, filepath.Join(root, artifact.KindSegments))) + require.Equal(t, 2, countFiles(t, filepath.Join(root, artifact.KindManifests))) + require.Equal(t, 2, countFiles(t, filepath.Join(root, artifact.KindCheckpoints))) + } + + autoGCAfterFolderSync(context.Background(), dataDir, target, time.Hour) + + // Only the live set remains, in both the local store and the shared target, + // so a later set-union exchange cannot re-propagate the deleted files. + for _, root := range []string{localOriginRoot, targetOriginRoot} { + require.Equal(t, 1, countFiles(t, filepath.Join(root, artifact.KindSegments))) + require.Equal(t, 1, countFiles(t, filepath.Join(root, artifact.KindManifests))) + require.Equal(t, 1, countFiles(t, filepath.Join(root, artifact.KindCheckpoints))) + } +} + +func TestAutoGCAfterFolderSyncSkipsLocalGCForNonFolderTarget(t *testing.T) { + for _, tc := range []struct { + name string + target string + }{ + {name: "http", target: "https://peer.example.test:8080"}, + {name: "s3", target: "s3://bucket/artifacts"}, + } { + t.Run(tc.name, func(t *testing.T) { + // A non-folder target cannot be pruned by the same operation. Use a + // placeholder folder for its remote history: neither side may be + // collected, or the next set-union exchange would download the + // remote-only files again. + untouched := t.TempDir() + dataDir, origin := autoGCTestStore(t, untouched) + localOriginRoot := filepath.Join(dataDir, "artifacts", origin) + untouchedOriginRoot := filepath.Join(untouched, origin) + + autoGCAfterFolderSync(context.Background(), dataDir, tc.target, time.Hour) + + // Both stores retain the superseded set until a transport can prune + // them together. + require.Equal(t, 2, countFiles(t, + filepath.Join(localOriginRoot, artifact.KindSegments))) + require.Equal(t, 2, countFiles(t, + filepath.Join(untouchedOriginRoot, artifact.KindSegments))) + }) + } +} diff --git a/cmd/agentsview/sync_test.go b/cmd/agentsview/sync_test.go index d9535695a..1b65e6f4b 100644 --- a/cmd/agentsview/sync_test.go +++ b/cmd/agentsview/sync_test.go @@ -15,6 +15,7 @@ import ( "path/filepath" "strconv" "strings" + "sync/atomic" "testing" "time" @@ -527,12 +528,55 @@ func TestDoSyncUsesDaemonRouteWhenWritableDaemonRunning(t *testing.T) { registerSyncRouteTestRuntime(t, env.DataDir, ts.URL) - hadFailures := doSync(SyncConfig{}) + hadFailures, err := doSync(SyncConfig{}) + require.NoError(t, err) require.False(t, hadFailures) assert.True(t, syncCalled) env.assertNoLocalDB(t) } +func TestDoSyncRejectsArtifactTargetWhenWritableDaemonRunning(t *testing.T) { + env := newSyncCLIEnv(t) + target := t.TempDir() + + var syncCalled bool + ts := syncRouteTestServer(t, func(w http.ResponseWriter, r *http.Request) { + syncCalled = true + writeDoneSSE(t, w, agentsync.SyncStats{Synced: 7}) + }) + + registerSyncRouteTestRuntime(t, env.DataDir, ts.URL) + + hadFailures, err := doSync(SyncConfig{ArtifactFolder: target}) + require.Error(t, err) + assert.False(t, hadFailures) + assert.False(t, syncCalled, "daemon sync must not consume an artifact target") + assert.Contains(t, err.Error(), "artifact sync") + assert.Contains(t, err.Error(), "writable daemon") + env.assertNoLocalDB(t) +} + +func TestSyncTransportArtifactTargetUsesDirectDBWithoutAutoStartingDaemon(t *testing.T) { + env := newSyncCLIEnv(t) + var autostartCalled bool + stubStartBackgroundServeForTransport(t, func( + context.Context, *config.Config, time.Duration, + ) (*DaemonRuntime, error) { + autostartCalled = true + return nil, errors.New("unexpected daemon autostart") + }) + + appCfg := config.Config{ + DataDir: env.DataDir, + } + tr, err := syncTransport(&appCfg, SyncConfig{ + ArtifactFolder: t.TempDir(), + }) + require.NoError(t, err) + assert.Equal(t, transportDirect, tr.Mode) + assert.False(t, autostartCalled) +} + func TestDoSyncFullUsesDaemonResyncRoute(t *testing.T) { env := newSyncCLIEnv(t) @@ -554,7 +598,9 @@ func TestDoSyncFullUsesDaemonResyncRoute(t *testing.T) { var hadFailures bool out := captureStdout(t, func() { - hadFailures = doSync(SyncConfig{Full: true}) + var err error + hadFailures, err = doSync(SyncConfig{Full: true}) + require.NoError(t, err) }) require.False(t, hadFailures) assert.True(t, resyncCalled) @@ -590,13 +636,14 @@ func TestDoSyncRemoteHostUsesDaemonRouteWhenWritableDaemonRunning(t *testing.T) ts := remoteSyncRouteTestServer(t, handler) registerSyncRouteTestRuntime(t, env.DataDir, ts.URL) - hadFailures := doSync(SyncConfig{ + hadFailures, err := doSync(SyncConfig{ Host: "devbox", User: "alice", Port: 2222, Full: true, }) + require.NoError(t, err) require.False(t, hadFailures) assert.False(t, got.IncludeLocal) assert.True(t, got.Full) @@ -627,10 +674,12 @@ func TestDoSyncRemoteHostPrintsDaemonProgress(t *testing.T) { registerSyncRouteTestRuntime(t, env.DataDir, ts.URL) var hadFailures bool + var err error out := captureStdout(t, func() { - hadFailures = doSync(SyncConfig{Host: "devbox"}) + hadFailures, err = doSync(SyncConfig{Host: "devbox"}) }) + require.NoError(t, err) require.False(t, hadFailures) assert.Contains(t, out, "Running sync with remotes via daemon...") assert.Contains(t, out, "Resolving agent directories on devbox") @@ -709,8 +758,9 @@ user = "robot" ts := remoteSyncRouteTestServer(t, handler) registerSyncRouteTestRuntime(t, env.DataDir, ts.URL) - hadFailures := doSync(SyncConfig{}) + hadFailures, err := doSync(SyncConfig{}) + require.NoError(t, err) require.False(t, hadFailures) assert.True(t, got.IncludeLocal) require.Len(t, got.Hosts, 1) @@ -931,7 +981,6 @@ func TestRemoteFailureDisplaySanitizesHTTPErrors(t *testing.T) { }) } } - func TestRunHTTPRemoteSyncReachesMirrorPath(t *testing.T) { manifestRequests := 0 ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -966,3 +1015,232 @@ func TestRunHTTPRemoteSyncReachesMirrorPath(t *testing.T) { assert.Equal(t, 1, manifestRequests, "configured DataDir must route HTTP sync through the manifest/mirror path") } + +func TestNewSyncCommandRegistersArtifactFolderFlag(t *testing.T) { + cmd := newSyncCommand() + flag := cmd.Flags().Lookup("artifact-folder") + require.NotNil(t, flag) + assert.Equal(t, "", flag.DefValue) + assert.Contains(t, flag.Usage, "local-first sync artifacts") + initFlag := cmd.Flags().Lookup("init") + require.NotNil(t, initFlag) + assert.Equal(t, "false", initFlag.DefValue) + assert.Contains(t, initFlag.Usage, "Initialize artifact sync") + assert.Contains(t, cmd.Long, "do not point this at the") + assert.Contains(t, cmd.Long, "Use --init with an artifact folder") + watchFlag := cmd.Flags().Lookup("watch") + require.NotNil(t, watchFlag) + assert.Equal(t, "false", watchFlag.DefValue) + assert.Contains(t, watchFlag.Usage, "Run artifact folder sync continuously") + assert.Equal(t, defaultWatchDebounce.String(), cmd.Flags().Lookup("debounce").DefValue) + assert.Equal(t, defaultWatchInterval.String(), cmd.Flags().Lookup("interval").DefValue) + assert.Contains(t, cmd.Long, "Use --watch with an artifact folder") +} + +func TestApplySyncArtifactTargetUsesPositionalArgument(t *testing.T) { + cfg := SyncConfig{} + err := applySyncArtifactTarget(&cfg, []string{"/tmp/agentsview-share"}, false) + require.NoError(t, err) + assert.Equal(t, "/tmp/agentsview-share", cfg.ArtifactFolder) +} + +func TestApplySyncArtifactTargetRejectsArgumentAndFlag(t *testing.T) { + cfg := SyncConfig{ArtifactFolder: "/tmp/from-flag"} + err := applySyncArtifactTarget(&cfg, []string{"/tmp/from-arg"}, true) + require.Error(t, err) + assert.Contains(t, err.Error(), "both as an argument") + assert.Equal(t, "/tmp/from-flag", cfg.ArtifactFolder) +} + +func TestValidateSyncConfigInitRequiresArtifactFolder(t *testing.T) { + err := validateSyncConfig(SyncConfig{Init: true}) + require.Error(t, err) + assert.Contains(t, err.Error(), "--init requires") +} + +func TestValidateSyncConfigInitRejectsHost(t *testing.T) { + err := validateSyncConfig(SyncConfig{ + Init: true, + Host: "remote", + ArtifactFolder: "/tmp/agentsview-share", + }) + require.Error(t, err) + assert.Contains(t, err.Error(), "cannot be combined with --host") +} + +func TestValidateSyncConfigInitAllowsArtifactFolder(t *testing.T) { + err := validateSyncConfig(SyncConfig{ + Init: true, + ArtifactFolder: "/tmp/agentsview-share", + }) + require.NoError(t, err) +} + +func TestValidateSyncConfigWatchRequiresArtifactFolder(t *testing.T) { + err := validateSyncConfig(SyncConfig{Watch: true}) + require.Error(t, err) + assert.Contains(t, err.Error(), "--watch requires") +} + +func TestValidateSyncConfigWatchRejectsHost(t *testing.T) { + err := validateSyncConfig(SyncConfig{ + Watch: true, + Host: "remote", + ArtifactFolder: "/tmp/agentsview-share", + }) + require.Error(t, err) + assert.Contains(t, err.Error(), "cannot be combined with --host") +} + +func TestValidateSyncConfigWatchAllowsArtifactFolder(t *testing.T) { + err := validateSyncConfig(SyncConfig{ + Watch: true, + ArtifactFolder: "/tmp/agentsview-share", + Debounce: time.Second, + Interval: time.Minute, + }) + require.NoError(t, err) +} + +func TestValidateSyncConfigRejectsHostWithArtifactTarget(t *testing.T) { + err := validateSyncConfig(SyncConfig{ + Host: "remote", + ArtifactFolder: "/tmp/agentsview-share", + }) + require.Error(t, err) + assert.Contains(t, err.Error(), "--host cannot be combined with an artifact target") +} + +func TestValidateSyncConfigAllowsHostWithoutArtifactTarget(t *testing.T) { + require.NoError(t, validateSyncConfig(SyncConfig{Host: "remote"})) +} + +func TestArtifactPeerTokenDoesNotReuseLocalAuthToken(t *testing.T) { + got := artifactPeerToken(SyncConfig{ + ArtifactFolder: "https://peer.example.test", + }) + assert.Empty(t, got) + + got = artifactPeerToken(SyncConfig{ + ArtifactFolder: "https://peer.example.test", + Token: "peer-secret", + }) + assert.Equal(t, "peer-secret", got) +} + +func TestSyncArtifactFolderPlumbsInsecurePeerOptIn(t *testing.T) { + target, requests := insecureArtifactPeerTarget(t) + dataDir := t.TempDir() + database, err := db.Open(filepath.Join(dataDir, "sessions.db")) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, database.Close()) }) + + _, err = syncArtifactFolder( + context.Background(), config.Config{DataDir: dataDir}, database, + target, "desk-a1b2c3", "", true, false, nil, + ) + + require.NoError(t, err) + assert.Positive(t, requests.Load(), + "one-shot sync must reach an explicitly allowed plaintext peer") +} + +func insecureArtifactPeerTarget(t *testing.T) (string, *atomic.Int32) { + t.Helper() + var host net.IP + addrs, err := net.InterfaceAddrs() + require.NoError(t, err) + for _, addr := range addrs { + ip, _, parseErr := net.ParseCIDR(addr.String()) + if parseErr == nil && ip.To4() != nil && !ip.IsLoopback() && !ip.IsUnspecified() { + host = ip + break + } + } + if host == nil { + t.Skip("no non-loopback IPv4 interface available for plaintext peer plumbing test") + } + listener, err := net.Listen("tcp4", "0.0.0.0:0") + require.NoError(t, err) + var requests atomic.Int32 + peer := &httptest.Server{ + Listener: listener, + Config: &http.Server{Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests.Add(1) + w.Header().Set("Content-Type", "application/json") + switch { + case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/origins"): + _, _ = io.WriteString(w, `{"origins":[]}`) + case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/index"): + _, _ = io.WriteString(w, `{"origin":"desk-a1b2c3"}`) + case r.Method == http.MethodPost: + w.WriteHeader(http.StatusCreated) + default: + http.NotFound(w, r) + } + })}, + } + peer.Start() + t.Cleanup(peer.Close) + t.Setenv("HTTP_PROXY", "") + t.Setenv("HTTPS_PROXY", "") + t.Setenv("NO_PROXY", "*") + port := listener.Addr().(*net.TCPAddr).Port + return "http://" + net.JoinHostPort(host.String(), strconv.Itoa(port)), &requests +} + +func TestSyncGCHelpDocumentsMaintenanceFlags(t *testing.T) { + help, err := executeCommand(newRootCommand(), "sync", "gc", "--help") + require.NoError(t, err) + for _, want := range []string{"Garbage collect", "--grace", "--dry-run"} { + assert.Contains(t, help, want) + } +} + +func TestSyncGCRejectsNonFolderTarget(t *testing.T) { + _, err := executeCommand(newRootCommand(), "sync", "gc", "https://example.test/artifacts") + require.Error(t, err) + assert.Contains(t, err.Error(), "local folder targets only") +} + +func TestSyncGCDryRunPrintsSummary(t *testing.T) { + out, err := executeCommand(newRootCommand(), "sync", "gc", "--dry-run", t.TempDir()) + require.NoError(t, err) + assert.Contains(t, out, "Artifact GC:") + assert.Contains(t, out, "would delete 0 artifact(s)") +} + +func TestArtifactFolderPusherPushExportsToTarget(t *testing.T) { + dataDir := t.TempDir() + target := t.TempDir() + database, err := db.Open(filepath.Join(dataDir, "sessions.db")) + require.NoError(t, err) + t.Cleanup(func() { database.Close() }) + + dbtest.SeedSession(t, database, "sess-1", "alpha", func(s *db.Session) { + s.MessageCount = 1 + s.UserMessageCount = 1 + }) + require.NoError(t, database.ReplaceSessionMessages("sess-1", []db.Message{ + {SessionID: "sess-1", Ordinal: 0, Role: "user", Content: "hello", ContentLength: 5}, + })) + + pusher := &artifactFolderPusher{ + appCfg: config.Config{DataDir: dataDir}, + database: database, + target: target, + origin: "desk-a1b2c3", + } + require.NoError(t, pusher.push(context.Background(), reasonChange)) + + checkpoints, err := filepath.Glob( + filepath.Join(target, "desk-a1b2c3", "checkpoints", "*.json"), + ) + require.NoError(t, err) + require.Len(t, checkpoints, 1) + manifests, err := filepath.Glob( + filepath.Join(target, "desk-a1b2c3", "manifests", "*.json.zst"), + ) + require.NoError(t, err) + assert.Len(t, manifests, 1) +} diff --git a/cmd/agentsview/sync_watch.go b/cmd/agentsview/sync_watch.go new file mode 100644 index 000000000..a85a049ea --- /dev/null +++ b/cmd/agentsview/sync_watch.go @@ -0,0 +1,209 @@ +package main + +import ( + "context" + "fmt" + "log" + "os" + "os/signal" + "syscall" + "time" + + "github.com/gofrs/flock" + "go.kenn.io/agentsview/internal/config" + "go.kenn.io/agentsview/internal/db" + "go.kenn.io/agentsview/internal/parser" + syncpkg "go.kenn.io/agentsview/internal/sync" + "go.kenn.io/kit/daemon" +) + +type artifactFolderPusher struct { + appCfg config.Config + database *db.DB + engine artifactWatchSyncer + target string + origin string + token string + allowInsecure bool + gcGrace time.Duration + gcEnabled bool + // baseline publishes first-run curation metadata (--init) on the next + // push. It stays set until a push succeeds so a failed initial exchange + // retries the baseline; AppendBaselineSnapshot skips already-covered + // fields, making the retry idempotent. Pushes run on a single loop + // goroutine, so no locking is needed. + baseline bool + onDataChanged func() +} + +type artifactWatchSyncer interface { + SyncAll(context.Context, syncpkg.ProgressFunc) syncpkg.SyncStats + FlushSignals() +} + +func newArtifactWatchEngine( + database *db.DB, appCfg config.Config, +) *syncpkg.Engine { + return syncpkg.NewEngine(database, syncpkg.EngineConfig{ + AgentDirs: appCfg.AgentDirs, + IncludeCwdPrefixes: appCfg.SyncIncludeCwdPrefixes, + Machine: "local", + BlockedResultCategories: appCfg.ResultContentBlockedCategories, + }) +} + +func (p *artifactFolderPusher) push( + ctx context.Context, reason pushReason, +) error { + if p.engine != nil { + // Startup already performed a full sync, and watcher change bursts have + // already applied their targeted paths. The periodic floor covers roots + // that could not be watched, while shutdown discovery recovers changes + // still waiting in the watcher's batching window before the final export. + if reason == reasonInterval || reason == reasonShutdown { + p.engine.SyncAll(ctx, nil) + } + // Export reads session rows outside a sync operation; flush + // debounced signal recomputes so manifests carry current signals. + p.engine.FlushSignals() + } + res, err := syncArtifactFolder( + ctx, p.appCfg, p.database, p.target, p.origin, p.token, + p.allowInsecure, p.baseline, p.onDataChanged, + ) + if err != nil { + return err + } + p.baseline = false + log.Printf( + "artifact watch: exported %d sessions, imported %d sessions, %d messages, %d metadata events (%s)", + res.ExportedSessions, res.ImportedSessions, res.ImportedMessages, + res.ImportedMetadata, reason, + ) + // Collect superseded artifacts on the periodic floor and at startup, not on + // every debounced change, to keep frequent edits cheap. + if p.gcEnabled && (reason == reasonStartup || reason == reasonInterval) { + autoGCAfterFolderSync(ctx, p.appCfg.DataDir, p.target, p.gcGrace) + } + return nil +} + +// runSyncWatch runs continuous artifact folder sync: an initial local sync and +// artifact exchange, then debounced file-change exchanges and a periodic floor. +func runSyncWatch(cfg SyncConfig) { + appCfg, err := config.LoadMinimal() + if err != nil { + log.Fatalf("loading config: %v", err) + } + if err := os.MkdirAll(appCfg.DataDir, 0o755); err != nil { + log.Fatalf("creating data dir: %v", err) + } + setupLogFileNamed(appCfg.DataDir, "artifact-watch.log") + + if cfg.ArtifactFolder == "" { + fatal("artifact watch: folder target is required") + } + + debounce := cfg.Debounce + if debounce <= 0 { + debounce = defaultWatchDebounce + } + interval := cfg.Interval + if interval <= 0 { + interval = defaultWatchInterval + } + + lockPath, err := (daemon.RuntimeStore{ + Dir: appCfg.DataDir, + Prefix: "artifact-watch", + }).LockPath() + if err != nil { + fatal("artifact watch: %v", err) + } + lock := flock.New(lockPath) + locked, err := lock.TryLock() + if err != nil { + fatal("artifact watch: locking %s: %v", lockPath, err) + } + if !locked { + fatal("artifact watch: already locked (%s)", lockPath) + } + defer func() { + if rerr := lock.Unlock(); rerr != nil { + log.Printf("artifact watch: releasing lock: %v", rerr) + } + }() + + applyClassifierConfig(appCfg) + database, writeLock := mustOpenWriteDB(context.Background(), appCfg) + defer closeWriteDB(database, writeLock) + + for _, def := range parser.Registry { + if !appCfg.IsUserConfigured(def.Type) { + continue + } + warnMissingDirs(appCfg.ResolveDirs(def.Type), string(def.Type)) + } + cleanResyncTemp(appCfg.DBPath) + + origin, err := resolveArtifactOrigin(appCfg, database) + if err != nil { + fatal("artifact watch origin: %v", err) + } + + ctx, stop := signal.NotifyContext( + context.Background(), os.Interrupt, syscall.SIGTERM, + ) + defer stop() + + engine := newArtifactWatchEngine(database, appCfg) + defer engine.Close() + + didResync := cfg.Full || database.NeedsResync() + if didResync { + engine.ResyncAll(ctx, nil) + } else { + engine.SyncAll(ctx, nil) + } + if ctx.Err() != nil { + return + } + + pusher := &artifactFolderPusher{ + appCfg: appCfg, + database: database, + engine: engine, + target: cfg.ArtifactFolder, + origin: origin, + token: artifactPeerToken(cfg), + allowInsecure: cfg.AllowInsecure, + gcGrace: cfg.GCGrace, + gcEnabled: !cfg.NoGC, + baseline: cfg.Init, + } + + log.Printf( + "artifact watch: starting (origin=%q target=%q debounce=%s interval=%s)", + origin, cfg.ArtifactFolder, debounce, interval, + ) + fmt.Printf( + "agentsview sync --watch: syncing artifacts as %q to %s "+ + "(debounce %s, floor %s)\n", + origin, cfg.ArtifactFolder, debounce, interval, + ) + + if err := pusher.push(ctx, reasonStartup); err != nil { + log.Printf("artifact watch: initial sync failed: %v", err) + } + + runWatchedSink(ctx, watchedSinkConfig{ + AppConfig: appCfg, + Engine: engine, + Debounce: debounce, + Interval: interval, + LogPrefix: "artifact watch", + Push: func(c context.Context, r pushReason) error { + return pusher.push(c, r) + }, + }) +} diff --git a/cmd/agentsview/sync_watch_test.go b/cmd/agentsview/sync_watch_test.go new file mode 100644 index 000000000..1e69efd26 --- /dev/null +++ b/cmd/agentsview/sync_watch_test.go @@ -0,0 +1,337 @@ +package main + +import ( + "context" + "os" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.kenn.io/agentsview/internal/artifact" + "go.kenn.io/agentsview/internal/config" + "go.kenn.io/agentsview/internal/db" + "go.kenn.io/agentsview/internal/parser" + syncpkg "go.kenn.io/agentsview/internal/sync" + "go.kenn.io/agentsview/internal/testjsonl" +) + +type countingArtifactWatchSyncer struct { + syncAllCalls int + flushCalls int + syncAllHasDeadline bool +} + +func (s *countingArtifactWatchSyncer) SyncAll( + ctx context.Context, _ syncpkg.ProgressFunc, +) syncpkg.SyncStats { + s.syncAllCalls++ + _, s.syncAllHasDeadline = ctx.Deadline() + return syncpkg.SyncStats{} +} + +func (s *countingArtifactWatchSyncer) FlushSignals() { + s.flushCalls++ +} + +func openWatchTestDB(t *testing.T) *db.DB { + t.Helper() + database, err := db.Open(filepath.Join(t.TempDir(), "test.db")) + require.NoError(t, err) + t.Cleanup(func() { database.Close() }) + return database +} + +func seedStarredSession(t *testing.T, database *db.DB, id string) { + t.Helper() + require.NoError(t, database.UpsertSession(db.Session{ + ID: id, + Project: "alpha", + Machine: "local", + Agent: "claude", + MessageCount: 1, + UserMessageCount: 1, + CreatedAt: "2026-06-14T01:02:03Z", + })) + require.NoError(t, database.ReplaceSessionMessages(id, []db.Message{ + {SessionID: id, Ordinal: 0, Role: "user", Content: "hello", ContentLength: 5}, + })) + starred, err := database.StarSession(id) + require.NoError(t, err, "StarSession") + require.True(t, starred, "session should be newly starred") +} + +func TestArtifactFolderPusherPublishesBaselineOnFirstSuccessfulPush(t *testing.T) { + dataDir := t.TempDir() + target := t.TempDir() + database := openWatchTestDB(t) + seedStarredSession(t, database, "sess-1") + + pusher := &artifactFolderPusher{ + appCfg: config.Config{DataDir: dataDir}, + database: database, + target: target, + origin: "laptop-a1b2c3", + baseline: true, + } + require.NoError(t, pusher.push(context.Background(), reasonStartup)) + assert.False(t, pusher.baseline, + "baseline must be published at most once after a successful push") + + events, err := filepath.Glob( + filepath.Join(target, "laptop-a1b2c3", "meta", "*"), + ) + require.NoError(t, err) + assert.NotEmpty(t, events, + "--init --watch must publish baseline metadata events to the target") +} + +func TestArtifactFolderPusherRetainsBaselineAfterFailedPush(t *testing.T) { + dataDir := t.TempDir() + // A regular file as the folder target makes the exchange fail. + target := filepath.Join(t.TempDir(), "not-a-dir") + require.NoError(t, os.WriteFile(target, []byte("x"), 0o600)) + database := openWatchTestDB(t) + seedStarredSession(t, database, "sess-1") + + pusher := &artifactFolderPusher{ + appCfg: config.Config{DataDir: dataDir}, + database: database, + target: target, + origin: "laptop-a1b2c3", + baseline: true, + } + require.Error(t, pusher.push(context.Background(), reasonStartup)) + assert.True(t, pusher.baseline, + "a failed push must keep the baseline pending for the next retry") +} + +func TestArtifactFolderPusherPlumbsInsecurePeerOptIn(t *testing.T) { + target, requests := insecureArtifactPeerTarget(t) + dataDir := t.TempDir() + database := openWatchTestDB(t) + pusher := &artifactFolderPusher{ + appCfg: config.Config{DataDir: dataDir}, + database: database, + target: target, + origin: "desk-a1b2c3", + allowInsecure: true, + } + + require.NoError(t, pusher.push(context.Background(), reasonStartup)) + assert.Positive(t, requests.Load(), + "watch sync must reach an explicitly allowed plaintext peer") +} + +func TestArtifactFolderPusherRunsFullDiscoveryForIntervalAndShutdown(t *testing.T) { + dataDir := t.TempDir() + target := t.TempDir() + database := openWatchTestDB(t) + syncer := &countingArtifactWatchSyncer{} + pusher := &artifactFolderPusher{ + appCfg: config.Config{DataDir: dataDir}, + database: database, + engine: syncer, + target: target, + origin: "desk-a1b2c3", + } + + for _, reason := range []pushReason{reasonStartup, reasonChange} { + require.NoError(t, pusher.push(context.Background(), reason)) + } + assert.Zero(t, syncer.syncAllCalls, + "startup and watcher-driven pushes already synchronized local files") + assert.Equal(t, 2, syncer.flushCalls, + "every export must flush pending signal recomputes") + + require.NoError(t, pusher.push(context.Background(), reasonShutdown)) + assert.Equal(t, 1, syncer.syncAllCalls, + "shutdown must discover changes still pending in the watcher batch") + require.NoError(t, pusher.push(context.Background(), reasonInterval)) + assert.Equal(t, 2, syncer.syncAllCalls, + "the periodic floor must discover changes from unwatched roots") + assert.Equal(t, 4, syncer.flushCalls) +} + +func TestArtifactFolderPusherFlushesPendingWatchBatchOnShutdown(t *testing.T) { + database := openWatchTestDB(t) + syncer := &countingArtifactWatchSyncer{} + pusher := &artifactFolderPusher{ + appCfg: config.Config{DataDir: t.TempDir()}, + database: database, + engine: syncer, + target: t.TempDir(), + origin: "desk-a1b2c3", + } + debounceArmed := make(chan struct{}) + neverFire := make(chan time.Time) + loop := &pushLoop{ + debounce: time.Hour, + dirty: make(chan struct{}, 1), + floor: make(chan time.Time), + after: func(time.Duration) <-chan time.Time { + close(debounceArmed) + return neverFire + }, + push: pusher.push, + flushTimeout: time.Second, + } + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan struct{}) + go func() { + loop.Run(ctx) + close(done) + }() + + loop.NotifyDirty() + select { + case <-debounceArmed: + case <-time.After(5 * time.Second): + require.FailNow(t, "pending watch batch did not arm the debounce") + } + cancel() + select { + case <-done: + case <-time.After(5 * time.Second): + require.FailNow(t, "shutdown flush did not complete") + } + + assert.Equal(t, 1, syncer.syncAllCalls, + "shutdown must replace the pending watcher batch with full discovery") + assert.True(t, syncer.syncAllHasDeadline, + "shutdown discovery must use the bounded flush context") +} + +func TestArtifactFolderPusherShutdownDiscoversPendingFilesystemChange(t *testing.T) { + claudeDir := t.TempDir() + projectDir := filepath.Join(claudeDir, "-Users-alice-work") + require.NoError(t, os.MkdirAll(projectDir, 0o755)) + content := testjsonl.NewSessionBuilder(). + AddClaudeUser("2026-01-01T00:00:00Z", "pending change", "/Users/alice/work/project"). + AddClaudeAssistant("2026-01-01T00:00:01Z", "saved"). + String() + require.NoError(t, os.WriteFile( + filepath.Join(projectDir, "pending-session.jsonl"), []byte(content), 0o644, + )) + + dataDir := t.TempDir() + target := t.TempDir() + database := openWatchTestDB(t) + appCfg := config.Config{ + DataDir: dataDir, + AgentDirs: map[parser.AgentType][]string{ + parser.AgentClaude: {claudeDir}, + }, + } + engine := newArtifactWatchEngine(database, appCfg) + t.Cleanup(engine.Close) + pusher := &artifactFolderPusher{ + appCfg: appCfg, + database: database, + engine: engine, + target: target, + origin: "desk-a1b2c3", + } + + require.NoError(t, pusher.push(context.Background(), reasonShutdown)) + + session, err := database.GetSession(context.Background(), "pending-session") + require.NoError(t, err) + require.NotNil(t, session, + "the shutdown exchange must ingest a change still pending in the watcher batch") + require.NotNil(t, session.FirstMessage) + assert.Equal(t, "pending change", *session.FirstMessage) + summary, err := artifact.CheckpointSummary(target, "desk-a1b2c3") + require.NoError(t, err) + require.True(t, summary.Found) + assert.Equal(t, 1, summary.SessionCount, + "the shutdown exchange must publish the newly ingested session") +} + +func TestArtifactWatchEngineHonorsConfiguredCwdPrefixes(t *testing.T) { + claudeDir := t.TempDir() + projectDir := filepath.Join(claudeDir, "-Users-alice-work") + require.NoError(t, os.MkdirAll(projectDir, 0o755)) + writeSession := func(name, cwd, prompt string) { + content := testjsonl.NewSessionBuilder(). + AddClaudeUser("2026-01-01T00:00:00Z", prompt, cwd). + AddClaudeAssistant("2026-01-01T00:00:01Z", "ok"). + String() + require.NoError(t, os.WriteFile( + filepath.Join(projectDir, name+".jsonl"), []byte(content), 0o644, + )) + } + writeSession("allowed-session", "/Users/alice/work/project", "allowed") + writeSession("blocked-session", "/Users/alice/personal/project", "blocked") + + database := openWatchTestDB(t) + engine := newArtifactWatchEngine(database, config.Config{ + AgentDirs: map[parser.AgentType][]string{ + parser.AgentClaude: {claudeDir}, + }, + SyncIncludeCwdPrefixes: []string{"/Users/alice/work"}, + }) + t.Cleanup(engine.Close) + stats := engine.SyncAll(context.Background(), nil) + require.Equal(t, 1, stats.Synced) + + allowed, err := database.GetSession(context.Background(), "allowed-session") + require.NoError(t, err) + require.NotNil(t, allowed) + assert.Equal(t, "allowed", *allowed.FirstMessage) + blocked, err := database.GetSession(context.Background(), "blocked-session") + require.NoError(t, err) + assert.Nil(t, blocked, + "watch mode must not ingest sessions outside sync_include_cwd_prefixes") +} + +func TestResolveArtifactOriginPromotesDBOriginIntoConfig(t *testing.T) { + dir := t.TempDir() + appCfg := config.Config{DataDir: dir} + database := openWatchTestDB(t) + require.NoError(t, artifact.AdoptOrigin(database, "laptop-a1b2c3"), + "seed DB-only origin") + + origin, err := resolveArtifactOrigin(appCfg, database) + require.NoError(t, err) + assert.Equal(t, "laptop-a1b2c3", origin, + "a DB-only origin must be reused, not replaced by a generated one") + + data, err := os.ReadFile(filepath.Join(dir, "config.toml")) + require.NoError(t, err) + assert.Contains(t, string(data), `artifact_origin_id = "laptop-a1b2c3"`, + "the DB origin must be promoted into config so serve adopts the same origin") +} + +func TestResolveArtifactOriginConfigWinsOverDB(t *testing.T) { + dir := t.TempDir() + appCfg := config.Config{DataDir: dir, ArtifactOriginID: "desktop-d4e5f6"} + database := openWatchTestDB(t) + require.NoError(t, artifact.AdoptOrigin(database, "laptop-a1b2c3"), + "seed DB origin") + + origin, err := resolveArtifactOrigin(appCfg, database) + require.NoError(t, err) + assert.Equal(t, "desktop-d4e5f6", origin) + + stored, err := artifact.StoredOrigin(database) + require.NoError(t, err) + assert.Equal(t, "desktop-d4e5f6", stored, + "the authoritative config origin must replace a divergent DB origin") +} + +func TestResolveArtifactOriginGeneratesWhenAbsentEverywhere(t *testing.T) { + dir := t.TempDir() + appCfg := config.Config{DataDir: dir} + database := openWatchTestDB(t) + + origin, err := resolveArtifactOrigin(appCfg, database) + require.NoError(t, err) + assert.Regexp(t, `^[a-z0-9]+(?:-[a-z0-9]+)*-[0-9a-f]{6}$`, origin) + + stored, err := artifact.StoredOrigin(database) + require.NoError(t, err) + assert.Equal(t, origin, stored, + "a directly generated CLI origin must be available to DB-only consumers") +} diff --git a/docs/artifact-sync.md b/docs/artifact-sync.md new file mode 100644 index 000000000..25de34197 --- /dev/null +++ b/docs/artifact-sync.md @@ -0,0 +1,248 @@ +--- +title: Trusted-Fleet Artifact Sync +description: Sync AgentsView archives between trusted personal machines without copying the live SQLite database +--- + +# Trusted-Fleet Artifact Sync + +Artifact sync exchanges immutable AgentsView artifacts between machines and +imports them into each machine's local SQLite archive. It is local-first: every +machine keeps its own complete database, and transports only move +content-addressed session artifacts plus metadata events. + +Use it for a fully trusted personal fleet: your laptop, desktop, home server, +NAS, or object-store bucket. Do not treat artifact sync as a team-sharing +security boundary. A peer that can write to the shared artifact target can +publish sessions and metadata for the fleet. + +## When To Use It + +Artifact sync makes sense when you want multiple machines to converge on the +same session archive without running PostgreSQL as the coordination point. + +It is a good fit for: + +- laptop plus desktop archives +- NAS, Syncthing, Dropbox, or rclone-backed rendezvous folders +- S3-compatible buckets such as MinIO, Backblaze B2, or AWS S3 +- trusted always-on AgentsView peers over HTTP + +Use [PostgreSQL Sync](/pg-sync/) or [DuckDB Mirror](/duckdb/) when you want a +read-only aggregation or analytics mirror. Those backends are still mirrors: +SQLite remains the local write/archive database, and artifact sync projects +foreign artifacts into ordinary SQLite rows before they can be pushed onward. + +## Quick Start + +Use a dedicated artifact share folder: + +```bash +agentsview sync --init /path/to/agentsview-artifacts +agentsview sync /path/to/agentsview-artifacts +``` + +Run `--init` once on each machine. It creates or adopts that machine's artifact +origin, backfills local sessions into the local artifact store, exchanges +artifacts with the target, and imports peer artifacts already present there. + +CLI artifact sync needs exclusive write access to the local archive. If a local +`agentsview` daemon is running, stop it first with `agentsview serve stop` and +retry. A running server still participates in a fleet as an +[HTTP peer](#http-peer) target for other machines. + +To keep a machine exchanging artifacts while it is online, run watch mode: + +```bash +agentsview sync --watch /path/to/agentsview-artifacts +``` + +Watch mode runs an initial local sync and artifact exchange, debounces local +session-file changes, retries failed exchanges on later changes or interval +ticks, and performs a final best-effort exchange on shutdown. + +## Targets + +### Folder + +```bash +agentsview sync /path/to/agentsview-artifacts +``` + +The folder may live on a local disk, NAS mount, Syncthing folder, Dropbox +folder, NFS share, or rclone-mounted bucket. The folder must be dedicated to +artifact sync. Do not point artifact sync at: + +- `AGENTSVIEW_DATA_DIR` +- the live SQLite database file or its WAL/SHM files +- a whole AgentsView data directory +- raw agent directories that contain live database files + +Copying the live SQLite database or the whole data directory with a general +file-sync tool is unsafe. Artifact sync exists specifically to avoid that. + +### HTTP Peer + +An AgentsView server can expose artifact exchange routes behind the existing +Bearer-token auth middleware: + +```bash +agentsview sync https://desktop:8080 --token +``` + +HTTP peer sync only sends an `Authorization` header when `--token` is provided. +It does not reuse the local server's `auth_token` for explicit peer URLs. HTTP +peer sync rejects redirects, so configure the final artifact API URL directly. +Credentials and artifact bodies are not forwarded to a redirect destination. + +Non-loopback peers require HTTPS by default. Plain `http://` remains available +for `localhost`, `127.0.0.0/8`, and `::1`. To connect to a remote plaintext peer +on a trusted test network, LAN, or VPN, opt in explicitly: + +```bash +agentsview sync http://desktop:8080 --token --allow-insecure +``` + +The override works for one-shot and `--watch` sync and logs a warning. It sends +the bearer token and full archive content without transport encryption, so use +HTTPS through a reverse proxy or VPN termination whenever possible. If you +expose a server beyond loopback, enable authentication and protect the token +like write access to the full archive. + +The HTTP client pulls every missing artifact from the peer and posts every local +artifact the peer is missing. Garbage collection of superseded artifacts on the +remote peer is the peer's own responsibility. + +### S3-Compatible Object Storage + +```bash +export AWS_ACCESS_KEY_ID=... +export AWS_SECRET_ACCESS_KEY=... +agentsview sync s3://my-bucket/agentsview +``` + +Credentials come from `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, and optional +`AWS_SESSION_TOKEN`. Region resolves from `AGENTSVIEW_S3_REGION`, then +`AWS_REGION`, defaulting to `us-east-1`. + +For MinIO, Backblaze B2, or another S3-compatible service, set +`AGENTSVIEW_S3_ENDPOINT`. A custom endpoint automatically uses path-style +addressing; `AGENTSVIEW_S3_PATH_STYLE=true` forces it otherwise. + +Custom endpoints default to HTTPS when no scheme is given. Plain `http://` is +accepted only for loopback hosts or when +`AGENTSVIEW_ALLOW_INSECURE_S3_ENDPOINT=true` is explicitly set. Use that +override only on a trusted test or private network because requests and artifact +content travel without TLS. Artifact sync rejects redirects. If a corrupt remote +object must be deleted so a later upload can heal the bucket, that deletion is +HTTPS-only, even when the insecure override permits other requests over HTTP. + +## How It Works + +Each install has a stable artifact origin ID. Locally owned sessions keep their +ordinary SQLite IDs and `machine='local'`. Foreign sessions are imported as +`~` with `machine=`. Source, parent, and +subagent relationship IDs are rewritten the same way before import, so SQLite, +PostgreSQL, and DuckDB see the same ordinary session graph after projection. + +The artifact store lives beside, not inside, the database: + +```text +$AGENTSVIEW_DATA_DIR/artifacts// + checkpoints/cp-.json + manifests/.json.zst + segments/.ndjson.zst + meta/-.json + raw/ +``` + +Artifact kinds: + +- **checkpoints** list the current manifest hash for each session published by + an origin +- **manifests** hold the canonical session header, usage events, and segment + references +- **segments** hold canonical message NDJSON +- **metadata events** record user edits such as rename, trash/restore, star, + pin/unpin, and delete-everywhere +- **raw artifacts** are optional source snapshots when a parser can provide a + safe regular-file snapshot + +All artifact files are immutable. Folder writes use no-replace semantics, and S3 +writes use conditional create, so repeated syncs are idempotent set-union +operations. + +## Metadata And Deletes + +A machine records metadata events only after it has an artifact origin, which it +gets by running `agentsview sync --init`, running any artifact sync, or +receiving a peer exchange. Until then curation stays local and writes nothing +under `artifacts/`; the `--init` baseline snapshot publishes the accumulated +curation state when the machine later joins a fleet. + +User curation converges through metadata events. Rename, trash/restore, star, +and pin changes are replayed deterministically with hybrid logical clocks. If +two peers edit the same metadata field close together, AgentsView records the +losing value in the local conflict log while still deriving one deterministic +current value. Conflicts and per-origin sync status are visible on the Peers +page in the UI, and a conflicted session shows a fork badge in its header. + +Emptying local trash is local-only. Fleet-wide permanent delete is explicit: +delete-everywhere writes a purge event and an exclusion tombstone so peers do +not resurrect the session from older artifacts. + +Checkpoint absence is never a deletion signal. A missing artifact, truncated +checkpoint, offline peer, or old target cannot remove local data. + +## Version And Failure Handling + +Artifact readers ignore unknown JSON fields. Unknown future metadata operations +are marked applied and skipped. Artifacts with a future format version are +deferred, not treated as successful imports, so older AgentsView versions keep +syncing the artifact kinds and versions they understand. + +Manifests that reference missing segments are also deferred. Import watermarks +advance only after all referenced content is hash-verified and applied. + +A corrupt artifact — a hash-mismatched segment or manifest, an undecodable +compressed file, or an unparseable checkpoint or metadata event — is quarantined +under a `.corrupt` suffix and skipped, so one damaged file never aborts a sync +or spreads to other machines. Read and export paths fall back to the previous +valid checkpoint, folder exchanges replace a quarantined copy when the share +still holds a valid one, and metadata events whose timestamps do not parse are +rejected at write time and skipped at replay. An object corrupt in place inside +an S3 bucket is eligible for remote deletion when a peer fetches it because it +is missing locally and validation fails. AgentsView attempts that deletion only +over HTTPS; if it succeeds, a later push from a valid holder can re-upload the +object. A corrupt remote checkpoint found while comparing a checkpoint already +present locally is retained for its owner to repair. Over permitted HTTP, +corrupt remote objects are retained and skipped; automatic deletion does not +occur. + +## Garbage Collection + +Folder syncs collect superseded artifacts automatically after each exchange, in +both the local artifact store and the folder target, once artifacts are older +than a seven-day grace window. Tune the window with `--gc-grace` or disable +automatic collection with `--no-gc`. + +The `agentsview sync gc` command runs the same conservative collection manually, +with `--dry-run` to preview: + +```bash +agentsview sync gc --dry-run /path/to/agentsview-artifacts +agentsview sync gc --grace 168h /path/to/agentsview-artifacts +``` + +GC keeps the latest checkpoint for each origin and every manifest, segment, and +raw artifact reachable from it. Origins without checkpoints are skipped rather +than interpreted as deleted. + +## Availability + +Two intermittent machines only sync directly while both are online and can reach +the same target. A NAS folder, always-on home server, S3-compatible bucket, +cloud-synced folder, or always-running AgentsView peer can act as a rendezvous. + +That rendezvous is a deployment convention, not a privileged architecture: +AgentsView still treats every participant as a peer and keeps the complete local +archive on each machine. diff --git a/docs/index.md b/docs/index.md index c9416c1f1..ca3a62f20 100644 --- a/docs/index.md +++ b/docs/index.md @@ -178,8 +178,10 @@ See [Activity](/activity/) for the full reference. AgentsView reads the session files that your [AI coding agents](/configuration/#session-discovery) leave on your machine and gives you a local-first desktop and web app to work with them. By default -everything stays on your machine. Optionally, [PostgreSQL sync](/pg-sync/) can -push session data to a shared database for team or multi-machine setups. +everything stays on your machine. Optionally, [artifact sync](/artifact-sync/) +can converge a trusted personal fleet without copying the live SQLite database, +and [PostgreSQL sync](/pg-sync/) can push session data to a shared database for +team or multi-machine dashboards.
diff --git a/docs/zensical.toml b/docs/zensical.toml index 3a8cf872e..5ed602cce 100644 --- a/docs/zensical.toml +++ b/docs/zensical.toml @@ -30,6 +30,7 @@ nav = [ {"Recall (Experimental)" = "recall.md"}, {"Configuration" = "configuration.md"}, {"Remote Access" = "remote-access.md"}, + {"Artifact Sync" = "artifact-sync.md"}, {"PostgreSQL Sync" = "pg-sync.md"}, {"DuckDB Mirror" = "duckdb.md"}, {"Changelog" = "changelog.md"}, diff --git a/frontend/messages/en.json b/frontend/messages/en.json index 8f4e8ad6f..de928b8f9 100644 --- a/frontend/messages/en.json +++ b/frontend/messages/en.json @@ -22,6 +22,7 @@ "nav_pinned": "Pinned", "nav_insights": "Insights", "nav_trash": "Trash", + "nav_peers": "Peers", "nav_search_sessions": "Search sessions...", "nav_search_sessions_shortcut": "Search sessions ({shortcut})", "header_transcript_normal_title": "Normal transcript - show all messages", @@ -187,6 +188,23 @@ "session_breadcrumb_find_in_session_shortcut": "Find in session (/)", "session_breadcrumb_rename": "Rename", "session_breadcrumb_delete": "Delete", + "session_breadcrumb_metadata_conflicts": "Metadata conflicts", + "session_breadcrumb_conflict_field_name": "Name", + "session_breadcrumb_conflict_field_trash": "Trash", + "session_breadcrumb_conflict_field_star": "Star", + "session_breadcrumb_conflict_field_delete_everywhere": "Delete everywhere", + "session_breadcrumb_conflict_field_pin": "Pin", + "session_breadcrumb_conflict_default_title": "Default title", + "session_breadcrumb_conflict_restored": "Restored", + "session_breadcrumb_conflict_in_trash": "In trash", + "session_breadcrumb_conflict_starred": "Starred", + "session_breadcrumb_conflict_unstarred": "Unstarred", + "session_breadcrumb_conflict_unpinned_target": "Unpinned {target}", + "session_breadcrumb_conflict_pinned_target": "Pinned {target}", + "session_breadcrumb_conflict_pinned_target_note": "Pinned {target}: {note}", + "session_breadcrumb_conflict_unknown_origin": "unknown origin", + "session_breadcrumb_conflict_current": "Current", + "session_breadcrumb_conflict_other": "Other", "session_breadcrumb_resumed_in": "Resumed in {target}", "session_breadcrumb_command_copied": "Command copied!", "session_breadcrumb_failed": "Failed", @@ -450,6 +468,57 @@ "sidebar_row_rename": "Rename", "sidebar_row_open_in_new_tab": "Open in new tab", "sidebar_row_delete": "Delete", + "peers_title": "Peers", + "peers_refresh_status": "Refresh peer status", + "peers_conflict_count_singular": "1 metadata conflict", + "peers_conflict_count_plural": "{count} metadata conflicts", + "peers_loading": "Loading peers...", + "peers_empty": "No peers yet", + "peers_unavailable": "Artifact sync is not available on this server.", + "peers_this_machine": "This machine", + "peers_in_sync": "In sync", + "peers_pending": "{count} pending", + "peers_local_sessions_title": "Sessions present in this database from this peer", + "peers_local_sessions": "{count} local", + "peers_published_sessions_title": "Sessions this peer has published", + "peers_published_sessions": "{count} published", + "peers_checkpoint_title": "Latest checkpoint sequence", + "peers_checkpoint": "checkpoint #{seq}", + "peers_last_published_title": "Last checkpoint published", + "peers_updated": "updated {time}", + "trash_title": "Trash", + "trash_empty_local_title": "Empty local trash", + "trash_emptying": "Emptying...", + "trash_empty_local": "Empty Local Trash", + "trash_loading": "Loading trash...", + "trash_empty": "Trash is empty", + "trash_empty_desc": "Deleted sessions will appear here.", + "trash_messages": [ + { + "declarations": [ + "input count", + "input countLabel", + "local countPlural = count: plural" + ], + "selectors": [ + "countPlural" + ], + "match": { + "countPlural=one": "{countLabel} msg", + "countPlural=other": "{countLabel} msgs" + } + } + ], + "trash_deleted": "deleted {time}", + "trash_restore_session": "Restore session", + "trash_restore": "Restore", + "trash_delete_everywhere_prompt": "Delete everywhere?", + "trash_confirm_delete_everywhere": "Confirm delete everywhere", + "trash_deleting": "Deleting...", + "trash_confirm": "Confirm", + "trash_cancel": "Cancel", + "trash_delete_everywhere": "Delete Everywhere", + "trash_delete_everywhere_title": "Delete everywhere", "shared_all_projects": "All Projects", "shared_project_filter_placeholder": "Filter projects...", "shared_select_project": "Select project", @@ -1621,33 +1690,6 @@ "system_boundary_stop_hook": "Stop hook feedback", "system_boundary_title": "System boundary: {subtype}", "system_boundary_show_content": "Show content", - "trash_loading": "Loading trash...", - "trash_empty": "Trash is empty", - "trash_empty_desc": "Deleted sessions will appear here.", - "trash_title": "Trash", - "trash_emptying": "Emptying...", - "trash_empty_trash": "Empty Trash", - "trash_msgs": [ - { - "declarations": [ - "input count", - "input countLabel", - "local countPlural = count: plural" - ], - "selectors": [ - "countPlural" - ], - "match": { - "countPlural=one": "{countLabel} msg", - "countPlural=other": "{countLabel} msgs" - } - } - ], - "trash_deleted_ago": "deleted {time}", - "trash_restore_session": "Restore session", - "trash_restore": "Restore", - "trash_permanently_delete": "Permanently delete", - "trash_delete_forever": "Delete Forever", "trends_term": "Term", "trends_per1k_messages": "Per 1k messages", "trends_count": "Count", diff --git a/frontend/messages/ko.json b/frontend/messages/ko.json index fbad5c3ab..f3fad279d 100644 --- a/frontend/messages/ko.json +++ b/frontend/messages/ko.json @@ -22,6 +22,7 @@ "nav_pinned": "고정됨", "nav_insights": "인사이트", "nav_trash": "휴지통", + "nav_peers": "피어", "nav_search_sessions": "세션 검색...", "nav_search_sessions_shortcut": "세션 검색 ({shortcut})", "header_transcript_normal_title": "일반 트랜스크립트 - 모든 메시지 표시", @@ -170,7 +171,9 @@ "input countLabel", "local countPlural = count: plural" ], - "selectors": ["countPlural"], + "selectors": [ + "countPlural" + ], "match": { "countPlural=other": "{countLabel}개 단계" } @@ -183,6 +186,23 @@ "session_breadcrumb_find_in_session_shortcut": "세션에서 찾기 (/)", "session_breadcrumb_rename": "이름 변경", "session_breadcrumb_delete": "삭제", + "session_breadcrumb_metadata_conflicts": "메타데이터 충돌", + "session_breadcrumb_conflict_field_name": "이름", + "session_breadcrumb_conflict_field_trash": "휴지통", + "session_breadcrumb_conflict_field_star": "고정", + "session_breadcrumb_conflict_field_delete_everywhere": "모든 기기에서 삭제", + "session_breadcrumb_conflict_field_pin": "메시지 고정", + "session_breadcrumb_conflict_default_title": "기본 제목", + "session_breadcrumb_conflict_restored": "복원됨", + "session_breadcrumb_conflict_in_trash": "휴지통에 있음", + "session_breadcrumb_conflict_starred": "고정됨", + "session_breadcrumb_conflict_unstarred": "고정 해제됨", + "session_breadcrumb_conflict_unpinned_target": "{target} 고정 해제됨", + "session_breadcrumb_conflict_pinned_target": "{target} 고정됨", + "session_breadcrumb_conflict_pinned_target_note": "{target} 고정됨: {note}", + "session_breadcrumb_conflict_unknown_origin": "알 수 없는 출처", + "session_breadcrumb_conflict_current": "현재", + "session_breadcrumb_conflict_other": "기타", "session_breadcrumb_resumed_in": "{target}에서 재개됨", "session_breadcrumb_command_copied": "명령어가 복사되었습니다!", "session_breadcrumb_failed": "실패", @@ -437,6 +457,56 @@ "sidebar_row_rename": "이름 변경", "sidebar_row_open_in_new_tab": "새 탭에서 열기", "sidebar_row_delete": "삭제", + "peers_title": "피어", + "peers_refresh_status": "피어 상태 새로 고침", + "peers_conflict_count_singular": "메타데이터 충돌 1건", + "peers_conflict_count_plural": "메타데이터 충돌 {count}건", + "peers_loading": "피어를 불러오는 중...", + "peers_empty": "피어가 없습니다", + "peers_unavailable": "이 서버에서는 아티팩트 동기화를 사용할 수 없습니다.", + "peers_this_machine": "이 컴퓨터", + "peers_in_sync": "동기화됨", + "peers_pending": "{count}개 대기 중", + "peers_local_sessions_title": "이 피어의 세션 중 이 데이터베이스에 있는 세션", + "peers_local_sessions": "로컬 {count}개", + "peers_published_sessions_title": "이 피어가 게시한 세션", + "peers_published_sessions": "게시됨 {count}개", + "peers_checkpoint_title": "최신 체크포인트 순번", + "peers_checkpoint": "체크포인트 #{seq}", + "peers_last_published_title": "마지막 체크포인트 게시 시각", + "peers_updated": "{time} 업데이트됨", + "trash_title": "휴지통", + "trash_empty_local_title": "로컬 휴지통 비우기", + "trash_emptying": "비우는 중...", + "trash_empty_local": "로컬 휴지통 비우기", + "trash_loading": "휴지통을 불러오는 중...", + "trash_empty": "휴지통이 비어 있습니다", + "trash_empty_desc": "삭제된 세션이 여기에 표시됩니다.", + "trash_messages": [ + { + "declarations": [ + "input count", + "input countLabel", + "local countPlural = count: plural" + ], + "selectors": [ + "countPlural" + ], + "match": { + "countPlural=other": "메시지 {countLabel}개" + } + } + ], + "trash_deleted": "{time} 삭제됨", + "trash_restore_session": "세션 복원", + "trash_restore": "복원", + "trash_delete_everywhere_prompt": "모든 기기에서 삭제할까요?", + "trash_confirm_delete_everywhere": "모든 기기에서 삭제 확인", + "trash_deleting": "삭제 중...", + "trash_confirm": "확인", + "trash_cancel": "취소", + "trash_delete_everywhere": "모든 기기에서 삭제", + "trash_delete_everywhere_title": "모든 기기에서 삭제", "shared_all_projects": "모든 프로젝트", "shared_project_filter_placeholder": "프로젝트 필터링...", "shared_select_project": "프로젝트 선택", @@ -576,7 +646,9 @@ "input countLabel", "local countPlural = count: plural" ], - "selectors": ["countPlural"], + "selectors": [ + "countPlural" + ], "match": { "countPlural=other": "세션 {countLabel}개" } @@ -1586,32 +1658,6 @@ "system_boundary_stop_hook": "중지 훅 피드백", "system_boundary_title": "시스템 경계: {subtype}", "system_boundary_show_content": "내용 표시", - "trash_loading": "휴지통을 불러오는 중...", - "trash_empty": "휴지통이 비어 있습니다", - "trash_empty_desc": "삭제된 세션이 여기에 표시됩니다.", - "trash_title": "휴지통", - "trash_emptying": "비우는 중...", - "trash_empty_trash": "휴지통 비우기", - "trash_msgs": [ - { - "declarations": [ - "input count", - "input countLabel", - "local countPlural = count: plural" - ], - "selectors": [ - "countPlural" - ], - "match": { - "countPlural=other": "메시지 {countLabel}개" - } - } - ], - "trash_deleted_ago": "{time} 삭제됨", - "trash_restore_session": "세션 복원", - "trash_restore": "복원", - "trash_permanently_delete": "영구 삭제", - "trash_delete_forever": "영구 삭제", "trends_term": "용어", "trends_per1k_messages": "메시지 1,000건당", "trends_count": "개수", diff --git a/frontend/messages/zh-CN.json b/frontend/messages/zh-CN.json index 6f0d623e3..9e778c49f 100644 --- a/frontend/messages/zh-CN.json +++ b/frontend/messages/zh-CN.json @@ -22,6 +22,7 @@ "nav_pinned": "已固定", "nav_insights": "洞察", "nav_trash": "回收站", + "nav_peers": "Peers", "nav_search_sessions": "搜索会话...", "nav_search_sessions_shortcut": "搜索会话 ({shortcut})", "header_transcript_normal_title": "普通 transcript - 显示所有消息", @@ -183,6 +184,23 @@ "session_breadcrumb_find_in_session_shortcut": "在会话中查找 (/)", "session_breadcrumb_rename": "重命名", "session_breadcrumb_delete": "删除", + "session_breadcrumb_metadata_conflicts": "元数据冲突", + "session_breadcrumb_conflict_field_name": "名称", + "session_breadcrumb_conflict_field_trash": "回收站", + "session_breadcrumb_conflict_field_star": "固定", + "session_breadcrumb_conflict_field_delete_everywhere": "全局删除", + "session_breadcrumb_conflict_field_pin": "固定", + "session_breadcrumb_conflict_default_title": "默认标题", + "session_breadcrumb_conflict_restored": "已恢复", + "session_breadcrumb_conflict_in_trash": "在回收站中", + "session_breadcrumb_conflict_starred": "已固定", + "session_breadcrumb_conflict_unstarred": "已取消固定", + "session_breadcrumb_conflict_unpinned_target": "已取消固定 {target}", + "session_breadcrumb_conflict_pinned_target": "已固定 {target}", + "session_breadcrumb_conflict_pinned_target_note": "已固定 {target}: {note}", + "session_breadcrumb_conflict_unknown_origin": "未知来源", + "session_breadcrumb_conflict_current": "当前", + "session_breadcrumb_conflict_other": "其他", "session_breadcrumb_resumed_in": "已在 {target} 中继续", "session_breadcrumb_command_copied": "命令已复制！", "session_breadcrumb_failed": "失败", @@ -437,6 +455,56 @@ "sidebar_row_rename": "重命名", "sidebar_row_open_in_new_tab": "在新标签页打开", "sidebar_row_delete": "删除", + "peers_title": "Peers", + "peers_refresh_status": "刷新 peer 状态", + "peers_conflict_count_singular": "1 个元数据冲突", + "peers_conflict_count_plural": "{count} 个元数据冲突", + "peers_loading": "正在加载 peers...", + "peers_empty": "暂无 peers", + "peers_unavailable": "此服务器不可用 artifact sync。", + "peers_this_machine": "此机器", + "peers_in_sync": "已同步", + "peers_pending": "{count} 个待处理", + "peers_local_sessions_title": "此数据库中来自该 peer 的会话", + "peers_local_sessions": "{count} 个本地", + "peers_published_sessions_title": "此 peer 已发布的会话", + "peers_published_sessions": "{count} 个已发布", + "peers_checkpoint_title": "最新 checkpoint 序号", + "peers_checkpoint": "checkpoint #{seq}", + "peers_last_published_title": "上次发布 checkpoint", + "peers_updated": "更新于 {time}", + "trash_title": "回收站", + "trash_empty_local_title": "清空本地回收站", + "trash_emptying": "正在清空...", + "trash_empty_local": "清空本地回收站", + "trash_loading": "正在加载回收站...", + "trash_empty": "回收站为空", + "trash_empty_desc": "已删除的会话会显示在这里。", + "trash_messages": [ + { + "declarations": [ + "input count", + "input countLabel", + "local countPlural = count: plural" + ], + "selectors": [ + "countPlural" + ], + "match": { + "countPlural=other": "{countLabel} 条消息" + } + } + ], + "trash_deleted": "删除于 {time}", + "trash_restore_session": "恢复会话", + "trash_restore": "恢复", + "trash_delete_everywhere_prompt": "全局删除？", + "trash_confirm_delete_everywhere": "确认全局删除", + "trash_deleting": "正在删除...", + "trash_confirm": "确认", + "trash_cancel": "取消", + "trash_delete_everywhere": "全局删除", + "trash_delete_everywhere_title": "全局删除", "shared_all_projects": "所有项目", "shared_project_filter_placeholder": "筛选项目...", "shared_select_project": "选择项目", @@ -1584,32 +1652,6 @@ "system_boundary_stop_hook": "停止钩子反馈", "system_boundary_title": "系统边界：{subtype}", "system_boundary_show_content": "显示内容", - "trash_loading": "正在加载回收站...", - "trash_empty": "回收站为空", - "trash_empty_desc": "已删除的会话会显示在这里。", - "trash_title": "回收站", - "trash_emptying": "正在清空...", - "trash_empty_trash": "清空回收站", - "trash_msgs": [ - { - "declarations": [ - "input count", - "input countLabel", - "local countPlural = count: plural" - ], - "selectors": [ - "countPlural" - ], - "match": { - "countPlural=other": "{countLabel} 条消息" - } - } - ], - "trash_deleted_ago": "{time} 删除", - "trash_restore_session": "恢复会话", - "trash_restore": "恢复", - "trash_permanently_delete": "永久删除", - "trash_delete_forever": "永久删除", "trends_term": "词项", "trends_per1k_messages": "每千条消息", "trends_count": "数量", diff --git a/frontend/messages/zh-TW.json b/frontend/messages/zh-TW.json index 513c6380c..9afe09ed4 100644 --- a/frontend/messages/zh-TW.json +++ b/frontend/messages/zh-TW.json @@ -22,6 +22,7 @@ "nav_pinned": "已固定", "nav_insights": "洞察", "nav_trash": "回收站", + "nav_peers": "Peers", "nav_search_sessions": "搜索會話...", "nav_search_sessions_shortcut": "搜索會話 ({shortcut})", "header_transcript_normal_title": "普通 transcript - 顯示所有消息", @@ -183,6 +184,23 @@ "session_breadcrumb_find_in_session_shortcut": "在會話中查找 (/)", "session_breadcrumb_rename": "重命名", "session_breadcrumb_delete": "刪除", + "session_breadcrumb_metadata_conflicts": "元資料衝突", + "session_breadcrumb_conflict_field_name": "名稱", + "session_breadcrumb_conflict_field_trash": "回收站", + "session_breadcrumb_conflict_field_star": "固定", + "session_breadcrumb_conflict_field_delete_everywhere": "全域刪除", + "session_breadcrumb_conflict_field_pin": "固定", + "session_breadcrumb_conflict_default_title": "預設標題", + "session_breadcrumb_conflict_restored": "已恢復", + "session_breadcrumb_conflict_in_trash": "在回收站中", + "session_breadcrumb_conflict_starred": "已固定", + "session_breadcrumb_conflict_unstarred": "已取消固定", + "session_breadcrumb_conflict_unpinned_target": "已取消固定 {target}", + "session_breadcrumb_conflict_pinned_target": "已固定 {target}", + "session_breadcrumb_conflict_pinned_target_note": "已固定 {target}: {note}", + "session_breadcrumb_conflict_unknown_origin": "未知來源", + "session_breadcrumb_conflict_current": "目前", + "session_breadcrumb_conflict_other": "其他", "session_breadcrumb_resumed_in": "已在 {target} 中繼續", "session_breadcrumb_command_copied": "命令已複製！", "session_breadcrumb_failed": "失敗", @@ -437,6 +455,56 @@ "sidebar_row_rename": "重命名", "sidebar_row_open_in_new_tab": "在新標籤頁打開", "sidebar_row_delete": "刪除", + "peers_title": "Peers", + "peers_refresh_status": "重新整理 peer 狀態", + "peers_conflict_count_singular": "1 個元資料衝突", + "peers_conflict_count_plural": "{count} 個元資料衝突", + "peers_loading": "正在加載 peers...", + "peers_empty": "暫無 peers", + "peers_unavailable": "此伺服器無法使用 artifact sync。", + "peers_this_machine": "此機器", + "peers_in_sync": "已同步", + "peers_pending": "{count} 個待處理", + "peers_local_sessions_title": "此資料庫中來自該 peer 的會話", + "peers_local_sessions": "{count} 個本地", + "peers_published_sessions_title": "此 peer 已發佈的會話", + "peers_published_sessions": "{count} 個已發佈", + "peers_checkpoint_title": "最新 checkpoint 序號", + "peers_checkpoint": "checkpoint #{seq}", + "peers_last_published_title": "上次發佈 checkpoint", + "peers_updated": "更新於 {time}", + "trash_title": "回收站", + "trash_empty_local_title": "清空本地回收站", + "trash_emptying": "正在清空...", + "trash_empty_local": "清空本地回收站", + "trash_loading": "正在加載回收站...", + "trash_empty": "回收站為空", + "trash_empty_desc": "已刪除的會話會顯示在這裡。", + "trash_messages": [ + { + "declarations": [ + "input count", + "input countLabel", + "local countPlural = count: plural" + ], + "selectors": [ + "countPlural" + ], + "match": { + "countPlural=other": "{countLabel} 條消息" + } + } + ], + "trash_deleted": "{time} 刪除", + "trash_restore_session": "恢復會話", + "trash_restore": "恢復", + "trash_delete_everywhere_prompt": "全域刪除？", + "trash_confirm_delete_everywhere": "確認全域刪除", + "trash_deleting": "正在刪除...", + "trash_confirm": "確認", + "trash_cancel": "取消", + "trash_delete_everywhere": "全域刪除", + "trash_delete_everywhere_title": "全域刪除", "shared_all_projects": "所有項目", "shared_project_filter_placeholder": "篩選項目...", "shared_select_project": "選擇項目", @@ -1584,32 +1652,6 @@ "system_boundary_stop_hook": "停止鉤子反饋", "system_boundary_title": "系統邊界：{subtype}", "system_boundary_show_content": "顯示內容", - "trash_loading": "正在加載回收站...", - "trash_empty": "回收站為空", - "trash_empty_desc": "已刪除的會話會顯示在這裡。", - "trash_title": "回收站", - "trash_emptying": "正在清空...", - "trash_empty_trash": "清空回收站", - "trash_msgs": [ - { - "declarations": [ - "input count", - "input countLabel", - "local countPlural = count: plural" - ], - "selectors": [ - "countPlural" - ], - "match": { - "countPlural=other": "{countLabel} 條消息" - } - } - ], - "trash_deleted_ago": "{time} 刪除", - "trash_restore_session": "恢復會話", - "trash_restore": "恢復", - "trash_permanently_delete": "永久刪除", - "trash_delete_forever": "永久刪除", "trends_term": "詞項", "trends_per1k_messages": "每千條消息", "trends_count": "數量", diff --git a/frontend/src/App.svelte b/frontend/src/App.svelte index 20999813d..0012501e2 100644 --- a/frontend/src/App.svelte +++ b/frontend/src/App.svelte @@ -64,6 +64,7 @@ import PinnedPage from "./lib/components/pinned/PinnedPage.svelte"; import TrashPage from "./lib/components/trash/TrashPage.svelte"; import RecentEditsPage from "./lib/components/recentedits/RecentEditsPage.svelte"; + import PeersPage from "./lib/components/peers/PeersPage.svelte"; import SettingsPage from "./lib/components/settings/SettingsPage.svelte"; import { sessions, filtersToParams } from "./lib/stores/sessions.svelte.js"; import { messages } from "./lib/stores/messages.svelte.js"; @@ -535,6 +536,10 @@
+{:else if router.route === "peers"} +
+ +
{:else if router.route === "settings"}
diff --git a/frontend/src/lib/api/generated/index.ts b/frontend/src/lib/api/generated/index.ts index 65131dd7e..e418c80c1 100644 --- a/frontend/src/lib/api/generated/index.ts +++ b/frontend/src/lib/api/generated/index.ts @@ -18,6 +18,10 @@ export type { AgentsResponse } from './models/AgentsResponse'; export type { AgentTotal } from './models/AgentTotal'; export type { ApiErrorResponse } from './models/ApiErrorResponse'; export type { ApplyWorktreeMappingsResponse } from './models/ApplyWorktreeMappingsResponse'; +export type { ArtifactOriginsResponse } from './models/ArtifactOriginsResponse'; +export type { ArtifactPeer } from './models/ArtifactPeer'; +export type { ArtifactPeersResponse } from './models/ArtifactPeersResponse'; +export type { ArtifactPostResponse } from './models/ArtifactPostResponse'; export type { BatchDeleteInputBody } from './models/BatchDeleteInputBody'; export type { BranchesResponse } from './models/BranchesResponse'; export type { BulkStarInputBody } from './models/BulkStarInputBody'; @@ -52,6 +56,7 @@ export type { DbHourOfWeekCell } from './models/DbHourOfWeekCell'; export type { DbHourOfWeekResponse } from './models/DbHourOfWeekResponse'; export type { DbInsight } from './models/DbInsight'; export type { DbMessage } from './models/DbMessage'; +export type { DbMetadataConflict } from './models/DbMetadataConflict'; export type { DbModelBreakdown } from './models/DbModelBreakdown'; export type { DbPeakContextDistribution } from './models/DbPeakContextDistribution'; export type { DbPercentiles } from './models/DbPercentiles'; @@ -149,6 +154,7 @@ export type { GithubConfigResponse } from './models/GithubConfigResponse'; export type { InsightCannedSessionFilters } from './models/InsightCannedSessionFilters'; export type { InsightsResponse } from './models/InsightsResponse'; export type { MachinesResponse } from './models/MachinesResponse'; +export type { MetadataConflictsResponse } from './models/MetadataConflictsResponse'; export type { ModelTotal } from './models/ModelTotal'; export type { Opener } from './models/Opener'; export type { OpenersResponse } from './models/OpenersResponse'; @@ -212,6 +218,7 @@ export type { WorktreeMappingsResponse } from './models/WorktreeMappingsResponse export { ActivityService } from './services/ActivityService'; export { AnalyticsService } from './services/AnalyticsService'; +export { ArtifactsService } from './services/ArtifactsService'; export { AssetsService } from './services/AssetsService'; export { ConfigService } from './services/ConfigService'; export { EmbeddingsService } from './services/EmbeddingsService'; diff --git a/frontend/src/lib/api/generated/models/ArtifactOriginsResponse.ts b/frontend/src/lib/api/generated/models/ArtifactOriginsResponse.ts new file mode 100644 index 000000000..076955347 --- /dev/null +++ b/frontend/src/lib/api/generated/models/ArtifactOriginsResponse.ts @@ -0,0 +1,7 @@ +/* generated using openapi-typescript-codegen -- do not edit */ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ +export type ArtifactOriginsResponse = { + origins: any[] | null; +}; diff --git a/frontend/src/lib/api/generated/models/ArtifactPeer.ts b/frontend/src/lib/api/generated/models/ArtifactPeer.ts new file mode 100644 index 000000000..bebe43eea --- /dev/null +++ b/frontend/src/lib/api/generated/models/ArtifactPeer.ts @@ -0,0 +1,12 @@ +/* generated using openapi-typescript-codegen -- do not edit */ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ +export type ArtifactPeer = { + checkpoint_seq: number; + is_local: boolean; + last_published?: string; + local_sessions: number; + origin: string; + published_sessions: number; +}; diff --git a/frontend/src/lib/api/generated/models/ArtifactPeersResponse.ts b/frontend/src/lib/api/generated/models/ArtifactPeersResponse.ts new file mode 100644 index 000000000..5e47fb005 --- /dev/null +++ b/frontend/src/lib/api/generated/models/ArtifactPeersResponse.ts @@ -0,0 +1,9 @@ +/* generated using openapi-typescript-codegen -- do not edit */ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ +export type ArtifactPeersResponse = { + conflict_count: number; + local_origin: string; + peers: any[] | null; +}; diff --git a/frontend/src/lib/api/generated/models/ArtifactPostResponse.ts b/frontend/src/lib/api/generated/models/ArtifactPostResponse.ts new file mode 100644 index 000000000..3caad6ae8 --- /dev/null +++ b/frontend/src/lib/api/generated/models/ArtifactPostResponse.ts @@ -0,0 +1,12 @@ +/* generated using openapi-typescript-codegen -- do not edit */ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ +export type ArtifactPostResponse = { + duplicate: boolean; + hash?: string; + kind: string; + name: string; + origin: string; + size: number; +}; diff --git a/frontend/src/lib/api/generated/models/DbMetadataConflict.ts b/frontend/src/lib/api/generated/models/DbMetadataConflict.ts new file mode 100644 index 000000000..66afad6d0 --- /dev/null +++ b/frontend/src/lib/api/generated/models/DbMetadataConflict.ts @@ -0,0 +1,18 @@ +/* generated using openapi-typescript-codegen -- do not edit */ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ +export type DbMetadataConflict = { + created_at: string; + field: string; + id: number; + losing_op: string; + losing_order_key: string; + losing_origin: string; + losing_value: string; + session_gid: string; + winning_op: string; + winning_order_key: string; + winning_origin: string; + winning_value: string; +}; diff --git a/frontend/src/lib/api/generated/models/DbTopSession.ts b/frontend/src/lib/api/generated/models/DbTopSession.ts index 11808f97d..ba2c1497a 100644 --- a/frontend/src/lib/api/generated/models/DbTopSession.ts +++ b/frontend/src/lib/api/generated/models/DbTopSession.ts @@ -15,4 +15,3 @@ export type DbTopSession = { started_at?: string; termination_status?: string; }; - diff --git a/frontend/src/lib/api/generated/models/MetadataConflictsResponse.ts b/frontend/src/lib/api/generated/models/MetadataConflictsResponse.ts new file mode 100644 index 000000000..1dc167d78 --- /dev/null +++ b/frontend/src/lib/api/generated/models/MetadataConflictsResponse.ts @@ -0,0 +1,7 @@ +/* generated using openapi-typescript-codegen -- do not edit */ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ +export type MetadataConflictsResponse = { + conflicts: any[] | null; +}; diff --git a/frontend/src/lib/api/generated/services/ArtifactsService.ts b/frontend/src/lib/api/generated/services/ArtifactsService.ts new file mode 100644 index 000000000..efa85b815 --- /dev/null +++ b/frontend/src/lib/api/generated/services/ArtifactsService.ts @@ -0,0 +1,188 @@ +/* generated using openapi-typescript-codegen -- do not edit */ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ +import type { ArtifactOriginsResponse } from '../models/ArtifactOriginsResponse'; +import type { ArtifactPeersResponse } from '../models/ArtifactPeersResponse'; +import type { ArtifactPostResponse } from '../models/ArtifactPostResponse'; +import type { CancelablePromise } from '../core/CancelablePromise'; +import { OpenAPI } from '../core/OpenAPI'; +import { request as __request } from '../core/request'; +export class ArtifactsService { + /** + * List artifact origins + * @returns ArtifactOriginsResponse OK + * @throws ApiError + */ + public static getApiV1ArtifactsOrigins(): CancelablePromise { + return __request(OpenAPI, { + method: 'GET', + url: '/api/v1/artifacts/origins', + errors: { + 400: `Bad Request`, + 401: `Unauthorized`, + 403: `Forbidden`, + 404: `Not Found`, + 409: `Conflict`, + 500: `Internal Server Error`, + 501: `Not Implemented`, + 502: `Bad Gateway`, + 503: `Service Unavailable`, + 504: `Gateway Timeout`, + }, + }); + } + /** + * List artifact peers + * @returns ArtifactPeersResponse OK + * @throws ApiError + */ + public static getApiV1ArtifactsPeers(): CancelablePromise { + return __request(OpenAPI, { + method: 'GET', + url: '/api/v1/artifacts/peers', + errors: { + 400: `Bad Request`, + 401: `Unauthorized`, + 403: `Forbidden`, + 404: `Not Found`, + 409: `Conflict`, + 500: `Internal Server Error`, + 501: `Not Implemented`, + 502: `Bad Gateway`, + 503: `Service Unavailable`, + 504: `Gateway Timeout`, + }, + }); + } + /** + * Get latest artifact checkpoint + * @returns string OK + * @throws ApiError + */ + public static getApiV1ArtifactsOriginCheckpoint({ + origin, + }: { + /** + * Artifact origin ID + */ + origin: string, + }): CancelablePromise { + return __request(OpenAPI, { + method: 'GET', + url: '/api/v1/artifacts/{origin}/checkpoint', + path: { + 'origin': origin, + }, + errors: { + 400: `Bad Request`, + 401: `Unauthorized`, + 403: `Forbidden`, + 404: `Not Found`, + 409: `Conflict`, + 422: `Unprocessable Entity`, + 500: `Internal Server Error`, + 501: `Not Implemented`, + 502: `Bad Gateway`, + 503: `Service Unavailable`, + 504: `Gateway Timeout`, + }, + }); + } + /** + * Get artifact + * @returns string OK + * @throws ApiError + */ + public static getApiV1ArtifactsOriginKindName({ + origin, + kind, + name, + }: { + /** + * Artifact origin ID + */ + origin: string, + /** + * Artifact kind + */ + kind: string, + /** + * Artifact filename or hash + */ + name: string, + }): CancelablePromise { + return __request(OpenAPI, { + method: 'GET', + url: '/api/v1/artifacts/{origin}/{kind}/{name}', + path: { + 'origin': origin, + 'kind': kind, + 'name': name, + }, + errors: { + 400: `Bad Request`, + 401: `Unauthorized`, + 403: `Forbidden`, + 404: `Not Found`, + 409: `Conflict`, + 422: `Unprocessable Entity`, + 500: `Internal Server Error`, + 501: `Not Implemented`, + 502: `Bad Gateway`, + 503: `Service Unavailable`, + 504: `Gateway Timeout`, + }, + }); + } + /** + * Post artifact + * @returns ArtifactPostResponse OK + * @throws ApiError + */ + public static postApiV1ArtifactsOriginKindName({ + origin, + kind, + name, + requestBody, + }: { + /** + * Artifact origin ID + */ + origin: string, + /** + * Artifact kind + */ + kind: string, + /** + * Artifact filename or hash + */ + name: string, + requestBody: Blob, + }): CancelablePromise { + return __request(OpenAPI, { + method: 'POST', + url: '/api/v1/artifacts/{origin}/{kind}/{name}', + path: { + 'origin': origin, + 'kind': kind, + 'name': name, + }, + body: requestBody, + mediaType: 'application/octet-stream', + errors: { + 400: `Bad Request`, + 401: `Unauthorized`, + 403: `Forbidden`, + 404: `Not Found`, + 409: `Conflict`, + 422: `Unprocessable Entity`, + 500: `Internal Server Error`, + 501: `Not Implemented`, + 502: `Bad Gateway`, + 503: `Service Unavailable`, + 504: `Gateway Timeout`, + }, + }); + } +} diff --git a/frontend/src/lib/api/generated/services/SessionsService.ts b/frontend/src/lib/api/generated/services/SessionsService.ts index 86730d5f3..14bca43ce 100644 --- a/frontend/src/lib/api/generated/services/SessionsService.ts +++ b/frontend/src/lib/api/generated/services/SessionsService.ts @@ -8,6 +8,7 @@ import type { DbSessionActivityResponse } from '../models/DbSessionActivityRespo import type { DbSessionTiming } from '../models/DbSessionTiming'; import type { DbSidebarSessionIndex } from '../models/DbSidebarSessionIndex'; import type { EmptyTrashResponse } from '../models/EmptyTrashResponse'; +import type { MetadataConflictsResponse } from '../models/MetadataConflictsResponse'; import type { OpenRequest } from '../models/OpenRequest'; import type { OpenSessionResponse } from '../models/OpenSessionResponse'; import type { OrdinalsResponse } from '../models/OrdinalsResponse'; @@ -847,6 +848,40 @@ export class SessionsService { }, }); } + /** + * List session metadata conflicts + * @returns MetadataConflictsResponse OK + * @throws ApiError + */ + public static getApiV1SessionsIdMetadataConflicts({ + id, + }: { + /** + * Session ID + */ + id: string, + }): CancelablePromise { + return __request(OpenAPI, { + method: 'GET', + url: '/api/v1/sessions/{id}/metadata-conflicts', + path: { + 'id': id, + }, + errors: { + 400: `Bad Request`, + 401: `Unauthorized`, + 403: `Forbidden`, + 404: `Not Found`, + 409: `Conflict`, + 422: `Unprocessable Entity`, + 500: `Internal Server Error`, + 501: `Not Implemented`, + 502: `Bad Gateway`, + 503: `Service Unavailable`, + 504: `Gateway Timeout`, + }, + }); + } /** * Open session directory * @returns OpenSessionResponse OK diff --git a/frontend/src/lib/components/layout/AppHeader.svelte b/frontend/src/lib/components/layout/AppHeader.svelte index 73a466a8a..e0568de7e 100644 --- a/frontend/src/lib/components/layout/AppHeader.svelte +++ b/frontend/src/lib/components/layout/AppHeader.svelte @@ -90,6 +90,7 @@ "insights", "trash", "recent-edits", + "peers", ] as const; const tabs: TopBarTab[] = $derived([ @@ -101,6 +102,7 @@ { id: "insights", label: m.nav_insights() }, { id: "trash", label: m.nav_trash() }, { id: "recent-edits", label: m.nav_recent_edits() }, + { id: "peers", label: m.nav_peers() }, ]); const activeTab = $derived( diff --git a/frontend/src/lib/components/layout/SessionBreadcrumb.svelte b/frontend/src/lib/components/layout/SessionBreadcrumb.svelte index b7beacac4..8eff03a3d 100644 --- a/frontend/src/lib/components/layout/SessionBreadcrumb.svelte +++ b/frontend/src/lib/components/layout/SessionBreadcrumb.svelte @@ -13,12 +13,14 @@ LinkIcon, SearchIcon, SquareTerminalIcon, + TriangleAlertIcon, } from "../../icons.js"; import { onMount } from "svelte"; import type { Session } from "../../api/types.js"; import { OpenersService, SessionsService, + type DbMetadataConflict, type ResumeRequest, type ResumeResponse, } from "../../api/generated/index"; @@ -65,6 +67,9 @@ let openFeedback = $state(""); let feedbackTimer: ReturnType | undefined; let sessionDir = $state(null); + let metadataConflicts = $state([]); + let conflictsOpen = $state(false); + let conflictRequestSeq = 0; interface Opener { id: string; @@ -129,6 +134,31 @@ }); }); + $effect(() => { + if (!session) { + metadataConflicts = []; + conflictsOpen = false; + conflictRequestSeq++; + return; + } + const id = session.id; + metadataConflicts = []; + conflictsOpen = false; + const seq = ++conflictRequestSeq; + configureGeneratedClient(); + SessionsService.getApiV1SessionsIdMetadataConflicts({ id }) + .then((res) => { + if (seq !== conflictRequestSeq || session?.id !== id) return; + metadataConflicts = Array.isArray(res.conflicts) + ? (res.conflicts as DbMetadataConflict[]) + : []; + }) + .catch(() => { + if (seq !== conflictRequestSeq) return; + metadataConflicts = []; + }); + }); + let sessionCost = $state(null); let sessionUsageBreakdownCount = $state(0); let sessionUsageBreakdown = $state([]); @@ -480,6 +510,90 @@ : m.session_breadcrumb_failed()); } + type ConflictSide = "winning" | "losing"; + + function conflictFieldLabel(field: string): string { + if (field === "display_name") return m.session_breadcrumb_conflict_field_name(); + if (field === "deleted_at") return m.session_breadcrumb_conflict_field_trash(); + if (field === "starred") return m.session_breadcrumb_conflict_field_star(); + if (field === "purge") return m.session_breadcrumb_conflict_field_delete_everywhere(); + if (field.startsWith("pin:")) return m.session_breadcrumb_conflict_field_pin(); + return field.replaceAll("_", " "); + } + + function parseJSONValue(value: string): Record | null { + if (!value) return null; + try { + const parsed = JSON.parse(value) as unknown; + return parsed !== null && typeof parsed === "object" + ? parsed as Record + : null; + } catch { + return null; + } + } + + function conflictSideValue( + conflict: DbMetadataConflict, + side: ConflictSide, + ): string { + const op = side === "winning" + ? conflict.winning_op + : conflict.losing_op; + const value = side === "winning" + ? conflict.winning_value + : conflict.losing_value; + if (conflict.field === "display_name") { + const parsed = parseJSONValue(value); + const displayName = parsed?.display_name; + if (typeof displayName === "string" && displayName.trim()) { + return displayName; + } + return m.session_breadcrumb_conflict_default_title(); + } + if (conflict.field === "deleted_at") { + if (op === "restore") return m.session_breadcrumb_conflict_restored(); + if (op === "soft_delete") return m.session_breadcrumb_conflict_in_trash(); + } + if (conflict.field === "starred") { + if (op === "star") return m.session_breadcrumb_conflict_starred(); + if (op === "unstar") return m.session_breadcrumb_conflict_unstarred(); + } + if (conflict.field === "purge") { + return op === "purge" + ? m.session_breadcrumb_conflict_field_delete_everywhere() + : op; + } + if (conflict.field.startsWith("pin:")) { + const parsed = parseJSONValue(value); + const ordinal = parsed?.ordinal; + const sourceUUID = parsed?.source_uuid; + const note = parsed?.note; + const target = typeof sourceUUID === "string" && sourceUUID + ? sourceUUID.slice(0, 8) + : typeof ordinal === "number" + ? `#${ordinal}` + : "pin"; + if (op === "unpin") { + return m.session_breadcrumb_conflict_unpinned_target({ target }); + } + if (typeof note === "string" && note.trim()) { + return m.session_breadcrumb_conflict_pinned_target_note({ target, note }); + } + return m.session_breadcrumb_conflict_pinned_target({ target }); + } + return value || op; + } + + function conflictOrigin( + conflict: DbMetadataConflict, + side: ConflictSide, + ): string { + return side === "winning" + ? conflict.winning_origin || m.session_breadcrumb_conflict_unknown_origin() + : conflict.losing_origin || m.session_breadcrumb_conflict_unknown_origin(); + } + async function handleOpenIn(opener: Opener) { if (!session) return; showOpenMenu = false; @@ -584,6 +698,9 @@ } else if (showOpenMenu) { showOpenMenu = false; e.preventDefault(); + } else if (conflictsOpen) { + conflictsOpen = false; + e.preventDefault(); } return; } @@ -615,6 +732,9 @@ if (!(target as HTMLElement).closest?.(".open-group")) { showOpenMenu = false; } + if (!(target as HTMLElement).closest?.(".conflict-group")) { + conflictsOpen = false; + } } @@ -704,6 +824,63 @@ > {getGradeLabel(session.health_grade)} + {#if metadataConflicts.length > 0} + + + {#if conflictsOpen} +
+
{m.session_breadcrumb_metadata_conflicts()}
+ {#each metadataConflicts as conflict (conflict.id)} +
+
+ {conflictFieldLabel(conflict.field)} +
+
+ {m.session_breadcrumb_conflict_current()} + + {conflictSideValue(conflict, "winning")} + + + {conflictOrigin(conflict, "winning")} + +
+
+ {m.session_breadcrumb_conflict_other()} + + {conflictSideValue(conflict, "losing")} + + + {conflictOrigin(conflict, "losing")} + +
+
+ {/each} +
+ {/if} + + {/if} {#if showDropdown} +
+ + {#if conflictCount > 0} +
+
+ {/if} + + {#if loading} +
+ + {m.peers_loading()} +
+ {:else if peers.length === 0} + + {#snippet icon()} + + {:else} +
+ {#each peers as peer (peer.origin)} + {@const state = syncState(peer)} +
+
+ {#if peer.is_local} +
+
+
+ {peer.origin} + {#if peer.is_local} + {m.peers_this_machine()} + {/if} + {#if state === "synced"} + {m.peers_in_sync()} + {:else if state === "behind"} + + {m.peers_pending({ + count: formatNumber(peer.published_sessions - peer.local_sessions), + })} + + {/if} +
+
+ + {m.peers_local_sessions({ count: formatNumber(peer.local_sessions) })} + + / + + {m.peers_published_sessions({ count: formatNumber(peer.published_sessions) })} + + {#if peer.checkpoint_seq > 0} + · + + {m.peers_checkpoint({ seq: String(peer.checkpoint_seq) })} + + {/if} + {#if peer.last_published} + · + + {m.peers_updated({ time: formatRelativeTime(peer.last_published) })} + + {/if} +
+
+
+ {/each} +
+ {/if} +
+ + diff --git a/frontend/src/lib/components/trash/TrashPage.svelte b/frontend/src/lib/components/trash/TrashPage.svelte index 8c269a0f6..f42228556 100644 --- a/frontend/src/lib/components/trash/TrashPage.svelte +++ b/frontend/src/lib/components/trash/TrashPage.svelte @@ -12,6 +12,8 @@ let trashedSessions: Session[] = $state([]); let loading = $state(true); let emptying = $state(false); + let confirmingDeleteId = $state(null); + let deletingId = $state(null); interface TrashResponse { sessions: Session[]; @@ -40,6 +42,7 @@ configureGeneratedClient(); await SessionsService.postApiV1SessionsIdRestore({ id }); trashedSessions = trashedSessions.filter((s) => s.id !== id); + if (confirmingDeleteId === id) confirmingDeleteId = null; sessions.clearRecentlyDeleted(id); sessions.invalidateFilterCaches(); sessions.load(); @@ -48,15 +51,27 @@ } } + function requestPermanentDelete(id: string) { + confirmingDeleteId = id; + } + + function cancelPermanentDelete(id: string) { + if (confirmingDeleteId === id) confirmingDeleteId = null; + } + async function permanentDelete(id: string) { + deletingId = id; try { configureGeneratedClient(); await SessionsService.deleteApiV1SessionsIdPermanent({ id }); trashedSessions = trashedSessions.filter((s) => s.id !== id); + if (confirmingDeleteId === id) confirmingDeleteId = null; sessions.clearRecentlyDeleted(id); sessions.invalidateFilterCaches(); } catch { // silently fail + } finally { + if (deletingId === id) deletingId = null; } } @@ -82,6 +97,22 @@
+
+
+ {#if loading}
{m.trash_loading()}
{:else if trashedSessions.length === 0} @@ -91,19 +122,6 @@ {/snippet} {:else} -
-
-
{#each trashedSessions as session (session.id)}
@@ -112,12 +130,12 @@
{session.agent} {session.project} - {m.trash_msgs({ + {m.trash_messages({ count: session.user_message_count, countLabel: session.user_message_count.toLocaleString(), })} {#if session.deleted_at} - {m.trash_deleted_ago({ time: formatRelativeTime(session.deleted_at) })} + {m.trash_deleted({ time: formatRelativeTime(session.deleted_at) })} {/if}
@@ -129,13 +147,32 @@ > {m.trash_restore()} - + {#if confirmingDeleteId === session.id} + {m.trash_delete_everywhere_prompt()} + + + {:else} + + {/if}
{/each} @@ -269,6 +306,7 @@ .trash-card-actions { display: flex; + align-items: center; gap: 6px; flex-shrink: 0; } @@ -304,4 +342,38 @@ .perm-delete-btn:hover { background: color-mix(in srgb, var(--accent-red, #e55) 8%, transparent); } + + .perm-delete-btn--confirm { + border-color: var(--accent-red, #e55); + } + + .delete-confirm-label { + color: var(--text-muted); + font-size: 11px; + font-weight: 500; + white-space: nowrap; + } + + .cancel-delete-btn { + font-size: 11px; + font-weight: 500; + color: var(--text-muted); + background: none; + border: 1px solid var(--border-muted); + border-radius: var(--radius-sm); + padding: 4px 10px; + cursor: pointer; + transition: background 0.12s, color 0.12s; + } + + .cancel-delete-btn:hover:not(:disabled) { + color: var(--text-secondary); + background: var(--bg-surface-hover); + } + + .perm-delete-btn:disabled, + .cancel-delete-btn:disabled { + cursor: default; + opacity: 0.6; + } diff --git a/frontend/src/lib/components/trash/TrashPage.test.ts b/frontend/src/lib/components/trash/TrashPage.test.ts new file mode 100644 index 000000000..80843a33b --- /dev/null +++ b/frontend/src/lib/components/trash/TrashPage.test.ts @@ -0,0 +1,152 @@ +// @vitest-environment jsdom +import { + afterEach, + beforeEach, + describe, + expect, + it, + vi, +} from "vite-plus/test"; +import { mount, tick, unmount } from "svelte"; +// @ts-ignore +import TrashPage from "./TrashPage.svelte"; +import type { Session } from "../../api/types.js"; +import { SessionsService } from "../../api/generated/index"; + +vi.mock("../../api/runtime.js", async (importOriginal) => { + const orig = + await importOriginal(); + return { + ...orig, + configureGeneratedClient: vi.fn(), + }; +}); + +vi.mock("../../api/generated/index", async (importOriginal) => { + const orig = + await importOriginal(); + return { + ...orig, + SessionsService: { + deleteApiV1SessionsIdPermanent: vi.fn(), + deleteApiV1Trash: vi.fn(), + getApiV1Trash: vi.fn(), + postApiV1SessionsIdRestore: vi.fn(), + }, + }; +}); + +vi.mock("../../stores/sessions.svelte.js", () => ({ + sessions: { + clearRecentlyDeleted: vi.fn(), + invalidateFilterCaches: vi.fn(), + load: vi.fn(), + }, +})); + +const sessionsService = SessionsService as unknown as { + deleteApiV1SessionsIdPermanent: ReturnType; + deleteApiV1Trash: ReturnType; + getApiV1Trash: ReturnType; + postApiV1SessionsIdRestore: ReturnType; +}; + +function makeTrashedSession(overrides: Partial = {}): Session { + return { + id: "s1", + project: "alpha", + machine: "local", + agent: "claude", + first_message: "build this", + display_name: null, + started_at: "2026-06-14T01:02:03Z", + ended_at: "2026-06-14T01:03:03Z", + created_at: "2026-06-14T01:02:03Z", + deleted_at: "2026-06-14T01:04:03Z", + message_count: 2, + user_message_count: 1, + total_output_tokens: 0, + peak_context_tokens: 0, + is_automated: false, + ...overrides, + } as Session; +} + +function buttonByText(label: string): HTMLButtonElement { + const button = Array.from(document.querySelectorAll("button")) + .find((el) => el.textContent?.trim() === label); + expect(button).toBeTruthy(); + return button as HTMLButtonElement; +} + +beforeEach(() => { + sessionsService.getApiV1Trash + .mockReset() + .mockResolvedValue({ sessions: [makeTrashedSession()] }); + sessionsService.deleteApiV1SessionsIdPermanent + .mockReset() + .mockResolvedValue(undefined); + sessionsService.deleteApiV1Trash + .mockReset() + .mockResolvedValue({ deleted: 1 }); + sessionsService.postApiV1SessionsIdRestore + .mockReset() + .mockResolvedValue(undefined); +}); + +afterEach(() => { + document.body.innerHTML = ""; +}); + +describe("TrashPage", () => { + it("requires confirmation before deleting a session everywhere", async () => { + const component = mount(TrashPage, { target: document.body }); + + await vi.waitFor(() => { + expect(buttonByText("Delete Everywhere")).toBeTruthy(); + }); + + buttonByText("Delete Everywhere").dispatchEvent( + new MouseEvent("click", { bubbles: true }), + ); + await tick(); + + expect( + sessionsService.deleteApiV1SessionsIdPermanent, + ).not.toHaveBeenCalled(); + expect(document.body.textContent).toContain("Delete everywhere?"); + + buttonByText("Confirm").dispatchEvent( + new MouseEvent("click", { bubbles: true }), + ); + + await vi.waitFor(() => { + expect( + sessionsService.deleteApiV1SessionsIdPermanent, + ).toHaveBeenCalledWith({ id: "s1" }); + }); + + unmount(component); + }); + + it("keeps empty trash on the local endpoint", async () => { + const component = mount(TrashPage, { target: document.body }); + + await vi.waitFor(() => { + expect(buttonByText("Empty Local Trash")).toBeTruthy(); + }); + + buttonByText("Empty Local Trash").dispatchEvent( + new MouseEvent("click", { bubbles: true }), + ); + + await vi.waitFor(() => { + expect(sessionsService.deleteApiV1Trash).toHaveBeenCalled(); + }); + expect( + sessionsService.deleteApiV1SessionsIdPermanent, + ).not.toHaveBeenCalled(); + + unmount(component); + }); +}); diff --git a/frontend/src/lib/i18n/i18n.test.ts b/frontend/src/lib/i18n/i18n.test.ts index 4a6950b43..98a79b655 100644 --- a/frontend/src/lib/i18n/i18n.test.ts +++ b/frontend/src/lib/i18n/i18n.test.ts @@ -141,7 +141,7 @@ describe("i18n locale selection", () => { count: 2, countLabel: "2", })).toBe("2 sessions"); - expect(m.trash_msgs({ + expect(m.trash_messages({ count: 1, countLabel: "1", })).toBe("1 msg"); diff --git a/frontend/src/lib/stores/router.svelte.ts b/frontend/src/lib/stores/router.svelte.ts index e6bc5cd24..3f9cafe14 100644 --- a/frontend/src/lib/stores/router.svelte.ts +++ b/frontend/src/lib/stores/router.svelte.ts @@ -7,6 +7,7 @@ export type Route = | "pinned" | "trash" | "recent-edits" + | "peers" | "settings"; const VALID_ROUTES: ReadonlySet = new Set([ @@ -18,6 +19,7 @@ const VALID_ROUTES: ReadonlySet = new Set([ "pinned", "trash", "recent-edits", + "peers", "settings", ]); diff --git a/frontend/src/lib/stores/router.test.ts b/frontend/src/lib/stores/router.test.ts index f70881788..813c0c024 100644 --- a/frontend/src/lib/stores/router.test.ts +++ b/frontend/src/lib/stores/router.test.ts @@ -68,6 +68,7 @@ describe("parsePath", () => { "insights", "pinned", "trash", + "peers", "settings", ]) { setURL(`/${route}`); diff --git a/internal/artifact/compression_test.go b/internal/artifact/compression_test.go new file mode 100644 index 000000000..18e71cd8e --- /dev/null +++ b/internal/artifact/compression_test.go @@ -0,0 +1,775 @@ +package artifact + +import ( + "bytes" + "context" + "crypto/sha256" + "fmt" + "io" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/klauspost/compress/zstd" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.kenn.io/agentsview/internal/db" +) + +func TestReadCompressedRejectsDecodedOutputAboveKindLimit(t *testing.T) { + tests := []struct { + name string + extension string + total int64 + }{ + {name: "manifest", extension: manifestExtension, total: 16<<20 + 1}, + {name: "segment", extension: segmentExtension, total: 64<<20 + 1}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + compressed, _ := compressedArtifactWithPadding(t, []byte("{}\n"), tt.total) + path := filepath.Join(t.TempDir(), "artifact"+tt.extension) + require.NoError(t, os.WriteFile(path, compressed, 0o644)) + + _, err := readCompressed(path) + require.Error(t, err) + assert.ErrorIs(t, err, errCorruptArtifact) + assert.Contains(t, err.Error(), "decoded output exceeds") + }) + } +} + +func TestReadCompressedLimitsTotalAcrossConcatenatedFrames(t *testing.T) { + first, _ := compressedArtifactWithPadding(t, []byte("{}\n"), 9<<20) + second, _ := compressedArtifactWithPadding(t, nil, 9<<20) + path := filepath.Join(t.TempDir(), "artifact"+manifestExtension) + require.NoError(t, os.WriteFile(path, append(first, second...), 0o644)) + + _, err := readCompressed(path) + require.Error(t, err) + assert.ErrorIs(t, err, errCorruptArtifact) + assert.Contains(t, err.Error(), "decoded output exceeds") +} + +func TestReadCompressedRejectsFrameAboveWriterWindow(t *testing.T) { + compressed, _ := compressedArtifactWithPadding( + t, []byte("{}\n"), 9<<20, + zstd.WithWindowSize(16<<20), + zstd.WithEncoderConcurrency(1), + ) + path := filepath.Join(t.TempDir(), "artifact"+segmentExtension) + require.NoError(t, os.WriteFile(path, compressed, 0o644)) + + _, err := readCompressed(path) + require.Error(t, err) + assert.ErrorIs(t, err, errCorruptArtifact) + assert.Contains(t, err.Error(), "window size exceeded") +} + +func TestWriteArtifactRejectsOversizedDecodedOutputWithoutWriting(t *testing.T) { + root := t.TempDir() + origin := "peer-a1b2c3" + segmentData, err := canonicalJSON(segmentMessage{ + Version: formatVersion, + Ordinal: 0, + Role: "user", + Content: "hello", + }) + require.NoError(t, err) + manifestData, err := canonicalJSON(manifest{ + Version: formatVersion, + Origin: origin, + NativeSessionID: "sess-1", + Session: manifestSession{ + ID: "sess-1", + Machine: origin, + }, + Segments: []string{strings64("a")}, + }) + require.NoError(t, err) + + tests := []struct { + name string + kind string + extension string + prefix []byte + total int64 + }{ + { + name: "manifest", kind: KindManifests, extension: manifestExtension, + prefix: manifestData, total: 16<<20 + 1, + }, + { + name: "segment", kind: KindSegments, extension: segmentExtension, + prefix: segmentData, total: 64<<20 + 1, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + compressed, hash := compressedArtifactWithPadding(t, tt.prefix, tt.total) + + _, err := WriteArtifact(root, origin, tt.kind, hash, compressed) + require.Error(t, err) + assert.ErrorIs(t, err, ErrArtifactInvalid) + assert.Contains(t, err.Error(), "decoded output exceeds") + assert.NoFileExists(t, filepath.Join(root, origin, tt.kind, hash+tt.extension)) + }) + } +} + +func TestWriteArtifactRejectsFrameAboveWriterWindowWithoutWriting(t *testing.T) { + root := t.TempDir() + origin := "peer-a1b2c3" + prefix, err := canonicalJSON(segmentMessage{ + Version: formatVersion, + Ordinal: 0, + Role: "user", + Content: "hello", + }) + require.NoError(t, err) + compressed, hash := compressedArtifactWithPadding( + t, prefix, 9<<20, + zstd.WithWindowSize(16<<20), + zstd.WithEncoderConcurrency(1), + ) + + _, err = WriteArtifact(root, origin, KindSegments, hash, compressed) + require.Error(t, err) + assert.ErrorIs(t, err, ErrArtifactInvalid) + assert.Contains(t, err.Error(), "window size exceeded") + assert.NoFileExists(t, filepath.Join(root, origin, KindSegments, hash+segmentExtension)) +} + +func TestWriteArtifactRejectsManifestStructuralAmplification(t *testing.T) { + origin := "peer-a1b2c3" + validHash := strings64("a") + tests := []struct { + name string + manifest manifest + wantError string + }{ + { + name: "duplicate segment references", + manifest: manifest{ + Segments: []string{validHash, validHash}, + }, + wantError: "duplicate segment reference", + }, + { + name: "too many segment references", + manifest: manifest{ + Segments: syntheticSegmentHashes(17), + }, + wantError: "segment reference limit", + }, + { + name: "too many usage events", + manifest: manifest{ + Segments: []string{validHash}, + UsageEvents: make([]artifactUsageEvent, 32_769), + }, + wantError: "usage event limit", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + root := t.TempDir() + m := tt.manifest + m.Version = formatVersion + m.Origin = origin + m.NativeSessionID = "sess-1" + m.Session = manifestSession{ID: "sess-1", Machine: origin} + data, err := canonicalJSON(m) + require.NoError(t, err) + hash := hashHex(data) + + _, err = WriteArtifact( + root, origin, KindManifests, hash, compressPeerTestData(t, data), + ) + require.Error(t, err) + assert.ErrorIs(t, err, ErrArtifactInvalid) + assert.Contains(t, err.Error(), tt.wantError) + assert.NoFileExists(t, filepath.Join( + root, origin, KindManifests, hash+manifestExtension, + )) + }) + } +} + +func TestWriteArtifactRejectsSegmentRecordAmplification(t *testing.T) { + root := t.TempDir() + origin := "peer-a1b2c3" + data := syntheticSegmentRecords(t, 4_097) + hash := hashHex(data) + + _, err := WriteArtifact( + root, origin, KindSegments, hash, compressPeerTestData(t, data), + ) + require.Error(t, err) + assert.ErrorIs(t, err, ErrArtifactInvalid) + assert.Contains(t, err.Error(), "message record limit") + assert.NoFileExists(t, filepath.Join( + root, origin, KindSegments, hash+segmentExtension, + )) +} + +func TestWriteArtifactRejectsSegmentNestedAmplificationWithoutWriting(t *testing.T) { + origin := "peer-a1b2c3" + tests := []struct { + name string + record segmentMessage + wantError string + }{ + { + name: "too many tool calls in one message", + record: segmentMessage{ + ToolCalls: make([]segmentToolCall, 257), + }, + wantError: "tool call limit", + }, + { + name: "too many result events in one tool call", + record: segmentMessage{ + ToolCalls: []segmentToolCall{{ + ResultEvents: make([]segmentResultEvent, 1_025), + }}, + }, + wantError: "result event limit", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + root := t.TempDir() + record := tt.record + record.Version = formatVersion + record.Ordinal = 7 + record.Role = "assistant" + data, err := canonicalJSON(record) + require.NoError(t, err) + hash := hashHex(data) + + _, err = WriteArtifact( + root, origin, KindSegments, hash, compressPeerTestData(t, data), + ) + require.Error(t, err) + assert.ErrorIs(t, err, ErrArtifactInvalid) + assert.Contains(t, err.Error(), tt.wantError) + assert.NoFileExists(t, filepath.Join( + root, origin, KindSegments, hash+segmentExtension, + )) + }) + } +} + +func TestWriteArtifactRejectsBlankSegmentRecordsWithoutWriting(t *testing.T) { + root := t.TempDir() + origin := "peer-a1b2c3" + data := bytes.Repeat([]byte("\n"), 4_097) + hash := hashHex(data) + + _, err := WriteArtifact( + root, origin, KindSegments, hash, compressPeerTestData(t, data), + ) + require.Error(t, err) + assert.ErrorIs(t, err, ErrArtifactInvalid) + assert.Contains(t, err.Error(), "blank message record") + assert.NoFileExists(t, filepath.Join( + root, origin, KindSegments, hash+segmentExtension, + )) +} + +func TestReadManifestRejectsDuplicateSegmentReferences(t *testing.T) { + originRoot := t.TempDir() + hash := strings64("a") + data, err := canonicalJSON(manifest{ + Version: formatVersion, + Origin: "laptop-a1b2c3", + NativeSessionID: "sess-1", + Session: manifestSession{ + ID: "sess-1", + Machine: "laptop-a1b2c3", + }, + Segments: []string{hash, hash}, + }) + require.NoError(t, err) + manifestHash := hashHex(data) + path := filepath.Join(originRoot, KindManifests, manifestHash+manifestExtension) + require.NoError(t, writeCompressed(path, data)) + + _, err = readManifest(originRoot, manifestHash) + require.Error(t, err) + assert.ErrorIs(t, err, errCorruptArtifact) + assert.Contains(t, err.Error(), "duplicate segment reference") + assert.NoFileExists(t, path) + assert.FileExists(t, path+quarantineSuffix) +} + +func TestReadSegmentRejectsRecordAmplification(t *testing.T) { + originRoot := t.TempDir() + data := syntheticSegmentRecords(t, 4_097) + hash := hashHex(data) + path := filepath.Join(originRoot, KindSegments, hash+segmentExtension) + require.NoError(t, writeCompressed(path, data)) + + _, err := readSegmentMessages(originRoot, hash) + require.Error(t, err) + assert.ErrorIs(t, err, errCorruptArtifact) + assert.Contains(t, err.Error(), "message record limit") + assert.NoFileExists(t, path) + assert.FileExists(t, path+quarantineSuffix) +} + +func TestReadSegmentQuarantinesNestedAmplification(t *testing.T) { + tests := []struct { + name string + record segmentMessage + wantError string + }{ + { + name: "too many tool calls in one message", + record: segmentMessage{ + ToolCalls: make([]segmentToolCall, 257), + }, + wantError: "tool call limit", + }, + { + name: "too many result events in one tool call", + record: segmentMessage{ + ToolCalls: []segmentToolCall{{ + ResultEvents: make([]segmentResultEvent, 1_025), + }}, + }, + wantError: "result event limit", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + originRoot := t.TempDir() + record := tt.record + record.Version = formatVersion + record.Ordinal = 7 + record.Role = "assistant" + data, err := canonicalJSON(record) + require.NoError(t, err) + hash := hashHex(data) + path := filepath.Join(originRoot, KindSegments, hash+segmentExtension) + require.NoError(t, writeCompressed(path, data)) + + _, err = readSegmentMessages(originRoot, hash) + require.Error(t, err) + assert.ErrorIs(t, err, errCorruptArtifact) + assert.Contains(t, err.Error(), tt.wantError) + assert.NoFileExists(t, path) + assert.FileExists(t, path+quarantineSuffix) + }) + } +} + +func TestReadSegmentQuarantinesBlankRecords(t *testing.T) { + originRoot := t.TempDir() + data := bytes.Repeat([]byte("\n"), 4_097) + hash := hashHex(data) + path := filepath.Join(originRoot, KindSegments, hash+segmentExtension) + require.NoError(t, writeCompressed(path, data)) + + _, err := readSegmentMessages(originRoot, hash) + require.Error(t, err) + assert.ErrorIs(t, err, errCorruptArtifact) + assert.Contains(t, err.Error(), "blank message record") + assert.NoFileExists(t, path) + assert.FileExists(t, path+quarantineSuffix) +} + +func TestReadManifestMessagesRejectsSessionMessageAmplification(t *testing.T) { + originRoot := t.TempDir() + m := manifest{Segments: make([]string, 0, 9)} + ordinal := 0 + for segment := range 9 { + count := 4_096 + if segment == 8 { + count = 1 + } + data := syntheticSegmentRecordsFrom(t, ordinal, count) + ordinal += count + hash := hashHex(data) + require.NoError(t, writeCompressed( + filepath.Join(originRoot, KindSegments, hash+segmentExtension), data, + )) + m.Segments = append(m.Segments, hash) + } + + _, err := readManifestMessages(originRoot, m) + require.Error(t, err) + assert.ErrorIs(t, err, errCorruptArtifact) + assert.Contains(t, err.Error(), "session message limit") +} + +func TestReadManifestMessagesRejectsSessionDecodedByteBudgetWithSmallLimit(t *testing.T) { + originRoot := t.TempDir() + m := manifest{} + totalBytes := 0 + for ordinal := range 2 { + data := syntheticSegmentRecordsFrom(t, ordinal, 1) + totalBytes += len(data) + hash := hashHex(data) + require.NoError(t, writeCompressed( + filepath.Join(originRoot, KindSegments, hash+segmentExtension), data, + )) + m.Segments = append(m.Segments, hash) + } + limits := productionArtifactLimits() + limits.sessionDecodedBytes = int64(totalBytes - 1) + + _, err := readManifestMessagesWithLimits(originRoot, m, limits) + require.Error(t, err) + assert.ErrorIs(t, err, errCorruptArtifact) + assert.Contains(t, err.Error(), "session decoded byte limit") +} + +func TestExportChunksOnMessageRecordLimit(t *testing.T) { + ctx := context.Background() + database := testDB(t) + root := t.TempDir() + origin := "laptop-a1b2c3" + seedSession(t, database, "sess-1", "alpha") + msgs := make([]db.Message, 4_097) + for i := range msgs { + msgs[i] = db.Message{SessionID: "sess-1", Ordinal: i, Role: "user"} + } + require.NoError(t, database.ReplaceSessionMessages("sess-1", msgs)) + + _, err := Export(ctx, database, root, origin) + require.NoError(t, err) + cp, err := readLatestCheckpoint(filepath.Join(root, origin)) + require.NoError(t, err) + require.NotNil(t, cp) + m, err := readManifest(filepath.Join(root, origin), cp.Sessions[origin+"~sess-1"]) + require.NoError(t, err) + require.Len(t, m.Segments, 2) + got, err := readManifestMessages(filepath.Join(root, origin), m) + require.NoError(t, err) + assert.Len(t, got, 4_097) +} + +func TestExportRejectsOversizedGeneratedManifestBeforePublication(t *testing.T) { + ctx := context.Background() + database := testDB(t) + root := t.TempDir() + origin := "laptop-a1b2c3" + seedSession(t, database, "sess-1", "alpha", func(sess *db.Session) { + first := strings.Repeat("x", int(manifestDecodedLimit)) + sess.FirstMessage = &first + }) + + _, err := Export(ctx, database, root, origin) + require.Error(t, err) + assert.Contains(t, err.Error(), "generated manifest exceeds") + assertNoPublishedArtifactFiles(t, root, origin) + state, err := database.GetSyncState(exportStateKey(origin, "sess-1")) + require.NoError(t, err) + assert.Empty(t, state) +} + +func TestExportRejectsSessionMessageAmplificationBeforePublication(t *testing.T) { + ctx := context.Background() + database := testDB(t) + root := t.TempDir() + origin := "laptop-a1b2c3" + seedSession(t, database, "sess-1", "alpha") + msgs := make([]db.Message, 32_769) + for i := range msgs { + msgs[i] = db.Message{SessionID: "sess-1", Ordinal: i, Role: "user"} + } + require.NoError(t, database.ReplaceSessionMessages("sess-1", msgs)) + + _, err := Export(ctx, database, root, origin) + require.Error(t, err) + assert.Contains(t, err.Error(), "session message limit") + assertNoPublishedArtifactFiles(t, root, origin) + state, err := database.GetSyncState(exportStateKey(origin, "sess-1")) + require.NoError(t, err) + assert.Empty(t, state) +} + +func TestExportRejectsUsageEventAmplificationBeforePublication(t *testing.T) { + ctx := context.Background() + database := testDB(t) + root := t.TempDir() + origin := "laptop-a1b2c3" + seedSession(t, database, "sess-1", "alpha") + events := make([]db.UsageEvent, 32_769) + for i := range events { + events[i] = db.UsageEvent{ + SessionID: "sess-1", + Source: "fixture", + DedupKey: fmt.Sprintf("usage-%d", i), + } + } + require.NoError(t, database.ReplaceSessionUsageEvents("sess-1", events)) + + _, err := Export(ctx, database, root, origin) + require.Error(t, err) + assert.Contains(t, err.Error(), "usage event limit") + assertNoPublishedArtifactFiles(t, root, origin) + state, err := database.GetSyncState(exportStateKey(origin, "sess-1")) + require.NoError(t, err) + assert.Empty(t, state) +} + +func TestExportSessionRejectsAggregateLimitsBeforeWritingWithSmallLimits(t *testing.T) { + tests := []struct { + name string + configure func(*artifactLimits) + wantError string + }{ + { + name: "decoded bytes", + configure: func(limits *artifactLimits) { + limits.sessionDecodedBytes = 1 + }, + wantError: "session decoded byte limit", + }, + { + name: "segment references", + configure: func(limits *artifactLimits) { + limits.segmentMessages = 1 + limits.manifestSegments = 1 + }, + wantError: "segment reference limit", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctx := context.Background() + database := testDB(t) + root := t.TempDir() + origin := "laptop-a1b2c3" + seedSession(t, database, "sess-1", "alpha") + limits := productionArtifactLimits() + tt.configure(&limits) + + _, _, err := exportSessionWithLimits( + ctx, database, filepath.Join(root, origin), origin, "sess-1", "", limits, + ) + require.Error(t, err) + assert.Contains(t, err.Error(), tt.wantError) + assert.Empty(t, globArtifacts( + t, root, origin, KindSegments, "*"+segmentExtension, + )) + assert.Empty(t, globArtifacts( + t, root, origin, KindManifests, "*"+manifestExtension, + )) + }) + } +} + +func TestImportQuarantinesOversizedSegmentWithoutAdvancingState(t *testing.T) { + ctx := context.Background() + root := t.TempDir() + origin := "laptop-a1b2c3" + localOrigin := "desktop-d4e5f6" + importDB := testDB(t) + originRoot := filepath.Join(root, origin) + + prefix, err := canonicalJSON(segmentMessage{ + Version: formatVersion, + Ordinal: 0, + Role: "user", + Content: "hello", + ContentLength: 5, + }) + require.NoError(t, err) + compressed, segmentHash := compressedArtifactWithPadding(t, prefix, 64<<20+1) + segmentPath := filepath.Join(originRoot, KindSegments, segmentHash+segmentExtension) + require.NoError(t, os.MkdirAll(filepath.Dir(segmentPath), 0o755)) + require.NoError(t, os.WriteFile(segmentPath, compressed, 0o644)) + + gid := origin + "~sess-1" + m := manifest{ + Version: formatVersion, + Origin: origin, + NativeSessionID: "sess-1", + Session: manifestSession{ + ID: "sess-1", + Machine: origin, + }, + Segments: []string{segmentHash}, + } + manifestData, err := canonicalJSON(m) + require.NoError(t, err) + manifestHash := hashHex(manifestData) + require.NoError(t, writeCompressed( + filepath.Join(originRoot, KindManifests, manifestHash+manifestExtension), + manifestData, + )) + writeCheckpoint(t, originRoot, checkpoint{ + Version: formatVersion, + Origin: origin, + Sequence: 1, + Sessions: map[string]string{gid: manifestHash}, + }) + + res, err := ImportDetailed(ctx, importDB, root, localOrigin) + require.NoError(t, err) + assert.False(t, res.Changed()) + state, err := importDB.GetSyncState(importStateKey(origin, gid)) + require.NoError(t, err) + assert.Empty(t, state) + got, err := importDB.GetSessionFull(ctx, gid) + require.NoError(t, err) + assert.Nil(t, got) + assert.NoFileExists(t, segmentPath) + assert.FileExists(t, segmentPath+quarantineSuffix) +} + +func TestExportChunksLargeMultiMessageSessionInOrder(t *testing.T) { + ctx := context.Background() + database := testDB(t) + root := t.TempDir() + origin := "laptop-a1b2c3" + seedSession(t, database, "sess-1", "alpha") + content := strings.Repeat("x", 9<<20) + msgs := make([]db.Message, 4) + for i := range msgs { + msgs[i] = db.Message{ + SessionID: "sess-1", + Ordinal: i, + Role: "user", + Content: content, + ContentLength: len(content), + } + } + require.NoError(t, database.ReplaceSessionMessages("sess-1", msgs)) + + count, err := Export(ctx, database, root, origin) + require.NoError(t, err) + assert.Equal(t, 1, count) + cp, err := readLatestCheckpoint(filepath.Join(root, origin)) + require.NoError(t, err) + require.NotNil(t, cp) + m, err := readManifest(filepath.Join(root, origin), cp.Sessions[origin+"~sess-1"]) + require.NoError(t, err) + require.Len(t, m.Segments, 2) + + got, err := readManifestMessages(filepath.Join(root, origin), m) + require.NoError(t, err) + require.Len(t, got, 4) + for i := range got { + assert.Equal(t, i, got[i].Ordinal) + assert.Equal(t, content, got[i].Content) + } +} + +func TestExportRejectsSingleEncodedRecordAboveReadableLimit(t *testing.T) { + ctx := context.Background() + database := testDB(t) + root := t.TempDir() + origin := "laptop-a1b2c3" + seedSession(t, database, "sess-1", "alpha") + content := strings.Repeat("x", 64<<20) + require.NoError(t, database.ReplaceSessionMessages("sess-1", []db.Message{{ + SessionID: "sess-1", + Ordinal: 0, + Role: "user", + Content: content, + ContentLength: len(content), + }})) + + _, err := Export(ctx, database, root, origin) + require.Error(t, err) + assert.Contains(t, err.Error(), "encoded message record") + assert.Contains(t, err.Error(), "67108864-byte readable limit") + assert.Empty(t, globArtifacts(t, root, origin, KindCheckpoints, "cp-*.json")) +} + +func TestExportPreservesSmallSingleSegmentHash(t *testing.T) { + ctx := context.Background() + database := testDB(t) + root := t.TempDir() + origin := "laptop-a1b2c3" + seedSession(t, database, "sess-1", "alpha") + msgs, err := database.GetAllMessages(ctx, "sess-1") + require.NoError(t, err) + segmentData, err := encodeSegment(canonicalMessages(msgs)) + require.NoError(t, err) + wantHash := hashHex(segmentData) + + _, err = Export(ctx, database, root, origin) + require.NoError(t, err) + cp, err := readLatestCheckpoint(filepath.Join(root, origin)) + require.NoError(t, err) + require.NotNil(t, cp) + m, err := readManifest(filepath.Join(root, origin), cp.Sessions[origin+"~sess-1"]) + require.NoError(t, err) + assert.Equal(t, []string{wantHash}, m.Segments) +} + +type repeatedByteReader byte + +func (r repeatedByteReader) Read(p []byte) (int, error) { + for i := range p { + p[i] = byte(r) + } + return len(p), nil +} + +func compressedArtifactWithPadding( + t *testing.T, + prefix []byte, + total int64, + opts ...zstd.EOption, +) ([]byte, string) { + t.Helper() + require.LessOrEqual(t, int64(len(prefix)), total) + var compressed bytes.Buffer + enc, err := zstd.NewWriter(&compressed, opts...) + require.NoError(t, err) + h := sha256.New() + w := io.MultiWriter(enc, h) + _, err = w.Write(prefix) + require.NoError(t, err) + _, err = io.CopyN(w, repeatedByteReader('\n'), total-int64(len(prefix))) + require.NoError(t, err) + require.NoError(t, enc.Close()) + return compressed.Bytes(), fmt.Sprintf("%x", h.Sum(nil)) +} + +func syntheticSegmentHashes(count int) []string { + hashes := make([]string, count) + for i := range hashes { + hashes[i] = fmt.Sprintf("%064x", i+1) + } + return hashes +} + +func syntheticSegmentRecords(t *testing.T, count int) []byte { + return syntheticSegmentRecordsFrom(t, 0, count) +} + +func syntheticSegmentRecordsFrom(t *testing.T, start, count int) []byte { + t.Helper() + var data bytes.Buffer + for i := range count { + record, err := canonicalJSON(segmentMessage{ + Version: formatVersion, + Ordinal: start + i, + Role: "user", + }) + require.NoError(t, err) + _, err = data.Write(record) + require.NoError(t, err) + } + return data.Bytes() +} + +func assertNoPublishedArtifactFiles(t *testing.T, root, origin string) { + t.Helper() + for _, kind := range []string{ + KindCheckpoints, KindManifests, KindSegments, KindMeta, KindRaw, + } { + entries, err := os.ReadDir(filepath.Join(root, origin, kind)) + require.NoError(t, err) + assert.Empty(t, entries, "unexpected published %s artifact", kind) + } +} diff --git a/internal/artifact/export_usage_test.go b/internal/artifact/export_usage_test.go new file mode 100644 index 000000000..73876614d --- /dev/null +++ b/internal/artifact/export_usage_test.go @@ -0,0 +1,92 @@ +package artifact + +import ( + "context" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.kenn.io/agentsview/internal/db" +) + +func TestExportIncludesUsageOnlySession(t *testing.T) { + ctx := context.Background() + root := t.TempDir() + origin := "laptop-a1b2c3" + database := testDB(t) + + // A usage-only session: owned, zero messages, with a usage event. The + // sidebar list filter (message_count > 0) hides it, so export must not rely + // on that path. + require.NoError(t, database.UpsertSession(db.Session{ + ID: "usage-1", + Project: "alpha", + Machine: "local", + Agent: "claude", + CreatedAt: "2026-06-14T01:02:03Z", + })) + require.NoError(t, database.ReplaceSessionUsageEvents("usage-1", []db.UsageEvent{{ + SessionID: "usage-1", + Source: "assistant", + Model: "claude-opus-4-8", + InputTokens: 10, + OutputTokens: 20, + OccurredAt: "2026-06-14T01:02:30Z", + DedupKey: "usage-1:0", + }})) + + exported, err := Export(ctx, database, root, origin) + require.NoError(t, err) + assert.Equal(t, 1, exported) + + gid := origin + "~usage-1" + cp, err := readLatestCheckpoint(filepath.Join(root, origin)) + require.NoError(t, err) + require.NotNil(t, cp) + assert.Contains(t, cp.Sessions, gid, "usage-only session must be in the checkpoint") + + // It imports into a peer as a real session with its usage events intact. + importDB := testDB(t) + res, err := ImportDetailed(ctx, importDB, root, "desktop-d4e5f6") + require.NoError(t, err) + assert.Equal(t, 1, res.Sessions) + + got, err := importDB.GetSessionFull(ctx, gid) + require.NoError(t, err) + require.NotNil(t, got) + events, err := importDB.GetUsageEvents(ctx, gid) + require.NoError(t, err) + require.Len(t, events, 1) + assert.Equal(t, 10, events[0].InputTokens) + assert.Equal(t, 20, events[0].OutputTokens) +} + +func TestExportSkipsDeletedAndForeignSessions(t *testing.T) { + ctx := context.Background() + root := t.TempDir() + origin := "laptop-a1b2c3" + database := testDB(t) + + seedSession(t, database, "owned", "alpha") + // A soft-deleted owned session and a foreign-owned session are both excluded. + seedSession(t, database, "trashed", "alpha") + require.NoError(t, database.SoftDeleteSession("trashed")) + require.NoError(t, database.UpsertSession(db.Session{ + ID: "foreign", + Project: "alpha", + Machine: "desktop-d4e5f6", + Agent: "claude", + CreatedAt: "2026-06-14T01:02:03Z", + })) + + _, err := Export(ctx, database, root, origin) + require.NoError(t, err) + + cp, err := readLatestCheckpoint(filepath.Join(root, origin)) + require.NoError(t, err) + require.NotNil(t, cp) + assert.Contains(t, cp.Sessions, origin+"~owned") + assert.NotContains(t, cp.Sessions, origin+"~trashed") + assert.NotContains(t, cp.Sessions, origin+"~foreign") +} diff --git a/internal/artifact/format_test.go b/internal/artifact/format_test.go new file mode 100644 index 000000000..7c6b95d80 --- /dev/null +++ b/internal/artifact/format_test.go @@ -0,0 +1,212 @@ +package artifact + +import ( + "encoding/json" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.kenn.io/agentsview/internal/db" +) + +func TestCanonicalCheckpointGolden(t *testing.T) { + cp := checkpoint{ + Version: formatVersion, + Origin: "laptop-a1b2c3", + Sequence: 7, + Sessions: map[string]string{ + "laptop-a1b2c3~sess-b": "b222", + "laptop-a1b2c3~sess-a": "a111", + }, + } + + data, err := canonicalJSON(cp) + require.NoError(t, err) + + assert.Equal(t, + "{\"origin\":\"laptop-a1b2c3\",\"seq\":7,\"sessions\":{\"laptop-a1b2c3~sess-a\":\"a111\",\"laptop-a1b2c3~sess-b\":\"b222\"},\"v\":1}\n", + string(data), + ) + assert.Equal(t, "56fd64d35ebd700bfa2a50d41a97857b871e033e5c1e1d02dea55c25c7df7655", hashHex(data)) +} + +func TestCanonicalManifestGolden(t *testing.T) { + cost := 0.03125 + ordinal := 2 + parent := "parent-1" + name := "Fixture" + raw := rawSourceRef{ + Hash: "raw123", + Size: 4096, + MediaType: "application/jsonl", + Path: "claude/session.jsonl", + } + m := manifest{ + Version: formatVersion, + Origin: "laptop-a1b2c3", + NativeSessionID: "sess-1", + Session: manifestSession{ + ID: "sess-1", + Project: "alpha", + Machine: "laptop-a1b2c3", + Agent: "claude", + FirstMessage: new("hello"), + StartedAt: new("2026-06-14T01:02:03Z"), + EndedAt: new("2026-06-14T01:03:03Z"), + MessageCount: 2, + UserMessageCount: 1, + ParentSessionID: &parent, + RelationshipType: "subagent", + TotalOutputTokens: 42, + CreatedAt: "2026-06-14T01:02:03Z", + }, + SessionName: &name, + Segments: []string{"seg222", "seg111"}, + UsageEvents: []artifactUsageEvent{ + { + MessageOrdinal: &ordinal, + Source: "fixture", + Model: "claude-test", + InputTokens: 11, + OutputTokens: 7, + CostUSD: &cost, + CostStatus: "known", + CostSource: "fixture", + OccurredAt: "2026-06-14T01:02:04Z", + DedupKey: "usage-1", + }, + }, + RawSource: &raw, + DataVersion: 99, + Generation: 3, + SessionHasToolCalls: true, + SessionHasContextData: true, + SessionQualitySignals: &manifestQualitySignals{ + Version: 3, + ShortPromptCount: 2, + UnstructuredStart: true, + MissingSuccessCriteriaCount: 4, + MissingVerificationCount: 5, + DuplicatePromptCount: 6, + NoCodeContextCount: 7, + RunawayToolLoopCount: 1, + }, + } + + data, err := canonicalJSON(m) + require.NoError(t, err) + + assert.Equal(t, + "{\"data_version\":99,\"generation\":3,\"native_session_id\":\"sess-1\",\"origin\":\"laptop-a1b2c3\",\"raw_source\":{\"hash\":\"raw123\",\"media_type\":\"application/jsonl\",\"path\":\"claude/session.jsonl\",\"size\":4096},\"segments\":[\"seg222\",\"seg111\"],\"session\":{\"agent\":\"claude\",\"compaction_count\":0,\"consecutive_failure_max\":0,\"created_at\":\"2026-06-14T01:02:03Z\",\"edit_churn_count\":0,\"ended_at\":\"2026-06-14T01:03:03Z\",\"ended_with_role\":\"\",\"final_failure_streak\":0,\"first_message\":\"hello\",\"has_peak_context_tokens\":false,\"has_total_output_tokens\":false,\"id\":\"sess-1\",\"is_automated\":false,\"machine\":\"laptop-a1b2c3\",\"message_count\":2,\"mid_task_compaction_count\":0,\"outcome\":\"\",\"outcome_confidence\":\"\",\"parent_session_id\":\"parent-1\",\"peak_context_tokens\":0,\"project\":\"alpha\",\"relationship_type\":\"subagent\",\"secret_leak_count\":0,\"started_at\":\"2026-06-14T01:02:03Z\",\"tool_failure_signal_count\":0,\"tool_retry_count\":0,\"total_output_tokens\":42,\"user_message_count\":1},\"session_has_context_data\":true,\"session_has_tool_calls\":true,\"session_name\":\"Fixture\",\"session_quality_signals\":{\"duplicate_prompt_count\":6,\"missing_success_criteria_count\":4,\"missing_verification_count\":5,\"no_code_context_count\":7,\"runaway_tool_loop_count\":1,\"short_prompt_count\":2,\"unstructured_start\":true,\"version\":3},\"usage_events\":[{\"cost_source\":\"fixture\",\"cost_status\":\"known\",\"cost_usd\":0.03125,\"dedup_key\":\"usage-1\",\"input_tokens\":11,\"message_ordinal\":2,\"model\":\"claude-test\",\"occurred_at\":\"2026-06-14T01:02:04Z\",\"output_tokens\":7,\"source\":\"fixture\"}],\"v\":1}\n", + string(data), + ) + assert.Equal(t, "1a563d1b1642cf850bb2253643d8cb628a91499cbf48607a6c40b15af01b4a6f", hashHex(data)) +} + +func TestCanonicalMessageSegmentGolden(t *testing.T) { + msgs := []db.Message{ + { + ID: 99, + SessionID: "sess-1", + Ordinal: 2, + Role: "assistant", + Content: "world", + ContentLength: 5, + Timestamp: "2026-06-14T01:02:05Z", + HasToolUse: true, + Model: "claude-test", + TokenUsage: json.RawMessage(`{"output":2,"input":1}`), + OutputTokens: 2, + HasOutputTokens: true, + ClaudeMessageID: "msg-1", + ClaudeRequestID: "req-1", + SourceType: "jsonl", + SourceSubtype: "assistant", + SourceUUID: "uuid-msg-1", + SourceParentUUID: "uuid-parent", + ToolCalls: []db.ToolCall{ + { + MessageID: 99, + SessionID: "sess-1", + ToolName: "Read", + Category: "file", + ToolUseID: "tool-1", + InputJSON: "{\"file_path\":\"README.md\"}", + FilePath: "README.md", + ResultContentLength: 12, + ResultContent: "file content", + SubagentSessionID: "child-1", + ResultEvents: []db.ToolResultEvent{ + { + ToolUseID: "tool-1", + AgentID: "agent-1", + SubagentSessionID: "child-1", + Source: "tool_result", + Status: "success", + Content: "done", + ContentLength: 4, + Timestamp: "2026-06-14T01:02:06Z", + EventIndex: 0, + }, + }, + }, + }, + }, + } + + data, err := encodeSegment(msgs) + require.NoError(t, err) + + assert.Equal(t, + "{\"claude_message_id\":\"msg-1\",\"claude_request_id\":\"req-1\",\"content\":\"world\",\"content_length\":5,\"has_output_tokens\":true,\"has_tool_use\":true,\"model\":\"claude-test\",\"ordinal\":2,\"output_tokens\":2,\"role\":\"assistant\",\"source_parent_uuid\":\"uuid-parent\",\"source_subtype\":\"assistant\",\"source_type\":\"jsonl\",\"source_uuid\":\"uuid-msg-1\",\"timestamp\":\"2026-06-14T01:02:05Z\",\"token_usage\":{\"input\":1,\"output\":2},\"tool_calls\":[{\"call_index\":0,\"category\":\"file\",\"file_path\":\"README.md\",\"input_json\":\"{\\\"file_path\\\":\\\"README.md\\\"}\",\"result_content\":\"file content\",\"result_content_length\":12,\"result_events\":[{\"agent_id\":\"agent-1\",\"content\":\"done\",\"content_length\":4,\"event_index\":0,\"source\":\"tool_result\",\"status\":\"success\",\"subagent_session_id\":\"child-1\",\"timestamp\":\"2026-06-14T01:02:06Z\",\"tool_use_id\":\"tool-1\"}],\"subagent_session_id\":\"child-1\",\"tool_name\":\"Read\",\"tool_use_id\":\"tool-1\"}],\"v\":1}\n", + string(data), + ) + assert.NotContains(t, string(data), `"id"`) + assert.NotContains(t, string(data), `"session_id"`) + assert.NotContains(t, string(data), `"message_id"`) + assert.Equal(t, "f46c1edbc77dab4eb15f43bcb3ce196243c784445b07e9135c223b5d58c6dea5", hashHex(data)) +} + +func TestCanonicalMetadataEventGolden(t *testing.T) { + value := json.RawMessage(`{"display_name":"Renamed session"}`) + note := "remember this" + event := metadataEvent{ + Version: formatVersion, + HLC: "2026-06-14T010203.000000001Z-laptop-a1b2c3", + Origin: "laptop-a1b2c3", + SessionGID: "desktop-d4e5f6~sess-1", + Op: "rename", + Value: value, + Pin: &MetadataPin{ + SourceUUID: "uuid-msg-1", + Ordinal: 2, + Note: ¬e, + }, + } + + data, err := canonicalJSON(event) + require.NoError(t, err) + + assert.Equal(t, + "{\"hlc\":\"2026-06-14T010203.000000001Z-laptop-a1b2c3\",\"op\":\"rename\",\"origin\":\"laptop-a1b2c3\",\"pin\":{\"note\":\"remember this\",\"ordinal\":2,\"source_uuid\":\"uuid-msg-1\"},\"session_gid\":\"desktop-d4e5f6~sess-1\",\"v\":1,\"value\":{\"display_name\":\"Renamed session\"}}\n", + string(data), + ) + assert.Equal(t, "fcb36d602e56fe1616ba6e2f86e973adde4ef547e0ecf280b37eb534b60e4b71", hashHex(data)) +} + +func TestCompressedArtifactsUseUncompressedContentHash(t *testing.T) { + data := []byte("{\"v\":1}\n") + hash := hashHex(data) + path := filepath.Join(t.TempDir(), hash+manifestExtension) + + require.NoError(t, writeCompressed(path, data)) + read, err := readCompressed(path) + require.NoError(t, err) + + assert.Equal(t, data, read) + assert.Equal(t, "2b4248702881de2f5638efe96b233de1c0dd9be5dd24ec35ad030d6b06aede9a", hash) + _, err = os.Stat(path) + require.NoError(t, err) +} diff --git a/internal/artifact/gc.go b/internal/artifact/gc.go new file mode 100644 index 000000000..03b2626eb --- /dev/null +++ b/internal/artifact/gc.go @@ -0,0 +1,583 @@ +package artifact + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "io/fs" + "os" + "path/filepath" + "sort" + "time" +) + +// GCOptions configures conservative artifact garbage collection. +type GCOptions struct { + Root string + Grace time.Duration + DryRun bool + Now time.Time + Logf func(string, ...any) +} + +// GCResult summarizes one artifact garbage collection scan. +type GCResult struct { + DryRun bool + Origins int + SkippedOrigins int + Scanned int + Candidates int + Eligible int + KeptByGrace int + Deleted int + BytesEligible int64 + BytesDeleted int64 +} + +type gcRef struct { + kind string + name string +} + +type gcCandidate struct { + path string + origin string + kind string + name string + size int64 + modTime time.Time +} + +type gcOriginResult struct { + skipped bool + scanned int + candidates []gcCandidate + kindRoots map[string]*os.Root +} + +// GarbageCollect deletes or reports superseded artifacts that are no longer +// reachable from an origin's latest checkpoint and its live manifests. +func GarbageCollect(ctx context.Context, opts GCOptions) (GCResult, error) { + if opts.Root == "" { + return GCResult{}, fmt.Errorf("artifact gc root is required") + } + if opts.Grace < 0 { + return GCResult{}, fmt.Errorf("artifact gc grace must be >= 0") + } + now := opts.Now + if now.IsZero() { + now = time.Now() + } + cutoff := now.Add(-opts.Grace) + + storeRoot, err := openArtifactRoot(opts.Root, "gc") + if err != nil { + return GCResult{}, fmt.Errorf("opening artifact gc root: %w", err) + } + defer storeRoot.Close() + origins, err := listGCOrigins(storeRoot) + if err != nil { + return GCResult{}, fmt.Errorf("listing artifact origins: %w", err) + } + res := GCResult{DryRun: opts.DryRun} + for _, origin := range origins { + if err := ctx.Err(); err != nil { + return res, err + } + originRoot, err := openArtifactSubroot(storeRoot, origin, "gc origin") + if err != nil { + return res, fmt.Errorf("opening artifact gc origin %s: %w", origin, err) + } + originRes, err := collectGCCandidates(ctx, originRoot, origin) + if err != nil { + _ = originRoot.Close() + if errors.Is(err, errIncompleteArtifact) || + errors.Is(err, errCorruptArtifact) || + errors.Is(err, errFutureArtifactVersion) { + res.Origins++ + res.SkippedOrigins++ + logGC(opts, "artifact gc: skipping %s: %v", origin, err) + continue + } + return res, fmt.Errorf("scanning %s: %w", origin, err) + } + res.Origins++ + if originRes.skipped { + closeGCKindRoots(originRes.kindRoots) + _ = originRoot.Close() + res.SkippedOrigins++ + logGC(opts, "artifact gc: skipping %s with no checkpoints", origin) + continue + } + res.Scanned += originRes.scanned + for _, cand := range originRes.candidates { + res.Candidates++ + if cand.modTime.After(cutoff) { + res.KeptByGrace++ + continue + } + res.Eligible++ + res.BytesEligible += cand.size + if opts.DryRun { + logGC(opts, "artifact gc: would delete %s (%d bytes)", cand.path, cand.size) + continue + } + if err := originRes.kindRoots[cand.kind].Remove(cand.name); err != nil { + if errors.Is(err, fs.ErrNotExist) { + continue + } + closeGCKindRoots(originRes.kindRoots) + _ = originRoot.Close() + return res, fmt.Errorf("deleting %s: %w", cand.path, err) + } + res.Deleted++ + res.BytesDeleted += cand.size + logGC(opts, "artifact gc: deleted %s (%d bytes)", cand.path, cand.size) + } + closeGCKindRoots(originRes.kindRoots) + if err := originRoot.Close(); err != nil { + return res, fmt.Errorf("closing artifact gc origin %s: %w", origin, err) + } + } + return res, nil +} + +func listGCOrigins(root *os.Root) ([]string, error) { + entries, err := fs.ReadDir(root.FS(), ".") + if err != nil { + return nil, err + } + origins := make([]string, 0, len(entries)) + for _, ent := range entries { + if validateOriginID(ent.Name()) != nil { + continue + } + info, err := root.Lstat(ent.Name()) + if err != nil { + return nil, err + } + if !info.IsDir() { + return nil, fmt.Errorf("artifact origin %s is not a directory", ent.Name()) + } + origins = append(origins, ent.Name()) + } + sort.Strings(origins) + return origins, nil +} + +func collectGCCandidates(ctx context.Context, root *os.Root, origin string) (gcOriginResult, error) { + kindRoots, err := openGCKindRoots(root) + if err != nil { + return gcOriginResult{}, err + } + keepRoots := false + defer func() { + if !keepRoots { + closeGCKindRoots(kindRoots) + } + }() + checkpoints, err := validGCCheckpointNames(kindRoots[KindCheckpoints]) + if err != nil { + return gcOriginResult{}, err + } + if len(checkpoints) == 0 { + return gcOriginResult{skipped: true}, nil + } + + latestName := checkpoints[len(checkpoints)-1] + data, err := kindRoots[KindCheckpoints].ReadFile(latestName) + if err != nil { + return gcOriginResult{}, fmt.Errorf("reading live checkpoint: %w", err) + } + var cp checkpoint + if err := json.Unmarshal(data, &cp); err != nil { + return gcOriginResult{}, fmt.Errorf("%w: decoding live checkpoint: %v", errCorruptArtifact, err) + } + if err := validateCheckpoint(&cp, origin); err != nil { + if !errors.Is(err, errFutureArtifactVersion) { + err = fmt.Errorf("%w: validating live checkpoint: %v", errCorruptArtifact, err) + } + return gcOriginResult{}, err + } + if err := validateCheckpointSequenceIdentity(cp, latestName); err != nil { + return gcOriginResult{}, fmt.Errorf( + "%w: validating live checkpoint identity: %v", errCorruptArtifact, err, + ) + } + live := map[gcRef]struct{}{ + {kind: KindCheckpoints, name: latestName}: {}, + } + + gids := make([]string, 0, len(cp.Sessions)) + for gid := range cp.Sessions { + gids = append(gids, gid) + } + sort.Strings(gids) + for _, gid := range gids { + if err := ctx.Err(); err != nil { + return gcOriginResult{}, err + } + manifestHash := cp.Sessions[gid] + if err := validateHashHex(manifestHash); err != nil { + return gcOriginResult{}, fmt.Errorf("live manifest %s: %w", gid, err) + } + manifestName := manifestHash + manifestExtension + live[gcRef{kind: KindManifests, name: manifestName}] = struct{}{} + + m, err := readGCManifest(kindRoots[KindManifests], manifestHash) + if err != nil { + return gcOriginResult{}, fmt.Errorf("reading live manifest %s: %w", manifestHash, err) + } + if err := validateManifest(m, origin, gid); err != nil { + if !errors.Is(err, errFutureArtifactVersion) { + err = fmt.Errorf("%w: validating live manifest: %v", errCorruptArtifact, err) + } + return gcOriginResult{}, err + } + if err := validateGCManifestMessagesWithLimits( + kindRoots[KindSegments], m, productionArtifactLimits(), + ); err != nil { + return gcOriginResult{}, fmt.Errorf("reading live segments for %s: %w", gid, err) + } + for _, segmentHash := range m.Segments { + live[gcRef{kind: KindSegments, name: segmentHash + segmentExtension}] = struct{}{} + } + if m.RawSource != nil && m.RawSource.Hash != "" { + if err := validateLiveRawSource(kindRoots[KindRaw], *m.RawSource); err != nil { + return gcOriginResult{}, fmt.Errorf("reading live raw source for %s: %w", gid, err) + } + live[gcRef{kind: KindRaw, name: m.RawSource.Hash}] = struct{}{} + } + } + + candidates, scanned, err := scanGCCandidates(ctx, root, kindRoots, origin, live) + if err != nil { + return gcOriginResult{}, err + } + keepRoots = true + return gcOriginResult{ + scanned: scanned, candidates: candidates, kindRoots: kindRoots, + }, nil +} + +func openGCKindRoots(root *os.Root) (map[string]*os.Root, error) { + roots := make(map[string]*os.Root, 4) + for _, kind := range []string{KindCheckpoints, KindManifests, KindSegments, KindRaw} { + info, err := root.Lstat(kind) + if errors.Is(err, fs.ErrNotExist) { + continue + } + if err != nil { + closeGCKindRoots(roots) + return nil, err + } + if !info.IsDir() { + closeGCKindRoots(roots) + return nil, fmt.Errorf("artifact kind %s is not a directory", kind) + } + kindRoot, err := openArtifactSubroot(root, kind, "gc kind") + if err != nil { + closeGCKindRoots(roots) + return nil, err + } + roots[kind] = kindRoot + } + return roots, nil +} + +func closeGCKindRoots(roots map[string]*os.Root) { + for _, root := range roots { + _ = root.Close() + } +} + +func readGCManifest(root *os.Root, hash string) (manifest, error) { + if root == nil { + return manifest{}, fmt.Errorf("%w: manifest %s", errIncompleteArtifact, hash) + } + name := hash + manifestExtension + compressed, err := root.ReadFile(name) + if err != nil { + if errors.Is(err, fs.ErrNotExist) { + return manifest{}, fmt.Errorf("%w: manifest %s", errIncompleteArtifact, hash) + } + return manifest{}, err + } + data, err := readCompressedBytes(compressed, manifestDecodedLimit) + if err != nil { + quarantineArtifactRoot(root, name) + return manifest{}, fmt.Errorf("%w: decoding manifest %s: %v", errCorruptArtifact, hash, err) + } + if got := hashHex(data); got != hash { + quarantineArtifactRoot(root, name) + return manifest{}, fmt.Errorf( + "%w: manifest %s hash mismatch: got %s", errCorruptArtifact, hash, got, + ) + } + m, err := decodeManifestWithLimits(data, productionArtifactLimits()) + if err != nil { + quarantineArtifactRoot(root, name) + return manifest{}, fmt.Errorf("%w: decoding manifest %s: %v", errCorruptArtifact, hash, err) + } + return m, nil +} + +func validateGCManifestMessagesWithLimits( + root *os.Root, m manifest, limits artifactLimits, +) error { + if err := validateManifestReferencesWithLimits(m, limits); err != nil { + return fmt.Errorf("%w: %v", errCorruptArtifact, err) + } + if root == nil && len(m.Segments) > 0 { + return fmt.Errorf("%w: segment %s", errIncompleteArtifact, m.Segments[0]) + } + var decodedBytes int64 + totalMessages := 0 + totalNested := nestedCollectionCounts{} + for _, hash := range m.Segments { + name := hash + segmentExtension + compressed, err := root.ReadFile(name) + if err != nil { + if errors.Is(err, fs.ErrNotExist) { + return fmt.Errorf("%w: segment %s", errIncompleteArtifact, hash) + } + return err + } + data, err := readCompressedBytes(compressed, segmentDecodedLimit) + if err != nil { + quarantineArtifactRoot(root, name) + return fmt.Errorf("%w: segment %s: %v", errCorruptArtifact, hash, err) + } + if got := hashHex(data); got != hash { + quarantineArtifactRoot(root, name) + return fmt.Errorf( + "%w: segment %s hash mismatch: got %s", errCorruptArtifact, hash, got, + ) + } + preflight, err := preflightSegmentData(data, limits) + if err != nil { + if errors.Is(err, errFutureArtifactVersion) { + return fmt.Errorf("segment %s: %w", hash, err) + } + quarantineArtifactRoot(root, name) + return fmt.Errorf("%w: segment %s: %v", errCorruptArtifact, hash, err) + } + segmentBytes := int64(len(data)) + if segmentBytes > limits.sessionDecodedBytes-decodedBytes { + return fmt.Errorf( + "%w: session decoded byte limit exceeded: limit %d", + errCorruptArtifact, limits.sessionDecodedBytes, + ) + } + if len(preflight.records) > limits.sessionMessages-totalMessages { + return fmt.Errorf( + "%w: session message limit exceeded: limit %d", + errCorruptArtifact, limits.sessionMessages, + ) + } + if exceedsCollectionLimit( + totalNested.toolCalls, preflight.nested.toolCalls, limits.sessionToolCalls, + ) { + return fmt.Errorf( + "%w: session tool call limit exceeded: limit %d", + errCorruptArtifact, limits.sessionToolCalls, + ) + } + if exceedsCollectionLimit( + totalNested.resultEvents, + preflight.nested.resultEvents, + limits.sessionResultEvents, + ) { + return fmt.Errorf( + "%w: session result event limit exceeded: limit %d", + errCorruptArtifact, limits.sessionResultEvents, + ) + } + messages, err := decodePreflightedSegment(preflight) + if err != nil { + quarantineArtifactRoot(root, name) + return fmt.Errorf("%w: segment %s: %v", errCorruptArtifact, hash, err) + } + decodedBytes += segmentBytes + totalMessages += len(messages) + totalNested.toolCalls += preflight.nested.toolCalls + totalNested.resultEvents += preflight.nested.resultEvents + } + return nil +} + +func validateLiveRawSource(root *os.Root, ref rawSourceRef) error { + if err := validateHashHex(ref.Hash); err != nil { + return err + } + if root == nil { + return fmt.Errorf("%w: raw source %s", errIncompleteArtifact, ref.Hash) + } + info, err := root.Lstat(ref.Hash) + if err != nil { + if errors.Is(err, fs.ErrNotExist) { + return fmt.Errorf("%w: raw source %s", errIncompleteArtifact, ref.Hash) + } + return err + } + if !info.Mode().IsRegular() { + return fmt.Errorf("%w: raw source %s is not a regular file", errCorruptArtifact, ref.Hash) + } + if ref.Size != 0 && info.Size() != ref.Size { + return fmt.Errorf( + "%w: raw source %s size mismatch: got %d, want %d", + errCorruptArtifact, ref.Hash, info.Size(), ref.Size, + ) + } + + file, err := root.Open(ref.Hash) + if err != nil { + if errors.Is(err, fs.ErrNotExist) { + return fmt.Errorf("%w: raw source %s", errIncompleteArtifact, ref.Hash) + } + return err + } + defer file.Close() + openedInfo, err := file.Stat() + if err != nil { + return err + } + if !openedInfo.Mode().IsRegular() || !os.SameFile(info, openedInfo) { + return fmt.Errorf("%w: raw source %s changed while opening", errCorruptArtifact, ref.Hash) + } + h := sha256.New() + size, err := io.Copy(h, file) + if err != nil { + return err + } + if ref.Size != 0 && size != ref.Size { + return fmt.Errorf( + "%w: raw source %s size mismatch while reading: got %d, want %d", + errCorruptArtifact, ref.Hash, size, ref.Size, + ) + } + if got := hex.EncodeToString(h.Sum(nil)); got != ref.Hash { + return fmt.Errorf( + "%w: raw source %s hash mismatch: got %s", + errCorruptArtifact, ref.Hash, got, + ) + } + return nil +} + +func validGCCheckpointNames(root *os.Root) ([]string, error) { + if root == nil { + return nil, nil + } + entries, err := fs.ReadDir(root.FS(), ".") + if err != nil { + return nil, err + } + valid := make([]string, 0, len(entries)) + for _, ent := range entries { + if _, err := checkpointSequence(ent.Name()); err == nil { + valid = append(valid, ent.Name()) + } + } + sort.Strings(valid) + return valid, nil +} + +func scanGCCandidates( + ctx context.Context, + originRoot *os.Root, + kindRoots map[string]*os.Root, + origin string, + live map[gcRef]struct{}, +) ([]gcCandidate, int, error) { + var candidates []gcCandidate + scanned := 0 + for _, spec := range []struct { + kind string + valid func(string) bool + }{ + {kind: KindCheckpoints, valid: isGCCheckpointName}, + {kind: KindManifests, valid: isGCManifestName}, + {kind: KindSegments, valid: isGCSegmentName}, + {kind: KindRaw, valid: isGCRawName}, + } { + dir := filepath.Join(originRoot.Name(), spec.kind) + kindRoot := kindRoots[spec.kind] + if kindRoot == nil { + continue + } + entries, err := fs.ReadDir(kindRoot.FS(), ".") + if err != nil { + if errors.Is(err, fs.ErrNotExist) { + continue + } + return nil, scanned, fmt.Errorf("reading %s: %w", dir, err) + } + for _, ent := range entries { + if err := ctx.Err(); err != nil { + return nil, scanned, err + } + if ent.IsDir() || isTempArtifactEntry(ent.Name()) || !spec.valid(ent.Name()) { + continue + } + info, err := ent.Info() + if err != nil { + return nil, scanned, fmt.Errorf("stat %s: %w", filepath.Join(dir, ent.Name()), err) + } + if !info.Mode().IsRegular() { + continue + } + scanned++ + if _, ok := live[gcRef{kind: spec.kind, name: ent.Name()}]; ok { + continue + } + candidates = append(candidates, gcCandidate{ + path: filepath.Join(dir, ent.Name()), + origin: origin, + kind: spec.kind, + name: ent.Name(), + size: info.Size(), + modTime: info.ModTime(), + }) + } + } + sort.Slice(candidates, func(i, j int) bool { + if candidates[i].origin != candidates[j].origin { + return candidates[i].origin < candidates[j].origin + } + if candidates[i].kind != candidates[j].kind { + return candidates[i].kind < candidates[j].kind + } + return candidates[i].name < candidates[j].name + }) + return candidates, scanned, nil +} + +func isGCCheckpointName(name string) bool { + _, err := checkpointSequence(name) + return err == nil +} + +func isGCManifestName(name string) bool { + _, _, err := normalizeHashName(name, manifestExtension) + return err == nil +} + +func isGCSegmentName(name string) bool { + _, _, err := normalizeHashName(name, segmentExtension) + return err == nil +} + +func isGCRawName(name string) bool { + return validateHashHex(name) == nil +} + +func logGC(opts GCOptions, format string, args ...any) { + if opts.Logf != nil { + opts.Logf(format, args...) + } +} diff --git a/internal/artifact/gc_test.go b/internal/artifact/gc_test.go new file mode 100644 index 000000000..f18b4ba38 --- /dev/null +++ b/internal/artifact/gc_test.go @@ -0,0 +1,630 @@ +package artifact + +import ( + "context" + "fmt" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.kenn.io/agentsview/internal/db" +) + +func TestGarbageCollectDeletesSupersededArtifactsAfterGrace(t *testing.T) { + root, oldPaths, livePaths := supersededArtifactFixture(t) + now := time.Unix(1_800_000_000, 0) + touchPaths(t, now.Add(-2*time.Hour), oldPaths...) + + res, err := GarbageCollect(context.Background(), GCOptions{ + Root: root, + Grace: time.Hour, + Now: now, + }) + require.NoError(t, err) + + assert.Equal(t, 1, res.Origins) + assert.Equal(t, 3, res.Candidates) + assert.Equal(t, 3, res.Eligible) + assert.Equal(t, 3, res.Deleted) + assert.Zero(t, res.KeptByGrace) + for _, path := range oldPaths { + assertNoFile(t, path) + } + for _, path := range livePaths { + assertFileExists(t, path) + } +} + +func TestGarbageCollectDryRunLogsAndKeepsArtifacts(t *testing.T) { + root, oldPaths, _ := supersededArtifactFixture(t) + now := time.Unix(1_800_000_000, 0) + touchPaths(t, now.Add(-2*time.Hour), oldPaths...) + + var logs []string + res, err := GarbageCollect(context.Background(), GCOptions{ + Root: root, + Grace: time.Hour, + Now: now, + DryRun: true, + Logf: func(format string, args ...any) { + logs = append(logs, fmt.Sprintf(format, args...)) + }, + }) + require.NoError(t, err) + + assert.Equal(t, 3, res.Candidates) + assert.Equal(t, 3, res.Eligible) + assert.Zero(t, res.Deleted) + require.NotEmpty(t, logs) + assert.Contains(t, logs[0], "would delete") + for _, path := range oldPaths { + assertFileExists(t, path) + } +} + +func TestGarbageCollectKeepsUnreferencedArtifactsWithinGrace(t *testing.T) { + root, oldPaths, _ := supersededArtifactFixture(t) + now := time.Unix(1_800_000_000, 0) + touchPaths(t, now.Add(-30*time.Minute), oldPaths...) + + res, err := GarbageCollect(context.Background(), GCOptions{ + Root: root, + Grace: time.Hour, + Now: now, + }) + require.NoError(t, err) + + assert.Equal(t, 3, res.Candidates) + assert.Zero(t, res.Eligible) + assert.Equal(t, 3, res.KeptByGrace) + assert.Zero(t, res.Deleted) + for _, path := range oldPaths { + assertFileExists(t, path) + } +} + +func TestGarbageCollectRejectsSymlinkedArtifactKindWithoutDeletingTarget(t *testing.T) { + fixture := newGCClosureFixture(t) + external := t.TempDir() + liveSegment := filepath.Base(fixture.latest.segment) + liveData, err := os.ReadFile(fixture.latest.segment) + require.NoError(t, err) + require.NoError(t, os.WriteFile(filepath.Join(external, liveSegment), liveData, 0o644)) + staleName := strings.Repeat("a", 64) + segmentExtension + stalePath := filepath.Join(external, staleName) + require.NoError(t, os.WriteFile(stalePath, []byte("outside"), 0o644)) + require.NoError(t, os.RemoveAll(filepath.Join(fixture.originRoot, KindSegments))) + require.NoError(t, os.Symlink(external, filepath.Join(fixture.originRoot, KindSegments))) + + _, err = GarbageCollect(context.Background(), GCOptions{ + Root: fixture.root, + Now: time.Now().Add(time.Hour), + }) + require.Error(t, err) + assert.FileExists(t, stalePath) +} + +func TestGarbageCollectRejectsSymlinkedOrigin(t *testing.T) { + external := newGCClosureFixture(t) + root := t.TempDir() + require.NoError(t, os.Symlink( + external.originRoot, filepath.Join(root, external.origin), + )) + + _, err := GarbageCollect(context.Background(), GCOptions{Root: root}) + require.Error(t, err) + for _, path := range external.oldPaths { + assert.FileExists(t, path) + } +} + +func TestGarbageCollectSkipsOriginsWithoutCheckpoints(t *testing.T) { + root := t.TempDir() + origin := "laptop-a1b2c3" + hash := "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + path := filepath.Join(root, origin, KindManifests, hash+manifestExtension) + require.NoError(t, os.MkdirAll(filepath.Dir(path), 0o755)) + require.NoError(t, os.WriteFile(path, []byte("orphan"), 0o644)) + + res, err := GarbageCollect(context.Background(), GCOptions{ + Root: root, + Grace: 0, + Now: time.Unix(1_800_000_000, 0), + }) + require.NoError(t, err) + + assert.Equal(t, 1, res.Origins) + assert.Equal(t, 1, res.SkippedOrigins) + assert.Zero(t, res.Candidates) + assert.Zero(t, res.Deleted) + assertFileExists(t, path) +} + +func TestGarbageCollectSkipsOriginWithCorruptLiveManifest(t *testing.T) { + ctx := context.Background() + root := t.TempDir() + origin := "laptop-a1b2c3" + database := testDB(t) + seedSession(t, database, "sess-1", "alpha") + + _, err := Export(ctx, database, root, origin) + require.NoError(t, err) + + manifests := globArtifacts(t, root, origin, "manifests", "*"+manifestExtension) + require.Len(t, manifests, 1) + require.NoError(t, os.Remove(manifests[0])) + require.NoError(t, writeCompressed(manifests[0], []byte("tampered"))) + segments := globArtifacts(t, root, origin, "segments", "*"+segmentExtension) + require.Len(t, segments, 1) + + res, err := GarbageCollect(ctx, GCOptions{Root: root, Grace: 0}) + require.NoError(t, err) + assert.Equal(t, 1, res.SkippedOrigins) + assert.Zero(t, res.Deleted) + assertFileExists(t, segments[0]) +} + +func TestGarbageCollectKeepsRawReferencedByLiveManifest(t *testing.T) { + root := t.TempDir() + origin := "laptop-a1b2c3" + originRoot := filepath.Join(root, origin) + for _, dir := range []string{KindCheckpoints, KindManifests, KindSegments, KindRaw} { + require.NoError(t, os.MkdirAll(filepath.Join(originRoot, dir), 0o755)) + } + + segmentData := []byte("{\"content\":\"hello\",\"ordinal\":0,\"role\":\"user\",\"v\":1}\n") + segmentHash := hashHex(segmentData) + require.NoError(t, writeCompressed( + filepath.Join(originRoot, KindSegments, segmentHash+segmentExtension), + segmentData, + )) + + liveRaw := []byte("live raw") + liveRawHash := hashHex(liveRaw) + require.NoError(t, os.WriteFile( + filepath.Join(originRoot, KindRaw, liveRawHash), + liveRaw, + 0o644, + )) + staleRaw := []byte("stale raw") + staleRawHash := hashHex(staleRaw) + staleRawPath := filepath.Join(originRoot, KindRaw, staleRawHash) + require.NoError(t, os.WriteFile(staleRawPath, staleRaw, 0o644)) + + gid := origin + "~sess-1" + m := manifest{ + Version: formatVersion, + Origin: origin, + NativeSessionID: "sess-1", + Session: manifestSession{ID: "sess-1", Machine: origin}, + Segments: []string{segmentHash}, + RawSource: &rawSourceRef{ + Hash: liveRawHash, + Size: int64(len(liveRaw)), + }, + DataVersion: 1, + Generation: 1, + } + manifestData, err := canonicalJSON(m) + require.NoError(t, err) + manifestHash := hashHex(manifestData) + require.NoError(t, writeCompressed( + filepath.Join(originRoot, KindManifests, manifestHash+manifestExtension), + manifestData, + )) + writeCheckpoint(t, originRoot, checkpoint{ + Version: formatVersion, + Origin: origin, + Sequence: 1, + Sessions: map[string]string{gid: manifestHash}, + }) + + res, err := GarbageCollect(context.Background(), GCOptions{ + Root: root, + Grace: 0, + Now: time.Unix(1_800_000_000, 0), + }) + require.NoError(t, err) + + assert.Equal(t, 1, res.Candidates) + assert.Equal(t, 1, res.Deleted) + assertNoFile(t, staleRawPath) + assertFileExists(t, filepath.Join(originRoot, KindRaw, liveRawHash)) +} + +func TestGarbageCollectSkipsOriginWhenLatestClosureIsIncompleteOrCorrupt(t *testing.T) { + tests := []struct { + name string + mutate func(*testing.T, *gcClosureFixture) + }{ + { + name: "missing segment", + mutate: func(t *testing.T, fixture *gcClosureFixture) { + t.Helper() + require.NoError(t, os.Remove(fixture.latest.segment)) + }, + }, + { + name: "corrupt segment", + mutate: func(t *testing.T, fixture *gcClosureFixture) { + t.Helper() + require.NoError(t, os.Remove(fixture.latest.segment)) + require.NoError(t, writeCompressed(fixture.latest.segment, []byte("corrupt\n"))) + }, + }, + { + name: "future segment", + mutate: func(t *testing.T, fixture *gcClosureFixture) { + t.Helper() + data, err := canonicalJSON(segmentMessage{ + Version: formatVersion + 1, + Ordinal: 0, + Role: "user", + Content: "future", + }) + require.NoError(t, err) + hash := hashHex(data) + require.NoError(t, writeCompressed( + filepath.Join(fixture.originRoot, KindSegments, hash+segmentExtension), + data, + )) + m := fixture.latest.manifest + m.Segments = []string{hash} + manifestData, err := canonicalJSON(m) + require.NoError(t, err) + manifestHash := hashHex(manifestData) + require.NoError(t, writeCompressed( + filepath.Join(fixture.originRoot, KindManifests, manifestHash+manifestExtension), + manifestData, + )) + writeCheckpoint(t, fixture.originRoot, checkpoint{ + Version: formatVersion, + Origin: fixture.origin, + Sequence: 2, + Sessions: map[string]string{fixture.gid: manifestHash}, + }) + }, + }, + { + name: "invalid manifest reference", + mutate: func(t *testing.T, fixture *gcClosureFixture) { + t.Helper() + m := fixture.latest.manifest + m.Segments = []string{"not-a-segment-hash"} + manifestData, err := canonicalJSON(m) + require.NoError(t, err) + manifestHash := hashHex(manifestData) + require.NoError(t, writeCompressed( + filepath.Join(fixture.originRoot, KindManifests, manifestHash+manifestExtension), + manifestData, + )) + writeCheckpoint(t, fixture.originRoot, checkpoint{ + Version: formatVersion, + Origin: fixture.origin, + Sequence: 2, + Sessions: map[string]string{fixture.gid: manifestHash}, + }) + }, + }, + { + name: "tool call amplification", + mutate: func(t *testing.T, fixture *gcClosureFixture) { + t.Helper() + data := nestedSegmentData(t, segmentMessage{ + ToolCalls: make([]segmentToolCall, 257), + }) + hash := hashHex(data) + require.NoError(t, writeCompressed( + filepath.Join(fixture.originRoot, KindSegments, hash+segmentExtension), + data, + )) + m := fixture.latest.manifest + m.Segments = []string{hash} + replaceLatestGCManifest(t, fixture, m) + }, + }, + { + name: "result event amplification", + mutate: func(t *testing.T, fixture *gcClosureFixture) { + t.Helper() + data := nestedSegmentData(t, segmentMessage{ + ToolCalls: []segmentToolCall{{ + ResultEvents: make([]segmentResultEvent, 1_025), + }}, + }) + hash := hashHex(data) + require.NoError(t, writeCompressed( + filepath.Join(fixture.originRoot, KindSegments, hash+segmentExtension), + data, + )) + m := fixture.latest.manifest + m.Segments = []string{hash} + replaceLatestGCManifest(t, fixture, m) + }, + }, + { + name: "session message amplification", + mutate: func(t *testing.T, fixture *gcClosureFixture) { + t.Helper() + m := fixture.latest.manifest + m.Segments = nil + ordinal := 0 + for segment := range 9 { + count := 4_096 + if segment == 8 { + count = 1 + } + data := syntheticSegmentRecordsFrom(t, ordinal, count) + ordinal += count + hash := hashHex(data) + require.NoError(t, writeCompressed( + filepath.Join(fixture.originRoot, KindSegments, hash+segmentExtension), + data, + )) + m.Segments = append(m.Segments, hash) + } + replaceLatestGCManifest(t, fixture, m) + }, + }, + { + name: "checkpoint sequence mismatch", + mutate: func(t *testing.T, fixture *gcClosureFixture) { + t.Helper() + manifestData, err := canonicalJSON(fixture.latest.manifest) + require.NoError(t, err) + data, err := canonicalJSON(checkpoint{ + Version: formatVersion, + Origin: fixture.origin, + Sequence: 1, + Sessions: map[string]string{ + fixture.gid: hashHex(manifestData), + }, + }) + require.NoError(t, err) + path := filepath.Join( + fixture.originRoot, KindCheckpoints, "cp-0000000002.json", + ) + require.NoError(t, os.Remove(path)) + require.NoError(t, writeFileAtomic(path, data, 0o644)) + }, + }, + { + name: "missing raw", + mutate: func(t *testing.T, fixture *gcClosureFixture) { + t.Helper() + require.NoError(t, os.Remove(fixture.latest.raw)) + }, + }, + { + name: "raw size mismatch", + mutate: func(t *testing.T, fixture *gcClosureFixture) { + t.Helper() + require.NoError(t, os.WriteFile(fixture.latest.raw, []byte("wrong size"), 0o644)) + }, + }, + { + name: "raw hash mismatch", + mutate: func(t *testing.T, fixture *gcClosureFixture) { + t.Helper() + require.NoError(t, os.WriteFile(fixture.latest.raw, []byte("bad raw bytes"), 0o644)) + }, + }, + { + name: "raw is not regular", + mutate: func(t *testing.T, fixture *gcClosureFixture) { + t.Helper() + require.NoError(t, os.Remove(fixture.latest.raw)) + require.NoError(t, os.Mkdir(fixture.latest.raw, 0o755)) + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + fixture := newGCClosureFixture(t) + now := time.Unix(1_800_000_000, 0) + touchPaths(t, now.Add(-2*time.Hour), fixture.oldPaths...) + tt.mutate(t, fixture) + + res, err := GarbageCollect(context.Background(), GCOptions{ + Root: fixture.root, Grace: 0, Now: now, + }) + require.NoError(t, err) + assert.Equal(t, 1, res.Origins) + assert.Equal(t, 1, res.SkippedOrigins) + assert.Zero(t, res.Candidates) + assert.Zero(t, res.Deleted) + for _, path := range fixture.oldPaths { + assertFileExists(t, path) + } + }) + } +} + +type gcClosureFixture struct { + root string + origin string + originRoot string + gid string + oldPaths []string + latest gcGeneration +} + +type gcGeneration struct { + manifest manifest + segment string + raw string +} + +func newGCClosureFixture(t *testing.T) *gcClosureFixture { + t.Helper() + fixture := &gcClosureFixture{ + root: t.TempDir(), + origin: "laptop-a1b2c3", + } + fixture.originRoot = filepath.Join(fixture.root, fixture.origin) + fixture.gid = fixture.origin + "~sess-1" + for _, dir := range []string{KindCheckpoints, KindManifests, KindSegments, KindRaw} { + require.NoError(t, os.MkdirAll(filepath.Join(fixture.originRoot, dir), 0o755)) + } + + old := writeGCGeneration(t, fixture, 1, "old message", []byte("old raw bytes")) + fixture.latest = writeGCGeneration(t, fixture, 2, "new message", []byte("new raw bytes")) + oldManifestData, err := canonicalJSON(old.manifest) + require.NoError(t, err) + fixture.oldPaths = []string{ + filepath.Join(fixture.originRoot, KindCheckpoints, "cp-0000000001.json"), + filepath.Join(fixture.originRoot, KindManifests, hashHex(oldManifestData)+manifestExtension), + old.segment, + old.raw, + } + return fixture +} + +func writeGCGeneration( + t *testing.T, + fixture *gcClosureFixture, + sequence int, + content string, + raw []byte, +) gcGeneration { + t.Helper() + segmentData, err := canonicalJSON(segmentMessage{ + Version: formatVersion, + Ordinal: 0, + Role: "user", + Content: content, + ContentLength: len(content), + }) + require.NoError(t, err) + segmentHash := hashHex(segmentData) + segmentPath := filepath.Join( + fixture.originRoot, KindSegments, segmentHash+segmentExtension, + ) + require.NoError(t, writeCompressed(segmentPath, segmentData)) + + rawHash := hashHex(raw) + rawPath := filepath.Join(fixture.originRoot, KindRaw, rawHash) + require.NoError(t, os.WriteFile(rawPath, raw, 0o644)) + m := manifest{ + Version: formatVersion, + Origin: fixture.origin, + NativeSessionID: "sess-1", + Session: manifestSession{ + ID: "sess-1", + Machine: fixture.origin, + }, + Segments: []string{segmentHash}, + RawSource: &rawSourceRef{ + Hash: rawHash, + Size: int64(len(raw)), + }, + DataVersion: sequence, + Generation: sequence, + } + manifestData, err := canonicalJSON(m) + require.NoError(t, err) + manifestHash := hashHex(manifestData) + require.NoError(t, writeCompressed( + filepath.Join(fixture.originRoot, KindManifests, manifestHash+manifestExtension), + manifestData, + )) + writeCheckpoint(t, fixture.originRoot, checkpoint{ + Version: formatVersion, + Origin: fixture.origin, + Sequence: sequence, + Sessions: map[string]string{fixture.gid: manifestHash}, + }) + return gcGeneration{manifest: m, segment: segmentPath, raw: rawPath} +} + +func replaceLatestGCManifest(t *testing.T, fixture *gcClosureFixture, m manifest) { + t.Helper() + manifestData, err := canonicalJSON(m) + require.NoError(t, err) + manifestHash := hashHex(manifestData) + require.NoError(t, writeCompressed( + filepath.Join(fixture.originRoot, KindManifests, manifestHash+manifestExtension), + manifestData, + )) + writeCheckpoint(t, fixture.originRoot, checkpoint{ + Version: formatVersion, + Origin: fixture.origin, + Sequence: 2, + Sessions: map[string]string{fixture.gid: manifestHash}, + }) +} + +func supersededArtifactFixture(t *testing.T) (string, []string, []string) { + t.Helper() + ctx := context.Background() + database := testDB(t) + root := t.TempDir() + origin := "laptop-a1b2c3" + gid := origin + "~sess-1" + originRoot := filepath.Join(root, origin) + + seedSession(t, database, "sess-1", "alpha") + _, err := Export(ctx, database, root, origin) + require.NoError(t, err) + firstCP, err := readLatestCheckpoint(originRoot) + require.NoError(t, err) + require.NotNil(t, firstCP) + firstManifestHash := firstCP.Sessions[gid] + require.NotEmpty(t, firstManifestHash) + firstManifest, err := readManifest(originRoot, firstManifestHash) + require.NoError(t, err) + require.Len(t, firstManifest.Segments, 1) + + require.NoError(t, database.ReplaceSessionMessages("sess-1", []db.Message{ + {SessionID: "sess-1", Ordinal: 0, Role: "user", Content: "hello", ContentLength: 5}, + {SessionID: "sess-1", Ordinal: 1, Role: "assistant", Content: "planet", ContentLength: 6}, + })) + _, err = Export(ctx, database, root, origin) + require.NoError(t, err) + latestCP, err := readLatestCheckpoint(originRoot) + require.NoError(t, err) + require.NotNil(t, latestCP) + latestManifestHash := latestCP.Sessions[gid] + require.NotEmpty(t, latestManifestHash) + require.NotEqual(t, firstManifestHash, latestManifestHash) + latestManifest, err := readManifest(originRoot, latestManifestHash) + require.NoError(t, err) + require.Len(t, latestManifest.Segments, 1) + require.NotEqual(t, firstManifest.Segments[0], latestManifest.Segments[0]) + + oldPaths := []string{ + filepath.Join(originRoot, KindCheckpoints, "cp-0000000001.json"), + filepath.Join(originRoot, KindManifests, firstManifestHash+manifestExtension), + filepath.Join(originRoot, KindSegments, firstManifest.Segments[0]+segmentExtension), + } + livePaths := []string{ + filepath.Join(originRoot, KindCheckpoints, "cp-0000000002.json"), + filepath.Join(originRoot, KindManifests, latestManifestHash+manifestExtension), + filepath.Join(originRoot, KindSegments, latestManifest.Segments[0]+segmentExtension), + } + return root, oldPaths, livePaths +} + +func touchPaths(t *testing.T, ts time.Time, paths ...string) { + t.Helper() + for _, path := range paths { + require.NoError(t, os.Chtimes(path, ts, ts)) + } +} + +func assertFileExists(t *testing.T, path string) { + t.Helper() + info, err := os.Stat(path) + require.NoError(t, err) + assert.True(t, info.Mode().IsRegular()) +} + +func assertNoFile(t *testing.T, path string) { + t.Helper() + _, err := os.Stat(path) + require.ErrorIs(t, err, os.ErrNotExist) +} diff --git a/internal/artifact/hlc.go b/internal/artifact/hlc.go new file mode 100644 index 000000000..8b2b4fc7b --- /dev/null +++ b/internal/artifact/hlc.go @@ -0,0 +1,261 @@ +package artifact + +import ( + "errors" + "fmt" + "strconv" + "strings" + "sync" + "time" +) + +const ( + metadataHLCStateKey = "artifact_metadata_hlc" + defaultMetadataHLCMaxDrift = 5 * time.Minute + // hlcWallLayout deliberately omits the ":" separators of RFC3339 so the + // rendered timestamp is safe to embed directly in artifact filenames. + // Windows forbids ":" in path components, and metadata event files are + // named after the HLC. Fixed-width fields keep the result lexicographically + // sortable and round-trippable through ParseHLCTimestamp. + hlcWallLayout = "2006-01-02T150405.000000000Z" + hlcLogicalWidth = 20 +) + +type hlcStateStore interface { + GetSyncState(key string) (string, error) + SetSyncState(key, value string) error +} + +// HLCTimestamp is a hybrid logical clock value for metadata events. +type HLCTimestamp struct { + WallTime time.Time + Logical uint64 +} + +// String formats the timestamp in a lexicographically sortable form. +func (t HLCTimestamp) String() string { + return fmt.Sprintf( + "%s-%0*d", + normalizeHLCWallTime(t.WallTime).Format(hlcWallLayout), + hlcLogicalWidth, + t.Logical, + ) +} + +// ParseHLCTimestamp parses a timestamp produced by HLCTimestamp.String. +func ParseHLCTimestamp(s string) (HLCTimestamp, error) { + idx := strings.LastIndex(s, "-") + if idx < 0 { + return HLCTimestamp{}, fmt.Errorf("invalid HLC timestamp %q: missing logical counter", s) + } + wallPart := s[:idx] + logicalPart := s[idx+1:] + if len(logicalPart) != hlcLogicalWidth || !isDecimal(logicalPart) { + return HLCTimestamp{}, fmt.Errorf("invalid HLC timestamp %q: logical counter must be %d digits", s, hlcLogicalWidth) + } + wall, err := time.Parse(hlcWallLayout, wallPart) + if err != nil { + return HLCTimestamp{}, fmt.Errorf("invalid HLC timestamp %q: %w", s, err) + } + logical, err := strconv.ParseUint(logicalPart, 10, 64) + if err != nil { + return HLCTimestamp{}, fmt.Errorf("invalid HLC timestamp %q: %w", s, err) + } + return HLCTimestamp{ + WallTime: normalizeHLCWallTime(wall), + Logical: logical, + }, nil +} + +// Compare returns -1, 0, or 1 when t is ordered before, equal to, or after other. +func (t HLCTimestamp) Compare(other HLCTimestamp) int { + wall := normalizeHLCWallTime(t.WallTime) + otherWall := normalizeHLCWallTime(other.WallTime) + switch { + case wall.Before(otherWall): + return -1 + case wall.After(otherWall): + return 1 + case t.Logical < other.Logical: + return -1 + case t.Logical > other.Logical: + return 1 + default: + return 0 + } +} + +// OrderingKey appends a deterministic tie-breaker, usually the artifact hash. +func (t HLCTimestamp) OrderingKey(tieBreaker string) string { + return t.String() + "-" + tieBreaker +} + +// HLCClockOptions configures a persisted metadata HLC clock. +type HLCClockOptions struct { + StateKey string + Now func() time.Time + MaxDrift time.Duration +} + +// HLCClock persists a monotonic hybrid logical clock in the sync-state store. +type HLCClock struct { + mu sync.Mutex + store hlcStateStore + stateKey string + now func() time.Time + maxDrift time.Duration +} + +// NewHLCClock returns a persisted metadata HLC clock. +func NewHLCClock(store hlcStateStore, opts HLCClockOptions) *HLCClock { + stateKey := opts.StateKey + if stateKey == "" { + stateKey = metadataHLCStateKey + } + now := opts.Now + if now == nil { + now = time.Now + } + maxDrift := opts.MaxDrift + if maxDrift <= 0 { + maxDrift = defaultMetadataHLCMaxDrift + } + return &HLCClock{ + store: store, + stateKey: stateKey, + now: now, + maxDrift: maxDrift, + } +} + +// Next returns and persists the next local metadata-event timestamp. +func (c *HLCClock) Next() (HLCTimestamp, error) { + c.mu.Lock() + defer c.mu.Unlock() + + last, ok, err := c.load() + if err != nil { + return HLCTimestamp{}, err + } + now := c.currentWallTime() + if ok { + if err := c.checkPersistedDrift(last, now); err != nil { + return HLCTimestamp{}, err + } + if !now.After(last.WallTime) { + next := HLCTimestamp{WallTime: last.WallTime, Logical: last.Logical + 1} + return next, c.persist(next) + } + } + next := HLCTimestamp{WallTime: now} + return next, c.persist(next) +} + +// Observe returns and persists a timestamp that is after the local and remote HLCs. +func (c *HLCClock) Observe(remote HLCTimestamp) (HLCTimestamp, error) { + c.mu.Lock() + defer c.mu.Unlock() + + last, ok, err := c.load() + if err != nil { + return HLCTimestamp{}, err + } + now := c.currentWallTime() + remote = HLCTimestamp{ + WallTime: normalizeHLCWallTime(remote.WallTime), + Logical: remote.Logical, + } + if ok { + if err := c.checkPersistedDrift(last, now); err != nil { + return HLCTimestamp{}, err + } + } + if remote.WallTime.After(now.Add(c.maxDrift)) { + return HLCTimestamp{}, fmt.Errorf( + "remote HLC wall time %s is more than %s ahead of local time %s", + remote.WallTime.Format(hlcWallLayout), + c.maxDrift, + now.Format(hlcWallLayout), + ) + } + + next := mergeHLC(HLCTimestamp{WallTime: now}, last, ok, remote) + return next, c.persist(next) +} + +func (c *HLCClock) load() (HLCTimestamp, bool, error) { + if c.store == nil { + return HLCTimestamp{}, false, errors.New("HLC state store is required") + } + raw, err := c.store.GetSyncState(c.stateKey) + if err != nil { + return HLCTimestamp{}, false, fmt.Errorf("reading HLC state: %w", err) + } + if strings.TrimSpace(raw) == "" { + return HLCTimestamp{}, false, nil + } + stamp, err := ParseHLCTimestamp(raw) + if err != nil { + return HLCTimestamp{}, false, fmt.Errorf("reading HLC state: %w", err) + } + return stamp, true, nil +} + +func (c *HLCClock) persist(stamp HLCTimestamp) error { + if err := c.store.SetSyncState(c.stateKey, stamp.String()); err != nil { + return fmt.Errorf("persisting HLC state: %w", err) + } + return nil +} + +func (c *HLCClock) currentWallTime() time.Time { + return normalizeHLCWallTime(c.now()) +} + +func (c *HLCClock) checkPersistedDrift(last HLCTimestamp, now time.Time) error { + if last.WallTime.After(now.Add(c.maxDrift)) { + return fmt.Errorf( + "persisted HLC wall time %s is more than %s ahead of local time %s", + last.WallTime.Format(hlcWallLayout), + c.maxDrift, + now.Format(hlcWallLayout), + ) + } + return nil +} + +func mergeHLC(now, last HLCTimestamp, hasLast bool, remote HLCTimestamp) HLCTimestamp { + maxWall := now.WallTime + if hasLast && last.WallTime.After(maxWall) { + maxWall = last.WallTime + } + if remote.WallTime.After(maxWall) { + maxWall = remote.WallTime + } + + next := HLCTimestamp{WallTime: maxWall} + if hasLast && last.WallTime.Equal(maxWall) { + next.Logical = last.Logical + } + if remote.WallTime.Equal(maxWall) && remote.Logical > next.Logical { + next.Logical = remote.Logical + } + if now.WallTime.Equal(maxWall) && next.Logical == 0 { + return next + } + next.Logical++ + return next +} + +func normalizeHLCWallTime(t time.Time) time.Time { + return t.UTC().Round(0) +} + +func isDecimal(s string) bool { + for _, r := range s { + if r < '0' || r > '9' { + return false + } + } + return s != "" +} diff --git a/internal/artifact/hlc_test.go b/internal/artifact/hlc_test.go new file mode 100644 index 000000000..067776953 --- /dev/null +++ b/internal/artifact/hlc_test.go @@ -0,0 +1,223 @@ +package artifact + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestHLCClockNextPersistsAcrossRestarts(t *testing.T) { + database := testDB(t) + now := fixedHLCTime() + + firstClock := NewHLCClock(database, HLCClockOptions{ + Now: func() time.Time { return now }, + MaxDrift: 5 * time.Minute, + }) + first, err := firstClock.Next() + require.NoError(t, err) + assert.Equal(t, now, first.WallTime) + assert.Equal(t, uint64(0), first.Logical) + + persisted, err := database.GetSyncState(metadataHLCStateKey) + require.NoError(t, err) + assert.Equal(t, "2026-06-14T010203.000000001Z-00000000000000000000", persisted) + + secondClock := NewHLCClock(database, HLCClockOptions{ + Now: func() time.Time { return now }, + MaxDrift: 5 * time.Minute, + }) + second, err := secondClock.Next() + require.NoError(t, err) + assert.Equal(t, now, second.WallTime) + assert.Equal(t, uint64(1), second.Logical) +} + +func TestHLCClockNextMonotonicCases(t *testing.T) { + base := fixedHLCTime() + + tests := []struct { + name string + last HLCTimestamp + now time.Time + want HLCTimestamp + }{ + { + name: "same wall time increments logical counter", + last: HLCTimestamp{WallTime: base, Logical: 7}, + now: base, + want: HLCTimestamp{WallTime: base, Logical: 8}, + }, + { + name: "physical time ahead resets logical counter", + last: HLCTimestamp{WallTime: base, Logical: 7}, + now: base.Add(time.Nanosecond), + want: HLCTimestamp{WallTime: base.Add(time.Nanosecond), Logical: 0}, + }, + { + name: "backward skew within bound increments logical counter", + last: HLCTimestamp{WallTime: base, Logical: 7}, + now: base.Add(-2 * time.Minute), + want: HLCTimestamp{WallTime: base, Logical: 8}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + database := testDB(t) + require.NoError(t, database.SetSyncState(metadataHLCStateKey, tt.last.String())) + clock := NewHLCClock(database, HLCClockOptions{ + Now: func() time.Time { return tt.now }, + MaxDrift: 5 * time.Minute, + }) + + got, err := clock.Next() + require.NoError(t, err) + + assert.Equal(t, 0, tt.want.Compare(got)) + persisted, err := database.GetSyncState(metadataHLCStateKey) + require.NoError(t, err) + assert.Equal(t, tt.want.String(), persisted) + }) + } +} + +func TestHLCClockNextRejectsBackwardSkewBeyondBound(t *testing.T) { + database := testDB(t) + base := fixedHLCTime() + last := HLCTimestamp{WallTime: base, Logical: 7} + require.NoError(t, database.SetSyncState(metadataHLCStateKey, last.String())) + clock := NewHLCClock(database, HLCClockOptions{ + Now: func() time.Time { return base.Add(-10 * time.Minute) }, + MaxDrift: 5 * time.Minute, + }) + + got, err := clock.Next() + require.Error(t, err) + + assert.Equal(t, HLCTimestamp{}, got) + assert.Contains(t, err.Error(), "persisted HLC wall time") + persisted, err := database.GetSyncState(metadataHLCStateKey) + require.NoError(t, err) + assert.Equal(t, last.String(), persisted) +} + +func TestHLCClockObserveCases(t *testing.T) { + base := fixedHLCTime() + + tests := []struct { + name string + last *HLCTimestamp + now time.Time + remote HLCTimestamp + want HLCTimestamp + }{ + { + name: "remote future within bound is absorbed", + now: base, + remote: HLCTimestamp{WallTime: base.Add(time.Minute), Logical: 3}, + want: HLCTimestamp{WallTime: base.Add(time.Minute), Logical: 4}, + }, + { + name: "same wall uses max logical counter", + last: &HLCTimestamp{WallTime: base, Logical: 5}, + now: base, + remote: HLCTimestamp{WallTime: base, Logical: 7}, + want: HLCTimestamp{WallTime: base, Logical: 8}, + }, + { + name: "local physical time wins when ahead", + last: &HLCTimestamp{WallTime: base, Logical: 7}, + now: base.Add(time.Minute), + remote: HLCTimestamp{WallTime: base.Add(30 * time.Second), Logical: 9}, + want: HLCTimestamp{WallTime: base.Add(time.Minute), Logical: 0}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + database := testDB(t) + if tt.last != nil { + require.NoError(t, database.SetSyncState(metadataHLCStateKey, tt.last.String())) + } + clock := NewHLCClock(database, HLCClockOptions{ + Now: func() time.Time { return tt.now }, + MaxDrift: 5 * time.Minute, + }) + + got, err := clock.Observe(tt.remote) + require.NoError(t, err) + + assert.Equal(t, 0, tt.want.Compare(got)) + persisted, err := database.GetSyncState(metadataHLCStateKey) + require.NoError(t, err) + assert.Equal(t, tt.want.String(), persisted) + }) + } +} + +func TestHLCClockObserveRejectsRemoteFutureBeyondBound(t *testing.T) { + database := testDB(t) + base := fixedHLCTime() + last := HLCTimestamp{WallTime: base, Logical: 7} + require.NoError(t, database.SetSyncState(metadataHLCStateKey, last.String())) + clock := NewHLCClock(database, HLCClockOptions{ + Now: func() time.Time { return base }, + MaxDrift: 5 * time.Minute, + }) + + got, err := clock.Observe(HLCTimestamp{ + WallTime: base.Add(10 * time.Minute), + Logical: 3, + }) + require.Error(t, err) + + assert.Equal(t, HLCTimestamp{}, got) + assert.Contains(t, err.Error(), "remote HLC wall time") + persisted, err := database.GetSyncState(metadataHLCStateKey) + require.NoError(t, err) + assert.Equal(t, last.String(), persisted) +} + +func TestHLCTimestampOrderingKeyAndParse(t *testing.T) { + base := fixedHLCTime() + stamp := HLCTimestamp{WallTime: base, Logical: 42} + + text := stamp.String() + parsed, err := ParseHLCTimestamp(text) + require.NoError(t, err) + + assert.Equal(t, "2026-06-14T010203.000000001Z-00000000000000000042", text) + assert.Equal(t, 0, stamp.Compare(parsed)) + assert.Equal(t, -1, stamp.Compare(HLCTimestamp{WallTime: base, Logical: 43})) + assert.Equal(t, -1, stamp.Compare(HLCTimestamp{WallTime: base.Add(time.Nanosecond), Logical: 0})) + assert.Equal(t, 1, stamp.Compare(HLCTimestamp{WallTime: base.Add(-time.Nanosecond), Logical: 99})) + assert.Less(t, stamp.OrderingKey("a111"), stamp.OrderingKey("b222")) + assert.Less(t, + HLCTimestamp{WallTime: base, Logical: 41}.OrderingKey("ffff"), + stamp.OrderingKey("0000"), + ) +} + +func TestParseHLCTimestampRejectsMalformedValues(t *testing.T) { + tests := []string{ + "", + "2026-06-14T010203Z-00000000000000000000", + "2026-06-14T010203.000000001Z-42", + "2026-06-14T010203.000000001Z-0000000000000000000x", + "2026-06-14T01:02:03.000000001Z-00000000000000000000", + } + + for _, tt := range tests { + t.Run(tt, func(t *testing.T) { + _, err := ParseHLCTimestamp(tt) + require.Error(t, err) + }) + } +} + +func fixedHLCTime() time.Time { + return time.Date(2026, 6, 14, 1, 2, 3, 1, time.UTC) +} diff --git a/internal/artifact/manifest_session.go b/internal/artifact/manifest_session.go new file mode 100644 index 000000000..395bd4da8 --- /dev/null +++ b/internal/artifact/manifest_session.go @@ -0,0 +1,227 @@ +package artifact + +import "go.kenn.io/agentsview/internal/db" + +// manifestSession is the manifest wire representation of a session row. It is +// deliberately a separate type from db.Session: manifest bytes feed content +// hashes, so this struct is part of the pinned artifact format and its +// serialized output can never change for an existing value. Keeping it apart +// from the internal DB type means adding a db.Session field cannot silently +// re-hash every exported manifest; extending THIS struct is an explicit wire +// format decision (see TestManifestSessionMatchesDBSessionWireFormat). +type manifestSession struct { + ID string `json:"id"` + Project string `json:"project"` + Machine string `json:"machine"` + Agent string `json:"agent"` + FirstMessage *string `json:"first_message"` + DisplayName *string `json:"display_name,omitempty"` + StartedAt *string `json:"started_at"` + EndedAt *string `json:"ended_at"` + MessageCount int `json:"message_count"` + UserMessageCount int `json:"user_message_count"` + ParentSessionID *string `json:"parent_session_id,omitempty"` + RelationshipType string `json:"relationship_type,omitempty"` + TotalOutputTokens int `json:"total_output_tokens"` + PeakContextTokens int `json:"peak_context_tokens"` + HasTotalOutputTokens bool `json:"has_total_output_tokens"` + HasPeakContextTokens bool `json:"has_peak_context_tokens"` + IsAutomated bool `json:"is_automated"` + + ToolFailureSignalCount int `json:"tool_failure_signal_count"` + ToolRetryCount int `json:"tool_retry_count"` + EditChurnCount int `json:"edit_churn_count"` + ConsecutiveFailureMax int `json:"consecutive_failure_max"` + Outcome string `json:"outcome"` + OutcomeConfidence string `json:"outcome_confidence"` + EndedWithRole string `json:"ended_with_role"` + FinalFailureStreak int `json:"final_failure_streak"` + SignalsPendingSince *string `json:"signals_pending_since,omitempty"` + CompactionCount int `json:"compaction_count"` + MidTaskCompactionCount int `json:"mid_task_compaction_count"` + ContextPressureMax *float64 `json:"context_pressure_max,omitempty"` + HealthScore *int `json:"health_score,omitempty"` + HealthGrade *string `json:"health_grade,omitempty"` + QualitySignals *manifestQualitySignals `json:"quality_signals,omitempty"` + SecretLeakCount int `json:"secret_leak_count"` + + Cwd string `json:"cwd,omitempty"` + GitBranch string `json:"git_branch,omitempty"` + SourceSessionID string `json:"source_session_id,omitempty"` + SourceVersion string `json:"source_version,omitempty"` + TranscriptFidelity string `json:"transcript_fidelity,omitempty"` + ParserMalformedLines int `json:"parser_malformed_lines,omitempty"` + IsTruncated bool `json:"is_truncated,omitempty"` + + DeletedAt *string `json:"deleted_at,omitempty"` + TerminationStatus *string `json:"termination_status,omitempty"` + FilePath *string `json:"file_path,omitempty"` + FileSize *int64 `json:"file_size,omitempty"` + FileMtime *int64 `json:"file_mtime,omitempty"` + FileInode *int64 `json:"file_inode,omitempty"` + FileDevice *int64 `json:"file_device,omitempty"` + FileHash *string `json:"file_hash,omitempty"` + LocalModifiedAt *string `json:"local_modified_at,omitempty"` + CreatedAt string `json:"created_at"` +} + +// manifestQualitySignals mirrors db.QualitySignals for the same reason +// manifestSession mirrors db.Session: it appears in hashed manifest bytes. +type manifestQualitySignals struct { + Version int `json:"version"` + ShortPromptCount int `json:"short_prompt_count"` + UnstructuredStart bool `json:"unstructured_start"` + MissingSuccessCriteriaCount int `json:"missing_success_criteria_count"` + MissingVerificationCount int `json:"missing_verification_count"` + DuplicatePromptCount int `json:"duplicate_prompt_count"` + NoCodeContextCount int `json:"no_code_context_count"` + RunawayToolLoopCount int `json:"runaway_tool_loop_count"` +} + +func manifestSessionFromDB(s db.Session) manifestSession { + return manifestSession{ + ID: s.ID, + Project: s.Project, + Machine: s.Machine, + Agent: s.Agent, + FirstMessage: s.FirstMessage, + DisplayName: s.DisplayName, + StartedAt: s.StartedAt, + EndedAt: s.EndedAt, + MessageCount: s.MessageCount, + UserMessageCount: s.UserMessageCount, + ParentSessionID: s.ParentSessionID, + RelationshipType: s.RelationshipType, + TotalOutputTokens: s.TotalOutputTokens, + PeakContextTokens: s.PeakContextTokens, + HasTotalOutputTokens: s.HasTotalOutputTokens, + HasPeakContextTokens: s.HasPeakContextTokens, + IsAutomated: s.IsAutomated, + + ToolFailureSignalCount: s.ToolFailureSignalCount, + ToolRetryCount: s.ToolRetryCount, + EditChurnCount: s.EditChurnCount, + ConsecutiveFailureMax: s.ConsecutiveFailureMax, + Outcome: s.Outcome, + OutcomeConfidence: s.OutcomeConfidence, + EndedWithRole: s.EndedWithRole, + FinalFailureStreak: s.FinalFailureStreak, + SignalsPendingSince: s.SignalsPendingSince, + CompactionCount: s.CompactionCount, + MidTaskCompactionCount: s.MidTaskCompactionCount, + ContextPressureMax: s.ContextPressureMax, + HealthScore: s.HealthScore, + HealthGrade: s.HealthGrade, + QualitySignals: manifestQualitySignalsFromDB(s.QualitySignals), + SecretLeakCount: s.SecretLeakCount, + + Cwd: s.Cwd, + GitBranch: s.GitBranch, + SourceSessionID: s.SourceSessionID, + SourceVersion: s.SourceVersion, + TranscriptFidelity: s.TranscriptFidelity, + ParserMalformedLines: s.ParserMalformedLines, + IsTruncated: s.IsTruncated, + + DeletedAt: s.DeletedAt, + TerminationStatus: s.TerminationStatus, + FilePath: s.FilePath, + FileSize: s.FileSize, + FileMtime: s.FileMtime, + FileInode: s.FileInode, + FileDevice: s.FileDevice, + FileHash: s.FileHash, + LocalModifiedAt: s.LocalModifiedAt, + CreatedAt: s.CreatedAt, + } +} + +func (m manifestSession) dbSession() db.Session { + return db.Session{ + ID: m.ID, + Project: m.Project, + Machine: m.Machine, + Agent: m.Agent, + FirstMessage: m.FirstMessage, + DisplayName: m.DisplayName, + StartedAt: m.StartedAt, + EndedAt: m.EndedAt, + MessageCount: m.MessageCount, + UserMessageCount: m.UserMessageCount, + ParentSessionID: m.ParentSessionID, + RelationshipType: m.RelationshipType, + TotalOutputTokens: m.TotalOutputTokens, + PeakContextTokens: m.PeakContextTokens, + HasTotalOutputTokens: m.HasTotalOutputTokens, + HasPeakContextTokens: m.HasPeakContextTokens, + IsAutomated: m.IsAutomated, + + ToolFailureSignalCount: m.ToolFailureSignalCount, + ToolRetryCount: m.ToolRetryCount, + EditChurnCount: m.EditChurnCount, + ConsecutiveFailureMax: m.ConsecutiveFailureMax, + Outcome: m.Outcome, + OutcomeConfidence: m.OutcomeConfidence, + EndedWithRole: m.EndedWithRole, + FinalFailureStreak: m.FinalFailureStreak, + SignalsPendingSince: m.SignalsPendingSince, + CompactionCount: m.CompactionCount, + MidTaskCompactionCount: m.MidTaskCompactionCount, + ContextPressureMax: m.ContextPressureMax, + HealthScore: m.HealthScore, + HealthGrade: m.HealthGrade, + QualitySignals: m.QualitySignals.dbQualitySignals(), + SecretLeakCount: m.SecretLeakCount, + + Cwd: m.Cwd, + GitBranch: m.GitBranch, + SourceSessionID: m.SourceSessionID, + SourceVersion: m.SourceVersion, + TranscriptFidelity: m.TranscriptFidelity, + ParserMalformedLines: m.ParserMalformedLines, + IsTruncated: m.IsTruncated, + + DeletedAt: m.DeletedAt, + TerminationStatus: m.TerminationStatus, + FilePath: m.FilePath, + FileSize: m.FileSize, + FileMtime: m.FileMtime, + FileInode: m.FileInode, + FileDevice: m.FileDevice, + FileHash: m.FileHash, + LocalModifiedAt: m.LocalModifiedAt, + CreatedAt: m.CreatedAt, + } +} + +func manifestQualitySignalsFromDB(qs *db.QualitySignals) *manifestQualitySignals { + if qs == nil { + return nil + } + return &manifestQualitySignals{ + Version: qs.Version, + ShortPromptCount: qs.ShortPromptCount, + UnstructuredStart: qs.UnstructuredStart, + MissingSuccessCriteriaCount: qs.MissingSuccessCriteriaCount, + MissingVerificationCount: qs.MissingVerificationCount, + DuplicatePromptCount: qs.DuplicatePromptCount, + NoCodeContextCount: qs.NoCodeContextCount, + RunawayToolLoopCount: qs.RunawayToolLoopCount, + } +} + +func (m *manifestQualitySignals) dbQualitySignals() *db.QualitySignals { + if m == nil { + return nil + } + return &db.QualitySignals{ + Version: m.Version, + ShortPromptCount: m.ShortPromptCount, + UnstructuredStart: m.UnstructuredStart, + MissingSuccessCriteriaCount: m.MissingSuccessCriteriaCount, + MissingVerificationCount: m.MissingVerificationCount, + DuplicatePromptCount: m.DuplicatePromptCount, + NoCodeContextCount: m.NoCodeContextCount, + RunawayToolLoopCount: m.RunawayToolLoopCount, + } +} diff --git a/internal/artifact/manifest_session_test.go b/internal/artifact/manifest_session_test.go new file mode 100644 index 000000000..7bec4a8f4 --- /dev/null +++ b/internal/artifact/manifest_session_test.go @@ -0,0 +1,96 @@ +package artifact + +import ( + "fmt" + "reflect" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "go.kenn.io/agentsview/internal/db" +) + +// TestManifestSessionMatchesDBSessionWireFormat pins the manifest wire DTO to +// the JSON-visible fields of db.Session with a fully populated value: every +// exported field gets a distinct non-zero value, so a missing, extra, or +// transposed DTO field changes the canonical JSON and fails the comparison. +// +// If this test fails after adding a field to db.Session, that is the wire +// format asking for a decision: adding the field to manifestSession changes +// every manifest hash fleet-wide (a re-export/re-import of all sessions), so +// extend the DTO only when the field is genuinely part of the session content +// contract; otherwise leave the DTO alone and update populateWireFixture's +// expectations here. +func TestManifestSessionMatchesDBSessionWireFormat(t *testing.T) { + var sess db.Session + populateWireFixture(t, reflect.ValueOf(&sess).Elem(), 1) + + want, err := canonicalJSON(sess) + require.NoError(t, err) + got, err := canonicalJSON(manifestSessionFromDB(sess)) + require.NoError(t, err) + assert.Equal(t, string(want), string(got), + "manifestSession must serialize byte-identically to db.Session") + + roundTrip, err := canonicalJSON(manifestSessionFromDB(sess).dbSession()) + require.NoError(t, err) + assert.Equal(t, string(want), string(roundTrip), + "converting to the wire DTO and back must preserve every wire-visible field") +} + +func TestManifestQualitySignalsMatchesDBWireFormat(t *testing.T) { + var qs db.QualitySignals + populateWireFixture(t, reflect.ValueOf(&qs).Elem(), 100) + + want, err := canonicalJSON(qs) + require.NoError(t, err) + dto := manifestQualitySignalsFromDB(&qs) + require.NotNil(t, dto) + got, err := canonicalJSON(*dto) + require.NoError(t, err) + assert.Equal(t, string(want), string(got)) + + roundTrip, err := canonicalJSON(*dto.dbQualitySignals()) + require.NoError(t, err) + assert.Equal(t, string(want), string(roundTrip)) + + assert.Nil(t, manifestQualitySignalsFromDB(nil)) +} + +// populateWireFixture fills every exported field of a struct with a distinct +// deterministic non-zero value so field transpositions are detectable. +func populateWireFixture(t *testing.T, v reflect.Value, seed int) { + t.Helper() + for i := 0; i < v.NumField(); i++ { + field := v.Field(i) + if !field.CanSet() { + continue + } + setWireFixtureValue(t, field, seed+i) + } +} + +func setWireFixtureValue(t *testing.T, field reflect.Value, n int) { + t.Helper() + switch field.Kind() { + case reflect.String: + field.SetString(fmt.Sprintf("value-%d", n)) + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + field.SetInt(int64(n)) + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: + field.SetUint(uint64(n)) + case reflect.Bool: + field.SetBool(true) + case reflect.Float32, reflect.Float64: + field.SetFloat(float64(n) + 0.5) + case reflect.Pointer: + elem := reflect.New(field.Type().Elem()) + setWireFixtureValue(t, elem.Elem(), n) + field.Set(elem) + case reflect.Struct: + populateWireFixture(t, field, n*10) + default: + t.Fatalf("populateWireFixture: unhandled field kind %s; teach the fixture about it", field.Kind()) + } +} diff --git a/internal/artifact/metadata.go b/internal/artifact/metadata.go new file mode 100644 index 000000000..ca159cd20 --- /dev/null +++ b/internal/artifact/metadata.go @@ -0,0 +1,481 @@ +package artifact + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "path/filepath" + "strings" + "sync" + "time" + + "go.kenn.io/agentsview/internal/db" + "go.kenn.io/agentsview/internal/parser" +) + +const metadataEventExtension = ".json" + +// errOriginNotAdopted reports that this machine has no artifact origin, i.e. +// it never opted into artifact sync via `sync --init`, a sync run, or a peer +// exchange. Recording paths treat it as "stay local"; explicit sync paths +// treat it as a hard error. +var errOriginNotAdopted = errors.New("artifact origin not adopted") + +type metadataSuppressionKey struct{} + +// Metadata operation names written into the metadata event ledger. +const ( + MetadataOpRename = "rename" + MetadataOpSoftDelete = "soft_delete" + MetadataOpRestore = "restore" + MetadataOpStar = "star" + MetadataOpUnstar = "unstar" + MetadataOpPin = "pin" + MetadataOpUnpin = "unpin" + MetadataOpPurge = "purge" +) + +// MetadataPin identifies a pinned message with stable source coordinates. +type MetadataPin struct { + SourceUUID string `json:"source_uuid,omitempty"` + Ordinal int `json:"ordinal"` + Note *string `json:"note,omitempty"` +} + +// MetadataEventInput describes a local user metadata mutation to append. +type MetadataEventInput struct { + SessionID string + Op string + Value json.RawMessage + Pin *MetadataPin +} + +// MetadataRecord describes a metadata artifact written to disk. +type MetadataRecord struct { + HLC string + Origin string + SessionGID string + Op string + Hash string + Path string +} + +// MetadataPublishedError reports that the artifact file was durably written, +// but local replay bookkeeping failed afterward. +type MetadataPublishedError struct { + Record MetadataRecord + Err error +} + +func (e *MetadataPublishedError) Error() string { + return fmt.Sprintf("metadata event published but local replay state was not recorded: %v", e.Err) +} + +func (e *MetadataPublishedError) Unwrap() error { + return e.Err +} + +// MetadataRecorderOptions configures metadata event artifact writes. +type MetadataRecorderOptions struct { + DataDir string + Origin string + Now func() time.Time + MaxDrift time.Duration +} + +// MetadataRecorder appends canonical metadata event artifacts. +type MetadataRecorder struct { + mu sync.Mutex + database *db.DB + root string + origin string + clock *HLCClock +} + +// NewMetadataRecorder creates a metadata event recorder for the local artifact store. +func NewMetadataRecorder(database *db.DB, opts MetadataRecorderOptions) *MetadataRecorder { + root := "" + if strings.TrimSpace(opts.DataDir) != "" { + root = filepath.Join(opts.DataDir, "artifacts") + } + return &MetadataRecorder{ + database: database, + root: root, + origin: strings.TrimSpace(opts.Origin), + clock: NewHLCClock(database, HLCClockOptions{ + Now: opts.Now, + MaxDrift: opts.MaxDrift, + }), + } +} + +// WithMetadataEventSuppression marks a context as replaying metadata events. +func WithMetadataEventSuppression(ctx context.Context) context.Context { + return context.WithValue(ctx, metadataSuppressionKey{}, true) +} + +// MetadataEventsSuppressed reports whether local metadata event writes are disabled. +func MetadataEventsSuppressed(ctx context.Context) bool { + suppressed, _ := ctx.Value(metadataSuppressionKey{}).(bool) + return suppressed +} + +// Append writes one metadata event artifact unless ctx is replay-suppressed. +func (r *MetadataRecorder) Append(ctx context.Context, input MetadataEventInput) (MetadataRecord, error) { + if MetadataEventsSuppressed(ctx) { + return MetadataRecord{}, nil + } + if r == nil { + return MetadataRecord{}, nil + } + if r.database == nil { + return MetadataRecord{}, errors.New("metadata recorder database is required") + } + if r.root == "" { + return MetadataRecord{}, errors.New("metadata recorder data dir is required") + } + if input.SessionID == "" { + return MetadataRecord{}, errors.New("metadata event session id is required") + } + if err := validateMetadataOp(input.Op); err != nil { + return MetadataRecord{}, err + } + origin, err := r.resolveOrigin() + if errors.Is(err, errOriginNotAdopted) { + // The machine never opted into artifact sync: curation stays local. + // If it joins a fleet later, the `sync --init` baseline snapshot + // publishes the accumulated local curation state. + return MetadataRecord{}, nil + } + if err != nil { + return MetadataRecord{}, err + } + stamp, err := r.clock.Next() + if err != nil { + return MetadataRecord{}, err + } + event := metadataEvent{ + Version: formatVersion, + HLC: stamp.String(), + Origin: origin, + SessionGID: MetadataSessionGID(origin, input.SessionID), + Op: input.Op, + Value: input.Value, + Pin: input.Pin, + } + data, err := canonicalJSON(event) + if err != nil { + return MetadataRecord{}, err + } + hash := hashHex(data) + orderKey := stamp.OrderingKey(hash) + projection, err := metadataProjection(metadataArtifact{ + orderKey: orderKey, + hash: hash, + hlc: event.HLC, + event: event, + }, origin) + if err != nil { + return MetadataRecord{}, err + } + path := filepath.Join(r.root, origin, "meta", orderKey+metadataEventExtension) + record := MetadataRecord{ + HLC: event.HLC, + Origin: origin, + SessionGID: event.SessionGID, + Op: event.Op, + Hash: hash, + Path: path, + } + if err := writeFileAtomic(path, data, 0o644); err != nil { + return MetadataRecord{}, fmt.Errorf("writing metadata event: %w", err) + } + // Record the local event in the LWW replay register only after the artifact + // exists. Otherwise a failed publish can leave hidden local state that wins + // future LWW comparisons for an event no peer can import. + if _, err := r.database.RecordLocalMetadataProjection(ctx, projection); err != nil { + return record, &MetadataPublishedError{ + Record: record, + Err: fmt.Errorf("recording local metadata replay state: %w", err), + } + } + return record, nil +} + +// RepairLocalSessionMetadata rebuilds local replay bookkeeping for already +// published local metadata artifacts without re-applying their visible +// mutations. +func (r *MetadataRecorder) RepairLocalSessionMetadata( + ctx context.Context, + sessionID string, + ops ...string, +) (int, error) { + if r == nil { + return 0, nil + } + if r.database == nil { + return 0, errors.New("metadata recorder database is required") + } + if r.root == "" { + return 0, errors.New("metadata recorder data dir is required") + } + if sessionID == "" { + return 0, errors.New("metadata event session id is required") + } + opSet := make(map[string]struct{}, len(ops)) + for _, op := range ops { + if err := validateMetadataOp(op); err != nil { + return 0, err + } + opSet[op] = struct{}{} + } + origin, err := r.resolveOrigin() + if errors.Is(err, errOriginNotAdopted) { + // No origin means no published local artifacts to repair against. + return 0, nil + } + if err != nil { + return 0, err + } + events, err := readMetadataArtifacts(filepath.Join(r.root, origin), origin, nil) + if err != nil { + return 0, err + } + sessionGID := MetadataSessionGID(origin, sessionID) + repaired := 0 + for _, art := range events { + if err := ctx.Err(); err != nil { + return repaired, err + } + if art.event.SessionGID != sessionGID { + continue + } + if len(opSet) > 0 { + if _, ok := opSet[art.event.Op]; !ok { + continue + } + } + if err := validateMetadataArtifactEvent(art, origin); err != nil { + if errors.Is(err, errFutureArtifactVersion) { + continue + } + return repaired, err + } + if err := validateMetadataOp(art.event.Op); err != nil { + return repaired, err + } + projection, err := metadataProjection(art, origin) + if err != nil { + return repaired, err + } + if _, err := r.database.RecordLocalMetadataProjection(ctx, projection); err != nil { + return repaired, fmt.Errorf("repairing local metadata replay state: %w", err) + } + repaired++ + } + return repaired, nil +} + +// AppendBaseline writes metadata events for existing local curation that +// predates artifact metadata recording. +func (r *MetadataRecorder) AppendBaseline(ctx context.Context) (int, error) { + if r == nil { + return 0, nil + } + if r.database == nil { + return 0, errors.New("metadata recorder database is required") + } + snap, err := r.database.MetadataBaselineSnapshot(ctx) + if err != nil { + return 0, err + } + return r.AppendBaselineSnapshot(ctx, snap) +} + +// AppendBaselineSnapshot writes metadata events from a previously captured +// curation snapshot. Callers that import peer artifacts before initialization +// should capture the snapshot before that import so newly imported rows cannot +// be re-published as local baseline metadata. +func (r *MetadataRecorder) AppendBaselineSnapshot( + ctx context.Context, + snap db.MetadataBaselineSnapshot, +) (int, error) { + if r == nil { + return 0, nil + } + if r.database == nil { + return 0, errors.New("metadata recorder database is required") + } + origin, err := r.resolveOrigin() + if err != nil { + return 0, err + } + written := 0 + for _, rename := range snap.Renames { + covered, err := r.baselineFieldCovered(ctx, origin, rename.SessionID, "display_name") + if err != nil { + return written, err + } + if covered { + continue + } + value, err := metadataRenameValue(rename.DisplayName) + if err != nil { + return written, err + } + if _, err := r.Append(ctx, MetadataEventInput{ + SessionID: rename.SessionID, + Op: MetadataOpRename, + Value: value, + }); err != nil { + return written, fmt.Errorf("writing baseline rename metadata: %w", err) + } + written++ + } + for _, sessionID := range snap.StarredSessionIDs { + covered, err := r.baselineFieldCovered(ctx, origin, sessionID, "starred") + if err != nil { + return written, err + } + if covered { + continue + } + if _, err := r.Append(ctx, MetadataEventInput{ + SessionID: sessionID, + Op: MetadataOpStar, + }); err != nil { + return written, fmt.Errorf("writing baseline star metadata: %w", err) + } + written++ + } + for _, sessionID := range snap.SoftDeletedIDs { + covered, err := r.baselineFieldCovered(ctx, origin, sessionID, "deleted_at") + if err != nil { + return written, err + } + if covered { + continue + } + if _, err := r.Append(ctx, MetadataEventInput{ + SessionID: sessionID, + Op: MetadataOpSoftDelete, + }); err != nil { + return written, fmt.Errorf("writing baseline soft-delete metadata: %w", err) + } + written++ + } + for _, pin := range snap.Pins { + metadataPin := MetadataPin{ + SourceUUID: pin.SourceUUID, + Ordinal: pin.Ordinal, + Note: pin.Note, + } + covered, err := r.baselineFieldCovered( + ctx, origin, pin.SessionID, "pin:"+metadataPinAnchor(metadataPin), + ) + if err != nil { + return written, err + } + if covered { + continue + } + if _, err := r.Append(ctx, MetadataEventInput{ + SessionID: pin.SessionID, + Op: MetadataOpPin, + Pin: &metadataPin, + }); err != nil { + return written, fmt.Errorf("writing baseline pin metadata: %w", err) + } + written++ + } + return written, nil +} + +func (r *MetadataRecorder) baselineFieldCovered( + ctx context.Context, + origin string, + sessionID string, + field string, +) (bool, error) { + _, ok, err := r.database.MetadataReplayStateOp( + ctx, MetadataSessionGID(origin, sessionID), field, + ) + if err != nil { + return false, fmt.Errorf("checking baseline metadata field %s: %w", field, err) + } + return ok, nil +} + +// Import reads every foreign origin under root and imports referenced sessions +// plus metadata events, advancing this recorder's HLC clock past observed +// remote HLCs so later local edits stay causally ahead of imported peers. +func (r *MetadataRecorder) Import(ctx context.Context, root string) (ImportResult, error) { + if r == nil || r.database == nil { + return ImportResult{}, errors.New("metadata recorder database is required") + } + origin, err := r.resolveOrigin() + if err != nil { + return ImportResult{}, err + } + return importDetailed(ctx, r.database, r.clock, root, origin) +} + +// MetadataSessionGID returns the global metadata target ID for a session. +func MetadataSessionGID(origin, sessionID string) string { + if host, _ := parser.StripHostPrefix(sessionID); host != "" { + return sessionID + } + return origin + "~" + sessionID +} + +// resolveOrigin returns the recorder's origin without ever creating one: the +// explicit option wins, then the origin persisted in DB sync state. A machine +// with no origin anywhere has not opted into artifact sync and gets +// errOriginNotAdopted. The empty result is not cached, so a recorder built +// before opt-in starts resolving the origin as soon as it is adopted. +func (r *MetadataRecorder) resolveOrigin() (string, error) { + r.mu.Lock() + defer r.mu.Unlock() + if r.origin != "" { + if err := validateOriginID(r.origin); err != nil { + return "", fmt.Errorf("metadata recorder origin: %w", err) + } + return r.origin, nil + } + origin, err := StoredOrigin(r.database) + if err != nil { + return "", err + } + if origin == "" { + return "", errOriginNotAdopted + } + r.origin = origin + return origin, nil +} + +func metadataRenameValue(displayName *string) (json.RawMessage, error) { + data, err := json.Marshal(struct { + DisplayName *string `json:"display_name"` + }{DisplayName: displayName}) + if err != nil { + return nil, err + } + return json.RawMessage(data), nil +} + +func validateMetadataOp(op string) error { + switch op { + case MetadataOpRename, + MetadataOpSoftDelete, + MetadataOpRestore, + MetadataOpStar, + MetadataOpUnstar, + MetadataOpPin, + MetadataOpUnpin, + MetadataOpPurge: + return nil + default: + return fmt.Errorf("unsupported metadata event op %q", op) + } +} diff --git a/internal/artifact/metadata_test.go b/internal/artifact/metadata_test.go new file mode 100644 index 000000000..c720ca018 --- /dev/null +++ b/internal/artifact/metadata_test.go @@ -0,0 +1,152 @@ +package artifact + +import ( + "context" + "encoding/json" + "os" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestMetadataRecorderAppendWritesCanonicalEvent(t *testing.T) { + database := testDB(t) + dataDir := t.TempDir() + now := fixedHLCTime() + recorder := NewMetadataRecorder(database, MetadataRecorderOptions{ + DataDir: dataDir, + Origin: "laptop-a1b2c3", + Now: func() time.Time { return now }, + }) + + value := json.RawMessage(`{"display_name":"Renamed session"}`) + record, err := recorder.Append(context.Background(), MetadataEventInput{ + SessionID: "sess-1", + Op: MetadataOpRename, + Value: value, + }) + require.NoError(t, err) + + assert.Equal(t, "2026-06-14T010203.000000001Z-00000000000000000000", record.HLC) + assert.Equal(t, "laptop-a1b2c3", record.Origin) + assert.Equal(t, "laptop-a1b2c3~sess-1", record.SessionGID) + assert.Equal(t, MetadataOpRename, record.Op) + + data, err := os.ReadFile(record.Path) + require.NoError(t, err) + assert.Equal(t, + "{\"hlc\":\"2026-06-14T010203.000000001Z-00000000000000000000\",\"op\":\"rename\",\"origin\":\"laptop-a1b2c3\",\"session_gid\":\"laptop-a1b2c3~sess-1\",\"v\":1,\"value\":{\"display_name\":\"Renamed session\"}}\n", + string(data), + ) + assert.Equal(t, hashHex(data), record.Hash) + assert.Equal(t, + filepath.Join(dataDir, "artifacts", "laptop-a1b2c3", "meta", record.HLC+"-"+record.Hash+".json"), + record.Path, + ) + // The metadata event filename must be safe on every supported OS, + // including Windows, which forbids these characters in path components. + assert.NotContains(t, filepath.Base(record.Path), ":", + "metadata filename must not contain ':' (invalid on Windows)") + for _, c := range `<>:"/\|?*` { + assert.NotContainsf(t, record.HLC, string(c), + "HLC %q must not contain %q (invalid in Windows filenames)", record.HLC, string(c)) + } +} + +func TestImportObservesRemoteHLCForLaterLocalEdits(t *testing.T) { + ctx := context.Background() + root := t.TempDir() + dataDir := t.TempDir() + localOrigin := "desktop-d4e5f6" + peerOrigin := "laptop-a1b2c3" + database := testDB(t) + seedSession(t, database, "sess-1", "alpha") + peerGID := localOrigin + "~sess-1" + + recorderNow := fixedHLCTime() + // A peer event whose wall time is ahead of the recorder's local clock but + // within the drift bound. + remoteStamp := HLCTimestamp{WallTime: recorderNow.Add(2 * time.Minute), Logical: 5} + writeMetadataArtifact(t, filepath.Join(root, peerOrigin), + replayRenameEvent(t, peerOrigin, peerGID, remoteStamp.String(), "Peer name")) + + recorder := NewMetadataRecorder(database, MetadataRecorderOptions{ + DataDir: dataDir, + Origin: localOrigin, + Now: func() time.Time { return recorderNow }, + }) + + imported, err := recorder.Import(ctx, root) + require.NoError(t, err) + assert.Equal(t, 1, imported.Metadata) + + // The next local edit must receive an HLC strictly after the observed + // remote HLC even though the local wall clock is behind it. + rec, err := recorder.Append(ctx, MetadataEventInput{ + SessionID: "sess-1", + Op: MetadataOpStar, + }) + require.NoError(t, err) + localStamp, err := ParseHLCTimestamp(rec.HLC) + require.NoError(t, err) + assert.Equal(t, 1, localStamp.Compare(remoteStamp), + "local HLC %s must be after observed remote HLC %s", rec.HLC, remoteStamp.String()) +} + +func TestMetadataRecorderWithoutOriginIsNoOp(t *testing.T) { + database := testDB(t) + dataDir := t.TempDir() + recorder := NewMetadataRecorder(database, MetadataRecorderOptions{DataDir: dataDir}) + + record, err := recorder.Append(context.Background(), MetadataEventInput{ + SessionID: "sess-1", + Op: MetadataOpStar, + }) + require.NoError(t, err) + assert.Equal(t, MetadataRecord{}, record) + + origin, err := StoredOrigin(database) + require.NoError(t, err) + assert.Empty(t, origin, + "recorder must not mint an origin for a machine that never opted into artifact sync") + _, err = os.Stat(filepath.Join(dataDir, "artifacts")) + assert.True(t, os.IsNotExist(err), + "no artifact store should be created without an origin") + + repaired, err := recorder.RepairLocalSessionMetadata(context.Background(), "sess-1") + require.NoError(t, err) + assert.Zero(t, repaired) +} + +func TestMetadataRecorderRecordsAfterOriginAdopted(t *testing.T) { + database := testDB(t) + dataDir := t.TempDir() + recorder := NewMetadataRecorder(database, MetadataRecorderOptions{DataDir: dataDir}) + + record, err := recorder.Append(context.Background(), MetadataEventInput{ + SessionID: "sess-1", + Op: MetadataOpStar, + }) + require.NoError(t, err) + assert.Empty(t, record.Origin) + + require.NoError(t, AdoptOrigin(database, "desk-a1b2c3")) + + record, err = recorder.Append(context.Background(), MetadataEventInput{ + SessionID: "sess-1", + Op: MetadataOpStar, + }) + require.NoError(t, err) + assert.Equal(t, "desk-a1b2c3", record.Origin) + _, err = os.Stat(record.Path) + require.NoError(t, err, + "recorder must start writing events once the origin is adopted, without reconstruction") +} + +func TestMetadataSessionGID(t *testing.T) { + assert.Equal(t, "desk-a1b2c3~sess-1", MetadataSessionGID("desk-a1b2c3", "sess-1")) + assert.Equal(t, "laptop-d4e5f6~sess-1", MetadataSessionGID("desk-a1b2c3", "laptop-d4e5f6~sess-1")) +} diff --git a/internal/artifact/nested_limits_test.go b/internal/artifact/nested_limits_test.go new file mode 100644 index 000000000..a140b1de5 --- /dev/null +++ b/internal/artifact/nested_limits_test.go @@ -0,0 +1,475 @@ +package artifact + +import ( + "bytes" + "context" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.kenn.io/agentsview/internal/db" +) + +func TestDecodeSegmentRejectsAggregateNestedLimitsWithSmallLimits(t *testing.T) { + tests := []struct { + name string + records []segmentMessage + configure func(*artifactLimits) + wantError string + }{ + { + name: "tool calls per segment", + records: []segmentMessage{ + {ToolCalls: []segmentToolCall{{}}}, + {ToolCalls: []segmentToolCall{{}}}, + }, + configure: func(limits *artifactLimits) { + limits.segmentToolCalls = 1 + }, + wantError: "segment tool call limit", + }, + { + name: "result events per segment", + records: []segmentMessage{ + {ToolCalls: []segmentToolCall{{ResultEvents: []segmentResultEvent{{}}}}}, + {ToolCalls: []segmentToolCall{{ResultEvents: []segmentResultEvent{{}}}}}, + }, + configure: func(limits *artifactLimits) { + limits.segmentResultEvents = 1 + }, + wantError: "segment result event limit", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + limits := productionArtifactLimits() + tt.configure(&limits) + data := nestedSegmentData(t, tt.records...) + + _, err := decodeSegmentWithLimits(data, limits) + require.Error(t, err) + assert.Contains(t, err.Error(), tt.wantError) + }) + } +} + +func TestReadManifestMessagesRejectsSessionNestedLimitsWithSmallLimits(t *testing.T) { + tests := []struct { + name string + record segmentMessage + configure func(*artifactLimits) + wantError string + }{ + { + name: "tool calls per session", + record: segmentMessage{ToolCalls: []segmentToolCall{{}}}, + configure: func(limits *artifactLimits) { + limits.sessionToolCalls = 1 + }, + wantError: "session tool call limit", + }, + { + name: "result events per session", + record: segmentMessage{ToolCalls: []segmentToolCall{{ + ResultEvents: []segmentResultEvent{{}}, + }}}, + configure: func(limits *artifactLimits) { + limits.sessionResultEvents = 1 + }, + wantError: "session result event limit", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + originRoot := t.TempDir() + m := manifest{} + for ordinal := range 2 { + record := tt.record + record.Ordinal = ordinal + record.Content = string(rune('a' + ordinal)) + data := nestedSegmentData(t, record) + hash := hashHex(data) + require.NoError(t, writeCompressed( + filepath.Join(originRoot, KindSegments, hash+segmentExtension), data, + )) + m.Segments = append(m.Segments, hash) + } + limits := productionArtifactLimits() + tt.configure(&limits) + + _, err := readManifestMessagesWithLimits(originRoot, m, limits) + require.Error(t, err) + assert.ErrorIs(t, err, errCorruptArtifact) + assert.Contains(t, err.Error(), tt.wantError) + }) + } +} + +func TestPeerArtifactDefersFutureNestedSchema(t *testing.T) { + root := t.TempDir() + origin := "peer-a1b2c3" + data := []byte(`{"v":2,"ordinal":{"future_shape":true},"tool_calls":{"future_shape":true}}` + "\n") + hash := hashHex(data) + compressed := compressPeerTestData(t, data) + + res, err := WriteArtifact(root, origin, KindSegments, hash, compressed) + require.NoError(t, err) + assert.Equal(t, hash+segmentExtension, res.Name) + stored, err := ReadArtifact(root, origin, KindSegments, hash) + require.NoError(t, err) + assert.Equal(t, compressed, stored.Data) +} + +func TestDecodeSegmentAcceptsCanonicalTrailingNewlineAndEmptySession(t *testing.T) { + record := nestedSegmentData(t, segmentMessage{}) + tests := []struct { + name string + data []byte + want int + }{ + {name: "canonical trailing newline", data: record, want: 1}, + {name: "zero byte empty segment", data: nil, want: 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + msgs, err := decodeSegment(tt.data) + require.NoError(t, err) + assert.Len(t, msgs, tt.want) + }) + } +} + +func TestImportQuarantinesNestedAmplificationWithoutAdvancingState(t *testing.T) { + tests := []struct { + name string + record segmentMessage + }{ + { + name: "too many tool calls", + record: segmentMessage{ + ToolCalls: make([]segmentToolCall, 257), + }, + }, + { + name: "too many result events", + record: segmentMessage{ + ToolCalls: []segmentToolCall{{ + ResultEvents: make([]segmentResultEvent, 1_025), + }}, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctx := context.Background() + root := t.TempDir() + origin := "laptop-a1b2c3" + localOrigin := "desktop-d4e5f6" + importDB := testDB(t) + gid, segmentPath := writeNestedImportFixture( + t, root, origin, tt.record, + ) + + res, err := ImportDetailed(ctx, importDB, root, localOrigin) + require.NoError(t, err) + assert.False(t, res.Changed()) + state, err := importDB.GetSyncState(importStateKey(origin, gid)) + require.NoError(t, err) + assert.Empty(t, state) + got, err := importDB.GetSessionFull(ctx, gid) + require.NoError(t, err) + assert.Nil(t, got) + assert.NoFileExists(t, segmentPath) + assert.FileExists(t, segmentPath+quarantineSuffix) + }) + } +} + +func TestExportRejectsNestedAmplificationBeforePublication(t *testing.T) { + tests := []struct { + name string + message db.Message + wantError string + }{ + { + name: "too many tool calls in one message", + message: db.Message{ + ToolCalls: make([]db.ToolCall, 257), + }, + wantError: "tool call limit exceeded for message ordinal 0", + }, + { + name: "too many result events in one tool call", + message: db.Message{ + ToolCalls: []db.ToolCall{{ + ResultEvents: make([]db.ToolResultEvent, 1_025), + }}, + }, + wantError: "result event limit exceeded for tool call 0 in message ordinal 0", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctx := context.Background() + database := testDB(t) + root := t.TempDir() + origin := "laptop-a1b2c3" + seedSession(t, database, "sess-1", "alpha") + message := tt.message + message.SessionID = "sess-1" + message.Ordinal = 0 + message.Role = "assistant" + require.NoError(t, database.ReplaceSessionMessages("sess-1", []db.Message{message})) + + _, err := Export(ctx, database, root, origin) + require.Error(t, err) + assert.Contains(t, err.Error(), tt.wantError) + assertNoPublishedArtifactFiles(t, root, origin) + state, err := database.GetSyncState(exportStateKey(origin, "sess-1")) + require.NoError(t, err) + assert.Empty(t, state) + }) + } +} + +func TestExportChunksOnAggregateNestedLimitsWithSmallLimits(t *testing.T) { + tests := []struct { + name string + resultEvents []db.ToolResultEvent + configure func(*artifactLimits) + }{ + { + name: "tool calls per segment", + configure: func(limits *artifactLimits) { + limits.segmentToolCalls = 2 + }, + }, + { + name: "result events per segment", + resultEvents: []db.ToolResultEvent{{EventIndex: 0}}, + configure: func(limits *artifactLimits) { + limits.segmentResultEvents = 2 + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctx := context.Background() + database := testDB(t) + root := t.TempDir() + origin := "laptop-a1b2c3" + originRoot := filepath.Join(root, origin) + for _, kind := range []string{KindManifests, KindSegments} { + require.NoError(t, os.MkdirAll(filepath.Join(originRoot, kind), 0o755)) + } + seedSession(t, database, "sess-1", "alpha") + msgs := make([]db.Message, 3) + for ordinal := range msgs { + msgs[ordinal] = db.Message{ + SessionID: "sess-1", + Ordinal: ordinal, + Role: "assistant", + ToolCalls: []db.ToolCall{{ + ResultEvents: tt.resultEvents, + }}, + } + } + require.NoError(t, database.ReplaceSessionMessages("sess-1", msgs)) + limits := productionArtifactLimits() + tt.configure(&limits) + + manifestHash, changed, err := exportSessionWithLimits( + ctx, database, originRoot, origin, "sess-1", "", limits, + ) + require.NoError(t, err) + assert.True(t, changed) + m, err := readManifest(originRoot, manifestHash) + require.NoError(t, err) + require.Len(t, m.Segments, 2) + got, err := readManifestMessages(originRoot, m) + require.NoError(t, err) + require.Len(t, got, 3) + for ordinal := range got { + assert.Equal(t, ordinal, got[ordinal].Ordinal) + require.Len(t, got[ordinal].ToolCalls, 1) + assert.Len(t, got[ordinal].ToolCalls[0].ResultEvents, len(tt.resultEvents)) + } + }) + } +} + +func TestExportRejectsMessageThatCannotFitNestedSegmentLimits(t *testing.T) { + tests := []struct { + name string + message db.Message + configure func(*artifactLimits) + wantError string + }{ + { + name: "tool calls cannot split across segments", + message: db.Message{ + ToolCalls: []db.ToolCall{{}, {}}, + }, + configure: func(limits *artifactLimits) { + limits.segmentToolCalls = 1 + }, + wantError: "2 tool calls", + }, + { + name: "one tool result history cannot split across segments", + message: db.Message{ + ToolCalls: []db.ToolCall{{ + ResultEvents: []db.ToolResultEvent{{EventIndex: 0}, {EventIndex: 1}}, + }}, + }, + configure: func(limits *artifactLimits) { + limits.segmentResultEvents = 1 + }, + wantError: "2 result events", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctx := context.Background() + database := testDB(t) + root := t.TempDir() + origin := "laptop-a1b2c3" + seedSession(t, database, "sess-1", "alpha") + message := tt.message + message.SessionID = "sess-1" + message.Ordinal = 0 + message.Role = "assistant" + require.NoError(t, database.ReplaceSessionMessages("sess-1", []db.Message{message})) + limits := productionArtifactLimits() + tt.configure(&limits) + + _, _, err := exportSessionWithLimits( + ctx, database, filepath.Join(root, origin), origin, "sess-1", "", limits, + ) + require.Error(t, err) + assert.Contains(t, err.Error(), "cannot fit in one segment") + assert.Contains(t, err.Error(), tt.wantError) + assert.Empty(t, globArtifacts( + t, root, origin, KindSegments, "*"+segmentExtension, + )) + assert.Empty(t, globArtifacts( + t, root, origin, KindManifests, "*"+manifestExtension, + )) + }) + } +} + +func TestExportRejectsSessionNestedLimitsBeforeWritingWithSmallLimits(t *testing.T) { + tests := []struct { + name string + configure func(*artifactLimits) + wantError string + }{ + { + name: "tool calls per session", + configure: func(limits *artifactLimits) { + limits.sessionToolCalls = 1 + }, + wantError: "session tool call limit", + }, + { + name: "result events per session", + configure: func(limits *artifactLimits) { + limits.sessionResultEvents = 1 + }, + wantError: "session result event limit", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctx := context.Background() + database := testDB(t) + root := t.TempDir() + origin := "laptop-a1b2c3" + originRoot := filepath.Join(root, origin) + seedSession(t, database, "sess-1", "alpha") + msgs := make([]db.Message, 2) + for ordinal := range msgs { + msgs[ordinal] = db.Message{ + SessionID: "sess-1", + Ordinal: ordinal, + Role: "assistant", + ToolCalls: []db.ToolCall{{ + ResultEvents: []db.ToolResultEvent{{EventIndex: 0}}, + }}, + } + } + require.NoError(t, database.ReplaceSessionMessages("sess-1", msgs)) + limits := productionArtifactLimits() + tt.configure(&limits) + + _, _, err := exportSessionWithLimits( + ctx, database, originRoot, origin, "sess-1", "", limits, + ) + require.Error(t, err) + assert.Contains(t, err.Error(), tt.wantError) + assert.Empty(t, globArtifacts( + t, root, origin, KindSegments, "*"+segmentExtension, + )) + assert.Empty(t, globArtifacts( + t, root, origin, KindManifests, "*"+manifestExtension, + )) + }) + } +} + +func nestedSegmentData(t *testing.T, records ...segmentMessage) []byte { + t.Helper() + var data bytes.Buffer + for ordinal := range records { + records[ordinal].Version = formatVersion + records[ordinal].Ordinal = ordinal + records[ordinal].Role = "assistant" + encoded, err := canonicalJSON(records[ordinal]) + require.NoError(t, err) + _, err = data.Write(encoded) + require.NoError(t, err) + } + return data.Bytes() +} + +func writeNestedImportFixture( + t *testing.T, + root, origin string, + record segmentMessage, +) (string, string) { + t.Helper() + originRoot := filepath.Join(root, origin) + data := nestedSegmentData(t, record) + segmentHash := hashHex(data) + segmentPath := filepath.Join(originRoot, KindSegments, segmentHash+segmentExtension) + require.NoError(t, writeCompressed(segmentPath, data)) + + gid := origin + "~sess-1" + m := manifest{ + Version: formatVersion, + Origin: origin, + NativeSessionID: "sess-1", + Session: manifestSession{ + ID: "sess-1", + Machine: origin, + }, + Segments: []string{segmentHash}, + } + manifestData, err := canonicalJSON(m) + require.NoError(t, err) + manifestHash := hashHex(manifestData) + require.NoError(t, writeCompressed( + filepath.Join(originRoot, KindManifests, manifestHash+manifestExtension), + manifestData, + )) + writeCheckpoint(t, originRoot, checkpoint{ + Version: formatVersion, + Origin: origin, + Sequence: 1, + Sessions: map[string]string{gid: manifestHash}, + }) + return gid, segmentPath +} diff --git a/internal/artifact/peer.go b/internal/artifact/peer.go new file mode 100644 index 000000000..76636762a --- /dev/null +++ b/internal/artifact/peer.go @@ -0,0 +1,743 @@ +package artifact + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "io" + "io/fs" + "log" + "maps" + "os" + "path/filepath" + "slices" + "sort" + "strings" + "time" + + "github.com/klauspost/compress/zstd" +) + +const ( + KindCheckpoints = "checkpoints" + KindManifests = "manifests" + KindSegments = "segments" + KindMeta = "meta" + KindRaw = "raw" +) + +var ( + ErrArtifactInvalid = errors.New("invalid artifact") + ErrArtifactNotFound = errors.New("artifact not found") + ErrArtifactConflict = errors.New("artifact conflict") +) + +// PeerArtifact is one immutable artifact file served through the peer API. +type PeerArtifact struct { + Origin string + Kind string + Name string + Hash string + ContentType string + Data []byte +} + +// PeerArtifactWrite describes the result of a peer artifact write. +type PeerArtifactWrite struct { + Origin string + Kind string + Name string + Hash string + Size int64 + Duplicate bool +} + +type peerArtifactSpec struct { + origin string + kind string + dir string + name string + hash string + path string + contentType string +} + +// ListOrigins returns valid origin directories in the artifact store. +func ListOrigins(root string) ([]string, error) { + if strings.TrimSpace(root) == "" { + return nil, fmt.Errorf("%w: artifact root is required", ErrArtifactInvalid) + } + entries, err := os.ReadDir(root) + if err != nil { + if errors.Is(err, fs.ErrNotExist) { + return []string{}, nil + } + return nil, err + } + origins := make([]string, 0, len(entries)) + for _, ent := range entries { + if !ent.IsDir() { + continue + } + origin := ent.Name() + if validateOriginID(origin) == nil { + origins = append(origins, origin) + } + } + sort.Strings(origins) + return origins, nil +} + +// OriginArtifactIndex lists the artifact filenames an origin currently holds, +// grouped by kind. It is the enumeration the HTTP peer transport needs for a +// set-union pull, since metadata events are not referenced by the checkpoint +// and therefore cannot be discovered from it. +type OriginArtifactIndex struct { + Origin string `json:"origin"` + Checkpoints []string `json:"checkpoints"` + Manifests []string `json:"manifests"` + Segments []string `json:"segments"` + Meta []string `json:"meta"` + Raw []string `json:"raw"` +} + +// ListArtifacts enumerates the valid artifact filenames an origin holds, grouped +// by kind, skipping temp files and entries that do not match a kind's naming +// rules. Missing kind directories yield empty lists, not errors. +func ListArtifacts(root, origin string) (OriginArtifactIndex, error) { + if strings.TrimSpace(root) == "" { + return OriginArtifactIndex{}, fmt.Errorf("%w: artifact root is required", ErrArtifactInvalid) + } + if err := validateOriginID(origin); err != nil { + return OriginArtifactIndex{}, fmt.Errorf("%w: %v", ErrArtifactInvalid, err) + } + idx := OriginArtifactIndex{Origin: origin} + var err error + if idx.Checkpoints, err = listArtifactKind(root, origin, KindCheckpoints); err != nil { + return OriginArtifactIndex{}, err + } + if idx.Manifests, err = listArtifactKind(root, origin, KindManifests); err != nil { + return OriginArtifactIndex{}, err + } + if idx.Segments, err = listArtifactKind(root, origin, KindSegments); err != nil { + return OriginArtifactIndex{}, err + } + if idx.Meta, err = listArtifactKind(root, origin, KindMeta); err != nil { + return OriginArtifactIndex{}, err + } + if idx.Raw, err = listArtifactKind(root, origin, KindRaw); err != nil { + return OriginArtifactIndex{}, err + } + return idx, nil +} + +func listArtifactKind(root, origin, kind string) ([]string, error) { + dir := filepath.Join(root, origin, kind) + entries, err := os.ReadDir(dir) + if err != nil { + if errors.Is(err, fs.ErrNotExist) { + return []string{}, nil + } + return nil, err + } + names := make([]string, 0, len(entries)) + for _, ent := range entries { + if ent.IsDir() || isTempArtifactEntry(ent.Name()) { + continue + } + if !validArtifactKindName(kind, ent.Name()) { + continue + } + names = append(names, ent.Name()) + } + sort.Strings(names) + return names, nil +} + +func validArtifactKindName(kind, name string) bool { + switch kind { + case KindCheckpoints: + return isGCCheckpointName(name) + case KindManifests: + return isGCManifestName(name) + case KindSegments: + return isGCSegmentName(name) + case KindRaw: + return isGCRawName(name) + case KindMeta: + _, _, err := normalizeMetadataName(name) + return err == nil + default: + return false + } +} + +// ReadLatestCheckpoint returns the newest checkpoint file for an origin. +func ReadLatestCheckpoint(root, origin string) (PeerArtifact, error) { + if strings.TrimSpace(root) == "" { + return PeerArtifact{}, fmt.Errorf("%w: artifact root is required", ErrArtifactInvalid) + } + if err := validateOriginID(origin); err != nil { + return PeerArtifact{}, fmt.Errorf("%w: %v", ErrArtifactInvalid, err) + } + originRoot := filepath.Join(root, origin) + path, err := latestCheckpointPath(originRoot) + if err != nil { + return PeerArtifact{}, err + } + data, err := os.ReadFile(path) + if err != nil { + if errors.Is(err, fs.ErrNotExist) { + return PeerArtifact{}, ErrArtifactNotFound + } + return PeerArtifact{}, err + } + if err := validateCheckpointData(data, origin, filepath.Base(path)); err != nil { + return PeerArtifact{}, err + } + return PeerArtifact{ + Origin: origin, + Kind: KindCheckpoints, + Name: filepath.Base(path), + ContentType: "application/json", + Data: data, + }, nil +} + +// OriginCheckpointSummary describes the latest valid checkpoint published by +// one origin, including its exact global-session-ID to manifest-hash mapping. +// Found is false when the origin has no compatible checkpoint yet. +type OriginCheckpointSummary struct { + Sequence int + SessionCount int + SessionManifests map[string]string + ModTime time.Time + Found bool +} + +// CheckpointSummary returns summary information about an origin's latest valid +// checkpoint without decoding any session bundles. It is the read side of the +// peers status view. A checkpoint that no longer validates is quarantined and +// the summary falls back to the newest valid one, so one corrupt file cannot +// make the peers view unusable. +func CheckpointSummary(root, origin string) (OriginCheckpointSummary, error) { + if strings.TrimSpace(root) == "" { + return OriginCheckpointSummary{}, fmt.Errorf("%w: artifact root is required", ErrArtifactInvalid) + } + if err := validateOriginID(origin); err != nil { + return OriginCheckpointSummary{}, fmt.Errorf("%w: %v", ErrArtifactInvalid, err) + } + paths, err := filepath.Glob(filepath.Join(root, origin, "checkpoints", "cp-*.json")) + if err != nil { + return OriginCheckpointSummary{}, err + } + sort.Strings(paths) + for _, path := range slices.Backward(paths) { + info, err := os.Stat(path) + if err != nil { + if errors.Is(err, fs.ErrNotExist) { + continue + } + return OriginCheckpointSummary{}, err + } + data, err := os.ReadFile(path) + if err != nil { + if errors.Is(err, fs.ErrNotExist) { + continue + } + return OriginCheckpointSummary{}, err + } + var cp checkpoint + if err := json.Unmarshal(data, &cp); err != nil { + log.Printf("artifact: skipping corrupt checkpoint %s in peer summary: %v", path, err) + quarantineArtifact(path) + continue + } + if err := validateCheckpoint(&cp, origin); err != nil { + if errors.Is(err, errFutureArtifactVersion) { + continue + } + log.Printf("artifact: skipping invalid checkpoint %s in peer summary: %v", path, err) + quarantineArtifact(path) + continue + } + if err := validateCheckpointSequenceIdentity(cp, filepath.Base(path)); err != nil { + log.Printf("artifact: skipping invalid checkpoint %s in peer summary: %v", path, err) + quarantineArtifact(path) + continue + } + return OriginCheckpointSummary{ + Sequence: cp.Sequence, + SessionCount: len(cp.Sessions), + SessionManifests: maps.Clone(cp.Sessions), + ModTime: info.ModTime(), + Found: true, + }, nil + } + return OriginCheckpointSummary{}, nil +} + +// ReadArtifact returns one artifact file by kind and name. +func ReadArtifact(root, origin, kind, name string) (PeerArtifact, error) { + spec, err := makePeerArtifactSpec(root, origin, kind, name) + if err != nil { + return PeerArtifact{}, err + } + return readArtifactSpec(spec) +} + +// ReadArtifactForServe returns one artifact for peer download. A stored file +// that fails content validation is quarantined and reported as not found: +// dropping it from the served set removes it from the origin index, so a peer +// that still holds a valid copy re-uploads it on a later push and the store +// heals, instead of every pull failing against the same bytes forever. A +// request whose origin, kind, or name is itself invalid is still rejected as +// invalid without touching any file. +func ReadArtifactForServe(root, origin, kind, name string) (PeerArtifact, error) { + spec, err := makePeerArtifactSpec(root, origin, kind, name) + if err != nil { + return PeerArtifact{}, err + } + art, err := readArtifactSpec(spec) + if err != nil && errors.Is(err, ErrArtifactInvalid) { + log.Printf("artifact: quarantining corrupt artifact %s/%s/%s requested by peer: %v", + origin, kind, name, err) + quarantineArtifact(spec.path) + return PeerArtifact{}, ErrArtifactNotFound + } + return art, err +} + +func readArtifactSpec(spec peerArtifactSpec) (PeerArtifact, error) { + data, err := os.ReadFile(spec.path) + if err != nil { + if errors.Is(err, fs.ErrNotExist) { + return PeerArtifact{}, ErrArtifactNotFound + } + return PeerArtifact{}, err + } + if err := validateArtifactData(spec, data); err != nil { + return PeerArtifact{}, err + } + return PeerArtifact{ + Origin: spec.origin, + Kind: spec.kind, + Name: spec.name, + Hash: spec.hash, + ContentType: spec.contentType, + Data: data, + }, nil +} + +// WriteArtifact verifies and stores one peer artifact. Existing identical +// content is accepted as a duplicate; existing different content is rejected. +func WriteArtifact(root, origin, kind, name string, data []byte) (PeerArtifactWrite, error) { + spec, err := makePeerArtifactSpec(root, origin, kind, name) + if err != nil { + return PeerArtifactWrite{}, err + } + if len(data) == 0 { + return PeerArtifactWrite{}, fmt.Errorf("%w: artifact body is empty", ErrArtifactInvalid) + } + if err := validateArtifactData(spec, data); err != nil { + return PeerArtifactWrite{}, err + } + if duplicate, err := existingArtifactMatches(spec.path, data); err != nil { + if isArtifactPathConflict(err) { + return PeerArtifactWrite{}, fmt.Errorf("%w: %v", ErrArtifactConflict, err) + } + return PeerArtifactWrite{}, err + } else if duplicate { + return PeerArtifactWrite{ + Origin: spec.origin, + Kind: spec.kind, + Name: spec.name, + Hash: spec.hash, + Size: int64(len(data)), + Duplicate: true, + }, nil + } + if err := writeFileAtomic(spec.path, data, 0o644); err != nil { + if isArtifactPathConflict(err) { + return PeerArtifactWrite{}, fmt.Errorf("%w: %v", ErrArtifactConflict, err) + } + return PeerArtifactWrite{}, err + } + return PeerArtifactWrite{ + Origin: spec.origin, + Kind: spec.kind, + Name: spec.name, + Hash: spec.hash, + Size: int64(len(data)), + }, nil +} + +func latestCheckpointPath(originRoot string) (string, error) { + paths, err := filepath.Glob(filepath.Join(originRoot, "checkpoints", "cp-*.json")) + if err != nil { + return "", err + } + if len(paths) == 0 { + return "", ErrArtifactNotFound + } + sort.Strings(paths) + return paths[len(paths)-1], nil +} + +func makePeerArtifactSpec(root, origin, kind, name string) (peerArtifactSpec, error) { + if strings.TrimSpace(root) == "" { + return peerArtifactSpec{}, fmt.Errorf("%w: artifact root is required", ErrArtifactInvalid) + } + if err := validateOriginID(origin); err != nil { + return peerArtifactSpec{}, fmt.Errorf("%w: %v", ErrArtifactInvalid, err) + } + if err := validateArtifactName(name); err != nil { + return peerArtifactSpec{}, err + } + spec, err := artifactSpecForKind(origin, kind, name) + if err != nil { + return peerArtifactSpec{}, err + } + spec.path = filepath.Join(root, origin, spec.dir, spec.name) + return spec, nil +} + +// artifactSpecForKind resolves an artifact's kind, canonical name, and +// content hash from its path identity alone, without requiring a valid origin +// id or store root, so on-disk files can be validated in place. +func artifactSpecForKind(origin, kind, name string) (peerArtifactSpec, error) { + kind = normalizeArtifactKind(kind) + spec := peerArtifactSpec{ + origin: origin, + kind: kind, + } + switch kind { + case KindCheckpoints: + filename, err := normalizeCheckpointName(name) + if err != nil { + return peerArtifactSpec{}, err + } + spec.dir = "checkpoints" + spec.name = filename + spec.contentType = "application/json" + case KindManifests: + filename, hash, err := normalizeHashName(name, manifestExtension) + if err != nil { + return peerArtifactSpec{}, err + } + spec.dir = "manifests" + spec.name = filename + spec.hash = hash + spec.contentType = "application/zstd" + case KindSegments: + filename, hash, err := normalizeHashName(name, segmentExtension) + if err != nil { + return peerArtifactSpec{}, err + } + spec.dir = "segments" + spec.name = filename + spec.hash = hash + spec.contentType = "application/zstd" + case KindMeta: + filename, hash, err := normalizeMetadataName(name) + if err != nil { + return peerArtifactSpec{}, err + } + spec.dir = "meta" + spec.name = filename + spec.hash = hash + spec.contentType = "application/json" + case KindRaw: + if err := validateHashHex(name); err != nil { + return peerArtifactSpec{}, err + } + spec.dir = "raw" + spec.name = name + spec.hash = name + spec.contentType = "application/octet-stream" + default: + return peerArtifactSpec{}, fmt.Errorf("%w: unsupported artifact kind %q", ErrArtifactInvalid, kind) + } + return spec, nil +} + +func normalizeArtifactKind(kind string) string { + switch strings.TrimSpace(strings.ToLower(kind)) { + case "checkpoint", "checkpoints": + return KindCheckpoints + case "manifest", "manifests": + return KindManifests + case "segment", "segments": + return KindSegments + case "meta", "metadata": + return KindMeta + case "raw": + return KindRaw + default: + return strings.TrimSpace(strings.ToLower(kind)) + } +} + +func validateArtifactName(name string) error { + if strings.TrimSpace(name) == "" { + return fmt.Errorf("%w: artifact name is required", ErrArtifactInvalid) + } + if strings.Contains(name, "/") || strings.Contains(name, "\\") || strings.Contains(name, "..") { + return fmt.Errorf("%w: invalid artifact name", ErrArtifactInvalid) + } + return nil +} + +func normalizeHashName(name, extension string) (filename, hash string, err error) { + hash = strings.TrimSuffix(name, extension) + if err := validateHashHex(hash); err != nil { + return "", "", err + } + return hash + extension, hash, nil +} + +func normalizeMetadataName(name string) (filename, hash string, err error) { + base := strings.TrimSuffix(name, metadataEventExtension) + idx := strings.LastIndex(base, "-") + if idx < 0 { + return "", "", fmt.Errorf("%w: metadata artifact missing hash suffix", ErrArtifactInvalid) + } + hash = base[idx+1:] + if err := validateHashHex(hash); err != nil { + return "", "", err + } + return base + metadataEventExtension, hash, nil +} + +func normalizeCheckpointName(name string) (string, error) { + base := strings.TrimSuffix(name, ".json") + if _, err := checkpointSequence(base + ".json"); err != nil { + return "", err + } + return base + ".json", nil +} + +func checkpointSequence(filename string) (int, error) { + base := strings.TrimSuffix(filename, ".json") + if len(base) != len("cp-0000000000") || !strings.HasPrefix(base, "cp-") { + return 0, fmt.Errorf("%w: invalid checkpoint name", ErrArtifactInvalid) + } + seq := 0 + for _, r := range base[len("cp-"):] { + if r < '0' || r > '9' { + return 0, fmt.Errorf("%w: invalid checkpoint name", ErrArtifactInvalid) + } + seq = seq*10 + int(r-'0') + } + if seq <= 0 { + return 0, fmt.Errorf("%w: invalid checkpoint sequence", ErrArtifactInvalid) + } + return seq, nil +} + +func validateHashHex(hash string) error { + if len(hash) != 64 { + return fmt.Errorf("%w: invalid artifact hash", ErrArtifactInvalid) + } + for _, r := range hash { + if (r < '0' || r > '9') && (r < 'a' || r > 'f') { + return fmt.Errorf("%w: invalid artifact hash", ErrArtifactInvalid) + } + } + return nil +} + +func validateArtifactData(spec peerArtifactSpec, data []byte) error { + switch spec.kind { + case KindCheckpoints: + return validateCheckpointData(data, spec.origin, spec.name) + case KindManifests: + return validateManifestArtifactData(data, spec.origin, spec.hash) + case KindSegments: + return validateSegmentArtifactData(data, spec.hash) + case KindMeta: + return validateMetadataArtifactData(data, spec.origin, spec.name, spec.hash) + case KindRaw: + if got := hashHex(data); got != spec.hash { + return fmt.Errorf("%w: raw artifact hash mismatch: got %s", ErrArtifactInvalid, got) + } + return nil + default: + return fmt.Errorf("%w: unsupported artifact kind %q", ErrArtifactInvalid, spec.kind) + } +} + +func validateCheckpointData(data []byte, origin, filename string) error { + var cp checkpoint + if err := json.Unmarshal(data, &cp); err != nil { + return fmt.Errorf("%w: decoding checkpoint: %v", ErrArtifactInvalid, err) + } + if cp.Version > formatVersion { + if cp.Origin != origin { + return fmt.Errorf( + "%w: checkpoint origin mismatch for %s: got %q", + ErrArtifactInvalid, origin, cp.Origin, + ) + } + if err := validateCheckpointSequenceIdentity(cp, filename); err != nil { + return fmt.Errorf("%w: %v", ErrArtifactInvalid, err) + } + if err := validateCheckpointReferences(&cp, origin); err != nil { + return fmt.Errorf("%w: %v", ErrArtifactInvalid, err) + } + return nil + } + if err := validateCheckpoint(&cp, origin); err != nil { + return fmt.Errorf("%w: %v", ErrArtifactInvalid, err) + } + if err := validateCheckpointSequenceIdentity(cp, filename); err != nil { + return fmt.Errorf("%w: %v", ErrArtifactInvalid, err) + } + return nil +} + +func validateCheckpointSequenceIdentity(cp checkpoint, filename string) error { + seq, err := checkpointSequence(filename) + if err != nil { + return err + } + if cp.Sequence != seq { + return fmt.Errorf( + "checkpoint sequence mismatch: name has %d, body has %d", + seq, cp.Sequence, + ) + } + return nil +} + +func validateManifestArtifactData(data []byte, origin, hash string) error { + decoded, err := readCompressedBytes(data, manifestDecodedLimit) + if err != nil { + return fmt.Errorf("%w: decoding manifest compression: %v", ErrArtifactInvalid, err) + } + if got := hashHex(decoded); got != hash { + return fmt.Errorf("%w: manifest hash mismatch: got %s", ErrArtifactInvalid, got) + } + m, err := decodeManifestWithLimits(decoded, productionArtifactLimits()) + if err != nil { + return fmt.Errorf("%w: decoding manifest: %v", ErrArtifactInvalid, err) + } + if m.Version > formatVersion { + if m.Origin != origin { + return fmt.Errorf("%w: manifest origin mismatch for %s: got %q", ErrArtifactInvalid, origin, m.Origin) + } + return nil + } + if m.Origin != origin { + return fmt.Errorf("%w: manifest origin mismatch for %s: got %q", ErrArtifactInvalid, origin, m.Origin) + } + if m.NativeSessionID == "" || m.Session.ID != m.NativeSessionID || m.Session.Machine != origin { + return fmt.Errorf("%w: manifest session identity mismatch", ErrArtifactInvalid) + } + if len(m.Segments) == 0 { + return fmt.Errorf("%w: manifest has no message segments", ErrArtifactInvalid) + } + if err := validateManifestReferences(m); err != nil { + return fmt.Errorf("%w: %v", ErrArtifactInvalid, err) + } + return nil +} + +func validateSegmentArtifactData(data []byte, hash string) error { + decoded, err := readCompressedBytes(data, segmentDecodedLimit) + if err != nil { + return fmt.Errorf("%w: decoding segment compression: %v", ErrArtifactInvalid, err) + } + if got := hashHex(decoded); got != hash { + return fmt.Errorf("%w: segment hash mismatch: got %s", ErrArtifactInvalid, got) + } + if _, err := decodeSegment(decoded); err != nil { + if errors.Is(err, errFutureArtifactVersion) { + if ferr := validateFutureSegmentData(decoded); ferr != nil { + return fmt.Errorf("%w: decoding segment: %v", ErrArtifactInvalid, ferr) + } + return nil + } + return fmt.Errorf("%w: decoding segment: %v", ErrArtifactInvalid, err) + } + return nil +} + +func validateFutureSegmentData(data []byte) error { + records, err := segmentRecords(data, maxSegmentMessages) + if err != nil { + return err + } + for _, line := range records { + var record struct { + Version int `json:"v"` + } + if err := json.Unmarshal(line, &record); err != nil { + return err + } + if record.Version <= formatVersion { + return fmt.Errorf("message segment has unsupported artifact version %d", record.Version) + } + } + return nil +} + +func validateMetadataArtifactData(data []byte, origin, filename, hash string) error { + if got := hashHex(data); got != hash { + return fmt.Errorf("%w: metadata artifact hash mismatch: got %s", ErrArtifactInvalid, got) + } + base := strings.TrimSuffix(filename, metadataEventExtension) + hlc := strings.TrimSuffix(base, "-"+hash) + var event metadataEvent + if err := json.Unmarshal(data, &event); err != nil { + return fmt.Errorf("%w: decoding metadata event: %v", ErrArtifactInvalid, err) + } + art := metadataArtifact{ + path: filename, + hlc: hlc, + hash: hash, + event: event, + } + if err := validateMetadataArtifactEvent(art, origin); err != nil { + if errors.Is(err, errFutureArtifactVersion) { + return nil + } + return fmt.Errorf("%w: %v", ErrArtifactInvalid, err) + } + // Unknown ops stay accepted for forward compatibility (replay marks + // them applied and skips them), but a known op must carry a payload + // that projects cleanly or replay could never apply it. + if err := validateMetadataOp(event.Op); err == nil { + if _, _, _, _, err := metadataProjectionFields(event); err != nil { + return fmt.Errorf("%w: metadata event payload: %v", ErrArtifactInvalid, err) + } + } + return nil +} + +func readCompressedBytes(data []byte, maxDecoded int64) ([]byte, error) { + dec, err := zstd.NewReader( + bytes.NewReader(data), + zstd.WithDecoderConcurrency(1), + zstd.WithDecoderLowmem(true), + zstd.WithDecoderMaxMemory(uint64(maxDecoded)), + zstd.WithDecoderMaxWindow(zstdMaxWindowSize), + ) + if err != nil { + return nil, err + } + defer dec.Close() + out, err := io.ReadAll(io.LimitReader(dec, maxDecoded+1)) + if err != nil { + return nil, err + } + if int64(len(out)) > maxDecoded { + return nil, fmt.Errorf("decoded output exceeds %d-byte limit", maxDecoded) + } + return out, nil +} + +func isArtifactPathConflict(err error) bool { + return errors.Is(err, errArtifactPathConflict) +} diff --git a/internal/artifact/peer_test.go b/internal/artifact/peer_test.go new file mode 100644 index 000000000..d9a4245c8 --- /dev/null +++ b/internal/artifact/peer_test.go @@ -0,0 +1,322 @@ +package artifact + +import ( + "bytes" + "context" + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/klauspost/compress/zstd" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "go.kenn.io/agentsview/internal/db" +) + +func TestPeerArtifactWriteReadManifestDuplicate(t *testing.T) { + root := t.TempDir() + origin := "peer-a1b2c3" + startedAt := "2026-06-14T01:02:03Z" + segmentData, err := encodeSegment([]db.Message{{ + Ordinal: 0, + Role: "user", + Content: "hello", + }}) + require.NoError(t, err) + segmentHash := hashHex(segmentData) + manifestData, err := canonicalJSON(manifest{ + Version: formatVersion, + Origin: origin, + NativeSessionID: "sess-1", + Session: manifestSession{ + ID: "sess-1", + Machine: origin, + Agent: "claude", + Project: "alpha", + StartedAt: &startedAt, + CreatedAt: "2026-06-14T01:02:03Z", + MessageCount: 1, + UserMessageCount: 1, + TotalOutputTokens: 0, + PeakContextTokens: 0, + }, + Segments: []string{segmentHash}, + DataVersion: 1, + Generation: 1, + }) + require.NoError(t, err) + manifestHash := hashHex(manifestData) + compressed := compressPeerTestData(t, manifestData) + + res, err := WriteArtifact(root, origin, "manifest", manifestHash, compressed) + require.NoError(t, err) + assert.False(t, res.Duplicate) + assert.Equal(t, KindManifests, res.Kind) + assert.Equal(t, manifestHash+manifestExtension, res.Name) + + res, err = WriteArtifact(root, origin, "manifests", manifestHash+manifestExtension, compressed) + require.NoError(t, err) + assert.True(t, res.Duplicate) + + got, err := ReadArtifact(root, origin, "manifests", manifestHash) + require.NoError(t, err) + assert.Equal(t, "application/zstd", got.ContentType) + assert.Equal(t, compressed, got.Data) +} + +func TestPeerArtifactRejectsHashMismatch(t *testing.T) { + root := t.TempDir() + origin := "peer-a1b2c3" + data := compressPeerTestData(t, []byte("not the named hash\n")) + + _, err := WriteArtifact(root, origin, "segments", strings64("0"), data) + require.Error(t, err) + assert.ErrorIs(t, err, ErrArtifactInvalid) +} + +func TestPeerArtifactCheckpointDuplicateAndConflict(t *testing.T) { + root := t.TempDir() + origin := "peer-a1b2c3" + first := []byte(`{"origin":"peer-a1b2c3","seq":1,"sessions":{},"v":1}` + "\n") + second := []byte(`{"origin":"peer-a1b2c3","seq":1,"sessions":{"peer-a1b2c3~sess-1":"` + strings64("a") + `"},"v":1}` + "\n") + + res, err := WriteArtifact(root, origin, "checkpoints", "cp-0000000001", first) + require.NoError(t, err) + assert.False(t, res.Duplicate) + + res, err = WriteArtifact(root, origin, "checkpoints", "cp-0000000001.json", first) + require.NoError(t, err) + assert.True(t, res.Duplicate) + + _, err = WriteArtifact(root, origin, "checkpoints", "cp-0000000001", second) + require.Error(t, err) + assert.ErrorIs(t, err, ErrArtifactConflict) +} + +func TestPeerArtifactRejectsInvalidReferences(t *testing.T) { + root := t.TempDir() + origin := "peer-a1b2c3" + + checkpointData := []byte(`{"origin":"peer-a1b2c3","seq":1,"sessions":{"peer-a1b2c3~sess-1":"../outside"},"v":1}` + "\n") + _, err := WriteArtifact(root, origin, KindCheckpoints, "cp-0000000001", checkpointData) + require.Error(t, err) + assert.ErrorIs(t, err, ErrArtifactInvalid) + + manifestData, err := canonicalJSON(manifest{ + Version: formatVersion, + Origin: origin, + NativeSessionID: "sess-1", + Session: manifestSession{ + ID: "sess-1", + Machine: origin, + Agent: "claude", + Project: "alpha", + CreatedAt: "2026-06-14T01:02:03Z", + }, + Segments: []string{"../outside"}, + }) + require.NoError(t, err) + manifestHash := hashHex(manifestData) + _, err = WriteArtifact( + root, + origin, + KindManifests, + manifestHash, + compressPeerTestData(t, manifestData), + ) + require.Error(t, err) + assert.ErrorIs(t, err, ErrArtifactInvalid) +} + +func TestPeerArtifactMetadataMustMatchOrigin(t *testing.T) { + root := t.TempDir() + origin := "peer-a1b2c3" + body := []byte(`{"hlc":"2026-06-14T010203.000000001Z-other-b2c3d4","op":"rename","origin":"other-b2c3d4","session_gid":"other-b2c3d4~sess-1","v":1,"value":{"display_name":"Remote"}}` + "\n") + hash := hashHex(body) + name := "2026-06-14T010203.000000001Z-other-b2c3d4-" + hash + + _, err := WriteArtifact(root, origin, "meta", name, body) + require.Error(t, err) + assert.ErrorIs(t, err, ErrArtifactInvalid) +} + +func TestPeerArtifactMetadataRejectsMalformedKnownOpPayload(t *testing.T) { + root := t.TempDir() + origin := "peer-a1b2c3" + hlc := "2026-06-14T010203.000000001Z-peer-a1b2c3" + + tests := []struct { + name string + op string + value json.RawMessage + }{ + {name: "pin missing payload", op: MetadataOpPin}, + {name: "unpin missing payload", op: MetadataOpUnpin}, + {name: "rename non-object value", op: MetadataOpRename, value: json.RawMessage(`[1,2]`)}, + {name: "rename missing value", op: MetadataOpRename}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + data, err := canonicalJSON(metadataEvent{ + Version: formatVersion, + HLC: hlc, + Origin: origin, + SessionGID: origin + "~sess-1", + Op: tt.op, + Value: tt.value, + }) + require.NoError(t, err) + name := hlc + "-" + hashHex(data) + _, err = WriteArtifact(root, origin, KindMeta, name, data) + require.Error(t, err) + assert.ErrorIs(t, err, ErrArtifactInvalid) + }) + } +} + +func TestPeerArtifactStoresFutureVersionArtifacts(t *testing.T) { + root := t.TempDir() + origin := "peer-a1b2c3" + + checkpointData, err := canonicalJSON(checkpoint{ + Version: formatVersion + 1, + Origin: origin, + Sequence: 1, + Sessions: map[string]string{ + origin + "~sess-1": strings64("a"), + }, + }) + require.NoError(t, err) + res, err := WriteArtifact(root, origin, KindCheckpoints, "cp-0000000001", checkpointData) + require.NoError(t, err) + assert.False(t, res.Duplicate) + + got, err := ReadArtifact(root, origin, KindCheckpoints, "cp-0000000001") + require.NoError(t, err) + assert.Equal(t, checkpointData, got.Data) + + segmentData, err := canonicalJSON(segmentMessage{ + Version: formatVersion + 1, + Ordinal: 0, + Role: "user", + Content: "future segment", + }) + require.NoError(t, err) + compressedSegment := compressPeerTestData(t, segmentData) + segmentHash := hashHex(segmentData) + res, err = WriteArtifact(root, origin, KindSegments, segmentHash, compressedSegment) + require.NoError(t, err) + assert.Equal(t, segmentHash+segmentExtension, res.Name) + got, err = ReadArtifact(root, origin, KindSegments, segmentHash) + require.NoError(t, err) + assert.Equal(t, compressedSegment, got.Data) + + futureManifestData, err := canonicalJSON(struct { + Version int `json:"v"` + Origin string `json:"origin"` + Future string `json:"future"` + }{ + Version: formatVersion + 1, + Origin: origin, + Future: "schema-owned-by-newer-peer", + }) + require.NoError(t, err) + compressedManifest := compressPeerTestData(t, futureManifestData) + manifestHash := hashHex(futureManifestData) + res, err = WriteArtifact(root, origin, KindManifests, manifestHash, compressedManifest) + require.NoError(t, err) + assert.Equal(t, manifestHash+manifestExtension, res.Name) + got, err = ReadArtifact(root, origin, KindManifests, manifestHash) + require.NoError(t, err) + assert.Equal(t, compressedManifest, got.Data) + + hlc := "2026-06-14T010203.000000001Z-peer-a1b2c3" + metadataData, err := canonicalJSON(metadataEvent{ + Version: formatVersion + 1, + HLC: hlc, + Origin: origin, + SessionGID: origin + "~sess-1", + Op: "future_op", + Value: json.RawMessage(`{"future":true}`), + }) + require.NoError(t, err) + metadataHash := hashHex(metadataData) + metadataName := hlc + "-" + metadataHash + res, err = WriteArtifact(root, origin, KindMeta, metadataName, metadataData) + require.NoError(t, err) + assert.Equal(t, metadataName+metadataEventExtension, res.Name) + got, err = ReadArtifact(root, origin, KindMeta, metadataName) + require.NoError(t, err) + assert.Equal(t, metadataData, got.Data) +} + +func TestPeerArtifactRejectsMixedFutureSegmentVersions(t *testing.T) { + root := t.TempDir() + origin := "peer-a1b2c3" + futureLine, err := canonicalJSON(segmentMessage{ + Version: formatVersion + 1, + Ordinal: 0, + Role: "user", + Content: "future", + }) + require.NoError(t, err) + currentLine, err := canonicalJSON(segmentMessage{ + Version: formatVersion, + Ordinal: 1, + Role: "assistant", + Content: "current", + }) + require.NoError(t, err) + segmentData := append(futureLine, currentLine...) + compressed := compressPeerTestData(t, segmentData) + hash := hashHex(segmentData) + + _, err = WriteArtifact(root, origin, KindSegments, hash, compressed) + require.Error(t, err) + assert.ErrorIs(t, err, ErrArtifactInvalid) +} + +func compressPeerTestData(t *testing.T, data []byte) []byte { + t.Helper() + var buf bytes.Buffer + enc, err := zstd.NewWriter(&buf) + require.NoError(t, err) + _, err = enc.Write(data) + require.NoError(t, err) + require.NoError(t, enc.Close()) + return buf.Bytes() +} + +func strings64(ch string) string { + return strings.Repeat(ch, 64) +} + +func TestCheckpointSummarySkipsAndQuarantinesCorruptLatestCheckpoint(t *testing.T) { + ctx := context.Background() + root := t.TempDir() + origin := "laptop-a1b2c3" + database := testDB(t) + seedSession(t, database, "sess-1", "alpha") + + _, err := Export(ctx, database, root, origin) + require.NoError(t, err) + seedSession(t, database, "sess-2", "beta") + _, err = Export(ctx, database, root, origin) + require.NoError(t, err) + + latest := filepath.Join(root, origin, "checkpoints", "cp-0000000002.json") + require.NoError(t, os.WriteFile(latest, []byte("not json"), 0o644)) + + // The summary falls back to the newest checkpoint that still decodes and + // quarantines the corrupt one, instead of failing the peers status view. + summary, err := CheckpointSummary(root, origin) + require.NoError(t, err) + require.True(t, summary.Found) + assert.Equal(t, 1, summary.Sequence) + assert.Equal(t, 1, summary.SessionCount) + assert.NoFileExists(t, latest) + assert.FileExists(t, latest+quarantineSuffix) +} diff --git a/internal/artifact/replay.go b/internal/artifact/replay.go new file mode 100644 index 000000000..5f007be63 --- /dev/null +++ b/internal/artifact/replay.go @@ -0,0 +1,313 @@ +package artifact + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io/fs" + "log" + "os" + "path/filepath" + "sort" + "strings" + + "go.kenn.io/agentsview/internal/db" +) + +type metadataArtifact struct { + path string + orderKey string + hash string + hlc string + event metadataEvent +} + +func replayMetadata( + ctx context.Context, + database *db.DB, + clock *HLCClock, + originRoot, origin, localOrigin string, + appliedEvents map[db.MetadataEventIdentity]struct{}, +) (int, error) { + events, err := readMetadataArtifacts(originRoot, origin, appliedEvents) + if err != nil { + return 0, err + } + changed := 0 + for _, art := range events { + if err := ctx.Err(); err != nil { + return changed, err + } + stamp, perr := ParseHLCTimestamp(art.hlc) + if perr != nil { + // A future format version may change the HLC shape; defer such + // events for an upgraded app instead of discarding them. + if art.event.Version > formatVersion { + continue + } + // LWW ordering is a raw order-key comparison, so an event whose + // HLC does not parse could sort above every real timestamp and + // win permanently. The artifact is immutable; mark it applied so + // it can never enter replay ordering. + log.Printf("artifact: skipping metadata event %s with invalid HLC: %v", art.path, perr) + if err := database.MarkMetadataEventApplied(ctx, origin, art.orderKey, art.hash); err != nil { + return changed, err + } + continue + } + // Advance the local HLC past this remote event before applying it, so a + // later local edit is causally ahead of the peer. If the clock cannot be + // advanced (the remote wall time is beyond the drift bound), defer the + // event rather than dragging local state to a value a future local edit + // could not out-order; a later run retries once wall time advances. + if clock != nil { + if _, oerr := clock.Observe(stamp); oerr != nil { + continue + } + } + if err := validateMetadataArtifactEvent(art, origin); err != nil { + if errors.Is(err, errFutureArtifactVersion) { + continue + } + return changed, err + } + if err := validateMetadataOp(art.event.Op); err != nil { + if err := database.MarkMetadataEventApplied(ctx, origin, art.orderKey, art.hash); err != nil { + return changed, err + } + continue + } + projection, err := metadataProjection(art, localOrigin) + if err != nil { + // A known op with a malformed payload can never become + // applicable: the artifact is immutable. Mark it applied so + // one bad event cannot permanently block the origin's replay. + log.Printf( + "artifact: skipping malformed metadata event %s: %v", + art.path, err, + ) + if err := database.MarkMetadataEventApplied(ctx, origin, art.orderKey, art.hash); err != nil { + return changed, err + } + continue + } + res, err := database.ApplyMetadataProjection(ctx, projection) + if err != nil { + // The event targets a session or message that is not durable + // locally yet. Skip only this event, leaving it unapplied so a + // later run retries it, and keep replaying the rest of the origin. + if errors.Is(err, db.ErrMetadataTargetUnavailable) { + continue + } + return changed, fmt.Errorf("replaying metadata event %s: %w", art.path, err) + } + if res.Applied || res.Conflict { + changed++ + } + } + return changed, nil +} + +func readMetadataArtifacts( + originRoot, origin string, + appliedEvents map[db.MetadataEventIdentity]struct{}, +) ([]metadataArtifact, error) { + paths, err := filepath.Glob(filepath.Join(originRoot, "meta", "*"+metadataEventExtension)) + if err != nil { + return nil, err + } + events := make([]metadataArtifact, 0, len(paths)) + for _, path := range paths { + orderKey, keyErr := metadataArtifactOrderKey(path) + if keyErr == nil { + identity := db.MetadataEventIdentity{Origin: origin, OrderKey: orderKey} + if _, ok := appliedEvents[identity]; ok { + continue + } + } + art, err := readMetadataArtifact(path) + if err != nil { + if errors.Is(err, fs.ErrNotExist) { + continue + } + if errors.Is(err, errCorruptArtifact) { + quarantineArtifact(path) + } + log.Printf("artifact: skipping unreadable metadata event %s for %s: %v", path, origin, err) + continue + } + events = append(events, art) + } + sort.Slice(events, func(i, j int) bool { + return events[i].orderKey < events[j].orderKey + }) + return events, nil +} + +func readMetadataArtifact(path string) (metadataArtifact, error) { + name, err := metadataArtifactOrderKey(path) + if err != nil { + return metadataArtifact{}, err + } + idx := strings.LastIndex(name, "-") + if idx < 0 { + return metadataArtifact{}, fmt.Errorf("metadata artifact %s missing hash suffix", filepath.Base(path)) + } + hlc := name[:idx] + hash := name[idx+1:] + data, err := os.ReadFile(path) + if err != nil { + return metadataArtifact{}, err + } + if got := hashHex(data); got != hash { + return metadataArtifact{}, fmt.Errorf("%w: metadata artifact %s hash mismatch: got %s", errCorruptArtifact, filepath.Base(path), got) + } + var event metadataEvent + if err := json.Unmarshal(data, &event); err != nil { + return metadataArtifact{}, fmt.Errorf("decoding metadata artifact %s: %w", filepath.Base(path), err) + } + return metadataArtifact{ + path: path, + orderKey: name, + hash: hash, + hlc: hlc, + event: event, + }, nil +} + +func metadataArtifactOrderKey(path string) (string, error) { + base := filepath.Base(path) + name := strings.TrimSuffix(base, metadataEventExtension) + if name == base { + return "", fmt.Errorf("metadata artifact %s missing %s extension", base, metadataEventExtension) + } + return name, nil +} + +func validateMetadataArtifactEvent(art metadataArtifact, origin string) error { + if art.event.HLC != art.hlc { + return fmt.Errorf("metadata event %s HLC mismatch: got %q", art.path, art.event.HLC) + } + if art.event.Origin != origin { + return fmt.Errorf( + "metadata event %s origin mismatch for %s: got %q", + art.path, origin, art.event.Origin, + ) + } + if art.event.SessionGID == "" { + return fmt.Errorf("metadata event %s has empty session GID", art.path) + } + if art.event.Version > formatVersion { + return fmt.Errorf( + "%w: metadata event %s has artifact version %d", + errFutureArtifactVersion, art.path, art.event.Version, + ) + } + if art.event.Version != formatVersion { + return fmt.Errorf( + "metadata event %s has unsupported artifact version %d", + art.path, art.event.Version, + ) + } + // Checked after the version gate: a future format may change the HLC + // shape, but a current-version event with an unparseable HLC would + // poison raw order-key LWW comparison and must never be accepted. + if _, err := ParseHLCTimestamp(art.hlc); err != nil { + return fmt.Errorf("metadata event %s has invalid HLC: %v", art.path, err) + } + return nil +} + +func metadataProjection(art metadataArtifact, localOrigin string) (db.MetadataProjection, error) { + event := art.event + field, value, displayName, pin, err := metadataProjectionFields(event) + if err != nil { + return db.MetadataProjection{}, err + } + return db.MetadataProjection{ + EventOrigin: event.Origin, + OrderKey: art.orderKey, + HLC: event.HLC, + ArtifactHash: art.hash, + SessionGID: event.SessionGID, + LocalSessionID: metadataLocalSessionID(localOrigin, event.SessionGID), + Field: field, + Op: event.Op, + Value: value, + DisplayName: displayName, + Pin: pin, + }, nil +} + +func metadataProjectionFields( + event metadataEvent, +) (field string, value string, displayName *string, pin *db.MetadataPinProjection, err error) { + switch event.Op { + case MetadataOpRename: + var payload struct { + DisplayName *string `json:"display_name"` + } + if err := json.Unmarshal(event.Value, &payload); err != nil { + return "", "", nil, nil, fmt.Errorf("decoding rename metadata value: %w", err) + } + value, err := metadataCanonicalValue(event.Value) + return "display_name", value, payload.DisplayName, nil, err + case MetadataOpSoftDelete, MetadataOpRestore: + return "deleted_at", event.Op, nil, nil, nil + case MetadataOpStar, MetadataOpUnstar: + return "starred", event.Op, nil, nil, nil + case MetadataOpPin, MetadataOpUnpin: + if event.Pin == nil { + return "", "", nil, nil, fmt.Errorf("%s metadata event missing pin payload", event.Op) + } + value, err := metadataCanonicalPin(*event.Pin) + if err != nil { + return "", "", nil, nil, err + } + return "pin:" + metadataPinAnchor(*event.Pin), value, nil, &db.MetadataPinProjection{ + SourceUUID: event.Pin.SourceUUID, + Ordinal: event.Pin.Ordinal, + Note: event.Pin.Note, + }, nil + case MetadataOpPurge: + return "purge", event.Op, nil, nil, nil + default: + return "", "", nil, nil, fmt.Errorf("unsupported metadata event op %q", event.Op) + } +} + +func metadataLocalSessionID(localOrigin, gid string) string { + prefix := localOrigin + "~" + if after, ok := strings.CutPrefix(gid, prefix); ok { + return after + } + return gid +} + +func metadataPinAnchor(pin MetadataPin) string { + if pin.SourceUUID != "" { + return "source_uuid:" + pin.SourceUUID + } + return fmt.Sprintf("ordinal:%d", pin.Ordinal) +} + +func metadataCanonicalValue(raw json.RawMessage) (string, error) { + if len(bytes.TrimSpace(raw)) == 0 { + return "", nil + } + data, err := canonicalJSON(raw) + if err != nil { + return "", err + } + return string(bytes.TrimSpace(data)), nil +} + +func metadataCanonicalPin(pin MetadataPin) (string, error) { + data, err := canonicalJSON(pin) + if err != nil { + return "", err + } + return string(bytes.TrimSpace(data)), nil +} diff --git a/internal/artifact/replay_test.go b/internal/artifact/replay_test.go new file mode 100644 index 000000000..cbbc1f926 --- /dev/null +++ b/internal/artifact/replay_test.go @@ -0,0 +1,770 @@ +package artifact + +import ( + "context" + "database/sql" + "encoding/json" + "os" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.kenn.io/agentsview/internal/db" +) + +func TestImportReplaysMetadataRenameDeterministically(t *testing.T) { + ctx := context.Background() + origin := "laptop-a1b2c3" + localOrigin := "desktop-d4e5f6" + gid := origin + "~sess-1" + stamp := replayTestHLC(0, 7) + events := []metadataEvent{ + replayRenameEvent(t, origin, gid, stamp, "Alpha"), + replayRenameEvent(t, origin, gid, stamp, "Beta"), + } + + for _, tc := range []struct { + name string + writeOrder []int + }{ + {name: "forward write order", writeOrder: []int{0, 1}}, + {name: "reverse write order", writeOrder: []int{1, 0}}, + } { + t.Run(tc.name, func(t *testing.T) { + root := t.TempDir() + exportDB := testDB(t) + importDB := testDB(t) + seedSession(t, exportDB, "sess-1", "alpha") + _, err := Export(ctx, exportDB, root, origin) + require.NoError(t, err) + + arts := make([]metadataArtifact, len(events)) + for _, idx := range tc.writeOrder { + arts[idx] = writeMetadataArtifact(t, filepath.Join(root, origin), events[idx]) + } + wantName := "Alpha" + if arts[1].orderKey > arts[0].orderKey { + wantName = "Beta" + } + + imported, messages, err := Import(ctx, importDB, root, localOrigin) + require.NoError(t, err) + assert.Equal(t, 1, imported) + assert.Equal(t, 2, messages) + + got, err := importDB.GetSession(ctx, gid) + require.NoError(t, err) + require.NotNil(t, got) + require.NotNil(t, got.DisplayName) + assert.Equal(t, wantName, *got.DisplayName) + assertMetadataConflictCount(t, importDB, gid, "display_name", 0) + assert.Equal(t, 2, metadataAppliedCount(t, importDB, origin)) + + imported, messages, err = Import(ctx, importDB, root, localOrigin) + require.NoError(t, err) + assert.Zero(t, imported) + assert.Zero(t, messages) + assertMetadataConflictCount(t, importDB, gid, "display_name", 0) + assert.Equal(t, 2, metadataAppliedCount(t, importDB, origin)) + }) + } +} + +func TestImportAppliesEarlierOriginMetadataToLaterOriginContentInOnePass(t *testing.T) { + ctx := context.Background() + metadataOrigin := "alpha-a1b2c3" + contentOrigin := "zulu-d4e5f6" + localOrigin := "desktop-c7d8e9" + root := t.TempDir() + exportDB := testDB(t) + importDB := testDB(t) + seedSession(t, exportDB, "sess-1", "alpha") + _, err := Export(ctx, exportDB, root, contentOrigin) + require.NoError(t, err) + + gid := contentOrigin + "~sess-1" + writeMetadataArtifact(t, filepath.Join(root, metadataOrigin), replayRenameEvent( + t, metadataOrigin, gid, replayTestHLC(0, 0), "Renamed before import", + )) + writeMetadataArtifact(t, filepath.Join(root, metadataOrigin), metadataEvent{ + Version: formatVersion, + HLC: replayTestHLC(time.Nanosecond, 0), + Origin: metadataOrigin, + SessionGID: gid, + Op: MetadataOpSoftDelete, + }) + + res, err := ImportDetailed(ctx, importDB, root, localOrigin) + require.NoError(t, err) + assert.Equal(t, 1, res.Sessions) + assert.Equal(t, 2, res.Messages) + assert.Equal(t, 2, res.Metadata) + + var displayName, deletedAt sql.NullString + err = importDB.Reader().QueryRowContext(ctx, + `SELECT display_name, deleted_at FROM sessions WHERE id = ?`, gid, + ).Scan(&displayName, &deletedAt) + require.NoError(t, err) + require.True(t, displayName.Valid) + assert.Equal(t, "Renamed before import", displayName.String) + require.True(t, deletedAt.Valid) + assert.Equal(t, "2026-06-14T01:02:03.000Z", deletedAt.String) + assert.Equal(t, 2, metadataAppliedCount(t, importDB, metadataOrigin)) +} + +func TestImportSkipsUnknownMetadataOpAndContinues(t *testing.T) { + ctx := context.Background() + origin := "laptop-a1b2c3" + localOrigin := "desktop-d4e5f6" + root := t.TempDir() + gid := origin + "~sess-1" + exportDB := testDB(t) + importDB := testDB(t) + seedSession(t, exportDB, "sess-1", "alpha") + _, err := Export(ctx, exportDB, root, origin) + require.NoError(t, err) + + unknown := metadataEvent{ + Version: formatVersion, + HLC: replayTestHLC(0, 0), + Origin: origin, + SessionGID: gid, + Op: "future_tag", + Value: json.RawMessage(`{"tag":"later"}`), + } + writeMetadataArtifact(t, filepath.Join(root, origin), unknown) + writeMetadataArtifact(t, filepath.Join(root, origin), + replayRenameEvent(t, origin, gid, replayTestHLC(time.Nanosecond, 0), "Known winner")) + + _, _, err = Import(ctx, importDB, root, localOrigin) + require.NoError(t, err) + + got, err := importDB.GetSession(ctx, gid) + require.NoError(t, err) + require.NotNil(t, got) + require.NotNil(t, got.DisplayName) + assert.Equal(t, "Known winner", *got.DisplayName) + assert.Equal(t, 2, metadataAppliedCount(t, importDB, origin)) +} + +// A known op with a malformed payload (here: pin without its pin +// payload) is immutable and can never become applicable, so replay +// must quarantine it as applied instead of aborting the origin. +func TestImportQuarantinesMalformedKnownOpAndContinues(t *testing.T) { + ctx := context.Background() + origin := "laptop-a1b2c3" + localOrigin := "desktop-d4e5f6" + root := t.TempDir() + gid := origin + "~sess-1" + exportDB := testDB(t) + importDB := testDB(t) + seedSession(t, exportDB, "sess-1", "alpha") + _, err := Export(ctx, exportDB, root, origin) + require.NoError(t, err) + + malformed := metadataEvent{ + Version: formatVersion, + HLC: replayTestHLC(0, 0), + Origin: origin, + SessionGID: gid, + Op: MetadataOpPin, + } + writeMetadataArtifact(t, filepath.Join(root, origin), malformed) + writeMetadataArtifact(t, filepath.Join(root, origin), + replayRenameEvent(t, origin, gid, replayTestHLC(time.Nanosecond, 0), "Known winner")) + + _, _, err = Import(ctx, importDB, root, localOrigin) + require.NoError(t, err) + + got, err := importDB.GetSession(ctx, gid) + require.NoError(t, err) + require.NotNil(t, got) + require.NotNil(t, got.DisplayName) + assert.Equal(t, "Known winner", *got.DisplayName) + // Both events are marked applied so replay never retries the + // malformed one. + assert.Equal(t, 2, metadataAppliedCount(t, importDB, origin)) + + // A later sync neither errors nor reprocesses the malformed event. + _, _, err = Import(ctx, importDB, root, localOrigin) + require.NoError(t, err) + assert.Equal(t, 2, metadataAppliedCount(t, importDB, origin)) +} + +func TestImportSkipsAndQuarantinesCorruptMetadataEvent(t *testing.T) { + ctx := context.Background() + origin := "laptop-a1b2c3" + localOrigin := "desktop-d4e5f6" + root := t.TempDir() + gid := origin + "~sess-1" + exportDB := testDB(t) + importDB := testDB(t) + seedSession(t, exportDB, "sess-1", "alpha") + _, err := Export(ctx, exportDB, root, origin) + require.NoError(t, err) + + valid := writeMetadataArtifact(t, filepath.Join(root, origin), + replayRenameEvent(t, origin, gid, replayTestHLC(0, 0), "Winner")) + // A corrupt event: well-formed name whose hash no longer matches the + // file bytes, as bit rot or an interrupted file-sync write produces. + corrupt := replayRenameEvent(t, origin, gid, replayTestHLC(time.Nanosecond, 0), "PWNED") + data, err := canonicalJSON(corrupt) + require.NoError(t, err) + stamp, err := ParseHLCTimestamp(corrupt.HLC) + require.NoError(t, err) + corruptPath := filepath.Join(root, origin, "meta", + stamp.OrderingKey(hashHex([]byte("other")))+metadataEventExtension) + require.NoError(t, writeFileAtomic(corruptPath, data, 0o644)) + + imported, messages, err := Import(ctx, importDB, root, localOrigin) + require.NoError(t, err) + assert.Equal(t, 1, imported) + assert.Equal(t, 2, messages) + + got, err := importDB.GetSession(ctx, gid) + require.NoError(t, err) + require.NotNil(t, got) + require.NotNil(t, got.DisplayName) + assert.Equal(t, "Winner", *got.DisplayName) + assert.Equal(t, 1, metadataAppliedCount(t, importDB, origin)) + + assert.NoFileExists(t, corruptPath) + assert.FileExists(t, corruptPath+quarantineSuffix) + assert.FileExists(t, valid.path) + + // A later sync neither errors nor reprocesses the corrupt event. + _, _, err = Import(ctx, importDB, root, localOrigin) + require.NoError(t, err) + assert.Equal(t, 1, metadataAppliedCount(t, importDB, origin)) +} + +func TestImportSkipsUnparseableHLCMetadataEvent(t *testing.T) { + ctx := context.Background() + origin := "laptop-a1b2c3" + localOrigin := "desktop-d4e5f6" + root := t.TempDir() + gid := origin + "~sess-1" + exportDB := testDB(t) + importDB := testDB(t) + seedSession(t, exportDB, "sess-1", "alpha") + _, err := Export(ctx, exportDB, root, origin) + require.NoError(t, err) + + // A forged event whose HLC does not parse but whose filename hash is + // valid. Its order key sorts lexicographically above every real + // timestamp, so if accepted it would win LWW permanently (#1034). + forgedHLC := "9999-99-99T999999.000000000Z-00000000000000000000" + forged := replayRenameEvent(t, origin, gid, forgedHLC, "PWNED") + data, err := canonicalJSON(forged) + require.NoError(t, err) + forgedPath := filepath.Join(root, origin, "meta", + forgedHLC+"-"+hashHex(data)+metadataEventExtension) + require.NoError(t, writeFileAtomic(forgedPath, data, 0o644)) + writeMetadataArtifact(t, filepath.Join(root, origin), + replayRenameEvent(t, origin, gid, replayTestHLC(0, 0), "Legit")) + + imported, messages, err := Import(ctx, importDB, root, localOrigin) + require.NoError(t, err) + assert.Equal(t, 1, imported) + assert.Equal(t, 2, messages) + + got, err := importDB.GetSession(ctx, gid) + require.NoError(t, err) + require.NotNil(t, got) + require.NotNil(t, got.DisplayName) + assert.Equal(t, "Legit", *got.DisplayName) + // Both events are marked applied so replay never retries the forged one. + assert.Equal(t, 2, metadataAppliedCount(t, importDB, origin)) + + // Repeat imports stay clean and never resurrect the forged value. + _, _, err = Import(ctx, importDB, root, localOrigin) + require.NoError(t, err) + got, err = importDB.GetSession(ctx, gid) + require.NoError(t, err) + require.NotNil(t, got) + require.NotNil(t, got.DisplayName) + assert.Equal(t, "Legit", *got.DisplayName) + assert.Equal(t, 2, metadataAppliedCount(t, importDB, origin)) +} + +func TestImportSoftDeleteUsesEventHLCForDeletedAt(t *testing.T) { + ctx := context.Background() + origin := "laptop-a1b2c3" + localOrigin := "desktop-d4e5f6" + root := t.TempDir() + gid := origin + "~sess-1" + exportDB := testDB(t) + importDB := testDB(t) + seedSession(t, exportDB, "sess-1", "alpha") + _, err := Export(ctx, exportDB, root, origin) + require.NoError(t, err) + + writeMetadataArtifact(t, filepath.Join(root, origin), metadataEvent{ + Version: formatVersion, + HLC: replayTestHLC(0, 0), + Origin: origin, + SessionGID: gid, + Op: MetadataOpSoftDelete, + }) + + _, _, err = Import(ctx, importDB, root, localOrigin) + require.NoError(t, err) + + var deletedAt sql.NullString + err = importDB.Reader().QueryRowContext(ctx, + `SELECT deleted_at FROM sessions WHERE id = ?`, gid).Scan(&deletedAt) + require.NoError(t, err) + require.True(t, deletedAt.Valid) + // Trash retention must be anchored to the author-side event time, not the + // moment each machine happened to import the event, or retention windows + // drift across the fleet. + assert.Equal(t, "2026-06-14T01:02:03.000Z", deletedAt.String) +} + +func TestWriteArtifactRejectsUnparseableHLCMetadataEvent(t *testing.T) { + root := t.TempDir() + origin := "laptop-a1b2c3" + gid := origin + "~sess-1" + forgedHLC := "9999-99-99T999999.000000000Z-00000000000000000000" + forged := replayRenameEvent(t, origin, gid, forgedHLC, "PWNED") + data, err := canonicalJSON(forged) + require.NoError(t, err) + name := forgedHLC + "-" + hashHex(data) + metadataEventExtension + + _, err = WriteArtifact(root, origin, KindMeta, name, data) + require.Error(t, err) + assert.ErrorIs(t, err, ErrArtifactInvalid) +} + +func TestImportDefersFutureVersionMetadataEventAndContinues(t *testing.T) { + ctx := context.Background() + origin := "laptop-a1b2c3" + localOrigin := "desktop-d4e5f6" + root := t.TempDir() + gid := origin + "~sess-1" + exportDB := testDB(t) + importDB := testDB(t) + seedSession(t, exportDB, "sess-1", "alpha") + _, err := Export(ctx, exportDB, root, origin) + require.NoError(t, err) + + future := metadataEvent{ + Version: formatVersion + 1, + HLC: replayTestHLC(0, 0), + Origin: origin, + SessionGID: gid, + Op: "future_tag", + Value: json.RawMessage(`{"tag":"later"}`), + } + writeMetadataArtifact(t, filepath.Join(root, origin), future) + writeMetadataArtifact(t, filepath.Join(root, origin), + replayRenameEvent(t, origin, gid, replayTestHLC(time.Nanosecond, 0), "Known winner")) + + _, _, err = Import(ctx, importDB, root, localOrigin) + require.NoError(t, err) + + got, err := importDB.GetSession(ctx, gid) + require.NoError(t, err) + require.NotNil(t, got) + require.NotNil(t, got.DisplayName) + assert.Equal(t, "Known winner", *got.DisplayName) + assert.Equal(t, 1, metadataAppliedCount(t, importDB, origin)) +} + +func TestImportObservesFutureVersionMetadataHLCBeforeDeferring(t *testing.T) { + ctx := context.Background() + origin := "laptop-a1b2c3" + localOrigin := "desktop-d4e5f6" + root := t.TempDir() + importDB := testDB(t) + now := fixedHLCTime() + remote := HLCTimestamp{WallTime: now.Add(time.Minute)} + event := metadataEvent{ + Version: formatVersion + 1, + HLC: remote.String(), + Origin: origin, + SessionGID: origin + "~sess-1", + Op: "future_tag", + Value: json.RawMessage(`{"tag":"later"}`), + } + writeMetadataArtifact(t, filepath.Join(root, origin), event) + + clock := NewHLCClock(importDB, HLCClockOptions{ + Now: func() time.Time { return now }, + }) + _, err := importDetailed(ctx, importDB, clock, root, localOrigin) + require.NoError(t, err) + + next, err := clock.Next() + require.NoError(t, err) + assert.Positive(t, next.Compare(remote), + "a local edit after deferring a future-version event must sort after that peer event") + assert.Equal(t, 0, metadataAppliedCount(t, importDB, origin)) +} + +func TestImportAlreadyAppliedMetadataDoesNotAdvanceResetHLC(t *testing.T) { + ctx := context.Background() + origin := "laptop-a1b2c3" + localOrigin := "desktop-d4e5f6" + root := t.TempDir() + exportDB := testDB(t) + importDB := testDB(t) + seedSession(t, exportDB, "sess-1", "alpha") + _, err := Export(ctx, exportDB, root, origin) + require.NoError(t, err) + + now := fixedHLCTime() + remote := HLCTimestamp{WallTime: now.Add(time.Minute)} + writeMetadataArtifact(t, filepath.Join(root, origin), + replayRenameEvent(t, origin, origin+"~sess-1", remote.String(), "Renamed")) + clock := NewHLCClock(importDB, HLCClockOptions{Now: func() time.Time { return now }}) + _, err = importDetailed(ctx, importDB, clock, root, localOrigin) + require.NoError(t, err) + assert.Equal(t, 1, metadataAppliedCount(t, importDB, origin)) + + older := HLCTimestamp{WallTime: now.Add(-time.Minute)}.String() + require.NoError(t, importDB.SetSyncState(metadataHLCStateKey, older)) + clock = NewHLCClock(importDB, HLCClockOptions{Now: func() time.Time { return now }}) + _, err = importDetailed(ctx, importDB, clock, root, localOrigin) + require.NoError(t, err) + + persisted, err := importDB.GetSyncState(metadataHLCStateKey) + require.NoError(t, err) + assert.Equal(t, older, persisted, + "an already-applied event must be skipped before its HLC is observed") +} + +func TestImportSkipsAlreadyAppliedMetadataBeforeReadingArtifact(t *testing.T) { + ctx := context.Background() + origin := "laptop-a1b2c3" + localOrigin := "desktop-d4e5f6" + root := t.TempDir() + exportDB := testDB(t) + importDB := testDB(t) + seedSession(t, exportDB, "sess-1", "alpha") + _, err := Export(ctx, exportDB, root, origin) + require.NoError(t, err) + art := writeMetadataArtifact(t, filepath.Join(root, origin), + replayRenameEvent(t, origin, origin+"~sess-1", replayTestHLC(0, 0), "Renamed")) + _, _, err = Import(ctx, importDB, root, localOrigin) + require.NoError(t, err) + assert.Equal(t, 1, metadataAppliedCount(t, importDB, origin)) + + require.NoError(t, os.WriteFile(art.path, []byte("corrupt after apply"), 0o644)) + _, _, err = Import(ctx, importDB, root, localOrigin) + require.NoError(t, err) + + assert.FileExists(t, art.path, + "applied identity should be skipped before reading the artifact bytes") + assert.NoFileExists(t, art.path+quarantineSuffix) +} + +func TestImportDoesNotAdvanceMetadataWatermarkWhenTargetMissing(t *testing.T) { + ctx := context.Background() + origin := "laptop-a1b2c3" + localOrigin := "desktop-d4e5f6" + root := t.TempDir() + originRoot := filepath.Join(root, origin) + importDB := testDB(t) + writeMetadataArtifact(t, originRoot, + replayRenameEvent(t, origin, localOrigin+"~sess-1", replayTestHLC(0, 0), "Remote rename")) + + imported, messages, err := Import(ctx, importDB, root, localOrigin) + require.NoError(t, err) + assert.Zero(t, imported) + assert.Zero(t, messages) + assert.Equal(t, 0, metadataAppliedCount(t, importDB, origin)) + + seedSession(t, importDB, "sess-1", "alpha") + imported, messages, err = Import(ctx, importDB, root, localOrigin) + require.NoError(t, err) + assert.Zero(t, imported) + assert.Zero(t, messages) + + got, err := importDB.GetSession(ctx, "sess-1") + require.NoError(t, err) + require.NotNil(t, got) + require.NotNil(t, got.DisplayName) + assert.Equal(t, "Remote rename", *got.DisplayName) + assert.Equal(t, 1, metadataAppliedCount(t, importDB, origin)) +} + +func TestImportReplaysMetadataPinAndUnpin(t *testing.T) { + ctx := context.Background() + origin := "laptop-a1b2c3" + localOrigin := "desktop-d4e5f6" + root := t.TempDir() + gid := origin + "~sess-1" + exportDB := testDB(t) + importDB := testDB(t) + seedSession(t, exportDB, "sess-1", "alpha") + require.NoError(t, exportDB.ReplaceSessionMessages("sess-1", []db.Message{ + { + SessionID: "sess-1", + Ordinal: 0, + Role: "user", + Content: "hello", + ContentLength: 5, + SourceUUID: "uuid-question", + }, + { + SessionID: "sess-1", + Ordinal: 1, + Role: "assistant", + Content: "world", + ContentLength: 5, + SourceUUID: "uuid-answer", + }, + })) + _, err := Export(ctx, exportDB, root, origin) + require.NoError(t, err) + + note := "remember" + writeMetadataArtifact(t, filepath.Join(root, origin), metadataEvent{ + Version: formatVersion, + HLC: replayTestHLC(0, 0), + Origin: origin, + SessionGID: gid, + Op: MetadataOpPin, + Pin: &MetadataPin{ + SourceUUID: "uuid-answer", + Ordinal: 1, + Note: ¬e, + }, + }) + _, _, err = Import(ctx, importDB, root, localOrigin) + require.NoError(t, err) + pins, err := importDB.ListPinnedMessages(ctx, gid, "") + require.NoError(t, err) + require.Len(t, pins, 1) + assert.Equal(t, 1, pins[0].Ordinal) + require.NotNil(t, pins[0].Note) + assert.Equal(t, note, *pins[0].Note) + + writeMetadataArtifact(t, filepath.Join(root, origin), metadataEvent{ + Version: formatVersion, + HLC: replayTestHLC(time.Nanosecond, 0), + Origin: origin, + SessionGID: gid, + Op: MetadataOpUnpin, + Pin: &MetadataPin{ + SourceUUID: "uuid-answer", + Ordinal: 1, + }, + }) + _, _, err = Import(ctx, importDB, root, localOrigin) + require.NoError(t, err) + pins, err = importDB.ListPinnedMessages(ctx, gid, "") + require.NoError(t, err) + assert.Empty(t, pins) + assert.Equal(t, 2, metadataAppliedCount(t, importDB, origin)) +} + +func TestLocalMetadataEditBeatsLowerHLCPeerEvent(t *testing.T) { + ctx := context.Background() + root := t.TempDir() + dataDir := t.TempDir() + localOrigin := "desktop-d4e5f6" + peerOrigin := "laptop-a1b2c3" + gid := localOrigin + "~sess-1" + + database := testDB(t) + seedSession(t, database, "sess-1", "alpha") + // Simulate the local rename handler: mutate the session, then record the + // metadata event artifact and replay register entry. + require.NoError(t, database.RenameSession("sess-1", new("Local name"))) + + now := fixedHLCTime().Add(time.Hour) + recorder := NewMetadataRecorder(database, MetadataRecorderOptions{ + DataDir: dataDir, + Origin: localOrigin, + Now: func() time.Time { return now }, + }) + value, err := json.Marshal(struct { + DisplayName string `json:"display_name"` + }{DisplayName: "Local name"}) + require.NoError(t, err) + localRec, err := recorder.Append(ctx, MetadataEventInput{ + SessionID: "sess-1", + Op: MetadataOpRename, + Value: value, + }) + require.NoError(t, err) + localOrderKey := localRec.HLC + "-" + localRec.Hash + + // A peer renames the same desktop-originated session at an earlier HLC. + peerArt := writeMetadataArtifact(t, filepath.Join(root, peerOrigin), + replayRenameEvent(t, peerOrigin, gid, replayTestHLC(0, 0), "Peer name")) + require.Less(t, peerArt.orderKey, localOrderKey) + + res, err := ImportDetailed(ctx, database, root, localOrigin) + require.NoError(t, err) + assert.Equal(t, 1, res.Metadata) + + got, err := database.GetSession(ctx, "sess-1") + require.NoError(t, err) + require.NotNil(t, got.DisplayName) + assert.Equal(t, "Local name", *got.DisplayName) + assertMetadataConflict(t, database, gid, "display_name", localOrderKey, peerArt.orderKey) +} + +func TestImportContinuesPastUnavailableMetadataTarget(t *testing.T) { + ctx := context.Background() + root := t.TempDir() + localOrigin := "desktop-d4e5f6" + peerOrigin := "laptop-a1b2c3" + database := testDB(t) + seedSession(t, database, "sess-present", "alpha") + missingGID := localOrigin + "~sess-missing" + presentGID := localOrigin + "~sess-present" + + // The earlier event targets a session that is not durable locally; the + // later event targets an existing session and must still apply. + writeMetadataArtifact(t, filepath.Join(root, peerOrigin), + replayRenameEvent(t, peerOrigin, missingGID, replayTestHLC(0, 0), "Missing target")) + writeMetadataArtifact(t, filepath.Join(root, peerOrigin), + replayRenameEvent(t, peerOrigin, presentGID, replayTestHLC(time.Nanosecond, 0), "Present target")) + + res, err := ImportDetailed(ctx, database, root, localOrigin) + require.NoError(t, err) + assert.Equal(t, 1, res.Metadata) + + got, err := database.GetSession(ctx, "sess-present") + require.NoError(t, err) + require.NotNil(t, got.DisplayName) + assert.Equal(t, "Present target", *got.DisplayName) + // Only the present-target event is marked applied; the unavailable one is + // left for a later run to retry. + assert.Equal(t, 1, metadataAppliedCount(t, database, peerOrigin)) + + // Once the missing session exists, a later import applies its event too. + seedSession(t, database, "sess-missing", "beta") + res, err = ImportDetailed(ctx, database, root, localOrigin) + require.NoError(t, err) + assert.Equal(t, 1, res.Metadata) + got, err = database.GetSession(ctx, "sess-missing") + require.NoError(t, err) + require.NotNil(t, got.DisplayName) + assert.Equal(t, "Missing target", *got.DisplayName) + assert.Equal(t, 2, metadataAppliedCount(t, database, peerOrigin)) +} + +func TestReplayDefersRemoteEventBeyondClockDrift(t *testing.T) { + ctx := context.Background() + root := t.TempDir() + dataDir := t.TempDir() + localOrigin := "desktop-d4e5f6" + peerOrigin := "laptop-a1b2c3" + gid := localOrigin + "~sess-1" + + database := testDB(t) + seedSession(t, database, "sess-1", "alpha") + + now := fixedHLCTime() + // A peer event whose wall time is an hour ahead — well beyond the default + // 5-minute drift bound, so the local clock cannot be advanced past it. + future := HLCTimestamp{WallTime: now.Add(time.Hour)} + writeMetadataArtifact(t, filepath.Join(root, peerOrigin), + replayRenameEvent(t, peerOrigin, gid, future.String(), "From the future")) + + recorder := NewMetadataRecorder(database, MetadataRecorderOptions{ + DataDir: dataDir, + Origin: localOrigin, + Now: func() time.Time { return now }, + }) + res, err := recorder.Import(ctx, root) + require.NoError(t, err) + assert.Zero(t, res.Metadata, "event beyond the drift bound must be deferred, not applied") + + // The local session is untouched and the event is left unapplied for a later + // run to retry once wall time catches up. + got, err := database.GetSession(ctx, "sess-1") + require.NoError(t, err) + require.NotNil(t, got) + require.NotNil(t, got.DisplayName) + assert.NotEqual(t, "From the future", *got.DisplayName, "deferred rename must not be applied") + assert.Equal(t, 0, metadataAppliedCount(t, database, peerOrigin)) +} + +func replayRenameEvent( + t *testing.T, + origin, gid, hlc, displayName string, +) metadataEvent { + t.Helper() + value, err := json.Marshal(struct { + DisplayName string `json:"display_name"` + }{ + DisplayName: displayName, + }) + require.NoError(t, err) + return metadataEvent{ + Version: formatVersion, + HLC: hlc, + Origin: origin, + SessionGID: gid, + Op: MetadataOpRename, + Value: value, + } +} + +func replayTestHLC(offset time.Duration, logical uint64) string { + return HLCTimestamp{WallTime: fixedHLCTime().Add(offset), Logical: logical}.String() +} + +func writeMetadataArtifact(t *testing.T, originRoot string, event metadataEvent) metadataArtifact { + t.Helper() + stamp, err := ParseHLCTimestamp(event.HLC) + require.NoError(t, err) + data, err := canonicalJSON(event) + require.NoError(t, err) + hash := hashHex(data) + path := filepath.Join(originRoot, "meta", stamp.OrderingKey(hash)+metadataEventExtension) + require.NoError(t, writeFileAtomic(path, data, 0o644)) + art, err := readMetadataArtifact(path) + require.NoError(t, err) + return art +} + +func assertMetadataConflict( + t *testing.T, + database *db.DB, + gid, field, wantWinning, wantLosing string, +) { + t.Helper() + var winning, losing string + err := database.Reader().QueryRowContext(context.Background(), + `SELECT winning_order_key, losing_order_key + FROM metadata_conflicts + WHERE session_gid = ? AND field = ?`, + gid, field, + ).Scan(&winning, &losing) + require.NoError(t, err) + assert.Equal(t, wantWinning, winning) + assert.Equal(t, wantLosing, losing) +} + +func assertMetadataConflictCount(t *testing.T, database *db.DB, gid, field string, want int) { + t.Helper() + var got int + err := database.Reader().QueryRowContext(context.Background(), + `SELECT COUNT(*) + FROM metadata_conflicts + WHERE session_gid = ? AND field = ?`, + gid, field, + ).Scan(&got) + require.NoError(t, err) + assert.Equal(t, want, got) +} + +func metadataAppliedCount(t *testing.T, database *db.DB, origin string) int { + t.Helper() + var count int + err := database.Reader().QueryRowContext(context.Background(), + `SELECT COUNT(*) FROM metadata_applied_events WHERE origin = ?`, + origin, + ).Scan(&count) + require.NoError(t, err) + return count +} diff --git a/internal/artifact/sync.go b/internal/artifact/sync.go new file mode 100644 index 000000000..ddaa7b6bf --- /dev/null +++ b/internal/artifact/sync.go @@ -0,0 +1,3041 @@ +package artifact + +import ( + "bytes" + "context" + "crypto/rand" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "io/fs" + "log" + "maps" + "net" + "os" + "path/filepath" + "reflect" + "slices" + "sort" + "strconv" + "strings" + "syscall" + "time" + + "github.com/klauspost/compress/zstd" + "go.kenn.io/agentsview/internal/config" + "go.kenn.io/agentsview/internal/db" +) + +const ( + formatVersion = 1 + originStateKey = "artifact_origin_id" + importStatePrefix = "artifact_import:" + exportStatePrefix = "artifact_export:" + tempFilePrefix = ".tmp-" + manifestExtension = ".json.zst" + segmentExtension = ".ndjson.zst" + + manifestDecodedLimit = int64(16 << 20) + segmentDecodedLimit = int64(64 << 20) + segmentTargetSize = int64(32 << 20) + // zstd.NewWriter documents an 8 MiB maximum default window. Matching it + // keeps existing package-written artifacts readable without accepting + // attacker-selected large decoder windows. + zstdMaxWindowSize = uint64(8 << 20) + + // Cardinality caps complement the byte caps: 4,096 records keeps one + // segment's decoded object graph bounded, while 32,768 records and 256 MiB + // leave ample room for unusually long sessions without letting many valid + // chunks amplify during aggregation. Sixteen references accommodate uneven + // 32 MiB chunks; the aggregate byte cap remains the final session bound. + maxManifestSegments = 16 + maxManifestUsageEvents = 32_768 + maxSegmentMessages = 4_096 + maxSessionMessages = 32_768 + maxSessionDecodedBytes = int64(256 << 20) + + // Nested collections need independent caps because compact empty objects can + // amplify far beyond the decoded byte budget when unmarshaled. A message may + // still describe unusually wide tool fan-out, and one tool may retain a long + // result history. Segment totals keep one decoded chunk modest; session totals + // allow eight full nested-budget segments, matching the message-count ratio. + maxMessageToolCalls = 256 + maxToolResultEvents = 1_024 + maxSegmentToolCalls = 8_192 + maxSegmentResultEvents = 32_768 + maxSessionToolCalls = 65_536 + maxSessionResultEvents = 262_144 +) + +// artifactLimits bounds decoded collection cardinality in addition to raw +// bytes. The production values are intentionally generous for real sessions +// while preventing small JSON records from amplifying into unbounded Go +// object graphs. +type artifactLimits struct { + manifestSegments int + manifestUsageEvents int + segmentMessages int + sessionMessages int + sessionDecodedBytes int64 + messageToolCalls int + toolResultEvents int + segmentToolCalls int + segmentResultEvents int + sessionToolCalls int + sessionResultEvents int +} + +func productionArtifactLimits() artifactLimits { + return artifactLimits{ + manifestSegments: maxManifestSegments, + manifestUsageEvents: maxManifestUsageEvents, + segmentMessages: maxSegmentMessages, + sessionMessages: maxSessionMessages, + sessionDecodedBytes: maxSessionDecodedBytes, + messageToolCalls: maxMessageToolCalls, + toolResultEvents: maxToolResultEvents, + segmentToolCalls: maxSegmentToolCalls, + segmentResultEvents: maxSegmentResultEvents, + sessionToolCalls: maxSessionToolCalls, + sessionResultEvents: maxSessionResultEvents, + } +} + +type nestedCollectionCounts struct { + toolCalls int + resultEvents int +} + +type segmentPreflight struct { + records [][]byte + nested nestedCollectionCounts +} + +func exceedsCollectionLimit(current, additional, limit int) bool { + return current > limit || additional > limit-current +} + +var errIncompleteArtifact = errors.New("incomplete artifact") + +// errCorruptArtifact reports an artifact whose bytes fail content validation: +// a hash mismatch or an undecodable compressed stream. Corrupt artifacts are +// quarantined and skipped so one bad file cannot abort sync permanently. +var errCorruptArtifact = errors.New("corrupt artifact") + +// quarantineSuffix is appended to a corrupt artifact's filename when it is +// quarantined. Quarantined files are ignored by every read path and are never +// mirrored by CopyUnion, so a re-fetched valid copy can take the original name. +const quarantineSuffix = ".corrupt" + +var errFutureArtifactVersion = errors.New("future artifact version") + +var writeFileAtomicBeforeCommit func(path string) +var writeFileAtomicLink = os.Link + +// SyncOptions configures a local-first artifact folder sync. +type SyncOptions struct { + DataDir string + Target string + Origin string + // Now is the wall-clock source for advancing the metadata HLC past + // observed remote events. When nil, time.Now is used. Sharing it with the + // local metadata recorder keeps import and local edits on one time base. + Now func() time.Time + // Token is the Bearer token for an HTTP peer target. It is ignored by + // folder and object-store targets. + Token string + // AllowInsecure permits plaintext HTTP to a non-loopback peer. Loopback + // HTTP remains allowed without this override. + AllowInsecure bool + // BaselineMetadata writes metadata events for existing local curation before + // exchanging artifacts. It is intended for first-time initialization. + BaselineMetadata bool + // OnDataChanged is called after a foreign import writes local rows. + OnDataChanged func() +} + +// Sync runs one artifact sync, selecting the transport from the target shape: +// an http(s):// URL uses the HTTP peer transport, anything else is treated as a +// local folder target. +func Sync(ctx context.Context, database *db.DB, opts SyncOptions) (SyncResult, error) { + if opts.Target == "" { + return SyncResult{}, errors.New("artifact sync target is required") + } + if IsHTTPTarget(opts.Target) { + tr, err := newHTTPTransport(opts.Target, opts.Token, opts.AllowInsecure) + if err != nil { + return SyncResult{}, err + } + return syncWithTransport(ctx, database, opts, tr) + } + if IsObjectTarget(opts.Target) { + tr, err := newObjectTransport(opts.Target, ObjectStoreOptionsFromEnv()) + if err != nil { + return SyncResult{}, err + } + return syncWithTransport(ctx, database, opts, tr) + } + return syncWithTransport(ctx, database, opts, &folderTransport{target: opts.Target}) +} + +// SyncResult summarizes a folder artifact sync run. +type SyncResult struct { + Origin string + ExportedSessions int + ImportedSessions int + ImportedMessages int + ImportedMetadata int +} + +// ImportResult summarizes local rows changed by artifact import. +type ImportResult struct { + Sessions int + Messages int + Metadata int + Deferred int +} + +// Changed reports whether the import wrote user-visible local data. +func (r ImportResult) Changed() bool { + return r.Sessions > 0 || r.Messages > 0 || r.Metadata > 0 +} + +// SyncFolder exports local sessions to the local artifact store, exchanges the +// store with target, and imports foreign origins from the exchanged artifacts. +func SyncFolder(ctx context.Context, database *db.DB, opts SyncOptions) (SyncResult, error) { + if opts.Target == "" { + return SyncResult{}, errors.New("artifact sync target is required") + } + return syncWithTransport(ctx, database, opts, &folderTransport{target: opts.Target}) +} + +// syncWithTransport runs one artifact sync over any transport: export local +// sessions, exchange the store with the remote via set-union, then import +// foreign origins. Folder, HTTP peer, and object-store targets differ only in +// the transport's Prepare and Exchange. +func syncWithTransport( + ctx context.Context, + database *db.DB, + opts SyncOptions, + tr Transport, +) (SyncResult, error) { + if opts.DataDir == "" { + return SyncResult{}, errors.New("artifact sync data dir is required") + } + localRoot := filepath.Join(opts.DataDir, "artifacts") + if err := tr.Prepare(ctx, localRoot); err != nil { + return SyncResult{}, err + } + if err := os.MkdirAll(localRoot, 0o755); err != nil { + return SyncResult{}, fmt.Errorf("creating local artifact store: %w", err) + } + origin := opts.Origin + if origin == "" { + var err error + origin, err = EnsureOrigin(database) + if err != nil { + return SyncResult{}, err + } + } else if err := validateOriginID(origin); err != nil { + return SyncResult{}, err + } + + clock := NewHLCClock(database, HLCClockOptions{Now: opts.Now}) + var imported ImportResult + var baselineSnapshot db.MetadataBaselineSnapshot + if opts.BaselineMetadata { + var err error + baselineSnapshot, err = database.MetadataBaselineSnapshot(ctx) + if err != nil { + return SyncResult{}, err + } + if err := tr.Exchange(ctx, localRoot); err != nil { + return SyncResult{}, err + } + preBaselineImported, err := importDetailed(ctx, database, clock, localRoot, origin) + if err != nil { + return SyncResult{}, err + } + imported.Sessions += preBaselineImported.Sessions + imported.Messages += preBaselineImported.Messages + imported.Metadata += preBaselineImported.Metadata + + recorder := NewMetadataRecorder(database, MetadataRecorderOptions{ + DataDir: opts.DataDir, + Origin: origin, + Now: opts.Now, + }) + if _, err := recorder.AppendBaselineSnapshot(ctx, baselineSnapshot); err != nil { + return SyncResult{}, err + } + } + exported, err := Export(ctx, database, localRoot, origin) + if err != nil { + return SyncResult{}, err + } + if err := tr.Exchange(ctx, localRoot); err != nil { + return SyncResult{}, err + } + postExportImported, err := importDetailed(ctx, database, clock, localRoot, origin) + if err != nil { + return SyncResult{}, err + } + imported.Sessions += postExportImported.Sessions + imported.Messages += postExportImported.Messages + imported.Metadata += postExportImported.Metadata + if imported.Changed() && opts.OnDataChanged != nil { + opts.OnDataChanged() + } + return SyncResult{ + Origin: origin, + ExportedSessions: exported, + ImportedSessions: imported.Sessions, + ImportedMessages: imported.Messages, + ImportedMetadata: imported.Metadata, + }, nil +} + +// EnsureOrigin returns the persisted origin ID, creating one when absent. +func EnsureOrigin(database *db.DB) (string, error) { + origin, err := StoredOrigin(database) + if err != nil { + return "", err + } + if origin != "" { + return origin, nil + } + origin, err = newOriginID() + if err != nil { + return "", err + } + if err := validateOriginID(origin); err != nil { + return "", fmt.Errorf("generated artifact origin: %w", err) + } + if err := database.SetSyncState(originStateKey, origin); err != nil { + return "", fmt.Errorf("persisting artifact origin: %w", err) + } + return origin, nil +} + +// AdoptOrigin persists origin as this machine's artifact origin in the database +// sync state so DB-derived lookups (EnsureOrigin and its callers) agree with the +// authoritative config origin. It validates the input and is idempotent: it only +// writes when the stored value differs. The config origin always wins, so a +// previously stored value is overwritten to converge on a single origin. +func AdoptOrigin(database *db.DB, origin string) error { + if err := validateOriginID(origin); err != nil { + return fmt.Errorf("adopting artifact origin: %w", err) + } + existing, err := StoredOrigin(database) + if err != nil { + return err + } + if existing == origin { + return nil + } + if err := database.SetSyncState(originStateKey, origin); err != nil { + return fmt.Errorf("persisting artifact origin: %w", err) + } + return nil +} + +// StoredOrigin returns the persisted origin ID without creating one. +func StoredOrigin(database *db.DB) (string, error) { + origin, err := database.GetSyncState(originStateKey) + if err != nil { + return "", fmt.Errorf("reading artifact origin: %w", err) + } + if origin != "" { + if err := validateOriginID(origin); err != nil { + return "", fmt.Errorf("stored artifact origin: %w", err) + } + return origin, nil + } + return "", nil +} + +type syncStateValueReader interface { + SyncStateValues(keys []string) (map[string]string, error) +} + +// ImportedSessionIDs returns the candidate session IDs with durable artifact +// import provenance. A foreign machine~id shape is shared by other import +// mechanisms, so callers must query the exact provenance keys rather than +// infer artifact ownership from the session row or scan all historical imports. +func ImportedSessionIDs( + database syncStateValueReader, candidateIDs []string, +) (map[string]struct{}, error) { + ids := make(map[string]struct{}) + if len(candidateIDs) == 0 { + return ids, nil + } + keys := make([]string, 0, len(candidateIDs)) + keyToID := make(map[string]string, len(candidateIDs)) + for _, gid := range candidateIDs { + origin, nativeID, ok := strings.Cut(gid, "~") + if !ok || origin == "" || nativeID == "" { + continue + } + key := importStateKey(origin, gid) + keys = append(keys, key) + keyToID[key] = gid + } + if len(keys) == 0 { + return ids, nil + } + states, err := database.SyncStateValues(keys) + if err != nil { + return nil, fmt.Errorf("reading artifact import provenance: %w", err) + } + for key := range states { + if gid, ok := keyToID[key]; ok { + ids[gid] = struct{}{} + } + } + return ids, nil +} + +// CountImportedCheckpointSessions returns how many sessions in one checkpoint +// have landed at the exact manifest version it publishes. Import provenance is +// independent of the session row's current lifecycle state, so a locally +// trashed session remains landed while a stale active row does not. +func CountImportedCheckpointSessions( + database syncStateValueReader, origin string, sessionManifests map[string]string, +) (int, error) { + if len(sessionManifests) == 0 { + return 0, nil + } + keys := make([]string, 0, len(sessionManifests)) + expectedByKey := make(map[string]string, len(sessionManifests)) + for gid, manifestHash := range sessionManifests { + key := importStateKey(origin, gid) + keys = append(keys, key) + expectedByKey[key] = manifestHash + } + states, err := database.SyncStateValues(keys) + if err != nil { + return 0, fmt.Errorf("reading checkpoint import provenance: %w", err) + } + landed := 0 + for key, expectedHash := range expectedByKey { + if states[key] == expectedHash { + landed++ + } + } + return landed, nil +} + +func newOriginID() (string, error) { + host, err := os.Hostname() + if err != nil || strings.TrimSpace(host) == "" { + host = "machine" + } + host = sanitizeOriginPart(host) + if host == "" || host == "local" { + host = "machine" + } + var suffix [3]byte + if _, err := rand.Read(suffix[:]); err != nil { + return "", fmt.Errorf("generating artifact origin suffix: %w", err) + } + return fmt.Sprintf("%s-%s", host, hex.EncodeToString(suffix[:])), nil +} + +func validateOriginID(origin string) error { + return config.ValidateArtifactOriginID(origin) +} + +func validateDisjointRoots(localRoot, target string) error { + localAbs, err := filepath.Abs(localRoot) + if err != nil { + return fmt.Errorf("resolving local artifact store: %w", err) + } + targetAbs, err := filepath.Abs(target) + if err != nil { + return fmt.Errorf("resolving artifact sync target: %w", err) + } + localAbs = filepath.Clean(localAbs) + targetAbs = filepath.Clean(targetAbs) + localCanonical, err := canonicalArtifactPath(localAbs) + if err != nil { + return fmt.Errorf("resolving local artifact store symlinks: %w", err) + } + targetCanonical, err := canonicalArtifactPath(targetAbs) + if err != nil { + return fmt.Errorf("resolving artifact sync target symlinks: %w", err) + } + if rootsOverlap(localAbs, targetAbs) || rootsOverlap(localCanonical, targetCanonical) { + return fmt.Errorf( + "artifact sync target %s must not overlap local artifact store %s", + targetCanonical, localCanonical, + ) + } + return nil +} + +func canonicalArtifactPath(path string) (string, error) { + missing := make([]string, 0, 2) + current := path + for { + resolved, err := filepath.EvalSymlinks(current) + if err == nil { + for _, part := range slices.Backward(missing) { + resolved = filepath.Join(resolved, part) + } + return filepath.Clean(resolved), nil + } + if !errors.Is(err, fs.ErrNotExist) { + return "", err + } + parent := filepath.Dir(current) + if parent == current { + return "", err + } + missing = append(missing, filepath.Base(current)) + current = parent + } +} + +func rootsOverlap(a, b string) bool { + return a == b || pathContains(a, b) || pathContains(b, a) +} + +func pathContains(parent, child string) bool { + rel, err := filepath.Rel(parent, child) + if err != nil { + return false + } + return rel != "." && rel != ".." && !strings.HasPrefix(rel, ".."+string(filepath.Separator)) && !filepath.IsAbs(rel) +} + +func sanitizeOriginPart(s string) string { + s = strings.ToLower(strings.TrimSpace(s)) + var b strings.Builder + lastDash := false + for _, r := range s { + ok := r >= 'a' && r <= 'z' || r >= '0' && r <= '9' + if ok { + b.WriteRune(r) + lastDash = false + continue + } + if !lastDash { + b.WriteByte('-') + lastDash = true + } + } + return strings.Trim(b.String(), "-") +} + +type checkpoint struct { + Version int `json:"v"` + Origin string `json:"origin"` + Sequence int `json:"seq"` + Sessions map[string]string `json:"sessions"` +} + +type manifest struct { + Version int `json:"v"` + Origin string `json:"origin"` + NativeSessionID string `json:"native_session_id"` + Session manifestSession `json:"session"` + SessionName *string `json:"session_name,omitempty"` + Segments []string `json:"segments"` + UsageEvents []artifactUsageEvent `json:"usage_events,omitempty"` + RawSource *rawSourceRef `json:"raw_source,omitempty"` + DataVersion int `json:"data_version"` + Generation int `json:"generation"` + // Signal state persisted on the session row but absent from the wire + // Session above, which mirrors only db.Session's JSON-visible fields. + // Carried explicitly so an imported session keeps its tool-call, context, + // and quality signal state instead of resetting to false/zero. Secret-scan + // state is deliberately not carried: findings live outside the manifest, + // so imported sessions are treated as unscanned (see rewriteForImport). + SessionHasToolCalls bool `json:"session_has_tool_calls,omitempty"` + SessionHasContextData bool `json:"session_has_context_data,omitempty"` + SessionQualitySignals *manifestQualitySignals `json:"session_quality_signals,omitempty"` +} + +type artifactUsageEvent struct { + MessageOrdinal *int `json:"message_ordinal,omitempty"` + Source string `json:"source"` + Model string `json:"model"` + InputTokens int `json:"input_tokens,omitempty"` + OutputTokens int `json:"output_tokens,omitempty"` + CacheCreationInputTokens int `json:"cache_creation_input_tokens,omitempty"` + CacheReadInputTokens int `json:"cache_read_input_tokens,omitempty"` + ReasoningTokens int `json:"reasoning_tokens,omitempty"` + CostUSD *float64 `json:"cost_usd,omitempty"` + CostStatus string `json:"cost_status,omitempty"` + CostSource string `json:"cost_source,omitempty"` + OccurredAt string `json:"occurred_at,omitempty"` + DedupKey string `json:"dedup_key,omitempty"` +} + +type rawSourceRef struct { + Hash string `json:"hash"` + Size int64 `json:"size"` + MediaType string `json:"media_type,omitempty"` + Path string `json:"path,omitempty"` +} + +type metadataEvent struct { + Version int `json:"v"` + HLC string `json:"hlc"` + Origin string `json:"origin"` + SessionGID string `json:"session_gid"` + Op string `json:"op"` + Value json.RawMessage `json:"value,omitempty"` + Pin *MetadataPin `json:"pin,omitempty"` +} + +type segmentMessage struct { + Version int `json:"v"` + Ordinal int `json:"ordinal"` + Role string `json:"role"` + Content string `json:"content"` + ThinkingText string `json:"thinking_text,omitempty"` + Timestamp string `json:"timestamp,omitempty"` + HasThinking bool `json:"has_thinking,omitempty"` + HasToolUse bool `json:"has_tool_use,omitempty"` + ContentLength int `json:"content_length,omitempty"` + Model string `json:"model,omitempty"` + TokenUsage json.RawMessage `json:"token_usage,omitempty"` + ContextTokens int `json:"context_tokens,omitempty"` + OutputTokens int `json:"output_tokens,omitempty"` + HasContextTokens bool `json:"has_context_tokens,omitempty"` + HasOutputTokens bool `json:"has_output_tokens,omitempty"` + ClaudeMessageID string `json:"claude_message_id,omitempty"` + ClaudeRequestID string `json:"claude_request_id,omitempty"` + ToolCalls []segmentToolCall `json:"tool_calls,omitempty"` + IsSystem bool `json:"is_system,omitempty"` + SourceType string `json:"source_type,omitempty"` + SourceSubtype string `json:"source_subtype,omitempty"` + SourceUUID string `json:"source_uuid,omitempty"` + SourceParentUUID string `json:"source_parent_uuid,omitempty"` + IsSidechain bool `json:"is_sidechain,omitempty"` + IsCompactBoundary bool `json:"is_compact_boundary,omitempty"` +} + +type segmentToolCall struct { + CallIndex int `json:"call_index"` + ToolName string `json:"tool_name"` + Category string `json:"category,omitempty"` + ToolUseID string `json:"tool_use_id,omitempty"` + InputJSON string `json:"input_json,omitempty"` + FilePath string `json:"file_path,omitempty"` + SkillName string `json:"skill_name,omitempty"` + ResultContentLength int `json:"result_content_length,omitempty"` + ResultContent string `json:"result_content,omitempty"` + SubagentSessionID string `json:"subagent_session_id,omitempty"` + ResultEvents []segmentResultEvent `json:"result_events,omitempty"` +} + +type segmentResultEvent struct { + ToolUseID string `json:"tool_use_id,omitempty"` + AgentID string `json:"agent_id,omitempty"` + SubagentSessionID string `json:"subagent_session_id,omitempty"` + Source string `json:"source"` + Status string `json:"status"` + Content string `json:"content"` + ContentLength int `json:"content_length,omitempty"` + Timestamp string `json:"timestamp,omitempty"` + EventIndex int `json:"event_index"` +} + +// Export writes current machine-owned sessions into root/origin. +func Export(ctx context.Context, database *db.DB, root, origin string) (int, error) { + if err := validateOriginID(origin); err != nil { + return 0, err + } + originRoot := filepath.Join(root, origin) + for _, dir := range []string{"checkpoints", "manifests", "segments", "meta", "raw"} { + if err := os.MkdirAll(filepath.Join(originRoot, dir), 0o755); err != nil { + return 0, fmt.Errorf("creating %s: %w", dir, err) + } + } + + // Enumerate owned sessions from a raw query rather than the sidebar list API + // so usage-only (zero-message) sessions are still exported. + ids, err := database.ListOwnedSessionIDsForExport(ctx) + if err != nil { + return 0, err + } + var exported int + sessions := map[string]string{} + for _, id := range ids { + stateKey := exportStateKey(origin, id) + prevHash, err := database.GetSyncState(stateKey) + if err != nil { + return 0, fmt.Errorf("reading export state for %s: %w", id, err) + } + hash, changed, err := exportSession(ctx, database, originRoot, origin, id, prevHash) + if err != nil { + return 0, err + } + sessions[origin+"~"+id] = hash + if changed { + if err := database.SetSyncState(stateKey, hash); err != nil { + return 0, fmt.Errorf("writing export state for %s: %w", id, err) + } + exported++ + } + } + if latestCheckpointMatches(originRoot, origin, sessions) { + return exported, nil + } + seq, err := nextCheckpointSequence(originRoot) + if err != nil { + return 0, err + } + cp := checkpoint{Version: formatVersion, Origin: origin, Sequence: seq, Sessions: sessions} + data, err := canonicalJSON(cp) + if err != nil { + return 0, err + } + checkpointPath := filepath.Join(originRoot, "checkpoints", fmt.Sprintf("cp-%010d.json", seq)) + if err := writeFileAtomic(checkpointPath, data, 0o644); err != nil { + return 0, fmt.Errorf("writing checkpoint: %w", err) + } + return exported, nil +} + +func latestCheckpointMatches(originRoot, origin string, sessions map[string]string) bool { + paths, err := filepath.Glob(filepath.Join(originRoot, "checkpoints", "cp-*.json")) + if err != nil || len(paths) == 0 { + return false + } + path := paths[len(paths)-1] + data, err := os.ReadFile(path) + if err != nil { + return false + } + var cp checkpoint + if err := json.Unmarshal(data, &cp); err != nil { + return false + } + if err := validateCheckpoint(&cp, origin); err != nil { + return false + } + if err := validateCheckpointSequenceIdentity(cp, filepath.Base(path)); err != nil { + return false + } + return maps.Equal(cp.Sessions, sessions) +} + +// nextCheckpointSequence derives the next checkpoint sequence from the +// highest existing checkpoint filename, quarantined ones included, so a +// corrupt latest checkpoint cannot block export and a sequence number that +// may already have been published to peers is never reused for different +// content. A latest live checkpoint whose body no longer parses is +// quarantined so read paths fall back to the previous valid checkpoint and a +// valid peer copy can heal it. +func nextCheckpointSequence(originRoot string) (int, error) { + dir := filepath.Join(originRoot, "checkpoints") + live, err := filepath.Glob(filepath.Join(dir, "cp-*.json")) + if err != nil { + return 0, err + } + quarantined, err := filepath.Glob(filepath.Join(dir, "cp-*.json"+quarantineSuffix)) + if err != nil { + return 0, err + } + sort.Strings(live) + if len(live) > 0 { + latest := live[len(live)-1] + data, err := os.ReadFile(latest) + if err != nil && !errors.Is(err, fs.ErrNotExist) { + return 0, err + } + var cp checkpoint + if err == nil { + if uerr := json.Unmarshal(data, &cp); uerr != nil { + log.Printf("artifact: skipping corrupt checkpoint %s: %v", latest, uerr) + quarantineArtifact(latest) + } + } + } + maxSeq := 0 + for _, p := range append(live, quarantined...) { + name := strings.TrimSuffix(filepath.Base(p), quarantineSuffix) + var seq int + if _, serr := fmt.Sscanf(name, "cp-%d.json", &seq); serr != nil { + continue + } + if seq > maxSeq { + maxSeq = seq + } + } + return maxSeq + 1, nil +} + +func exportStateKey(origin, sessionID string) string { + return exportStatePrefix + origin + ":" + sessionID +} + +func exportSession( + ctx context.Context, + database *db.DB, + originRoot, origin, sessionID, prevManifestHash string, +) (string, bool, error) { + return exportSessionWithLimits( + ctx, database, originRoot, origin, sessionID, prevManifestHash, + productionArtifactLimits(), + ) +} + +func exportSessionWithLimits( + ctx context.Context, + database *db.DB, + originRoot, origin, sessionID, prevManifestHash string, + limits artifactLimits, +) (string, bool, error) { + sess, err := database.GetSessionFull(ctx, sessionID) + if err != nil { + return "", false, fmt.Errorf("loading session %s for artifact export: %w", sessionID, err) + } + if sess == nil { + return "", false, fmt.Errorf("session %s disappeared during artifact export", sessionID) + } + msgs, err := database.GetAllMessages(ctx, sessionID) + if err != nil { + return "", false, fmt.Errorf("loading messages for artifact export %s: %w", sessionID, err) + } + usageEvents, err := database.GetUsageEvents(ctx, sessionID) + if err != nil { + return "", false, fmt.Errorf("loading usage events for artifact export %s: %w", sessionID, err) + } + if len(msgs) > limits.sessionMessages { + return "", false, fmt.Errorf( + "session message limit exceeded for %s: got %d, limit %d", + sessionID, len(msgs), limits.sessionMessages, + ) + } + if len(usageEvents) > limits.manifestUsageEvents { + return "", false, fmt.Errorf( + "manifest usage event limit exceeded for %s: got %d, limit %d", + sessionID, len(usageEvents), limits.manifestUsageEvents, + ) + } + if err := validateExportNestedCollections(msgs, limits); err != nil { + return "", false, fmt.Errorf("validating nested collections for %s: %w", sessionID, err) + } + + wireMessages := canonicalMessages(msgs) + segmentHashes := make([]string, 0, 1) + seenSegmentHashes := make(map[string]struct{}) + var sessionDecodedBytes int64 + if err := forEachEncodedSegmentWithLimits(wireMessages, limits, func(data []byte) error { + if len(segmentHashes) >= limits.manifestSegments { + return fmt.Errorf( + "manifest segment reference limit exceeded for %s: limit %d", + sessionID, limits.manifestSegments, + ) + } + segmentBytes := int64(len(data)) + if segmentBytes > limits.sessionDecodedBytes-sessionDecodedBytes { + return fmt.Errorf( + "session decoded byte limit exceeded for %s: limit %d", + sessionID, limits.sessionDecodedBytes, + ) + } + segmentHash := hashHex(data) + if _, ok := seenSegmentHashes[segmentHash]; ok { + return fmt.Errorf("generated duplicate segment reference %s", segmentHash) + } + seenSegmentHashes[segmentHash] = struct{}{} + segmentHashes = append(segmentHashes, segmentHash) + sessionDecodedBytes += segmentBytes + return nil + }); err != nil { + return "", false, err + } + + wireSession := manifestSessionFromDB(*sess) + wireSession.Machine = origin + normalizeManifestSessionLocalState(&wireSession) + m := manifest{ + Version: formatVersion, + Origin: origin, + NativeSessionID: sessionID, + Session: wireSession, + SessionName: sess.SessionName, + Segments: segmentHashes, + UsageEvents: canonicalUsageEvents(usageEvents), + DataVersion: sess.DataVersion, + Generation: 1, + SessionHasToolCalls: sess.HasToolCalls, + SessionHasContextData: sess.HasContextData, + SessionQualitySignals: manifestQualitySignalsFromDB(sess.StoredQualitySignals()), + } + manifestData, err := canonicalJSON(m) + if err != nil { + return "", false, err + } + if int64(len(manifestData)) > manifestDecodedLimit { + return "", false, fmt.Errorf( + "generated manifest exceeds %d-byte readable limit: got %d bytes", + manifestDecodedLimit, len(manifestData), + ) + } + manifestHash := hashHex(manifestData) + if compatible, err := previousManifestMatchesAfterLocalStateNormalization( + originRoot, origin, sessionID, prevManifestHash, manifestHash, + ); err != nil { + return "", false, err + } else if compatible { + return prevManifestHash, false, nil + } + manifestPath := filepath.Join(originRoot, "manifests", manifestHash+manifestExtension) + if prevManifestHash == manifestHash { + _, valid, err := readValidExportArtifacts( + originRoot, origin, sessionID, manifestHash, + ) + if err != nil { + return "", false, err + } + if valid { + return manifestHash, false, nil + } + } + segmentIndex := 0 + if err := forEachEncodedSegmentWithLimits(wireMessages, limits, func(data []byte) error { + segmentHash := hashHex(data) + if segmentIndex >= len(segmentHashes) || segmentHashes[segmentIndex] != segmentHash { + return errors.New("message segment encoding changed between export passes") + } + segmentIndex++ + if err := healComputedExportSegment(originRoot, segmentHash); err != nil { + return fmt.Errorf("validating computed segment: %w", err) + } + segmentPath := filepath.Join(originRoot, "segments", segmentHash+segmentExtension) + if err := writeCompressed(segmentPath, data); err != nil { + return fmt.Errorf("writing segment: %w", err) + } + return nil + }); err != nil { + return "", false, fmt.Errorf("exporting segments for %s: %w", sessionID, err) + } + if err := writeCompressed(manifestPath, manifestData); err != nil { + return "", false, fmt.Errorf("writing manifest for %s: %w", sessionID, err) + } + return manifestHash, true, nil +} + +// healComputedExportSegment validates only the segment path derived from the +// current local messages. Corrupt bytes are quarantined by the shared reader so +// the subsequent immutable write can recreate them. Missing artifacts are +// expected; semantic errors such as a future segment version remain untouched +// and surface to the caller. +func healComputedExportSegment(originRoot, segmentHash string) error { + _, err := readManifestMessages(originRoot, manifest{ + Segments: []string{segmentHash}, + }) + if err == nil || errors.Is(err, errIncompleteArtifact) || + errors.Is(err, errCorruptArtifact) { + return nil + } + return err +} + +func normalizeManifestSessionLocalState(sess *manifestSession) { + // Keep non-content, machine-local state out of the canonical manifest so a + // source-only change to it does not alter the content hash and trigger a + // re-import that clears the importer's local findings. secret_leak_count is + // import-discarded secret state (see rewriteForImport); local_modified_at is + // the local sync watermark, which import ignores (the importer stamps its + // own) -- and a secret rescan bumps both even when no exported message + // content changed. The file_* fields are source-file bookkeeping that + // import clears (see clearImportedSessionSourceState); a touch, move, or + // re-download of the source file changes them without changing any + // exported content. + sess.SecretLeakCount = 0 + sess.LocalModifiedAt = nil + sess.FilePath = nil + sess.FileSize = nil + sess.FileMtime = nil + sess.FileInode = nil + sess.FileDevice = nil + sess.FileHash = nil +} + +func previousManifestMatchesAfterLocalStateNormalization( + originRoot, origin, sessionID, prevManifestHash, currentManifestHash string, +) (bool, error) { + if prevManifestHash == "" || prevManifestHash == currentManifestHash { + return false, nil + } + prev, valid, err := readValidExportArtifacts( + originRoot, origin, sessionID, prevManifestHash, + ) + if err != nil { + return false, err + } + if !valid { + return false, nil + } + normalizeManifestSessionLocalState(&prev.Session) + data, err := canonicalJSON(prev) + if err != nil { + return false, err + } + return hashHex(data) == currentManifestHash, nil +} + +func readValidExportArtifacts( + originRoot, origin, sessionID, manifestHash string, +) (manifest, bool, error) { + m, err := readManifest(originRoot, manifestHash) + if err != nil { + if errors.Is(err, errIncompleteArtifact) || errors.Is(err, errCorruptArtifact) { + return manifest{}, false, nil + } + return manifest{}, false, err + } + if err := validateManifest(m, origin, origin+"~"+sessionID); err != nil { + if errors.Is(err, errFutureArtifactVersion) { + return manifest{}, false, nil + } + quarantineArtifact(filepath.Join( + originRoot, KindManifests, manifestHash+manifestExtension, + )) + return manifest{}, false, nil + } + if _, err := readManifestMessages(originRoot, m); err != nil { + if errors.Is(err, errIncompleteArtifact) || + errors.Is(err, errCorruptArtifact) || + errors.Is(err, errFutureArtifactVersion) { + return manifest{}, false, nil + } + return manifest{}, false, err + } + return m, true, nil +} + +// Import reads every foreign origin under root and imports referenced sessions. +func Import(ctx context.Context, database *db.DB, root, localOrigin string) (int, int, error) { + res, err := ImportDetailed(ctx, database, root, localOrigin) + return res.Sessions, res.Messages, err +} + +// ImportDetailed reads every foreign origin under root and imports referenced +// sessions plus metadata events. +func ImportDetailed(ctx context.Context, database *db.DB, root, localOrigin string) (ImportResult, error) { + return importDetailed(ctx, database, nil, root, localOrigin) +} + +// importDetailed imports foreign origins, advancing clock past observed remote +// metadata HLCs. When clock is nil a default clock backed by database is used so +// the persisted metadata clock is still advanced. +func importDetailed( + ctx context.Context, + database *db.DB, + clock *HLCClock, + root, localOrigin string, +) (ImportResult, error) { + if clock == nil { + clock = NewHLCClock(database, HLCClockOptions{}) + } + entries, err := os.ReadDir(root) + if err != nil { + if errors.Is(err, fs.ErrNotExist) { + return ImportResult{}, nil + } + return ImportResult{}, fmt.Errorf("reading artifact roots: %w", err) + } + appliedEvents, err := database.MetadataAppliedEventIdentities(ctx) + if err != nil { + return ImportResult{}, err + } + foreignOrigins := make([]string, 0, len(entries)) + for _, ent := range entries { + if ent.IsDir() && ent.Name() != localOrigin { + foreignOrigins = append(foreignOrigins, ent.Name()) + } + } + var res ImportResult + for _, origin := range foreignOrigins { + originRes, err := importOriginContent( + ctx, database, filepath.Join(root, origin), origin, + ) + if err != nil { + return res, err + } + res.Sessions += originRes.Sessions + res.Messages += originRes.Messages + res.Deferred += originRes.Deferred + } + for _, origin := range foreignOrigins { + metadata, err := replayMetadata( + ctx, database, clock, filepath.Join(root, origin), origin, + localOrigin, appliedEvents, + ) + if err != nil { + return res, err + } + res.Metadata += metadata + } + return res, nil +} + +func importOriginContent( + ctx context.Context, + database *db.DB, + originRoot, origin string, +) (ImportResult, error) { + cp, err := readLatestCompatibleCheckpoint(originRoot, origin) + if err != nil { + return ImportResult{}, fmt.Errorf("reading checkpoint for %s: %w", origin, err) + } + var res ImportResult + if cp != nil { + keys := make([]string, 0, len(cp.Sessions)) + for k := range cp.Sessions { + keys = append(keys, k) + } + sort.Strings(keys) + + for _, gid := range keys { + if err := ctx.Err(); err != nil { + return res, err + } + manifestHash := cp.Sessions[gid] + stateKey := importStateKey(origin, gid) + prevHash, err := database.GetSyncState(stateKey) + if err != nil { + return res, fmt.Errorf("reading import state for %s: %w", gid, err) + } + if prevHash == manifestHash { + continue + } + m, err := readManifest(originRoot, manifestHash) + if err != nil { + if errors.Is(err, errIncompleteArtifact) { + res.Deferred++ + continue + } + if errors.Is(err, errCorruptArtifact) { + log.Printf("artifact: skipping session %s from %s: %v", gid, origin, err) + res.Deferred++ + continue + } + return res, err + } + if err := validateManifest(m, origin, gid); err != nil { + if errors.Is(err, errFutureArtifactVersion) { + continue + } + return res, err + } + msgs, err := readManifestMessages(originRoot, m) + if err != nil { + if errors.Is(err, errIncompleteArtifact) || errors.Is(err, errFutureArtifactVersion) { + if errors.Is(err, errIncompleteArtifact) { + res.Deferred++ + } + continue + } + if errors.Is(err, errCorruptArtifact) { + log.Printf("artifact: skipping session %s from %s: %v", gid, origin, err) + res.Deferred++ + continue + } + return res, err + } + write := rewriteForImport(m, msgs) + writeRes, err := database.WriteSessionBatchAtomic([]db.SessionBatchWrite{write}) + if err != nil { + if errors.Is(err, db.ErrSessionExcluded) || errors.Is(err, db.ErrSessionTrashed) { + continue + } + return res, fmt.Errorf("importing artifact session %s: %w", gid, err) + } + res.Sessions += writeRes.WrittenSessions + res.Messages += writeRes.WrittenMessages + if _, err := database.ReapplyMetadataReplayState(ctx, gid, write.Session.ID); err != nil { + return res, fmt.Errorf("reapplying metadata after importing artifact session %s: %w", gid, err) + } + if err := database.SetSyncState(stateKey, manifestHash); err != nil { + return res, fmt.Errorf("writing import state for %s: %w", gid, err) + } + } + } + return res, nil +} + +func importStateKey(origin, gid string) string { + return importStatePrefix + origin + ":" + gid +} + +func validateCheckpoint(cp *checkpoint, origin string) error { + if cp.Version > formatVersion { + return fmt.Errorf( + "%w: checkpoint for %s has artifact version %d", + errFutureArtifactVersion, origin, cp.Version, + ) + } + if cp.Version != formatVersion { + return fmt.Errorf( + "checkpoint for %s has unsupported artifact version %d", + origin, cp.Version, + ) + } + if cp.Origin != origin { + return fmt.Errorf( + "checkpoint origin mismatch for %s: got %q", + origin, cp.Origin, + ) + } + return validateCheckpointReferences(cp, origin) +} + +func validateCheckpointReferences(cp *checkpoint, origin string) error { + for gid, manifestHash := range cp.Sessions { + if gid == "" { + return fmt.Errorf("checkpoint for %s contains empty session id", origin) + } + if !strings.HasPrefix(gid, origin+"~") { + return fmt.Errorf( + "checkpoint session %s does not belong to origin %s", + gid, origin, + ) + } + if strings.TrimSpace(manifestHash) == "" { + return fmt.Errorf("checkpoint session %s has empty manifest hash", gid) + } + if err := validateHashHex(manifestHash); err != nil { + return fmt.Errorf("checkpoint session %s has invalid manifest hash: %w", gid, err) + } + } + return nil +} + +func validateManifest(m manifest, origin, gid string) error { + if m.Version > formatVersion { + return fmt.Errorf( + "%w: manifest %s has artifact version %d", + errFutureArtifactVersion, gid, m.Version, + ) + } + if m.Version != formatVersion { + return fmt.Errorf( + "manifest %s has unsupported artifact version %d", + gid, m.Version, + ) + } + if m.Origin != origin { + return fmt.Errorf( + "manifest origin mismatch for %s: got %q", + gid, m.Origin, + ) + } + if m.NativeSessionID == "" { + return fmt.Errorf("manifest %s has empty native session id", gid) + } + expectedGID := origin + "~" + m.NativeSessionID + if gid != expectedGID { + return fmt.Errorf( + "manifest session id mismatch: checkpoint has %s, manifest has %s", + gid, expectedGID, + ) + } + if m.Session.ID != m.NativeSessionID { + return fmt.Errorf( + "manifest %s session row id mismatch: got %q", + gid, m.Session.ID, + ) + } + if m.Session.Machine != origin { + return fmt.Errorf( + "manifest %s session row machine mismatch: got %q", + gid, m.Session.Machine, + ) + } + if len(m.Segments) == 0 { + return fmt.Errorf("manifest %s has no message segments", gid) + } + if err := validateManifestReferences(m); err != nil { + return err + } + return nil +} + +func validateManifestReferences(m manifest) error { + return validateManifestReferencesWithLimits(m, productionArtifactLimits()) +} + +func validateManifestReferencesWithLimits(m manifest, limits artifactLimits) error { + if len(m.Segments) > limits.manifestSegments { + return fmt.Errorf( + "manifest segment reference limit exceeded: got %d, limit %d", + len(m.Segments), limits.manifestSegments, + ) + } + seen := make(map[string]struct{}, len(m.Segments)) + for _, segmentHash := range m.Segments { + if err := validateHashHex(segmentHash); err != nil { + return fmt.Errorf("manifest segment has invalid hash: %w", err) + } + if _, ok := seen[segmentHash]; ok { + return fmt.Errorf("manifest has duplicate segment reference %s", segmentHash) + } + seen[segmentHash] = struct{}{} + } + if len(m.UsageEvents) > limits.manifestUsageEvents { + return fmt.Errorf( + "manifest usage event limit exceeded: got %d, limit %d", + len(m.UsageEvents), limits.manifestUsageEvents, + ) + } + if m.RawSource != nil && m.RawSource.Hash != "" { + if err := validateHashHex(m.RawSource.Hash); err != nil { + return fmt.Errorf("manifest raw source has invalid hash: %w", err) + } + } + return nil +} + +func rewriteForImport(m manifest, msgs []db.Message) db.SessionBatchWrite { + importedID := m.Origin + "~" + m.NativeSessionID + sess := m.Session.dbSession() + sess.ID = importedID + sess.Machine = m.Origin + sess.SessionName = m.SessionName + clearImportedSessionSourceState(&sess) + // Restore signal state dropped from the Session JSON; signalsFromSession + // reads these fields below to persist the imported session's signal columns. + sess.HasToolCalls = m.SessionHasToolCalls + sess.HasContextData = m.SessionHasContextData + sess.ApplyQualitySignals(m.SessionQualitySignals.dbQualitySignals()) + // Secret findings are not carried in the manifest, so an imported session has + // no finding rows. Treat it as unscanned rather than trusting the source scan: + // clear the rules version (json:"-", so already absent) and the leak count + // (carried in the Session JSON) so the count stays consistent with the zero + // findings and `secrets scan --backfill` rescans it with local rules. Stamping + // it scanned-at-source-version would make backfill (secrets_rules_version != + // current) skip a secret-bearing session, leaving no revealable findings. + sess.SecretsRulesVersion = "" + sess.SecretLeakCount = 0 + sess.SourceSessionID = prefixImportedSessionID(m.Origin, sess.SourceSessionID) + if sess.ParentSessionID != nil { + prefixed := prefixImportedSessionID(m.Origin, *sess.ParentSessionID) + sess.ParentSessionID = &prefixed + } + for i := range msgs { + msgs[i].ID = 0 + msgs[i].SessionID = importedID + for j := range msgs[i].ToolCalls { + msgs[i].ToolCalls[j].MessageID = 0 + msgs[i].ToolCalls[j].SessionID = importedID + msgs[i].ToolCalls[j].SubagentSessionID = prefixImportedSessionID( + m.Origin, + msgs[i].ToolCalls[j].SubagentSessionID, + ) + for k := range msgs[i].ToolCalls[j].ResultEvents { + ev := &msgs[i].ToolCalls[j].ResultEvents[k] + ev.SubagentSessionID = prefixImportedSessionID(m.Origin, ev.SubagentSessionID) + } + } + } + usageEvents := dbUsageEvents(m.UsageEvents, importedID) + return db.SessionBatchWrite{ + Session: sess, + Messages: msgs, + UsageEvents: usageEvents, + Signals: signalsFromSession(sess), + DataVersion: m.DataVersion, + ReplaceMessages: true, + } +} + +func clearImportedSessionSourceState(sess *db.Session) { + sess.FilePath = nil + sess.FileSize = nil + sess.FileMtime = nil + sess.NextOrdinal = 0 + sess.LastEntryUUID = nil + sess.FileInode = nil + sess.FileDevice = nil + sess.FileHash = nil +} + +func prefixImportedSessionID(origin, id string) string { + if id == "" || strings.Contains(id, "~") { + return id + } + return origin + "~" + id +} + +func signalsFromSession(s db.Session) db.SessionSignalUpdate { + update := db.SessionSignalUpdate{ + ToolFailureSignalCount: s.ToolFailureSignalCount, + ToolRetryCount: s.ToolRetryCount, + EditChurnCount: s.EditChurnCount, + ConsecutiveFailureMax: s.ConsecutiveFailureMax, + Outcome: s.Outcome, + OutcomeConfidence: s.OutcomeConfidence, + EndedWithRole: s.EndedWithRole, + FinalFailureStreak: s.FinalFailureStreak, + SignalsPendingSince: s.SignalsPendingSince, + CompactionCount: s.CompactionCount, + MidTaskCompactionCount: s.MidTaskCompactionCount, + ContextPressureMax: s.ContextPressureMax, + HealthScore: s.HealthScore, + HealthGrade: s.HealthGrade, + HasToolCalls: s.HasToolCalls, + HasContextData: s.HasContextData, + SecretLeakCount: s.SecretLeakCount, + SecretsRulesVersion: s.SecretsRulesVersion, + } + if qs := s.StoredQualitySignals(); qs != nil { + update.QualitySignals = *qs + } + return update +} + +func readLatestCheckpoint(originRoot string) (*checkpoint, error) { + paths, err := filepath.Glob(filepath.Join(originRoot, "checkpoints", "cp-*.json")) + if err != nil { + return nil, err + } + if len(paths) == 0 { + return nil, nil + } + sort.Strings(paths) + data, err := os.ReadFile(paths[len(paths)-1]) + if err != nil { + return nil, err + } + var cp checkpoint + if err := json.Unmarshal(data, &cp); err != nil { + return nil, err + } + return &cp, nil +} + +func readLatestCompatibleCheckpoint(originRoot, origin string) (*checkpoint, error) { + paths, err := filepath.Glob(filepath.Join(originRoot, "checkpoints", "cp-*.json")) + if err != nil { + return nil, err + } + sort.Strings(paths) + for _, v := range slices.Backward(paths) { + data, err := os.ReadFile(v) + if err != nil { + if errors.Is(err, fs.ErrNotExist) { + continue + } + return nil, err + } + var cp checkpoint + if err := json.Unmarshal(data, &cp); err != nil { + log.Printf("artifact: skipping corrupt checkpoint %s: %v", v, err) + quarantineArtifact(v) + continue + } + if err := validateCheckpoint(&cp, origin); err != nil { + if errors.Is(err, errFutureArtifactVersion) { + continue + } + log.Printf("artifact: skipping invalid checkpoint %s: %v", v, err) + quarantineArtifact(v) + continue + } + if err := validateCheckpointSequenceIdentity(cp, filepath.Base(v)); err != nil { + log.Printf("artifact: skipping invalid checkpoint %s: %v", v, err) + quarantineArtifact(v) + continue + } + return &cp, nil + } + return nil, nil +} + +func readManifest(originRoot, hash string) (manifest, error) { + if err := validateHashHex(hash); err != nil { + return manifest{}, err + } + path := filepath.Join(originRoot, "manifests", hash+manifestExtension) + data, err := readCompressed(path) + if err != nil { + if errors.Is(err, fs.ErrNotExist) { + return manifest{}, fmt.Errorf("%w: manifest %s", errIncompleteArtifact, hash) + } + if errors.Is(err, errCorruptArtifact) { + quarantineArtifact(path) + return manifest{}, fmt.Errorf("manifest %s: %w", hash, err) + } + return manifest{}, fmt.Errorf("reading manifest %s: %w", hash, err) + } + if got := hashHex(data); got != hash { + quarantineArtifact(path) + return manifest{}, fmt.Errorf("%w: manifest %s hash mismatch: got %s", errCorruptArtifact, hash, got) + } + m, err := decodeManifestWithLimits(data, productionArtifactLimits()) + if err != nil { + quarantineArtifact(path) + return manifest{}, fmt.Errorf("%w: decoding manifest %s: %v", errCorruptArtifact, hash, err) + } + return m, nil +} + +func decodeManifestWithLimits(data []byte, limits artifactLimits) (manifest, error) { + var envelope struct { + Version int `json:"v"` + Origin string `json:"origin"` + Segments json.RawMessage `json:"segments"` + UsageEvents json.RawMessage `json:"usage_events"` + } + if err := json.Unmarshal(data, &envelope); err != nil { + return manifest{}, err + } + // Future manifests are retained for forward compatibility. Reading only + // their scalar header avoids allocating collections whose schema this + // version does not understand. + if envelope.Version > formatVersion { + return manifest{Version: envelope.Version, Origin: envelope.Origin}, nil + } + if err := preflightManifestCollections( + envelope.Segments, envelope.UsageEvents, limits, + ); err != nil { + return manifest{}, err + } + var m manifest + if err := json.Unmarshal(data, &m); err != nil { + return manifest{}, err + } + return m, nil +} + +func preflightManifestCollections( + segments, usageEvents json.RawMessage, + limits artifactLimits, +) error { + if err := preflightSegmentReferences(segments, limits.manifestSegments); err != nil { + return err + } + return preflightJSONArrayCount( + usageEvents, "manifest usage event", limits.manifestUsageEvents, + ) +} + +func preflightSegmentReferences(data json.RawMessage, limit int) error { + trimmed := bytes.TrimSpace(data) + if len(trimmed) == 0 || bytes.Equal(trimmed, []byte("null")) { + return nil + } + dec := json.NewDecoder(bytes.NewReader(trimmed)) + token, err := dec.Token() + if err != nil { + return err + } + if token != json.Delim('[') { + return errors.New("manifest segments must be an array") + } + seen := make(map[string]struct{}, min(limit, 16)) + count := 0 + for dec.More() { + if count >= limit { + return fmt.Errorf("manifest segment reference limit exceeded: limit %d", limit) + } + var hash string + if err := dec.Decode(&hash); err != nil { + return fmt.Errorf("decoding manifest segment reference: %w", err) + } + if _, ok := seen[hash]; ok { + return fmt.Errorf("manifest has duplicate segment reference %s", hash) + } + seen[hash] = struct{}{} + count++ + } + _, err = dec.Token() + return err +} + +func preflightJSONArrayCount(data json.RawMessage, name string, limit int) error { + _, err := countJSONArrayElements(data, name, limit) + return err +} + +func countJSONArrayElements( + data json.RawMessage, + name string, + limit int, +) (int, error) { + trimmed := bytes.TrimSpace(data) + if len(trimmed) == 0 || bytes.Equal(trimmed, []byte("null")) { + return 0, nil + } + dec := json.NewDecoder(bytes.NewReader(trimmed)) + token, err := dec.Token() + if err != nil { + return 0, err + } + if token != json.Delim('[') { + return 0, fmt.Errorf("%ss must be an array", name) + } + count := 0 + for dec.More() { + if count >= limit { + return 0, fmt.Errorf("%s limit exceeded: limit %d", name, limit) + } + var value json.RawMessage + if err := dec.Decode(&value); err != nil { + return 0, fmt.Errorf("decoding %s: %w", name, err) + } + count++ + } + if _, err := dec.Token(); err != nil { + return 0, err + } + return count, nil +} + +func readManifestMessages(originRoot string, m manifest) ([]db.Message, error) { + return readManifestMessagesWithLimits(originRoot, m, productionArtifactLimits()) +} + +func readManifestMessagesWithLimits( + originRoot string, + m manifest, + limits artifactLimits, +) ([]db.Message, error) { + var msgs []db.Message + err := walkManifestMessagesWithLimits( + originRoot, m, limits, + func(segmentMsgs []db.Message) { + msgs = append(msgs, segmentMsgs...) + }, + ) + return msgs, err +} + +func walkManifestMessagesWithLimits( + originRoot string, + m manifest, + limits artifactLimits, + visit func([]db.Message), +) error { + if err := validateManifestReferencesWithLimits(m, limits); err != nil { + return fmt.Errorf("%w: %v", errCorruptArtifact, err) + } + var decodedBytes int64 + totalMessages := 0 + totalNested := nestedCollectionCounts{} + for _, segmentHash := range m.Segments { + segment, err := readSegmentPreflightWithLimits( + originRoot, segmentHash, limits, + ) + if err != nil { + return err + } + segmentBytes := int64(len(segment.data)) + if segmentBytes > limits.sessionDecodedBytes-decodedBytes { + return fmt.Errorf( + "%w: session decoded byte limit exceeded: limit %d", + errCorruptArtifact, limits.sessionDecodedBytes, + ) + } + if len(segment.preflight.records) > limits.sessionMessages-totalMessages { + return fmt.Errorf( + "%w: session message limit exceeded: limit %d", + errCorruptArtifact, limits.sessionMessages, + ) + } + if exceedsCollectionLimit( + totalNested.toolCalls, + segment.preflight.nested.toolCalls, + limits.sessionToolCalls, + ) { + return fmt.Errorf( + "%w: session tool call limit exceeded: limit %d", + errCorruptArtifact, limits.sessionToolCalls, + ) + } + if exceedsCollectionLimit( + totalNested.resultEvents, + segment.preflight.nested.resultEvents, + limits.sessionResultEvents, + ) { + return fmt.Errorf( + "%w: session result event limit exceeded: limit %d", + errCorruptArtifact, limits.sessionResultEvents, + ) + } + segmentMsgs, err := decodeStoredSegment(segment) + if err != nil { + return err + } + decodedBytes += segmentBytes + totalMessages += len(segmentMsgs) + totalNested.toolCalls += segment.preflight.nested.toolCalls + totalNested.resultEvents += segment.preflight.nested.resultEvents + if visit != nil { + visit(segmentMsgs) + } + } + return nil +} + +func readSegmentMessages(originRoot, segmentHash string) ([]db.Message, error) { + msgs, _, err := readSegmentMessagesWithLimits( + originRoot, segmentHash, productionArtifactLimits(), + ) + return msgs, err +} + +func readSegmentMessagesWithLimits( + originRoot, segmentHash string, + limits artifactLimits, +) ([]db.Message, int64, error) { + segment, err := readSegmentPreflightWithLimits(originRoot, segmentHash, limits) + if err != nil { + return nil, 0, err + } + segmentMsgs, err := decodeStoredSegment(segment) + if err != nil { + return nil, 0, err + } + return segmentMsgs, int64(len(segment.data)), nil +} + +type storedSegmentPreflight struct { + path string + hash string + data []byte + preflight segmentPreflight +} + +func readSegmentPreflightWithLimits( + originRoot, segmentHash string, + limits artifactLimits, +) (storedSegmentPreflight, error) { + if err := validateHashHex(segmentHash); err != nil { + return storedSegmentPreflight{}, fmt.Errorf("manifest segment has invalid hash: %w", err) + } + path := filepath.Join(originRoot, "segments", segmentHash+segmentExtension) + data, err := readCompressed(path) + if err != nil { + if errors.Is(err, fs.ErrNotExist) { + return storedSegmentPreflight{}, fmt.Errorf( + "%w: segment %s", errIncompleteArtifact, segmentHash, + ) + } + if errors.Is(err, errCorruptArtifact) { + quarantineArtifact(path) + return storedSegmentPreflight{}, fmt.Errorf("segment %s: %w", segmentHash, err) + } + return storedSegmentPreflight{}, fmt.Errorf("reading segment %s: %w", segmentHash, err) + } + if got := hashHex(data); got != segmentHash { + quarantineArtifact(path) + return storedSegmentPreflight{}, fmt.Errorf( + "%w: segment %s hash mismatch: got %s", + errCorruptArtifact, segmentHash, got, + ) + } + preflight, err := preflightSegmentData(data, limits) + if err != nil { + if errors.Is(err, errFutureArtifactVersion) { + return storedSegmentPreflight{}, fmt.Errorf("segment %s: %w", segmentHash, err) + } + quarantineArtifact(path) + return storedSegmentPreflight{}, fmt.Errorf( + "%w: segment %s: %v", errCorruptArtifact, segmentHash, err, + ) + } + return storedSegmentPreflight{ + path: path, hash: segmentHash, data: data, preflight: preflight, + }, nil +} + +func decodeStoredSegment(segment storedSegmentPreflight) ([]db.Message, error) { + segmentMsgs, err := decodePreflightedSegment(segment.preflight) + if err == nil { + return segmentMsgs, nil + } + quarantineArtifact(segment.path) + return nil, fmt.Errorf("%w: segment %s: %v", errCorruptArtifact, segment.hash, err) +} + +func canonicalMessages(msgs []db.Message) []db.Message { + out := make([]db.Message, len(msgs)) + for i, msg := range msgs { + msg.ID = 0 + msg.SessionID = "" + if len(msg.ToolCalls) > 0 { + calls := make([]db.ToolCall, len(msg.ToolCalls)) + copy(calls, msg.ToolCalls) + for j := range calls { + calls[j].MessageID = 0 + calls[j].SessionID = "" + } + msg.ToolCalls = calls + } + out[i] = msg + } + return out +} + +func canonicalUsageEvents(events []db.UsageEvent) []artifactUsageEvent { + out := make([]artifactUsageEvent, len(events)) + for i, ev := range events { + out[i] = artifactUsageEvent{ + MessageOrdinal: ev.MessageOrdinal, + Source: ev.Source, + Model: ev.Model, + InputTokens: ev.InputTokens, + OutputTokens: ev.OutputTokens, + CacheCreationInputTokens: ev.CacheCreationInputTokens, + CacheReadInputTokens: ev.CacheReadInputTokens, + ReasoningTokens: ev.ReasoningTokens, + CostUSD: ev.CostUSD, + CostStatus: ev.CostStatus, + CostSource: ev.CostSource, + OccurredAt: ev.OccurredAt, + DedupKey: ev.DedupKey, + } + } + return out +} + +func validateExportNestedCollections(msgs []db.Message, limits artifactLimits) error { + total := nestedCollectionCounts{} + for _, msg := range msgs { + messageNested, err := dbMessageNestedCounts(msg, limits) + if err != nil { + return err + } + if err := validateMessageFitsSegment(msg.Ordinal, messageNested, limits); err != nil { + return err + } + if exceedsCollectionLimit( + total.toolCalls, messageNested.toolCalls, limits.sessionToolCalls, + ) { + return fmt.Errorf( + "session tool call limit exceeded at message ordinal %d: limit %d", + msg.Ordinal, limits.sessionToolCalls, + ) + } + if exceedsCollectionLimit( + total.resultEvents, messageNested.resultEvents, limits.sessionResultEvents, + ) { + return fmt.Errorf( + "session result event limit exceeded at message ordinal %d: limit %d", + msg.Ordinal, limits.sessionResultEvents, + ) + } + total.toolCalls += messageNested.toolCalls + total.resultEvents += messageNested.resultEvents + } + return nil +} + +func dbMessageNestedCounts( + msg db.Message, + limits artifactLimits, +) (nestedCollectionCounts, error) { + if len(msg.ToolCalls) > limits.messageToolCalls { + return nestedCollectionCounts{}, fmt.Errorf( + "tool call limit exceeded for message ordinal %d: got %d, limit %d", + msg.Ordinal, len(msg.ToolCalls), limits.messageToolCalls, + ) + } + counts := nestedCollectionCounts{toolCalls: len(msg.ToolCalls)} + for toolIndex, call := range msg.ToolCalls { + if len(call.ResultEvents) > limits.toolResultEvents { + return nestedCollectionCounts{}, fmt.Errorf( + "result event limit exceeded for tool call %d in message ordinal %d: got %d, limit %d", + toolIndex, msg.Ordinal, len(call.ResultEvents), limits.toolResultEvents, + ) + } + counts.resultEvents += len(call.ResultEvents) + } + return counts, nil +} + +func validateMessageFitsSegment( + ordinal int, + counts nestedCollectionCounts, + limits artifactLimits, +) error { + if counts.toolCalls > limits.segmentToolCalls { + return fmt.Errorf( + "message ordinal %d cannot fit in one segment: got %d tool calls, segment limit %d", + ordinal, counts.toolCalls, limits.segmentToolCalls, + ) + } + if counts.resultEvents > limits.segmentResultEvents { + return fmt.Errorf( + "message ordinal %d cannot fit in one segment: got %d result events, segment limit %d", + ordinal, counts.resultEvents, limits.segmentResultEvents, + ) + } + return nil +} + +func encodeSegment(msgs []db.Message) ([]byte, error) { + var buf bytes.Buffer + for _, msg := range msgs { + data, err := canonicalJSON(segmentMessageFromDB(msg)) + if err != nil { + return nil, fmt.Errorf("encoding message segment: %w", err) + } + buf.Write(data) + } + return buf.Bytes(), nil +} + +// forEachEncodedSegment emits deterministic NDJSON chunks while bounding each +// chunk to the size accepted by readers. Chunks normally stop near +// segmentTargetSize; a single larger record remains intact up to the hard +// readable limit so message records are never split. +func forEachEncodedSegmentWithLimits( + msgs []db.Message, + limits artifactLimits, + emit func([]byte) error, +) error { + var buf bytes.Buffer + segmentMessages := 0 + segmentNested := nestedCollectionCounts{} + flush := func() error { + if err := emit(buf.Bytes()); err != nil { + return err + } + buf.Reset() + segmentMessages = 0 + segmentNested = nestedCollectionCounts{} + return nil + } + for _, msg := range msgs { + messageNested, err := dbMessageNestedCounts(msg, limits) + if err != nil { + return err + } + if err := validateMessageFitsSegment(msg.Ordinal, messageNested, limits); err != nil { + return err + } + data, err := canonicalJSON(segmentMessageFromDB(msg)) + if err != nil { + return fmt.Errorf("encoding message segment: %w", err) + } + if int64(len(data)) > segmentDecodedLimit { + return fmt.Errorf( + "encoded message record at ordinal %d exceeds %d-byte readable limit", + msg.Ordinal, segmentDecodedLimit, + ) + } + if buf.Len() > 0 && (int64(buf.Len()+len(data)) > segmentTargetSize || + segmentMessages >= limits.segmentMessages || + exceedsCollectionLimit( + segmentNested.toolCalls, + messageNested.toolCalls, + limits.segmentToolCalls, + ) || exceedsCollectionLimit( + segmentNested.resultEvents, + messageNested.resultEvents, + limits.segmentResultEvents, + )) { + if err := flush(); err != nil { + return err + } + } + _, _ = buf.Write(data) + segmentMessages++ + segmentNested.toolCalls += messageNested.toolCalls + segmentNested.resultEvents += messageNested.resultEvents + } + if buf.Len() > 0 || len(msgs) == 0 { + return flush() + } + return nil +} + +func decodeSegment(data []byte) ([]db.Message, error) { + return decodeSegmentWithLimits(data, productionArtifactLimits()) +} + +func decodeSegmentWithLimits(data []byte, limits artifactLimits) ([]db.Message, error) { + preflight, err := preflightSegmentData(data, limits) + if err != nil { + return nil, err + } + return decodePreflightedSegment(preflight) +} + +func decodePreflightedSegment(preflight segmentPreflight) ([]db.Message, error) { + msgs := make([]db.Message, 0, len(preflight.records)) + for _, line := range preflight.records { + var record segmentMessage + if err := json.Unmarshal(line, &record); err != nil { + return nil, fmt.Errorf("decoding message segment: %w", err) + } + msgs = append(msgs, record.dbMessage()) + } + return msgs, nil +} + +func segmentRecords(data []byte, limit int) ([][]byte, error) { + capacity := min(max(limit, 0), 64) + records := make([][]byte, 0, capacity) + remaining := data + lineNumber := 0 + for len(remaining) > 0 { + lineNumber++ + newline := bytes.IndexByte(remaining, '\n') + line := remaining + if newline >= 0 { + line = remaining[:newline] + remaining = remaining[newline+1:] + } else { + remaining = nil + } + if len(records) >= limit { + return nil, fmt.Errorf( + "message record limit exceeded: limit %d per segment", limit, + ) + } + if len(bytes.TrimSpace(line)) == 0 { + return nil, fmt.Errorf("blank message record at line %d", lineNumber) + } + records = append(records, line) + } + return records, nil +} + +func preflightSegmentData(data []byte, limits artifactLimits) (segmentPreflight, error) { + records, err := segmentRecords(data, limits.segmentMessages) + if err != nil { + return segmentPreflight{}, err + } + preflight := segmentPreflight{records: records} + for _, line := range records { + var header struct { + Version int `json:"v"` + } + if err := json.Unmarshal(line, &header); err != nil { + return segmentPreflight{}, fmt.Errorf("decoding message segment header: %w", err) + } + if header.Version > formatVersion { + return segmentPreflight{}, fmt.Errorf( + "%w: message segment has artifact version %d", + errFutureArtifactVersion, header.Version, + ) + } + if header.Version != formatVersion { + return segmentPreflight{}, fmt.Errorf( + "message segment has unsupported artifact version %d", + header.Version, + ) + } + messageNested, err := preflightMessageNestedCollections(line, limits) + if err != nil { + return segmentPreflight{}, err + } + if exceedsCollectionLimit( + preflight.nested.toolCalls, + messageNested.toolCalls, + limits.segmentToolCalls, + ) { + return segmentPreflight{}, fmt.Errorf( + "segment tool call limit exceeded: limit %d", limits.segmentToolCalls, + ) + } + if exceedsCollectionLimit( + preflight.nested.resultEvents, + messageNested.resultEvents, + limits.segmentResultEvents, + ) { + return segmentPreflight{}, fmt.Errorf( + "segment result event limit exceeded: limit %d", + limits.segmentResultEvents, + ) + } + preflight.nested.toolCalls += messageNested.toolCalls + preflight.nested.resultEvents += messageNested.resultEvents + } + return preflight, nil +} + +func preflightMessageNestedCollections( + line []byte, + limits artifactLimits, +) (nestedCollectionCounts, error) { + var envelope struct { + Ordinal int `json:"ordinal"` + ToolCalls json.RawMessage `json:"tool_calls"` + } + if err := json.Unmarshal(line, &envelope); err != nil { + return nestedCollectionCounts{}, fmt.Errorf( + "decoding message segment collections: %w", err, + ) + } + return preflightToolCallCollections(envelope.ToolCalls, envelope.Ordinal, limits) +} + +func preflightToolCallCollections( + data json.RawMessage, + ordinal int, + limits artifactLimits, +) (nestedCollectionCounts, error) { + trimmed := bytes.TrimSpace(data) + if len(trimmed) == 0 || bytes.Equal(trimmed, []byte("null")) { + return nestedCollectionCounts{}, nil + } + dec := json.NewDecoder(bytes.NewReader(trimmed)) + token, err := dec.Token() + if err != nil { + return nestedCollectionCounts{}, err + } + if token != json.Delim('[') { + return nestedCollectionCounts{}, errors.New("message tool_calls must be an array") + } + counts := nestedCollectionCounts{} + for dec.More() { + if counts.toolCalls >= limits.messageToolCalls { + return nestedCollectionCounts{}, fmt.Errorf( + "tool call limit exceeded for message ordinal %d: limit %d per message", + ordinal, limits.messageToolCalls, + ) + } + var toolEnvelope struct { + ResultEvents json.RawMessage `json:"result_events"` + } + if err := dec.Decode(&toolEnvelope); err != nil { + return nestedCollectionCounts{}, fmt.Errorf( + "decoding tool call %d in message ordinal %d: %w", + counts.toolCalls, ordinal, err, + ) + } + resultEvents, err := countJSONArrayElements( + toolEnvelope.ResultEvents, "result event", limits.toolResultEvents, + ) + if err != nil { + return nestedCollectionCounts{}, fmt.Errorf( + "preflighting tool call %d in message ordinal %d: %w", + counts.toolCalls, ordinal, err, + ) + } + counts.toolCalls++ + counts.resultEvents += resultEvents + } + if _, err := dec.Token(); err != nil { + return nestedCollectionCounts{}, err + } + return counts, nil +} + +func dbUsageEvents(events []artifactUsageEvent, sessionID string) []db.UsageEvent { + out := make([]db.UsageEvent, len(events)) + for i, ev := range events { + out[i] = db.UsageEvent{ + SessionID: sessionID, + MessageOrdinal: ev.MessageOrdinal, + Source: ev.Source, + Model: ev.Model, + InputTokens: ev.InputTokens, + OutputTokens: ev.OutputTokens, + CacheCreationInputTokens: ev.CacheCreationInputTokens, + CacheReadInputTokens: ev.CacheReadInputTokens, + ReasoningTokens: ev.ReasoningTokens, + CostUSD: ev.CostUSD, + CostStatus: ev.CostStatus, + CostSource: ev.CostSource, + OccurredAt: ev.OccurredAt, + DedupKey: ev.DedupKey, + } + } + return out +} + +func segmentMessageFromDB(msg db.Message) segmentMessage { + record := segmentMessage{ + Version: formatVersion, + Ordinal: msg.Ordinal, + Role: msg.Role, + Content: msg.Content, + ThinkingText: msg.ThinkingText, + Timestamp: msg.Timestamp, + HasThinking: msg.HasThinking, + HasToolUse: msg.HasToolUse, + ContentLength: msg.ContentLength, + Model: msg.Model, + TokenUsage: msg.TokenUsage, + ContextTokens: msg.ContextTokens, + OutputTokens: msg.OutputTokens, + HasContextTokens: msg.HasContextTokens, + HasOutputTokens: msg.HasOutputTokens, + ClaudeMessageID: msg.ClaudeMessageID, + ClaudeRequestID: msg.ClaudeRequestID, + IsSystem: msg.IsSystem, + SourceType: msg.SourceType, + SourceSubtype: msg.SourceSubtype, + SourceUUID: msg.SourceUUID, + SourceParentUUID: msg.SourceParentUUID, + IsSidechain: msg.IsSidechain, + IsCompactBoundary: msg.IsCompactBoundary, + } + if len(msg.ToolCalls) > 0 { + record.ToolCalls = make([]segmentToolCall, len(msg.ToolCalls)) + for i, call := range msg.ToolCalls { + record.ToolCalls[i] = segmentToolCall{ + CallIndex: i, + ToolName: call.ToolName, + Category: call.Category, + ToolUseID: call.ToolUseID, + InputJSON: call.InputJSON, + FilePath: call.FilePath, + SkillName: call.SkillName, + ResultContentLength: call.ResultContentLength, + ResultContent: call.ResultContent, + SubagentSessionID: call.SubagentSessionID, + } + if len(call.ResultEvents) > 0 { + record.ToolCalls[i].ResultEvents = make([]segmentResultEvent, len(call.ResultEvents)) + for j, ev := range call.ResultEvents { + record.ToolCalls[i].ResultEvents[j] = segmentResultEvent{ + ToolUseID: ev.ToolUseID, + AgentID: ev.AgentID, + SubagentSessionID: ev.SubagentSessionID, + Source: ev.Source, + Status: ev.Status, + Content: ev.Content, + ContentLength: ev.ContentLength, + Timestamp: ev.Timestamp, + EventIndex: ev.EventIndex, + } + } + } + } + } + return record +} + +func (m segmentMessage) dbMessage() db.Message { + msg := db.Message{ + Ordinal: m.Ordinal, + Role: m.Role, + Content: m.Content, + ThinkingText: m.ThinkingText, + Timestamp: m.Timestamp, + HasThinking: m.HasThinking, + HasToolUse: m.HasToolUse, + ContentLength: m.ContentLength, + Model: m.Model, + TokenUsage: m.TokenUsage, + ContextTokens: m.ContextTokens, + OutputTokens: m.OutputTokens, + HasContextTokens: m.HasContextTokens, + HasOutputTokens: m.HasOutputTokens, + ClaudeMessageID: m.ClaudeMessageID, + ClaudeRequestID: m.ClaudeRequestID, + IsSystem: m.IsSystem, + SourceType: m.SourceType, + SourceSubtype: m.SourceSubtype, + SourceUUID: m.SourceUUID, + SourceParentUUID: m.SourceParentUUID, + IsSidechain: m.IsSidechain, + IsCompactBoundary: m.IsCompactBoundary, + } + if len(m.ToolCalls) > 0 { + msg.ToolCalls = make([]db.ToolCall, len(m.ToolCalls)) + for i, call := range m.ToolCalls { + msg.ToolCalls[i] = db.ToolCall{ + ToolName: call.ToolName, + Category: call.Category, + ToolUseID: call.ToolUseID, + InputJSON: call.InputJSON, + FilePath: call.FilePath, + SkillName: call.SkillName, + ResultContentLength: call.ResultContentLength, + ResultContent: call.ResultContent, + SubagentSessionID: call.SubagentSessionID, + } + if len(call.ResultEvents) > 0 { + msg.ToolCalls[i].ResultEvents = make([]db.ToolResultEvent, len(call.ResultEvents)) + for j, ev := range call.ResultEvents { + msg.ToolCalls[i].ResultEvents[j] = db.ToolResultEvent{ + ToolUseID: ev.ToolUseID, + AgentID: ev.AgentID, + SubagentSessionID: ev.SubagentSessionID, + Source: ev.Source, + Status: ev.Status, + Content: ev.Content, + ContentLength: ev.ContentLength, + Timestamp: ev.Timestamp, + EventIndex: ev.EventIndex, + } + } + } + } + } + return msg +} + +func canonicalJSON(v any) ([]byte, error) { + var buf bytes.Buffer + if err := writeCanonicalJSON(&buf, reflect.ValueOf(v)); err != nil { + return nil, fmt.Errorf("encoding canonical artifact JSON: %w", err) + } + buf.WriteByte('\n') + return buf.Bytes(), nil +} + +func writeCanonicalJSON(buf *bytes.Buffer, v reflect.Value) error { + if !v.IsValid() { + buf.WriteString("null") + return nil + } + if v.Kind() == reflect.Interface { + if v.IsNil() { + buf.WriteString("null") + return nil + } + return writeCanonicalJSON(buf, v.Elem()) + } + if v.Kind() == reflect.Pointer { + if v.IsNil() { + buf.WriteString("null") + return nil + } + return writeCanonicalJSON(buf, v.Elem()) + } + if v.Type() == reflect.TypeFor[json.RawMessage]() { + raw := v.Interface().(json.RawMessage) + if len(raw) == 0 { + buf.WriteString("null") + return nil + } + dec := json.NewDecoder(bytes.NewReader(raw)) + dec.UseNumber() + var decoded any + if err := dec.Decode(&decoded); err != nil { + return err + } + return writeCanonicalJSON(buf, reflect.ValueOf(decoded)) + } + if v.Type() == reflect.TypeFor[json.Number]() { + buf.WriteString(v.Interface().(json.Number).String()) + return nil + } + switch v.Kind() { + case reflect.Bool: + buf.WriteString(strconv.FormatBool(v.Bool())) + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + buf.WriteString(strconv.FormatInt(v.Int(), 10)) + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: + buf.WriteString(strconv.FormatUint(v.Uint(), 10)) + case reflect.Float32, reflect.Float64: + data, err := json.Marshal(v.Interface()) + if err != nil { + return err + } + buf.Write(data) + case reflect.String: + data, err := json.Marshal(v.String()) + if err != nil { + return err + } + buf.Write(data) + case reflect.Slice, reflect.Array: + buf.WriteByte('[') + for i := 0; i < v.Len(); i++ { + if i > 0 { + buf.WriteByte(',') + } + if err := writeCanonicalJSON(buf, v.Index(i)); err != nil { + return err + } + } + buf.WriteByte(']') + case reflect.Map: + return writeCanonicalMap(buf, v) + case reflect.Struct: + return writeCanonicalStruct(buf, v) + default: + return fmt.Errorf("unsupported canonical JSON kind %s", v.Kind()) + } + return nil +} + +func writeCanonicalMap(buf *bytes.Buffer, v reflect.Value) error { + if v.IsNil() { + buf.WriteString("null") + return nil + } + if v.Type().Key().Kind() != reflect.String { + return fmt.Errorf("unsupported canonical map key type %s", v.Type().Key()) + } + keys := make([]string, 0, v.Len()) + for _, key := range v.MapKeys() { + keys = append(keys, key.String()) + } + sort.Strings(keys) + buf.WriteByte('{') + for i, key := range keys { + if i > 0 { + buf.WriteByte(',') + } + keyData, err := json.Marshal(key) + if err != nil { + return err + } + buf.Write(keyData) + buf.WriteByte(':') + if err := writeCanonicalJSON(buf, v.MapIndex(reflect.ValueOf(key))); err != nil { + return err + } + } + buf.WriteByte('}') + return nil +} + +type canonicalField struct { + name string + value reflect.Value +} + +func writeCanonicalStruct(buf *bytes.Buffer, v reflect.Value) error { + fields := make([]canonicalField, 0, v.NumField()) + t := v.Type() + for i := 0; i < v.NumField(); i++ { + field := t.Field(i) + if field.PkgPath != "" { + continue + } + name, omitEmpty, skip := jsonField(field) + if skip { + continue + } + value := v.Field(i) + if omitEmpty && isCanonicalEmpty(value) { + continue + } + fields = append(fields, canonicalField{name: name, value: value}) + } + sort.Slice(fields, func(i, j int) bool { + return fields[i].name < fields[j].name + }) + + buf.WriteByte('{') + for i, field := range fields { + if i > 0 { + buf.WriteByte(',') + } + name, err := json.Marshal(field.name) + if err != nil { + return err + } + buf.Write(name) + buf.WriteByte(':') + if err := writeCanonicalJSON(buf, field.value); err != nil { + return err + } + } + buf.WriteByte('}') + return nil +} + +func jsonField(field reflect.StructField) (name string, omitEmpty bool, skip bool) { + name = field.Name + tag := field.Tag.Get("json") + if tag == "-" { + return "", false, true + } + if tag == "" { + return name, false, false + } + parts := strings.Split(tag, ",") + if parts[0] != "" { + name = parts[0] + } + for _, opt := range parts[1:] { + if opt == "omitempty" { + omitEmpty = true + } + } + return name, omitEmpty, false +} + +func isCanonicalEmpty(v reflect.Value) bool { + if !v.IsValid() { + return true + } + switch v.Kind() { + case reflect.Array: + return v.Len() == 0 + case reflect.Map, reflect.Slice, reflect.String: + return v.Len() == 0 + case reflect.Bool: + return !v.Bool() + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + return v.Int() == 0 + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: + return v.Uint() == 0 + case reflect.Float32, reflect.Float64: + return v.Float() == 0 + case reflect.Interface, reflect.Pointer: + return v.IsNil() + } + return false +} + +func hashHex(data []byte) string { + sum := sha256.Sum256(data) + return hex.EncodeToString(sum[:]) +} + +func writeCompressed(path string, data []byte) error { + var buf bytes.Buffer + enc, err := zstd.NewWriter(&buf) + if err != nil { + return err + } + if _, err := enc.Write(data); err != nil { + enc.Close() + return err + } + if err := enc.Close(); err != nil { + return err + } + return writeFileAtomic(path, buf.Bytes(), 0o644) +} + +func readCompressed(path string) ([]byte, error) { + data, err := os.ReadFile(path) + if err != nil { + return nil, err + } + limit, err := compressedArtifactDecodedLimit(path) + if err != nil { + return nil, fmt.Errorf("%w: %v", errCorruptArtifact, err) + } + out, err := readCompressedBytes(data, limit) + if err != nil { + return nil, fmt.Errorf("%w: %v", errCorruptArtifact, err) + } + return out, nil +} + +func compressedArtifactDecodedLimit(path string) (int64, error) { + switch { + case strings.HasSuffix(path, manifestExtension): + return manifestDecodedLimit, nil + case strings.HasSuffix(path, segmentExtension): + return segmentDecodedLimit, nil + default: + return 0, fmt.Errorf("unknown compressed artifact extension for %s", filepath.Base(path)) + } +} + +// errArtifactPathConflict reports that an immutable artifact path already +// holds different content. Callers detect it with errors.Is. +var errArtifactPathConflict = errors.New("artifact path conflict") + +// quarantineArtifact renames a corrupt artifact aside so read paths treat it +// as missing and a valid copy can be re-fetched under the original name. The +// rename is best-effort: on failure the file stays in place and is rescanned. +func quarantineArtifact(path string) { + dst := path + quarantineSuffix + _ = os.Remove(dst) + err := os.Rename(path, dst) + switch { + case err == nil: + log.Printf("artifact: quarantined corrupt artifact %s", path) + case !errors.Is(err, fs.ErrNotExist): + log.Printf("artifact: quarantining %s: %v", path, err) + } +} + +func writeFileAtomic(path string, data []byte, perm fs.FileMode) error { + if done, err := existingArtifactMatches(path, data); err != nil || done { + return err + } + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + return err + } + tmp, err := os.CreateTemp(filepath.Dir(path), tempFilePrefix+"*") + if err != nil { + return err + } + tmpName := tmp.Name() + defer os.Remove(tmpName) + if _, err := tmp.Write(data); err != nil { + tmp.Close() + return err + } + if err := tmp.Chmod(perm); err != nil { + tmp.Close() + return err + } + if err := tmp.Close(); err != nil { + return err + } + if done, err := existingArtifactMatches(path, data); err != nil || done { + return err + } + if writeFileAtomicBeforeCommit != nil { + writeFileAtomicBeforeCommit(path) + } + if err := writeFileAtomicLink(tmpName, path); err != nil { + if errors.Is(err, fs.ErrExist) { + if done, matchErr := existingArtifactMatches(path, data); matchErr != nil || done { + return matchErr + } + return fmt.Errorf("%w at %s", errArtifactPathConflict, path) + } + if isHardLinkUnsupported(err) { + return writeFileNoReplace(path, data, perm) + } + return err + } + return nil +} + +func isHardLinkUnsupported(err error) bool { + return errors.Is(err, syscall.ENOTSUP) || + errors.Is(err, syscall.EOPNOTSUPP) || + errors.Is(err, syscall.EXDEV) || + errors.Is(err, syscall.EPERM) +} + +func writeFileNoReplace(path string, data []byte, perm fs.FileMode) error { + file, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_EXCL, perm) + if err != nil { + if errors.Is(err, fs.ErrExist) { + if done, matchErr := existingArtifactMatches(path, data); matchErr != nil || done { + return matchErr + } + return fmt.Errorf("%w at %s", errArtifactPathConflict, path) + } + return err + } + created := true + defer func() { + if created { + _ = os.Remove(path) + } + }() + if _, err := file.Write(data); err != nil { + _ = file.Close() + return err + } + if err := file.Chmod(perm); err != nil { + _ = file.Close() + return err + } + if err := file.Close(); err != nil { + return err + } + created = false + return nil +} + +func existingArtifactMatches(path string, data []byte) (bool, error) { + info, err := os.Lstat(path) + if err != nil { + if errors.Is(err, fs.ErrNotExist) { + return false, nil + } + return false, err + } + if info.IsDir() { + return false, fmt.Errorf("artifact destination %s is a directory", path) + } + if !info.Mode().IsRegular() { + return false, fmt.Errorf("artifact destination %s is not a regular file", path) + } + existing, err := os.ReadFile(path) + if err != nil { + return false, err + } + if bytes.Equal(existing, data) { + return true, nil + } + return false, fmt.Errorf("%w at %s", errArtifactPathConflict, path) +} + +// CopyUnion copies files from src into dst without deleting files that only +// exist in dst. Existing identical files are left in place. +func CopyUnion(src, dst string) error { + srcRoot, err := openArtifactRoot(src, "source") + if err != nil { + return err + } + defer srcRoot.Close() + dstRoot, err := openArtifactRoot(dst, "destination") + if err != nil { + return err + } + defer dstRoot.Close() + if err := validateDisjointRoots(srcRoot.Name(), dstRoot.Name()); err != nil { + return err + } + + return fs.WalkDir(srcRoot.FS(), ".", func(path string, ent fs.DirEntry, walkErr error) error { + if walkErr != nil { + return walkErr + } + if isTempArtifactEntry(ent.Name()) { + if ent.IsDir() { + return filepath.SkipDir + } + return nil + } + if !ent.IsDir() && strings.HasSuffix(ent.Name(), quarantineSuffix) { + return nil + } + if path == "." { + return nil + } + rel := filepath.FromSlash(path) + if ent.IsDir() { + return dstRoot.MkdirAll(rel, 0o755) + } + info, err := ent.Info() + if err != nil { + return err + } + if !info.Mode().IsRegular() { + return fmt.Errorf( + "artifact source %s is not a regular file", + filepath.Join(srcRoot.Name(), rel), + ) + } + return copyUnionFile(srcRoot, dstRoot, rel) + }) +} + +func openArtifactRoot(path, role string) (*os.Root, error) { + info, err := os.Lstat(path) + if err != nil { + return nil, err + } + if !info.IsDir() { + return nil, fmt.Errorf("artifact %s root %s is not a directory", role, path) + } + root, err := os.OpenRoot(path) + if err != nil { + return nil, fmt.Errorf("opening artifact %s root: %w", role, err) + } + openedInfo, err := root.Stat(".") + if err != nil { + _ = root.Close() + return nil, fmt.Errorf("stating artifact %s root: %w", role, err) + } + currentInfo, err := os.Lstat(path) + if err != nil { + _ = root.Close() + return nil, err + } + if !currentInfo.IsDir() || !os.SameFile(openedInfo, currentInfo) { + _ = root.Close() + return nil, fmt.Errorf("artifact %s root %s changed while opening", role, path) + } + return root, nil +} + +func openArtifactSubroot(parent *os.Root, rel, role string) (*os.Root, error) { + info, err := parent.Lstat(rel) + if err != nil { + return nil, err + } + if !info.IsDir() { + return nil, fmt.Errorf("artifact %s %s is not a directory", role, rel) + } + root, err := parent.OpenRoot(rel) + if err != nil { + return nil, fmt.Errorf("opening artifact %s: %w", role, err) + } + openedInfo, err := root.Stat(".") + if err != nil { + _ = root.Close() + return nil, fmt.Errorf("stating artifact %s: %w", role, err) + } + currentInfo, err := parent.Lstat(rel) + if err != nil { + _ = root.Close() + return nil, err + } + if !currentInfo.IsDir() || !os.SameFile(openedInfo, currentInfo) { + _ = root.Close() + return nil, fmt.Errorf("artifact %s %s changed while opening", role, rel) + } + return root, nil +} + +// copyUnionFile copies one artifact file into the destination store. Corrupt +// content-addressed artifacts are never mirrored: an invalid source is skipped +// and an invalid destination under a valid source's name is repaired, so one +// bad file cannot spread between stores or wedge the exchange. +func copyUnionFile(srcRoot, dstRoot *os.Root, rel string) error { + path := filepath.Join(srcRoot.Name(), rel) + srcData, info, err := readRootRegularFile(srcRoot, rel) + if err != nil { + return err + } + to := filepath.Join(dstRoot.Name(), rel) + existing, err := dstRoot.Lstat(rel) + if err != nil && !errors.Is(err, fs.ErrNotExist) { + return err + } + if err == nil { + if existing.IsDir() { + return fmt.Errorf("artifact destination %s is a directory", to) + } + if !existing.Mode().IsRegular() { + return fmt.Errorf("artifact destination %s is not a regular file", to) + } + if existing.Size() == info.Size() { + dstData, err := dstRoot.ReadFile(rel) + if err != nil { + return err + } + if bytes.Equal(srcData, dstData) { + return nil + } + } + return reconcileArtifactConflict(path, srcData, dstRoot, rel, info.Mode().Perm()) + } + if known, verr := validateKnownArtifact(rel, srcData); known && verr != nil { + // Quarantine at detection instead of leaving the corrupt file in + // place: the export "unchanged" fast path only checks that the + // content-addressed files exist, so an owned artifact left corrupt + // would never be regenerated, and an imported one would never be + // re-fetched. Renaming it away lets both recover on the next round. + log.Printf("artifact: not mirroring corrupt artifact %s: %v", path, verr) + quarantineArtifactRoot(srcRoot, rel) + return nil + } + return writeFileAtomicRoot(dstRoot, rel, srcData, info.Mode().Perm()) +} + +func readRootRegularFile(root *os.Root, rel string) ([]byte, fs.FileInfo, error) { + file, err := root.Open(rel) + if err != nil { + return nil, nil, err + } + defer file.Close() + info, err := file.Stat() + if err != nil { + return nil, nil, err + } + if !info.Mode().IsRegular() { + return nil, nil, fmt.Errorf( + "artifact source %s is not a regular file", filepath.Join(root.Name(), rel), + ) + } + data, err := io.ReadAll(file) + if err != nil { + return nil, nil, err + } + return data, info, nil +} + +func quarantineArtifactRoot(root *os.Root, rel string) { + dst := rel + quarantineSuffix + _ = root.Remove(dst) + err := root.Rename(rel, dst) + path := filepath.Join(root.Name(), rel) + switch { + case err == nil: + log.Printf("artifact: quarantined corrupt artifact %s", path) + case !errors.Is(err, fs.ErrNotExist): + log.Printf("artifact: quarantining %s: %v", path, err) + } +} + +// reconcileArtifactConflict handles a same-name, different-content pair. For a +// recognized artifact whose source validates and destination does not, the +// destination is repaired in place; a corrupt source is skipped instead of +// mirrored. Everything else keeps the write-once conflict error. +func reconcileArtifactConflict( + path string, + srcData []byte, + dstRoot *os.Root, + rel string, + perm fs.FileMode, +) error { + to := filepath.Join(dstRoot.Name(), rel) + known, srcErr := validateKnownArtifact(rel, srcData) + if !known { + return fmt.Errorf("%w at %s", errArtifactPathConflict, to) + } + if srcErr != nil { + log.Printf("artifact: not mirroring corrupt artifact %s: %v", path, srcErr) + return nil + } + dstData, err := dstRoot.ReadFile(rel) + if err != nil { + return err + } + if _, dstErr := validateKnownArtifact(rel, dstData); dstErr != nil { + log.Printf("artifact: repairing corrupt artifact %s from %s", to, path) + return replaceFileAtomicRoot(dstRoot, rel, srcData, perm) + } + return fmt.Errorf("%w at %s", errArtifactPathConflict, to) +} + +func writeFileAtomicRoot(root *os.Root, rel string, data []byte, perm fs.FileMode) error { + if done, err := existingArtifactMatchesRoot(root, rel, data); err != nil || done { + return err + } + if err := root.MkdirAll(filepath.Dir(rel), 0o755); err != nil { + return err + } + tmp, tmpRel, err := createRootTemp(root, filepath.Dir(rel)) + if err != nil { + return err + } + defer func() { _ = root.Remove(tmpRel) }() + if err := writeAndCloseArtifact(tmp, data, perm); err != nil { + return err + } + if done, err := existingArtifactMatchesRoot(root, rel, data); err != nil || done { + return err + } + if err := root.Link(tmpRel, rel); err != nil { + if errors.Is(err, fs.ErrExist) { + if done, matchErr := existingArtifactMatchesRoot(root, rel, data); matchErr != nil || done { + return matchErr + } + return fmt.Errorf("%w at %s", errArtifactPathConflict, filepath.Join(root.Name(), rel)) + } + if isHardLinkUnsupported(err) { + return writeFileNoReplaceRoot(root, rel, data, perm) + } + return err + } + return nil +} + +func replaceFileAtomicRoot(root *os.Root, rel string, data []byte, perm fs.FileMode) error { + if err := root.MkdirAll(filepath.Dir(rel), 0o755); err != nil { + return err + } + tmp, tmpRel, err := createRootTemp(root, filepath.Dir(rel)) + if err != nil { + return err + } + defer func() { _ = root.Remove(tmpRel) }() + if err := writeAndCloseArtifact(tmp, data, perm); err != nil { + return err + } + return root.Rename(tmpRel, rel) +} + +func createRootTemp(root *os.Root, dir string) (*os.File, string, error) { + for range 100 { + var suffix [8]byte + if _, err := rand.Read(suffix[:]); err != nil { + return nil, "", err + } + rel := filepath.Join(dir, tempFilePrefix+hex.EncodeToString(suffix[:])) + file, err := root.OpenFile(rel, os.O_RDWR|os.O_CREATE|os.O_EXCL, 0o600) + if err == nil { + return file, rel, nil + } + if !errors.Is(err, fs.ErrExist) { + return nil, "", err + } + } + return nil, "", errors.New("creating temporary artifact file: too many collisions") +} + +func writeAndCloseArtifact(file *os.File, data []byte, perm fs.FileMode) error { + if _, err := file.Write(data); err != nil { + _ = file.Close() + return err + } + if err := file.Chmod(perm); err != nil { + _ = file.Close() + return err + } + return file.Close() +} + +func writeFileNoReplaceRoot(root *os.Root, rel string, data []byte, perm fs.FileMode) error { + file, err := root.OpenFile(rel, os.O_WRONLY|os.O_CREATE|os.O_EXCL, perm) + if err != nil { + if errors.Is(err, fs.ErrExist) { + if done, matchErr := existingArtifactMatchesRoot(root, rel, data); matchErr != nil || done { + return matchErr + } + return fmt.Errorf("%w at %s", errArtifactPathConflict, filepath.Join(root.Name(), rel)) + } + return err + } + return writeAndCloseArtifact(file, data, perm) +} + +func existingArtifactMatchesRoot(root *os.Root, rel string, data []byte) (bool, error) { + info, err := root.Lstat(rel) + if err != nil { + if errors.Is(err, fs.ErrNotExist) { + return false, nil + } + return false, err + } + path := filepath.Join(root.Name(), rel) + if info.IsDir() { + return false, fmt.Errorf("artifact destination %s is a directory", path) + } + if !info.Mode().IsRegular() { + return false, fmt.Errorf("artifact destination %s is not a regular file", path) + } + existing, err := root.ReadFile(rel) + if err != nil { + return false, err + } + if bytes.Equal(existing, data) { + return true, nil + } + return false, fmt.Errorf("%w at %s", errArtifactPathConflict, path) +} + +// validateKnownArtifact validates one store file's bytes against its +// path-derived identity. known is false when rel does not name a recognized +// origin/kind/name artifact; such files are not validatable and are mirrored +// as-is for forward compatibility. +func validateKnownArtifact(rel string, data []byte) (known bool, err error) { + parts := strings.Split(filepath.ToSlash(rel), "/") + if len(parts) != 3 { + return false, nil + } + spec, err := artifactSpecForKind(parts[0], parts[1], parts[2]) + if err != nil { + return false, nil + } + return true, validateArtifactData(spec, data) +} + +func isTempArtifactEntry(name string) bool { + return strings.HasPrefix(name, tempFilePrefix) +} + +// IsFolderTarget reports whether target is a local filesystem target rather +// than a future HTTP or object-store target. +func IsFolderTarget(target string) bool { + if target == "" || strings.Contains(target, "://") { + return false + } + if isWindowsDrivePath(target) { + return true + } + _, _, err := net.SplitHostPort(target) + return err != nil +} + +func isWindowsDrivePath(target string) bool { + if len(target) < 3 || target[1] != ':' { + return false + } + c := target[0] + if (c < 'A' || c > 'Z') && (c < 'a' || c > 'z') { + return false + } + return target[2] == '\\' || target[2] == '/' +} diff --git a/internal/artifact/sync_test.go b/internal/artifact/sync_test.go new file mode 100644 index 000000000..4c02f03a0 --- /dev/null +++ b/internal/artifact/sync_test.go @@ -0,0 +1,2707 @@ +package artifact + +import ( + "context" + "errors" + "fmt" + "os" + "path/filepath" + "strings" + "syscall" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.kenn.io/agentsview/internal/db" +) + +func TestEnsureOriginPersists(t *testing.T) { + database := testDB(t) + + first, err := EnsureOrigin(database) + require.NoError(t, err) + require.NotEmpty(t, first) + require.NotEqual(t, "local", first) + + second, err := EnsureOrigin(database) + require.NoError(t, err) + assert.Equal(t, first, second) +} + +type recordingSyncStateValueReader struct { + states map[string]string + keys []string + calls int +} + +func (r *recordingSyncStateValueReader) SyncStateValues( + keys []string, +) (map[string]string, error) { + r.calls++ + r.keys = append([]string(nil), keys...) + result := make(map[string]string) + for _, key := range keys { + if value := r.states[key]; value != "" { + result[key] = value + } + } + return result, nil +} + +func TestImportedSessionIDsReadsOnlyCandidateProvenance(t *testing.T) { + reader := &recordingSyncStateValueReader{states: map[string]string{ + "artifact_import:desk-a1b2c3:desk-a1b2c3~one": "manifest-one", + "artifact_import:laptop-d4e5f6:laptop-d4e5f6~two": "manifest-two", + }} + + got, err := ImportedSessionIDs(reader, []string{ + "desk-a1b2c3~one", + "local-session", + "phone-112233~missing", + }) + require.NoError(t, err) + assert.Equal(t, map[string]struct{}{"desk-a1b2c3~one": {}}, got) + assert.Equal(t, []string{ + "artifact_import:desk-a1b2c3:desk-a1b2c3~one", + "artifact_import:phone-112233:phone-112233~missing", + }, reader.keys) + assert.Equal(t, 1, reader.calls) + + empty, err := ImportedSessionIDs(reader, nil) + require.NoError(t, err) + assert.Empty(t, empty) + assert.Equal(t, 1, reader.calls, + "a no-candidate push must not query artifact provenance") +} + +func TestIsFolderTargetAcceptsWindowsDrivePaths(t *testing.T) { + tests := []struct { + name string + target string + want bool + }{ + {"windows backslash path", `C:\Users\runner\artifacts`, true}, + {"windows slash path", `C:/Users/runner/artifacts`, true}, + {"posix path", "/tmp/agentsview-artifacts", true}, + {"relative path", "artifacts", true}, + {"http peer", "https://peer.example.test/artifacts", false}, + {"s3 target", "s3://bucket/artifacts", false}, + {"host port", "localhost:8080", false}, + {"empty", "", false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.want, IsFolderTarget(tt.target)) + }) + } +} + +func TestAdoptOriginPersistsConfigOrigin(t *testing.T) { + database := testDB(t) + + require.NoError(t, AdoptOrigin(database, "desk-a1b2c3")) + + stored, err := StoredOrigin(database) + require.NoError(t, err) + assert.Equal(t, "desk-a1b2c3", stored) + + // EnsureOrigin and its callers now agree with the adopted origin instead + // of generating a divergent DB-only value. + ensured, err := EnsureOrigin(database) + require.NoError(t, err) + assert.Equal(t, "desk-a1b2c3", ensured) +} + +func TestAdoptOriginIsIdempotent(t *testing.T) { + database := testDB(t) + + require.NoError(t, AdoptOrigin(database, "desk-a1b2c3")) + require.NoError(t, AdoptOrigin(database, "desk-a1b2c3")) + + stored, err := StoredOrigin(database) + require.NoError(t, err) + assert.Equal(t, "desk-a1b2c3", stored) +} + +func TestAdoptOriginOverwritesDivergentDBOrigin(t *testing.T) { + database := testDB(t) + + // Simulate the pre-fix state: the recorder generated a DB-only origin + // before the authoritative config origin existed. + stale, err := EnsureOrigin(database) + require.NoError(t, err) + require.NotEqual(t, "desk-a1b2c3", stale) + + require.NoError(t, AdoptOrigin(database, "desk-a1b2c3")) + + stored, err := StoredOrigin(database) + require.NoError(t, err) + assert.Equal(t, "desk-a1b2c3", stored) +} + +func TestAdoptOriginRejectsInvalidOrigin(t *testing.T) { + database := testDB(t) + + err := AdoptOrigin(database, "../outside") + require.Error(t, err) + assert.Contains(t, err.Error(), "adopting artifact origin") + + stored, err := StoredOrigin(database) + require.NoError(t, err) + assert.Empty(t, stored) +} + +func TestEnsureOriginRejectsInvalidPersistedOrigin(t *testing.T) { + database := testDB(t) + require.NoError(t, database.SetSyncState(originStateKey, "../outside")) + + origin, err := EnsureOrigin(database) + require.Error(t, err) + assert.Empty(t, origin) + assert.Contains(t, err.Error(), "stored artifact origin") + assert.Contains(t, err.Error(), "invalid artifact origin") +} + +func TestSyncFolderRoundTripImportsForeignSession(t *testing.T) { + ctx := context.Background() + share := t.TempDir() + aData := t.TempDir() + bData := t.TempDir() + aDB := testDB(t) + bDB := testDB(t) + + require.NoError(t, aDB.SetSyncState(originStateKey, "laptop-a1b2c3")) + require.NoError(t, bDB.SetSyncState(originStateKey, "desktop-d4e5f6")) + seedSession(t, aDB, "sess-1", "alpha") + + aRes, err := SyncFolder(ctx, aDB, SyncOptions{DataDir: aData, Target: share}) + require.NoError(t, err) + assert.Equal(t, "laptop-a1b2c3", aRes.Origin) + assert.Equal(t, 1, aRes.ExportedSessions) + + bRes, err := SyncFolder(ctx, bDB, SyncOptions{DataDir: bData, Target: share}) + require.NoError(t, err) + assert.Equal(t, 1, bRes.ImportedSessions) + assert.Equal(t, 2, bRes.ImportedMessages) + + bRes, err = SyncFolder(ctx, bDB, SyncOptions{DataDir: bData, Target: share}) + require.NoError(t, err) + assert.Zero(t, bRes.ImportedSessions) + assert.Zero(t, bRes.ImportedMessages) + + got, err := bDB.GetSessionFull(ctx, "laptop-a1b2c3~sess-1") + require.NoError(t, err) + require.NotNil(t, got) + assert.Equal(t, "laptop-a1b2c3", got.Machine) + assert.Equal(t, "alpha", got.Project) + + msgs, err := bDB.GetAllMessages(ctx, got.ID) + require.NoError(t, err) + require.Len(t, msgs, 2) + assert.Equal(t, "user", msgs[0].Role) + assert.Equal(t, "hello", msgs[0].Content) + assert.Equal(t, "assistant", msgs[1].Role) + assert.Equal(t, "world", msgs[1].Content) + assert.Equal(t, filepath.Join(bData, "artifacts", "laptop-a1b2c3"), filepath.Join(bData, "artifacts", got.Machine)) +} + +func TestSyncFolderRoundTripPreservesSessionName(t *testing.T) { + ctx := context.Background() + share := t.TempDir() + aData := t.TempDir() + bData := t.TempDir() + aDB := testDB(t) + bDB := testDB(t) + + require.NoError(t, aDB.SetSyncState(originStateKey, "laptop-a1b2c3")) + require.NoError(t, bDB.SetSyncState(originStateKey, "desktop-d4e5f6")) + sessionName := "Parser Provided Title" + seedSession(t, aDB, "sess-1", "alpha", func(s *db.Session) { + s.SessionName = &sessionName + }) + + _, err := SyncFolder(ctx, aDB, SyncOptions{DataDir: aData, Target: share}) + require.NoError(t, err) + + bRes, err := SyncFolder(ctx, bDB, SyncOptions{DataDir: bData, Target: share}) + require.NoError(t, err) + require.Equal(t, 1, bRes.ImportedSessions) + + got, err := bDB.GetSessionFull(ctx, "laptop-a1b2c3~sess-1") + require.NoError(t, err) + require.NotNil(t, got) + require.NotNil(t, got.SessionName) + assert.Equal(t, sessionName, *got.SessionName) +} + +func TestSyncFolderInitBaselineMetadataConvergesCuration(t *testing.T) { + ctx := context.Background() + share := t.TempDir() + aData := t.TempDir() + bData := t.TempDir() + aDB := testDB(t) + bDB := testDB(t) + + require.NoError(t, AdoptOrigin(aDB, "laptop-a1b2c3")) + require.NoError(t, AdoptOrigin(bDB, "desktop-d4e5f6")) + seedSession(t, aDB, "sess-1", "alpha") + require.NoError(t, aDB.ReplaceSessionMessages("sess-1", []db.Message{ + { + SessionID: "sess-1", + Ordinal: 0, + Role: "user", + Content: "hello", + ContentLength: 5, + SourceUUID: "uuid-question", + }, + { + SessionID: "sess-1", + Ordinal: 1, + Role: "assistant", + Content: "world", + ContentLength: 5, + SourceUUID: "uuid-answer", + }, + })) + displayName := "Already renamed" + require.NoError(t, aDB.RenameSession("sess-1", &displayName)) + starred, err := aDB.StarSession("sess-1") + require.NoError(t, err) + require.True(t, starred) + msgs, err := aDB.GetAllMessages(ctx, "sess-1") + require.NoError(t, err) + require.Len(t, msgs, 2) + note := "already pinned" + _, err = aDB.PinMessage("sess-1", msgs[1].ID, ¬e) + require.NoError(t, err) + + _, err = SyncFolder(ctx, aDB, SyncOptions{ + DataDir: aData, + Target: share, + BaselineMetadata: true, + }) + require.NoError(t, err) + res, err := SyncFolder(ctx, bDB, SyncOptions{ + DataDir: bData, + Target: share, + }) + require.NoError(t, err) + assert.Equal(t, 1, res.ImportedSessions) + assert.Equal(t, 2, res.ImportedMessages) + assert.Equal(t, 3, res.ImportedMetadata) + + gid := "laptop-a1b2c3~sess-1" + got, err := bDB.GetSessionFull(ctx, gid) + require.NoError(t, err) + require.NotNil(t, got) + require.NotNil(t, got.DisplayName) + assert.Equal(t, displayName, *got.DisplayName) + stars, err := bDB.ListStarredSessionIDs(ctx) + require.NoError(t, err) + assert.Equal(t, []string{gid}, stars) + pins, err := bDB.ListPinnedMessages(ctx, gid, "") + require.NoError(t, err) + require.Len(t, pins, 1) + assert.Equal(t, 1, pins[0].Ordinal) + require.NotNil(t, pins[0].Note) + assert.Equal(t, note, *pins[0].Note) + + op, ok, err := bDB.MetadataReplayStateOp(ctx, gid, "display_name") + require.NoError(t, err) + assert.True(t, ok) + assert.Equal(t, MetadataOpRename, op) + op, ok, err = bDB.MetadataReplayStateOp(ctx, gid, "starred") + require.NoError(t, err) + assert.True(t, ok) + assert.Equal(t, MetadataOpStar, op) + op, ok, err = bDB.MetadataReplayStateOp(ctx, gid, "pin:source_uuid:uuid-answer") + require.NoError(t, err) + assert.True(t, ok) + assert.Equal(t, MetadataOpPin, op) +} + +func TestSyncFolderInitBaselineMetadataConvergesTrash(t *testing.T) { + ctx := context.Background() + share := t.TempDir() + aData := t.TempDir() + bData := t.TempDir() + aDB := testDB(t) + bDB := testDB(t) + + require.NoError(t, AdoptOrigin(aDB, "laptop-a1b2c3")) + require.NoError(t, AdoptOrigin(bDB, "desktop-d4e5f6")) + seedSession(t, aDB, "sess-1", "alpha") + + _, err := SyncFolder(ctx, aDB, SyncOptions{ + DataDir: aData, + Target: share, + }) + require.NoError(t, err) + _, err = SyncFolder(ctx, bDB, SyncOptions{ + DataDir: bData, + Target: share, + }) + require.NoError(t, err) + + require.NoError(t, aDB.SoftDeleteSession("sess-1")) + + _, err = SyncFolder(ctx, aDB, SyncOptions{ + DataDir: aData, + Target: share, + BaselineMetadata: true, + }) + require.NoError(t, err) + res, err := SyncFolder(ctx, bDB, SyncOptions{ + DataDir: bData, + Target: share, + }) + require.NoError(t, err) + assert.Equal(t, 1, res.ImportedMetadata) + + gid := "laptop-a1b2c3~sess-1" + got, err := bDB.GetSessionFull(ctx, gid) + require.NoError(t, err) + require.NotNil(t, got) + require.NotNil(t, got.DeletedAt) + op, ok, err := bDB.MetadataReplayStateOp(ctx, gid, "deleted_at") + require.NoError(t, err) + assert.True(t, ok) + assert.Equal(t, MetadataOpSoftDelete, op) +} + +func TestSyncFolderInitBaselinesCurationOfTrashedSession(t *testing.T) { + ctx := context.Background() + share := t.TempDir() + aData := t.TempDir() + bData := t.TempDir() + aDB := testDB(t) + bDB := testDB(t) + + require.NoError(t, AdoptOrigin(aDB, "laptop-a1b2c3")) + require.NoError(t, AdoptOrigin(bDB, "desktop-d4e5f6")) + seedSession(t, aDB, "sess-1", "alpha") + require.NoError(t, aDB.ReplaceSessionMessages("sess-1", []db.Message{{ + SessionID: "sess-1", Ordinal: 0, Role: "user", Content: "hello", + ContentLength: 5, SourceUUID: "uuid-question", + }})) + displayName := "Renamed before trash" + require.NoError(t, aDB.RenameSession("sess-1", &displayName)) + starred, err := aDB.StarSession("sess-1") + require.NoError(t, err) + require.True(t, starred) + msgs, err := aDB.GetAllMessages(ctx, "sess-1") + require.NoError(t, err) + require.Len(t, msgs, 1) + note := "pinned before trash" + _, err = aDB.PinMessage("sess-1", msgs[0].ID, ¬e) + require.NoError(t, err) + require.NoError(t, aDB.SoftDeleteSession("sess-1")) + + // The session sits in trash when the machine first opts in: its curation + // must still baseline, or a later restore reaches peers without it. + _, err = SyncFolder(ctx, aDB, SyncOptions{ + DataDir: aData, + Target: share, + BaselineMetadata: true, + }) + require.NoError(t, err) + _, err = SyncFolder(ctx, bDB, SyncOptions{DataDir: bData, Target: share}) + require.NoError(t, err) + + // Restoring on A publishes the session content; B then converges on a + // visible session that kept its pre-init name, star, and pin. + _, err = aDB.RestoreSession("sess-1") + require.NoError(t, err) + recorder := NewMetadataRecorder(aDB, MetadataRecorderOptions{ + DataDir: aData, + Origin: "laptop-a1b2c3", + }) + _, err = recorder.Append(ctx, MetadataEventInput{ + SessionID: "sess-1", + Op: MetadataOpRestore, + }) + require.NoError(t, err) + _, err = SyncFolder(ctx, aDB, SyncOptions{DataDir: aData, Target: share}) + require.NoError(t, err) + _, err = SyncFolder(ctx, bDB, SyncOptions{DataDir: bData, Target: share}) + require.NoError(t, err) + + gid := "laptop-a1b2c3~sess-1" + got, err := bDB.GetSessionFull(ctx, gid) + require.NoError(t, err) + require.NotNil(t, got) + assert.Nil(t, got.DeletedAt) + require.NotNil(t, got.DisplayName) + assert.Equal(t, displayName, *got.DisplayName) + stars, err := bDB.ListStarredSessionIDs(ctx) + require.NoError(t, err) + assert.Equal(t, []string{gid}, stars) + pins, err := bDB.ListPinnedMessages(ctx, gid, "") + require.NoError(t, err) + require.Len(t, pins, 1) + require.NotNil(t, pins[0].Note) + assert.Equal(t, note, *pins[0].Note) +} + +func TestSyncFolderImportClearsSourceFileState(t *testing.T) { + ctx := context.Background() + share := t.TempDir() + aData := t.TempDir() + bData := t.TempDir() + aDB := testDB(t) + bDB := testDB(t) + sourcePath := filepath.Join(t.TempDir(), "shared-session.jsonl") + peerHash := strings64("a") + localHash := strings64("b") + lastEntryUUID := "entry-99" + + require.NoError(t, aDB.SetSyncState(originStateKey, "laptop-a1b2c3")) + require.NoError(t, bDB.SetSyncState(originStateKey, "desktop-d4e5f6")) + seedSession(t, aDB, "sess-1", "alpha", func(s *db.Session) { + s.FilePath = &sourcePath + s.FileSize = new(int64) + *s.FileSize = 4096 + s.FileMtime = new(int64) + *s.FileMtime = 200 + s.NextOrdinal = 99 + s.LastEntryUUID = &lastEntryUUID + s.FileInode = new(int64) + *s.FileInode = 12345 + s.FileDevice = new(int64) + *s.FileDevice = 67890 + s.FileHash = &peerHash + }) + seedSession(t, bDB, "local-sess", "alpha", func(s *db.Session) { + s.FilePath = &sourcePath + s.FileMtime = new(int64) + *s.FileMtime = 100 + s.FileHash = &localHash + }) + + _, err := SyncFolder(ctx, aDB, SyncOptions{DataDir: aData, Target: share}) + require.NoError(t, err) + res, err := SyncFolder(ctx, bDB, SyncOptions{DataDir: bData, Target: share}) + require.NoError(t, err) + require.Equal(t, 1, res.ImportedSessions) + + imported, err := bDB.GetSessionFull(ctx, "laptop-a1b2c3~sess-1") + require.NoError(t, err) + require.NotNil(t, imported) + assert.Nil(t, imported.FilePath) + assert.Nil(t, imported.FileSize) + assert.Nil(t, imported.FileMtime) + assert.Zero(t, imported.NextOrdinal) + assert.Nil(t, imported.LastEntryUUID) + assert.Nil(t, imported.FileInode) + assert.Nil(t, imported.FileDevice) + assert.Nil(t, imported.FileHash) + + ids, err := bDB.ListSessionIDsByFilePath(sourcePath, "claude") + require.NoError(t, err) + assert.Equal(t, []string{"local-sess"}, ids) + gotHash, ok := bDB.GetFileHashByPath(sourcePath) + require.True(t, ok) + assert.Equal(t, localHash, gotHash) +} + +func TestSyncFolderRoundTripPreservesSessionSignals(t *testing.T) { + ctx := context.Background() + share := t.TempDir() + aData := t.TempDir() + bData := t.TempDir() + aDB := testDB(t) + bDB := testDB(t) + + require.NoError(t, aDB.SetSyncState(originStateKey, "laptop-a1b2c3")) + require.NoError(t, bDB.SetSyncState(originStateKey, "desktop-d4e5f6")) + seedSession(t, aDB, "sess-1", "alpha") + + // Signal columns are written outside UpsertSession, so seed them through + // the same writer paths the live app uses. These include fields the Session + // JSON drops (json:"-"): has_tool_calls, has_context_data, and the quality + // scalars. Secret-scan state is seeded too, but unlike the others it is + // deliberately not carried across import (asserted below). + require.NoError(t, aDB.UpdateSessionSignals("sess-1", db.SessionSignalUpdate{ + HasToolCalls: true, + HasContextData: true, + Outcome: "success", + QualitySignals: db.QualitySignals{ + Version: 3, + ShortPromptCount: 2, + UnstructuredStart: true, + RunawayToolLoopCount: 1, + }, + })) + require.NoError(t, aDB.ReplaceSessionSecretFindings("sess-1", nil, 0, "rules-v7")) + + _, err := SyncFolder(ctx, aDB, SyncOptions{DataDir: aData, Target: share}) + require.NoError(t, err) + + bRes, err := SyncFolder(ctx, bDB, SyncOptions{DataDir: bData, Target: share}) + require.NoError(t, err) + require.Equal(t, 1, bRes.ImportedSessions) + + got, err := bDB.GetSessionFull(ctx, "laptop-a1b2c3~sess-1") + require.NoError(t, err) + require.NotNil(t, got) + assert.True(t, got.HasToolCalls, "has_tool_calls should survive the round trip") + assert.True(t, got.HasContextData, "has_context_data should survive the round trip") + assert.Equal(t, "success", got.Outcome) + // Secret findings are not carried in the manifest, so the imported session + // is treated as unscanned: the source rules version is dropped so + // `secrets scan --backfill` rescans it with local rules. + assert.Empty(t, got.SecretsRulesVersion, + "secret-scan state must not be restored without findings") + + qs := got.StoredQualitySignals() + require.NotNil(t, qs, "quality signals should survive the round trip") + assert.Equal(t, 3, qs.Version) + assert.Equal(t, 2, qs.ShortPromptCount) + assert.True(t, qs.UnstructuredStart) + assert.Equal(t, 1, qs.RunawayToolLoopCount) +} + +func TestSyncFolderRoundTripRewritesForeignRelationshipIDs(t *testing.T) { + ctx := context.Background() + share := t.TempDir() + aData := t.TempDir() + bData := t.TempDir() + aDB := testDB(t) + bDB := testDB(t) + + require.NoError(t, aDB.SetSyncState(originStateKey, "laptop-a1b2c3")) + require.NoError(t, bDB.SetSyncState(originStateKey, "desktop-d4e5f6")) + seedSession(t, aDB, "source-1", "alpha") + seedSession(t, aDB, "parent-1", "alpha") + seedSession(t, aDB, "child-1", "alpha") + parentID := "parent-1" + seedSession(t, aDB, "sess-1", "alpha", func(s *db.Session) { + s.SourceSessionID = "source-1" + s.ParentSessionID = &parentID + }) + require.NoError(t, aDB.ReplaceSessionMessages("sess-1", []db.Message{ + { + SessionID: "sess-1", + Ordinal: 0, + Role: "assistant", + Content: "delegating", + ContentLength: 10, + ToolCalls: []db.ToolCall{{ + ToolName: "Task", + Category: "Task", + ToolUseID: "toolu_1", + SubagentSessionID: "child-1", + ResultEvents: []db.ToolResultEvent{{ + ToolUseID: "toolu_1", + AgentID: "agent-1", + SubagentSessionID: "child-1", + Source: "tool_result", + Status: "success", + Content: "done", + ContentLength: 4, + EventIndex: 0, + }}, + }}, + }, + })) + + _, err := SyncFolder(ctx, aDB, SyncOptions{DataDir: aData, Target: share}) + require.NoError(t, err) + _, err = SyncFolder(ctx, bDB, SyncOptions{DataDir: bData, Target: share}) + require.NoError(t, err) + + got, err := bDB.GetSessionFull(ctx, "laptop-a1b2c3~sess-1") + require.NoError(t, err) + require.NotNil(t, got) + assert.Equal(t, "laptop-a1b2c3~source-1", got.SourceSessionID) + require.NotNil(t, got.ParentSessionID) + assert.Equal(t, "laptop-a1b2c3~parent-1", *got.ParentSessionID) + + msgs, err := bDB.GetAllMessages(ctx, got.ID) + require.NoError(t, err) + require.Len(t, msgs, 1) + require.Len(t, msgs[0].ToolCalls, 1) + assert.Equal(t, "laptop-a1b2c3~child-1", msgs[0].ToolCalls[0].SubagentSessionID) + require.Len(t, msgs[0].ToolCalls[0].ResultEvents, 1) + assert.Equal(t, "laptop-a1b2c3~child-1", + msgs[0].ToolCalls[0].ResultEvents[0].SubagentSessionID) +} + +// Recent Edits and edit file grouping read the persisted +// tool_calls.file_path column, so the segment format must carry it: +// imported foreign sessions get no parse-time re-derivation and the +// one-time file_path backfill has already run on existing databases. +func TestSyncFolderRoundTripPreservesToolCallFilePath(t *testing.T) { + ctx := context.Background() + share := t.TempDir() + aData := t.TempDir() + bData := t.TempDir() + aDB := testDB(t) + bDB := testDB(t) + + require.NoError(t, aDB.SetSyncState(originStateKey, "laptop-a1b2c3")) + require.NoError(t, bDB.SetSyncState(originStateKey, "desktop-d4e5f6")) + seedSession(t, aDB, "sess-1", "alpha") + require.NoError(t, aDB.ReplaceSessionMessages("sess-1", []db.Message{ + { + SessionID: "sess-1", + Ordinal: 0, + Role: "assistant", + Content: "editing", + ContentLength: 7, + ToolCalls: []db.ToolCall{{ + ToolName: "Edit", + Category: "Edit", + ToolUseID: "toolu_1", + InputJSON: `{"file_path":"src/app.go"}`, + FilePath: "src/app.go", + }}, + }, + })) + + _, err := SyncFolder(ctx, aDB, SyncOptions{DataDir: aData, Target: share}) + require.NoError(t, err) + _, err = SyncFolder(ctx, bDB, SyncOptions{DataDir: bData, Target: share}) + require.NoError(t, err) + + msgs, err := bDB.GetAllMessages(ctx, "laptop-a1b2c3~sess-1") + require.NoError(t, err) + require.Len(t, msgs, 1) + require.Len(t, msgs[0].ToolCalls, 1) + assert.Equal(t, "src/app.go", msgs[0].ToolCalls[0].FilePath) +} + +// TestSyncFolderImportLeavesScannedSessionBackfillable verifies that a session +// scanned for secrets at the source (rules version, a finding row, a nonzero +// leak count) is imported as unscanned, because the manifest carries no finding +// rows. The imported session must have no leak count and no rules version, so it +// stays a `secrets scan --backfill` candidate even when the source rules version +// is current on the importing machine. Stamping it scanned-at-source-version +// would skip a secret-bearing session, leaving a leak count with no revealable +// findings. +func TestSyncFolderImportLeavesScannedSessionBackfillable(t *testing.T) { + ctx := context.Background() + share := t.TempDir() + aData := t.TempDir() + bData := t.TempDir() + aDB := testDB(t) + bDB := testDB(t) + + require.NoError(t, aDB.SetSyncState(originStateKey, "laptop-a1b2c3")) + require.NoError(t, bDB.SetSyncState(originStateKey, "desktop-d4e5f6")) + seedSession(t, aDB, "sess-1", "alpha") + + // The source session is fully scanned: a finding row, a nonzero leak count, + // and a rules version that is also current on the importing machine below. + const rulesVersion = "rules-current" + findings := []db.SecretFinding{{ + SessionID: "sess-1", + RuleName: "aws-access-key", + Confidence: "definite", + LocationKind: "message", + MessageOrdinal: 1, + MatchStart: 4, + MatchEnd: 24, + RedactedMatch: "AKIA…MPLE", + RulesVersion: rulesVersion, + }} + require.NoError(t, aDB.ReplaceSessionSecretFindings("sess-1", findings, 1, rulesVersion)) + + _, err := SyncFolder(ctx, aDB, SyncOptions{DataDir: aData, Target: share}) + require.NoError(t, err) + + bRes, err := SyncFolder(ctx, bDB, SyncOptions{DataDir: bData, Target: share}) + require.NoError(t, err) + require.Equal(t, 1, bRes.ImportedSessions) + + const importedID = "laptop-a1b2c3~sess-1" + got, err := bDB.GetSessionFull(ctx, importedID) + require.NoError(t, err) + require.NotNil(t, got) + // Imported session must be unscanned so its state is consistent with the zero + // findings carried in the manifest. + assert.Empty(t, got.SecretsRulesVersion, "imported session must not be stamped scanned") + assert.Zero(t, got.SecretLeakCount, "imported session must not claim leaks without findings") + + // With the source rules version current on the importing machine, backfill + // must still treat the imported session as a candidate (secrets_rules_version + // "" != current) instead of skipping it. + cands, err := bDB.SecretScanCandidates(ctx, db.SecretScanCandidateFilter{ + CurrentVersion: rulesVersion, + OnlyStale: true, + }) + require.NoError(t, err) + assert.Contains(t, cands, importedID, + "secret-bearing imported session must be a backfill candidate, not skipped") +} + +// TestSyncFolderSourceLeakCountChangeKeepsLocalFindings verifies that a +// source-side secret rescan that changes only secret_leak_count (not message +// content) does not alter the artifact manifest hash, so the importer neither +// re-imports the session nor clears the findings it scanned locally. +// secret_leak_count is the only secret field carried in the Session JSON, and +// import discards secret-scan state, so it must not influence the +// content-addressed manifest. +func TestSyncFolderSourceLeakCountChangeKeepsLocalFindings(t *testing.T) { + ctx := context.Background() + share := t.TempDir() + aData := t.TempDir() + bData := t.TempDir() + aDB := testDB(t) + bDB := testDB(t) + + require.NoError(t, aDB.SetSyncState(originStateKey, "laptop-a1b2c3")) + require.NoError(t, bDB.SetSyncState(originStateKey, "desktop-d4e5f6")) + seedSession(t, aDB, "sess-1", "alpha") + + // First round trip: A exports sess-1 (no secrets yet), B imports it. + _, err := SyncFolder(ctx, aDB, SyncOptions{DataDir: aData, Target: share}) + require.NoError(t, err) + bRes, err := SyncFolder(ctx, bDB, SyncOptions{DataDir: bData, Target: share}) + require.NoError(t, err) + require.Equal(t, 1, bRes.ImportedSessions) + + const importedID = "laptop-a1b2c3~sess-1" + + // B scans the imported session locally and records a finding. + bFinding := []db.SecretFinding{{ + SessionID: importedID, RuleName: "aws-access-key", Confidence: "definite", + LocationKind: "message", MessageOrdinal: 0, MatchStart: 4, MatchEnd: 24, + RedactedMatch: "AKIA…MPLE", RulesVersion: "rules-b", + }} + require.NoError(t, bDB.ReplaceSessionSecretFindings(importedID, bFinding, 1, "rules-b")) + + // A rescans sess-1: only secret_leak_count changes (0 -> 1); the message + // content A exports is untouched. + aFinding := []db.SecretFinding{{ + SessionID: "sess-1", RuleName: "aws-access-key", Confidence: "definite", + LocationKind: "message", MessageOrdinal: 0, MatchStart: 4, MatchEnd: 24, + RedactedMatch: "AKIA…MPLE", RulesVersion: "rules-a", + }} + require.NoError(t, aDB.ReplaceSessionSecretFindings("sess-1", aFinding, 1, "rules-a")) + + // Second round trip after the source-only rescan. + _, err = SyncFolder(ctx, aDB, SyncOptions{DataDir: aData, Target: share}) + require.NoError(t, err) + bRes, err = SyncFolder(ctx, bDB, SyncOptions{DataDir: bData, Target: share}) + require.NoError(t, err) + assert.Zero(t, bRes.ImportedSessions, + "a source-only leak-count change must not re-import the session") + + // B's locally scanned findings and scan state survive. + got, err := bDB.SessionSecretFindings(ctx, importedID) + require.NoError(t, err) + assert.Len(t, got, 1, "importer's local secret findings must not be cleared") + + sess, err := bDB.GetSessionFull(ctx, importedID) + require.NoError(t, err) + require.NotNil(t, sess) + assert.Equal(t, 1, sess.SecretLeakCount, "importer's leak count preserved") + assert.Equal(t, "rules-b", sess.SecretsRulesVersion, "importer's scan version preserved") +} + +func TestExportSessionReusesPreNormalizationManifestHashForIgnoredLocalFields(t *testing.T) { + ctx := context.Background() + database := testDB(t) + root := t.TempDir() + origin := "laptop-a1b2c3" + originRoot := filepath.Join(root, origin) + seedSession(t, database, "sess-1", "alpha") + + finding := []db.SecretFinding{{ + SessionID: "sess-1", RuleName: "aws-access-key", Confidence: "definite", + LocationKind: "message", MessageOrdinal: 0, MatchStart: 0, MatchEnd: 5, + RedactedMatch: "hello", RulesVersion: "rules-a", + }} + require.NoError(t, database.ReplaceSessionSecretFindings("sess-1", finding, 1, "rules-a")) + + prevManifestHash := writePreNormalizationManifest(t, ctx, database, originRoot, origin, "sess-1") + require.NoError(t, database.SetSyncState(exportStateKey(origin, "sess-1"), prevManifestHash)) + + gotHash, changed, err := exportSession(ctx, database, originRoot, origin, "sess-1", prevManifestHash) + require.NoError(t, err) + assert.Equal(t, prevManifestHash, gotHash) + assert.False(t, changed, + "an old manifest that only differs by ignored local fields should remain the export watermark") + + manifests := globArtifacts(t, root, origin, "manifests", "*"+manifestExtension) + require.Len(t, manifests, 1) + assert.Equal(t, prevManifestHash+manifestExtension, filepath.Base(manifests[0])) +} + +func TestExportSessionRejectsCorruptPreNormalizationFastPath(t *testing.T) { + ctx := context.Background() + database := testDB(t) + root := t.TempDir() + origin := "laptop-a1b2c3" + originRoot := filepath.Join(root, origin) + seedSession(t, database, "sess-1", "alpha") + require.NoError(t, database.ReplaceSessionSecretFindings("sess-1", []db.SecretFinding{{ + SessionID: "sess-1", RuleName: "test", Confidence: "definite", + LocationKind: "message", MessageOrdinal: 0, + MatchStart: 0, MatchEnd: 5, RedactedMatch: "hello", RulesVersion: "rules-a", + }}, 1, "rules-a")) + + prevHash := writePreNormalizationManifest( + t, ctx, database, originRoot, origin, "sess-1", + ) + prev, err := readManifest(originRoot, prevHash) + require.NoError(t, err) + require.Len(t, prev.Segments, 1) + segmentPath := filepath.Join( + originRoot, KindSegments, prev.Segments[0]+segmentExtension, + ) + require.NoError(t, os.WriteFile(segmentPath, []byte("corrupt"), 0o644)) + + gotHash, changed, err := exportSession( + ctx, database, originRoot, origin, "sess-1", prevHash, + ) + require.NoError(t, err) + assert.NotEqual(t, prevHash, gotHash) + assert.True(t, changed) + assert.FileExists(t, segmentPath+quarantineSuffix) + manifest, err := readManifest(originRoot, gotHash) + require.NoError(t, err) + _, err = readManifestMessages(originRoot, manifest) + require.NoError(t, err) +} + +func TestExportSessionQuarantinesSemanticInvalidPreNormalizationManifest(t *testing.T) { + ctx := context.Background() + database := testDB(t) + root := t.TempDir() + origin := "laptop-a1b2c3" + originRoot := filepath.Join(root, origin) + seedSession(t, database, "sess-1", "alpha") + require.NoError(t, database.ReplaceSessionSecretFindings("sess-1", []db.SecretFinding{{ + SessionID: "sess-1", RuleName: "test", Confidence: "definite", + LocationKind: "message", MessageOrdinal: 0, + MatchStart: 0, MatchEnd: 5, RedactedMatch: "hello", RulesVersion: "rules-a", + }}, 1, "rules-a")) + + previousHash := writePreNormalizationManifest( + t, ctx, database, originRoot, origin, "sess-1", + ) + previous, err := readManifest(originRoot, previousHash) + require.NoError(t, err) + previous.Session.Machine = "wrong-origin" + invalidData, err := canonicalJSON(previous) + require.NoError(t, err) + invalidHash := hashHex(invalidData) + invalidPath := filepath.Join( + originRoot, KindManifests, invalidHash+manifestExtension, + ) + require.NoError(t, writeCompressed(invalidPath, invalidData)) + + decoded, err := readManifest(originRoot, invalidHash) + require.NoError(t, err, "precondition: manifest must decode and match its content hash") + assert.Equal(t, "wrong-origin", decoded.Session.Machine) + + gotHash, changed, err := exportSession( + ctx, database, originRoot, origin, "sess-1", invalidHash, + ) + require.NoError(t, err) + assert.NotEqual(t, invalidHash, gotHash) + assert.True(t, changed) + assert.NoFileExists(t, invalidPath) + assert.FileExists(t, invalidPath+quarantineSuffix) + + regenerated, err := readManifest(originRoot, gotHash) + require.NoError(t, err) + require.NoError(t, validateManifest( + regenerated, origin, origin+"~sess-1", + )) + _, err = readManifestMessages(originRoot, regenerated) + require.NoError(t, err) +} + +func TestExportSessionHealsInvalidManifestAndComputedSegmentTogether(t *testing.T) { + ctx := context.Background() + database := testDB(t) + root := t.TempDir() + origin := "laptop-a1b2c3" + originRoot := filepath.Join(root, origin) + seedSession(t, database, "sess-1", "alpha") + require.NoError(t, database.ReplaceSessionSecretFindings("sess-1", []db.SecretFinding{{ + SessionID: "sess-1", RuleName: "test", Confidence: "definite", + LocationKind: "message", MessageOrdinal: 0, + MatchStart: 0, MatchEnd: 5, RedactedMatch: "hello", RulesVersion: "rules-a", + }}, 1, "rules-a")) + + previousHash := writePreNormalizationManifest( + t, ctx, database, originRoot, origin, "sess-1", + ) + previous, err := readManifest(originRoot, previousHash) + require.NoError(t, err) + require.Len(t, previous.Segments, 1) + computedSegmentPath := filepath.Join( + originRoot, KindSegments, previous.Segments[0]+segmentExtension, + ) + + decoyData, err := encodeSegment([]db.Message{{ + Ordinal: 0, Role: "user", Content: "decoy", ContentLength: 5, + }}) + require.NoError(t, err) + decoyHash := hashHex(decoyData) + decoyPath := filepath.Join( + originRoot, KindSegments, decoyHash+segmentExtension, + ) + require.NoError(t, writeCompressed(decoyPath, decoyData)) + previous.Session.Machine = "wrong-origin" + previous.Segments = []string{decoyHash} + invalidData, err := canonicalJSON(previous) + require.NoError(t, err) + invalidHash := hashHex(invalidData) + invalidPath := filepath.Join( + originRoot, KindManifests, invalidHash+manifestExtension, + ) + require.NoError(t, writeCompressed(invalidPath, invalidData)) + + require.NoError(t, os.Remove(computedSegmentPath)) + require.NoError(t, os.WriteFile(computedSegmentPath, []byte("corrupt"), 0o644)) + + gotHash, changed, err := exportSession( + ctx, database, originRoot, origin, "sess-1", invalidHash, + ) + require.NoError(t, err) + assert.True(t, changed) + assert.NoFileExists(t, invalidPath) + assert.FileExists(t, invalidPath+quarantineSuffix) + assert.FileExists(t, computedSegmentPath) + assert.FileExists(t, computedSegmentPath+quarantineSuffix) + assert.FileExists(t, decoyPath, + "an invalid manifest must not make export follow its untrusted segment hash") + assert.NoFileExists(t, decoyPath+quarantineSuffix) + + regenerated, err := readManifest(originRoot, gotHash) + require.NoError(t, err) + _, err = readManifestMessages(originRoot, regenerated) + require.NoError(t, err) +} + +func TestReadValidExportArtifactsDefersFutureManifestWithoutQuarantine(t *testing.T) { + originRoot := t.TempDir() + origin := "laptop-a1b2c3" + segmentData, err := encodeSegment([]db.Message{{ + Ordinal: 0, Role: "user", Content: "hello", ContentLength: 5, + }}) + require.NoError(t, err) + segmentHash := hashHex(segmentData) + require.NoError(t, writeCompressed( + filepath.Join(originRoot, KindSegments, segmentHash+segmentExtension), + segmentData, + )) + m := manifest{ + Version: formatVersion + 1, + Origin: origin, + NativeSessionID: "sess-1", + Session: manifestSession{ + ID: "sess-1", + Machine: origin, + }, + Segments: []string{segmentHash}, + } + manifestData, err := canonicalJSON(m) + require.NoError(t, err) + manifestHash := hashHex(manifestData) + manifestPath := filepath.Join( + originRoot, KindManifests, manifestHash+manifestExtension, + ) + require.NoError(t, writeCompressed(manifestPath, manifestData)) + + _, valid, err := readValidExportArtifacts( + originRoot, origin, "sess-1", manifestHash, + ) + require.NoError(t, err) + assert.False(t, valid) + assert.FileExists(t, manifestPath) + assert.NoFileExists(t, manifestPath+quarantineSuffix) +} + +func TestReadValidExportArtifactsKeepsManifestForUnavailableSegment(t *testing.T) { + tests := []struct { + name string + writeSegment func(t *testing.T, originRoot string) (string, string) + segmentPresent bool + segmentQuarantined bool + }{ + { + name: "missing", + writeSegment: func(t *testing.T, originRoot string) (string, string) { + t.Helper() + hash := strings.Repeat("a", 64) + return hash, filepath.Join( + originRoot, KindSegments, hash+segmentExtension, + ) + }, + }, + { + name: "corrupt", + writeSegment: func(t *testing.T, originRoot string) (string, string) { + t.Helper() + hash := strings.Repeat("b", 64) + path := filepath.Join( + originRoot, KindSegments, hash+segmentExtension, + ) + require.NoError(t, writeCompressed(path, []byte("tampered"))) + return hash, path + }, + segmentQuarantined: true, + }, + { + name: "future version", + writeSegment: func(t *testing.T, originRoot string) (string, string) { + t.Helper() + data, err := canonicalJSON(segmentMessage{ + Version: formatVersion + 1, + Ordinal: 0, + Role: "user", + Content: "hello", + ContentLength: 5, + }) + require.NoError(t, err) + hash := hashHex(data) + path := filepath.Join( + originRoot, KindSegments, hash+segmentExtension, + ) + require.NoError(t, writeCompressed(path, data)) + return hash, path + }, + segmentPresent: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + originRoot := t.TempDir() + origin := "laptop-a1b2c3" + segmentHash, segmentPath := tt.writeSegment(t, originRoot) + m := manifest{ + Version: formatVersion, + Origin: origin, + NativeSessionID: "sess-1", + Session: manifestSession{ + ID: "sess-1", + Machine: origin, + }, + Segments: []string{segmentHash}, + } + manifestData, err := canonicalJSON(m) + require.NoError(t, err) + manifestHash := hashHex(manifestData) + manifestPath := filepath.Join( + originRoot, KindManifests, manifestHash+manifestExtension, + ) + require.NoError(t, writeCompressed(manifestPath, manifestData)) + + _, valid, err := readValidExportArtifacts( + originRoot, origin, "sess-1", manifestHash, + ) + require.NoError(t, err) + assert.False(t, valid) + assert.FileExists(t, manifestPath) + assert.NoFileExists(t, manifestPath+quarantineSuffix) + if tt.segmentPresent { + assert.FileExists(t, segmentPath) + } else { + assert.NoFileExists(t, segmentPath) + } + if tt.segmentQuarantined { + assert.FileExists(t, segmentPath+quarantineSuffix) + } else { + assert.NoFileExists(t, segmentPath+quarantineSuffix) + } + }) + } +} + +func TestSyncFolderNotifiesWhenImportWritesData(t *testing.T) { + ctx := context.Background() + share := t.TempDir() + aData := t.TempDir() + bData := t.TempDir() + aDB := testDB(t) + bDB := testDB(t) + + require.NoError(t, aDB.SetSyncState(originStateKey, "laptop-a1b2c3")) + require.NoError(t, bDB.SetSyncState(originStateKey, "desktop-d4e5f6")) + seedSession(t, aDB, "sess-1", "alpha") + + _, err := SyncFolder(ctx, aDB, SyncOptions{DataDir: aData, Target: share}) + require.NoError(t, err) + + changes := 0 + _, err = SyncFolder(ctx, bDB, SyncOptions{ + DataDir: bData, + Target: share, + OnDataChanged: func() { changes++ }, + }) + require.NoError(t, err) + assert.Equal(t, 1, changes) + + _, err = SyncFolder(ctx, bDB, SyncOptions{ + DataDir: bData, + Target: share, + OnDataChanged: func() { changes++ }, + }) + require.NoError(t, err) + assert.Equal(t, 1, changes) +} + +func TestSyncFolderUsesProvidedOrigin(t *testing.T) { + ctx := context.Background() + share := t.TempDir() + dataDir := t.TempDir() + database := testDB(t) + seedSession(t, database, "sess-1", "alpha") + + res, err := SyncFolder(ctx, database, SyncOptions{ + DataDir: dataDir, + Target: share, + Origin: "configured-a1b2c3", + }) + require.NoError(t, err) + + assert.Equal(t, "configured-a1b2c3", res.Origin) + persisted, err := database.GetSyncState(originStateKey) + require.NoError(t, err) + assert.Empty(t, persisted) + manifests := globArtifacts(t, share, "configured-a1b2c3", "manifests", "*"+manifestExtension) + assert.Len(t, manifests, 1) +} + +func TestImportMaintainsFTS(t *testing.T) { + ctx := context.Background() + root := t.TempDir() + origin := "laptop-a1b2c3" + exportDB := testDB(t) + importDB := testDB(t) + if !importDB.HasFTS() { + t.Skip("FTS unavailable") + } + seedSession(t, exportDB, "sess-1", "alpha") + + _, err := Export(ctx, exportDB, root, origin) + require.NoError(t, err) + imported, messages, err := Import(ctx, importDB, root, "desktop-d4e5f6") + require.NoError(t, err) + require.Equal(t, 1, imported) + require.Equal(t, 2, messages) + + page, err := importDB.Search(ctx, db.SearchFilter{Query: "world", Limit: 10}) + require.NoError(t, err) + require.Len(t, page.Results, 1) + assert.Equal(t, origin+"~sess-1", page.Results[0].SessionID) +} + +func TestImportPreservesPinsAndStatsOnRewrite(t *testing.T) { + ctx := context.Background() + root := t.TempDir() + origin := "laptop-a1b2c3" + exportDB := testDB(t) + importDB := testDB(t) + seedSession(t, exportDB, "sess-1", "alpha") + + _, err := Export(ctx, exportDB, root, origin) + require.NoError(t, err) + imported, messages, err := Import(ctx, importDB, root, "desktop-d4e5f6") + require.NoError(t, err) + require.Equal(t, 1, imported) + require.Equal(t, 2, messages) + + gid := origin + "~sess-1" + importedMsgs, err := importDB.GetAllMessages(ctx, gid) + require.NoError(t, err) + require.Len(t, importedMsgs, 2) + note := "keep this pin" + _, err = importDB.PinMessage(gid, importedMsgs[1].ID, ¬e) + require.NoError(t, err) + + require.NoError(t, exportDB.ReplaceSessionMessages("sess-1", []db.Message{ + {SessionID: "sess-1", Ordinal: 0, Role: "user", Content: "hello", ContentLength: 5}, + {SessionID: "sess-1", Ordinal: 1, Role: "assistant", Content: "planet", ContentLength: 6}, + })) + _, err = Export(ctx, exportDB, root, origin) + require.NoError(t, err) + imported, messages, err = Import(ctx, importDB, root, "desktop-d4e5f6") + require.NoError(t, err) + require.Equal(t, 1, imported) + require.Equal(t, 2, messages) + + pins, err := importDB.ListPinnedMessages(ctx, gid, "") + require.NoError(t, err) + require.Len(t, pins, 1) + assert.Equal(t, 1, pins[0].Ordinal) + require.NotNil(t, pins[0].Note) + assert.Equal(t, note, *pins[0].Note) + + allPins, err := importDB.ListPinnedMessages(ctx, "", "") + require.NoError(t, err) + require.Len(t, allPins, 1) + assert.Equal(t, gid, allPins[0].SessionID) + require.NotNil(t, allPins[0].Content) + assert.Equal(t, "planet", *allPins[0].Content) + + stats, err := importDB.GetStats(ctx, false, false) + require.NoError(t, err) + assert.Equal(t, 1, stats.SessionCount) + assert.Equal(t, 2, stats.MessageCount) + assert.Equal(t, 1, stats.ProjectCount) + assert.Equal(t, 1, stats.MachineCount) +} + +func TestImportDoesNotAdvanceStateForExcludedOrTrashedSessions(t *testing.T) { + ctx := context.Background() + root := t.TempDir() + origin := "laptop-a1b2c3" + exportDB := testDB(t) + seedSession(t, exportDB, "sess-1", "alpha") + _, err := Export(ctx, exportDB, root, origin) + require.NoError(t, err) + + gid := origin + "~sess-1" + tests := []struct { + name string + seed func(*testing.T, *db.DB) + }{ + { + name: "excluded", + seed: func(t *testing.T, database *db.DB) { + t.Helper() + seedSession(t, database, gid, "alpha", func(s *db.Session) { + s.Machine = origin + }) + require.NoError(t, database.DeleteSession(gid)) + }, + }, + { + name: "trashed", + seed: func(t *testing.T, database *db.DB) { + t.Helper() + seedSession(t, database, gid, "alpha", func(s *db.Session) { + s.Machine = origin + }) + require.NoError(t, database.SoftDeleteSession(gid)) + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + importDB := testDB(t) + tt.seed(t, importDB) + + imported, messages, err := Import(ctx, importDB, root, "desktop-d4e5f6") + require.NoError(t, err) + assert.Zero(t, imported) + assert.Zero(t, messages) + + state, err := importDB.GetSyncState(importStateKey(origin, gid)) + require.NoError(t, err) + assert.Empty(t, state) + }) + } +} + +func TestSyncFolderRetriesIncompleteForeignArtifacts(t *testing.T) { + ctx := context.Background() + share := t.TempDir() + aData := t.TempDir() + bData := t.TempDir() + aDB := testDB(t) + bDB := testDB(t) + + require.NoError(t, aDB.SetSyncState(originStateKey, "laptop-a1b2c3")) + require.NoError(t, bDB.SetSyncState(originStateKey, "desktop-d4e5f6")) + seedSession(t, aDB, "sess-1", "alpha") + + _, err := SyncFolder(ctx, aDB, SyncOptions{DataDir: aData, Target: share}) + require.NoError(t, err) + segments := globArtifacts(t, share, "laptop-a1b2c3", "segments", "*"+segmentExtension) + require.Len(t, segments, 1) + require.NoError(t, os.Remove(segments[0])) + + res, err := SyncFolder(ctx, bDB, SyncOptions{DataDir: bData, Target: share}) + require.NoError(t, err) + assert.Zero(t, res.ImportedSessions) + assert.Zero(t, res.ImportedMessages) + + got, err := bDB.GetSessionFull(ctx, "laptop-a1b2c3~sess-1") + require.NoError(t, err) + assert.Nil(t, got) + + require.NoError(t, CopyUnion(filepath.Join(aData, "artifacts"), share)) + res, err = SyncFolder(ctx, bDB, SyncOptions{DataDir: bData, Target: share}) + require.NoError(t, err) + assert.Equal(t, 1, res.ImportedSessions) + assert.Equal(t, 2, res.ImportedMessages) +} + +func TestSyncFolderSkipsCheckpointWithMissingManifest(t *testing.T) { + ctx := context.Background() + share := t.TempDir() + aData := t.TempDir() + bData := t.TempDir() + aDB := testDB(t) + bDB := testDB(t) + + require.NoError(t, aDB.SetSyncState(originStateKey, "laptop-a1b2c3")) + require.NoError(t, bDB.SetSyncState(originStateKey, "desktop-d4e5f6")) + seedSession(t, aDB, "sess-1", "alpha") + + _, err := SyncFolder(ctx, aDB, SyncOptions{DataDir: aData, Target: share}) + require.NoError(t, err) + manifests := globArtifacts(t, share, "laptop-a1b2c3", "manifests", "*"+manifestExtension) + require.Len(t, manifests, 1) + require.NoError(t, os.Remove(manifests[0])) + + res, err := SyncFolder(ctx, bDB, SyncOptions{DataDir: bData, Target: share}) + require.NoError(t, err) + assert.Zero(t, res.ImportedSessions) + assert.Zero(t, res.ImportedMessages) +} + +func TestImportQuarantinesMismatchedCheckpointOrigin(t *testing.T) { + ctx := context.Background() + root := t.TempDir() + origin := "laptop-a1b2c3" + database := testDB(t) + seedSession(t, database, "sess-1", "alpha") + + _, err := Export(ctx, database, root, origin) + require.NoError(t, err) + + originRoot := filepath.Join(root, origin) + cp, err := readLatestCheckpoint(originRoot) + require.NoError(t, err) + require.NotNil(t, cp) + cp.Origin = "spoofed-origin" + writeCheckpoint(t, originRoot, *cp) + checkpointPath := filepath.Join( + originRoot, KindCheckpoints, "cp-0000000001.json", + ) + + importDB := testDB(t) + imported, messages, err := Import(ctx, importDB, root, "desktop-d4e5f6") + require.NoError(t, err) + assert.Zero(t, imported) + assert.Zero(t, messages) + assert.NoFileExists(t, checkpointPath) + assert.FileExists(t, checkpointPath+quarantineSuffix) + + got, err := importDB.GetSessionFull(ctx, origin+"~sess-1") + require.NoError(t, err) + assert.Nil(t, got) +} + +func TestImportRejectsMismatchedManifestOrigin(t *testing.T) { + ctx := context.Background() + root := t.TempDir() + origin := "laptop-a1b2c3" + database := testDB(t) + seedSession(t, database, "sess-1", "alpha") + + _, err := Export(ctx, database, root, origin) + require.NoError(t, err) + + originRoot := filepath.Join(root, origin) + cp, err := readLatestCheckpoint(originRoot) + require.NoError(t, err) + require.NotNil(t, cp) + gid := origin + "~sess-1" + manifestHash := cp.Sessions[gid] + require.NotEmpty(t, manifestHash) + + m, err := readManifest(originRoot, manifestHash) + require.NoError(t, err) + m.Origin = "spoofed-origin" + manifestData, err := canonicalJSON(m) + require.NoError(t, err) + spoofedHash := hashHex(manifestData) + require.NoError(t, writeCompressed(filepath.Join(originRoot, "manifests", spoofedHash+manifestExtension), manifestData)) + cp.Sessions[gid] = spoofedHash + writeCheckpoint(t, originRoot, *cp) + + importDB := testDB(t) + imported, messages, err := Import(ctx, importDB, root, "desktop-d4e5f6") + require.Error(t, err) + assert.Contains(t, err.Error(), "manifest origin mismatch") + assert.Zero(t, imported) + assert.Zero(t, messages) + + got, err := importDB.GetSessionFull(ctx, gid) + require.NoError(t, err) + assert.Nil(t, got) +} + +func TestImportSkipsAndQuarantinesCorruptManifest(t *testing.T) { + ctx := context.Background() + root := t.TempDir() + origin := "laptop-a1b2c3" + database := testDB(t) + seedSession(t, database, "sess-1", "alpha") + seedSession(t, database, "sess-2", "beta") + + _, err := Export(ctx, database, root, origin) + require.NoError(t, err) + + originRoot := filepath.Join(root, origin) + cp, err := readLatestCheckpoint(originRoot) + require.NoError(t, err) + require.NotNil(t, cp) + manifestHash := cp.Sessions[origin+"~sess-1"] + require.NotEmpty(t, manifestHash) + m, err := readManifest(originRoot, manifestHash) + require.NoError(t, err) + m.Session.Project = "tampered" + data, err := canonicalJSON(m) + require.NoError(t, err) + path := filepath.Join(originRoot, "manifests", manifestHash+manifestExtension) + require.NoError(t, os.Remove(path)) + require.NoError(t, writeCompressed(path, data)) + + importDB := testDB(t) + imported, messages, err := Import(ctx, importDB, root, "desktop-d4e5f6") + require.NoError(t, err) + assert.Equal(t, 1, imported) + assert.Equal(t, 2, messages) + + got, err := importDB.GetSessionFull(ctx, origin+"~sess-1") + require.NoError(t, err) + assert.Nil(t, got) + got, err = importDB.GetSessionFull(ctx, origin+"~sess-2") + require.NoError(t, err) + assert.NotNil(t, got) + + assert.NoFileExists(t, path) + assert.FileExists(t, path+quarantineSuffix) + + // A later sync neither errors nor resurrects the corrupt manifest. + imported, messages, err = Import(ctx, importDB, root, "desktop-d4e5f6") + require.NoError(t, err) + assert.Zero(t, imported) + assert.Zero(t, messages) +} + +func TestImportQuarantinesInvalidCheckpointManifestReference(t *testing.T) { + ctx := context.Background() + root := t.TempDir() + origin := "laptop-a1b2c3" + importDB := testDB(t) + originRoot := filepath.Join(root, origin) + writeCheckpoint(t, originRoot, checkpoint{ + Version: formatVersion, + Origin: origin, + Sequence: 1, + Sessions: map[string]string{ + origin + "~sess-1": "../outside", + }, + }) + checkpointPath := filepath.Join( + originRoot, KindCheckpoints, "cp-0000000001.json", + ) + + imported, messages, err := Import(ctx, importDB, root, "desktop-d4e5f6") + require.NoError(t, err) + assert.Zero(t, imported) + assert.Zero(t, messages) + assert.NoFileExists(t, checkpointPath) + assert.FileExists(t, checkpointPath+quarantineSuffix) +} + +func TestImportSkipsAndQuarantinesCorruptSegment(t *testing.T) { + ctx := context.Background() + root := t.TempDir() + origin := "laptop-a1b2c3" + database := testDB(t) + seedSession(t, database, "sess-1", "alpha") + seedSession(t, database, "sess-2", "beta") + // Distinct messages so the two sessions do not share one + // content-addressed segment. + require.NoError(t, database.ReplaceSessionMessages("sess-2", []db.Message{ + {SessionID: "sess-2", Ordinal: 0, Role: "user", Content: "howdy", ContentLength: 5}, + {SessionID: "sess-2", Ordinal: 1, Role: "assistant", Content: "hi there", ContentLength: 8}, + })) + + _, err := Export(ctx, database, root, origin) + require.NoError(t, err) + + originRoot := filepath.Join(root, origin) + cp, err := readLatestCheckpoint(originRoot) + require.NoError(t, err) + require.NotNil(t, cp) + manifestHash := cp.Sessions[origin+"~sess-1"] + require.NotEmpty(t, manifestHash) + m, err := readManifest(originRoot, manifestHash) + require.NoError(t, err) + require.Len(t, m.Segments, 1) + segmentPath := filepath.Join(originRoot, "segments", m.Segments[0]+segmentExtension) + require.NoError(t, os.Remove(segmentPath)) + require.NoError(t, writeCompressed(segmentPath, []byte("{\"v\":1,\"ordinal\":0,\"role\":\"user\",\"content\":\"tampered\"}\n"))) + + importDB := testDB(t) + imported, messages, err := Import(ctx, importDB, root, "desktop-d4e5f6") + require.NoError(t, err) + assert.Equal(t, 1, imported) + assert.Equal(t, 2, messages) + + got, err := importDB.GetSessionFull(ctx, origin+"~sess-1") + require.NoError(t, err) + assert.Nil(t, got) + got, err = importDB.GetSessionFull(ctx, origin+"~sess-2") + require.NoError(t, err) + assert.NotNil(t, got) + + assert.NoFileExists(t, segmentPath) + assert.FileExists(t, segmentPath+quarantineSuffix) +} + +func TestImportFallsBackPastCorruptLatestCheckpoint(t *testing.T) { + ctx := context.Background() + root := t.TempDir() + origin := "laptop-a1b2c3" + database := testDB(t) + seedSession(t, database, "sess-1", "alpha") + + _, err := Export(ctx, database, root, origin) + require.NoError(t, err) + + originRoot := filepath.Join(root, origin) + corruptPath := filepath.Join(originRoot, "checkpoints", "cp-0000000002.json") + require.NoError(t, os.WriteFile(corruptPath, []byte("not json"), 0o644)) + + importDB := testDB(t) + imported, messages, err := Import(ctx, importDB, root, "desktop-d4e5f6") + require.NoError(t, err) + assert.Equal(t, 1, imported) + assert.Equal(t, 2, messages) + + got, err := importDB.GetSessionFull(ctx, origin+"~sess-1") + require.NoError(t, err) + assert.NotNil(t, got) + + assert.NoFileExists(t, corruptPath) + assert.FileExists(t, corruptPath+quarantineSuffix) +} + +func TestImportSkipsAndQuarantinesUndecodableManifest(t *testing.T) { + ctx := context.Background() + root := t.TempDir() + origin := "laptop-a1b2c3" + database := testDB(t) + seedSession(t, database, "sess-1", "alpha") + seedSession(t, database, "sess-2", "beta") + + _, err := Export(ctx, database, root, origin) + require.NoError(t, err) + + // A manifest whose bytes hash to the checkpoint reference but do not + // decode as JSON: hash verification passes, only the decode fails. + originRoot := filepath.Join(root, origin) + junk := []byte("not json") + junkHash := hashHex(junk) + path := filepath.Join(originRoot, "manifests", junkHash+manifestExtension) + require.NoError(t, writeCompressed(path, junk)) + cp, err := readLatestCheckpoint(originRoot) + require.NoError(t, err) + require.NotNil(t, cp) + cp.Sessions[origin+"~sess-1"] = junkHash + writeCheckpoint(t, originRoot, *cp) + + importDB := testDB(t) + imported, messages, err := Import(ctx, importDB, root, "desktop-d4e5f6") + require.NoError(t, err) + assert.Equal(t, 1, imported) + assert.Equal(t, 2, messages) + + got, err := importDB.GetSessionFull(ctx, origin+"~sess-1") + require.NoError(t, err) + assert.Nil(t, got) + got, err = importDB.GetSessionFull(ctx, origin+"~sess-2") + require.NoError(t, err) + assert.NotNil(t, got) + + assert.NoFileExists(t, path) + assert.FileExists(t, path+quarantineSuffix) +} + +func TestImportSkipsAndQuarantinesUndecodableSegment(t *testing.T) { + ctx := context.Background() + root := t.TempDir() + origin := "laptop-a1b2c3" + database := testDB(t) + seedSession(t, database, "sess-1", "alpha") + seedSession(t, database, "sess-2", "beta") + + _, err := Export(ctx, database, root, origin) + require.NoError(t, err) + + // A segment whose bytes hash to the manifest reference but do not decode + // as NDJSON: hash verification passes, only the decode fails. + originRoot := filepath.Join(root, origin) + junk := []byte("not ndjson\n") + junkHash := hashHex(junk) + segPath := filepath.Join(originRoot, "segments", junkHash+segmentExtension) + require.NoError(t, writeCompressed(segPath, junk)) + + cp, err := readLatestCheckpoint(originRoot) + require.NoError(t, err) + require.NotNil(t, cp) + m, err := readManifest(originRoot, cp.Sessions[origin+"~sess-1"]) + require.NoError(t, err) + m.Segments = []string{junkHash} + data, err := canonicalJSON(m) + require.NoError(t, err) + manifestHash := hashHex(data) + require.NoError(t, writeCompressed( + filepath.Join(originRoot, "manifests", manifestHash+manifestExtension), data)) + cp.Sessions[origin+"~sess-1"] = manifestHash + writeCheckpoint(t, originRoot, *cp) + + importDB := testDB(t) + imported, messages, err := Import(ctx, importDB, root, "desktop-d4e5f6") + require.NoError(t, err) + assert.Equal(t, 1, imported) + assert.Equal(t, 2, messages) + + got, err := importDB.GetSessionFull(ctx, origin+"~sess-1") + require.NoError(t, err) + assert.Nil(t, got) + got, err = importDB.GetSessionFull(ctx, origin+"~sess-2") + require.NoError(t, err) + assert.NotNil(t, got) + + assert.NoFileExists(t, segPath) + assert.FileExists(t, segPath+quarantineSuffix) +} + +func TestExportAdvancesPastCorruptLatestCheckpoint(t *testing.T) { + ctx := context.Background() + root := t.TempDir() + origin := "laptop-a1b2c3" + database := testDB(t) + seedSession(t, database, "sess-1", "alpha") + + _, err := Export(ctx, database, root, origin) + require.NoError(t, err) + + originRoot := filepath.Join(root, origin) + corruptPath := filepath.Join(originRoot, "checkpoints", "cp-0000000001.json") + require.NoError(t, os.WriteFile(corruptPath, []byte("not json"), 0o644)) + + seedSession(t, database, "sess-2", "beta") + _, err = Export(ctx, database, root, origin) + require.NoError(t, err, "corrupt latest checkpoint must not block export") + + // The corrupt body is quarantined so read paths fall back and a valid + // peer copy can heal it, and its sequence number is not reused: the name + // may already have been published to peers with the original content. + assert.NoFileExists(t, corruptPath) + assert.FileExists(t, corruptPath+quarantineSuffix) + cp, err := readLatestCheckpoint(originRoot) + require.NoError(t, err) + require.NotNil(t, cp) + assert.Equal(t, 2, cp.Sequence) + assert.Contains(t, cp.Sessions, origin+"~sess-1") + assert.Contains(t, cp.Sessions, origin+"~sess-2") +} + +func TestImportRejectsInvalidManifestSegmentReference(t *testing.T) { + ctx := context.Background() + root := t.TempDir() + origin := "laptop-a1b2c3" + importDB := testDB(t) + originRoot := filepath.Join(root, origin) + m := manifest{ + Version: formatVersion, + Origin: origin, + NativeSessionID: "sess-1", + Session: manifestSession{ + ID: "sess-1", + Machine: origin, + Agent: "claude", + Project: "alpha", + CreatedAt: "2026-06-14T01:02:03Z", + }, + Segments: []string{"../outside"}, + } + data, err := canonicalJSON(m) + require.NoError(t, err) + manifestHash := hashHex(data) + require.NoError(t, writeCompressed(filepath.Join(originRoot, "manifests", manifestHash+manifestExtension), data)) + writeCheckpoint(t, originRoot, checkpoint{ + Version: formatVersion, + Origin: origin, + Sequence: 1, + Sessions: map[string]string{ + origin + "~sess-1": manifestHash, + }, + }) + + imported, messages, err := Import(ctx, importDB, root, "desktop-d4e5f6") + require.Error(t, err) + assert.ErrorIs(t, err, ErrArtifactInvalid) + assert.Contains(t, err.Error(), "invalid artifact hash") + assert.Zero(t, imported) + assert.Zero(t, messages) +} + +func TestSyncFolderRejectsOverlappingRoots(t *testing.T) { + ctx := context.Background() + + tests := []struct { + name string + target func(string) string + }{ + { + name: "target is data dir", + target: func(dataDir string) string { + return dataDir + }, + }, + { + name: "target is artifact store", + target: func(dataDir string) string { + return filepath.Join(dataDir, "artifacts") + }, + }, + { + name: "target inside artifact store", + target: func(dataDir string) string { + return filepath.Join(dataDir, "artifacts", "share") + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + database := testDB(t) + dataDir := t.TempDir() + + _, err := SyncFolder(ctx, database, SyncOptions{ + DataDir: dataDir, + Target: tt.target(dataDir), + }) + require.Error(t, err) + assert.Contains(t, err.Error(), "must not overlap") + }) + } +} + +func TestSyncFolderRejectsSymlinkedAncestorResolvingInsideArtifactStore(t *testing.T) { + dataDir := t.TempDir() + localRoot := filepath.Join(dataDir, "artifacts") + targetDir := filepath.Join(localRoot, "shared") + require.NoError(t, os.MkdirAll(targetDir, 0o755)) + alias := filepath.Join(t.TempDir(), "alias") + require.NoError(t, os.Symlink(localRoot, alias)) + + err := validateDisjointRoots(localRoot, filepath.Join(alias, "shared")) + require.Error(t, err) + assert.Contains(t, err.Error(), "must not overlap") +} + +func TestExportSkipsUnchangedCheckpoint(t *testing.T) { + ctx := context.Background() + database := testDB(t) + root := t.TempDir() + origin := "laptop-a1b2c3" + seedSession(t, database, "sess-1", "alpha") + + count, err := Export(ctx, database, root, origin) + require.NoError(t, err) + assert.Equal(t, 1, count) + count, err = Export(ctx, database, root, origin) + require.NoError(t, err) + assert.Zero(t, count) + + checkpoints, err := filepath.Glob(filepath.Join(root, origin, "checkpoints", "cp-*.json")) + require.NoError(t, err) + assert.Equal(t, []string{ + filepath.Join(root, origin, "checkpoints", "cp-0000000001.json"), + }, checkpoints) + + manifests := globArtifacts(t, root, origin, "manifests", "*"+manifestExtension) + assert.Len(t, manifests, 1) +} + +func TestExportUnchangedWhenOnlySourceFileBookkeepingChanges(t *testing.T) { + ctx := context.Background() + database := testDB(t) + root := t.TempDir() + origin := "laptop-a1b2c3" + fileFields := func(path string, size, mtime, inode, device int64, hash string) func(*db.Session) { + return func(s *db.Session) { + s.FilePath = &path + s.FileSize = &size + s.FileMtime = &mtime + s.FileInode = &inode + s.FileDevice = &device + s.FileHash = &hash + } + } + seedSession(t, database, "sess-1", "alpha", + fileFields("/old/sess.jsonl", 4096, 100, 11, 22, strings64("a"))) + + count, err := Export(ctx, database, root, origin) + require.NoError(t, err) + require.Equal(t, 1, count) + cp, err := readLatestCheckpoint(filepath.Join(root, origin)) + require.NoError(t, err) + require.NotNil(t, cp) + firstHash := cp.Sessions[origin+"~sess-1"] + require.NotEmpty(t, firstHash) + + // Touch/move the source file: every file_* bookkeeping column changes + // while the exported session content stays identical. + seedSession(t, database, "sess-1", "alpha", + fileFields("/new/sess.jsonl", 8192, 200, 33, 44, strings64("b"))) + + count, err = Export(ctx, database, root, origin) + require.NoError(t, err) + assert.Zero(t, count, + "source-file bookkeeping changes must not re-export the session") + cp, err = readLatestCheckpoint(filepath.Join(root, origin)) + require.NoError(t, err) + require.NotNil(t, cp) + assert.Equal(t, firstHash, cp.Sessions[origin+"~sess-1"], + "manifest hash must not change when only file bookkeeping changes") +} + +func TestExportEmitsNewManifestAfterDataVersionChange(t *testing.T) { + ctx := context.Background() + database := testDB(t) + root := t.TempDir() + origin := "laptop-a1b2c3" + seedSession(t, database, "sess-1", "alpha") + + count, err := Export(ctx, database, root, origin) + require.NoError(t, err) + require.Equal(t, 1, count) + cp, err := readLatestCheckpoint(filepath.Join(root, origin)) + require.NoError(t, err) + require.NotNil(t, cp) + firstHash := cp.Sessions[origin+"~sess-1"] + require.NotEmpty(t, firstHash) + + require.NoError(t, database.SetSessionDataVersion("sess-1", 42)) + count, err = Export(ctx, database, root, origin) + require.NoError(t, err) + assert.Equal(t, 1, count) + cp, err = readLatestCheckpoint(filepath.Join(root, origin)) + require.NoError(t, err) + require.NotNil(t, cp) + nextHash := cp.Sessions[origin+"~sess-1"] + require.NotEmpty(t, nextHash) + assert.NotEqual(t, firstHash, nextHash) + + m, err := readManifest(filepath.Join(root, origin), nextHash) + require.NoError(t, err) + assert.Equal(t, 42, m.DataVersion) +} + +func TestExportIncludesLocalOwnedSessionClasses(t *testing.T) { + ctx := context.Background() + database := testDB(t) + root := t.TempDir() + origin := "laptop-a1b2c3" + seedSession(t, database, "file-sess", "alpha") + seedSession(t, database, "claude-ai-sess", "bravo", func(s *db.Session) { + s.Agent = "claude-ai" + }) + seedSession(t, database, "upload-sess", "charlie", func(s *db.Session) { + s.Agent = "upload" + }) + seedSession(t, database, "orphan-sess", "delta", func(s *db.Session) { + s.SourceSessionID = "missing-source" + }) + + count, err := Export(ctx, database, root, origin) + require.NoError(t, err) + assert.Equal(t, 4, count) + + cp, err := readLatestCheckpoint(filepath.Join(root, origin)) + require.NoError(t, err) + require.NotNil(t, cp) + assert.Contains(t, cp.Sessions, origin+"~file-sess") + assert.Contains(t, cp.Sessions, origin+"~claude-ai-sess") + assert.Contains(t, cp.Sessions, origin+"~upload-sess") + assert.Contains(t, cp.Sessions, origin+"~orphan-sess") +} + +func TestImportDefersFutureVersionCheckpoint(t *testing.T) { + ctx := context.Background() + root := t.TempDir() + origin := "laptop-a1b2c3" + importDB := testDB(t) + cp := checkpoint{ + Version: formatVersion + 1, + Origin: origin, + Sequence: 1, + Sessions: map[string]string{ + origin + "~sess-1": "future-manifest", + }, + } + originRoot := filepath.Join(root, origin) + require.NoError(t, os.MkdirAll(filepath.Join(originRoot, "checkpoints"), 0o755)) + data, err := canonicalJSON(cp) + require.NoError(t, err) + require.NoError(t, writeFileAtomic( + filepath.Join(originRoot, "checkpoints", "cp-0000000001.json"), + data, + 0o644, + )) + + res, err := ImportDetailed(ctx, importDB, root, "desktop-d4e5f6") + require.NoError(t, err) + assert.False(t, res.Changed()) +} + +func TestImportUsesLatestCompatibleCheckpointBeforeFutureCheckpoint(t *testing.T) { + ctx := context.Background() + root := t.TempDir() + origin := "laptop-a1b2c3" + localOrigin := "desktop-d4e5f6" + exportDB := testDB(t) + importDB := testDB(t) + seedSession(t, exportDB, "sess-1", "alpha") + + _, err := Export(ctx, exportDB, root, origin) + require.NoError(t, err) + originRoot := filepath.Join(root, origin) + writeCheckpoint(t, originRoot, checkpoint{ + Version: formatVersion + 1, + Origin: origin, + Sequence: 2, + Sessions: map[string]string{ + origin + "~future": "future-manifest", + }, + }) + + res, err := ImportDetailed(ctx, importDB, root, localOrigin) + require.NoError(t, err) + assert.Equal(t, 1, res.Sessions) + assert.Equal(t, 2, res.Messages) +} + +func TestImportQuarantinesInvalidCheckpointAndFallsBack(t *testing.T) { + tests := []struct { + name string + mutate func(checkpoint) checkpoint + }{ + { + name: "origin mismatch", + mutate: func(cp checkpoint) checkpoint { + cp.Origin = "wrong-origin" + return cp + }, + }, + { + name: "filename sequence mismatch", + mutate: func(cp checkpoint) checkpoint { + cp.Sequence = 99 + return cp + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctx := context.Background() + root := t.TempDir() + origin := "laptop-a1b2c3" + exportDB := testDB(t) + importDB := testDB(t) + seedSession(t, exportDB, "sess-1", "alpha") + + _, err := Export(ctx, exportDB, root, origin) + require.NoError(t, err) + originRoot := filepath.Join(root, origin) + first, err := readLatestCheckpoint(originRoot) + require.NoError(t, err) + require.NotNil(t, first) + invalid := tt.mutate(checkpoint{ + Version: formatVersion, + Origin: origin, + Sequence: 2, + Sessions: first.Sessions, + }) + invalidPath := filepath.Join( + originRoot, KindCheckpoints, "cp-0000000002.json", + ) + data, err := canonicalJSON(invalid) + require.NoError(t, err) + require.NoError(t, writeFileAtomic(invalidPath, data, 0o644)) + + res, err := ImportDetailed(ctx, importDB, root, "desktop-d4e5f6") + require.NoError(t, err) + assert.Equal(t, 1, res.Sessions) + assert.FileExists(t, invalidPath+quarantineSuffix) + assert.NoFileExists(t, invalidPath) + }) + } +} + +func TestImportDefersFutureVersionManifest(t *testing.T) { + ctx := context.Background() + root := t.TempDir() + origin := "laptop-a1b2c3" + importDB := testDB(t) + originRoot := filepath.Join(root, origin) + m := manifest{ + Version: formatVersion + 1, + Origin: origin, + NativeSessionID: "sess-1", + Session: manifestSession{ + ID: "sess-1", + Machine: origin, + }, + Segments: []string{"future-segment"}, + } + data, err := canonicalJSON(m) + require.NoError(t, err) + hash := hashHex(data) + require.NoError(t, writeCompressed(filepath.Join(originRoot, "manifests", hash+manifestExtension), data)) + writeCheckpoint(t, originRoot, checkpoint{ + Version: formatVersion, + Origin: origin, + Sequence: 1, + Sessions: map[string]string{ + origin + "~sess-1": hash, + }, + }) + + res, err := ImportDetailed(ctx, importDB, root, "desktop-d4e5f6") + require.NoError(t, err) + assert.False(t, res.Changed()) + state, err := importDB.GetSyncState(importStateKey(origin, origin+"~sess-1")) + require.NoError(t, err) + assert.Empty(t, state) +} + +func TestImportDefersFutureVersionSegment(t *testing.T) { + ctx := context.Background() + root := t.TempDir() + origin := "laptop-a1b2c3" + importDB := testDB(t) + originRoot := filepath.Join(root, origin) + segmentData, err := canonicalJSON(segmentMessage{ + Version: formatVersion + 1, + Ordinal: 0, + Role: "user", + Content: "hello", + ContentLength: 5, + }) + require.NoError(t, err) + segmentHash := hashHex(segmentData) + require.NoError(t, writeCompressed(filepath.Join(originRoot, "segments", segmentHash+segmentExtension), segmentData)) + m := manifest{ + Version: formatVersion, + Origin: origin, + NativeSessionID: "sess-1", + Session: manifestSession{ + ID: "sess-1", + Machine: origin, + }, + Segments: []string{segmentHash}, + } + manifestData, err := canonicalJSON(m) + require.NoError(t, err) + manifestHash := hashHex(manifestData) + require.NoError(t, writeCompressed(filepath.Join(originRoot, "manifests", manifestHash+manifestExtension), manifestData)) + writeCheckpoint(t, originRoot, checkpoint{ + Version: formatVersion, + Origin: origin, + Sequence: 1, + Sessions: map[string]string{ + origin + "~sess-1": manifestHash, + }, + }) + + res, err := ImportDetailed(ctx, importDB, root, "desktop-d4e5f6") + require.NoError(t, err) + assert.False(t, res.Changed()) + state, err := importDB.GetSyncState(importStateKey(origin, origin+"~sess-1")) + require.NoError(t, err) + assert.Empty(t, state) +} + +func TestExportScrubsUnstableArtifactIDs(t *testing.T) { + ctx := context.Background() + database := testDB(t) + root := t.TempDir() + origin := "laptop-a1b2c3" + seedSession(t, database, "sess-1", "alpha") + require.NoError(t, database.ReplaceSessionUsageEvents("sess-1", []db.UsageEvent{ + { + SessionID: "sess-1", + Source: "fixture", + Model: "claude-test", + InputTokens: 1, + OccurredAt: "2026-06-14T01:02:04Z", + DedupKey: "usage-1", + }, + })) + + _, err := Export(ctx, database, root, origin) + require.NoError(t, err) + cp, err := readLatestCheckpoint(filepath.Join(root, origin)) + require.NoError(t, err) + require.NotNil(t, cp) + manifestHash := cp.Sessions[origin+"~sess-1"] + require.NotEmpty(t, manifestHash) + + m, err := readManifest(filepath.Join(root, origin), manifestHash) + require.NoError(t, err) + require.Len(t, m.UsageEvents, 1) + manifestData, err := canonicalJSON(m) + require.NoError(t, err) + assert.NotContains(t, string(manifestData), `"ID"`) + assert.NotContains(t, string(manifestData), `"SessionID"`) + + msgs, err := readManifestMessages(filepath.Join(root, origin), m) + require.NoError(t, err) + require.Len(t, msgs, 2) + for _, msg := range msgs { + assert.Zero(t, msg.ID) + assert.Empty(t, msg.SessionID) + } +} + +func TestExportRejectsInvalidOriginBeforeCreatingPaths(t *testing.T) { + ctx := context.Background() + database := testDB(t) + parent := t.TempDir() + root := filepath.Join(parent, "artifacts") + seedSession(t, database, "sess-1", "alpha") + + count, err := Export(ctx, database, root, "../outside") + require.Error(t, err) + assert.Zero(t, count) + assert.Contains(t, err.Error(), "invalid artifact origin") + + _, statErr := os.Stat(filepath.Join(parent, "outside")) + assert.ErrorIs(t, statErr, os.ErrNotExist) +} + +func TestExportSkipsForeignSessions(t *testing.T) { + ctx := context.Background() + database := testDB(t) + dataDir := t.TempDir() + share := t.TempDir() + seedSession(t, database, "other~sess-1", "alpha", func(s *db.Session) { + s.Machine = "other" + }) + seedSession(t, database, "legacy-remote~sess-2", "bravo", func(s *db.Session) { + s.Machine = "remote" + }) + + res, err := SyncFolder(ctx, database, SyncOptions{DataDir: dataDir, Target: share}) + require.NoError(t, err) + assert.Zero(t, res.ExportedSessions) + + origin := res.Origin + manifests := globArtifacts(t, filepath.Join(dataDir, "artifacts"), origin, "manifests", "*"+manifestExtension) + assert.Empty(t, manifests) +} + +func TestCopyUnionRejectsPathConflicts(t *testing.T) { + src := t.TempDir() + dst := t.TempDir() + // An unrecognized artifact shape cannot be content-validated, so the + // write-once conflict error is preserved as-is. + rel := filepath.Join("origin", "blobs", "data.bin") + require.NoError(t, os.MkdirAll(filepath.Join(src, filepath.Dir(rel)), 0o755)) + require.NoError(t, os.MkdirAll(filepath.Join(dst, filepath.Dir(rel)), 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(src, rel), []byte("alpha"), 0o644)) + require.NoError(t, os.WriteFile(filepath.Join(dst, rel), []byte("bravo"), 0o644)) + + err := CopyUnion(src, dst) + require.Error(t, err) + assert.ErrorIs(t, err, errArtifactPathConflict) +} + +func TestCopyUnionDetectsCheckpointSequenceConflict(t *testing.T) { + src := t.TempDir() + dst := t.TempDir() + rel := filepath.Join("origin", "checkpoints", "cp-0000000001.json") + require.NoError(t, os.MkdirAll(filepath.Join(src, filepath.Dir(rel)), 0o755)) + require.NoError(t, os.MkdirAll(filepath.Join(dst, filepath.Dir(rel)), 0o755)) + hashA := hashHex([]byte("a")) + hashB := hashHex([]byte("b")) + srcData, err := canonicalJSON(checkpoint{ + Version: formatVersion, Origin: "origin", Sequence: 1, + Sessions: map[string]string{"origin~a": hashA}, + }) + require.NoError(t, err) + dstData, err := canonicalJSON(checkpoint{ + Version: formatVersion, Origin: "origin", Sequence: 1, + Sessions: map[string]string{"origin~b": hashB}, + }) + require.NoError(t, err) + require.NoError(t, os.WriteFile(filepath.Join(src, rel), srcData, 0o644)) + require.NoError(t, os.WriteFile(filepath.Join(dst, rel), dstData, 0o644)) + + // Two valid checkpoints diverging under the same name is real divergence, + // not corruption, and must keep failing loudly. + err = CopyUnion(src, dst) + require.Error(t, err) + assert.ErrorIs(t, err, errArtifactPathConflict) +} + +func TestWriteFileAtomicIsWriteOnce(t *testing.T) { + path := filepath.Join(t.TempDir(), "origin", "segments", "artifact.ndjson.zst") + first := []byte("first") + second := []byte("second") + + require.NoError(t, writeFileAtomic(path, first, 0o644)) + require.NoError(t, writeFileAtomic(path, first, 0o644)) + err := writeFileAtomic(path, second, 0o644) + require.Error(t, err) + assert.ErrorIs(t, err, errArtifactPathConflict) + got, readErr := os.ReadFile(path) + require.NoError(t, readErr) + assert.Equal(t, first, got) +} + +func TestWriteFileAtomicDoesNotReplaceFileCreatedDuringCommit(t *testing.T) { + path := filepath.Join(t.TempDir(), "origin", "segments", "artifact.ndjson.zst") + first := []byte("first") + second := []byte("second") + + writeFileAtomicBeforeCommit = func(commitPath string) { + require.Equal(t, path, commitPath) + require.NoError(t, os.WriteFile(path, first, 0o644)) + } + t.Cleanup(func() { writeFileAtomicBeforeCommit = nil }) + + err := writeFileAtomic(path, second, 0o644) + require.Error(t, err) + assert.ErrorIs(t, err, errArtifactPathConflict) + got, readErr := os.ReadFile(path) + require.NoError(t, readErr) + assert.Equal(t, first, got) +} + +func TestWriteFileAtomicFallsBackWhenHardLinksUnsupported(t *testing.T) { + path := filepath.Join(t.TempDir(), "origin", "segments", "artifact.ndjson.zst") + data := []byte("payload") + + origLink := writeFileAtomicLink + writeFileAtomicLink = func(_, _ string) error { + return &os.LinkError{Op: "link", Err: syscall.ENOTSUP} + } + t.Cleanup(func() { writeFileAtomicLink = origLink }) + + require.NoError(t, writeFileAtomic(path, data, 0o644)) + got, err := os.ReadFile(path) + require.NoError(t, err) + assert.Equal(t, data, got) +} + +func TestCopyUnionRejectsNonRegularSources(t *testing.T) { + src := t.TempDir() + dst := t.TempDir() + target := filepath.Join(t.TempDir(), "target.txt") + require.NoError(t, os.WriteFile(target, []byte("secret"), 0o644)) + link := filepath.Join(src, "link.txt") + if err := os.Symlink(target, link); err != nil { + t.Skipf("symlink creation unavailable: %v", err) + } + + err := CopyUnion(src, dst) + require.Error(t, err) + assert.Contains(t, err.Error(), "not a regular file") +} + +func TestCopyUnionRejectsSourceReplacedByEscapingSymlink(t *testing.T) { + src := t.TempDir() + dst := t.TempDir() + rel := filepath.Join("origin", "blobs", "data.bin") + path := filepath.Join(src, rel) + require.NoError(t, os.MkdirAll(filepath.Dir(path), 0o755)) + require.NoError(t, os.WriteFile(path, []byte("artifact"), 0o644)) + secret := filepath.Join(t.TempDir(), "secret.txt") + require.NoError(t, os.WriteFile(secret, []byte("private"), 0o600)) + require.NoError(t, os.Remove(path)) + if err := os.Symlink(secret, path); err != nil { + t.Skipf("symlink creation unavailable: %v", err) + } + srcRoot, err := os.OpenRoot(src) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, srcRoot.Close()) }) + dstRoot, err := os.OpenRoot(dst) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, dstRoot.Close()) }) + + err = copyUnionFile(srcRoot, dstRoot, rel) + + require.Error(t, err) + assert.NoFileExists(t, filepath.Join(dst, rel)) +} + +func TestCopyUnionRejectsNonRegularDestinations(t *testing.T) { + src := t.TempDir() + dst := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(src, "artifact.txt"), []byte("secret"), 0o644)) + target := filepath.Join(t.TempDir(), "target.txt") + require.NoError(t, os.WriteFile(target, []byte("secret"), 0o644)) + link := filepath.Join(dst, "artifact.txt") + if err := os.Symlink(target, link); err != nil { + t.Skipf("symlink creation unavailable: %v", err) + } + + err := CopyUnion(src, dst) + require.Error(t, err) + assert.Contains(t, err.Error(), "not a regular file") +} + +func TestCopyUnionConfinesDestinationSymlinksToRoot(t *testing.T) { + src := t.TempDir() + dst := t.TempDir() + outside := t.TempDir() + rel := filepath.Join("origin", "blobs", "data.bin") + require.NoError(t, os.MkdirAll(filepath.Join(src, filepath.Dir(rel)), 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(src, rel), []byte("artifact"), 0o644)) + if err := os.Symlink(outside, filepath.Join(dst, "origin")); err != nil { + t.Skipf("symlink creation unavailable: %v", err) + } + + err := CopyUnion(src, dst) + + require.Error(t, err) + assert.NoFileExists(t, filepath.Join(outside, "blobs", "data.bin")) +} + +func TestCopyUnionSkipsAtomicTempFiles(t *testing.T) { + src := t.TempDir() + dst := t.TempDir() + rel := filepath.Join("origin", "segments", "artifact.ndjson.zst") + tmpRel := filepath.Join("origin", "segments", tempFilePrefix+"leftover") + require.NoError(t, os.MkdirAll(filepath.Join(src, "origin", "segments"), 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(src, rel), []byte("artifact"), 0o644)) + require.NoError(t, os.WriteFile(filepath.Join(src, tmpRel), []byte("partial"), 0o644)) + + require.NoError(t, CopyUnion(src, dst)) + + got, err := os.ReadFile(filepath.Join(dst, rel)) + require.NoError(t, err) + assert.Equal(t, []byte("artifact"), got) + _, err = os.Stat(filepath.Join(dst, tmpRel)) + assert.True(t, os.IsNotExist(err), "temp artifact should not be copied") +} + +func TestCopyUnionRepeatedExchangeLeavesExistingArtifactsAndCopiesMissing(t *testing.T) { + src := t.TempDir() + dst := t.TempDir() + existingRel := filepath.Join("origin", "manifests", "existing.json.zst") + missingRel := filepath.Join("origin", "checkpoints", "cp-0000000002.json") + require.NoError(t, os.MkdirAll(filepath.Join(src, filepath.Dir(existingRel)), 0o755)) + require.NoError(t, os.MkdirAll(filepath.Join(src, filepath.Dir(missingRel)), 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(src, existingRel), []byte("manifest"), 0o644)) + + require.NoError(t, CopyUnion(src, dst)) + dstExisting := filepath.Join(dst, existingRel) + pinnedMtime := time.Unix(1_700_000_000, 0) + require.NoError(t, os.Chtimes(dstExisting, pinnedMtime, pinnedMtime)) + checkpointData, err := canonicalJSON(checkpoint{Version: formatVersion, Origin: "origin", Sequence: 2}) + require.NoError(t, err) + require.NoError(t, os.WriteFile(filepath.Join(src, missingRel), checkpointData, 0o644)) + require.NoError(t, os.WriteFile(filepath.Join(src, "origin", tempFilePrefix+"leftover"), []byte("partial"), 0o644)) + + require.NoError(t, CopyUnion(src, dst)) + + info, err := os.Stat(dstExisting) + require.NoError(t, err) + assert.True(t, info.ModTime().Equal(pinnedMtime), "existing artifact should not be rewritten") + got, err := os.ReadFile(filepath.Join(dst, missingRel)) + require.NoError(t, err) + assert.Equal(t, checkpointData, got) + _, err = os.Stat(filepath.Join(dst, "origin", tempFilePrefix+"leftover")) + assert.True(t, os.IsNotExist(err), "stale temp file should not be copied while resuming") +} + +func TestCopyUnionSkipsCorruptAndQuarantinedArtifacts(t *testing.T) { + ctx := context.Background() + src := t.TempDir() + dst := t.TempDir() + origin := "laptop-a1b2c3" + database := testDB(t) + seedSession(t, database, "sess-1", "alpha") + + _, err := Export(ctx, database, src, origin) + require.NoError(t, err) + + originRoot := filepath.Join(src, origin) + segments := globArtifacts(t, src, origin, "segments", "*"+segmentExtension) + require.Len(t, segments, 1) + require.NoError(t, os.Remove(segments[0])) + require.NoError(t, writeCompressed(segments[0], []byte("tampered"))) + quarantined := filepath.Join(originRoot, "manifests", "leftover.json.zst"+quarantineSuffix) + require.NoError(t, os.WriteFile(quarantined, []byte("junk"), 0o644)) + // A forged metadata event with a valid hash but an unparseable HLC must + // not propagate between stores (#1034). + forgedHLC := "9999-99-99T999999.000000000Z-00000000000000000000" + forged := replayRenameEvent(t, origin, origin+"~sess-1", forgedHLC, "PWNED") + forgedData, err := canonicalJSON(forged) + require.NoError(t, err) + forgedName := forgedHLC + "-" + hashHex(forgedData) + metadataEventExtension + require.NoError(t, writeFileAtomic( + filepath.Join(originRoot, "meta", forgedName), forgedData, 0o644)) + + require.NoError(t, CopyUnion(src, dst)) + + assert.NoFileExists(t, filepath.Join(dst, origin, "segments", filepath.Base(segments[0]))) + assert.NoFileExists(t, filepath.Join(dst, origin, "manifests", "leftover.json.zst"+quarantineSuffix)) + assert.NoFileExists(t, filepath.Join(dst, origin, "meta", forgedName)) + assert.Len(t, globArtifacts(t, dst, origin, "manifests", "*"+manifestExtension), 1) + assert.Len(t, globArtifacts(t, dst, origin, "checkpoints", "cp-*.json"), 1) +} + +func TestCopyUnionQuarantinesCorruptSourceAndExportRegenerates(t *testing.T) { + ctx := context.Background() + src := t.TempDir() + dst := t.TempDir() + origin := "laptop-a1b2c3" + database := testDB(t) + seedSession(t, database, "sess-1", "alpha") + + _, err := Export(ctx, database, src, origin) + require.NoError(t, err) + + segments := globArtifacts(t, src, origin, "segments", "*"+segmentExtension) + require.Len(t, segments, 1) + require.NoError(t, os.Remove(segments[0])) + require.NoError(t, writeCompressed(segments[0], []byte("tampered"))) + + require.NoError(t, CopyUnion(src, dst)) + + // The corrupt source is quarantined at detection, not left in place, so + // the export "unchanged" fast path stops seeing it as present. + assert.NoFileExists(t, segments[0]) + assert.FileExists(t, segments[0]+quarantineSuffix) + assert.NoFileExists(t, filepath.Join(dst, origin, "segments", filepath.Base(segments[0]))) + + // The next export regenerates the segment instead of treating the session + // as unchanged, and the following exchange publishes it. + _, err = Export(ctx, database, src, origin) + require.NoError(t, err) + assert.FileExists(t, segments[0]) + require.NoError(t, CopyUnion(src, dst)) + assert.FileExists(t, filepath.Join(dst, origin, "segments", filepath.Base(segments[0]))) +} + +func TestExportQuarantinesAndRegeneratesCorruptUnchangedArtifacts(t *testing.T) { + ctx := context.Background() + for _, kind := range []string{KindManifests, KindSegments} { + t.Run(kind, func(t *testing.T) { + root := t.TempDir() + origin := "laptop-a1b2c3" + database := testDB(t) + seedSession(t, database, "sess-1", "alpha") + _, err := Export(ctx, database, root, origin) + require.NoError(t, err) + + pattern := "*" + manifestExtension + if kind == KindSegments { + pattern = "*" + segmentExtension + } + paths := globArtifacts(t, root, origin, kind, pattern) + require.Len(t, paths, 1) + path := paths[0] + require.NoError(t, os.WriteFile(path, []byte("corrupt in place"), 0o644)) + + exported, err := Export(ctx, database, root, origin) + require.NoError(t, err) + assert.Equal(t, 1, exported, + "regeneration must count the session as changed") + assert.FileExists(t, path) + assert.FileExists(t, path+quarantineSuffix) + + manifestPaths := globArtifacts( + t, root, origin, KindManifests, "*"+manifestExtension, + ) + require.Len(t, manifestPaths, 1) + manifestHash := strings.TrimSuffix( + filepath.Base(manifestPaths[0]), manifestExtension, + ) + manifest, err := readManifest(filepath.Join(root, origin), manifestHash) + require.NoError(t, err) + messages, err := readManifestMessages(filepath.Join(root, origin), manifest) + require.NoError(t, err) + require.Len(t, messages, 2) + assert.Equal(t, "hello", messages[0].Content) + }) + } +} + +func TestCopyUnionRepairsCorruptDestinationArtifact(t *testing.T) { + ctx := context.Background() + src := t.TempDir() + dst := t.TempDir() + origin := "laptop-a1b2c3" + database := testDB(t) + seedSession(t, database, "sess-1", "alpha") + + _, err := Export(ctx, database, src, origin) + require.NoError(t, err) + require.NoError(t, CopyUnion(src, dst)) + + segments := globArtifacts(t, dst, origin, "segments", "*"+segmentExtension) + require.Len(t, segments, 1) + require.NoError(t, os.Remove(segments[0])) + require.NoError(t, writeCompressed(segments[0], []byte("tampered"))) + + require.NoError(t, CopyUnion(src, dst)) + + want, err := os.ReadFile(filepath.Join(src, origin, "segments", filepath.Base(segments[0]))) + require.NoError(t, err) + got, err := os.ReadFile(segments[0]) + require.NoError(t, err) + assert.Equal(t, want, got, "valid source should repair corrupt destination") +} + +func TestSyncFolderHealsCorruptSegmentInShare(t *testing.T) { + ctx := context.Background() + share := t.TempDir() + aData := t.TempDir() + bData := t.TempDir() + aDB := testDB(t) + bDB := testDB(t) + + require.NoError(t, aDB.SetSyncState(originStateKey, "laptop-a1b2c3")) + require.NoError(t, bDB.SetSyncState(originStateKey, "desktop-d4e5f6")) + seedSession(t, aDB, "sess-1", "alpha") + + _, err := SyncFolder(ctx, aDB, SyncOptions{DataDir: aData, Target: share}) + require.NoError(t, err) + + // Corrupt A's segment in the share, as an interrupted file-sync write would. + segments := globArtifacts(t, share, "laptop-a1b2c3", "segments", "*"+segmentExtension) + require.Len(t, segments, 1) + require.NoError(t, os.Remove(segments[0])) + require.NoError(t, writeCompressed(segments[0], []byte("tampered"))) + + // B tolerates the corrupt share: sync succeeds, the poison is not + // mirrored into B's local store, and repeat runs stay healthy. + res, err := SyncFolder(ctx, bDB, SyncOptions{DataDir: bData, Target: share}) + require.NoError(t, err) + assert.Zero(t, res.ImportedSessions) + assert.NoFileExists(t, filepath.Join(bData, "artifacts", "laptop-a1b2c3", "segments", filepath.Base(segments[0]))) + _, err = SyncFolder(ctx, bDB, SyncOptions{DataDir: bData, Target: share}) + require.NoError(t, err) + + // A still holds the valid copy and repairs the share on its next sync. + _, err = SyncFolder(ctx, aDB, SyncOptions{DataDir: aData, Target: share}) + require.NoError(t, err) + want, err := os.ReadFile(filepath.Join(aData, "artifacts", "laptop-a1b2c3", "segments", filepath.Base(segments[0]))) + require.NoError(t, err) + got, err := os.ReadFile(segments[0]) + require.NoError(t, err) + require.Equal(t, want, got, "publisher should repair the corrupt share copy") + + // With the share healed, B converges. + res, err = SyncFolder(ctx, bDB, SyncOptions{DataDir: bData, Target: share}) + require.NoError(t, err) + assert.Equal(t, 1, res.ImportedSessions) + assert.Equal(t, 2, res.ImportedMessages) +} + +func globArtifacts(t *testing.T, root, origin, kind, pattern string) []string { + t.Helper() + paths, err := filepath.Glob(filepath.Join(root, origin, kind, pattern)) + require.NoError(t, err) + return paths +} + +func writeCheckpoint(t *testing.T, originRoot string, cp checkpoint) { + t.Helper() + data, err := canonicalJSON(cp) + require.NoError(t, err) + path := filepath.Join(originRoot, "checkpoints", fmt.Sprintf("cp-%010d.json", cp.Sequence)) + if err := os.Remove(path); err != nil && !errors.Is(err, os.ErrNotExist) { + require.NoError(t, err) + } + require.NoError(t, writeFileAtomic(path, data, 0o644)) +} + +func writePreNormalizationManifest( + t *testing.T, + ctx context.Context, + database *db.DB, + originRoot, origin, sessionID string, +) string { + t.Helper() + sess, err := database.GetSessionFull(ctx, sessionID) + require.NoError(t, err) + require.NotNil(t, sess) + msgs, err := database.GetAllMessages(ctx, sessionID) + require.NoError(t, err) + usageEvents, err := database.GetUsageEvents(ctx, sessionID) + require.NoError(t, err) + segmentData, err := encodeSegment(canonicalMessages(msgs)) + require.NoError(t, err) + segmentHash := hashHex(segmentData) + require.NoError(t, writeCompressed( + filepath.Join(originRoot, "segments", segmentHash+segmentExtension), + segmentData, + )) + + wireSession := manifestSessionFromDB(*sess) + wireSession.Machine = origin + require.NotZero(t, wireSession.SecretLeakCount, "precondition: old manifest carries source scan count") + require.NotNil(t, wireSession.LocalModifiedAt, "precondition: old manifest carries local watermark") + m := manifest{ + Version: formatVersion, + Origin: origin, + NativeSessionID: sessionID, + Session: wireSession, + SessionName: sess.SessionName, + Segments: []string{segmentHash}, + UsageEvents: canonicalUsageEvents(usageEvents), + DataVersion: sess.DataVersion, + Generation: 1, + SessionHasToolCalls: sess.HasToolCalls, + SessionHasContextData: sess.HasContextData, + SessionQualitySignals: manifestQualitySignalsFromDB(sess.StoredQualitySignals()), + } + manifestData, err := canonicalJSON(m) + require.NoError(t, err) + manifestHash := hashHex(manifestData) + require.NoError(t, writeCompressed( + filepath.Join(originRoot, "manifests", manifestHash+manifestExtension), + manifestData, + )) + return manifestHash +} + +func testDB(t *testing.T) *db.DB { + t.Helper() + database, err := db.Open(filepath.Join(t.TempDir(), "test.db")) + require.NoError(t, err) + t.Cleanup(func() { database.Close() }) + return database +} + +func seedSession(t *testing.T, database *db.DB, id, project string, opts ...func(*db.Session)) { + t.Helper() + sess := db.Session{ + ID: id, + Project: project, + Machine: "local", + Agent: "claude", + MessageCount: 2, + UserMessageCount: 1, + FirstMessage: new("hello"), + StartedAt: new("2026-06-14T01:02:03Z"), + EndedAt: new("2026-06-14T01:03:03Z"), + SessionName: new("Test Session"), + CreatedAt: "2026-06-14T01:02:03Z", + } + for _, opt := range opts { + opt(&sess) + } + require.NoError(t, database.UpsertSession(sess)) + require.NoError(t, database.ReplaceSessionMessages(id, []db.Message{ + {SessionID: id, Ordinal: 0, Role: "user", Content: "hello", ContentLength: 5}, + {SessionID: id, Ordinal: 1, Role: "assistant", Content: "world", ContentLength: 5}, + })) +} diff --git a/internal/artifact/transport.go b/internal/artifact/transport.go new file mode 100644 index 000000000..922d6e20e --- /dev/null +++ b/internal/artifact/transport.go @@ -0,0 +1,126 @@ +package artifact + +import ( + "bytes" + "context" + "errors" + "fmt" + "log" + "os" + "path/filepath" +) + +// Transport exchanges immutable, content-addressed artifacts between the local +// store and a remote target using set-union semantics: publish local-only +// artifacts to the remote and fetch remote-only artifacts into the local store. +// Because every artifact is write-once and content-addressed, exchange is +// order-independent and idempotent, so folder, HTTP peer, and object-store +// targets are interchangeable behind this interface. +type Transport interface { + // Prepare validates the target against the local store and creates any + // remote-side structure required before exchange. It runs before the local + // export so a misconfigured target fails fast. + Prepare(ctx context.Context, localRoot string) error + // Exchange performs the set-union publish (local-only -> remote) and fetch + // (remote-only -> local). + Exchange(ctx context.Context, localRoot string) error +} + +// folderTransport exchanges artifacts with a local filesystem folder: a synced +// share such as Syncthing, Dropbox, NFS, or an rclone mount. +type folderTransport struct { + target string +} + +func (t *folderTransport) Prepare(_ context.Context, localRoot string) error { + if err := validateDisjointRoots(localRoot, t.target); err != nil { + return err + } + if err := os.MkdirAll(t.target, 0o755); err != nil { + return fmt.Errorf("creating artifact sync target: %w", err) + } + return nil +} + +func (t *folderTransport) Exchange(_ context.Context, localRoot string) error { + if err := CopyUnion(localRoot, t.target); err != nil { + return fmt.Errorf("publishing artifacts: %w", err) + } + if err := CopyUnion(t.target, localRoot); err != nil { + return fmt.Errorf("fetching artifacts: %w", err) + } + return nil +} + +// reconcileCommonCheckpoint guards the HTTP and S3 exchanges against silent +// checkpoint divergence: checkpoints are sequence-named rather than +// content-addressed, so a rebuilt store or an accidentally shared origin id +// can hold different bytes under the same name, which name-set comparison +// alone would skip forever. The highest common checkpoint name is compared by +// content each exchange: a corrupt local copy is quarantined (the caller +// refreshes its index so the remote copy is re-fetched), a corrupt remote +// copy is left for its owner to repair, and a valid-but-divergent pair fails +// loudly with the same conflict error the folder transport raises. Divergence +// buried below the highest common sequence is not scanned; import only trusts +// the latest compatible checkpoint, so it cannot change the imported state. +func reconcileCommonCheckpoint( + localRoot, origin string, + local, remote OriginArtifactIndex, + fetch func(name string) ([]byte, error), +) (quarantined bool, err error) { + name := highestCommonCheckpoint(local, remote) + if name == "" { + return false, nil + } + remoteData, err := fetch(name) + if err != nil { + if errors.Is(err, ErrArtifactNotFound) { + // The remote no longer serves its copy, typically after + // quarantining a corrupt one; nothing to compare this round. + return false, nil + } + return false, err + } + art, err := ReadArtifact(localRoot, origin, KindCheckpoints, name) + if err != nil { + if errors.Is(err, ErrArtifactInvalid) { + log.Printf("artifact: quarantining corrupt local checkpoint %s/%s: %v", origin, name, err) + quarantineArtifact(filepath.Join(localRoot, origin, KindCheckpoints, name)) + return true, nil + } + if errors.Is(err, ErrArtifactNotFound) { + return false, nil + } + return false, err + } + if bytes.Equal(art.Data, remoteData) { + return false, nil + } + spec, err := artifactSpecForKind(origin, KindCheckpoints, name) + if err != nil { + return false, err + } + if verr := validateArtifactData(spec, remoteData); verr != nil { + log.Printf("artifact: remote holds corrupt checkpoint %s/%s: %v", origin, name, verr) + return false, nil + } + return false, fmt.Errorf( + "%w: checkpoint %s/%s differs between the local store and the remote; "+ + "was this origin's artifact store rebuilt or its origin id reused?", + errArtifactPathConflict, origin, name, + ) +} + +func highestCommonCheckpoint(local, remote OriginArtifactIndex) string { + names := make(map[string]struct{}, len(local.Checkpoints)) + for _, name := range local.Checkpoints { + names[name] = struct{}{} + } + best := "" + for _, name := range remote.Checkpoints { + if _, ok := names[name]; ok && name > best { + best = name + } + } + return best +} diff --git a/internal/artifact/transport_http.go b/internal/artifact/transport_http.go new file mode 100644 index 000000000..deb03fd99 --- /dev/null +++ b/internal/artifact/transport_http.go @@ -0,0 +1,334 @@ +package artifact + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "log" + "net/http" + "net/url" + "path/filepath" + "strings" + "time" +) + +const ( + artifactAPIPath = "/api/v1/artifacts" + httpTransportTimeout = 120 * time.Second + httpTransportMaxErrLen = 512 +) + +// IsHTTPTarget reports whether target is an HTTP(S) peer URL rather than a local +// folder target. +func IsHTTPTarget(target string) bool { + return strings.HasPrefix(target, "http://") || strings.HasPrefix(target, "https://") +} + +// httpTransport exchanges artifacts with a remote agentsview peer over its +// authenticated HTTP artifact API: list origins, list an origin's artifact +// index, get an artifact by name, and post an artifact. Content addressing makes +// "does the peer have X" a name-set comparison, so exchange is a stateless, +// idempotent set-union in both directions. +type httpTransport struct { + base string + origin string + token string + client *http.Client +} + +func newHTTPTransport(target, token string, allowInsecure bool) (*httpTransport, error) { + u, err := url.Parse(strings.TrimRight(target, "/")) + if err != nil { + return nil, fmt.Errorf("parsing peer URL %q: %w", target, err) + } + if u.Scheme != "http" && u.Scheme != "https" { + return nil, fmt.Errorf("peer target must be http:// or https://: %q", target) + } + if u.Host == "" { + return nil, fmt.Errorf("peer target is missing a host: %q", target) + } + if u.Scheme == "http" && !allowInsecure && !isLoopbackEndpointHost(u.Hostname()) { + return nil, fmt.Errorf( + "insecure artifact peer %q requires HTTPS", + target, + ) + } + if u.Scheme == "http" && allowInsecure && !isLoopbackEndpointHost(u.Hostname()) { + log.Printf( + "warning: artifact sync to %q uses plaintext HTTP; credentials and archive content are not encrypted in transit", + target, + ) + } + base := strings.TrimRight(u.String(), "/") + if !strings.HasSuffix(base, artifactAPIPath) { + base += artifactAPIPath + } + peerOrigin := (&url.URL{Scheme: u.Scheme, Host: u.Host}).String() + return &httpTransport{ + base: base, + origin: peerOrigin, + token: token, + client: &http.Client{ + Timeout: httpTransportTimeout, + CheckRedirect: func(_ *http.Request, _ []*http.Request) error { + return http.ErrUseLastResponse + }, + }, + }, nil +} + +// Prepare validates that the peer is reachable and authenticated before the +// local export runs, so a wrong URL or token fails fast. +func (t *httpTransport) Prepare(ctx context.Context, _ string) error { + if _, err := t.listOrigins(ctx); err != nil { + return fmt.Errorf("connecting to artifact peer: %w", err) + } + return nil +} + +func (t *httpTransport) Exchange(ctx context.Context, localRoot string) error { + if err := t.pull(ctx, localRoot); err != nil { + return fmt.Errorf("fetching artifacts from peer: %w", err) + } + if err := t.push(ctx, localRoot); err != nil { + return fmt.Errorf("publishing artifacts to peer: %w", err) + } + return nil +} + +// pull fetches every artifact the peer holds that is missing locally. +func (t *httpTransport) pull(ctx context.Context, localRoot string) error { + origins, err := t.listOrigins(ctx) + if err != nil { + return err + } + for _, origin := range origins { + remote, err := t.getIndex(ctx, origin) + if err != nil { + return err + } + local, err := ListArtifacts(localRoot, origin) + if err != nil { + return err + } + quarantined, err := reconcileCommonCheckpoint(localRoot, origin, local, remote, + func(name string) ([]byte, error) { + return t.getArtifact(ctx, origin, KindCheckpoints, name) + }) + if err != nil { + return err + } + if quarantined { + if local, err = ListArtifacts(localRoot, origin); err != nil { + return err + } + } + for _, item := range missingItems(remote, local) { + data, err := t.getArtifact(ctx, origin, item.kind, item.name) + if err != nil { + if errors.Is(err, ErrArtifactNotFound) { + log.Printf("artifact: peer no longer serves %s/%s/%s; skipping", + origin, item.kind, item.name) + continue + } + return err + } + if _, err := WriteArtifact(localRoot, origin, item.kind, item.name, data); err != nil { + if errors.Is(err, ErrArtifactInvalid) { + log.Printf("artifact: skipping corrupt artifact %s/%s/%s from peer: %v", + origin, item.kind, item.name, err) + continue + } + return err + } + } + } + return nil +} + +// push uploads every local artifact the peer is missing. +func (t *httpTransport) push(ctx context.Context, localRoot string) error { + origins, err := ListOrigins(localRoot) + if err != nil { + return err + } + for _, origin := range origins { + local, err := ListArtifacts(localRoot, origin) + if err != nil { + return err + } + remote, err := t.getIndex(ctx, origin) + if err != nil { + return err + } + for _, item := range missingItems(local, remote) { + art, err := ReadArtifact(localRoot, origin, item.kind, item.name) + if err != nil { + if skipUnpublishableArtifact(localRoot, origin, item, err) { + continue + } + return err + } + if err := t.postArtifact(ctx, origin, item.kind, item.name, art.Data); err != nil { + return err + } + } + } + return nil +} + +type artifactItem struct { + kind string + name string +} + +// skipUnpublishableArtifact reports whether a local artifact that failed to +// read before publishing should be skipped: a corrupt file is quarantined so +// a valid copy can be re-fetched, and a file already removed (quarantined or +// collected by a concurrent run) is simply gone. Anything else aborts. +func skipUnpublishableArtifact(localRoot, origin string, item artifactItem, err error) bool { + if errors.Is(err, ErrArtifactInvalid) { + log.Printf("artifact: not publishing corrupt artifact %s/%s/%s: %v", + origin, item.kind, item.name, err) + quarantineArtifact(filepath.Join(localRoot, origin, item.kind, item.name)) + return true + } + return errors.Is(err, ErrArtifactNotFound) +} + +func indexItems(idx OriginArtifactIndex) []artifactItem { + items := make([]artifactItem, 0, + len(idx.Checkpoints)+len(idx.Manifests)+len(idx.Segments)+len(idx.Meta)+len(idx.Raw)) + for _, group := range []struct { + kind string + names []string + }{ + {KindSegments, idx.Segments}, + {KindRaw, idx.Raw}, + {KindManifests, idx.Manifests}, + {KindMeta, idx.Meta}, + {KindCheckpoints, idx.Checkpoints}, + } { + for _, name := range group.names { + items = append(items, artifactItem{kind: group.kind, name: name}) + } + } + return items +} + +// missingItems returns artifacts present in have but absent from other. +func missingItems(have, other OriginArtifactIndex) []artifactItem { + present := make(map[artifactItem]struct{}) + for _, it := range indexItems(other) { + present[it] = struct{}{} + } + var out []artifactItem + for _, it := range indexItems(have) { + if _, ok := present[it]; !ok { + out = append(out, it) + } + } + return out +} + +func (t *httpTransport) listOrigins(ctx context.Context) ([]string, error) { + var resp struct { + Origins []string `json:"origins"` + } + if err := t.getJSON(ctx, t.base+"/origins", &resp); err != nil { + return nil, err + } + return resp.Origins, nil +} + +func (t *httpTransport) getIndex(ctx context.Context, origin string) (OriginArtifactIndex, error) { + var idx OriginArtifactIndex + if err := t.getJSON(ctx, t.base+"/"+url.PathEscape(origin)+"/index", &idx); err != nil { + return OriginArtifactIndex{}, err + } + return idx, nil +} + +func (t *httpTransport) getArtifact(ctx context.Context, origin, kind, name string) ([]byte, error) { + u := t.base + "/" + url.PathEscape(origin) + "/" + url.PathEscape(kind) + "/" + url.PathEscape(name) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, u, nil) + if err != nil { + return nil, err + } + t.authorize(req) + resp, err := t.client.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + if resp.StatusCode == http.StatusNotFound { + // The peer listed this artifact but no longer serves it, typically + // because it quarantined a corrupt copy between index and fetch. + return nil, fmt.Errorf("%w: %s", ErrArtifactNotFound, resp.Status) + } + if resp.StatusCode != http.StatusOK { + return nil, httpStatusError(resp) + } + return io.ReadAll(resp.Body) +} + +func (t *httpTransport) postArtifact(ctx context.Context, origin, kind, name string, data []byte) error { + u := t.base + "/" + url.PathEscape(origin) + "/" + url.PathEscape(kind) + "/" + url.PathEscape(name) + req, err := http.NewRequestWithContext(ctx, http.MethodPost, u, bytes.NewReader(data)) + if err != nil { + return err + } + req.Header.Set("Content-Type", "application/octet-stream") + req.Header.Set("Origin", t.origin) + t.authorize(req) + resp, err := t.client.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusCreated { + return httpStatusError(resp) + } + _, _ = io.Copy(io.Discard, resp.Body) + return nil +} + +func (t *httpTransport) getJSON(ctx context.Context, u string, out any) error { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, u, nil) + if err != nil { + return err + } + t.authorize(req) + resp, err := t.client.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return httpStatusError(resp) + } + return json.NewDecoder(resp.Body).Decode(out) +} + +func (t *httpTransport) authorize(req *http.Request) { + if t.token != "" { + req.Header.Set("Authorization", "Bearer "+t.token) + } +} + +func httpStatusError(resp *http.Response) error { + body, _ := io.ReadAll(io.LimitReader(resp.Body, httpTransportMaxErrLen)) + msg := strings.TrimSpace(string(body)) + if resp.StatusCode == http.StatusUnauthorized { + return fmt.Errorf("%w: peer rejected the bearer token (401)", errHTTPPeer) + } + if msg == "" { + return fmt.Errorf("%w: %s", errHTTPPeer, resp.Status) + } + return fmt.Errorf("%w: %s: %s", errHTTPPeer, resp.Status, msg) +} + +var errHTTPPeer = errors.New("artifact peer request failed") diff --git a/internal/artifact/transport_http_test.go b/internal/artifact/transport_http_test.go new file mode 100644 index 000000000..d76a58cf6 --- /dev/null +++ b/internal/artifact/transport_http_test.go @@ -0,0 +1,378 @@ +package artifact + +import ( + "bytes" + "context" + "encoding/json" + "io" + "log" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "sort" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.kenn.io/agentsview/internal/db" +) + +// fakeArtifactPeer is an in-memory peer implementing the artifact API surface +// the HTTP transport exchanges against: origin listing, per-origin index, and +// artifact get/post. +type fakeArtifactPeer struct { + mu sync.Mutex + arts map[string][]byte // "origin/kind/name" -> bytes + posts []string +} + +func newFakeArtifactPeer() *fakeArtifactPeer { + return &fakeArtifactPeer{arts: map[string][]byte{}} +} + +func (p *fakeArtifactPeer) put(origin, kind, name string, data []byte) { + p.mu.Lock() + defer p.mu.Unlock() + p.arts[origin+"/"+kind+"/"+name] = append([]byte(nil), data...) +} + +func (p *fakeArtifactPeer) has(origin, kind, name string) bool { + p.mu.Lock() + defer p.mu.Unlock() + _, ok := p.arts[origin+"/"+kind+"/"+name] + return ok +} + +func (p *fakeArtifactPeer) postedKinds() []string { + p.mu.Lock() + defer p.mu.Unlock() + kinds := make([]string, 0, len(p.posts)) + for _, key := range p.posts { + parts := strings.SplitN(key, "/", 3) + if len(parts) == 3 { + kinds = append(kinds, parts[1]) + } + } + return kinds +} + +func (p *fakeArtifactPeer) ServeHTTP(w http.ResponseWriter, r *http.Request) { + rest := strings.TrimPrefix(r.URL.Path, artifactAPIPath+"/") + p.mu.Lock() + defer p.mu.Unlock() + if rest == "origins" { + seen := map[string]bool{} + for key := range p.arts { + seen[strings.SplitN(key, "/", 2)[0]] = true + } + origins := make([]string, 0, len(seen)) + for origin := range seen { + origins = append(origins, origin) + } + sort.Strings(origins) + _ = json.NewEncoder(w).Encode(map[string]any{"origins": origins}) + return + } + parts := strings.Split(rest, "/") + if len(parts) == 2 && parts[1] == "index" { + idx := OriginArtifactIndex{Origin: parts[0]} + for key := range p.arts { + kp := strings.SplitN(key, "/", 3) + if kp[0] != parts[0] { + continue + } + switch kp[1] { + case KindCheckpoints: + idx.Checkpoints = append(idx.Checkpoints, kp[2]) + case KindManifests: + idx.Manifests = append(idx.Manifests, kp[2]) + case KindSegments: + idx.Segments = append(idx.Segments, kp[2]) + case KindMeta: + idx.Meta = append(idx.Meta, kp[2]) + case KindRaw: + idx.Raw = append(idx.Raw, kp[2]) + } + } + _ = json.NewEncoder(w).Encode(idx) + return + } + if len(parts) == 3 { + switch r.Method { + case http.MethodGet: + data, ok := p.arts[rest] + if !ok { + http.NotFound(w, r) + return + } + _, _ = w.Write(data) + case http.MethodPost: + data, _ := io.ReadAll(r.Body) + p.arts[rest] = data + p.posts = append(p.posts, rest) + w.WriteHeader(http.StatusCreated) + } + return + } + http.NotFound(w, r) +} + +func TestHTTPTransportRequiresTLSForNonLoopbackPeers(t *testing.T) { + tests := []struct { + name string + target string + wantErr bool + }{ + {name: "public HTTP", target: "http://203.0.113.10:8080", wantErr: true}, + {name: "hostname HTTP", target: "http://peer.example.test:8080", wantErr: true}, + {name: "public HTTPS", target: "https://peer.example.test:8443"}, + {name: "localhost HTTP", target: "http://localhost:8080"}, + {name: "IPv4 loopback HTTP", target: "http://127.0.0.1:8080"}, + {name: "IPv6 loopback HTTP", target: "http://[::1]:8080"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tr, err := newHTTPTransport(tt.target, "", false) + if tt.wantErr { + require.Error(t, err) + assert.Contains(t, err.Error(), "requires HTTPS") + assert.Nil(t, tr) + return + } + require.NoError(t, err) + assert.NotNil(t, tr) + }) + } +} + +func TestHTTPTransportAllowsExplicitRemotePlaintextOptIn(t *testing.T) { + var logs bytes.Buffer + previousOutput := log.Writer() + log.SetOutput(&logs) + t.Cleanup(func() { log.SetOutput(previousOutput) }) + + tr, err := newHTTPTransport("http://peer.example.test:8080", "", true) + + require.NoError(t, err) + require.NotNil(t, tr) + assert.Equal(t, "http://peer.example.test:8080"+artifactAPIPath, tr.base) + assert.Contains(t, logs.String(), "warning") + assert.Contains(t, logs.String(), "plaintext HTTP") +} + +func TestHTTPTransportPrepareHonorsCanceledSync(t *testing.T) { + var requests atomic.Int32 + peer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + requests.Add(1) + _ = json.NewEncoder(w).Encode(map[string]any{"origins": []string{}}) + })) + t.Cleanup(peer.Close) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + _, err := Sync(ctx, testDB(t), SyncOptions{ + DataDir: t.TempDir(), + Target: peer.URL, + Origin: "laptop-a1b2c3", + }) + + require.ErrorIs(t, err, context.Canceled) + assert.Zero(t, requests.Load(), "canceled preparation must not contact the peer") +} + +func TestHTTPTransportPullSkipsCorruptRemoteArtifact(t *testing.T) { + origin := "desktop-d4e5f6" + remoteRoot := exportStore(t, origin, func(database *db.DB) { + seedSession(t, database, "sess-7", "beta") + }) + remoteIdx, err := ListArtifacts(remoteRoot, origin) + require.NoError(t, err) + peer := newFakeArtifactPeer() + for _, item := range indexItems(remoteIdx) { + art, err := ReadArtifact(remoteRoot, origin, item.kind, item.name) + require.NoError(t, err) + peer.put(origin, item.kind, item.name, art.Data) + } + corruptName := hashHex([]byte("corrupt")) + segmentExtension + peer.put(origin, KindSegments, corruptName, []byte("garbage")) + + srv := httptest.NewServer(peer) + t.Cleanup(srv.Close) + tr, err := newHTTPTransport(srv.URL, "", false) + require.NoError(t, err) + localRoot := filepath.Join(t.TempDir(), "artifacts") + require.NoError(t, tr.Exchange(context.Background(), localRoot)) + + gotIdx, err := ListArtifacts(localRoot, origin) + require.NoError(t, err) + assert.ElementsMatch(t, indexItems(remoteIdx), indexItems(gotIdx)) + assert.NoFileExists(t, filepath.Join(localRoot, origin, KindSegments, corruptName)) +} + +func TestHTTPTransportPushSkipsAndQuarantinesCorruptLocalArtifact(t *testing.T) { + origin := "laptop-a1b2c3" + localRoot := exportStore(t, origin, func(database *db.DB) { + seedSession(t, database, "sess-1", "alpha") + }) + validIdx, err := ListArtifacts(localRoot, origin) + require.NoError(t, err) + corruptName := hashHex([]byte("junk")) + segmentExtension + corruptPath := filepath.Join(localRoot, origin, KindSegments, corruptName) + require.NoError(t, os.WriteFile(corruptPath, []byte("garbage"), 0o644)) + + peer := newFakeArtifactPeer() + srv := httptest.NewServer(peer) + t.Cleanup(srv.Close) + tr, err := newHTTPTransport(srv.URL, "", false) + require.NoError(t, err) + require.NoError(t, tr.Exchange(context.Background(), localRoot)) + + for _, item := range indexItems(validIdx) { + assert.True(t, peer.has(origin, item.kind, item.name), + "expected %s/%s on the peer", item.kind, item.name) + } + assert.False(t, peer.has(origin, KindSegments, corruptName)) + assert.NoFileExists(t, corruptPath) + assert.FileExists(t, corruptPath+quarantineSuffix) +} + +func TestHTTPTransportPushPublishesDependenciesBeforeCheckpoint(t *testing.T) { + localRoot := exportStore(t, "laptop-a1b2c3", func(database *db.DB) { + seedSession(t, database, "sess-1", "alpha") + }) + peer := newFakeArtifactPeer() + server := httptest.NewServer(peer) + t.Cleanup(server.Close) + transport, err := newHTTPTransport(server.URL, "", false) + require.NoError(t, err) + + require.NoError(t, transport.Exchange(context.Background(), localRoot)) + + assert.Equal(t, + []string{KindSegments, KindManifests, KindCheckpoints}, + peer.postedKinds(), + ) +} + +func TestHTTPTransportExchangeDetectsDivergentCheckpoint(t *testing.T) { + origin := "laptop-a1b2c3" + localRoot := exportStore(t, origin, func(database *db.DB) { + seedSession(t, database, "sess-1", "alpha") + }) + checkpoints := globArtifacts(t, localRoot, origin, KindCheckpoints, "cp-*.json") + require.Len(t, checkpoints, 1) + name := filepath.Base(checkpoints[0]) + + // The peer holds a different, equally valid checkpoint under the same + // sequence name, as a rebuilt store under a reused origin id would. + divergent, err := canonicalJSON(checkpoint{ + Version: formatVersion, Origin: origin, Sequence: 1, + Sessions: map[string]string{origin + "~other": hashHex([]byte("other"))}, + }) + require.NoError(t, err) + peer := newFakeArtifactPeer() + peer.put(origin, KindCheckpoints, name, divergent) + + srv := httptest.NewServer(peer) + t.Cleanup(srv.Close) + tr, err := newHTTPTransport(srv.URL, "", false) + require.NoError(t, err) + + err = tr.Exchange(context.Background(), localRoot) + require.Error(t, err) + assert.ErrorIs(t, err, errArtifactPathConflict) +} + +func TestHTTPTransportExchangeRepairsCorruptLocalCheckpoint(t *testing.T) { + origin := "laptop-a1b2c3" + localRoot := exportStore(t, origin, func(database *db.DB) { + seedSession(t, database, "sess-1", "alpha") + }) + checkpoints := globArtifacts(t, localRoot, origin, KindCheckpoints, "cp-*.json") + require.Len(t, checkpoints, 1) + name := filepath.Base(checkpoints[0]) + valid, err := os.ReadFile(checkpoints[0]) + require.NoError(t, err) + + peer := newFakeArtifactPeer() + peer.put(origin, KindCheckpoints, name, valid) + require.NoError(t, os.WriteFile(checkpoints[0], []byte("not json"), 0o644)) + + srv := httptest.NewServer(peer) + t.Cleanup(srv.Close) + tr, err := newHTTPTransport(srv.URL, "", false) + require.NoError(t, err) + require.NoError(t, tr.Exchange(context.Background(), localRoot)) + + got, err := os.ReadFile(checkpoints[0]) + require.NoError(t, err) + assert.Equal(t, valid, got, "corrupt local checkpoint should be re-fetched from the peer") +} + +func TestHTTPTransportPostArtifactSetsPeerOrigin(t *testing.T) { + var gotOrigin string + peer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotOrigin = r.Header.Get("Origin") + w.WriteHeader(http.StatusCreated) + })) + defer peer.Close() + + tr, err := newHTTPTransport(peer.URL+"/api/v1/artifacts", "", false) + require.NoError(t, err) + + err = tr.postArtifact(context.Background(), "peer-a1b2c3", KindSegments, strings64("a"), []byte("artifact")) + require.NoError(t, err) + + assert.Equal(t, peer.URL, gotOrigin) +} + +func TestHTTPTransportRejectsRedirectedArtifactPost(t *testing.T) { + type requestCapture struct { + authorization string + body []byte + readErr error + } + + var destinationReached atomic.Bool + destination := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + destinationReached.Store(true) + w.WriteHeader(http.StatusCreated) + })) + t.Cleanup(destination.Close) + + captured := make(chan requestCapture, 1) + source := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body, err := io.ReadAll(r.Body) + captured <- requestCapture{ + authorization: r.Header.Get("Authorization"), + body: body, + readErr: err, + } + http.Redirect(w, r, destination.URL, http.StatusTemporaryRedirect) + })) + t.Cleanup(source.Close) + + tr, err := newHTTPTransport(source.URL, "peer-secret", false) + require.NoError(t, err) + tr.client.Transport = source.Client().Transport + + err = tr.postArtifact(context.Background(), "peer-a1b2c3", KindSegments, strings64("a"), []byte("artifact-secret")) + require.Error(t, err) + assert.ErrorIs(t, err, errHTTPPeer) + var got requestCapture + select { + case got = <-captured: + case <-time.After(time.Second): + require.FailNow(t, "redirect source was not reached", "timed out waiting for the artifact POST") + } + require.NoError(t, got.readErr) + assert.Equal(t, "Bearer peer-secret", got.authorization) + assert.Equal(t, []byte("artifact-secret"), got.body) + assert.False(t, destinationReached.Load()) +} diff --git a/internal/artifact/transport_s3.go b/internal/artifact/transport_s3.go new file mode 100644 index 000000000..dc7f3ae41 --- /dev/null +++ b/internal/artifact/transport_s3.go @@ -0,0 +1,661 @@ +package artifact + +import ( + "bytes" + "context" + "crypto/hmac" + "crypto/sha256" + "encoding/hex" + "encoding/xml" + "errors" + "fmt" + "io" + "log" + "net" + "net/http" + "net/url" + "os" + "sort" + "strconv" + "strings" + "time" +) + +// emptyPayloadSHA256 is the hex SHA-256 of the empty string, the payload hash +// SigV4 uses for requests without a body (GET and ListObjectsV2). +const emptyPayloadSHA256 = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + +var errObjectStore = errors.New("artifact object store request failed") + +// IsObjectTarget reports whether target is an S3-compatible object-store URL +// rather than a local folder or HTTP peer target. +func IsObjectTarget(target string) bool { + return strings.HasPrefix(target, "s3://") +} + +// ObjectStoreOptions configures the S3-compatible object-store transport. It +// carries static credentials plus the addressing details needed to talk to AWS +// S3 or a compatible service such as MinIO or Backblaze B2. +type ObjectStoreOptions struct { + // Endpoint is the service base URL. Empty means real AWS S3, addressed in + // virtual-host style at https://s3..amazonaws.com. A non-empty value + // (host or full URL) forces path-style addressing. + Endpoint string + Region string + AccessKeyID string + SecretAccessKey string + SessionToken string + // AllowInsecureEndpoint permits a non-loopback custom HTTP endpoint. + AllowInsecureEndpoint bool + // PathStyle forces path-style addressing (//) instead of + // virtual-host addressing (.host/). It is implied by a custom + // Endpoint. + PathStyle bool +} + +// ObjectStoreOptionsFromEnv reads object-store credentials and addressing from +// the environment, preferring agentsview-specific variables over the standard +// AWS ones for region and endpoint. Region defaults to us-east-1, and a custom +// endpoint forces path-style addressing because MinIO, B2, and local test +// servers do not support virtual-host buckets. +func ObjectStoreOptionsFromEnv() ObjectStoreOptions { + region := os.Getenv("AGENTSVIEW_S3_REGION") + if region == "" { + region = os.Getenv("AWS_REGION") + } + if region == "" { + region = "us-east-1" + } + endpoint := os.Getenv("AGENTSVIEW_S3_ENDPOINT") + pathStyle := os.Getenv("AGENTSVIEW_S3_PATH_STYLE") == "true" + allowInsecureEndpoint := false + switch strings.ToLower(strings.TrimSpace(os.Getenv("AGENTSVIEW_ALLOW_INSECURE_S3_ENDPOINT"))) { + case "1", "true", "yes": + allowInsecureEndpoint = true + } + if endpoint != "" { + pathStyle = true + } + return ObjectStoreOptions{ + Endpoint: endpoint, + Region: region, + AccessKeyID: os.Getenv("AWS_ACCESS_KEY_ID"), + SecretAccessKey: os.Getenv("AWS_SECRET_ACCESS_KEY"), + SessionToken: os.Getenv("AWS_SESSION_TOKEN"), + AllowInsecureEndpoint: allowInsecureEndpoint, + PathStyle: pathStyle, + } +} + +// s3Transport exchanges artifacts with an S3-compatible bucket using the same +// set-union semantics as the folder and HTTP peer transports: list every object +// under the prefix once, fetch remote-only artifacts into the local store, and +// upload local-only artifacts to the bucket. Content addressing makes "does the +// bucket have X" a name-set comparison, so exchange is stateless and idempotent. +type s3Transport struct { + bucket string + prefix string // no leading or trailing slash; may be empty + endpoint *url.URL + pathStyle bool + opts ObjectStoreOptions + client *http.Client +} + +func newObjectTransport(target string, opts ObjectStoreOptions) (*s3Transport, error) { + if !IsObjectTarget(target) { + return nil, fmt.Errorf("object store target must be s3://: %q", target) + } + rest := strings.TrimPrefix(target, "s3://") + bucket, prefix, _ := strings.Cut(rest, "/") + prefix = strings.Trim(prefix, "/") + if bucket == "" { + return nil, fmt.Errorf("object store target is missing a bucket: %q", target) + } + if opts.AccessKeyID == "" || opts.SecretAccessKey == "" { + return nil, errors.New("object store target requires AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY") + } + region := opts.Region + if region == "" { + region = "us-east-1" + opts.Region = region + } + var endpoint *url.URL + if opts.Endpoint == "" { + endpoint = &url.URL{Scheme: "https", Host: "s3." + region + ".amazonaws.com"} + } else { + raw := opts.Endpoint + if !strings.Contains(raw, "://") { + raw = "https://" + raw + } + u, err := url.Parse(raw) + if err != nil { + return nil, fmt.Errorf("parsing object store endpoint %q: %w", opts.Endpoint, err) + } + if u.Host == "" { + return nil, fmt.Errorf("object store endpoint is missing a host: %q", opts.Endpoint) + } + scheme := strings.ToLower(u.Scheme) + switch scheme { + case "https": + case "http": + if !opts.AllowInsecureEndpoint && !isLoopbackEndpointHost(u.Hostname()) { + return nil, fmt.Errorf("insecure S3 endpoint %q requires HTTPS or AGENTSVIEW_ALLOW_INSECURE_S3_ENDPOINT", opts.Endpoint) + } + default: + return nil, fmt.Errorf("object store endpoint %q uses unsupported scheme %q; only http and https are allowed", opts.Endpoint, u.Scheme) + } + endpoint = &url.URL{Scheme: scheme, Host: u.Host} + opts.PathStyle = true + } + return &s3Transport{ + bucket: bucket, + prefix: prefix, + endpoint: endpoint, + pathStyle: opts.PathStyle, + opts: opts, + client: &http.Client{ + Timeout: httpTransportTimeout, + CheckRedirect: func(_ *http.Request, _ []*http.Request) error { + return http.ErrUseLastResponse + }, + }, + }, nil +} + +func isLoopbackEndpointHost(host string) bool { + if strings.EqualFold(host, "localhost") { + return true + } + if address, _, ok := strings.Cut(host, "%"); ok { + host = address + } + ip := net.ParseIP(host) + return ip != nil && ip.IsLoopback() +} + +// Prepare verifies the bucket is reachable and the credentials are accepted +// before the local export runs, so a wrong endpoint or key fails fast. +func (t *s3Transport) Prepare(ctx context.Context, _ string) error { + if _, err := t.listPage(ctx, "", 1); err != nil { + return fmt.Errorf("connecting to object store: %w", err) + } + return nil +} + +func (t *s3Transport) Exchange(ctx context.Context, localRoot string) error { + remote, err := t.listRemote(ctx) + if err != nil { + return fmt.Errorf("listing object store artifacts: %w", err) + } + if err := t.pull(ctx, localRoot, remote); err != nil { + return fmt.Errorf("fetching artifacts from object store: %w", err) + } + if err := t.push(ctx, localRoot, remote); err != nil { + return fmt.Errorf("publishing artifacts to object store: %w", err) + } + return nil +} + +// pull fetches every artifact the bucket holds that is missing locally. +func (t *s3Transport) pull(ctx context.Context, localRoot string, remote map[string]OriginArtifactIndex) error { + origins := make([]string, 0, len(remote)) + for origin := range remote { + origins = append(origins, origin) + } + sort.Strings(origins) + for _, origin := range origins { + local, err := ListArtifacts(localRoot, origin) + if err != nil { + return err + } + quarantined, err := reconcileCommonCheckpoint(localRoot, origin, local, remote[origin], + func(name string) ([]byte, error) { + return t.getObject(ctx, t.objectKey(origin, KindCheckpoints, name)) + }) + if err != nil { + return err + } + if quarantined { + if local, err = ListArtifacts(localRoot, origin); err != nil { + return err + } + } + for _, item := range missingItems(remote[origin], local) { + data, err := t.getObject(ctx, t.objectKey(origin, item.kind, item.name)) + if err != nil { + return err + } + if _, err := WriteArtifact(localRoot, origin, item.kind, item.name, data); err != nil { + if errors.Is(err, ErrArtifactInvalid) { + // Delete the corrupt object so its name stops masking a + // valid copy: push only uploads names the bucket is + // missing, so a corrupt object left in place would never + // be re-uploaded by a peer that still holds good bytes. + // Content validation tolerates future format versions, so + // only deterministic corruption reaches this branch. A + // failed delete keeps the plain skip. + log.Printf("artifact: detected corrupt artifact %s/%s/%s in object store: %v", + origin, item.kind, item.name, err) + if derr := t.deleteObject(ctx, t.objectKey(origin, item.kind, item.name)); derr != nil { + log.Printf("artifact: deleting corrupt object %s/%s/%s: %v", + origin, item.kind, item.name, derr) + } + continue + } + return err + } + } + } + return nil +} + +// push uploads every local artifact the bucket is missing. +func (t *s3Transport) push(ctx context.Context, localRoot string, remote map[string]OriginArtifactIndex) error { + origins, err := ListOrigins(localRoot) + if err != nil { + return err + } + for _, origin := range origins { + local, err := ListArtifacts(localRoot, origin) + if err != nil { + return err + } + for _, item := range missingItems(local, remote[origin]) { + art, err := ReadArtifact(localRoot, origin, item.kind, item.name) + if err != nil { + if skipUnpublishableArtifact(localRoot, origin, item, err) { + continue + } + return err + } + if err := t.putObject(ctx, t.objectKey(origin, item.kind, item.name), art.Data); err != nil { + return err + } + } + } + return nil +} + +// listRemote walks the whole prefix once via paginated ListObjectsV2 and groups +// the valid artifact keys into a per-origin index, the enumeration both pull and +// push compare against. +func (t *s3Transport) listRemote(ctx context.Context) (map[string]OriginArtifactIndex, error) { + indexes := map[string]*OriginArtifactIndex{} + token := "" + for { + result, err := t.listPage(ctx, token, 0) + if err != nil { + return nil, err + } + for _, c := range result.Contents { + t.indexKey(indexes, c.Key) + } + if !result.IsTruncated || result.NextContinuationToken == "" { + break + } + token = result.NextContinuationToken + } + out := make(map[string]OriginArtifactIndex, len(indexes)) + for origin, idx := range indexes { + out[origin] = *idx + } + return out, nil +} + +// indexKey parses one object key into origin/kind/name and records it in the +// per-origin index, skipping keys that do not have exactly three path segments +// under the prefix or whose origin, kind, and name fail the same validation +// ListArtifacts applies locally. +func (t *s3Transport) indexKey(indexes map[string]*OriginArtifactIndex, key string) { + rel := key + if t.prefix != "" { + p := t.prefix + "/" + if !strings.HasPrefix(key, p) { + return + } + rel = key[len(p):] + } + parts := strings.Split(rel, "/") + if len(parts) != 3 { + return + } + origin, kind, name := parts[0], parts[1], parts[2] + if validateOriginID(origin) != nil { + return + } + if !validArtifactKindName(kind, name) { + return + } + idx := indexes[origin] + if idx == nil { + idx = &OriginArtifactIndex{Origin: origin} + indexes[origin] = idx + } + switch kind { + case KindCheckpoints: + idx.Checkpoints = append(idx.Checkpoints, name) + case KindManifests: + idx.Manifests = append(idx.Manifests, name) + case KindSegments: + idx.Segments = append(idx.Segments, name) + case KindMeta: + idx.Meta = append(idx.Meta, name) + case KindRaw: + idx.Raw = append(idx.Raw, name) + } +} + +// objectKey builds the bucket key for one artifact, joining the optional prefix +// with origin/kind/name. +func (t *s3Transport) objectKey(origin, kind, name string) string { + parts := make([]string, 0, 4) + if t.prefix != "" { + parts = append(parts, t.prefix) + } + parts = append(parts, origin, kind, name) + return strings.Join(parts, "/") +} + +// listBucketResult is the subset of the ListObjectsV2 response we consume. +type listBucketResult struct { + XMLName xml.Name `xml:"ListBucketResult"` + IsTruncated bool `xml:"IsTruncated"` + NextContinuationToken string `xml:"NextContinuationToken"` + Contents []struct { + Key string `xml:"Key"` + } `xml:"Contents"` +} + +// listPage performs one ListObjectsV2 request. A maxKeys of 0 leaves the limit +// to the server default; the reachability check in Prepare uses 1. +func (t *s3Transport) listPage(ctx context.Context, token string, maxKeys int) (listBucketResult, error) { + q := url.Values{} + q.Set("list-type", "2") + if t.prefix != "" { + q.Set("prefix", t.prefix+"/") + } + if token != "" { + q.Set("continuation-token", token) + } + if maxKeys > 0 { + q.Set("max-keys", strconv.Itoa(maxKeys)) + } + req, err := t.newRequest(ctx, http.MethodGet, "", q, nil) + if err != nil { + return listBucketResult{}, err + } + resp, err := t.client.Do(req) + if err != nil { + return listBucketResult{}, err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return listBucketResult{}, t.statusError(resp) + } + body, err := io.ReadAll(resp.Body) + if err != nil { + return listBucketResult{}, err + } + var result listBucketResult + if err := xml.Unmarshal(body, &result); err != nil { + return listBucketResult{}, fmt.Errorf("decoding object store listing: %w", err) + } + return result, nil +} + +func (t *s3Transport) getObject(ctx context.Context, key string) ([]byte, error) { + req, err := t.newRequest(ctx, http.MethodGet, key, nil, nil) + if err != nil { + return nil, err + } + resp, err := t.client.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return nil, t.statusError(resp) + } + return io.ReadAll(resp.Body) +} + +func (t *s3Transport) putObject(ctx context.Context, key string, data []byte) error { + req, err := t.newRequest(ctx, http.MethodPut, key, nil, data) + if err != nil { + return err + } + // Write-once: refuse to overwrite an existing object. Artifacts are + // immutable and content-addressed, so a collision means either a harmless + // re-upload of identical bytes or a genuine origin-ID conflict that must be + // surfaced, not silently merged. + req.Header.Set("If-None-Match", "*") + resp, err := t.client.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + switch resp.StatusCode { + case http.StatusOK, http.StatusCreated: + _, _ = io.Copy(io.Discard, resp.Body) + return nil + case http.StatusPreconditionFailed: + _, _ = io.Copy(io.Discard, resp.Body) + return t.reconcileExistingObject(ctx, key, data) + default: + return t.statusError(resp) + } +} + +// deleteObject removes one object; a missing key counts as success. +func (t *s3Transport) deleteObject(ctx context.Context, key string) error { + if t.endpoint.Scheme != "https" { + return fmt.Errorf("refusing to delete object through insecure S3 endpoint %q", t.endpoint.String()) + } + req, err := t.newRequest(ctx, http.MethodDelete, key, nil, nil) + if err != nil { + return err + } + resp, err := t.client.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + switch resp.StatusCode { + case http.StatusOK, http.StatusNoContent, http.StatusNotFound: + _, _ = io.Copy(io.Discard, resp.Body) + return nil + default: + return t.statusError(resp) + } +} + +// reconcileExistingObject handles a write-once collision: identical content is +// an accepted duplicate, differing content is a conflict. +func (t *s3Transport) reconcileExistingObject(ctx context.Context, key string, data []byte) error { + existing, err := t.getObject(ctx, key) + if err != nil { + return fmt.Errorf("comparing conflicting object %s: %w", key, err) + } + if bytes.Equal(existing, data) { + return nil + } + return fmt.Errorf("%w: object %s already exists with different content", errObjectStore, key) +} + +// newRequest builds and signs one object-store request. An empty key targets the +// bucket itself (used for listing). A nil body sends no payload and signs the +// empty-payload hash; a non-nil body signs its real SHA-256. +func (t *s3Transport) newRequest(ctx context.Context, method, key string, query url.Values, body []byte) (*http.Request, error) { + host := t.endpoint.Host + var rawPath string + if t.pathStyle { + rawPath = "/" + t.bucket + if key != "" { + rawPath += "/" + key + } + } else { + host = t.bucket + "." + t.endpoint.Host + rawPath = "/" + key + } + u := &url.URL{ + Scheme: t.endpoint.Scheme, + Host: host, + Path: rawPath, + RawPath: s3EncodePath(rawPath), + RawQuery: canonicalQueryString(query), + } + var reader io.Reader + payloadHash := emptyPayloadSHA256 + if body != nil { + reader = bytes.NewReader(body) + payloadHash = hashHex(body) + } + req, err := http.NewRequestWithContext(ctx, method, u.String(), reader) + if err != nil { + return nil, err + } + if body != nil { + req.Header.Set("Content-Type", "application/octet-stream") + } + signRequest(req, payloadHash, t.opts, time.Now()) + return req, nil +} + +func (t *s3Transport) statusError(resp *http.Response) error { + body, _ := io.ReadAll(io.LimitReader(resp.Body, httpTransportMaxErrLen)) + msg := strings.TrimSpace(string(body)) + if msg == "" { + return fmt.Errorf("%w: %s", errObjectStore, resp.Status) + } + return fmt.Errorf("%w: %s: %s", errObjectStore, resp.Status, msg) +} + +// signRequest applies AWS Signature Version 4 for the "s3" service to req, +// setting the x-amz-date, x-amz-content-sha256, optional x-amz-security-token, +// and Authorization headers. payloadSHA256Hex must be the hex SHA-256 of the +// request body (the empty-payload hash for bodyless requests). +func signRequest(req *http.Request, payloadSHA256Hex string, opts ObjectStoreOptions, now time.Time) { + now = now.UTC() + amzDate := now.Format("20060102T150405Z") + dateStamp := now.Format("20060102") + + req.Header.Set("X-Amz-Date", amzDate) + req.Header.Set("X-Amz-Content-Sha256", payloadSHA256Hex) + if opts.SessionToken != "" { + req.Header.Set("X-Amz-Security-Token", opts.SessionToken) + } + + // Sign the host header plus every x-amz-* header we set. They are already in + // lowercase canonical form, so we sort by name to build the canonical block. + type header struct{ name, value string } + headers := []header{ + {"host", req.URL.Host}, + {"x-amz-content-sha256", payloadSHA256Hex}, + {"x-amz-date", amzDate}, + } + if opts.SessionToken != "" { + headers = append(headers, header{"x-amz-security-token", opts.SessionToken}) + } + sort.Slice(headers, func(i, j int) bool { return headers[i].name < headers[j].name }) + + var canonicalHeaders strings.Builder + signedNames := make([]string, 0, len(headers)) + for _, h := range headers { + canonicalHeaders.WriteString(h.name) + canonicalHeaders.WriteByte(':') + canonicalHeaders.WriteString(strings.TrimSpace(h.value)) + canonicalHeaders.WriteByte('\n') + signedNames = append(signedNames, h.name) + } + signedHeaders := strings.Join(signedNames, ";") + + canonicalRequest := strings.Join([]string{ + req.Method, + req.URL.EscapedPath(), + req.URL.RawQuery, + canonicalHeaders.String(), + signedHeaders, + payloadSHA256Hex, + }, "\n") + + scope := dateStamp + "/" + opts.Region + "/s3/aws4_request" + stringToSign := strings.Join([]string{ + "AWS4-HMAC-SHA256", + amzDate, + scope, + hashHex([]byte(canonicalRequest)), + }, "\n") + + signingKey := sigV4SigningKey(opts.SecretAccessKey, dateStamp, opts.Region, "s3") + signature := hex.EncodeToString(hmacSHA256(signingKey, stringToSign)) + + req.Header.Set("Authorization", fmt.Sprintf( + "AWS4-HMAC-SHA256 Credential=%s/%s, SignedHeaders=%s, Signature=%s", + opts.AccessKeyID, scope, signedHeaders, signature, + )) +} + +func sigV4SigningKey(secret, dateStamp, region, service string) []byte { + kDate := hmacSHA256([]byte("AWS4"+secret), dateStamp) + kRegion := hmacSHA256(kDate, region) + kService := hmacSHA256(kRegion, service) + return hmacSHA256(kService, "aws4_request") +} + +func hmacSHA256(key []byte, data string) []byte { + h := hmac.New(sha256.New, key) + h.Write([]byte(data)) + return h.Sum(nil) +} + +// canonicalQueryString renders query parameters in the sorted, S3-encoded form +// the SigV4 canonical request requires. +func canonicalQueryString(q url.Values) string { + if len(q) == 0 { + return "" + } + keys := make([]string, 0, len(q)) + for k := range q { + keys = append(keys, k) + } + sort.Strings(keys) + parts := make([]string, 0, len(q)) + for _, k := range keys { + values := append([]string(nil), q[k]...) + sort.Strings(values) + for _, v := range values { + parts = append(parts, s3URIEncode(k)+"="+s3URIEncode(v)) + } + } + return strings.Join(parts, "&") +} + +// s3EncodePath URI-encodes a path, encoding each segment per S3 rules while +// preserving the '/' separators. +func s3EncodePath(path string) string { + segments := strings.Split(path, "/") + for i, seg := range segments { + segments[i] = s3URIEncode(seg) + } + return strings.Join(segments, "/") +} + +// s3URIEncode percent-encodes s following the AWS SigV4 rules: only the +// unreserved characters A-Z a-z 0-9 - _ . ~ pass through unencoded, and every +// other byte is encoded as uppercase %XX (notably space becomes %20, not '+'). +func s3URIEncode(s string) string { + var b strings.Builder + for i := 0; i < len(s); i++ { + c := s[i] + if (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || (c >= '0' && c <= '9') || + c == '-' || c == '_' || c == '.' || c == '~' { + b.WriteByte(c) + continue + } + b.WriteByte('%') + const hexDigits = "0123456789ABCDEF" + b.WriteByte(hexDigits[c>>4]) + b.WriteByte(hexDigits[c&0xf]) + } + return b.String() +} diff --git a/internal/artifact/transport_s3_miniotest_test.go b/internal/artifact/transport_s3_miniotest_test.go new file mode 100644 index 000000000..f21376f76 --- /dev/null +++ b/internal/artifact/transport_s3_miniotest_test.go @@ -0,0 +1,151 @@ +//go:build miniotest + +package artifact + +import ( + "context" + "fmt" + "net/http" + "os" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/testcontainers/testcontainers-go" + "github.com/testcontainers/testcontainers-go/wait" +) + +// TestS3TransportMinIORoundTrip exercises the object-store transport against a +// real MinIO server in a container, validating that the hand-rolled SigV4 +// signing is accepted by a genuine S3 implementation end to end: create bucket, +// push a producer's artifacts, list them, and pull them into a fresh store. +func TestS3TransportMinIORoundTrip(t *testing.T) { + ctx := context.Background() + endpoint, accessKey, secretKey := startMinIO(t, ctx) + + tr, err := newObjectTransport("s3://agentsview/sync", ObjectStoreOptions{ + Endpoint: endpoint, + Region: "us-east-1", + AccessKeyID: accessKey, + SecretAccessKey: secretKey, + AllowInsecureEndpoint: true, + PathStyle: true, + }) + require.NoError(t, err) + requireCreateBucket(t, ctx, tr) + + // Producer store: one exported session plus a star metadata event. + origin := "laptop-a1b2c3" + prod := testDB(t) + seedSession(t, prod, "sess-1", "alpha") + prodDir := t.TempDir() + prodRoot := filepath.Join(prodDir, "artifacts") + _, err = Export(ctx, prod, prodRoot, origin) + require.NoError(t, err) + recorder := NewMetadataRecorder(prod, MetadataRecorderOptions{DataDir: prodDir, Origin: origin}) + _, err = recorder.Append(ctx, MetadataEventInput{SessionID: "sess-1", Op: MetadataOpStar}) + require.NoError(t, err) + + want, err := ListArtifacts(prodRoot, origin) + require.NoError(t, err) + require.NotEmpty(t, want.Manifests) + require.NotEmpty(t, want.Meta) + + // Push to MinIO, then confirm the bucket lists exactly the producer's set. + require.NoError(t, tr.Prepare(context.Background(), prodRoot)) + require.NoError(t, tr.Exchange(ctx, prodRoot)) + + remote, err := tr.listRemote(ctx) + require.NoError(t, err) + require.Contains(t, remote, origin) + assertIndexEqual(t, want, remote[origin]) + + // A fresh consumer store pulls every artifact back. + consDir := t.TempDir() + consRoot := filepath.Join(consDir, "artifacts") + require.NoError(t, os.MkdirAll(consRoot, 0o755)) + require.NoError(t, tr.Exchange(ctx, consRoot)) + + got, err := ListArtifacts(consRoot, origin) + require.NoError(t, err) + assertIndexEqual(t, want, got) + + // Re-running is a no-op set-union: nothing new to fetch or upload. + require.NoError(t, tr.Exchange(ctx, consRoot)) + got, err = ListArtifacts(consRoot, origin) + require.NoError(t, err) + assertIndexEqual(t, want, got) +} + +func assertIndexEqual(t *testing.T, want, got OriginArtifactIndex) { + t.Helper() + assert.ElementsMatch(t, want.Checkpoints, got.Checkpoints, "checkpoints") + assert.ElementsMatch(t, want.Manifests, got.Manifests, "manifests") + assert.ElementsMatch(t, want.Segments, got.Segments, "segments") + assert.ElementsMatch(t, want.Meta, got.Meta, "meta") + assert.ElementsMatch(t, want.Raw, got.Raw, "raw") +} + +// requireCreateBucket issues a signed CreateBucket request through the transport, +// reusing the same SigV4 path under test. An already-owned bucket is fine. +func requireCreateBucket(t *testing.T, ctx context.Context, tr *s3Transport) { + t.Helper() + var lastStatus string + for attempt := 0; attempt < 10; attempt++ { + req, err := tr.newRequest(ctx, http.MethodPut, "", nil, nil) + require.NoError(t, err) + resp, err := tr.client.Do(req) + require.NoError(t, err) + lastStatus = resp.Status + status := resp.StatusCode + resp.Body.Close() + // 200 = created, 409 = already owned. 503 can still occur briefly while + // MinIO finishes initializing; retry those. + if status == http.StatusOK || status == http.StatusConflict { + return + } + if status != http.StatusServiceUnavailable { + break + } + time.Sleep(500 * time.Millisecond) + } + require.FailNowf(t, "create bucket failed", "last status: %s", lastStatus) +} + +func startMinIO(t *testing.T, ctx context.Context) (endpoint, accessKey, secretKey string) { + t.Helper() + const user, pass = "minioadmin", "minioadmin" + container, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{ + ContainerRequest: testcontainers.ContainerRequest{ + Image: "minio/minio:RELEASE.2025-07-23T15-54-02Z", + ExposedPorts: []string{"9000/tcp"}, + Env: map[string]string{ + "MINIO_ROOT_USER": user, + "MINIO_ROOT_PASSWORD": pass, + }, + Cmd: []string{"server", "/data"}, + // "ready" (not "live") signals MinIO can actually serve S3 requests; + // "live" only means the process is up and races CreateBucket to 503. + WaitingFor: wait.ForHTTP("/minio/health/ready"). + WithPort("9000/tcp"). + WithStartupTimeout(2 * time.Minute), + }, + Started: true, + }) + if err != nil { + t.Skipf("could not start MinIO container (is Docker available?): %v", err) + } + t.Cleanup(func() { + stopCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + _ = container.Terminate(stopCtx) + }) + + host, err := container.Host(ctx) + require.NoError(t, err) + port, err := container.MappedPort(ctx, "9000/tcp") + require.NoError(t, err) + return fmt.Sprintf("http://%s:%s", host, port.Port()), user, pass +} diff --git a/internal/artifact/transport_s3_test.go b/internal/artifact/transport_s3_test.go new file mode 100644 index 000000000..474cacaa0 --- /dev/null +++ b/internal/artifact/transport_s3_test.go @@ -0,0 +1,660 @@ +package artifact + +import ( + "context" + "encoding/xml" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "sort" + "strconv" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.kenn.io/agentsview/internal/db" +) + +// mockS3 is an in-memory, path-style S3-compatible server backing a single +// bucket. It implements just enough of ListObjectsV2, GetObject, PutObject, and +// DeleteObject to exercise the object-store transport, and verifies that +// requests arrive signed (Authorization plus x-amz-date) without re-validating +// the signature. +type mockS3 struct { + t *testing.T + bucket string + pageSize int + + mu sync.Mutex + objects map[string][]byte + deletes int +} + +func newMockS3(t *testing.T, bucket string, pageSize int) *mockS3 { + return &mockS3{ + t: t, + bucket: bucket, + pageSize: pageSize, + objects: map[string][]byte{}, + } +} + +func (m *mockS3) put(key string, data []byte) { + m.mu.Lock() + defer m.mu.Unlock() + m.objects[key] = append([]byte(nil), data...) +} + +func (m *mockS3) has(key string) bool { + m.mu.Lock() + defer m.mu.Unlock() + _, ok := m.objects[key] + return ok +} + +func (m *mockS3) deleteCount() int { + m.mu.Lock() + defer m.mu.Unlock() + return m.deletes +} + +func (m *mockS3) ServeHTTP(w http.ResponseWriter, r *http.Request) { + assert.True(m.t, strings.HasPrefix(r.Header.Get("Authorization"), "AWS4-HMAC-SHA256"), + "request must carry a SigV4 Authorization header") + assert.NotEmpty(m.t, r.Header.Get("X-Amz-Date"), "request must carry an x-amz-date header") + + bucketPath := "/" + m.bucket + if r.Method == http.MethodGet && r.URL.Path == bucketPath && r.URL.Query().Get("list-type") == "2" { + m.list(w, r) + return + } + if !strings.HasPrefix(r.URL.Path, bucketPath+"/") { + http.Error(w, "not found", http.StatusNotFound) + return + } + key := strings.TrimPrefix(r.URL.Path, bucketPath+"/") + switch r.Method { + case http.MethodGet: + m.mu.Lock() + data, ok := m.objects[key] + m.mu.Unlock() + if !ok { + http.Error(w, "no such key", http.StatusNotFound) + return + } + _, _ = w.Write(data) + case http.MethodPut: + body := make([]byte, 0) + buf := make([]byte, 4096) + for { + n, err := r.Body.Read(buf) + body = append(body, buf[:n]...) + if err != nil { + break + } + } + // Honor the write-once conditional: reject when the key already exists. + if r.Header.Get("If-None-Match") == "*" && m.has(key) { + http.Error(w, "precondition failed", http.StatusPreconditionFailed) + return + } + m.put(key, body) + w.WriteHeader(http.StatusOK) + case http.MethodDelete: + m.mu.Lock() + m.deletes++ + delete(m.objects, key) + m.mu.Unlock() + w.WriteHeader(http.StatusNoContent) + default: + http.Error(w, "method not allowed", http.StatusMethodNotAllowed) + } +} + +func (m *mockS3) list(w http.ResponseWriter, r *http.Request) { + prefix := r.URL.Query().Get("prefix") + token := r.URL.Query().Get("continuation-token") + + m.mu.Lock() + keys := make([]string, 0, len(m.objects)) + for k := range m.objects { + if strings.HasPrefix(k, prefix) { + keys = append(keys, k) + } + } + m.mu.Unlock() + sort.Strings(keys) + + start := 0 + if token != "" { + start, _ = strconv.Atoi(token) + } + pageSize := m.pageSize + if pageSize <= 0 { + pageSize = 1000 + } + end := start + pageSize + truncated := end < len(keys) + if end > len(keys) { + end = len(keys) + } + + type contentsXML struct { + Key string `xml:"Key"` + } + type resultXML struct { + XMLName xml.Name `xml:"ListBucketResult"` + IsTruncated bool `xml:"IsTruncated"` + Contents []contentsXML `xml:"Contents"` + NextContinuationToken string `xml:"NextContinuationToken,omitempty"` + } + out := resultXML{IsTruncated: truncated} + for _, k := range keys[start:end] { + out.Contents = append(out.Contents, contentsXML{Key: k}) + } + if truncated { + out.NextContinuationToken = strconv.Itoa(end) + } + w.Header().Set("Content-Type", "application/xml") + require.NoError(m.t, xml.NewEncoder(w).Encode(out)) +} + +func testObjectOptions(endpoint string) ObjectStoreOptions { + return ObjectStoreOptions{ + Endpoint: endpoint, + Region: "us-east-1", + AccessKeyID: "AKIDEXAMPLE", + SecretAccessKey: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", + PathStyle: true, + } +} + +func TestS3TransportPrepareHonorsCanceledSync(t *testing.T) { + var requests atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + requests.Add(1) + _, _ = w.Write([]byte("")) + })) + t.Cleanup(server.Close) + transport, err := newObjectTransport("s3://bucket/arts", testObjectOptions(server.URL)) + require.NoError(t, err) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + _, err = syncWithTransport(ctx, testDB(t), SyncOptions{ + DataDir: t.TempDir(), + Target: "s3://bucket/arts", + Origin: "laptop-a1b2c3", + }, transport) + + require.ErrorIs(t, err, context.Canceled) + assert.Zero(t, requests.Load(), "canceled preparation must not contact the object store") +} + +// exportStore exports one origin's sessions into a fresh artifact store and +// returns the artifact root. +func exportStore(t *testing.T, origin string, seed func(*db.DB)) string { + t.Helper() + database := testDB(t) + seed(database) + root := filepath.Join(t.TempDir(), "artifacts") + _, err := Export(context.Background(), database, root, origin) + require.NoError(t, err) + return root +} + +func TestS3TransportPushRoundTrip(t *testing.T) { + origin := "laptop-a1b2c3" + database := testDB(t) + seedSession(t, database, "sess-1", "alpha") + + dataDir := t.TempDir() + localRoot := filepath.Join(dataDir, "artifacts") + _, err := Export(context.Background(), database, localRoot, origin) + require.NoError(t, err) + + // Append a metadata event so a meta artifact is part of the push. + rec := NewMetadataRecorder(database, MetadataRecorderOptions{ + DataDir: dataDir, + Origin: origin, + Now: func() time.Time { return fixedHLCTime() }, + }) + _, err = database.StarSession("sess-1") + require.NoError(t, err) + _, err = rec.Append(context.Background(), MetadataEventInput{ + SessionID: "sess-1", + Op: MetadataOpStar, + }) + require.NoError(t, err) + + mock := newMockS3(t, "bucket", 0) + srv := httptest.NewServer(mock) + t.Cleanup(srv.Close) + + tr, err := newObjectTransport("s3://bucket/arts", testObjectOptions(srv.URL)) + require.NoError(t, err) + require.NoError(t, tr.Prepare(context.Background(), localRoot)) + require.NoError(t, tr.Exchange(context.Background(), localRoot)) + + idx, err := ListArtifacts(localRoot, origin) + require.NoError(t, err) + items := indexItems(idx) + require.NotEmpty(t, items) + assert.NotEmpty(t, idx.Meta, "the star event should have produced a meta artifact") + for _, item := range items { + key := "arts/" + origin + "/" + item.kind + "/" + item.name + assert.True(t, mock.has(key), "expected object %q in bucket", key) + } +} + +func TestS3TransportPullRoundTrip(t *testing.T) { + origin := "desktop-d4e5f6" + + // Produce a populated store for the origin and upload it into the bucket so + // the transport must pull it down into an empty local store. + remoteRoot := exportStore(t, origin, func(database *db.DB) { + seedSession(t, database, "sess-7", "beta") + seedSession(t, database, "sess-8", "beta") + }) + remoteIdx, err := ListArtifacts(remoteRoot, origin) + require.NoError(t, err) + uploaded := indexItems(remoteIdx) + require.NotEmpty(t, uploaded) + + // Use a small page size and more than one object to exercise the + // continuation-token pagination path. + mock := newMockS3(t, "bucket", 2) + for _, item := range uploaded { + art, err := ReadArtifact(remoteRoot, origin, item.kind, item.name) + require.NoError(t, err) + mock.put("arts/"+origin+"/"+item.kind+"/"+item.name, art.Data) + } + require.Greater(t, len(uploaded), 2, "need multiple pages to test pagination") + + srv := httptest.NewServer(mock) + t.Cleanup(srv.Close) + + localRoot := filepath.Join(t.TempDir(), "artifacts") + tr, err := newObjectTransport("s3://bucket/arts", testObjectOptions(srv.URL)) + require.NoError(t, err) + require.NoError(t, tr.Exchange(context.Background(), localRoot)) + + gotIdx, err := ListArtifacts(localRoot, origin) + require.NoError(t, err) + assert.ElementsMatch(t, indexItems(remoteIdx), indexItems(gotIdx)) +} + +func TestS3TransportPullRetainsCorruptRemoteArtifactOverHTTP(t *testing.T) { + origin := "desktop-d4e5f6" + remoteRoot := exportStore(t, origin, func(database *db.DB) { + seedSession(t, database, "sess-7", "beta") + }) + remoteIdx, err := ListArtifacts(remoteRoot, origin) + require.NoError(t, err) + uploaded := indexItems(remoteIdx) + mock := newMockS3(t, "bucket", 0) + for _, item := range uploaded { + art, err := ReadArtifact(remoteRoot, origin, item.kind, item.name) + require.NoError(t, err) + mock.put("arts/"+origin+"/"+item.kind+"/"+item.name, art.Data) + } + corruptName := hashHex([]byte("corrupt")) + segmentExtension + mock.put("arts/"+origin+"/segments/"+corruptName, []byte("garbage")) + + srv := httptest.NewServer(mock) + t.Cleanup(srv.Close) + localRoot := filepath.Join(t.TempDir(), "artifacts") + tr, err := newObjectTransport("s3://bucket/arts", testObjectOptions(srv.URL)) + require.NoError(t, err) + require.NoError(t, tr.Exchange(context.Background(), localRoot)) + + gotIdx, err := ListArtifacts(localRoot, origin) + require.NoError(t, err) + assert.ElementsMatch(t, uploaded, indexItems(gotIdx)) + assert.NoFileExists(t, filepath.Join(localRoot, origin, KindSegments, corruptName)) + assert.True(t, mock.has("arts/"+origin+"/segments/"+corruptName)) + assert.Zero(t, mock.deleteCount()) +} + +func TestS3TransportPullDeletesCorruptRemoteObjectSoPushHeals(t *testing.T) { + origin := "desktop-d4e5f6" + ownerRoot := exportStore(t, origin, func(database *db.DB) { + seedSession(t, database, "sess-7", "beta") + }) + ownerIdx, err := ListArtifacts(ownerRoot, origin) + require.NoError(t, err) + require.Len(t, ownerIdx.Segments, 1) + segKey := "arts/" + origin + "/segments/" + ownerIdx.Segments[0] + + mock := newMockS3(t, "bucket", 0) + for _, item := range indexItems(ownerIdx) { + art, err := ReadArtifact(ownerRoot, origin, item.kind, item.name) + require.NoError(t, err) + mock.put("arts/"+origin+"/"+item.kind+"/"+item.name, art.Data) + } + // The bucket copy is corrupted in place: its name still lists, so pushes + // from valid holders would otherwise skip it forever. + mock.put(segKey, []byte("garbage")) + + srv := httptest.NewTLSServer(mock) + t.Cleanup(srv.Close) + + // An empty peer's pull fails to validate the object and deletes it, so + // the name stops masking the valid copy. + emptyRoot := filepath.Join(t.TempDir(), "artifacts") + tr, err := newObjectTransport("s3://bucket/arts", testObjectOptions(srv.URL)) + require.NoError(t, err) + tr.client.Transport = srv.Client().Transport + require.NoError(t, tr.Exchange(context.Background(), emptyRoot)) + assert.False(t, mock.has(segKey), "corrupt object deleted from the bucket") + assert.Equal(t, 1, mock.deleteCount(), "expected one DELETE for the corrupt object") + + // The owner's next exchange re-uploads its valid copy, and the empty + // peer's next pull completes its store. + ownerTr, err := newObjectTransport("s3://bucket/arts", testObjectOptions(srv.URL)) + require.NoError(t, err) + ownerTr.client.Transport = srv.Client().Transport + require.NoError(t, ownerTr.Exchange(context.Background(), ownerRoot)) + assert.True(t, mock.has(segKey), "valid copy re-uploaded") + + require.NoError(t, tr.Exchange(context.Background(), emptyRoot)) + gotIdx, err := ListArtifacts(emptyRoot, origin) + require.NoError(t, err) + assert.ElementsMatch(t, indexItems(ownerIdx), indexItems(gotIdx)) +} + +func TestS3TransportRejectsRedirect(t *testing.T) { + var sourceReached atomic.Bool + var destinationReached atomic.Bool + destination := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + destinationReached.Store(true) + w.Header().Set("Content-Type", "application/xml") + _, _ = w.Write([]byte("false")) + })) + t.Cleanup(destination.Close) + + source := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + sourceReached.Store(true) + http.Redirect(w, r, destination.URL, http.StatusTemporaryRedirect) + })) + t.Cleanup(source.Close) + + tr, err := newObjectTransport("s3://bucket/arts", testObjectOptions(source.URL)) + require.NoError(t, err) + tr.client.Transport = source.Client().Transport + + _, err = tr.listPage(context.Background(), "", 0) + require.Error(t, err) + assert.True(t, sourceReached.Load()) + assert.False(t, destinationReached.Load()) +} + +func TestS3TransportPushSkipsAndQuarantinesCorruptLocalArtifact(t *testing.T) { + origin := "laptop-a1b2c3" + localRoot := exportStore(t, origin, func(database *db.DB) { + seedSession(t, database, "sess-1", "alpha") + }) + validIdx, err := ListArtifacts(localRoot, origin) + require.NoError(t, err) + corruptName := hashHex([]byte("junk")) + segmentExtension + corruptPath := filepath.Join(localRoot, origin, KindSegments, corruptName) + require.NoError(t, os.WriteFile(corruptPath, []byte("garbage"), 0o644)) + + mock := newMockS3(t, "bucket", 0) + srv := httptest.NewServer(mock) + t.Cleanup(srv.Close) + tr, err := newObjectTransport("s3://bucket/arts", testObjectOptions(srv.URL)) + require.NoError(t, err) + require.NoError(t, tr.Exchange(context.Background(), localRoot)) + + for _, item := range indexItems(validIdx) { + assert.True(t, mock.has("arts/"+origin+"/"+item.kind+"/"+item.name), + "expected object %s/%s in bucket", item.kind, item.name) + } + assert.False(t, mock.has("arts/"+origin+"/segments/"+corruptName)) + assert.NoFileExists(t, corruptPath) + assert.FileExists(t, corruptPath+quarantineSuffix) +} + +func TestS3TransportExchangeDetectsDivergentCheckpoint(t *testing.T) { + origin := "laptop-a1b2c3" + localRoot := exportStore(t, origin, func(database *db.DB) { + seedSession(t, database, "sess-1", "alpha") + }) + checkpoints := globArtifacts(t, localRoot, origin, KindCheckpoints, "cp-*.json") + require.Len(t, checkpoints, 1) + name := filepath.Base(checkpoints[0]) + + divergent, err := canonicalJSON(checkpoint{ + Version: formatVersion, Origin: origin, Sequence: 1, + Sessions: map[string]string{origin + "~other": hashHex([]byte("other"))}, + }) + require.NoError(t, err) + mock := newMockS3(t, "bucket", 0) + mock.put("arts/"+origin+"/"+KindCheckpoints+"/"+name, divergent) + + srv := httptest.NewServer(mock) + t.Cleanup(srv.Close) + tr, err := newObjectTransport("s3://bucket/arts", testObjectOptions(srv.URL)) + require.NoError(t, err) + + err = tr.Exchange(context.Background(), localRoot) + require.Error(t, err) + assert.ErrorIs(t, err, errArtifactPathConflict) +} + +func TestS3TransportExchangeRepairsCorruptLocalCheckpoint(t *testing.T) { + origin := "laptop-a1b2c3" + localRoot := exportStore(t, origin, func(database *db.DB) { + seedSession(t, database, "sess-1", "alpha") + }) + checkpoints := globArtifacts(t, localRoot, origin, KindCheckpoints, "cp-*.json") + require.Len(t, checkpoints, 1) + name := filepath.Base(checkpoints[0]) + valid, err := os.ReadFile(checkpoints[0]) + require.NoError(t, err) + + mock := newMockS3(t, "bucket", 0) + mock.put("arts/"+origin+"/"+KindCheckpoints+"/"+name, valid) + require.NoError(t, os.WriteFile(checkpoints[0], []byte("not json"), 0o644)) + + srv := httptest.NewServer(mock) + t.Cleanup(srv.Close) + tr, err := newObjectTransport("s3://bucket/arts", testObjectOptions(srv.URL)) + require.NoError(t, err) + require.NoError(t, tr.Exchange(context.Background(), localRoot)) + + got, err := os.ReadFile(checkpoints[0]) + require.NoError(t, err) + assert.Equal(t, valid, got, "corrupt local checkpoint should be re-fetched from the bucket") +} + +func TestS3TransportWriteOnceRejectsDivergentContent(t *testing.T) { + mock := newMockS3(t, "bucket", 0) + srv := httptest.NewServer(mock) + t.Cleanup(srv.Close) + tr, err := newObjectTransport("s3://bucket/arts", testObjectOptions(srv.URL)) + require.NoError(t, err) + ctx := context.Background() + key := "arts/laptop-a1b2c3/raw/deadbeef" + + // First write creates the object. + require.NoError(t, tr.putObject(ctx, key, []byte("one"))) + // An identical re-write is an accepted duplicate, not an error. + require.NoError(t, tr.putObject(ctx, key, []byte("one"))) + // Divergent content at the same key is a conflict, never a silent overwrite. + err = tr.putObject(ctx, key, []byte("two")) + require.Error(t, err) + assert.ErrorIs(t, err, errObjectStore) + // The original content is preserved. + got, err := tr.getObject(ctx, key) + require.NoError(t, err) + assert.Equal(t, []byte("one"), got) +} + +func TestIsObjectTarget(t *testing.T) { + tests := []struct { + name string + target string + want bool + }{ + {"s3 url", "s3://bucket/prefix", true}, + {"s3 bucket only", "s3://bucket", true}, + {"http peer", "http://example.com", false}, + {"https peer", "https://example.com", false}, + {"folder path", "/var/data/share", false}, + {"empty", "", false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.want, IsObjectTarget(tt.target)) + }) + } +} + +func TestNewObjectTransport(t *testing.T) { + creds := ObjectStoreOptions{ + Region: "us-east-1", + AccessKeyID: "AK", + SecretAccessKey: "SK", + } + + t.Run("missing bucket", func(t *testing.T) { + _, err := newObjectTransport("s3://", creds) + require.Error(t, err) + assert.Contains(t, err.Error(), "bucket") + }) + + t.Run("missing credentials", func(t *testing.T) { + _, err := newObjectTransport("s3://bucket/prefix", ObjectStoreOptions{Region: "us-east-1"}) + require.Error(t, err) + assert.Contains(t, err.Error(), "AWS_ACCESS_KEY_ID") + }) + + t.Run("not an object target", func(t *testing.T) { + _, err := newObjectTransport("https://example.com", creds) + require.Error(t, err) + }) + + t.Run("parses bucket and prefix", func(t *testing.T) { + tr, err := newObjectTransport("s3://bucket/some/prefix/", creds) + require.NoError(t, err) + assert.Equal(t, "bucket", tr.bucket) + assert.Equal(t, "some/prefix", tr.prefix) + assert.Equal(t, "s3.us-east-1.amazonaws.com", tr.endpoint.Host) + assert.False(t, tr.pathStyle, "real AWS defaults to virtual-host addressing") + }) + + t.Run("custom endpoint forces path style", func(t *testing.T) { + tr, err := newObjectTransport("s3://bucket", ObjectStoreOptions{ + Endpoint: "http://localhost:9000", + Region: "us-east-1", + AccessKeyID: "AK", + SecretAccessKey: "SK", + }) + require.NoError(t, err) + assert.True(t, tr.pathStyle) + assert.Equal(t, "localhost:9000", tr.endpoint.Host) + assert.Empty(t, tr.prefix) + }) + + t.Run("rejects insecure remote endpoint by default", func(t *testing.T) { + options := creds + options.Endpoint = "http://minio.lan:9000" + + _, err := newObjectTransport("s3://bucket", options) + require.Error(t, err) + assert.Contains(t, err.Error(), "insecure S3 endpoint") + assert.Contains(t, err.Error(), "AGENTSVIEW_ALLOW_INSECURE_S3_ENDPOINT") + }) + + t.Run("allows opted-in insecure remote endpoint", func(t *testing.T) { + options := creds + options.Endpoint = "http://minio.lan:9000" + options.AllowInsecureEndpoint = true + + tr, err := newObjectTransport("s3://bucket", options) + require.NoError(t, err) + assert.Equal(t, "http", tr.endpoint.Scheme) + }) + + for _, endpoint := range []string{ + "http://localhost:9000", + "http://LOCALHOST:9000", + "http://127.0.0.1:9000", + "http://[::1]:9000", + } { + t.Run("allows loopback endpoint "+endpoint, func(t *testing.T) { + options := creds + options.Endpoint = endpoint + + tr, err := newObjectTransport("s3://bucket", options) + require.NoError(t, err) + assert.Equal(t, "http", tr.endpoint.Scheme) + }) + } + + t.Run("bare host defaults to HTTPS", func(t *testing.T) { + options := creds + options.Endpoint = "minio.lan:9000" + + tr, err := newObjectTransport("s3://bucket", options) + require.NoError(t, err) + assert.Equal(t, "https", tr.endpoint.Scheme) + }) + + t.Run("rejects unsupported endpoint scheme", func(t *testing.T) { + options := creds + options.Endpoint = "ftp://minio.lan" + + _, err := newObjectTransport("s3://bucket", options) + require.Error(t, err) + assert.Contains(t, err.Error(), "ftp") + }) +} + +func TestObjectStoreOptionsFromEnvAllowsInsecureEndpoint(t *testing.T) { + for _, value := range []string{"1", "true", "yes", "YES"} { + t.Run(value, func(t *testing.T) { + t.Setenv("AWS_ACCESS_KEY_ID", "AK") + t.Setenv("AWS_SECRET_ACCESS_KEY", "SK") + t.Setenv("AGENTSVIEW_S3_ENDPOINT", "http://minio.lan:9000") + t.Setenv("AGENTSVIEW_ALLOW_INSECURE_S3_ENDPOINT", value) + + tr, err := newObjectTransport("s3://bucket", ObjectStoreOptionsFromEnv()) + require.NoError(t, err) + assert.Equal(t, "http", tr.endpoint.Scheme) + }) + } +} + +func TestObjectStoreOptionsFromEnvRejectsInvalidInsecureEndpointOverride(t *testing.T) { + tests := []struct { + name string + value string + }{ + {name: "empty", value: ""}, + {name: "zero", value: "0"}, + {name: "false", value: "false"}, + {name: "no", value: "no"}, + {name: "typo", value: "treu"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Setenv("AWS_ACCESS_KEY_ID", "AK") + t.Setenv("AWS_SECRET_ACCESS_KEY", "SK") + t.Setenv("AGENTSVIEW_S3_ENDPOINT", "http://minio.lan:9000") + t.Setenv("AGENTSVIEW_ALLOW_INSECURE_S3_ENDPOINT", tt.value) + + _, err := newObjectTransport("s3://bucket", ObjectStoreOptionsFromEnv()) + require.Error(t, err) + assert.Contains(t, err.Error(), "insecure S3 endpoint") + assert.Contains(t, err.Error(), "AGENTSVIEW_ALLOW_INSECURE_S3_ENDPOINT") + }) + } +} diff --git a/internal/artifact/twoinstance_test.go b/internal/artifact/twoinstance_test.go new file mode 100644 index 000000000..7b361fbcd --- /dev/null +++ b/internal/artifact/twoinstance_test.go @@ -0,0 +1,344 @@ +package artifact + +import ( + "context" + "database/sql" + "encoding/json" + "slices" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.kenn.io/agentsview/internal/db" +) + +// syncInstance models one machine in a two-instance artifact-sync harness: its +// own database, data dir, origin, and metadata recorder, all driven through the +// public folder-sync API against a shared target folder. +type syncInstance struct { + t *testing.T + db *db.DB + dataDir string + origin string + now time.Time + rec *MetadataRecorder +} + +func newSyncInstance(t *testing.T, origin string) *syncInstance { + t.Helper() + in := &syncInstance{ + t: t, + db: testDB(t), + dataDir: t.TempDir(), + origin: origin, + now: fixedHLCTime(), + } + in.rec = NewMetadataRecorder(in.db, MetadataRecorderOptions{ + DataDir: in.dataDir, + Origin: origin, + Now: func() time.Time { return in.now }, + }) + return in +} + +// at sets the wall clock used for the instance's next local metadata edits. +func (in *syncInstance) at(ts time.Time) *syncInstance { + in.now = ts + return in +} + +// sync exports local sessions to the shared target, exchanges the union both +// ways, and imports foreign origins, mirroring `agentsview sync `. +func (in *syncInstance) sync(target string) SyncResult { + in.t.Helper() + res, err := SyncFolder(context.Background(), in.db, SyncOptions{ + DataDir: in.dataDir, + Target: target, + Origin: in.origin, + Now: func() time.Time { return in.now }, + }) + require.NoError(in.t, err) + return res +} + +// syncInit mirrors `agentsview sync --init ` for baseline metadata +// initialization. +func (in *syncInstance) syncInit(target string) SyncResult { + in.t.Helper() + res, err := SyncFolder(context.Background(), in.db, SyncOptions{ + DataDir: in.dataDir, + Target: target, + Origin: in.origin, + Now: func() time.Time { return in.now }, + BaselineMetadata: true, + }) + require.NoError(in.t, err) + return res +} + +// rename mirrors the rename handler: mutate the local row, then append the +// metadata event (which also records the local LWW register entry). +func (in *syncInstance) rename(localID, name string) { + in.t.Helper() + require.NoError(in.t, in.db.RenameSession(localID, &name)) + value, err := json.Marshal(struct { + DisplayName string `json:"display_name"` + }{DisplayName: name}) + require.NoError(in.t, err) + _, err = in.rec.Append(context.Background(), MetadataEventInput{ + SessionID: localID, + Op: MetadataOpRename, + Value: value, + }) + require.NoError(in.t, err) +} + +// star mirrors the star handler: star the local row, then append the event. +func (in *syncInstance) star(localID string) { + in.t.Helper() + _, err := in.db.StarSession(localID) + require.NoError(in.t, err) + _, err = in.rec.Append(context.Background(), MetadataEventInput{ + SessionID: localID, + Op: MetadataOpStar, + }) + require.NoError(in.t, err) +} + +// purge mirrors the permanent-delete handler: soft delete, permanently delete +// from trash, then append the purge event. +func (in *syncInstance) purge(localID string) { + in.t.Helper() + require.NoError(in.t, in.db.SoftDeleteSession(localID)) + _, err := in.db.DeleteSessionIfTrashed(localID) + require.NoError(in.t, err) + _, err = in.rec.Append(context.Background(), MetadataEventInput{ + SessionID: localID, + Op: MetadataOpPurge, + }) + require.NoError(in.t, err) +} + +func (in *syncInstance) displayName(t *testing.T, id string) *string { + t.Helper() + got, err := in.db.GetSession(context.Background(), id) + require.NoError(t, err) + require.NotNil(t, got) + return got.DisplayName +} + +func (in *syncInstance) requireSession(t *testing.T, id string) *db.Session { + t.Helper() + got, err := in.db.GetSession(context.Background(), id) + require.NoError(t, err) + require.NotNil(t, got, "session %s should exist", id) + return got +} + +func (in *syncInstance) isStarred(t *testing.T, id string) bool { + t.Helper() + ids, err := in.db.ListStarredSessionIDs(context.Background()) + require.NoError(t, err) + return slices.Contains(ids, id) +} + +func TestTwoInstanceSessionAndRenamePropagate(t *testing.T) { + target := t.TempDir() + a := newSyncInstance(t, "laptop-a1b2c3") + b := newSyncInstance(t, "desktop-d4e5f6") + gid := a.origin + "~sess-1" + + seedSession(t, a.db, "sess-1", "alpha") + a.at(fixedHLCTime()).rename("sess-1", "Renamed on A") + + a.sync(target) + res := b.sync(target) + assert.Equal(t, 1, res.ImportedSessions) + assert.GreaterOrEqual(t, res.ImportedMessages, 1) + assert.Equal(t, 1, res.ImportedMetadata) + + got, err := b.db.GetSession(context.Background(), gid) + require.NoError(t, err) + require.NotNil(t, got) + require.NotNil(t, got.DisplayName) + assert.Equal(t, "Renamed on A", *got.DisplayName) + assert.Equal(t, a.origin, got.Machine) + + // Re-syncing is idempotent: no new rows, no new conflicts. + res = b.sync(target) + assert.Zero(t, res.ImportedSessions) + assert.Zero(t, res.ImportedMetadata) +} + +func TestTwoInstanceConcurrentRenameConverges(t *testing.T) { + target := t.TempDir() + a := newSyncInstance(t, "laptop-a1b2c3") + b := newSyncInstance(t, "desktop-d4e5f6") + gid := a.origin + "~sess-1" + bLocalID := gid // the session is foreign on B, keyed by its global id + + // Share the session from A to B first so both can edit it. + seedSession(t, a.db, "sess-1", "alpha") + a.sync(target) + b.sync(target) + b.requireSession(t, bLocalID) + + // Both rename the same session concurrently. A uses the later HLC (within the + // clock drift bound so the receiver can observe it), so A wins deterministically + // on both machines. + b.at(fixedHLCTime()).rename(bLocalID, "Renamed on B") + a.at(fixedHLCTime().Add(time.Minute)).rename("sess-1", "Renamed on A") + + // Exchange until quiescent: A publishes its edit, B pulls it (A wins on B), + // then A pulls B's losing edit and records the conflict. + a.sync(target) + b.sync(target) + a.sync(target) + + require.NotNil(t, a.displayName(t, "sess-1")) + require.NotNil(t, b.displayName(t, bLocalID)) + assert.Equal(t, "Renamed on A", *a.displayName(t, "sess-1")) + assert.Equal(t, "Renamed on A", *b.displayName(t, bLocalID)) + + // Both machines independently record exactly one losing edit for the field. + assertMetadataConflictCount(t, a.db, gid, "display_name", 1) + assertMetadataConflictCount(t, b.db, gid, "display_name", 1) +} + +func TestSyncInitDoesNotLetBaselineOutrankExistingPeerMetadata(t *testing.T) { + target := t.TempDir() + a := newSyncInstance(t, "laptop-a1b2c3") + b := newSyncInstance(t, "desktop-d4e5f6") + gid := a.origin + "~sess-1" + + seedSession(t, a.db, "sess-1", "alpha") + a.sync(target) + b.sync(target) + b.requireSession(t, gid) + + peerName := "Peer newer title" + b.at(fixedHLCTime().Add(time.Minute)).rename(gid, peerName) + b.sync(target) + + staleLocalName := "Stale local title" + require.NoError(t, a.db.RenameSession("sess-1", &staleLocalName)) + + res := a.at(fixedHLCTime().Add(2 * time.Minute)).syncInit(target) + assert.Equal(t, 1, res.ImportedMetadata) + require.NotNil(t, a.displayName(t, "sess-1")) + assert.Equal(t, peerName, *a.displayName(t, "sess-1")) + + b.sync(target) + require.NotNil(t, b.displayName(t, gid)) + assert.Equal(t, peerName, *b.displayName(t, gid)) + assertMetadataConflictCount(t, a.db, gid, "display_name", 0) + assertMetadataConflictCount(t, b.db, gid, "display_name", 0) +} + +func TestSyncInitDoesNotBaselineRowsCreatedByPreBaselineImport(t *testing.T) { + target := t.TempDir() + a := newSyncInstance(t, "laptop-a1b2c3") + b := newSyncInstance(t, "desktop-d4e5f6") + gid := b.origin + "~sess-1" + + seedSession(t, b.db, "sess-1", "alpha") + b.sync(target) + + require.NoError(t, a.db.Update(func(tx *sql.Tx) error { + _, err := tx.Exec(` + CREATE TRIGGER test_imported_display_name + AFTER INSERT ON sessions + WHEN NEW.id = 'desktop-d4e5f6~sess-1' + BEGIN + UPDATE sessions + SET display_name = 'Imported stale title' + WHERE id = NEW.id; + END`) + return err + })) + + res := a.syncInit(target) + assert.Equal(t, 1, res.ImportedSessions) + assert.Equal(t, 0, res.ImportedMetadata) + require.NotNil(t, a.displayName(t, gid)) + assert.Equal(t, "Imported stale title", *a.displayName(t, gid)) + + _, ok, err := a.db.MetadataReplayStateOp(context.Background(), gid, "display_name") + require.NoError(t, err) + assert.False(t, ok, "import-created curation must not become local baseline metadata") + assert.Empty(t, globArtifacts(t, target, a.origin, "meta", "*"+metadataEventExtension)) +} + +func TestSyncReappliesMetadataReplayStateAfterManifestRefresh(t *testing.T) { + target := t.TempDir() + a := newSyncInstance(t, "laptop-a1b2c3") + b := newSyncInstance(t, "desktop-d4e5f6") + gid := b.origin + "~sess-1" + + sourceName := "Source title" + seedSession(t, b.db, "sess-1", "alpha", func(s *db.Session) { + s.SessionName = &sourceName + }) + b.sync(target) + a.sync(target) + a.requireSession(t, gid) + + localName := "Local winning title" + a.at(fixedHLCTime().Add(time.Minute)).rename(gid, localName) + require.NotNil(t, a.displayName(t, gid)) + assert.Equal(t, localName, *a.displayName(t, gid)) + + require.NoError(t, a.db.Update(func(tx *sql.Tx) error { + _, err := tx.Exec(`UPDATE sessions SET display_name = NULL WHERE id = ?`, gid) + return err + })) + refreshedSourceName := "Refreshed source title" + seedSession(t, b.db, "sess-1", "alpha", func(s *db.Session) { + s.SessionName = &refreshedSourceName + }) + b.sync(target) + + a.sync(target) + require.NotNil(t, a.displayName(t, gid)) + assert.Equal(t, localName, *a.displayName(t, gid)) +} + +func TestTwoInstanceStarAndPurgePropagate(t *testing.T) { + target := t.TempDir() + a := newSyncInstance(t, "laptop-a1b2c3") + b := newSyncInstance(t, "desktop-d4e5f6") + gid := a.origin + "~sess-1" + + seedSession(t, a.db, "sess-1", "alpha") + a.sync(target) + b.sync(target) + b.requireSession(t, gid) + + // B stars the shared session; the star converges back to A. + b.at(fixedHLCTime()).star(gid) + b.sync(target) + a.sync(target) + assert.True(t, a.isStarred(t, "sess-1")) + assert.True(t, b.isStarred(t, gid)) + + // A purges the session; the purge tombstone propagates to B and blocks + // re-import of the now-superseded manifest. The HLC stays within the drift + // bound so B can observe it on import. + a.at(fixedHLCTime().Add(time.Minute)).purge("sess-1") + a.sync(target) + b.sync(target) + + gotA, err := a.db.GetSession(context.Background(), "sess-1") + require.NoError(t, err) + assert.Nil(t, gotA) + gotB, err := b.db.GetSession(context.Background(), gid) + require.NoError(t, err) + assert.Nil(t, gotB) + + // A later sync must not resurrect the purged session on B. + b.sync(target) + gotB, err = b.db.GetSession(context.Background(), gid) + require.NoError(t, err) + assert.Nil(t, gotB) +} diff --git a/internal/config/config.go b/internal/config/config.go index 6017767fa..54500d756 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -4,7 +4,9 @@ import ( "bytes" "crypto/rand" "encoding/base64" + "encoding/hex" "encoding/json" + "errors" "flag" "fmt" "log" @@ -437,6 +439,7 @@ type Config struct { GithubToken string `json:"github_token,omitempty" toml:"github_token"` Terminal TerminalConfig `json:"terminal,omitempty" toml:"terminal"` AuthToken string `json:"auth_token,omitempty" toml:"auth_token"` + ArtifactOriginID string `json:"artifact_origin_id,omitempty" toml:"artifact_origin_id"` RequireAuth bool `json:"require_auth" toml:"require_auth"` NoBrowser bool `json:"no_browser" toml:"no_browser"` DisableUpdateCheck bool `json:"disable_update_check" toml:"disable_update_check"` @@ -966,6 +969,7 @@ func (c *Config) applyConfigTOML(data string) error { ResultContentBlockedCategories []string `toml:"result_content_blocked_categories"` Terminal TerminalConfig `toml:"terminal"` AuthToken string `toml:"auth_token"` + ArtifactOriginID string `toml:"artifact_origin_id"` RequireAuth bool `toml:"require_auth"` RemoteAccess bool `toml:"remote_access"` DisableUpdateCheck bool `toml:"disable_update_check"` @@ -1036,6 +1040,9 @@ func (c *Config) applyConfigTOML(data string) error { if file.AuthToken != "" && c.AuthToken == "" { c.AuthToken = file.AuthToken } + if file.ArtifactOriginID != "" { + c.ArtifactOriginID = file.ArtifactOriginID + } c.RequireAuth = file.RequireAuth || file.RemoteAccess c.DisableUpdateCheck = file.DisableUpdateCheck if meta.IsDefined("default_pg") { @@ -1627,6 +1634,11 @@ func finalize(cfg *Config) error { if err := cfg.Vector.Validate(); err != nil { return err } + if cfg.ArtifactOriginID != "" { + if err := ValidateArtifactOriginID(cfg.ArtifactOriginID); err != nil { + return fmt.Errorf("invalid artifact origin id: %w", err) + } + } return nil } @@ -2408,6 +2420,11 @@ func (c *Config) SaveSettings(patch map[string]any) error { c.AuthToken = s } } + if v, ok := patch["artifact_origin_id"]; ok { + if s, ok := v.(string); ok { + c.ArtifactOriginID = s + } + } if v, ok := patch["require_auth"]; ok { if b, ok := v.(bool); ok { c.RequireAuth = b @@ -2447,6 +2464,176 @@ func (c *Config) EnsureAuthToken() error { }) } +// EnsureArtifactOriginID generates and persists a stable artifact sync origin +// ID if one does not already exist. +func (c *Config) EnsureArtifactOriginID() (string, error) { + if c.ArtifactOriginID != "" { + if err := ValidateArtifactOriginID(c.ArtifactOriginID); err != nil { + return "", fmt.Errorf("stored artifact origin: %w", err) + } + return c.ArtifactOriginID, nil + } + + var origin string + if err := c.withConfigLock(func() error { + existing, err := c.readConfigMap() + if err != nil { + return err + } + if stored, ok := existing["artifact_origin_id"].(string); ok && stored != "" { + if err := ValidateArtifactOriginID(stored); err != nil { + return fmt.Errorf("stored artifact origin: %w", err) + } + c.ArtifactOriginID = stored + origin = stored + return nil + } + + machineName, err := c.artifactOriginMachineName() + if err != nil { + return err + } + generated, err := newArtifactOriginID(machineName) + if err != nil { + return err + } + if err := ValidateArtifactOriginID(generated); err != nil { + return fmt.Errorf("generated artifact origin: %w", err) + } + + existing["artifact_origin_id"] = generated + if err := c.writeConfigMap(existing); err != nil { + return err + } + c.ArtifactOriginID = generated + origin = generated + return nil + }); err != nil { + return "", err + } + return origin, nil +} + +// AdoptArtifactOriginID persists origin as this machine's artifact sync +// origin unless the config file already records one, in which case the +// recorded origin wins and is returned. Artifact sync uses this to promote an +// origin that exists only in database sync state -- for example one minted by +// an incoming peer exchange before the config ever initialized an origin -- +// so the machine keeps publishing under a single origin instead of generating +// a competing config origin that would strand earlier metadata events. +func (c *Config) AdoptArtifactOriginID(origin string) (string, error) { + if err := ValidateArtifactOriginID(origin); err != nil { + return "", fmt.Errorf("adopting artifact origin: %w", err) + } + if c.ArtifactOriginID != "" { + if err := ValidateArtifactOriginID(c.ArtifactOriginID); err != nil { + return "", fmt.Errorf("stored artifact origin: %w", err) + } + return c.ArtifactOriginID, nil + } + + adopted := origin + if err := c.withConfigLock(func() error { + existing, err := c.readConfigMap() + if err != nil { + return err + } + if stored, ok := existing["artifact_origin_id"].(string); ok && stored != "" { + if err := ValidateArtifactOriginID(stored); err != nil { + return fmt.Errorf("stored artifact origin: %w", err) + } + c.ArtifactOriginID = stored + adopted = stored + return nil + } + + existing["artifact_origin_id"] = origin + if err := c.writeConfigMap(existing); err != nil { + return err + } + c.ArtifactOriginID = origin + return nil + }); err != nil { + return "", err + } + return adopted, nil +} + +func (c *Config) artifactOriginMachineName() (string, error) { + pgMachine := strings.TrimSpace(c.PG.MachineName) + if pgMachine != "" { + if pgMachine == "local" { + return "", fmt.Errorf( + "machine name %q is reserved; choose a different pg.machine_name", + pgMachine, + ) + } + return c.PG.MachineName, nil + } + host, err := os.Hostname() + if err != nil || strings.TrimSpace(host) == "" { + return "machine", nil + } + return host, nil +} + +func newArtifactOriginID(machine string) (string, error) { + base := sanitizeArtifactOriginPart(machine) + if base == "" || base == "local" { + base = "machine" + } + var suffix [3]byte + if _, err := rand.Read(suffix[:]); err != nil { + return "", fmt.Errorf("generating artifact origin suffix: %w", err) + } + return fmt.Sprintf("%s-%s", base, hex.EncodeToString(suffix[:])), nil +} + +func sanitizeArtifactOriginPart(s string) string { + s = strings.ToLower(strings.TrimSpace(s)) + var b strings.Builder + lastDash := false + for _, r := range s { + ok := r >= 'a' && r <= 'z' || r >= '0' && r <= '9' + if ok { + b.WriteRune(r) + lastDash = false + continue + } + if !lastDash { + b.WriteByte('-') + lastDash = true + } + } + return strings.Trim(b.String(), "-") +} + +// ValidateArtifactOriginID checks the persisted single-writer origin prefix. +func ValidateArtifactOriginID(origin string) error { + if origin == "" { + return errors.New("artifact origin is required") + } + if origin != strings.TrimSpace(origin) { + return fmt.Errorf("invalid artifact origin %q", origin) + } + if origin == "local" { + return fmt.Errorf("invalid artifact origin %q", origin) + } + if strings.ContainsAny(origin, `/\`) || filepath.Base(origin) != origin { + return fmt.Errorf("invalid artifact origin %q", origin) + } + if strings.HasPrefix(origin, "-") || strings.HasSuffix(origin, "-") { + return fmt.Errorf("invalid artifact origin %q", origin) + } + for _, r := range origin { + if r >= 'a' && r <= 'z' || r >= '0' && r <= '9' || r == '-' { + continue + } + return fmt.Errorf("invalid artifact origin %q", origin) + } + return nil +} + // SaveGithubToken persists the GitHub token to the config file. func (c *Config) SaveGithubToken(token string) error { return c.withConfigLock(func() error { diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 885986600..1d0d3395a 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -13,6 +13,7 @@ import ( "time" "github.com/BurntSushi/toml" + "github.com/gofrs/flock" "github.com/spf13/pflag" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -474,6 +475,221 @@ func TestLoad_PublicURLMergedIntoOrigins(t *testing.T) { assert.Equal(t, "https://viewer.example.test", strings.Join(cfg.PublicOrigins, ",")) } +func TestLoad_ArtifactOriginIDFromConfigFile(t *testing.T) { + tmp := setupTestEnv(t) + writeConfig(t, tmp, map[string]any{ + "artifact_origin_id": "desk-abcdef", + }) + + cfg, err := LoadMinimal() + require.NoError(t, err) + + assert.Equal(t, "desk-abcdef", cfg.ArtifactOriginID) +} + +func TestLoad_ArtifactOriginIDRejectsInvalid(t *testing.T) { + tmp := setupTestEnv(t) + writeConfig(t, tmp, map[string]any{ + "artifact_origin_id": "local", + }) + + _, err := LoadMinimal() + require.Error(t, err) + assert.Contains(t, err.Error(), "invalid artifact origin id") +} + +func TestEnsureArtifactOriginIDPersists(t *testing.T) { + tmp := setupTestEnv(t) + cfg, err := LoadMinimal() + require.NoError(t, err) + + origin, err := cfg.EnsureArtifactOriginID() + require.NoError(t, err) + assert.Regexp(t, `^[a-z0-9]+(?:-[a-z0-9]+)*-[0-9a-f]{6}$`, origin) + + data, err := os.ReadFile(filepath.Join(tmp, configFileName)) + require.NoError(t, err) + assert.Contains(t, string(data), `artifact_origin_id = "`+origin+`"`) + + reloaded, err := LoadMinimal() + require.NoError(t, err) + assert.Equal(t, origin, reloaded.ArtifactOriginID) + again, err := reloaded.EnsureArtifactOriginID() + require.NoError(t, err) + assert.Equal(t, origin, again) +} + +func TestEnsureArtifactOriginIDUsesConfiguredMachineName(t *testing.T) { + tmp := setupTestEnv(t) + writeConfig(t, tmp, map[string]any{ + "pg": map[string]any{ + "machine_name": "Desk Box", + }, + }) + cfg, err := LoadMinimal() + require.NoError(t, err) + + origin, err := cfg.EnsureArtifactOriginID() + require.NoError(t, err) + + assert.Regexp(t, `^desk-box-[0-9a-f]{6}$`, origin) +} + +func TestEnsureArtifactOriginIDRejectsReservedMachineName(t *testing.T) { + tmp := setupTestEnv(t) + writeConfig(t, tmp, map[string]any{ + "pg": map[string]any{ + "machine_name": "local", + }, + }) + cfg, err := LoadMinimal() + require.NoError(t, err) + + origin, err := cfg.EnsureArtifactOriginID() + require.Error(t, err) + assert.Empty(t, origin) + assert.Contains(t, err.Error(), "reserved") +} + +func TestEnsureArtifactOriginIDDoesNotRewriteExistingOrigin(t *testing.T) { + tmp := setupTestEnv(t) + writeConfig(t, tmp, map[string]any{ + "artifact_origin_id": "original-a1b2c3", + "pg": map[string]any{ + "machine_name": "new-machine", + }, + }) + cfg, err := LoadMinimal() + require.NoError(t, err) + + origin, err := cfg.EnsureArtifactOriginID() + require.NoError(t, err) + + assert.Equal(t, "original-a1b2c3", origin) +} + +func TestAdoptArtifactOriginIDPersists(t *testing.T) { + tmp := setupTestEnv(t) + cfg, err := LoadMinimal() + require.NoError(t, err) + + adopted, err := cfg.AdoptArtifactOriginID("laptop-a1b2c3") + require.NoError(t, err) + assert.Equal(t, "laptop-a1b2c3", adopted) + + data, err := os.ReadFile(filepath.Join(tmp, configFileName)) + require.NoError(t, err) + assert.Contains(t, string(data), `artifact_origin_id = "laptop-a1b2c3"`) + + reloaded, err := LoadMinimal() + require.NoError(t, err) + origin, err := reloaded.EnsureArtifactOriginID() + require.NoError(t, err) + assert.Equal(t, "laptop-a1b2c3", origin, + "later ensure must reuse the adopted origin instead of generating") +} + +func TestAdoptArtifactOriginIDKeepsExistingFileOrigin(t *testing.T) { + tmp := setupTestEnv(t) + cfg, err := LoadMinimal() + require.NoError(t, err) + + // The config file gains an origin after load but before adoption; the + // recorded origin wins. + writeConfig(t, tmp, map[string]any{ + "artifact_origin_id": "desktop-d4e5f6", + }) + + adopted, err := cfg.AdoptArtifactOriginID("laptop-a1b2c3") + require.NoError(t, err) + assert.Equal(t, "desktop-d4e5f6", adopted) + assert.Equal(t, "desktop-d4e5f6", cfg.ArtifactOriginID) +} + +func TestAdoptArtifactOriginIDKeepsInMemoryOrigin(t *testing.T) { + setupTestEnv(t) + cfg, err := LoadMinimal() + require.NoError(t, err) + cfg.ArtifactOriginID = "desktop-d4e5f6" + + adopted, err := cfg.AdoptArtifactOriginID("laptop-a1b2c3") + require.NoError(t, err) + assert.Equal(t, "desktop-d4e5f6", adopted) +} + +func TestAdoptArtifactOriginIDRejectsInvalidOrigin(t *testing.T) { + setupTestEnv(t) + cfg, err := LoadMinimal() + require.NoError(t, err) + + _, err = cfg.AdoptArtifactOriginID("local") + require.Error(t, err) + assert.Empty(t, cfg.ArtifactOriginID) +} + +func TestEnsureArtifactOriginIDReusesOriginWrittenBeforeLock(t *testing.T) { + tmp := setupTestEnv(t) + cfg, err := LoadMinimal() + require.NoError(t, err) + + lock := flock.New(cfg.configPath() + ".lock") + require.NoError(t, lock.Lock()) + t.Cleanup(func() { + _ = lock.Unlock() + }) + + type ensureResult struct { + origin string + err error + } + done := make(chan ensureResult, 1) + go func() { + origin, err := cfg.EnsureArtifactOriginID() + done <- ensureResult{origin: origin, err: err} + }() + + select { + case res := <-done: + require.Failf(t, "EnsureArtifactOriginID ignored config lock", + "origin=%q err=%v", res.origin, res.err) + case <-time.After(250 * time.Millisecond): + } + + writeConfig(t, tmp, map[string]any{ + "artifact_origin_id": "winner-a1b2c3", + }) + require.NoError(t, lock.Unlock()) + + select { + case res := <-done: + require.NoError(t, res.err) + assert.Equal(t, "winner-a1b2c3", res.origin) + assert.Equal(t, "winner-a1b2c3", cfg.ArtifactOriginID) + case <-time.After(2 * time.Second): + require.Fail(t, "EnsureArtifactOriginID did not finish after lock release") + } +} + +func TestMigrateJSONToTOMLPreservesArtifactOriginID(t *testing.T) { + tmp := setupTestEnv(t) + jsonPath := filepath.Join(tmp, "config.json") + require.NoError(t, os.WriteFile( + jsonPath, + []byte(`{"artifact_origin_id":"desk-abcdef"}`), + 0o600, + )) + + cfg, err := LoadMinimal() + require.NoError(t, err) + + assert.Equal(t, "desk-abcdef", cfg.ArtifactOriginID) + _, err = os.Stat(jsonPath + ".bak") + require.NoError(t, err) + data, err := os.ReadFile(filepath.Join(tmp, configFileName)) + require.NoError(t, err) + assert.Contains(t, string(data), `artifact_origin_id = "desk-abcdef"`) +} + func TestLoad_ProxyConfigFromFile(t *testing.T) { cfg := loadMinimalWithConfig(t, map[string]any{ "public_url": "https://viewer.example.test", diff --git a/internal/db/db.go b/internal/db/db.go index 62aca4b03..466628c3d 100644 --- a/internal/db/db.go +++ b/internal/db/db.go @@ -1639,6 +1639,47 @@ func (db *DB) migrateColumns() error { return err } + if _, err := w.Exec(` + CREATE TABLE IF NOT EXISTS metadata_applied_events ( + origin TEXT NOT NULL, + order_key TEXT NOT NULL, + artifact_hash TEXT NOT NULL, + applied_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')), + PRIMARY KEY (origin, order_key) + ); + CREATE TABLE IF NOT EXISTS metadata_replay_state ( + session_gid TEXT NOT NULL, + field TEXT NOT NULL, + order_key TEXT NOT NULL, + hlc TEXT NOT NULL, + artifact_hash TEXT NOT NULL, + origin TEXT NOT NULL, + op TEXT NOT NULL, + value TEXT NOT NULL DEFAULT '', + updated_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')), + PRIMARY KEY (session_gid, field) + ); + CREATE TABLE IF NOT EXISTS metadata_conflicts ( + id INTEGER PRIMARY KEY, + session_gid TEXT NOT NULL, + field TEXT NOT NULL, + winning_order_key TEXT NOT NULL, + losing_order_key TEXT NOT NULL, + winning_origin TEXT NOT NULL, + losing_origin TEXT NOT NULL, + winning_op TEXT NOT NULL, + losing_op TEXT NOT NULL, + winning_value TEXT NOT NULL DEFAULT '', + losing_value TEXT NOT NULL DEFAULT '', + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')), + UNIQUE(session_gid, field, winning_order_key, losing_order_key) + ); + `); err != nil { + return fmt.Errorf( + "creating metadata replay tables: %w", err, + ) + } + if err := db.ensureUsageEventsSchemaLocked(w); err != nil { return err } @@ -2885,6 +2926,47 @@ func (db *DB) GetSyncState(key string) (string, error) { return value, err } +// SyncStateValues reads the non-empty values for exact sync-state keys. Queries +// are bounded so callers can bulk-resolve state without exceeding SQLite's +// historical variable limit. +func (db *DB) SyncStateValues(keys []string) (map[string]string, error) { + states := map[string]string{} + const batchSize = 900 + for start := 0; start < len(keys); start += batchSize { + end := min(start+batchSize, len(keys)) + batch := keys[start:end] + placeholders := strings.TrimSuffix(strings.Repeat("?,", len(batch)), ",") + args := make([]any, len(batch)) + for i, key := range batch { + args[i] = key + } + rows, err := db.getReader().Query( + `SELECT key, value FROM pg_sync_state + WHERE key IN (`+placeholders+`) AND value <> ''`, + args..., + ) + if err != nil { + return nil, err + } + for rows.Next() { + var key, value string + if err := rows.Scan(&key, &value); err != nil { + rows.Close() + return nil, err + } + states[key] = value + } + if err := rows.Err(); err != nil { + rows.Close() + return nil, err + } + if err := rows.Close(); err != nil { + return nil, err + } + } + return states, nil +} + // SetSyncState writes a value to the pg_sync_state table. func (db *DB) SetSyncState(key, value string) error { db.mu.Lock() diff --git a/internal/db/db_test.go b/internal/db/db_test.go index be473bb31..59502aa8c 100644 --- a/internal/db/db_test.go +++ b/internal/db/db_test.go @@ -892,6 +892,47 @@ func TestInsertMessages_PreservesToolResultEvents(t *testing.T) { assert.Equal(t, "subagent_notification", tc.ResultEvents[1].Source, "result event 1 source") } +func TestSessionSubagentSessionIDs(t *testing.T) { + d := testDB(t) + insertSession(t, d, "s-sub", "proj") + require.NoError(t, d.InsertMessages([]Message{ + { + SessionID: "s-sub", Ordinal: 0, Role: "assistant", + Content: "spawn", HasToolUse: true, + ToolCalls: []ToolCall{ + { + SessionID: "s-sub", ToolName: "Task", Category: "Task", + ToolUseID: "call-1", SubagentSessionID: "sub-a", + ResultEvents: []ToolResultEvent{ + { + ToolUseID: "call-1", SubagentSessionID: "sub-a", + Source: "subagent", Status: "ok", EventIndex: 0, + }, + { + ToolUseID: "call-1", SubagentSessionID: "sub-b", + Source: "subagent", Status: "ok", EventIndex: 1, + }, + }, + }, + { + SessionID: "s-sub", ToolName: "Read", Category: "file", + ToolUseID: "call-2", + }, + }, + }, + }), "InsertMessages") + + ids, err := d.SessionSubagentSessionIDs("s-sub") + require.NoError(t, err, "SessionSubagentSessionIDs") + // "sub-a" appears on both the tool call and a result event (deduped); + // "sub-b" only on a result event; the empty subagent id is excluded. + assert.ElementsMatch(t, []string{"sub-a", "sub-b"}, ids) + + none, err := d.SessionSubagentSessionIDs("missing") + require.NoError(t, err, "SessionSubagentSessionIDs missing") + assert.Empty(t, none) +} + func TestOpenPreservesDataAtCurrentVersion(t *testing.T) { dir := t.TempDir() path := filepath.Join(dir, "test.db") @@ -4384,12 +4425,17 @@ func TestStarSession(t *testing.T) { assert.Equal(t, []string{"s1"}, ids, "listed = %v, want [s1]", ids) // Unstar. - err = d.UnstarSession("s1") + removed, err := d.UnstarSession("s1") require.NoError(t, err, "UnstarSession") + assert.True(t, removed, "UnstarSession should report removed star") ids, err = d.ListStarredSessionIDs(ctx) require.NoError(t, err, "ListStarredSessionIDs after unstar") assert.Empty(t, ids, "listed after unstar = %v, want []", ids) + removed, err = d.UnstarSession("s1") + require.NoError(t, err, "UnstarSession no-op") + assert.False(t, removed, "UnstarSession should report no-op") + // Star non-existent session returns false (no FK error). ok, err = d.StarSession("nonexistent") require.NoError(t, err, "StarSession nonexistent") @@ -4403,8 +4449,13 @@ func TestBulkStarSessions(t *testing.T) { insertSession(t, d, "s2", "proj") // Bulk star with mix of valid and invalid IDs. - err := d.BulkStarSessions([]string{"s1", "s2", "nonexistent"}) + starred, err := d.BulkStarSessions([]string{"s1", "s2", "nonexistent"}) require.NoError(t, err, "BulkStarSessions") + assert.ElementsMatch(t, []string{"s1", "s2"}, starred, "starred ids returned") + + starred, err = d.BulkStarSessions([]string{"s1", "s2"}) + require.NoError(t, err, "BulkStarSessions already starred") + assert.Empty(t, starred, "already-starred ids should not be returned") ids, err := d.ListStarredSessionIDs(ctx) require.NoError(t, err, "ListStarredSessionIDs") @@ -4679,6 +4730,16 @@ func TestCopySyncStateFrom_OnlyCopiesDurablePGKeys(t *testing.T) { srcDB := testDBAtPath(t, srcPath, "src") require.NoError(t, srcDB.SetSyncState("pg_push_marker_id", "marker-123"), "seed source marker") + require.NoError(t, srcDB.SetSyncState("artifact_origin_id", "laptop-a1b2c3"), + "seed source artifact origin") + require.NoError(t, srcDB.SetSyncState("artifact_metadata_hlc", "hlc-42"), + "seed source artifact hlc") + require.NoError(t, + srcDB.SetSyncState("artifact_import:peer-b4c5d6:peer-b4c5d6~sess-1", "hash-imp"), + "seed source import watermark") + require.NoError(t, + srcDB.SetSyncState("artifact_export:laptop-a1b2c3:sess-2", "hash-exp"), + "seed source export watermark") require.NoError(t, srcDB.SetSyncState("last_sync_started_at", "old-start"), "seed source started") require.NoError(t, srcDB.SetSyncState("last_sync_finished_at", "old-finish"), @@ -4700,6 +4761,18 @@ func TestCopySyncStateFrom_OnlyCopiesDurablePGKeys(t *testing.T) { require.NoError(t, err, "GetSyncState pg_push_marker_id") assert.Equal(t, "marker-123", gotMarker) + durableArtifactKeys := map[string]string{ + "artifact_origin_id": "laptop-a1b2c3", + "artifact_metadata_hlc": "hlc-42", + "artifact_import:peer-b4c5d6:peer-b4c5d6~sess-1": "hash-imp", + "artifact_export:laptop-a1b2c3:sess-2": "hash-exp", + } + for key, want := range durableArtifactKeys { + got, err := dstDB.GetSyncState(key) + require.NoError(t, err, "GetSyncState %s", key) + assert.Equal(t, want, got, "artifact key %s must survive the copy", key) + } + gotStarted, err := dstDB.GetSyncState("last_sync_started_at") require.NoError(t, err, "GetSyncState last_sync_started_at") assert.Equal(t, "new-start", gotStarted) @@ -4732,6 +4805,99 @@ func TestCopySyncStateFrom_PropagatesErrors(t *testing.T) { assert.Equal(t, "safe", got) } +func TestCopyMetadataReplayFrom(t *testing.T) { + dir := t.TempDir() + ctx := context.Background() + + srcPath := filepath.Join(dir, "src.db") + srcDB := testDBAtPath(t, srcPath, "src") + + // Seed the LWW register and applied-event markers without touching + // session rows, exactly as local curation bookkeeping does. + winner := MetadataProjection{ + EventOrigin: "laptop-a1b2c3", + OrderKey: "0000000002", + HLC: "hlc-2", + ArtifactHash: "hash-2", + SessionGID: "laptop-a1b2c3~sess-1", + LocalSessionID: "sess-1", + Field: "display_name", + Op: "rename", + Value: `{"display_name":"kept"}`, + } + res, err := srcDB.RecordLocalMetadataProjection(ctx, winner) + require.NoError(t, err, "record winning projection") + require.True(t, res.Applied, "winning projection applied") + + // A stale peer event loses and records a conflict row. + loser := winner + loser.EventOrigin = "peer-b4c5d6" + loser.OrderKey = "0000000001" + loser.HLC = "hlc-1" + loser.ArtifactHash = "hash-1" + loser.Value = `{"display_name":"stale"}` + res, err = srcDB.RecordLocalMetadataProjection(ctx, loser) + require.NoError(t, err, "record losing projection") + require.True(t, res.Conflict, "losing projection records a conflict") + require.NoError(t, srcDB.Close(), "Close src") + + dstPath := filepath.Join(dir, "dst.db") + dstDB := testDBAtPath(t, dstPath, "dst") + defer dstDB.Close() + + require.NoError(t, dstDB.CopyMetadataReplayFrom(srcPath), + "CopyMetadataReplayFrom") + + for _, ev := range []MetadataProjection{winner, loser} { + applied, err := dstDB.MetadataEventApplied(ctx, ev.EventOrigin, ev.OrderKey) + require.NoError(t, err, "MetadataEventApplied %s/%s", ev.EventOrigin, ev.OrderKey) + assert.True(t, applied, + "applied-event marker %s/%s must survive the copy", + ev.EventOrigin, ev.OrderKey) + } + + op, ok, err := dstDB.MetadataReplayStateOp(ctx, winner.SessionGID, winner.Field) + require.NoError(t, err, "MetadataReplayStateOp") + require.True(t, ok, "replay state must survive the copy") + assert.Equal(t, "rename", op) + + conflicts, err := dstDB.ListMetadataConflicts(ctx, []string{winner.SessionGID}) + require.NoError(t, err, "ListMetadataConflicts") + require.Len(t, conflicts, 1, "conflict row must survive the copy") + assert.Equal(t, "laptop-a1b2c3", conflicts[0].WinningOrigin) + assert.Equal(t, "peer-b4c5d6", conflicts[0].LosingOrigin) +} + +func TestCopyMetadataReplayFrom_NoSourceTables(t *testing.T) { + dir := t.TempDir() + ctx := context.Background() + + // Simulate an older source database that predates the metadata + // replay tables. + srcPath := filepath.Join(dir, "src.db") + srcDB := testDBAtPath(t, srcPath, "src") + for _, table := range []string{ + "metadata_applied_events", + "metadata_replay_state", + "metadata_conflicts", + } { + _, err := srcDB.getWriter().Exec("DROP TABLE " + table) + require.NoError(t, err, "drop %s", table) + } + require.NoError(t, srcDB.Close(), "Close src") + + dstPath := filepath.Join(dir, "dst.db") + dstDB := testDBAtPath(t, dstPath, "dst") + defer dstDB.Close() + + require.NoError(t, dstDB.CopyMetadataReplayFrom(srcPath), + "CopyMetadataReplayFrom with missing source tables") + + applied, err := dstDB.MetadataEventApplied(ctx, "laptop-a1b2c3", "0000000001") + require.NoError(t, err, "MetadataEventApplied") + assert.False(t, applied) +} + func TestCopySessionMetadataFrom(t *testing.T) { dir := t.TempDir() ctx := context.Background() diff --git a/internal/db/messages.go b/internal/db/messages.go index f4ca876f0..1b681cf2e 100644 --- a/internal/db/messages.go +++ b/internal/db/messages.go @@ -363,6 +363,26 @@ func (db *DB) GetAllMessages( return msgs, nil } +// GetMessageForMetadataPin returns only the stable fields needed to publish a +// pin metadata event. It deliberately avoids loading message content, tool +// calls, and tool-result events for an otherwise single-row lookup. +func (db *DB) GetMessageForMetadataPin( + ctx context.Context, sessionID string, messageID int64, +) (*Message, error) { + row := db.getReader().QueryRowContext(ctx, ` + SELECT id, session_id, ordinal, COALESCE(source_uuid, '') + FROM messages + WHERE session_id = ? AND id = ?`, sessionID, messageID) + var msg Message + if err := row.Scan(&msg.ID, &msg.SessionID, &msg.Ordinal, &msg.SourceUUID); err != nil { + if errors.Is(err, sql.ErrNoRows) { + return nil, nil + } + return nil, fmt.Errorf("querying message metadata for pin: %w", err) + } + return &msg, nil +} + // EmbeddableUnit is one embedding document: a single embeddable user // message, or a run of contiguous embeddable assistant messages. type EmbeddableUnit struct { @@ -2132,6 +2152,40 @@ func (db *DB) ToolResultEventFingerprintWithTimestampNormalizer( return b.String(), rows.Err() } +// SessionSubagentSessionIDs returns the distinct non-empty subagent_session_id +// values referenced by a session's tool calls and tool result events. Used by +// PG push to decide whether a session whose content otherwise matches PG still +// needs its message rows replaced so subagent links can be re-resolved. +func (db *DB) SessionSubagentSessionIDs(sessionID string) ([]string, error) { + rows, err := db.getReader().Query(` + SELECT DISTINCT subagent_session_id FROM ( + SELECT subagent_session_id FROM tool_calls + WHERE session_id = ? + UNION + SELECT subagent_session_id FROM tool_result_events + WHERE session_id = ? + ) + WHERE subagent_session_id IS NOT NULL AND subagent_session_id != ''`, + sessionID, sessionID, + ) + if err != nil { + return nil, fmt.Errorf( + "querying subagent session ids for %s: %w", sessionID, err, + ) + } + defer rows.Close() + + var ids []string + for rows.Next() { + var id string + if err := rows.Scan(&id); err != nil { + return nil, err + } + ids = append(ids, id) + } + return ids, rows.Err() +} + // GetMessageByOrdinal returns a single message by session ID and ordinal. func (db *DB) GetMessageByOrdinal( sessionID string, ordinal int, diff --git a/internal/db/metadata_baseline.go b/internal/db/metadata_baseline.go new file mode 100644 index 000000000..c57b74804 --- /dev/null +++ b/internal/db/metadata_baseline.go @@ -0,0 +1,140 @@ +package db + +import ( + "context" + "database/sql" + "fmt" +) + +// MetadataBaselineSnapshot captures existing local user curation that predates +// artifact metadata event recording. +type MetadataBaselineSnapshot struct { + Renames []MetadataBaselineRename + StarredSessionIDs []string + SoftDeletedIDs []string + Pins []MetadataBaselinePin +} + +// MetadataBaselineRename is one session display-name override. +type MetadataBaselineRename struct { + SessionID string + DisplayName *string +} + +// MetadataBaselinePin is one pinned message represented in metadata-event +// coordinates. +type MetadataBaselinePin struct { + SessionID string + SourceUUID string + Ordinal int + Note *string +} + +// MetadataBaselineSnapshot returns the current curation rows that need baseline +// metadata events during artifact sync initialization. +func (db *DB) MetadataBaselineSnapshot(ctx context.Context) (MetadataBaselineSnapshot, error) { + var snap MetadataBaselineSnapshot + + // Curation queries do not filter on deleted_at: a session sitting in + // trash at opt-in still baselines its name, star, and pins, so a later + // restore reaches peers with them instead of only the soft delete. + renameRows, err := db.getReader().QueryContext(ctx, ` + SELECT id, display_name + FROM sessions + WHERE display_name IS NOT NULL + ORDER BY id`) + if err != nil { + return snap, fmt.Errorf("listing baseline renames: %w", err) + } + defer renameRows.Close() + for renameRows.Next() { + var sessionID string + var displayName string + if err := renameRows.Scan(&sessionID, &displayName); err != nil { + return snap, fmt.Errorf("scanning baseline rename: %w", err) + } + displayNameCopy := displayName + snap.Renames = append(snap.Renames, MetadataBaselineRename{ + SessionID: sessionID, + DisplayName: &displayNameCopy, + }) + } + if err := renameRows.Err(); err != nil { + return snap, fmt.Errorf("iterating baseline renames: %w", err) + } + + starRows, err := db.getReader().QueryContext(ctx, ` + SELECT ss.session_id + FROM starred_sessions ss + JOIN sessions s + ON s.id = ss.session_id + ORDER BY ss.session_id`) + if err != nil { + return snap, fmt.Errorf("listing baseline stars: %w", err) + } + defer starRows.Close() + for starRows.Next() { + var sessionID string + if err := starRows.Scan(&sessionID); err != nil { + return snap, fmt.Errorf("scanning baseline star: %w", err) + } + snap.StarredSessionIDs = append(snap.StarredSessionIDs, sessionID) + } + if err := starRows.Err(); err != nil { + return snap, fmt.Errorf("iterating baseline stars: %w", err) + } + + deletedRows, err := db.getReader().QueryContext(ctx, ` + SELECT id + FROM sessions + WHERE deleted_at IS NOT NULL + ORDER BY id`) + if err != nil { + return snap, fmt.Errorf("listing baseline soft deletes: %w", err) + } + defer deletedRows.Close() + for deletedRows.Next() { + var sessionID string + if err := deletedRows.Scan(&sessionID); err != nil { + return snap, fmt.Errorf("scanning baseline soft delete: %w", err) + } + snap.SoftDeletedIDs = append(snap.SoftDeletedIDs, sessionID) + } + if err := deletedRows.Err(); err != nil { + return snap, fmt.Errorf("iterating baseline soft deletes: %w", err) + } + + pinRows, err := db.getReader().QueryContext(ctx, ` + SELECT p.session_id, COALESCE(m.source_uuid, ''), + m.ordinal, p.note + FROM pinned_messages p + JOIN sessions s + ON s.id = p.session_id + JOIN messages m + ON m.id = p.message_id + AND m.session_id = p.session_id + ORDER BY p.session_id, m.ordinal, p.id`) + if err != nil { + return snap, fmt.Errorf("listing baseline pins: %w", err) + } + defer pinRows.Close() + for pinRows.Next() { + var pin MetadataBaselinePin + var note sql.NullString + if err := pinRows.Scan( + &pin.SessionID, &pin.SourceUUID, &pin.Ordinal, ¬e, + ); err != nil { + return snap, fmt.Errorf("scanning baseline pin: %w", err) + } + if note.Valid { + noteCopy := note.String + pin.Note = ¬eCopy + } + snap.Pins = append(snap.Pins, pin) + } + if err := pinRows.Err(); err != nil { + return snap, fmt.Errorf("iterating baseline pins: %w", err) + } + + return snap, nil +} diff --git a/internal/db/metadata_baseline_test.go b/internal/db/metadata_baseline_test.go new file mode 100644 index 000000000..85857f75b --- /dev/null +++ b/internal/db/metadata_baseline_test.go @@ -0,0 +1,52 @@ +package db + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// A session that was renamed, starred, and pinned before the machine first +// opted into artifact sync must baseline that curation even while it sits in +// trash: only the soft delete would otherwise publish, and a later restore +// would reach peers without the name, star, or pin. +func TestMetadataBaselineSnapshotIncludesTrashedSessions(t *testing.T) { + d := testDB(t) + ctx := context.Background() + + require.NoError(t, d.UpsertSession(Session{ + ID: "s1", Project: "proj", Machine: "local", Agent: "claude", + MessageCount: 1, CreatedAt: "2026-01-01T00:00:00Z", + })) + require.NoError(t, d.InsertMessages([]Message{{ + SessionID: "s1", Ordinal: 0, Role: "user", Content: "hi", + ContentLength: 2, SourceUUID: "uuid-1", + }})) + name := "Kept name" + require.NoError(t, d.RenameSession("s1", &name)) + starred, err := d.StarSession("s1") + require.NoError(t, err) + require.True(t, starred) + msgs, err := d.GetAllMessages(ctx, "s1") + require.NoError(t, err) + require.Len(t, msgs, 1) + note := "kept pin" + _, err = d.PinMessage("s1", msgs[0].ID, ¬e) + require.NoError(t, err) + require.NoError(t, d.SoftDeleteSession("s1")) + + snap, err := d.MetadataBaselineSnapshot(ctx) + require.NoError(t, err) + + require.Len(t, snap.Renames, 1) + assert.Equal(t, "s1", snap.Renames[0].SessionID) + require.NotNil(t, snap.Renames[0].DisplayName) + assert.Equal(t, name, *snap.Renames[0].DisplayName) + assert.Equal(t, []string{"s1"}, snap.StarredSessionIDs) + assert.Equal(t, []string{"s1"}, snap.SoftDeletedIDs) + require.Len(t, snap.Pins, 1) + assert.Equal(t, "s1", snap.Pins[0].SessionID) + assert.Equal(t, "uuid-1", snap.Pins[0].SourceUUID) +} diff --git a/internal/db/metadata_replay.go b/internal/db/metadata_replay.go new file mode 100644 index 000000000..d01f9a15f --- /dev/null +++ b/internal/db/metadata_replay.go @@ -0,0 +1,830 @@ +package db + +import ( + "context" + "database/sql" + "encoding/json" + "errors" + "fmt" + "strings" + "time" +) + +// ErrMetadataTargetUnavailable means a metadata event depends on session or +// message content that is not durable locally yet. +var ErrMetadataTargetUnavailable = errors.New("metadata target unavailable") + +// MetadataPinProjection identifies a pinned message during metadata replay. +type MetadataPinProjection struct { + SourceUUID string `json:"source_uuid,omitempty"` + Ordinal int `json:"ordinal"` + Note *string `json:"note,omitempty"` +} + +// MetadataProjection is one decoded artifact metadata event ready for replay. +type MetadataProjection struct { + EventOrigin string + OrderKey string + HLC string + ArtifactHash string + SessionGID string + LocalSessionID string + Field string + Op string + Value string + DisplayName *string + Pin *MetadataPinProjection +} + +// MetadataApplyResult summarizes how replay handled an event. +type MetadataApplyResult struct { + Applied bool + Skipped bool + Conflict bool + Duplicate bool +} + +// MetadataConflict is a losing metadata value recorded during deterministic +// replay. +type MetadataConflict struct { + ID int64 `json:"id"` + SessionGID string `json:"session_gid"` + Field string `json:"field"` + WinningOrderKey string `json:"winning_order_key"` + LosingOrderKey string `json:"losing_order_key"` + WinningOrigin string `json:"winning_origin"` + LosingOrigin string `json:"losing_origin"` + WinningOp string `json:"winning_op"` + LosingOp string `json:"losing_op"` + WinningValue string `json:"winning_value"` + LosingValue string `json:"losing_value"` + CreatedAt string `json:"created_at"` +} + +type metadataReplayState struct { + OrderKey string + HLC string + ArtifactHash string + Origin string + Op string + Value string +} + +// MetadataEventIdentity identifies one immutable artifact metadata event. +type MetadataEventIdentity struct { + Origin string + OrderKey string +} + +// MetadataAppliedEventIdentities bulk-loads the events already durably handled +// by metadata replay. +func (db *DB) MetadataAppliedEventIdentities( + ctx context.Context, +) (map[MetadataEventIdentity]struct{}, error) { + rows, err := db.getReader().QueryContext(ctx, + `SELECT origin, order_key FROM metadata_applied_events`, + ) + if err != nil { + return nil, fmt.Errorf("listing applied metadata events: %w", err) + } + defer rows.Close() + + identities := make(map[MetadataEventIdentity]struct{}) + for rows.Next() { + var identity MetadataEventIdentity + if err := rows.Scan(&identity.Origin, &identity.OrderKey); err != nil { + return nil, fmt.Errorf("scanning applied metadata event: %w", err) + } + identities[identity] = struct{}{} + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("iterating applied metadata events: %w", err) + } + return identities, nil +} + +// MetadataEventApplied reports whether an artifact metadata event was already +// durably handled. +func (db *DB) MetadataEventApplied(ctx context.Context, origin, orderKey string) (bool, error) { + var exists int + err := db.getReader().QueryRowContext(ctx, + `SELECT 1 FROM metadata_applied_events + WHERE origin = ? AND order_key = ?`, + origin, orderKey, + ).Scan(&exists) + if err == sql.ErrNoRows { + return false, nil + } + if err != nil { + return false, fmt.Errorf("checking metadata event %s/%s: %w", origin, orderKey, err) + } + return true, nil +} + +// ListMetadataConflicts returns conflict rows for one or more global session +// identifiers. +func (db *DB) ListMetadataConflicts( + ctx context.Context, + sessionGIDs []string, +) ([]MetadataConflict, error) { + ids := uniqueNonEmptyStrings(sessionGIDs) + if len(ids) == 0 { + return []MetadataConflict{}, nil + } + placeholders := strings.TrimRight(strings.Repeat("?,", len(ids)), ",") + args := make([]any, len(ids)) + for i, id := range ids { + args[i] = id + } + rows, err := db.getReader().QueryContext(ctx, + `SELECT id, session_gid, field, winning_order_key, losing_order_key, + winning_origin, losing_origin, winning_op, losing_op, + winning_value, losing_value, created_at + FROM metadata_conflicts + WHERE session_gid IN (`+placeholders+`) + AND winning_origin <> losing_origin + ORDER BY created_at DESC, id DESC`, + args..., + ) + if err != nil { + return nil, fmt.Errorf("listing metadata conflicts: %w", err) + } + defer rows.Close() + + conflicts := []MetadataConflict{} + for rows.Next() { + var c MetadataConflict + if err := rows.Scan( + &c.ID, &c.SessionGID, &c.Field, + &c.WinningOrderKey, &c.LosingOrderKey, + &c.WinningOrigin, &c.LosingOrigin, + &c.WinningOp, &c.LosingOp, + &c.WinningValue, &c.LosingValue, + &c.CreatedAt, + ); err != nil { + return nil, fmt.Errorf("scanning metadata conflict: %w", err) + } + conflicts = append(conflicts, c) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("iterating metadata conflicts: %w", err) + } + return conflicts, nil +} + +// CountMetadataConflicts returns the total number of recorded metadata +// conflicts across all sessions. +func (db *DB) CountMetadataConflicts(ctx context.Context) (int, error) { + var count int + err := db.getReader().QueryRowContext(ctx, + `SELECT COUNT(*) FROM metadata_conflicts + WHERE winning_origin <> losing_origin`, + ).Scan(&count) + if err != nil { + return 0, fmt.Errorf("counting metadata conflicts: %w", err) + } + return count, nil +} + +// MarkMetadataEventApplied records a metadata event that was intentionally +// skipped, such as an unknown future op. +func (db *DB) MarkMetadataEventApplied(ctx context.Context, origin, orderKey, hash string) error { + db.mu.Lock() + defer db.mu.Unlock() + _, err := db.getWriter().ExecContext(ctx, + `INSERT OR IGNORE INTO metadata_applied_events + (origin, order_key, artifact_hash) + VALUES (?, ?, ?)`, + origin, orderKey, hash, + ) + if err != nil { + return fmt.Errorf("marking metadata event %s/%s applied: %w", origin, orderKey, err) + } + return nil +} + +// ApplyMetadataProjection applies one known metadata event if it wins the +// per-field LWW register, recording conflicts and the applied-event marker in +// the same transaction. +func (db *DB) ApplyMetadataProjection( + ctx context.Context, + ev MetadataProjection, +) (MetadataApplyResult, error) { + return db.applyMetadataProjection(ctx, ev, true) +} + +// RecordLocalMetadataProjection records the LWW register, conflict rows, and +// applied-event marker for a locally-originated metadata event whose session +// mutation the caller has already applied. It runs the same per-field LWW +// bookkeeping as replay but does not re-apply the mutation, so a later peer +// event with a lower order key cannot silently overwrite a newer local edit. +func (db *DB) RecordLocalMetadataProjection( + ctx context.Context, + ev MetadataProjection, +) (MetadataApplyResult, error) { + return db.applyMetadataProjection(ctx, ev, false) +} + +// MetadataReplayStateOp returns the current LWW operation recorded for a +// metadata field. +func (db *DB) MetadataReplayStateOp( + ctx context.Context, + sessionGID string, + field string, +) (string, bool, error) { + var op string + err := db.getReader().QueryRowContext(ctx, + `SELECT op FROM metadata_replay_state + WHERE session_gid = ? AND field = ?`, + sessionGID, field, + ).Scan(&op) + if err == sql.ErrNoRows { + return "", false, nil + } + if err != nil { + return "", false, fmt.Errorf("reading metadata replay state: %w", err) + } + return op, true, nil +} + +// ReapplyMetadataReplayState reapplies the current visible metadata projection +// for a session from the durable replay register. It does not alter LWW state or +// applied-event markers; it only repairs fields that content import may have +// overwritten or invalidated while replacing session/message rows. +func (db *DB) ReapplyMetadataReplayState( + ctx context.Context, + sessionGID string, + localSessionID string, +) (int, error) { + if err := db.requireWritable(); err != nil { + return 0, err + } + if strings.TrimSpace(sessionGID) == "" || strings.TrimSpace(localSessionID) == "" { + return 0, errors.New("metadata replay session id is required") + } + + db.mu.Lock() + defer db.mu.Unlock() + tx, err := db.getWriter().BeginTx(ctx, nil) + if err != nil { + return 0, fmt.Errorf("begin metadata reapply tx: %w", err) + } + defer func() { _ = tx.Rollback() }() + + rows, err := tx.QueryContext(ctx, + `SELECT field, order_key, hlc, artifact_hash, origin, op, value + FROM metadata_replay_state + WHERE session_gid = ? + ORDER BY field`, + sessionGID, + ) + if err != nil { + return 0, fmt.Errorf("reading metadata replay state: %w", err) + } + projections := []MetadataProjection{} + for rows.Next() { + var field string + var state metadataReplayState + if err := rows.Scan( + &field, &state.OrderKey, &state.HLC, &state.ArtifactHash, + &state.Origin, &state.Op, &state.Value, + ); err != nil { + rows.Close() + return 0, fmt.Errorf("scanning metadata replay state: %w", err) + } + ev, err := metadataReplayStateProjection(sessionGID, localSessionID, field, state) + if err != nil { + rows.Close() + return 0, err + } + projections = append(projections, ev) + } + if err := rows.Close(); err != nil { + return 0, fmt.Errorf("closing metadata replay state rows: %w", err) + } + if err := rows.Err(); err != nil { + return 0, fmt.Errorf("iterating metadata replay state: %w", err) + } + + applied := 0 + for _, ev := range projections { + if err := ctx.Err(); err != nil { + return applied, err + } + if err := applyMetadataProjectionTx(ctx, tx, ev); err != nil { + if errors.Is(err, ErrMetadataTargetUnavailable) { + continue + } + return applied, fmt.Errorf("reapplying metadata replay state: %w", err) + } + applied++ + } + if err := tx.Commit(); err != nil { + return applied, fmt.Errorf("commit metadata reapply tx: %w", err) + } + return applied, nil +} + +func (db *DB) applyMetadataProjection( + ctx context.Context, + ev MetadataProjection, + applyMutation bool, +) (MetadataApplyResult, error) { + if ev.EventOrigin == "" || ev.OrderKey == "" || ev.ArtifactHash == "" { + return MetadataApplyResult{}, errors.New("metadata projection event identity is required") + } + if ev.SessionGID == "" || ev.LocalSessionID == "" || ev.Field == "" || ev.Op == "" { + return MetadataApplyResult{}, errors.New("metadata projection target is required") + } + + db.mu.Lock() + defer db.mu.Unlock() + tx, err := db.getWriter().BeginTx(ctx, nil) + if err != nil { + return MetadataApplyResult{}, fmt.Errorf("begin metadata replay tx: %w", err) + } + defer func() { _ = tx.Rollback() }() + + already, err := metadataEventAppliedTx(ctx, tx, ev.EventOrigin, ev.OrderKey) + if err != nil { + return MetadataApplyResult{}, err + } + if already { + if err := tx.Commit(); err != nil { + return MetadataApplyResult{}, fmt.Errorf("commit metadata replay duplicate: %w", err) + } + return MetadataApplyResult{Skipped: true, Duplicate: true}, nil + } + + current, hasCurrent, err := metadataReplayStateTx(ctx, tx, ev.SessionGID, ev.Field) + if err != nil { + return MetadataApplyResult{}, err + } + result := MetadataApplyResult{} + if hasCurrent && ev.OrderKey <= current.OrderKey { + if metadataStateDiffers(current.Op, current.Value, ev.Op, ev.Value) && + metadataConflictOriginsDiffer(current.Origin, ev.EventOrigin) { + if err := insertMetadataConflictTx(ctx, tx, metadataConflict{ + sessionGID: ev.SessionGID, + field: ev.Field, + winningOrderKey: current.OrderKey, + losingOrderKey: ev.OrderKey, + winningOrigin: current.Origin, + losingOrigin: ev.EventOrigin, + winningOp: current.Op, + losingOp: ev.Op, + winningValue: current.Value, + losingValue: ev.Value, + }); err != nil { + return MetadataApplyResult{}, err + } + result.Conflict = true + } + if err := markMetadataEventAppliedTx(ctx, tx, ev.EventOrigin, ev.OrderKey, ev.ArtifactHash); err != nil { + return MetadataApplyResult{}, err + } + if err := tx.Commit(); err != nil { + return MetadataApplyResult{}, fmt.Errorf("commit metadata replay loser: %w", err) + } + result.Skipped = true + return result, nil + } + + if hasCurrent && metadataStateDiffers(current.Op, current.Value, ev.Op, ev.Value) && + metadataConflictOriginsDiffer(ev.EventOrigin, current.Origin) { + if err := insertMetadataConflictTx(ctx, tx, metadataConflict{ + sessionGID: ev.SessionGID, + field: ev.Field, + winningOrderKey: ev.OrderKey, + losingOrderKey: current.OrderKey, + winningOrigin: ev.EventOrigin, + losingOrigin: current.Origin, + winningOp: ev.Op, + losingOp: current.Op, + winningValue: ev.Value, + losingValue: current.Value, + }); err != nil { + return MetadataApplyResult{}, err + } + result.Conflict = true + } + if applyMutation { + if err := applyMetadataProjectionTx(ctx, tx, ev); err != nil { + return MetadataApplyResult{}, err + } + } + if err := upsertMetadataReplayStateTx(ctx, tx, ev); err != nil { + return MetadataApplyResult{}, err + } + if err := markMetadataEventAppliedTx(ctx, tx, ev.EventOrigin, ev.OrderKey, ev.ArtifactHash); err != nil { + return MetadataApplyResult{}, err + } + if err := tx.Commit(); err != nil { + return MetadataApplyResult{}, fmt.Errorf("commit metadata replay: %w", err) + } + result.Applied = true + return result, nil +} + +func metadataEventAppliedTx(ctx context.Context, tx *sql.Tx, origin, orderKey string) (bool, error) { + var exists int + err := tx.QueryRowContext(ctx, + `SELECT 1 FROM metadata_applied_events + WHERE origin = ? AND order_key = ?`, + origin, orderKey, + ).Scan(&exists) + if err == sql.ErrNoRows { + return false, nil + } + if err != nil { + return false, fmt.Errorf("checking metadata event %s/%s: %w", origin, orderKey, err) + } + return true, nil +} + +func metadataReplayStateTx( + ctx context.Context, + tx *sql.Tx, + sessionGID, field string, +) (metadataReplayState, bool, error) { + var state metadataReplayState + err := tx.QueryRowContext(ctx, + `SELECT order_key, hlc, artifact_hash, origin, op, value + FROM metadata_replay_state + WHERE session_gid = ? AND field = ?`, + sessionGID, field, + ).Scan( + &state.OrderKey, &state.HLC, &state.ArtifactHash, + &state.Origin, &state.Op, &state.Value, + ) + if err == sql.ErrNoRows { + return metadataReplayState{}, false, nil + } + if err != nil { + return metadataReplayState{}, false, fmt.Errorf("reading metadata replay state: %w", err) + } + return state, true, nil +} + +func metadataReplayStateProjection( + sessionGID string, + localSessionID string, + field string, + state metadataReplayState, +) (MetadataProjection, error) { + ev := MetadataProjection{ + EventOrigin: state.Origin, + OrderKey: state.OrderKey, + HLC: state.HLC, + ArtifactHash: state.ArtifactHash, + SessionGID: sessionGID, + LocalSessionID: localSessionID, + Field: field, + Op: state.Op, + Value: state.Value, + } + switch state.Op { + case "rename": + var payload struct { + DisplayName *string `json:"display_name"` + } + if err := json.Unmarshal([]byte(state.Value), &payload); err != nil { + return MetadataProjection{}, fmt.Errorf("decoding rename metadata replay state: %w", err) + } + ev.DisplayName = payload.DisplayName + case "pin", "unpin": + var pin MetadataPinProjection + if err := json.Unmarshal([]byte(state.Value), &pin); err != nil { + return MetadataProjection{}, fmt.Errorf("decoding pin metadata replay state: %w", err) + } + ev.Pin = &pin + } + return ev, nil +} + +// metadataEventWallTime renders a replayed event's HLC wall-clock portion in +// the sessions.deleted_at column format, so trash retention is anchored to +// when the deletion happened on the authoring machine rather than when each +// peer imported the event. The HLC shape ("-<20-digit logical>", wall +// layout without ":" separators) is pinned by the artifact format contract in +// internal/artifact. Returns "" when the HLC has no parseable wall portion. +func metadataEventWallTime(hlc string) string { + idx := strings.LastIndex(hlc, "-") + if idx <= 0 { + return "" + } + wall, err := time.Parse("2006-01-02T150405.000000000Z", hlc[:idx]) + if err != nil { + return "" + } + return wall.UTC().Format("2006-01-02T15:04:05.000Z") +} + +func applyMetadataProjectionTx(ctx context.Context, tx *sql.Tx, ev MetadataProjection) error { + switch ev.Op { + case "rename": + if err := requireMetadataSessionTx(ctx, tx, ev.LocalSessionID); err != nil { + return err + } + _, err := tx.ExecContext(ctx, + `UPDATE sessions + SET display_name = ?, + local_modified_at = strftime('%Y-%m-%dT%H:%M:%fZ','now') + WHERE id = ?`, + ev.DisplayName, ev.LocalSessionID, + ) + return err + case "soft_delete": + if err := requireMetadataSessionTx(ctx, tx, ev.LocalSessionID); err != nil { + return err + } + _, err := tx.ExecContext(ctx, + `UPDATE sessions + SET deleted_at = COALESCE(deleted_at, NULLIF(?, ''), strftime('%Y-%m-%dT%H:%M:%fZ','now')), + local_modified_at = strftime('%Y-%m-%dT%H:%M:%fZ','now') + WHERE id = ?`, + metadataEventWallTime(ev.HLC), ev.LocalSessionID, + ) + return err + case "restore": + if err := requireMetadataSessionTx(ctx, tx, ev.LocalSessionID); err != nil { + return err + } + _, err := tx.ExecContext(ctx, + `UPDATE sessions + SET deleted_at = NULL, + local_modified_at = strftime('%Y-%m-%dT%H:%M:%fZ','now') + WHERE id = ?`, + ev.LocalSessionID, + ) + return err + case "star": + if err := requireMetadataSessionTx(ctx, tx, ev.LocalSessionID); err != nil { + return err + } + _, err := tx.ExecContext(ctx, + `INSERT OR IGNORE INTO starred_sessions (session_id) + VALUES (?)`, + ev.LocalSessionID, + ) + return err + case "unstar": + if err := requireMetadataSessionTx(ctx, tx, ev.LocalSessionID); err != nil { + return err + } + _, err := tx.ExecContext(ctx, + `DELETE FROM starred_sessions WHERE session_id = ?`, + ev.LocalSessionID, + ) + return err + case "pin": + if err := requireMetadataSessionTx(ctx, tx, ev.LocalSessionID); err != nil { + return err + } + if ev.Pin == nil { + return errors.New("pin metadata event missing pin payload") + } + return applyMetadataPinTx(ctx, tx, ev.LocalSessionID, *ev.Pin) + case "unpin": + if err := requireMetadataSessionTx(ctx, tx, ev.LocalSessionID); err != nil { + return err + } + if ev.Pin == nil { + return errors.New("unpin metadata event missing pin payload") + } + return unpinMetadataTx(ctx, tx, ev.LocalSessionID, *ev.Pin) + case "purge": + return applyMetadataPurgeTx(ctx, tx, ev.LocalSessionID) + default: + return fmt.Errorf("unsupported metadata op %q", ev.Op) + } +} + +func applyMetadataPinTx(ctx context.Context, tx *sql.Tx, sessionID string, pin MetadataPinProjection) error { + msg, ok, err := metadataPinTargetTx(ctx, tx, sessionID, pin) + if err != nil { + return err + } + if !ok { + return fmt.Errorf("%w: pin target %s ordinal %d", + ErrMetadataTargetUnavailable, sessionID, pin.Ordinal) + } + _, err = tx.ExecContext(ctx, + `INSERT INTO pinned_messages (session_id, message_id, ordinal, note) + VALUES (?, ?, ?, ?) + ON CONFLICT(session_id, message_id) DO UPDATE SET note = excluded.note`, + sessionID, msg.id, msg.ordinal, pin.Note, + ) + return err +} + +func applyMetadataPurgeTx(ctx context.Context, tx *sql.Tx, sessionID string) error { + aliasIDs, err := sessionAliasIDsTx(tx, "id = ?", sessionID) + if err != nil { + return err + } + if _, err := tx.ExecContext(ctx, + `INSERT OR IGNORE INTO excluded_sessions (id) VALUES (?)`, + sessionID, + ); err != nil { + return err + } + for _, aliasID := range aliasIDs { + if err := excludeSessionIDTx(tx, aliasID); err != nil { + return fmt.Errorf("excluding metadata purge alias %s: %w", aliasID, err) + } + } + _, err = tx.ExecContext(ctx, + `DELETE FROM sessions WHERE id = ?`, + sessionID, + ) + return err +} + +func requireMetadataSessionTx(ctx context.Context, tx *sql.Tx, id string) error { + var exists int + err := tx.QueryRowContext(ctx, + `SELECT 1 FROM sessions WHERE id = ?`, + id, + ).Scan(&exists) + if err == sql.ErrNoRows { + return fmt.Errorf("%w: session %s", ErrMetadataTargetUnavailable, id) + } + if err != nil { + return fmt.Errorf("checking metadata session %s: %w", id, err) + } + return nil +} + +type metadataPinTarget struct { + id int64 + ordinal int +} + +func metadataPinTargetTx( + ctx context.Context, + tx *sql.Tx, + sessionID string, + pin MetadataPinProjection, +) (metadataPinTarget, bool, error) { + if pin.SourceUUID != "" { + target, ok, err := metadataPinTargetByQueryTx(ctx, tx, + `SELECT id, ordinal FROM messages + WHERE session_id = ? AND source_uuid = ? + ORDER BY ordinal LIMIT 1`, + sessionID, pin.SourceUUID, + ) + if err != nil || ok { + return target, ok, err + } + } + return metadataPinTargetByQueryTx(ctx, tx, + `SELECT id, ordinal FROM messages + WHERE session_id = ? AND ordinal = ? + ORDER BY id LIMIT 1`, + sessionID, pin.Ordinal, + ) +} + +func metadataPinTargetByQueryTx( + ctx context.Context, + tx *sql.Tx, + query string, + args ...any, +) (metadataPinTarget, bool, error) { + var target metadataPinTarget + err := tx.QueryRowContext(ctx, query, args...).Scan(&target.id, &target.ordinal) + if err == sql.ErrNoRows { + return metadataPinTarget{}, false, nil + } + if err != nil { + return metadataPinTarget{}, false, fmt.Errorf("finding metadata pin target: %w", err) + } + return target, true, nil +} + +func unpinMetadataTx( + ctx context.Context, + tx *sql.Tx, + sessionID string, + pin MetadataPinProjection, +) error { + if pin.SourceUUID != "" { + res, err := tx.ExecContext(ctx, + `DELETE FROM pinned_messages + WHERE session_id = ? + AND message_id IN ( + SELECT id FROM messages + WHERE session_id = ? AND source_uuid = ? + )`, + sessionID, sessionID, pin.SourceUUID, + ) + if err != nil { + return err + } + if n, _ := res.RowsAffected(); n > 0 { + return nil + } + } + _, err := tx.ExecContext(ctx, + `DELETE FROM pinned_messages + WHERE session_id = ? + AND message_id IN ( + SELECT id FROM messages + WHERE session_id = ? AND ordinal = ? + )`, + sessionID, sessionID, pin.Ordinal, + ) + return err +} + +func upsertMetadataReplayStateTx(ctx context.Context, tx *sql.Tx, ev MetadataProjection) error { + _, err := tx.ExecContext(ctx, + `INSERT INTO metadata_replay_state + (session_gid, field, order_key, hlc, artifact_hash, origin, op, value, updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, strftime('%Y-%m-%dT%H:%M:%fZ','now')) + ON CONFLICT(session_gid, field) DO UPDATE SET + order_key = excluded.order_key, + hlc = excluded.hlc, + artifact_hash = excluded.artifact_hash, + origin = excluded.origin, + op = excluded.op, + value = excluded.value, + updated_at = excluded.updated_at`, + ev.SessionGID, ev.Field, ev.OrderKey, ev.HLC, ev.ArtifactHash, + ev.EventOrigin, ev.Op, ev.Value, + ) + if err != nil { + return fmt.Errorf("upserting metadata replay state: %w", err) + } + return nil +} + +func markMetadataEventAppliedTx( + ctx context.Context, + tx *sql.Tx, + origin, orderKey, hash string, +) error { + _, err := tx.ExecContext(ctx, + `INSERT OR IGNORE INTO metadata_applied_events + (origin, order_key, artifact_hash) + VALUES (?, ?, ?)`, + origin, orderKey, hash, + ) + if err != nil { + return fmt.Errorf("marking metadata event %s/%s applied: %w", origin, orderKey, err) + } + return nil +} + +type metadataConflict struct { + sessionGID string + field string + winningOrderKey string + losingOrderKey string + winningOrigin string + losingOrigin string + winningOp string + losingOp string + winningValue string + losingValue string +} + +func insertMetadataConflictTx(ctx context.Context, tx *sql.Tx, c metadataConflict) error { + _, err := tx.ExecContext(ctx, + `INSERT OR IGNORE INTO metadata_conflicts + (session_gid, field, winning_order_key, losing_order_key, + winning_origin, losing_origin, winning_op, losing_op, + winning_value, losing_value) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, + c.sessionGID, c.field, c.winningOrderKey, c.losingOrderKey, + c.winningOrigin, c.losingOrigin, c.winningOp, c.losingOp, + c.winningValue, c.losingValue, + ) + if err != nil { + return fmt.Errorf("inserting metadata conflict: %w", err) + } + return nil +} + +func metadataStateDiffers(aOp, aValue, bOp, bValue string) bool { + return aOp != bOp || aValue != bValue +} + +func metadataConflictOriginsDiffer(winningOrigin, losingOrigin string) bool { + return winningOrigin != losingOrigin +} + +func uniqueNonEmptyStrings(values []string) []string { + seen := make(map[string]bool, len(values)) + unique := make([]string, 0, len(values)) + for _, value := range values { + value = strings.TrimSpace(value) + if value == "" || seen[value] { + continue + } + seen[value] = true + unique = append(unique, value) + } + return unique +} diff --git a/internal/db/metadata_replay_test.go b/internal/db/metadata_replay_test.go new file mode 100644 index 000000000..44494e1fa --- /dev/null +++ b/internal/db/metadata_replay_test.go @@ -0,0 +1,254 @@ +package db + +import ( + "context" + "database/sql" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestApplyMetadataProjectionSessionOps(t *testing.T) { + ctx := context.Background() + d := testDB(t) + insertSession(t, d, "s1", "alpha") + + applyMetadataProjectionForTest(t, d, MetadataProjection{ + EventOrigin: "laptop-a1b2c3", + OrderKey: "0001-a", + HLC: "0001", + ArtifactHash: "a", + SessionGID: "desktop-d4e5f6~s1", + LocalSessionID: "s1", + Field: "starred", + Op: "star", + Value: "star", + }) + assert.Equal(t, 1, metadataTableCount(t, d, "starred_sessions", "session_id = 's1'")) + + applyMetadataProjectionForTest(t, d, MetadataProjection{ + EventOrigin: "laptop-a1b2c3", + OrderKey: "0002-a", + HLC: "0002", + ArtifactHash: "a", + SessionGID: "desktop-d4e5f6~s1", + LocalSessionID: "s1", + Field: "starred", + Op: "unstar", + Value: "unstar", + }) + assert.Equal(t, 0, metadataTableCount(t, d, "starred_sessions", "session_id = 's1'")) + + applyMetadataProjectionForTest(t, d, MetadataProjection{ + EventOrigin: "laptop-a1b2c3", + OrderKey: "0003-a", + HLC: "0003", + ArtifactHash: "a", + SessionGID: "desktop-d4e5f6~s1", + LocalSessionID: "s1", + Field: "deleted_at", + Op: "soft_delete", + Value: "soft_delete", + }) + got, err := d.GetSession(ctx, "s1") + require.NoError(t, err) + assert.Nil(t, got) + + applyMetadataProjectionForTest(t, d, MetadataProjection{ + EventOrigin: "laptop-a1b2c3", + OrderKey: "0004-a", + HLC: "0004", + ArtifactHash: "a", + SessionGID: "desktop-d4e5f6~s1", + LocalSessionID: "s1", + Field: "deleted_at", + Op: "restore", + Value: "restore", + }) + got, err = d.GetSession(ctx, "s1") + require.NoError(t, err) + require.NotNil(t, got) + assert.Equal(t, "s1", got.ID) + + applyMetadataProjectionForTest(t, d, MetadataProjection{ + EventOrigin: "laptop-a1b2c3", + OrderKey: "0005-a", + HLC: "0005", + ArtifactHash: "a", + SessionGID: "desktop-d4e5f6~s1", + LocalSessionID: "s1", + Field: "purge", + Op: "purge", + Value: "purge", + }) + got, err = d.GetSessionFull(ctx, "s1") + require.NoError(t, err) + assert.Nil(t, got) + assert.Equal(t, 1, metadataTableCount(t, d, "excluded_sessions", "id = 's1'")) +} + +func TestApplyMetadataProjectionRequiresPinTargetBeforeMarkingApplied(t *testing.T) { + d := testDB(t) + insertSession(t, d, "s1", "alpha") + + result, err := d.ApplyMetadataProjection(context.Background(), MetadataProjection{ + EventOrigin: "laptop-a1b2c3", + OrderKey: "0001-a", + HLC: "0001", + ArtifactHash: "a", + SessionGID: "desktop-d4e5f6~s1", + LocalSessionID: "s1", + Field: "pin:source_uuid:missing", + Op: "pin", + Value: `{"ordinal":1,"source_uuid":"missing"}`, + Pin: &MetadataPinProjection{ + SourceUUID: "missing", + Ordinal: 1, + }, + }) + + require.ErrorIs(t, err, ErrMetadataTargetUnavailable) + assert.False(t, result.Applied) + applied, checkErr := d.MetadataEventApplied(context.Background(), "laptop-a1b2c3", "0001-a") + require.NoError(t, checkErr) + assert.False(t, applied) + assert.Equal(t, 0, metadataTableCount(t, d, "metadata_replay_state", "session_gid = 'desktop-d4e5f6~s1'")) +} + +func TestApplyMetadataProjectionDoesNotConflictSameOriginSequentialEdits(t *testing.T) { + d := testDB(t) + insertSession(t, d, "s1", "alpha") + firstName := "one" + secondName := "two" + + events := []MetadataProjection{ + { + EventOrigin: "laptop-a1b2c3", + OrderKey: "0001-a", + HLC: "0001", + ArtifactHash: "a", + SessionGID: "desktop-d4e5f6~s1", + LocalSessionID: "s1", + Field: "starred", + Op: "star", + Value: "star", + }, + { + EventOrigin: "laptop-a1b2c3", + OrderKey: "0002-a", + HLC: "0002", + ArtifactHash: "b", + SessionGID: "desktop-d4e5f6~s1", + LocalSessionID: "s1", + Field: "starred", + Op: "unstar", + Value: "unstar", + }, + { + EventOrigin: "laptop-a1b2c3", + OrderKey: "0003-a", + HLC: "0003", + ArtifactHash: "c", + SessionGID: "desktop-d4e5f6~s1", + LocalSessionID: "s1", + Field: "display_name", + Op: "rename", + Value: `{"display_name":"one"}`, + DisplayName: &firstName, + }, + { + EventOrigin: "laptop-a1b2c3", + OrderKey: "0004-a", + HLC: "0004", + ArtifactHash: "d", + SessionGID: "desktop-d4e5f6~s1", + LocalSessionID: "s1", + Field: "display_name", + Op: "rename", + Value: `{"display_name":"two"}`, + DisplayName: &secondName, + }, + } + + for _, ev := range events { + result, err := d.ApplyMetadataProjection(context.Background(), ev) + require.NoError(t, err) + assert.True(t, result.Applied) + assert.False(t, result.Conflict) + } + + assert.Equal(t, 0, metadataTableCount(t, d, "metadata_conflicts", "1 = 1")) +} + +func TestMetadataConflictQueriesIgnoreSameOriginRows(t *testing.T) { + ctx := context.Background() + d := testDB(t) + _, err := d.getWriter().ExecContext(ctx, + `INSERT INTO metadata_conflicts + (session_gid, field, winning_order_key, losing_order_key, + winning_origin, losing_origin, winning_op, losing_op, + winning_value, losing_value) + VALUES + ('desktop-d4e5f6~s1', 'display_name', '0002-a', '0001-a', + 'desktop-d4e5f6', 'desktop-d4e5f6', 'rename', 'rename', + '{"display_name":"two"}', '{"display_name":"one"}'), + ('desktop-d4e5f6~s1', 'display_name', '0003-b', '0002-a', + 'laptop-a1b2c3', 'desktop-d4e5f6', 'rename', 'rename', + '{"display_name":"peer"}', '{"display_name":"two"}')`, + ) + require.NoError(t, err) + + conflicts, err := d.ListMetadataConflicts(ctx, []string{"desktop-d4e5f6~s1"}) + require.NoError(t, err) + require.Len(t, conflicts, 1) + assert.Equal(t, "laptop-a1b2c3", conflicts[0].WinningOrigin) + assert.Equal(t, "desktop-d4e5f6", conflicts[0].LosingOrigin) + + count, err := d.CountMetadataConflicts(ctx) + require.NoError(t, err) + assert.Equal(t, 1, count) +} + +func TestApplyMetadataProjectionPurgeExcludesFallbackAlias(t *testing.T) { + d := testDB(t) + filePath := "/tmp/vibe/session_20260616_083518_abc123/messages.jsonl" + insertSession(t, d, "vibe:canonical-1", "alpha", func(s *Session) { + s.Agent = "vibe" + s.FilePath = &filePath + }) + + applyMetadataProjectionForTest(t, d, MetadataProjection{ + EventOrigin: "laptop-a1b2c3", + OrderKey: "0001-a", + HLC: "0001", + ArtifactHash: "a", + SessionGID: "laptop-a1b2c3~vibe:canonical-1", + LocalSessionID: "vibe:canonical-1", + Field: "purge", + Op: "purge", + Value: "purge", + }) + + assert.Equal(t, 1, metadataTableCount(t, d, "excluded_sessions", "id = 'vibe:canonical-1'")) + assert.Equal(t, 1, metadataTableCount(t, d, "excluded_sessions", "id = 'vibe:session_20260616_083518_abc123'")) +} + +func applyMetadataProjectionForTest(t *testing.T, d *DB, ev MetadataProjection) { + t.Helper() + result, err := d.ApplyMetadataProjection(context.Background(), ev) + require.NoError(t, err) + assert.True(t, result.Applied) + assert.False(t, result.Duplicate) +} + +func metadataTableCount(t *testing.T, d *DB, table, where string) int { + t.Helper() + var count int + err := d.Reader().QueryRow("SELECT COUNT(*) FROM " + table + " WHERE " + where).Scan(&count) + if err == sql.ErrNoRows { + return 0 + } + require.NoError(t, err) + return count +} diff --git a/internal/db/orphaned.go b/internal/db/orphaned.go index 935db50fd..33428e4d3 100644 --- a/internal/db/orphaned.go +++ b/internal/db/orphaned.go @@ -309,9 +309,13 @@ func (d *DB) CopyTrashedDataFrom(sourcePath string) (int, error) { return count, nil } -// CopySyncStateFrom copies pg_sync_state rows from the source database into the -// current database. ResyncAll uses this to preserve durable local sync metadata -// such as the PG push owner marker across the temp-DB swap. +// CopySyncStateFrom copies durable pg_sync_state rows from the source database +// into the current database. ResyncAll uses this to preserve durable local sync +// metadata across the temp-DB swap: the PG push owner marker plus the artifact +// ledger state written by internal/artifact (origin identity, metadata HLC, +// and per-session import/export watermarks, all keyed with an "artifact_" +// prefix). Transient bookkeeping such as last_sync_* timestamps is +// deliberately left behind so the rebuilt DB reports its own sync times. func (d *DB) CopySyncStateFrom(sourcePath string) error { d.mu.Lock() defer d.mu.Unlock() @@ -347,13 +351,83 @@ func (d *DB) CopySyncStateFrom(sourcePath string) error { _, err = conn.ExecContext(ctx, ` INSERT OR REPLACE INTO main.pg_sync_state (key, value) SELECT key, value FROM old_db.pg_sync_state - WHERE key = 'pg_push_marker_id'`) + WHERE key = 'pg_push_marker_id' + OR key LIKE 'artifact\_%' ESCAPE '\'`) if err != nil { return fmt.Errorf("copying sync state: %w", err) } return nil } +// CopyMetadataReplayFrom copies the durable artifact metadata replay tables +// (metadata_applied_events, metadata_replay_state, metadata_conflicts) from +// the source database. ResyncAll uses this so previously applied peer +// metadata events are not replayed against an empty LWW register after a +// full rebuild, which would let old events overwrite newer local state. +func (d *DB) CopyMetadataReplayFrom(sourcePath string) error { + d.mu.Lock() + defer d.mu.Unlock() + + ctx := context.Background() + conn, err := d.getWriter().Conn(ctx) + if err != nil { + return fmt.Errorf("acquiring connection: %w", err) + } + defer conn.Close() + + if _, err := conn.ExecContext( + ctx, "ATTACH DATABASE ? AS old_db", sourcePath, + ); err != nil { + return fmt.Errorf("attaching source db: %w", err) + } + defer func() { + _, _ = execWithoutCancel(ctx, conn, "DETACH DATABASE old_db") + }() + + copies := []struct { + table string + columns string + }{ + { + "metadata_applied_events", + "origin, order_key, artifact_hash, applied_at", + }, + { + "metadata_replay_state", + "session_gid, field, order_key, hlc, artifact_hash, " + + "origin, op, value, updated_at", + }, + { + "metadata_conflicts", + "session_gid, field, winning_order_key, losing_order_key, " + + "winning_origin, losing_origin, winning_op, losing_op, " + + "winning_value, losing_value, created_at", + }, + } + for _, c := range copies { + // Older databases may predate the metadata replay tables. + var tableExists int + err := conn.QueryRowContext(ctx, + "SELECT 1 FROM old_db.sqlite_master WHERE type='table' AND name=?", + c.table, + ).Scan(&tableExists) + if err != nil { + if errors.Is(err, sql.ErrNoRows) { + continue + } + return fmt.Errorf("probing %s table: %w", c.table, err) + } + if _, err := conn.ExecContext(ctx, fmt.Sprintf( + `INSERT OR IGNORE INTO main.%s (%s) + SELECT %s FROM old_db.%s`, + c.table, c.columns, c.columns, c.table, + )); err != nil { + return fmt.Errorf("copying %s: %w", c.table, err) + } + } + return nil +} + // CopyExcludedSessionsFrom copies the excluded_sessions table // from the source DB so permanently deleted sessions survive // full DB rebuilds. The source must not have active connections. diff --git a/internal/db/read_only_test.go b/internal/db/read_only_test.go index 03c9dd1cc..d84028939 100644 --- a/internal/db/read_only_test.go +++ b/internal/db/read_only_test.go @@ -234,7 +234,8 @@ func TestOpenReadOnlyWriteMethodsReturnErrReadOnly(t *testing.T) { return readonly.InsertMessages(nil) }) requireReadOnlyOp(t, "BulkStarSessions", func() error { - return readonly.BulkStarSessions(nil) + _, err := readonly.BulkStarSessions(nil) + return err }) requireReadOnlyOp(t, "DeleteParserExcludedSessions", func() error { _, err := readonly.DeleteParserExcludedSessions(nil) diff --git a/internal/db/schema.sql b/internal/db/schema.sql index 2978024aa..bc723a072 100644 --- a/internal/db/schema.sql +++ b/internal/db/schema.sql @@ -516,6 +516,48 @@ CREATE TABLE IF NOT EXISTS pg_sync_state ( value TEXT NOT NULL ); +-- Metadata replay: durable record of handled artifact events. This lets +-- import scan the small append-only feed repeatedly without replaying +-- duplicates or relying on an unsafe max-HLC watermark. +CREATE TABLE IF NOT EXISTS metadata_applied_events ( + origin TEXT NOT NULL, + order_key TEXT NOT NULL, + artifact_hash TEXT NOT NULL, + applied_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')), + PRIMARY KEY (origin, order_key) +); + +-- Per-field LWW winners for metadata replay. +CREATE TABLE IF NOT EXISTS metadata_replay_state ( + session_gid TEXT NOT NULL, + field TEXT NOT NULL, + order_key TEXT NOT NULL, + hlc TEXT NOT NULL, + artifact_hash TEXT NOT NULL, + origin TEXT NOT NULL, + op TEXT NOT NULL, + value TEXT NOT NULL DEFAULT '', + updated_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')), + PRIMARY KEY (session_gid, field) +); + +-- Losing metadata values that were overridden by deterministic LWW replay. +CREATE TABLE IF NOT EXISTS metadata_conflicts ( + id INTEGER PRIMARY KEY, + session_gid TEXT NOT NULL, + field TEXT NOT NULL, + winning_order_key TEXT NOT NULL, + losing_order_key TEXT NOT NULL, + winning_origin TEXT NOT NULL, + losing_origin TEXT NOT NULL, + winning_op TEXT NOT NULL, + losing_op TEXT NOT NULL, + winning_value TEXT NOT NULL DEFAULT '', + losing_value TEXT NOT NULL DEFAULT '', + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')), + UNIQUE(session_gid, field, winning_order_key, losing_order_key) +); + -- Model pricing for cost calculation CREATE TABLE IF NOT EXISTS model_pricing ( model_pattern TEXT PRIMARY KEY, diff --git a/internal/db/sessions.go b/internal/db/sessions.go index 55e1b57a4..c5ccf8fc2 100644 --- a/internal/db/sessions.go +++ b/internal/db/sessions.go @@ -2205,6 +2205,35 @@ func sqliteLikeEscape(value string) string { return value } +// ListOwnedSessionIDsForExport returns the IDs of locally-owned, non-deleted +// sessions for artifact export, ordered by id. Unlike ListSessions it does not +// apply the sidebar visibility filter (message_count > 0), so zero-message +// usage-only sessions are still published. +func (db *DB) ListOwnedSessionIDsForExport(ctx context.Context) ([]string, error) { + rows, err := db.getReader().QueryContext(ctx, + `SELECT id FROM sessions + WHERE machine = 'local' AND deleted_at IS NULL + ORDER BY id`, + ) + if err != nil { + return nil, fmt.Errorf("listing sessions for artifact export: %w", err) + } + defer rows.Close() + + var ids []string + for rows.Next() { + var id string + if err := rows.Scan(&id); err != nil { + return nil, fmt.Errorf("scanning export session ID: %w", err) + } + ids = append(ids, id) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("iterating export session IDs: %w", err) + } + return ids, nil +} + // GetDataVersionByPath returns the minimum data_version for // sessions matching a file_path. Returns 0 when no session // exists for the path. @@ -2604,6 +2633,32 @@ func (db *DB) GetBranches( return branches, rows.Err() } +// MachineSessionCounts returns the number of non-deleted sessions per machine, +// keyed by machine name. Child sessions are included so the count reflects the +// full corpus owned by or imported from each origin. +func (db *DB) MachineSessionCounts(ctx context.Context) (map[string]int, error) { + rows, err := db.getReader().QueryContext(ctx, + `SELECT machine, COUNT(*) FROM sessions + WHERE deleted_at IS NULL + GROUP BY machine`, + ) + if err != nil { + return nil, fmt.Errorf("counting sessions per machine: %w", err) + } + defer rows.Close() + + counts := map[string]int{} + for rows.Next() { + var machine string + var count int + if err := rows.Scan(&machine, &count); err != nil { + return nil, fmt.Errorf("scanning machine session count: %w", err) + } + counts[machine] = count + } + return counts, rows.Err() +} + // scanSessionRows iterates rows and scans each using // scanSessionRow. func scanSessionRows(rows *sql.Rows) ([]Session, error) { @@ -2750,8 +2805,16 @@ func (db *DB) SoftDeleteSession(id string) error { // deleted_at. Sessions that are already soft-deleted are skipped. // Returns the count of newly deleted rows. func (db *DB) SoftDeleteSessions(ids []string) (int, error) { + deleted, err := db.SoftDeleteSessionsReturningIDs(ids) + return len(deleted), err +} + +// SoftDeleteSessionsReturningIDs marks multiple sessions as deleted by setting +// deleted_at and returns the IDs that were newly deleted. Sessions that are +// already soft-deleted are skipped. +func (db *DB) SoftDeleteSessionsReturningIDs(ids []string) ([]string, error) { if len(ids) == 0 { - return 0, nil + return []string{}, nil } db.mu.Lock() @@ -2759,11 +2822,11 @@ func (db *DB) SoftDeleteSessions(ids []string) (int, error) { tx, err := db.getWriter().Begin() if err != nil { - return 0, fmt.Errorf("beginning soft-delete tx: %w", err) + return nil, fmt.Errorf("beginning soft-delete tx: %w", err) } defer func() { _ = tx.Rollback() }() - total := 0 + deleted := make([]string, 0, len(ids)) const batchSize = 500 for i := 0; i < len(ids); i += batchSize { end := min(i+batchSize, len(ids)) @@ -2775,24 +2838,100 @@ func (db *DB) SoftDeleteSessions(ids []string) (int, error) { } placeholders := strings.Repeat(",?", len(batch))[1:] - res, err := tx.Exec( + rows, err := tx.Query( + `SELECT id FROM sessions + WHERE id IN (`+placeholders+`) AND deleted_at IS NULL`, + args..., + ) + if err != nil { + return nil, fmt.Errorf("loading soft-delete batch ids: %w", err) + } + batchIDs := map[string]struct{}{} + for rows.Next() { + var id string + if err := rows.Scan(&id); err != nil { + _ = rows.Close() + return nil, fmt.Errorf("scanning soft-delete batch id: %w", err) + } + batchIDs[id] = struct{}{} + } + if err := rows.Close(); err != nil { + return nil, fmt.Errorf("closing soft-delete batch ids: %w", err) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("iterating soft-delete batch ids: %w", err) + } + for _, id := range batch { + if _, ok := batchIDs[id]; ok { + deleted = append(deleted, id) + delete(batchIDs, id) + } + } + + if _, err := tx.Exec( `UPDATE sessions SET deleted_at = strftime('%Y-%m-%dT%H:%M:%fZ','now'), local_modified_at = strftime('%Y-%m-%dT%H:%M:%fZ','now') WHERE id IN (`+placeholders+`) AND deleted_at IS NULL`, args..., - ) - if err != nil { - return 0, fmt.Errorf("soft-deleting batch: %w", err) + ); err != nil { + return nil, fmt.Errorf("soft-deleting batch: %w", err) } - n, _ := res.RowsAffected() - total += int(n) } if err := tx.Commit(); err != nil { - return 0, fmt.Errorf("committing soft-delete tx: %w", err) + return nil, fmt.Errorf("committing soft-delete tx: %w", err) } - return total, nil + return deleted, nil +} + +// TrashedSessionIDs returns requested sessions that currently exist in the +// trash. The result preserves first-seen input order and omits duplicates. +func (db *DB) TrashedSessionIDs(ids []string) ([]string, error) { + if len(ids) == 0 { + return []string{}, nil + } + trashed := make(map[string]struct{}, len(ids)) + const batchSize = 500 + for i := 0; i < len(ids); i += batchSize { + end := min(i+batchSize, len(ids)) + batch := ids[i:end] + args := make([]any, len(batch)) + for j, id := range batch { + args[j] = id + } + placeholders := strings.Repeat(",?", len(batch))[1:] + rows, err := db.getReader().Query( + `SELECT id FROM sessions + WHERE id IN (`+placeholders+`) AND deleted_at IS NOT NULL`, + args..., + ) + if err != nil { + return nil, fmt.Errorf("loading trashed session ids: %w", err) + } + for rows.Next() { + var id string + if err := rows.Scan(&id); err != nil { + _ = rows.Close() + return nil, fmt.Errorf("scanning trashed session id: %w", err) + } + trashed[id] = struct{}{} + } + if err := rows.Close(); err != nil { + return nil, fmt.Errorf("closing trashed session ids: %w", err) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("iterating trashed session ids: %w", err) + } + } + out := make([]string, 0, len(trashed)) + for _, id := range ids { + if _, ok := trashed[id]; ok { + out = append(out, id) + delete(trashed, id) + } + } + return out, nil } // RestoreSession clears deleted_at, making the session visible again. diff --git a/internal/db/starred.go b/internal/db/starred.go index 926ec3412..0b04faf70 100644 --- a/internal/db/starred.go +++ b/internal/db/starred.go @@ -3,6 +3,7 @@ package db import ( "context" "database/sql" + "errors" "fmt" ) @@ -39,18 +40,22 @@ func (db *DB) StarSession(sessionID string) (bool, error) { return true, nil // already starred } -// UnstarSession removes a session's star. -func (db *DB) UnstarSession(sessionID string) error { +// UnstarSession removes a session's star and reports whether a row was removed. +func (db *DB) UnstarSession(sessionID string) (bool, error) { db.mu.Lock() defer db.mu.Unlock() - _, err := db.getWriter().Exec( + res, err := db.getWriter().Exec( "DELETE FROM starred_sessions WHERE session_id = ?", sessionID, ) if err != nil { - return fmt.Errorf("unstarring session %s: %w", sessionID, err) + return false, fmt.Errorf("unstarring session %s: %w", sessionID, err) } - return nil + n, err := res.RowsAffected() + if err != nil { + return false, fmt.Errorf("checking unstar result for %s: %w", sessionID, err) + } + return n > 0, nil } // ListStarredSessionIDs returns all starred session IDs. @@ -78,12 +83,12 @@ func (db *DB) ListStarredSessionIDs( // BulkStarSessions stars multiple sessions in a single transaction. // Used for migrating localStorage stars to the database. -func (db *DB) BulkStarSessions(sessionIDs []string) error { +func (db *DB) BulkStarSessions(sessionIDs []string) ([]string, error) { if err := db.requireWritable(); err != nil { - return err + return nil, err } if len(sessionIDs) == 0 { - return nil + return nil, nil } db.mu.Lock() @@ -91,27 +96,50 @@ func (db *DB) BulkStarSessions(sessionIDs []string) error { tx, err := db.getWriter().Begin() if err != nil { - return fmt.Errorf("beginning transaction: %w", err) + return nil, fmt.Errorf("beginning transaction: %w", err) } defer func() { _ = tx.Rollback() }() - // Use INSERT ... SELECT ... WHERE EXISTS so that stale IDs - // (sessions pruned or deleted from disk) are silently skipped - // instead of causing a foreign key violation that aborts the - // entire migration transaction. - stmt, err := tx.Prepare(` - INSERT OR IGNORE INTO starred_sessions (session_id) - SELECT ? WHERE EXISTS (SELECT 1 FROM sessions WHERE id = ?)`) + // Check existence separately from the insert so stale IDs (sessions pruned + // or deleted from disk) are silently skipped instead of aborting the + // migration transaction, and so the caller learns which sessions were + // actually starred and need a converging metadata event. + exists, err := tx.Prepare(`SELECT 1 FROM sessions WHERE id = ?`) + if err != nil { + return nil, fmt.Errorf("preparing existence statement: %w", err) + } + defer exists.Close() + insert, err := tx.Prepare( + `INSERT OR IGNORE INTO starred_sessions (session_id) VALUES (?)`) if err != nil { - return fmt.Errorf("preparing statement: %w", err) + return nil, fmt.Errorf("preparing statement: %w", err) } - defer stmt.Close() + defer insert.Close() + starred := make([]string, 0, len(sessionIDs)) for _, id := range sessionIDs { - if _, err := stmt.Exec(id, id); err != nil { - return fmt.Errorf("starring session %s: %w", id, err) + var one int + switch err := exists.QueryRow(id).Scan(&one); { + case errors.Is(err, sql.ErrNoRows): + continue + case err != nil: + return nil, fmt.Errorf("checking session %s: %w", id, err) + } + res, err := insert.Exec(id) + if err != nil { + return nil, fmt.Errorf("starring session %s: %w", id, err) + } + rowsAffected, err := res.RowsAffected() + if err != nil { + return nil, fmt.Errorf("checking star insert result for %s: %w", id, err) + } + if rowsAffected > 0 { + starred = append(starred, id) } } - return tx.Commit() + if err := tx.Commit(); err != nil { + return nil, fmt.Errorf("committing star transaction: %w", err) + } + return starred, nil } diff --git a/internal/db/store.go b/internal/db/store.go index 7b3d27f7b..97b36056f 100644 --- a/internal/db/store.go +++ b/internal/db/store.go @@ -40,6 +40,7 @@ type Store interface { GetMessages(ctx context.Context, sessionID string, from, limit int, asc bool) ([]Message, error) GetMessagesWindow(ctx context.Context, sessionID string, w MessageWindow) ([]Message, error) GetAllMessages(ctx context.Context, sessionID string) ([]Message, error) + GetMessageForMetadataPin(ctx context.Context, sessionID string, messageID int64) (*Message, error) GetSessionActivity(ctx context.Context, sessionID string) (*SessionActivityResponse, error) // Timing. @@ -58,6 +59,9 @@ type Store interface { GetSessionVersion(id string) (count int, version int64, ok bool) // Metadata. + ListMetadataConflicts(ctx context.Context, sessionGIDs []string) ([]MetadataConflict, error) + CountMetadataConflicts(ctx context.Context) (int, error) + MachineSessionCounts(ctx context.Context) (map[string]int, error) GetStats(ctx context.Context, excludeOneShot, excludeAutomated bool) (Stats, error) GetProjects(ctx context.Context, excludeOneShot, excludeAutomated bool) ([]ProjectInfo, error) GetAgents(ctx context.Context, excludeOneShot, excludeAutomated bool) ([]AgentInfo, error) @@ -92,9 +96,9 @@ type Store interface { // Stars. StarSession(sessionID string) (bool, error) - UnstarSession(sessionID string) error + UnstarSession(sessionID string) (bool, error) ListStarredSessionIDs(ctx context.Context) ([]string, error) - BulkStarSessions(sessionIDs []string) error + BulkStarSessions(sessionIDs []string) ([]string, error) // Pins. PinMessage(sessionID string, messageID int64, note *string) (int64, error) @@ -128,6 +132,7 @@ type Store interface { RenameSession(id string, displayName *string) error SoftDeleteSession(id string) error SoftDeleteSessions(ids []string) (int, error) + SoftDeleteSessionsReturningIDs(ids []string) ([]string, error) RestoreSession(id string) (int64, error) DeleteSessionIfTrashed(id string) (int64, error) ListTrashedSessions(ctx context.Context) ([]Session, error) diff --git a/internal/db/store_contract_test.go b/internal/db/store_contract_test.go index 8eee23dd6..6ad31bba0 100644 --- a/internal/db/store_contract_test.go +++ b/internal/db/store_contract_test.go @@ -129,6 +129,7 @@ func TestStoreContract(t *testing.T) { {"stars_and_pins", contractStarsAndPins}, {"analytics_trends_and_usage", contractAnalyticsTrendsAndUsage}, {"local_only_methods", contractLocalOnlyMethods}, + {"machine_counts_and_conflicts", contractMachineCountsAndConflicts}, } for _, backend := range storeContractBackends() { @@ -243,6 +244,32 @@ func contractSessionsCursorFiltersAndDates( require.Equal(t, []string{"linux", "mac"}, machines) } +func contractMachineCountsAndConflicts( + t *testing.T, + store Store, + _ storeContractFixture, + _ storeContractBackend, +) { + t.Helper() + ctx := context.Background() + + counts, err := store.MachineSessionCounts(ctx) + require.NoError(t, err) + // The seed places non-deleted sessions on both machines. + require.Positive(t, counts["linux"]) + require.Positive(t, counts["mac"]) + for machine := range counts { + require.Contains(t, []string{"linux", "mac"}, machine, + "unexpected machine in session counts: %q", machine) + } + + // No conflicts are seeded; every backend reports zero (read-only + // mirrors do not carry the local metadata ledger at all). + conflicts, err := store.CountMetadataConflicts(ctx) + require.NoError(t, err) + require.Equal(t, 0, conflicts) +} + func contractMessagesOrderingAndToolResults( t *testing.T, store Store, @@ -385,13 +412,23 @@ func contractStarsAndPins( ok, err := store.StarSession(fixture.alphaID) require.NoError(t, err) require.True(t, ok) - require.NoError(t, store.BulkStarSessions([]string{fixture.gammaID, "missing-session"})) + bulkStarred, err := store.BulkStarSessions([]string{fixture.gammaID, "missing-session"}) + require.NoError(t, err) + require.Equal(t, []string{fixture.gammaID}, bulkStarred) + bulkStarred, err = store.BulkStarSessions([]string{fixture.alphaID, fixture.gammaID}) + require.NoError(t, err) + require.Empty(t, bulkStarred) stars, err := store.ListStarredSessionIDs(ctx) require.NoError(t, err) require.ElementsMatch(t, []string{fixture.alphaID, fixture.gammaID}, stars) - require.NoError(t, store.UnstarSession(fixture.gammaID)) + removed, err := store.UnstarSession(fixture.gammaID) + require.NoError(t, err) + require.True(t, removed) + removed, err = store.UnstarSession(fixture.gammaID) + require.NoError(t, err) + require.False(t, removed) stars, err = store.ListStarredSessionIDs(ctx) require.NoError(t, err) require.Equal(t, []string{fixture.alphaID}, stars) diff --git a/internal/db/sync_state_test.go b/internal/db/sync_state_test.go new file mode 100644 index 000000000..8e04ebc75 --- /dev/null +++ b/internal/db/sync_state_test.go @@ -0,0 +1,31 @@ +package db + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestSyncStateValuesReadsOnlyExactKeysAcrossBatches(t *testing.T) { + database := testDB(t) + for i := range 905 { + key := fmt.Sprintf("artifact_import:desk:desk~%04d", i) + require.NoError(t, database.SetSyncState(key, fmt.Sprintf("hash-%04d", i))) + } + require.NoError(t, database.SetSyncState("unrelated", "keep-out")) + + got, err := database.SyncStateValues([]string{ + "artifact_import:desk:desk~0000", + "artifact_import:desk:desk~0899", + "artifact_import:desk:desk~0904", + "missing", + }) + require.NoError(t, err) + assert.Equal(t, map[string]string{ + "artifact_import:desk:desk~0000": "hash-0000", + "artifact_import:desk:desk~0899": "hash-0899", + "artifact_import:desk:desk~0904": "hash-0904", + }, got) +} diff --git a/internal/duckdb/curation.go b/internal/duckdb/curation.go index d0bfd5ab3..c51fa9330 100644 --- a/internal/duckdb/curation.go +++ b/internal/duckdb/curation.go @@ -12,8 +12,8 @@ func (s *Store) StarSession(sessionID string) (bool, error) { return false, db.ErrReadOnly } -func (s *Store) UnstarSession(sessionID string) error { - return db.ErrReadOnly +func (s *Store) UnstarSession(sessionID string) (bool, error) { + return false, db.ErrReadOnly } func (s *Store) ListStarredSessionIDs(ctx context.Context) ([]string, error) { @@ -35,8 +35,8 @@ func (s *Store) ListStarredSessionIDs(ctx context.Context) ([]string, error) { return ids, rows.Err() } -func (s *Store) BulkStarSessions(sessionIDs []string) error { - return db.ErrReadOnly +func (s *Store) BulkStarSessions(sessionIDs []string) ([]string, error) { + return nil, db.ErrReadOnly } func (s *Store) PinMessage(sessionID string, messageID int64, note *string) (int64, error) { diff --git a/internal/duckdb/messages.go b/internal/duckdb/messages.go index dadaf0ea2..f4e700278 100644 --- a/internal/duckdb/messages.go +++ b/internal/duckdb/messages.go @@ -3,6 +3,7 @@ package duckdb import ( "context" "database/sql" + "errors" "fmt" "slices" "strings" @@ -236,6 +237,25 @@ func (s *Store) GetAllMessages(ctx context.Context, sessionID string) ([]db.Mess return msgs, nil } +// GetMessageForMetadataPin returns only the stable message identity fields +// needed for metadata pin events. +func (s *Store) GetMessageForMetadataPin( + ctx context.Context, sessionID string, messageID int64, +) (*db.Message, error) { + row := s.queryRowContext(ctx, ` + SELECT id, session_id, ordinal, COALESCE(source_uuid, '') + FROM messages + WHERE session_id = ? AND id = ?`, sessionID, messageID) + var msg db.Message + if err := row.Scan(&msg.ID, &msg.SessionID, &msg.Ordinal, &msg.SourceUUID); err != nil { + if errors.Is(err, sql.ErrNoRows) { + return nil, nil + } + return nil, fmt.Errorf("querying message metadata for pin: %w", err) + } + return &msg, nil +} + func scanMessages(rows *sql.Rows) ([]db.Message, error) { var msgs []db.Message for rows.Next() { diff --git a/internal/duckdb/metadata.go b/internal/duckdb/metadata.go new file mode 100644 index 000000000..dd2736052 --- /dev/null +++ b/internal/duckdb/metadata.go @@ -0,0 +1,22 @@ +package duckdb + +import ( + "context" + + "go.kenn.io/agentsview/internal/db" +) + +// ListMetadataConflicts returns no rows for DuckDB read mode because the local +// artifact metadata ledger is not part of the analytical mirror. +func (s *Store) ListMetadataConflicts( + context.Context, + []string, +) ([]db.MetadataConflict, error) { + return []db.MetadataConflict{}, nil +} + +// CountMetadataConflicts returns zero for DuckDB read mode because the local +// artifact metadata ledger is not part of the analytical mirror. +func (s *Store) CountMetadataConflicts(context.Context) (int, error) { + return 0, nil +} diff --git a/internal/duckdb/store.go b/internal/duckdb/store.go index c5f61b37f..4e2cad097 100644 --- a/internal/duckdb/store.go +++ b/internal/duckdb/store.go @@ -654,6 +654,31 @@ func (s *Store) GetAgents(ctx context.Context, excludeOneShot, excludeAutomated return out, rows.Err() } +// MachineSessionCounts returns the number of non-deleted sessions per machine, +// keyed by machine name. +func (s *Store) MachineSessionCounts(ctx context.Context) (map[string]int, error) { + rows, err := s.duck.QueryContext(ctx, + `SELECT machine, COUNT(*) FROM sessions + WHERE deleted_at IS NULL + GROUP BY machine`, + ) + if err != nil { + return nil, err + } + defer rows.Close() + + counts := map[string]int{} + for rows.Next() { + var machine string + var count int + if err := rows.Scan(&machine, &count); err != nil { + return nil, err + } + counts[machine] = count + } + return counts, rows.Err() +} + func (s *Store) GetMachines(ctx context.Context, excludeOneShot, excludeAutomated bool) ([]string, error) { rows, err := s.queryContext(ctx, `SELECT DISTINCT machine FROM sessions WHERE `+ diff --git a/internal/duckdb/store_contract_test.go b/internal/duckdb/store_contract_test.go index bc506d5ad..5e1a87fdd 100644 --- a/internal/duckdb/store_contract_test.go +++ b/internal/duckdb/store_contract_test.go @@ -233,6 +233,15 @@ func duckContractSessionsCursorsAndMetadata( machines, err := store.GetMachines(ctx, false, false) require.NoError(t, err) require.Equal(t, []string{"test-machine"}, machines) + + counts, err := store.MachineSessionCounts(ctx) + require.NoError(t, err) + require.Len(t, counts, 1) + require.Positive(t, counts["test-machine"]) + + conflicts, err := store.CountMetadataConflicts(ctx) + require.NoError(t, err) + require.Equal(t, 0, conflicts) } func duckContractMessagesSearchAndSecrets( @@ -310,8 +319,10 @@ func duckContractReadOnlyCuration( ok, err := store.StarSession(fixture.betaID) require.ErrorIs(t, err, db.ErrReadOnly) require.False(t, ok) - require.ErrorIs(t, store.UnstarSession(fixture.alphaID), db.ErrReadOnly) - require.ErrorIs(t, store.BulkStarSessions([]string{fixture.betaID}), db.ErrReadOnly) + _, err = store.UnstarSession(fixture.alphaID) + require.ErrorIs(t, err, db.ErrReadOnly) + _, bulkErr := store.BulkStarSessions([]string{fixture.betaID}) + require.ErrorIs(t, bulkErr, db.ErrReadOnly) pinID, err := store.PinMessage(fixture.alphaID, 1, nil) require.ErrorIs(t, err, db.ErrReadOnly) diff --git a/internal/duckdb/store_test.go b/internal/duckdb/store_test.go index 0d27bbfa6..f9d42c79b 100644 --- a/internal/duckdb/store_test.go +++ b/internal/duckdb/store_test.go @@ -688,8 +688,10 @@ func TestStoreCurationMethods(t *testing.T) { ok, err := store.StarSession(fixture.betaID) require.ErrorIs(t, err, db.ErrReadOnly) assert.False(t, ok) - require.ErrorIs(t, store.BulkStarSessions([]string{fixture.betaID}), db.ErrReadOnly) - require.ErrorIs(t, store.UnstarSession(fixture.alphaID), db.ErrReadOnly) + _, bulkErr := store.BulkStarSessions([]string{fixture.betaID}) + require.ErrorIs(t, bulkErr, db.ErrReadOnly) + _, err = store.UnstarSession(fixture.alphaID) + require.ErrorIs(t, err, db.ErrReadOnly) starred, err = store.ListStarredSessionIDs(ctx) require.NoError(t, err) assert.Equal(t, []string{fixture.alphaID}, starred) diff --git a/internal/duckdb/stubs.go b/internal/duckdb/stubs.go index 91ea32e52..c8c445345 100644 --- a/internal/duckdb/stubs.go +++ b/internal/duckdb/stubs.go @@ -15,9 +15,12 @@ func (s *Store) GetInsight(_ context.Context, _ int64) (*db.Insight, error) { re func (s *Store) GetCachedInsight(_ context.Context, _ string) (*db.Insight, error) { return nil, nil } -func (s *Store) RenameSession(_ string, _ *string) error { return db.ErrReadOnly } -func (s *Store) SoftDeleteSession(_ string) error { return db.ErrReadOnly } -func (s *Store) SoftDeleteSessions(_ []string) (int, error) { return 0, db.ErrReadOnly } +func (s *Store) RenameSession(_ string, _ *string) error { return db.ErrReadOnly } +func (s *Store) SoftDeleteSession(_ string) error { return db.ErrReadOnly } +func (s *Store) SoftDeleteSessions(_ []string) (int, error) { return 0, db.ErrReadOnly } +func (s *Store) SoftDeleteSessionsReturningIDs(_ []string) ([]string, error) { + return nil, db.ErrReadOnly +} func (s *Store) RestoreSession(_ string) (int64, error) { return 0, db.ErrReadOnly } func (s *Store) DeleteSessionIfTrashed(_ string) (int64, error) { return 0, db.ErrReadOnly } func (s *Store) ListTrashedSessions(_ context.Context) ([]db.Session, error) { diff --git a/internal/e2e/artifact_sync_test.go b/internal/e2e/artifact_sync_test.go new file mode 100644 index 000000000..5d406b3b1 --- /dev/null +++ b/internal/e2e/artifact_sync_test.go @@ -0,0 +1,344 @@ +//go:build e2e + +package e2e + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "net/url" + "os" + "path/filepath" + "sort" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "go.kenn.io/agentsview/internal/artifact" + "go.kenn.io/agentsview/internal/config" + "go.kenn.io/agentsview/internal/db" + "go.kenn.io/agentsview/internal/dbtest" + "go.kenn.io/agentsview/internal/parser" + "go.kenn.io/agentsview/internal/server" + syncpkg "go.kenn.io/agentsview/internal/sync" +) + +const e2eToken = "artifact-e2e-token" + +func TestArtifactSyncTwoInstanceFolderAndHTTP(t *testing.T) { + ctx := context.Background() + root := preservedWorkspace(t) + share := filepath.Join(root, "share") + require.NoError(t, os.MkdirAll(share, 0o755)) + + a := openE2ENode(t, filepath.Join(root, "node-a"), "laptop-a1b2c3") + defer a.Close() + b := openE2ENode(t, filepath.Join(root, "node-b"), "desktop-d4e5f6") + defer b.Close() + + seedE2ESession(t, a.DB, "sess-1", "alpha", "world") + + folderSync(t, a, share) + folderSync(t, b, share) + + importedID := a.Origin + "~sess-1" + assertSessionProject(t, b.DB, importedID, "alpha") + assertSearchFinds(t, b.DB, "world", importedID) + + renameWithMetadata(t, a, "sess-1", "Alpha from laptop") + folderSync(t, a, share) + folderSync(t, b, share) + assertSessionDisplayName(t, b.DB, importedID, "Alpha from laptop") + + renameWithMetadata(t, b, importedID, "Bravo from desktop") + folderSync(t, b, share) + folderSync(t, a, share) + assertSessionDisplayName(t, a.DB, "sess-1", "Bravo from desktop") + + b.Close() + b = openE2ENode(t, filepath.Join(root, "node-b"), "desktop-d4e5f6") + defer b.Close() + + require.NoError(t, a.DB.ReplaceSessionMessages("sess-1", []db.Message{ + {SessionID: "sess-1", Ordinal: 0, Role: "user", Content: "hello", ContentLength: 5}, + {SessionID: "sess-1", Ordinal: 1, Role: "assistant", Content: "planet", ContentLength: 6}, + })) + _, err := artifact.Export(ctx, a.DB, filepath.Join(a.DataDir, "artifacts"), a.Origin) + require.NoError(t, err) + postOriginArtifacts(t, a, b, a.Origin) + + assertMessagesContain(t, b.DB, importedID, "planet") + assertSearchFinds(t, b.DB, "planet", importedID) + + writeForeignRename(t, b, "writer-a1b2c3", importedID, "Fork one") + writeForeignRename(t, b, "writer-b1b2c3", importedID, "Fork two") + imported, err := artifact.ImportDetailed( + ctx, + b.DB, + filepath.Join(b.DataDir, "artifacts"), + b.Origin, + ) + require.NoError(t, err) + assert.NotZero(t, imported.Metadata) + + conflicts, err := b.DB.ListMetadataConflicts(ctx, []string{importedID}) + require.NoError(t, err) + require.NotEmpty(t, conflicts) + assert.Equal(t, "display_name", conflicts[0].Field) + + apiConflicts := getMetadataConflicts(t, b, importedID) + require.NotEmpty(t, apiConflicts.Conflicts) + assert.Equal(t, importedID, apiConflicts.Conflicts[0].SessionGID) +} + +type e2eNode struct { + DataDir string + DBPath string + Origin string + DB *db.DB + Server *httptest.Server +} + +func openE2ENode(t *testing.T, dataDir, origin string) *e2eNode { + t.Helper() + require.NoError(t, os.MkdirAll(dataDir, 0o755)) + dbPath := filepath.Join(dataDir, "sessions.db") + database, err := db.Open(dbPath) + require.NoError(t, err) + + emptyAgentDir := filepath.Join(dataDir, "empty-agent-dir") + require.NoError(t, os.MkdirAll(emptyAgentDir, 0o755)) + broadcaster := server.NewBroadcaster(0) + engine := syncpkg.NewEngine(database, syncpkg.EngineConfig{ + AgentDirs: map[parser.AgentType][]string{ + parser.AgentClaude: {emptyAgentDir}, + }, + Machine: origin, + Emitter: broadcaster, + }) + cfg := config.Config{ + Host: "127.0.0.1", + Port: 0, + DataDir: dataDir, + DBPath: dbPath, + WriteTimeout: 30 * time.Second, + RequireAuth: true, + AuthToken: e2eToken, + ArtifactOriginID: origin, + } + srv := server.New(cfg, database, engine, server.WithBroadcaster(broadcaster)) + return &e2eNode{ + DataDir: dataDir, + DBPath: dbPath, + Origin: origin, + DB: database, + Server: httptest.NewServer(srv.Handler()), + } +} + +func (n *e2eNode) Close() { + if n == nil { + return + } + if n.Server != nil { + n.Server.Close() + n.Server = nil + } + if n.DB != nil { + n.DB.Close() + n.DB = nil + } +} + +func preservedWorkspace(t *testing.T) string { + t.Helper() + root, err := os.MkdirTemp("", "agentsview-artifact-e2e-*") + require.NoError(t, err) + t.Cleanup(func() { + if t.Failed() { + t.Logf("preserved artifact sync e2e workspace: %s", root) + return + } + require.NoError(t, os.RemoveAll(root)) + }) + return root +} + +func seedE2ESession(t *testing.T, database *db.DB, id, project, assistantText string) { + t.Helper() + started := "2026-06-14T01:02:03Z" + ended := "2026-06-14T01:03:03Z" + first := "hello" + dbtest.SeedSession(t, database, id, project, func(s *db.Session) { + s.MessageCount = 2 + s.UserMessageCount = 1 + s.FirstMessage = &first + s.StartedAt = &started + s.EndedAt = &ended + }) + require.NoError(t, database.ReplaceSessionMessages(id, []db.Message{ + {SessionID: id, Ordinal: 0, Role: "user", Content: "hello", ContentLength: 5}, + {SessionID: id, Ordinal: 1, Role: "assistant", Content: assistantText, ContentLength: len(assistantText)}, + })) +} + +func folderSync(t *testing.T, n *e2eNode, share string) artifact.SyncResult { + t.Helper() + res, err := artifact.SyncFolder(context.Background(), n.DB, artifact.SyncOptions{ + DataDir: n.DataDir, + Target: share, + Origin: n.Origin, + }) + require.NoError(t, err) + return res +} + +func renameWithMetadata(t *testing.T, n *e2eNode, sessionID, displayName string) { + t.Helper() + require.NoError(t, n.DB.RenameSession(sessionID, &displayName)) + // A local edit records its own replay register in this node's db, exactly as + // the rename handler does. + appendRenameArtifact(t, n.DB, n.DataDir, n.Origin, sessionID, displayName) +} + +func writeForeignRename(t *testing.T, n *e2eNode, origin, sessionID, displayName string) { + t.Helper() + // A foreign origin's event arrives as an artifact file written by another + // machine. Record it through a throwaway db so it is not pre-marked applied + // in this node's db, leaving the real import to replay it. + scratch, err := db.Open(filepath.Join(t.TempDir(), "scratch.db")) + require.NoError(t, err) + t.Cleanup(func() { scratch.Close() }) + appendRenameArtifact(t, scratch, n.DataDir, origin, sessionID, displayName) +} + +func appendRenameArtifact(t *testing.T, database *db.DB, dataDir, origin, sessionID, displayName string) { + t.Helper() + value, err := json.Marshal(struct { + DisplayName *string `json:"display_name"` + }{DisplayName: &displayName}) + require.NoError(t, err) + recorder := artifact.NewMetadataRecorder(database, artifact.MetadataRecorderOptions{ + DataDir: dataDir, + Origin: origin, + }) + _, err = recorder.Append(context.Background(), artifact.MetadataEventInput{ + SessionID: sessionID, + Op: artifact.MetadataOpRename, + Value: json.RawMessage(value), + }) + require.NoError(t, err) +} + +func postOriginArtifacts(t *testing.T, from, to *e2eNode, origin string) { + t.Helper() + for _, kind := range []string{ + artifact.KindSegments, + artifact.KindRaw, + artifact.KindManifests, + artifact.KindMeta, + artifact.KindCheckpoints, + } { + pattern := filepath.Join(from.DataDir, "artifacts", origin, kind, "*") + paths, err := filepath.Glob(pattern) + require.NoError(t, err) + sort.Strings(paths) + for _, path := range paths { + info, err := os.Stat(path) + require.NoError(t, err) + if !info.Mode().IsRegular() { + continue + } + postArtifact(t, to, origin, kind, filepath.Base(path), path) + } + } +} + +func postArtifact(t *testing.T, to *e2eNode, origin, kind, name, path string) { + t.Helper() + body, err := os.ReadFile(path) + require.NoError(t, err) + req, err := http.NewRequest( + http.MethodPost, + to.Server.URL+"/api/v1/artifacts/"+origin+"/"+kind+"/"+url.PathEscape(name), + bytes.NewReader(body), + ) + require.NoError(t, err) + req.Header.Set("Authorization", "Bearer "+e2eToken) + req.Header.Set("Content-Type", "application/octet-stream") + resp, err := http.DefaultClient.Do(req) + require.NoError(t, err) + defer resp.Body.Close() + require.Equal(t, http.StatusOK, resp.StatusCode) +} + +type metadataConflictsResponse struct { + Conflicts []db.MetadataConflict `json:"conflicts"` +} + +func getMetadataConflicts(t *testing.T, n *e2eNode, sessionID string) metadataConflictsResponse { + t.Helper() + req, err := http.NewRequest( + http.MethodGet, + n.Server.URL+"/api/v1/sessions/"+url.PathEscape(sessionID)+"/metadata-conflicts", + nil, + ) + require.NoError(t, err) + req.Header.Set("Authorization", "Bearer "+e2eToken) + resp, err := http.DefaultClient.Do(req) + require.NoError(t, err) + defer resp.Body.Close() + require.Equal(t, http.StatusOK, resp.StatusCode) + var out metadataConflictsResponse + require.NoError(t, json.NewDecoder(resp.Body).Decode(&out)) + return out +} + +func assertSessionProject(t *testing.T, database *db.DB, sessionID, project string) { + t.Helper() + sess, err := database.GetSessionFull(context.Background(), sessionID) + require.NoError(t, err) + require.NotNil(t, sess) + assert.Equal(t, project, sess.Project) +} + +func assertSessionDisplayName(t *testing.T, database *db.DB, sessionID, displayName string) { + t.Helper() + sess, err := database.GetSessionFull(context.Background(), sessionID) + require.NoError(t, err) + require.NotNil(t, sess) + require.NotNil(t, sess.DisplayName) + assert.Equal(t, displayName, *sess.DisplayName) +} + +func assertMessagesContain(t *testing.T, database *db.DB, sessionID, text string) { + t.Helper() + msgs, err := database.GetAllMessages(context.Background(), sessionID) + require.NoError(t, err) + for _, msg := range msgs { + if msg.Content == text { + return + } + } + require.Fail(t, fmt.Sprintf("session %s messages did not contain %q", sessionID, text)) +} + +func assertSearchFinds(t *testing.T, database *db.DB, query, sessionID string) { + t.Helper() + page, err := database.Search(context.Background(), db.SearchFilter{ + Query: query, + Limit: 10, + }) + require.NoError(t, err) + for _, result := range page.Results { + if result.SessionID == sessionID { + return + } + } + require.Fail(t, fmt.Sprintf("search %q did not find %s", query, sessionID)) +} diff --git a/internal/postgres/collision_pgtest_test.go b/internal/postgres/collision_pgtest_test.go index e1ea596d8..cc46da747 100644 --- a/internal/postgres/collision_pgtest_test.go +++ b/internal/postgres/collision_pgtest_test.go @@ -4,15 +4,873 @@ package postgres import ( "context" + "database/sql" "path/filepath" "testing" + "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "go.kenn.io/agentsview/internal/artifact" "go.kenn.io/agentsview/internal/db" ) +func TestPushArtifactNativeAndImportedCopiesShareOriginIdentity(t *testing.T) { + pgURL := testPGURL(t) + ctx := context.Background() + const originA = "origin-a1b2c3" + const originB = "origin-b4c5d6" + const nativeID = "native-id" + const canonicalID = originA + "~" + nativeID + const childID = "child-id" + const canonicalChildID = originA + "~" + childID + + for _, tc := range []struct { + name string + schema string + importerFirst bool + }{ + { + name: "origin pushes first", + schema: "agentsview_artifact_identity_origin_first_test", + }, + { + name: "importer pushes first", + schema: "agentsview_artifact_identity_importer_first_test", + importerFirst: true, + }, + } { + t.Run(tc.name, func(t *testing.T) { + pg, err := Open(pgURL, tc.schema, true) + require.NoError(t, err, "Open") + t.Cleanup(func() { require.NoError(t, pg.Close()) }) + _, err = pg.Exec(`DROP SCHEMA IF EXISTS ` + tc.schema + ` CASCADE`) + require.NoError(t, err, "drop schema") + require.NoError(t, EnsureSchema(ctx, pg, tc.schema), "EnsureSchema") + + originDB, err := db.Open(filepath.Join(t.TempDir(), "origin.db")) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, originDB.Close()) }) + require.NoError(t, artifact.AdoptOrigin(originDB, originA)) + require.NoError(t, originDB.UpsertSession(db.Session{ + ID: nativeID, Project: "alpha", Machine: "local", Agent: "claude", + MessageCount: 1, UserMessageCount: 1, + CreatedAt: "2026-01-01T00:00:00Z", + })) + require.NoError(t, originDB.ReplaceSessionMessages(nativeID, []db.Message{{ + SessionID: nativeID, Ordinal: 0, Role: "user", + Content: "hello", ContentLength: 5, + }})) + parentID := nativeID + require.NoError(t, originDB.UpsertSession(db.Session{ + ID: childID, Project: "alpha", Machine: "local", Agent: "claude", + MessageCount: 1, UserMessageCount: 1, + CreatedAt: "2026-01-01T00:00:01Z", ParentSessionID: &parentID, + })) + require.NoError(t, originDB.ReplaceSessionMessages(childID, []db.Message{{ + SessionID: childID, Ordinal: 0, Role: "user", + Content: "child", ContentLength: 5, + }})) + + artifactRoot := t.TempDir() + _, err = artifact.Export(ctx, originDB, artifactRoot, originA) + require.NoError(t, err) + importerDB, err := db.Open(filepath.Join(t.TempDir(), "importer.db")) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, importerDB.Close()) }) + require.NoError(t, artifact.AdoptOrigin(importerDB, originB)) + imported, messages, err := artifact.Import(ctx, importerDB, artifactRoot, originB) + require.NoError(t, err) + require.Equal(t, 2, imported) + require.Equal(t, 2, messages) + + // Advance the origin after the importer captured its artifact snapshot. + // An origin-first push must not be rolled back when that stale replica + // subsequently pushes the same canonical owner marker. + require.NoError(t, originDB.UpsertSession(db.Session{ + ID: nativeID, Project: "fresh-origin", Machine: "local", Agent: "claude", + MessageCount: 1, UserMessageCount: 1, + CreatedAt: "2026-01-01T00:00:00Z", + })) + require.NoError(t, originDB.ReplaceSessionMessages(nativeID, []db.Message{{ + SessionID: nativeID, Ordinal: 0, Role: "user", + Content: "fresh", ContentLength: 5, + }})) + + originSync := &Sync{ + pg: pg, local: originDB, machine: "host-a", + schema: tc.schema, schemaDone: true, + } + importerSync := &Sync{ + pg: pg, local: importerDB, machine: "host-b", + schema: tc.schema, schemaDone: true, + } + pushers := []*Sync{originSync, importerSync} + if tc.importerFirst { + pushers[0], pushers[1] = pushers[1], pushers[0] + } + for _, pusher := range pushers { + result, pushErr := pusher.Push(ctx, false, nil) + require.NoError(t, pushErr) + assert.Zero(t, result.Errors) + assert.Zero(t, result.SkippedConflicts) + } + + var id, machine, ownerMarker string + err = pg.QueryRowContext(ctx, ` + SELECT id, machine, owner_marker + FROM sessions + WHERE id IN ($1, $2) + `, nativeID, canonicalID).Scan(&id, &machine, &ownerMarker) + require.NoError(t, err) + assert.Equal(t, canonicalID, id) + assert.Equal(t, originA, machine) + assert.Equal(t, artifactOwnerMarkerPrefix+originA, ownerMarker) + var project, content string + require.NoError(t, pg.QueryRowContext(ctx, ` + SELECT s.project, m.content + FROM sessions s + JOIN messages m ON m.session_id = s.id + WHERE s.id = $1 AND m.ordinal = 0 + `, canonicalID).Scan(&project, &content)) + assert.Equal(t, "fresh-origin", project) + assert.Equal(t, "fresh", content) + var parent string + require.NoError(t, pg.QueryRowContext(ctx, ` + SELECT parent_session_id FROM sessions WHERE id = $1 + `, canonicalChildID).Scan(&parent)) + assert.Equal(t, canonicalID, parent, + "artifact relationships must resolve through the canonical origin identity") + + for _, pusher := range []*Sync{importerSync, originSync} { + result, pushErr := pusher.Push(ctx, true, nil) + require.NoError(t, pushErr) + assert.Zero(t, result.Errors) + assert.Zero(t, result.SkippedConflicts) + } + var count int + require.NoError(t, pg.QueryRowContext(ctx, ` + SELECT COUNT(*) FROM sessions WHERE id IN ($1, $2) + `, nativeID, canonicalID).Scan(&count)) + assert.Equal(t, 1, count, + "native and imported copies must keep one PG row across repeated pushes") + }) + } +} + +func TestPushSSHShapedSessionDoesNotAdoptArtifactOwnership(t *testing.T) { + pgURL := testPGURL(t) + ctx := context.Background() + const schema = "agentsview_artifact_provenance_guard_test" + const remoteOrigin = "origin-a1b2c3" + const localOrigin = "origin-b4c5d6" + const sessionID = remoteOrigin + "~native-id" + + pg, err := Open(pgURL, schema, true) + require.NoError(t, err, "Open") + t.Cleanup(func() { require.NoError(t, pg.Close()) }) + _, err = pg.Exec(`DROP SCHEMA IF EXISTS ` + schema + ` CASCADE`) + require.NoError(t, err, "drop schema") + require.NoError(t, EnsureSchema(ctx, pg, schema), "EnsureSchema") + + localDB, err := db.Open(filepath.Join(t.TempDir(), "local.db")) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, localDB.Close()) }) + require.NoError(t, artifact.AdoptOrigin(localDB, localOrigin)) + require.NoError(t, localDB.UpsertSession(db.Session{ + ID: sessionID, Project: "ssh-copy", Machine: remoteOrigin, Agent: "claude", + MessageCount: 1, UserMessageCount: 1, + CreatedAt: "2026-01-01T00:00:00Z", + })) + require.NoError(t, localDB.ReplaceSessionMessages(sessionID, []db.Message{{ + SessionID: sessionID, Ordinal: 0, Role: "user", + Content: "ssh copy", ContentLength: 8, + }})) + + _, err = pg.ExecContext(ctx, ` + INSERT INTO sessions ( + id, machine, owner_marker, project, agent, created_at + ) VALUES ($1, $2, $3, $4, $5, NOW()) + `, sessionID, remoteOrigin, artifactOwnerMarkerPrefix+remoteOrigin, + "genuine-artifact", "claude") + require.NoError(t, err, "seed genuine artifact-owned row") + + syncer := &Sync{ + pg: pg, local: localDB, machine: "ssh-importer", + schema: schema, schemaDone: true, + } + result, err := syncer.Push(ctx, false, nil) + require.NoError(t, err) + assert.Equal(t, 1, result.SkippedConflicts, + "an SSH-shaped row without artifact import provenance must retain legacy collision protection") + assert.Zero(t, result.SessionsPushed) + + var project, ownerMarker string + require.NoError(t, pg.QueryRowContext(ctx, ` + SELECT project, owner_marker FROM sessions WHERE id = $1 + `, sessionID).Scan(&project, &ownerMarker)) + assert.Equal(t, "genuine-artifact", project) + assert.Equal(t, artifactOwnerMarkerPrefix+remoteOrigin, ownerMarker) +} + +func TestPushArtifactOriginAdoptionReusesLegacyBareRows(t *testing.T) { + pgURL := testPGURL(t) + ctx := context.Background() + const schema = "agentsview_artifact_origin_upgrade_test" + const originA = "origin-a1b2c3" + const originB = "origin-b4c5d6" + const parentID = "native-parent" + const childID = "native-child" + const canonicalParentID = originA + "~" + parentID + const canonicalChildID = originA + "~" + childID + const stateScope = "work" + + pg, err := Open(pgURL, schema, true) + require.NoError(t, err, "Open") + t.Cleanup(func() { require.NoError(t, pg.Close()) }) + _, err = pg.Exec(`DROP SCHEMA IF EXISTS ` + schema + ` CASCADE`) + require.NoError(t, err, "drop schema") + require.NoError(t, EnsureSchema(ctx, pg, schema), "EnsureSchema") + + originDB, err := db.Open(filepath.Join(t.TempDir(), "origin.db")) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, originDB.Close()) }) + require.NoError(t, originDB.UpsertSession(db.Session{ + ID: parentID, Project: "alpha", Machine: "local", Agent: "claude", + MessageCount: 1, UserMessageCount: 1, + CreatedAt: "2026-01-01T00:00:00Z", + })) + require.NoError(t, originDB.ReplaceSessionMessages(parentID, []db.Message{{ + SessionID: parentID, Ordinal: 0, Role: "user", + Content: "parent", ContentLength: 6, SourceUUID: "parent-source", + }})) + parentRef := parentID + require.NoError(t, originDB.UpsertSession(db.Session{ + ID: childID, Project: "alpha", Machine: "local", Agent: "claude", + MessageCount: 1, UserMessageCount: 1, + CreatedAt: "2026-01-01T00:00:01Z", ParentSessionID: &parentRef, + })) + require.NoError(t, originDB.ReplaceSessionMessages(childID, []db.Message{{ + SessionID: childID, Ordinal: 0, Role: "user", + Content: "child", ContentLength: 5, SourceUUID: "child-source", + }})) + + originSync := &Sync{ + pg: pg, local: originDB, machine: "host-a", + schema: schema, schemaDone: true, + syncState: newScopedSyncStateStore(originDB, stateScope, false), + syncStateTarget: stateScope, + } + first, err := originSync.Push(ctx, false, nil) + require.NoError(t, err) + require.Equal(t, 2, first.SessionsPushed) + watermark, err := originDB.GetSyncState("last_push_at:" + stateScope) + require.NoError(t, err) + require.NotEmpty(t, watermark, "precondition: initial push established incremental state") + // Simulate a pusher upgraded from a build predating identity-mode state. + require.NoError(t, originDB.SetSyncState( + "pg_artifact_identity_v1:"+stateScope, "", + )) + + _, err = pg.ExecContext(ctx, + `UPDATE sessions SET display_name = 'PG title' WHERE id = $1`, + parentID, + ) + require.NoError(t, err, "seed PG-local display name") + _, err = pg.ExecContext(ctx, + `INSERT INTO starred_sessions (session_id) VALUES ($1)`, + parentID, + ) + require.NoError(t, err, "seed PG-local star") + _, err = pg.ExecContext(ctx, ` + INSERT INTO pinned_messages ( + session_id, message_id, ordinal, source_uuid, note + ) + SELECT $1, ordinal, ordinal, COALESCE(source_uuid, ''), 'PG pin' + FROM messages WHERE session_id = $1 AND ordinal = 0 + `, parentID) + require.NoError(t, err, "seed PG-local pin") + + before, err := originDB.GetSessionFull(ctx, parentID) + require.NoError(t, err) + require.NotNil(t, before) + require.NoError(t, artifact.AdoptOrigin(originDB, originA)) + afterAdopt, err := originDB.GetSessionFull(ctx, parentID) + require.NoError(t, err) + require.NotNil(t, afterAdopt) + assert.Equal(t, before.LocalModifiedAt, afterAdopt.LocalModifiedAt, + "origin adoption must not need to mutate session timestamps") + + upgraded, err := originSync.Push(ctx, false, nil) + require.NoError(t, err) + assert.Equal(t, 2, upgraded.SessionsPushed, + "identity-mode change must revisit unchanged sessions") + assert.Zero(t, upgraded.Errors) + assert.Zero(t, upgraded.SkippedConflicts) + + var stableBareRows int + require.NoError(t, pg.QueryRowContext(ctx, ` + SELECT COUNT(*) FROM sessions + WHERE id IN ($1, $2) AND machine = $3 AND owner_marker = $4 + `, parentID, childID, originA, artifactOwnerMarkerPrefix+originA).Scan(&stableBareRows)) + assert.Equal(t, 2, stableBareRows, + "same-owner legacy bare rows must upgrade in place to stable artifact ownership") + var canonicalRows int + require.NoError(t, pg.QueryRowContext(ctx, ` + SELECT COUNT(*) FROM sessions WHERE id IN ($1, $2) + `, canonicalParentID, canonicalChildID).Scan(&canonicalRows)) + assert.Zero(t, canonicalRows, "upgrade must not migrate primary keys or duplicate rows") + assert.Equal(t, parentID, pgParentSessionID(t, ctx, pg, childID)) + assertPGArtifactUpgradeCuration(t, ctx, pg, parentID) + mode, err := originDB.GetSyncState("pg_artifact_identity_v1:" + stateScope) + require.NoError(t, err) + assert.Equal(t, artifactOwnerMarkerPrefix+originA, mode, + "successful push must persist the target-scoped identity mode") + + artifactRoot := t.TempDir() + _, err = artifact.Export(ctx, originDB, artifactRoot, originA) + require.NoError(t, err) + importerDB, err := db.Open(filepath.Join(t.TempDir(), "importer.db")) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, importerDB.Close()) }) + require.NoError(t, artifact.AdoptOrigin(importerDB, originB)) + imported, messages, err := artifact.Import(ctx, importerDB, artifactRoot, originB) + require.NoError(t, err) + require.Equal(t, 2, imported) + require.Equal(t, 2, messages) + + importerSync := &Sync{ + pg: pg, local: importerDB, machine: "host-b", + schema: schema, schemaDone: true, + syncState: newScopedSyncStateStore(importerDB, stateScope, false), + syncStateTarget: stateScope, + } + importResult, err := importerSync.Push(ctx, false, nil) + require.NoError(t, err) + assert.Zero(t, importResult.Errors) + assert.Zero(t, importResult.SkippedConflicts) + require.NoError(t, pg.QueryRowContext(ctx, ` + SELECT COUNT(*) FROM sessions + WHERE id IN ($1, $2, $3, $4) + `, parentID, childID, canonicalParentID, canonicalChildID).Scan(&stableBareRows)) + assert.Equal(t, 2, stableBareRows, + "an imported copy must reuse the stable legacy bare aliases") + assert.Equal(t, parentID, pgParentSessionID(t, ctx, pg, childID)) + assertPGArtifactUpgradeCuration(t, ctx, pg, parentID) + + _, err = pg.ExecContext(ctx, ` + UPDATE sessions SET parent_session_id = $1 WHERE id = $2 + `, canonicalParentID, childID) + require.NoError(t, err, "seed stale canonical relationship") + time.Sleep(5 * time.Millisecond) + name := "Imported child" + require.NoError(t, importerDB.RenameSession(canonicalChildID, &name)) + replicaResult, err := importerSync.Push(ctx, false, nil) + require.NoError(t, err) + assert.Zero(t, replicaResult.SessionsPushed, + "an imported replica must not update an existing canonical alias") + assert.Zero(t, replicaResult.SkippedConflicts) + assert.Equal(t, canonicalParentID, pgParentSessionID(t, ctx, pg, childID), + "a stale replica must leave the canonical row unchanged") + + originName := "Origin child" + require.NoError(t, originDB.RenameSession(childID, &originName)) + fallbackResult, err := originSync.Push(ctx, false, nil) + require.NoError(t, err) + assert.Equal(t, 1, fallbackResult.SessionsPushed, + "the modified origin child should use committed-PG relationship fallback") + assert.Zero(t, fallbackResult.SkippedConflicts) + assert.Equal(t, parentID, pgParentSessionID(t, ctx, pg, childID), + "origin relationship fallback must resolve the stable bare parent alias") + assertPGArtifactUpgradeCuration(t, ctx, pg, parentID) +} + +func TestPushArtifactOriginAdoptionConvergesImporterFirst(t *testing.T) { + pgURL := testPGURL(t) + ctx := context.Background() + const schema = "agentsview_artifact_origin_upgrade_importer_first_test" + const originA = "origin-a1b2c3" + const originB = "origin-b4c5d6" + const parentID = "native-parent" + const childID = "native-child" + const canonicalParentID = originA + "~" + parentID + const canonicalChildID = originA + "~" + childID + const referenceID = "pg-reference-holder" + const stateScope = "work" + const sourceDisplayName = "Source title" + const localDisplayName = "PG title" + const pinCreatedAt = "2026-01-04T00:00:00Z" + const updatedSourceDisplay = "Origin renamed child" + const canonicalChildDeletedAt = "2026-01-05T00:00:00Z" + + pg, err := Open(pgURL, schema, true) + require.NoError(t, err, "Open") + t.Cleanup(func() { require.NoError(t, pg.Close()) }) + _, err = pg.Exec(`DROP SCHEMA IF EXISTS ` + schema + ` CASCADE`) + require.NoError(t, err, "drop schema") + require.NoError(t, EnsureSchema(ctx, pg, schema), "EnsureSchema") + + originDB, err := db.Open(filepath.Join(t.TempDir(), "origin.db")) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, originDB.Close()) }) + require.NoError(t, originDB.UpsertSession(db.Session{ + ID: parentID, Project: "alpha", Machine: "local", Agent: "claude", + MessageCount: 1, UserMessageCount: 1, + CreatedAt: "2026-01-01T00:00:00Z", + })) + require.NoError(t, originDB.ReplaceSessionMessages(parentID, []db.Message{{ + SessionID: parentID, Ordinal: 0, Role: "user", + Content: "parent", ContentLength: 6, SourceUUID: "parent-source", + }})) + parentRef := parentID + require.NoError(t, originDB.UpsertSession(db.Session{ + ID: childID, Project: "alpha", Machine: "local", Agent: "claude", + MessageCount: 1, UserMessageCount: 0, HasToolCalls: true, + CreatedAt: "2026-01-01T00:00:01Z", + ParentSessionID: &parentRef, + SourceSessionID: parentID, + })) + require.NoError(t, originDB.ReplaceSessionMessages(childID, []db.Message{{ + SessionID: childID, Ordinal: 0, Role: "assistant", + Content: "child", ContentLength: 5, SourceUUID: "child-source", + HasToolUse: true, + ToolCalls: []db.ToolCall{{ + ToolName: "subagent", Category: "Task", ToolUseID: "call-parent", + SubagentSessionID: parentID, + ResultEvents: []db.ToolResultEvent{{ + ToolUseID: "call-parent", SubagentSessionID: parentID, + Source: "tool_result", Status: "completed", Content: "done", + ContentLength: 4, + }}, + }}, + }})) + + originSync := &Sync{ + pg: pg, local: originDB, machine: "host-a", + schema: schema, schemaDone: true, + syncState: newScopedSyncStateStore(originDB, stateScope, false), + syncStateTarget: stateScope, + } + legacyResult, err := originSync.Push(ctx, false, nil) + require.NoError(t, err) + require.Equal(t, 2, legacyResult.SessionsPushed) + require.NoError(t, originDB.SetSyncState( + "pg_artifact_identity_v1:"+stateScope, "", + )) + + _, err = pg.ExecContext(ctx, ` + UPDATE sessions + SET display_name = $1, + source_display_name = $2 + WHERE id = $3 + `, localDisplayName, sourceDisplayName, parentID) + require.NoError(t, err, "seed PG-local session curation") + _, err = pg.ExecContext(ctx, ` + INSERT INTO starred_sessions (session_id, created_at) + VALUES ($1, '2026-01-04T00:00:00Z'::timestamptz) + `, parentID) + require.NoError(t, err, "seed PG-local star") + _, err = pg.ExecContext(ctx, ` + INSERT INTO pinned_messages ( + session_id, message_id, ordinal, source_uuid, note, created_at + ) + VALUES ($1, 0, 0, 'parent-source', 'PG pin', $2::timestamptz) + `, parentID, pinCreatedAt) + require.NoError(t, err, "seed PG-local pin") + + require.NoError(t, artifact.AdoptOrigin(originDB, originA)) + artifactRoot := t.TempDir() + _, err = artifact.Export(ctx, originDB, artifactRoot, originA) + require.NoError(t, err) + importerDB, err := db.Open(filepath.Join(t.TempDir(), "importer.db")) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, importerDB.Close()) }) + require.NoError(t, artifact.AdoptOrigin(importerDB, originB)) + imported, messages, err := artifact.Import( + ctx, importerDB, artifactRoot, originB, + ) + require.NoError(t, err) + require.Equal(t, 2, imported) + require.Equal(t, 2, messages) + + importerSync := &Sync{ + pg: pg, local: importerDB, machine: "host-b", + schema: schema, schemaDone: true, + syncState: newScopedSyncStateStore(importerDB, stateScope, false), + syncStateTarget: stateScope, + } + importerResult, err := importerSync.Push(ctx, false, nil) + require.NoError(t, err) + assert.Zero(t, importerResult.Errors) + assert.Zero(t, importerResult.SkippedConflicts) + assertPGArtifactNativeRows(t, ctx, pg, parentID, canonicalParentID, 2) + assertPGArtifactNativeRows(t, ctx, pg, childID, canonicalChildID, 2) + _, err = pg.ExecContext(ctx, ` + UPDATE sessions + SET deleted_at = $1::timestamptz, + source_deleted_at = NULL + WHERE id = $2 + `, canonicalChildDeletedAt, canonicalChildID) + require.NoError(t, err, "seed newer canonical PG curation") + require.NoError(t, originDB.RenameSession( + childID, new(updatedSourceDisplay), + )) + require.NoError(t, originDB.SoftDeleteSession(parentID)) + updatedParent, err := originDB.GetSessionFull(ctx, parentID) + require.NoError(t, err) + require.NotNil(t, updatedParent) + require.NotNil(t, updatedParent.DeletedAt) + updatedSourceDeletedAt, ok := ParseSQLiteTimestamp(*updatedParent.DeletedAt) + require.True(t, ok, "parse source soft-delete timestamp") + wantSourceDeletedAt := updatedSourceDeletedAt.UTC().Format(time.RFC3339) + + _, err = pg.ExecContext(ctx, ` + INSERT INTO sessions ( + id, machine, owner_marker, project, agent, created_at, + parent_session_id, source_session_id + ) VALUES ($1, 'pg-only', 'pg-only-owner', 'alpha', 'claude', NOW(), $2, $2) + `, referenceID, parentID) + require.NoError(t, err, "seed PG-only session relationships") + _, err = pg.ExecContext(ctx, ` + INSERT INTO messages (session_id, ordinal, role, content) + VALUES ($1, 0, 'assistant', 'reference') + `, referenceID) + require.NoError(t, err, "seed PG-only message") + _, err = pg.ExecContext(ctx, ` + INSERT INTO tool_calls ( + session_id, tool_name, category, call_index, tool_use_id, + subagent_session_id, message_ordinal + ) VALUES ($1, 'subagent', 'Task', 0, 'pg-call', $2, 0) + `, referenceID, parentID) + require.NoError(t, err, "seed PG-only tool call relationship") + _, err = pg.ExecContext(ctx, ` + INSERT INTO tool_result_events ( + session_id, tool_call_message_ordinal, call_index, + tool_use_id, subagent_session_id, source, status, + content, content_length, event_index + ) VALUES ($1, 0, 0, 'pg-call', $2, 'tool_result', + 'completed', 'done', 4, 0) + `, referenceID, parentID) + require.NoError(t, err, "seed PG-only tool result relationship") + + upgradeResult, err := originSync.Push(ctx, false, nil) + require.NoError(t, err) + assert.Equal(t, 2, upgradeResult.SessionsPushed, + "identity-mode change must revisit unchanged origin sessions") + assert.Zero(t, upgradeResult.Errors) + assert.Zero(t, upgradeResult.SkippedConflicts) + assertPGArtifactNativeRows(t, ctx, pg, parentID, canonicalParentID, 1) + assertPGArtifactNativeRows(t, ctx, pg, childID, canonicalChildID, 1) + assertPGArtifactStableOwner(t, ctx, pg, canonicalParentID, originA) + assertPGArtifactStableOwner(t, ctx, pg, canonicalChildID, originA) + assertPGArtifactUpgradeCurationDetails( + t, ctx, pg, canonicalParentID, + localDisplayName, sourceDisplayName, + wantSourceDeletedAt, wantSourceDeletedAt, pinCreatedAt, + ) + assertPGArtifactUpdatedSourceDisplay( + t, ctx, pg, canonicalChildID, + updatedSourceDisplay, canonicalChildDeletedAt, + ) + assertPGArtifactUpgradeRelationships( + t, ctx, pg, canonicalParentID, canonicalChildID, referenceID, + ) + + for _, pusher := range []*Sync{importerSync, originSync} { + repeated, pushErr := pusher.Push(ctx, true, nil) + require.NoError(t, pushErr) + assert.Zero(t, repeated.Errors) + assert.Zero(t, repeated.SkippedConflicts) + } + assertPGArtifactNativeRows(t, ctx, pg, parentID, canonicalParentID, 1) + assertPGArtifactNativeRows(t, ctx, pg, childID, canonicalChildID, 1) + assertPGArtifactStableOwner(t, ctx, pg, canonicalParentID, originA) + assertPGArtifactStableOwner(t, ctx, pg, canonicalChildID, originA) + assertPGArtifactUpgradeCurationAfterRepeatedPushes( + t, ctx, pg, canonicalParentID, + localDisplayName, wantSourceDeletedAt, pinCreatedAt, + ) + assertPGArtifactUpgradeRelationships( + t, ctx, pg, canonicalParentID, canonicalChildID, referenceID, + ) +} + +func TestPushArtifactOriginAdoptionIgnoresForeignBareAlias(t *testing.T) { + pgURL := testPGURL(t) + ctx := context.Background() + const schema = "agentsview_artifact_origin_upgrade_proof_test" + const origin = "origin-a1b2c3" + const nativeID = "native-id" + const canonicalID = origin + "~" + nativeID + + pg, err := Open(pgURL, schema, true) + require.NoError(t, err, "Open") + t.Cleanup(func() { require.NoError(t, pg.Close()) }) + _, err = pg.Exec(`DROP SCHEMA IF EXISTS ` + schema + ` CASCADE`) + require.NoError(t, err, "drop schema") + require.NoError(t, EnsureSchema(ctx, pg, schema), "EnsureSchema") + + localDB, err := db.Open(filepath.Join(t.TempDir(), "local.db")) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, localDB.Close()) }) + require.NoError(t, artifact.AdoptOrigin(localDB, origin)) + require.NoError(t, localDB.UpsertSession(db.Session{ + ID: nativeID, Project: "local-project", Machine: "local", Agent: "claude", + MessageCount: 1, UserMessageCount: 1, + CreatedAt: "2026-01-01T00:00:00Z", + })) + require.NoError(t, localDB.ReplaceSessionMessages(nativeID, []db.Message{{ + SessionID: nativeID, Ordinal: 0, Role: "user", + Content: "local", ContentLength: 5, + }})) + + _, err = pg.ExecContext(ctx, ` + INSERT INTO sessions ( + id, machine, owner_marker, project, agent, created_at + ) VALUES + ($1, $2, $3, 'canonical-before', 'claude', NOW()), + ($4, 'legacy-host', 'different-random-marker', + 'legacy-before', 'claude', NOW()) + `, canonicalID, origin, artifactOwnerMarkerPrefix+origin, nativeID) + require.NoError(t, err, "seed unproven duplicate pair") + + syncer := &Sync{ + pg: pg, local: localDB, machine: "host-a", + schema: schema, schemaDone: true, + } + result, err := syncer.Push(ctx, true, nil) + require.NoError(t, err) + assert.Zero(t, result.Errors) + assert.Zero(t, result.SkippedConflicts) + assert.Equal(t, 1, result.SessionsPushed) + + rows, err := pg.QueryContext(ctx, ` + SELECT id, project FROM sessions + WHERE id IN ($1, $2) ORDER BY id + `, nativeID, canonicalID) + require.NoError(t, err) + defer rows.Close() + projects := map[string]string{} + for rows.Next() { + var id, project string + require.NoError(t, rows.Scan(&id, &project)) + projects[id] = project + } + require.NoError(t, rows.Err()) + assert.Equal(t, map[string]string{ + nativeID: "legacy-before", + canonicalID: "local-project", + }, projects, "a foreign bare collision must not block the stable canonical row") +} + +func assertPGArtifactNativeRows( + t *testing.T, ctx context.Context, pg *sql.DB, + bareID, canonicalID string, want int, +) { + t.Helper() + var count int + require.NoError(t, pg.QueryRowContext(ctx, ` + SELECT COUNT(*) FROM sessions WHERE id IN ($1, $2) + `, bareID, canonicalID).Scan(&count)) + assert.Equal(t, want, count) +} + +func assertPGArtifactStableOwner( + t *testing.T, ctx context.Context, pg *sql.DB, + sessionID, origin string, +) { + t.Helper() + var machine, ownerMarker string + require.NoError(t, pg.QueryRowContext(ctx, ` + SELECT machine, owner_marker FROM sessions WHERE id = $1 + `, sessionID).Scan(&machine, &ownerMarker)) + assert.Equal(t, origin, machine) + assert.Equal(t, artifactOwnerMarkerPrefix+origin, ownerMarker) +} + +func assertPGArtifactUpgradeCurationDetails( + t *testing.T, ctx context.Context, pg *sql.DB, sessionID, + wantDisplay, wantSourceDisplay, wantDeleted, wantSourceDeleted, + wantPinCreated string, +) { + t.Helper() + assertPGArtifactSessionCuration( + t, ctx, pg, sessionID, + wantDisplay, wantSourceDisplay, wantDeleted, wantSourceDeleted, + ) + + var stars int + require.NoError(t, pg.QueryRowContext(ctx, ` + SELECT COUNT(*) FROM starred_sessions WHERE session_id = $1 + `, sessionID).Scan(&stars)) + assert.Equal(t, 1, stars) + + var ordinal, messageID int + var sourceUUID, note string + var createdAt time.Time + require.NoError(t, pg.QueryRowContext(ctx, ` + SELECT message_id, ordinal, source_uuid, note, created_at + FROM pinned_messages WHERE session_id = $1 + `, sessionID).Scan( + &messageID, &ordinal, &sourceUUID, ¬e, &createdAt, + )) + assert.Equal(t, 0, messageID) + assert.Equal(t, 0, ordinal) + assert.Equal(t, "parent-source", sourceUUID) + assert.Equal(t, "PG pin", note) + assert.Equal(t, wantPinCreated, createdAt.UTC().Format(time.RFC3339)) +} + +func assertPGArtifactSessionCuration( + t *testing.T, ctx context.Context, pg *sql.DB, sessionID, + wantDisplay, wantSourceDisplay, wantDeleted, wantSourceDeleted string, +) { + t.Helper() + var displayName, sourceDisplayName string + var deletedAt, sourceDeletedAt sql.NullTime + require.NoError(t, pg.QueryRowContext(ctx, ` + SELECT display_name, source_display_name, deleted_at, source_deleted_at + FROM sessions WHERE id = $1 + `, sessionID).Scan( + &displayName, &sourceDisplayName, &deletedAt, &sourceDeletedAt, + )) + assert.Equal(t, wantDisplay, displayName) + assert.Equal(t, wantSourceDisplay, sourceDisplayName) + if assert.True(t, deletedAt.Valid, "deleted_at") { + assert.Equal(t, wantDeleted, deletedAt.Time.UTC().Format(time.RFC3339)) + } + if assert.True(t, sourceDeletedAt.Valid, "source_deleted_at") { + assert.Equal(t, wantSourceDeleted, + sourceDeletedAt.Time.UTC().Format(time.RFC3339)) + } +} + +func assertPGArtifactUpdatedSourceDisplay( + t *testing.T, ctx context.Context, pg *sql.DB, sessionID, + wantDisplay, wantDeleted string, +) { + t.Helper() + var displayName, sourceDisplayName sql.NullString + var deletedAt time.Time + var sourceDeletedAt sql.NullTime + require.NoError(t, pg.QueryRowContext(ctx, ` + SELECT display_name, source_display_name, deleted_at, source_deleted_at + FROM sessions WHERE id = $1 + `, sessionID).Scan( + &displayName, &sourceDisplayName, &deletedAt, &sourceDeletedAt, + )) + if assert.True(t, displayName.Valid, "display_name") { + assert.Equal(t, wantDisplay, displayName.String) + } + if assert.True(t, sourceDisplayName.Valid, "source_display_name") { + assert.Equal(t, wantDisplay, sourceDisplayName.String) + } + assert.Equal(t, wantDeleted, deletedAt.UTC().Format(time.RFC3339)) + assert.False(t, sourceDeletedAt.Valid, + "canonical delete override must retain the current source baseline") +} + +func assertPGArtifactUpgradeCurationAfterRepeatedPushes( + t *testing.T, ctx context.Context, pg *sql.DB, sessionID, + wantDisplay, wantDeleted, wantPinCreated string, +) { + t.Helper() + var displayName string + var deletedAt time.Time + require.NoError(t, pg.QueryRowContext(ctx, ` + SELECT display_name, deleted_at + FROM sessions WHERE id = $1 + `, sessionID).Scan(&displayName, &deletedAt)) + assert.Equal(t, wantDisplay, displayName) + assert.Equal(t, wantDeleted, deletedAt.UTC().Format(time.RFC3339)) + + var stars int + require.NoError(t, pg.QueryRowContext(ctx, ` + SELECT COUNT(*) FROM starred_sessions WHERE session_id = $1 + `, sessionID).Scan(&stars)) + assert.Equal(t, 1, stars) + + var ordinal, messageID int + var sourceUUID, note string + var createdAt time.Time + require.NoError(t, pg.QueryRowContext(ctx, ` + SELECT message_id, ordinal, source_uuid, note, created_at + FROM pinned_messages WHERE session_id = $1 + `, sessionID).Scan( + &messageID, &ordinal, &sourceUUID, ¬e, &createdAt, + )) + assert.Equal(t, 0, messageID) + assert.Equal(t, 0, ordinal) + assert.Equal(t, "parent-source", sourceUUID) + assert.Equal(t, "PG pin", note) + assert.Equal(t, wantPinCreated, createdAt.UTC().Format(time.RFC3339)) +} + +func assertPGArtifactUpgradeRelationships( + t *testing.T, ctx context.Context, pg *sql.DB, + canonicalParentID, canonicalChildID, referenceID string, +) { + t.Helper() + var parentID, sourceID string + require.NoError(t, pg.QueryRowContext(ctx, ` + SELECT parent_session_id, source_session_id + FROM sessions WHERE id = $1 + `, canonicalChildID).Scan(&parentID, &sourceID)) + assert.Equal(t, canonicalParentID, parentID) + assert.Equal(t, canonicalParentID, sourceID) + require.NoError(t, pg.QueryRowContext(ctx, ` + SELECT parent_session_id, source_session_id + FROM sessions WHERE id = $1 + `, referenceID).Scan(&parentID, &sourceID)) + assert.Equal(t, canonicalParentID, parentID) + assert.Equal(t, canonicalParentID, sourceID) + + var toolCallSubagentID string + require.NoError(t, pg.QueryRowContext(ctx, ` + SELECT subagent_session_id FROM tool_calls WHERE session_id = $1 + `, referenceID).Scan(&toolCallSubagentID)) + assert.Equal(t, canonicalParentID, toolCallSubagentID, "tool_calls") + var toolResultSubagentID string + require.NoError(t, pg.QueryRowContext(ctx, ` + SELECT subagent_session_id + FROM tool_result_events WHERE session_id = $1 + `, referenceID).Scan(&toolResultSubagentID)) + assert.Equal(t, canonicalParentID, toolResultSubagentID, "tool_result_events") +} + +func pgParentSessionID( + t *testing.T, ctx context.Context, pg *sql.DB, sessionID string, +) string { + t.Helper() + var parentID string + require.NoError(t, pg.QueryRowContext(ctx, ` + SELECT COALESCE(parent_session_id, '') FROM sessions WHERE id = $1 + `, sessionID).Scan(&parentID)) + return parentID +} + +func assertPGArtifactUpgradeCuration( + t *testing.T, ctx context.Context, pg *sql.DB, sessionID string, +) { + t.Helper() + var displayName string + require.NoError(t, pg.QueryRowContext(ctx, ` + SELECT COALESCE(display_name, '') FROM sessions WHERE id = $1 + `, sessionID).Scan(&displayName)) + assert.Equal(t, "PG title", displayName) + var stars, pins int + require.NoError(t, pg.QueryRowContext(ctx, ` + SELECT COUNT(*) FROM starred_sessions WHERE session_id = $1 + `, sessionID).Scan(&stars)) + require.NoError(t, pg.QueryRowContext(ctx, ` + SELECT COUNT(*) FROM pinned_messages + WHERE session_id = $1 AND note = 'PG pin' + `, sessionID).Scan(&pins)) + assert.Equal(t, 1, stars) + assert.Equal(t, 1, pins) +} + // TestPushSessionGuardsAgainstCrossMachineCollision verifies that when two // machines share the same session ID (from dotfile sync, directory restore, etc.), // the second machine's push is skipped if the session is already owned by a @@ -84,7 +942,10 @@ func TestPushSessionGuardsAgainstCrossMachineCollision(t *testing.T) { // Execute pushSession. tx, err := pg.BeginTx(ctx, nil) require.NoError(t, err, "BeginTx") - err = sync.pushSession(ctx, tx, sess, markerID, nil) + err = sync.pushSession(ctx, tx, sess, pushedSessionIdentity{ + ID: sess.ID, + Machine: sess.Machine, + }, markerID, nil) require.ErrorIs(t, err, errSessionOwnershipConflict, "pushSession should return ownership conflict sentinel") require.NoError(t, tx.Commit(), "Commit") @@ -156,7 +1017,10 @@ func TestPushSessionAllowsMachineRenameForSameOwnerMarker(t *testing.T) { tx, err := pg.BeginTx(ctx, nil) require.NoError(t, err, "BeginTx") - require.NoError(t, sync.pushSession(ctx, tx, sess, markerID, nil), "pushSession") + require.NoError(t, sync.pushSession(ctx, tx, sess, pushedSessionIdentity{ + ID: sess.ID, + Machine: "renamed-host", + }, markerID, nil), "pushSession") require.NoError(t, tx.Commit(), "Commit") var machine, ownerMarker string @@ -168,6 +1032,214 @@ func TestPushSessionAllowsMachineRenameForSameOwnerMarker(t *testing.T) { assert.Equal(t, markerID, ownerMarker) } +// TestPushResolvesRelationshipIDsToPrefixedTargets verifies that when a +// referenced session is pushed under a collision-avoidance prefix, the +// relationship ids pointing at it -- source_session_id, parent_session_id, and +// a tool-call subagent_session_id -- are rewritten to the prefixed id so child +// and subagent rows link to the right PG session instead of a foreign machine's +// row or a dangling id. A non-colliding parent keeps its bare id. +func TestPushResolvesRelationshipIDsToPrefixedTargets(t *testing.T) { + pgURL := testPGURL(t) + + const schema = "agentsview_relationship_resolution_test" + pg, err := Open(pgURL, schema, true) + require.NoError(t, err, "Open") + defer pg.Close() + + ctx := context.Background() + _, err = pg.Exec(`DROP SCHEMA IF EXISTS ` + schema + ` CASCADE`) + require.NoError(t, err, "drop schema") + require.NoError(t, EnsureSchema(ctx, pg, schema), "EnsureSchema") + + localDB, err := db.Open(filepath.Join(t.TempDir(), "local.db")) + require.NoError(t, err, "db.Open") + defer localDB.Close() + + sync := &Sync{ + pg: pg, + local: localDB, + machine: "machine-b", + schema: schema, + schemaDone: true, + } + + // A foreign machine already owns the bare "shared" id, so machine-b's + // "shared" session must be pushed under the "machine-b~shared" prefix. + _, err = pg.ExecContext(ctx, ` + INSERT INTO sessions ( + id, machine, owner_marker, project, agent, created_at + ) VALUES ($1, $2, $3, $4, $5, NOW()) + `, "shared", "machine-a", "foreign-owner", "test-proj", "claude") + require.NoError(t, err, "insert foreign-owned shared session") + + sharedParent := "shared" + plainParent := "plain" + sessions := []db.Session{ + {ID: "shared", Project: "test-proj", Machine: "machine-b", + Agent: "claude", MessageCount: 1, CreatedAt: "2026-01-01T00:00:00Z"}, + {ID: "plain", Project: "test-proj", Machine: "machine-b", + Agent: "claude", MessageCount: 1, CreatedAt: "2026-01-01T00:00:00Z"}, + {ID: "child-shared", Project: "test-proj", Machine: "machine-b", + Agent: "claude", MessageCount: 1, CreatedAt: "2026-01-01T00:00:00Z", + SourceSessionID: "shared", ParentSessionID: &sharedParent}, + {ID: "child-plain", Project: "test-proj", Machine: "machine-b", + Agent: "claude", MessageCount: 1, CreatedAt: "2026-01-01T00:00:00Z", + ParentSessionID: &plainParent}, + } + for _, s := range sessions { + require.NoError(t, localDB.UpsertSession(s), "UpsertSession "+s.ID) + } + + // child-shared references the colliding "shared" id from a tool call too. + require.NoError(t, localDB.InsertMessages([]db.Message{{ + SessionID: "child-shared", Ordinal: 0, Role: "assistant", + Content: "spawning", HasToolUse: true, + ToolCalls: []db.ToolCall{{ + ToolName: "subagent", Category: "Task", + SubagentSessionID: "shared", + }}, + }}), "InsertMessages child-shared") + for _, id := range []string{"shared", "plain", "child-plain"} { + require.NoError(t, localDB.InsertMessages([]db.Message{{ + SessionID: id, Ordinal: 0, Role: "user", + Content: "hi", ContentLength: 2, + }}), "InsertMessages "+id) + } + + _, err = sync.Push(ctx, false, nil) + require.NoError(t, err, "Push") + + const prefixedShared = "machine-b~shared" + var n int + require.NoError(t, pg.QueryRowContext(ctx, + `SELECT COUNT(*) FROM sessions WHERE id = $1 AND machine = $2`, + prefixedShared, "machine-b").Scan(&n), "count prefixed shared") + assert.Equal(t, 1, n, "machine-b's shared session stored under prefixed id") + + var source, parent string + require.NoError(t, pg.QueryRowContext(ctx, + `SELECT source_session_id, parent_session_id + FROM sessions WHERE id = $1`, + "child-shared").Scan(&source, &parent), "read child-shared relations") + assert.Equal(t, prefixedShared, source, "source_session_id resolved") + assert.Equal(t, prefixedShared, parent, "parent_session_id resolved") + + var subagent string + require.NoError(t, pg.QueryRowContext(ctx, + `SELECT subagent_session_id FROM tool_calls WHERE session_id = $1`, + "child-shared").Scan(&subagent), "read child-shared subagent link") + assert.Equal(t, prefixedShared, subagent, "subagent_session_id resolved") + + var plainParentGot string + require.NoError(t, pg.QueryRowContext(ctx, + `SELECT parent_session_id FROM sessions WHERE id = $1`, + "child-plain").Scan(&plainParentGot), "read child-plain parent") + assert.Equal(t, "plain", plainParentGot, "non-colliding parent stays bare") +} + +// TestPushRepairsStaleSubagentLinkOnIncrementalPush verifies that an +// incremental push repairs a PG tool-call subagent link left at its unprefixed +// local id by a push that predated the collision: the parent's tool call was +// pushed while "sub-1" was still unclaimed, and the subagent session only +// later collided with a foreign owner and moved under "machine-b~sub-1". The +// local rows never change, so both the session candidacy fingerprint and the +// message fast path would otherwise skip the parent and never rewrite the +// link; only the resolved subagent id distinguishes it from PG. +func TestPushRepairsStaleSubagentLinkOnIncrementalPush(t *testing.T) { + pgURL := testPGURL(t) + + const schema = "agentsview_stale_subagent_repair_test" + pg, err := Open(pgURL, schema, true) + require.NoError(t, err, "Open") + defer pg.Close() + + ctx := context.Background() + _, err = pg.Exec(`DROP SCHEMA IF EXISTS ` + schema + ` CASCADE`) + require.NoError(t, err, "drop schema") + require.NoError(t, EnsureSchema(ctx, pg, schema), "EnsureSchema") + + localDB, err := db.Open(filepath.Join(t.TempDir(), "local.db")) + require.NoError(t, err, "db.Open") + defer localDB.Close() + + sync := &Sync{ + pg: pg, + local: localDB, + machine: "machine-b", + schema: schema, + schemaDone: true, + } + + // The parent references "sub-1" before any session by that id exists in + // PG or locally, so the first push writes the tool-call link at its bare + // local id. + require.NoError(t, localDB.UpsertSession(db.Session{ + ID: "parent-1", Project: "proj", Machine: "machine-b", Agent: "claude", + MessageCount: 1, CreatedAt: "2026-01-01T00:00:00Z", + }), "UpsertSession parent-1") + require.NoError(t, localDB.InsertMessages([]db.Message{{ + SessionID: "parent-1", Ordinal: 0, Role: "assistant", + Content: "spawning", HasToolUse: true, + ToolCalls: []db.ToolCall{{ + ToolName: "subagent", Category: "Task", SubagentSessionID: "sub-1", + }}, + }}), "InsertMessages parent-1") + + _, err = sync.Push(ctx, false, nil) + require.NoError(t, err, "first Push") + + subagent := func() string { + var s string + require.NoError(t, pg.QueryRowContext(ctx, + `SELECT subagent_session_id FROM tool_calls WHERE session_id = $1`, + "parent-1").Scan(&s), "read parent-1 subagent link") + return s + } + require.Equal(t, "sub-1", subagent(), + "first push keeps the bare link while the id is unclaimed") + + // A foreign machine claims the bare "sub-1" id, then machine-b's subagent + // session appears locally and is pushed under the "machine-b~sub-1" + // prefix. The parent's local row is unchanged, so its PG link now points + // at the foreign machine's session. + _, err = pg.ExecContext(ctx, ` + INSERT INTO sessions ( + id, machine, owner_marker, project, agent, created_at + ) VALUES ($1, $2, $3, $4, $5, NOW()) + `, "sub-1", "machine-a", "foreign-owner", "proj", "claude") + require.NoError(t, err, "insert foreign-owned subagent session") + + require.NoError(t, localDB.UpsertSession(db.Session{ + ID: "sub-1", Project: "proj", Machine: "machine-b", Agent: "claude", + MessageCount: 1, CreatedAt: "2026-01-01T00:00:00Z", + }), "UpsertSession sub-1") + require.NoError(t, localDB.InsertMessages([]db.Message{{ + SessionID: "sub-1", Ordinal: 0, Role: "user", Content: "hi", ContentLength: 2, + }}), "InsertMessages sub-1") + + const prefixedSub = "machine-b~sub-1" + _, err = sync.Push(ctx, false, nil) + require.NoError(t, err, "second Push") + var n int + require.NoError(t, pg.QueryRowContext(ctx, + `SELECT COUNT(*) FROM sessions WHERE id = $1 AND machine = $2`, + prefixedSub, "machine-b").Scan(&n), "count prefixed sub-1") + require.Equal(t, 1, n, "machine-b's subagent stored under prefixed id") + require.Equal(t, "sub-1", subagent(), + "precondition: parent was not a candidate, so its link is stale") + + // Re-list the parent without touching its message content, so only the + // resolved subagent id distinguishes it from what was last pushed. + require.NoError(t, localDB.BumpLocalModifiedAt("parent-1"), + "mark parent-1 modified") + + res, err := sync.Push(ctx, false, nil) + require.NoError(t, err, "third Push") + assert.Zero(t, res.Errors, "third push should report no failures") + assert.Equal(t, prefixedSub, subagent(), + "incremental push repairs the stale subagent link") +} + func TestPushSessionAdoptsLegacyLocalSentinelRow(t *testing.T) { pgURL := testPGURL(t) @@ -215,7 +1287,10 @@ func TestPushSessionAdoptsLegacyLocalSentinelRow(t *testing.T) { require.NoError(t, err, "BeginTx") markerID, err := sync.pushMarkerID() require.NoError(t, err, "pushMarkerID") - require.NoError(t, sync.pushSession(ctx, tx, sess, markerID, nil), "pushSession") + require.NoError(t, sync.pushSession(ctx, tx, sess, pushedSessionIdentity{ + ID: sess.ID, + Machine: "host-a", + }, markerID, nil), "pushSession") require.NoError(t, tx.Commit(), "Commit") var machine, ownerMarker string @@ -226,3 +1301,352 @@ func TestPushSessionAdoptsLegacyLocalSentinelRow(t *testing.T) { assert.Equal(t, "host-a", machine) assert.Equal(t, markerID, ownerMarker) } + +// TestResolveOwnedPushIDReusesLegacyPrefixAfterRename verifies that the shared +// id resolver (used by both resolvePushedSessionIdentity and +// relationshipResolver.lookup) reuses a row owned under a prior machine prefix. +// A pusher that once stored a colliding session under "old-host~id" and later +// renamed to "new-host" must resolve back to "old-host~id" instead of minting a +// duplicate "new-host~id". +func TestResolveOwnedPushIDReusesLegacyPrefixAfterRename(t *testing.T) { + pgURL := testPGURL(t) + + const schema = "agentsview_legacy_prefix_resolve_test" + pg, err := Open(pgURL, schema, true) + require.NoError(t, err, "Open") + defer pg.Close() + + ctx := context.Background() + _, err = pg.Exec(`DROP SCHEMA IF EXISTS ` + schema + ` CASCADE`) + require.NoError(t, err, "drop schema") + require.NoError(t, EnsureSchema(ctx, pg, schema), "EnsureSchema") + + localDB, err := db.Open(filepath.Join(t.TempDir(), "local.db")) + require.NoError(t, err, "db.Open") + defer localDB.Close() + + sync := &Sync{ + pg: pg, + local: localDB, + machine: "new-host", + schema: schema, + schemaDone: true, + } + markerID, err := sync.pushMarkerID() + require.NoError(t, err, "pushMarkerID") + + // A foreign machine owns the bare id, which is why this pusher stored its + // session under the old machine prefix before the rename. + _, err = pg.ExecContext(ctx, ` + INSERT INTO sessions (id, machine, owner_marker, project, agent, created_at) + VALUES ($1, $2, $3, $4, $5, NOW())`, + "sess-x", "foreign", "foreign-owner", "proj", "claude") + require.NoError(t, err, "insert foreign bare row") + _, err = pg.ExecContext(ctx, ` + INSERT INTO sessions (id, machine, owner_marker, project, agent, created_at) + VALUES ($1, $2, $3, $4, $5, NOW())`, + "old-host~sess-x", "old-host", markerID, "proj", "claude") + require.NoError(t, err, "insert owned legacy-prefixed row") + + identity := pushedSessionIdentity{Machine: "new-host"} + got, err := sync.resolveOwnedPushIdentityID( + ctx, "sess-x", identity, markerID, []string{"old-host"}, + ) + require.NoError(t, err) + assert.Equal(t, "old-host~sess-x", got, + "a renamed pusher must reuse the row it owns under the old machine prefix") + + // Without the old machine name, the resolver cannot find the owned row and + // would mint a duplicate under the new prefix -- the blind spot being fixed. + got, err = sync.resolveOwnedPushIdentityID(ctx, "sess-x", identity, markerID, nil) + require.NoError(t, err) + assert.Equal(t, "new-host~sess-x", got, + "precondition: without the legacy machine the resolver duplicates the row") +} + +// TestPushReusesLegacyPrefixedRowAfterRename verifies the end-to-end push: +// after a machine rename, a session previously stored under the old machine +// prefix is updated in place rather than duplicated under the new prefix. +func TestPushReusesLegacyPrefixedRowAfterRename(t *testing.T) { + pgURL := testPGURL(t) + + const schema = "agentsview_legacy_prefix_push_test" + pg, err := Open(pgURL, schema, true) + require.NoError(t, err, "Open") + defer pg.Close() + + ctx := context.Background() + _, err = pg.Exec(`DROP SCHEMA IF EXISTS ` + schema + ` CASCADE`) + require.NoError(t, err, "drop schema") + require.NoError(t, EnsureSchema(ctx, pg, schema), "EnsureSchema") + + localDB, err := db.Open(filepath.Join(t.TempDir(), "local.db")) + require.NoError(t, err, "db.Open") + defer localDB.Close() + + sync := &Sync{ + pg: pg, + local: localDB, + machine: "new-host", + schema: schema, + schemaDone: true, + } + markerID, err := sync.pushMarkerID() + require.NoError(t, err, "pushMarkerID") + + // Record that this marker last pushed as "old-host", so the rename push + // treats "old-host" as a legacy machine prefix. + _, err = pg.ExecContext(ctx, ` + INSERT INTO sync_metadata (key, value) VALUES ($1, $2) + ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value`, + pushMarkerKeyPrefix+markerID, "old-host") + require.NoError(t, err, "seed push marker machine") + + // A foreign machine owns the bare id; this pusher's session lives under the + // old machine prefix from before the rename. + _, err = pg.ExecContext(ctx, ` + INSERT INTO sessions (id, machine, owner_marker, project, agent, created_at) + VALUES ($1, $2, $3, $4, $5, NOW())`, + "sess-x", "foreign", "foreign-owner", "proj", "claude") + require.NoError(t, err, "insert foreign bare row") + _, err = pg.ExecContext(ctx, ` + INSERT INTO sessions (id, machine, owner_marker, project, agent, created_at) + VALUES ($1, $2, $3, $4, $5, NOW())`, + "old-host~sess-x", "old-host", markerID, "proj", "claude") + require.NoError(t, err, "insert owned legacy-prefixed row") + + sess := db.Session{ + ID: "sess-x", Project: "proj", Machine: "local", Agent: "claude", + MessageCount: 1, CreatedAt: "2026-01-01T00:00:00Z", + } + require.NoError(t, localDB.UpsertSession(sess), "UpsertSession") + require.NoError(t, localDB.InsertMessages([]db.Message{{ + SessionID: "sess-x", Ordinal: 0, Role: "user", Content: "hi", ContentLength: 2, + }}), "InsertMessages") + + _, err = sync.Push(ctx, false, nil) + require.NoError(t, err, "Push") + + var dup int + require.NoError(t, pg.QueryRowContext(ctx, + `SELECT COUNT(*) FROM sessions WHERE id = $1`, "new-host~sess-x").Scan(&dup), + "count new-prefix duplicate") + assert.Equal(t, 0, dup, "rename must not create a duplicate under the new prefix") + + // The legacy-prefixed row was updated in place: the machine column reflects + // the rename and the pushed message landed there. + var machine string + require.NoError(t, pg.QueryRowContext(ctx, + `SELECT machine FROM sessions WHERE id = $1`, "old-host~sess-x").Scan(&machine), + "read reused row machine") + assert.Equal(t, "new-host", machine, "owned legacy row updated in place") + var msgs int + require.NoError(t, pg.QueryRowContext(ctx, + `SELECT COUNT(*) FROM messages WHERE session_id = $1`, "old-host~sess-x").Scan(&msgs), + "count reused row messages") + assert.Equal(t, 1, msgs, "message pushed to the reused legacy row") + + var foreignOwner string + require.NoError(t, pg.QueryRowContext(ctx, + `SELECT owner_marker FROM sessions WHERE id = $1`, "sess-x").Scan(&foreignOwner), + "read foreign bare row") + assert.Equal(t, "foreign-owner", foreignOwner, "foreign bare row not adopted") +} + +// TestPushSkipsPrefixedConflictWithoutAbortingBatch verifies that a single +// per-session ownership conflict on the current-machine prefixed id is skipped +// and reported, while unrelated sessions in the same push still go through. The +// conflict must not fail the whole push from identity pre-resolution. +func TestPushSkipsPrefixedConflictWithoutAbortingBatch(t *testing.T) { + pgURL := testPGURL(t) + + const schema = "agentsview_prefixed_conflict_skip_test" + pg, err := Open(pgURL, schema, true) + require.NoError(t, err, "Open") + defer pg.Close() + + ctx := context.Background() + _, err = pg.Exec(`DROP SCHEMA IF EXISTS ` + schema + ` CASCADE`) + require.NoError(t, err, "drop schema") + require.NoError(t, EnsureSchema(ctx, pg, schema), "EnsureSchema") + + localDB, err := db.Open(filepath.Join(t.TempDir(), "local.db")) + require.NoError(t, err, "db.Open") + defer localDB.Close() + + sync := &Sync{ + pg: pg, + local: localDB, + machine: "new-host", + schema: schema, + schemaDone: true, + } + markerID, err := sync.pushMarkerID() + require.NoError(t, err, "pushMarkerID") + + // A different owner holds both the bare id and the current-machine prefixed + // id, so "conf-1" has nowhere to land and must be skipped as a conflict. + for _, id := range []string{"conf-1", "new-host~conf-1"} { + _, err = pg.ExecContext(ctx, ` + INSERT INTO sessions (id, machine, owner_marker, project, agent, created_at) + VALUES ($1, $2, $3, $4, $5, NOW())`, + id, "other-host", "other-owner", "proj", "claude") + require.NoError(t, err, "insert foreign "+id) + } + + for _, s := range []db.Session{ + {ID: "conf-1", Project: "proj", Machine: "new-host", Agent: "claude", + MessageCount: 1, CreatedAt: "2026-01-01T00:00:00Z"}, + {ID: "clean-1", Project: "proj", Machine: "new-host", Agent: "claude", + MessageCount: 1, CreatedAt: "2026-01-01T00:00:00Z"}, + } { + require.NoError(t, localDB.UpsertSession(s), "UpsertSession "+s.ID) + require.NoError(t, localDB.InsertMessages([]db.Message{{ + SessionID: s.ID, Ordinal: 0, Role: "user", Content: "hi", ContentLength: 2, + }}), "InsertMessages "+s.ID) + } + + res, err := sync.Push(ctx, false, nil) + require.NoError(t, err, "a per-session conflict must not fail the whole push") + assert.Equal(t, 1, res.SkippedConflicts, "the conflicting session is reported as skipped") + + var clean int + require.NoError(t, pg.QueryRowContext(ctx, + `SELECT COUNT(*) FROM sessions WHERE id = $1 AND owner_marker = $2`, + "clean-1", markerID).Scan(&clean), "count clean session") + assert.Equal(t, 1, clean, "unrelated session pushed despite the conflict") + + var owner string + require.NoError(t, pg.QueryRowContext(ctx, + `SELECT owner_marker FROM sessions WHERE id = $1`, "new-host~conf-1").Scan(&owner), + "read foreign prefixed row") + assert.Equal(t, "other-owner", owner, "foreign prefixed row not overwritten") +} + +func TestPushSkipsRelationshipsToPrefixedOwnershipConflict(t *testing.T) { + pgURL := testPGURL(t) + + const schema = "agentsview_prefixed_conflict_relationship_test" + pg, err := Open(pgURL, schema, true) + require.NoError(t, err, "Open") + defer pg.Close() + + ctx := context.Background() + _, err = pg.Exec(`DROP SCHEMA IF EXISTS ` + schema + ` CASCADE`) + require.NoError(t, err, "drop schema") + require.NoError(t, EnsureSchema(ctx, pg, schema), "EnsureSchema") + + localDB, err := db.Open(filepath.Join(t.TempDir(), "local.db")) + require.NoError(t, err, "db.Open") + defer localDB.Close() + + sync := &Sync{ + pg: pg, + local: localDB, + machine: "new-host", + schema: schema, + schemaDone: true, + } + + for _, id := range []string{"conf-1", "new-host~conf-1"} { + _, err = pg.ExecContext(ctx, ` + INSERT INTO sessions (id, machine, owner_marker, project, agent, created_at) + VALUES ($1, $2, $3, $4, $5, NOW())`, + id, "other-host", "other-owner", "proj", "claude") + require.NoError(t, err, "insert foreign "+id) + } + + sourceID := "conf-1" + for _, s := range []db.Session{ + {ID: "conf-1", Project: "proj", Machine: "new-host", Agent: "claude", + MessageCount: 1, CreatedAt: "2026-01-01T00:00:00Z"}, + {ID: "child-1", Project: "proj", Machine: "new-host", Agent: "claude", + SourceSessionID: sourceID, MessageCount: 1, CreatedAt: "2026-01-01T00:00:00Z"}, + } { + require.NoError(t, localDB.UpsertSession(s), "UpsertSession "+s.ID) + require.NoError(t, localDB.InsertMessages([]db.Message{{ + SessionID: s.ID, Ordinal: 0, Role: "user", Content: "hi", ContentLength: 2, + }}), "InsertMessages "+s.ID) + } + + res, err := sync.Push(ctx, false, nil) + require.NoError(t, err, "relationship to a per-session conflict must not fail the whole push") + assert.Equal(t, 2, res.SkippedConflicts, + "the conflicted session and the dependent relationship session are skipped") + assert.Zero(t, res.Errors) + + var childRows int + require.NoError(t, pg.QueryRowContext(ctx, + `SELECT COUNT(*) FROM sessions WHERE id = $1`, "child-1").Scan(&childRows), + "count child session") + assert.Equal(t, 0, childRows, "dependent session must not be pushed with a foreign source link") + + var foreignRefs int + require.NoError(t, pg.QueryRowContext(ctx, + `SELECT COUNT(*) FROM sessions WHERE source_session_id = $1`, + "new-host~conf-1").Scan(&foreignRefs), "count references to foreign prefixed row") + assert.Equal(t, 0, foreignRefs, "no pushed row may point at the foreign prefixed session") +} + +func TestPushSkipsRelationshipsToAlreadyPrefixedOwnershipConflict(t *testing.T) { + pgURL := testPGURL(t) + + const schema = "agentsview_already_prefixed_conflict_relationship_test" + pg, err := Open(pgURL, schema, true) + require.NoError(t, err, "Open") + defer pg.Close() + + ctx := context.Background() + _, err = pg.Exec(`DROP SCHEMA IF EXISTS ` + schema + ` CASCADE`) + require.NoError(t, err, "drop schema") + require.NoError(t, EnsureSchema(ctx, pg, schema), "EnsureSchema") + + localDB, err := db.Open(filepath.Join(t.TempDir(), "local.db")) + require.NoError(t, err, "db.Open") + defer localDB.Close() + + sync := &Sync{ + pg: pg, + local: localDB, + machine: "new-host", + schema: schema, + schemaDone: true, + } + + const conflictedID = "new-host~conf-1" + _, err = pg.ExecContext(ctx, ` + INSERT INTO sessions (id, machine, owner_marker, project, agent, created_at) + VALUES ($1, $2, $3, $4, $5, NOW())`, + conflictedID, "other-host", "other-owner", "proj", "claude") + require.NoError(t, err, "insert foreign already-prefixed row") + + for _, s := range []db.Session{ + {ID: conflictedID, Project: "proj", Machine: "new-host", Agent: "claude", + MessageCount: 1, CreatedAt: "2026-01-01T00:00:00Z"}, + {ID: "child-1", Project: "proj", Machine: "new-host", Agent: "claude", + SourceSessionID: conflictedID, MessageCount: 1, CreatedAt: "2026-01-01T00:00:00Z"}, + } { + require.NoError(t, localDB.UpsertSession(s), "UpsertSession "+s.ID) + require.NoError(t, localDB.InsertMessages([]db.Message{{ + SessionID: s.ID, Ordinal: 0, Role: "user", Content: "hi", ContentLength: 2, + }}), "InsertMessages "+s.ID) + } + + res, err := sync.Push(ctx, false, nil) + require.NoError(t, err, "already-prefixed relationship conflict must not fail the whole push") + assert.Equal(t, 2, res.SkippedConflicts, + "the conflicted already-prefixed session and its dependent are skipped") + assert.Zero(t, res.Errors) + + var childRows int + require.NoError(t, pg.QueryRowContext(ctx, + `SELECT COUNT(*) FROM sessions WHERE id = $1`, "child-1").Scan(&childRows), + "count child session") + assert.Equal(t, 0, childRows, "dependent session must not be pushed with a foreign source link") + + var foreignRefs int + require.NoError(t, pg.QueryRowContext(ctx, + `SELECT COUNT(*) FROM sessions WHERE source_session_id = $1`, + conflictedID).Scan(&foreignRefs), "count references to foreign already-prefixed row") + assert.Equal(t, 0, foreignRefs, "no pushed row may point at the foreign already-prefixed session") +} diff --git a/internal/postgres/curation.go b/internal/postgres/curation.go index 9d1b96ebf..1bb59fddd 100644 --- a/internal/postgres/curation.go +++ b/internal/postgres/curation.go @@ -3,6 +3,7 @@ package postgres import ( "context" "database/sql" + "errors" "fmt" "time" @@ -43,15 +44,20 @@ func (s *Store) StarSession(sessionID string) (bool, error) { } // UnstarSession removes a session star from the shared PG dashboard -// metadata. -func (s *Store) UnstarSession(sessionID string) error { - if _, err := s.pg.Exec( +// metadata and reports whether a row was removed. +func (s *Store) UnstarSession(sessionID string) (bool, error) { + res, err := s.pg.Exec( `DELETE FROM starred_sessions WHERE session_id = $1`, sessionID, - ); err != nil { - return fmt.Errorf("unstarring session %s: %w", sessionID, err) + ) + if err != nil { + return false, fmt.Errorf("unstarring session %s: %w", sessionID, err) } - return nil + n, err := res.RowsAffected() + if err != nil { + return false, fmt.Errorf("checking unstar result for %s: %w", sessionID, err) + } + return n > 0, nil } // ListStarredSessionIDs returns shared PG-starred session IDs. @@ -80,35 +86,60 @@ func (s *Store) ListStarredSessionIDs( // BulkStarSessions stars multiple existing sessions in one transaction. // Unknown session IDs are skipped. -func (s *Store) BulkStarSessions(sessionIDs []string) error { +func (s *Store) BulkStarSessions(sessionIDs []string) ([]string, error) { if len(sessionIDs) == 0 { - return nil + return nil, nil } tx, err := s.pg.Begin() if err != nil { - return fmt.Errorf("beginning star transaction: %w", err) + return nil, fmt.Errorf("beginning star transaction: %w", err) } defer func() { _ = tx.Rollback() }() - stmt, err := tx.Prepare(` + // Check existence separately from the insert so stale IDs are skipped and + // the caller learns which sessions were actually starred, mirroring the + // SQLite backend. + exists, err := tx.Prepare(`SELECT 1 FROM sessions WHERE id = $1`) + if err != nil { + return nil, fmt.Errorf("preparing existence statement: %w", err) + } + defer exists.Close() + insert, err := tx.Prepare(` INSERT INTO starred_sessions (session_id) - SELECT $1 WHERE EXISTS ( - SELECT 1 FROM sessions WHERE id = $1 - ) + VALUES ($1) ON CONFLICT (session_id) DO NOTHING`) if err != nil { - return fmt.Errorf("preparing star statement: %w", err) + return nil, fmt.Errorf("preparing star statement: %w", err) } - defer stmt.Close() + defer insert.Close() + starred := make([]string, 0, len(sessionIDs)) for _, id := range sessionIDs { - if _, err := stmt.Exec(id); err != nil { - return fmt.Errorf("starring session %s: %w", id, err) + var one int + switch err := exists.QueryRow(id).Scan(&one); { + case errors.Is(err, sql.ErrNoRows): + continue + case err != nil: + return nil, fmt.Errorf("checking session %s: %w", id, err) + } + res, err := insert.Exec(id) + if err != nil { + return nil, fmt.Errorf("starring session %s: %w", id, err) + } + rowsAffected, err := res.RowsAffected() + if err != nil { + return nil, fmt.Errorf("checking star insert result for %s: %w", id, err) + } + if rowsAffected > 0 { + starred = append(starred, id) } } - return tx.Commit() + if err := tx.Commit(); err != nil { + return nil, fmt.Errorf("committing star transaction: %w", err) + } + return starred, nil } // PinMessage creates or updates a shared PG pin for a message. PG diff --git a/internal/postgres/curation_pgtest_test.go b/internal/postgres/curation_pgtest_test.go index 784924c3a..c424903d5 100644 --- a/internal/postgres/curation_pgtest_test.go +++ b/internal/postgres/curation_pgtest_test.go @@ -69,9 +69,12 @@ func TestStoreStarsAndPins(t *testing.T) { ok, err = store.StarSession("missing") require.NoError(t, err, "StarSession missing") assert.False(t, ok, "StarSession missing") - require.NoError(t, store.BulkStarSessions( - []string{"cur-star-2", "missing"}, - ), "BulkStarSessions") + bulkStarred, err := store.BulkStarSessions([]string{"cur-star-2", "missing"}) + require.NoError(t, err, "BulkStarSessions") + assert.Equal(t, []string{"cur-star-2"}, bulkStarred, "starred ids returned") + bulkStarred, err = store.BulkStarSessions([]string{"cur-star-1", "cur-star-2"}) + require.NoError(t, err, "BulkStarSessions already starred") + assert.Empty(t, bulkStarred, "already-starred ids should not be returned") ids, err := store.ListStarredSessionIDs(ctx) require.NoError(t, err, "ListStarredSessionIDs") @@ -83,7 +86,12 @@ func TestStoreStarsAndPins(t *testing.T) { for _, id := range ids { assert.True(t, wantStars[id], "unexpected starred id %q in %v", id, ids) } - require.NoError(t, store.UnstarSession("cur-star-1"), "UnstarSession") + removed, err := store.UnstarSession("cur-star-1") + require.NoError(t, err, "UnstarSession") + require.True(t, removed, "UnstarSession") + removed, err = store.UnstarSession("cur-star-1") + require.NoError(t, err, "UnstarSession no-op") + require.False(t, removed, "UnstarSession no-op") ids, err = store.ListStarredSessionIDs(ctx) require.NoError(t, err, "ListStarredSessionIDs after unstar") require.Len(t, ids, 1) @@ -247,6 +255,109 @@ func TestPushPreservesMultiplePGPinsBySourceUUID(t *testing.T) { assert.Equal(t, 3, pin.Ordinal) } +func TestPushBackfillsLegacyPinSourceUUIDBeforeOrdinalShift(t *testing.T) { + pgURL := testPGURL(t) + cleanPGSchema(t, pgURL) + t.Cleanup(func() { cleanPGSchema(t, pgURL) }) + + local := testDB(t) + ps, err := New( + pgURL, "agentsview", local, + "curation-machine", true, + SyncOptions{}, + ) + require.NoError(t, err, "New sync") + defer ps.Close() + + ctx := context.Background() + require.NoError(t, ps.EnsureSchema(ctx), "EnsureSchema") + + sess := db.Session{ + ID: "pg-legacy-pin-shift", + Project: "proj-curation", + Machine: "local", + Agent: "codex", + MessageCount: 2, + UserMessageCount: 1, + CreatedAt: "2026-05-01T00:00:00Z", + } + require.NoError(t, local.UpsertSession(sess), "UpsertSession first") + require.NoError(t, local.InsertMessages([]db.Message{ + { + SessionID: "pg-legacy-pin-shift", + Ordinal: 0, + Role: "user", + Content: "question", + SourceUUID: "uuid-question", + }, + { + SessionID: "pg-legacy-pin-shift", + Ordinal: 1, + Role: "assistant", + Content: "answer", + SourceUUID: "uuid-answer", + }, + }), "InsertMessages first") + _, err = ps.Push(ctx, false, nil) + require.NoError(t, err, "Push first") + + store, err := NewStore(pgURL, "agentsview", true) + require.NoError(t, err, "NewStore") + defer store.Close() + + note := "legacy pin" + _, err = store.PinMessage("pg-legacy-pin-shift", 1, ¬e) + require.NoError(t, err, "PinMessage") + _, err = ps.pg.ExecContext(ctx, ` + UPDATE pinned_messages + SET source_uuid = '' + WHERE session_id = $1 AND message_id = $2`, + "pg-legacy-pin-shift", 1, + ) + require.NoError(t, err, "clear source_uuid to simulate legacy pin") + + sess.MessageCount = 3 + require.NoError(t, local.UpsertSession(sess), "UpsertSession second") + require.NoError(t, local.ReplaceSessionMessages( + "pg-legacy-pin-shift", + []db.Message{ + { + SessionID: "pg-legacy-pin-shift", + Ordinal: 0, + Role: "user", + Content: "question", + SourceUUID: "uuid-question", + }, + { + SessionID: "pg-legacy-pin-shift", + Ordinal: 1, + Role: "user", + Content: "[compact]", + SourceUUID: "uuid-boundary", + IsCompactBoundary: true, + }, + { + SessionID: "pg-legacy-pin-shift", + Ordinal: 2, + Role: "assistant", + Content: "answer", + SourceUUID: "uuid-answer", + }, + }, + ), "ReplaceSessionMessages") + + _, err = ps.Push(ctx, true, nil) + require.NoError(t, err, "Push rewrite") + + pins, err := store.ListPinnedMessages(ctx, "pg-legacy-pin-shift", "") + require.NoError(t, err, "ListPinnedMessages") + require.Len(t, pins, 1) + assert.Equal(t, int64(2), pins[0].MessageID) + assert.Equal(t, 2, pins[0].Ordinal) + require.NotNil(t, pins[0].Note) + assert.Equal(t, note, *pins[0].Note) +} + func TestReconcilePinnedMessagesPrefersCurrentTargetPin(t *testing.T) { pgURL := testPGURL(t) diff --git a/internal/postgres/integration_test.go b/internal/postgres/integration_test.go index 84e6f7b63..c905ca0c7 100644 --- a/internal/postgres/integration_test.go +++ b/internal/postgres/integration_test.go @@ -163,7 +163,7 @@ func TestPushSecretFindingsReportsChange(t *testing.T) { pushOnce := func() bool { tx, err := ps.pg.BeginTx(ctx, nil) require.NoError(t, err, "begin tx") - changed, err := ps.pushSecretFindings(ctx, tx, sessID) + changed, err := ps.pushSecretFindings(ctx, tx, sessID, sessID) if err != nil { _ = tx.Rollback() t.Fatalf("pushSecretFindings: %v", err) diff --git a/internal/postgres/messages.go b/internal/postgres/messages.go index 481bb8ed9..451500155 100644 --- a/internal/postgres/messages.go +++ b/internal/postgres/messages.go @@ -2,6 +2,8 @@ package postgres import ( "context" + "database/sql" + "errors" "fmt" "slices" "strings" @@ -249,6 +251,28 @@ func (s *Store) GetAllMessages( return msgs, nil } +// GetMessageForMetadataPin returns the stable message identity fields used by +// metadata pin events. PostgreSQL message IDs exposed through the API are +// ordinals, matching PinMessage. +func (s *Store) GetMessageForMetadataPin( + ctx context.Context, sessionID string, messageID int64, +) (*db.Message, error) { + var msg db.Message + err := s.pg.QueryRowContext(ctx, ` + SELECT ordinal, session_id, ordinal, COALESCE(source_uuid, '') + FROM messages + WHERE session_id = $1 AND ordinal = $2`, sessionID, messageID).Scan( + &msg.ID, &msg.SessionID, &msg.Ordinal, &msg.SourceUUID, + ) + if errors.Is(err, sql.ErrNoRows) { + return nil, nil + } + if err != nil { + return nil, fmt.Errorf("querying message metadata for pin: %w", err) + } + return &msg, nil +} + // SearchSession performs ILIKE substring search within a single // session's messages, returning matching ordinals. func (s *Store) SearchSession( diff --git a/internal/postgres/metadata.go b/internal/postgres/metadata.go new file mode 100644 index 000000000..6d052f618 --- /dev/null +++ b/internal/postgres/metadata.go @@ -0,0 +1,22 @@ +package postgres + +import ( + "context" + + "go.kenn.io/agentsview/internal/db" +) + +// ListMetadataConflicts returns no rows for PostgreSQL read mode because the +// local artifact metadata ledger is not part of the shared SQL mirror. +func (s *Store) ListMetadataConflicts( + context.Context, + []string, +) ([]db.MetadataConflict, error) { + return []db.MetadataConflict{}, nil +} + +// CountMetadataConflicts returns zero for PostgreSQL read mode because the local +// artifact metadata ledger is not part of the shared SQL mirror. +func (s *Store) CountMetadataConflicts(context.Context) (int, error) { + return 0, nil +} diff --git a/internal/postgres/name_source_pgtest_test.go b/internal/postgres/name_source_pgtest_test.go index dad3987a1..c8701dd74 100644 --- a/internal/postgres/name_source_pgtest_test.go +++ b/internal/postgres/name_source_pgtest_test.go @@ -62,7 +62,10 @@ func TestPushSessionNameRoundTrip(t *testing.T) { // Push via pushSession directly. tx, err := pg.BeginTx(ctx, nil) require.NoError(t, err, "BeginTx") - if err := sync.pushSession(ctx, tx, sess, markerID, nil); err != nil { + if err := sync.pushSession(ctx, tx, sess, pushedSessionIdentity{ + ID: sess.ID, + Machine: sess.Machine, + }, markerID, nil); err != nil { _ = tx.Rollback() t.Fatalf("pushSession: %v", err) } @@ -105,7 +108,10 @@ func TestPushSessionNameRoundTrip(t *testing.T) { tx2, err := pg.BeginTx(ctx, nil) require.NoError(t, err, "BeginTx (second)") - if err := sync.pushSession(ctx, tx2, sess, markerID, nil); err != nil { + if err := sync.pushSession(ctx, tx2, sess, pushedSessionIdentity{ + ID: sess.ID, + Machine: sess.Machine, + }, markerID, nil); err != nil { _ = tx2.Rollback() t.Fatalf("pushSession (second): %v", err) } diff --git a/internal/postgres/push.go b/internal/postgres/push.go index 7e6bcec85..11f7075fe 100644 --- a/internal/postgres/push.go +++ b/internal/postgres/push.go @@ -14,8 +14,10 @@ import ( "slices" "sort" "strings" + "sync" "time" + "go.kenn.io/agentsview/internal/artifact" "go.kenn.io/agentsview/internal/db" "go.kenn.io/agentsview/internal/export" ) @@ -24,6 +26,9 @@ const ( lastPushBoundaryStateKey = "last_push_boundary_state" lastPushTargetFingerprintKey = "pg_target_fingerprint_v1" sessionAliasBackfillStateKey = "pg_session_alias_backfill_v1" + artifactIdentityModeStateKey = "pg_artifact_identity_v1" + artifactOwnerMarkerPrefix = "artifact-origin:" + legacyArtifactIdentityMode = "legacy" ) // pushMarkerIDStateKey names the local sync-state entry holding this DB's @@ -37,6 +42,7 @@ const ( var errSessionOwnershipConflict = errors.New("session ownership conflict") var errSessionExcluded = errors.New("session excluded") +var errArtifactReplicaExists = errors.New("artifact replica already exists") type pushBoundaryState struct { Cutoff string `json:"cutoff"` @@ -167,6 +173,26 @@ func (s *Sync) Push( if err != nil { return result, err } + localArtifactOrigin, err := artifact.StoredOrigin(s.local) + if err != nil { + return result, err + } + artifactIdentityMode := currentArtifactIdentityMode(localArtifactOrigin) + storedArtifactIdentityMode, err := state.GetSyncState( + artifactIdentityModeStateKey, + ) + if err != nil { + return result, fmt.Errorf( + "reading %s: %w", artifactIdentityModeStateKey, err, + ) + } + if lastPush != "" && storedArtifactIdentityMode != artifactIdentityMode { + log.Printf( + "pgsync: artifact identity mode changed; forcing full push", + ) + lastPush = "" + full = true + } markerMachine, markerMachineAliases, markerExists, err := s.pgPushMarkerMachineState(ctx, markerID) if err != nil { return result, err @@ -281,6 +307,7 @@ func (s *Sync) Push( var priorFingerprints map[string]string sessionFingerprints := make(map[string]string, len(sessionByID)) + sessionIdentities := make(map[string]pushedSessionIdentity, len(sessionByID)) if !full { var bErr error priorFingerprints, _, _, bErr = readBoundaryAndFingerprints( @@ -322,8 +349,32 @@ func (s *Sync) Push( } } + artifactImportedSessions, err := artifact.ImportedSessionIDs( + s.local, mapKeys(sessionByID), + ) + if err != nil { + return result, err + } + ctx, err = s.preloadPGSessionOwners(ctx, s.pushIdentityOwnerCandidateIDs( + sessionByID, artifactImportedSessions, localArtifactOrigin, + markerID, legacyMarkerMachines, + )) + if err != nil { + return result, err + } + for id, sess := range sessionByID { + _, artifactImported := artifactImportedSessions[sess.ID] + identity, err := s.resolvePushedSessionIdentity( + ctx, sess, localArtifactOrigin, artifactImported, + markerID, legacyMarkerMachines, + ) + if err != nil { + return result, err + } + sessionIdentities[id] = identity + } if err := purgePGExcludedPushSessions( - ctx, s.pg, sessionByID, + ctx, s.pg, sessionByID, sessionIdentities, ); err != nil { return result, err } @@ -336,10 +387,14 @@ func (s *Sync) Push( "computing local usage event fingerprints: %w", err, ) } - // The fingerprint loop issues several local queries per candidate - // session; on a full push that covers every session and runs for - // minutes, so it reports its own progress phase rather than sitting - // silent until the first batch lands. + if err := s.markRelationshipConflicts( + ctx, sessionByID, sessionIdentities, markerID, legacyMarkerMachines, + ); err != nil { + return result, err + } + // Fingerprint preparation resolves relationship targets and reads local + // dependency state in bounded batches. Full pushes can spend minutes here, + // so report this phase rather than staying silent until writes begin. log.Printf("pgsync: computing push fingerprints for %d candidate session(s)", len(sessionByID)) reportPrepare := func(done int) { @@ -363,6 +418,36 @@ func (s *Sync) Push( return result, err } for _, id := range chunk { + sess := sessionByID[id] + identity := sessionIdentities[id] + prepared++ + if identity.Conflict { + if prepared%pushPrepareProgressStride == 0 { + reportPrepare(prepared) + } + continue + } + // Resolve relationship ids (source/parent) to the ids their target + // sessions are stored under in PG once every identity is known, so the + // fingerprint and the written row agree and child rows link correctly + // even when a target was pushed under a collision-avoidance prefix. + resolver := s.newRelationshipResolver( + sessionIdentities, identity, + markerID, legacyMarkerMachines, + ) + if sess.SourceSessionID != "" || sess.ParentSessionID != nil { + resolvedSource, err := resolver.resolve(ctx, sess.SourceSessionID) + if err != nil { + return result, err + } + sess.SourceSessionID = resolvedSource + resolvedParent, err := resolver.resolvePtr(ctx, sess.ParentSessionID) + if err != nil { + return result, err + } + sess.ParentSessionID = resolvedParent + sessionByID[id] = sess + } usageFP, usageKnown := usageFingerprints[id] dependencyFP, err := depState.dependencyFingerprint( s.local, id, usageFP, usageKnown, @@ -373,12 +458,14 @@ func (s *Sync) Push( id, err, ) } - sess := sessionByID[id] + subagentFP, err := resolver.resolvedSubagentLinkFingerprint(ctx, id) + if err != nil { + return result, err + } sessionFingerprints[id] = sessionPushFingerprint( - sess, pushedSessionMachine(sess, s.machine), - usageFP, markerID, dependencyFP, + sess, identity.ID, identity.Machine, + usageFP, identity.effectiveOwnerMarker(markerID), dependencyFP+subagentFP, ) - prepared++ if prepared%pushPrepareProgressStride == 0 { reportPrepare(prepared) } @@ -388,6 +475,9 @@ func (s *Sync) Push( if len(priorFingerprints) > 0 { for id := range sessionByID { + if sessionIdentities[id].Conflict { + continue + } if priorFingerprints[id] == sessionFingerprints[id] { delete(sessionByID, id) } @@ -438,6 +528,11 @@ func (s *Sync) Push( ); err != nil { return result, err } + if err := completeArtifactIdentityMode( + state, artifactIdentityMode, result, + ); err != nil { + return result, err + } result.Vectors, err = s.runVectorPushPhase(ctx, full, nil, onProgress) if err != nil { return result, err @@ -446,61 +541,32 @@ func (s *Sync) Push( return result, nil } - var pushed []db.Session - // Sessions whose individual retry also failed: their PG sessions/messages - // rows are stale or absent, so the vector phase must not push their newer - // local vectors ahead of them. - var failedSessions map[string]struct{} - const batchSize = 50 - for i := 0; i < len(sessions); i += batchSize { - end := min(i+batchSize, len(sessions)) - batch := sessions[i:end] - - batchResult, err := s.pushBatch( - ctx, batch, full, markerID, legacyMarkerMachines, - usageFingerprints, &pushed, - ) - if err != nil { - return result, err - } - if batchResult.ok { - result.SessionsPushed += batchResult.sessions - result.MessagesPushed += batchResult.messages - result.SkippedConflicts += batchResult.skippedConflicts - } else { - // Batch failed — retry each session individually - // so one bad session doesn't block the rest. - for _, sess := range batch { - sr, retryErr := s.pushBatch( - ctx, []db.Session{sess}, - full, markerID, legacyMarkerMachines, - usageFingerprints, &pushed, - ) - if retryErr != nil { - return result, retryErr - } - if sr.ok { - result.SessionsPushed += sr.sessions - result.MessagesPushed += sr.messages - result.SkippedConflicts += sr.skippedConflicts - } else { - result.Errors++ - if failedSessions == nil { - failedSessions = make(map[string]struct{}) - } - failedSessions[sess.ID] = struct{}{} - } - } - } - if onProgress != nil { - onProgress(PushProgress{ - SessionsDone: end, - SessionsTotal: len(sessions), - MessagesDone: result.MessagesPushed, - SkippedConflicts: result.SkippedConflicts, - Errors: result.Errors, - }) - } + sink := pgSessionSink{ + sync: s, + full: full, + markerID: markerID, + legacyMarkerMachines: legacyMarkerMachines, + sessionUsageFingerprints: usageFingerprints, + identities: sessionIdentities, + } + pushed, err := drainSessionBatches( + ctx, sessions, sink, &result, onProgress, + ) + if err != nil { + return result, err + } + // Sessions not written by the session phase include failed retries and + // ownership conflicts. Their vectors must not advance ahead of the + // sessions/messages rows they depend on. + failedSessions := make(map[string]struct{}, len(sessions)-len(pushed)) + for _, sess := range sessions { + failedSessions[sess.ID] = struct{}{} + } + for _, sess := range pushed { + delete(failedSessions, sess.ID) + } + if len(failedSessions) == 0 { + failedSessions = nil } if s.isFiltered() { @@ -543,6 +609,11 @@ func (s *Sync) Push( ); err != nil { return result, err } + if err := completeArtifactIdentityMode( + state, artifactIdentityMode, result, + ); err != nil { + return result, err + } result.Vectors, err = s.runVectorPushPhase(ctx, full, failedSessions, onProgress) if err != nil { return result, err @@ -628,6 +699,99 @@ func filterProjectIdentityObservations( return out } +// sessionBatchSink persists batches of sessions to a target during a push. +// Extracting this seam keeps the batch/retry/progress orchestration +// (drainSessionBatches) free of any SQL: PostgreSQL is one sink today, and the +// artifact exporter can become another without duplicating the loop. +type sessionBatchSink interface { + // writeBatch persists batch atomically and appends successfully written + // sessions to *pushed. It returns ok=false (and no error) when the batch + // failed for a reason the caller should recover from by retrying each + // session individually. A non-nil error is fatal and aborts the push. + writeBatch( + ctx context.Context, batch []db.Session, pushed *[]db.Session, + ) (batchResult, error) +} + +// pgSessionSink writes batches to PostgreSQL via Sync.pushBatch. It binds the +// per-push parameters (full mode, push marker identity) so the orchestration +// does not need to thread them through. +type pgSessionSink struct { + sync *Sync + full bool + markerID string + legacyMarkerMachines []string + sessionUsageFingerprints map[string]string + identities map[string]pushedSessionIdentity +} + +func (p pgSessionSink) writeBatch( + ctx context.Context, batch []db.Session, pushed *[]db.Session, +) (batchResult, error) { + return p.sync.pushBatch( + ctx, batch, p.full, p.markerID, p.legacyMarkerMachines, + p.sessionUsageFingerprints, pushed, p.identities, + ) +} + +// drainSessionBatches pushes sessions through sink in fixed-size batches, +// accumulating counts into result and reporting progress after each batch. +// When a batch fails (ok=false) it retries each session individually so one +// bad session does not block the rest; sessions that still fail are counted as +// errors. It returns the sessions successfully written, in push order. +func drainSessionBatches( + ctx context.Context, + sessions []db.Session, + sink sessionBatchSink, + result *PushResult, + onProgress func(PushProgress), +) ([]db.Session, error) { + var pushed []db.Session + const batchSize = 50 + for i := 0; i < len(sessions); i += batchSize { + end := min(i+batchSize, len(sessions)) + batch := sessions[i:end] + + batchResult, err := sink.writeBatch(ctx, batch, &pushed) + if err != nil { + return pushed, err + } + if batchResult.ok { + result.SessionsPushed += batchResult.sessions + result.MessagesPushed += batchResult.messages + result.SkippedConflicts += batchResult.skippedConflicts + } else { + // Batch failed — retry each session individually + // so one bad session doesn't block the rest. + for _, sess := range batch { + sr, retryErr := sink.writeBatch( + ctx, []db.Session{sess}, &pushed, + ) + if retryErr != nil { + return pushed, retryErr + } + if sr.ok { + result.SessionsPushed += sr.sessions + result.MessagesPushed += sr.messages + result.SkippedConflicts += sr.skippedConflicts + } else { + result.Errors++ + } + } + } + if onProgress != nil { + onProgress(PushProgress{ + SessionsDone: end, + SessionsTotal: len(sessions), + MessagesDone: result.MessagesPushed, + SkippedConflicts: result.SkippedConflicts, + Errors: result.Errors, + }) + } + } + return pushed, nil +} + // pgPushMarkerMachineState reports whether this host's push marker is present // in PG and returns the current machine plus legacy machine aliases stored with // the marker. @@ -886,11 +1050,12 @@ func (s *Sync) pushBatch( legacyMarkerMachines []string, sessionUsageFingerprints map[string]string, pushed *[]db.Session, + identities map[string]pushedSessionIdentity, ) (batchResult, error) { preloadComparisons := len(batch) > 0 && !full result, err := s.pushBatchAttempt( ctx, batch, full, markerID, legacyMarkerMachines, - sessionUsageFingerprints, pushed, preloadComparisons, + sessionUsageFingerprints, pushed, identities, preloadComparisons, ) if err == nil || !errors.Is(err, errPushComparisonPreload) { return result, err @@ -902,7 +1067,7 @@ func (s *Sync) pushBatch( ) return s.pushBatchAttempt( ctx, batch, full, markerID, legacyMarkerMachines, - sessionUsageFingerprints, pushed, false, + sessionUsageFingerprints, pushed, identities, false, ) } @@ -914,6 +1079,7 @@ func (s *Sync) pushBatchAttempt( legacyMarkerMachines []string, sessionUsageFingerprints map[string]string, pushed *[]db.Session, + identities map[string]pushedSessionIdentity, preloadComparisons bool, ) (batchResult, error) { tx, err := s.pg.BeginTx(ctx, nil) @@ -928,7 +1094,17 @@ func (s *Sync) pushBatchAttempt( skippedConflicts := 0 sessionIDs := make([]string, 0, len(batch)) for _, sess := range batch { - sessionIDs = append(sessionIDs, sess.ID) + identity := identities[sess.ID] + if identity.Conflict { + continue + } + if identity.ID == "" { + identity = pushedSessionIdentity{ + ID: sess.ID, + Machine: pushedSessionMachine(sess, s.machine), + } + } + sessionIDs = append(sessionIDs, identity.ID) } comparisons := (*pushMessageComparison)(nil) if preloadComparisons && len(sessionIDs) > 0 { @@ -945,9 +1121,23 @@ func (s *Sync) pushBatchAttempt( } for _, sess := range batch { + identity := identities[sess.ID] + if identity.Conflict { + skippedConflicts++ + continue + } + if identity.ID == "" { + identity = pushedSessionIdentity{ + ID: sess.ID, + Machine: pushedSessionMachine(sess, s.machine), + } + } if err := s.pushSession( - ctx, tx, sess, markerID, legacyMarkerMachines, + ctx, tx, sess, identity, markerID, legacyMarkerMachines, ); err != nil { + if errors.Is(err, errArtifactReplicaExists) { + continue + } if errors.Is(err, errSessionOwnershipConflict) { skippedConflicts++ continue @@ -964,8 +1154,11 @@ func (s *Sync) pushBatchAttempt( return batchResult{}, nil } + resolver := s.newRelationshipResolver( + identities, identity, markerID, legacyMarkerMachines, + ) msgCount, err := s.pushMessages( - ctx, tx, sess.ID, full, + ctx, tx, sess.ID, identity.ID, resolver, full, sessionUsageFingerprints, comparisons, ) if err != nil { @@ -978,7 +1171,9 @@ func (s *Sync) pushBatchAttempt( return batchResult{}, nil } - findingsChanged, err := s.pushSecretFindings(ctx, tx, sess.ID) + findingsChanged, err := s.pushSecretFindings( + ctx, tx, sess.ID, identity.ID, + ) if err != nil { log.Printf( "pgsync: secret findings %s: %v", @@ -998,11 +1193,11 @@ func (s *Sync) pushBatchAttempt( UPDATE sessions SET updated_at = NOW() WHERE id = $1`, - sess.ID, + identity.ID, ); err != nil { log.Printf( "pgsync: bumping updated_at %s: %v", - sess.ID, err, + identity.ID, err, ) _ = tx.Rollback() *pushed = (*pushed)[:len(*pushed)-n] @@ -1154,6 +1349,25 @@ func completeSessionAliasBackfill( return markSessionAliasBackfillDone(local) } +func currentArtifactIdentityMode(origin string) string { + if origin == "" { + return legacyArtifactIdentityMode + } + return artifactOwnerMarkerPrefix + origin +} + +func completeArtifactIdentityMode( + local syncStateStore, mode string, result PushResult, +) error { + if result.Errors > 0 { + return nil + } + if err := local.SetSyncState(artifactIdentityModeStateKey, mode); err != nil { + return fmt.Errorf("updating %s: %w", artifactIdentityModeStateKey, err) + } + return nil +} + func persistPushTargetFingerprint( local syncStateStore, fingerprint string, @@ -1367,12 +1581,19 @@ func pgExcludedSessionIDsQuery(ids []string) (string, []any) { } func purgePGExcludedPushSessions( - ctx context.Context, pg *sql.DB, sessionByID map[string]db.Session, + ctx context.Context, + pg *sql.DB, + sessionByID map[string]db.Session, + identities map[string]pushedSessionIdentity, ) error { tombstoneIDsBySession := make(map[string][]string, len(sessionByID)) candidateIDs := []string{} for id, sess := range sessionByID { - tombstoneIDs := pgSessionTombstoneIDs(sess) + identity := identities[id] + if identity.Conflict || identity.ID == "" { + continue + } + tombstoneIDs := pgSessionTombstoneIDsForPushedID(sess, identity.ID) tombstoneIDsBySession[id] = tombstoneIDs candidateIDs = append(candidateIDs, tombstoneIDs...) } @@ -1437,9 +1658,9 @@ func deletePGExcludedSessionRows( } func deletePGSessionIfExcluded( - ctx context.Context, tx *sql.Tx, sess db.Session, + ctx context.Context, tx *sql.Tx, sess db.Session, pushedID string, ) (bool, error) { - ids := pgSessionTombstoneIDs(sess) + ids := pgSessionTombstoneIDsForPushedID(sess, pushedID) excluded, err := readPGExcludedSessionIDs(ctx, tx, ids) if err != nil { return false, err @@ -1456,17 +1677,25 @@ func deletePGSessionIfExcluded( return true, nil } +func pgSessionTombstoneIDsForPushedID(sess db.Session, pushedID string) []string { + if pushedID != "" { + sess.ID = pushedID + } + return pgSessionTombstoneIDs(sess) +} + // sessionPushFingerprint builds the change-detection fingerprint for a -// session. pushedMachine is the value pushSession actually writes to PG -// (pushedSessionMachine), not the raw sess.Machine: a "local"/empty sentinel -// row is written under the fallback machine, so the fingerprint must track the -// fallback to force a re-push when s.machine changes. +// session. pushedID and pushedMachine are the values pushSession actually +// writes to PG. They may differ from sess.ID/sess.Machine when a native +// session ID collides across machines or a "local"/empty sentinel row is +// written under the fallback machine, so the fingerprint must track them to +// force a re-push when the resolved PG identity changes. func sessionPushFingerprint( - sess db.Session, pushedMachine, + sess db.Session, pushedID, pushedMachine, usageEventFingerprint, ownerMarker, dependencyFingerprint string, ) string { fields := []string{ - sess.ID, + pushedID, sess.Project, pushedMachine, ownerMarker, @@ -1527,21 +1756,681 @@ func sessionPushFingerprint( sess.SecretsRulesVersion, usageEventFingerprint, } - var b strings.Builder - for _, f := range fields { - fmt.Fprintf(&b, "%d:%s", len(f), f) + var b strings.Builder + for _, f := range fields { + fmt.Fprintf(&b, "%d:%s", len(f), f) + } + return b.String() +} + +// pushedSessionMachine resolves the machine field for a PG row. Old rows +// pushed before this fix with machine="local" will be repaired gradually as +// each session is modified (message count change, etc.) and re-fingerprinted. +func pushedSessionMachine(sess db.Session, fallbackMachine string) string { + if sess.Machine != "" && sess.Machine != "local" { + return sess.Machine + } + return fallbackMachine +} + +func artifactPushIdentity( + sess db.Session, localOrigin string, artifactImported bool, +) (id, machine, ownerMarker string, ok bool) { + origin := "" + switch { + case localOrigin != "" && (sess.Machine == "" || sess.Machine == "local"): + origin = localOrigin + case localOrigin != "" && artifactImported && + sess.Machine != "" && sess.Machine != "local" && + strings.HasPrefix(sess.ID, sess.Machine+"~"): + origin = sess.Machine + default: + return "", "", "", false + } + return prefixedSessionID(origin, sess.ID), origin, + artifactOwnerMarkerPrefix + origin, true +} + +type pushedSessionIdentity struct { + ID string + Machine string + OwnerMarker string + LegacyOwnerMarkers []string + ArtifactReplica bool + AliasIDs []string + LegacyDuplicateID string + Conflict bool +} + +func (i pushedSessionIdentity) effectiveOwnerMarker(fallback string) string { + if i.OwnerMarker != "" { + return i.OwnerMarker + } + return fallback +} + +func (s *Sync) markRelationshipConflicts( + ctx context.Context, + sessionByID map[string]db.Session, + identities map[string]pushedSessionIdentity, + markerID string, + legacyMarkerMachines []string, +) error { + changed := true + for changed { + changed = false + for id, sess := range sessionByID { + identity := identities[id] + if identity.Conflict { + continue + } + resolver := s.newRelationshipResolver( + identities, identity, markerID, legacyMarkerMachines, + ) + if sess.SourceSessionID != "" { + if _, err := resolver.resolve(ctx, sess.SourceSessionID); err != nil { + if errors.Is(err, errSessionOwnershipConflict) { + identity.Conflict = true + identities[id] = identity + changed = true + continue + } + return err + } + } + if sess.ParentSessionID != nil && *sess.ParentSessionID != "" { + if _, err := resolver.resolve(ctx, *sess.ParentSessionID); err != nil { + if errors.Is(err, errSessionOwnershipConflict) { + identity.Conflict = true + identities[id] = identity + changed = true + continue + } + return err + } + } + if _, err := resolver.sessionNeedsSubagentRewrite(ctx, id); err != nil { + if errors.Is(err, errSessionOwnershipConflict) { + identity.Conflict = true + identities[id] = identity + changed = true + continue + } + return err + } + } + } + return nil +} + +func initialPushedSessionIdentity( + sess db.Session, + fallbackMachine string, + localArtifactOrigin string, + artifactImported bool, + markerID string, +) (pushedSessionIdentity, bool) { + identity := pushedSessionIdentity{ + ID: sess.ID, + Machine: pushedSessionMachine(sess, fallbackMachine), + } + if id, machine, ownerMarker, ok := artifactPushIdentity( + sess, localArtifactOrigin, artifactImported, + ); ok { + identity.ID = id + identity.Machine = machine + identity.OwnerMarker = ownerMarker + identity.LegacyOwnerMarkers = []string{markerID} + identity.ArtifactReplica = artifactImported + identity.AliasIDs = artifactPushAliasIDs(sess, id, machine) + return identity, true + } + return identity, false +} + +// resolvePushedSessionIdentity decides the PG id a local session is stored +// under. A session this sync owns -- by matching push marker, or an adoptable +// legacy/ownerless row (see sameSessionOwner) -- is updated in place: an +// existing row under the current or any prior machine prefix is reused, so +// machine renames and marker adoption keep updating the same row instead of +// creating a duplicate. Only a bare id already held by a different owner +// collides; that session is stored under the current machine prefix so both +// rows coexist instead of ping-ponging (issue 655). A collision is not rejected +// here: pushSession skips the conflicting row, so one conflicting session never +// fails the whole push. +func (s *Sync) resolvePushedSessionIdentity( + ctx context.Context, + sess db.Session, + localArtifactOrigin string, + artifactImported bool, + markerID string, + legacyMarkerMachines []string, +) (pushedSessionIdentity, error) { + identity, artifactIdentity := initialPushedSessionIdentity( + sess, s.machine, localArtifactOrigin, artifactImported, markerID, + ) + canonicalID := identity.ID + id, err := s.resolveOwnedPushIdentityID( + ctx, identity.ID, identity, markerID, legacyMarkerMachines, + ) + if err != nil { + if errors.Is(err, errSessionOwnershipConflict) { + identity.ID = "" + identity.Conflict = true + return identity, nil + } + return pushedSessionIdentity{}, err + } + identity.ID = id + if artifactIdentity && id == canonicalID { + legacyDuplicateID, conflict, resolveErr := + s.artifactLegacyDuplicateCandidate( + ctx, canonicalID, identity, markerID, + ) + if resolveErr != nil { + return pushedSessionIdentity{}, resolveErr + } + if conflict { + identity.ID = "" + identity.Conflict = true + return identity, nil + } + identity.LegacyDuplicateID = legacyDuplicateID + } + return identity, nil +} + +func (s *Sync) pushIdentityOwnerCandidateIDs( + sessionByID map[string]db.Session, + artifactImportedSessions map[string]struct{}, + localArtifactOrigin string, + markerID string, + legacyMarkerMachines []string, +) []string { + ids := make(map[string]struct{}, len(sessionByID)*3) + for _, sess := range sessionByID { + _, artifactImported := artifactImportedSessions[sess.ID] + identity, _ := initialPushedSessionIdentity( + sess, s.machine, localArtifactOrigin, artifactImported, markerID, + ) + ids[identity.ID] = struct{}{} + for _, machine := range pushIDMachinePrefixes( + identity.Machine, legacyMarkerMachines, + ) { + candidateID := prefixedSessionID(machine, identity.ID) + if candidateID != identity.ID { + ids[candidateID] = struct{}{} + } + } + for _, aliasID := range uniqueNonEmptyStrings(identity.AliasIDs) { + ids[aliasID] = struct{}{} + } + } + result := make([]string, 0, len(ids)) + for id := range ids { + if id != "" { + result = append(result, id) + } + } + sort.Strings(result) + return result +} + +// artifactLegacyDuplicateCandidate recognizes the narrow upgrade state where +// an importer already created the stable artifact id while this origin still +// owns its pre-artifact bare row. Both rows must already have the exact owners +// expected for that history. A foreign-owned bare alias is an unrelated id +// collision and is ignored; only a current-marker alias with invalid canonical +// ownership is surfaced as a conflict rather than guessed safe to merge. +func (s *Sync) artifactLegacyDuplicateCandidate( + ctx context.Context, + canonicalID string, + identity pushedSessionIdentity, + markerID string, +) (legacyDuplicateID string, conflict bool, err error) { + if canonicalID == "" || identity.OwnerMarker == "" || markerID == "" { + return "", false, nil + } + canonicalMachine, canonicalOwnerMarker, canonicalExists, canonicalErr := + s.pgSessionOwner(ctx, canonicalID) + if canonicalErr != nil { + return "", false, canonicalErr + } + if !canonicalExists { + return "", false, nil + } + for _, aliasID := range uniqueNonEmptyStrings(identity.AliasIDs) { + if aliasID == canonicalID { + continue + } + _, aliasOwnerMarker, aliasExists, aliasErr := s.pgSessionOwner( + ctx, aliasID, + ) + if aliasErr != nil { + return "", false, aliasErr + } + if !aliasExists { + continue + } + if aliasOwnerMarker != markerID { + continue + } + if canonicalMachine != identity.Machine || + canonicalOwnerMarker != identity.OwnerMarker { + return "", true, nil + } + return aliasID, false, nil + } + return "", false, nil +} + +func (s *Sync) resolveOwnedPushIdentityID( + ctx context.Context, + localID string, + identity pushedSessionIdentity, + markerID string, + legacyMarkerMachines []string, +) (string, error) { + machine := identity.Machine + currentPrefixedID := prefixedSessionID(machine, localID) + currentPrefixConflict := false + for _, candidate := range pushIDMachinePrefixes(machine, legacyMarkerMachines) { + prefixedID := prefixedSessionID(candidate, localID) + if prefixedID == localID { + continue + } + existingMachine, ownerMarker, ok, err := s.pgSessionOwner(ctx, prefixedID) + if err != nil { + return "", err + } + if ok && samePushedSessionOwner( + ownerMarker, existingMachine, identity, markerID, legacyMarkerMachines, + ) { + return prefixedID, nil + } + if ok && prefixedID == currentPrefixedID { + currentPrefixConflict = true + } + } + existingMachine, ownerMarker, ok, err := s.pgSessionOwner(ctx, localID) + if err != nil { + return "", err + } + if ok && samePushedSessionOwner( + ownerMarker, existingMachine, identity, markerID, legacyMarkerMachines, + ) { + return localID, nil + } + for _, aliasID := range uniqueNonEmptyStrings(identity.AliasIDs) { + if aliasID == localID { + continue + } + aliasMachine, aliasOwnerMarker, aliasExists, aliasErr := s.pgSessionOwner( + ctx, aliasID, + ) + if aliasErr != nil { + return "", aliasErr + } + if aliasExists && samePushedSessionOwner( + aliasOwnerMarker, aliasMachine, identity, + markerID, legacyMarkerMachines, + ) { + return aliasID, nil + } + } + if ok && !samePushedSessionOwner( + ownerMarker, existingMachine, identity, markerID, legacyMarkerMachines, + ) { + if localID == currentPrefixedID || currentPrefixConflict { + return "", errSessionOwnershipConflict + } + return prefixedSessionID(machine, localID), nil + } + return localID, nil +} + +func artifactPushAliasIDs( + sess db.Session, canonicalID, origin string, +) []string { + aliasID := sess.ID + if sess.Machine != "" && sess.Machine != "local" { + aliasID = strings.TrimPrefix(sess.ID, origin+"~") + } + if aliasID == "" || aliasID == canonicalID { + return nil + } + return []string{aliasID} +} + +func artifactCanonicalAliasIDs(origin, canonicalID string) []string { + prefix := origin + "~" + if origin == "" || !strings.HasPrefix(canonicalID, prefix) { + return nil + } + aliasID := strings.TrimPrefix(canonicalID, prefix) + if aliasID == "" || aliasID == canonicalID { + return nil + } + return []string{aliasID} +} + +// pushIDMachinePrefixes lists the machine names whose id prefixes identify rows +// this owner may already hold: the current machine first, then prior machine +// names (legacy marker machines) the same marker pushed under before a rename. +// The current machine is not repeated when it also appears in the legacy set. +func pushIDMachinePrefixes(machine string, legacyMarkerMachines []string) []string { + prefixes := make([]string, 0, len(legacyMarkerMachines)+1) + if machine != "" { + prefixes = append(prefixes, machine) + } + for _, m := range legacyMarkerMachines { + if m == machine || m == "" { + continue + } + prefixes = append(prefixes, m) + } + return prefixes +} + +type pgSessionOwnerRecord struct { + machine string + ownerMarker string + exists bool +} + +type pgSessionOwnerCache struct { + mu sync.Mutex + entries map[string]pgSessionOwnerRecord +} + +type pgSessionOwnerCacheContextKey struct{} + +func (c *pgSessionOwnerCache) get(id string) (pgSessionOwnerRecord, bool) { + c.mu.Lock() + defer c.mu.Unlock() + record, ok := c.entries[id] + return record, ok +} + +func (c *pgSessionOwnerCache) put(id string, record pgSessionOwnerRecord) { + c.mu.Lock() + defer c.mu.Unlock() + c.entries[id] = record +} + +// preloadPGSessionOwners resolves a candidate set in one PG round trip and +// records both hits and misses. pgSessionOwner reuses this cache and memoizes +// any relationship targets discovered later in the same push. +func (s *Sync) preloadPGSessionOwners( + ctx context.Context, ids []string, +) (context.Context, error) { + unique := uniqueNonEmptyStrings(ids) + cache := &pgSessionOwnerCache{ + entries: make(map[string]pgSessionOwnerRecord, len(unique)), + } + for _, id := range unique { + cache.entries[id] = pgSessionOwnerRecord{} + } + if len(unique) == 0 { + return context.WithValue(ctx, pgSessionOwnerCacheContextKey{}, cache), nil + } + rows, err := s.pg.QueryContext(ctx, ` + SELECT id, machine, owner_marker + FROM sessions + WHERE id = ANY($1)`, unique) + if err != nil { + return ctx, fmt.Errorf("preloading pg session owners: %w", err) + } + defer rows.Close() + for rows.Next() { + var id, machine string + var ownerMarker sql.NullString + if err := rows.Scan(&id, &machine, &ownerMarker); err != nil { + return ctx, fmt.Errorf("scanning pg session owner: %w", err) + } + cache.entries[id] = pgSessionOwnerRecord{ + machine: machine, ownerMarker: ownerMarker.String, exists: true, + } + } + if err := rows.Err(); err != nil { + return ctx, fmt.Errorf("iterating pg session owners: %w", err) + } + return context.WithValue(ctx, pgSessionOwnerCacheContextKey{}, cache), nil +} + +// pgSessionOwner returns the machine and owner_marker of a PG session row, and +// whether it exists. owner_marker is empty for legacy rows pushed before the +// marker model. +func (s *Sync) pgSessionOwner( + ctx context.Context, + id string, +) (string, string, bool, error) { + cache, _ := ctx.Value(pgSessionOwnerCacheContextKey{}).(*pgSessionOwnerCache) + if cache != nil { + if record, ok := cache.get(id); ok { + return record.machine, record.ownerMarker, record.exists, nil + } + } + var machine string + var ownerMarker sql.NullString + err := s.pg.QueryRowContext(ctx, + `SELECT machine, owner_marker FROM sessions WHERE id = $1`, + id, + ).Scan(&machine, &ownerMarker) + if err != nil { + if errors.Is(err, sql.ErrNoRows) { + if cache != nil { + cache.put(id, pgSessionOwnerRecord{}) + } + return "", "", false, nil + } + return "", "", false, fmt.Errorf( + "reading pg session owner for %s: %w", + id, err, + ) + } + if cache != nil { + cache.put(id, pgSessionOwnerRecord{ + machine: machine, ownerMarker: ownerMarker.String, exists: true, + }) + } + return machine, ownerMarker.String, true, nil +} + +func prefixedSessionID(machine, id string) string { + if machine == "" || id == "" { + return id + } + prefix := machine + "~" + if strings.HasPrefix(id, prefix) { + return id + } + return prefix + id +} + +// relationshipResolver maps the local session ids that appear in a session's +// relationship fields (source/parent) and in tool-call subagent links to the +// ids those target sessions are stored under in PG. When a target session was +// pushed under a collision-avoidance prefix (machine~id), the unprefixed local +// id would dangle or point at a different machine's session; resolving keeps +// child and subagent rows linked to the correct PG row. +// +// A child session and the sessions it references (parent, source, subagents) +// originate on the same machine, so the resolver is scoped to one machine and +// mirrors resolvePushedSessionIdentity's ownership rule. +type relationshipResolver struct { + sync *Sync + identities map[string]pushedSessionIdentity + identity pushedSessionIdentity + markerID string + legacyMarkerMachines []string + cache map[string]string +} + +func (s *Sync) newRelationshipResolver( + identities map[string]pushedSessionIdentity, + identity pushedSessionIdentity, + markerID string, + legacyMarkerMachines []string, +) relationshipResolver { + return relationshipResolver{ + sync: s, + identities: identities, + identity: identity, + markerID: markerID, + legacyMarkerMachines: legacyMarkerMachines, + cache: make(map[string]string), + } +} + +// resolve maps a local session id to the id it is stored under in PG. It +// prefers the in-run identity map (which already reflects collision prefixing +// for every session in this push) and falls back to committed PG state for +// targets outside the push window. +func (r relationshipResolver) resolve( + ctx context.Context, localID string, +) (string, error) { + if localID == "" { + return "", nil + } + if identity, ok := r.identities[localID]; ok { + if identity.Conflict { + return "", errSessionOwnershipConflict + } + if identity.ID != "" { + return identity.ID, nil + } + } + if resolved, ok := r.cache[localID]; ok { + return resolved, nil + } + resolved, err := r.lookup(ctx, localID) + if err != nil { + return "", err + } + r.cache[localID] = resolved + return resolved, nil +} + +// resolvePtr resolves a relationship id held behind a pointer, preserving nil +// and returning the original pointer when the value is unchanged. +func (r relationshipResolver) resolvePtr( + ctx context.Context, localID *string, +) (*string, error) { + if localID == nil || *localID == "" { + return localID, nil + } + resolved, err := r.resolve(ctx, *localID) + if err != nil { + return nil, err + } + if resolved == *localID { + return localID, nil + } + return &resolved, nil +} + +// lookup resolves an id absent from the in-run identity map by consulting +// committed PG state, sharing resolveOwnedPushIdentityID with identity +// resolution: it reuses a row owned under the current or any legacy machine +// prefix (so a target pushed before a rename still resolves), returns the +// current prefix when the bare id is held by another owner, and otherwise keeps +// the bare id. +func (r relationshipResolver) lookup( + ctx context.Context, localID string, +) (string, error) { + identity := r.identity + if r.identity.OwnerMarker != "" { + localID = prefixedSessionID(r.identity.Machine, localID) + identity.AliasIDs = artifactCanonicalAliasIDs( + r.identity.Machine, localID, + ) + } + return r.sync.resolveOwnedPushIdentityID( + ctx, localID, identity, r.markerID, r.legacyMarkerMachines, + ) +} + +// rewriteSubagentIDs resolves the subagent session link on each tool call and +// result event in msgs in place so they reference the PG ids of the subagent +// sessions rather than their unprefixed local ids. +func (r relationshipResolver) rewriteSubagentIDs( + ctx context.Context, msgs []db.Message, +) error { + for i := range msgs { + for j := range msgs[i].ToolCalls { + tc := &msgs[i].ToolCalls[j] + resolved, err := r.resolve(ctx, tc.SubagentSessionID) + if err != nil { + return err + } + tc.SubagentSessionID = resolved + for k := range tc.ResultEvents { + ev := &tc.ResultEvents[k] + resolvedEv, err := r.resolve(ctx, ev.SubagentSessionID) + if err != nil { + return err + } + ev.SubagentSessionID = resolvedEv + } + } + } + return nil +} + +// sessionNeedsSubagentRewrite reports whether any subagent link in the local +// session resolves to a different PG id than its stored local value. The push +// fast path compares local and PG fingerprints built from the unprefixed local +// ids, so a row already in PG with a stale unprefixed subagent id would match +// and skip the rewrite; this check forces message replacement in that case. +func (r relationshipResolver) sessionNeedsSubagentRewrite( + ctx context.Context, localSessionID string, +) (bool, error) { + ids, err := r.sync.local.SessionSubagentSessionIDs(localSessionID) + if err != nil { + return false, fmt.Errorf( + "reading subagent session ids for %s: %w", localSessionID, err, + ) + } + for _, id := range ids { + resolved, err := r.resolve(ctx, id) + if err != nil { + return false, err + } + if resolved != id { + return true, nil + } } - return b.String() + return false, nil } -// pushedSessionMachine resolves the machine field for a PG row. Old rows -// pushed before this fix with machine="local" will be repaired gradually as -// each session is modified (message count change, etc.) and re-fingerprinted. -func pushedSessionMachine(sess db.Session, fallbackMachine string) string { - if sess.Machine != "" && sess.Machine != "local" { - return sess.Machine +// resolvedSubagentLinkFingerprint fingerprints the PG ids the session's +// subagent links resolve to. The dependency fingerprint is built from the +// local rows, which keep their unprefixed ids, so a collision alias that +// appears after the session was last pushed would leave the stored fingerprint +// unchanged and the fast path would skip the session with its PG tool-call +// links stale; folding the resolved ids in forces that re-push. Sessions +// without subagent links contribute an empty string so their fingerprints are +// unaffected. +func (r relationshipResolver) resolvedSubagentLinkFingerprint( + ctx context.Context, localSessionID string, +) (string, error) { + ids, err := r.sync.local.SessionSubagentSessionIDs(localSessionID) + if err != nil { + return "", fmt.Errorf( + "reading subagent session ids for %s: %w", localSessionID, err, + ) } - return fallbackMachine + var b strings.Builder + for _, id := range ids { + resolved, err := r.resolve(ctx, id) + if err != nil { + return "", err + } + b.WriteString("\x1f") + b.WriteString(resolved) + } + return b.String(), nil } func sameSessionOwner( @@ -1563,6 +2452,29 @@ func sameSessionOwner( return existingMachine == pushedMachine } +func samePushedSessionOwner( + existingOwnerMarker, existingMachine string, + identity pushedSessionIdentity, + markerID string, + legacyMarkerMachines []string, +) bool { + if identity.OwnerMarker == "" { + return sameSessionOwner( + existingOwnerMarker, existingMachine, markerID, + identity.Machine, legacyMarkerMachines, + ) + } + if existingOwnerMarker != "" { + return existingOwnerMarker == identity.OwnerMarker || + slices.Contains(identity.LegacyOwnerMarkers, existingOwnerMarker) + } + // Ownerless artifact rows are legacy-compatible only when PG's marker + // history proves this pusher previously wrote under that machine name. + // Matching the artifact origin alone would let an importer adopt a row it + // did not create. + return slices.Contains(legacyMarkerMachines, existingMachine) +} + func stringValue(value *string) string { if value == nil { return "" @@ -1617,34 +2529,48 @@ func nilStrTS(s *string) any { // local-only and used solely by the sync engine to detect // re-parsed sessions. func (s *Sync) pushSession( - ctx context.Context, tx *sql.Tx, sess db.Session, markerID string, + ctx context.Context, tx *sql.Tx, sess db.Session, + identity pushedSessionIdentity, markerID string, legacyMarkerMachines []string, ) error { + if identity.LegacyDuplicateID != "" { + if err := verifyArtifactLegacyDuplicateOwnership( + ctx, tx, identity, markerID, + ); err != nil { + return err + } + } createdAt, _ := ParseSQLiteTimestamp(sess.CreatedAt) isAutomated := sess.IsAutomated - pushedMachine := pushedSessionMachine(sess, s.machine) + ownerMarker := identity.effectiveOwnerMarker(markerID) var existingMachine sql.NullString var existingOwnerMarker sql.NullString checkErr := tx.QueryRowContext(ctx, - `SELECT machine, owner_marker FROM sessions WHERE id = $1`, sess.ID, + `SELECT machine, owner_marker FROM sessions WHERE id = $1`, + identity.ID, ).Scan(&existingMachine, &existingOwnerMarker) if checkErr != nil && !errors.Is(checkErr, sql.ErrNoRows) { - return fmt.Errorf("checking session ownership %s: %w", sess.ID, checkErr) + return fmt.Errorf( + "checking session ownership %s: %w", identity.ID, checkErr, + ) } - if checkErr == nil && !sameSessionOwner( + if checkErr == nil && !samePushedSessionOwner( existingOwnerMarker.String, existingMachine.String, + identity, markerID, - pushedMachine, legacyMarkerMachines, ) { log.Printf( "pgsync: session %s: skipping — already owned by machine %q, "+ "this pusher is %q; sync from the origin machine to update", - sess.ID, existingMachine.String, pushedMachine, + identity.ID, existingMachine.String, identity.Machine, ) return errSessionOwnershipConflict } + if checkErr == nil && identity.ArtifactReplica { + return errArtifactReplicaExists + } if legacyMarkerMachines == nil { legacyMarkerMachines = []string{} } @@ -1652,6 +2578,10 @@ func (s *Sync) pushSession( if err != nil { return fmt.Errorf("encoding legacy marker machines: %w", err) } + legacyOwnerMarkersJSON, err := json.Marshal(identity.LegacyOwnerMarkers) + if err != nil { + return fmt.Errorf("encoding legacy owner markers: %w", err) + } result, err := tx.ExecContext(ctx, ` INSERT INTO sessions ( id, machine, owner_marker, project, agent, @@ -1779,7 +2709,10 @@ func (s *Sync) pushSession( SELECT jsonb_array_elements_text($59::jsonb) )) ) - OR sessions.owner_marker = EXCLUDED.owner_marker) + OR sessions.owner_marker = EXCLUDED.owner_marker + OR sessions.owner_marker IN ( + SELECT jsonb_array_elements_text($60::jsonb) + )) AND NOT EXISTS ( SELECT 1 FROM excluded_sessions WHERE id = EXCLUDED.id @@ -1840,7 +2773,7 @@ func (s *Sync) pushSession( OR sessions.duplicate_prompt_count IS DISTINCT FROM EXCLUDED.duplicate_prompt_count OR sessions.no_code_context_count IS DISTINCT FROM EXCLUDED.no_code_context_count OR sessions.runaway_tool_loop_count IS DISTINCT FROM EXCLUDED.runaway_tool_loop_count)`, - sess.ID, pushedMachine, markerID, + identity.ID, identity.Machine, ownerMarker, sanitizePG(sess.Project), sess.Agent, nilStr(sess.FirstMessage), @@ -1880,12 +2813,13 @@ func (s *Sync) pushSession( sess.NoCodeContextCount, sess.RunawayToolLoopCount, sanitizePG(sess.TranscriptFidelity), string(legacyMarkerMachinesJSON), + string(legacyOwnerMarkersJSON), ) if err != nil { return err } if rowsAffected, rowsErr := result.RowsAffected(); rowsErr == nil && rowsAffected == 0 { - excluded, excludedErr := deletePGSessionIfExcluded(ctx, tx, sess) + excluded, excludedErr := deletePGSessionIfExcluded(ctx, tx, sess, identity.ID) if excludedErr != nil { return excludedErr } @@ -1893,7 +2827,8 @@ func (s *Sync) pushSession( return errSessionExcluded } refreshErr := tx.QueryRowContext(ctx, - `SELECT machine, owner_marker FROM sessions WHERE id = $1`, sess.ID, + `SELECT machine, owner_marker FROM sessions WHERE id = $1`, + identity.ID, ).Scan(&existingMachine, &existingOwnerMarker) if refreshErr != nil { // The guarded upsert changed no rows and we cannot @@ -1907,30 +2842,234 @@ func (s *Sync) pushSession( sess.ID, refreshErr, ) } - if !sameSessionOwner( + if !samePushedSessionOwner( existingOwnerMarker.String, existingMachine.String, - markerID, pushedMachine, legacyMarkerMachines, + identity, markerID, legacyMarkerMachines, ) { log.Printf( "pgsync: session %s: skipping — already owned by machine %q, this pusher is %q; sync from the origin machine to update", - sess.ID, existingMachine.String, pushedMachine, + identity.ID, existingMachine.String, identity.Machine, ) return errSessionOwnershipConflict } } - excluded, excludedErr := deletePGSessionIfExcluded(ctx, tx, sess) + excluded, excludedErr := deletePGSessionIfExcluded(ctx, tx, sess, identity.ID) if excludedErr != nil { return excludedErr } if excluded { return errSessionExcluded } - if err := replacePGSessionAliases(ctx, tx, sess); err != nil { + if identity.LegacyDuplicateID != "" { + if err := consolidateArtifactLegacyDuplicate( + ctx, tx, identity, + ); err != nil { + return err + } + } + aliasSession := sess + aliasSession.ID = identity.ID + if err := replacePGSessionAliases(ctx, tx, aliasSession); err != nil { return err } return nil } +type pgSessionOwnership struct { + machine string + ownerMarker string +} + +// verifyArtifactLegacyDuplicateOwnership locks both sides of an artifact +// upgrade merge before pushSession changes either row. The canonical row must +// have the stable artifact owner and machine, while the bare row must still be +// owned by this pusher's exact pre-artifact random marker. +func verifyArtifactLegacyDuplicateOwnership( + ctx context.Context, + tx *sql.Tx, + identity pushedSessionIdentity, + markerID string, +) error { + if identity.ID == "" || identity.LegacyDuplicateID == "" || + identity.ID == identity.LegacyDuplicateID || + identity.OwnerMarker == "" || markerID == "" { + return fmt.Errorf( + "%w: invalid artifact duplicate consolidation identity", + errSessionOwnershipConflict, + ) + } + + rows, err := tx.QueryContext(ctx, ` + SELECT id, machine, owner_marker + FROM sessions + WHERE id IN ($1, $2) + ORDER BY id + FOR UPDATE + `, identity.ID, identity.LegacyDuplicateID) + if err != nil { + return fmt.Errorf( + "locking artifact duplicate sessions: %w", err, + ) + } + owners := make(map[string]pgSessionOwnership, 2) + for rows.Next() { + var id string + var owner pgSessionOwnership + if err := rows.Scan(&id, &owner.machine, &owner.ownerMarker); err != nil { + _ = rows.Close() + return fmt.Errorf( + "reading artifact duplicate ownership: %w", err, + ) + } + owners[id] = owner + } + if err := rows.Close(); err != nil { + return fmt.Errorf( + "closing artifact duplicate ownership rows: %w", err, + ) + } + if err := rows.Err(); err != nil { + return fmt.Errorf( + "reading artifact duplicate ownership rows: %w", err, + ) + } + + canonical, canonicalOK := owners[identity.ID] + legacy, legacyOK := owners[identity.LegacyDuplicateID] + if !canonicalOK || !legacyOK || + canonical.machine != identity.Machine || + canonical.ownerMarker != identity.OwnerMarker || + legacy.ownerMarker != markerID { + return fmt.Errorf( + "%w: artifact duplicate ownership changed before consolidation", + errSessionOwnershipConflict, + ) + } + return nil +} + +// consolidateArtifactLegacyDuplicate transfers PG-local state to the stable +// canonical artifact row, rewrites references that would otherwise dangle, +// and removes the proven legacy duplicate. The caller holds row locks from +// verifyArtifactLegacyDuplicateOwnership for the duration of this transaction. +func consolidateArtifactLegacyDuplicate( + ctx context.Context, + tx *sql.Tx, + identity pushedSessionIdentity, +) error { + result, err := tx.ExecContext(ctx, ` + UPDATE sessions AS canonical + SET display_name = CASE + WHEN legacy.display_name IS DISTINCT FROM + legacy.source_display_name + THEN legacy.display_name + ELSE canonical.display_name + END, + source_display_name = CASE + WHEN legacy.display_name IS DISTINCT FROM + legacy.source_display_name + THEN legacy.source_display_name + ELSE canonical.source_display_name + END, + deleted_at = CASE + WHEN legacy.deleted_at IS DISTINCT FROM + legacy.source_deleted_at + THEN legacy.deleted_at + ELSE canonical.deleted_at + END, + source_deleted_at = CASE + WHEN legacy.deleted_at IS DISTINCT FROM + legacy.source_deleted_at + THEN legacy.source_deleted_at + ELSE canonical.source_deleted_at + END + FROM sessions AS legacy + WHERE canonical.id = $1 AND legacy.id = $2 + `, identity.ID, identity.LegacyDuplicateID) + if err != nil { + return fmt.Errorf("copying artifact duplicate session curation: %w", err) + } + if rowsAffected, rowsErr := result.RowsAffected(); rowsErr != nil { + return fmt.Errorf("counting artifact duplicate curation rows: %w", rowsErr) + } else if rowsAffected != 1 { + return fmt.Errorf( + "copying artifact duplicate session curation affected %d rows", + rowsAffected, + ) + } + + if _, err := tx.ExecContext(ctx, ` + INSERT INTO starred_sessions (session_id, created_at) + SELECT $1, created_at + FROM starred_sessions + WHERE session_id = $2 + ON CONFLICT (session_id) DO UPDATE SET + created_at = EXCLUDED.created_at + `, identity.ID, identity.LegacyDuplicateID); err != nil { + return fmt.Errorf("copying artifact duplicate star: %w", err) + } + if _, err := tx.ExecContext(ctx, ` + INSERT INTO pinned_messages ( + session_id, message_id, ordinal, source_uuid, note, created_at + ) + SELECT $1, message_id, ordinal, source_uuid, note, created_at + FROM pinned_messages + WHERE session_id = $2 + ON CONFLICT (session_id, message_id) DO UPDATE SET + ordinal = EXCLUDED.ordinal, + source_uuid = EXCLUDED.source_uuid, + note = EXCLUDED.note, + created_at = EXCLUDED.created_at + `, identity.ID, identity.LegacyDuplicateID); err != nil { + return fmt.Errorf("copying artifact duplicate pins: %w", err) + } + + if _, err := tx.ExecContext(ctx, ` + UPDATE sessions + SET parent_session_id = $1 + WHERE parent_session_id = $2 + `, identity.ID, identity.LegacyDuplicateID); err != nil { + return fmt.Errorf("rewriting artifact duplicate parent links: %w", err) + } + if _, err := tx.ExecContext(ctx, ` + UPDATE sessions + SET source_session_id = $1 + WHERE source_session_id = $2 + `, identity.ID, identity.LegacyDuplicateID); err != nil { + return fmt.Errorf("rewriting artifact duplicate source links: %w", err) + } + if _, err := tx.ExecContext(ctx, ` + UPDATE tool_calls + SET subagent_session_id = $1 + WHERE subagent_session_id = $2 + `, identity.ID, identity.LegacyDuplicateID); err != nil { + return fmt.Errorf("rewriting artifact duplicate tool-call links: %w", err) + } + if _, err := tx.ExecContext(ctx, ` + UPDATE tool_result_events + SET subagent_session_id = $1 + WHERE subagent_session_id = $2 + `, identity.ID, identity.LegacyDuplicateID); err != nil { + return fmt.Errorf("rewriting artifact duplicate tool-result links: %w", err) + } + + result, err = tx.ExecContext(ctx, ` + DELETE FROM sessions WHERE id = $1 + `, identity.LegacyDuplicateID) + if err != nil { + return fmt.Errorf("deleting artifact legacy duplicate: %w", err) + } + if rowsAffected, rowsErr := result.RowsAffected(); rowsErr != nil { + return fmt.Errorf("counting deleted artifact duplicate rows: %w", rowsErr) + } else if rowsAffected != 1 { + return fmt.Errorf( + "deleting artifact legacy duplicate affected %d rows", + rowsAffected, + ) + } + return nil +} + // pushMessages replaces a session's messages and tool calls // in PG. It skips the replacement when the PG message count // already matches the local count, avoiding redundant work @@ -1938,12 +3077,14 @@ func (s *Sync) pushSession( func (s *Sync) pushMessages( ctx context.Context, tx *sql.Tx, - sessionID string, + localSessionID string, + pgSessionID string, + resolver relationshipResolver, full bool, sessionUsageFingerprints map[string]string, comparisons *pushMessageComparison, ) (int, error) { - localCount, err := s.local.MessageCount(sessionID) + localCount, err := s.local.MessageCount(localSessionID) if err != nil { return 0, fmt.Errorf( "counting local messages: %w", err, @@ -1952,7 +3093,7 @@ func (s *Sync) pushMessages( if localCount == 0 { if _, err := tx.ExecContext(ctx, `DELETE FROM tool_result_events WHERE session_id = $1`, - sessionID, + pgSessionID, ); err != nil { return 0, fmt.Errorf( "deleting stale pg tool_result_events: %w", err, @@ -1960,7 +3101,7 @@ func (s *Sync) pushMessages( } if _, err := tx.ExecContext(ctx, `DELETE FROM tool_calls WHERE session_id = $1`, - sessionID, + pgSessionID, ); err != nil { return 0, fmt.Errorf( "deleting stale pg tool_calls: %w", err, @@ -1968,7 +3109,7 @@ func (s *Sync) pushMessages( } if _, err := tx.ExecContext(ctx, `DELETE FROM messages WHERE session_id = $1`, - sessionID, + pgSessionID, ); err != nil { return 0, fmt.Errorf( "deleting stale pg messages: %w", err, @@ -1979,11 +3120,13 @@ func (s *Sync) pushMessages( // state.db-only session) with zero messages. Sync them here // too so their cost reaches PG instead of being dropped with // the rest of the message-replace path below. - if err := s.replaceUsageEvents(ctx, tx, sessionID); err != nil { + if err := s.replaceUsageEvents( + ctx, tx, localSessionID, pgSessionID, + ); err != nil { return 0, err } if err := reconcilePinnedMessages( - ctx, tx, sessionID, + ctx, tx, pgSessionID, ); err != nil { return 0, err } @@ -1991,7 +3134,7 @@ func (s *Sync) pushMessages( } pgAgg, pgToolAgg, hasPreloadedComparisons := comparisonAggregates( - sessionID, comparisons, + pgSessionID, comparisons, ) if !hasPreloadedComparisons { if err := tx.QueryRowContext(ctx, @@ -2006,7 +3149,7 @@ func (s *Sync) pushMessages( ) FROM messages WHERE session_id = $1`, - sessionID, + pgSessionID, ).Scan( &pgAgg.Count, &pgAgg.Sum, &pgAgg.Max, &pgAgg.Min, @@ -2021,7 +3164,7 @@ func (s *Sync) pushMessages( COALESCE(SUM(result_content_length), 0) FROM tool_calls WHERE session_id = $1`, - sessionID, + pgSessionID, ).Scan(&pgToolAgg.Count, &pgToolAgg.Sum); err != nil { return 0, fmt.Errorf( "counting pg tool_calls: %w", err, @@ -2030,182 +3173,198 @@ func (s *Sync) pushMessages( } if !full && pgAgg.Count == localCount && pgAgg.Count > 0 { - localFP := pushLocalMessageFingerprint{} - - localFP.Sum, localFP.Max, localFP.Min, err = s.local.MessageContentFingerprint( - sessionID, - ) - if err != nil { - return 0, fmt.Errorf( - "computing local content fingerprint: %w", - err, - ) - } - localFP.ContentHashFP, err = s.local.MessageContentHashFingerprint( - sessionID, - ) - if err != nil { - return 0, fmt.Errorf( - "computing local content hash fingerprint: %w", - err, - ) - } - localFP.RoleTimeFP, err = localMessageRoleTimePGFingerprint( - s.local, sessionID, - ) - if err != nil { - return 0, fmt.Errorf( - "computing local role/time fingerprint: %w", - err, - ) - } - localFP.FlagsFP, err = s.local.MessageFlagsFingerprint(sessionID) - if err != nil { - return 0, fmt.Errorf( - "computing local message flags fingerprint: %w", - err, - ) - } - localFP.SystemFP, err = s.local.SystemMessageFingerprint(sessionID) - if err != nil { - return 0, fmt.Errorf( - "computing local system message fingerprint: %w", err, - ) - } - localFP.ToolCallCount, err = s.local.ToolCallCount(sessionID) - if err != nil { - return 0, fmt.Errorf( - "counting local tool_calls: %w", err, - ) - } - localFP.ToolCallSum, err = s.local.ToolCallContentFingerprint( - sessionID, - ) - if err != nil { - return 0, fmt.Errorf( - "computing local tool_call content fingerprint: %w", - err, - ) - } - localFP.ToolCallFP, err = s.local.ToolCallFingerprint(sessionID) - if err != nil { - return 0, fmt.Errorf( - "computing local tool_call fingerprint: %w", err, - ) - } - localFP.ToolResultFP, err = localToolResultEventPGFingerprint( - s.local, sessionID, + // A row already in PG with a stale unprefixed subagent id matches the + // local fingerprint (both unprefixed), so the skip below would never + // repair it. Force replacement when any subagent link now resolves to a + // different pushed id. + subagentRewrite, err := resolver.sessionNeedsSubagentRewrite( + ctx, localSessionID, ) if err != nil { - return 0, fmt.Errorf( - "computing local tool_result_event fingerprint: %w", err, - ) - } - localFP.TokenFP, err = s.local.MessageTokenFingerprint(sessionID) - if err != nil { - return 0, fmt.Errorf( - "computing local token fingerprint: %w", - err, - ) + return 0, err } + if !subagentRewrite { + localFP := pushLocalMessageFingerprint{} - usageFromMap := false - if sessionUsageFingerprints != nil { - var ok bool - localFP.UsageEventFP, ok = sessionUsageFingerprints[sessionID] - usageFromMap = ok - } - if !usageFromMap { - localFP.UsageEventFP, err = s.local.UsageEventFingerprint(sessionID) + localFP.Sum, localFP.Max, localFP.Min, err = s.local.MessageContentFingerprint( + localSessionID, + ) if err != nil { return 0, fmt.Errorf( - "computing local usage event fingerprint: %w", + "computing local content fingerprint: %w", err, ) } - } - - if comparisons == nil { - pgContentHashFP, err := pgMessageContentHashFingerprint( - ctx, tx, sessionID, + localFP.ContentHashFP, err = s.local.MessageContentHashFingerprint( + localSessionID, ) if err != nil { return 0, fmt.Errorf( - "computing pg content hash fingerprint: %w", + "computing local content hash fingerprint: %w", err, ) } - pgRoleTimeFP, err := pgMessageRoleTimeFingerprint( - ctx, tx, sessionID, + localFP.RoleTimeFP, err = localMessageRoleTimePGFingerprint( + s.local, localSessionID, ) if err != nil { return 0, fmt.Errorf( - "computing pg role/time fingerprint: %w", + "computing local role/time fingerprint: %w", err, ) } - pgFlagsFP, err := pgMessageFlagsFingerprint(ctx, tx, sessionID) + localFP.FlagsFP, err = s.local.MessageFlagsFingerprint(localSessionID) if err != nil { return 0, fmt.Errorf( - "computing pg message flags fingerprint: %w", + "computing local message flags fingerprint: %w", err, ) } - pgTokenFP, err := pgMessageTokenFingerprint(ctx, tx, sessionID) + localFP.SystemFP, err = s.local.SystemMessageFingerprint(localSessionID) if err != nil { return 0, fmt.Errorf( - "computing pg token fingerprint: %w", - err, + "computing local system message fingerprint: %w", err, ) } - pgTCFP, err := pgToolCallFingerprint(ctx, tx, sessionID) + localFP.ToolCallCount, err = s.local.ToolCallCount(localSessionID) if err != nil { return 0, fmt.Errorf( - "computing pg tool_call fingerprint: %w", - err, + "counting local tool_calls: %w", err, ) } - pgResultFP, err := pgToolResultEventFingerprint(ctx, tx, sessionID) + localFP.ToolCallSum, err = s.local.ToolCallContentFingerprint( + localSessionID, + ) if err != nil { return 0, fmt.Errorf( - "computing pg tool_result_event fingerprint: %w", + "computing local tool_call content fingerprint: %w", err, ) } - pgUsageFP, err := pgUsageEventFingerprint(ctx, tx, sessionID) + localFP.ToolCallFP, err = s.local.ToolCallFingerprint(localSessionID) + if err != nil { + return 0, fmt.Errorf( + "computing local tool_call fingerprint: %w", err, + ) + } + localFP.ToolResultFP, err = localToolResultEventPGFingerprint( + s.local, localSessionID, + ) + if err != nil { + return 0, fmt.Errorf( + "computing local tool_result_event fingerprint: %w", err, + ) + } + localFP.TokenFP, err = s.local.MessageTokenFingerprint(localSessionID) if err != nil { return 0, fmt.Errorf( - "computing pg usage event fingerprint: %w", + "computing local token fingerprint: %w", err, ) } - if localFP.Sum == pgAgg.Sum && - localFP.Max == pgAgg.Max && - localFP.Min == pgAgg.Min && - localFP.ContentHashFP == pgContentHashFP && - localFP.RoleTimeFP == pgRoleTimeFP && - localFP.FlagsFP == pgFlagsFP && - localFP.SystemFP == pgAgg.SysFP && - localFP.ToolCallCount == pgToolAgg.Count && - localFP.ToolCallSum == pgToolAgg.Sum && - localFP.ToolCallFP == pgTCFP && - localFP.ToolResultFP == pgResultFP && - localFP.TokenFP == pgTokenFP && - localFP.UsageEventFP == pgUsageFP { + usageFromMap := false + if sessionUsageFingerprints != nil { + var ok bool + localFP.UsageEventFP, ok = sessionUsageFingerprints[localSessionID] + usageFromMap = ok + } + if !usageFromMap { + localFP.UsageEventFP, err = s.local.UsageEventFingerprint(localSessionID) + if err != nil { + return 0, fmt.Errorf( + "computing local usage event fingerprint: %w", + err, + ) + } + } + + if comparisons == nil { + pgContentHashFP, err := pgMessageContentHashFingerprint( + ctx, tx, pgSessionID, + ) + if err != nil { + return 0, fmt.Errorf( + "computing pg content hash fingerprint: %w", + err, + ) + } + pgRoleTimeFP, err := pgMessageRoleTimeFingerprint( + ctx, tx, pgSessionID, + ) + if err != nil { + return 0, fmt.Errorf( + "computing pg role/time fingerprint: %w", + err, + ) + } + pgFlagsFP, err := pgMessageFlagsFingerprint(ctx, tx, pgSessionID) + if err != nil { + return 0, fmt.Errorf( + "computing pg message flags fingerprint: %w", + err, + ) + } + pgTokenFP, err := pgMessageTokenFingerprint(ctx, tx, pgSessionID) + if err != nil { + return 0, fmt.Errorf( + "computing pg token fingerprint: %w", + err, + ) + } + pgTCFP, err := pgToolCallFingerprint(ctx, tx, pgSessionID) + if err != nil { + return 0, fmt.Errorf( + "computing pg tool_call fingerprint: %w", + err, + ) + } + pgResultFP, err := pgToolResultEventFingerprint(ctx, tx, pgSessionID) + if err != nil { + return 0, fmt.Errorf( + "computing pg tool_result_event fingerprint: %w", + err, + ) + } + pgUsageFP, err := pgUsageEventFingerprint(ctx, tx, pgSessionID) + if err != nil { + return 0, fmt.Errorf( + "computing pg usage event fingerprint: %w", + err, + ) + } + + if localFP.Sum == pgAgg.Sum && + localFP.Max == pgAgg.Max && + localFP.Min == pgAgg.Min && + localFP.ContentHashFP == pgContentHashFP && + localFP.RoleTimeFP == pgRoleTimeFP && + localFP.FlagsFP == pgFlagsFP && + localFP.SystemFP == pgAgg.SysFP && + localFP.ToolCallCount == pgToolAgg.Count && + localFP.ToolCallSum == pgToolAgg.Sum && + localFP.ToolCallFP == pgTCFP && + localFP.ToolResultFP == pgResultFP && + localFP.TokenFP == pgTokenFP && + localFP.UsageEventFP == pgUsageFP { + return 0, nil + } + } else if shouldSkipSessionMessages( + pgSessionID, localCount, localFP, full, comparisons, + ) { return 0, nil } - } else if shouldSkipSessionMessages( - sessionID, localCount, localFP, full, comparisons, - ) { - return 0, nil } } + if err := backfillLegacyPinnedMessageSourceUUIDs(ctx, tx, pgSessionID); err != nil { + return 0, err + } + if _, err := tx.ExecContext(ctx, ` DELETE FROM tool_result_events WHERE session_id = $1 - `, sessionID); err != nil { + `, pgSessionID); err != nil { return 0, fmt.Errorf( "deleting pg tool_result_events: %w", err, ) @@ -2213,7 +3372,7 @@ func (s *Sync) pushMessages( if _, err := tx.ExecContext(ctx, ` DELETE FROM tool_calls WHERE session_id = $1 - `, sessionID); err != nil { + `, pgSessionID); err != nil { return 0, fmt.Errorf( "deleting pg tool_calls: %w", err, ) @@ -2221,12 +3380,14 @@ func (s *Sync) pushMessages( if _, err := tx.ExecContext(ctx, ` DELETE FROM messages WHERE session_id = $1 - `, sessionID); err != nil { + `, pgSessionID); err != nil { return 0, fmt.Errorf( "deleting pg messages: %w", err, ) } - if err := s.replaceUsageEvents(ctx, tx, sessionID); err != nil { + if err := s.replaceUsageEvents( + ctx, tx, localSessionID, pgSessionID, + ); err != nil { return 0, err } @@ -2234,7 +3395,7 @@ func (s *Sync) pushMessages( startOrdinal := 0 for { msgs, err := s.local.GetMessages( - ctx, sessionID, startOrdinal, + ctx, localSessionID, startOrdinal, db.MaxMessageLimit, true, ) if err != nil { @@ -2251,23 +3412,29 @@ func (s *Sync) pushMessages( return count, fmt.Errorf( "pushMessages %s: ordinal did not "+ "advance (start=%d, last=%d)", - sessionID, startOrdinal, + localSessionID, startOrdinal, msgs[len(msgs)-1].Ordinal, ) } + if err := resolver.rewriteSubagentIDs(ctx, msgs); err != nil { + return count, fmt.Errorf( + "resolving subagent session ids: %w", err, + ) + } + if err := bulkInsertMessages( - ctx, tx, sessionID, msgs, + ctx, tx, pgSessionID, msgs, ); err != nil { return count, err } if err := bulkInsertToolCalls( - ctx, tx, sessionID, msgs, + ctx, tx, pgSessionID, msgs, ); err != nil { return count, err } if err := bulkInsertToolResultEvents( - ctx, tx, sessionID, msgs, + ctx, tx, pgSessionID, msgs, ); err != nil { return count, err } @@ -2275,7 +3442,7 @@ func (s *Sync) pushMessages( startOrdinal = nextOrdinal } - if err := reconcilePinnedMessages(ctx, tx, sessionID); err != nil { + if err := reconcilePinnedMessages(ctx, tx, pgSessionID); err != nil { return count, err } @@ -2289,19 +3456,22 @@ func (s *Sync) pushMessages( // zero-message and the normal message-replace paths in pushMessages call // this so a session's cost always reaches PG. func (s *Sync) replaceUsageEvents( - ctx context.Context, tx *sql.Tx, sessionID string, + ctx context.Context, tx *sql.Tx, + localSessionID string, pgSessionID string, ) error { if _, err := tx.ExecContext(ctx, ` DELETE FROM usage_events WHERE session_id = $1 - `, sessionID); err != nil { + `, pgSessionID); err != nil { return fmt.Errorf("deleting pg usage_events: %w", err) } - usageEvents, err := s.local.GetUsageEvents(ctx, sessionID) + usageEvents, err := s.local.GetUsageEvents(ctx, localSessionID) if err != nil { return fmt.Errorf("reading local usage events: %w", err) } - if err := bulkInsertUsageEvents(ctx, tx, usageEvents); err != nil { + if err := bulkInsertUsageEvents( + ctx, tx, pgSessionID, usageEvents, + ); err != nil { return err } return nil @@ -2310,20 +3480,8 @@ func (s *Sync) replaceUsageEvents( func reconcilePinnedMessages( ctx context.Context, tx *sql.Tx, sessionID string, ) error { - if _, err := tx.ExecContext(ctx, ` - UPDATE pinned_messages p - SET source_uuid = m.source_uuid - FROM messages m - WHERE p.session_id = $1 - AND m.session_id = p.session_id - AND m.ordinal = p.message_id - AND p.source_uuid = '' - AND m.source_uuid <> ''`, - sessionID, - ); err != nil { - return fmt.Errorf( - "backfilling pg pin source_uuid: %w", err, - ) + if err := backfillLegacyPinnedMessageSourceUUIDs(ctx, tx, sessionID); err != nil { + return err } // Move shifted source-backed pins out of the real ordinal range @@ -2480,6 +3638,27 @@ func reconcilePinnedMessages( return nil } +func backfillLegacyPinnedMessageSourceUUIDs( + ctx context.Context, tx *sql.Tx, sessionID string, +) error { + if _, err := tx.ExecContext(ctx, ` + UPDATE pinned_messages p + SET source_uuid = m.source_uuid + FROM messages m + WHERE p.session_id = $1 + AND m.session_id = p.session_id + AND m.ordinal = p.message_id + AND p.source_uuid = '' + AND m.source_uuid <> ''`, + sessionID, + ); err != nil { + return fmt.Errorf( + "backfilling pg pin source_uuid: %w", err, + ) + } + return nil +} + func pgMessageTokenFingerprint( ctx context.Context, tx *sql.Tx, sessionID string, ) (string, error) { @@ -2848,7 +4027,8 @@ func bulkInsertMessages( } func bulkInsertUsageEvents( - ctx context.Context, tx *sql.Tx, events []db.UsageEvent, + ctx context.Context, tx *sql.Tx, + sessionID string, events []db.UsageEvent, ) error { if len(events) == 0 { return nil @@ -2891,7 +4071,7 @@ func bulkInsertUsageEvents( cost = *ev.CostUSD } args = append(args, - ev.SessionID, + sessionID, ordinal, sanitizePG(ev.Source), sanitizePG(ev.Model), @@ -3127,31 +4307,32 @@ func bulkInsertToolResultEvents( // sessions.secrets_rules_version is pushed by pushSession alongside // the rest of the session columns. func (s *Sync) pushSecretFindings( - ctx context.Context, tx *sql.Tx, sessionID string, + ctx context.Context, tx *sql.Tx, + localSessionID string, pgSessionID string, ) (bool, error) { res, err := tx.ExecContext(ctx, `DELETE FROM secret_findings WHERE session_id = $1`, - sessionID, + pgSessionID, ) if err != nil { return false, fmt.Errorf( "deleting pg secret_findings for %s: %w", - sessionID, err, + pgSessionID, err, ) } deleted, err := res.RowsAffected() if err != nil { return false, fmt.Errorf( "counting deleted secret_findings for %s: %w", - sessionID, err, + pgSessionID, err, ) } - findings, err := s.local.SessionSecretFindings(ctx, sessionID) + findings, err := s.local.SessionSecretFindings(ctx, localSessionID) if err != nil { return false, fmt.Errorf( "reading local secret_findings for %s: %w", - sessionID, err, + localSessionID, err, ) } if len(findings) == 0 { @@ -3184,7 +4365,7 @@ func (s *Sync) pushSecretFindings( p+5, p+6, p+7, p+8, p+9, p+10, p+11, ) args = append(args, - f.SessionID, f.RuleName, f.Confidence, + pgSessionID, f.RuleName, f.Confidence, f.LocationKind, f.MessageOrdinal, f.CallIndex, f.EventIndex, f.MatchStart, f.MatchEnd, f.MatchIndex, @@ -3196,7 +4377,7 @@ func (s *Sync) pushSecretFindings( ); err != nil { return false, fmt.Errorf( "bulk inserting secret_findings for %s: %w", - sessionID, err, + pgSessionID, err, ) } } diff --git a/internal/postgres/push_pgtest_test.go b/internal/postgres/push_pgtest_test.go index 4c03b7f5f..017bf5821 100644 --- a/internal/postgres/push_pgtest_test.go +++ b/internal/postgres/push_pgtest_test.go @@ -373,7 +373,10 @@ func TestPushSessionTerminationStatus(t *testing.T) { t.Helper() tx, err := pg.BeginTx(ctx, nil) require.NoError(t, err, "BeginTx") - if err := sync.pushSession(ctx, tx, s, markerID, nil); err != nil { + if err := sync.pushSession(ctx, tx, s, pushedSessionIdentity{ + ID: s.ID, + Machine: s.Machine, + }, markerID, nil); err != nil { _ = tx.Rollback() t.Fatalf("pushSession: %v", err) } @@ -439,7 +442,12 @@ func TestPushSessionPreservesSourceMachine(t *testing.T) { require.NoError(t, err, "BeginTx") markerID, err := sync.pushMarkerID() require.NoError(t, err, "pushMarkerID") - require.NoError(t, sync.pushSession(ctx, tx, remoteSession, markerID, nil), "pushSession") + require.NoError(t, sync.pushSession( + ctx, tx, remoteSession, pushedSessionIdentity{ + ID: remoteSession.ID, + Machine: remoteSession.Machine, + }, markerID, nil, + ), "pushSession") require.NoError(t, tx.Commit(), "Commit") var got string @@ -788,7 +796,7 @@ func TestPushSessionSkipsPGExcludedSession(t *testing.T) { Machine: "test-machine", Agent: "claude", CreatedAt: "2026-01-01T00:00:00Z", - }, markerID, nil) + }, pushedSessionIdentity{ID: sessionID, Machine: "test-machine"}, markerID, nil) require.ErrorIs(t, err, errSessionExcluded) require.NoError(t, tx.Rollback(), "Rollback") @@ -1549,6 +1557,272 @@ func TestPushIncrementalWithOnlyForeignMachineSessions(t *testing.T) { "second incremental push must not rewrite the session") } +func TestPushPreservesMixedLocalAndImportedMachinesForPGStore(t *testing.T) { + pgURL := testPGURL(t) + + const schema = "agentsview_push_mixed_machine_attribution_test" + pg, err := Open(pgURL, schema, true) + require.NoError(t, err, "Open") + defer pg.Close() + + ctx := context.Background() + _, err = pg.Exec(`DROP SCHEMA IF EXISTS ` + schema + ` CASCADE`) + require.NoError(t, err, "drop schema") + require.NoError(t, EnsureSchema(ctx, pg, schema), "EnsureSchema") + + localDB := testDB(t) + sync := &Sync{ + pg: pg, + local: localDB, + machine: "laptop-a1b2c3", + schema: schema, + schemaDone: true, + } + + localSess := db.Session{ + ID: "local-session-001", + Project: "local-proj", + Machine: "local", + Agent: "claude", + MessageCount: 1, + CreatedAt: "2026-01-01T00:00:00Z", + } + importedSess := db.Session{ + ID: "desktop-d4e5f6~foreign-session-001", + Project: "foreign-proj", + Machine: "desktop-d4e5f6", + Agent: "codex", + MessageCount: 1, + CreatedAt: "2026-01-01T00:01:00Z", + } + for _, sess := range []db.Session{localSess, importedSess} { + require.NoError(t, localDB.UpsertSession(sess), "UpsertSession %s", sess.ID) + require.NoError(t, localDB.InsertMessages([]db.Message{{ + SessionID: sess.ID, + Ordinal: 0, + Role: "assistant", + Content: "hello from " + sess.Machine, + ContentLength: len("hello from " + sess.Machine), + }}), "InsertMessages %s", sess.ID) + } + + res, err := sync.Push(ctx, false, nil) + require.NoError(t, err, "first Push") + assert.Zero(t, res.Errors, "first push should report no failures") + assert.Equal(t, 2, res.SessionsPushed) + + rows, err := pg.Query(`SELECT id, machine FROM sessions`) + require.NoError(t, err, "querying pushed machines") + defer rows.Close() + gotMachines := map[string]string{} + for rows.Next() { + var id, machine string + require.NoError(t, rows.Scan(&id, &machine), "scanning pushed machine") + gotMachines[id] = machine + } + require.NoError(t, rows.Err(), "iterating pushed machines") + assert.Equal(t, "laptop-a1b2c3", gotMachines[localSess.ID]) + assert.Equal(t, "desktop-d4e5f6", gotMachines[importedSess.ID]) + + store, err := NewStore(pgURL, schema, true) + require.NoError(t, err, "NewStore") + defer store.Close() + machines, err := store.GetMachines(ctx, false, false) + require.NoError(t, err, "GetMachines") + assert.ElementsMatch(t, []string{"desktop-d4e5f6", "laptop-a1b2c3"}, machines) + + importedPage, err := store.ListSessions(ctx, db.SessionFilter{ + Machine: "desktop-d4e5f6", + Limit: 10, + }) + require.NoError(t, err, "ListSessions imported machine") + require.Equal(t, 1, importedPage.Total) + require.Len(t, importedPage.Sessions, 1) + assert.Equal(t, importedSess.ID, importedPage.Sessions[0].ID) + assert.Equal(t, "desktop-d4e5f6", importedPage.Sessions[0].Machine) + + localPage, err := store.ListSessions(ctx, db.SessionFilter{ + Machine: "laptop-a1b2c3", + Limit: 10, + }) + require.NoError(t, err, "ListSessions local machine") + require.Equal(t, 1, localPage.Total) + require.Len(t, localPage.Sessions, 1) + assert.Equal(t, localSess.ID, localPage.Sessions[0].ID) + assert.Equal(t, "laptop-a1b2c3", localPage.Sessions[0].Machine) + + res, err = sync.Push(ctx, false, nil) + require.NoError(t, err, "second Push") + assert.Zero(t, res.Errors, "second push should report no failures") + assert.Zero(t, res.SessionsPushed, "unchanged mixed-machine sessions should not be re-pushed") +} + +func TestPushKeepsSameNativeIDDistinctAcrossMachines(t *testing.T) { + pgURL := testPGURL(t) + + const schema = "agentsview_push_native_id_collision_test" + pg, err := Open(pgURL, schema, true) + require.NoError(t, err, "Open") + defer pg.Close() + + ctx := context.Background() + _, err = pg.Exec(`DROP SCHEMA IF EXISTS ` + schema + ` CASCADE`) + require.NoError(t, err, "drop schema") + require.NoError(t, EnsureSchema(ctx, pg, schema), "EnsureSchema") + + const nativeID = "shared-native-session-001" + const machineA = "laptop-a1b2c3" + const machineB = "desktop-d4e5f6" + const projectA = "laptop-proj" + const projectB = "desktop-proj" + pgIDA := nativeID + pgIDB := prefixedSessionID(machineB, nativeID) + + localA := testDB(t) + localB := testDB(t) + syncA := &Sync{ + pg: pg, + local: localA, + machine: machineA, + schema: schema, + schemaDone: true, + } + syncB := &Sync{ + pg: pg, + local: localB, + machine: machineB, + schema: schema, + schemaDone: true, + } + + seed := func(local *db.DB, project, content string) db.Session { + t.Helper() + sess := db.Session{ + ID: nativeID, + Project: project, + Machine: "local", + Agent: "claude", + MessageCount: 1, + CreatedAt: "2026-01-01T00:00:00Z", + } + require.NoError(t, local.UpsertSession(sess), "UpsertSession") + require.NoError(t, local.InsertMessages([]db.Message{{ + SessionID: nativeID, + Ordinal: 0, + Role: "assistant", + Content: content, + ContentLength: len(content), + }}), "InsertMessages") + return sess + } + sessA := seed(localA, projectA, "from laptop") + sessB := seed(localB, projectB, "from desktop") + + res, err := syncA.Push(ctx, false, nil) + require.NoError(t, err, "Push A") + assert.Zero(t, res.Errors, "first A push should report no failures") + assert.Equal(t, 1, res.SessionsPushed) + + res, err = syncB.Push(ctx, false, nil) + require.NoError(t, err, "Push B") + assert.Zero(t, res.Errors, "first B push should report no failures") + assert.Equal(t, 1, res.SessionsPushed) + + rows, err := pg.Query( + `SELECT id, machine, project FROM sessions ORDER BY id`, + ) + require.NoError(t, err, "querying pushed collision rows") + defer rows.Close() + got := map[string]db.Session{} + for rows.Next() { + var row db.Session + require.NoError(t, rows.Scan( + &row.ID, &row.Machine, &row.Project, + ), "scanning collision row") + got[row.ID] = row + } + require.NoError(t, rows.Err(), "iterating collision rows") + require.Len(t, got, 2) + assert.Equal(t, machineA, got[pgIDA].Machine) + assert.Equal(t, projectA, got[pgIDA].Project) + assert.Equal(t, machineB, got[pgIDB].Machine) + assert.Equal(t, projectB, got[pgIDB].Project) + + var contentA, contentB string + require.NoError(t, pg.QueryRow( + `SELECT content FROM messages WHERE session_id = $1 AND ordinal = 0`, + pgIDA, + ).Scan(&contentA), "reading A message") + require.NoError(t, pg.QueryRow( + `SELECT content FROM messages WHERE session_id = $1 AND ordinal = 0`, + pgIDB, + ).Scan(&contentB), "reading B message") + assert.Equal(t, "from laptop", contentA) + assert.Equal(t, "from desktop", contentB) + + store, err := NewStore(pgURL, schema, true) + require.NoError(t, err, "NewStore") + defer store.Close() + pageA, err := store.ListSessions(ctx, db.SessionFilter{ + Machine: machineA, + Limit: 10, + }) + require.NoError(t, err, "ListSessions A") + require.Equal(t, 1, pageA.Total) + require.Len(t, pageA.Sessions, 1) + assert.Equal(t, pgIDA, pageA.Sessions[0].ID) + assert.Equal(t, projectA, pageA.Sessions[0].Project) + pageB, err := store.ListSessions(ctx, db.SessionFilter{ + Machine: machineB, + Limit: 10, + }) + require.NoError(t, err, "ListSessions B") + require.Equal(t, 1, pageB.Total) + require.Len(t, pageB.Sessions, 1) + assert.Equal(t, pgIDB, pageB.Sessions[0].ID) + assert.Equal(t, projectB, pageB.Sessions[0].Project) + + res, err = syncA.Push(ctx, false, nil) + require.NoError(t, err, "second Push A") + assert.Zero(t, res.Errors, "second A push should report no failures") + assert.Zero(t, res.SessionsPushed) + res, err = syncB.Push(ctx, false, nil) + require.NoError(t, err, "second Push B") + assert.Zero(t, res.Errors, "second B push should report no failures") + assert.Zero(t, res.SessionsPushed) + + var rowCount int + require.NoError(t, pg.QueryRow( + `SELECT COUNT(*) FROM sessions WHERE id IN ($1, $2)`, + pgIDA, pgIDB, + ).Scan(&rowCount), "counting collision rows") + assert.Equal(t, 2, rowCount) + + updatedB := sessB + updatedB.Project = "desktop-proj-updated" + require.NoError(t, localB.UpsertSession(updatedB), "update B session") + // UpsertSession does not write local_modified_at, so mark the session + // modified explicitly; otherwise the incremental push will not re-list it. + require.NoError(t, localB.BumpLocalModifiedAt(nativeID), + "mark updated B session modified") + res, err = syncB.Push(ctx, false, nil) + require.NoError(t, err, "updated Push B") + assert.Zero(t, res.Errors, "updated B push should report no failures") + assert.Equal(t, 1, res.SessionsPushed) + + var projectAfterA, projectAfterB string + require.NoError(t, pg.QueryRow( + `SELECT project FROM sessions WHERE id = $1`, + pgIDA, + ).Scan(&projectAfterA), "reading A project after B update") + require.NoError(t, pg.QueryRow( + `SELECT project FROM sessions WHERE id = $1`, + pgIDB, + ).Scan(&projectAfterB), "reading B project after update") + assert.Equal(t, sessA.Project, projectAfterA) + assert.Equal(t, updatedB.Project, projectAfterB) +} + // TestPushDetectsResetWhenCompetingMachineRowsExist verifies that a PG reset is // detected even when another pusher has repopulated rows under a machine value // this host also writes. The local session carries Machine "remote-host" (as a @@ -2058,11 +2332,18 @@ func TestPushReportsSkippedConflicts(t *testing.T) { schemaDone: true, } - const sessID = "conflict-001" + // A same-id collision against a different live machine is resolved by + // storing the session under a machine-prefixed id, not skipped. A skipped + // conflict now arises only when the id cannot be re-prefixed: an imported + // foreign-origin session already carries its origin's prefix, so when PG + // holds that id under a different owner marker -- e.g. a third machine + // imported and pushed the same session first -- this push must leave the + // row to its owner and report it as a skipped conflict. + const sessID = "machine-a~conflict-001" require.NoError(t, localDB.UpsertSession(db.Session{ ID: sessID, Project: "proj", - Machine: "machine-b", + Machine: "machine-a", Agent: "claude", MessageCount: 1, CreatedAt: "2026-01-01T00:00:00Z", @@ -2087,4 +2368,18 @@ func TestPushReportsSkippedConflicts(t *testing.T) { assert.Zero(t, res.Errors, "push should not report failed sessions") assert.Zero(t, res.SessionsPushed, "conflicting session should not be counted as pushed") assert.Equal(t, 1, res.SkippedConflicts, "skipped conflicts should be observable in PushResult") + + // The conflicting row is left untouched and no doubly-prefixed row is + // created in this pusher's namespace. + var ownerMarker string + require.NoError(t, pg.QueryRow( + `SELECT owner_marker FROM sessions WHERE id = $1`, sessID, + ).Scan(&ownerMarker), "reading conflicting row owner") + assert.Equal(t, "other-owner", ownerMarker) + var doublyPrefixed int + require.NoError(t, pg.QueryRow( + `SELECT COUNT(*) FROM sessions WHERE id = $1`, + prefixedSessionID("machine-b", sessID), + ).Scan(&doublyPrefixed), "counting doubly-prefixed rows") + assert.Zero(t, doublyPrefixed) } diff --git a/internal/postgres/push_test.go b/internal/postgres/push_test.go index 153552d3f..459faad81 100644 --- a/internal/postgres/push_test.go +++ b/internal/postgres/push_test.go @@ -6,6 +6,7 @@ import ( "database/sql/driver" "encoding/json" "errors" + "fmt" "io" "path/filepath" "strings" @@ -301,6 +302,23 @@ func TestSessionAliasBackfillForcesOneFullPush(t *testing.T) { assert.False(t, needed) } +func TestArtifactIdentityModePersistsOnlyAfterSuccessfulPush(t *testing.T) { + store := &syncStateStoreStub{values: map[string]string{ + artifactIdentityModeStateKey: legacyArtifactIdentityMode, + }} + mode := artifactOwnerMarkerPrefix + "origin-a1b2c3" + + require.NoError(t, completeArtifactIdentityMode( + store, mode, PushResult{Errors: 1}, + )) + assert.Equal(t, legacyArtifactIdentityMode, + store.values[artifactIdentityModeStateKey], + "a partial push must leave the prior mode so the next run retries the transition") + + require.NoError(t, completeArtifactIdentityMode(store, mode, PushResult{})) + assert.Equal(t, mode, store.values[artifactIdentityModeStateKey]) +} + func TestCompleteSessionAliasBackfillMarksDoneUnlessErrors(t *testing.T) { for _, tc := range []struct { name string @@ -503,6 +521,125 @@ func TestSessionAliasBackfillKeysStayFilteredForPushState(t *testing.T) { assert.Empty(t, store.values["last_push_at:work"]) } +// scriptedSink is a sessionBatchSink that records calls and writes every +// session except those named in fail. It mirrors pushBatch semantics: a +// multi-session batch containing a failing session rolls back as a unit +// (ok=false) so the driver retries each session individually, and a failing +// single-session batch returns ok=false. A session named in fatal makes the +// batch containing it return a fatal error. +type scriptedSink struct { + fail map[string]bool + fatal string + calls [][]string +} + +func (s *scriptedSink) writeBatch( + _ context.Context, batch []db.Session, pushed *[]db.Session, +) (batchResult, error) { + ids := make([]string, len(batch)) + for i, sess := range batch { + ids[i] = sess.ID + } + s.calls = append(s.calls, ids) + for _, sess := range batch { + if sess.ID == s.fatal { + return batchResult{}, errors.New("fatal sink error") + } + if s.fail[sess.ID] { + // Whole batch rolls back without writing. + return batchResult{ok: false}, nil + } + } + msgs := 0 + for _, sess := range batch { + *pushed = append(*pushed, sess) + msgs += 2 + } + return batchResult{ok: true, sessions: len(batch), messages: msgs}, nil +} + +func sessionsWithIDs(ids ...string) []db.Session { + out := make([]db.Session, len(ids)) + for i, id := range ids { + out[i] = db.Session{ID: id} + } + return out +} + +func TestDrainSessionBatchesChunksAndReportsProgress(t *testing.T) { + var ids []string + for i := range 120 { + ids = append(ids, fmt.Sprintf("s%03d", i)) + } + sessions := sessionsWithIDs(ids...) + sink := &scriptedSink{} + + var result PushResult + var progress []PushProgress + pushed, err := drainSessionBatches( + context.Background(), sessions, sink, &result, + func(p PushProgress) { progress = append(progress, p) }, + ) + require.NoError(t, err) + + // Batched in chunks of 50: 50 + 50 + 20. + require.Len(t, sink.calls, 3) + assert.Len(t, sink.calls[0], 50) + assert.Len(t, sink.calls[1], 50) + assert.Len(t, sink.calls[2], 20) + + assert.Equal(t, 120, result.SessionsPushed) + assert.Equal(t, 240, result.MessagesPushed) + assert.Equal(t, 0, result.Errors) + assert.Len(t, pushed, 120) + + require.Len(t, progress, 3) + assert.Equal(t, PushProgress{SessionsDone: 50, SessionsTotal: 120, MessagesDone: 100}, progress[0]) + assert.Equal(t, PushProgress{SessionsDone: 100, SessionsTotal: 120, MessagesDone: 200}, progress[1]) + assert.Equal(t, PushProgress{SessionsDone: 120, SessionsTotal: 120, MessagesDone: 240}, progress[2]) +} + +func TestDrainSessionBatchesRetriesFailedBatchIndividually(t *testing.T) { + sessions := sessionsWithIDs("a", "b", "c") + sink := &scriptedSink{fail: map[string]bool{"b": true}} + + var result PushResult + pushed, err := drainSessionBatches( + context.Background(), sessions, sink, &result, nil, + ) + require.NoError(t, err) + + // First the whole batch (rolls back), then each session individually. + require.Len(t, sink.calls, 4) + assert.Equal(t, []string{"a", "b", "c"}, sink.calls[0]) + assert.Equal(t, []string{"a"}, sink.calls[1]) + assert.Equal(t, []string{"b"}, sink.calls[2]) + assert.Equal(t, []string{"c"}, sink.calls[3]) + + assert.Equal(t, 2, result.SessionsPushed) + assert.Equal(t, 4, result.MessagesPushed) + assert.Equal(t, 1, result.Errors) + pushedIDs := make([]string, len(pushed)) + for i, sess := range pushed { + pushedIDs[i] = sess.ID + } + assert.Equal(t, []string{"a", "c"}, pushedIDs) +} + +func TestDrainSessionBatchesFatalErrorAborts(t *testing.T) { + sessions := sessionsWithIDs("a", "b", "c") + sink := &scriptedSink{fatal: "b"} + + var result PushResult + pushed, err := drainSessionBatches( + context.Background(), sessions, sink, &result, nil, + ) + require.Error(t, err) + assert.Equal(t, 0, result.SessionsPushed) + assert.Equal(t, 0, result.Errors) + assert.Empty(t, pushed) +} + func TestReadPushBoundaryStateValidity(t *testing.T) { const cutoff = "2026-03-11T12:34:56.123Z" @@ -586,6 +723,97 @@ func TestPGExcludedSessionIDsQueryUsesSingleArrayParameter(t *testing.T) { ) } +func TestArtifactPushIdentityCanonicalizesNativeAndImportedCopies(t *testing.T) { + tests := []struct { + name string + session db.Session + localOrigin string + imported bool + wantID string + wantMachine string + wantOwner string + wantOK bool + }{ + { + name: "locally owned artifact session", + session: db.Session{ID: "native-id", Machine: "local"}, + localOrigin: "origin-a1b2c3", + wantID: "origin-a1b2c3~native-id", + wantMachine: "origin-a1b2c3", + wantOwner: "artifact-origin:origin-a1b2c3", + wantOK: true, + }, + { + name: "imported artifact session", + session: db.Session{ID: "origin-a1b2c3~native-id", Machine: "origin-a1b2c3"}, + localOrigin: "origin-b4c5d6", + imported: true, + wantID: "origin-a1b2c3~native-id", + wantMachine: "origin-a1b2c3", + wantOwner: "artifact-origin:origin-a1b2c3", + wantOK: true, + }, + { + name: "ssh-shaped session without artifact provenance stays legacy", + session: db.Session{ID: "origin-a1b2c3~native-id", Machine: "origin-a1b2c3"}, + localOrigin: "origin-b4c5d6", + wantOK: false, + }, + { + name: "no artifact origin preserves legacy collision behavior", + session: db.Session{ID: "native-id", Machine: "local"}, + wantOK: false, + }, + { + name: "prefixed foreign session without local artifact opt in stays legacy", + session: db.Session{ID: "remote-host~native-id", Machine: "remote-host"}, + wantOK: false, + }, + { + name: "unrelated foreign machine id is not treated as imported", + session: db.Session{ID: "native-id", Machine: "remote-host"}, + localOrigin: "origin-b4c5d6", + wantOK: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + id, machine, owner, ok := artifactPushIdentity( + tt.session, tt.localOrigin, tt.imported, + ) + assert.Equal(t, tt.wantOK, ok) + assert.Equal(t, tt.wantID, id) + assert.Equal(t, tt.wantMachine, machine) + assert.Equal(t, tt.wantOwner, owner) + }) + } +} + +func TestArtifactPushOwnershipAdoptsOnlyCurrentPushersLegacyMarker(t *testing.T) { + identity := pushedSessionIdentity{ + Machine: "origin-a1b2c3", + OwnerMarker: "artifact-origin:origin-a1b2c3", + LegacyOwnerMarkers: []string{"this-pusher-marker"}, + } + + assert.True(t, samePushedSessionOwner( + "artifact-origin:origin-a1b2c3", "origin-a1b2c3", + identity, "this-pusher-marker", nil, + )) + assert.True(t, samePushedSessionOwner( + "this-pusher-marker", "origin-a1b2c3", + identity, "this-pusher-marker", nil, + )) + assert.False(t, samePushedSessionOwner( + "another-pusher-marker", "origin-a1b2c3", + identity, "this-pusher-marker", nil, + )) + assert.False(t, samePushedSessionOwner( + "", "origin-a1b2c3", identity, "this-pusher-marker", nil, + ), "matching an artifact origin must not let an importer seize an ownerless row") +} + func TestDeletePGExcludedSessionRowsUsesSingleArrayParameter(t *testing.T) { execer := &capturePGExec{} @@ -636,6 +864,7 @@ func TestPushSessionRechecksExclusionAfterSuccessfulUpsert(t *testing.T) { Agent: "claude", CreatedAt: "2026-01-01T00:00:00Z", }, + pushedSessionIdentity{ID: "sess-race", Machine: "push-machine"}, "marker", nil, ) @@ -669,6 +898,7 @@ func TestPushSessionStoresVibeFallbackAlias(t *testing.T) { CreatedAt: "2026-01-01T00:00:00Z", FilePath: &filePath, }, + pushedSessionIdentity{ID: "vibe:canonical-uuid", Machine: "push-machine"}, "marker", nil, ) @@ -680,6 +910,95 @@ func TestPushSessionStoresVibeFallbackAlias(t *testing.T) { require.NoError(t, tx.Rollback(), "Rollback") } +func TestPushSessionStoresAliasUnderResolvedPGID(t *testing.T) { + state := &pushSessionProbeState{aliases: map[string]string{}} + pg := newPushSessionProbeDB(t, state) + tx, err := pg.BeginTx(context.Background(), nil) + require.NoError(t, err, "BeginTx") + + sessionDir := filepath.Join( + t.TempDir(), + "session_20260616_083518_alias1", + ) + filePath := filepath.Join(sessionDir, "messages.jsonl") + syncer := &Sync{machine: "push-machine"} + err = syncer.pushSession( + context.Background(), tx, + db.Session{ + ID: "vibe:canonical-uuid", + Project: "proj", + Machine: "push-machine", + Agent: "vibe", + CreatedAt: "2026-01-01T00:00:00Z", + FilePath: &filePath, + }, + pushedSessionIdentity{ + ID: "push-machine~vibe:canonical-uuid", + Machine: "push-machine", + }, + "marker", nil, + ) + + require.NoError(t, err, "pushSession") + assert.Empty(t, state.aliases["vibe:canonical-uuid"]) + assert.Equal(t, + "push-machine~vibe:session_20260616_083518_alias1", + state.aliases["push-machine~vibe:canonical-uuid"], + ) + require.NoError(t, tx.Rollback(), "Rollback") +} + +func TestPushSessionIgnoresBareTombstoneForResolvedPGID(t *testing.T) { + state := &pushSessionProbeState{ + aliases: map[string]string{}, + existingExcluded: map[string]bool{ + "vibe:canonical-deleted": true, + }, + excludedIDs: map[string]bool{}, + } + pg := newPushSessionProbeDB(t, state) + tx, err := pg.BeginTx(context.Background(), nil) + require.NoError(t, err, "BeginTx") + + sessionDir := filepath.Join( + t.TempDir(), + "session_20260616_083518_bare01", + ) + filePath := filepath.Join(sessionDir, "messages.jsonl") + syncer := &Sync{machine: "push-machine"} + err = syncer.pushSession( + context.Background(), tx, + db.Session{ + ID: "vibe:canonical-deleted", + Project: "proj", + Machine: "push-machine", + Agent: "vibe", + CreatedAt: "2026-01-01T00:00:00Z", + FilePath: &filePath, + }, + pushedSessionIdentity{ + ID: "push-machine~vibe:canonical-deleted", + Machine: "push-machine", + }, + "marker", nil, + ) + + require.NoError(t, err, "pushSession") + assert.False(t, + state.excludedIDs["vibe:canonical-deleted"], + "resolved pushes must not adopt another owner's bare tombstone", + ) + assert.False(t, + state.deletedExcluded, + "resolved pushes must not delete another owner's bare row", + ) + assert.Equal(t, + "push-machine~vibe:session_20260616_083518_bare01", + state.aliases["push-machine~vibe:canonical-deleted"], + ) + require.NoError(t, tx.Rollback(), "Rollback") +} + func TestPushSessionExcludesVibeFallbackAliasWhenCanonicalExcluded(t *testing.T) { state := &pushSessionProbeState{ existingExcluded: map[string]bool{ @@ -707,6 +1026,7 @@ func TestPushSessionExcludesVibeFallbackAliasWhenCanonicalExcluded(t *testing.T) CreatedAt: "2026-01-01T00:00:00Z", FilePath: &filePath, }, + pushedSessionIdentity{ID: "vibe:canonical-deleted", Machine: "push-machine"}, "marker", nil, ) @@ -745,6 +1065,7 @@ func TestPushSessionSkipsVibeCanonicalWhenFallbackAliasExcluded(t *testing.T) { CreatedAt: "2026-01-01T00:00:00Z", FilePath: &filePath, }, + pushedSessionIdentity{ID: "vibe:canonical-active", Machine: "push-machine"}, "marker", nil, ) @@ -782,9 +1103,15 @@ func TestPurgePGExcludedPushSessionsChecksDerivedAliases(t *testing.T) { FilePath: &filePath, }, } + identities := map[string]pushedSessionIdentity{ + "vibe:canonical-unchanged": { + ID: "vibe:canonical-unchanged", + Machine: "push-machine", + }, + } err := purgePGExcludedPushSessions( - context.Background(), pg, sessionByID, + context.Background(), pg, sessionByID, identities, ) require.NoError(t, err, "purgePGExcludedPushSessions") @@ -799,6 +1126,51 @@ func TestPurgePGExcludedPushSessionsChecksDerivedAliases(t *testing.T) { assert.Equal(t, 0, state.upserts) } +func TestPurgePGExcludedPushSessionsUsesResolvedPGID(t *testing.T) { + state := &pushSessionProbeState{ + existingExcluded: map[string]bool{ + "vibe:canonical-foreign": true, + }, + excludedIDs: map[string]bool{}, + } + pg := newPushSessionProbeDB(t, state) + + sessionDir := filepath.Join( + t.TempDir(), + "session_20260616_083518_resolved", + ) + filePath := filepath.Join(sessionDir, "messages.jsonl") + sessionByID := map[string]db.Session{ + "vibe:canonical-foreign": { + ID: "vibe:canonical-foreign", + Project: "proj", + Machine: "push-machine", + Agent: "vibe", + CreatedAt: "2026-01-01T00:00:00Z", + FilePath: &filePath, + }, + } + identities := map[string]pushedSessionIdentity{ + "vibe:canonical-foreign": { + ID: "push-machine~vibe:canonical-foreign", + Machine: "push-machine", + }, + } + + err := purgePGExcludedPushSessions( + context.Background(), pg, sessionByID, identities, + ) + + require.NoError(t, err, "purgePGExcludedPushSessions") + assert.Contains(t, sessionByID, "vibe:canonical-foreign") + assert.Empty(t, state.excludedIDs) + assert.False(t, + state.deletedExcluded, + "resolved purge must not delete another owner's bare row", + ) + assert.Equal(t, 1, state.exclusionChecks) +} + type pushSessionProbeDriver struct{} type pushSessionProbeConn struct { @@ -822,6 +1194,13 @@ type pushSessionProbeState struct { aliases map[string]string excludedIDs map[string]bool existingExcluded map[string]bool + ownerQueries int + owners map[string]pushSessionProbeOwner +} + +type pushSessionProbeOwner struct { + machine string + marker string } var ( @@ -938,7 +1317,29 @@ func (c *pushSessionProbeConn) QueryContext( defer c.state.mu.Unlock() switch { + case strings.Contains(normalized, "select id, machine, owner_marker"): + c.state.ownerQueries++ + values := [][]driver.Value{} + for _, id := range namedValueStrings(args) { + if owner, ok := c.state.owners[id]; ok { + values = append(values, []driver.Value{id, owner.machine, owner.marker}) + } + } + return &pushSessionProbeRows{ + columns: []string{"id", "machine", "owner_marker"}, + values: values, + }, nil case strings.Contains(normalized, "select machine, owner_marker"): + c.state.ownerQueries++ + if len(args) > 0 { + id, _ := args[0].Value.(string) + if owner, ok := c.state.owners[id]; ok { + return &pushSessionProbeRows{ + columns: []string{"machine", "owner_marker"}, + values: [][]driver.Value{{owner.machine, owner.marker}}, + }, nil + } + } return &pushSessionProbeRows{ columns: []string{"machine", "owner_marker"}, }, nil @@ -975,6 +1376,71 @@ func (c *pushSessionProbeConn) QueryContext( } } +func TestPreloadPGSessionOwnersUsesOneQueryAndCachesMisses(t *testing.T) { + state := &pushSessionProbeState{owners: map[string]pushSessionProbeOwner{ + "owned-a": {machine: "desk", marker: "marker-a"}, + "owned-b": {machine: "laptop", marker: "marker-b"}, + }} + sync := &Sync{pg: newPushSessionProbeDB(t, state)} + + ctx, err := sync.preloadPGSessionOwners( + context.Background(), []string{"owned-a", "owned-b", "missing"}, + ) + require.NoError(t, err) + for _, tc := range []struct { + id string + machine string + marker string + exists bool + }{ + {id: "owned-a", machine: "desk", marker: "marker-a", exists: true}, + {id: "owned-b", machine: "laptop", marker: "marker-b", exists: true}, + {id: "missing"}, + } { + machine, marker, exists, lookupErr := sync.pgSessionOwner(ctx, tc.id) + require.NoError(t, lookupErr) + assert.Equal(t, tc.machine, machine) + assert.Equal(t, tc.marker, marker) + assert.Equal(t, tc.exists, exists) + } + assert.Equal(t, 1, state.ownerQueries, + "preloaded hits and misses must use one owner query") + + _, _, exists, err := sync.pgSessionOwner(ctx, "late-miss") + require.NoError(t, err) + assert.False(t, exists) + _, _, exists, err = sync.pgSessionOwner(ctx, "late-miss") + require.NoError(t, err) + assert.False(t, exists) + assert.Equal(t, 2, state.ownerQueries, + "an owner first discovered after preload must be memoized") +} + +func TestPushIdentityOwnerCandidateIDsCoverLegacyAndArtifactAliases(t *testing.T) { + sync := &Sync{machine: "desk"} + sessions := map[string]db.Session{ + "plain": { + ID: "plain", Machine: "local", + }, + "remote-a1b2c3~imported": { + ID: "remote-a1b2c3~imported", Machine: "remote-a1b2c3", + }, + } + imported := map[string]struct{}{"remote-a1b2c3~imported": {}} + + got := sync.pushIdentityOwnerCandidateIDs( + sessions, imported, "desk-origin", "marker", []string{"old-desk"}, + ) + assert.ElementsMatch(t, []string{ + "desk-origin~plain", + "old-desk~desk-origin~plain", + "plain", + "remote-a1b2c3~imported", + "old-desk~remote-a1b2c3~imported", + "imported", + }, got) +} + func (pushSessionProbeTx) Commit() error { return nil } func (pushSessionProbeTx) Rollback() error { return nil } @@ -1003,7 +1469,7 @@ func TestSessionPushFingerprintDiffers(t *testing.T) { CreatedAt: "2026-03-11T12:00:00Z", } - fp1 := sessionPushFingerprint(base, base.Machine, "", "", "") + fp1 := sessionPushFingerprint(base, base.ID, base.Machine, "", "", "") tests := []struct { name string @@ -1099,13 +1565,14 @@ func TestSessionPushFingerprintDiffers(t *testing.T) { for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { modified := tc.modify(base) - fp2 := sessionPushFingerprint(modified, modified.Machine, "", "", "") + fp2 := sessionPushFingerprint( + modified, modified.ID, modified.Machine, "", "", "") require.NotEqual(t, fp1, fp2, "fingerprint should differ after %s", tc.name) }) } - assert.Equal(t, fp1, sessionPushFingerprint(base, base.Machine, "", "", ""), + assert.Equal(t, fp1, sessionPushFingerprint(base, base.ID, base.Machine, "", "", ""), "identical sessions should produce identical fingerprints") } @@ -1125,24 +1592,24 @@ func TestSessionPushFingerprintIgnoresVolatileStatFields(t *testing.T) { LocalModifiedAt: &localModifiedAt, CreatedAt: "2026-03-11T12:00:00Z", } - baseFP := sessionPushFingerprint(base, base.Machine, "", "", "deps") + baseFP := sessionPushFingerprint(base, base.ID, base.Machine, "", "", "deps") statOnlyMtime := int64(1700000001000000000) statOnlyModifiedAt := "2026-03-11T12:00:01.000Z" statOnly := base statOnly.FileMtime = &statOnlyMtime statOnly.LocalModifiedAt = &statOnlyModifiedAt - assert.Equal(t, baseFP, sessionPushFingerprint(statOnly, statOnly.Machine, "", "", "deps"), + assert.Equal(t, baseFP, sessionPushFingerprint(statOnly, statOnly.ID, statOnly.Machine, "", "", "deps"), "file stat churn should not change push candidacy") contentChanged := statOnly contentChanged.MessageCount++ assert.NotEqual(t, baseFP, - sessionPushFingerprint(contentChanged, contentChanged.Machine, "", "", "deps"), + sessionPushFingerprint(contentChanged, contentChanged.ID, contentChanged.Machine, "", "", "deps"), "content changes should still change push candidacy") assert.NotEqual(t, baseFP, - sessionPushFingerprint(statOnly, statOnly.Machine, "", "", "changed-deps"), + sessionPushFingerprint(statOnly, statOnly.ID, statOnly.Machine, "", "", "changed-deps"), "dependent row changes should still change push candidacy") } @@ -1181,7 +1648,7 @@ func TestLocalSessionDependencyPushFingerprintTracksMessageEditsWithoutFileHash( CreatedAt: "2026-03-11T12:00:00Z", } fpBefore := sessionPushFingerprint( - session, session.Machine, "", "", depsBefore, + session, session.ID, session.Machine, "", "", depsBefore, ) require.NoError(t, localDB.ReplaceSessionMessages(sessID, []db.Message{{ @@ -1195,7 +1662,7 @@ func TestLocalSessionDependencyPushFingerprintTracksMessageEditsWithoutFileHash( ) require.NoError(t, err) fpAfter := sessionPushFingerprint( - session, session.Machine, "", "", depsAfter, + session, session.ID, session.Machine, "", "", depsAfter, ) assert.NotEqual(t, depsBefore, depsAfter) @@ -1216,12 +1683,29 @@ func TestSessionPushFingerprintIncludesUsageEventFingerprint( CreatedAt: "2026-03-11T12:00:00Z", } - withoutUsage := sessionPushFingerprint(base, base.Machine, "", "", "") - withUsage := sessionPushFingerprint(base, base.Machine, "usage-fp", "", "") + withoutUsage := sessionPushFingerprint(base, base.ID, base.Machine, "", "", "") + withUsage := sessionPushFingerprint(base, base.ID, base.Machine, "usage-fp", "", "") assert.NotEqual(t, withoutUsage, withUsage, "usage event fingerprint should affect session fingerprint") } +func TestSessionPushFingerprintTracksResolvedID(t *testing.T) { + base := db.Session{ + ID: "sess-001", + Project: "proj", + Machine: "laptop", + Agent: "claude", + CreatedAt: "2026-03-11T12:00:00Z", + } + + native := sessionPushFingerprint(base, base.ID, base.Machine, "", "", "") + prefixed := sessionPushFingerprint( + base, prefixedSessionID(base.Machine, base.ID), base.Machine, "", "", "", + ) + assert.NotEqual(t, native, prefixed, + "resolved PG id must affect session fingerprint") +} + func TestSessionPushFingerprintTracksResolvedMachine(t *testing.T) { sentinel := db.Session{ ID: "sess-001", @@ -1231,9 +1715,11 @@ func TestSessionPushFingerprintTracksResolvedMachine(t *testing.T) { CreatedAt: "2026-03-11T12:00:00Z", } fpA := sessionPushFingerprint( - sentinel, pushedSessionMachine(sentinel, "host-a"), "", "", "") + sentinel, sentinel.ID, + pushedSessionMachine(sentinel, "host-a"), "", "", "") fpB := sessionPushFingerprint( - sentinel, pushedSessionMachine(sentinel, "host-b"), "", "", "") + sentinel, sentinel.ID, + pushedSessionMachine(sentinel, "host-b"), "", "", "") assert.NotEqual(t, fpA, fpB, "sentinel session fingerprint must change with the fallback machine") @@ -1245,9 +1731,11 @@ func TestSessionPushFingerprintTracksResolvedMachine(t *testing.T) { CreatedAt: "2026-03-11T12:00:00Z", } fp1 := sessionPushFingerprint( - real, pushedSessionMachine(real, "host-a"), "", "", "") + real, real.ID, + pushedSessionMachine(real, "host-a"), "", "", "") fp2 := sessionPushFingerprint( - real, pushedSessionMachine(real, "host-b"), "", "", "") + real, real.ID, + pushedSessionMachine(real, "host-b"), "", "", "") assert.Equal(t, fp1, fp2, "a session with a real machine ignores the fallback") } @@ -1291,6 +1779,46 @@ func TestPushedSessionMachine(t *testing.T) { } } +func TestPrefixedSessionID(t *testing.T) { + tests := []struct { + name string + machine string + id string + want string + }{ + { + name: "prefixes native id", + machine: "host-a", + id: "sess-001", + want: "host-a~sess-001", + }, + { + name: "keeps already prefixed id", + machine: "host-a", + id: "host-a~sess-001", + want: "host-a~sess-001", + }, + { + name: "keeps empty machine", + machine: "", + id: "sess-001", + want: "sess-001", + }, + { + name: "keeps empty id", + machine: "host-a", + id: "", + want: "", + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + assert.Equal(t, tc.want, prefixedSessionID(tc.machine, tc.id)) + }) + } +} + func TestSessionPushFingerprintNoFieldCollisions( t *testing.T, ) { @@ -1305,8 +1833,8 @@ func TestSessionPushFingerprintNoFieldCollisions( CreatedAt: "2026-03-11T12:00:00Z", } assert.NotEqual(t, - sessionPushFingerprint(s1, s1.Machine, "", "", ""), - sessionPushFingerprint(s2, s2.Machine, "", "", ""), + sessionPushFingerprint(s1, s1.ID, s1.Machine, "", "", ""), + sessionPushFingerprint(s2, s2.ID, s2.Machine, "", "", ""), "length-prefixed fingerprints should not collide") } @@ -1591,7 +2119,16 @@ func TestFinalizePushStateMergesPriorFingerprints( require.NoError(t, finalizePushState( store, cutoff, cycle2Sessions, priorFingerprints, - map[string]string{"sess-002": sessionPushFingerprint(cycle2Sessions[0], cycle2Sessions[0].Machine, "", "", "")}, + map[string]string{ + "sess-002": sessionPushFingerprint( + cycle2Sessions[0], + cycle2Sessions[0].ID, + cycle2Sessions[0].Machine, + "", + "", + "", + ), + }, )) raw := store.values[lastPushBoundaryStateKey] diff --git a/internal/postgres/schema.go b/internal/postgres/schema.go index af09dc0d4..16ba6d60c 100644 --- a/internal/postgres/schema.go +++ b/internal/postgres/schema.go @@ -219,10 +219,6 @@ CREATE INDEX IF NOT EXISTS idx_pinned_session CREATE INDEX IF NOT EXISTS idx_pinned_created ON pinned_messages (created_at DESC); -CREATE INDEX IF NOT EXISTS idx_pinned_source_uuid - ON pinned_messages (session_id, source_uuid) - WHERE source_uuid <> ''; - CREATE TABLE IF NOT EXISTS model_pricing ( model_pattern TEXT PRIMARY KEY, input_per_mtok DOUBLE PRECISION NOT NULL DEFAULT 0, @@ -693,6 +689,11 @@ func EnsureSchema( `thinking_text TEXT NOT NULL DEFAULT ''`, "adding messages.thinking_text", }, + { + "pinned_messages", "source_uuid", + `source_uuid TEXT NOT NULL DEFAULT ''`, + "adding pinned_messages.source_uuid", + }, { "sessions", "termination_status", `termination_status TEXT`, @@ -870,12 +871,17 @@ func createPartialIndexesPG(ctx context.Context, db *sql.DB) error { ON sessions(cwd) WHERE cwd != ''`, `CREATE INDEX IF NOT EXISTS idx_sessions_project_git_branch ON sessions(project, git_branch) WHERE git_branch != ''`, + `CREATE INDEX IF NOT EXISTS idx_sessions_source_session + ON sessions(source_session_id) WHERE source_session_id != ''`, `CREATE INDEX IF NOT EXISTS idx_messages_compact_boundary ON messages(session_id, ordinal) WHERE is_compact_boundary = TRUE`, `CREATE INDEX IF NOT EXISTS idx_messages_sidechain ON messages(session_id) WHERE is_sidechain = TRUE`, `CREATE INDEX IF NOT EXISTS idx_messages_source_uuid ON messages(source_uuid) WHERE source_uuid != ''`, + `CREATE INDEX IF NOT EXISTS idx_pinned_source_uuid + ON pinned_messages(session_id, source_uuid) + WHERE source_uuid <> ''`, `CREATE INDEX IF NOT EXISTS idx_messages_usage_covering ON messages(timestamp, session_id, ordinal, model, claude_message_id, claude_request_id) @@ -889,6 +895,12 @@ func createPartialIndexesPG(ctx context.Context, db *sql.DB) error { // SQLite partial index so legacy schemas migrate cleanly. `CREATE INDEX IF NOT EXISTS idx_tool_calls_file_path ON tool_calls(file_path) WHERE file_path IS NOT NULL`, + `CREATE INDEX IF NOT EXISTS idx_tool_calls_subagent_session + ON tool_calls(subagent_session_id) + WHERE subagent_session_id IS NOT NULL`, + `CREATE INDEX IF NOT EXISTS idx_tool_result_events_subagent_session + ON tool_result_events(subagent_session_id) + WHERE subagent_session_id IS NOT NULL`, // idx_messages_session_role backs the dense-flow unit-range boundary // fetch (user ordinals by session), mirroring the SQLite index. `CREATE INDEX IF NOT EXISTS idx_messages_session_role diff --git a/internal/postgres/schema_pgtest_test.go b/internal/postgres/schema_pgtest_test.go index 491cc9af2..103138155 100644 --- a/internal/postgres/schema_pgtest_test.go +++ b/internal/postgres/schema_pgtest_test.go @@ -123,3 +123,54 @@ func TestToolCallsFilePathIndex(t *testing.T) { require.NoError(t, err, "checking idx_tool_calls_file_path") assert.True(t, exists, "idx_tool_calls_file_path index missing") } + +func TestEnsureSchemaMigratesPinnedMessageSourceUUIDBeforeIndex(t *testing.T) { + pgURL := testPGURL(t) + cleanSchemaTestPG(t, pgURL) + t.Cleanup(func() { cleanSchemaTestPG(t, pgURL) }) + + pg, err := Open(pgURL, schemaTestSchema, true) + require.NoError(t, err, "connecting to pg") + defer pg.Close() + + ctx := context.Background() + _, err = pg.ExecContext(ctx, + `CREATE SCHEMA IF NOT EXISTS `+schemaTestSchema) + require.NoError(t, err, "creating test schema") + _, err = pg.ExecContext(ctx, ` + CREATE TABLE pinned_messages ( + id BIGSERIAL PRIMARY KEY, + session_id TEXT NOT NULL, + message_id INT NOT NULL, + ordinal INT NOT NULL, + note TEXT, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + UNIQUE (session_id, message_id) + )`) + require.NoError(t, err, "creating legacy pinned_messages table") + + require.NoError(t, EnsureSchema(ctx, pg, schemaTestSchema), + "EnsureSchema should add source_uuid before creating its index") + + var columnExists bool + err = pg.QueryRowContext(ctx, ` + SELECT EXISTS ( + SELECT 1 FROM information_schema.columns + WHERE table_schema = $1 + AND table_name = 'pinned_messages' + AND column_name = 'source_uuid' + )`, schemaTestSchema).Scan(&columnExists) + require.NoError(t, err, "checking pinned_messages.source_uuid") + assert.True(t, columnExists, "pinned_messages.source_uuid missing") + + var indexExists bool + err = pg.QueryRowContext(ctx, ` + SELECT EXISTS ( + SELECT 1 FROM pg_indexes + WHERE schemaname = $1 + AND tablename = 'pinned_messages' + AND indexname = 'idx_pinned_source_uuid' + )`, schemaTestSchema).Scan(&indexExists) + require.NoError(t, err, "checking idx_pinned_source_uuid") + assert.True(t, indexExists, "idx_pinned_source_uuid index missing") +} diff --git a/internal/postgres/schema_test.go b/internal/postgres/schema_test.go index 7b0f6ab98..89193cc75 100644 --- a/internal/postgres/schema_test.go +++ b/internal/postgres/schema_test.go @@ -6,6 +6,7 @@ import ( "database/sql/driver" "errors" "io" + "slices" "strings" "sync" "testing" @@ -109,19 +110,20 @@ func (c *schemaProbeConn) Begin() (driver.Tx, error) { func (c *schemaProbeConn) ExecContext( _ context.Context, query string, args []driver.NamedValue, ) (driver.Result, error) { + normalized := strings.ToLower(query) + if strings.Contains(normalized, "idx_pinned_source_uuid") && + c.state.hasColumn("pinned_messages", "session_id") && + !c.state.hasColumn("pinned_messages", "source_uuid") { + return nil, errors.New(`ERROR: column "source_uuid" does not exist (SQLSTATE 42703)`) + } c.state.mu.Lock() c.state.execs = append(c.state.execs, query) c.state.execArgs = append( c.state.execArgs, append([]driver.NamedValue(nil), args...), ) c.state.mu.Unlock() - normalized := strings.ToLower(query) if strings.Contains(normalized, "alter table") { - c.state.mu.Lock() - c.state.alterTableExecs = append( - c.state.alterTableExecs, query, - ) - c.state.mu.Unlock() + c.state.recordAlterTable(query) } if strings.Contains(normalized, "insert into sync_metadata") && len(args) > 0 { @@ -136,6 +138,65 @@ func (c *schemaProbeConn) ExecContext( return driver.RowsAffected(0), nil } +func (s *schemaProbeState) hasColumn(table, column string) bool { + s.mu.Lock() + defer s.mu.Unlock() + return slices.Contains(s.existingColumnNames[table], column) +} + +func (s *schemaProbeState) recordAlterTable(query string) { + s.mu.Lock() + defer s.mu.Unlock() + s.alterTableExecs = append(s.alterTableExecs, query) + + table, ok := alterTableName(query) + if !ok { + return + } + if s.existingColumnNames == nil { + s.existingColumnNames = map[string][]string{} + } + for _, column := range alterTableColumns(query) { + exists := slices.Contains(s.existingColumnNames[table], column) + if !exists { + s.existingColumnNames[table] = append( + s.existingColumnNames[table], column, + ) + } + } +} + +func alterTableName(query string) (string, bool) { + const prefix = `ALTER TABLE "` + _, after, ok := strings.Cut(query, prefix) + if !ok { + return "", false + } + rest := after + before0, _, ok0 := strings.Cut(rest, `"`) + if !ok0 { + return "", false + } + return before0, true +} + +func alterTableColumns(query string) []string { + const marker = "ADD COLUMN IF NOT EXISTS " + parts := strings.Split(query, marker) + if len(parts) < 2 { + return nil + } + columns := make([]string, 0, len(parts)-1) + for _, part := range parts[1:] { + fields := strings.Fields(part) + if len(fields) == 0 { + continue + } + columns = append(columns, strings.Trim(fields[0], `",`)) + } + return columns +} + func (c *schemaProbeConn) QueryContext( _ context.Context, query string, args []driver.NamedValue, ) (driver.Rows, error) { @@ -769,6 +830,12 @@ func TestEnsureSchemaCreatesSessionTraversalIndex(t *testing.T) { assert.Contains(t, state.executedSQL(), "CREATE INDEX IF NOT EXISTS idx_sessions_parent") + assert.Contains(t, state.executedSQL(), + "CREATE INDEX IF NOT EXISTS idx_sessions_source_session") + assert.Contains(t, state.executedSQL(), + "CREATE INDEX IF NOT EXISTS idx_tool_calls_subagent_session") + assert.Contains(t, state.executedSQL(), + "CREATE INDEX IF NOT EXISTS idx_tool_result_events_subagent_session") } func TestEnsureSchemaGroupsMissingColumnMigrationsByTable(t *testing.T) { @@ -808,6 +875,9 @@ func TestEnsureSchemaGroupsMissingColumnMigrationsByTable(t *testing.T) { "tool_calls": { "call_index", "file_path", }, + "pinned_messages": { + "source_uuid", + }, }) require.NoError(t, EnsureSchema(context.Background(), db, "agentsview")) @@ -819,3 +889,25 @@ func TestEnsureSchemaGroupsMissingColumnMigrationsByTable(t *testing.T) { // it contributes no ALTER. assert.Equal(t, 2, state.alterTableExecCount(), "ALTER TABLE execs") } + +func TestEnsureSchemaMigratesPinnedMessageSourceUUID(t *testing.T) { + db, state := newSchemaProbeDB(t, map[string][]string{ + "sessions": { + "has_total_output_tokens", + "has_peak_context_tokens", + }, + "messages": { + "has_context_tokens", + "has_output_tokens", + }, + "pinned_messages": { + "id", "session_id", "message_id", "ordinal", + "note", "created_at", + }, + }) + + require.NoError(t, EnsureSchema(context.Background(), db, "agentsview")) + + assert.Contains(t, state.executedSQL(), + "ALTER TABLE \"pinned_messages\" ADD COLUMN IF NOT EXISTS source_uuid TEXT NOT NULL DEFAULT ''") +} diff --git a/internal/postgres/sessions.go b/internal/postgres/sessions.go index 7af2e9fb5..adf162d51 100644 --- a/internal/postgres/sessions.go +++ b/internal/postgres/sessions.go @@ -1088,6 +1088,31 @@ func (s *Store) GetAgents( } // GetMachines returns distinct machine names. +// MachineSessionCounts returns the number of non-deleted sessions per machine, +// keyed by machine name. +func (s *Store) MachineSessionCounts(ctx context.Context) (map[string]int, error) { + rows, err := s.pg.QueryContext(ctx, + `SELECT machine, COUNT(*) FROM sessions + WHERE deleted_at IS NULL + GROUP BY machine`, + ) + if err != nil { + return nil, err + } + defer rows.Close() + + counts := map[string]int{} + for rows.Next() { + var machine string + var count int + if err := rows.Scan(&machine, &count); err != nil { + return nil, err + } + counts[machine] = count + } + return counts, rows.Err() +} + func (s *Store) GetMachines( ctx context.Context, excludeOneShot, excludeAutomated bool, diff --git a/internal/postgres/store.go b/internal/postgres/store.go index b214188d5..072e43a88 100644 --- a/internal/postgres/store.go +++ b/internal/postgres/store.go @@ -451,6 +451,51 @@ func (s *Store) SoftDeleteSessions(ids []string) (int, error) { return total, nil } +// SoftDeleteSessionsReturningIDs moves multiple sessions to the trash and +// returns the IDs that changed state. +func (s *Store) SoftDeleteSessionsReturningIDs(ids []string) ([]string, error) { + if len(ids) == 0 { + return nil, nil + } + deleted := make([]string, 0, len(ids)) + const batchSize = 500 + for start := 0; start < len(ids); start += batchSize { + end := min(start+batchSize, len(ids)) + pb := ¶mBuilder{} + placeholders := make([]string, 0, end-start) + for _, id := range ids[start:end] { + placeholders = append(placeholders, pb.add(id)) + } + rows, err := s.pg.Query( + `UPDATE sessions + SET deleted_at = NOW(), + updated_at = NOW() + WHERE id IN (`+strings.Join(placeholders, ",")+ + `) AND deleted_at IS NULL + RETURNING id`, + pb.args..., + ) + if err != nil { + return deleted, mapPGWriteError("soft deleting sessions", err) + } + for rows.Next() { + var id string + if err := rows.Scan(&id); err != nil { + _ = rows.Close() + return deleted, fmt.Errorf("scanning soft deleted session: %w", err) + } + deleted = append(deleted, id) + } + if err := rows.Close(); err != nil { + return deleted, fmt.Errorf("closing soft deleted session rows: %w", err) + } + if err := rows.Err(); err != nil { + return deleted, fmt.Errorf("iterating soft deleted sessions: %w", err) + } + } + return deleted, nil +} + // RestoreSession restores a trashed session. func (s *Store) RestoreSession(id string) (int64, error) { res, err := s.pg.Exec( diff --git a/internal/postgres/sync.go b/internal/postgres/sync.go index d70f122dc..1322ac09b 100644 --- a/internal/postgres/sync.go +++ b/internal/postgres/sync.go @@ -52,6 +52,7 @@ func (s *scopedSyncStateStore) ensureMigration() error { "last_push_at", lastPushBoundaryStateKey, lastPushTargetFingerprintKey, + artifactIdentityModeStateKey, } { scopedKey := s.scopedKey(key) scopedValue, err := s.base.GetSyncState(scopedKey) diff --git a/internal/server/artifact_http_transport_test.go b/internal/server/artifact_http_transport_test.go new file mode 100644 index 000000000..53b8e2d38 --- /dev/null +++ b/internal/server/artifact_http_transport_test.go @@ -0,0 +1,192 @@ +package server_test + +import ( + "context" + "encoding/json" + "net/http/httptest" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "go.kenn.io/agentsview/internal/artifact" + "go.kenn.io/agentsview/internal/db" + "go.kenn.io/agentsview/internal/dbtest" +) + +// newClientNode opens a client-only artifact store (a db plus data dir) that +// drives artifact.Sync against an HTTP peer. +func newClientNode(t *testing.T, sessionID, project string) (*db.DB, string) { + t.Helper() + dir := t.TempDir() + database, err := db.Open(filepath.Join(dir, "client.db")) + require.NoError(t, err) + t.Cleanup(func() { database.Close() }) + dbtest.SeedSession(t, database, sessionID, project, func(s *db.Session) { + s.MessageCount = 2 + s.UserMessageCount = 1 + }) + require.NoError(t, database.ReplaceSessionMessages(sessionID, []db.Message{ + {SessionID: sessionID, Ordinal: 0, Role: "user", Content: "hello", ContentLength: 5}, + {SessionID: sessionID, Ordinal: 1, Role: "assistant", Content: "world", ContentLength: 5}, + })) + return database, dir +} + +func TestArtifactHTTPTransportSyncsSessionsAndMetadata(t *testing.T) { + ctx := context.Background() + const token = "secret" + const aOrigin = "laptop-a1b2c3" + + // Node B: a real server exposing the artifact peer API behind auth. + te := setup(t, withAuth(token), withArtifactOrigin("desktop-d4e5f6")) + peer := httptest.NewServer(te.srv.Handler()) + defer peer.Close() + + aDB, aDir := newClientNode(t, "sess-1", "alpha") + + syncToPeer := func() { + _, err := artifact.Sync(ctx, aDB, artifact.SyncOptions{ + DataDir: aDir, + Target: peer.URL, + Origin: aOrigin, + Token: token, + }) + require.NoError(t, err) + } + + // A pushes its session over HTTP; B imports it on receipt. + syncToPeer() + importedID := aOrigin + "~sess-1" + gotB, err := te.db.GetSession(ctx, importedID) + require.NoError(t, err) + require.NotNil(t, gotB, "peer should import the pushed session") + assert.Equal(t, "alpha", gotB.Project) + + // A renames the session and syncs again; the metadata event is enumerated + // via the index route, posted, and replayed on B. + display := "Renamed on A" + require.NoError(t, aDB.RenameSession("sess-1", &display)) + recorder := artifact.NewMetadataRecorder(aDB, artifact.MetadataRecorderOptions{ + DataDir: aDir, + Origin: aOrigin, + }) + value, err := json.Marshal(struct { + DisplayName *string `json:"display_name"` + }{DisplayName: &display}) + require.NoError(t, err) + _, err = recorder.Append(ctx, artifact.MetadataEventInput{ + SessionID: "sess-1", + Op: artifact.MetadataOpRename, + Value: value, + }) + require.NoError(t, err) + + syncToPeer() + gotB, err = te.db.GetSession(ctx, importedID) + require.NoError(t, err) + require.NotNil(t, gotB) + require.NotNil(t, gotB.DisplayName) + assert.Equal(t, display, *gotB.DisplayName) +} + +func TestArtifactHTTPTransportPullsRemoteSessions(t *testing.T) { + ctx := context.Background() + const token = "secret" + const bOrigin = "desktop-d4e5f6" + + // Node B owns a session but has not run a separate artifact publisher. + te := setup(t, withAuth(token), withArtifactOrigin(bOrigin)) + peer := httptest.NewServer(te.srv.Handler()) + defer peer.Close() + dbtest.SeedSession(t, te.db, "remote-1", "bravo", func(s *db.Session) { + s.MessageCount = 2 + s.UserMessageCount = 1 + }) + require.NoError(t, te.db.ReplaceSessionMessages("remote-1", []db.Message{ + {SessionID: "remote-1", Ordinal: 0, Role: "user", Content: "ping", ContentLength: 4}, + {SessionID: "remote-1", Ordinal: 1, Role: "assistant", Content: "pong", ContentLength: 4}, + })) + displayName := "Renamed before HTTP publishing" + require.NoError(t, te.db.RenameSession("remote-1", &displayName)) + + aDB, aDir := newClientNode(t, "sess-1", "alpha") + _, err := artifact.Sync(ctx, aDB, artifact.SyncOptions{ + DataDir: aDir, + Target: peer.URL, + Origin: "laptop-a1b2c3", + Token: token, + }) + require.NoError(t, err) + + // A pulled B's session. + gotA, err := aDB.GetSession(ctx, bOrigin+"~remote-1") + require.NoError(t, err) + require.NotNil(t, gotA, "client should pull the remote session") + assert.Equal(t, "bravo", gotA.Project) + require.NotNil(t, gotA.DisplayName) + assert.Equal(t, displayName, *gotA.DisplayName) +} + +func TestArtifactHTTPTransportRejectsBadToken(t *testing.T) { + te := setup(t, withAuth("secret"), withArtifactOrigin("desktop-d4e5f6")) + peer := httptest.NewServer(te.srv.Handler()) + defer peer.Close() + aDB, aDir := newClientNode(t, "sess-1", "alpha") + + _, err := artifact.Sync(context.Background(), aDB, artifact.SyncOptions{ + DataDir: aDir, + Target: peer.URL, + Origin: "laptop-a1b2c3", + Token: "wrong", + }) + require.Error(t, err) + assert.Contains(t, err.Error(), "peer") +} + +func TestArtifactHTTPTransportPullRepairsCorruptOwnedArtifact(t *testing.T) { + ctx := context.Background() + const token = "secret" + const bOrigin = "desktop-d4e5f6" + + te := setup(t, withAuth(token), withArtifactOrigin(bOrigin)) + peer := httptest.NewServer(te.srv.Handler()) + defer peer.Close() + dbtest.SeedSession(t, te.db, "remote-1", "bravo", func(s *db.Session) { + s.MessageCount = 2 + s.UserMessageCount = 1 + }) + require.NoError(t, te.db.ReplaceSessionMessages("remote-1", []db.Message{ + {SessionID: "remote-1", Ordinal: 0, Role: "user", Content: "ping", ContentLength: 4}, + {SessionID: "remote-1", Ordinal: 1, Role: "assistant", Content: "pong", ContentLength: 4}, + })) + _, err := artifact.Export(ctx, te.db, filepath.Join(te.dataDir, "artifacts"), bOrigin) + require.NoError(t, err) + + // Corrupt the stored segment so the real server's GET-side content + // validation fails for it. + segments, err := filepath.Glob(filepath.Join( + te.dataDir, "artifacts", bOrigin, "segments", "*")) + require.NoError(t, err) + require.Len(t, segments, 1) + require.NoError(t, os.WriteFile(segments[0], []byte("corrupt"), 0o644)) + + // Peer discovery refreshes owned artifacts, so the corrupt segment is + // quarantined and regenerated before the client enumerates the store. + aDB, aDir := newClientNode(t, "sess-1", "alpha") + _, err = artifact.Sync(ctx, aDB, artifact.SyncOptions{ + DataDir: aDir, + Target: peer.URL, + Origin: "laptop-a1b2c3", + Token: token, + }) + require.NoError(t, err) + got, err := aDB.GetSession(ctx, bOrigin+"~remote-1") + require.NoError(t, err) + require.NotNil(t, got) + assert.Equal(t, "bravo", got.Project) + assert.FileExists(t, segments[0]) + assert.FileExists(t, segments[0]+".corrupt") +} diff --git a/internal/server/artifact_peer_test.go b/internal/server/artifact_peer_test.go new file mode 100644 index 000000000..b6a8b99cd --- /dev/null +++ b/internal/server/artifact_peer_test.go @@ -0,0 +1,610 @@ +package server_test + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "net/http" + "net/http/httptest" + "net/url" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "go.kenn.io/agentsview/internal/artifact" + "go.kenn.io/agentsview/internal/config" + "go.kenn.io/agentsview/internal/db" + "go.kenn.io/agentsview/internal/dbtest" + "go.kenn.io/agentsview/internal/server" +) + +type artifactOriginsBody struct { + Origins []string `json:"origins"` +} + +type artifactPostBody struct { + Origin string `json:"origin"` + Kind string `json:"kind"` + Name string `json:"name"` + Hash string `json:"hash,omitempty"` + Size int64 `json:"size"` + Duplicate bool `json:"duplicate"` +} + +type artifactRemoteStore struct { + db.Store +} + +func (artifactRemoteStore) ReadOnly() bool { return true } + +func (artifactRemoteStore) MachineSessionCounts( + context.Context, +) (map[string]int, error) { + return map[string]int{}, nil +} + +func (artifactRemoteStore) CountMetadataConflicts(context.Context) (int, error) { + return 0, nil +} + +func TestArtifactPeerRoutesRequireBearerAuthWhenConfigured(t *testing.T) { + te := setup(t, withAuth("secret")) + + w := artifactPeerRequest(t, te, http.MethodGet, "/api/v1/artifacts/origins", nil, "") + assertStatus(t, w, http.StatusUnauthorized) + + w = artifactPeerRequest(t, te, http.MethodGet, "/api/v1/artifacts/origins", nil, "secret") + assertStatus(t, w, http.StatusOK) +} + +func TestArtifactPeerRoutesPostDuplicateAndFetch(t *testing.T) { + te := setup(t, withAuth("secret")) + origin := "peer-a1b2c3" + metadataBody, metadataName := peerMetadataArtifact( + origin, + "2026-06-14T010203.000000001Z-00000000000000000000", + ) + + w := artifactPeerRequest( + t, te, http.MethodPost, + "/api/v1/artifacts/"+origin+"/meta/"+url.PathEscape(metadataName), + metadataBody, "secret", + ) + assertStatus(t, w, http.StatusOK) + posted := decode[artifactPostBody](t, w) + assert.False(t, posted.Duplicate) + assert.Equal(t, "meta", posted.Kind) + assert.Equal(t, metadataName, posted.Name) + + w = artifactPeerRequest( + t, te, http.MethodPost, + "/api/v1/artifacts/"+origin+"/meta/"+url.PathEscape(metadataName), + metadataBody, "secret", + ) + assertStatus(t, w, http.StatusOK) + posted = decode[artifactPostBody](t, w) + assert.True(t, posted.Duplicate) + + w = artifactPeerRequest( + t, te, http.MethodGet, + "/api/v1/artifacts/"+origin+"/meta/"+url.PathEscape(metadataName), + nil, "secret", + ) + assertStatus(t, w, http.StatusOK) + assert.Equal(t, "application/json", w.Header().Get("Content-Type")) + assert.Equal(t, metadataBody, w.Body.Bytes()) + + checkpoint := []byte(`{"origin":"peer-a1b2c3","seq":1,"sessions":{},"v":1}` + "\n") + w = artifactPeerRequest( + t, te, http.MethodPost, + "/api/v1/artifacts/"+origin+"/checkpoints/cp-0000000001", + checkpoint, "secret", + ) + assertStatus(t, w, http.StatusOK) + + w = artifactPeerRequest( + t, te, http.MethodGet, + "/api/v1/artifacts/"+origin+"/checkpoint", + nil, "secret", + ) + assertStatus(t, w, http.StatusOK) + assert.Equal(t, checkpoint, w.Body.Bytes()) + + w = artifactPeerRequest(t, te, http.MethodGet, "/api/v1/artifacts/origins", nil, "secret") + assertStatus(t, w, http.StatusOK) + origins := decode[artifactOriginsBody](t, w) + assert.Contains(t, origins.Origins, origin) +} + +func TestArtifactPeerPostRejectsRemoteStoreBeforeWrite(t *testing.T) { + dir := tempDirWithRetryCleanup(t) + cfg := config.Config{ + Host: "127.0.0.1", + Port: 0, + DataDir: dir, + WriteTimeout: 30 * time.Second, + } + srv := server.New(cfg, artifactRemoteStore{}, nil) + te := &testEnv{ + srv: srv, + handler: wrapTestHandler(cfg, srv.Handler()), + dataDir: dir, + } + origin := "peer-a1b2c3" + metadataBody, metadataName := peerMetadataArtifact( + origin, + "2026-06-14T010203.000000001Z-00000000000000000000", + ) + + w := artifactPeerRequest( + t, te, http.MethodPost, + "/api/v1/artifacts/"+origin+"/meta/"+url.PathEscape(metadataName), + metadataBody, "", + ) + + assertStatus(t, w, http.StatusNotImplemented) + assertArtifactMissing(t, dir, origin, "meta", metadataName) +} + +func TestArtifactPeerReadRoutesRejectRemoteStoreBeforeReadingFiles(t *testing.T) { + dir := tempDirWithRetryCleanup(t) + cfg := config.Config{ + Host: "127.0.0.1", + Port: 0, + DataDir: dir, + WriteTimeout: 30 * time.Second, + } + origin := "peer-a1b2c3" + checkpoint := []byte(`{"origin":"peer-a1b2c3","seq":1,"sessions":{},"v":1}` + "\n") + _, err := artifact.WriteArtifact( + filepath.Join(dir, "artifacts"), + origin, + "checkpoints", + "cp-0000000001", + checkpoint, + ) + require.NoError(t, err) + + srv := server.New(cfg, artifactRemoteStore{}, nil) + te := &testEnv{ + srv: srv, + handler: wrapTestHandler(cfg, srv.Handler()), + dataDir: dir, + } + + for _, path := range []string{ + "/api/v1/artifacts/origins", + "/api/v1/artifacts/peers", + "/api/v1/artifacts/" + origin + "/index", + "/api/v1/artifacts/" + origin + "/checkpoint", + "/api/v1/artifacts/" + origin + "/checkpoints/cp-0000000001", + } { + w := artifactPeerRequest(t, te, http.MethodGet, path, nil, "") + assertStatus(t, w, http.StatusNotImplemented) + } +} + +func TestArtifactPeerPostRejectsReadOnlySQLiteBeforeWrite(t *testing.T) { + dir := tempDirWithRetryCleanup(t) + dbPath := filepath.Join(dir, "test.db") + writable, err := db.Open(dbPath) + require.NoError(t, err) + require.NoError(t, writable.Close()) + readonly, err := db.OpenReadOnly(dbPath) + require.NoError(t, err) + t.Cleanup(func() { readonly.Close() }) + + cfg := config.Config{ + Host: "127.0.0.1", + Port: 0, + DataDir: dir, + DBPath: dbPath, + ArtifactOriginID: "desktop-d4e5f6", + WriteTimeout: 30 * time.Second, + } + srv := server.New(cfg, readonly, nil) + te := &testEnv{ + srv: srv, + handler: wrapTestHandler(cfg, srv.Handler()), + db: readonly, + dataDir: dir, + } + origin := "peer-a1b2c3" + metadataBody, metadataName := peerMetadataArtifact( + origin, + "2026-06-14T010203.000000001Z-00000000000000000000", + ) + + w := artifactPeerRequest( + t, te, http.MethodPost, + "/api/v1/artifacts/"+origin+"/meta/"+url.PathEscape(metadataName), + metadataBody, "", + ) + + assertStatus(t, w, http.StatusNotImplemented) + assertArtifactMissing(t, dir, origin, "meta", metadataName) +} + +type artifactPeerBody struct { + Origin string `json:"origin"` + IsLocal bool `json:"is_local"` + CheckpointSeq int `json:"checkpoint_seq"` + PublishedSessions int `json:"published_sessions"` + LocalSessions int `json:"local_sessions"` + LastPublished string `json:"last_published"` +} + +type artifactPeersBody struct { + LocalOrigin string `json:"local_origin"` + Peers []artifactPeerBody `json:"peers"` + ConflictCount int `json:"conflict_count"` +} + +func TestArtifactPeersStatus(t *testing.T) { + local := "desktop-d4e5f6" + te := setup(t, withArtifactOrigin(local)) + ctx := context.Background() + artifactRoot := filepath.Join(te.dataDir, "artifacts") + first := "hi" + + // Two owned sessions, exported so the local origin gets a checkpoint. + dbtest.SeedSession(t, te.db, "local-1", "proj", func(s *db.Session) { s.FirstMessage = &first }) + dbtest.SeedSession(t, te.db, "local-2", "proj", func(s *db.Session) { s.FirstMessage = &first }) + exported, err := artifact.Export(ctx, te.db, artifactRoot, local) + require.NoError(t, err) + require.Equal(t, 2, exported) + + // A foreign peer publishes one session that the server imports. + origin := "peer-a1b2c3" + peerRoot := t.TempDir() + peerDB, err := db.Open(filepath.Join(t.TempDir(), "peer.db")) + require.NoError(t, err) + t.Cleanup(func() { peerDB.Close() }) + dbtest.SeedSession(t, peerDB, "sess-1", "alpha", func(s *db.Session) { s.FirstMessage = &first }) + require.NoError(t, peerDB.ReplaceSessionMessages("sess-1", []db.Message{ + {SessionID: "sess-1", Ordinal: 0, Role: "user", Content: "hello", ContentLength: 5}, + })) + _, err = artifact.Export(ctx, peerDB, peerRoot, origin) + require.NoError(t, err) + postArtifactFile(t, te, origin, "segments", oneArtifactPath(t, peerRoot, origin, "segments", "*")) + postArtifactFile(t, te, origin, "manifests", oneArtifactPath(t, peerRoot, origin, "manifests", "*")) + postArtifactFile(t, te, origin, "checkpoints", oneArtifactPath(t, peerRoot, origin, "checkpoints", "*")) + + w := artifactPeerRequest(t, te, http.MethodGet, "/api/v1/artifacts/peers", nil, "") + assertStatus(t, w, http.StatusOK) + body := decode[artifactPeersBody](t, w) + + assert.Equal(t, local, body.LocalOrigin) + assert.Equal(t, 0, body.ConflictCount) + require.Len(t, body.Peers, 2) + + byOrigin := map[string]artifactPeerBody{} + for _, p := range body.Peers { + byOrigin[p.Origin] = p + } + + localPeer, ok := byOrigin[local] + require.True(t, ok, "local origin present in peers") + assert.True(t, localPeer.IsLocal) + assert.Equal(t, 2, localPeer.PublishedSessions) + assert.Equal(t, 2, localPeer.LocalSessions) + assert.NotEmpty(t, localPeer.LastPublished) + + peer, ok := byOrigin[origin] + require.True(t, ok, "foreign origin present in peers") + assert.False(t, peer.IsLocal) + assert.Equal(t, 1, peer.PublishedSessions) + assert.Equal(t, 1, peer.LocalSessions) + assert.Equal(t, 1, peer.CheckpointSeq) +} + +func TestArtifactPeersStatusUsesLatestCheckpointImportProvenance(t *testing.T) { + te := setup(t, withArtifactOrigin("desktop-d4e5f6")) + ctx := context.Background() + firstMessage := "hello" + + // This peer is fully imported, then its local row is trashed. The import + // provenance still proves the published manifest landed successfully. + trashedOrigin := "trashed-a1b2c3" + trashedRoot := t.TempDir() + trashedDB, err := db.Open(filepath.Join(t.TempDir(), "trashed-peer.db")) + require.NoError(t, err) + t.Cleanup(func() { trashedDB.Close() }) + dbtest.SeedSession(t, trashedDB, "sess-1", "alpha", func(s *db.Session) { + s.FirstMessage = &firstMessage + }) + require.NoError(t, trashedDB.ReplaceSessionMessages("sess-1", []db.Message{ + {SessionID: "sess-1", Ordinal: 0, Role: "user", Content: "hello", ContentLength: 5}, + })) + _, err = artifact.Export(ctx, trashedDB, trashedRoot, trashedOrigin) + require.NoError(t, err) + postArtifactFile(t, te, trashedOrigin, "segments", + oneArtifactPath(t, trashedRoot, trashedOrigin, "segments", "*")) + postArtifactFile(t, te, trashedOrigin, "manifests", + oneArtifactPath(t, trashedRoot, trashedOrigin, "manifests", "*")) + postArtifactFile(t, te, trashedOrigin, "checkpoints", + oneArtifactPath(t, trashedRoot, trashedOrigin, "checkpoints", "*")) + require.NoError(t, te.db.SoftDeleteSession(trashedOrigin+"~sess-1")) + + // This peer's first manifest landed, but its latest checkpoint references a + // newer manifest that has not arrived. The active stale row is not current. + staleOrigin := "stale-d4e5f6" + staleRoot := t.TempDir() + staleDB, err := db.Open(filepath.Join(t.TempDir(), "stale-peer.db")) + require.NoError(t, err) + t.Cleanup(func() { staleDB.Close() }) + dbtest.SeedSession(t, staleDB, "sess-1", "before", func(s *db.Session) { + s.FirstMessage = &firstMessage + }) + require.NoError(t, staleDB.ReplaceSessionMessages("sess-1", []db.Message{ + {SessionID: "sess-1", Ordinal: 0, Role: "user", Content: "hello", ContentLength: 5}, + })) + _, err = artifact.Export(ctx, staleDB, staleRoot, staleOrigin) + require.NoError(t, err) + postArtifactFile(t, te, staleOrigin, "segments", + oneArtifactPath(t, staleRoot, staleOrigin, "segments", "*")) + postArtifactFile(t, te, staleOrigin, "manifests", + oneArtifactPath(t, staleRoot, staleOrigin, "manifests", "*")) + postArtifactFile(t, te, staleOrigin, "checkpoints", + filepath.Join(staleRoot, staleOrigin, "checkpoints", "cp-0000000001.json")) + + dbtest.SeedSession(t, staleDB, "sess-1", "after", func(s *db.Session) { + s.FirstMessage = &firstMessage + }) + exported, err := artifact.Export(ctx, staleDB, staleRoot, staleOrigin) + require.NoError(t, err) + require.Equal(t, 1, exported) + postArtifactFile(t, te, staleOrigin, "checkpoints", + filepath.Join(staleRoot, staleOrigin, "checkpoints", "cp-0000000002.json")) + require.NoError(t, te.db.UpsertSession(db.Session{ + ID: staleOrigin + "~unrelated", + Project: "unrelated", + Machine: staleOrigin, + Agent: "claude", + MessageCount: 1, + UserMessageCount: 1, + CreatedAt: "2026-07-15T12:00:00Z", + }), "seed an active same-origin row absent from the checkpoint") + + w := artifactPeerRequest(t, te, http.MethodGet, "/api/v1/artifacts/peers", nil, "") + assertStatus(t, w, http.StatusOK) + body := decode[artifactPeersBody](t, w) + byOrigin := make(map[string]artifactPeerBody, len(body.Peers)) + for _, peer := range body.Peers { + byOrigin[peer.Origin] = peer + } + + trashedPeer, ok := byOrigin[trashedOrigin] + require.True(t, ok) + assert.Equal(t, 1, trashedPeer.PublishedSessions) + assert.Equal(t, 1, trashedPeer.LocalSessions, + "a locally trashed row remains landed when its manifest provenance matches") + stalePeer, ok := byOrigin[staleOrigin] + require.True(t, ok) + assert.Equal(t, 1, stalePeer.PublishedSessions) + assert.Equal(t, 0, stalePeer.LocalSessions, + "stale and checkpoint-unrelated active rows must not count as landed") +} + +func TestArtifactPeersStatusPublishesEmptyLocalOrigin(t *testing.T) { + te := setup(t, withArtifactOrigin("desktop-d4e5f6")) + // Discovery publishes an explicit empty checkpoint for a configured origin. + w := artifactPeerRequest(t, te, http.MethodGet, "/api/v1/artifacts/peers", nil, "") + assertStatus(t, w, http.StatusOK) + body := decode[artifactPeersBody](t, w) + assert.Equal(t, "desktop-d4e5f6", body.LocalOrigin) + require.Len(t, body.Peers, 1) + assert.True(t, body.Peers[0].IsLocal) + assert.Equal(t, 0, body.Peers[0].PublishedSessions) + assert.Equal(t, 1, body.Peers[0].CheckpointSeq) + assert.NotEmpty(t, body.Peers[0].LastPublished) +} + +func TestArtifactPeerPostRejectsHashMismatch(t *testing.T) { + te := setup(t, withAuth("secret")) + origin := "peer-a1b2c3" + metadataBody, _ := peerMetadataArtifact( + origin, + "2026-06-14T010203.000000001Z-00000000000000000000", + ) + badName := "2026-06-14T010203.000000001Z-peer-a1b2c3-" + strings.Repeat("0", 64) + + w := artifactPeerRequest( + t, te, http.MethodPost, + "/api/v1/artifacts/"+origin+"/meta/"+url.PathEscape(badName), + metadataBody, "secret", + ) + assertStatus(t, w, http.StatusBadRequest) +} + +func TestArtifactPeerPostImportsAndEmitsDataChanged(t *testing.T) { + te := setup(t, withArtifactOrigin("desktop-d4e5f6")) + origin := "peer-a1b2c3" + artifactRoot := t.TempDir() + peerDB, err := db.Open(filepath.Join(t.TempDir(), "peer.db")) + require.NoError(t, err) + t.Cleanup(func() { peerDB.Close() }) + + first := "hello" + started := "2026-06-14T01:02:03Z" + ended := "2026-06-14T01:03:03Z" + dbtest.SeedSession(t, peerDB, "sess-1", "alpha", func(s *db.Session) { + s.MessageCount = 2 + s.UserMessageCount = 1 + s.FirstMessage = &first + s.StartedAt = &started + s.EndedAt = &ended + }) + require.NoError(t, peerDB.ReplaceSessionMessages("sess-1", []db.Message{ + {SessionID: "sess-1", Ordinal: 0, Role: "user", Content: "hello", ContentLength: 5}, + {SessionID: "sess-1", Ordinal: 1, Role: "assistant", Content: "world", ContentLength: 5}, + })) + _, err = artifact.Export(context.Background(), peerDB, artifactRoot, origin) + require.NoError(t, err) + + postArtifactFile(t, te, origin, "segments", + oneArtifactPath(t, artifactRoot, origin, "segments", "*")) + postArtifactFile(t, te, origin, "manifests", + oneArtifactPath(t, artifactRoot, origin, "manifests", "*")) + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + req := httptest.NewRequest(http.MethodGet, "/api/v1/events", nil).WithContext(ctx) + stream := &flushRecorder{ResponseRecorder: httptest.NewRecorder()} + done := make(chan struct{}) + go func() { + te.handler.ServeHTTP(stream, req) + close(done) + }() + time.Sleep(100 * time.Millisecond) + + postArtifactFile(t, te, origin, "checkpoints", + oneArtifactPath(t, artifactRoot, origin, "checkpoints", "*")) + te.waitForSSEEvent(t, stream, "data_changed", 3*time.Second) + + // Live clients only refresh the session index on the "sessions" + // scope and only invalidate hydrated session details on the + // "messages" scope; an import needs both. + assert.Eventually(t, func() bool { + scopes := dataChangedScopes(stream) + return scopes["messages"] && scopes["sessions"] + }, 3*time.Second, 10*time.Millisecond, + "import must emit data_changed with both messages and sessions scopes") + + got, err := te.db.GetSession(context.Background(), origin+"~sess-1") + require.NoError(t, err) + require.NotNil(t, got) + assert.Equal(t, origin, got.Machine) + assert.Equal(t, "alpha", got.Project) + + cancel() + <-done +} + +// dataChangedScopes collects the scope payloads of every data_changed +// event written to the SSE stream so far. +func dataChangedScopes(w *flushRecorder) map[string]bool { + scopes := make(map[string]bool) + for _, e := range parseSSE(w.BodyString()) { + if e.Event != "data_changed" { + continue + } + var payload struct { + Scope string `json:"scope"` + } + if json.Unmarshal([]byte(e.Data), &payload) == nil { + scopes[payload.Scope] = true + } + } + return scopes +} + +func artifactPeerRequest( + t *testing.T, + te *testEnv, + method string, + path string, + body []byte, + token string, +) *httptest.ResponseRecorder { + t.Helper() + req := httptest.NewRequest(method, path, bytes.NewReader(body)) + req.Header.Set("Content-Type", "application/octet-stream") + if token != "" { + req.Header.Set("Authorization", "Bearer "+token) + } + w := httptest.NewRecorder() + te.handler.ServeHTTP(w, req) + return w +} + +func postArtifactFile( + t *testing.T, + te *testEnv, + origin string, + kind string, + path string, +) { + t.Helper() + body, err := os.ReadFile(path) + require.NoError(t, err) + w := artifactPeerRequest( + t, te, http.MethodPost, + "/api/v1/artifacts/"+origin+"/"+kind+"/"+url.PathEscape(filepath.Base(path)), + body, "", + ) + assertStatus(t, w, http.StatusOK) +} + +func assertArtifactMissing( + t *testing.T, + dataDir string, + origin string, + kind string, + name string, +) { + t.Helper() + path := filepath.Join(dataDir, "artifacts", origin, kind, name) + _, err := os.Stat(path) + assert.True(t, errors.Is(err, os.ErrNotExist), + "artifact should not have been written at %s", path) +} + +func oneArtifactPath( + t *testing.T, + root string, + origin string, + kind string, + pattern string, +) string { + t.Helper() + paths, err := filepath.Glob(filepath.Join(root, origin, kind, pattern)) + require.NoError(t, err) + require.Len(t, paths, 1) + return paths[0] +} + +func peerMetadataArtifact(origin, hlc string) ([]byte, string) { + body := []byte(`{"hlc":"` + hlc + `","op":"rename","origin":"` + origin + `","session_gid":"` + origin + `~sess-1","v":1,"value":{"display_name":"Remote"}}` + "\n") + sum := sha256.Sum256(body) + hash := hex.EncodeToString(sum[:]) + return body, hlc + "-" + hash + ".json" +} + +func TestArtifactPeersStatusToleratesCorruptLatestCheckpoint(t *testing.T) { + local := "desktop-d4e5f6" + te := setup(t, withArtifactOrigin(local)) + ctx := context.Background() + artifactRoot := filepath.Join(te.dataDir, "artifacts") + first := "hi" + + dbtest.SeedSession(t, te.db, "local-1", "proj", func(s *db.Session) { s.FirstMessage = &first }) + _, err := artifact.Export(ctx, te.db, artifactRoot, local) + require.NoError(t, err) + dbtest.SeedSession(t, te.db, "local-2", "proj", func(s *db.Session) { s.FirstMessage = &first }) + _, err = artifact.Export(ctx, te.db, artifactRoot, local) + require.NoError(t, err) + + latest := filepath.Join(artifactRoot, local, "checkpoints", "cp-0000000002.json") + require.NoError(t, os.WriteFile(latest, []byte("not json"), 0o644)) + + // One corrupt newest checkpoint must not make the peers page unusable: + // discovery quarantines it and publishes the current two-session state at + // the next unused sequence. + w := artifactPeerRequest(t, te, http.MethodGet, "/api/v1/artifacts/peers", nil, "") + assertStatus(t, w, http.StatusOK) + body := decode[artifactPeersBody](t, w) + require.Len(t, body.Peers, 1) + assert.Equal(t, 3, body.Peers[0].CheckpointSeq) + assert.Equal(t, 2, body.Peers[0].PublishedSessions) + assert.NoFileExists(t, latest) + assert.FileExists(t, latest+".corrupt") +} diff --git a/internal/server/bulk_star_metadata_test.go b/internal/server/bulk_star_metadata_test.go new file mode 100644 index 000000000..b1e26e100 --- /dev/null +++ b/internal/server/bulk_star_metadata_test.go @@ -0,0 +1,70 @@ +package server_test + +import ( + "context" + "net/http" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "go.kenn.io/agentsview/internal/artifact" +) + +func TestBulkStarAppendsMetadataEvents(t *testing.T) { + te := setup(t, withArtifactOrigin("desktop-d4e5f6")) + te.seedSession(t, "s1", "alpha", 2) + te.seedSession(t, "s2", "beta", 2) + + w := te.requestJSON(t, http.MethodPost, "/api/v1/starred/bulk", + `{"session_ids":["s1","s2","missing"]}`) + require.Equal(t, http.StatusNoContent, w.Code, "body: %s", w.Body.String()) + + // Both existing sessions are starred; the missing one is skipped. + w = te.get(t, "/api/v1/starred") + require.Equal(t, http.StatusOK, w.Code, "body: %s", w.Body.String()) + list := decode[starredHandlerResponse](t, w) + assert.ElementsMatch(t, []string{"s1", "s2"}, list.SessionIDs) + + // A star metadata event artifact was written for each session actually + // starred, so the migrated stars converge through artifact sync. The missing + // session produces no event. + metaDir := filepath.Join(te.dataDir, "artifacts", "desktop-d4e5f6", "meta") + entries, err := os.ReadDir(metaDir) + require.NoError(t, err) + assert.Len(t, entries, 2, "one star event per existing session") +} + +func TestBulkStarRetriesRepairPublishedMetadataState(t *testing.T) { + te := setup(t, withArtifactOrigin("desktop-d4e5f6")) + te.seedSession(t, "s1", "alpha", 2) + + execTestDDL(t, te, ` +CREATE TRIGGER fail_metadata_replay_state_insert +BEFORE INSERT ON metadata_replay_state +BEGIN + SELECT RAISE(FAIL, 'forced metadata replay failure'); +END; +`) + + w := te.requestJSON(t, http.MethodPost, "/api/v1/starred/bulk", + `{"session_ids":["s1"]}`) + require.Equal(t, http.StatusInternalServerError, w.Code, "body: %s", w.Body.String()) + ids, err := te.db.ListStarredSessionIDs(context.Background()) + require.NoError(t, err) + assert.Equal(t, []string{"s1"}, ids) + assert.Equal(t, 0, serverMetadataTableCount(t, te, "metadata_replay_state", "session_gid = 'desktop-d4e5f6~s1'")) + events := readMetadataEvents(t, te) + require.Len(t, events, 1) + assert.Equal(t, artifact.MetadataOpStar, events[0].Op) + + execTestDDL(t, te, `DROP TRIGGER fail_metadata_replay_state_insert`) + w = te.requestJSON(t, http.MethodPost, "/api/v1/starred/bulk", + `{"session_ids":["s1"]}`) + require.Equal(t, http.StatusNoContent, w.Code, "body: %s", w.Body.String()) + assert.Equal(t, artifact.MetadataOpStar, + serverMetadataReplayOp(t, te, "desktop-d4e5f6~s1", "starred")) + assert.Len(t, readMetadataEvents(t, te), 1) +} diff --git a/internal/server/huma_route_groups.go b/internal/server/huma_route_groups.go index 1f9c99657..2056776b9 100644 --- a/internal/server/huma_route_groups.go +++ b/internal/server/huma_route_groups.go @@ -27,6 +27,7 @@ func (s *Server) registerTypedAPIRoutes() { s.registerImportRoutes() s.registerAssetRoutes() s.registerEmbeddingsRoutes() + s.registerArtifactRoutes() } type routeGroup struct { diff --git a/internal/server/huma_routes_artifacts.go b/internal/server/huma_routes_artifacts.go new file mode 100644 index 000000000..01fa6f706 --- /dev/null +++ b/internal/server/huma_routes_artifacts.go @@ -0,0 +1,400 @@ +package server + +import ( + "context" + "errors" + "net/http" + "path/filepath" + "time" + + "go.kenn.io/agentsview/internal/artifact" + "go.kenn.io/agentsview/internal/db" +) + +func (s *Server) registerArtifactRoutes() { + group := newRouteGroup(s.api, "/api/v1/artifacts", "Artifacts") + + get(s, group, "/origins", "List artifact origins", s.humaListArtifactOrigins) + get(s, group, "/peers", "List artifact peers", s.humaListArtifactPeers) + get(s, group, "/{origin}/index", "List artifact index for an origin", s.humaGetArtifactIndex) + raw(s, group, http.MethodGet, "/{origin}/checkpoint", "Get latest artifact checkpoint", s.humaGetArtifactCheckpoint) + raw(s, group, http.MethodGet, "/{origin}/{kind}/{name}", "Get artifact", s.humaGetArtifact) + post(s, group, "/{origin}/{kind}/{name}", "Post artifact", s.humaPostArtifact) +} + +type artifactOriginInput struct { + Origin string `path:"origin" required:"true" doc:"Artifact origin ID"` +} + +type artifactPathInput struct { + Origin string `path:"origin" required:"true" doc:"Artifact origin ID"` + Kind string `path:"kind" required:"true" doc:"Artifact kind"` + Name string `path:"name" required:"true" doc:"Artifact filename or hash"` +} + +type artifactPostInput struct { + Origin string `path:"origin" required:"true" doc:"Artifact origin ID"` + Kind string `path:"kind" required:"true" doc:"Artifact kind"` + Name string `path:"name" required:"true" doc:"Artifact filename or hash"` + RawBody []byte `contentType:"application/octet-stream"` +} + +type artifactOriginsResponse struct { + Origins []string `json:"origins"` +} + +// artifactPeer is one origin's status in the peers view: what it has published +// (from its latest checkpoint) and how much of it has landed locally. +type artifactPeer struct { + Origin string `json:"origin"` + IsLocal bool `json:"is_local"` + CheckpointSeq int `json:"checkpoint_seq"` + PublishedSessions int `json:"published_sessions"` + LocalSessions int `json:"local_sessions"` + LastPublished string `json:"last_published,omitempty"` +} + +type artifactPeersResponse struct { + LocalOrigin string `json:"local_origin"` + Peers []artifactPeer `json:"peers"` + ConflictCount int `json:"conflict_count"` +} + +type artifactPostResponse struct { + Origin string `json:"origin"` + Kind string `json:"kind"` + Name string `json:"name"` + Hash string `json:"hash,omitempty"` + Size int64 `json:"size"` + Duplicate bool `json:"duplicate"` +} + +func (s *Server) artifactRoot() (string, error) { + if s.cfg.DataDir == "" { + return "", apiError(http.StatusServiceUnavailable, "artifact store not configured") + } + return filepath.Join(s.cfg.DataDir, "artifacts"), nil +} + +func (s *Server) artifactCapableRoot() (string, error) { + if err := s.requireLocalArtifactStore(); err != nil { + return "", err + } + return s.artifactRoot() +} + +func (s *Server) humaListArtifactOrigins( + ctx context.Context, + _ *emptyInput, +) (*jsonOutput[artifactOriginsResponse], error) { + root, err := s.artifactCapableRoot() + if err != nil { + return nil, err + } + if err := s.publishLocalArtifacts(ctx, root); err != nil { + return nil, err + } + origins, err := artifact.ListOrigins(root) + if err != nil { + return nil, artifactRouteError("list artifact origins", err) + } + return &jsonOutput[artifactOriginsResponse]{ + Body: artifactOriginsResponse{Origins: origins}, + }, nil +} + +func (s *Server) humaGetArtifactIndex( + _ context.Context, + in *artifactOriginInput, +) (*jsonOutput[artifact.OriginArtifactIndex], error) { + root, err := s.artifactCapableRoot() + if err != nil { + return nil, err + } + index, err := artifact.ListArtifacts(root, in.Origin) + if err != nil { + return nil, artifactRouteError("list artifact index", err) + } + return &jsonOutput[artifact.OriginArtifactIndex]{Body: index}, nil +} + +// localArtifactOrigin returns this machine's artifact origin without creating +// one. It prefers the configured origin and falls back to the persisted DB +// value so read-only callers never mint a new identity. +func (s *Server) localArtifactOrigin() string { + if s.cfg.ArtifactOriginID != "" { + return s.cfg.ArtifactOriginID + } + if local, ok := s.db.(*db.DB); ok { + if origin, err := artifact.StoredOrigin(local); err == nil { + return origin + } + } + return "" +} + +func (s *Server) humaListArtifactPeers( + ctx context.Context, + _ *emptyInput, +) (*jsonOutput[artifactPeersResponse], error) { + root, err := s.artifactCapableRoot() + if err != nil { + return nil, err + } + if err := s.publishLocalArtifacts(ctx, root); err != nil { + return nil, err + } + origins, err := artifact.ListOrigins(root) + if err != nil { + return nil, artifactRouteError("list artifact origins", err) + } + localDB, err := s.writableArtifactImportDB() + if err != nil { + return nil, err + } + conflicts, err := s.db.CountMetadataConflicts(ctx) + if err != nil { + return nil, internalError("count metadata conflicts", err) + } + + localOrigin := s.localArtifactOrigin() + // Always surface this machine even before its first export. + seen := map[string]bool{} + ordered := make([]string, 0, len(origins)+1) + if localOrigin != "" { + ordered = append(ordered, localOrigin) + seen[localOrigin] = true + } + for _, origin := range origins { + if seen[origin] { + continue + } + seen[origin] = true + ordered = append(ordered, origin) + } + + peers := make([]artifactPeer, 0, len(ordered)) + for _, origin := range ordered { + summary, err := artifact.CheckpointSummary(root, origin) + if err != nil { + return nil, artifactRouteError("read artifact checkpoint", err) + } + isLocal := origin == localOrigin + var landedSessions int + if isLocal { + // publishLocalArtifacts just refreshed this origin, so every session + // in its latest checkpoint is present locally at that manifest. + landedSessions = summary.SessionCount + } else { + landedSessions, err = artifact.CountImportedCheckpointSessions( + localDB, origin, summary.SessionManifests, + ) + if err != nil { + return nil, internalError("count landed artifact sessions", err) + } + } + last := "" + if summary.Found { + last = summary.ModTime.UTC().Format(time.RFC3339) + } + peers = append(peers, artifactPeer{ + Origin: origin, + IsLocal: isLocal, + CheckpointSeq: summary.Sequence, + PublishedSessions: summary.SessionCount, + LocalSessions: landedSessions, + LastPublished: last, + }) + } + + return &jsonOutput[artifactPeersResponse]{ + Body: artifactPeersResponse{ + LocalOrigin: localOrigin, + Peers: peers, + ConflictCount: conflicts, + }, + }, nil +} + +// publishLocalArtifacts refreshes the server's owned origin immediately before +// peer discovery. HTTP transports begin every exchange with origin discovery, +// so this makes the server a publisher without requiring a separate folder +// sync process while keeping individual artifact reads side-effect free. +func (s *Server) publishLocalArtifacts(ctx context.Context, root string) error { + local, err := s.writableArtifactImportDB() + if err != nil { + return err + } + origin := s.localArtifactOrigin() + if origin == "" { + return nil + } + + s.lockSessionLifecycle() + defer s.sessionLifecycleMu.Unlock() + if !s.artifactBaselineDone && s.metadata != nil { + if _, err := s.metadata.AppendBaseline(ctx); err != nil { + return artifactRouteError("baseline local artifact metadata", err) + } + s.artifactBaselineDone = true + } + if s.engine != nil { + s.engine.FlushSignals() + } + if _, err := artifact.Export(ctx, local, root, origin); err != nil { + return artifactRouteError("export local artifacts", err) + } + return nil +} + +func (s *Server) humaGetArtifactCheckpoint( + _ context.Context, + in *artifactOriginInput, +) (*bytesOutput, error) { + root, err := s.artifactCapableRoot() + if err != nil { + return nil, err + } + art, err := artifact.ReadLatestCheckpoint(root, in.Origin) + if err != nil { + return nil, artifactRouteError("get artifact checkpoint", err) + } + return artifactBytesOutput(art), nil +} + +func (s *Server) humaGetArtifact( + _ context.Context, + in *artifactPathInput, +) (*bytesOutput, error) { + root, err := s.artifactCapableRoot() + if err != nil { + return nil, err + } + art, err := artifact.ReadArtifactForServe(root, in.Origin, in.Kind, in.Name) + if err != nil { + return nil, artifactRouteError("get artifact", err) + } + return artifactBytesOutput(art), nil +} + +func (s *Server) humaPostArtifact( + ctx context.Context, + in *artifactPostInput, +) (*jsonOutput[artifactPostResponse], error) { + local, err := s.writableArtifactImportDB() + if err != nil { + return nil, err + } + root, err := s.artifactRoot() + if err != nil { + return nil, err + } + res, err := artifact.WriteArtifact(root, in.Origin, in.Kind, in.Name, in.RawBody) + if err != nil { + return nil, artifactRouteError("post artifact", err) + } + s.lockSessionLifecycle() + defer s.sessionLifecycleMu.Unlock() + if res.Kind == artifact.KindCheckpoints || res.Kind == artifact.KindMeta || s.artifactImportPending { + importRes, err := s.importPeerArtifacts(ctx, local, root) + if err != nil { + return nil, err + } + s.artifactImportPending = importRes.Deferred > 0 + } + return &jsonOutput[artifactPostResponse]{ + Body: artifactPostResponse{ + Origin: res.Origin, + Kind: res.Kind, + Name: res.Name, + Hash: res.Hash, + Size: res.Size, + Duplicate: res.Duplicate, + }, + }, nil +} + +func (s *Server) writableArtifactImportDB() (*db.DB, error) { + if err := s.requireLocalArtifactStore(); err != nil { + return nil, err + } + return s.db.(*db.DB), nil +} + +func (s *Server) requireLocalArtifactStore() error { + local, ok := s.db.(*db.DB) + if !ok { + return apiError(http.StatusNotImplemented, + "artifact routes are not available in remote mode") + } + if local.ReadOnly() { + return apiError(http.StatusNotImplemented, + "artifact routes are not available in read-only mode") + } + return nil +} + +func (s *Server) importPeerArtifacts( + ctx context.Context, local *db.DB, root string, +) (artifact.ImportResult, error) { + var res artifact.ImportResult + var err error + if s.metadata != nil { + // An incoming peer exchange is this machine's opt-in to artifact + // sync, so mint the origin here. The recorder itself never creates + // one: ordinary curation stays ledger-free until a machine opts in. + if s.cfg.ArtifactOriginID == "" { + if _, err := artifact.EnsureOrigin(local); err != nil { + return artifact.ImportResult{}, internalError("artifact import origin", err) + } + } + // Route through the recorder so import and local appends share one HLC + // clock, keeping later local edits causally ahead of imported peers. + res, err = s.metadata.Import(ctx, root) + } else { + localOrigin := s.cfg.ArtifactOriginID + if localOrigin == "" { + localOrigin, err = artifact.EnsureOrigin(local) + if err != nil { + return artifact.ImportResult{}, internalError("artifact import origin", err) + } + } + res, err = artifact.ImportDetailed(ctx, local, root, localOrigin) + } + if err != nil { + return artifact.ImportResult{}, artifactRouteError("import peer artifacts", err) + } + if res.Changed() && s.broadcaster != nil { + // Imports add sessions and apply curation metadata, so live + // clients need the session-index refresh that only the + // "sessions" scope triggers; "messages" additionally + // invalidates hydrated session details and cached signal + // detail. Emit "sessions" last so a coalesced burst resolves + // to the index refresh. + s.broadcaster.Emit("messages") + s.broadcaster.Emit("sessions") + } + return res, nil +} + +func artifactBytesOutput(art artifact.PeerArtifact) *bytesOutput { + return &bytesOutput{ + ContentType: art.ContentType, + NoSniff: "nosniff", + CacheControl: "no-store", + Body: art.Data, + } +} + +func artifactRouteError(logPrefix string, err error) error { + switch { + case errors.Is(err, artifact.ErrArtifactInvalid): + return apiError(http.StatusBadRequest, err.Error()) + case errors.Is(err, artifact.ErrArtifactNotFound): + return apiError(http.StatusNotFound, "artifact not found") + case errors.Is(err, artifact.ErrArtifactConflict): + return apiError(http.StatusConflict, "artifact conflict") + default: + return internalError(logPrefix, err) + } +} diff --git a/internal/server/huma_routes_metadata_internal_test.go b/internal/server/huma_routes_metadata_internal_test.go index a658370a6..e2509c89a 100644 --- a/internal/server/huma_routes_metadata_internal_test.go +++ b/internal/server/huma_routes_metadata_internal_test.go @@ -2,11 +2,21 @@ package server import ( "context" + "crypto/sha256" + "encoding/hex" + "errors" + "slices" + "sync" + "sync/atomic" "testing" + "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "go.kenn.io/agentsview/internal/artifact" "go.kenn.io/agentsview/internal/config" + "go.kenn.io/agentsview/internal/db" + "go.kenn.io/agentsview/internal/dbtest" "go.kenn.io/agentsview/internal/service" ) @@ -36,3 +46,968 @@ func TestHumaGetSessionStatsUsesServerGitHubToken(t *testing.T) { require.NoError(t, err) assert.Equal(t, "server-token", spy.got.GHToken) } + +func TestHumaBatchDeleteRestoresOnlyUnpublishedNewDeletions(t *testing.T) { + tests := []struct { + name string + failure error + wantDeleted []string + }{ + { + name: "ordinary append failure restores current and later", + failure: errors.New("artifact write failed"), + wantDeleted: []string{"already-trashed", "s1"}, + }, + { + name: "published failure keeps current and restores later", + failure: &artifact.MetadataPublishedError{ + Err: errors.New("replay bookkeeping failed"), + }, + wantDeleted: []string{"already-trashed", "s1", "s2"}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + database := dbtest.OpenTestDB(t) + for _, id := range []string{"already-trashed", "s1", "s2", "s3"} { + dbtest.SeedSession(t, database, id, "alpha") + } + require.NoError(t, database.SoftDeleteSession("already-trashed")) + + var appended []string + srv := &Server{ + db: database, + metadataAppend: func( + _ context.Context, input artifact.MetadataEventInput, + ) error { + appended = append(appended, input.SessionID) + if input.SessionID == "s2" { + return tt.failure + } + return nil + }, + } + in := &batchDeleteInput{} + in.Body.SessionIDs = []string{"already-trashed", "s1", "s2", "s3"} + + _, err := srv.humaBatchDeleteSessions(context.Background(), in) + + require.Error(t, err) + assert.Equal(t, []string{"s1", "s2"}, appended, + "publication must stop at the first failed event") + for _, id := range []string{"already-trashed", "s1", "s2", "s3"} { + session, getErr := database.GetSessionFull(context.Background(), id) + require.NoError(t, getErr) + require.NotNil(t, session) + assert.Equal(t, containsString(tt.wantDeleted, id), session.DeletedAt != nil, + "unexpected trash state for %s", id) + } + }) + } +} + +func containsString(values []string, want string) bool { + return slices.Contains(values, want) +} + +type batchRestoreStore struct { + db.Store + restored []string + failID string + failErr error +} + +func (s *batchRestoreStore) RestoreSession(id string) (int64, error) { + s.restored = append(s.restored, id) + if id == s.failID { + return 0, s.failErr + } + return 1, nil +} + +func TestRestoreBatchDeletedSessionsJoinsFailuresAndContinues(t *testing.T) { + cause := errors.New("metadata append failed") + restoreErr := errors.New("restore failed") + store := &batchRestoreStore{failID: "s2", failErr: restoreErr} + srv := &Server{db: store} + + err := srv.restoreBatchDeletedSessions([]string{"s2", "s3"}, cause) + + assert.ErrorIs(t, err, cause) + assert.ErrorIs(t, err, restoreErr) + assert.Equal(t, []string{"s2", "s3"}, store.restored, + "a restoration failure must not prevent later sessions from being restored") +} + +func TestRestoreLifecycleWaitsForFailedRestoreCompensation(t *testing.T) { + const origin = "desk-a1b2c3" + database := dbtest.OpenTestDB(t) + dbtest.SeedSession(t, database, "s1", "alpha") + require.NoError(t, database.SoftDeleteSession("s1")) + recordSoftDeleteReplayState(t, database, origin, "s1") + + firstAppendStarted := make(chan struct{}) + secondAppendStarted := make(chan struct{}) + secondLockAttempted := make(chan struct{}) + releaseFirstAppend := make(chan struct{}) + var appendCalls atomic.Int32 + var lockAttempts atomic.Int32 + srv := &Server{ + db: database, + cfg: config.Config{ArtifactOriginID: origin}, + beforeSessionLifecycleLock: func() { + if lockAttempts.Add(1) == 2 { + close(secondLockAttempted) + } + }, + metadataAppend: func( + _ context.Context, input artifact.MetadataEventInput, + ) error { + if input.Op != artifact.MetadataOpRestore { + return errors.New("unexpected metadata op") + } + switch appendCalls.Add(1) { + case 1: + close(firstAppendStarted) + <-releaseFirstAppend + return errors.New("artifact write failed") + case 2: + close(secondAppendStarted) + return nil + default: + return errors.New("unexpected extra metadata append") + } + }, + } + + firstResult := make(chan error, 1) + go func() { + _, err := srv.humaRestoreSession(context.Background(), &idPathInput{ID: "s1"}) + firstResult <- err + }() + <-firstAppendStarted + + secondResult := make(chan error, 1) + go func() { + _, err := srv.humaRestoreSession(context.Background(), &idPathInput{ID: "s1"}) + secondResult <- err + }() + <-secondLockAttempted + + interleaved := false + select { + case <-secondAppendStarted: + interleaved = true + case <-time.After(500 * time.Millisecond): + } + close(releaseFirstAppend) + firstErr := <-firstResult + secondErr := <-secondResult + + assert.False(t, interleaved, + "a second restore must not publish before the first restore compensates") + require.Error(t, firstErr) + require.NoError(t, secondErr) + assert.Equal(t, int32(2), appendCalls.Load()) + session, err := database.GetSessionFull(context.Background(), "s1") + require.NoError(t, err) + require.NotNil(t, session) + assert.Nil(t, session.DeletedAt, + "the later successful restore must remain visible locally") +} + +func TestPermanentDeleteLifecycleExcludesConcurrentRestore(t *testing.T) { + const origin = "desk-a1b2c3" + database := dbtest.OpenTestDB(t) + dbtest.SeedSession(t, database, "s1", "alpha") + require.NoError(t, database.SoftDeleteSession("s1")) + recordSoftDeleteReplayState(t, database, origin, "s1") + + firstAppendStarted := make(chan struct{}) + secondAppendStarted := make(chan struct{}) + secondLockAttempted := make(chan struct{}) + releaseFirstAppend := make(chan struct{}) + var appendCalls atomic.Int32 + var lockAttempts atomic.Int32 + srv := &Server{ + db: database, + cfg: config.Config{ArtifactOriginID: origin}, + beforeSessionLifecycleLock: func() { + if lockAttempts.Add(1) == 2 { + close(secondLockAttempted) + } + }, + metadataAppend: func( + _ context.Context, input artifact.MetadataEventInput, + ) error { + switch appendCalls.Add(1) { + case 1: + if input.Op != artifact.MetadataOpPurge { + return errors.New("unexpected first metadata op") + } + close(firstAppendStarted) + <-releaseFirstAppend + return nil + case 2: + if input.Op != artifact.MetadataOpRestore { + return errors.New("unexpected second metadata op") + } + close(secondAppendStarted) + return nil + default: + return errors.New("unexpected extra metadata append") + } + }, + } + + purgeResult := make(chan error, 1) + go func() { + _, err := srv.humaPermanentDeleteSession( + context.Background(), &idPathInput{ID: "s1"}, + ) + purgeResult <- err + }() + <-firstAppendStarted + + restoreResult := make(chan error, 1) + go func() { + _, err := srv.humaRestoreSession(context.Background(), &idPathInput{ID: "s1"}) + restoreResult <- err + }() + <-secondLockAttempted + + interleaved := false + select { + case <-secondAppendStarted: + interleaved = true + case <-time.After(500 * time.Millisecond): + } + close(releaseFirstAppend) + purgeErr := <-purgeResult + restoreErr := <-restoreResult + + assert.False(t, interleaved, + "restore must not publish while a purge has reserved the trashed session") + require.NoError(t, purgeErr) + require.Error(t, restoreErr) + assert.Equal(t, int32(1), appendCalls.Load()) + session, err := database.GetSessionFull(context.Background(), "s1") + require.NoError(t, err) + assert.Nil(t, session, + "a durable purge must delete the local session before restore can run") +} + +func TestPermanentDeleteLifecycleExcludesConcurrentPeerRestore(t *testing.T) { + const ( + localOrigin = "desk-a1b2c3" + peerOrigin = "peer-b2c3d4" + ) + database := dbtest.OpenTestDB(t) + dbtest.SeedSession(t, database, "s1", "alpha") + require.NoError(t, database.SoftDeleteSession("s1")) + recordSoftDeleteReplayState(t, database, localOrigin, "s1") + + firstAppendStarted := make(chan struct{}) + secondLockAttempted := make(chan struct{}) + releaseFirstAppend := make(chan struct{}) + var appendCalls atomic.Int32 + var lockAttempts atomic.Int32 + srv := &Server{ + db: database, + cfg: config.Config{ + ArtifactOriginID: localOrigin, + DataDir: t.TempDir(), + }, + beforeSessionLifecycleLock: func() { + if lockAttempts.Add(1) == 2 { + close(secondLockAttempted) + } + }, + metadataAppend: func( + _ context.Context, input artifact.MetadataEventInput, + ) error { + if appendCalls.Add(1) != 1 || input.Op != artifact.MetadataOpPurge { + return errors.New("unexpected metadata append") + } + close(firstAppendStarted) + <-releaseFirstAppend + return nil + }, + } + + purgeResult := make(chan error, 1) + go func() { + _, err := srv.humaPermanentDeleteSession( + context.Background(), &idPathInput{ID: "s1"}, + ) + purgeResult <- err + }() + <-firstAppendStarted + + body, name := peerRestoreArtifact( + peerOrigin, artifact.MetadataSessionGID(localOrigin, "s1"), + ) + peerResult := make(chan error, 1) + go func() { + _, err := srv.humaPostArtifact(context.Background(), &artifactPostInput{ + Origin: peerOrigin, + Kind: artifact.KindMeta, + Name: name, + RawBody: body, + }) + peerResult <- err + }() + + peerCompletedBeforeRelease := false + var peerErr error + select { + case <-secondLockAttempted: + select { + case peerErr = <-peerResult: + peerCompletedBeforeRelease = true + case <-time.After(500 * time.Millisecond): + } + case peerErr = <-peerResult: + peerCompletedBeforeRelease = true + case <-time.After(2 * time.Second): + require.FailNow(t, "peer restore reached neither lifecycle lock nor completion") + } + close(releaseFirstAppend) + purgeErr := <-purgeResult + if !peerCompletedBeforeRelease { + peerErr = <-peerResult + } + + assert.False(t, peerCompletedBeforeRelease, + "peer restore import must wait for the durable purge to delete locally") + require.NoError(t, purgeErr) + require.NoError(t, peerErr) + assert.Equal(t, int32(1), appendCalls.Load()) + session, err := database.GetSessionFull(context.Background(), "s1") + require.NoError(t, err) + assert.Nil(t, session, + "peer restore must not interleave between purge publication and deletion") +} + +func TestArtifactContentPostDefersDatabaseImport(t *testing.T) { + ctx := context.Background() + const peerOrigin = "peer-b2c3d4" + peerDB := dbtest.OpenTestDB(t) + dbtest.SeedSession(t, peerDB, "s1", "alpha") + peerRoot := t.TempDir() + _, err := artifact.Export(ctx, peerDB, peerRoot, peerOrigin) + require.NoError(t, err) + index, err := artifact.ListArtifacts(peerRoot, peerOrigin) + require.NoError(t, err) + require.Len(t, index.Segments, 1) + segment, err := artifact.ReadArtifact( + peerRoot, peerOrigin, artifact.KindSegments, index.Segments[0], + ) + require.NoError(t, err) + + local := dbtest.OpenTestDB(t) + server := &Server{ + db: local, + cfg: config.Config{DataDir: t.TempDir()}, + } + _, err = server.humaPostArtifact(ctx, &artifactPostInput{ + Origin: peerOrigin, + Kind: artifact.KindSegments, + Name: segment.Name, + RawBody: segment.Data, + }) + require.NoError(t, err) + + localOrigin, err := artifact.StoredOrigin(local) + require.NoError(t, err) + assert.Empty(t, localOrigin, + "content-only upload must not trigger a full import or enroll the receiver") +} + +func TestArtifactDependencyPostRetriesDeferredCheckpointImport(t *testing.T) { + ctx := context.Background() + const peerOrigin = "peer-b2c3d4" + peerDB := dbtest.OpenTestDB(t) + dbtest.SeedSession(t, peerDB, "s1", "alpha") + peerRoot := t.TempDir() + _, err := artifact.Export(ctx, peerDB, peerRoot, peerOrigin) + require.NoError(t, err) + index, err := artifact.ListArtifacts(peerRoot, peerOrigin) + require.NoError(t, err) + require.Len(t, index.Checkpoints, 1) + require.Len(t, index.Manifests, 1) + require.Len(t, index.Segments, 1) + + local := dbtest.OpenTestDB(t) + server := &Server{db: local, cfg: config.Config{DataDir: t.TempDir()}} + post := func(kind, name string) { + t.Helper() + art, readErr := artifact.ReadArtifact(peerRoot, peerOrigin, kind, name) + require.NoError(t, readErr) + _, postErr := server.humaPostArtifact(ctx, &artifactPostInput{ + Origin: peerOrigin, Kind: kind, Name: name, RawBody: art.Data, + }) + require.NoError(t, postErr) + } + + post(artifact.KindCheckpoints, index.Checkpoints[0]) + post(artifact.KindManifests, index.Manifests[0]) + got, err := local.GetSession(ctx, peerOrigin+"~s1") + require.NoError(t, err) + assert.Nil(t, got) + + post(artifact.KindSegments, index.Segments[0]) + got, err = local.GetSession(ctx, peerOrigin+"~s1") + require.NoError(t, err) + require.NotNil(t, got) + assert.Equal(t, "alpha", got.Project) +} + +func peerRestoreArtifact(origin, sessionGID string) ([]byte, string) { + const hlc = "2026-07-10T010203.000000001Z-00000000000000000000" + body := []byte(`{"hlc":"` + hlc + `","op":"restore","origin":"` + origin + + `","session_gid":"` + sessionGID + `","v":1}` + "\n") + sum := sha256.Sum256(body) + hash := hex.EncodeToString(sum[:]) + return body, hlc + "-" + hash + ".json" +} + +func recordSoftDeleteReplayState(t *testing.T, database *db.DB, origin, sessionID string) { + t.Helper() + _, err := database.RecordLocalMetadataProjection(context.Background(), db.MetadataProjection{ + EventOrigin: origin, + OrderKey: "0001-soft-delete", + HLC: "0001", + ArtifactHash: "soft-delete", + SessionGID: artifact.MetadataSessionGID(origin, sessionID), + LocalSessionID: sessionID, + Field: "deleted_at", + Op: artifact.MetadataOpSoftDelete, + Value: artifact.MetadataOpSoftDelete, + }) + require.NoError(t, err) +} + +type curationAppendGate struct { + firstStarted chan struct{} + secondStarted chan struct{} + releaseFirst chan struct{} + releaseOnce sync.Once + calls atomic.Int32 + inputs [2]artifact.MetadataEventInput +} + +func newCurationAppendGate() *curationAppendGate { + return &curationAppendGate{ + firstStarted: make(chan struct{}), + secondStarted: make(chan struct{}), + releaseFirst: make(chan struct{}), + } +} + +func (g *curationAppendGate) append( + _ context.Context, input artifact.MetadataEventInput, +) error { + call := g.calls.Add(1) + if call <= int32(len(g.inputs)) { + g.inputs[call-1] = input + } + switch call { + case 1: + close(g.firstStarted) + <-g.releaseFirst + return errors.New("artifact write failed") + case 2: + close(g.secondStarted) + return nil + default: + return errors.New("unexpected extra metadata append") + } +} + +func (g *curationAppendGate) release() { + g.releaseOnce.Do(func() { close(g.releaseFirst) }) +} + +func curationMetadataRecorder(t *testing.T, database *db.DB) *artifact.MetadataRecorder { + t.Helper() + return artifact.NewMetadataRecorder(database, artifact.MetadataRecorderOptions{ + DataDir: t.TempDir(), + Origin: "desk-a1b2c3", + }) +} + +func waitForSecondCurationRequest( + t *testing.T, + secondLockAttempted <-chan struct{}, + secondAppendStarted <-chan struct{}, +) bool { + t.Helper() + select { + case <-secondLockAttempted: + select { + case <-secondAppendStarted: + return true + case <-time.After(500 * time.Millisecond): + return false + } + case <-secondAppendStarted: + return true + case <-time.After(2 * time.Second): + require.FailNow(t, "second curation request reached neither lifecycle lock nor metadata append") + return false + } +} + +func TestRenameLifecycleWaitsForFailedAppendCompensation(t *testing.T) { + database := dbtest.OpenTestDB(t) + original := "original" + dbtest.SeedSession(t, database, "s1", "alpha", func(session *db.Session) { + session.DisplayName = &original + }) + + gate := newCurationAppendGate() + t.Cleanup(gate.release) + secondLockAttempted := make(chan struct{}) + var lockAttempts atomic.Int32 + srv := &Server{ + db: database, + beforeSessionLifecycleLock: func() { + if lockAttempts.Add(1) == 2 { + close(secondLockAttempted) + } + }, + metadataAppend: gate.append, + } + + firstName := "first" + firstResult := make(chan error, 1) + go func() { + _, err := srv.humaRenameSession(context.Background(), &renameSessionInput{ + ID: "s1", + Body: renameRequest{DisplayName: &firstName}, + }) + firstResult <- err + }() + <-gate.firstStarted + + secondName := "second" + secondResult := make(chan error, 1) + go func() { + _, err := srv.humaRenameSession(context.Background(), &renameSessionInput{ + ID: "s1", + Body: renameRequest{DisplayName: &secondName}, + }) + secondResult <- err + }() + + interleaved := waitForSecondCurationRequest( + t, secondLockAttempted, gate.secondStarted, + ) + gate.release() + firstErr := <-firstResult + secondErr := <-secondResult + + assert.False(t, interleaved, + "a later rename must not publish before the failed rename compensates") + require.Error(t, firstErr) + require.NoError(t, secondErr) + assert.Equal(t, int32(2), gate.calls.Load()) + assert.Equal(t, artifact.MetadataOpRename, gate.inputs[0].Op) + assert.Equal(t, artifact.MetadataOpRename, gate.inputs[1].Op) + assert.JSONEq(t, `{"display_name":"second"}`, string(gate.inputs[1].Value)) + session, err := database.GetSession(context.Background(), "s1") + require.NoError(t, err) + require.NotNil(t, session) + require.NotNil(t, session.DisplayName) + assert.Equal(t, "second", *session.DisplayName, + "SQLite must retain the value represented by the later durable rename") +} + +func TestStarLifecycleWaitsForFailedAppendCompensation(t *testing.T) { + database := dbtest.OpenTestDB(t) + dbtest.SeedSession(t, database, "s1", "alpha") + + gate := newCurationAppendGate() + t.Cleanup(gate.release) + secondLockAttempted := make(chan struct{}) + var lockAttempts atomic.Int32 + srv := &Server{ + db: database, + metadata: curationMetadataRecorder(t, database), + beforeSessionLifecycleLock: func() { + if lockAttempts.Add(1) == 2 { + close(secondLockAttempted) + } + }, + metadataAppend: gate.append, + } + + firstResult := make(chan error, 1) + go func() { + _, err := srv.humaStarSession(context.Background(), &idPathInput{ID: "s1"}) + firstResult <- err + }() + <-gate.firstStarted + + secondResult := make(chan error, 1) + go func() { + _, err := srv.humaStarSession(context.Background(), &idPathInput{ID: "s1"}) + secondResult <- err + }() + + interleaved := waitForSecondCurationRequest( + t, secondLockAttempted, gate.secondStarted, + ) + gate.release() + firstErr := <-firstResult + secondErr := <-secondResult + + assert.False(t, interleaved, + "a later star must not publish before the failed star compensates") + require.Error(t, firstErr) + require.NoError(t, secondErr) + assert.Equal(t, int32(2), gate.calls.Load()) + assert.Equal(t, artifact.MetadataOpStar, gate.inputs[0].Op) + assert.Equal(t, artifact.MetadataOpStar, gate.inputs[1].Op) + starred, err := database.ListStarredSessionIDs(context.Background()) + require.NoError(t, err) + assert.Equal(t, []string{"s1"}, starred, + "SQLite must retain the star represented by the later durable event") +} + +func seedCurationPinMessage(t *testing.T, database *db.DB) int64 { + t.Helper() + dbtest.SeedSession(t, database, "s1", "alpha") + message := dbtest.UserMsg("s1", 0, "investigate") + message.SourceUUID = "message-a1b2c3" + dbtest.SeedMessages(t, database, message) + messages, err := database.GetAllMessages(context.Background(), "s1") + require.NoError(t, err) + require.Len(t, messages, 1) + return messages[0].ID +} + +type metadataPinPointLookupStore struct { + db.Store + message *db.Message + fullReads int + pointReads int + lookupSessionID string + lookupMessageID int64 +} + +func (s *metadataPinPointLookupStore) GetAllMessages( + _ context.Context, _ string, +) ([]db.Message, error) { + s.fullReads++ + return nil, errors.New("full transcript read must not be used for pin metadata") +} + +func (s *metadataPinPointLookupStore) GetMessageForMetadataPin( + _ context.Context, sessionID string, messageID int64, +) (*db.Message, error) { + s.pointReads++ + s.lookupSessionID = sessionID + s.lookupMessageID = messageID + return s.message, nil +} + +func TestMetadataPinUsesPointMessageLookup(t *testing.T) { + store := &metadataPinPointLookupStore{message: &db.Message{ + ID: 42, + SessionID: "s1", + Ordinal: 7, + SourceUUID: "message-a1b2c3", + }} + srv := &Server{db: store} + note := "remember" + + pin, err := srv.metadataPinForMessage( + context.Background(), "s1", 42, ¬e, + ) + require.NoError(t, err) + require.NotNil(t, pin) + assert.Equal(t, "message-a1b2c3", pin.SourceUUID) + assert.Equal(t, 7, pin.Ordinal) + require.NotNil(t, pin.Note) + assert.Equal(t, "remember", *pin.Note) + assert.Equal(t, 1, store.pointReads) + assert.Zero(t, store.fullReads) + assert.Equal(t, "s1", store.lookupSessionID) + assert.Equal(t, int64(42), store.lookupMessageID) +} + +func TestPinLifecycleWaitsForFailedAppendCompensation(t *testing.T) { + database := dbtest.OpenTestDB(t) + messageID := seedCurationPinMessage(t, database) + + gate := newCurationAppendGate() + t.Cleanup(gate.release) + secondLockAttempted := make(chan struct{}) + var lockAttempts atomic.Int32 + srv := &Server{ + db: database, + metadata: curationMetadataRecorder(t, database), + beforeSessionLifecycleLock: func() { + if lockAttempts.Add(1) == 2 { + close(secondLockAttempted) + } + }, + metadataAppend: gate.append, + } + + firstNote := "first" + firstResult := make(chan error, 1) + go func() { + _, err := srv.humaPinMessage(context.Background(), &pinMessageInput{ + ID: "s1", + MessageID: messageID, + Body: pinRequest{Note: &firstNote}, + }) + firstResult <- err + }() + <-gate.firstStarted + + secondNote := "second" + secondResult := make(chan error, 1) + go func() { + _, err := srv.humaPinMessage(context.Background(), &pinMessageInput{ + ID: "s1", + MessageID: messageID, + Body: pinRequest{Note: &secondNote}, + }) + secondResult <- err + }() + + interleaved := waitForSecondCurationRequest( + t, secondLockAttempted, gate.secondStarted, + ) + gate.release() + firstErr := <-firstResult + secondErr := <-secondResult + + assert.False(t, interleaved, + "a later pin must not publish before the failed pin compensates") + require.Error(t, firstErr) + require.NoError(t, secondErr) + assert.Equal(t, int32(2), gate.calls.Load()) + assert.Equal(t, artifact.MetadataOpPin, gate.inputs[0].Op) + assert.Equal(t, artifact.MetadataOpPin, gate.inputs[1].Op) + require.NotNil(t, gate.inputs[1].Pin) + require.NotNil(t, gate.inputs[1].Pin.Note) + assert.Equal(t, "second", *gate.inputs[1].Pin.Note) + pins, err := database.ListPinnedMessages(context.Background(), "s1", "") + require.NoError(t, err) + require.Len(t, pins, 1, + "SQLite must retain the pin represented by the later durable event") + require.NotNil(t, pins[0].Note) + assert.Equal(t, "second", *pins[0].Note) +} + +func assertCurationMutationWaitsForLifecycle( + t *testing.T, + srv *Server, + wantOp string, + mutate func() error, + assertBefore func(), + assertAfter func(), +) { + t.Helper() + lockAttempted := make(chan struct{}) + appended := make(chan artifact.MetadataEventInput, 2) + var lockAttempts atomic.Int32 + srv.beforeSessionLifecycleLock = func() { + if lockAttempts.Add(1) == 1 { + close(lockAttempted) + } + } + srv.metadataAppend = func( + _ context.Context, input artifact.MetadataEventInput, + ) error { + appended <- input + return nil + } + + srv.sessionLifecycleMu.Lock() + locked := true + defer func() { + if locked { + srv.sessionLifecycleMu.Unlock() + } + }() + + result := make(chan error, 1) + go func() { result <- mutate() }() + + completedBeforeLock := false + var mutationErr error + select { + case <-lockAttempted: + case mutationErr = <-result: + completedBeforeLock = true + case <-time.After(2 * time.Second): + require.FailNow(t, "curation mutation reached neither lifecycle lock nor completion") + } + assert.False(t, completedBeforeLock, + "curation mutation must wait for the shared lifecycle boundary") + assertBefore() + + srv.sessionLifecycleMu.Unlock() + locked = false + if !completedBeforeLock { + mutationErr = <-result + } + require.NoError(t, mutationErr) + assertAfter() + assert.Equal(t, int32(1), lockAttempts.Load()) + select { + case input := <-appended: + assert.Equal(t, wantOp, input.Op) + case <-time.After(2 * time.Second): + require.FailNow(t, "curation mutation did not append metadata") + } + select { + case input := <-appended: + assert.Fail(t, "curation mutation appended extra metadata", "op: %s", input.Op) + default: + } +} + +func TestUnstarMutationWaitsForSessionLifecycle(t *testing.T) { + database := dbtest.OpenTestDB(t) + dbtest.SeedSession(t, database, "s1", "alpha") + starred, err := database.StarSession("s1") + require.NoError(t, err) + require.True(t, starred) + srv := &Server{db: database} + + assertStarred := func(want []string) func() { + return func() { + ids, listErr := database.ListStarredSessionIDs(context.Background()) + require.NoError(t, listErr) + assert.ElementsMatch(t, want, ids) + } + } + assertCurationMutationWaitsForLifecycle( + t, + srv, + artifact.MetadataOpUnstar, + func() error { + _, handlerErr := srv.humaUnstarSession( + context.Background(), &idPathInput{ID: "s1"}, + ) + return handlerErr + }, + assertStarred([]string{"s1"}), + assertStarred([]string{}), + ) +} + +func TestBulkStarMutationWaitsForSessionLifecycle(t *testing.T) { + database := dbtest.OpenTestDB(t) + dbtest.SeedSession(t, database, "s1", "alpha") + srv := &Server{db: database} + in := &bulkStarInput{} + in.Body.SessionIDs = []string{"s1"} + + assertStarred := func(want []string) func() { + return func() { + ids, err := database.ListStarredSessionIDs(context.Background()) + require.NoError(t, err) + assert.ElementsMatch(t, want, ids) + } + } + assertCurationMutationWaitsForLifecycle( + t, + srv, + artifact.MetadataOpStar, + func() error { + _, err := srv.humaBulkStar(context.Background(), in) + return err + }, + assertStarred([]string{}), + assertStarred([]string{"s1"}), + ) +} + +func TestBulkStarEmptyInputDoesNotWaitForSessionLifecycle(t *testing.T) { + lockAttempted := make(chan struct{}) + var appendCalls atomic.Int32 + srv := &Server{ + beforeSessionLifecycleLock: func() { close(lockAttempted) }, + metadataAppend: func( + _ context.Context, _ artifact.MetadataEventInput, + ) error { + appendCalls.Add(1) + return nil + }, + } + srv.sessionLifecycleMu.Lock() + locked := true + defer func() { + if locked { + srv.sessionLifecycleMu.Unlock() + } + }() + + result := make(chan error, 1) + go func() { + _, err := srv.humaBulkStar(context.Background(), &bulkStarInput{}) + result <- err + }() + + select { + case err := <-result: + require.NoError(t, err) + case <-lockAttempted: + srv.sessionLifecycleMu.Unlock() + locked = false + <-result + require.FailNow(t, "empty bulk star waited for the lifecycle boundary") + case <-time.After(2 * time.Second): + require.FailNow(t, "empty bulk star did not complete") + } + assert.Equal(t, int32(0), appendCalls.Load()) +} + +func TestUnpinMutationWaitsForSessionLifecycle(t *testing.T) { + database := dbtest.OpenTestDB(t) + messageID := seedCurationPinMessage(t, database) + note := "keep" + _, err := database.PinMessage("s1", messageID, ¬e) + require.NoError(t, err) + srv := &Server{ + db: database, + metadata: curationMetadataRecorder(t, database), + } + + assertPinned := func(want bool) func() { + return func() { + pins, listErr := database.ListPinnedMessages(context.Background(), "s1", "") + require.NoError(t, listErr) + if !want { + assert.Empty(t, pins) + return + } + require.Len(t, pins, 1) + require.NotNil(t, pins[0].Note) + assert.Equal(t, "keep", *pins[0].Note) + } + } + assertCurationMutationWaitsForLifecycle( + t, + srv, + artifact.MetadataOpUnpin, + func() error { + _, handlerErr := srv.humaUnpinMessage(context.Background(), &messagePathInput{ + ID: "s1", + MessageID: messageID, + }) + return handlerErr + }, + assertPinned(true), + assertPinned(false), + ) +} diff --git a/internal/server/huma_routes_pins.go b/internal/server/huma_routes_pins.go index 13eb81ce2..f0cdfec40 100644 --- a/internal/server/huma_routes_pins.go +++ b/internal/server/huma_routes_pins.go @@ -2,8 +2,11 @@ package server import ( "context" + "errors" + "fmt" "net/http" + "go.kenn.io/agentsview/internal/artifact" "go.kenn.io/agentsview/internal/db" ) @@ -63,9 +66,25 @@ func (s *Server) humaListSessionPins( } func (s *Server) humaPinMessage( - _ context.Context, + ctx context.Context, in *pinMessageInput, ) (*createdOutput[pinMessageResponse], error) { + s.lockSessionLifecycle() + defer s.sessionLifecycleMu.Unlock() + + var prior *db.PinnedMessage + var pin *artifact.MetadataPin + if s.metadata != nil { + var err error + prior, err = s.findPinnedMessage(ctx, in.ID, in.MessageID) + if err != nil { + return nil, internalError("pin message prior state", err) + } + pin, err = s.metadataPinForMessage(ctx, in.ID, in.MessageID, in.Body.Note) + if err != nil { + return nil, internalError("pin message metadata lookup", err) + } + } id, err := s.db.PinMessage(in.ID, in.MessageID, in.Body.Note) if err != nil { if handled := handleHumaReadOnly(err); handled != nil { @@ -77,6 +96,20 @@ func (s *Server) humaPinMessage( return nil, apiError(http.StatusBadRequest, "message does not belong to this session") } + if pin != nil { + if err := s.appendMetadataEvent(ctx, artifact.MetadataEventInput{ + SessionID: in.ID, + Op: artifact.MetadataOpPin, + Pin: pin, + }); err != nil { + var publishedErr *artifact.MetadataPublishedError + if errors.As(err, &publishedErr) { + return nil, internalError("pin message metadata event", err) + } + return nil, internalError("pin message metadata event", + s.restorePinState(in.ID, in.MessageID, prior, err)) + } + } return &createdOutput[pinMessageResponse]{ Status: http.StatusCreated, Body: pinMessageResponse{ID: id}, @@ -84,14 +117,82 @@ func (s *Server) humaPinMessage( } func (s *Server) humaUnpinMessage( - _ context.Context, + ctx context.Context, in *messagePathInput, ) (*noContentOutput, error) { + s.lockSessionLifecycle() + defer s.sessionLifecycleMu.Unlock() + + var prior *db.PinnedMessage + var pin *artifact.MetadataPin + if s.metadata != nil { + var err error + prior, err = s.findPinnedMessage(ctx, in.ID, in.MessageID) + if err != nil { + return nil, internalError("unpin message prior state", err) + } + pin, err = s.metadataPinForMessage(ctx, in.ID, in.MessageID, nil) + if err != nil { + return nil, internalError("unpin message metadata lookup", err) + } + } if err := s.db.UnpinMessage(in.ID, in.MessageID); err != nil { if handled := handleHumaReadOnly(err); handled != nil { return nil, handled } return nil, internalError("unpin message", err) } + if pin != nil { + if err := s.appendMetadataEvent(ctx, artifact.MetadataEventInput{ + SessionID: in.ID, + Op: artifact.MetadataOpUnpin, + Pin: pin, + }); err != nil { + var publishedErr *artifact.MetadataPublishedError + if errors.As(err, &publishedErr) { + return nil, internalError("unpin message metadata event", err) + } + return nil, internalError("unpin message metadata event", + s.restorePinState(in.ID, in.MessageID, prior, err)) + } + } return &noContentOutput{Status: http.StatusNoContent}, nil } + +// findPinnedMessage returns the current pinned_messages row for the +// message, or nil when the message is not pinned. +func (s *Server) findPinnedMessage( + ctx context.Context, sessionID string, messageID int64, +) (*db.PinnedMessage, error) { + pins, err := s.db.ListPinnedMessages(ctx, sessionID, "") + if err != nil { + return nil, err + } + for i := range pins { + if pins[i].MessageID == messageID { + return &pins[i], nil + } + } + return nil, nil +} + +// restorePinState puts the pinned_messages row for the message back to +// prior after a pre-publish metadata failure, so local pin state never +// diverges from the durable ledger. It returns baseErr joined with any +// restore failure. +func (s *Server) restorePinState( + sessionID string, messageID int64, prior *db.PinnedMessage, baseErr error, +) error { + if prior != nil { + if _, err := s.db.PinMessage(sessionID, messageID, prior.Note); err != nil { + return errors.Join(baseErr, + fmt.Errorf("restore pin after metadata failure: %w", err)) + } + return baseErr + } + if err := s.db.UnpinMessage(sessionID, messageID); err != nil { + return errors.Join(baseErr, + fmt.Errorf("remove pin after metadata failure: %w", err)) + } + return baseErr +} diff --git a/internal/server/huma_routes_sessions.go b/internal/server/huma_routes_sessions.go index 45240f2ec..500a96803 100644 --- a/internal/server/huma_routes_sessions.go +++ b/internal/server/huma_routes_sessions.go @@ -15,6 +15,7 @@ import ( "time" "github.com/danielgtaylor/huma/v2" + "go.kenn.io/agentsview/internal/artifact" "go.kenn.io/agentsview/internal/db" "go.kenn.io/agentsview/internal/parser" "go.kenn.io/agentsview/internal/service" @@ -34,6 +35,7 @@ func (s *Server) registerSessionRoutes() { get(s, group, "/sessions/{id}/activity", "Get session activity", s.humaGetSessionActivity) get(s, group, "/sessions/{id}/timing", "Get session timing", s.humaSessionTiming) get(s, group, "/sessions/{id}/usage", "Get session usage", s.humaSessionUsage) + get(s, group, "/sessions/{id}/metadata-conflicts", "List session metadata conflicts", s.humaListMetadataConflicts) stream(s, group, http.MethodGet, "/sessions/{id}/watch", "Watch session events", s.humaWatchSession) stream(s, group, http.MethodGet, "/events", "Watch server events", s.humaEvents) raw(s, group, http.MethodGet, "/sessions/{id}/export", "Export session as HTML", s.humaExportSession) @@ -547,6 +549,43 @@ type emptyTrashResponse struct { Deleted int `json:"deleted"` } +type metadataConflictsResponse struct { + Conflicts []db.MetadataConflict `json:"conflicts"` +} + +func (s *Server) humaListMetadataConflicts( + ctx context.Context, + in *idPathInput, +) (*jsonOutput[metadataConflictsResponse], error) { + session, err := s.db.GetSessionFull(ctx, in.ID) + if err != nil { + return nil, internalError("metadata conflict session lookup", err) + } + if session == nil { + return nil, apiError(http.StatusNotFound, "session not found") + } + gids := []string{in.ID} + if localDB, ok := s.db.(*db.DB); ok && !strings.Contains(in.ID, "~") { + origin, err := artifact.StoredOrigin(localDB) + if err != nil { + return nil, internalError("read artifact origin", err) + } + if origin != "" { + gids = append(gids, artifact.MetadataSessionGID(origin, in.ID)) + } + } + conflicts, err := s.db.ListMetadataConflicts(ctx, gids) + if err != nil { + return nil, internalError("list metadata conflicts", err) + } + if conflicts == nil { + conflicts = []db.MetadataConflict{} + } + return &jsonOutput[metadataConflictsResponse]{ + Body: metadataConflictsResponse{Conflicts: conflicts}, + }, nil +} + func (s *Server) humaGetSessionDir( ctx context.Context, in *idPathInput, @@ -657,6 +696,9 @@ func (s *Server) humaRenameSession( ctx context.Context, in *renameSessionInput, ) (*jsonOutput[*db.Session], error) { + s.lockSessionLifecycle() + defer s.sessionLifecycleMu.Unlock() + session, err := s.db.GetSession(ctx, in.ID) if err != nil { return nil, internalError("rename session lookup", err) @@ -674,6 +716,27 @@ func (s *Server) humaRenameSession( } return nil, internalError("rename session", err) } + value, err := renameMetadataValue(displayName) + if err != nil { + return nil, internalError("rename session metadata value", err) + } + if err := s.appendMetadataEvent(ctx, artifact.MetadataEventInput{ + SessionID: in.ID, + Op: artifact.MetadataOpRename, + Value: value, + }); err != nil { + var publishedErr *artifact.MetadataPublishedError + if errors.As(err, &publishedErr) { + return nil, internalError("rename session metadata event", err) + } + if restoreErr := s.db.RenameSession(in.ID, session.DisplayName); restoreErr != nil { + return nil, internalError( + "rename session metadata event", + errors.Join(err, fmt.Errorf("restore display name after metadata failure: %w", restoreErr)), + ) + } + return nil, internalError("rename session metadata event", err) + } updated, err := s.db.GetSession(ctx, in.ID) if err != nil { @@ -689,6 +752,9 @@ func (s *Server) humaDeleteSession( ctx context.Context, in *idPathInput, ) (*noContentOutput, error) { + s.lockSessionLifecycle() + defer s.sessionLifecycleMu.Unlock() + session, err := s.db.GetSessionFull(ctx, in.ID) if err != nil { return nil, internalError("delete session lookup", err) @@ -702,6 +768,32 @@ func (s *Server) humaDeleteSession( } return nil, internalError("soft delete session", err) } + if err := s.appendMetadataEvent(ctx, artifact.MetadataEventInput{ + SessionID: in.ID, + Op: artifact.MetadataOpSoftDelete, + }); err != nil { + var publishedErr *artifact.MetadataPublishedError + if errors.As(err, &publishedErr) { + return nil, internalError("soft delete session metadata event", err) + } + // Only undo a trashing this request performed: SoftDeleteSession + // is a no-op on an already-trashed session, and restoring one of + // those would revert an earlier, ledger-backed deletion. + if session.DeletedAt == nil { + if n, restoreErr := s.db.RestoreSession(in.ID); restoreErr != nil { + return nil, internalError( + "soft delete session metadata event", + errors.Join(err, fmt.Errorf("restore session after metadata failure: %w", restoreErr)), + ) + } else if n == 0 { + return nil, internalError( + "soft delete session metadata event", + errors.Join(err, fmt.Errorf("restore session after metadata failure: session %q not in trash", in.ID)), + ) + } + } + return nil, internalError("soft delete session metadata event", err) + } return &noContentOutput{Status: http.StatusNoContent}, nil } @@ -711,26 +803,107 @@ type batchDeleteInput struct { } } +type trashedSessionIDStore interface { + TrashedSessionIDs(ids []string) ([]string, error) +} + +type excludedSessionStore interface { + IsSessionExcluded(id string) bool +} + func (s *Server) humaBatchDeleteSessions( - _ context.Context, + ctx context.Context, in *batchDeleteInput, ) (*noContentOutput, error) { if len(in.Body.SessionIDs) == 0 { return &noContentOutput{Status: http.StatusNoContent}, nil } - if _, err := s.db.SoftDeleteSessions(in.Body.SessionIDs); err != nil { + s.lockSessionLifecycle() + defer s.sessionLifecycleMu.Unlock() + + deletedIDs, err := s.db.SoftDeleteSessionsReturningIDs(in.Body.SessionIDs) + if err != nil { if handled := handleHumaReadOnly(err); handled != nil { return nil, handled } return nil, internalError("batch delete sessions", err) } + newlyDeleted := make(map[string]struct{}, len(deletedIDs)) + for _, id := range deletedIDs { + newlyDeleted[id] = struct{}{} + } + for i, id := range deletedIDs { + if err := s.appendMetadataEvent(ctx, artifact.MetadataEventInput{ + SessionID: id, + Op: artifact.MetadataOpSoftDelete, + }); err != nil { + rollbackStart := i + var publishedErr *artifact.MetadataPublishedError + if errors.As(err, &publishedErr) { + rollbackStart++ + } + rollback := make([]string, 0, len(deletedIDs)-rollbackStart) + for _, rollbackID := range deletedIDs[rollbackStart:] { + if _, ok := newlyDeleted[rollbackID]; ok { + rollback = append(rollback, rollbackID) + } + } + return nil, internalError( + "batch delete session metadata event", + s.restoreBatchDeletedSessions(rollback, err), + ) + } + } + store, ok := s.db.(trashedSessionIDStore) + if !ok { + return &noContentOutput{Status: http.StatusNoContent}, nil + } + trashedIDs, err := store.TrashedSessionIDs(in.Body.SessionIDs) + if err != nil { + return nil, internalError("batch delete retry lookup", err) + } + for _, id := range trashedIDs { + if _, ok := newlyDeleted[id]; ok { + continue + } + if err := s.ensureLocalMetadataEvent(ctx, artifact.MetadataEventInput{ + SessionID: id, + Op: artifact.MetadataOpSoftDelete, + }, "deleted_at", artifact.MetadataOpSoftDelete); err != nil { + return nil, internalError("batch delete metadata repair", err) + } + } return &noContentOutput{Status: http.StatusNoContent}, nil } +func (s *Server) restoreBatchDeletedSessions(ids []string, cause error) error { + errs := []error{cause} + for _, id := range ids { + n, err := s.db.RestoreSession(id) + if err != nil { + errs = append(errs, + fmt.Errorf("restore session %s after metadata failure: %w", id, err)) + continue + } + if n == 0 { + errs = append(errs, + fmt.Errorf("restore session %s after metadata failure: session not in trash", id)) + } + } + return errors.Join(errs...) +} + func (s *Server) humaRestoreSession( - _ context.Context, + ctx context.Context, in *idPathInput, ) (*noContentOutput, error) { + s.lockSessionLifecycle() + defer s.sessionLifecycleMu.Unlock() + + metadataInput := artifact.MetadataEventInput{ + SessionID: in.ID, + Op: artifact.MetadataOpRestore, + } n, err := s.db.RestoreSession(in.ID) if err != nil { if handled := handleHumaReadOnly(err); handled != nil { @@ -739,25 +912,116 @@ func (s *Server) humaRestoreSession( return nil, internalError("restore session", err) } if n == 0 { + session, err := s.db.GetSessionFull(ctx, in.ID) + if err != nil { + return nil, internalError("restore session retry lookup", err) + } + if session != nil && session.DeletedAt == nil { + op, ok, err := s.metadataReplayStateOp(ctx, in.ID, "deleted_at") + if err != nil { + return nil, internalError("restore session metadata retry lookup", err) + } + if !ok || op != artifact.MetadataOpSoftDelete { + return nil, apiError(http.StatusNotFound, "session not found or not in trash") + } + _, err = s.repairLocalMetadataEvent(ctx, metadataInput) + if err != nil { + return nil, internalError("restore session metadata repair", err) + } + op, ok, err = s.metadataReplayStateOp(ctx, in.ID, "deleted_at") + if err != nil { + return nil, internalError("restore session metadata retry lookup", err) + } + if !ok || op != artifact.MetadataOpRestore { + if err := s.appendMetadataEvent(ctx, metadataInput); err != nil { + return nil, internalError("restore session metadata event", err) + } + } + return &noContentOutput{Status: http.StatusNoContent}, nil + } return nil, apiError(http.StatusNotFound, "session not found or not in trash") } + if err := s.appendMetadataEvent(ctx, metadataInput); err != nil { + var publishedErr *artifact.MetadataPublishedError + if errors.As(err, &publishedErr) { + return nil, internalError("restore session metadata event", err) + } + deletedIDs, rollbackErr := s.db.SoftDeleteSessionsReturningIDs([]string{in.ID}) + if rollbackErr != nil { + return nil, internalError( + "restore session metadata event", + errors.Join(err, fmt.Errorf("return session to trash after metadata failure: %w", rollbackErr)), + ) + } + if len(deletedIDs) != 1 || deletedIDs[0] != in.ID { + return nil, internalError( + "restore session metadata event", + errors.Join(err, fmt.Errorf("return session %q to trash after metadata failure: session no longer visible", in.ID)), + ) + } + return nil, internalError("restore session metadata event", err) + } return &noContentOutput{Status: http.StatusNoContent}, nil } func (s *Server) humaPermanentDeleteSession( - _ context.Context, + ctx context.Context, in *idPathInput, ) (*noContentOutput, error) { + s.lockSessionLifecycle() + defer s.sessionLifecycleMu.Unlock() + + metadataInput := artifact.MetadataEventInput{ + SessionID: in.ID, + Op: artifact.MetadataOpPurge, + } + session, err := s.db.GetSessionFull(ctx, in.ID) + if err != nil { + return nil, internalError("permanent delete session lookup", err) + } + if session == nil { + if store, ok := s.db.(excludedSessionStore); ok && store.IsSessionExcluded(in.ID) { + repaired, err := s.repairLocalMetadataEvent(ctx, metadataInput) + if err != nil { + return nil, internalError("permanent delete session metadata repair", err) + } + if repaired == 0 { + if err := s.appendMetadataEvent(ctx, metadataInput); err != nil { + return nil, internalError("permanent delete session metadata event", err) + } + } + return &noContentOutput{Status: http.StatusNoContent}, nil + } + return nil, apiError(http.StatusConflict, "session not found or not in trash") + } + if session.DeletedAt == nil { + return nil, apiError(http.StatusConflict, "session not found or not in trash") + } + metadataErr := s.ensureLocalMetadataEvent( + ctx, metadataInput, "purge", artifact.MetadataOpPurge, + ) + if metadataErr != nil { + var publishedErr *artifact.MetadataPublishedError + if !errors.As(metadataErr, &publishedErr) { + return nil, internalError("permanent delete session metadata event", metadataErr) + } + } n, err := s.db.DeleteSessionIfTrashed(in.ID) if err != nil { if handled := handleHumaReadOnly(err); handled != nil { return nil, handled } - return nil, internalError("permanent delete session", err) + return nil, internalError( + "permanent delete session", + errors.Join(metadataErr, err), + ) } if n == 0 { return nil, apiError(http.StatusConflict, "session not found or not in trash") } + if metadataErr != nil { + return nil, internalError("permanent delete session metadata event", metadataErr) + } return &noContentOutput{Status: http.StatusNoContent}, nil } @@ -776,6 +1040,9 @@ func (s *Server) humaEmptyTrash( _ context.Context, _ *emptyInput, ) (*jsonOutput[emptyTrashResponse], error) { + s.lockSessionLifecycle() + defer s.sessionLifecycleMu.Unlock() + count, err := s.db.EmptyTrash() if err != nil { if handled := handleHumaReadOnly(err); handled != nil { diff --git a/internal/server/huma_routes_starred.go b/internal/server/huma_routes_starred.go index 1235123b4..3b8284705 100644 --- a/internal/server/huma_routes_starred.go +++ b/internal/server/huma_routes_starred.go @@ -2,7 +2,12 @@ package server import ( "context" + "errors" + "fmt" "net/http" + "slices" + + "go.kenn.io/agentsview/internal/artifact" ) func (s *Server) registerStarredRoutes() { @@ -39,9 +44,23 @@ func (s *Server) humaListStarred( } func (s *Server) humaStarSession( - _ context.Context, + ctx context.Context, in *idPathInput, ) (*noContentOutput, error) { + s.lockSessionLifecycle() + defer s.sessionLifecycleMu.Unlock() + + // Prior state decides rollback: StarSession reports success for an + // already-starred session too, and that star must survive a failed + // metadata append. + wasStarred := false + if s.metadata != nil { + var err error + wasStarred, err = s.sessionStarred(ctx, in.ID) + if err != nil { + return nil, internalError("star session prior state", err) + } + } ok, err := s.db.StarSession(in.ID) if err != nil { if handled := handleHumaReadOnly(err); handled != nil { @@ -52,34 +71,167 @@ func (s *Server) humaStarSession( if !ok { return nil, apiError(http.StatusNotFound, "session not found") } + if err := s.appendMetadataEvent(ctx, artifact.MetadataEventInput{ + SessionID: in.ID, + Op: artifact.MetadataOpStar, + }); err != nil { + var publishedErr *artifact.MetadataPublishedError + if errors.As(err, &publishedErr) { + return nil, internalError("star session metadata event", err) + } + if !wasStarred { + if _, removeErr := s.db.UnstarSession(in.ID); removeErr != nil { + return nil, internalError( + "star session metadata event", + errors.Join(err, fmt.Errorf("remove star after metadata failure: %w", removeErr)), + ) + } + } + return nil, internalError("star session metadata event", err) + } return &noContentOutput{Status: http.StatusNoContent}, nil } +// sessionStarred reports whether the session is currently starred. +func (s *Server) sessionStarred(ctx context.Context, id string) (bool, error) { + ids, err := s.db.ListStarredSessionIDs(ctx) + if err != nil { + return false, err + } + if slices.Contains(ids, id) { + return true, nil + } + return false, nil +} + func (s *Server) humaUnstarSession( - _ context.Context, + ctx context.Context, in *idPathInput, ) (*noContentOutput, error) { - if err := s.db.UnstarSession(in.ID); err != nil { + s.lockSessionLifecycle() + defer s.sessionLifecycleMu.Unlock() + + removed, err := s.db.UnstarSession(in.ID) + if err != nil { if handled := handleHumaReadOnly(err); handled != nil { return nil, handled } return nil, internalError("unstar session", err) } + if !removed { + if _, err := s.repairLocalMetadataEvent(ctx, artifact.MetadataEventInput{ + SessionID: in.ID, + Op: artifact.MetadataOpUnstar, + }); err != nil { + return nil, internalError("unstar session metadata repair", err) + } + return &noContentOutput{Status: http.StatusNoContent}, nil + } + if err := s.appendMetadataEvent(ctx, artifact.MetadataEventInput{ + SessionID: in.ID, + Op: artifact.MetadataOpUnstar, + }); err != nil { + var publishedErr *artifact.MetadataPublishedError + if errors.As(err, &publishedErr) { + return nil, internalError("unstar session metadata event", err) + } + if restored, restoreErr := s.db.StarSession(in.ID); restoreErr != nil { + return nil, internalError( + "unstar session metadata event", + errors.Join(err, fmt.Errorf("restore star after metadata failure: %w", restoreErr)), + ) + } else if !restored { + return nil, internalError( + "unstar session metadata event", + errors.Join(err, fmt.Errorf("restore star after metadata failure: session %q not found", in.ID)), + ) + } + return nil, internalError("unstar session metadata event", err) + } return &noContentOutput{Status: http.StatusNoContent}, nil } func (s *Server) humaBulkStar( - _ context.Context, + ctx context.Context, in *bulkStarInput, ) (*noContentOutput, error) { if len(in.Body.SessionIDs) == 0 { return &noContentOutput{Status: http.StatusNoContent}, nil } - if err := s.db.BulkStarSessions(in.Body.SessionIDs); err != nil { + s.lockSessionLifecycle() + defer s.sessionLifecycleMu.Unlock() + + starred, err := s.db.BulkStarSessions(in.Body.SessionIDs) + if err != nil { if handled := handleHumaReadOnly(err); handled != nil { return nil, handled } return nil, internalError("bulk star", err) } + newlyStarred := make(map[string]struct{}, len(starred)) + // Emit one star event per session actually starred so localStorage star + // migration converges through artifact sync, matching single-session star. + for i, id := range starred { + newlyStarred[id] = struct{}{} + if err := s.appendMetadataEvent(ctx, artifact.MetadataEventInput{ + SessionID: id, + Op: artifact.MetadataOpStar, + }); err != nil { + // Stars whose events are already in the ledger stay; the + // rest were created by this request without a ledger event + // to sync them, so they are removed. A published error + // means the failed event itself is durably recorded, so + // its star stays too. + rollback := starred[i:] + var publishedErr *artifact.MetadataPublishedError + if errors.As(err, &publishedErr) { + rollback = starred[i+1:] + } + return nil, internalError("bulk star metadata event", + s.rollbackBulkStar(rollback, err)) + } + } + starredIDs, err := s.db.ListStarredSessionIDs(ctx) + if err != nil { + return nil, internalError("bulk star metadata repair", err) + } + starredNow := make(map[string]struct{}, len(starredIDs)) + for _, id := range starredIDs { + starredNow[id] = struct{}{} + } + seenRetry := map[string]struct{}{} + for _, id := range in.Body.SessionIDs { + if _, ok := newlyStarred[id]; ok { + continue + } + if _, ok := starredNow[id]; !ok { + continue + } + if _, ok := seenRetry[id]; ok { + continue + } + seenRetry[id] = struct{}{} + if err := s.ensureLocalMetadataEvent(ctx, artifact.MetadataEventInput{ + SessionID: id, + Op: artifact.MetadataOpStar, + }, "starred", artifact.MetadataOpStar); err != nil { + return nil, internalError("bulk star metadata repair", err) + } + } return &noContentOutput{Status: http.StatusNoContent}, nil } + +// rollbackBulkStar removes stars this request created but never +// recorded in the ledger, so local state does not run ahead of the +// artifact log. The returned error joins the append failure with any +// rollback failures. +func (s *Server) rollbackBulkStar(ids []string, cause error) error { + errs := []error{cause} + for _, id := range ids { + if _, err := s.db.UnstarSession(id); err != nil { + errs = append(errs, + fmt.Errorf("remove star %s after metadata failure: %w", id, err)) + } + } + return errors.Join(errs...) +} diff --git a/internal/server/metadata_events.go b/internal/server/metadata_events.go new file mode 100644 index 000000000..d220e553e --- /dev/null +++ b/internal/server/metadata_events.go @@ -0,0 +1,115 @@ +package server + +import ( + "context" + "encoding/json" + "fmt" + + "go.kenn.io/agentsview/internal/artifact" +) + +func (s *Server) appendMetadataEvent( + ctx context.Context, + input artifact.MetadataEventInput, +) error { + if artifact.MetadataEventsSuppressed(ctx) { + return nil + } + if s.metadataAppend != nil { + return s.metadataAppend(ctx, input) + } + if s.metadata == nil { + return nil + } + _, err := s.metadata.Append(ctx, input) + return err +} + +func (s *Server) repairLocalMetadataEvent( + ctx context.Context, + input artifact.MetadataEventInput, +) (int, error) { + if s.metadata == nil { + return 0, nil + } + return s.metadata.RepairLocalSessionMetadata(ctx, input.SessionID, input.Op) +} + +func (s *Server) ensureLocalMetadataEvent( + ctx context.Context, + input artifact.MetadataEventInput, + field string, + wantOp string, +) error { + if s.metadata == nil { + if s.metadataAppend != nil { + return s.appendMetadataEvent(ctx, input) + } + return nil + } + if _, err := s.repairLocalMetadataEvent(ctx, input); err != nil { + return err + } + op, ok, err := s.metadataReplayStateOp(ctx, input.SessionID, field) + if err != nil { + return err + } + if ok && op == wantOp { + return nil + } + return s.appendMetadataEvent(ctx, input) +} + +type metadataReplayStateStore interface { + MetadataReplayStateOp(ctx context.Context, sessionGID string, field string) (string, bool, error) +} + +func (s *Server) metadataReplayStateOp( + ctx context.Context, + sessionID string, + field string, +) (string, bool, error) { + store, ok := s.db.(metadataReplayStateStore) + if !ok { + return "", false, nil + } + origin := s.localArtifactOrigin() + if origin == "" { + return "", false, nil + } + return store.MetadataReplayStateOp(ctx, artifact.MetadataSessionGID(origin, sessionID), field) +} + +func renameMetadataValue(displayName *string) (json.RawMessage, error) { + data, err := json.Marshal(struct { + DisplayName *string `json:"display_name"` + }{DisplayName: displayName}) + if err != nil { + return nil, err + } + return json.RawMessage(data), nil +} + +func (s *Server) metadataPinForMessage( + ctx context.Context, + sessionID string, + messageID int64, + note *string, +) (*artifact.MetadataPin, error) { + msg, err := s.db.GetMessageForMetadataPin(ctx, sessionID, messageID) + if err != nil { + return nil, fmt.Errorf("loading message for metadata pin: %w", err) + } + if msg == nil { + return nil, nil + } + pin := &artifact.MetadataPin{ + SourceUUID: msg.SourceUUID, + Ordinal: msg.Ordinal, + } + if note != nil { + noteCopy := *note + pin.Note = ¬eCopy + } + return pin, nil +} diff --git a/internal/server/metadata_events_test.go b/internal/server/metadata_events_test.go new file mode 100644 index 000000000..e50aa7268 --- /dev/null +++ b/internal/server/metadata_events_test.go @@ -0,0 +1,992 @@ +package server_test + +import ( + "context" + "database/sql" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "sort" + "strings" + "testing" + + _ "github.com/mattn/go-sqlite3" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "go.kenn.io/agentsview/internal/artifact" + "go.kenn.io/agentsview/internal/config" + "go.kenn.io/agentsview/internal/db" +) + +type recordedMetadataEvent struct { + Version int `json:"v"` + HLC string `json:"hlc"` + Origin string `json:"origin"` + SessionGID string `json:"session_gid"` + Op string `json:"op"` + Value map[string]any `json:"value,omitempty"` + Pin *recordedMetadataPin `json:"pin,omitempty"` +} + +type recordedMetadataPin struct { + SourceUUID string `json:"source_uuid,omitempty"` + Ordinal int `json:"ordinal,omitempty"` + Note *string `json:"note,omitempty"` +} + +func withArtifactOrigin(origin string) setupOption { + return func(c *config.Config) { c.ArtifactOriginID = origin } +} + +func TestMetadataEventsAppendForUserMutations(t *testing.T) { + te := setup(t, withArtifactOrigin("desk-a1b2c3")) + te.seedSession(t, "s1", "alpha", 2) + te.seedMessages(t, "s1", 2, func(i int, m *db.Message) { + if i == 1 { + m.SourceUUID = "uuid-answer" + } + }) + msgs, err := te.db.GetAllMessages(context.Background(), "s1") + require.NoError(t, err) + require.Len(t, msgs, 2) + messageID := msgs[1].ID + + w := te.patch(t, "/api/v1/sessions/s1/rename", `{"display_name":"Pinned investigation"}`) + require.Equal(t, http.StatusOK, w.Code, "body: %s", w.Body.String()) + + w = te.put(t, "/api/v1/sessions/s1/star", `{}`) + require.Equal(t, http.StatusNoContent, w.Code, "body: %s", w.Body.String()) + + w = te.del(t, "/api/v1/sessions/s1/star") + require.Equal(t, http.StatusNoContent, w.Code, "body: %s", w.Body.String()) + + w = te.post(t, fmt.Sprintf("/api/v1/sessions/s1/messages/%d/pin", messageID), `{"note":"remember"}`) + require.Equal(t, http.StatusCreated, w.Code, "body: %s", w.Body.String()) + + w = te.del(t, fmt.Sprintf("/api/v1/sessions/s1/messages/%d/pin", messageID)) + require.Equal(t, http.StatusNoContent, w.Code, "body: %s", w.Body.String()) + + w = te.del(t, "/api/v1/sessions/s1") + require.Equal(t, http.StatusNoContent, w.Code, "body: %s", w.Body.String()) + + w = te.post(t, "/api/v1/sessions/s1/restore", `{}`) + require.Equal(t, http.StatusNoContent, w.Code, "body: %s", w.Body.String()) + + w = te.del(t, "/api/v1/sessions/s1") + require.Equal(t, http.StatusNoContent, w.Code, "body: %s", w.Body.String()) + + w = te.del(t, "/api/v1/sessions/s1/permanent") + require.Equal(t, http.StatusNoContent, w.Code, "body: %s", w.Body.String()) + + events := readMetadataEvents(t, te) + require.Len(t, events, 9) + assert.Equal(t, []string{ + artifact.MetadataOpRename, + artifact.MetadataOpStar, + artifact.MetadataOpUnstar, + artifact.MetadataOpPin, + artifact.MetadataOpUnpin, + artifact.MetadataOpSoftDelete, + artifact.MetadataOpRestore, + artifact.MetadataOpSoftDelete, + artifact.MetadataOpPurge, + }, metadataOps(events)) + for _, event := range events { + assert.Equal(t, 1, event.Version) + assert.NotEmpty(t, event.HLC) + assert.Equal(t, "desk-a1b2c3", event.Origin) + assert.Equal(t, "desk-a1b2c3~s1", event.SessionGID) + } + assert.Equal(t, "Pinned investigation", events[0].Value["display_name"]) + require.NotNil(t, events[3].Pin) + assert.Equal(t, "uuid-answer", events[3].Pin.SourceUUID) + assert.Equal(t, 1, events[3].Pin.Ordinal) + require.NotNil(t, events[3].Pin.Note) + assert.Equal(t, "remember", *events[3].Pin.Note) + require.NotNil(t, events[4].Pin) + assert.Equal(t, "uuid-answer", events[4].Pin.SourceUUID) + assert.Equal(t, 1, events[4].Pin.Ordinal) + assert.Nil(t, events[4].Pin.Note) +} + +func TestMetadataEventsAppendWithoutSyncEngine(t *testing.T) { + te := setupNoSyncMode(t) + require.NoError(t, artifact.AdoptOrigin(te.db, "nosync-a1b2c3")) + te.seedSession(t, "s1", "alpha", 2) + + w := te.patch(t, "/api/v1/sessions/s1/rename", `{"display_name":"No sync title"}`) + require.Equal(t, http.StatusOK, w.Code, "body: %s", w.Body.String()) + + events := readMetadataEvents(t, te) + require.Len(t, events, 1) + assert.Equal(t, artifact.MetadataOpRename, events[0].Op) + assert.Equal(t, "nosync-a1b2c3", events[0].Origin) + assert.Equal(t, "nosync-a1b2c3~s1", events[0].SessionGID) +} + +func TestMetadataEventsNotRecordedWithoutOptIn(t *testing.T) { + te := setup(t) + te.seedSession(t, "s1", "alpha", 2) + + w := te.patch(t, "/api/v1/sessions/s1/rename", `{"display_name":"Local only"}`) + require.Equal(t, http.StatusOK, w.Code, "body: %s", w.Body.String()) + w = te.put(t, "/api/v1/sessions/s1/star", `{}`) + require.Equal(t, http.StatusNoContent, w.Code, "body: %s", w.Body.String()) + + assert.Empty(t, readMetadataEvents(t, te), + "curation must not write ledger events before the machine opts into artifact sync") + origin, err := artifact.StoredOrigin(te.db) + require.NoError(t, err) + assert.Empty(t, origin, + "curation must not mint an artifact origin before opt-in") +} + +func TestMetadataEventsEmptyTrashStaysLocalOnly(t *testing.T) { + te := setup(t, withArtifactOrigin("desk-a1b2c3")) + for _, id := range []string{"s1", "s2"} { + te.seedSession(t, id, "alpha", 2) + w := te.del(t, "/api/v1/sessions/"+id) + require.Equal(t, http.StatusNoContent, w.Code, "delete %s body: %s", id, w.Body.String()) + } + + w := te.del(t, "/api/v1/trash") + require.Equal(t, http.StatusOK, w.Code, "body: %s", w.Body.String()) + + events := readMetadataEvents(t, te) + require.Len(t, events, 2) + assert.Equal(t, []string{ + artifact.MetadataOpSoftDelete, + artifact.MetadataOpSoftDelete, + }, metadataOps(events)) +} + +func TestMetadataEventsBatchDeleteRecordsNewAndUnrecordedTrashedSessions(t *testing.T) { + te := setup(t, withArtifactOrigin("desk-a1b2c3")) + for _, id := range []string{"s1", "s2", "s3"} { + te.seedSession(t, id, "alpha", 2) + } + require.NoError(t, te.db.SoftDeleteSession("s3")) + + w := te.requestJSON(t, http.MethodPost, "/api/v1/sessions/batch-delete", + `{"session_ids":["s1","s2","s3","missing"]}`) + require.Equal(t, http.StatusNoContent, w.Code, "body: %s", w.Body.String()) + + events := readMetadataEvents(t, te) + require.Len(t, events, 3) + assert.Equal(t, []string{ + artifact.MetadataOpSoftDelete, + artifact.MetadataOpSoftDelete, + artifact.MetadataOpSoftDelete, + }, metadataOps(events)) + assert.ElementsMatch(t, []string{ + "desk-a1b2c3~s1", + "desk-a1b2c3~s2", + "desk-a1b2c3~s3", + }, []string{events[0].SessionGID, events[1].SessionGID, events[2].SessionGID}) +} + +func TestMetadataEventsBatchDeleteRetriesAlreadyDeletedSessions(t *testing.T) { + te := setup(t, withArtifactOrigin("desk-a1b2c3")) + for _, id := range []string{"s1", "s2"} { + te.seedSession(t, id, "alpha", 2) + require.NoError(t, te.db.SoftDeleteSession(id)) + } + + w := te.requestJSON(t, http.MethodPost, "/api/v1/sessions/batch-delete", + `{"session_ids":["s1","s2","missing"]}`) + require.Equal(t, http.StatusNoContent, w.Code, "body: %s", w.Body.String()) + + events := readMetadataEvents(t, te) + require.Len(t, events, 2) + assert.Equal(t, []string{ + artifact.MetadataOpSoftDelete, + artifact.MetadataOpSoftDelete, + }, metadataOps(events)) + assert.ElementsMatch(t, []string{ + "desk-a1b2c3~s1", + "desk-a1b2c3~s2", + }, []string{events[0].SessionGID, events[1].SessionGID}) +} + +func TestMetadataEventsBatchDeleteRepairsPublishedFailureOnRetry(t *testing.T) { + te := setup(t, withArtifactOrigin("desk-a1b2c3")) + for _, id := range []string{"s1", "s2"} { + te.seedSession(t, id, "alpha", 2) + } + execTestDDL(t, te, ` +CREATE TRIGGER fail_s1_metadata_replay_state_insert +BEFORE INSERT ON metadata_replay_state +WHEN NEW.session_gid = 'desk-a1b2c3~s1' +BEGIN + SELECT RAISE(FAIL, 'forced s1 metadata replay failure'); +END; +`) + + w := te.requestJSON(t, http.MethodPost, "/api/v1/sessions/batch-delete", + `{"session_ids":["s1","s2"]}`) + require.Equal(t, http.StatusInternalServerError, w.Code, "body: %s", w.Body.String()) + + s1, err := te.db.GetSessionFull(context.Background(), "s1") + require.NoError(t, err) + require.NotNil(t, s1) + assert.NotNil(t, s1.DeletedAt, + "published failure keeps the session whose artifact is durable in trash") + s2, err := te.db.GetSessionFull(context.Background(), "s2") + require.NoError(t, err) + require.NotNil(t, s2) + assert.Nil(t, s2.DeletedAt, + "later unpublished sessions are restored after the failure") + assert.Equal(t, 0, serverMetadataTableCount( + t, te, "metadata_replay_state", + "session_gid = 'desk-a1b2c3~s1' AND field = 'deleted_at'", + )) + firstEvents := readMetadataEvents(t, te) + require.Len(t, firstEvents, 1) + assert.Equal(t, "desk-a1b2c3~s1", firstEvents[0].SessionGID) + + execTestDDL(t, te, `DROP TRIGGER fail_s1_metadata_replay_state_insert`) + w = te.requestJSON(t, http.MethodPost, "/api/v1/sessions/batch-delete", + `{"session_ids":["s1","s2"]}`) + require.Equal(t, http.StatusNoContent, w.Code, "body: %s", w.Body.String()) + + for _, id := range []string{"s1", "s2"} { + sess, getErr := te.db.GetSessionFull(context.Background(), id) + require.NoError(t, getErr) + require.NotNil(t, sess) + assert.NotNil(t, sess.DeletedAt, "retry must trash %s", id) + assert.Equal(t, artifact.MetadataOpSoftDelete, + serverMetadataReplayOp(t, te, "desk-a1b2c3~"+id, "deleted_at")) + } + events := readMetadataEvents(t, te) + require.Len(t, events, 2, + "retry must repair the first artifact instead of publishing a duplicate") + counts := map[string]int{} + for _, event := range events { + counts[event.SessionGID]++ + } + assert.Equal(t, map[string]int{ + "desk-a1b2c3~s1": 1, + "desk-a1b2c3~s2": 1, + }, counts) +} + +func TestMetadataEventsUnstarOnlyRecordsRemovedStars(t *testing.T) { + te := setup(t, withArtifactOrigin("desk-a1b2c3")) + te.seedSession(t, "s1", "alpha", 2) + + w := te.del(t, "/api/v1/sessions/missing/star") + require.Equal(t, http.StatusNoContent, w.Code, "body: %s", w.Body.String()) + w = te.del(t, "/api/v1/sessions/s1/star") + require.Equal(t, http.StatusNoContent, w.Code, "body: %s", w.Body.String()) + assert.Empty(t, readMetadataEvents(t, te)) + + ok, err := te.db.StarSession("s1") + require.NoError(t, err) + require.True(t, ok) + w = te.del(t, "/api/v1/sessions/s1/star") + require.Equal(t, http.StatusNoContent, w.Code, "body: %s", w.Body.String()) + + events := readMetadataEvents(t, te) + require.Len(t, events, 1) + assert.Equal(t, artifact.MetadataOpUnstar, events[0].Op) + assert.Equal(t, "desk-a1b2c3~s1", events[0].SessionGID) +} + +func TestMetadataEventsUnstarRestoresStarWhenArtifactWriteFails(t *testing.T) { + te := setup(t, withArtifactOrigin("desk-a1b2c3")) + te.seedSession(t, "s1", "alpha", 2) + ok, err := te.db.StarSession("s1") + require.NoError(t, err) + require.True(t, ok) + + metaDir := filepath.Join(te.dataDir, "artifacts", "desk-a1b2c3", "meta") + require.NoError(t, os.MkdirAll(filepath.Dir(metaDir), 0o755)) + require.NoError(t, os.WriteFile(metaDir, []byte("not a directory"), 0o644)) + + w := te.del(t, "/api/v1/sessions/s1/star") + require.Equal(t, http.StatusInternalServerError, w.Code, "body: %s", w.Body.String()) + ids, err := te.db.ListStarredSessionIDs(context.Background()) + require.NoError(t, err) + assert.Equal(t, []string{"s1"}, ids) + assert.Equal(t, 0, serverMetadataTableCount(t, te, "metadata_replay_state", "session_gid = 'desk-a1b2c3~s1'")) + assert.Equal(t, 0, serverMetadataTableCount(t, te, "metadata_applied_events", "origin = 'desk-a1b2c3'")) + + require.NoError(t, os.Remove(metaDir)) + w = te.del(t, "/api/v1/sessions/s1/star") + require.Equal(t, http.StatusNoContent, w.Code, "body: %s", w.Body.String()) + ids, err = te.db.ListStarredSessionIDs(context.Background()) + require.NoError(t, err) + assert.Empty(t, ids) + + events := readMetadataEvents(t, te) + require.Len(t, events, 1) + assert.Equal(t, artifact.MetadataOpUnstar, events[0].Op) + assert.Equal(t, "desk-a1b2c3~s1", events[0].SessionGID) +} + +func TestMetadataEventsUnstarDoesNotRestoreStarWhenArtifactPublished(t *testing.T) { + te := setup(t, withArtifactOrigin("desk-a1b2c3")) + te.seedSession(t, "s1", "alpha", 2) + ok, err := te.db.StarSession("s1") + require.NoError(t, err) + require.True(t, ok) + + execTestDDL(t, te, ` +CREATE TRIGGER fail_metadata_replay_state_insert +BEFORE INSERT ON metadata_replay_state +BEGIN + SELECT RAISE(FAIL, 'forced metadata replay failure'); +END; +`) + + w := te.del(t, "/api/v1/sessions/s1/star") + require.Equal(t, http.StatusInternalServerError, w.Code, "body: %s", w.Body.String()) + ids, err := te.db.ListStarredSessionIDs(context.Background()) + require.NoError(t, err) + assert.Empty(t, ids) + assert.Equal(t, 0, serverMetadataTableCount(t, te, "metadata_replay_state", "session_gid = 'desk-a1b2c3~s1'")) + assert.Equal(t, 0, serverMetadataTableCount(t, te, "metadata_applied_events", "origin = 'desk-a1b2c3'")) + + events := readMetadataEvents(t, te) + require.Len(t, events, 1) + assert.Equal(t, artifact.MetadataOpUnstar, events[0].Op) + assert.Equal(t, "desk-a1b2c3~s1", events[0].SessionGID) +} + +func TestMetadataEventsNoopUnstarRepairsPublishedArtifactState(t *testing.T) { + te := setup(t, withArtifactOrigin("desk-a1b2c3")) + te.seedSession(t, "s1", "alpha", 2) + + w := te.put(t, "/api/v1/sessions/s1/star", `{}`) + require.Equal(t, http.StatusNoContent, w.Code, "body: %s", w.Body.String()) + assert.Equal(t, artifact.MetadataOpStar, + serverMetadataReplayOp(t, te, "desk-a1b2c3~s1", "starred")) + + execTestDDL(t, te, ` +CREATE TRIGGER fail_metadata_replay_state_insert +BEFORE INSERT ON metadata_replay_state +BEGIN + SELECT RAISE(FAIL, 'forced metadata replay failure'); +END; +`) + + w = te.del(t, "/api/v1/sessions/s1/star") + require.Equal(t, http.StatusInternalServerError, w.Code, "body: %s", w.Body.String()) + assert.Equal(t, artifact.MetadataOpStar, + serverMetadataReplayOp(t, te, "desk-a1b2c3~s1", "starred")) + unstarOrderKey := metadataEventOrderKey(t, te, artifact.MetadataOpUnstar) + + execTestDDL(t, te, `DROP TRIGGER fail_metadata_replay_state_insert`) + w = te.del(t, "/api/v1/sessions/s1/star") + require.Equal(t, http.StatusNoContent, w.Code, "body: %s", w.Body.String()) + assert.Equal(t, artifact.MetadataOpUnstar, + serverMetadataReplayOp(t, te, "desk-a1b2c3~s1", "starred")) + + remoteHLC, remoteHash := splitMetadataOrderKey(t, unstarOrderKey) + _, err := te.db.ApplyMetadataProjection(context.Background(), db.MetadataProjection{ + EventOrigin: "peer-b2c3d4", + OrderKey: unstarOrderKey, + HLC: remoteHLC, + ArtifactHash: remoteHash, + SessionGID: "desk-a1b2c3~s1", + LocalSessionID: "s1", + Field: "starred", + Op: artifact.MetadataOpStar, + Value: artifact.MetadataOpStar, + }) + require.NoError(t, err) + ids, err := te.db.ListStarredSessionIDs(context.Background()) + require.NoError(t, err) + assert.Empty(t, ids) +} + +func TestMetadataEventsStarRollsBackWhenArtifactWriteFails(t *testing.T) { + te := setup(t, withArtifactOrigin("desk-a1b2c3")) + te.seedSession(t, "s1", "alpha", 2) + metaDir := breakMetadataArtifactDir(t, te) + + w := te.put(t, "/api/v1/sessions/s1/star", `{}`) + require.Equal(t, http.StatusInternalServerError, w.Code, "body: %s", w.Body.String()) + ids, err := te.db.ListStarredSessionIDs(context.Background()) + require.NoError(t, err) + assert.Empty(t, ids, "failed metadata publish must roll back the star") + + require.NoError(t, os.Remove(metaDir)) + w = te.put(t, "/api/v1/sessions/s1/star", `{}`) + require.Equal(t, http.StatusNoContent, w.Code, "body: %s", w.Body.String()) + events := readMetadataEvents(t, te) + require.Len(t, events, 1) + assert.Equal(t, artifact.MetadataOpStar, events[0].Op) + + // A pre-existing star survives a later failed re-star: the failed + // append changed nothing this request needs to undo. + // (breakMetadataArtifactDir also wipes recorded events.) + breakMetadataArtifactDir(t, te) + w = te.put(t, "/api/v1/sessions/s1/star", `{}`) + require.Equal(t, http.StatusInternalServerError, w.Code, "body: %s", w.Body.String()) + ids, err = te.db.ListStarredSessionIDs(context.Background()) + require.NoError(t, err) + assert.Equal(t, []string{"s1"}, ids, + "failed re-star must not remove the pre-existing star") +} + +func TestMetadataEventsStarKeepsStarWhenArtifactPublished(t *testing.T) { + te := setup(t, withArtifactOrigin("desk-a1b2c3")) + te.seedSession(t, "s1", "alpha", 2) + + execTestDDL(t, te, ` +CREATE TRIGGER fail_metadata_replay_state_insert +BEFORE INSERT ON metadata_replay_state +BEGIN + SELECT RAISE(FAIL, 'forced metadata replay failure'); +END; +`) + + w := te.put(t, "/api/v1/sessions/s1/star", `{}`) + require.Equal(t, http.StatusInternalServerError, w.Code, "body: %s", w.Body.String()) + ids, err := te.db.ListStarredSessionIDs(context.Background()) + require.NoError(t, err) + assert.Equal(t, []string{"s1"}, ids, + "published metadata event must keep the local star") + + events := readMetadataEvents(t, te) + require.Len(t, events, 1) + assert.Equal(t, artifact.MetadataOpStar, events[0].Op) +} + +func TestMetadataEventsBulkStarRollsBackWhenArtifactWriteFails(t *testing.T) { + te := setup(t, withArtifactOrigin("desk-a1b2c3")) + te.seedSession(t, "s1", "alpha", 2) + te.seedSession(t, "s2", "alpha", 2) + ok, err := te.db.StarSession("s1") + require.NoError(t, err) + require.True(t, ok) + + breakMetadataArtifactDir(t, te) + w := te.requestJSON(t, http.MethodPost, "/api/v1/starred/bulk", + `{"session_ids":["s1","s2"]}`) + require.Equal(t, http.StatusInternalServerError, w.Code, "body: %s", w.Body.String()) + + ids, err := te.db.ListStarredSessionIDs(context.Background()) + require.NoError(t, err) + assert.Equal(t, []string{"s1"}, ids, + "failed metadata publish must roll back only the stars this request created") + assert.Empty(t, readMetadataEvents(t, te)) +} + +func TestMetadataEventsRenameRestoresNameWhenArtifactWriteFails(t *testing.T) { + te := setup(t, withArtifactOrigin("desk-a1b2c3")) + te.seedSession(t, "s1", "alpha", 2) + w := te.patch(t, "/api/v1/sessions/s1/rename", `{"display_name":"keep"}`) + require.Equal(t, http.StatusOK, w.Code, "body: %s", w.Body.String()) + + breakMetadataArtifactDir(t, te) + w = te.patch(t, "/api/v1/sessions/s1/rename", `{"display_name":"replace"}`) + require.Equal(t, http.StatusInternalServerError, w.Code, "body: %s", w.Body.String()) + + session, err := te.db.GetSession(context.Background(), "s1") + require.NoError(t, err) + require.NotNil(t, session) + require.NotNil(t, session.DisplayName) + assert.Equal(t, "keep", *session.DisplayName, + "failed metadata publish must restore the prior display name") + // breakMetadataArtifactDir also wiped the first rename's event, so + // no events remain after the rolled-back rename. + assert.Empty(t, readMetadataEvents(t, te)) +} + +func TestMetadataEventsDeleteRestoresSessionWhenArtifactWriteFails(t *testing.T) { + te := setup(t, withArtifactOrigin("desk-a1b2c3")) + te.seedSession(t, "s1", "alpha", 2) + metaDir := breakMetadataArtifactDir(t, te) + + w := te.del(t, "/api/v1/sessions/s1") + require.Equal(t, http.StatusInternalServerError, w.Code, "body: %s", w.Body.String()) + session, err := te.db.GetSessionFull(context.Background(), "s1") + require.NoError(t, err) + require.NotNil(t, session) + assert.Nil(t, session.DeletedAt, + "failed metadata publish must restore the soft-deleted session") + + require.NoError(t, os.Remove(metaDir)) + w = te.del(t, "/api/v1/sessions/s1") + require.Equal(t, http.StatusNoContent, w.Code, "body: %s", w.Body.String()) + + events := readMetadataEvents(t, te) + require.Len(t, events, 1) + assert.Equal(t, artifact.MetadataOpSoftDelete, events[0].Op) +} + +func seedPinTestMessage(t *testing.T, te *testEnv) int64 { + t.Helper() + te.seedSession(t, "s1", "alpha", 2) + te.seedMessages(t, "s1", 2) + msgs, err := te.db.GetAllMessages(context.Background(), "s1") + require.NoError(t, err) + require.Len(t, msgs, 2) + return msgs[1].ID +} + +func breakMetadataArtifactDir(t *testing.T, te *testEnv) string { + t.Helper() + metaDir := filepath.Join(te.dataDir, "artifacts", "desk-a1b2c3", "meta") + require.NoError(t, os.RemoveAll(metaDir)) + require.NoError(t, os.MkdirAll(filepath.Dir(metaDir), 0o755)) + require.NoError(t, os.WriteFile(metaDir, []byte("not a directory"), 0o644)) + return metaDir +} + +func TestMetadataEventsPinRollsBackWhenArtifactWriteFails(t *testing.T) { + te := setup(t, withArtifactOrigin("desk-a1b2c3")) + messageID := seedPinTestMessage(t, te) + metaDir := breakMetadataArtifactDir(t, te) + pinPath := fmt.Sprintf("/api/v1/sessions/s1/messages/%d/pin", messageID) + + w := te.post(t, pinPath, `{"note":"remember"}`) + require.Equal(t, http.StatusInternalServerError, w.Code, "body: %s", w.Body.String()) + pins, err := te.db.ListPinnedMessages(context.Background(), "s1", "") + require.NoError(t, err) + assert.Empty(t, pins, "failed metadata publish must roll back the pin") + assert.Equal(t, 0, serverMetadataTableCount(t, te, "metadata_replay_state", "session_gid = 'desk-a1b2c3~s1'")) + assert.Equal(t, 0, serverMetadataTableCount(t, te, "metadata_applied_events", "origin = 'desk-a1b2c3'")) + + require.NoError(t, os.Remove(metaDir)) + w = te.post(t, pinPath, `{"note":"remember"}`) + require.Equal(t, http.StatusCreated, w.Code, "body: %s", w.Body.String()) + pins, err = te.db.ListPinnedMessages(context.Background(), "s1", "") + require.NoError(t, err) + require.Len(t, pins, 1) + + events := readMetadataEvents(t, te) + require.Len(t, events, 1) + assert.Equal(t, artifact.MetadataOpPin, events[0].Op) +} + +func TestMetadataEventsRepinRestoresPriorNoteWhenArtifactWriteFails(t *testing.T) { + te := setup(t, withArtifactOrigin("desk-a1b2c3")) + messageID := seedPinTestMessage(t, te) + pinPath := fmt.Sprintf("/api/v1/sessions/s1/messages/%d/pin", messageID) + + w := te.post(t, pinPath, `{"note":"keep"}`) + require.Equal(t, http.StatusCreated, w.Code, "body: %s", w.Body.String()) + + breakMetadataArtifactDir(t, te) + w = te.post(t, pinPath, `{"note":"replace"}`) + require.Equal(t, http.StatusInternalServerError, w.Code, "body: %s", w.Body.String()) + + pins, err := te.db.ListPinnedMessages(context.Background(), "s1", "") + require.NoError(t, err) + require.Len(t, pins, 1) + require.NotNil(t, pins[0].Note) + assert.Equal(t, "keep", *pins[0].Note, + "failed re-pin must restore the prior note") +} + +func TestMetadataEventsUnpinRestoresPinWhenArtifactWriteFails(t *testing.T) { + te := setup(t, withArtifactOrigin("desk-a1b2c3")) + messageID := seedPinTestMessage(t, te) + pinPath := fmt.Sprintf("/api/v1/sessions/s1/messages/%d/pin", messageID) + + w := te.post(t, pinPath, `{"note":"remember"}`) + require.Equal(t, http.StatusCreated, w.Code, "body: %s", w.Body.String()) + + metaDir := breakMetadataArtifactDir(t, te) + w = te.del(t, pinPath) + require.Equal(t, http.StatusInternalServerError, w.Code, "body: %s", w.Body.String()) + pins, err := te.db.ListPinnedMessages(context.Background(), "s1", "") + require.NoError(t, err) + require.Len(t, pins, 1, "failed metadata publish must restore the pin") + require.NotNil(t, pins[0].Note) + assert.Equal(t, "remember", *pins[0].Note) + + require.NoError(t, os.Remove(metaDir)) + w = te.del(t, pinPath) + require.Equal(t, http.StatusNoContent, w.Code, "body: %s", w.Body.String()) + pins, err = te.db.ListPinnedMessages(context.Background(), "s1", "") + require.NoError(t, err) + assert.Empty(t, pins) + + events := readMetadataEvents(t, te) + require.Len(t, events, 1) + assert.Equal(t, artifact.MetadataOpUnpin, events[0].Op) +} + +func TestMetadataEventsPinKeepsPinWhenArtifactPublished(t *testing.T) { + te := setup(t, withArtifactOrigin("desk-a1b2c3")) + messageID := seedPinTestMessage(t, te) + pinPath := fmt.Sprintf("/api/v1/sessions/s1/messages/%d/pin", messageID) + + execTestDDL(t, te, ` +CREATE TRIGGER fail_metadata_replay_state_insert +BEFORE INSERT ON metadata_replay_state +BEGIN + SELECT RAISE(FAIL, 'forced metadata replay failure'); +END; +`) + + w := te.post(t, pinPath, `{"note":"remember"}`) + require.Equal(t, http.StatusInternalServerError, w.Code, "body: %s", w.Body.String()) + + pins, err := te.db.ListPinnedMessages(context.Background(), "s1", "") + require.NoError(t, err) + require.Len(t, pins, 1, + "published metadata event must keep the local pin") + + events := readMetadataEvents(t, te) + require.Len(t, events, 1) + assert.Equal(t, artifact.MetadataOpPin, events[0].Op) +} + +func TestMetadataEventsPermanentDeleteRetriesExcludedSession(t *testing.T) { + te := setup(t, withArtifactOrigin("desk-a1b2c3")) + te.seedSession(t, "s1", "alpha", 2) + + w := te.del(t, "/api/v1/sessions/s1") + require.Equal(t, http.StatusNoContent, w.Code, "body: %s", w.Body.String()) + + execTestDDL(t, te, ` +CREATE TRIGGER fail_metadata_replay_state_insert +BEFORE INSERT ON metadata_replay_state +BEGIN + SELECT RAISE(FAIL, 'forced metadata replay failure'); +END; +`) + + w = te.del(t, "/api/v1/sessions/s1/permanent") + require.Equal(t, http.StatusInternalServerError, w.Code, "body: %s", w.Body.String()) + got, err := te.db.GetSessionFull(context.Background(), "s1") + require.NoError(t, err) + assert.Nil(t, got) + assert.True(t, te.db.IsSessionExcluded("s1")) + deleteMetadataEventsByOp(t, te, artifact.MetadataOpPurge) + + execTestDDL(t, te, `DROP TRIGGER fail_metadata_replay_state_insert`) + w = te.del(t, "/api/v1/sessions/s1/permanent") + require.Equal(t, http.StatusNoContent, w.Code, "body: %s", w.Body.String()) + + events := readMetadataEvents(t, te) + assert.Equal(t, []string{ + artifact.MetadataOpSoftDelete, + artifact.MetadataOpPurge, + }, metadataOps(events)) + assert.Equal(t, artifact.MetadataOpPurge, + serverMetadataReplayOp(t, te, "desk-a1b2c3~s1", "purge")) +} + +func TestMetadataEventsPermanentDeleteRetainsSessionWhenArtifactWriteFails(t *testing.T) { + te := setup(t, withArtifactOrigin("desk-a1b2c3")) + te.seedSession(t, "s1", "alpha", 2) + + w := te.del(t, "/api/v1/sessions/s1") + require.Equal(t, http.StatusNoContent, w.Code, "body: %s", w.Body.String()) + metaDir := breakMetadataArtifactDir(t, te) + + w = te.del(t, "/api/v1/sessions/s1/permanent") + require.Equal(t, http.StatusInternalServerError, w.Code, "body: %s", w.Body.String()) + session, err := te.db.GetSessionFull(context.Background(), "s1") + require.NoError(t, err) + require.NotNil(t, session, "purge must not remove the only local copy before publication") + assert.NotNil(t, session.DeletedAt) + assert.False(t, te.db.IsSessionExcluded("s1")) + assert.Empty(t, readMetadataEvents(t, te)) + + require.NoError(t, os.Remove(metaDir)) + w = te.del(t, "/api/v1/sessions/s1/permanent") + require.Equal(t, http.StatusNoContent, w.Code, "body: %s", w.Body.String()) + session, err = te.db.GetSessionFull(context.Background(), "s1") + require.NoError(t, err) + assert.Nil(t, session) + assert.True(t, te.db.IsSessionExcluded("s1")) + events := readMetadataEvents(t, te) + require.Len(t, events, 1) + assert.Equal(t, artifact.MetadataOpPurge, events[0].Op) +} + +func TestMetadataEventsRestoreRepairsPublishedArtifactState(t *testing.T) { + te := setup(t, withArtifactOrigin("desk-a1b2c3")) + te.seedSession(t, "s1", "alpha", 2) + + w := te.del(t, "/api/v1/sessions/s1") + require.Equal(t, http.StatusNoContent, w.Code, "body: %s", w.Body.String()) + assert.Equal(t, artifact.MetadataOpSoftDelete, + serverMetadataReplayOp(t, te, "desk-a1b2c3~s1", "deleted_at")) + + execTestDDL(t, te, ` +CREATE TRIGGER fail_metadata_replay_state_insert +BEFORE INSERT ON metadata_replay_state +BEGIN + SELECT RAISE(FAIL, 'forced metadata replay failure'); +END; +`) + + w = te.post(t, "/api/v1/sessions/s1/restore", `{}`) + require.Equal(t, http.StatusInternalServerError, w.Code, "body: %s", w.Body.String()) + restored, err := te.db.GetSessionFull(context.Background(), "s1") + require.NoError(t, err) + require.NotNil(t, restored) + assert.Nil(t, restored.DeletedAt) + assert.Equal(t, artifact.MetadataOpSoftDelete, + serverMetadataReplayOp(t, te, "desk-a1b2c3~s1", "deleted_at")) + restoreOrderKey := metadataEventOrderKey(t, te, artifact.MetadataOpRestore) + + execTestDDL(t, te, `DROP TRIGGER fail_metadata_replay_state_insert`) + w = te.post(t, "/api/v1/sessions/s1/restore", `{}`) + require.Equal(t, http.StatusNoContent, w.Code, "body: %s", w.Body.String()) + assert.Equal(t, artifact.MetadataOpRestore, + serverMetadataReplayOp(t, te, "desk-a1b2c3~s1", "deleted_at")) + + restoreHLC, _ := splitMetadataOrderKey(t, restoreOrderKey) + olderHash := strings.Repeat("0", 64) + _, err = te.db.ApplyMetadataProjection(context.Background(), db.MetadataProjection{ + EventOrigin: "peer-b2c3d4", + OrderKey: restoreHLC + "-" + olderHash, + HLC: restoreHLC, + ArtifactHash: olderHash, + SessionGID: "desk-a1b2c3~s1", + LocalSessionID: "s1", + Field: "deleted_at", + Op: artifact.MetadataOpSoftDelete, + Value: artifact.MetadataOpSoftDelete, + }) + require.NoError(t, err) + restored, err = te.db.GetSessionFull(context.Background(), "s1") + require.NoError(t, err) + require.NotNil(t, restored) + assert.Nil(t, restored.DeletedAt) +} + +func TestMetadataEventsRestoreRetrashesSessionWhenArtifactWriteFails(t *testing.T) { + te := setup(t, withArtifactOrigin("desk-a1b2c3")) + te.seedSession(t, "s1", "alpha", 2) + + w := te.del(t, "/api/v1/sessions/s1") + require.Equal(t, http.StatusNoContent, w.Code, "body: %s", w.Body.String()) + breakMetadataArtifactDir(t, te) + + w = te.post(t, "/api/v1/sessions/s1/restore", `{}`) + require.Equal(t, http.StatusInternalServerError, w.Code, "body: %s", w.Body.String()) + session, err := te.db.GetSessionFull(context.Background(), "s1") + require.NoError(t, err) + require.NotNil(t, session) + assert.NotNil(t, session.DeletedAt, + "failed pre-publication restore must return the session to trash") + assert.Equal(t, artifact.MetadataOpSoftDelete, + serverMetadataReplayOp(t, te, "desk-a1b2c3~s1", "deleted_at")) + assert.Empty(t, readMetadataEvents(t, te)) +} + +func TestMetadataEventsRestoreRetriesWithoutPublishedArtifact(t *testing.T) { + te := setup(t, withArtifactOrigin("desk-a1b2c3")) + te.seedSession(t, "s1", "alpha", 2) + + w := te.del(t, "/api/v1/sessions/s1") + require.Equal(t, http.StatusNoContent, w.Code, "body: %s", w.Body.String()) + + execTestDDL(t, te, ` +CREATE TRIGGER fail_metadata_replay_state_insert +BEFORE INSERT ON metadata_replay_state +BEGIN + SELECT RAISE(FAIL, 'forced metadata replay failure'); +END; +`) + + w = te.post(t, "/api/v1/sessions/s1/restore", `{}`) + require.Equal(t, http.StatusInternalServerError, w.Code, "body: %s", w.Body.String()) + restored, err := te.db.GetSessionFull(context.Background(), "s1") + require.NoError(t, err) + require.NotNil(t, restored) + assert.Nil(t, restored.DeletedAt) + deleteMetadataEventsByOp(t, te, artifact.MetadataOpRestore) + + execTestDDL(t, te, `DROP TRIGGER fail_metadata_replay_state_insert`) + w = te.post(t, "/api/v1/sessions/s1/restore", `{}`) + require.Equal(t, http.StatusNoContent, w.Code, "body: %s", w.Body.String()) + + events := readMetadataEvents(t, te) + assert.Equal(t, []string{ + artifact.MetadataOpSoftDelete, + artifact.MetadataOpRestore, + }, metadataOps(events)) + assert.Equal(t, artifact.MetadataOpRestore, + serverMetadataReplayOp(t, te, "desk-a1b2c3~s1", "deleted_at")) +} + +func TestMetadataEventsRestoreRetryPublishesWhenOlderRestoreLoses(t *testing.T) { + te := setup(t, withArtifactOrigin("desk-a1b2c3")) + te.seedSession(t, "s1", "alpha", 2) + + w := te.del(t, "/api/v1/sessions/s1") + require.Equal(t, http.StatusNoContent, w.Code, "body: %s", w.Body.String()) + w = te.post(t, "/api/v1/sessions/s1/restore", `{}`) + require.Equal(t, http.StatusNoContent, w.Code, "body: %s", w.Body.String()) + olderRestore := metadataEventOrderKey(t, te, artifact.MetadataOpRestore) + + w = te.del(t, "/api/v1/sessions/s1") + require.Equal(t, http.StatusNoContent, w.Code, "body: %s", w.Body.String()) + assert.Equal(t, artifact.MetadataOpSoftDelete, + serverMetadataReplayOp(t, te, "desk-a1b2c3~s1", "deleted_at")) + + execTestDDL(t, te, ` +CREATE TRIGGER fail_metadata_replay_state_insert +BEFORE INSERT ON metadata_replay_state +BEGIN + SELECT RAISE(FAIL, 'forced metadata replay failure'); +END; +`) + + w = te.post(t, "/api/v1/sessions/s1/restore", `{}`) + require.Equal(t, http.StatusInternalServerError, w.Code, "body: %s", w.Body.String()) + assert.Equal(t, artifact.MetadataOpSoftDelete, + serverMetadataReplayOp(t, te, "desk-a1b2c3~s1", "deleted_at")) + for _, key := range metadataEventOrderKeys(t, te, artifact.MetadataOpRestore) { + if key != olderRestore { + deleteMetadataEventOrderKey(t, te, key) + } + } + + execTestDDL(t, te, `DROP TRIGGER fail_metadata_replay_state_insert`) + w = te.post(t, "/api/v1/sessions/s1/restore", `{}`) + require.Equal(t, http.StatusNoContent, w.Code, "body: %s", w.Body.String()) + + assert.Equal(t, artifact.MetadataOpRestore, + serverMetadataReplayOp(t, te, "desk-a1b2c3~s1", "deleted_at")) + assert.Len(t, metadataEventOrderKeys(t, te, artifact.MetadataOpRestore), 2) +} + +func TestMetadataEventsSuppressedDuringReplay(t *testing.T) { + te := setup(t, withArtifactOrigin("desk-a1b2c3")) + te.seedSession(t, "s1", "alpha", 2) + + ctx := artifact.WithMetadataEventSuppression(context.Background()) + req := httptest.NewRequest( + http.MethodPatch, + "/api/v1/sessions/s1/rename", + strings.NewReader(`{"display_name":"Replay name"}`), + ).WithContext(ctx) + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Origin", "http://127.0.0.1:0") + w := httptest.NewRecorder() + te.handler.ServeHTTP(w, req) + require.Equal(t, http.StatusOK, w.Code, "body: %s", w.Body.String()) + + renamed, err := te.db.GetSession(context.Background(), "s1") + require.NoError(t, err) + require.NotNil(t, renamed) + require.NotNil(t, renamed.DisplayName) + assert.Equal(t, "Replay name", *renamed.DisplayName) + assert.Empty(t, readMetadataEvents(t, te)) +} + +// execTestDDL runs schema DDL (failure-injection triggers) on the test +// database through a short-lived write connection. The server's Reader() pool +// opens with mode=ro, so tests cannot install triggers through it. +func execTestDDL(t *testing.T, te *testEnv, stmt string) { + t.Helper() + raw, err := sql.Open("sqlite3", "file:"+te.db.Path()+"?_busy_timeout=5000") + require.NoError(t, err) + defer func() { require.NoError(t, raw.Close()) }() + _, err = raw.Exec(stmt) + require.NoError(t, err) +} + +func readMetadataEvents(t *testing.T, te *testEnv) []recordedMetadataEvent { + t.Helper() + paths, err := filepath.Glob(filepath.Join(te.dataDir, "artifacts", "*", "meta", "*.json")) + require.NoError(t, err) + sort.Strings(paths) + events := make([]recordedMetadataEvent, 0, len(paths)) + for _, path := range paths { + data, err := os.ReadFile(path) + require.NoError(t, err) + var event recordedMetadataEvent + require.NoError(t, json.Unmarshal(data, &event)) + events = append(events, event) + } + return events +} + +func metadataEventOrderKey(t *testing.T, te *testEnv, op string) string { + t.Helper() + keys := metadataEventOrderKeys(t, te, op) + require.NotEmpty(t, keys, "metadata event op %s not found", op) + return keys[0] +} + +func metadataEventOrderKeys(t *testing.T, te *testEnv, op string) []string { + t.Helper() + paths, err := filepath.Glob(filepath.Join(te.dataDir, "artifacts", "*", "meta", "*.json")) + require.NoError(t, err) + sort.Strings(paths) + keys := make([]string, 0) + for _, path := range paths { + data, err := os.ReadFile(path) + require.NoError(t, err) + var event recordedMetadataEvent + require.NoError(t, json.Unmarshal(data, &event)) + if event.Op == op { + keys = append(keys, strings.TrimSuffix(filepath.Base(path), ".json")) + } + } + return keys +} + +func splitMetadataOrderKey(t *testing.T, orderKey string) (string, string) { + t.Helper() + idx := strings.LastIndex(orderKey, "-") + require.NotEqual(t, -1, idx, "order key %q missing hash suffix", orderKey) + return orderKey[:idx], orderKey[idx+1:] +} + +func deleteMetadataEventOrderKey(t *testing.T, te *testEnv, orderKey string) { + t.Helper() + paths, err := filepath.Glob(filepath.Join(te.dataDir, "artifacts", "*", "meta", orderKey+".json")) + require.NoError(t, err) + require.Len(t, paths, 1) + require.NoError(t, os.Remove(paths[0])) +} + +func deleteMetadataEventsByOp(t *testing.T, te *testEnv, op string) { + t.Helper() + paths, err := filepath.Glob(filepath.Join(te.dataDir, "artifacts", "*", "meta", "*.json")) + require.NoError(t, err) + for _, path := range paths { + data, err := os.ReadFile(path) + require.NoError(t, err) + var event recordedMetadataEvent + require.NoError(t, json.Unmarshal(data, &event)) + if event.Op == op { + require.NoError(t, os.Remove(path)) + } + } +} + +func metadataOps(events []recordedMetadataEvent) []string { + ops := make([]string, len(events)) + for i, event := range events { + ops[i] = event.Op + } + return ops +} + +func serverMetadataReplayOp(t *testing.T, te *testEnv, sessionGID, field string) string { + t.Helper() + var op string + err := te.db.Reader().QueryRow( + `SELECT op FROM metadata_replay_state WHERE session_gid = ? AND field = ?`, + sessionGID, field, + ).Scan(&op) + require.NoError(t, err) + return op +} + +func serverMetadataTableCount(t *testing.T, te *testEnv, table, where string) int { + t.Helper() + var count int + err := te.db.Reader().QueryRow("SELECT COUNT(*) FROM " + table + " WHERE " + where).Scan(&count) + require.NoError(t, err) + return count +} diff --git a/internal/server/server.go b/internal/server/server.go index 1a3e2f2ed..3f91bcf2c 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -19,6 +19,7 @@ import ( "github.com/danielgtaylor/huma/v2" "github.com/danielgtaylor/huma/v2/adapters/humago" + "go.kenn.io/agentsview/internal/artifact" "go.kenn.io/agentsview/internal/config" "go.kenn.io/agentsview/internal/db" "go.kenn.io/agentsview/internal/insight" @@ -49,18 +50,23 @@ const ( // Server is the HTTP server that serves the SPA and REST API. type Server struct { - mu gosync.RWMutex - cfg config.Config - db db.Store - engine *sync.Engine - onDemandEngine *sync.Engine - sessions service.SessionService - broadcaster *Broadcaster - mux *http.ServeMux - api huma.API - httpSrv *http.Server - version VersionInfo - dataDir string + mu gosync.RWMutex + sessionLifecycleMu gosync.Mutex + artifactImportPending bool + artifactBaselineDone bool + cfg config.Config + db db.Store + engine *sync.Engine + onDemandEngine *sync.Engine + sessions service.SessionService + broadcaster *Broadcaster + metadata *artifact.MetadataRecorder + metadataAppend func(context.Context, artifact.MetadataEventInput) error + mux *http.ServeMux + api huma.API + httpSrv *http.Server + version VersionInfo + dataDir string // baseCtx, when set, is used as the base context for all // incoming requests. Cancelling it causes SSE handlers to @@ -78,6 +84,9 @@ type Server struct { // handler, used only by tests to guarantee handlers // exceed a short timeout. Zero in production. handlerDelay time.Duration + // beforeSessionLifecycleLock observes lock attempts in concurrency tests. + // Production servers leave it nil. + beforeSessionLifecycleLock func() // updateCheckFn is the function called to check for // updates. Defaults to update.CheckForUpdate; tests @@ -113,6 +122,13 @@ type Server struct { vectorPushSource postgres.VectorPushSource } +func (s *Server) lockSessionLifecycle() { + if s.beforeSessionLifecycleLock != nil { + s.beforeSessionLifecycleLock() + } + s.sessionLifecycleMu.Lock() +} + // New creates a new Server. func New( cfg config.Config, database db.Store, engine *sync.Engine, @@ -157,6 +173,12 @@ func New( spaFS: dist, spaHandler: http.FileServerFS(dist), } + if local, ok := database.(*db.DB); ok && !local.ReadOnly() && cfg.DataDir != "" { + s.metadata = artifact.NewMetadataRecorder(local, artifact.MetadataRecorderOptions{ + DataDir: cfg.DataDir, + Origin: cfg.ArtifactOriginID, + }) + } for _, opt := range opts { opt(s) } diff --git a/internal/server/session_mgmt_test.go b/internal/server/session_mgmt_test.go index ea4512d82..df27bda53 100644 --- a/internal/server/session_mgmt_test.go +++ b/internal/server/session_mgmt_test.go @@ -2,6 +2,7 @@ package server_test import ( "context" + "database/sql" "net/http" "testing" @@ -19,6 +20,10 @@ type emptyTrashHandlerResponse struct { Deleted int `json:"deleted"` } +type metadataConflictsHandlerResponse struct { + Conflicts []db.MetadataConflict `json:"conflicts"` +} + func TestSessionManagementRenameHandler(t *testing.T) { te := setup(t) te.seedSession(t, "s1", "alpha", 2) @@ -94,3 +99,40 @@ func TestSessionManagementEmptyTrashHandler(t *testing.T) { trash := decode[trashHandlerResponse](t, w) assert.Empty(t, trash.Sessions) } + +func TestSessionManagementMetadataConflictsHandler(t *testing.T) { + te := setup(t) + te.seedSession(t, "s1", "alpha", 2) + require.NoError(t, te.db.SetSyncState("artifact_origin_id", "desktop-d4e5f6")) + require.NoError(t, te.db.Update(func(tx *sql.Tx) error { + _, err := tx.Exec( + `INSERT INTO metadata_conflicts + (session_gid, field, winning_order_key, losing_order_key, + winning_origin, losing_origin, winning_op, losing_op, + winning_value, losing_value) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, + "desktop-d4e5f6~s1", + "display_name", + "2026-06-14T01:02:03.000000002Z-00000000000000000000-bbb", + "2026-06-14T01:02:03.000000002Z-00000000000000000000-aaa", + "desktop-d4e5f6", + "laptop-a1b2c3", + "rename", + "rename", + `{"display_name":"Winner"}`, + `{"display_name":"Other"}`, + ) + return err + })) + + w := te.get(t, "/api/v1/sessions/s1/metadata-conflicts") + require.Equal(t, http.StatusOK, w.Code, "body: %s", w.Body.String()) + resp := decode[metadataConflictsHandlerResponse](t, w) + require.Len(t, resp.Conflicts, 1) + assert.Equal(t, "display_name", resp.Conflicts[0].Field) + assert.Equal(t, `{"display_name":"Winner"}`, resp.Conflicts[0].WinningValue) + + w = te.get(t, "/api/v1/sessions/missing/metadata-conflicts") + require.Equal(t, http.StatusNotFound, w.Code, "body: %s", w.Body.String()) + assertErrorResponse(t, w, "session not found") +} diff --git a/internal/sync/engine.go b/internal/sync/engine.go index a5f84df5c..584907df4 100644 --- a/internal/sync/engine.go +++ b/internal/sync/engine.go @@ -1459,6 +1459,29 @@ func (e *Engine) resyncAllLocked( return stats } + // Copy the artifact metadata replay tables so peer metadata + // events keep their durable LWW state across the swap. Losing + // this state would let already-applied peer events replay and + // overwrite newer local curation, so failure aborts the swap + // just like the sync state copy above. + if err := newDB.CopyMetadataReplayFrom(origPath); err != nil { + log.Printf("resync: copy metadata replay state: %v", err) + stats.Aborted = true + stats.Warnings = append(stats.Warnings, + "metadata replay copy failed, aborting swap: "+err.Error(), + ) + newDB.Close() + removeTempDB(tempPath) + restoreSkipCache() + if rerr := origDB.Reopen(); rerr != nil { + log.Printf("resync: recovery reopen: %v", rerr) + } + e.mu.Lock() + e.lastSyncStats = stats + e.mu.Unlock() + return stats + } + // Copy insights into newDB from the quiesced old DB file. tInsights := time.Now() reportResyncPhase( diff --git a/internal/sync/engine_integration_test.go b/internal/sync/engine_integration_test.go index ff26305f0..dbc2515a2 100644 --- a/internal/sync/engine_integration_test.go +++ b/internal/sync/engine_integration_test.go @@ -2245,7 +2245,9 @@ func TestSyncEngineHashSkip(t *testing.T) { different := testjsonl.NewSessionBuilder(). AddClaudeUser(tsZero, "msg2"). String() - os.WriteFile(path, []byte(different), 0o644) + require.NoError(t, os.WriteFile(path, []byte(different), 0o644), "rewrite changed session") + future := time.Unix(0, mtime).Add(time.Second) + require.NoError(t, os.Chtimes(path, future, future), "advance changed file mtime") // Third sync — mtime changed → re-synced runSyncAndAssert(t, env.engine, sync.SyncStats{TotalSessions: 1 + 0, Synced: 1, Skipped: 0}) @@ -4968,6 +4970,8 @@ func TestSyncPathsOpenCodeStorageChildUpdateAdvancesSessionMtime( `{"id":"part-a1","sessionID":"oc-storage-mtime","messageID":"msg-a1","type":"text","text":"updated reply","time":{"created":1704067201000}}`, ), 0o644) require.NoError(t, err, "rewrite part") + childMtime := time.Unix(0, initialMtime).Add(time.Second) + require.NoError(t, os.Chtimes(partPath, childMtime, childMtime), "advance part mtime") err = os.Chtimes( sessionPath, time.Unix(0, sessionMtime), @@ -5742,6 +5746,8 @@ func TestSyncPathsMiMoCodeStorageIgnoresStaleSessionSkipCache(t *testing.T) { t, sessionID, "msg-a1", "part-a1", "rewritten mimo reply", 1704067201000, ) + childMtime := sessionMtime.Add(time.Second) + require.NoError(t, os.Chtimes(partPath, childMtime, childMtime), "advance part mtime") require.NoError(t, os.Chtimes(sessionPath, sessionMtime, sessionMtime), "restore session mtime") @@ -11125,6 +11131,67 @@ func TestResyncAllPreservesPGPushMarkerID(t *testing.T) { assert.Equal(t, "marker-123", got) } +func TestResyncAllPreservesArtifactMetadataState(t *testing.T) { + env := setupTestEnv(t) + ctx := context.Background() + + content := testjsonl.NewSessionBuilder(). + AddClaudeUser(tsEarly, "hello"). + AddClaudeAssistant(tsZeroS5, "hi"). + String() + env.writeClaudeSession(t, "proj", "sess.jsonl", content) + env.engine.SyncAll(ctx, nil) + + artifactState := map[string]string{ + "artifact_origin_id": "laptop-a1b2c3", + "artifact_metadata_hlc": "hlc-42", + "artifact_import:peer-b4c5d6:peer-b4c5d6~sess-9": "hash-imp", + "artifact_export:laptop-a1b2c3:sess.jsonl": "hash-exp", + } + for key, value := range artifactState { + require.NoError(t, env.db.SetSyncState(key, value), + "SetSyncState %s", key) + } + + projection := db.MetadataProjection{ + EventOrigin: "peer-b4c5d6", + OrderKey: "0000000001", + HLC: "hlc-1", + ArtifactHash: "hash-1", + SessionGID: "peer-b4c5d6~sess-9", + LocalSessionID: "peer-b4c5d6~sess-9", + Field: "display_name", + Op: "rename", + Value: `{"display_name":"peer name"}`, + } + _, err := env.db.RecordLocalMetadataProjection(ctx, projection) + require.NoError(t, err, "RecordLocalMetadataProjection") + + stats := env.engine.ResyncAll(ctx, nil) + require.False(t, stats.Aborted, "ResyncAll aborted: %+v", stats) + + for key, want := range artifactState { + got, err := env.db.GetSyncState(key) + require.NoError(t, err, "GetSyncState %s after resync", key) + assert.Equal(t, want, got, + "artifact sync state %s must survive resync", key) + } + + applied, err := env.db.MetadataEventApplied( + ctx, projection.EventOrigin, projection.OrderKey, + ) + require.NoError(t, err, "MetadataEventApplied after resync") + assert.True(t, applied, + "applied peer metadata event must survive resync") + + op, ok, err := env.db.MetadataReplayStateOp( + ctx, projection.SessionGID, projection.Field, + ) + require.NoError(t, err, "MetadataReplayStateOp after resync") + require.True(t, ok, "metadata replay state must survive resync") + assert.Equal(t, "rename", op) +} + func TestOpenCodeExcludedSessionsAreSkipped(t *testing.T) { env := setupSingleAgentTestEnv(t, parser.AgentOpenCode)