From 70741f73ddaf4c42cf3aef6b3cff4c038d83e5f4 Mon Sep 17 00:00:00 2001 From: Khaliq Date: Tue, 24 Mar 2026 14:58:49 +0100 Subject: [PATCH 01/10] Add bulk API, workspace export, WebSocket push, mount hardening, and E2E infrastructure Extends the Go server with bulk seed/export endpoints, WebSocket file-change notifications, and binary file support. Hardens mount sync with conflict resolution and bidirectional sync. Adds E2E test script, workflow definitions, design docs, and updated TypeScript SDK. Co-Authored-By: Claude Opus 4.6 (1M context) --- .gitignore | 3 +- cmd/relayfile-mount/main.go | 19 + docs/bulk-export-design.md | 747 ++++++++++++++++++ docs/knowledge-graph-spec.md | 834 ++++++++++++++++++++ docs/mount-hardening-plan.md | 295 +++++++ go.mod | 2 + go.sum | 2 + internal/httpapi/server.go | 212 +++++ internal/httpapi/server_test.go | 427 +++++++++- internal/httpapi/websocket.go | 126 +++ internal/mountsync/syncer.go | 176 ++++- internal/mountsync/syncer_test.go | 120 +++ internal/relayfile/store.go | 305 ++++++- internal/relayfile/store_test.go | 415 ++++++++-- openapi/relayfile-v1.openapi.yaml | 195 +++++ package-lock.json | 587 ++++++++++++++ package.json | 10 + scripts/e2e.ts | 519 ++++++++++++ sdk/relayfile-sdk/src/client.ts | 187 ++++- sdk/relayfile-sdk/src/index.ts | 13 +- sdk/relayfile-sdk/src/types.ts | 38 + workflows/relayfile-bulk-and-export.ts | 420 ++++++++++ workflows/relayfile-ci-and-publish.ts | 358 +++++++++ workflows/relayfile-cloud-server.ts | 392 +++++++++ workflows/relayfile-developer-experience.ts | 401 ++++++++++ workflows/relayfile-landing-page.ts | 267 +++++++ 26 files changed, 6949 insertions(+), 121 deletions(-) create mode 100644 docs/bulk-export-design.md create mode 100644 docs/knowledge-graph-spec.md create mode 100644 docs/mount-hardening-plan.md create mode 100644 internal/httpapi/websocket.go create mode 100644 package-lock.json create mode 100644 package.json create mode 100644 scripts/e2e.ts create mode 100644 workflows/relayfile-bulk-and-export.ts create mode 100644 workflows/relayfile-ci-and-publish.ts create mode 100644 workflows/relayfile-cloud-server.ts create mode 100644 workflows/relayfile-developer-experience.ts create mode 100644 workflows/relayfile-landing-page.ts diff --git a/.gitignore b/.gitignore index e5b7a09c..b0b3308a 100644 --- a/.gitignore +++ b/.gitignore @@ -22,8 +22,7 @@ .DS_Store Thumbs.db -# Trajectories -.trajectories/ +.agent-relay # Node (SDK) node_modules/ diff --git a/cmd/relayfile-mount/main.go b/cmd/relayfile-mount/main.go index ed022a9f..06c60490 100644 --- a/cmd/relayfile-mount/main.go +++ b/cmd/relayfile-mount/main.go @@ -27,6 +27,7 @@ func main() { interval := flag.Duration("interval", durationEnv("RELAYFILE_MOUNT_INTERVAL", 2*time.Second), "sync interval") intervalJitter := flag.Float64("interval-jitter", floatEnv("RELAYFILE_MOUNT_INTERVAL_JITTER", 0.2), "sync interval jitter ratio (0.0-1.0)") timeout := flag.Duration("timeout", durationEnv("RELAYFILE_MOUNT_TIMEOUT", 15*time.Second), "per-sync timeout") + websocketEnabled := flag.Bool("websocket", boolEnv("RELAYFILE_MOUNT_WEBSOCKET", true), "enable websocket event streaming when available") once := flag.Bool("once", false, "run one sync cycle and exit") flag.Parse() @@ -54,6 +55,7 @@ func main() { EventProvider: strings.TrimSpace(*eventProvider), LocalRoot: *localDir, StateFile: *stateFile, + WebSocket: boolPtr(*websocketEnabled), Logger: log.Default(), }) if err != nil { @@ -126,6 +128,23 @@ func floatEnv(name string, fallback float64) float64 { return value } +func boolEnv(name string, fallback bool) bool { + raw := strings.TrimSpace(os.Getenv(name)) + if raw == "" { + return fallback + } + value, err := strconv.ParseBool(raw) + if err != nil { + log.Printf("invalid %s=%q, using fallback %t", name, raw, fallback) + return fallback + } + return value +} + +func boolPtr(value bool) *bool { + return &value +} + func clampJitterRatio(value float64) float64 { if value < 0 { return 0 diff --git a/docs/bulk-export-design.md b/docs/bulk-export-design.md new file mode 100644 index 00000000..5bd9c856 --- /dev/null +++ b/docs/bulk-export-design.md @@ -0,0 +1,747 @@ +# Bulk Seed, Export, WebSocket & Binary Support — Design Doc + +**Status:** Draft +**Date:** 2026-03-24 + +--- + +## Overview + +This document specifies four new capabilities for the RelayFile HTTP API: + +1. **Bulk Seed** — atomic multi-file write in a single request +2. **Workspace Export** — full workspace snapshot in tar, JSON, or patch format +3. **WebSocket Events** — real-time push of filesystem change events +4. **Binary File Support** — base64-encoded content for non-text files + +All endpoints live under the existing `/v1/workspaces/{workspaceId}/fs/` namespace and follow the same auth, correlation-ID, and rate-limiting conventions as the current API. + +--- + +## 1. Bulk Seed + +### Endpoint + +``` +POST /v1/workspaces/{workspaceId}/fs/bulk +``` + +**Scope:** `fs:write` + +### Purpose + +Populate or update many files in a single atomic request. Useful for initial workspace seeding from a provider sync, migration tooling, or batch imports. + +### Request — JSON + +``` +Content-Type: application/json +``` + +```json +{ + "files": [ + { + "path": "/docs/readme.md", + "contentType": "text/markdown", + "content": "# Hello", + "encoding": "utf-8", + "semantics": { + "properties": { "stage": "active" }, + "relations": [], + "permissions": [], + "comments": [] + } + } + ] +} +``` + +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| `files` | array | yes | Array of file objects to write | +| `files[].path` | string | yes | Virtual filesystem path (must start with `/`) | +| `files[].contentType` | string | no | MIME type (default: `text/markdown`) | +| `files[].content` | string | yes | File content (plain text or base64-encoded) | +| `files[].encoding` | string | no | `"utf-8"` (default) or `"base64"` | +| `files[].semantics` | object | no | Optional semantic metadata (properties, relations, permissions, comments) | + +**Limits:** + +- Maximum 500 files per request +- Total request body capped at 50 MB +- Individual file content capped at 10 MB + +### Request — Multipart tar.gz + +``` +Content-Type: multipart/form-data +``` + +| Part | Type | Description | +|------|------|-------------| +| `archive` | file (application/gzip) | A `.tar.gz` containing files rooted at `/` | +| `metadata` | application/json (optional) | JSON object mapping paths to semantic metadata | + +The tar archive is extracted with paths mapped directly to the virtual filesystem. Directory entries are created implicitly. The optional `metadata` part allows attaching semantics: + +```json +{ + "/docs/readme.md": { + "contentType": "text/markdown", + "semantics": { + "properties": { "stage": "active" } + } + } +} +``` + +### Atomicity + +- All files are written within a single store transaction. +- If any file fails validation (invalid path, exceeds size), the entire batch is rejected. +- Each successfully written file produces one filesystem event (`file.created` or `file.updated`). +- A single `If-Match: *` semantic is used — bulk seed always overwrites existing content (create-or-update). + +### Response + +**Success (200):** + +```json +{ + "imported": 12, + "skipped": 0, + "errors": [], + "correlationId": "corr-abc123" +} +``` + +**Partial failure (207 Multi-Status):** + +Returned only if the `allowPartial=true` query parameter is set. Without it, any error rejects the entire batch. + +```json +{ + "imported": 10, + "skipped": 0, + "errors": [ + { "path": "/bad/path", "error": "invalid path: must start with /" }, + { "path": "/too/large.bin", "error": "content exceeds 10 MB limit" } + ], + "correlationId": "corr-abc123" +} +``` + +### Store Method + +```go +type BulkWriteFile struct { + Path string + ContentType string + Content string + Encoding string // "utf-8" or "base64" + Semantics FileSemantics +} + +type BulkWriteResult struct { + Imported int + Skipped int + Errors []BulkWriteError +} + +type BulkWriteError struct { + Path string + Error string +} + +func (s *Store) BulkWrite(workspaceID string, files []BulkWriteFile, allowPartial bool, correlationID string) (BulkWriteResult, error) +``` + +Implementation notes: +- Acquires store lock once for the entire batch. +- Validates all files before writing any (unless `allowPartial`). +- Generates a revision per file, emits one event per file. +- Queues writeback operations per file if the workspace has provider bindings. + +### OpenAPI Addition + +```yaml +/v1/workspaces/{workspaceId}/fs/bulk: + post: + tags: [Filesystem] + operationId: bulkSeedFiles + summary: Atomically write multiple files in a single request + security: + - BearerAuth: [fs:write] + parameters: + - $ref: '#/components/parameters/WorkspaceId' + - $ref: '#/components/parameters/CorrelationId' + - name: allowPartial + in: query + required: false + schema: + type: boolean + default: false + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/BulkSeedRequest' + multipart/form-data: + schema: + type: object + properties: + archive: + type: string + format: binary + metadata: + type: string + format: json + responses: + '200': + description: All files imported successfully + '207': + description: Partial success (allowPartial=true) + '400': + $ref: '#/components/responses/BadRequest' + '401': + $ref: '#/components/responses/Unauthorized' + '403': + $ref: '#/components/responses/Forbidden' + '413': + $ref: '#/components/responses/PayloadTooLarge' + '429': + $ref: '#/components/responses/RateLimited' +``` + +--- + +## 2. Workspace Export + +### Endpoint + +``` +GET /v1/workspaces/{workspaceId}/fs/export +``` + +**Scope:** `fs:read` + +### Purpose + +Download the entire workspace filesystem as a single artifact. Useful for backups, migrations, offline analysis, and audit. + +### Query Parameters + +| Param | Type | Default | Description | +|-------|------|---------|-------------| +| `format` | string | `json` | One of: `tar`, `json`, `patch` | +| `path` | string | `/` | Subtree to export (prefix filter) | +| `includeSemantics` | boolean | `true` | Include semantic metadata in export | + +### Format: `tar` + +**Response:** `Content-Type: application/gzip` + +Returns a gzip-compressed tar archive. File paths in the archive mirror the virtual filesystem paths. Binary files are stored as raw bytes. A `.relayfile-manifest.json` file is included at the root containing metadata: + +```json +{ + "workspaceId": "ws-123", + "exportedAt": "2026-03-24T12:00:00Z", + "fileCount": 42, + "files": { + "/docs/readme.md": { + "revision": "rev-abc", + "contentType": "text/markdown", + "semantics": { "properties": { "stage": "active" } } + } + } +} +``` + +### Format: `json` + +**Response:** `Content-Type: application/json` + +```json +{ + "workspaceId": "ws-123", + "exportedAt": "2026-03-24T12:00:00Z", + "files": [ + { + "path": "/docs/readme.md", + "revision": "rev-abc", + "contentType": "text/markdown", + "content": "# Hello", + "encoding": "utf-8", + "semantics": { + "properties": { "stage": "active" }, + "relations": [], + "permissions": [], + "comments": [] + } + } + ] +} +``` + +Binary files are included with `"encoding": "base64"` and base64-encoded content. + +### Format: `patch` + +**Response:** `Content-Type: text/plain` + +Returns a unified diff of all files against an empty tree, similar to `git diff --no-index /dev/null`. Each file produces a diff hunk: + +```diff +--- /dev/null ++++ a/docs/readme.md +@@ -0,0 +1,1 @@ ++# Hello +``` + +Binary files are represented as: + +``` +Binary file /images/logo.png added (4.2 KB, base64) +``` + +### Permission Filtering + +The export respects the caller's effective permissions. Files the caller cannot read (per ACL rules) are silently excluded. The response includes only visible files. + +### Store Method + +```go +func (s *Store) ExportWorkspace(workspaceID string, pathPrefix string) ([]File, error) +``` + +Returns all files under the given path prefix. The HTTP handler is responsible for format conversion (tar assembly, patch generation). + +### Streaming + +For large workspaces, the tar and patch formats stream directly to the response writer without buffering the entire archive in memory. The JSON format buffers because it requires a valid JSON array. + +For workspaces exceeding 10,000 files or 500 MB total content, the server returns `413 Payload Too Large` with a suggestion to use path-scoped exports. + +### OpenAPI Addition + +```yaml +/v1/workspaces/{workspaceId}/fs/export: + get: + tags: [Filesystem] + operationId: exportWorkspace + summary: Export workspace files as tar, JSON, or patch + security: + - BearerAuth: [fs:read] + parameters: + - $ref: '#/components/parameters/WorkspaceId' + - $ref: '#/components/parameters/CorrelationId' + - name: format + in: query + required: false + schema: + type: string + enum: [tar, json, patch] + default: json + - name: path + in: query + required: false + schema: + type: string + default: / + - name: includeSemantics + in: query + required: false + schema: + type: boolean + default: true + responses: + '200': + description: Workspace export + content: + application/json: + schema: + $ref: '#/components/schemas/ExportJsonResponse' + application/gzip: + schema: + type: string + format: binary + text/plain: + schema: + type: string + '401': + $ref: '#/components/responses/Unauthorized' + '403': + $ref: '#/components/responses/Forbidden' + '413': + $ref: '#/components/responses/PayloadTooLarge' + '429': + $ref: '#/components/responses/RateLimited' +``` + +--- + +## 3. WebSocket Events + +### Endpoint + +``` +GET /v1/workspaces/{workspaceId}/fs/ws +``` + +**Scope:** `fs:read` + +### Purpose + +Real-time push of filesystem change events over a persistent WebSocket connection. Replaces polling the `/fs/events` endpoint for latency-sensitive consumers. + +### Connection Lifecycle + +1. Client sends HTTP GET with `Upgrade: websocket` header. +2. Server validates JWT bearer token (same as REST endpoints). +3. On successful upgrade, server sends a `connected` frame. +4. Client optionally sends `subscribe` messages to filter events. +5. Server pushes events as they occur. +6. Either side can close the connection. + +### Server-to-Client Messages + +**Connected:** + +```json +{ + "type": "connected", + "workspaceId": "ws-123", + "serverTime": "2026-03-24T12:00:00Z" +} +``` + +**Event:** + +```json +{ + "type": "file.created", + "path": "/docs/readme.md", + "revision": "rev-abc", + "provider": "notion", + "origin": "provider_sync", + "correlationId": "corr-xyz", + "timestamp": "2026-03-24T12:00:01Z" +} +``` + +Event types mirror the existing `FilesystemEvent` schema: +- `file.created` +- `file.updated` +- `file.deleted` +- `dir.created` +- `dir.deleted` +- `sync.error` +- `sync.ignored` +- `sync.suppressed` +- `sync.stale` +- `writeback.failed` +- `writeback.succeeded` + +**Ping:** + +```json +{ "type": "ping", "ts": "2026-03-24T12:00:30Z" } +``` + +Server sends pings every 30 seconds. Client must respond with `pong` within 10 seconds or the connection is closed. + +### Client-to-Server Messages + +**Subscribe (filter):** + +```json +{ + "type": "subscribe", + "filter": { + "paths": ["/docs/*"], + "eventTypes": ["file.created", "file.updated"], + "providers": ["notion"] + } +} +``` + +All filter fields are optional. If omitted, the client receives all events. Filters use simple glob matching for paths (`*` matches any segment, `**` matches recursively). + +**Pong:** + +```json +{ "type": "pong" } +``` + +**Unsubscribe (reset to all):** + +```json +{ "type": "unsubscribe" } +``` + +### Cursor Resumption + +On connect, clients can provide a `cursor` query parameter to resume from a known position: + +``` +GET /v1/workspaces/{workspaceId}/fs/ws?cursor=evt-456 +``` + +The server replays missed events from the cursor position before switching to live push. If the cursor is too old (events have been pruned), the server sends an error frame and falls back to live-only: + +```json +{ + "type": "error", + "code": "cursor_expired", + "message": "Requested cursor is no longer available. Receiving live events only." +} +``` + +### Polling Fallback + +For environments that don't support WebSocket (some proxies, serverless), clients should fall back to the existing `GET /v1/workspaces/{workspaceId}/fs/events` endpoint with cursor-based polling. No server-side changes needed for this — it's a client implementation concern. + +### Implementation Notes + +- Use `gorilla/websocket` or `nhooyr.io/websocket` for the WebSocket upgrade. +- The store emits events to an internal channel; the WebSocket handler fans out to connected clients. +- Each workspace has an independent fan-out group. Connections are tracked in a `sync.Map`. +- Max connections per workspace: 50 (configurable). +- Max connections per agent per workspace: 5. +- Idle timeout: 5 minutes with no subscribe/pong activity. + +### Store Integration + +No new store method required. The WebSocket handler subscribes to the existing event emission path. A new internal interface is added: + +```go +type EventSubscriber interface { + Subscribe(workspaceID string, filter EventFilter) (<-chan FilesystemEvent, func()) +} + +type EventFilter struct { + Paths []string // glob patterns + EventTypes []string + Providers []string +} +``` + +The `Store` implements `EventSubscriber`. The returned channel receives events; the returned `func()` is called to unsubscribe. + +### OpenAPI Addition + +```yaml +/v1/workspaces/{workspaceId}/fs/ws: + get: + tags: [Events] + operationId: websocketEvents + summary: Real-time filesystem events over WebSocket + description: | + Upgrades to a WebSocket connection for real-time event push. + Falls back to polling /fs/events for non-WebSocket environments. + security: + - BearerAuth: [fs:read] + parameters: + - $ref: '#/components/parameters/WorkspaceId' + - name: cursor + in: query + required: false + schema: + type: string + responses: + '101': + description: WebSocket upgrade successful + '401': + $ref: '#/components/responses/Unauthorized' + '403': + $ref: '#/components/responses/Forbidden' + '429': + $ref: '#/components/responses/RateLimited' +``` + +--- + +## 4. Binary File Support + +### Purpose + +Support storing and retrieving non-text files (images, PDFs, compiled assets) through the same filesystem API by allowing base64-encoded content. + +### Write Path + +The `encoding` field is added to file write requests: + +```json +{ + "contentType": "image/png", + "content": "iVBORw0KGgoAAAANSUhEUgAA...", + "encoding": "base64" +} +``` + +| Field | Values | Default | Description | +|-------|--------|---------|-------------| +| `encoding` | `"utf-8"`, `"base64"` | `"utf-8"` | How `content` is encoded in the JSON payload | + +When `encoding` is `"base64"`: +- The server validates that `content` is valid base64. +- Content is stored as the raw base64 string in the store. +- The `encoding` value is persisted in file metadata. +- Size limits apply to the decoded size, not the base64 representation. + +### Read Path + +The read response includes the `encoding` field: + +```json +{ + "path": "/images/logo.png", + "revision": "rev-abc", + "contentType": "image/png", + "content": "iVBORw0KGgoAAAANSUhEUgAA...", + "encoding": "base64", + "semantics": {} +} +``` + +Detection logic: +- If the file was written with `encoding: "base64"`, it is returned with `encoding: "base64"`. +- If no encoding was specified at write time, it defaults to `"utf-8"`. +- The `contentType` field is informational and does not affect encoding detection. + +### Store Changes + +The `File` struct gains an `Encoding` field: + +```go +type File struct { + Path string + Revision string + ContentType string + Content string + Encoding string // "utf-8" or "base64" + Provider string + ProviderObjectID string + LastEditedAt string + Semantics FileSemantics +} +``` + +The `WriteRequest` struct gains a matching field: + +```go +type WriteRequest struct { + // ... existing fields ... + Encoding string // "utf-8" or "base64" +} +``` + +Storage is unchanged — content is always stored as a Go `string`. For base64 files, the string contains the base64 representation. No decoding happens at the store layer. + +### Validation + +- On write with `encoding: "base64"`: validate that content is valid standard base64 (RFC 4648). Reject with `400 bad_request` if invalid. +- Decoded size must not exceed the per-file content limit (10 MB decoded). +- Base64 content size in the JSON payload is approximately 4/3 of the decoded size. + +### Content Type Detection + +The server does not auto-detect content types. The caller must provide the correct `contentType`. Common binary types: + +- `image/png`, `image/jpeg`, `image/gif`, `image/svg+xml` +- `application/pdf` +- `application/octet-stream` (generic binary) + +### Impact on Other Endpoints + +| Endpoint | Change | +|----------|--------| +| `PUT /fs/file` | Accepts `encoding` field | +| `GET /fs/file` | Returns `encoding` field | +| `GET /fs/query` | No change (doesn't return content) | +| `GET /fs/tree` | No change | +| `GET /fs/export` | Includes `encoding` per file; tar format stores decoded bytes | +| `POST /fs/bulk` | Accepts `encoding` per file | +| WebSocket events | No change (events don't include content) | + +### OpenAPI Changes + +Add `encoding` to `FileWriteRequest`: + +```yaml +FileWriteRequest: + type: object + properties: + # ... existing ... + encoding: + type: string + enum: [utf-8, base64] + default: utf-8 + description: Content encoding. Use base64 for binary files. +``` + +Add `encoding` to `FileReadResponse`: + +```yaml +FileReadResponse: + type: object + properties: + # ... existing ... + encoding: + type: string + enum: [utf-8, base64] + default: utf-8 +``` + +--- + +## HTTP Handler Routing + +New routes added to `ServeHTTP`: + +```go +case len(parts) == 5 && parts[3] == "fs" && parts[4] == "bulk" && r.Method == http.MethodPost: + requiredScope = "fs:write" + route = "bulk_write" + +case len(parts) == 5 && parts[3] == "fs" && parts[4] == "export" && r.Method == http.MethodGet: + requiredScope = "fs:read" + route = "export" + +case len(parts) == 5 && parts[3] == "fs" && parts[4] == "ws" && r.Method == http.MethodGet: + requiredScope = "fs:read" + route = "ws_events" +``` + +--- + +## Migration & Compatibility + +- All changes are additive — no existing endpoints are modified. +- The `encoding` field defaults to `"utf-8"`, so existing clients see no change. +- Files written before binary support have no `encoding` metadata and default to `"utf-8"` on read. +- The WebSocket endpoint is independent of the polling events endpoint; both remain available. + +--- + +## Error Codes + +| Code | HTTP Status | Description | +|------|-------------|-------------| +| `bulk_too_large` | 413 | Bulk request exceeds file count or size limit | +| `invalid_base64` | 400 | Content is not valid base64 when encoding=base64 | +| `export_too_large` | 413 | Workspace exceeds export size threshold | +| `ws_limit_reached` | 429 | WebSocket connection limit per workspace exceeded | +| `cursor_expired` | N/A (WS frame) | Resume cursor no longer available | + +--- + +## Security Considerations + +- **Bulk seed** uses the same permission checks as single-file writes. Each file in the batch is validated against effective ACL rules. +- **Export** filters files by caller permissions — no information leakage. +- **WebSocket** validates JWT on upgrade. Token expiry is checked periodically (every 60s); expired tokens trigger graceful close. +- **Binary content** is stored as base64 strings, not raw bytes, avoiding encoding issues in JSON transport. No server-side execution of binary content. diff --git a/docs/knowledge-graph-spec.md b/docs/knowledge-graph-spec.md new file mode 100644 index 00000000..bd77475e --- /dev/null +++ b/docs/knowledge-graph-spec.md @@ -0,0 +1,834 @@ +# RelayFile Knowledge Graph Spec v2.0 + +## 1. Document Status + +- Status: Draft +- Date: 2026-03-12 +- Scope: Derivation graph, typed knowledge schema, and validation workers on top of RelayFile v1 +- Audience: relayfile, relay-cloud, SDK, and agent framework implementers +- Depends on: `relayfile-v1-spec.md` + +## 2. Problem Statement + +RelayFile v1 solves bidirectional sync: external sources stay in sync with the VFS, and agent writes propagate back to providers. But sync is not understanding. When an agent synthesizes three source documents into a strategy memo, v1 has no record of that derivation. When one of those sources updates, nothing flags the memo as potentially stale. When two active decisions contradict each other, nothing detects the conflict. + +Organizations accumulate context across dozens of systems. The value isn't in mirroring that context into files — v1 already does that. The value is in making the relationships between pieces of context queryable, the provenance of derived knowledge traceable, and the validity of synthesized beliefs automatically maintained. + +Without this, agents can traverse a workspace beautifully and still fail to determine which parts are live, which are stale, which are binding, and which were always just one person's interpretation. + +## 3. Goals + +1. Track derivation relationships between files as a first-class DAG (directed acyclic graph). +2. Cascade invalidation when upstream sources change — by infrastructure, not agent discipline. +3. Provide a typed knowledge schema so agents can distinguish decisions from assumptions from meeting notes. +4. Run background validation workers that detect staleness, contradictions, and expired claims. +5. Record agent corrections and human overrides as structured feedback the system can learn from. +6. Expose all of the above through the existing REST API and SDK patterns. + +## 4. Non-Goals + +1. Full knowledge-graph query language (SPARQL, Cypher). The graph is traversable via REST, not via arbitrary queries. +2. Automated rewriting of stale content. Validators flag; agents or humans decide what to do. +3. Cross-workspace graph relationships. Each workspace graph is self-contained in v1. +4. Replacing the VFS. The graph is metadata layered on top of the existing filesystem model. + +## 5. Design Principles + +1. **Graph as metadata, not a separate store.** Derivation edges, types, and validation state live in the existing state model alongside files. No second database. +2. **Invalidation over regeneration.** When a source changes, mark downstream nodes as `needs_revalidation`. Don't silently rewrite them. +3. **Schema is opt-in but rewarded.** Untyped files still work. Typed files get validation, cascade tracking, and richer query support. +4. **Agents declare derivations at write time.** The system doesn't infer relationships — the writing agent states them explicitly, just like imports in code. +5. **Humans stay in the loop.** Validators surface problems. Agents can propose fixes. Humans (or authorized agents) approve changes to authoritative content. + +--- + +## 6. Data Model Extensions + +### 6.1 Derivation Edges + +A derivation edge records that one file was produced from one or more other files. + +```go +type DerivationEdge struct { + TargetPath string `json:"targetPath"` // the derived file + SourcePath string `json:"sourcePath"` // one upstream dependency + SourceRev string `json:"sourceRevision"` // revision of source at derivation time + DerivedAt string `json:"derivedAt"` // timestamp + DerivedBy string `json:"derivedBy"` // agent name or "human" + Confidence string `json:"confidence"` // "high", "medium", "low" + Relationship string `json:"relationship"` // "synthesized_from", "summarized_from", "extracted_from", "supersedes" +} +``` + +Stored per-workspace as `DerivationEdges map[string][]DerivationEdge` keyed by `targetPath`. The reverse index `DownstreamIndex map[string][]string` maps `sourcePath → []targetPath` for cascade traversal. + +### 6.2 Knowledge Types + +A knowledge type defines the semantic shape of a file — what metadata it must carry, what states it can be in, and how it participates in validation. + +```go +type KnowledgeType struct { + Name string `json:"name"` + RequiredProps []string `json:"requiredProperties"` // e.g. ["status", "decided_by"] + AllowedStatus []string `json:"allowedStatus"` // e.g. ["active", "superseded", "draft"] + DefaultStatus string `json:"defaultStatus"` + Mutable bool `json:"mutable"` // false = only updated by sync + ExpiryField string `json:"expiryField,omitempty"` // property key for TTL + ValidatesOn []string `json:"validatesOn"` // triggers: ["source_change", "schedule", "expiry"] +} +``` + +Stored per-workspace as `KnowledgeTypes map[string]KnowledgeType`. Files reference their type via `semantics.properties["type"]`. + +Built-in types shipped with every workspace: + +| Type | Required Properties | Allowed Status | Notes | +|------|-------------------|---------------|-------| +| `source_mirror` | `provider`, `provider_object_id` | `synced`, `stale` | Immutable by agents. Updated only by sync pipeline. | +| `decision` | `status`, `decided_by`, `decided_at` | `active`, `superseded`, `draft` | Must declare `supersedes` relation if replacing another decision. | +| `synthesis` | `derived_from`, `confidence`, `last_validated` | `current`, `needs_revalidation`, `stale` | Must have at least one derivation edge. | +| `assumption` | `asserted_by`, `context` | `active`, `expired`, `disproven` | Optional `expiry` property for TTL. | +| `correction` | `corrects`, `corrected_by`, `reason` | `active` | Records human/agent override of a prior belief. | +| `sop` | `owner`, `last_reviewed` | `active`, `needs_review`, `archived` | Standard operating procedures. | + +Custom types can be registered per workspace via the API. + +### 6.3 Validation State + +Every file with a knowledge type and derivation edges carries validation state. + +```go +type ValidationState struct { + Status string `json:"status"` // "valid", "needs_revalidation", "stale", "conflict", "expired" + LastValidatedAt string `json:"lastValidatedAt"` + LastValidatedRev string `json:"lastValidatedRevision"` // revision of this file when last validated + InvalidatedBy string `json:"invalidatedBy,omitempty"` // path of source that triggered invalidation + InvalidatedAt string `json:"invalidatedAt,omitempty"` +} +``` + +Stored per-file in `semantics.properties` as flattened keys: +- `_validation_status` +- `_validation_last_validated_at` +- `_validation_last_validated_rev` +- `_validation_invalidated_by` +- `_validation_invalidated_at` + +The `_` prefix marks system-managed properties that agents can read but not write directly. + +### 6.4 Validation Records + +Output of validation workers. Immutable audit trail. + +```go +type ValidationRecord struct { + RecordID string `json:"recordId"` + WorkspaceID string `json:"workspaceId"` + FilePath string `json:"filePath"` + ValidationType string `json:"validationType"` // "staleness", "consistency", "expiry", "schema" + Result string `json:"result"` // "valid", "stale", "conflict", "expired", "schema_violation" + Evidence []string `json:"evidence"` // paths or descriptions of why + SuggestedAction string `json:"suggestedAction,omitempty"` // optional remediation hint + CreatedAt string `json:"createdAt"` + CorrelationID string `json:"correlationId"` +} +``` + +Stored per-workspace as an append-only log: `ValidationRecords []ValidationRecord`. + +### 6.5 Correction Records + +When a human or authorized agent rejects an agent's derived content and provides the correct version, the system records the correction. + +```go +type CorrectionRecord struct { + RecordID string `json:"recordId"` + FilePath string `json:"filePath"` + CorrectedBy string `json:"correctedBy"` // agent name or "human" + Reason string `json:"reason"` + OldRevision string `json:"oldRevision"` + NewRevision string `json:"newRevision"` + CreatedAt string `json:"createdAt"` +} +``` + +Agents can query corrections to avoid repeating the same mistake. An agent about to synthesize from source X can check: "has a previous synthesis from X been corrected? what was the correction?" + +--- + +## 7. Derivation Graph API + +All endpoints are workspace-scoped under `/v1/workspaces/{workspaceId}/graph/...`. + +Required scope: `graph:read` for queries, `graph:write` for mutations. + +### 7.1 Declare Derivation + +`POST /graph/edges` + +Called by agents at write time to declare that a file was derived from one or more sources. + +Request: + +```json +{ + "targetPath": "/synthesis/q1-strategy.md", + "sources": [ + { + "path": "/external/Sales/q1-numbers.csv", + "revision": "rev_abc" + }, + { + "path": "/external/Product/roadmap.md", + "revision": "rev_def" + } + ], + "relationship": "synthesized_from", + "confidence": "high" +} +``` + +Response (`201`): + +```json +{ + "edgeCount": 2, + "targetPath": "/synthesis/q1-strategy.md", + "validationStatus": "valid" +} +``` + +Behavior: +1. Validates that all source paths exist and revisions match current state. +2. Creates one `DerivationEdge` per source. +3. Updates the reverse `DownstreamIndex`. +4. Sets initial validation state to `valid` with current timestamp. +5. If the target file has a knowledge type of `synthesis`, enforces that at least one derivation edge exists. +6. Rejects cycles (target cannot be an ancestor of itself). + +### 7.2 Query Upstream + +`GET /graph/upstream?path=/synthesis/q1-strategy.md&depth=1` + +Returns all sources that the given file was derived from, up to `depth` levels (default 1, max 10). + +Response: + +```json +{ + "path": "/synthesis/q1-strategy.md", + "upstream": [ + { + "path": "/external/Sales/q1-numbers.csv", + "revision": "rev_abc", + "currentRevision": "rev_xyz", + "drifted": true, + "relationship": "synthesized_from", + "depth": 1 + }, + { + "path": "/external/Product/roadmap.md", + "revision": "rev_def", + "currentRevision": "rev_def", + "drifted": false, + "relationship": "synthesized_from", + "depth": 1 + } + ] +} +``` + +The `drifted` field compares `sourceRevision` at derivation time against the current revision. This is a lightweight staleness check without running full validation. + +### 7.3 Query Downstream + +`GET /graph/downstream?path=/external/Sales/q1-numbers.csv&depth=1` + +Returns all files derived from the given source, up to `depth` levels. + +Response: + +```json +{ + "path": "/external/Sales/q1-numbers.csv", + "downstream": [ + { + "path": "/synthesis/q1-strategy.md", + "relationship": "synthesized_from", + "validationStatus": "needs_revalidation", + "depth": 1 + } + ] +} +``` + +### 7.4 Remove Derivation + +`DELETE /graph/edges?targetPath=/synthesis/q1-strategy.md&sourcePath=/external/Sales/q1-numbers.csv` + +Removes a specific edge. If no `sourcePath` is given, removes all edges for the target. + +### 7.5 Graph Stats + +`GET /graph/stats` + +Response: + +```json +{ + "totalEdges": 847, + "totalTypedFiles": 312, + "validationSummary": { + "valid": 280, + "needs_revalidation": 22, + "stale": 8, + "conflict": 2, + "expired": 0 + }, + "topSources": [ + { "path": "/external/Sales/q1-numbers.csv", "downstreamCount": 14 } + ] +} +``` + +--- + +## 8. Knowledge Type API + +### 8.1 Register Type + +`POST /types` + +Request: + +```json +{ + "name": "competitive_intel", + "requiredProperties": ["source_url", "captured_by", "captured_at"], + "allowedStatus": ["current", "outdated", "unverified"], + "defaultStatus": "unverified", + "mutable": true, + "expiryField": "expires_at", + "validatesOn": ["schedule", "expiry"] +} +``` + +Response (`201`): + +```json +{ + "name": "competitive_intel", + "created": true +} +``` + +### 8.2 List Types + +`GET /types` + +Returns all registered knowledge types for the workspace, including built-ins. + +### 8.3 Get Type + +`GET /types/{typeName}` + +### 8.4 Delete Type + +`DELETE /types/{typeName}` + +Fails if any files in the workspace reference this type. Returns `409` with a list of files using the type. + +### 8.5 Schema Validation on Write + +When a file is written via `PUT /fs/file` and `semantics.properties["type"]` is set to a registered knowledge type: + +1. **Required properties check**: All properties listed in `requiredProperties` must be present in `semantics.properties`. Missing properties → `422` with details. +2. **Status validation**: If `semantics.properties["status"]` is set, it must be in `allowedStatus`. Invalid status → `422`. +3. **Default status**: If no status is set, the `defaultStatus` is applied automatically. +4. **Immutability check**: If `mutable: false`, only the sync pipeline can update the file. Agent writes → `403`. + +This validation is opt-in: files without a `type` property are unconstrained, same as v1. + +--- + +## 9. Validation Workers + +Three background worker types run per workspace. They subscribe to the event stream and execute on triggers. + +### 9.1 Staleness Validator + +**Trigger**: `file.updated` or `file.created` event on any file that appears in the `DownstreamIndex` (i.e., a source file has changed). + +**Behavior**: + +1. Look up all downstream paths from `DownstreamIndex[updatedPath]`. +2. For each downstream file: + a. Compare the `sourceRevision` in the derivation edge against the source's current revision. + b. If they differ, set `_validation_status = "needs_revalidation"` and `_validation_invalidated_by = updatedPath`. + c. Emit a `validation.invalidated` event on the change feed. + d. Write a `ValidationRecord` with type `staleness`. +3. Walk transitive downstream edges (depth-limited to 5) and repeat. + +**Does NOT**: Rewrite or regenerate the downstream file. Only marks it. + +### 9.2 Consistency Validator + +**Trigger**: Scheduled sweep (configurable interval, default every 6 hours) or on-demand via `POST /validation/run?type=consistency`. + +**Behavior**: + +1. For every pair of files sharing the same knowledge type and `status = "active"`: + a. If type is `decision` and both have the same `scope` or `topic` property, flag as potential conflict. + b. Write a `ValidationRecord` with type `consistency`, evidence listing both paths. + c. Set `_validation_status = "conflict"` on both files. + d. Emit `validation.conflict` events. + +2. For every `synthesis` that references a `decision` with `status = "superseded"`: + a. Flag the synthesis as referencing stale authority. + b. Write a `ValidationRecord`. + +**Design note**: Consistency validation is intentionally coarse in v1. It catches structural contradictions (two active decisions on the same topic, references to superseded content) but does not do semantic analysis of content. Semantic contradiction detection is a future extension point where an LLM validator could be plugged in. + +### 9.3 Expiry Validator + +**Trigger**: Scheduled sweep (configurable interval, default every 1 hour) or on-demand. + +**Behavior**: + +1. Query all files where the knowledge type has an `expiryField` set. +2. For each file, read `semantics.properties[expiryField]`. +3. If the expiry timestamp is in the past: + a. Set `_validation_status = "expired"`. + b. Set `semantics.properties["status"] = "expired"`. + c. Write a `ValidationRecord` with type `expiry`. + d. Emit `validation.expired` event. + +### 9.4 Schema Validator + +**Trigger**: On file write (inline, not async) and scheduled sweep. + +**Behavior**: + +1. For each file with a `type` property referencing a registered `KnowledgeType`: + a. Check required properties are present. + b. Check status is in allowed set. + c. If violations found, write a `ValidationRecord` with type `schema`. + +The inline check on write (§8.5) prevents new violations. The sweep catches files that became invalid due to type definition changes. + +--- + +## 10. Cascade Invalidation + +When a source file updates, the staleness validator walks the derivation graph downstream. This is the core mechanism that replaces "maintenance discipline" with infrastructure. + +### 10.1 Cascade Rules + +1. **Direct invalidation**: If file A is derived from file B, and B updates, A is marked `needs_revalidation`. +2. **Transitive invalidation**: If file C is derived from file A, and A is marked `needs_revalidation` due to B updating, C is also marked `needs_revalidation`. Depth limit: 5 levels. +3. **Selective cascade**: If a file has multiple sources and only one changes, the file is still marked `needs_revalidation` — but the `invalidatedBy` field records which specific source triggered it. +4. **Revalidation clears cascade**: When a `needs_revalidation` file is rewritten (new revision with updated derivation edges pointing to current source revisions), its validation state resets to `valid` and downstream cascade stops. + +### 10.2 Cascade Events + +Every cascade step emits a `validation.invalidated` event on the change feed: + +```json +{ + "eventId": "evt_500", + "type": "validation.invalidated", + "path": "/synthesis/q1-strategy.md", + "revision": "rev_current", + "origin": "validation_worker", + "metadata": { + "invalidatedBy": "/external/Sales/q1-numbers.csv", + "cascadeDepth": 1, + "validationType": "staleness" + }, + "correlationId": "corr_cascade_123", + "timestamp": "2026-03-12T10:00:00Z" +} +``` + +Agents subscribed to the event stream can listen for `validation.*` events and take action. + +--- + +## 11. Corrections API + +### 11.1 Record Correction + +`POST /corrections` + +Called when a human or authorized agent rejects derived content and provides the right answer. + +Request: + +```json +{ + "filePath": "/synthesis/q1-strategy.md", + "reason": "The synthesis incorrectly concluded Q1 revenue was flat. The sales CSV shows 12% growth when the APAC region is included.", + "correctedRevision": "rev_new" +} +``` + +The `correctedRevision` must be a revision that already exists (the correction is the new file content, written via the normal `PUT /fs/file` flow first). + +Response (`201`): + +```json +{ + "recordId": "corr_456", + "filePath": "/synthesis/q1-strategy.md", + "correctedBy": "human", + "createdAt": "2026-03-12T10:30:00Z" +} +``` + +### 11.2 Query Corrections + +`GET /corrections?path=/synthesis/q1-strategy.md` +`GET /corrections?sourcePath=/external/Sales/q1-numbers.csv` + +The second form returns all corrections on files derived from a given source — useful for an agent to check "have previous syntheses from this source been corrected?" before generating new content. + +Response: + +```json +{ + "corrections": [ + { + "recordId": "corr_456", + "filePath": "/synthesis/q1-strategy.md", + "correctedBy": "human", + "reason": "The synthesis incorrectly concluded Q1 revenue was flat...", + "oldRevision": "rev_old", + "newRevision": "rev_new", + "createdAt": "2026-03-12T10:30:00Z" + } + ] +} +``` + +--- + +## 12. SDK Extensions + +Package: `@agent-relay/relayfile-sdk` + +```ts +// Derivation graph +interface RelayFileClient { + // ... existing v1 methods ... + + // Graph + declareDerivation(workspaceId: string, input: DeclareDerivationInput): Promise; + queryUpstream(workspaceId: string, path: string, depth?: number): Promise; + queryDownstream(workspaceId: string, path: string, depth?: number): Promise; + removeDerivation(workspaceId: string, targetPath: string, sourcePath?: string): Promise; + getGraphStats(workspaceId: string): Promise; + + // Knowledge types + registerType(workspaceId: string, type: KnowledgeTypeInput): Promise; + listTypes(workspaceId: string): Promise; + getType(workspaceId: string, typeName: string): Promise; + deleteType(workspaceId: string, typeName: string): Promise; + + // Validation + getValidationRecords(workspaceId: string, opts?: ValidationQueryOpts): Promise; + triggerValidation(workspaceId: string, type: ValidationType): Promise; + + // Corrections + recordCorrection(workspaceId: string, input: CorrectionInput): Promise; + queryCorrections(workspaceId: string, opts?: CorrectionQueryOpts): Promise; +} + +type DeclareDerivationInput = { + targetPath: string; + sources: { path: string; revision: string }[]; + relationship: 'synthesized_from' | 'summarized_from' | 'extracted_from' | 'supersedes'; + confidence: 'high' | 'medium' | 'low'; +}; + +type ValidationQueryOpts = { + path?: string; + type?: 'staleness' | 'consistency' | 'expiry' | 'schema'; + result?: string; + since?: string; + cursor?: string; + limit?: number; +}; + +type CorrectionQueryOpts = { + path?: string; + sourcePath?: string; + cursor?: string; + limit?: number; +}; +``` + +### 12.1 Derived Write Helper + +Convenience method that combines file write + derivation declaration in one call: + +```ts +async function writeDerived( + client: RelayFileClient, + workspaceId: string, + input: { + path: string; + baseRevision: string; + content: string; + contentType?: string; + sources: { path: string; revision: string }[]; + relationship: string; + confidence: string; + properties?: Record; + } +): Promise<{ writeResult: WriteQueuedResponse; derivationResult: DerivationResult }>; +``` + +This is the primary API agents should use when writing derived content. It ensures the derivation edge is always declared atomically with the write. + +--- + +## 13. Event Stream Extensions + +New event types on the existing `/fs/events` change feed: + +| Event Type | Trigger | Metadata | +|-----------|---------|----------| +| `derivation.declared` | New derivation edge created | `targetPath`, `sourcePaths`, `relationship` | +| `derivation.removed` | Derivation edge deleted | `targetPath`, `sourcePath` | +| `validation.invalidated` | Cascade invalidation fired | `invalidatedBy`, `cascadeDepth`, `validationType` | +| `validation.conflict` | Consistency validator found conflict | `conflictingPaths`, `reason` | +| `validation.expired` | Expiry validator triggered | `expiryField`, `expiredAt` | +| `validation.resolved` | File revalidated (new revision clears invalid state) | `previousStatus` | +| `correction.recorded` | Human/agent correction recorded | `correctedBy`, `reason` | + +Agents can filter the event stream by type: `GET /fs/events?type=validation.*` to subscribe only to validation events. + +--- + +## 14. Auth Scope Extensions + +New JWT scopes: + +| Scope | Grants | +|-------|--------| +| `graph:read` | Query upstream/downstream, graph stats | +| `graph:write` | Declare/remove derivation edges | +| `types:read` | List/get knowledge types | +| `types:write` | Register/delete knowledge types | +| `validation:read` | Read validation records | +| `validation:trigger` | Trigger on-demand validation sweeps | +| `corrections:read` | Query correction records | +| `corrections:write` | Record corrections | + +--- + +## 15. Persisted State Extensions + +Additions to the workspace state: + +```go +type WorkspaceState struct { + // ... existing v1 fields ... + + // Knowledge graph + DerivationEdges map[string][]DerivationEdge `json:"derivationEdges,omitempty"` // targetPath -> edges + DownstreamIndex map[string][]string `json:"downstreamIndex,omitempty"` // sourcePath -> []targetPath + KnowledgeTypes map[string]KnowledgeType `json:"knowledgeTypes,omitempty"` + ValidationRecords []ValidationRecord `json:"validationRecords,omitempty"` + CorrectionRecords []CorrectionRecord `json:"correctionRecords,omitempty"` +} +``` + +For Postgres backend, the snapshot table already stores the full workspace state as JSON. No schema migration needed for v1 of this feature. If validation records grow large, they can be moved to a separate table in a future optimization. + +--- + +## 16. Agent Workflow Example + +End-to-end flow showing how the system works in practice: + +### Step 1: Source files sync from providers (existing v1) + +Salesforce webhook → envelope → VFS: +- `/external/Sales/q1-pipeline.csv` (type: `source_mirror`, revision: `rev_1`) +- `/external/Sales/q1-closed.csv` (type: `source_mirror`, revision: `rev_2`) + +### Step 2: Agent synthesizes + +Agent reads both files, produces a strategy doc, and writes it using `writeDerived`: + +```ts +await client.writeDerived(workspaceId, { + path: '/synthesis/q1-revenue-analysis.md', + baseRevision: null, // new file + content: '# Q1 Revenue Analysis\n...', + sources: [ + { path: '/external/Sales/q1-pipeline.csv', revision: 'rev_1' }, + { path: '/external/Sales/q1-closed.csv', revision: 'rev_2' } + ], + relationship: 'synthesized_from', + confidence: 'high', + properties: { type: 'synthesis', status: 'current' } +}); +``` + +### Step 3: Source updates + +Salesforce pipeline data updates. Webhook fires, VFS updates `/external/Sales/q1-pipeline.csv` to `rev_3`. + +### Step 4: Cascade invalidation + +Staleness validator: +1. Sees `file.updated` on `/external/Sales/q1-pipeline.csv`. +2. Looks up `DownstreamIndex["/external/Sales/q1-pipeline.csv"]` → `["/synthesis/q1-revenue-analysis.md"]`. +3. Compares: derivation edge says `sourceRevision: "rev_1"`, current is `"rev_3"` → drifted. +4. Sets `_validation_status = "needs_revalidation"` on the synthesis. +5. Emits `validation.invalidated` event. + +### Step 5: Agent reacts + +Agent subscribed to `validation.*` events sees the invalidation. It: +1. Queries `GET /graph/upstream?path=/synthesis/q1-revenue-analysis.md` to see what changed. +2. Queries `GET /corrections?sourcePath=/external/Sales/q1-pipeline.csv` to check for past mistakes. +3. Re-reads the updated sources. +4. Rewrites the synthesis with `writeDerived`, pointing to current revisions. +5. Validation state resets to `valid`. + +### Step 6: Human corrects + +Human reviews the synthesis, finds a mistake, edits via `PUT /fs/file`, then records the correction: + +```ts +await client.recordCorrection(workspaceId, { + filePath: '/synthesis/q1-revenue-analysis.md', + reason: 'APAC numbers were excluded from the regional breakdown', + correctedRevision: 'rev_corrected' +}); +``` + +Next time the agent synthesizes from these sources, it checks corrections first and avoids repeating the error. + +--- + +## 17. Validation Worker Configuration + +Per-workspace configuration stored in workspace settings: + +```json +{ + "validation": { + "staleness": { + "enabled": true, + "maxCascadeDepth": 5, + "batchSize": 50 + }, + "consistency": { + "enabled": true, + "intervalSeconds": 21600, + "scopes": ["decision", "sop"] + }, + "expiry": { + "enabled": true, + "intervalSeconds": 3600 + }, + "schema": { + "enabled": true, + "enforceOnWrite": true, + "sweepIntervalSeconds": 86400 + } + } +} +``` + +Workers run as goroutines within the relayfile process, sharing the same state backend. They use the existing event stream as their trigger mechanism — no additional queue infrastructure needed. + +--- + +## 18. SLOs + +| Metric | Target | +|--------|--------| +| Cascade invalidation after source update | < 5s p95 | +| Graph upstream/downstream query (depth 1) | < 50ms p95 | +| Graph upstream/downstream query (depth 5) | < 200ms p95 | +| Consistency sweep (1000 typed files) | < 30s | +| Expiry sweep (1000 typed files) | < 10s | +| Schema validation on write | < 10ms p95 (inline) | + +--- + +## 19. Testing Strategy + +1. **Unit tests**: Derivation edge CRUD, cycle detection, cascade walk, type validation, expiry checks. +2. **Integration tests**: Source update → cascade invalidation → event emission → agent revalidation flow. +3. **Consistency tests**: Seed workspace with known contradictions, verify consistency validator catches them. +4. **Correction tests**: Write → correct → re-derive cycle preserves correction history. +5. **Performance tests**: Graph traversal at 10k edges, cascade at 5 levels deep, sweep at 10k typed files. +6. **Regression tests**: Ensure v1 behavior is unchanged for workspaces that don't use graph features. + +--- + +## 20. Rollout Plan + +### Phase 1: Derivation Graph (2 weeks) + +1. `DerivationEdge` and `DownstreamIndex` data structures. +2. Graph CRUD API endpoints. +3. `writeDerived` SDK helper. +4. Cycle detection. +5. `derivation.declared` and `derivation.removed` events. + +Exit criteria: Agent can declare derivations and query the graph. + +### Phase 2: Knowledge Types + Schema Validation (2 weeks) + +1. `KnowledgeType` registry and API. +2. Built-in types. +3. Schema validation on write (inline). +4. Schema validation sweep (background). +5. Type-aware query extensions. + +Exit criteria: Files with types are validated on write. Query API filters by type. + +### Phase 3: Validation Workers + Cascade (2 weeks) + +1. Staleness validator with cascade. +2. Expiry validator. +3. `validation.*` events on change feed. +4. Validation records API. +5. Worker configuration. + +Exit criteria: Source update triggers cascade invalidation within SLO. Expired files are flagged automatically. + +### Phase 4: Corrections + Consistency (2 weeks) + +1. Corrections API. +2. Consistency validator. +3. SDK extensions for correction query. +4. End-to-end agent workflow validation. + +Exit criteria: Full workflow from §16 runs end-to-end. Corrections are queryable by source path. + +--- + +## 21. Open Decisions + +1. **Validation record retention**: How long to keep validation records before compaction? Propose 90 days with configurable retention. +2. **Semantic contradiction detection**: Should the consistency validator call an LLM to detect content-level contradictions, or stay purely structural? Propose structural-only in v1, with an extension point for LLM validators. +3. **Cross-workspace edges**: Some organizations will want derivations across workspaces (e.g., a company-wide strategy doc referenced by team-level syntheses). Defer to v2. +4. **Derivation inference**: Should the system attempt to auto-detect derivations by analyzing file content for references? Propose no — explicit declaration only. Inference is a future agent capability, not a platform feature. +5. **Correction learning**: How should agents consume corrections at scale? Simple query is v1. A future "correction embedding" or "mistake pattern" index could help agents generalize from specific corrections. + +--- + +## 22. References + +1. `relayfile-v1-spec.md` — base spec this extends +2. `architecture-ascii.md` — system topology +3. `internal/relayfile/store.go` — current data model and VFS implementation +4. `sdk/relayfile-sdk/src/types.ts` — current SDK type contracts diff --git a/docs/mount-hardening-plan.md b/docs/mount-hardening-plan.md new file mode 100644 index 00000000..83eac4c9 --- /dev/null +++ b/docs/mount-hardening-plan.md @@ -0,0 +1,295 @@ +# Mount Client Hardening Plan + +This document captures the research and proposed implementation strategy for the three hardening areas identified in the README: **true FUSE integration**, **inode cache semantics**, and **conflict translation**. + +--- + +## 1. Current State + +The mount client (`cmd/relayfile-mount` + `internal/mountsync`) uses a **polling-based sync loop**: + +- Every N seconds (default 2s, with jitter), it runs `SyncOnce()`. +- **Push phase**: walks the local directory, detects hash changes, and PUTs modified files with `If-Match` optimistic concurrency. +- **Pull phase**: either fetches the full remote tree or uses an incremental event cursor to download changed/deleted files. +- State is persisted to `.relayfile-mount-state.json` (revision, content hash, dirty flag per file). +- Conflict on write is detected via HTTP 409 and the file is marked `Dirty` for retry on the next cycle. + +### Limitations + +| Area | Current Behavior | Problem | +|------|-----------------|---------| +| Filesystem interface | Polling + `os.ReadFile` / `os.WriteFile` | Agent writes are invisible until the next poll; no instant notification; race window between poll cycles | +| Cache coherence | SHA-256 hash comparison per file per cycle | No expiration, no TTL; stale reads possible between polls; full-tree walk is O(n) for large workspaces | +| Conflict handling | HTTP 409 -> mark dirty, log, skip overwrite | No user-visible conflict artifact; no merge strategy; dirty flag silently retried forever | + +--- + +## 2. True FUSE Integration + +### 2.1 Goal + +Replace the polling-based local mirror with a true FUSE (Filesystem in Userspace) filesystem so that: + +- Agent reads hit the remote API on-demand (or from cache) without waiting for a sync cycle. +- Agent writes are captured immediately by the kernel VFS layer, not discovered on the next poll. +- The local directory behaves like a real mounted filesystem (`ls`, `cat`, `vim`, etc. all work). + +### 2.2 Recommended Library + +**[`bazil.org/fuse`](https://github.com/bazil/fuse)** (pure Go, well-maintained, macOS + Linux). + +Alternative: [`hanwen/go-fuse`](https://github.com/hanwen/go-fuse) (higher performance pathfs/nodefs API, used by rclone). If macOS support is critical (agents primarily run on macOS), `bazil.org/fuse` has broader macOS compatibility. For Linux-only production, `hanwen/go-fuse` is more performant. + +Platform dependency: macOS requires [macFUSE](https://osxfuse.github.io/); Linux requires `libfuse` / `fuse3`. Docker containers can use `--device /dev/fuse --cap-add SYS_ADMIN`. + +### 2.3 Architecture + +``` +Agent process (read/write files) + | + v + kernel VFS <--> FUSE kernel module + | + v + relayfile-fuse (userspace daemon) + | + +-- Lookup/GetAttr/ReadDir --> cached tree metadata + +-- Read --> content cache or remote ReadFile + +-- Write/Create --> buffer + WriteFile with If-Match + +-- Unlink/Rename --> DeleteFile / WriteFile + +-- Fsync --> flush pending writes to remote +``` + +### 2.4 Implementation Phases + +**Phase 1: Read-only FUSE mount** + +1. Add `bazil.org/fuse` dependency. +2. Implement a `fuseFS` struct satisfying `fuse.FS` / `fuse.Node` / `fuse.Handle` interfaces. +3. `Lookup` and `ReadDirAll` call `ListTree` (cached, see Section 3). +4. `Read` calls `ReadFile` (cached). +5. `GetAttr` returns synthetic inode numbers, sizes derived from content length, and timestamps from event feed. +6. Mount with `fuse.Mount(mountpoint, fuseFS, ...)`. +7. Retain the existing `mountsync` polling path as a fallback (`--mode=poll|fuse`, default `fuse`). + +**Phase 2: Read-write FUSE mount** + +1. `Write` / `Create` buffer content in memory (or a local spool directory). +2. `Fsync` or `Release` triggers `WriteFile` with `If-Match` against the cached revision. +3. `Unlink` triggers `DeleteFile`. +4. `Rename` implemented as read-source + write-dest + delete-source (the REST API has no rename primitive). +5. On conflict (409), apply conflict translation (Section 4). + +**Phase 3: Graceful degradation** + +1. If FUSE is unavailable (no macFUSE, no /dev/fuse), auto-fall back to polling mode with a log warning. +2. `--fuse-fallback=poll` flag to control behavior. +3. Docker Compose updated: add `--device /dev/fuse` and `SYS_ADMIN` capability to the mountsync container. + +### 2.5 File Layout + +``` +cmd/relayfile-mount/main.go (add --mode=fuse|poll flag) +internal/mountfuse/fs.go (fuse.FS, root node) +internal/mountfuse/dir.go (directory node) +internal/mountfuse/file.go (file node, read/write handles) +internal/mountfuse/cache.go (inode + content cache, see Section 3) +internal/mountfuse/conflict.go (conflict translation, see Section 4) +internal/mountfuse/fs_test.go (unit tests with mock RemoteClient) +``` + +--- + +## 3. Inode Cache Semantics + +### 3.1 Goal + +Provide a coherent, performant cache layer so FUSE operations do not hit the remote API on every `stat`, `readdir`, or `read` call, while bounding staleness. + +### 3.2 Cache Design + +#### 3.2.1 Metadata Cache (Tree / Attr) + +| Property | Value | +|----------|-------| +| Key | remote path | +| Value | `{inode, mode, size, revision, contentType, mtime, children}` | +| TTL | configurable, default 5s (`RELAYFILE_CACHE_METADATA_TTL`) | +| Invalidation | event feed poll (background goroutine), explicit `Fsync`, write-through on local mutations | +| Eviction | LRU with max entries (default 10,000; `RELAYFILE_CACHE_MAX_ENTRIES`) | + +#### 3.2.2 Content Cache + +| Property | Value | +|----------|-------| +| Key | remote path + revision | +| Value | file content bytes | +| TTL | revision-pinned (immutable per revision); evict on LRU pressure | +| Max size | configurable, default 256 MB (`RELAYFILE_CACHE_MAX_BYTES`) | +| Storage | in-memory for small files; optional disk spool for files > 1 MB | + +#### 3.2.3 Inode Allocation + +FUSE requires stable inode numbers. Strategy: + +- Root directory = inode 1. +- Hash remote path to uint64 (FNV-1a) for deterministic, collision-resistant inode assignment. +- Maintain a `map[string]uint64` (path -> inode) persisted across mount sessions in the state file. +- On collision (extremely rare with FNV-1a over path strings), linear probe to next available inode. + +#### 3.2.4 Background Refresh + +A background goroutine polls the event feed (same cursor mechanism as current `pullRemoteIncremental`) and: + +1. Invalidates metadata cache entries for changed paths. +2. Pre-fetches updated content for hot files (files accessed in the last TTL window). +3. Updates inode map for newly created / deleted paths. + +Poll interval: `RELAYFILE_CACHE_REFRESH_INTERVAL` (default 1s, separate from the legacy sync interval). + +#### 3.2.5 Kernel Cache Hints + +- Set `fuse.Attr.Valid` to metadata TTL so the kernel caches `stat` results. +- Use `fuse.OpenDirectIO` for files where instant coherence matters (opt-in via `.relayfile.acl` property). +- Use `fuse.OpenKeepCache` when revision has not changed since last open. + +### 3.3 Cache Coherence Contract + +| Operation | Guarantee | +|-----------|-----------| +| `stat` / `getattr` | Fresh within metadata TTL | +| `readdir` | Fresh within metadata TTL | +| `read` | Correct for the revision seen at `open` time (revision-pinned) | +| `write` | Write-through; cache updated immediately on successful PUT | +| `fsync` | All pending writes flushed; cache refreshed from remote | + +--- + +## 4. Conflict Translation + +### 4.1 Goal + +When a local write conflicts with a concurrent remote change (HTTP 409), produce a visible, recoverable artifact instead of silently retrying. + +### 4.2 Current Behavior + +On 409, the syncer: +1. Logs "conflict writing ". +2. Marks the tracked file as `Dirty: true`. +3. On the next cycle, retries the write (which will likely conflict again if the remote changed). + +This creates a silent infinite retry loop with no user visibility. + +### 4.3 Proposed Conflict Resolution Strategy + +#### 4.3.1 Conflict File Generation + +When a write returns 409: + +1. Fetch the current remote file (content + revision). +2. Save the local version as `.LOCAL.` (e.g., `guide.LOCAL.md`). +3. Save the remote version as `.REMOTE.` (e.g., `guide.REMOTE.md`). +4. Overwrite the original file with the remote version (so the canonical path always reflects server truth). +5. Create a `.relayfile-conflicts.json` manifest in the directory: + +```json +{ + "conflicts": [ + { + "path": "/docs/guide.md", + "localFile": "guide.LOCAL.md", + "remoteFile": "guide.REMOTE.md", + "localRevision": "rev_abc", + "remoteRevision": "rev_xyz", + "detectedAt": "2026-03-11T21:55:00Z" + } + ] +} +``` + +6. Log a prominent warning: `CONFLICT: /docs/guide.md -- local and remote versions saved as guide.LOCAL.md / guide.REMOTE.md`. + +#### 4.3.2 Conflict Resolution by Agent / User + +An agent or user resolves the conflict by: + +1. Editing the canonical file (`guide.md`) to the desired content. +2. Deleting `guide.LOCAL.md` and `guide.REMOTE.md`. +3. On the next sync cycle, the syncer detects: + - Canonical file changed -> push to remote. + - Conflict files deleted -> remove from manifest. + - Manifest empty -> delete `.relayfile-conflicts.json`. + +#### 4.3.3 Conflict Policies (Configurable) + +| Policy | Behavior | Flag | +|--------|----------|------| +| `conflict-files` (default) | Generate LOCAL/REMOTE conflict files as described above | `--conflict-strategy=conflict-files` | +| `remote-wins` | Silently overwrite local with remote; no conflict files | `--conflict-strategy=remote-wins` | +| `local-wins` | Force-push local content (retry with latest remote revision as base) | `--conflict-strategy=local-wins` | +| `fail` | Stop sync cycle and exit with error code | `--conflict-strategy=fail` | + +Environment variable: `RELAYFILE_CONFLICT_STRATEGY`. + +#### 4.3.4 FUSE-Specific Conflict Behavior + +In FUSE mode, conflicts are detected at `Fsync` / `Release` time: + +1. The `Write` syscall buffers content locally (always succeeds from the agent's perspective). +2. `Fsync` or `Release` attempts the PUT with `If-Match`. +3. On 409, the conflict strategy is applied: + - `conflict-files`: conflict artifacts materialized; `Fsync` returns `syscall.EBUSY` or `0` (configurable). + - `remote-wins`: local buffer discarded; `Fsync` returns `0`. + - `local-wins`: retry with fetched revision; `Fsync` returns `0`. + - `fail`: `Fsync` returns `syscall.EBUSY`. + +--- + +## 5. Implementation Roadmap + +| Phase | Scope | Estimated Effort | Dependencies | +|-------|-------|-----------------|--------------| +| **P0** | Conflict translation in current polling syncer | 1-2 days | None | +| **P1** | Inode cache + background event refresh | 2-3 days | None | +| **P2** | Read-only FUSE mount | 3-5 days | P1, bazil.org/fuse | +| **P3** | Read-write FUSE mount with conflict translation | 3-5 days | P0, P2 | +| **P4** | Docker/CI integration, macFUSE detection, fallback | 1-2 days | P2 | + +Total estimated effort: 10-17 days. + +### P0 can be done independently and immediately improves the polling mount. + +--- + +## 6. Configuration Summary + +| Variable | Default | Description | +|----------|---------|-------------| +| `RELAYFILE_MOUNT_MODE` | `fuse` | `fuse` or `poll` | +| `RELAYFILE_FUSE_FALLBACK` | `poll` | fallback when FUSE unavailable | +| `RELAYFILE_CONFLICT_STRATEGY` | `conflict-files` | `conflict-files`, `remote-wins`, `local-wins`, `fail` | +| `RELAYFILE_CACHE_METADATA_TTL` | `5s` | metadata cache TTL | +| `RELAYFILE_CACHE_MAX_ENTRIES` | `10000` | max cached metadata entries | +| `RELAYFILE_CACHE_MAX_BYTES` | `268435456` (256 MB) | max content cache size | +| `RELAYFILE_CACHE_REFRESH_INTERVAL` | `1s` | background event poll interval | + +--- + +## 7. Testing Strategy + +- **Unit tests**: mock `RemoteClient` for cache hit/miss/eviction, conflict file generation, inode allocation stability. +- **Integration tests**: spin up `relayfile` in-memory, mount via FUSE in a temp directory, perform read/write/conflict scenarios. +- **E2E tests**: extend `scripts/live-e2e.sh` to exercise FUSE mount mode with Docker `--device /dev/fuse`. +- **Benchmarks**: measure FUSE latency vs. polling latency for read-heavy and write-heavy workloads. + +--- + +## 8. Risks and Mitigations + +| Risk | Impact | Mitigation | +|------|--------|------------| +| macFUSE unavailable on agent machines | FUSE mode unusable | Auto-fallback to polling; clear error messages | +| FUSE kernel deadlocks on buggy operations | Mount hangs | Timeouts on all remote calls; `SIGTERM` handler unmounts cleanly | +| High-frequency writes overwhelm API | 429 / degraded performance | Write coalescing in FUSE buffer; respect Retry-After; backpressure via `Fsync` latency | +| Inode number collisions | Kernel confusion | FNV-1a + linear probe; collision rate is negligible for path-based keys | +| Content cache memory pressure | OOM | Bounded LRU with configurable max; disk spool for large files | diff --git a/go.mod b/go.mod index c4f273ed..563c85e3 100644 --- a/go.mod +++ b/go.mod @@ -3,3 +3,5 @@ module github.com/agentworkforce/relayfile go 1.22 require github.com/lib/pq v1.10.9 + +require nhooyr.io/websocket v1.8.17 // indirect diff --git a/go.sum b/go.sum index aeddeae3..86f247c2 100644 --- a/go.sum +++ b/go.sum @@ -1,2 +1,4 @@ github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +nhooyr.io/websocket v1.8.17 h1:KEVeLJkUywCKVsnLIDlD/5gtayKp8VoCkksHCGGfT9Y= +nhooyr.io/websocket v1.8.17/go.mod h1:rN9OFWIUwuxg4fR5tELlYC04bXYowCP9GX47ivo2l+c= diff --git a/internal/httpapi/server.go b/internal/httpapi/server.go index 207a8dba..a4dda08f 100644 --- a/internal/httpapi/server.go +++ b/internal/httpapi/server.go @@ -1,12 +1,16 @@ package httpapi import ( + "archive/tar" + "compress/gzip" + "encoding/base64" "encoding/json" "errors" "fmt" "io" "math" "net/http" + "path/filepath" "sort" "strconv" "strings" @@ -130,6 +134,15 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { case len(parts) == 5 && parts[3] == "fs" && parts[4] == "tree" && r.Method == http.MethodGet: requiredScope = "fs:read" route = "tree" + case len(parts) == 5 && parts[3] == "fs" && parts[4] == "ws" && r.Method == http.MethodGet: + requiredScope = "fs:read" + route = "fs_ws" + case len(parts) == 5 && parts[3] == "fs" && parts[4] == "bulk" && r.Method == http.MethodPost: + requiredScope = "fs:write" + route = "bulk_write" + case len(parts) == 5 && parts[3] == "fs" && parts[4] == "export" && r.Method == http.MethodGet: + requiredScope = "fs:read" + route = "export" case len(parts) == 5 && parts[3] == "fs" && parts[4] == "file" && r.Method == http.MethodGet: requiredScope = "fs:read" route = "read_file" @@ -189,6 +202,11 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { return } + if route == "fs_ws" { + s.handleFileEventsWebSocket(w, r, workspaceID) + return + } + claims, authErr := authorizeBearer(r.Header.Get("Authorization"), s.cfg.JWTSecret, workspaceID, requiredScope, time.Now().UTC()) if authErr != nil { writeError(w, authErr.status, authErr.code, authErr.message, getCorrelationID(r)) @@ -215,6 +233,10 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { switch route { case "tree": s.handleTree(w, r, workspaceID, correlationID, claims) + case "bulk_write": + s.handleBulkWrite(w, r, workspaceID, correlationID, claims) + case "export": + s.handleExport(w, r, workspaceID, correlationID, claims) case "read_file": s.handleReadFile(w, r, workspaceID, correlationID, claims) case "write_file": @@ -1324,6 +1346,96 @@ func (s *Server) handleReadFile(w http.ResponseWriter, r *http.Request, workspac writeJSON(w, http.StatusOK, file) } +func (s *Server) handleBulkWrite(w http.ResponseWriter, r *http.Request, workspaceID, correlationID string, claims tokenClaims) { + var body struct { + Files []relayfile.BulkWriteFile `json:"files"` + } + if !s.decodeJSONBody(w, r, correlationID, &body) { + return + } + if len(body.Files) == 0 { + writeError(w, http.StatusBadRequest, "bad_request", "missing files", correlationID) + return + } + + allowed := make([]relayfile.BulkWriteFile, 0, len(body.Files)) + errorsOut := make([]relayfile.BulkWriteError, 0) + for _, file := range body.Files { + path := normalizeRoutePath(file.Path) + _, readErr := s.store.ReadFile(workspaceID, path) + if readErr == nil { + existingPermissions := s.store.ResolveFilePermissions(workspaceID, path, true) + if !filePermissionAllows(existingPermissions, workspaceID, claims) { + errorsOut = append(errorsOut, relayfile.BulkWriteError{ + Path: path, + Code: "forbidden", + Message: "file access denied by permission policy", + }) + continue + } + } else if readErr == relayfile.ErrNotFound { + inheritedPermissions := s.store.ResolveFilePermissions(workspaceID, path, false) + if !filePermissionAllows(inheritedPermissions, workspaceID, claims) { + errorsOut = append(errorsOut, relayfile.BulkWriteError{ + Path: path, + Code: "forbidden", + Message: "file access denied by permission policy", + }) + continue + } + } + allowed = append(allowed, file) + } + + written, storeErrors := s.store.BulkWrite(workspaceID, allowed) + errorsOut = append(errorsOut, storeErrors...) + writeJSON(w, http.StatusAccepted, map[string]any{ + "written": written, + "errorCount": len(errorsOut), + "errors": errorsOut, + "correlationId": correlationID, + }) +} + +func (s *Server) handleExport(w http.ResponseWriter, r *http.Request, workspaceID, correlationID string, claims tokenClaims) { + format := strings.ToLower(strings.TrimSpace(r.URL.Query().Get("format"))) + if format == "" { + format = "json" + } + + files, err := s.store.ExportWorkspace(workspaceID) + if err != nil { + if err == relayfile.ErrInvalidInput { + writeError(w, http.StatusBadRequest, "bad_request", err.Error(), correlationID) + return + } + writeError(w, http.StatusInternalServerError, "internal_error", err.Error(), correlationID) + return + } + + visible := make([]relayfile.File, 0, len(files)) + for _, file := range files { + effectivePermissions := s.store.ResolveFilePermissions(workspaceID, file.Path, true) + if !filePermissionAllows(effectivePermissions, workspaceID, claims) { + continue + } + visible = append(visible, file) + } + + switch format { + case "json": + writeJSON(w, http.StatusOK, visible) + case "tar": + if err := s.writeTarExport(w, visible); err != nil { + writeError(w, http.StatusInternalServerError, "internal_error", err.Error(), correlationID) + } + case "patch": + s.writePatchExport(w, visible) + default: + writeError(w, http.StatusBadRequest, "bad_request", "invalid format", correlationID) + } +} + func (s *Server) handleWriteFile(w http.ResponseWriter, r *http.Request, workspaceID, correlationID string, claims tokenClaims) { path := r.URL.Query().Get("path") if path == "" { @@ -1354,6 +1466,7 @@ func (s *Server) handleWriteFile(w http.ResponseWriter, r *http.Request, workspa var body struct { ContentType string `json:"contentType"` Content string `json:"content"` + Encoding string `json:"encoding"` Semantics semanticJSONInput `json:"semantics"` } if !s.decodeJSONBody(w, r, correlationID, &body) { @@ -1366,6 +1479,7 @@ func (s *Server) handleWriteFile(w http.ResponseWriter, r *http.Request, workspa IfMatch: ifMatch, ContentType: body.ContentType, Content: body.Content, + Encoding: body.Encoding, Semantics: relayfile.FileSemantics{ Properties: stringPropertiesFromAny(body.Semantics.Properties), Relations: body.Semantics.Relations, @@ -1891,6 +2005,104 @@ func stringPropertiesFromAny(values map[string]any) map[string]string { return out } +func (s *Server) writeTarExport(w http.ResponseWriter, files []relayfile.File) error { + type tarFile struct { + name string + modTime time.Time + content []byte + } + prepared := make([]tarFile, 0, len(files)) + for _, file := range files { + content, err := decodeExportContent(file) + if err != nil { + return err + } + name := strings.TrimPrefix(filepath.Clean(file.Path), "/") + if name == "." || name == "" { + name = "root" + } + prepared = append(prepared, tarFile{ + name: name, + modTime: parseFileTime(file.LastEditedAt), + content: content, + }) + } + + w.Header().Set("Content-Type", "application/gzip") + w.Header().Set("Content-Disposition", `attachment; filename="workspace-export.tar.gz"`) + w.WriteHeader(http.StatusOK) + + gz := gzip.NewWriter(w) + defer gz.Close() + tw := tar.NewWriter(gz) + defer tw.Close() + + for _, file := range prepared { + header := &tar.Header{ + Name: file.name, + Mode: 0o644, + Size: int64(len(file.content)), + ModTime: file.modTime, + } + if err := tw.WriteHeader(header); err != nil { + return err + } + if _, err := tw.Write(file.content); err != nil { + return err + } + } + return nil +} + +func (s *Server) writePatchExport(w http.ResponseWriter, files []relayfile.File) { + var builder strings.Builder + for _, file := range files { + builder.WriteString("--- /dev/null\n") + builder.WriteString("+++ b") + builder.WriteString(file.Path) + builder.WriteString("\n") + if file.Encoding == "base64" { + builder.WriteString("@@ -0,0 +1 @@\n") + builder.WriteString("+[binary content omitted; encoding=base64]\n") + continue + } + lines := strings.Split(file.Content, "\n") + builder.WriteString(fmt.Sprintf("@@ -0,0 +1,%d @@\n", len(lines))) + for _, line := range lines { + builder.WriteString("+") + builder.WriteString(line) + builder.WriteString("\n") + } + } + w.Header().Set("Content-Type", "text/x-diff; charset=utf-8") + w.WriteHeader(http.StatusOK) + _, _ = io.WriteString(w, builder.String()) +} + +func decodeExportContent(file relayfile.File) ([]byte, error) { + if file.Encoding != "base64" { + return []byte(file.Content), nil + } + if decoded, err := decodeBase64String(file.Content); err == nil { + return decoded, nil + } + return nil, relayfile.ErrInvalidInput +} + +func decodeBase64String(value string) ([]byte, error) { + if decoded, err := base64.StdEncoding.DecodeString(value); err == nil { + return decoded, nil + } + return base64.RawStdEncoding.DecodeString(value) +} + +func parseFileTime(value string) time.Time { + if parsed, err := time.Parse(time.RFC3339Nano, strings.TrimSpace(value)); err == nil { + return parsed + } + return time.Unix(0, 0).UTC() +} + func writeJSON(w http.ResponseWriter, status int, data any) { w.Header().Set("Content-Type", "application/json") w.WriteHeader(status) diff --git a/internal/httpapi/server_test.go b/internal/httpapi/server_test.go index bc0f3a79..9c17c06f 100644 --- a/internal/httpapi/server_test.go +++ b/internal/httpapi/server_test.go @@ -1,12 +1,16 @@ package httpapi import ( + "archive/tar" "bytes" + "compress/gzip" + "context" "crypto/hmac" "crypto/sha256" "encoding/base64" "encoding/json" "fmt" + "io" "net/http" "net/http/httptest" "strings" @@ -15,6 +19,8 @@ import ( "time" "github.com/agentworkforce/relayfile/internal/relayfile" + "nhooyr.io/websocket" + "nhooyr.io/websocket/wsjson" ) func TestAuthRequired(t *testing.T) { @@ -66,6 +72,81 @@ func TestDashboardRouteRejectsNonGET(t *testing.T) { } } +func TestFileEventsWebSocketCatchUpAndPingPong(t *testing.T) { + store := relayfile.NewStoreWithOptions(relayfile.StoreOptions{DisableWorkers: true}) + t.Cleanup(store.Close) + + if _, err := store.WriteFile(relayfile.WriteRequest{ + WorkspaceID: "ws_socket", + Path: "/notion/Docs/One.md", + IfMatch: "0", + ContentType: "text/markdown", + Content: "# one", + CorrelationID: "corr_socket_1", + }); err != nil { + t.Fatalf("seed write failed: %v", err) + } + + server := httptest.NewServer(NewServer(store)) + defer server.Close() + + token := mustTestJWT(t, "dev-secret", "ws_socket", "Worker1", []string{"fs:read"}, time.Now().Add(time.Hour)) + wsURL := "ws" + strings.TrimPrefix(server.URL, "http") + "/v1/workspaces/ws_socket/fs/ws?token=" + token + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + conn, _, err := websocket.Dial(ctx, wsURL, nil) + if err != nil { + t.Fatalf("websocket dial failed: %v", err) + } + defer conn.Close(websocket.StatusNormalClosure, "") + + var catchUp map[string]any + if err := wsjson.Read(ctx, conn, &catchUp); err != nil { + t.Fatalf("read catch-up event failed: %v", err) + } + if catchUp["type"] != "file.created" { + t.Fatalf("expected file.created catch-up event, got %v", catchUp["type"]) + } + if catchUp["path"] != "/notion/Docs/One.md" { + t.Fatalf("unexpected catch-up path: %v", catchUp["path"]) + } + + if err := wsjson.Write(ctx, conn, map[string]any{"type": "ping"}); err != nil { + t.Fatalf("write ping failed: %v", err) + } + var pong map[string]any + if err := wsjson.Read(ctx, conn, &pong); err != nil { + t.Fatalf("read pong failed: %v", err) + } + if pong["type"] != "pong" { + t.Fatalf("expected pong response, got %v", pong["type"]) + } + + if _, err := store.WriteFile(relayfile.WriteRequest{ + WorkspaceID: "ws_socket", + Path: "/notion/Docs/Two.md", + IfMatch: "0", + ContentType: "text/markdown", + Content: "# two", + CorrelationID: "corr_socket_2", + }); err != nil { + t.Fatalf("live write failed: %v", err) + } + + var live map[string]any + if err := wsjson.Read(ctx, conn, &live); err != nil { + t.Fatalf("read live event failed: %v", err) + } + if live["type"] != "file.created" { + t.Fatalf("expected file.created live event, got %v", live["type"]) + } + if live["path"] != "/notion/Docs/Two.md" { + t.Fatalf("unexpected live event path: %v", live["path"]) + } +} + func TestLifecycleAndConflicts(t *testing.T) { server := NewServer(relayfile.NewStore()) token := mustTestJWT(t, "dev-secret", "ws_1", "Worker1", []string{"fs:read", "fs:write", "ops:read", "sync:read"}, time.Now().Add(time.Hour)) @@ -264,6 +345,326 @@ func TestWriteFilePayloadTooLarge(t *testing.T) { } } +func TestBinaryEncodingRoundTripAndExport(t *testing.T) { + server := NewServerWithConfig(relayfile.NewStoreWithOptions(relayfile.StoreOptions{DisableWorkers: true}), ServerConfig{}) + token := mustTestJWT(t, "dev-secret", "ws_binary", "Worker1", []string{"fs:read", "fs:write"}, time.Now().Add(time.Hour)) + encoded := base64.StdEncoding.EncodeToString([]byte{0x00, 0x7f, 0xff, 0x10}) + + writeResp := doRequest(t, server, request{ + method: http.MethodPut, + path: "/v1/workspaces/ws_binary/fs/file?path=/external/blob.bin", + headers: map[string]string{ + "Authorization": "Bearer " + token, + "X-Correlation-Id": "corr_binary_write", + "If-Match": "0", + }, + body: map[string]any{ + "contentType": "application/octet-stream", + "content": encoded, + "encoding": "base64", + }, + }) + if writeResp.Code != http.StatusAccepted { + t.Fatalf("expected 202 on binary write, got %d (%s)", writeResp.Code, writeResp.Body.String()) + } + + readResp := doRequest(t, server, request{ + method: http.MethodGet, + path: "/v1/workspaces/ws_binary/fs/file?path=/external/blob.bin", + headers: map[string]string{ + "Authorization": "Bearer " + token, + "X-Correlation-Id": "corr_binary_read", + }, + }) + if readResp.Code != http.StatusOK { + t.Fatalf("expected 200 on binary read, got %d (%s)", readResp.Code, readResp.Body.String()) + } + var file relayfile.File + if err := json.NewDecoder(readResp.Body).Decode(&file); err != nil { + t.Fatalf("decode binary read response: %v", err) + } + if file.Encoding != "base64" { + t.Fatalf("expected base64 encoding in read response, got %+v", file) + } + + exportResp := doRequest(t, server, request{ + method: http.MethodGet, + path: "/v1/workspaces/ws_binary/fs/export?format=tar", + headers: map[string]string{ + "Authorization": "Bearer " + token, + "X-Correlation-Id": "corr_binary_export", + }, + }) + if exportResp.Code != http.StatusOK { + t.Fatalf("expected 200 on tar export, got %d (%s)", exportResp.Code, exportResp.Body.String()) + } + gzr, err := gzip.NewReader(bytes.NewReader(exportResp.Body.Bytes())) + if err != nil { + t.Fatalf("gzip reader: %v", err) + } + defer gzr.Close() + tr := tar.NewReader(gzr) + header, err := tr.Next() + if err != nil { + t.Fatalf("tar next: %v", err) + } + if header.Name != "external/blob.bin" { + t.Fatalf("unexpected tar entry name: %s", header.Name) + } + content, err := io.ReadAll(tr) + if err != nil { + t.Fatalf("read tar content: %v", err) + } + if !bytes.Equal(content, []byte{0x00, 0x7f, 0xff, 0x10}) { + t.Fatalf("unexpected tar content bytes: %v", content) + } +} + +func TestBulkWriteAndJSONExportEndpoints(t *testing.T) { + server := NewServer(relayfile.NewStoreWithOptions(relayfile.StoreOptions{DisableWorkers: true})) + token := mustTestJWT(t, "dev-secret", "ws_bulk_api", "Worker1", []string{"fs:read", "fs:write"}, time.Now().Add(time.Hour)) + + bulkResp := doRequest(t, server, request{ + method: http.MethodPost, + path: "/v1/workspaces/ws_bulk_api/fs/bulk", + headers: map[string]string{ + "Authorization": "Bearer " + token, + "X-Correlation-Id": "corr_bulk", + }, + body: map[string]any{ + "files": []map[string]any{ + { + "path": "/external/A.md", + "contentType": "text/markdown", + "content": "# A", + }, + { + "path": "/external/B.md", + "contentType": "text/markdown", + "content": "# B", + }, + }, + }, + }) + if bulkResp.Code != http.StatusAccepted { + t.Fatalf("expected 202 on bulk write, got %d (%s)", bulkResp.Code, bulkResp.Body.String()) + } + var bulkPayload map[string]any + if err := json.NewDecoder(bulkResp.Body).Decode(&bulkPayload); err != nil { + t.Fatalf("decode bulk response: %v", err) + } + if int(bulkPayload["written"].(float64)) != 2 { + t.Fatalf("expected written=2, got %v", bulkPayload["written"]) + } + + exportResp := doRequest(t, server, request{ + method: http.MethodGet, + path: "/v1/workspaces/ws_bulk_api/fs/export?format=json", + headers: map[string]string{ + "Authorization": "Bearer " + token, + "X-Correlation-Id": "corr_export_json", + }, + }) + if exportResp.Code != http.StatusOK { + t.Fatalf("expected 200 on json export, got %d (%s)", exportResp.Code, exportResp.Body.String()) + } + var files []relayfile.File + if err := json.NewDecoder(exportResp.Body).Decode(&files); err != nil { + t.Fatalf("decode export response: %v", err) + } + if len(files) != 2 { + t.Fatalf("expected 2 exported files, got %d", len(files)) + } + if files[0].Path != "/external/A.md" || files[1].Path != "/external/B.md" { + t.Fatalf("unexpected exported file ordering: %+v", files) + } +} + +func TestBulkWriteEndpoint(t *testing.T) { + server := NewServer(relayfile.NewStoreWithOptions(relayfile.StoreOptions{DisableWorkers: true})) + token := mustTestJWT(t, "dev-secret", "ws_bulk_endpoint", "Worker1", []string{"fs:read", "fs:write"}, time.Now().Add(time.Hour)) + + files := make([]map[string]any, 0, 5) + for i := 0; i < 5; i++ { + files = append(files, map[string]any{ + "path": fmt.Sprintf("/external/Bulk-%d.md", i), + "contentType": "text/markdown", + "content": fmt.Sprintf("# file %d", i), + }) + } + + resp := doRequest(t, server, request{ + method: http.MethodPost, + path: "/v1/workspaces/ws_bulk_endpoint/fs/bulk", + headers: map[string]string{ + "Authorization": "Bearer " + token, + "X-Correlation-Id": "corr_bulk_endpoint", + }, + body: map[string]any{ + "files": files, + }, + }) + if resp.Code != http.StatusAccepted { + t.Fatalf("expected 202 on bulk write, got %d (%s)", resp.Code, resp.Body.String()) + } + + var payload map[string]any + if err := json.NewDecoder(resp.Body).Decode(&payload); err != nil { + t.Fatalf("decode bulk response: %v", err) + } + if int(payload["written"].(float64)) != 5 { + t.Fatalf("expected written=5, got %v", payload["written"]) + } + if int(payload["errorCount"].(float64)) != 0 { + t.Fatalf("expected errorCount=0, got %v", payload["errorCount"]) + } +} + +func TestExportJSON(t *testing.T) { + store := relayfile.NewStoreWithOptions(relayfile.StoreOptions{DisableWorkers: true}) + t.Cleanup(store.Close) + + if written, errs := store.BulkWrite("ws_export_json", []relayfile.BulkWriteFile{ + {Path: "/external/A.md", ContentType: "text/markdown", Content: "# A"}, + {Path: "/external/B.txt", ContentType: "text/plain", Content: "B"}, + }); written != 2 || len(errs) != 0 { + t.Fatalf("seed bulk write failed: written=%d errs=%+v", written, errs) + } + + server := NewServer(store) + token := mustTestJWT(t, "dev-secret", "ws_export_json", "Worker1", []string{"fs:read"}, time.Now().Add(time.Hour)) + + resp := doRequest(t, server, request{ + method: http.MethodGet, + path: "/v1/workspaces/ws_export_json/fs/export?format=json", + headers: map[string]string{ + "Authorization": "Bearer " + token, + "X-Correlation-Id": "corr_export_json_case", + }, + }) + if resp.Code != http.StatusOK { + t.Fatalf("expected 200 on json export, got %d (%s)", resp.Code, resp.Body.String()) + } + + var files []relayfile.File + if err := json.NewDecoder(resp.Body).Decode(&files); err != nil { + t.Fatalf("decode export json response: %v", err) + } + if len(files) != 2 { + t.Fatalf("expected 2 exported files, got %d", len(files)) + } + if files[0].Path != "/external/A.md" || files[0].Content != "# A" { + t.Fatalf("unexpected first exported file: %+v", files[0]) + } + if files[1].Path != "/external/B.txt" || files[1].Content != "B" { + t.Fatalf("unexpected second exported file: %+v", files[1]) + } +} + +func TestExportTar(t *testing.T) { + store := relayfile.NewStoreWithOptions(relayfile.StoreOptions{DisableWorkers: true}) + t.Cleanup(store.Close) + + if written, errs := store.BulkWrite("ws_export_tar", []relayfile.BulkWriteFile{ + { + Path: "/external/blob.bin", + ContentType: "application/octet-stream", + Content: base64.StdEncoding.EncodeToString([]byte{0x01, 0x02, 0x03, 0x04}), + Encoding: "base64", + }, + }); written != 1 || len(errs) != 0 { + t.Fatalf("seed bulk write failed: written=%d errs=%+v", written, errs) + } + + server := NewServer(store) + token := mustTestJWT(t, "dev-secret", "ws_export_tar", "Worker1", []string{"fs:read"}, time.Now().Add(time.Hour)) + + resp := doRequest(t, server, request{ + method: http.MethodGet, + path: "/v1/workspaces/ws_export_tar/fs/export?format=tar", + headers: map[string]string{ + "Authorization": "Bearer " + token, + "X-Correlation-Id": "corr_export_tar_case", + }, + }) + if resp.Code != http.StatusOK { + t.Fatalf("expected 200 on tar export, got %d (%s)", resp.Code, resp.Body.String()) + } + if got := resp.Header().Get("Content-Type"); got != "application/gzip" { + t.Fatalf("expected application/gzip content type, got %q", got) + } + + gzr, err := gzip.NewReader(bytes.NewReader(resp.Body.Bytes())) + if err != nil { + t.Fatalf("gzip reader: %v", err) + } + defer gzr.Close() + tr := tar.NewReader(gzr) + + header, err := tr.Next() + if err != nil { + t.Fatalf("tar next: %v", err) + } + if header.Name != "external/blob.bin" { + t.Fatalf("unexpected tar entry name: %s", header.Name) + } + content, err := io.ReadAll(tr) + if err != nil { + t.Fatalf("read tar content: %v", err) + } + if !bytes.Equal(content, []byte{0x01, 0x02, 0x03, 0x04}) { + t.Fatalf("unexpected tar content bytes: %v", content) + } +} + +func TestWebSocketEvents(t *testing.T) { + store := relayfile.NewStoreWithOptions(relayfile.StoreOptions{DisableWorkers: true}) + t.Cleanup(store.Close) + + server := httptest.NewServer(NewServer(store)) + defer server.Close() + + token := mustTestJWT(t, "dev-secret", "ws_websocket_events", "Worker1", []string{"fs:read", "fs:write"}, time.Now().Add(time.Hour)) + wsURL := "ws" + strings.TrimPrefix(server.URL, "http") + "/v1/workspaces/ws_websocket_events/fs/ws?token=" + token + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + conn, _, err := websocket.Dial(ctx, wsURL, nil) + if err != nil { + t.Fatalf("websocket dial failed: %v", err) + } + defer conn.Close(websocket.StatusNormalClosure, "") + + writeResp := doRequest(t, NewServer(store), request{ + method: http.MethodPut, + path: "/v1/workspaces/ws_websocket_events/fs/file?path=/external/Event.md", + headers: map[string]string{ + "Authorization": "Bearer " + token, + "X-Correlation-Id": "corr_ws_event_write", + "If-Match": "0", + }, + body: map[string]any{ + "contentType": "text/markdown", + "content": "# event", + }, + }) + if writeResp.Code != http.StatusAccepted { + t.Fatalf("expected 202 write, got %d (%s)", writeResp.Code, writeResp.Body.String()) + } + + var event map[string]any + if err := wsjson.Read(ctx, conn, &event); err != nil { + t.Fatalf("read websocket event failed: %v", err) + } + if event["type"] != "file.created" { + t.Fatalf("expected file.created event, got %v", event["type"]) + } + if event["path"] != "/external/Event.md" { + t.Fatalf("unexpected event path: %v", event["path"]) + } +} + func TestIfMatchQuotedETagAccepted(t *testing.T) { server := NewServer(relayfile.NewStore()) token := mustTestJWT(t, "dev-secret", "ws_ifmatch", "Worker1", []string{"fs:read", "fs:write"}, time.Now().Add(time.Hour)) @@ -4156,8 +4557,8 @@ func TestGenericWebhookIngestionMultipleProviders(t *testing.T) { token := mustTestJWT(t, "dev-secret", "ws_1", "Worker1", []string{"fs:read", "fs:write", "sync:read", "sync:trigger"}, time.Now().Add(time.Hour)) testCases := []struct { - name string - provider string + name string + provider string eventType string }{ { @@ -4217,9 +4618,9 @@ func TestGenericWebhookIngestionMissingFields(t *testing.T) { { name: "Missing provider", body: map[string]any{ - "event_type": "file.updated", - "path": "/doc.md", - "data": map[string]any{}, + "event_type": "file.updated", + "path": "/doc.md", + "data": map[string]any{}, "delivery_id": "evt_1", "timestamp": time.Now().UTC().Format(time.RFC3339), }, @@ -4228,9 +4629,9 @@ func TestGenericWebhookIngestionMissingFields(t *testing.T) { { name: "Missing event_type", body: map[string]any{ - "provider": "notion", - "path": "/doc.md", - "data": map[string]any{}, + "provider": "notion", + "path": "/doc.md", + "data": map[string]any{}, "delivery_id": "evt_1", "timestamp": time.Now().UTC().Format(time.RFC3339), }, @@ -4239,9 +4640,9 @@ func TestGenericWebhookIngestionMissingFields(t *testing.T) { { name: "Missing path", body: map[string]any{ - "provider": "notion", - "event_type": "file.updated", - "data": map[string]any{}, + "provider": "notion", + "event_type": "file.updated", + "data": map[string]any{}, "delivery_id": "evt_1", "timestamp": time.Now().UTC().Format(time.RFC3339), }, @@ -4313,8 +4714,8 @@ func TestGenericPassthroughForUnknownProvider(t *testing.T) { "event_type": "file.updated", "path": "/data/record_123.json", "data": map[string]any{ - "content": `{"id": 123, "name": "Test Record"}`, - "contentType": "application/json", + "content": `{"id": 123, "name": "Test Record"}`, + "contentType": "application/json", "providerObjectId": "obj_123", }, "delivery_id": "evt_custom_1", diff --git a/internal/httpapi/websocket.go b/internal/httpapi/websocket.go new file mode 100644 index 00000000..3a15b732 --- /dev/null +++ b/internal/httpapi/websocket.go @@ -0,0 +1,126 @@ +package httpapi + +import ( + "context" + "errors" + "net/http" + "strings" + "time" + + "github.com/agentworkforce/relayfile/internal/relayfile" + "nhooyr.io/websocket" + "nhooyr.io/websocket/wsjson" +) + +type fileEventMessage struct { + Type string `json:"type"` + Path string `json:"path,omitempty"` + Revision string `json:"revision,omitempty"` + Timestamp string `json:"timestamp,omitempty"` +} + +type websocketClientMessage struct { + Type string `json:"type"` +} + +func (s *Server) handleFileEventsWebSocket(w http.ResponseWriter, r *http.Request, workspaceID string) { + claims, authErr := authorizeBearer("Bearer "+strings.TrimSpace(r.URL.Query().Get("token")), s.cfg.JWTSecret, workspaceID, "fs:read", time.Now().UTC()) + if authErr != nil { + writeError(w, authErr.status, authErr.code, authErr.message, "") + return + } + if s.rateLimiter != nil { + key := workspaceID + "|" + claims.AgentName + if !s.rateLimiter.allow(key, time.Now().UTC()) { + writeError(w, http.StatusTooManyRequests, "rate_limited", "rate limit exceeded", "") + return + } + } + + conn, err := websocket.Accept(w, r, &websocket.AcceptOptions{ + InsecureSkipVerify: true, + }) + if err != nil { + return + } + defer conn.Close(websocket.StatusNormalClosure, "") + + ctx := r.Context() + subscriptionCh := make(chan relayfile.Event, 256) + unsubscribe := s.store.Subscribe(workspaceID, subscriptionCh) + defer unsubscribe() + + catchUp, err := s.store.GetRecentEvents(workspaceID, 100) + if err != nil { + _ = conn.Close(websocket.StatusInternalError, "failed to load catch-up events") + return + } + for _, event := range catchUp { + if err := s.writeWebSocketEvent(ctx, conn, event); err != nil { + return + } + } + + controlCh := make(chan fileEventMessage, 8) + readErrCh := make(chan error, 1) + go s.readWebSocketMessages(ctx, conn, controlCh, readErrCh) + + for { + select { + case <-ctx.Done(): + _ = conn.Close(websocket.StatusGoingAway, "request context canceled") + return + case err := <-readErrCh: + if err != nil && websocket.CloseStatus(err) == -1 { + _ = conn.Close(websocket.StatusInternalError, "websocket read failed") + } + return + case msg := <-controlCh: + if err := wsjson.Write(ctx, conn, msg); err != nil { + return + } + case event := <-subscriptionCh: + if err := s.writeWebSocketEvent(ctx, conn, event); err != nil { + return + } + } + } +} + +func (s *Server) readWebSocketMessages(ctx context.Context, conn *websocket.Conn, controlCh chan<- fileEventMessage, readErrCh chan<- error) { + defer close(readErrCh) + for { + var msg websocketClientMessage + err := wsjson.Read(ctx, conn, &msg) + if err != nil { + if websocket.CloseStatus(err) == websocket.StatusNormalClosure || + websocket.CloseStatus(err) == websocket.StatusGoingAway { + readErrCh <- nil + return + } + if errors.Is(err, context.Canceled) { + readErrCh <- nil + return + } + readErrCh <- err + return + } + if strings.EqualFold(strings.TrimSpace(msg.Type), "ping") { + select { + case controlCh <- fileEventMessage{Type: "pong", Timestamp: time.Now().UTC().Format(time.RFC3339Nano)}: + case <-ctx.Done(): + readErrCh <- nil + return + } + } + } +} + +func (s *Server) writeWebSocketEvent(ctx context.Context, conn *websocket.Conn, event relayfile.Event) error { + return wsjson.Write(ctx, conn, fileEventMessage{ + Type: event.Type, + Path: event.Path, + Revision: event.Revision, + Timestamp: event.Timestamp, + }) +} diff --git a/internal/mountsync/syncer.go b/internal/mountsync/syncer.go index ae05e070..9e2f28ef 100644 --- a/internal/mountsync/syncer.go +++ b/internal/mountsync/syncer.go @@ -17,7 +17,11 @@ import ( "sort" "strconv" "strings" + "sync" "time" + + "nhooyr.io/websocket" + "nhooyr.io/websocket/wsjson" ) var ErrConflict = errors.New("revision conflict") @@ -272,6 +276,7 @@ type SyncerOptions struct { LocalRoot string StateFile string EventProvider string + WebSocket *bool Logger Logger } @@ -289,6 +294,10 @@ type Syncer struct { logger Logger state mountState loaded bool + websocket bool + wsConn *websocket.Conn + wsCancel context.CancelFunc + mu sync.Mutex } type mountState struct { @@ -309,6 +318,13 @@ type localSnapshot struct { Hash string } +type websocketEvent struct { + Type string `json:"type"` + Path string `json:"path,omitempty"` + Revision string `json:"revision,omitempty"` + Timestamp string `json:"timestamp,omitempty"` +} + func NewSyncer(client RemoteClient, opts SyncerOptions) (*Syncer, error) { if client == nil { return nil, fmt.Errorf("client is required") @@ -337,6 +353,10 @@ func NewSyncer(client RemoteClient, opts SyncerOptions) (*Syncer, error) { if err := os.MkdirAll(localRoot, 0o755); err != nil { return nil, err } + websocketEnabled := true + if opts.WebSocket != nil { + websocketEnabled = *opts.WebSocket + } return &Syncer{ client: client, workspace: workspace, @@ -344,6 +364,7 @@ func NewSyncer(client RemoteClient, opts SyncerOptions) (*Syncer, error) { localRoot: localRoot, stateFile: stateFile, eventProvider: eventProvider, + websocket: websocketEnabled, logger: opts.Logger, state: mountState{ Files: map[string]trackedFile{}, @@ -352,9 +373,23 @@ func NewSyncer(client RemoteClient, opts SyncerOptions) (*Syncer, error) { } func (s *Syncer) SyncOnce(ctx context.Context) error { + s.mu.Lock() + defer s.mu.Unlock() + if err := s.loadState(); err != nil { return err } + + // Connect websocket outside the lock to avoid deadlock with readWebSocketLoop. + needsWS := s.websocket && s.wsConn == nil + if needsWS { + s.mu.Unlock() + if err := s.connectWebSocket(ctx); err != nil { + s.logf("websocket unavailable; using polling sync: %v", err) + } + s.mu.Lock() + } + conflicted, err := s.pushLocal(ctx) if err != nil { return err @@ -365,6 +400,114 @@ func (s *Syncer) SyncOnce(ctx context.Context) error { return s.saveState() } +func (s *Syncer) connectWebSocket(ctx context.Context) error { + s.mu.Lock() + if !s.websocket || s.wsConn != nil { + s.mu.Unlock() + return nil + } + s.mu.Unlock() + + httpClient, ok := s.client.(*HTTPClient) + if !ok { + return nil + } + + wsURL, err := httpClient.websocketURL(s.workspace) + if err != nil { + return err + } + conn, _, err := websocket.Dial(ctx, wsURL, nil) + if err != nil { + return err + } + + readCtx, cancel := context.WithCancel(context.Background()) + + s.mu.Lock() + s.wsConn = conn + s.wsCancel = cancel + s.mu.Unlock() + + go s.readWebSocketLoop(readCtx, conn) + return nil +} + +func (s *Syncer) readWebSocketLoop(ctx context.Context, conn *websocket.Conn) { + defer s.handleWebSocketDisconnect(conn) + + for { + var event websocketEvent + if err := wsjson.Read(ctx, conn, &event); err != nil { + if websocket.CloseStatus(err) == websocket.StatusNormalClosure || + websocket.CloseStatus(err) == websocket.StatusGoingAway || + errors.Is(err, context.Canceled) { + return + } + s.logf("websocket read failed; falling back to polling: %v", err) + return + } + if err := s.applyWebSocketEvent(ctx, event); err != nil { + s.logf("websocket event apply failed for %s: %v", strings.TrimSpace(event.Path), err) + } + } +} + +func (s *Syncer) applyWebSocketEvent(ctx context.Context, event websocketEvent) error { + switch eventType := strings.TrimSpace(event.Type); eventType { + case "", "pong": + return nil + case "file.created", "file.updated": + remotePath := normalizeRemotePath(event.Path) + if remotePath == "/" || !isUnderRemoteRoot(s.remoteRoot, remotePath) { + return nil + } + file, err := s.client.ReadFile(ctx, s.workspace, remotePath) + if err != nil { + var httpErr *HTTPError + if errors.As(err, &httpErr) && httpErr.StatusCode == http.StatusNotFound { + return nil + } + return err + } + s.mu.Lock() + defer s.mu.Unlock() + if err := s.applyRemoteFile(remotePath, file, nil); err != nil { + return err + } + return s.saveState() + case "file.deleted": + remotePath := normalizeRemotePath(event.Path) + if remotePath == "/" || !isUnderRemoteRoot(s.remoteRoot, remotePath) { + return nil + } + s.mu.Lock() + defer s.mu.Unlock() + if err := s.applyRemoteDelete(remotePath, nil); err != nil { + return err + } + return s.saveState() + default: + return nil + } +} + +func (s *Syncer) handleWebSocketDisconnect(conn *websocket.Conn) { + _ = conn.Close(websocket.StatusNormalClosure, "") + + s.mu.Lock() + defer s.mu.Unlock() + + if s.wsConn != conn { + return + } + if s.wsCancel != nil { + s.wsCancel() + s.wsCancel = nil + } + s.wsConn = nil +} + func (s *Syncer) pullRemote(ctx context.Context, conflicted map[string]struct{}) error { if s.state.EventsCursor != "" { nextCursor, err := s.pullRemoteIncremental(ctx, conflicted, s.state.EventsCursor) @@ -545,8 +688,10 @@ func (s *Syncer) resolveLatestEventCursor(ctx context.Context) (string, error) { } func (s *Syncer) applyRemoteFile(remotePath string, file RemoteFile, conflicted map[string]struct{}) error { - if _, skip := conflicted[remotePath]; skip { - return nil + if conflicted != nil { + if _, skip := conflicted[remotePath]; skip { + return nil + } } if tracked, ok := s.state.Files[remotePath]; ok && tracked.Dirty { return nil @@ -584,8 +729,10 @@ func (s *Syncer) applyRemoteFile(remotePath string, file RemoteFile, conflicted } func (s *Syncer) applyRemoteDelete(remotePath string, conflicted map[string]struct{}) error { - if _, skip := conflicted[remotePath]; skip { - return nil + if conflicted != nil { + if _, skip := conflicted[remotePath]; skip { + return nil + } } tracked, ok := s.state.Files[remotePath] if !ok || tracked.Dirty { @@ -907,6 +1054,27 @@ func correlationID() string { return fmt.Sprintf("mount_%d", time.Now().UnixNano()) } +func (c *HTTPClient) websocketURL(workspaceID string) (string, error) { + base, err := url.Parse(c.baseURL) + if err != nil { + return "", err + } + switch base.Scheme { + case "http": + base.Scheme = "ws" + case "https": + base.Scheme = "wss" + case "ws", "wss": + default: + return "", fmt.Errorf("unsupported base url scheme %q", base.Scheme) + } + base.Path = fmt.Sprintf("/v1/workspaces/%s/fs/ws", url.PathEscape(workspaceID)) + q := url.Values{} + q.Set("token", c.token) + base.RawQuery = q.Encode() + return base.String(), nil +} + func (c *HTTPClient) retryDelay(attempt int, retryAfterHeader string) time.Duration { maxDelay := c.maxDelay if maxDelay <= 0 { diff --git a/internal/mountsync/syncer_test.go b/internal/mountsync/syncer_test.go index 5ccb47d0..a74c3320 100644 --- a/internal/mountsync/syncer_test.go +++ b/internal/mountsync/syncer_test.go @@ -1,13 +1,25 @@ package mountsync import ( + "bytes" "context" + "crypto/hmac" + "crypto/sha256" + "encoding/base64" + "encoding/json" "fmt" + "io" + "net/http" + "net/http/httptest" "os" "path/filepath" "sort" "strings" "testing" + "time" + + "github.com/agentworkforce/relayfile/internal/httpapi" + "github.com/agentworkforce/relayfile/internal/relayfile" ) func TestSyncOncePullsRemoteAndPushesLocalEdits(t *testing.T) { @@ -330,6 +342,71 @@ func TestSyncOnceFallsBackToFullPullWhenEventsUnavailable(t *testing.T) { } } +func TestBulkSeedThenSync(t *testing.T) { + store := relayfile.NewStoreWithOptions(relayfile.StoreOptions{DisableWorkers: true}) + t.Cleanup(store.Close) + + api := httptest.NewServer(httpapi.NewServer(store)) + defer api.Close() + + workspaceID := "ws_mount_bulk_seed" + token := mustMountsyncTestJWT(t, "dev-secret", workspaceID, "MountSync", []string{"fs:read", "fs:write"}, time.Now().Add(time.Hour)) + + body, err := json.Marshal(map[string]any{ + "files": []map[string]any{ + { + "path": "/notion/Docs/A.md", + "contentType": "text/markdown", + "content": "# A", + }, + { + "path": "/notion/Docs/B.md", + "contentType": "text/markdown", + "content": "# B", + }, + }, + }) + if err != nil { + t.Fatalf("marshal bulk seed body failed: %v", err) + } + + req, err := http.NewRequest(http.MethodPost, api.URL+"/v1/workspaces/"+workspaceID+"/fs/bulk", bytes.NewReader(body)) + if err != nil { + t.Fatalf("new bulk request failed: %v", err) + } + req.Header.Set("Authorization", "Bearer "+token) + req.Header.Set("X-Correlation-Id", "corr_mount_bulk_seed") + req.Header.Set("Content-Type", "application/json") + + resp, err := api.Client().Do(req) + if err != nil { + t.Fatalf("bulk seed request failed: %v", err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusAccepted { + payload, _ := io.ReadAll(resp.Body) + t.Fatalf("expected 202 from bulk seed, got %d (%s)", resp.StatusCode, string(payload)) + } + + localDir := t.TempDir() + client := NewHTTPClient(api.URL, token, api.Client()) + syncer, err := NewSyncer(client, SyncerOptions{ + WorkspaceID: workspaceID, + RemoteRoot: "/notion", + LocalRoot: localDir, + }) + if err != nil { + t.Fatalf("new syncer failed: %v", err) + } + + if err := syncer.SyncOnce(context.Background()); err != nil { + t.Fatalf("sync once failed: %v", err) + } + + assertLocalFileContent(t, filepath.Join(localDir, "Docs", "A.md"), "# A") + assertLocalFileContent(t, filepath.Join(localDir, "Docs", "B.md"), "# B") +} + func TestRemoteToLocalAndLocalToRemotePath(t *testing.T) { localRoot := filepath.Join("tmp", "mirror") localPath, err := remoteToLocalPath(localRoot, "/notion", "/notion/Folder/File.md") @@ -531,6 +608,49 @@ func (c *fakeClient) DeleteFile(ctx context.Context, workspaceID, path, baseRevi return nil } +func assertLocalFileContent(t *testing.T, path, want string) { + t.Helper() + data, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read local file %s failed: %v", path, err) + } + if string(data) != want { + t.Fatalf("expected %s to contain %q, got %q", path, want, string(data)) + } +} + +func mustMountsyncTestJWT(t *testing.T, secret, workspaceID, agentName string, scopes []string, exp time.Time) string { + t.Helper() + + headerBytes, err := json.Marshal(map[string]any{ + "alg": "HS256", + "typ": "JWT", + }) + if err != nil { + t.Fatalf("marshal jwt header: %v", err) + } + payloadBytes, err := json.Marshal(map[string]any{ + "workspace_id": workspaceID, + "agent_name": agentName, + "scopes": scopes, + "exp": exp.Unix(), + "aud": "relayfile", + }) + if err != nil { + t.Fatalf("marshal jwt payload: %v", err) + } + + h := base64.RawURLEncoding.EncodeToString(headerBytes) + p := base64.RawURLEncoding.EncodeToString(payloadBytes) + signingInput := h + "." + p + + mac := hmac.New(sha256.New, []byte(secret)) + _, _ = mac.Write([]byte(signingInput)) + signature := base64.RawURLEncoding.EncodeToString(mac.Sum(nil)) + + return signingInput + "." + signature +} + func (c *fakeClient) appendEvent(eventType, path, revision string) { c.eventCounter++ c.events = append(c.events, FilesystemEvent{ diff --git a/internal/relayfile/store.go b/internal/relayfile/store.go index 242ea3e6..0ac50990 100644 --- a/internal/relayfile/store.go +++ b/internal/relayfile/store.go @@ -2,6 +2,7 @@ package relayfile import ( "context" + "encoding/base64" "encoding/json" "errors" "fmt" @@ -72,6 +73,7 @@ type File struct { Revision string `json:"revision"` ContentType string `json:"contentType"` Content string `json:"content"` + Encoding string `json:"encoding,omitempty"` Provider string `json:"provider,omitempty"` ProviderObjectID string `json:"providerObjectId,omitempty"` LastEditedAt string `json:"lastEditedAt,omitempty"` @@ -100,10 +102,24 @@ type WriteRequest struct { IfMatch string ContentType string Content string + Encoding string Semantics FileSemantics CorrelationID string } +type BulkWriteFile struct { + Path string `json:"path"` + ContentType string `json:"contentType"` + Content string `json:"content"` + Encoding string `json:"encoding"` +} + +type BulkWriteError struct { + Path string `json:"path"` + Code string `json:"code"` + Message string `json:"message"` +} + type DeleteRequest struct { WorkspaceID string Path string @@ -345,6 +361,7 @@ type WebhookEnvelopeRequest struct { type Store struct { mu sync.RWMutex queueMu sync.Mutex + subscriberMu sync.RWMutex workspaces map[string]*workspaceState revCounter uint64 opCounter uint64 @@ -379,6 +396,8 @@ type Store struct { providerMaxConcurrency int providerSemMu sync.Mutex providerSemaphores map[string]chan struct{} + subscribers map[string]map[uint64]chan<- Event + subscriberCounter uint64 closed chan struct{} queueCtx context.Context queueCancel context.CancelFunc @@ -773,6 +792,7 @@ func NewStoreWithOptions(opts StoreOptions) *Store { maxStoredEnvelopes: maxStoredEnvelopes, providerMaxConcurrency: providerMaxConcurrency, providerSemaphores: map[string]chan struct{}{}, + subscribers: map[string]map[uint64]chan<- Event{}, closed: make(chan struct{}), queueCtx: queueCtx, queueCancel: queueCancel, @@ -906,6 +926,9 @@ func (s *Store) seedQueuedIndexesFromQueues() { func (s *Store) Close() { s.closeOnce.Do(func() { close(s.closed) + s.subscriberMu.Lock() + s.subscribers = map[string]map[uint64]chan<- Event{} + s.subscriberMu.Unlock() if s.queueCancel != nil { s.queueCancel() } @@ -1147,6 +1170,13 @@ func (s *Store) WriteFile(req WriteRequest) (WriteResult, error) { if req.WorkspaceID == "" || req.Path == "" { return WriteResult{}, ErrInvalidInput } + encoding, err := normalizeEncoding(req.Encoding) + if err != nil { + return WriteResult{}, err + } + if err := validateEncodedContent(req.Content, encoding); err != nil { + return WriteResult{}, err + } s.mu.Lock() ws := s.ensureWorkspaceLocked(req.WorkspaceID) @@ -1171,6 +1201,7 @@ func (s *Store) WriteFile(req WriteRequest) (WriteResult, error) { Revision: revision, ContentType: contentType, Content: req.Content, + Encoding: encoding, Provider: provider, LastEditedAt: now, Semantics: semantics, @@ -1195,6 +1226,7 @@ func (s *Store) WriteFile(req WriteRequest) (WriteResult, error) { existing.Revision = revision existing.ContentType = contentType existing.Content = req.Content + existing.Encoding = encoding existing.LastEditedAt = now if !isZeroSemantics(semantics) { existing.Semantics = semantics @@ -1208,6 +1240,113 @@ func (s *Store) WriteFile(req WriteRequest) (WriteResult, error) { return result, nil } +func (s *Store) BulkWrite(workspaceID string, files []BulkWriteFile) (int, []BulkWriteError) { + if strings.TrimSpace(workspaceID) == "" { + return 0, []BulkWriteError{{ + Code: "invalid_input", + Message: ErrInvalidInput.Error(), + }} + } + if len(files) == 0 { + return 0, nil + } + + type queuedTask struct { + task writebackTask + } + + s.mu.Lock() + ws := s.ensureWorkspaceLocked(workspaceID) + now := time.Now().UTC() + written := 0 + errorsOut := make([]BulkWriteError, 0) + tasks := make([]queuedTask, 0, len(files)) + + for _, input := range files { + path := normalizePath(input.Path) + if strings.TrimSpace(input.Path) == "" { + errorsOut = append(errorsOut, BulkWriteError{ + Path: input.Path, + Code: "invalid_path", + Message: "missing path", + }) + continue + } + encoding, err := normalizeEncoding(input.Encoding) + if err != nil { + errorsOut = append(errorsOut, BulkWriteError{ + Path: path, + Code: "invalid_encoding", + Message: err.Error(), + }) + continue + } + if err := validateEncodedContent(input.Content, encoding); err != nil { + errorsOut = append(errorsOut, BulkWriteError{ + Path: path, + Code: "invalid_content", + Message: err.Error(), + }) + continue + } + contentType := strings.TrimSpace(input.ContentType) + if contentType == "" { + contentType = "text/markdown" + } + _, existed := ws.Files[path] + revision := s.nextRevisionLocked() + file := ws.Files[path] + file.Path = path + file.Revision = revision + file.ContentType = contentType + file.Content = input.Content + file.Encoding = encoding + if file.Provider == "" { + file.Provider = inferProviderFromPath(path) + } + file.LastEditedAt = now.Format(time.RFC3339Nano) + ws.Files[path] = file + + eventType := "file.created" + if existed { + eventType = "file.updated" + } + result, task := s.recordWriteLocked(ws, path, revision, eventType, file.Provider, "") + _ = result + tasks = append(tasks, queuedTask{task: task}) + written++ + } + + _ = s.saveLocked() + s.mu.Unlock() + + for _, queued := range tasks { + s.enqueueWriteback(queued.task) + } + return written, errorsOut +} + +func (s *Store) ExportWorkspace(workspaceID string) ([]File, error) { + if strings.TrimSpace(workspaceID) == "" { + return nil, ErrInvalidInput + } + s.mu.RLock() + defer s.mu.RUnlock() + + ws, ok := s.workspaces[workspaceID] + if !ok { + return []File{}, nil + } + files := make([]File, 0, len(ws.Files)) + for _, file := range ws.Files { + files = append(files, file) + } + sort.Slice(files, func(i, j int) bool { + return files[i].Path < files[j].Path + }) + return files, nil +} + func (s *Store) DeleteFile(req DeleteRequest) (WriteResult, error) { if req.IfMatch == "" { return WriteResult{}, ErrMissingPrecondition @@ -1295,6 +1434,61 @@ func (s *Store) GetEvents(workspaceID, provider, cursor string, limit int) (Even return EventFeed{Events: chunk, NextCursor: nextCursor}, nil } +func (s *Store) GetRecentEvents(workspaceID string, limit int) ([]Event, error) { + if workspaceID == "" { + return []Event{}, ErrInvalidInput + } + if limit <= 0 { + return []Event{}, nil + } + s.mu.RLock() + defer s.mu.RUnlock() + + ws, ok := s.workspaces[workspaceID] + if !ok || len(ws.Events) == 0 { + return []Event{}, nil + } + start := len(ws.Events) - limit + if start < 0 { + start = 0 + } + return append([]Event(nil), ws.Events[start:]...), nil +} + +func (s *Store) Subscribe(workspaceID string, ch chan<- Event) func() { + workspaceID = strings.TrimSpace(workspaceID) + if workspaceID == "" || ch == nil { + return func() {} + } + s.subscriberMu.Lock() + s.subscriberCounter++ + subID := s.subscriberCounter + if s.subscribers == nil { + s.subscribers = map[string]map[uint64]chan<- Event{} + } + if s.subscribers[workspaceID] == nil { + s.subscribers[workspaceID] = map[uint64]chan<- Event{} + } + s.subscribers[workspaceID][subID] = ch + s.subscriberMu.Unlock() + + var once sync.Once + return func() { + once.Do(func() { + s.subscriberMu.Lock() + defer s.subscriberMu.Unlock() + subscribers := s.subscribers[workspaceID] + if subscribers == nil { + return + } + delete(subscribers, subID) + if len(subscribers) == 0 { + delete(s.subscribers, workspaceID) + } + }) + } +} + func (s *Store) GetOperation(workspaceID, opID string) (OperationStatus, error) { s.mu.RLock() defer s.mu.RUnlock() @@ -1517,8 +1711,7 @@ func (s *Store) ListSyncStatuses(provider string) map[string]SyncStatus { if normalizedProvider != "" { providerNames = append(providerNames, normalizedProvider) } else { - providerSet := map[string]struct{}{ - } + providerSet := map[string]struct{}{} if workspaceAggregate != nil { for providerName := range workspaceAggregate.providers { if providerName == "" { @@ -2193,11 +2386,11 @@ func (s *Store) GetPendingWritebacks(workspaceID string) []map[string]any { for _, item := range items { if item.WorkspaceID == workspaceID { result = append(result, map[string]any{ - "id": item.OpID, - "workspaceId": item.WorkspaceID, - "path": item.Path, - "revision": item.Revision, - "correlationId": item.CorrelationID, + "id": item.OpID, + "workspaceId": item.WorkspaceID, + "path": item.Path, + "revision": item.Revision, + "correlationId": item.CorrelationID, }) } } @@ -2511,6 +2704,7 @@ func (s *Store) nextEventIDLocked() string { func (s *Store) recordWriteLocked(ws *workspaceState, path, revision, eventType, provider, correlationID string) (WriteResult, writebackTask) { if provider == "" { } + workspaceID := s.workspaceIDForStateLocked(ws) opID := s.nextOperationIDLocked() op := OperationStatus{ OpID: opID, @@ -2534,19 +2728,13 @@ func (s *Store) recordWriteLocked(ws *workspaceState, path, revision, eventType, CorrelationID: correlationID, Timestamp: time.Now().UTC().Format(time.RFC3339Nano), } - ws.Events = append(ws.Events, event) + s.appendWorkspaceEventLocked(workspaceID, ws, event) result := WriteResult{OpID: opID, Status: "queued", TargetRevision: revision} result.Writeback.Provider = provider result.Writeback.State = "pending" - task := writebackTask{WorkspaceID: "", OpID: opID, Path: path, Revision: revision, CorrelationID: correlationID} - for wsID, candidate := range s.workspaces { - if candidate == ws { - task.WorkspaceID = wsID - break - } - } + task := writebackTask{WorkspaceID: workspaceID, OpID: opID, Path: path, Revision: revision, CorrelationID: correlationID} return result, task } @@ -2735,7 +2923,7 @@ func (s *Store) processEnvelope(envelopeID string) { // If no provider-specific adapter found, we'll use generic pass-through later // This allows any provider to submit webhooks if s.shouldSuppressEnvelopeLocked(req, time.Now().UTC()) { - ws.Events = append(ws.Events, Event{ + s.appendWorkspaceEventLocked(req.WorkspaceID, ws, Event{ EventID: s.nextEventIDLocked(), Type: "sync.suppressed", Path: "/", @@ -2785,7 +2973,7 @@ func (s *Store) processEnvelope(envelopeID string) { errText := parseErr.Error() retryDelay := time.Duration(0) if attempt >= s.maxEnvelopeAttempts { - ws.Events = append(ws.Events, Event{ + s.appendWorkspaceEventLocked(req.WorkspaceID, ws, Event{ EventID: s.nextEventIDLocked(), Type: "sync.error", Path: "/", @@ -2825,7 +3013,7 @@ func (s *Store) processEnvelope(envelopeID string) { } for _, action := range actions { if s.isStaleProviderActionLocked(ws, req.Provider, action, req.ReceivedAt) { - ws.Events = append(ws.Events, Event{ + s.appendWorkspaceEventLocked(req.WorkspaceID, ws, Event{ EventID: s.nextEventIDLocked(), Type: "sync.stale", Path: normalizePath(action.Path), @@ -2844,7 +3032,7 @@ func (s *Store) processEnvelope(envelopeID string) { case ActionFileDelete: s.applyProviderDeleteLocked(ws, req.Provider, action, req.CorrelationID) default: - ws.Events = append(ws.Events, Event{ + s.appendWorkspaceEventLocked(req.WorkspaceID, ws, Event{ EventID: s.nextEventIDLocked(), Type: "sync.ignored", Path: "/", @@ -2966,7 +3154,7 @@ func (s *Store) processWriteback(task writebackTask) { op.NextAttemptAt = nil op.ProviderResult = map[string]any{"providerRevision": task.Revision} ws.Ops[task.OpID] = op - ws.Events = append(ws.Events, Event{ + s.appendWorkspaceEventLocked(task.WorkspaceID, ws, Event{ EventID: s.nextEventIDLocked(), Type: "writeback.succeeded", Path: task.Path, @@ -2987,7 +3175,7 @@ func (s *Store) processWriteback(task writebackTask) { op.Status = "dead_lettered" op.NextAttemptAt = nil ws.Ops[task.OpID] = op - ws.Events = append(ws.Events, Event{ + s.appendWorkspaceEventLocked(task.WorkspaceID, ws, Event{ EventID: s.nextEventIDLocked(), Type: "writeback.failed", Path: task.Path, @@ -3019,6 +3207,7 @@ func (s *Store) processWriteback(task writebackTask) { } func (s *Store) applyProviderUpsertLocked(ws *workspaceState, provider string, action ApplyAction, correlationID string) { + workspaceID := s.workspaceIDForStateLocked(ws) path := normalizePath(action.Path) objectID := strings.TrimSpace(action.ProviderObjectID) if path == "/" && objectID != "" { @@ -3043,7 +3232,7 @@ func (s *Store) applyProviderUpsertLocked(ws *workspaceState, provider string, a key := providerObjectKey(provider, objectID) if previousPath, ok := ws.ProviderIndex[key]; ok && previousPath != path { delete(ws.Files, previousPath) - ws.Events = append(ws.Events, Event{ + s.appendWorkspaceEventLocked(workspaceID, ws, Event{ EventID: s.nextEventIDLocked(), Type: "file.deleted", Path: previousPath, @@ -3067,6 +3256,7 @@ func (s *Store) applyProviderUpsertLocked(ws *workspaceState, provider string, a Revision: revision, ContentType: contentType, Content: content, + Encoding: "", Provider: provider, ProviderObjectID: objectID, LastEditedAt: now, @@ -3084,7 +3274,7 @@ func (s *Store) applyProviderUpsertLocked(ws *workspaceState, provider string, a // Keep update event for sync observability, but still revision-incremented. } } - ws.Events = append(ws.Events, Event{ + s.appendWorkspaceEventLocked(workspaceID, ws, Event{ EventID: s.nextEventIDLocked(), Type: fsEvent, Path: path, @@ -3097,6 +3287,7 @@ func (s *Store) applyProviderUpsertLocked(ws *workspaceState, provider string, a } func (s *Store) applyProviderDeleteLocked(ws *workspaceState, provider string, action ApplyAction, correlationID string) { + workspaceID := s.workspaceIDForStateLocked(ws) objectID := action.ProviderObjectID path := normalizePath(action.Path) now := time.Now().UTC().Format(time.RFC3339Nano) @@ -3116,7 +3307,7 @@ func (s *Store) applyProviderDeleteLocked(ws *workspaceState, provider string, a if objectID != "" { delete(ws.ProviderIndex, providerObjectKey(provider, objectID)) } - ws.Events = append(ws.Events, Event{ + s.appendWorkspaceEventLocked(workspaceID, ws, Event{ EventID: s.nextEventIDLocked(), Type: "file.deleted", Path: path, @@ -3493,6 +3684,31 @@ func normalizeFailureCode(errText string) string { } } +func normalizeEncoding(raw string) (string, error) { + encoding := strings.ToLower(strings.TrimSpace(raw)) + switch encoding { + case "", "utf-8", "utf8": + return "utf-8", nil + case "base64": + return "base64", nil + default: + return "", ErrInvalidInput + } +} + +func validateEncodedContent(content, encoding string) error { + if encoding != "base64" { + return nil + } + if _, err := base64.StdEncoding.DecodeString(content); err == nil { + return nil + } + if _, err := base64.RawStdEncoding.DecodeString(content); err == nil { + return nil + } + return ErrInvalidInput +} + func providerWatermarkKey(provider string, action ApplyAction) string { if action.ProviderObjectID != "" { return "object:" + providerObjectKey(provider, action.ProviderObjectID) @@ -3693,6 +3909,47 @@ func (s *Store) rebuildCoalesceIndexLocked() { } } +func (s *Store) workspaceIDForStateLocked(target *workspaceState) string { + for wsID, candidate := range s.workspaces { + if candidate == target { + return wsID + } + } + return "" +} + +func (s *Store) appendWorkspaceEventLocked(workspaceID string, ws *workspaceState, event Event) { + if ws == nil { + return + } + ws.Events = append(ws.Events, event) + s.publishEvent(workspaceID, event) +} + +func (s *Store) publishEvent(workspaceID string, event Event) { + workspaceID = strings.TrimSpace(workspaceID) + if workspaceID == "" { + return + } + s.subscriberMu.RLock() + subscribers := s.subscribers[workspaceID] + if len(subscribers) == 0 { + s.subscriberMu.RUnlock() + return + } + targets := make([]chan<- Event, 0, len(subscribers)) + for _, ch := range subscribers { + targets = append(targets, ch) + } + s.subscriberMu.RUnlock() + for _, ch := range targets { + select { + case ch <- event: + default: + } + } +} + func (s *Store) pruneProcessedEnvelopesLocked() { if s.maxStoredEnvelopes <= 0 { return diff --git a/internal/relayfile/store_test.go b/internal/relayfile/store_test.go index ca500ae9..7e0b6ed9 100644 --- a/internal/relayfile/store_test.go +++ b/internal/relayfile/store_test.go @@ -2,6 +2,7 @@ package relayfile import ( "context" + "encoding/base64" "encoding/json" "errors" "fmt" @@ -229,6 +230,64 @@ func (m *memoryStateBackend) Save(state *persistedState) error { return nil } +func TestStoreSubscribePublishesAndUnsubscribes(t *testing.T) { + store := NewStoreWithOptions(StoreOptions{DisableWorkers: true}) + t.Cleanup(store.Close) + + events := make(chan Event, 4) + unsubscribe := store.Subscribe("ws_subscribe", events) + + write, err := store.WriteFile(WriteRequest{ + WorkspaceID: "ws_subscribe", + Path: "/external/Doc.md", + IfMatch: "0", + ContentType: "text/markdown", + Content: "# hello", + CorrelationID: "corr_subscribe_1", + }) + if err != nil { + t.Fatalf("write failed: %v", err) + } + if write.TargetRevision == "" { + t.Fatalf("expected target revision") + } + + select { + case event := <-events: + if event.Type != "file.created" { + t.Fatalf("expected file.created event, got %s", event.Type) + } + if event.Path != "/external/Doc.md" { + t.Fatalf("unexpected event path: %s", event.Path) + } + case <-time.After(time.Second): + t.Fatalf("timed out waiting for subscribed event") + } + + unsubscribe() + + file, err := store.ReadFile("ws_subscribe", "/external/Doc.md") + if err != nil { + t.Fatalf("read failed: %v", err) + } + if _, err := store.WriteFile(WriteRequest{ + WorkspaceID: "ws_subscribe", + Path: "/external/Doc.md", + IfMatch: file.Revision, + ContentType: "text/markdown", + Content: "# updated", + CorrelationID: "corr_subscribe_2", + }); err != nil { + t.Fatalf("second write failed: %v", err) + } + + select { + case event := <-events: + t.Fatalf("unexpected event after unsubscribe: %+v", event) + case <-time.After(100 * time.Millisecond): + } +} + func TestStoreWriteReadConflictDeleteLifecycle(t *testing.T) { store := NewStore() t.Cleanup(store.Close) @@ -335,6 +394,222 @@ func TestStoreUsesCustomStateBackend(t *testing.T) { } } +func TestStoreBulkWriteAndExportWorkspace(t *testing.T) { + store := NewStoreWithOptions(StoreOptions{DisableWorkers: true}) + t.Cleanup(store.Close) + + written, errs := store.BulkWrite("ws_bulk", []BulkWriteFile{ + { + Path: "/external/Seed.md", + ContentType: "text/markdown", + Content: "# seed", + }, + { + Path: "/external/blob.bin", + ContentType: "application/octet-stream", + Content: base64.StdEncoding.EncodeToString([]byte{0x00, 0x01, 0x02, 0x03}), + Encoding: "base64", + }, + }) + if written != 2 { + t.Fatalf("expected 2 written files, got %d", written) + } + if len(errs) != 0 { + t.Fatalf("expected no bulk write errors, got %+v", errs) + } + + exported, err := store.ExportWorkspace("ws_bulk") + if err != nil { + t.Fatalf("export failed: %v", err) + } + if len(exported) != 2 { + t.Fatalf("expected 2 exported files, got %d", len(exported)) + } + + byPath := map[string]File{} + for _, file := range exported { + byPath[file.Path] = file + } + if byPath["/external/Seed.md"].Content != "# seed" { + t.Fatalf("unexpected exported content: %+v", byPath["/external/Seed.md"]) + } + if byPath["/external/blob.bin"].Encoding != "base64" { + t.Fatalf("expected base64 encoding for binary file, got %+v", byPath["/external/blob.bin"]) + } +} + +func TestBulkWrite(t *testing.T) { + store := NewStoreWithOptions(StoreOptions{DisableWorkers: true}) + t.Cleanup(store.Close) + + events := make(chan Event, 16) + unsubscribe := store.Subscribe("ws_bulk_write", events) + defer unsubscribe() + + files := make([]BulkWriteFile, 0, 10) + expectedPaths := make(map[string]struct{}, 10) + for i := 0; i < 10; i++ { + path := fmt.Sprintf("/external/File-%02d.md", i) + files = append(files, BulkWriteFile{ + Path: path, + ContentType: "text/markdown", + Content: fmt.Sprintf("# file %d", i), + }) + expectedPaths[path] = struct{}{} + } + + written, errs := store.BulkWrite("ws_bulk_write", files) + if written != 10 { + t.Fatalf("expected 10 written files, got %d", written) + } + if len(errs) != 0 { + t.Fatalf("expected no bulk write errors, got %+v", errs) + } + + tree, err := store.ListTree("ws_bulk_write", "/external", 1, "") + if err != nil { + t.Fatalf("list tree failed: %v", err) + } + if len(tree.Entries) != 10 { + t.Fatalf("expected 10 tree entries, got %d", len(tree.Entries)) + } + for _, entry := range tree.Entries { + if entry.Type != "file" { + t.Fatalf("expected file entry, got %+v", entry) + } + if _, ok := expectedPaths[entry.Path]; !ok { + t.Fatalf("unexpected tree path: %s", entry.Path) + } + } + + received := make(map[string]Event, 10) + deadline := time.After(time.Second) + for len(received) < 10 { + select { + case event := <-events: + received[event.Path] = event + case <-deadline: + t.Fatalf("timed out waiting for bulk write events; received %d", len(received)) + } + } + for path := range expectedPaths { + event, ok := received[path] + if !ok { + t.Fatalf("missing event for path %s", path) + } + if event.Type != "file.created" { + t.Fatalf("expected file.created for %s, got %s", path, event.Type) + } + } +} + +func TestBulkWriteOverwrite(t *testing.T) { + store := NewStoreWithOptions(StoreOptions{DisableWorkers: true}) + t.Cleanup(store.Close) + + initial := []BulkWriteFile{ + {Path: "/external/A.md", ContentType: "text/markdown", Content: "# v1"}, + {Path: "/external/B.md", ContentType: "text/markdown", Content: "# v1"}, + } + written, errs := store.BulkWrite("ws_bulk_overwrite", initial) + if written != len(initial) { + t.Fatalf("expected %d initial writes, got %d", len(initial), written) + } + if len(errs) != 0 { + t.Fatalf("expected no initial errors, got %+v", errs) + } + + updated := []BulkWriteFile{ + {Path: "/external/A.md", ContentType: "text/markdown", Content: "# v2"}, + {Path: "/external/B.md", ContentType: "text/markdown", Content: "# v3"}, + } + written, errs = store.BulkWrite("ws_bulk_overwrite", updated) + if written != len(updated) { + t.Fatalf("expected %d overwrite writes, got %d", len(updated), written) + } + if len(errs) != 0 { + t.Fatalf("expected no overwrite errors, got %+v", errs) + } + + for _, file := range updated { + got, err := store.ReadFile("ws_bulk_overwrite", file.Path) + if err != nil { + t.Fatalf("read %s failed: %v", file.Path, err) + } + if got.Content != file.Content { + t.Fatalf("expected latest content %q for %s, got %q", file.Content, file.Path, got.Content) + } + } +} + +func TestExportWorkspace(t *testing.T) { + store := NewStoreWithOptions(StoreOptions{DisableWorkers: true}) + t.Cleanup(store.Close) + + seed := []BulkWriteFile{ + {Path: "/external/A.md", ContentType: "text/markdown", Content: "# A"}, + {Path: "/external/B.md", ContentType: "text/plain", Content: "B"}, + {Path: "/external/C.json", ContentType: "application/json", Content: `{"ok":true}`}, + } + written, errs := store.BulkWrite("ws_export", seed) + if written != len(seed) { + t.Fatalf("expected %d written files, got %d", len(seed), written) + } + if len(errs) != 0 { + t.Fatalf("expected no bulk write errors, got %+v", errs) + } + + exported, err := store.ExportWorkspace("ws_export") + if err != nil { + t.Fatalf("export workspace failed: %v", err) + } + if len(exported) != len(seed) { + t.Fatalf("expected %d exported files, got %d", len(seed), len(exported)) + } + + for i, file := range exported { + if file.Path != seed[i].Path { + t.Fatalf("expected path %s at index %d, got %s", seed[i].Path, i, file.Path) + } + if file.Content != seed[i].Content { + t.Fatalf("expected content %q for %s, got %q", seed[i].Content, file.Path, file.Content) + } + if file.ContentType != seed[i].ContentType { + t.Fatalf("expected content type %q for %s, got %q", seed[i].ContentType, file.Path, file.ContentType) + } + } +} + +func TestBinaryEncoding(t *testing.T) { + store := NewStoreWithOptions(StoreOptions{DisableWorkers: true}) + t.Cleanup(store.Close) + + encoded := base64.StdEncoding.EncodeToString([]byte{0x00, 0x7f, 0xff, 0x10}) + written, errs := store.BulkWrite("ws_binary_encoding", []BulkWriteFile{{ + Path: "/external/blob.bin", + ContentType: "application/octet-stream", + Content: encoded, + Encoding: "base64", + }}) + if written != 1 { + t.Fatalf("expected 1 written file, got %d", written) + } + if len(errs) != 0 { + t.Fatalf("expected no bulk write errors, got %+v", errs) + } + + file, err := store.ReadFile("ws_binary_encoding", "/external/blob.bin") + if err != nil { + t.Fatalf("read binary file failed: %v", err) + } + if file.Content != encoded { + t.Fatalf("expected encoded content %q, got %q", encoded, file.Content) + } + if file.Encoding != "base64" { + t.Fatalf("expected base64 encoding, got %q", file.Encoding) + } +} + func TestStoreUsesCustomQueues(t *testing.T) { envelopeQueue := &countingEnvelopeQueue{ inner: NewInMemoryEnvelopeQueue(4), @@ -364,7 +639,7 @@ func TestStoreUsesCustomQueues(t *testing.T) { _, err = store.IngestEnvelope(WebhookEnvelopeRequest{ EnvelopeID: "env_custom_queue_1", WorkspaceID: "ws_custom_queue", - Provider: "external", + Provider: "external", DeliveryID: "delivery_custom_queue_1", ReceivedAt: time.Now().UTC().Format(time.RFC3339Nano), Payload: map[string]any{"event_type": "file.created", "providerObjectId": "obj_custom_queue_1", "path": "/external/Queue.md", "content": "# queue"}, @@ -443,7 +718,7 @@ func TestEnqueueEnvelopeDeduplicatesQueuedEnvelopeID(t *testing.T) { _, err := store.IngestEnvelope(WebhookEnvelopeRequest{ EnvelopeID: "env_queue_dedupe_1", WorkspaceID: "ws_queue_dedupe", - Provider: "external", + Provider: "external", DeliveryID: "delivery_queue_dedupe_1", ReceivedAt: time.Now().UTC().Format(time.RFC3339Nano), Payload: map[string]any{"event_type": "file.created", "providerObjectId": "obj_queue_dedupe_1", "path": "/external/Q.md", "content": "# q"}, @@ -1026,10 +1301,10 @@ func TestProviderUpsertWithoutPathUsesObjectIdentity(t *testing.T) { DeliveryID: "delivery_object_identity_1", ReceivedAt: firstReceivedAt, Payload: map[string]any{ - "event_type": "file.created", - "providerObjectId": "obj_identity_1", - "path": "/external/ObjectIdentity.md", - "content": "# initial", + "event_type": "file.created", + "providerObjectId": "obj_identity_1", + "path": "/external/ObjectIdentity.md", + "content": "# initial", }, CorrelationID: "corr_object_identity_1", }) @@ -1059,9 +1334,9 @@ func TestProviderUpsertWithoutPathUsesObjectIdentity(t *testing.T) { DeliveryID: "delivery_object_identity_2", ReceivedAt: secondReceivedAt, Payload: map[string]any{ - "event_type": "file.updated", - "providerObjectId": "obj_identity_1", - "content": "# updated", + "event_type": "file.updated", + "providerObjectId": "obj_identity_1", + "content": "# updated", }, CorrelationID: "corr_object_identity_2", }) @@ -1494,13 +1769,13 @@ func TestSuppressionMarkersPersistAcrossRestart(t *testing.T) { DeliveryID: "delivery_loop_persist_1", ReceivedAt: time.Now().UTC().Format(time.RFC3339Nano), Payload: map[string]any{ - "event_type": "file.updated", - "providerObjectId": "obj_loop_persist_1", - "path": "/external/LoopPersist.md", - "content": "# echo", - "origin": "relayfile", - "opId": write.OpID, - "correlationId": "corr_loop_persist_1", + "event_type": "file.updated", + "providerObjectId": "obj_loop_persist_1", + "path": "/external/LoopPersist.md", + "content": "# echo", + "origin": "relayfile", + "opId": write.OpID, + "correlationId": "corr_loop_persist_1", }, CorrelationID: "corr_loop_persist_2", }) @@ -1533,7 +1808,7 @@ func TestGetSyncStatusMarksProviderLaggingFromPendingEnvelope(t *testing.T) { _, err := store.IngestEnvelope(WebhookEnvelopeRequest{ EnvelopeID: "env_sync_lag_1", WorkspaceID: "ws_sync_lag", - Provider: "external", + Provider: "external", DeliveryID: "delivery_sync_lag_1", ReceivedAt: time.Now().UTC().Add(-2 * time.Minute).Format(time.RFC3339Nano), Payload: map[string]any{"type": "sync"}, @@ -1756,7 +2031,7 @@ func TestListSyncStatusesAggregatesWorkspacesAndAppliesProviderFilter(t *testing if _, err := store.IngestEnvelope(WebhookEnvelopeRequest{ EnvelopeID: "env_sync_list_1", WorkspaceID: "ws_sync_list_1", - Provider: "external", + Provider: "external", DeliveryID: "delivery_sync_list_1", ReceivedAt: time.Now().UTC().Add(-2 * time.Minute).Format(time.RFC3339Nano), Payload: map[string]any{"type": "sync"}, @@ -2241,7 +2516,7 @@ func TestStoreIngestEnvelopeAndReplayOp(t *testing.T) { resp, err := store.IngestEnvelope(WebhookEnvelopeRequest{ EnvelopeID: "env_1", WorkspaceID: "ws_env", - Provider: "external", + Provider: "external", DeliveryID: "delivery_1", ReceivedAt: receivedAt, Payload: map[string]any{"type": "sync"}, @@ -2258,7 +2533,7 @@ func TestStoreIngestEnvelopeAndReplayOp(t *testing.T) { dup, err := store.IngestEnvelope(WebhookEnvelopeRequest{ EnvelopeID: "env_2", WorkspaceID: "ws_env", - Provider: "external", + Provider: "external", DeliveryID: "delivery_1", ReceivedAt: receivedAt, Payload: map[string]any{"type": "sync"}, @@ -2372,10 +2647,10 @@ func TestReplayEnvelopeReprocessesProcessedEnvelope(t *testing.T) { DeliveryID: "delivery_replay_1", ReceivedAt: receivedAt, Payload: map[string]any{ - "event_type": "file.updated", + "event_type": "file.updated", "providerObjectId": "obj_replay_1", - "path": "/external/Replay.md", - "content": "# replay", + "path": "/external/Replay.md", + "content": "# replay", }, CorrelationID: "corr_replay_1", }) @@ -2862,7 +3137,7 @@ func TestReplayEnvelopeForWorkspaceRejectsWorkspaceMismatch(t *testing.T) { _, err := store.IngestEnvelope(WebhookEnvelopeRequest{ EnvelopeID: "env_ws_scope_1", WorkspaceID: "ws_scope_a", - Provider: "external", + Provider: "external", DeliveryID: "delivery_ws_scope_1", ReceivedAt: time.Now().UTC().Format(time.RFC3339), Payload: map[string]any{"type": "sync"}, @@ -2885,7 +3160,7 @@ func TestReplayEnvelopeForWorkspaceRejectsNonDeadLetterEnvelope(t *testing.T) { _, err := store.IngestEnvelope(WebhookEnvelopeRequest{ EnvelopeID: "env_ws_scope_2", WorkspaceID: "ws_scope_a", - Provider: "external", + Provider: "external", DeliveryID: "delivery_ws_scope_2", ReceivedAt: time.Now().UTC().Format(time.RFC3339), Payload: map[string]any{"type": "sync"}, @@ -2913,11 +3188,11 @@ func TestEnvelopePipelineAppliesGenericUpsertMoveDelete(t *testing.T) { DeliveryID: "delivery_pipe_1", ReceivedAt: receivedAt, Payload: map[string]any{ - "event_type": "file.updated", - "path": "/external/Engineering/FromWebhook.md", + "event_type": "file.updated", + "path": "/external/Engineering/FromWebhook.md", "providerObjectId": "obj_obj_1", - "content": "# from webhook", - "contentType": "text/markdown", + "content": "# from webhook", + "contentType": "text/markdown", }, CorrelationID: "corr_pipe_1", }) @@ -2933,11 +3208,11 @@ func TestEnvelopePipelineAppliesGenericUpsertMoveDelete(t *testing.T) { DeliveryID: "delivery_pipe_2", ReceivedAt: receivedAt, Payload: map[string]any{ - "event_type": "file.updated", - "path": "/external/Engineering/Moved.md", + "event_type": "file.updated", + "path": "/external/Engineering/Moved.md", "providerObjectId": "obj_obj_1", - "content": "# moved", - "contentType": "text/markdown", + "content": "# moved", + "contentType": "text/markdown", }, CorrelationID: "corr_pipe_2", }) @@ -2954,8 +3229,8 @@ func TestEnvelopePipelineAppliesGenericUpsertMoveDelete(t *testing.T) { DeliveryID: "delivery_pipe_3", ReceivedAt: receivedAt, Payload: map[string]any{ - "event_type": "file.deleted", - "path": "/external/Engineering/Moved.md", + "event_type": "file.deleted", + "path": "/external/Engineering/Moved.md", "providerObjectId": "obj_obj_1", }, CorrelationID: "corr_pipe_3", @@ -3313,10 +3588,10 @@ func TestWebhookBurstCoalescingKeepsSinglePendingEnvelope(t *testing.T) { DeliveryID: fmt.Sprintf("delivery_burst_%d", i), ReceivedAt: receivedAt.Add(time.Duration(i) * 10 * time.Millisecond).Format(time.RFC3339Nano), Payload: map[string]any{ - "event_type": "file.updated", + "event_type": "file.updated", "providerObjectId": "obj_burst_1", - "path": "/external/Burst.md", - "content": fmt.Sprintf("# burst %d", i), + "path": "/external/Burst.md", + "content": fmt.Sprintf("# burst %d", i), }, CorrelationID: fmt.Sprintf("corr_burst_%d", i), }) @@ -3357,10 +3632,10 @@ func TestRebuildCoalesceIndexPrefersLatestPendingEnvelope(t *testing.T) { DeliveryID: "delivery_rebuild_old", ReceivedAt: "2026-01-01T10:00:00Z", Payload: map[string]any{ - "event_type": "file.updated", + "event_type": "file.updated", "providerObjectId": "obj_rebuild_1", - "path": "/external/Rebuild.md", - "content": "# old", + "path": "/external/Rebuild.md", + "content": "# old", }, } newReq := WebhookEnvelopeRequest{ @@ -3370,10 +3645,10 @@ func TestRebuildCoalesceIndexPrefersLatestPendingEnvelope(t *testing.T) { DeliveryID: "delivery_rebuild_new", ReceivedAt: "2026-01-01T10:00:01Z", Payload: map[string]any{ - "event_type": "file.updated", + "event_type": "file.updated", "providerObjectId": "obj_rebuild_1", - "path": "/external/Rebuild.md", - "content": "# new", + "path": "/external/Rebuild.md", + "content": "# new", }, } coalesceKey := coalesceObjectKey(oldReq) @@ -3408,7 +3683,7 @@ func TestProcessedEnvelopeRetentionPrunesOldestProcessed(t *testing.T) { _, err := store.IngestEnvelope(WebhookEnvelopeRequest{ EnvelopeID: fmt.Sprintf("env_retention_%d", i), WorkspaceID: "ws_retention", - Provider: "external", + Provider: "external", DeliveryID: fmt.Sprintf("delivery_retention_%d", i), ReceivedAt: time.Now().UTC().Add(time.Duration(i) * time.Second).Format(time.RFC3339Nano), Payload: map[string]any{"type": "sync"}, @@ -3477,7 +3752,7 @@ func TestEnvelopeRetentionDoesNotPruneDeadLetters(t *testing.T) { _, err = store.IngestEnvelope(WebhookEnvelopeRequest{ EnvelopeID: "env_retention_ok_1", WorkspaceID: "ws_retention_dead", - Provider: "external", + Provider: "external", DeliveryID: "delivery_retention_ok_1", ReceivedAt: time.Now().UTC().Add(time.Second).Format(time.RFC3339Nano), Payload: map[string]any{"type": "sync"}, @@ -3512,7 +3787,7 @@ func TestEnvelopeQueueBackpressureAndIngressStatus(t *testing.T) { _, err := store.IngestEnvelope(WebhookEnvelopeRequest{ EnvelopeID: "env_backpressure_1", WorkspaceID: "ws_backpressure", - Provider: "external", + Provider: "external", DeliveryID: "delivery_backpressure_1", ReceivedAt: receivedAt, Payload: map[string]any{"type": "sync"}, @@ -3525,7 +3800,7 @@ func TestEnvelopeQueueBackpressureAndIngressStatus(t *testing.T) { _, err = store.IngestEnvelope(WebhookEnvelopeRequest{ EnvelopeID: "env_backpressure_2", WorkspaceID: "ws_backpressure", - Provider: "external", + Provider: "external", DeliveryID: "delivery_backpressure_2", ReceivedAt: receivedAt, Payload: map[string]any{"type": "sync"}, @@ -3571,7 +3846,7 @@ func TestGetIngressStatusForProviderFiltersWorkspaceMetrics(t *testing.T) { _, err := store.IngestEnvelope(WebhookEnvelopeRequest{ EnvelopeID: "env_ingress_provider_1", WorkspaceID: "ws_ingress_provider", - Provider: "external", + Provider: "external", DeliveryID: "delivery_ingress_provider_1", ReceivedAt: receivedAt, Payload: map[string]any{"type": "sync"}, @@ -3628,7 +3903,7 @@ func TestListIngressStatusesAggregatesWorkspaces(t *testing.T) { _, err := store.IngestEnvelope(WebhookEnvelopeRequest{ EnvelopeID: "env_ingress_list_1", WorkspaceID: "ws_ingress_list_1", - Provider: "external", + Provider: "external", DeliveryID: "delivery_ingress_list_1", ReceivedAt: receivedAt, Payload: map[string]any{"type": "sync"}, @@ -3681,7 +3956,7 @@ func TestEnvelopeDedupedIngressStatusCounter(t *testing.T) { _, err := store.IngestEnvelope(WebhookEnvelopeRequest{ EnvelopeID: "env_dedupe_1", WorkspaceID: "ws_dedupe_status", - Provider: "external", + Provider: "external", DeliveryID: "delivery_dedupe_1", ReceivedAt: receivedAt, Payload: map[string]any{"type": "sync"}, @@ -3694,7 +3969,7 @@ func TestEnvelopeDedupedIngressStatusCounter(t *testing.T) { _, err = store.IngestEnvelope(WebhookEnvelopeRequest{ EnvelopeID: "env_dedupe_2", WorkspaceID: "ws_dedupe_status", - Provider: "external", + Provider: "external", DeliveryID: "delivery_dedupe_1", ReceivedAt: receivedAt, Payload: map[string]any{"type": "sync"}, @@ -3739,7 +4014,7 @@ func TestIngressStatusReportsOldestPendingAge(t *testing.T) { _, err := store.IngestEnvelope(WebhookEnvelopeRequest{ EnvelopeID: "env_age_1", WorkspaceID: "ws_age", - Provider: "external", + Provider: "external", DeliveryID: "delivery_age_1", ReceivedAt: time.Now().UTC().Add(-2 * time.Minute).Format(time.RFC3339), Payload: map[string]any{"type": "sync"}, @@ -3951,10 +4226,10 @@ func TestPendingEnvelopeIsRecoveredAfterRestart(t *testing.T) { DeliveryID: "delivery_recovery_1", ReceivedAt: receivedAt, Payload: map[string]any{ - "event_type": "file.updated", + "event_type": "file.updated", "providerObjectId": "obj_recovery_1", - "path": "/external/Recovered.md", - "content": "# recovered", + "path": "/external/Recovered.md", + "content": "# recovered", }, CorrelationID: "corr_recovery_1", }) @@ -4072,13 +4347,13 @@ func TestLoopSuppressionSuppressesProviderEchoWithinWindow(t *testing.T) { DeliveryID: "delivery_loop_echo_1", ReceivedAt: time.Now().UTC().Format(time.RFC3339Nano), Payload: map[string]any{ - "event_type": "file.updated", - "providerObjectId": "obj_loop_1", - "path": "/external/Loop.md", - "content": "# echoed", - "origin": "relayfile", - "opId": write.OpID, - "correlationId": "corr_loop_1", + "event_type": "file.updated", + "providerObjectId": "obj_loop_1", + "path": "/external/Loop.md", + "content": "# echoed", + "origin": "relayfile", + "opId": write.OpID, + "correlationId": "corr_loop_1", }, CorrelationID: "corr_loop_echo_1", }) @@ -4153,13 +4428,13 @@ func TestLoopSuppressionWindowExpiryAllowsProviderApply(t *testing.T) { DeliveryID: "delivery_loop_expiry_1", ReceivedAt: time.Now().UTC().Format(time.RFC3339Nano), Payload: map[string]any{ - "event_type": "file.updated", - "providerObjectId": "obj_loop_expiry_1", - "path": "/external/LoopExpiry.md", - "content": "# echoed", - "origin": "relayfile", - "opId": write.OpID, - "correlationId": "corr_loop_expiry_1", + "event_type": "file.updated", + "providerObjectId": "obj_loop_expiry_1", + "path": "/external/LoopExpiry.md", + "content": "# echoed", + "origin": "relayfile", + "opId": write.OpID, + "correlationId": "corr_loop_expiry_1", }, CorrelationID: "corr_loop_expiry_echo_1", }) diff --git a/openapi/relayfile-v1.openapi.yaml b/openapi/relayfile-v1.openapi.yaml index 48720a67..33b0b44a 100644 --- a/openapi/relayfile-v1.openapi.yaml +++ b/openapi/relayfile-v1.openapi.yaml @@ -26,6 +26,27 @@ tags: security: - BearerAuth: [] paths: + /health: + get: + tags: [Admin] + operationId: healthCheck + summary: Health check endpoint + description: Returns a simple status indicator for load balancers and monitoring. + security: [] + responses: + '200': + description: Service is healthy + content: + application/json: + schema: + type: object + additionalProperties: false + required: [status] + properties: + status: + type: string + enum: [ok] + /v1/workspaces/{workspaceId}/fs/tree: get: tags: [Filesystem] @@ -83,6 +104,123 @@ paths: '500': $ref: '#/components/responses/InternalError' + /v1/workspaces/{workspaceId}/fs/ws: + get: + tags: [Events] + operationId: fileEventsWebSocket + summary: WebSocket stream of real-time filesystem change events + description: | + Opens a WebSocket connection that streams filesystem events for the + workspace in real time. Authentication is via a `token` query parameter + containing a valid JWT (same format as the Bearer token). + + The server sends JSON messages with `type`, `path`, `revision`, and + `timestamp` fields. The client may send `{"type":"ping"}` to keep the + connection alive; the server responds with `{"type":"pong"}`. + security: + - BearerAuth: [fs:read] + parameters: + - $ref: '#/components/parameters/WorkspaceId' + - name: token + in: query + required: true + description: JWT token for authentication (same as Bearer token value) + schema: + type: string + responses: + '101': + description: WebSocket upgrade successful + '401': + $ref: '#/components/responses/Unauthorized' + '429': + $ref: '#/components/responses/RateLimited' + + /v1/workspaces/{workspaceId}/fs/bulk: + post: + tags: [Filesystem] + operationId: bulkWriteFiles + summary: Write multiple files in a single request + description: | + Writes multiple files atomically within a workspace. Each file is + individually permission-checked; files that fail permission checks are + reported in the `errors` array while permitted files are written. + security: + - BearerAuth: [fs:write] + parameters: + - $ref: '#/components/parameters/WorkspaceId' + - $ref: '#/components/parameters/CorrelationId' + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/BulkWriteRequest' + responses: + '202': + description: Bulk write accepted + content: + application/json: + schema: + $ref: '#/components/schemas/BulkWriteResponse' + '400': + $ref: '#/components/responses/BadRequest' + '401': + $ref: '#/components/responses/Unauthorized' + '403': + $ref: '#/components/responses/Forbidden' + '429': + $ref: '#/components/responses/RateLimited' + '500': + $ref: '#/components/responses/InternalError' + + /v1/workspaces/{workspaceId}/fs/export: + get: + tags: [Filesystem] + operationId: exportWorkspace + summary: Export all visible files in a workspace + description: | + Exports all files visible to the caller under effective permission rules. + Supports multiple output formats: JSON (default), tar (gzipped tarball), + or patch (unified diff format). + security: + - BearerAuth: [fs:read] + parameters: + - $ref: '#/components/parameters/WorkspaceId' + - $ref: '#/components/parameters/CorrelationId' + - name: format + in: query + required: false + schema: + type: string + enum: [json, tar, patch] + default: json + responses: + '200': + description: Export data in requested format + content: + application/json: + schema: + type: array + items: + $ref: '#/components/schemas/FileReadResponse' + application/gzip: + schema: + type: string + format: binary + text/plain: + schema: + type: string + '400': + $ref: '#/components/responses/BadRequest' + '401': + $ref: '#/components/responses/Unauthorized' + '403': + $ref: '#/components/responses/Forbidden' + '429': + $ref: '#/components/responses/RateLimited' + '500': + $ref: '#/components/responses/InternalError' + /v1/workspaces/{workspaceId}/fs/file: get: tags: [Filesystem] @@ -2301,6 +2439,63 @@ components: type: string description: Correlation ID for tracing + BulkWriteRequest: + type: object + additionalProperties: false + required: [files] + properties: + files: + type: array + minItems: 1 + items: + $ref: '#/components/schemas/BulkWriteFile' + + BulkWriteFile: + type: object + additionalProperties: false + required: [path, contentType, content] + properties: + path: + type: string + pattern: '^/.*' + contentType: + type: string + content: + type: string + encoding: + type: string + description: Content encoding, e.g. "base64" for binary data + + BulkWriteError: + type: object + additionalProperties: false + required: [path, code, message] + properties: + path: + type: string + code: + type: string + message: + type: string + + BulkWriteResponse: + type: object + additionalProperties: false + required: [written, errorCount, errors, correlationId] + properties: + written: + type: integer + minimum: 0 + errorCount: + type: integer + minimum: 0 + errors: + type: array + items: + $ref: '#/components/schemas/BulkWriteError' + correlationId: + type: string + ConflictErrorResponse: allOf: - $ref: '#/components/schemas/ErrorResponse' diff --git a/package-lock.json b/package-lock.json new file mode 100644 index 00000000..b62db861 --- /dev/null +++ b/package-lock.json @@ -0,0 +1,587 @@ +{ + "name": "relayfile", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "dependencies": { + "@agent-relay/sdk": "latest" + } + }, + "node_modules/@agent-relay/config": { + "version": "3.2.15", + "resolved": "https://registry.npmjs.org/@agent-relay/config/-/config-3.2.15.tgz", + "integrity": "sha512-oIgspBgO2T9ulauilHmYm5ATk30Jd1YyCGAy67CbELOznF27XuAKfaGHuMHjMIRMhj/PI7VG89DhYaGyHk7O4w==", + "dependencies": { + "zod": "^3.23.8", + "zod-to-json-schema": "^3.23.1" + } + }, + "node_modules/@agent-relay/sdk": { + "version": "3.2.15", + "resolved": "https://registry.npmjs.org/@agent-relay/sdk/-/sdk-3.2.15.tgz", + "integrity": "sha512-JLw7gMeX5CiXOQnZBFnRXQaPgSTGdXZp2j6HvHSh0vuCERsFWF0fFpk86ybFaFVlkrpB3qmcLe5hnyUt6wj2CA==", + "dependencies": { + "@agent-relay/config": "3.2.15", + "@relaycast/sdk": "^1.0.0", + "@sinclair/typebox": "^0.34.48", + "chalk": "^4.1.2", + "listr2": "^10.2.1", + "ws": "^8.18.3", + "yaml": "^2.7.0" + }, + "peerDependencies": { + "@anthropic-ai/claude-agent-sdk": ">=0.1.0", + "@google/adk": ">=0.5.0", + "@langchain/langgraph": ">=1.2.0", + "@mariozechner/pi-coding-agent": ">=0.50.0", + "@openai/agents": ">=0.7.0", + "ai": ">=5.0.0", + "crewai": ">=1.0.0" + }, + "peerDependenciesMeta": { + "@anthropic-ai/claude-agent-sdk": { + "optional": true + }, + "@google/adk": { + "optional": true + }, + "@langchain/langgraph": { + "optional": true + }, + "@mariozechner/pi-coding-agent": { + "optional": true + }, + "@openai/agents": { + "optional": true + }, + "ai": { + "optional": true + }, + "crewai": { + "optional": true + } + } + }, + "node_modules/@relaycast/sdk": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/@relaycast/sdk/-/sdk-1.0.0.tgz", + "integrity": "sha512-s01xslec5xyDXxxkVDTJyHpRhzqlXC2gVoglvhu+HK1h5JeOKq13AFlhe2MszkxjJAQ0HJ36MItWXuGogbRdOg==", + "dependencies": { + "@relaycast/types": "1.0.0", + "zod": "^4.3.6" + } + }, + "node_modules/@relaycast/sdk/node_modules/zod": { + "version": "4.3.6", + "resolved": "https://registry.npmjs.org/zod/-/zod-4.3.6.tgz", + "integrity": "sha512-rftlrkhHZOcjDwkGlnUtZZkvaPHCsDATp4pGpuOOMDaTdDDXF91wuVDJoWoPsKX/3YPQ5fHuF3STjcYyKr+Qhg==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/colinhacks" + } + }, + "node_modules/@relaycast/types": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/@relaycast/types/-/types-1.0.0.tgz", + "integrity": "sha512-f9DnZ91jro+NX3CypIPhTyBzGpmA2ZRE3zu/3aPAAe7JGVrHTcOq0Or9s/pzxSc49m5y6p7vXi9+TYgDeL/xWA==", + "dependencies": { + "zod": "^4.3.6" + } + }, + "node_modules/@relaycast/types/node_modules/zod": { + "version": "4.3.6", + "resolved": "https://registry.npmjs.org/zod/-/zod-4.3.6.tgz", + "integrity": "sha512-rftlrkhHZOcjDwkGlnUtZZkvaPHCsDATp4pGpuOOMDaTdDDXF91wuVDJoWoPsKX/3YPQ5fHuF3STjcYyKr+Qhg==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/colinhacks" + } + }, + "node_modules/@sinclair/typebox": { + "version": "0.34.48", + "resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.34.48.tgz", + "integrity": "sha512-kKJTNuK3AQOrgjjotVxMrCn1sUJwM76wMszfq1kdU4uYVJjvEWuFQ6HgvLt4Xz3fSmZlTOxJ/Ie13KnIcWQXFA==", + "license": "MIT" + }, + "node_modules/ansi-escapes": { + "version": "7.3.0", + "resolved": "https://registry.npmjs.org/ansi-escapes/-/ansi-escapes-7.3.0.tgz", + "integrity": "sha512-BvU8nYgGQBxcmMuEeUEmNTvrMVjJNSH7RgW24vXexN4Ven6qCvy4TntnvlnwnMLTVlcRQQdbRY8NKnaIoeWDNg==", + "license": "MIT", + "dependencies": { + "environment": "^1.0.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/ansi-regex": { + "version": "6.2.2", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.2.2.tgz", + "integrity": "sha512-Bq3SmSpyFHaWjPk8If9yc6svM8c56dB5BAtW4Qbw5jHTwwXXcTLoRMkpDJp6VL0XzlWaCHTXrkFURMYmD0sLqg==", + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/ansi-regex?sponsor=1" + } + }, + "node_modules/ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "license": "MIT", + "dependencies": { + "color-convert": "^2.0.1" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/chalk": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", + "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", + "license": "MIT", + "dependencies": { + "ansi-styles": "^4.1.0", + "supports-color": "^7.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/chalk?sponsor=1" + } + }, + "node_modules/cli-cursor": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/cli-cursor/-/cli-cursor-5.0.0.tgz", + "integrity": "sha512-aCj4O5wKyszjMmDT4tZj93kxyydN/K5zPWSCe6/0AV/AA1pqe5ZBIw0a2ZfPQV7lL5/yb5HsUreJ6UFAF1tEQw==", + "license": "MIT", + "dependencies": { + "restore-cursor": "^5.0.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/cli-truncate": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/cli-truncate/-/cli-truncate-5.2.0.tgz", + "integrity": "sha512-xRwvIOMGrfOAnM1JYtqQImuaNtDEv9v6oIYAs4LIHwTiKee8uwvIi363igssOC0O5U04i4AlENs79LQLu9tEMw==", + "license": "MIT", + "dependencies": { + "slice-ansi": "^8.0.0", + "string-width": "^8.2.0" + }, + "engines": { + "node": ">=20" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "license": "MIT", + "dependencies": { + "color-name": "~1.1.4" + }, + "engines": { + "node": ">=7.0.0" + } + }, + "node_modules/color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", + "license": "MIT" + }, + "node_modules/emoji-regex": { + "version": "10.6.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-10.6.0.tgz", + "integrity": "sha512-toUI84YS5YmxW219erniWD0CIVOo46xGKColeNQRgOzDorgBi1v4D71/OFzgD9GO2UGKIv1C3Sp8DAn0+j5w7A==", + "license": "MIT" + }, + "node_modules/environment": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/environment/-/environment-1.1.0.tgz", + "integrity": "sha512-xUtoPkMggbz0MPyPiIWr1Kp4aeWJjDZ6SMvURhimjdZgsRuDplF5/s9hcgGhyXMhs+6vpnuoiZ2kFiu3FMnS8Q==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/eventemitter3": { + "version": "5.0.4", + "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-5.0.4.tgz", + "integrity": "sha512-mlsTRyGaPBjPedk6Bvw+aqbsXDtoAyAzm5MO7JgU+yVRyMQ5O8bD4Kcci7BS85f93veegeCPkL8R4GLClnjLFw==", + "license": "MIT" + }, + "node_modules/get-east-asian-width": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/get-east-asian-width/-/get-east-asian-width-1.5.0.tgz", + "integrity": "sha512-CQ+bEO+Tva/qlmw24dCejulK5pMzVnUOFOijVogd3KQs07HnRIgp8TGipvCCRT06xeYEbpbgwaCxglFyiuIcmA==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/has-flag": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", + "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/is-fullwidth-code-point": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-5.1.0.tgz", + "integrity": "sha512-5XHYaSyiqADb4RnZ1Bdad6cPp8Toise4TzEjcOYDHZkTCbKgiUl7WTUCpNWHuxmDt91wnsZBc9xinNzopv3JMQ==", + "license": "MIT", + "dependencies": { + "get-east-asian-width": "^1.3.1" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/listr2": { + "version": "10.2.1", + "resolved": "https://registry.npmjs.org/listr2/-/listr2-10.2.1.tgz", + "integrity": "sha512-7I5knELsJKTUjXG+A6BkKAiGkW1i25fNa/xlUl9hFtk15WbE9jndA89xu5FzQKrY5llajE1hfZZFMILXkDHk/Q==", + "license": "MIT", + "dependencies": { + "cli-truncate": "^5.2.0", + "eventemitter3": "^5.0.4", + "log-update": "^6.1.0", + "rfdc": "^1.4.1", + "wrap-ansi": "^10.0.0" + }, + "engines": { + "node": ">=22.13.0" + } + }, + "node_modules/log-update": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/log-update/-/log-update-6.1.0.tgz", + "integrity": "sha512-9ie8ItPR6tjY5uYJh8K/Zrv/RMZ5VOlOWvtZdEHYSTFKZfIBPQa9tOAEeAWhd+AnIneLJ22w5fjOYtoutpWq5w==", + "license": "MIT", + "dependencies": { + "ansi-escapes": "^7.0.0", + "cli-cursor": "^5.0.0", + "slice-ansi": "^7.1.0", + "strip-ansi": "^7.1.0", + "wrap-ansi": "^9.0.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/log-update/node_modules/ansi-styles": { + "version": "6.2.3", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-6.2.3.tgz", + "integrity": "sha512-4Dj6M28JB+oAH8kFkTLUo+a2jwOFkuqb3yucU0CANcRRUbxS0cP0nZYCGjcc3BNXwRIsUVmDGgzawme7zvJHvg==", + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/log-update/node_modules/slice-ansi": { + "version": "7.1.2", + "resolved": "https://registry.npmjs.org/slice-ansi/-/slice-ansi-7.1.2.tgz", + "integrity": "sha512-iOBWFgUX7caIZiuutICxVgX1SdxwAVFFKwt1EvMYYec/NWO5meOJ6K5uQxhrYBdQJne4KxiqZc+KptFOWFSI9w==", + "license": "MIT", + "dependencies": { + "ansi-styles": "^6.2.1", + "is-fullwidth-code-point": "^5.0.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/chalk/slice-ansi?sponsor=1" + } + }, + "node_modules/log-update/node_modules/string-width": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-7.2.0.tgz", + "integrity": "sha512-tsaTIkKW9b4N+AEj+SVA+WhJzV7/zMhcSu78mLKWSk7cXMOSHsBKFWUs0fWwq8QyK3MgJBQRX6Gbi4kYbdvGkQ==", + "license": "MIT", + "dependencies": { + "emoji-regex": "^10.3.0", + "get-east-asian-width": "^1.0.0", + "strip-ansi": "^7.1.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/log-update/node_modules/wrap-ansi": { + "version": "9.0.2", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-9.0.2.tgz", + "integrity": "sha512-42AtmgqjV+X1VpdOfyTGOYRi0/zsoLqtXQckTmqTeybT+BDIbM/Guxo7x3pE2vtpr1ok6xRqM9OpBe+Jyoqyww==", + "license": "MIT", + "dependencies": { + "ansi-styles": "^6.2.1", + "string-width": "^7.0.0", + "strip-ansi": "^7.1.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, + "node_modules/mimic-function": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/mimic-function/-/mimic-function-5.0.1.tgz", + "integrity": "sha512-VP79XUPxV2CigYP3jWwAUFSku2aKqBH7uTAapFWCBqutsbmDo96KY5o8uh6U+/YSIn5OxJnXp73beVkpqMIGhA==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/onetime": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/onetime/-/onetime-7.0.0.tgz", + "integrity": "sha512-VXJjc87FScF88uafS3JllDgvAm+c/Slfz06lorj2uAY34rlUu0Nt+v8wreiImcrgAjjIHp1rXpTDlLOGw29WwQ==", + "license": "MIT", + "dependencies": { + "mimic-function": "^5.0.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/restore-cursor": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/restore-cursor/-/restore-cursor-5.1.0.tgz", + "integrity": "sha512-oMA2dcrw6u0YfxJQXm342bFKX/E4sG9rbTzO9ptUcR/e8A33cHuvStiYOwH7fszkZlZ1z/ta9AAoPk2F4qIOHA==", + "license": "MIT", + "dependencies": { + "onetime": "^7.0.0", + "signal-exit": "^4.1.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/rfdc": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/rfdc/-/rfdc-1.4.1.tgz", + "integrity": "sha512-q1b3N5QkRUWUl7iyylaaj3kOpIT0N2i9MqIEQXP73GVsN9cw3fdx8X63cEmWhJGi2PPCF23Ijp7ktmd39rawIA==", + "license": "MIT" + }, + "node_modules/signal-exit": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-4.1.0.tgz", + "integrity": "sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==", + "license": "ISC", + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/slice-ansi": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/slice-ansi/-/slice-ansi-8.0.0.tgz", + "integrity": "sha512-stxByr12oeeOyY2BlviTNQlYV5xOj47GirPr4yA1hE9JCtxfQN0+tVbkxwCtYDQWhEKWFHsEK48ORg5jrouCAg==", + "license": "MIT", + "dependencies": { + "ansi-styles": "^6.2.3", + "is-fullwidth-code-point": "^5.1.0" + }, + "engines": { + "node": ">=20" + }, + "funding": { + "url": "https://github.com/chalk/slice-ansi?sponsor=1" + } + }, + "node_modules/slice-ansi/node_modules/ansi-styles": { + "version": "6.2.3", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-6.2.3.tgz", + "integrity": "sha512-4Dj6M28JB+oAH8kFkTLUo+a2jwOFkuqb3yucU0CANcRRUbxS0cP0nZYCGjcc3BNXwRIsUVmDGgzawme7zvJHvg==", + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/string-width": { + "version": "8.2.0", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-8.2.0.tgz", + "integrity": "sha512-6hJPQ8N0V0P3SNmP6h2J99RLuzrWz2gvT7VnK5tKvrNqJoyS9W4/Fb8mo31UiPvy00z7DQXkP2hnKBVav76thw==", + "license": "MIT", + "dependencies": { + "get-east-asian-width": "^1.5.0", + "strip-ansi": "^7.1.2" + }, + "engines": { + "node": ">=20" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/strip-ansi": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.2.0.tgz", + "integrity": "sha512-yDPMNjp4WyfYBkHnjIRLfca1i6KMyGCtsVgoKe/z1+6vukgaENdgGBZt+ZmKPc4gavvEZ5OgHfHdrazhgNyG7w==", + "license": "MIT", + "dependencies": { + "ansi-regex": "^6.2.2" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/strip-ansi?sponsor=1" + } + }, + "node_modules/supports-color": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", + "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", + "license": "MIT", + "dependencies": { + "has-flag": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/wrap-ansi": { + "version": "10.0.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-10.0.0.tgz", + "integrity": "sha512-SGcvg80f0wUy2/fXES19feHMz8E0JoXv2uNgHOu4Dgi2OrCy1lqwFYEJz1BLbDI0exjPMe/ZdzZ/YpGECBG/aQ==", + "license": "MIT", + "dependencies": { + "ansi-styles": "^6.2.3", + "string-width": "^8.2.0", + "strip-ansi": "^7.1.2" + }, + "engines": { + "node": ">=20" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, + "node_modules/wrap-ansi/node_modules/ansi-styles": { + "version": "6.2.3", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-6.2.3.tgz", + "integrity": "sha512-4Dj6M28JB+oAH8kFkTLUo+a2jwOFkuqb3yucU0CANcRRUbxS0cP0nZYCGjcc3BNXwRIsUVmDGgzawme7zvJHvg==", + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/ws": { + "version": "8.20.0", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.20.0.tgz", + "integrity": "sha512-sAt8BhgNbzCtgGbt2OxmpuryO63ZoDk/sqaB/znQm94T4fCEsy/yV+7CdC1kJhOU9lboAEU7R3kquuycDoibVA==", + "license": "MIT", + "engines": { + "node": ">=10.0.0" + }, + "peerDependencies": { + "bufferutil": "^4.0.1", + "utf-8-validate": ">=5.0.2" + }, + "peerDependenciesMeta": { + "bufferutil": { + "optional": true + }, + "utf-8-validate": { + "optional": true + } + } + }, + "node_modules/yaml": { + "version": "2.8.3", + "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.8.3.tgz", + "integrity": "sha512-AvbaCLOO2Otw/lW5bmh9d/WEdcDFdQp2Z2ZUH3pX9U2ihyUY0nvLv7J6TrWowklRGPYbB/IuIMfYgxaCPg5Bpg==", + "license": "ISC", + "bin": { + "yaml": "bin.mjs" + }, + "engines": { + "node": ">= 14.6" + }, + "funding": { + "url": "https://github.com/sponsors/eemeli" + } + }, + "node_modules/zod": { + "version": "3.25.76", + "resolved": "https://registry.npmjs.org/zod/-/zod-3.25.76.tgz", + "integrity": "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/colinhacks" + } + }, + "node_modules/zod-to-json-schema": { + "version": "3.25.1", + "resolved": "https://registry.npmjs.org/zod-to-json-schema/-/zod-to-json-schema-3.25.1.tgz", + "integrity": "sha512-pM/SU9d3YAggzi6MtR4h7ruuQlqKtad8e9S0fmxcMi+ueAK5Korys/aWcV9LIIHTVbj01NdzxcnXSN+O74ZIVA==", + "license": "ISC", + "peerDependencies": { + "zod": "^3.25 || ^4" + } + } + } +} diff --git a/package.json b/package.json new file mode 100644 index 00000000..0c34ed82 --- /dev/null +++ b/package.json @@ -0,0 +1,10 @@ +{ + "private": true, + "description": "relayfile — real-time filesystem for humans and agents", + "scripts": { + "workflow": "agent-relay run" + }, + "dependencies": { + "@agent-relay/sdk": "latest" + } +} diff --git a/scripts/e2e.ts b/scripts/e2e.ts new file mode 100644 index 00000000..5d80a871 --- /dev/null +++ b/scripts/e2e.ts @@ -0,0 +1,519 @@ +#!/usr/bin/env npx tsx +/** + * Relayfile E2E Smoke Test + * + * Spins up a relayfile server + two mount daemons, then exercises the full + * read/write/sync cycle between two agents sharing the same workspace. + * + * Usage: + * npx tsx scripts/e2e.ts # default (interactive) + * npx tsx scripts/e2e.ts --ci # CI mode (shorter pauses) + * npx tsx scripts/e2e.ts --continue-on-failure # keep running after failures + */ + +import { execSync, spawn, ChildProcess } from 'node:child_process'; +import { createHmac } from 'node:crypto'; +import { existsSync, readFileSync, writeFileSync, unlinkSync, mkdirSync, rmSync } from 'node:fs'; +import { join } from 'node:path'; +import { tmpdir } from 'node:os'; + +// --------------------------------------------------------------------------- +// Config +// --------------------------------------------------------------------------- +const flags = new Set(process.argv.slice(2).filter((a) => a.startsWith('--'))); +const CI = flags.has('--ci') || !!process.env.CI; +const CONTINUE_ON_FAILURE = flags.has('--continue-on-failure'); + +const PORT = 9090; +const BASE_URL = `http://127.0.0.1:${PORT}`; +const WORKSPACE = 'e2e-test'; +const JWT_SECRET = 'test-secret'; + +// --------------------------------------------------------------------------- +// Terminal colors +// --------------------------------------------------------------------------- +const R = '\x1b[0m'; +const B = '\x1b[1m'; +const DIM = '\x1b[2m'; +const RED = '\x1b[31m'; +const GREEN = '\x1b[32m'; +const YELLOW = '\x1b[33m'; +const CYAN = '\x1b[36m'; + +function ts() { + return `${DIM}${new Date().toISOString().slice(11, 23)}${R}`; +} + +function log(icon: string, msg: string) { + console.log(`${ts()} ${icon} ${msg}`); +} + +function step(label: string) { + console.log(`\n${ts()} ${YELLOW}${B}▸ ${label}${R}`); +} + +function ok(msg: string) { + log('✅', `${GREEN}${msg}${R}`); +} + +function fail(msg: string) { + log('❌', `${RED}${msg}${R}`); +} + +function sleep(ms: number) { + return new Promise((r) => setTimeout(r, ms)); +} + +// --------------------------------------------------------------------------- +// JWT generation (mirrors generate-dev-token.sh) +// --------------------------------------------------------------------------- +function base64url(buf: Buffer): string { + return buf.toString('base64').replace(/\+/g, '-').replace(/\//g, '_').replace(/=+$/, ''); +} + +function generateToken(workspaceId: string, agentName: string, scopes: string[], expSeconds: number): string { + const header = { alg: 'HS256', typ: 'JWT' }; + const payload = { + workspace_id: workspaceId, + agent_name: agentName, + scopes, + exp: Math.floor(Date.now() / 1000) + expSeconds, + aud: 'relayfile', + }; + const h = base64url(Buffer.from(JSON.stringify(header))); + const p = base64url(Buffer.from(JSON.stringify(payload))); + const sig = createHmac('sha256', JWT_SECRET).update(`${h}.${p}`).digest(); + return `${h}.${p}.${base64url(sig)}`; +} + +// --------------------------------------------------------------------------- +// Process management +// --------------------------------------------------------------------------- +const children: ChildProcess[] = []; + +function killAll() { + for (const child of children) { + try { + child.kill('SIGTERM'); + } catch {} + } +} + +function spawnTracked(cmd: string, args: string[], env?: Record): ChildProcess { + const child = spawn(cmd, args, { + env: { ...process.env, ...env }, + stdio: ['ignore', 'pipe', 'pipe'], + }); + children.push(child); + return child; +} + +// --------------------------------------------------------------------------- +// HTTP helpers +// --------------------------------------------------------------------------- +let correlationCounter = 0; +function nextCorrelationId(): string { + return `e2e_${Date.now()}_${++correlationCounter}`; +} + +const TOKEN = generateToken(WORKSPACE, 'e2e', ['fs:read', 'fs:write', 'sync:read', 'ops:read'], 3600); + +async function api(method: string, path: string, body?: unknown, headers?: Record): Promise<{ status: number; data: any }> { + const url = `${BASE_URL}${path}`; + const opts: RequestInit = { + method, + headers: { + Authorization: `Bearer ${TOKEN}`, + 'X-Correlation-Id': nextCorrelationId(), + 'Content-Type': 'application/json', + ...headers, + }, + }; + if (body !== undefined) { + opts.body = JSON.stringify(body); + } + const resp = await fetch(url, opts); + const text = await resp.text(); + let data: any; + try { + data = JSON.parse(text); + } catch { + data = text; + } + return { status: resp.status, data }; +} + +// --------------------------------------------------------------------------- +// Wait helpers +// --------------------------------------------------------------------------- +async function waitForHealth(timeoutMs: number): Promise { + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + try { + const resp = await fetch(`${BASE_URL}/health`); + if (resp.ok) return; + } catch {} + await sleep(200); + } + throw new Error(`Health check timed out after ${timeoutMs}ms`); +} + +async function waitForFile(filePath: string, timeoutMs: number): Promise { + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + if (existsSync(filePath)) return; + await sleep(200); + } + throw new Error(`Timed out waiting for file: ${filePath}`); +} + +async function waitForFileContent(filePath: string, expected: string, timeoutMs: number): Promise { + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + if (existsSync(filePath)) { + const content = readFileSync(filePath, 'utf-8'); + if (content.includes(expected)) return; + } + await sleep(200); + } + const actual = existsSync(filePath) ? readFileSync(filePath, 'utf-8') : ''; + throw new Error(`Timed out waiting for content "${expected}" in ${filePath}. Actual: "${actual}"`); +} + +async function waitForFileAbsent(filePath: string, timeoutMs: number): Promise { + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + if (!existsSync(filePath)) return; + await sleep(200); + } + throw new Error(`Timed out waiting for file to disappear: ${filePath}`); +} + +function assert(condition: boolean, msg: string): void { + if (!condition) throw new Error(`Assertion failed: ${msg}`); +} + +// --------------------------------------------------------------------------- +// Main +// --------------------------------------------------------------------------- +async function main() { + console.log(` +${B}${CYAN}╔══════════════════════════════════════════════╗ +║ Relayfile E2E Smoke Test ║ +╚══════════════════════════════════════════════╝${R} +`); + log('🌐', `Server: ${B}${BASE_URL}${R}`); + log('⚙️ ', `Mode: ${B}${CI ? 'CI (auto)' : 'Interactive'}${R}`); + log('🧭', `Flow: ${B}${CONTINUE_ON_FAILURE ? 'Continue on failure' : 'Fail fast (default)'}${R}`); + + const passed: string[] = []; + const failed: string[] = []; + + const SYNC_WAIT = CI ? 8_000 : 12_000; + const MOUNT_WAIT = CI ? 10_000 : 15_000; + + // Temp dirs + const tmpBase = join(tmpdir(), `relayfile-e2e-${Date.now()}`); + const agentADir = join(tmpBase, 'agent-a'); + const agentBDir = join(tmpBase, 'agent-b'); + mkdirSync(agentADir, { recursive: true }); + mkdirSync(agentBDir, { recursive: true }); + + async function run(name: string, fn: () => Promise) { + try { + await fn(); + ok(name); + passed.push(name); + } catch (err) { + const msg = err instanceof Error ? err.message : String(err); + fail(`${name}: ${msg}`); + failed.push(name); + if (!CONTINUE_ON_FAILURE) { + throw new Error(`${name} failed: ${msg}`); + } + } + } + + try { + // ------------------------------------------------------------------ + // Build + // ------------------------------------------------------------------ + step('Building relayfile and relayfile-mount'); + execSync('go build -o relayfile ./cmd/relayfile && go build -o relayfile-mount ./cmd/relayfile-mount', { + cwd: process.cwd(), + stdio: 'inherit', + }); + ok('Build succeeded'); + + // ------------------------------------------------------------------ + // Start server + // ------------------------------------------------------------------ + step('Starting relayfile server'); + const server = spawnTracked('./relayfile', [], { + RELAYFILE_ADDR: `:${PORT}`, + RELAYFILE_BACKEND_PROFILE: 'memory', + RELAYFILE_JWT_SECRET: JWT_SECRET, + RELAYFILE_EXTERNAL_WRITEBACK: 'false', + }); + server.stderr?.on('data', (d: Buffer) => { + const line = d.toString().trim(); + if (line) log('📡', `${DIM}[server] ${line}${R}`); + }); + + await waitForHealth(10_000); + ok('Server healthy'); + + // ------------------------------------------------------------------ + // Start mount daemons + // ------------------------------------------------------------------ + step('Starting mount daemons'); + const mountA = spawnTracked('./relayfile-mount', [ + '--base-url', BASE_URL, + '--workspace', WORKSPACE, + '--local-dir', agentADir, + '--token', TOKEN, + '--interval', '1s', + ]); + mountA.stderr?.on('data', (d: Buffer) => { + const line = d.toString().trim(); + if (line) log('🔄', `${DIM}[mount-a] ${line}${R}`); + }); + + const mountB = spawnTracked('./relayfile-mount', [ + '--base-url', BASE_URL, + '--workspace', WORKSPACE, + '--local-dir', agentBDir, + '--token', TOKEN, + '--interval', '1s', + ]); + mountB.stderr?.on('data', (d: Buffer) => { + const line = d.toString().trim(); + if (line) log('🔄', `${DIM}[mount-b] ${line}${R}`); + }); + + // Give daemons a moment to start + await sleep(1000); + ok('Mount daemons started'); + + // ------------------------------------------------------------------ + // Test: Seed files via API + // ------------------------------------------------------------------ + await run('Seed files via API', async () => { + step('Seeding files via API'); + + const files = [ + { path: '/src/index.ts', content: 'export const version = "1.0";' }, + { path: '/src/utils.ts', content: 'export function add(a: number, b: number) { return a + b; }' }, + { path: '/package.json', content: '{"name":"test","version":"1.0.0"}' }, + ]; + + for (const f of files) { + const resp = await api('PUT', `/v1/workspaces/${WORKSPACE}/fs/file?path=${encodeURIComponent(f.path)}`, { + contentType: 'text/plain', + content: f.content, + }, { 'If-Match': '0' }); + assert(resp.status === 200 || resp.status === 201 || resp.status === 202, + `Seed ${f.path} failed with status ${resp.status}: ${JSON.stringify(resp.data)}`); + log('📝', `Seeded ${f.path}`); + } + + const tree = await api('GET', `/v1/workspaces/${WORKSPACE}/fs/tree?path=/&depth=10`); + assert(tree.status === 200, `Tree listing failed: ${tree.status}`); + const entries = tree.data.entries || []; + assert(entries.length >= 3, `Expected >= 3 tree entries, got ${entries.length}`); + }); + + // ------------------------------------------------------------------ + // Test: Mount daemon pulls files to Agent A + // ------------------------------------------------------------------ + await run('Mount pulls files to Agent A', async () => { + step('Waiting for Agent A to sync'); + await waitForFile(join(agentADir, 'src', 'index.ts'), MOUNT_WAIT); + await waitForFile(join(agentADir, 'src', 'utils.ts'), MOUNT_WAIT); + await waitForFile(join(agentADir, 'package.json'), MOUNT_WAIT); + + const content = readFileSync(join(agentADir, 'src', 'index.ts'), 'utf-8'); + assert(content.includes('version = "1.0"'), `Agent A index.ts content mismatch: ${content}`); + }); + + // ------------------------------------------------------------------ + // Test: Mount daemon pulls files to Agent B + // ------------------------------------------------------------------ + await run('Mount pulls files to Agent B', async () => { + step('Waiting for Agent B to sync'); + await waitForFile(join(agentBDir, 'src', 'index.ts'), MOUNT_WAIT); + await waitForFile(join(agentBDir, 'src', 'utils.ts'), MOUNT_WAIT); + await waitForFile(join(agentBDir, 'package.json'), MOUNT_WAIT); + + const content = readFileSync(join(agentBDir, 'src', 'index.ts'), 'utf-8'); + assert(content.includes('version = "1.0"'), `Agent B index.ts content mismatch: ${content}`); + }); + + // ------------------------------------------------------------------ + // Test: Agent A writes a file, Agent B sees it + // ------------------------------------------------------------------ + await run('Agent A write syncs to Agent B', async () => { + step('Agent A writes new-feature.ts'); + const featureContent = 'export const feature = true;'; + mkdirSync(join(agentADir, 'src'), { recursive: true }); + writeFileSync(join(agentADir, 'src', 'new-feature.ts'), featureContent); + + // Wait for Agent A daemon to push + log('⏳', 'Waiting for Agent A to push...'); + await sleep(SYNC_WAIT); + + // Verify via API + const resp = await api('GET', `/v1/workspaces/${WORKSPACE}/fs/file?path=${encodeURIComponent('/src/new-feature.ts')}`); + assert(resp.status === 200, `API read new-feature.ts failed: ${resp.status} ${JSON.stringify(resp.data)}`); + assert(resp.data.content.includes('feature = true'), `API content mismatch: ${resp.data.content}`); + + // Wait for Agent B daemon to pull + log('⏳', 'Waiting for Agent B to pull...'); + await waitForFileContent(join(agentBDir, 'src', 'new-feature.ts'), 'feature = true', MOUNT_WAIT); + }); + + // ------------------------------------------------------------------ + // Test: Agent B edits a file, Agent A sees it + // ------------------------------------------------------------------ + await run('Agent B edit syncs to Agent A', async () => { + step('Agent B edits index.ts'); + writeFileSync(join(agentBDir, 'src', 'index.ts'), 'export const version = "2.0";'); + + log('⏳', 'Waiting for sync cycle...'); + await waitForFileContent(join(agentADir, 'src', 'index.ts'), 'version = "2.0"', SYNC_WAIT + MOUNT_WAIT); + }); + + // ------------------------------------------------------------------ + // Test: Events feed + // ------------------------------------------------------------------ + await run('Events feed contains entries', async () => { + step('Checking events feed'); + const resp = await api('GET', `/v1/workspaces/${WORKSPACE}/fs/events?limit=50`); + assert(resp.status === 200, `Events feed failed: ${resp.status}`); + const events = resp.data.events || []; + assert(events.length > 0, 'Events feed is empty'); + + const types = events.map((e: any) => e.type); + assert(types.includes('file.created'), `No file.created events. Types: ${types.join(', ')}`); + log('📊', `Found ${events.length} events: ${[...new Set(types)].join(', ')}`); + }); + + // ------------------------------------------------------------------ + // Test: Revision conflict detection + // ------------------------------------------------------------------ + await run('Revision conflict detection', async () => { + step('Testing revision conflict'); + const readResp = await api('GET', `/v1/workspaces/${WORKSPACE}/fs/file?path=${encodeURIComponent('/src/index.ts')}`); + assert(readResp.status === 200, `Read index.ts failed: ${readResp.status}`); + + const wrongRevision = 'wrong-rev-123'; + const writeResp = await api('PUT', `/v1/workspaces/${WORKSPACE}/fs/file?path=${encodeURIComponent('/src/index.ts')}`, { + contentType: 'text/plain', + content: 'should fail', + }, { 'If-Match': wrongRevision }); + assert(writeResp.status === 409, `Expected 409 Conflict, got ${writeResp.status}: ${JSON.stringify(writeResp.data)}`); + log('🔒', `Conflict correctly returned: ${writeResp.data.code}`); + }); + + // ------------------------------------------------------------------ + // Test: File deletion syncs + // ------------------------------------------------------------------ + await run('File deletion syncs', async () => { + step('Agent A deletes utils.ts'); + const utilsPath = join(agentADir, 'src', 'utils.ts'); + assert(existsSync(utilsPath), 'utils.ts should exist before deletion'); + unlinkSync(utilsPath); + + log('⏳', 'Waiting for deletion to sync...'); + await waitForFileAbsent(join(agentBDir, 'src', 'utils.ts'), SYNC_WAIT + MOUNT_WAIT); + + // Also verify via API + const resp = await api('GET', `/v1/workspaces/${WORKSPACE}/fs/file?path=${encodeURIComponent('/src/utils.ts')}`); + assert(resp.status === 404, `Expected 404 for deleted file, got ${resp.status}`); + }); + + // ------------------------------------------------------------------ + // Test: Bulk file performance + // ------------------------------------------------------------------ + await run('Bulk file sync performance', async () => { + step('Writing 50 files to Agent A workspace'); + const BULK_COUNT = 50; + const bulkDir = join(agentADir, 'bulk'); + mkdirSync(bulkDir, { recursive: true }); + + const startWrite = Date.now(); + for (let i = 0; i < BULK_COUNT; i++) { + writeFileSync(join(bulkDir, `file-${String(i).padStart(3, '0')}.ts`), `export const n = ${i};`); + } + log('📝', `Wrote ${BULK_COUNT} files in ${Date.now() - startWrite}ms`); + + // Wait for all files to appear in Agent B + const startSync = Date.now(); + const bulkTimeout = CI ? 60_000 : 90_000; + const deadline = Date.now() + bulkTimeout; + + while (Date.now() < deadline) { + let count = 0; + for (let i = 0; i < BULK_COUNT; i++) { + if (existsSync(join(agentBDir, 'bulk', `file-${String(i).padStart(3, '0')}.ts`))) { + count++; + } + } + if (count >= BULK_COUNT) break; + if (Date.now() >= deadline) { + throw new Error(`Only ${count}/${BULK_COUNT} files synced within ${bulkTimeout}ms`); + } + await sleep(500); + } + + const syncLatency = Date.now() - startSync; + log('⏱️ ', `${BULK_COUNT} files synced to Agent B in ${B}${syncLatency}ms${R}`); + }); + + } finally { + // ------------------------------------------------------------------ + // Teardown + // ------------------------------------------------------------------ + step('Teardown'); + killAll(); + + // Wait briefly for processes to exit + await sleep(500); + + try { + rmSync(tmpBase, { recursive: true, force: true }); + log('🗑️ ', 'Cleaned up temp dirs'); + } catch {} + + // Summary + console.log(` +${B}${CYAN}╔══════════════════════════════════════════════╗ +║ Summary ║ +╚══════════════════════════════════════════════╝${R} +`); + if (passed.length > 0) { + log('✅', `${GREEN}${B}${passed.length} passed${R}`); + for (const t of passed) { + log(' ', `${GREEN}• ${t}${R}`); + } + } + if (failed.length > 0) { + console.log(); + log('❌', `${RED}${B}${failed.length} failed${R}`); + for (const t of failed) { + log(' ', `${RED}• ${t}${R}`); + } + } + console.log(); + + if (failed.length > 0) { + process.exit(1); + } + } +} + +main().catch((err) => { + fail(err instanceof Error ? err.message : String(err)); + killAll(); + process.exit(1); +}); diff --git a/sdk/relayfile-sdk/src/client.ts b/sdk/relayfile-sdk/src/client.ts index bf7189d3..31e28f6a 100644 --- a/sdk/relayfile-sdk/src/client.ts +++ b/sdk/relayfile-sdk/src/client.ts @@ -1,6 +1,8 @@ import { - type AdminSyncStatusResponse, type AdminIngressStatusResponse, + type AdminSyncStatusResponse, + type BulkWriteInput, + type BulkWriteResponse, type BackendStatusResponse, type AckResponse, type DeleteFileInput, @@ -8,13 +10,15 @@ import { type DeadLetterFeedResponse, type ErrorResponse, type EventFeedResponse, + type ExportJsonResponse, + type ExportOptions, type FileQueryResponse, type FileReadResponse, + type FilesystemEvent, type GetEventsOptions, - type GetAdminSyncStatusOptions, type GetAdminIngressStatusOptions, + type GetAdminSyncStatusOptions, type GetOperationsOptions, - type QueryFilesOptions, type GetSyncDeadLettersOptions, type GetSyncIngressStatusOptions, type GetSyncStatusOptions, @@ -22,6 +26,7 @@ import { type OperationFeedResponse, type OperationStatusResponse, type QueuedResponse, + type QueryFilesOptions, type SyncIngressStatusResponse, type SyncStatusResponse, type TreeResponse, @@ -60,6 +65,24 @@ interface NormalizedRetryOptions { jitterRatio: number; } +type WebSocketEventName = "event" | "error" | "open" | "close"; +type WebSocketHandlerMap = { + event: (event: FilesystemEvent) => void; + error: (event: Event | Error) => void; + open: (event: Event) => void; + close: (event: CloseEvent) => void; +}; + +export interface WebSocketConnection { + close(code?: number, reason?: string): void; + on(event: TEventName, handler: WebSocketHandlerMap[TEventName]): () => void; +} + +export interface ConnectWebSocketOptions { + token?: string; + onEvent?: (event: FilesystemEvent) => void; +} + const DEFAULT_RETRY_OPTIONS: NormalizedRetryOptions = { maxRetries: 3, baseDelayMs: 100, @@ -102,12 +125,98 @@ async function resolveToken(tokenProvider: AccessTokenProvider): Promise return tokenProvider; } +function resolveSyncToken(tokenProvider: AccessTokenProvider): string { + if (typeof tokenProvider === "function") { + const token = tokenProvider(); + if (typeof token !== "string") { + throw new Error("connectWebSocket requires a synchronous token provider or an explicit token option."); + } + return token; + } + return tokenProvider; +} + function createAbortError(): Error { const err = new Error("The operation was aborted."); err.name = "AbortError"; return err; } +function createBlobFromResponse(response: Response): Promise { + if (typeof response.blob === "function") { + return response.blob(); + } + return response.arrayBuffer().then((buffer) => new Blob([buffer], { type: response.headers.get("content-type") ?? undefined })); +} + +class RelayFileWebSocketConnection implements WebSocketConnection { + private readonly socket: WebSocket; + private readonly handlers: { + [K in WebSocketEventName]: Set; + } = { + event: new Set(), + error: new Set(), + open: new Set(), + close: new Set() + }; + + constructor(socket: WebSocket, onEvent?: (event: FilesystemEvent) => void) { + this.socket = socket; + if (onEvent) { + this.handlers.event.add(onEvent); + } + + socket.addEventListener("open", (event) => { + for (const handler of this.handlers.open) { + handler(event); + } + }); + + socket.addEventListener("close", (event) => { + for (const handler of this.handlers.close) { + handler(event); + } + }); + + socket.addEventListener("error", (event) => { + const errorEvent = event instanceof ErrorEvent && event.error instanceof Error ? event.error : event; + for (const handler of this.handlers.error) { + handler(errorEvent); + } + }); + + socket.addEventListener("message", (event) => { + if (typeof event.data !== "string") { + return; + } + let parsed: FilesystemEvent; + try { + parsed = JSON.parse(event.data) as FilesystemEvent; + } catch (error) { + const parseError = error instanceof Error ? error : new Error("Failed to parse WebSocket event payload."); + for (const handler of this.handlers.error) { + handler(parseError); + } + return; + } + for (const handler of this.handlers.event) { + handler(parsed); + } + }); + } + + close(code?: number, reason?: string): void { + this.socket.close(code, reason); + } + + on(event: TEventName, handler: WebSocketHandlerMap[TEventName]): () => void { + this.handlers[event].add(handler); + return () => { + this.handlers[event].delete(handler); + }; + } +} + export class RelayFileClient { private readonly baseUrl: string; private readonly tokenProvider: AccessTokenProvider; @@ -185,12 +294,25 @@ export class RelayFileClient { body: { contentType: input.contentType ?? "text/markdown", content: input.content, + encoding: input.encoding, semantics: input.semantics }, signal: input.signal }); } + async bulkWrite(input: BulkWriteInput): Promise { + return this.request({ + method: "POST", + path: `/v1/workspaces/${encodeURIComponent(input.workspaceId)}/fs/bulk`, + correlationId: input.correlationId, + body: { + files: input.files + }, + signal: input.signal + }); + } + async deleteFile(input: DeleteFileInput): Promise { const query = buildQuery({ path: input.path }); return this.request({ @@ -218,6 +340,38 @@ export class RelayFileClient { }); } + async exportWorkspace(options: ExportOptions): Promise { + const format = options.format ?? "json"; + const query = buildQuery({ format }); + const correlationId = options.correlationId ?? generateCorrelationId(); + const response = await this.performRequest({ + method: "GET", + path: `/v1/workspaces/${encodeURIComponent(options.workspaceId)}/fs/export${query}`, + correlationId, + signal: options.signal, + accept: format === "json" ? "application/json" : "*/*" + }); + + if (format === "json") { + return this.readPayload(response) as Promise; + } + + return createBlobFromResponse(response); + } + + connectWebSocket( + workspaceId: string, + options: ConnectWebSocketOptions = {} + ): WebSocketConnection { + const token = options.token ?? resolveSyncToken(this.tokenProvider); + const url = new URL(`${this.baseUrl}/v1/workspaces/${encodeURIComponent(workspaceId)}/fs/ws`); + url.protocol = url.protocol === "https:" ? "wss:" : "ws:"; + url.searchParams.set("token", token); + + const socket = new WebSocket(url.toString()); + return new RelayFileWebSocketConnection(socket, options.onEvent); + } + async getOp( workspaceId: string, opId: string, @@ -456,21 +610,40 @@ export class RelayFileClient { correlationId?: string; signal?: AbortSignal; }): Promise { + const response = await this.performRequest(params); + return this.readPayload(response) as Promise; + } + + private async performRequest(params: { + method: string; + path: string; + headers?: Record; + body?: unknown; + correlationId?: string; + signal?: AbortSignal; + accept?: string; + }): Promise { const correlationId = params.correlationId ?? generateCorrelationId(); const baseHeaders: Record = { - "Content-Type": "application/json", "X-Correlation-Id": correlationId, ...params.headers }; + if (params.body !== undefined) { + baseHeaders["Content-Type"] = "application/json"; + } + if (params.accept) { + baseHeaders["Accept"] = params.accept; + } if (this.userAgent) { baseHeaders["User-Agent"] = this.userAgent; } + let retries = 0; const url = `${this.baseUrl}${params.path}`; for (;;) { const token = await resolveToken(this.tokenProvider); const headers: Record = { - "Authorization": `Bearer ${token}`, + Authorization: `Bearer ${token}`, ...baseHeaders }; const requestInit: RequestInit = { @@ -494,11 +667,11 @@ export class RelayFileClient { throw error; } - const payload = await this.readPayload(response); if (response.ok) { - return payload as T; + return response; } + const payload = await this.readPayload(response); if (this.shouldRetryStatus(response.status, retries, params.signal)) { retries += 1; await this.sleep(this.computeRetryDelayMs(retries, response.headers.get("retry-after")), params.signal); diff --git a/sdk/relayfile-sdk/src/index.ts b/sdk/relayfile-sdk/src/index.ts index be43d6ea..45f27a99 100644 --- a/sdk/relayfile-sdk/src/index.ts +++ b/sdk/relayfile-sdk/src/index.ts @@ -1,4 +1,9 @@ -export { RelayFileClient, type RelayFileRetryOptions } from "./client.js"; +export { + RelayFileClient, + type ConnectWebSocketOptions, + type RelayFileRetryOptions, + type WebSocketConnection +} from "./client.js"; export { InvalidStateError, PayloadTooLargeError, @@ -23,12 +28,18 @@ export type { AdminSyncAlertType, AdminSyncStatusResponse, BackendStatusResponse, + BulkWriteFile, + BulkWriteInput, + BulkWriteResponse, ConflictErrorResponse, DeleteFileInput, DeadLetterFeedResponse, DeadLetterItem, ErrorResponse, EventFeedResponse, + ExportFormat, + ExportJsonResponse, + ExportOptions, FileQueryItem, FileQueryResponse, FileReadResponse, diff --git a/sdk/relayfile-sdk/src/types.ts b/sdk/relayfile-sdk/src/types.ts index abf6a8ff..b681003a 100644 --- a/sdk/relayfile-sdk/src/types.ts +++ b/sdk/relayfile-sdk/src/types.ts @@ -32,6 +32,7 @@ export interface FileReadResponse { revision: string; contentType: string; content: string; + encoding?: "utf-8" | "base64"; provider?: string; providerObjectId?: string; lastEditedAt?: string; @@ -44,6 +45,31 @@ export interface FileWriteRequest { semantics?: FileSemantics; } +export interface BulkWriteFile { + path: string; + contentType?: string; + content: string; + encoding?: "utf-8" | "base64"; +} + +export interface BulkWriteInput { + workspaceId: string; + files: BulkWriteFile[]; + correlationId?: string; + signal?: AbortSignal; +} + +export interface BulkWriteResponse { + written: number; + errorCount: number; + errors: Array<{ + path: string; + code: string; + message: string; + }>; + correlationId: string; +} + export interface FileQueryItem { path: string; revision: string; @@ -106,6 +132,17 @@ export interface EventFeedResponse { nextCursor: string | null; } +export type ExportFormat = "tar" | "json" | "patch"; + +export interface ExportOptions { + workspaceId: string; + format?: ExportFormat; + correlationId?: string; + signal?: AbortSignal; +} + +export type ExportJsonResponse = FileReadResponse[]; + export type OperationStatus = | "pending" | "running" @@ -440,6 +477,7 @@ export interface WriteFileInput { baseRevision: string; content: string; contentType?: string; + encoding?: "utf-8" | "base64"; semantics?: FileSemantics; correlationId?: string; signal?: AbortSignal; diff --git a/workflows/relayfile-bulk-and-export.ts b/workflows/relayfile-bulk-and-export.ts new file mode 100644 index 00000000..62bf02dc --- /dev/null +++ b/workflows/relayfile-bulk-and-export.ts @@ -0,0 +1,420 @@ +/** + * relayfile-bulk-and-export.ts + * + * Adds bulk seeding and workspace export to relayfile. + * These are prerequisites for cloud workflow integration: + * - Bulk seed: upload 200+ project files in a single API call (tarball or batch) + * - Export: download the entire workspace as a tar or git patch + * + * Also adds WebSocket push for real-time events (upgrade from polling) + * and binary file support. + * + * Run: agent-relay run workflows/relayfile-bulk-and-export.ts + */ + +import { workflow } from '@agent-relay/sdk/workflows'; + +const RELAYFILE = '/Users/khaliqgant/Projects/AgentWorkforce/relayfile'; + +async function main() { +const result = await workflow('relayfile-bulk-and-export') + .description('Add bulk seed, workspace export, WebSocket events, and binary file support to relayfile') + .pattern('dag') + .channel('wf-relayfile-bulk') + .maxConcurrency(3) + .timeout(3_600_000) + + .agent('architect', { + cli: 'claude', + preset: 'lead', + role: 'Design API endpoints, review implementation', + cwd: RELAYFILE, + }) + .agent('go-backend', { + cli: 'codex', + preset: 'worker', + role: 'Implement Go server endpoints and store methods', + cwd: RELAYFILE, + }) + .agent('go-websocket', { + cli: 'codex', + preset: 'worker', + role: 'Implement WebSocket event streaming', + cwd: RELAYFILE, + }) + .agent('sdk-dev', { + cli: 'codex', + preset: 'worker', + role: 'Update TypeScript SDK with new methods', + cwd: RELAYFILE, + }) + .agent('test-writer', { + cli: 'codex', + preset: 'worker', + role: 'Write tests for new functionality', + cwd: RELAYFILE, + }) + + // ── Phase 1: Read existing code ──────────────────────────────────── + + .step('read-store', { + type: 'deterministic', + command: `grep -n "func.*Store\\b\\|func.*Write\\|func.*Read\\|func.*Delete\\|func.*List\\|func.*Event" ${RELAYFILE}/internal/relayfile/store.go | head -40`, + captureOutput: true, + }) + + .step('read-http-server', { + type: 'deterministic', + command: `cat ${RELAYFILE}/internal/httpapi/server.go`, + captureOutput: true, + }) + + .step('read-openapi', { + type: 'deterministic', + command: `cat ${RELAYFILE}/openapi/relayfile-v1.openapi.yaml`, + captureOutput: true, + }) + + .step('read-ts-sdk', { + type: 'deterministic', + command: `cat ${RELAYFILE}/sdk/relayfile-sdk/src/client.ts`, + captureOutput: true, + }) + + .step('read-ts-types', { + type: 'deterministic', + command: `cat ${RELAYFILE}/sdk/relayfile-sdk/src/types.ts`, + captureOutput: true, + }) + + .step('read-syncer', { + type: 'deterministic', + command: `cat ${RELAYFILE}/internal/mountsync/syncer.go`, + captureOutput: true, + }) + + .step('read-store-full', { + type: 'deterministic', + command: `cat ${RELAYFILE}/internal/relayfile/store.go`, + captureOutput: true, + }) + + // ── Phase 2: Architect designs the new endpoints ─────────────────── + + .step('design-endpoints', { + agent: 'architect', + dependsOn: ['read-store', 'read-http-server', 'read-openapi'], + task: `Design the bulk seed, export, WebSocket, and binary support endpoints. + +Current store methods: +{{steps.read-store.output}} + +Current HTTP server: +{{steps.read-http-server.output}} + +Current OpenAPI spec: +{{steps.read-openapi.output}} + +Write a design doc at ${RELAYFILE}/docs/bulk-export-design.md: + +1. **Bulk Seed** — POST /v1/workspaces/{workspaceId}/fs/bulk + - Accept: application/json with array of files: + { files: [{ path: string, contentType: string, content: string }] } + - Also accept: multipart/form-data with a tar.gz file + - Atomic — all files written in one operation, single event per file + - Returns: { imported: number, errors: [{path, error}] } + - Store method: BulkWrite(workspaceId string, files []BulkWriteFile) (int, []error) + +2. **Workspace Export** — GET /v1/workspaces/{workspaceId}/fs/export + - Query params: format=tar|json|patch + - format=tar: returns application/gzip tarball of all files + - format=json: returns JSON array of all files with content + - format=patch: returns a unified diff against an empty tree (like git diff) + - Store method: ExportWorkspace(workspaceId string) ([]File, error) + +3. **WebSocket Events** — GET /v1/workspaces/{workspaceId}/fs/ws + - Upgrade to WebSocket + - Server pushes JSON events: { type: "file.created"|"file.updated"|"file.deleted", path, revision, timestamp } + - Client can send: { type: "subscribe", filter?: { paths?: string[] } } + - Falls back to polling for environments without WebSocket support + +4. **Binary File Support** + - Content field supports base64 encoding for binary files + - New field on write: encoding?: "utf-8" | "base64" (default: "utf-8") + - On read: detect content type, return encoding field + - Store: content stored as-is (string), encoding tracked in metadata + +Write the complete design.`, + verification: { type: 'exit_code' }, + }) + + // ── Phase 3: Parallel implementation ────────────────────────────── + + .step('implement-bulk-and-export', { + agent: 'go-backend', + dependsOn: ['design-endpoints', 'read-store-full', 'read-http-server'], + task: `Implement bulk seed, export, and binary file support in the Go server. + +Design: +{{steps.design-endpoints.output}} + +Current store: +{{steps.read-store-full.output}} + +Current HTTP server: +{{steps.read-http-server.output}} + +Changes: + +1. Edit ${RELAYFILE}/internal/relayfile/store.go — add methods: + - BulkWrite(workspaceID string, files []BulkWriteFile) (int, []BulkWriteError) + BulkWriteFile: { Path, ContentType, Content, Encoding string } + Writes all files, emitting events for each. No If-Match check (seed overwrites). + - ExportWorkspace(workspaceID string) ([]File, error) + Returns all files in the workspace with content. + +2. Edit ${RELAYFILE}/internal/httpapi/server.go — add route handlers: + - POST /v1/workspaces/{workspaceId}/fs/bulk → handleBulkWrite + Parse JSON body { files: [...] }, call store.BulkWrite, return summary. + - GET /v1/workspaces/{workspaceId}/fs/export → handleExport + Query param format=tar|json|patch. + For tar: use archive/tar + compress/gzip to stream a tarball. + For json: return JSON array of files. + For patch: generate unified diff format. + - Update handleWriteFile and handleReadFile for binary encoding support: + Accept "encoding" field in write request body. + Include "encoding" field in read response when base64. + +3. Add encoding field to File struct in store.go: + Encoding string \`json:"encoding,omitempty"\` // "utf-8" (default) or "base64" + +Write all changes to disk.`, + verification: { type: 'exit_code' }, + }) + + .step('implement-websocket', { + agent: 'go-websocket', + dependsOn: ['design-endpoints', 'read-store-full', 'read-http-server'], + task: `Implement WebSocket event streaming in the Go server. + +Design: +{{steps.design-endpoints.output}} + +Current store (for event subscription): +{{steps.read-store-full.output}} + +Current HTTP server: +{{steps.read-http-server.output}} + +Changes: + +1. Add dependency: go get nhooyr.io/websocket (lightweight WebSocket library for Go) + Or use golang.org/x/net/websocket. Pick whichever is simpler. + +2. Edit ${RELAYFILE}/internal/httpapi/server.go — add WebSocket handler: + - Route: /v1/workspaces/{workspaceId}/fs/ws + - Upgrade HTTP to WebSocket + - Auth: validate JWT from query param ?token= (WebSocket can't send headers) + - On connect: send all recent events (last 100) as catch-up + - Subscribe to store events for this workspace + - Push events as JSON messages: { type, path, revision, timestamp } + - Handle client messages: { type: "ping" } → respond with { type: "pong" } + - Clean disconnect on context cancellation + +3. Add event subscription to store: + - Add Subscribe(workspaceID string, ch chan<- Event) func() + Returns an unsubscribe function. + - On each Write/Delete, fan out to subscribers. + - Use sync.Map or mutex-protected map of workspace -> []chan. + +4. Create ${RELAYFILE}/internal/httpapi/websocket.go for the handler to keep server.go clean. + +Write all changes to disk.`, + verification: { type: 'exit_code' }, + }) + + .step('implement-sdk-updates', { + agent: 'sdk-dev', + dependsOn: ['design-endpoints', 'read-ts-sdk', 'read-ts-types'], + task: `Update the TypeScript SDK with new methods for bulk seed, export, and WebSocket. + +Design: +{{steps.design-endpoints.output}} + +Current SDK client: +{{steps.read-ts-sdk.output}} + +Current SDK types: +{{steps.read-ts-types.output}} + +Changes: + +1. Edit ${RELAYFILE}/sdk/relayfile-sdk/src/types.ts — add: + - BulkWriteFile: { path: string; contentType?: string; content: string; encoding?: "utf-8" | "base64" } + - BulkWriteInput: { workspaceId: string; files: BulkWriteFile[]; correlationId?: string; signal?: AbortSignal } + - BulkWriteResponse: { imported: number; errors: { path: string; error: string }[] } + - ExportFormat: "tar" | "json" | "patch" + - ExportOptions: { workspaceId: string; format?: ExportFormat; correlationId?: string; signal?: AbortSignal } + - ExportJsonResponse: { files: FileReadResponse[] } + - FilesystemEvent (already exists? add if missing): { eventId, type, path, revision, timestamp } + - Add encoding?: "utf-8" | "base64" to WriteFileInput and FileReadResponse + +2. Edit ${RELAYFILE}/sdk/relayfile-sdk/src/client.ts — add methods: + - bulkWrite(input: BulkWriteInput): Promise + POST /v1/workspaces/{workspaceId}/fs/bulk with JSON body + - exportWorkspace(options: ExportOptions): Promise + GET /v1/workspaces/{workspaceId}/fs/export?format=... + For json: parse as JSON. For tar/patch: return raw response. + - connectWebSocket(workspaceId: string, options?: { token?: string, onEvent?: (event: FilesystemEvent) => void }): WebSocketConnection + Connect to ws:// or wss:// endpoint + Return object with { close(), on(event, handler) } + +3. Re-export new types from ${RELAYFILE}/sdk/relayfile-sdk/src/index.ts + +Write all changes to disk.`, + verification: { type: 'exit_code' }, + }) + + // ── Phase 4: Update mount syncer for WebSocket ───────────────────── + + .step('update-syncer-websocket', { + agent: 'go-backend', + dependsOn: ['implement-websocket', 'read-syncer'], + task: `Update the mount syncer to use WebSocket events when available, falling back to polling. + +Current syncer: +{{steps.read-syncer.output}} + +Edit ${RELAYFILE}/internal/mountsync/syncer.go: + +1. Add a WebSocket mode to Syncer: + - New field: wsConn (WebSocket connection, nil if not connected) + - On SyncOnce(): if wsConn is nil, try to connect to ws:// endpoint + - If WebSocket is available: listen for events and apply them immediately (no polling) + - If WebSocket fails or disconnects: fall back to existing polling behavior + - This makes the daemon opportunistically real-time when the server supports it + +2. Add private method connectWebSocket(ctx) error: + - Dial ws://{baseURL}/v1/workspaces/{workspace}/fs/ws?token={token} + - Start goroutine to read events and apply via applyRemoteFile/applyRemoteDelete + - On disconnect: set wsConn = nil, fall back to polling + +3. Update the relayfile-mount CLI (cmd/relayfile-mount/main.go): + - Add --websocket flag (default: true) to enable/disable WebSocket mode + - When WebSocket mode is on and polling interval is set, use WebSocket for + real-time events and polling as a safety net (reconciliation every N intervals) + +Write all changes to disk. Keep the polling fallback working — WebSocket is an optimization, not a requirement.`, + verification: { type: 'exit_code' }, + }) + + // ── Phase 5: Tests ───────────────────────────────────────────────── + + .step('write-tests', { + agent: 'test-writer', + dependsOn: ['implement-bulk-and-export', 'implement-websocket', 'implement-sdk-updates'], + task: `Write Go tests for the new functionality. + +Create/update these test files: + +1. ${RELAYFILE}/internal/relayfile/store_test.go — add tests: + - TestBulkWrite: seed 10 files, verify all present in tree, verify events emitted + - TestBulkWriteOverwrite: bulk write same paths twice, verify latest content + - TestExportWorkspace: seed files, export, verify all returned with content + - TestBinaryEncoding: write base64 content, read back, verify encoding field + +2. ${RELAYFILE}/internal/httpapi/server_test.go — add HTTP tests: + - TestBulkWriteEndpoint: POST /fs/bulk with 5 files, verify 200 + imported count + - TestExportJSON: seed files, GET /fs/export?format=json, verify response + - TestExportTar: seed files, GET /fs/export?format=tar, verify gzip response + - TestWebSocketEvents: connect WebSocket, write a file via API, verify event received + +3. ${RELAYFILE}/internal/mountsync/syncer_test.go — add sync test: + - TestBulkSeedThenSync: use bulk API to seed, run SyncOnce, verify local files + +Follow the existing test patterns in each file. +Use the standard Go testing package + testify if already a dependency, otherwise stdlib. + +Write all test files to disk.`, + verification: { type: 'exit_code' }, + }) + + // ── Phase 6: Verify ─────────────────────────────────────────────── + + .step('verify-files', { + type: 'deterministic', + dependsOn: ['write-tests', 'update-syncer-websocket'], + command: `cd ${RELAYFILE} && echo "=== New/modified files ===" && \ +grep -rl "BulkWrite\|ExportWorkspace\|handleBulkWrite\|handleExport\|websocket\|WebSocket" internal/ --include="*.go" | sort && \ +echo "" && echo "=== SDK updates ===" && \ +grep -c "bulkWrite\|exportWorkspace\|connectWebSocket" sdk/relayfile-sdk/src/client.ts && \ +echo "" && echo "=== Build check ===" && \ +go build ./... 2>&1 | tail -10; echo "EXIT: $?"`, + captureOutput: true, + failOnError: false, + }) + + .step('run-tests', { + type: 'deterministic', + dependsOn: ['verify-files'], + command: `cd ${RELAYFILE} && go test ./... -count=1 2>&1 | tail -30; echo "EXIT: $?"`, + captureOutput: true, + failOnError: false, + }) + + .step('fix-issues', { + agent: 'architect', + dependsOn: ['run-tests'], + task: `Fix any build or test failures. + +Test output: +{{steps.run-tests.output}} + +Build output: +{{steps.verify-files.output}} + +If EXIT: 0 for both, summarize what was added. +Otherwise read the failing files and fix them. +Run go test ./... again to verify.`, + verification: { type: 'exit_code' }, + }) + + // ── Phase 7: Update OpenAPI spec ────────────────────────────────── + + .step('update-openapi', { + agent: 'architect', + dependsOn: ['fix-issues'], + task: `Update the OpenAPI spec to include the new endpoints. + +Read the current spec: cat ${RELAYFILE}/openapi/relayfile-v1.openapi.yaml + +Add these endpoints to the spec: + +1. POST /v1/workspaces/{workspaceId}/fs/bulk + - requestBody: { files: [{ path, contentType, content, encoding? }] } + - responses: 200 { imported, errors } + +2. GET /v1/workspaces/{workspaceId}/fs/export + - parameters: format (tar|json|patch) + - responses: 200 (varies by format) + +3. GET /v1/workspaces/{workspaceId}/fs/ws + - description: WebSocket upgrade for real-time events + +4. Update PUT /fs/file to include encoding field +5. Update GET /fs/file response to include encoding field + +Write the updated spec to disk.`, + verification: { type: 'exit_code' }, + }) + + .onError('retry', { maxRetries: 1, retryDelayMs: 10_000 }) + .run({ + cwd: RELAYFILE, + onEvent: (e: any) => console.log(`[${e.type}] ${e.stepName ?? e.step ?? ''} ${e.error ?? ''}`.trim()), + }); + +console.log(`\nBulk + Export workflow: ${result.status}`); +} + +main().catch(console.error); diff --git a/workflows/relayfile-ci-and-publish.ts b/workflows/relayfile-ci-and-publish.ts new file mode 100644 index 00000000..d419d0f8 --- /dev/null +++ b/workflows/relayfile-ci-and-publish.ts @@ -0,0 +1,358 @@ +/** + * relayfile-ci-and-publish.ts + * + * Sets up GitHub Actions CI/CD for relayfile: + * - CI: Go tests + TS SDK typecheck + E2E test on every PR + * - npm publish with provenance for the TS SDK (workflow_dispatch) + * - Go binary releases (cross-compile, GitHub Releases, checksums) + * - CF Workers deployment (wrangler deploy on main push) + * + * Models after relaycast's publish-npm.yml for provenance + versioning. + * + * Run: agent-relay run workflows/relayfile-ci-and-publish.ts + */ + +import { workflow } from '@agent-relay/sdk/workflows'; + +const RELAYFILE = '/Users/khaliqgant/Projects/AgentWorkforce/relayfile'; +const RELAYCAST = '/Users/khaliqgant/Projects/AgentWorkforce/relaycast'; + +async function main() { +const result = await workflow('relayfile-ci-and-publish') + .description('Set up CI/CD: Go tests, npm publish with provenance, Go binary releases, CF Workers deploy') + .pattern('dag') + .channel('wf-relayfile-ci') + .maxConcurrency(3) + .timeout(3_600_000) + + .agent('ci-lead', { + cli: 'claude', + preset: 'lead', + role: 'Design CI/CD pipeline, review workflows', + cwd: RELAYFILE, + }) + .agent('ci-implementer', { + cli: 'codex', + preset: 'worker', + role: 'Write GitHub Actions workflow files', + cwd: RELAYFILE, + }) + .agent('sdk-implementer', { + cli: 'codex', + preset: 'worker', + role: 'Prepare SDK package for npm publishing', + cwd: RELAYFILE, + }) + + // ── Phase 1: Read reference patterns ─────────────────────────────── + + .step('read-relaycast-ci', { + type: 'deterministic', + command: `cat ${RELAYCAST}/.github/workflows/ci.yml`, + captureOutput: true, + }) + + .step('read-relaycast-publish', { + type: 'deterministic', + command: `cat ${RELAYCAST}/.github/workflows/publish-npm.yml`, + captureOutput: true, + }) + + .step('read-relaycast-deploy', { + type: 'deterministic', + command: `cat ${RELAYCAST}/.github/workflows/deploy.yml`, + captureOutput: true, + }) + + .step('read-relaycast-binary-release', { + type: 'deterministic', + command: `cat ${RELAYCAST}/.github/workflows/local-binary-release.yml`, + captureOutput: true, + }) + + .step('read-sdk-package', { + type: 'deterministic', + command: `cat ${RELAYFILE}/sdk/relayfile-sdk/package.json 2>/dev/null || echo "no package.json"`, + captureOutput: true, + }) + + .step('read-existing-github', { + type: 'deterministic', + command: `find ${RELAYFILE}/.github -type f 2>/dev/null | sort || echo "no .github dir"`, + captureOutput: true, + }) + + // ── Phase 2: Design the pipeline ─────────────────────────────────── + + .step('design-ci', { + agent: 'ci-lead', + dependsOn: [ + 'read-relaycast-ci', 'read-relaycast-publish', 'read-relaycast-deploy', + 'read-relaycast-binary-release', 'read-existing-github', + ], + task: `Design the CI/CD pipeline for relayfile. We need 4 GitHub Actions workflows. + +Reference — relaycast CI: +{{steps.read-relaycast-ci.output}} + +Reference — relaycast npm publish (THIS IS THE KEY PATTERN for provenance): +{{steps.read-relaycast-publish.output}} + +Reference — relaycast deploy: +{{steps.read-relaycast-deploy.output}} + +Reference — relaycast binary release: +{{steps.read-relaycast-binary-release.output}} + +Write a design doc at ${RELAYFILE}/docs/ci-cd-design.md covering: + +1. **ci.yml** — runs on every PR and push to main: + - Go tests: go test ./... (all packages) + - Go build: go build ./cmd/relayfile ./cmd/relayfile-mount ./cmd/relayfile-cli + - TS SDK: cd sdk/relayfile-sdk && npm ci && npm run build && npx tsc --noEmit + - E2E test: start Go server, run scripts/e2e.ts --ci + - CF Workers typecheck: cd packages/server && npx tsc --noEmit (if it exists) + +2. **publish-npm.yml** — workflow_dispatch (manual trigger like relaycast): + - MUST use provenance: npm publish --access public --provenance + - MUST have permissions: contents: write, id-token: write (OIDC for provenance) + - Version bump: patch/minor/major/prerelease + - Dry run option + - NPM dist-tag: latest/next/beta/alpha + - Build SDK, run tests, publish to npm as @relayfile/sdk + - Create git tag + GitHub Release + - CRITICAL: registry-url must be "https://registry.npmjs.org" in setup-node + - CRITICAL: NODE_AUTH_TOKEN secret must be set (npm automation token) + +3. **release-binaries.yml** — triggered on tag push (v*): + - Cross-compile Go binaries: darwin/linux x amd64/arm64 + - Three binaries: relayfile (server), relayfile-mount, relayfile-cli + - Create checksums (sha256) + - Upload to GitHub Release + - Build + push Docker image to GHCR + +4. **deploy-workers.yml** — triggered on push to main (if packages/server exists): + - wrangler deploy for the CF Workers production server + - D1 migrations: wrangler d1 migrations apply + - Only runs if packages/server/ has changes + +Write the complete design with all secrets/permissions needed.`, + verification: { type: 'exit_code' }, + }) + + // ── Phase 3: Implementation (parallel) ───────────────────────────── + + .step('implement-ci-workflow', { + agent: 'ci-implementer', + dependsOn: ['design-ci', 'read-relaycast-ci'], + task: `Create the CI workflow. + +Design: +{{steps.design-ci.output}} + +Relaycast CI reference: +{{steps.read-relaycast-ci.output}} + +Create ${RELAYFILE}/.github/workflows/ci.yml: + +- Trigger: pull_request, push to main +- Jobs: + 1. go-test: setup Go 1.22, run go test ./... + 2. go-build: build all 3 binaries + 3. sdk-typecheck: setup Node 22, cd sdk/relayfile-sdk, npm ci, npm run build, tsc --noEmit + 4. e2e: depends on go-build, start server, run e2e.ts --ci + 5. workers-typecheck (conditional): if packages/server exists, tsc --noEmit + +Use matrix for Go version if desired. Cache go modules and npm. + +Write to disk.`, + verification: { type: 'exit_code' }, + }) + + .step('implement-publish-workflow', { + agent: 'ci-implementer', + dependsOn: ['design-ci', 'read-relaycast-publish'], + task: `Create the npm publish workflow with provenance. + +Design: +{{steps.design-ci.output}} + +Relaycast publish reference (follow this pattern closely): +{{steps.read-relaycast-publish.output}} + +Create ${RELAYFILE}/.github/workflows/publish-npm.yml: + +Key requirements: +- workflow_dispatch with inputs: version (patch/minor/major/prerelease), custom_version, dry_run, tag +- permissions: contents: write, id-token: write +- concurrency group to prevent parallel publishes +- Steps: + 1. Checkout + 2. Setup Node 22 with registry-url: "https://registry.npmjs.org" + 3. Install deps: cd sdk/relayfile-sdk && npm ci + 4. Version bump: npm version {type} --no-git-tag-version + 5. Build: npm run build + 6. Test: npx tsc --noEmit + 7. Dry run check (if dry_run) + 8. Publish: npm publish --access public --provenance --tag {tag} --ignore-scripts + 9. Commit version bump + create git tag + 10. Create GitHub Release + +The NODE_AUTH_TOKEN env var is auto-set by setup-node when registry-url is configured. +The repo needs an NPM_TOKEN secret (automation token from npmjs.com). + +env for publish step: + NODE_AUTH_TOKEN: \${{ secrets.NPM_TOKEN }} + +Write to disk. This is the most important workflow — provenance is non-negotiable.`, + verification: { type: 'exit_code' }, + }) + + .step('implement-release-workflow', { + agent: 'ci-implementer', + dependsOn: ['design-ci', 'read-relaycast-binary-release'], + task: `Create the Go binary release workflow. + +Design: +{{steps.design-ci.output}} + +Reference: +{{steps.read-relaycast-binary-release.output}} + +Create ${RELAYFILE}/.github/workflows/release-binaries.yml: + +- Trigger: push tags v* +- Matrix: os (linux, darwin) x arch (amd64, arm64) +- Steps: + 1. Checkout + 2. Setup Go 1.22 + 3. Cross-compile 3 binaries: + CGO_ENABLED=0 GOOS={os} GOARCH={arch} go build -o relayfile-{os}-{arch} ./cmd/relayfile + CGO_ENABLED=0 GOOS={os} GOARCH={arch} go build -o relayfile-mount-{os}-{arch} ./cmd/relayfile-mount + CGO_ENABLED=0 GOOS={os} GOARCH={arch} go build -o relayfile-cli-{os}-{arch} ./cmd/relayfile-cli + 4. Generate SHA256 checksums + 5. Upload to GitHub Release (use softprops/action-gh-release) + +Also add a Docker build job: + - Build from Dockerfile + - Push to ghcr.io/agentworkforce/relayfile:latest and :v{version} + +Write to disk.`, + verification: { type: 'exit_code' }, + }) + + .step('prepare-sdk-package', { + agent: 'sdk-implementer', + dependsOn: ['design-ci', 'read-sdk-package'], + task: `Prepare the SDK package.json for npm publishing with provenance. + +Current package.json: +{{steps.read-sdk-package.output}} + +Update ${RELAYFILE}/sdk/relayfile-sdk/package.json: + +{ + "name": "@relayfile/sdk", + "version": "0.1.0", + "description": "TypeScript SDK for relayfile — real-time filesystem for humans and agents", + "main": "dist/index.js", + "module": "dist/index.js", + "types": "dist/index.d.ts", + "type": "module", + "files": ["dist", "README.md", "LICENSE"], + "scripts": { + "build": "tsc", + "typecheck": "tsc --noEmit", + "prepublishOnly": "npm run build" + }, + "publishConfig": { + "access": "public", + "provenance": true + }, + "repository": { + "type": "git", + "url": "https://github.com/AgentWorkforce/relayfile", + "directory": "sdk/relayfile-sdk" + }, + "license": "MIT", + "keywords": ["relayfile", "filesystem", "sync", "agent", "collaboration"], + "engines": { "node": ">=18" } +} + +Also create/update ${RELAYFILE}/sdk/relayfile-sdk/tsconfig.json: +{ + "compilerOptions": { + "target": "ES2022", + "module": "ESNext", + "moduleResolution": "bundler", + "declaration": true, + "declarationMap": true, + "sourceMap": true, + "outDir": "dist", + "rootDir": "src", + "strict": true, + "esModuleInterop": true, + "skipLibCheck": true + }, + "include": ["src"] +} + +Create ${RELAYFILE}/sdk/relayfile-sdk/README.md with: +- Package name + description +- Install: npm install @relayfile/sdk +- Quick example: create client, list tree, read/write file +- Link to full docs + +Verify it builds: cd sdk/relayfile-sdk && npm run build + +Write all files to disk.`, + verification: { type: 'exit_code' }, + }) + + // ── Phase 4: Verify ─────────────────────────────────────────────── + + .step('verify-all', { + type: 'deterministic', + dependsOn: [ + 'implement-ci-workflow', 'implement-publish-workflow', + 'implement-release-workflow', 'prepare-sdk-package', + ], + command: `cd ${RELAYFILE} && echo "=== GitHub Actions ===" && \ +find .github/workflows -name "*.yml" | sort && \ +echo "" && echo "=== Provenance check ===" && \ +grep -c "provenance" .github/workflows/publish-npm.yml && \ +grep -c "id-token: write" .github/workflows/publish-npm.yml && \ +echo "" && echo "=== SDK builds ===" && \ +cd sdk/relayfile-sdk && npm install 2>&1 | tail -1 && npm run build 2>&1 | tail -3; echo "EXIT: $?"`, + captureOutput: true, + failOnError: false, + }) + + .step('fix-issues', { + agent: 'ci-lead', + dependsOn: ['verify-all'], + task: `Review and fix any issues. + +Results: +{{steps.verify-all.output}} + +Verify: +1. All 4 workflow files exist +2. publish-npm.yml has provenance enabled (--provenance flag AND id-token: write permission) +3. SDK builds cleanly +4. Workflow YAML is valid (no syntax errors) + +Fix any issues found.`, + verification: { type: 'exit_code' }, + }) + + .onError('retry', { maxRetries: 1, retryDelayMs: 10_000 }) + .run({ + cwd: RELAYFILE, + onEvent: (e: any) => console.log(`[${e.type}] ${e.stepName ?? e.step ?? ''} ${e.error ?? ''}`.trim()), + }); + +console.log(`\nCI/CD workflow: ${result.status}`); +} + +main().catch(console.error); diff --git a/workflows/relayfile-cloud-server.ts b/workflows/relayfile-cloud-server.ts new file mode 100644 index 00000000..9a5cf315 --- /dev/null +++ b/workflows/relayfile-cloud-server.ts @@ -0,0 +1,392 @@ +/** + * relayfile-cloud-server.ts + * + * Creates a separate private repo (AgentWorkforce/relayfile-cloud) for the + * Cloudflare Workers production server. The hosted server is closed-source + * while the Go server, mount daemon, CLI, and SDK remain open-source. + * + * Repo structure: + * relayfile (public): + * - Go server (local dev/testing) + * - Go mount daemon + * - Go CLI + * - TS SDK + * - OpenAPI spec + * - Docs, landing page + * + * relayfile-cloud (private): + * - CF Workers server (Hono + DO + R2 + D1) + * - Wrangler config + * - Deploy workflows + * - Production secrets management + * - Imports @relayfile/sdk for shared types + * + * The hosted server implements the same OpenAPI contract as the Go server. + * Both must pass the same E2E test suite. + * + * Run: agent-relay run workflows/relayfile-cloud-server.ts + */ + +import { workflow } from '@agent-relay/sdk/workflows'; + +const RELAYFILE = '/Users/khaliqgant/Projects/AgentWorkforce/relayfile'; +const RELAYCAST = '/Users/khaliqgant/Projects/AgentWorkforce/relaycast'; +const HOSTED = '/Users/khaliqgant/Projects/AgentWorkforce/relayfile-cloud'; + +async function main() { +const result = await workflow('relayfile-cloud-server') + .description('Create private repo for closed-source CF Workers hosted server') + .pattern('dag') + .channel('wf-relayfile-cloud') + .maxConcurrency(3) + .timeout(3_600_000) + + .agent('architect', { + cli: 'claude', + preset: 'lead', + role: 'Design repo structure, separation of concerns, shared types strategy', + cwd: RELAYFILE, + }) + .agent('server-dev', { + cli: 'codex', + preset: 'worker', + role: 'Implement the CF Workers server', + cwd: HOSTED, + }) + .agent('do-dev', { + cli: 'codex', + preset: 'worker', + role: 'Implement WorkspaceDO durable object', + cwd: HOSTED, + }) + .agent('infra-dev', { + cli: 'codex', + preset: 'worker', + role: 'Wrangler config, deploy workflows, secrets', + cwd: HOSTED, + }) + .agent('integrator', { + cli: 'claude', + preset: 'lead', + role: 'Wire everything together, verify against OpenAPI contract', + cwd: HOSTED, + }) + + // ── Phase 1: Read reference code ─────────────────────────────────── + + .step('read-go-store', { + type: 'deterministic', + command: `cat ${RELAYFILE}/internal/relayfile/store.go`, + captureOutput: true, + }) + + .step('read-go-http', { + type: 'deterministic', + command: `cat ${RELAYFILE}/internal/httpapi/server.go`, + captureOutput: true, + }) + + .step('read-openapi', { + type: 'deterministic', + command: `cat ${RELAYFILE}/openapi/relayfile-v1.openapi.yaml`, + captureOutput: true, + }) + + .step('read-relaycast-wrangler', { + type: 'deterministic', + command: `cat ${RELAYCAST}/wrangler.toml`, + captureOutput: true, + }) + + .step('read-relaycast-worker', { + type: 'deterministic', + command: `cat ${RELAYCAST}/packages/server/src/worker.ts`, + captureOutput: true, + }) + + .step('read-relaycast-channel-do', { + type: 'deterministic', + command: `head -200 ${RELAYCAST}/packages/server/src/durable-objects/channel.ts`, + captureOutput: true, + }) + + .step('read-relaycast-deploy-workflow', { + type: 'deterministic', + command: `cat ${RELAYCAST}/.github/workflows/deploy.yml`, + captureOutput: true, + }) + + .step('read-sdk-types', { + type: 'deterministic', + command: `cat ${RELAYFILE}/sdk/relayfile-sdk/src/types.ts`, + captureOutput: true, + }) + + // ── Phase 2: Create the private repo structure ───────────────────── + + .step('create-repo', { + type: 'deterministic', + command: `mkdir -p ${HOSTED} && cd ${HOSTED} && \ +git init 2>/dev/null || true && \ +echo "Repo initialized at ${HOSTED}"`, + captureOutput: true, + }) + + .step('design-repo', { + agent: 'architect', + dependsOn: [ + 'create-repo', 'read-go-store', 'read-go-http', 'read-openapi', + 'read-relaycast-wrangler', 'read-relaycast-worker', 'read-sdk-types', + ], + task: `Design the relayfile-cloud repo structure and scaffold it. + +The goal: a private repo containing ONLY the CF Workers production server. +Everything open-source stays in the public relayfile repo. + +Reference — relaycast's structure: +{{steps.read-relaycast-wrangler.output}} +{{steps.read-relaycast-worker.output}} + +OpenAPI contract (the server must implement this): +{{steps.read-openapi.output}} + +SDK types (import from @relayfile/sdk for shared types): +{{steps.read-sdk-types.output}} + +Go store (the logic to port): +{{steps.read-go-store.output}} + +Create these files at ${HOSTED}/: + +1. package.json: + - name: "relayfile-cloud" + - private: true + - dependencies: hono, @relayfile/sdk (for shared types) + - devDependencies: @cloudflare/workers-types, wrangler, typescript + +2. tsconfig.json + +3. wrangler.toml — following relaycast's pattern: + - name: relayfile-api + - D1: relayfile + - R2: relayfile-content + - Queues: relayfile-envelopes, relayfile-writeback + - DOs: WorkspaceDO + - Staging and preview environments + +4. src/env.ts — AppEnv with CF bindings + +5. src/worker.ts — Hono app entry, export WorkspaceDO + +6. src/types.ts — server-internal types (not in SDK) + Import shared types from @relayfile/sdk where possible. + +7. .gitignore: node_modules, dist, .wrangler + +8. README.md: + "# relayfile-cloud + Private Cloudflare Workers server for relayfile. + Implements the same API as the open-source Go server. + + ## Development + npm install + npm run dev # wrangler dev + npm run deploy # wrangler deploy + + ## Contract + The OpenAPI spec lives in the public repo: github.com/AgentWorkforce/relayfile/openapi/ + Both servers must pass the same E2E test suite." + +Write all files. Create directories as needed.`, + verification: { type: 'exit_code' }, + }) + + .step('verify-scaffold', { + type: 'deterministic', + dependsOn: ['design-repo'], + command: `cd ${HOSTED} && \ +[ -f package.json ] && [ -f wrangler.toml ] && [ -f src/worker.ts ] && [ -f src/env.ts ] && \ +echo "Scaffold OK" || (echo "MISSING FILES" && ls -la src/ && exit 1)`, + failOnError: true, + captureOutput: true, + }) + + // ── Phase 3: Implement (parallel) ────────────────────────────────── + + .step('implement-workspace-do', { + agent: 'do-dev', + dependsOn: ['verify-scaffold', 'read-go-store', 'read-relaycast-channel-do'], + task: `Implement WorkspaceDO at ${HOSTED}/src/durable-objects/workspace.ts. + +This is the core — one DO per workspace, holding the file tree. + +Go store logic to port: +{{steps.read-go-store.output}} + +Relaycast DO pattern to follow: +{{steps.read-relaycast-channel-do.output}} + +The WorkspaceDO must: + +1. Use DO SQLite storage for file metadata: + - files table: path, revision, content_type, content_ref (R2 key), size, encoding, updated_at, semantics_json + - events table: event_id, type, path, revision, origin, provider, correlation_id, timestamp + +2. Store file content in R2 (passed via env binding): + - Key format: {workspaceId}/{path}@{revision} + - DO stores content_ref pointing to R2 + +3. Methods (called via DO fetch): + - listTree(path, depth, cursor) + - readFile(path) — metadata from SQLite, content from R2 + - writeFile(path, ifMatch, content, contentType, encoding, semantics) + - Check revision conflict via ifMatch + - Increment revision counter + - Store content in R2, metadata in SQLite + - Emit event + - deleteFile(path, ifMatch) + - listEvents(provider?, cursor?, limit?) + - queryFiles(options) + - bulkWrite(files[]) — batch write without revision checks (for seeding) + - exportWorkspace(format) — export all files + +4. WebSocket hibernation for real-time event push + +5. Alarm handler for writeback retry scheduling + +Import shared types from @relayfile/sdk where they match. + +Write complete implementation.`, + verification: { type: 'exit_code' }, + }) + + .step('implement-routes', { + agent: 'server-dev', + dependsOn: ['verify-scaffold', 'read-go-http', 'read-openapi'], + task: `Implement Hono routes at ${HOSTED}/src/routes/. + +Go HTTP handlers to port: +{{steps.read-go-http.output}} + +OpenAPI spec: +{{steps.read-openapi.output}} + +Create route files matching the Go server's endpoints: +1. src/routes/fs.ts — tree, file CRUD, events, query, bulk, export, WebSocket +2. src/routes/sync.ts — sync status, ingress, dead-letter +3. src/routes/webhooks.ts — webhook ingestion +4. src/routes/ops.ts — operation tracking +5. src/routes/admin.ts — admin endpoints +6. src/routes/health.ts — GET /health + +Each route: +- Extracts workspaceId from URL params +- Gets WorkspaceDO stub via env.WORKSPACE_DO.idFromName(workspaceId) +- Forwards request to DO +- Returns JSON matching the OpenAPI spec + +Also create src/middleware/auth.ts — JWT verification (same as Go server's auth). + +Write all files.`, + verification: { type: 'exit_code' }, + }) + + .step('implement-deploy', { + agent: 'infra-dev', + dependsOn: ['verify-scaffold', 'read-relaycast-deploy-workflow'], + task: `Create deployment infrastructure. + +Relaycast deploy workflow: +{{steps.read-relaycast-deploy-workflow.output}} + +Create: + +1. ${HOSTED}/.github/workflows/deploy.yml: + - Push to main → deploy to production + - Push to staging branch → deploy to staging + - PR → deploy to preview environment + - Steps: install, typecheck, wrangler deploy + - D1 migrations before deploy + - Secrets: CLOUDFLARE_API_TOKEN, CLOUDFLARE_ACCOUNT_ID + +2. ${HOSTED}/.github/workflows/ci.yml: + - PR trigger: typecheck, lint + - Run E2E tests against wrangler dev (or Go server as reference) + +3. ${HOSTED}/src/db/migrations/0001_init.sql: + D1 migration for cross-workspace metadata: + - workspace_stats table + - dead_letters table + +4. Update wrangler.toml with: + - Staging environment + - Preview environment + - D1 migrations directory + +Write all files.`, + verification: { type: 'exit_code' }, + }) + + // ── Phase 4: Wire together ───────────────────────────────────────── + + .step('integrate', { + agent: 'integrator', + dependsOn: ['implement-workspace-do', 'implement-routes', 'implement-deploy'], + task: `Wire everything together in the hosted repo. + +1. Update src/worker.ts: + - Import and mount all routes + - Export WorkspaceDO + - Export queue consumers if any + - Add CORS and error handling middleware + +2. Install deps and typecheck: + cd ${HOSTED} && npm install && npx tsc --noEmit + +3. Fix any type errors. + +4. Verify wrangler config is valid: + npx wrangler deploy --dry-run 2>&1 | tail -20 + +5. Add a contract test: create ${HOSTED}/tests/contract.test.ts + that imports the OpenAPI spec from the public repo and verifies + all endpoints are implemented in the routes. + +Fix all issues until it type-checks cleanly.`, + verification: { type: 'exit_code' }, + }) + + .step('typecheck', { + type: 'deterministic', + dependsOn: ['integrate'], + command: `cd ${HOSTED} && npm install 2>&1 | tail -3 && npx tsc --noEmit 2>&1 | tail -10; echo "EXIT: $?"`, + captureOutput: true, + failOnError: false, + }) + + .step('fix-final', { + agent: 'integrator', + dependsOn: ['typecheck'], + task: `Fix any remaining issues. + +Typecheck: +{{steps.typecheck.output}} + +If EXIT: 0, create the GitHub repo: + gh repo create AgentWorkforce/relayfile-cloud --private --description "Hosted Cloudflare Workers server for relayfile" + cd ${HOSTED} && git add -A && git commit -m "initial: CF Workers server for relayfile" && git remote add origin git@github.com:AgentWorkforce/relayfile-cloud.git && git push -u origin main + +If there are errors, fix them first.`, + verification: { type: 'exit_code' }, + }) + + .onError('retry', { maxRetries: 1, retryDelayMs: 10_000 }) + .run({ + cwd: RELAYFILE, + onEvent: (e: any) => console.log(`[${e.type}] ${e.stepName ?? e.step ?? ''} ${e.error ?? ''}`.trim()), + }); + +console.log(`\nHosted server workflow: ${result.status}`); +} + +main().catch(console.error); diff --git a/workflows/relayfile-developer-experience.ts b/workflows/relayfile-developer-experience.ts new file mode 100644 index 00000000..682d2860 --- /dev/null +++ b/workflows/relayfile-developer-experience.ts @@ -0,0 +1,401 @@ +/** + * relayfile-developer-experience.ts + * + * Makes relayfile usable by humans — not just agents. + * Adds workspace CLI, auth flow, installable mount daemon, + * and user-facing documentation. + * + * After this workflow, a developer can: + * relayfile login + * relayfile workspace create my-project + * relayfile mount my-project ./src + * # colleague on another machine: + * relayfile mount my-project ./src + * # both see the same files, edits sync in real-time + * + * Run: agent-relay run workflows/relayfile-developer-experience.ts + */ + +import { workflow } from '@agent-relay/sdk/workflows'; + +const RELAYFILE = '/Users/khaliqgant/Projects/AgentWorkforce/relayfile'; + +async function main() { +const result = await workflow('relayfile-developer-experience') + .description('Make relayfile usable by humans — CLI, auth, install, docs') + .pattern('dag') + .channel('wf-relayfile-dx') + .maxConcurrency(3) + .timeout(3_600_000) + + .agent('dx-lead', { + cli: 'claude', + preset: 'lead', + role: 'Design the developer experience, CLI UX, auth flow', + cwd: RELAYFILE, + }) + .agent('cli-dev', { + cli: 'codex', + preset: 'worker', + role: 'Implement the relayfile CLI in Go', + cwd: RELAYFILE, + }) + .agent('docs-writer', { + cli: 'codex', + preset: 'worker', + role: 'Write user-facing documentation and guides', + cwd: RELAYFILE, + }) + .agent('install-dev', { + cli: 'codex', + preset: 'worker', + role: 'Build install scripts, Homebrew formula, GitHub releases', + cwd: RELAYFILE, + }) + + // ── Phase 1: Read existing code ──────────────────────────────────── + + .step('read-mount-cmd', { + type: 'deterministic', + command: `cat ${RELAYFILE}/cmd/relayfile-mount/main.go`, + captureOutput: true, + }) + + .step('read-server-cmd', { + type: 'deterministic', + command: `cat ${RELAYFILE}/cmd/relayfile/main.go`, + captureOutput: true, + }) + + .step('read-auth', { + type: 'deterministic', + command: `cat ${RELAYFILE}/internal/httpapi/auth.go`, + captureOutput: true, + }) + + .step('read-token-script', { + type: 'deterministic', + command: `cat ${RELAYFILE}/scripts/generate-dev-token.sh`, + captureOutput: true, + }) + + .step('read-readme', { + type: 'deterministic', + command: `cat ${RELAYFILE}/README.md`, + captureOutput: true, + }) + + .step('read-openapi', { + type: 'deterministic', + command: `cat ${RELAYFILE}/openapi/relayfile-v1.openapi.yaml | head -100`, + captureOutput: true, + }) + + // ── Phase 2: Design the CLI ──────────────────────────────────────── + + .step('design-cli', { + agent: 'dx-lead', + dependsOn: ['read-mount-cmd', 'read-auth', 'read-token-script'], + task: `Design the relayfile CLI UX. + +Current mount command: +{{steps.read-mount-cmd.output}} + +Current auth: +{{steps.read-auth.output}} + +Token generation: +{{steps.read-token-script.output}} + +Write a design doc at ${RELAYFILE}/docs/cli-design.md: + +**Commands:** + +1. relayfile login [--server URL] + - Opens browser to OAuth flow (or accepts API key) + - Stores credentials at ~/.relayfile/credentials.json + - Validates by calling GET /health on the server + - Default server: https://relayfile-api.agentworkforce.workers.dev + +2. relayfile workspace create + - Creates a new workspace + - Returns workspace ID + - Stores in ~/.relayfile/workspaces.json + +3. relayfile workspace list + - Lists all workspaces you have access to + +4. relayfile workspace delete + - Deletes a workspace (with confirmation) + +5. relayfile mount [local-dir] + - Mounts a workspace to a local directory (default: current dir) + - Starts relayfile-mount daemon in foreground (Ctrl+C to stop) + - Shows sync activity in real-time + - Auto-detects server URL from ~/.relayfile/credentials.json + +6. relayfile seed [dir] + - Bulk uploads a local directory to a workspace + - Uses bulk API for speed + - Shows progress bar + +7. relayfile export [--format tar|json|patch] [--output file] + - Downloads workspace contents + +8. relayfile status + - Shows workspace stats: file count, last activity, connected agents + +**Config file: ~/.relayfile/credentials.json** +{ + "server": "https://relayfile-api.agentworkforce.workers.dev", + "token": "eyJ...", + "refreshToken": "...", + "expiresAt": "..." +} + +**Auth flow options:** +- Option A: API key (simple, for agents and CI) +- Option B: OAuth via browser (for humans, like gh auth login) +- Option C: Token from environment variable RELAYFILE_TOKEN (for CI/CD) + +Start with A and C. OAuth can come later. + +Write the complete design doc.`, + verification: { type: 'exit_code' }, + }) + + // ── Phase 3: Implementation (parallel) ───────────────────────────── + + .step('implement-cli', { + agent: 'cli-dev', + dependsOn: ['design-cli', 'read-mount-cmd', 'read-server-cmd'], + task: `Implement the relayfile CLI as a unified Go binary. + +Design: +{{steps.design-cli.output}} + +Current mount command: +{{steps.read-mount-cmd.output}} + +The relayfile binary should be a single Go binary with subcommands. +Merge the existing relayfile-mount functionality as the "mount" subcommand. + +Create ${RELAYFILE}/cmd/relayfile-cli/main.go: + +Use Go's flag package or a lightweight CLI library (cobra is fine if already a dep, otherwise use flag with manual subcommand routing). + +Subcommands: + +1. relayfile login --server URL --token TOKEN + - If --token provided, store it directly + - If no --token, prompt for API key + - Validate: GET {server}/health + - Store: ~/.relayfile/credentials.json + +2. relayfile workspace create NAME + - Load credentials + - POST to create workspace (or just mint a token for the workspace name) + - For MVP: workspaces are created on-demand when you first write to them + - Store workspace name in ~/.relayfile/workspaces.json + +3. relayfile workspace list + - Load credentials + - GET /v1/admin/workspaces or list from local config + +4. relayfile mount WORKSPACE [LOCAL_DIR] + - Load credentials from ~/.relayfile/credentials.json + - Default LOCAL_DIR to current directory + - Reuse the existing mountsync.Syncer logic from internal/mountsync + - Run in foreground, show sync activity + - Handle Ctrl+C gracefully + +5. relayfile seed WORKSPACE [DIR] + - Load credentials + - Walk DIR, read all files + - POST /v1/workspaces/{ws}/fs/bulk with file contents + - Show progress: "Seeding 142/250 files..." + +6. relayfile export WORKSPACE --format FORMAT --output FILE + - GET /v1/workspaces/{ws}/fs/export?format=FORMAT + - Write to FILE or stdout + +7. relayfile status WORKSPACE + - GET /v1/workspaces/{ws}/sync/status + - Print file count, last activity + +Also update the Makefile/build: +- go build -o relayfile ./cmd/relayfile-cli +- Keep the server binary as: go build -o relayfile-server ./cmd/relayfile +- Keep the mount-only binary as: go build -o relayfile-mount ./cmd/relayfile-mount + +Write all files to disk.`, + verification: { type: 'exit_code' }, + }) + + .step('implement-install', { + agent: 'install-dev', + dependsOn: ['implement-cli'], + task: `Create installation scripts and release automation. + +Create: + +1. ${RELAYFILE}/scripts/install.sh + - curl-based installer: curl -fsSL https://relayfile.dev/install.sh | sh + - Detects OS (darwin/linux) and arch (amd64/arm64) + - Downloads the latest binary from GitHub releases + - Installs to /usr/local/bin/relayfile + - Verifies checksum + +2. ${RELAYFILE}/Makefile + - build: builds all three binaries (relayfile-cli, relayfile-server, relayfile-mount) + - build-all: cross-compile for darwin/linux x amd64/arm64 + - install: builds and copies to /usr/local/bin + - test: runs go test ./... + - release: creates release artifacts with checksums + +3. ${RELAYFILE}/.github/workflows/release.yml + - Triggered on tag push (v*) + - Cross-compile for darwin/linux x amd64/arm64 + - Create GitHub release with binaries + checksums + - Build and push Docker image + +4. ${RELAYFILE}/Formula/relayfile.rb (Homebrew formula template) + - Install the CLI binary + - Depends on: nothing (static Go binary) + +Write all files to disk.`, + verification: { type: 'exit_code' }, + }) + + .step('write-docs', { + agent: 'docs-writer', + dependsOn: ['design-cli', 'read-readme', 'read-openapi'], + task: `Write user-facing documentation. + +CLI design: +{{steps.design-cli.output}} + +Current README: +{{steps.read-readme.output}} + +Create/update: + +1. Update ${RELAYFILE}/README.md: + - Add "Quick Start" section at the top: + Install: curl -fsSL https://relayfile.dev/install.sh | sh + Login: relayfile login --server http://localhost:9090 --token dev-token + Seed: relayfile seed my-project ./src + Mount: relayfile mount my-project ./src + - Keep existing technical content below + - Add "Collaborate" section showing two-machine setup + +2. Create ${RELAYFILE}/docs/guides/getting-started.md: + - Prerequisites (Go for local server, or use hosted) + - Start the server locally: go run ./cmd/relayfile + - Login and create a workspace + - Seed files from a project + - Mount on two machines + - Watch files sync + +3. Create ${RELAYFILE}/docs/guides/cloud-integration.md: + - How relayfile integrates with Agent Relay cloud workflows + - Automatic workspace creation per workflow run + - relayfile-mount in sandbox snapshots + - How agents share files via relayfile + +4. Create ${RELAYFILE}/docs/guides/collaboration.md: + - Two humans collaborating: + Machine A: relayfile mount project-x ./src + Machine B: relayfile mount project-x ./src + Edit a file on A → appears on B in ~1-2 seconds + - Human + agent collaboration: + Human mounts locally, agent mounts in cloud sandbox + Both see the same files + - Conflict resolution: what happens when both edit the same file + +5. Create ${RELAYFILE}/docs/api-reference.md: + - Document all REST endpoints with curl examples + - Group by: Filesystem, Sync, Webhooks, Operations, Admin + - Include auth header examples + +Write all docs to disk.`, + verification: { type: 'exit_code' }, + }) + + // ── Phase 4: SDK publish prep ────────────────────────────────────── + + .step('prepare-sdk-publish', { + agent: 'cli-dev', + dependsOn: ['implement-cli'], + task: `Prepare the TypeScript SDK for npm publishing. + +Read the current SDK: cat ${RELAYFILE}/sdk/relayfile-sdk/package.json + +Update ${RELAYFILE}/sdk/relayfile-sdk/package.json: +- name: "@relayfile/sdk" (or "relayfile-sdk") +- version: "0.1.0" +- main: "dist/index.js" +- types: "dist/index.d.ts" +- files: ["dist"] +- scripts: + build: "tsc" + prepublishOnly: "npm run build" +- repository, license, description fields + +Create ${RELAYFILE}/sdk/relayfile-sdk/tsconfig.json: +- target: ES2022 +- module: ESNext +- moduleResolution: bundler +- declaration: true +- outDir: dist +- rootDir: src + +Verify it builds: cd sdk/relayfile-sdk && npm run build + +Write changes to disk.`, + verification: { type: 'exit_code' }, + }) + + // ── Phase 5: Verify everything ───────────────────────────────────── + + .step('verify-all', { + type: 'deterministic', + dependsOn: ['implement-cli', 'implement-install', 'write-docs', 'prepare-sdk-publish'], + command: `cd ${RELAYFILE} && echo "=== CLI builds ===" && \ +go build ./cmd/relayfile-cli 2>&1 && echo "CLI: OK" || echo "CLI: FAIL" && \ +echo "" && echo "=== Docs ===" && \ +for f in docs/guides/getting-started.md docs/guides/collaboration.md docs/guides/cloud-integration.md docs/api-reference.md docs/cli-design.md; do + [ -f "$f" ] && echo "$f: OK" || echo "$f: MISSING" +done && \ +echo "" && echo "=== Install script ===" && \ +[ -f scripts/install.sh ] && echo "install.sh: OK" || echo "install.sh: MISSING" && \ +echo "" && echo "=== SDK builds ===" && \ +cd sdk/relayfile-sdk && npm install 2>&1 | tail -1 && npx tsc --noEmit 2>&1 | tail -3; echo "EXIT: $?"`, + captureOutput: true, + failOnError: false, + }) + + .step('fix-issues', { + agent: 'dx-lead', + dependsOn: ['verify-all'], + task: `Fix any remaining issues. + +Results: +{{steps.verify-all.output}} + +Fix any build failures or missing files. +Review the docs for clarity and completeness. +Verify the README quick start actually works by reading the commands.`, + verification: { type: 'exit_code' }, + }) + + .onError('retry', { maxRetries: 1, retryDelayMs: 10_000 }) + .run({ + cwd: RELAYFILE, + onEvent: (e: any) => console.log(`[${e.type}] ${e.stepName ?? e.step ?? ''} ${e.error ?? ''}`.trim()), + }); + +console.log(`\nDeveloper experience workflow: ${result.status}`); +} + +main().catch(console.error); diff --git a/workflows/relayfile-landing-page.ts b/workflows/relayfile-landing-page.ts new file mode 100644 index 00000000..ecdd8881 --- /dev/null +++ b/workflows/relayfile-landing-page.ts @@ -0,0 +1,267 @@ +/** + * relayfile-landing-page.ts + * + * Builds a landing page for relayfile — "Real-time filesystem for humans and agents." + * + * The site lives at relayfile/site/ and deploys to Cloudflare Pages. + * Uses Astro + Tailwind (same stack as relaycast's site) for a fast, + * static marketing site. + * + * Run: agent-relay run workflows/relayfile-landing-page.ts + */ + +import { workflow } from '@agent-relay/sdk/workflows'; + +const RELAYFILE = '/Users/khaliqgant/Projects/AgentWorkforce/relayfile'; +const RELAYCAST_SITE = '/Users/khaliqgant/Projects/AgentWorkforce/relaycast/site'; + +async function main() { +const result = await workflow('relayfile-landing-page') + .description('Build the relayfile landing page — real-time filesystem for humans and agents') + .pattern('dag') + .channel('wf-relayfile-landing') + .maxConcurrency(3) + .timeout(3_600_000) + + .agent('designer', { + cli: 'claude', + preset: 'lead', + role: 'Content strategy, copy, page structure, design direction', + cwd: RELAYFILE, + }) + .agent('frontend', { + cli: 'codex', + preset: 'worker', + role: 'Implement Astro + Tailwind site', + cwd: RELAYFILE, + }) + .agent('illustrator', { + cli: 'codex', + preset: 'worker', + role: 'Create SVG diagrams and visual assets', + cwd: RELAYFILE, + }) + + // ── Phase 1: Read existing context ───────────────────────────────── + + .step('read-readme', { + type: 'deterministic', + command: `cat ${RELAYFILE}/README.md`, + captureOutput: true, + }) + + .step('read-spec', { + type: 'deterministic', + command: `cat ${RELAYFILE}/docs/relayfile-v1-spec.md`, + captureOutput: true, + }) + + .step('read-openapi', { + type: 'deterministic', + command: `head -100 ${RELAYFILE}/openapi/relayfile-v1.openapi.yaml`, + captureOutput: true, + }) + + .step('read-relaycast-site', { + type: 'deterministic', + command: `ls ${RELAYCAST_SITE}/src/ 2>/dev/null && cat ${RELAYCAST_SITE}/package.json 2>/dev/null | head -20 || echo "No relaycast site found"`, + captureOutput: true, + }) + + .step('read-architecture-doc', { + type: 'deterministic', + command: `cat ${RELAYFILE}/docs/architecture-ascii.md`, + captureOutput: true, + }) + + // ── Phase 2: Designer creates content and structure ──────────────── + + .step('design-site', { + agent: 'designer', + dependsOn: ['read-readme', 'read-spec', 'read-openapi', 'read-architecture-doc'], + task: `Create the content and page structure for the relayfile landing page. + +relayfile README: +{{steps.read-readme.output}} + +Architecture: +{{steps.read-architecture-doc.output}} + +Spec highlights: +{{steps.read-spec.output}} + +Write ${RELAYFILE}/site/CONTENT.md with the complete page content: + +**Hero section:** +- Headline: "Real-time filesystem for humans and agents" +- Subhead: "A revision-controlled, programmable filesystem that syncs everywhere. Mount it locally, in the cloud, or in a sandbox — everyone sees the same files." +- CTA: "Get Started" → docs, "View on GitHub" → repo + +**Problem section: "Files are the universal interface"** +- Every tool, every agent, every developer works with files +- But sharing files across machines, sandboxes, and agents is broken +- Git is async. FUSE volumes are slow. Cloud drives have no revision control. +- There's no real-time, programmable filesystem that just works everywhere. + +**Solution section: "relayfile"** +Three columns: +1. Mount anywhere — "relayfile-mount syncs a local directory to a shared workspace. Run it on your laptop, in a Docker container, or in a cloud sandbox. Agents and humans see the same files." +2. Revision-controlled — "Every write is tracked with a revision. Conflicts are detected, not silently overwritten. Full event history of who changed what." +3. Programmable — "REST API for reads, writes, queries. Webhook ingestion for external sources. Writeback queues for bidirectional sync. Build workflows on top of files." + +**Use cases section:** +1. "Multi-agent coding" — Backend and frontend agents work on the same codebase simultaneously. Changes sync in real-time. +2. "Human + agent collaboration" — Watch an agent work in real-time on your local machine. Edit a file to course-correct. The agent picks it up immediately. +3. "Cross-machine dev" — Mount the same workspace on your laptop and your cloud dev environment. No git push/pull ceremony. +4. "Tool integration" — Notion pages, GitHub files, Linear tickets — all projected into one filesystem via webhooks + writeback. + +**How it works section:** +Visual diagram showing: + Developer laptop ←→ relayfile ←→ Cloud sandbox (Agent A) + ←→ Cloud sandbox (Agent B) + +**API preview section:** +Show 3 curl examples: write a file, read a file, list events + +**Architecture section:** +- Cloudflare Workers + Durable Objects + R2 +- One DO per workspace — zero-conflict single-writer coordination +- Event stream via WebSocket for real-time push +- Mount daemon is a single Go binary — bake into any image + +**Footer:** +- GitHub link, docs link, "Built by Agent Workforce" + +Write the complete CONTENT.md with all copy.`, + verification: { type: 'exit_code' }, + }) + + // ── Phase 3: Build the site (parallel) ───────────────────────────── + + .step('scaffold-site', { + agent: 'frontend', + dependsOn: ['design-site', 'read-relaycast-site'], + task: `Scaffold the Astro + Tailwind site at ${RELAYFILE}/site/. + +Reference relaycast's site structure: +{{steps.read-relaycast-site.output}} + +Create: + +1. ${RELAYFILE}/site/package.json: + - Dependencies: astro, @astrojs/tailwind, tailwindcss + - Scripts: dev, build, preview + +2. ${RELAYFILE}/site/astro.config.mjs: + - Integrations: tailwind + - Output: static + +3. ${RELAYFILE}/site/tailwind.config.cjs: + - Dark mode, zinc/cyan color scheme (match the agent workforce brand) + +4. ${RELAYFILE}/site/tsconfig.json + +5. ${RELAYFILE}/site/src/layouts/Layout.astro: + - HTML shell with dark bg (#09090b), meta tags, favicon + - Font: Inter or system-ui + +6. ${RELAYFILE}/site/src/pages/index.astro: + - Import Layout, render all sections from the content doc + - Read content: cat ${RELAYFILE}/site/CONTENT.md + - Hero with gradient text (cyan → blue) + - Sections with prose styling + - API preview with syntax-highlighted code blocks + - Responsive: mobile-first, max-w-6xl container + +7. ${RELAYFILE}/site/src/components/Hero.astro +8. ${RELAYFILE}/site/src/components/FeatureGrid.astro +9. ${RELAYFILE}/site/src/components/UseCases.astro +10. ${RELAYFILE}/site/src/components/ApiPreview.astro +11. ${RELAYFILE}/site/src/components/Architecture.astro +12. ${RELAYFILE}/site/src/components/Footer.astro + +Use Tailwind utility classes. Dark theme throughout. +Zinc-950 backgrounds, zinc-100 text, cyan-400 accents. + +Write all files to disk.`, + verification: { type: 'exit_code' }, + }) + + .step('create-diagrams', { + agent: 'illustrator', + dependsOn: ['design-site', 'read-architecture-doc'], + task: `Create SVG diagrams for the landing page. + +Architecture reference: +{{steps.read-architecture-doc.output}} + +Content plan: +{{steps.design-site.output}} + +Create these SVG files at ${RELAYFILE}/site/public/: + +1. diagram-sync.svg — Shows the sync flow: + Three boxes: "Your Laptop", "relayfile", "Cloud Sandbox" + Bidirectional arrows between them + Inside each box: a file tree icon + Style: dark background (#09090b), cyan (#22d3ee) lines, white text, rounded corners + Keep it clean and minimal — no gradients, no shadows + +2. diagram-architecture.svg — Shows the stack: + Layers: "Cloudflare Workers" → "Durable Object (per workspace)" → "R2 (content)" / "D1 (metadata)" + Side: "Mount Daemon" connecting to "Workers" via HTTP/WebSocket + Style: same dark theme + +3. diagram-agents.svg — Shows multi-agent collaboration: + Center: "relayfile workspace" + Around it: "Backend Agent", "Frontend Agent", "Human Developer" + All connected with sync arrows + File change events flowing between them + Style: same dark theme + +Each SVG should be self-contained, ~400x250px, viewBox-based (scales well). +Use basic SVG elements (rect, text, line, path). No external dependencies. + +Write all SVGs to disk.`, + verification: { type: 'exit_code' }, + }) + + // ── Phase 4: Assemble and verify ────────────────────────────────── + + .step('verify-site', { + type: 'deterministic', + dependsOn: ['scaffold-site', 'create-diagrams'], + command: `cd ${RELAYFILE}/site && \ +echo "=== Files ===" && find src public -type f | sort && \ +echo "" && echo "=== Install ===" && npm install 2>&1 | tail -3 && \ +echo "" && echo "=== Build ===" && npx astro build 2>&1 | tail -10; echo "EXIT: $?"`, + captureOutput: true, + failOnError: false, + }) + + .step('fix-build', { + agent: 'designer', + dependsOn: ['verify-site'], + task: `Fix any build errors in the site. + +Build output: +{{steps.verify-site.output}} + +If EXIT: 0, the site builds. Summarize what was created. +If there are errors, read the failing files and fix them. +Run npx astro build again to verify. + +Also review the content — make sure the copy reads well and the sections flow logically.`, + verification: { type: 'exit_code' }, + }) + + .onError('retry', { maxRetries: 1, retryDelayMs: 10_000 }) + .run({ + cwd: RELAYFILE, + onEvent: (e: any) => console.log(`[${e.type}] ${e.stepName ?? e.step ?? ''} ${e.error ?? ''}`.trim()), + }); + +console.log(`\nLanding page workflow: ${result.status}`); +} + +main().catch(console.error); From 0139fc01b029c9e19becf678ccaeaf7d63ff05f9 Mon Sep 17 00:00:00 2001 From: Khaliq Date: Tue, 24 Mar 2026 15:18:15 +0100 Subject: [PATCH 02/10] Add CLI, Homebrew formula, CI release workflow, install script, and docs Adds relayfile-cli with login/mount/seed/export commands, Homebrew tap formula, GitHub Actions release workflow, install script, and user-facing docs (API reference, CLI design, guides). Updates .gitignore to exclude compiled binaries and agent tool configs. Co-Authored-By: Claude Opus 4.6 (1M context) --- .github/workflows/release.yml | 74 +++ .gitignore | 13 + Formula/relayfile.rb | 33 + Makefile | 69 ++ README.md | 62 ++ cmd/relayfile-cli/main.go | 940 ++++++++++++++++++++++++++++ docs/api-reference.md | 417 ++++++++++++ docs/cli-design.md | 424 +++++++++++++ docs/guides/cloud-integration.md | 88 +++ docs/guides/collaboration.md | 83 +++ docs/guides/getting-started.md | 130 ++++ scripts/install.sh | 132 ++++ sdk/relayfile-sdk/package-lock.json | 1 + sdk/relayfile-sdk/package.json | 13 +- sdk/relayfile-sdk/tsconfig.json | 14 +- 15 files changed, 2476 insertions(+), 17 deletions(-) create mode 100644 .github/workflows/release.yml create mode 100644 Formula/relayfile.rb create mode 100644 Makefile create mode 100644 cmd/relayfile-cli/main.go create mode 100644 docs/api-reference.md create mode 100644 docs/cli-design.md create mode 100644 docs/guides/cloud-integration.md create mode 100644 docs/guides/collaboration.md create mode 100644 docs/guides/getting-started.md create mode 100755 scripts/install.sh diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 00000000..78e4eed0 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,74 @@ +name: release + +on: + push: + tags: + - "v*" + +permissions: + contents: write + packages: write + +jobs: + release: + runs-on: ubuntu-latest + env: + VERSION: ${{ github.ref_name }} + steps: + - name: Check out source + uses: actions/checkout@v4 + + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version-file: go.mod + + - name: Run tests + run: make test + + - name: Build release artifacts + run: make release VERSION="${VERSION}" + + - name: Create GitHub release + uses: softprops/action-gh-release@v2 + with: + generate_release_notes: true + files: | + dist/*.tar.gz + dist/checksums.txt + + docker: + runs-on: ubuntu-latest + needs: release + steps: + - name: Check out source + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to GHCR + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract Docker metadata + id: meta + uses: docker/metadata-action@v5 + with: + images: ghcr.io/${{ github.repository }} + tags: | + type=ref,event=tag + type=raw,value=latest + + - name: Build and push Docker image + uses: docker/build-push-action@v6 + with: + context: . + file: ./Dockerfile + platforms: linux/amd64,linux/arm64 + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} diff --git a/.gitignore b/.gitignore index b0b3308a..7fa58fce 100644 --- a/.gitignore +++ b/.gitignore @@ -35,3 +35,16 @@ dist/ # Local mount mirror /.livefs/ + +# Compiled binaries +/relayfile +/relayfile-cli +/relayfile-mount +/relayfile-server + +# Agent tool configs +.factory/ +.gemini/ +.trajectories/ +.mcp.json +opencode.json diff --git a/Formula/relayfile.rb b/Formula/relayfile.rb new file mode 100644 index 00000000..036fa6cf --- /dev/null +++ b/Formula/relayfile.rb @@ -0,0 +1,33 @@ +class Relayfile < Formula + desc "CLI for RelayFile collaborative file workspaces" + homepage "https://github.com/agentworkforce/relayfile" + version "0.0.0" + + on_macos do + if Hardware::CPU.arm? + url "https://github.com/agentworkforce/relayfile/releases/download/v#{version}/relayfile_darwin_arm64.tar.gz" + sha256 "REPLACE_WITH_DARWIN_ARM64_SHA256" + else + url "https://github.com/agentworkforce/relayfile/releases/download/v#{version}/relayfile_darwin_amd64.tar.gz" + sha256 "REPLACE_WITH_DARWIN_AMD64_SHA256" + end + end + + on_linux do + if Hardware::CPU.arm? + url "https://github.com/agentworkforce/relayfile/releases/download/v#{version}/relayfile_linux_arm64.tar.gz" + sha256 "REPLACE_WITH_LINUX_ARM64_SHA256" + else + url "https://github.com/agentworkforce/relayfile/releases/download/v#{version}/relayfile_linux_amd64.tar.gz" + sha256 "REPLACE_WITH_LINUX_AMD64_SHA256" + end + end + + def install + bin.install "relayfile" + end + + test do + system "#{bin}/relayfile", "help" + end +end diff --git a/Makefile b/Makefile new file mode 100644 index 00000000..7918aa59 --- /dev/null +++ b/Makefile @@ -0,0 +1,69 @@ +SHELL := /bin/sh + +GO ?= go +BIN_DIR ?= bin +DIST_DIR ?= dist +INSTALL_DIR ?= /usr/local/bin +VERSION ?= $(shell git describe --tags --always --dirty 2>/dev/null || echo dev) +COMMIT ?= $(shell git rev-parse --short HEAD 2>/dev/null || echo unknown) +LDFLAGS ?= -s -w -X main.version=$(VERSION) -X main.commit=$(COMMIT) +GOFLAGS ?= + +CLI_PKG := ./cmd/relayfile-cli +SERVER_PKG := ./cmd/relayfile +MOUNT_PKG := ./cmd/relayfile-mount + +CLI_BIN := relayfile +SERVER_BIN := relayfile-server +MOUNT_BIN := relayfile-mount + +TARGETS := darwin/amd64 darwin/arm64 linux/amd64 linux/arm64 +RELEASE_BINS := $(CLI_BIN) $(SERVER_BIN) $(MOUNT_BIN) + +.PHONY: build build-all install test release clean + +build: + mkdir -p $(BIN_DIR) + CGO_ENABLED=0 $(GO) build $(GOFLAGS) -ldflags "$(LDFLAGS)" -o $(BIN_DIR)/$(CLI_BIN) $(CLI_PKG) + CGO_ENABLED=0 $(GO) build $(GOFLAGS) -ldflags "$(LDFLAGS)" -o $(BIN_DIR)/$(SERVER_BIN) $(SERVER_PKG) + CGO_ENABLED=0 $(GO) build $(GOFLAGS) -ldflags "$(LDFLAGS)" -o $(BIN_DIR)/$(MOUNT_BIN) $(MOUNT_PKG) + +build-all: + rm -rf $(DIST_DIR) + mkdir -p $(DIST_DIR) + set -eu; \ + for target in $(TARGETS); do \ + goos=$${target%/*}; \ + goarch=$${target#*/}; \ + out_dir="$(DIST_DIR)/$${goos}_$${goarch}"; \ + mkdir -p "$${out_dir}"; \ + CGO_ENABLED=0 GOOS=$${goos} GOARCH=$${goarch} $(GO) build $(GOFLAGS) -ldflags "$(LDFLAGS)" -o "$${out_dir}/$(CLI_BIN)" $(CLI_PKG); \ + CGO_ENABLED=0 GOOS=$${goos} GOARCH=$${goarch} $(GO) build $(GOFLAGS) -ldflags "$(LDFLAGS)" -o "$${out_dir}/$(SERVER_BIN)" $(SERVER_PKG); \ + CGO_ENABLED=0 GOOS=$${goos} GOARCH=$${goarch} $(GO) build $(GOFLAGS) -ldflags "$(LDFLAGS)" -o "$${out_dir}/$(MOUNT_BIN)" $(MOUNT_PKG); \ + done + +install: build + install -d $(INSTALL_DIR) + install $(BIN_DIR)/$(CLI_BIN) $(INSTALL_DIR)/$(CLI_BIN) + install $(BIN_DIR)/$(SERVER_BIN) $(INSTALL_DIR)/$(SERVER_BIN) + install $(BIN_DIR)/$(MOUNT_BIN) $(INSTALL_DIR)/$(MOUNT_BIN) + +test: + $(GO) test ./... + +release: clean build-all + set -eu; \ + for target in $(TARGETS); do \ + goos=$${target%/*}; \ + goarch=$${target#*/}; \ + work_dir="$(DIST_DIR)/$${goos}_$${goarch}"; \ + for bin in $(RELEASE_BINS); do \ + archive="$(DIST_DIR)/$${bin}_$${goos}_$${goarch}.tar.gz"; \ + LC_ALL=C tar -C "$${work_dir}" -czf "$${archive}" "$${bin}"; \ + done; \ + done + cd $(DIST_DIR) && \ + (if command -v sha256sum >/dev/null 2>&1; then sha256sum ./*.tar.gz > checksums.txt; else shasum -a 256 ./*.tar.gz > checksums.txt; fi) + +clean: + rm -rf $(BIN_DIR) $(DIST_DIR) diff --git a/README.md b/README.md index 021ec01f..2691718c 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,61 @@ Queue-first virtual filesystem-over-REST that ingests noisy external webhooks, projects a file tree, and executes conflict-safe writeback with retries, dead-lettering, and replay. +## Quick Start + +Install the RelayFile CLI: + +```bash +curl -fsSL https://relayfile.dev/install.sh | sh +``` + +Log in to a local RelayFile server: + +```bash +relayfile login --server http://localhost:9090 --token dev-token +``` + +Seed a workspace from an existing project: + +```bash +relayfile seed my-project ./src +``` + +Mount that workspace into a local directory: + +```bash +relayfile mount my-project ./src +``` + +For a step-by-step walkthrough, see `docs/guides/getting-started.md`. + +## Collaborate + +RelayFile is designed for shared files between humans and agents. + +Two-machine setup: + +```bash +# Machine A +relayfile mount project-x ./src + +# Machine B +relayfile mount project-x ./src +``` + +When someone edits a file on Machine A, the change appears on Machine B after the next sync cycle, typically in about 1-2 seconds with the default interval. + +Human + agent setup: + +- Mount the same workspace on your laptop with `relayfile mount`. +- Mount the workspace inside an agent sandbox with `relayfile-mount` or the user-facing `relayfile mount` workflow. +- Both sides read and write the same virtual project tree through RelayFile. + +More collaboration examples: + +- `docs/guides/collaboration.md` +- `docs/guides/cloud-integration.md` + ## What this service does - Exposes a workspace-scoped filesystem API (`/fs/tree`, `/fs/file`, `/fs/events`). @@ -251,6 +306,13 @@ Mount client env vars: - `RELAYFILE_MOUNT_INTERVAL_JITTER` - `RELAYFILE_MOUNT_TIMEOUT` +User-facing guides: + +- `docs/guides/getting-started.md` +- `docs/guides/collaboration.md` +- `docs/guides/cloud-integration.md` +- `docs/api-reference.md` + ## SDK TypeScript SDK lives in `sdk/relayfile-sdk`. diff --git a/cmd/relayfile-cli/main.go b/cmd/relayfile-cli/main.go new file mode 100644 index 00000000..b24138f2 --- /dev/null +++ b/cmd/relayfile-cli/main.go @@ -0,0 +1,940 @@ +package main + +import ( + "bufio" + "bytes" + "context" + "crypto/rand" + "encoding/base64" + "encoding/hex" + "encoding/json" + "errors" + "flag" + "fmt" + "io" + "log" + mathrand "math/rand" + "mime" + "net/http" + "net/url" + "os" + "os/signal" + "path/filepath" + "sort" + "strconv" + "strings" + "syscall" + "time" + "unicode/utf8" + + "github.com/agentworkforce/relayfile/internal/mountsync" +) + +const ( + defaultServerURL = "http://127.0.0.1:8080" + configDirName = ".relayfile" +) + +type credentials struct { + Server string `json:"server"` + Token string `json:"token"` + UpdatedAt string `json:"updatedAt,omitempty"` +} + +type workspaceCatalog struct { + Workspaces []workspaceRecord `json:"workspaces"` +} + +type workspaceRecord struct { + Name string `json:"name"` + CreatedAt string `json:"createdAt"` + LastUsedAt string `json:"lastUsedAt,omitempty"` +} + +type apiClient struct { + baseURL string + token string + httpClient *http.Client +} + +type bulkWriteRequest struct { + Files []bulkWriteFile `json:"files"` +} + +type bulkWriteFile struct { + Path string `json:"path"` + ContentType string `json:"contentType"` + Content string `json:"content"` + Encoding string `json:"encoding,omitempty"` +} + +type bulkWriteResponse struct { + Written int `json:"written"` + ErrorCount int `json:"errorCount"` + Errors []bulkWriteError `json:"errors"` + CorrelationID string `json:"correlationId"` +} + +type bulkWriteError struct { + Path string `json:"path"` + Code string `json:"code"` + Message string `json:"message"` +} + +type syncStatusResponse struct { + WorkspaceID string `json:"workspaceId"` + Providers []syncProviderStatus `json:"providers"` +} + +type syncProviderStatus struct { + Provider string `json:"provider"` + Status string `json:"status"` + Cursor *string `json:"cursor"` + WatermarkTs *string `json:"watermarkTs"` + LagSeconds int `json:"lagSeconds"` + LastError *string `json:"lastError"` + FailureCodes map[string]int `json:"failureCodes"` + DeadLetteredEnvelopes int `json:"deadLetteredEnvelopes"` + DeadLetteredOps int `json:"deadLetteredOps"` +} + +type exportedFile struct { + Path string `json:"path"` + Revision string `json:"revision"` + ContentType string `json:"contentType"` + Content string `json:"content"` + Encoding string `json:"encoding,omitempty"` + LastEdited string `json:"lastEditedAt,omitempty"` +} + +type adminWorkspaceList struct { + Workspaces []string `json:"workspaces"` +} + +type apiError struct { + StatusCode int + Code string + Message string +} + +func (e *apiError) Error() string { + if e.Code == "" { + return fmt.Sprintf("http %d: %s", e.StatusCode, e.Message) + } + return fmt.Sprintf("http %d %s: %s", e.StatusCode, e.Code, e.Message) +} + +func main() { + log.SetFlags(0) + if err := run(os.Args[1:], os.Stdin, os.Stdout, os.Stderr); err != nil { + fmt.Fprintln(os.Stderr, "error:", err) + os.Exit(1) + } +} + +func run(args []string, stdin io.Reader, stdout, stderr io.Writer) error { + if len(args) == 0 { + printUsage(stderr) + return nil + } + + switch args[0] { + case "login": + return runLogin(args[1:], stdin, stdout) + case "workspace": + return runWorkspace(args[1:], stdout) + case "mount": + return runMount(args[1:], stdout) + case "seed": + return runSeed(args[1:], stdout) + case "export": + return runExport(args[1:], stdout) + case "status": + return runStatus(args[1:], stdout) + case "help", "-h", "--help": + printUsage(stdout) + return nil + default: + printUsage(stderr) + return fmt.Errorf("unknown subcommand %q", args[0]) + } +} + +func printUsage(w io.Writer) { + fmt.Fprintln(w, `relayfile is the RelayFile CLI. + +Usage: + relayfile login --server URL [--token TOKEN] + relayfile workspace create NAME + relayfile workspace list + relayfile mount WORKSPACE [LOCAL_DIR] [flags] + relayfile seed WORKSPACE [DIR] + relayfile export WORKSPACE --format FORMAT [--output FILE] + relayfile status WORKSPACE + +Subcommands: + login Store credentials in ~/.relayfile/credentials.json + workspace Manage locally tracked workspaces + mount Mirror a remote workspace to a local directory + seed Upload a directory tree with bulk writes + export Export a workspace as json, tar, or patch + status Show sync status for a workspace`) +} + +func runLogin(args []string, stdin io.Reader, stdout io.Writer) error { + fs := flag.NewFlagSet("login", flag.ContinueOnError) + fs.SetOutput(io.Discard) + server := fs.String("server", envOrDefault("RELAYFILE_SERVER", envOrDefault("RELAYFILE_BASE_URL", defaultServerURL)), "relayfile server URL") + token := fs.String("token", strings.TrimSpace(os.Getenv("RELAYFILE_TOKEN")), "API token") + if err := fs.Parse(args); err != nil { + return err + } + + serverValue := strings.TrimSpace(*server) + if serverValue == "" { + serverValue = defaultServerURL + } + tokenValue := strings.TrimSpace(*token) + if tokenValue == "" { + prompted, err := promptLine(stdin, stdout, "API key: ") + if err != nil { + return err + } + tokenValue = strings.TrimSpace(prompted) + } + if tokenValue == "" { + return errors.New("token is required") + } + + client := &http.Client{Timeout: 10 * time.Second} + req, err := http.NewRequest(http.MethodGet, strings.TrimRight(serverValue, "/")+"/health", nil) + if err != nil { + return err + } + resp, err := client.Do(req) + if err != nil { + return fmt.Errorf("health check failed: %w", err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("health check failed with status %d", resp.StatusCode) + } + + creds := credentials{ + Server: strings.TrimRight(serverValue, "/"), + Token: tokenValue, + UpdatedAt: time.Now().UTC().Format(time.RFC3339), + } + if err := saveCredentials(creds); err != nil { + return err + } + fmt.Fprintf(stdout, "Stored credentials for %s\n", creds.Server) + return nil +} + +func runWorkspace(args []string, stdout io.Writer) error { + if len(args) == 0 { + return errors.New("workspace subcommand is required: create or list") + } + switch args[0] { + case "create": + return runWorkspaceCreate(args[1:], stdout) + case "list": + return runWorkspaceList(args[1:], stdout) + default: + return fmt.Errorf("unknown workspace subcommand %q", args[0]) + } +} + +func runWorkspaceCreate(args []string, stdout io.Writer) error { + fs := flag.NewFlagSet("workspace create", flag.ContinueOnError) + fs.SetOutput(io.Discard) + if err := fs.Parse(args); err != nil { + return err + } + if fs.NArg() != 1 { + return errors.New("usage: relayfile workspace create NAME") + } + name := strings.TrimSpace(fs.Arg(0)) + if name == "" { + return errors.New("workspace name is required") + } + if err := upsertWorkspace(name); err != nil { + return err + } + fmt.Fprintf(stdout, "Registered workspace %s in %s\n", name, workspacesPath()) + return nil +} + +func runWorkspaceList(args []string, stdout io.Writer) error { + fs := flag.NewFlagSet("workspace list", flag.ContinueOnError) + fs.SetOutput(io.Discard) + server := fs.String("server", "", "relayfile server URL override") + token := fs.String("token", "", "relayfile token override") + if err := fs.Parse(args); err != nil { + return err + } + creds, _ := loadCredentials() + + client, err := newAPIClient(resolveServer(*server, creds), resolveToken(*token, creds)) + if err == nil { + var remote adminWorkspaceList + err = client.getJSON(context.Background(), "/v1/admin/workspaces", &remote) + if err == nil && len(remote.Workspaces) > 0 { + for _, name := range remote.Workspaces { + fmt.Fprintln(stdout, name) + } + return nil + } + } + + catalog, err := loadWorkspaceCatalog() + if err != nil { + return err + } + for _, workspace := range catalog.Workspaces { + fmt.Fprintln(stdout, workspace.Name) + } + return nil +} + +func runMount(args []string, stdout io.Writer) error { + fs := flag.NewFlagSet("mount", flag.ContinueOnError) + fs.SetOutput(io.Discard) + + creds, _ := loadCredentials() + baseURL := fs.String("server", resolveServer("", creds), "relayfile server URL") + token := fs.String("token", resolveToken("", creds), "bearer token") + remotePath := fs.String("remote-path", envOrDefault("RELAYFILE_REMOTE_PATH", "/"), "remote root path") + eventProvider := fs.String("provider", strings.TrimSpace(os.Getenv("RELAYFILE_MOUNT_PROVIDER")), "event provider filter") + stateFile := fs.String("state-file", strings.TrimSpace(os.Getenv("RELAYFILE_MOUNT_STATE_FILE")), "state file path") + interval := fs.Duration("interval", durationEnv("RELAYFILE_MOUNT_INTERVAL", 2*time.Second), "sync interval") + intervalJitter := fs.Float64("interval-jitter", floatEnv("RELAYFILE_MOUNT_INTERVAL_JITTER", 0.2), "sync interval jitter ratio (0.0-1.0)") + timeout := fs.Duration("timeout", durationEnv("RELAYFILE_MOUNT_TIMEOUT", 15*time.Second), "per-sync timeout") + websocketEnabled := fs.Bool("websocket", boolEnv("RELAYFILE_MOUNT_WEBSOCKET", true), "enable websocket event streaming when available") + once := fs.Bool("once", false, "run one sync cycle and exit") + if err := fs.Parse(args); err != nil { + return err + } + if fs.NArg() < 1 || fs.NArg() > 2 { + return errors.New("usage: relayfile mount WORKSPACE [LOCAL_DIR]") + } + + workspaceID := strings.TrimSpace(fs.Arg(0)) + localDir := "." + if fs.NArg() == 2 { + localDir = fs.Arg(1) + } + absLocalDir, err := filepath.Abs(localDir) + if err != nil { + return err + } + + if strings.TrimSpace(*token) == "" { + return errors.New("token is required; run relayfile login or pass --token") + } + if *interval <= 0 { + *interval = 2 * time.Second + } + if *timeout <= 0 { + *timeout = 15 * time.Second + } + *intervalJitter = clampJitterRatio(*intervalJitter) + + client := mountsync.NewHTTPClient(*baseURL, *token, &http.Client{Timeout: *timeout}) + syncer, err := mountsync.NewSyncer(client, mountsync.SyncerOptions{ + WorkspaceID: workspaceID, + RemoteRoot: *remotePath, + EventProvider: strings.TrimSpace(*eventProvider), + LocalRoot: absLocalDir, + StateFile: *stateFile, + WebSocket: boolPtr(*websocketEnabled), + Logger: log.Default(), + }) + if err != nil { + return fmt.Errorf("failed to initialize mount syncer: %w", err) + } + if err := upsertWorkspace(workspaceID); err != nil { + return err + } + + rootCtx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer stop() + + runSync := func() { + ctx, cancel := context.WithTimeout(rootCtx, *timeout) + defer cancel() + if err := syncer.SyncOnce(ctx); err != nil { + log.Printf("mount sync cycle failed: %v", err) + return + } + log.Printf("mount sync cycle completed") + } + + runSync() + if *once { + return nil + } + + rng := mathrand.New(mathrand.NewSource(time.Now().UnixNano())) + timer := time.NewTimer(jitteredIntervalWithSample(*interval, *intervalJitter, rng.Float64())) + defer timer.Stop() + for { + select { + case <-rootCtx.Done(): + log.Printf("mount sync stopping: %v", rootCtx.Err()) + return nil + case <-timer.C: + runSync() + timer.Reset(jitteredIntervalWithSample(*interval, *intervalJitter, rng.Float64())) + } + } +} + +func runSeed(args []string, stdout io.Writer) error { + fs := flag.NewFlagSet("seed", flag.ContinueOnError) + fs.SetOutput(io.Discard) + server := fs.String("server", "", "relayfile server URL override") + token := fs.String("token", "", "relayfile token override") + if err := fs.Parse(args); err != nil { + return err + } + if fs.NArg() < 1 || fs.NArg() > 2 { + return errors.New("usage: relayfile seed WORKSPACE [DIR]") + } + + creds, err := loadCredentials() + if err != nil { + return err + } + client, err := newAPIClient(resolveServer(*server, creds), resolveToken(*token, creds)) + if err != nil { + return err + } + + workspaceID := strings.TrimSpace(fs.Arg(0)) + dir := "." + if fs.NArg() == 2 { + dir = fs.Arg(1) + } + root, err := filepath.Abs(dir) + if err != nil { + return err + } + + files, err := collectSeedFiles(root) + if err != nil { + return err + } + if len(files) == 0 { + fmt.Fprintln(stdout, "No files to seed") + return nil + } + + for i := range files { + fmt.Fprintf(stdout, "Seeding %d/%d files...\n", i+1, len(files)) + } + + var response bulkWriteResponse + if err := client.postJSON(context.Background(), fmt.Sprintf("/v1/workspaces/%s/fs/bulk", url.PathEscape(workspaceID)), bulkWriteRequest{Files: files}, &response); err != nil { + return err + } + if err := upsertWorkspace(workspaceID); err != nil { + return err + } + fmt.Fprintf(stdout, "Seeded %d files", response.Written) + if response.ErrorCount > 0 { + fmt.Fprintf(stdout, " with %d errors", response.ErrorCount) + } + fmt.Fprintln(stdout) + for _, item := range response.Errors { + fmt.Fprintf(stdout, "%s: %s (%s)\n", item.Path, item.Message, item.Code) + } + return nil +} + +func runExport(args []string, stdout io.Writer) error { + fs := flag.NewFlagSet("export", flag.ContinueOnError) + fs.SetOutput(io.Discard) + server := fs.String("server", "", "relayfile server URL override") + token := fs.String("token", "", "relayfile token override") + format := fs.String("format", "json", "export format: json, tar, or patch") + output := fs.String("output", "-", "output file path or - for stdout") + if err := fs.Parse(args); err != nil { + return err + } + if fs.NArg() != 1 { + return errors.New("usage: relayfile export WORKSPACE --format FORMAT [--output FILE]") + } + + creds, err := loadCredentials() + if err != nil { + return err + } + client, err := newAPIClient(resolveServer(*server, creds), resolveToken(*token, creds)) + if err != nil { + return err + } + + workspaceID := strings.TrimSpace(fs.Arg(0)) + path := fmt.Sprintf("/v1/workspaces/%s/fs/export?format=%s", url.PathEscape(workspaceID), url.QueryEscape(strings.ToLower(strings.TrimSpace(*format)))) + body, _, err := client.getBytes(context.Background(), path) + if err != nil { + return err + } + if err := upsertWorkspace(workspaceID); err != nil { + return err + } + if strings.TrimSpace(*output) == "" || *output == "-" { + _, err = stdout.Write(body) + return err + } + return os.WriteFile(*output, body, 0o644) +} + +func runStatus(args []string, stdout io.Writer) error { + fs := flag.NewFlagSet("status", flag.ContinueOnError) + fs.SetOutput(io.Discard) + server := fs.String("server", "", "relayfile server URL override") + token := fs.String("token", "", "relayfile token override") + if err := fs.Parse(args); err != nil { + return err + } + if fs.NArg() != 1 { + return errors.New("usage: relayfile status WORKSPACE") + } + + creds, err := loadCredentials() + if err != nil { + return err + } + client, err := newAPIClient(resolveServer(*server, creds), resolveToken(*token, creds)) + if err != nil { + return err + } + workspaceID := strings.TrimSpace(fs.Arg(0)) + + var status syncStatusResponse + if err := client.getJSON(context.Background(), fmt.Sprintf("/v1/workspaces/%s/sync/status", url.PathEscape(workspaceID)), &status); err != nil { + return err + } + if err := upsertWorkspace(workspaceID); err != nil { + return err + } + + fileCountText := "unknown" + var exported []exportedFile + if err := client.getJSON(context.Background(), fmt.Sprintf("/v1/workspaces/%s/fs/export?format=json", url.PathEscape(workspaceID)), &exported); err == nil { + fileCountText = strconv.Itoa(len(exported)) + } + + fmt.Fprintf(stdout, "Workspace: %s\n", status.WorkspaceID) + fmt.Fprintf(stdout, "File count: %s\n", fileCountText) + if len(status.Providers) == 0 { + fmt.Fprintln(stdout, "Providers: none") + return nil + } + for _, provider := range status.Providers { + lastActivity := "" + if provider.WatermarkTs != nil { + lastActivity = *provider.WatermarkTs + } + if lastActivity == "" { + lastActivity = "unknown" + } + fmt.Fprintf(stdout, "%s: %s", provider.Provider, provider.Status) + if provider.LagSeconds > 0 { + fmt.Fprintf(stdout, " lag=%ds", provider.LagSeconds) + } + fmt.Fprintf(stdout, " last_activity=%s", lastActivity) + if provider.LastError != nil && strings.TrimSpace(*provider.LastError) != "" { + fmt.Fprintf(stdout, " error=%q", *provider.LastError) + } + fmt.Fprintln(stdout) + } + return nil +} + +func newAPIClient(server, token string) (*apiClient, error) { + server = strings.TrimSpace(server) + token = strings.TrimSpace(token) + if server == "" { + server = defaultServerURL + } + if token == "" { + return nil, errors.New("token is required; run relayfile login or pass --token") + } + return &apiClient{ + baseURL: strings.TrimRight(server, "/"), + token: token, + httpClient: &http.Client{Timeout: 30 * time.Second}, + }, nil +} + +func (c *apiClient) getJSON(ctx context.Context, path string, out any) error { + body, _, err := c.do(ctx, http.MethodGet, path, nil) + if err != nil { + return err + } + if out == nil || len(body) == 0 { + return nil + } + return json.Unmarshal(body, out) +} + +func (c *apiClient) postJSON(ctx context.Context, path string, input, out any) error { + bodyBytes, err := json.Marshal(input) + if err != nil { + return err + } + body, _, err := c.do(ctx, http.MethodPost, path, bodyBytes) + if err != nil { + return err + } + if out == nil || len(body) == 0 { + return nil + } + return json.Unmarshal(body, out) +} + +func (c *apiClient) getBytes(ctx context.Context, path string) ([]byte, string, error) { + return c.do(ctx, http.MethodGet, path, nil) +} + +func (c *apiClient) do(ctx context.Context, method, path string, body []byte) ([]byte, string, error) { + var reader io.Reader + if len(body) > 0 { + reader = bytes.NewReader(body) + } + req, err := http.NewRequestWithContext(ctx, method, c.baseURL+path, reader) + if err != nil { + return nil, "", err + } + req.Header.Set("Authorization", "Bearer "+c.token) + req.Header.Set("X-Correlation-Id", correlationID()) + if len(body) > 0 { + req.Header.Set("Content-Type", "application/json") + } + + resp, err := c.httpClient.Do(req) + if err != nil { + return nil, "", err + } + defer resp.Body.Close() + + payload, err := io.ReadAll(resp.Body) + if err != nil { + return nil, "", err + } + if resp.StatusCode >= 200 && resp.StatusCode < 300 { + return payload, resp.Header.Get("Content-Type"), nil + } + + var apiErr struct { + Code string `json:"code"` + Message string `json:"message"` + } + if len(payload) > 0 { + _ = json.Unmarshal(payload, &apiErr) + } + if apiErr.Message == "" { + apiErr.Message = strings.TrimSpace(string(payload)) + } + return nil, "", &apiError{ + StatusCode: resp.StatusCode, + Code: apiErr.Code, + Message: apiErr.Message, + } +} + +func collectSeedFiles(root string) ([]bulkWriteFile, error) { + files := make([]bulkWriteFile, 0) + err := filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() { + return nil + } + info, err := d.Info() + if err != nil { + return err + } + if !info.Mode().IsRegular() { + return nil + } + rel, err := filepath.Rel(root, path) + if err != nil { + return err + } + content, err := os.ReadFile(path) + if err != nil { + return err + } + entry := bulkWriteFile{ + Path: "/" + filepath.ToSlash(rel), + ContentType: detectContentType(path, content), + } + if utf8.Valid(content) { + entry.Content = string(content) + } else { + entry.Content = base64.StdEncoding.EncodeToString(content) + entry.Encoding = "base64" + } + files = append(files, entry) + return nil + }) + if err != nil { + return nil, err + } + sort.Slice(files, func(i, j int) bool { + return files[i].Path < files[j].Path + }) + return files, nil +} + +func detectContentType(path string, content []byte) string { + extType := strings.TrimSpace(mime.TypeByExtension(strings.ToLower(filepath.Ext(path)))) + if extType != "" { + return extType + } + if len(content) == 0 { + return "application/octet-stream" + } + return http.DetectContentType(content) +} + +func promptLine(stdin io.Reader, stdout io.Writer, prompt string) (string, error) { + if _, err := io.WriteString(stdout, prompt); err != nil { + return "", err + } + reader := bufio.NewReader(stdin) + value, err := reader.ReadString('\n') + if err != nil && !errors.Is(err, io.EOF) { + return "", err + } + return strings.TrimSpace(value), nil +} + +func configDir() string { + home, err := os.UserHomeDir() + if err != nil { + return configDirName + } + return filepath.Join(home, configDirName) +} + +func credentialsPath() string { + return filepath.Join(configDir(), "credentials.json") +} + +func workspacesPath() string { + return filepath.Join(configDir(), "workspaces.json") +} + +func ensureConfigDir() error { + return os.MkdirAll(configDir(), 0o755) +} + +func saveCredentials(creds credentials) error { + if err := ensureConfigDir(); err != nil { + return err + } + payload, err := json.MarshalIndent(creds, "", " ") + if err != nil { + return err + } + payload = append(payload, '\n') + return os.WriteFile(credentialsPath(), payload, 0o600) +} + +func loadCredentials() (credentials, error) { + var creds credentials + payload, err := os.ReadFile(credentialsPath()) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return creds, fmt.Errorf("credentials not found at %s; run relayfile login", credentialsPath()) + } + return creds, err + } + if err := json.Unmarshal(payload, &creds); err != nil { + return creds, fmt.Errorf("parse %s: %w", credentialsPath(), err) + } + return creds, nil +} + +func loadWorkspaceCatalog() (workspaceCatalog, error) { + var catalog workspaceCatalog + payload, err := os.ReadFile(workspacesPath()) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return workspaceCatalog{}, nil + } + return catalog, err + } + if err := json.Unmarshal(payload, &catalog); err != nil { + return catalog, fmt.Errorf("parse %s: %w", workspacesPath(), err) + } + sort.Slice(catalog.Workspaces, func(i, j int) bool { + return catalog.Workspaces[i].Name < catalog.Workspaces[j].Name + }) + return catalog, nil +} + +func saveWorkspaceCatalog(catalog workspaceCatalog) error { + if err := ensureConfigDir(); err != nil { + return err + } + sort.Slice(catalog.Workspaces, func(i, j int) bool { + return catalog.Workspaces[i].Name < catalog.Workspaces[j].Name + }) + payload, err := json.MarshalIndent(catalog, "", " ") + if err != nil { + return err + } + payload = append(payload, '\n') + return os.WriteFile(workspacesPath(), payload, 0o644) +} + +func upsertWorkspace(name string) error { + name = strings.TrimSpace(name) + if name == "" { + return errors.New("workspace name is required") + } + catalog, err := loadWorkspaceCatalog() + if err != nil { + return err + } + now := time.Now().UTC().Format(time.RFC3339) + for i := range catalog.Workspaces { + if catalog.Workspaces[i].Name == name { + catalog.Workspaces[i].LastUsedAt = now + return saveWorkspaceCatalog(catalog) + } + } + catalog.Workspaces = append(catalog.Workspaces, workspaceRecord{ + Name: name, + CreatedAt: now, + LastUsedAt: now, + }) + return saveWorkspaceCatalog(catalog) +} + +func resolveServer(flagValue string, creds credentials) string { + if value := strings.TrimSpace(flagValue); value != "" { + return value + } + if value := strings.TrimSpace(os.Getenv("RELAYFILE_SERVER")); value != "" { + return value + } + if value := strings.TrimSpace(os.Getenv("RELAYFILE_BASE_URL")); value != "" { + return value + } + if value := strings.TrimSpace(creds.Server); value != "" { + return value + } + return defaultServerURL +} + +func resolveToken(flagValue string, creds credentials) string { + if value := strings.TrimSpace(flagValue); value != "" { + return value + } + if value := strings.TrimSpace(os.Getenv("RELAYFILE_TOKEN")); value != "" { + return value + } + return strings.TrimSpace(creds.Token) +} + +func envOrDefault(name, fallback string) string { + value := strings.TrimSpace(os.Getenv(name)) + if value == "" { + return fallback + } + return value +} + +func durationEnv(name string, fallback time.Duration) time.Duration { + raw := strings.TrimSpace(os.Getenv(name)) + if raw == "" { + return fallback + } + value, err := time.ParseDuration(raw) + if err != nil { + log.Printf("invalid %s=%q, using fallback %s", name, raw, fallback.String()) + return fallback + } + return value +} + +func floatEnv(name string, fallback float64) float64 { + raw := strings.TrimSpace(os.Getenv(name)) + if raw == "" { + return fallback + } + value, err := strconv.ParseFloat(raw, 64) + if err != nil { + log.Printf("invalid %s=%q, using fallback %f", name, raw, fallback) + return fallback + } + return value +} + +func boolEnv(name string, fallback bool) bool { + raw := strings.TrimSpace(os.Getenv(name)) + if raw == "" { + return fallback + } + value, err := strconv.ParseBool(raw) + if err != nil { + log.Printf("invalid %s=%q, using fallback %t", name, raw, fallback) + return fallback + } + return value +} + +func boolPtr(value bool) *bool { + return &value +} + +func clampJitterRatio(value float64) float64 { + if value < 0 { + return 0 + } + if value > 1 { + return 1 + } + return value +} + +func jitteredIntervalWithSample(base time.Duration, jitterRatio, sample float64) time.Duration { + if base <= 0 { + return 0 + } + jitterRatio = clampJitterRatio(jitterRatio) + if jitterRatio == 0 { + return base + } + if sample < 0 { + sample = 0 + } else if sample > 1 { + sample = 1 + } + factor := 1 + ((sample*2)-1)*jitterRatio + if factor < 0 { + factor = 0 + } + delay := time.Duration(float64(base) * factor) + if delay < time.Millisecond { + return time.Millisecond + } + return delay +} + +func correlationID() string { + var buf [16]byte + if _, err := rand.Read(buf[:]); err == nil { + return "corr_" + hex.EncodeToString(buf[:]) + } + return fmt.Sprintf("corr_%d", time.Now().UnixNano()) +} diff --git a/docs/api-reference.md b/docs/api-reference.md new file mode 100644 index 00000000..48f7d8de --- /dev/null +++ b/docs/api-reference.md @@ -0,0 +1,417 @@ +# RelayFile API Reference + +This document summarizes the current RelayFile HTTP surface with runnable `curl` examples. For the full schema, see `openapi/relayfile-v1.openapi.yaml`. + +## Base Variables + +Set these once for the examples below: + +```bash +export RELAYFILE_BASE_URL=http://127.0.0.1:8080 +export RELAYFILE_WORKSPACE=ws_live +export RELAYFILE_TOKEN="$(./scripts/generate-dev-token.sh ${RELAYFILE_WORKSPACE})" +export RELAYFILE_CORRELATION_ID="corr_$(date +%s)" +``` + +Authenticated requests usually include: + +```bash +-H "Authorization: Bearer ${RELAYFILE_TOKEN}" \ +-H "X-Correlation-Id: ${RELAYFILE_CORRELATION_ID}" +``` + +Endpoints that are intended for internal service-to-service ingress may require different auth or signing behavior in production. The examples here focus on local development and contract discovery. + +## Health + +### `GET /health` + +Returns a simple liveness response. This endpoint does not require auth. + +```bash +curl -sS "${RELAYFILE_BASE_URL}/health" | jq . +``` + +## Filesystem + +### `GET /v1/workspaces/{workspaceId}/fs/tree` + +List files and directories under a path. + +```bash +curl -sS \ + -H "Authorization: Bearer ${RELAYFILE_TOKEN}" \ + -H "X-Correlation-Id: ${RELAYFILE_CORRELATION_ID}" \ + "${RELAYFILE_BASE_URL}/v1/workspaces/${RELAYFILE_WORKSPACE}/fs/tree?path=/&depth=2" | jq . +``` + +### `GET /v1/workspaces/{workspaceId}/fs/file` + +Read one file. + +```bash +curl -sS \ + -H "Authorization: Bearer ${RELAYFILE_TOKEN}" \ + -H "X-Correlation-Id: ${RELAYFILE_CORRELATION_ID}" \ + "${RELAYFILE_BASE_URL}/v1/workspaces/${RELAYFILE_WORKSPACE}/fs/file?path=/README.md" | jq . +``` + +### `PUT /v1/workspaces/{workspaceId}/fs/file` + +Create or update one file. + +```bash +curl -sS -X PUT \ + -H "Authorization: Bearer ${RELAYFILE_TOKEN}" \ + -H "X-Correlation-Id: ${RELAYFILE_CORRELATION_ID}" \ + -H "Content-Type: application/json" \ + "${RELAYFILE_BASE_URL}/v1/workspaces/${RELAYFILE_WORKSPACE}/fs/file" \ + -d '{ + "path": "/docs/guide.md", + "content": "# agent guide", + "contentType": "text/markdown" + }' | jq . +``` + +### `DELETE /v1/workspaces/{workspaceId}/fs/file` + +Delete one file. + +```bash +curl -sS -X DELETE \ + -H "Authorization: Bearer ${RELAYFILE_TOKEN}" \ + -H "X-Correlation-Id: ${RELAYFILE_CORRELATION_ID}" \ + "${RELAYFILE_BASE_URL}/v1/workspaces/${RELAYFILE_WORKSPACE}/fs/file?path=/docs/guide.md" | jq . +``` + +### `GET /v1/workspaces/{workspaceId}/fs/events` + +Read the filesystem event feed for a workspace. + +```bash +curl -sS \ + -H "Authorization: Bearer ${RELAYFILE_TOKEN}" \ + -H "X-Correlation-Id: ${RELAYFILE_CORRELATION_ID}" \ + "${RELAYFILE_BASE_URL}/v1/workspaces/${RELAYFILE_WORKSPACE}/fs/events?path=/&limit=20" | jq . +``` + +### `GET /v1/workspaces/{workspaceId}/fs/query` + +Run a structured metadata query. + +```bash +curl -sS \ + -H "Authorization: Bearer ${RELAYFILE_TOKEN}" \ + -H "X-Correlation-Id: ${RELAYFILE_CORRELATION_ID}" \ + "${RELAYFILE_BASE_URL}/v1/workspaces/${RELAYFILE_WORKSPACE}/fs/query?path=/documents&property.topic=investments&relation=db_investments&permission=scope:fs:read&limit=20" | jq . +``` + +### `POST /v1/workspaces/{workspaceId}/fs/bulk` + +Write many files in one request. + +```bash +curl -sS -X POST \ + -H "Authorization: Bearer ${RELAYFILE_TOKEN}" \ + -H "X-Correlation-Id: ${RELAYFILE_CORRELATION_ID}" \ + -H "Content-Type: application/json" \ + "${RELAYFILE_BASE_URL}/v1/workspaces/${RELAYFILE_WORKSPACE}/fs/bulk" \ + -d '{ + "files": [ + { + "path": "/src/app.js", + "content": "console.log(\"hello\");", + "contentType": "application/javascript" + }, + { + "path": "/README.md", + "content": "# demo workspace", + "contentType": "text/markdown" + } + ] + }' | jq . +``` + +### `GET /v1/workspaces/{workspaceId}/fs/export` + +Export visible files from a workspace. + +JSON export: + +```bash +curl -sS \ + -H "Authorization: Bearer ${RELAYFILE_TOKEN}" \ + -H "X-Correlation-Id: ${RELAYFILE_CORRELATION_ID}" \ + "${RELAYFILE_BASE_URL}/v1/workspaces/${RELAYFILE_WORKSPACE}/fs/export?format=json" | jq . +``` + +Tar export: + +```bash +curl -sS \ + -H "Authorization: Bearer ${RELAYFILE_TOKEN}" \ + -H "X-Correlation-Id: ${RELAYFILE_CORRELATION_ID}" \ + "${RELAYFILE_BASE_URL}/v1/workspaces/${RELAYFILE_WORKSPACE}/fs/export?format=tar" \ + -o relayfile-export.tar.gz +``` + +Patch export: + +```bash +curl -sS \ + -H "Authorization: Bearer ${RELAYFILE_TOKEN}" \ + -H "X-Correlation-Id: ${RELAYFILE_CORRELATION_ID}" \ + "${RELAYFILE_BASE_URL}/v1/workspaces/${RELAYFILE_WORKSPACE}/fs/export?format=patch" +``` + +### `GET /v1/workspaces/{workspaceId}/fs/ws` + +Open a WebSocket stream for real-time filesystem changes. + +This endpoint upgrades to WebSocket rather than returning a normal REST body. The token is passed as a query parameter. + +```bash +wscat -c "ws://127.0.0.1:8080/v1/workspaces/${RELAYFILE_WORKSPACE}/fs/ws?token=${RELAYFILE_TOKEN}" +``` + +## Sync + +### `GET /v1/workspaces/{workspaceId}/sync/status` + +Show workspace sync status. + +```bash +curl -sS \ + -H "Authorization: Bearer ${RELAYFILE_TOKEN}" \ + -H "X-Correlation-Id: ${RELAYFILE_CORRELATION_ID}" \ + "${RELAYFILE_BASE_URL}/v1/workspaces/${RELAYFILE_WORKSPACE}/sync/status" | jq . +``` + +### `GET /v1/workspaces/{workspaceId}/sync/ingress` + +Inspect ingress queue and reliability counters for one workspace. + +```bash +curl -sS \ + -H "Authorization: Bearer ${RELAYFILE_TOKEN}" \ + -H "X-Correlation-Id: ${RELAYFILE_CORRELATION_ID}" \ + "${RELAYFILE_BASE_URL}/v1/workspaces/${RELAYFILE_WORKSPACE}/sync/ingress" | jq . +``` + +### `GET /v1/workspaces/{workspaceId}/sync/dead-letter` + +List dead-letter envelopes. + +```bash +curl -sS \ + -H "Authorization: Bearer ${RELAYFILE_TOKEN}" \ + -H "X-Correlation-Id: ${RELAYFILE_CORRELATION_ID}" \ + "${RELAYFILE_BASE_URL}/v1/workspaces/${RELAYFILE_WORKSPACE}/sync/dead-letter?limit=20" | jq . +``` + +### `GET /v1/workspaces/{workspaceId}/sync/dead-letter/{envelopeId}` + +Fetch one dead-letter envelope in detail. + +```bash +curl -sS \ + -H "Authorization: Bearer ${RELAYFILE_TOKEN}" \ + -H "X-Correlation-Id: ${RELAYFILE_CORRELATION_ID}" \ + "${RELAYFILE_BASE_URL}/v1/workspaces/${RELAYFILE_WORKSPACE}/sync/dead-letter/envelope_123" | jq . +``` + +### `POST /v1/workspaces/{workspaceId}/sync/dead-letter/{envelopeId}/replay` + +Replay one dead-letter envelope. + +```bash +curl -sS -X POST \ + -H "Authorization: Bearer ${RELAYFILE_TOKEN}" \ + -H "X-Correlation-Id: ${RELAYFILE_CORRELATION_ID}" \ + "${RELAYFILE_BASE_URL}/v1/workspaces/${RELAYFILE_WORKSPACE}/sync/dead-letter/envelope_123/replay" | jq . +``` + +### `POST /v1/workspaces/{workspaceId}/sync/dead-letter/{envelopeId}/ack` + +Mark one dead-letter envelope as acknowledged. + +```bash +curl -sS -X POST \ + -H "Authorization: Bearer ${RELAYFILE_TOKEN}" \ + -H "X-Correlation-Id: ${RELAYFILE_CORRELATION_ID}" \ + "${RELAYFILE_BASE_URL}/v1/workspaces/${RELAYFILE_WORKSPACE}/sync/dead-letter/envelope_123/ack" | jq . +``` + +### `POST /v1/workspaces/{workspaceId}/sync/refresh` + +Trigger a sync refresh for the workspace. + +```bash +curl -sS -X POST \ + -H "Authorization: Bearer ${RELAYFILE_TOKEN}" \ + -H "X-Correlation-Id: ${RELAYFILE_CORRELATION_ID}" \ + "${RELAYFILE_BASE_URL}/v1/workspaces/${RELAYFILE_WORKSPACE}/sync/refresh" | jq . +``` + +## Webhooks + +### `POST /v1/workspaces/{workspaceId}/webhooks/ingest` + +Submit a provider-agnostic webhook into a workspace. + +```bash +curl -sS -X POST \ + -H "Authorization: Bearer ${RELAYFILE_TOKEN}" \ + -H "X-Correlation-Id: webhook_test_1" \ + -H "Content-Type: application/json" \ + "${RELAYFILE_BASE_URL}/v1/workspaces/${RELAYFILE_WORKSPACE}/webhooks/ingest" \ + -d '{ + "provider": "salesforce", + "event_type": "file.updated", + "path": "/salesforce/Account_123", + "data": { + "content": "Account details", + "contentType": "text/plain" + }, + "delivery_id": "sf_evt_123" + }' | jq . +``` + +### `POST /v1/internal/webhook-envelopes` + +Submit an internal webhook envelope. In production this is intended for trusted internal callers. + +```bash +curl -sS -X POST \ + -H "Authorization: Bearer ${RELAYFILE_TOKEN}" \ + -H "X-Correlation-Id: ${RELAYFILE_CORRELATION_ID}" \ + -H "Content-Type: application/json" \ + "${RELAYFILE_BASE_URL}/v1/internal/webhook-envelopes" \ + -d '{ + "workspace_id": "'"${RELAYFILE_WORKSPACE}"'", + "provider": "internal", + "event_type": "file.updated", + "path": "/docs/guide.md", + "data": { + "content": "# internal update", + "contentType": "text/markdown" + }, + "delivery_id": "internal_evt_123" + }' | jq . +``` + +### `GET /v1/workspaces/{workspaceId}/writeback/pending` + +List outbound writeback items waiting for an external provider. + +```bash +curl -sS \ + -H "Authorization: Bearer ${RELAYFILE_TOKEN}" \ + -H "X-Correlation-Id: ${RELAYFILE_CORRELATION_ID}" \ + "${RELAYFILE_BASE_URL}/v1/workspaces/${RELAYFILE_WORKSPACE}/writeback/pending" | jq . +``` + +### `POST /v1/workspaces/{workspaceId}/writeback/{itemId}/ack` + +Acknowledge a processed writeback item. + +```bash +curl -sS -X POST \ + -H "Authorization: Bearer ${RELAYFILE_TOKEN}" \ + -H "X-Correlation-Id: ${RELAYFILE_CORRELATION_ID}" \ + "${RELAYFILE_BASE_URL}/v1/workspaces/${RELAYFILE_WORKSPACE}/writeback/writeback_123/ack" | jq . +``` + +## Operations + +### `GET /v1/workspaces/{workspaceId}/ops` + +List operations for a workspace. + +```bash +curl -sS \ + -H "Authorization: Bearer ${RELAYFILE_TOKEN}" \ + -H "X-Correlation-Id: ${RELAYFILE_CORRELATION_ID}" \ + "${RELAYFILE_BASE_URL}/v1/workspaces/${RELAYFILE_WORKSPACE}/ops?limit=20" | jq . +``` + +### `GET /v1/workspaces/{workspaceId}/ops/{opId}` + +Fetch one operation. + +```bash +curl -sS \ + -H "Authorization: Bearer ${RELAYFILE_TOKEN}" \ + -H "X-Correlation-Id: ${RELAYFILE_CORRELATION_ID}" \ + "${RELAYFILE_BASE_URL}/v1/workspaces/${RELAYFILE_WORKSPACE}/ops/op_123" | jq . +``` + +### `POST /v1/workspaces/{workspaceId}/ops/{opId}/replay` + +Replay one operation. + +```bash +curl -sS -X POST \ + -H "Authorization: Bearer ${RELAYFILE_TOKEN}" \ + -H "X-Correlation-Id: ${RELAYFILE_CORRELATION_ID}" \ + "${RELAYFILE_BASE_URL}/v1/workspaces/${RELAYFILE_WORKSPACE}/ops/op_123/replay" | jq . +``` + +## Admin + +### `GET /v1/admin/backends` + +Show active backend configuration and storage types. + +```bash +curl -sS \ + -H "Authorization: Bearer ${RELAYFILE_TOKEN}" \ + "${RELAYFILE_BASE_URL}/v1/admin/backends" | jq . +``` + +### `GET /v1/admin/ingress` + +Inspect cross-workspace ingress backlog, counters, and alerts. + +```bash +curl -sS \ + -H "Authorization: Bearer ${RELAYFILE_TOKEN}" \ + "${RELAYFILE_BASE_URL}/v1/admin/ingress?includeWorkspaces=true&includeAlerts=true&limit=20" | jq . +``` + +### `GET /v1/admin/sync` + +Inspect cross-workspace sync health, dead-letter totals, and alert state. + +```bash +curl -sS \ + -H "Authorization: Bearer ${RELAYFILE_TOKEN}" \ + "${RELAYFILE_BASE_URL}/v1/admin/sync?includeWorkspaces=true&includeAlerts=true&limit=20" | jq . +``` + +### `POST /v1/admin/replay/envelope/{envelopeId}` + +Replay an envelope from the admin plane. + +```bash +curl -sS -X POST \ + -H "Authorization: Bearer ${RELAYFILE_TOKEN}" \ + "${RELAYFILE_BASE_URL}/v1/admin/replay/envelope/envelope_123" | jq . +``` + +### `POST /v1/admin/replay/op/{opId}` + +Replay an operation from the admin plane. + +```bash +curl -sS -X POST \ + -H "Authorization: Bearer ${RELAYFILE_TOKEN}" \ + "${RELAYFILE_BASE_URL}/v1/admin/replay/op/op_123" | jq . +``` + +## Notes On Auth + +- Most workspace and admin endpoints use Bearer token auth. +- The OpenAPI contract defines scope-aware auth such as `fs:read` and `fs:write`. +- Internal ingress may use service-to-service signing in production. +- `X-Correlation-Id` is recommended for traceability and auditability across requests. diff --git a/docs/cli-design.md b/docs/cli-design.md new file mode 100644 index 00000000..e9921cb1 --- /dev/null +++ b/docs/cli-design.md @@ -0,0 +1,424 @@ +# RelayFile CLI — Design Doc + +**Status:** Draft +**Date:** 2026-03-24 + +--- + +## Overview + +The `relayfile` CLI is the primary interface for humans and CI systems to interact with RelayFile workspaces. It wraps the RelayFile HTTP API and the existing `relayfile-mount` sync engine into a single, ergonomic command-line tool. + +### Design principles + +- **Minimal flags, sensible defaults.** The happy path should require as few arguments as possible. +- **Config file over flags over env vars.** Credentials and server URL are stored once via `relayfile login`; individual commands inherit them automatically. Environment variables (`RELAYFILE_TOKEN`, etc.) override config for CI/CD use. +- **Composable with pipes and scripts.** All commands emit structured JSON when `--json` is passed; human-readable tables otherwise. +- **No implicit destructive actions.** Deletes require confirmation unless `--yes` is passed. + +--- + +## Authentication + +### Supported methods (in priority order) + +| Priority | Method | Use case | +|----------|--------|----------| +| 1 | `--token` flag | One-off override | +| 2 | `RELAYFILE_TOKEN` env var | CI/CD pipelines | +| 3 | `~/.relayfile/credentials.json` | Interactive use after `relayfile login` | + +### Auth flow: API key (v1) + +``` +relayfile login --server https://relayfile-api.agentworkforce.workers.dev +``` + +1. Prompts the user for an API key (paste from dashboard or generate via API). +2. Validates the key by calling `GET /health` on the target server. +3. Writes `~/.relayfile/credentials.json`. + +### Auth flow: OAuth via browser (future, v2) + +Same as `gh auth login` — opens a browser, completes device-code flow, stores refresh token. Not in scope for v1; the credential file schema reserves fields for it. + +### Credential file: `~/.relayfile/credentials.json` + +```json +{ + "server": "https://relayfile-api.agentworkforce.workers.dev", + "token": "eyJ...", + "refreshToken": null, + "expiresAt": null +} +``` + +- `server` — base URL for all API calls. Default: `https://relayfile-api.agentworkforce.workers.dev`. +- `token` — Bearer JWT or API key. +- `refreshToken` / `expiresAt` — reserved for OAuth flow (v2). Null for API-key auth. +- File permissions: `0600` (user-only read/write). + +--- + +## Commands + +### `relayfile login` + +Authenticate and store credentials. + +``` +relayfile login [--server URL] +``` + +| Flag | Default | Description | +|------|---------|-------------| +| `--server` | `https://relayfile-api.agentworkforce.workers.dev` | Server base URL | + +**Behavior:** + +1. Prompt for API key (stdin, or `--token` flag for non-interactive). +2. Call `GET /health` to validate connectivity. +3. Call a token-validated endpoint (e.g., `GET /v1/workspaces` with limit=1) to verify the token is accepted. +4. Write `~/.relayfile/credentials.json` with `0600` permissions. +5. Print confirmation: `Logged in to as `. + +**Errors:** +- Server unreachable: `Error: cannot reach — check the URL and your network.` +- Invalid token: `Error: server rejected the token (HTTP 401). Check your API key.` + +--- + +### `relayfile workspace create` + +Create a new workspace. + +``` +relayfile workspace create +``` + +| Argument | Required | Description | +|----------|----------|-------------| +| `name` | Yes | Human-readable workspace name | + +**Behavior:** + +1. `POST /v1/workspaces` with `{ "name": "" }`. +2. Store the workspace ID mapping in `~/.relayfile/workspaces.json`. +3. Print the workspace ID. + +**`~/.relayfile/workspaces.json`:** + +```json +{ + "workspaces": { + "my-project": { + "id": "ws_abc123", + "server": "https://relayfile-api.agentworkforce.workers.dev", + "createdAt": "2026-03-24T12:00:00Z" + } + }, + "default": "my-project" +} +``` + +The first workspace created becomes the default. Use `relayfile workspace use ` to switch. + +--- + +### `relayfile workspace list` + +List accessible workspaces. + +``` +relayfile workspace list [--json] +``` + +**Behavior:** + +1. `GET /v1/workspaces` (paginated). +2. Print table: `ID | Name | Files | Last Activity`. +3. With `--json`, emit the raw API response. + +--- + +### `relayfile workspace delete` + +Delete a workspace. + +``` +relayfile workspace delete [--yes] +``` + +| Flag | Default | Description | +|------|---------|-------------| +| `--yes` | `false` | Skip confirmation prompt | + +**Behavior:** + +1. Resolve `` to a workspace ID via `~/.relayfile/workspaces.json` or the API. +2. Prompt: `Delete workspace "" (ws_abc123)? This cannot be undone. [y/N]` +3. `DELETE /v1/workspaces/{workspaceId}`. +4. Remove entry from `~/.relayfile/workspaces.json`. + +--- + +### `relayfile mount` + +Mount a workspace to a local directory, syncing changes in real-time. + +``` +relayfile mount [local-dir] +``` + +| Argument | Required | Default | Description | +|----------|----------|---------|-------------| +| `workspace` | Yes | — | Workspace name or ID | +| `local-dir` | No | `.` (cwd) | Local directory to sync into | + +| Flag | Default | Description | +|------|---------|-------------| +| `--interval` | `2s` | Polling interval between sync cycles | +| `--once` | `false` | Run a single sync cycle and exit | +| `--provider` | (none) | Filter events by provider name | +| `--no-websocket` | `false` | Disable WebSocket streaming, poll only | + +**Behavior:** + +1. Resolve workspace name to ID. +2. Create `local-dir` if it doesn't exist. +3. Load or initialize state file at `/.relayfile-mount-state.json`. +4. Start the existing `mountsync.Syncer` engine (reuses `internal/mountsync`). +5. Run in foreground. Print sync activity to stderr: + ``` + [mount] syncing ws_abc123 -> ./my-project + [mount] + /src/main.go (1.2 KB) + [mount] ~ /README.md (updated) + [mount] waiting for changes... + ``` +6. `Ctrl+C` (SIGINT/SIGTERM) triggers graceful shutdown. + +**Relationship to `relayfile-mount`:** This command replaces the standalone `relayfile-mount` binary for end users. The `cmd/relayfile-mount` binary remains available for backwards compatibility and for deployments that need a minimal single-purpose daemon. + +--- + +### `relayfile seed` + +Bulk-upload a local directory into a workspace. + +``` +relayfile seed [dir] +``` + +| Argument | Required | Default | Description | +|----------|----------|---------|-------------| +| `workspace` | Yes | — | Workspace name or ID | +| `dir` | No | `.` (cwd) | Local directory to upload | + +| Flag | Default | Description | +|------|---------|-------------| +| `--exclude` | (none) | Glob patterns to exclude (repeatable) | +| `--dry-run` | `false` | List files that would be uploaded without uploading | +| `--batch-size` | `50` | Files per bulk API request | + +**Behavior:** + +1. Walk `dir`, respecting `.gitignore` and `--exclude` patterns. +2. For each batch of files, call `POST /v1/workspaces/{workspaceId}/fs/bulk`. +3. Print progress: `[seed] 142/350 files uploaded...` +4. On completion: `[seed] done — 350 files uploaded to ws_abc123`. + +**Uses the Bulk Seed API** defined in `docs/bulk-export-design.md`. + +--- + +### `relayfile export` + +Download a workspace snapshot. + +``` +relayfile export [--format tar|json|patch] [--output file] +``` + +| Flag | Default | Description | +|------|---------|-------------| +| `--format` | `tar` | Export format | +| `--output` | stdout | Output file path (or `-` for stdout) | +| `--path` | `/` | Export only files under this path | + +**Behavior:** + +1. Resolve workspace. +2. `GET /v1/workspaces/{workspaceId}/fs/export?format=&path=`. +3. Stream response to `--output` or stdout. +4. Print to stderr: `[export] downloaded 42 files (1.3 MB) from ws_abc123`. + +**Uses the Workspace Export API** defined in `docs/bulk-export-design.md`. + +--- + +### `relayfile status` + +Show workspace status and stats. + +``` +relayfile status [workspace] +``` + +| Argument | Required | Default | Description | +|----------|----------|---------|-------------| +| `workspace` | No | default workspace | Workspace name or ID | + +**Behavior:** + +1. Resolve workspace (uses default if omitted). +2. Fetch workspace metadata and tree stats. +3. Print: + ``` + Workspace: my-project (ws_abc123) + Server: https://relayfile-api.agentworkforce.workers.dev + Files: 142 + Last event: 2026-03-24T11:42:00Z (3 minutes ago) + Agents: compose-agent, reviewer-bot + ``` + +--- + +## Global flags + +These flags apply to all commands: + +| Flag | Env var | Default | Description | +|------|---------|---------|-------------| +| `--server` | `RELAYFILE_SERVER` | from credentials | Server base URL | +| `--token` | `RELAYFILE_TOKEN` | from credentials | Bearer token | +| `--workspace` | `RELAYFILE_WORKSPACE` | from workspaces.json default | Workspace name or ID | +| `--json` | — | `false` | Output JSON instead of human-readable text | +| `--verbose` | — | `false` | Enable debug logging to stderr | + +--- + +## Config directory structure + +``` +~/.relayfile/ + credentials.json # server URL + token (0600) + workspaces.json # workspace name -> ID mapping + default +``` + +--- + +## Token resolution + +The CLI resolves tokens in this order, first match wins: + +1. `--token` flag +2. `RELAYFILE_TOKEN` environment variable +3. `~/.relayfile/credentials.json` → `token` field + +If no token is found, the CLI prints: + +``` +Error: not authenticated. Run 'relayfile login' or set RELAYFILE_TOKEN. +``` + +--- + +## Server URL resolution + +1. `--server` flag +2. `RELAYFILE_SERVER` environment variable +3. `~/.relayfile/credentials.json` → `server` field +4. Default: `https://relayfile-api.agentworkforce.workers.dev` + +--- + +## Exit codes + +| Code | Meaning | +|------|---------| +| 0 | Success | +| 1 | General error (network, server error, invalid input) | +| 2 | Authentication failure (no token, token rejected) | +| 3 | Resource not found (workspace doesn't exist) | +| 130 | Interrupted (Ctrl+C) | + +--- + +## Implementation plan + +### Phase 1 — Core (ship first) + +1. `relayfile login` — API key auth, credential storage +2. `relayfile mount` — wraps existing `mountsync` engine +3. `relayfile seed` — wraps bulk API +4. `relayfile export` — wraps export API +5. `relayfile status` — workspace info + +### Phase 2 — Workspace management + +6. `relayfile workspace create` +7. `relayfile workspace list` +8. `relayfile workspace delete` + +### Phase 3 — Polish + +9. OAuth browser flow +10. Auto-update / version check +11. Shell completions (bash, zsh, fish) +12. `relayfile doctor` — diagnose connectivity and auth issues + +### Build target + +The CLI compiles to a single static binary via `cmd/relayfile-cli/main.go` using Go's `cobra` or `flag` subcommand pattern. Distribution via: + +- `go install github.com/agentworkforce/relayfile/cmd/relayfile-cli@latest` +- GitHub Releases (goreleaser) +- Homebrew tap (future) + +--- + +## Relationship to existing binaries + +| Binary | Purpose | Status | +|--------|---------|--------| +| `cmd/relayfile/main.go` | HTTP API server | Unchanged | +| `cmd/relayfile-mount/main.go` | Standalone mount daemon | Kept for backwards compat | +| `cmd/relayfile-cli/main.go` | **New** — user-facing CLI | This design | + +The CLI imports `internal/mountsync` directly for the `mount` subcommand rather than shelling out to `relayfile-mount`. + +--- + +## Examples + +```bash +# First-time setup +relayfile login +# Enter API key: ******** +# Logged in to https://relayfile-api.agentworkforce.workers.dev as my-agent + +# Seed a project +relayfile seed my-workspace ./src +# [seed] 350 files uploaded to ws_abc123 + +# Mount and watch +relayfile mount my-workspace ./local-mirror +# [mount] syncing ws_abc123 -> ./local-mirror +# [mount] + /src/main.go (1.2 KB) +# [mount] waiting for changes... +# ^C +# [mount] stopped + +# Export a snapshot +relayfile export my-workspace --format tar --output backup.tar +# [export] downloaded 350 files (4.2 MB) from ws_abc123 + +# Check status +relayfile status my-workspace +# Workspace: my-workspace (ws_abc123) +# Files: 350 +# Last event: 2 minutes ago + +# CI/CD usage (no login needed) +RELAYFILE_TOKEN=eyJ... relayfile seed my-workspace ./dist +``` diff --git a/docs/guides/cloud-integration.md b/docs/guides/cloud-integration.md new file mode 100644 index 00000000..ec403e97 --- /dev/null +++ b/docs/guides/cloud-integration.md @@ -0,0 +1,88 @@ +# RelayFile Cloud Integration + +RelayFile is the shared filesystem layer for Agent Relay style cloud workflows. It gives each workflow run a workspace-backed tree that humans, automations, and cloud agents can mount at the same time. + +## How RelayFile Fits Into Cloud Workflows + +A typical Agent Relay flow looks like this: + +1. A workflow run starts in the cloud. +2. The orchestration layer creates or resolves a RelayFile workspace for that run. +3. Agents mount the workspace into their sandbox snapshot. +4. Humans can mount the same workspace locally. +5. Everyone reads and writes the same project tree through RelayFile. + +This makes the filesystem a first-class workflow artifact instead of a side effect hidden inside one machine or one agent process. + +## Automatic Workspace Creation Per Workflow Run + +Cloud orchestrators can create a fresh workspace for every workflow run so that: + +- each run is isolated from other runs +- files remain attached to the run that produced them +- replay and audit tooling can refer back to a stable workspace ID + +The user-facing CLI model is: + +```bash +relayfile workspace create workflow-2026-03-24-1234 +relayfile seed workflow-2026-03-24-1234 ./bootstrap +``` + +In hosted automation, the same lifecycle is usually driven by the control plane rather than by a human at a terminal. + +## relayfile-mount In Sandbox Snapshots + +Inside a cloud sandbox, `relayfile-mount` acts as the local bridge between the sandbox filesystem and the RelayFile API. + +Typical sandbox mount: + +```bash +go run ./cmd/relayfile-mount \ + --base-url https://relayfile.agent-relay.com \ + --workspace ws_123 \ + --remote-path / \ + --local-dir /workspace \ + --token "$RELAYFILE_TOKEN" +``` + +The sandbox sees ordinary files under `/workspace`, but the source of truth is the RelayFile workspace. That means: + +- sandbox snapshots can be short-lived without losing shared files +- agents can restart and reconnect to the same workspace +- multiple agents can work against the same directory tree without shipping tarballs around + +## How Agents Share Files Via RelayFile + +RelayFile turns file exchange into normal filesystem work: + +- Agent A writes `/notes/plan.md` +- Agent B mounts the same workspace and sees `/notes/plan.md` +- A human reviews or edits the same file locally +- writeback, event feeds, and operation logs preserve the sync history + +This is especially useful for: + +- generated code or patch files +- logs and structured artifacts +- prompts, plans, and handoff notes +- exported bundles and review outputs + +## Shared Human And Agent Workspaces + +A practical pattern is: + +1. Human mounts `project-x` locally. +2. Cloud agent mounts `project-x` inside its sandbox. +3. Human edits requirements or reviews generated code. +4. Agent writes implementation changes back into the same tree. +5. Both sides see updates after the next sync cycle. + +That avoids manual upload, download, and copy-paste loops. + +## Operational Notes + +- Use a unique workspace per workflow run when isolation matters more than continuity. +- Reuse a long-lived workspace when a team wants a shared persistent project tree. +- Use the sync, ops, and admin endpoints from `docs/api-reference.md` to inspect ingestion health, dead letters, and replay status. +- For external providers, submit inbound events through `POST /v1/workspaces/{workspaceId}/webhooks/ingest` and consume outbound work through the writeback queue endpoints. diff --git a/docs/guides/collaboration.md b/docs/guides/collaboration.md new file mode 100644 index 00000000..aa3e3b83 --- /dev/null +++ b/docs/guides/collaboration.md @@ -0,0 +1,83 @@ +# Collaboration With RelayFile + +RelayFile is built for shared editing across machines, services, and agent sandboxes. The core model is simple: everyone mounts the same workspace, and RelayFile keeps the local directories converged. + +## Two Humans Collaborating + +Mount the same workspace on both machines. + +Machine A: + +```bash +relayfile mount project-x ./src +``` + +Machine B: + +```bash +relayfile mount project-x ./src +``` + +Typical experience: + +1. Person A edits a file on Machine A. +2. RelayFile uploads the change to the shared workspace. +3. Machine B pulls the update on the next sync cycle. +4. The file appears on Machine B in about 1-2 seconds with the default settings. + +This works well for: + +- pair programming across two laptops +- keeping a staging machine aligned with a development machine +- reviewing generated artifacts without sending archives around + +## Human And Agent Collaboration + +RelayFile also supports shared work between a local human environment and a cloud agent sandbox. + +Example setup: + +- Human mounts the workspace locally: `relayfile mount project-x ./src` +- Agent mounts the same workspace in the cloud sandbox +- Both sides read and write the same files + +Common pattern: + +1. Human seeds a project and starts a mount locally. +2. Agent mounts the same workspace remotely. +3. Agent writes code, notes, or generated artifacts. +4. Human reviews or edits those files locally. +5. The updated files flow back to the agent on the next sync cycle. + +This removes the usual upload-download loop between human terminals and remote agent runs. + +## Conflict Resolution + +Conflicts happen when two writers edit the same file before the system can reconcile the changes. + +RelayFile is designed around conflict-safe writeback and optimistic concurrency: + +- each write is associated with file revision state +- concurrent changes are detected instead of silently overwritten +- sync and operation feeds show what happened +- retry and dead-letter handling keep failures observable instead of hidden + +What to expect in practice: + +1. Machine A and Machine B both edit `src/main.go`. +2. One change lands first. +3. The later write is compared against the current revision. +4. If the write is stale, RelayFile treats it as a conflict rather than blindly replacing the newer version. +5. Clients or operators can inspect the resulting operation and replay state. + +Best practices: + +- Prefer separate files for parallel work when possible. +- Seed once, then keep mounts running so each collaborator is working from a current tree. +- Use the status, event, and ops APIs when you need to diagnose a sync race. + +## Related Guides + +- Setup and first sync: `docs/guides/getting-started.md` +- Cloud workflow model: `docs/guides/cloud-integration.md` +- Full HTTP contract: `docs/api-reference.md` diff --git a/docs/guides/getting-started.md b/docs/guides/getting-started.md new file mode 100644 index 00000000..011489ae --- /dev/null +++ b/docs/guides/getting-started.md @@ -0,0 +1,130 @@ +# Getting Started With RelayFile + +RelayFile gives multiple machines and agents a shared workspace-backed filesystem. A common flow is: + +1. Start a RelayFile server locally or point the CLI at a hosted deployment. +2. Log in and create a workspace. +3. Seed the workspace from an existing project. +4. Mount that workspace on one or more machines. +5. Watch edits propagate through the shared tree. + +## Prerequisites + +- For local development, install Go so you can run `go run ./cmd/relayfile`. +- For hosted usage, you only need the RelayFile CLI plus a server URL and token. +- If you want to mount the same workspace on multiple machines, each machine needs network access to the same RelayFile server. + +## Start The Server Locally + +Run the API server from the repository root: + +```bash +go run ./cmd/relayfile +``` + +By default the service listens on `http://127.0.0.1:8080`. + +If you want durable local state instead of in-memory state, use: + +```bash +RELAYFILE_BACKEND_PROFILE=durable-local \ +RELAYFILE_DATA_DIR=.data \ +go run ./cmd/relayfile +``` + +If you are using a hosted deployment instead, skip this step and replace `http://127.0.0.1:8080` in the examples below with your hosted server URL. + +## Log In And Create A Workspace + +Log in once so the CLI can store the server URL and token: + +```bash +relayfile login --server http://127.0.0.1:8080 --token dev-token +``` + +Create a workspace for the project you want to sync: + +```bash +relayfile workspace create my-project +``` + +List available workspaces: + +```bash +relayfile workspace list +``` + +If your environment already assigns you a workspace ID, you can use that directly instead of creating one. + +## Seed Files From A Project + +Upload an existing local directory into the workspace: + +```bash +relayfile seed my-project ./src +``` + +Useful variants: + +```bash +relayfile seed my-project . +relayfile seed my-project ./src --exclude node_modules --exclude .git +relayfile seed my-project ./src --dry-run +``` + +`seed` is the fastest way to populate a workspace before collaborators or agents mount it. + +## Mount On Two Machines + +After seeding, mount the same workspace on each machine that should participate. + +Machine A: + +```bash +relayfile mount my-project ./src +``` + +Machine B: + +```bash +relayfile mount my-project ./src +``` + +Both mounts point at the same remote workspace. RelayFile continuously syncs changes between the local directory and the server-backed virtual tree. + +If you want a one-shot sync instead of a long-running mount: + +```bash +relayfile mount my-project ./src --once +``` + +## Watch Files Sync + +With both mounts running: + +1. Edit `./src/README.md` on Machine A. +2. Save the file. +3. Wait for the next sync cycle. +4. The updated file appears on Machine B. + +With the default polling interval, changes usually arrive in about 1-2 seconds. + +If you want to watch the workspace status while testing: + +```bash +relayfile status my-project +``` + +For low-level API testing, you can also inspect the event feed directly: + +```bash +curl -sS \ + -H "Authorization: Bearer dev-token" \ + "http://127.0.0.1:8080/v1/workspaces/my-project/fs/events?path=/&limit=20" +``` + +## Next Steps + +- Collaboration patterns: `docs/guides/collaboration.md` +- Cloud workflow integration: `docs/guides/cloud-integration.md` +- REST contract details: `docs/api-reference.md` diff --git a/scripts/install.sh b/scripts/install.sh new file mode 100755 index 00000000..5c2912ae --- /dev/null +++ b/scripts/install.sh @@ -0,0 +1,132 @@ +#!/bin/sh + +set -eu + +REPO="agentworkforce/relayfile" +CLI_ASSET_PREFIX="relayfile" +INSTALL_PATH="${INSTALL_PATH:-/usr/local/bin/relayfile}" +TMP_DIR="$(mktemp -d)" +RELEASE_TAG="${RELAYFILE_VERSION:-}" + +cleanup() { + rm -rf "$TMP_DIR" +} +trap cleanup EXIT INT TERM + +log() { + printf '%s\n' "$*" >&2 +} + +require_cmd() { + if ! command -v "$1" >/dev/null 2>&1; then + log "missing required command: $1" + exit 1 + fi +} + +detect_os() { + case "$(uname -s)" in + Darwin) printf 'darwin' ;; + Linux) printf 'linux' ;; + *) + log "unsupported operating system: $(uname -s)" + exit 1 + ;; + esac +} + +detect_arch() { + case "$(uname -m)" in + x86_64|amd64) printf 'amd64' ;; + arm64|aarch64) printf 'arm64' ;; + *) + log "unsupported architecture: $(uname -m)" + exit 1 + ;; + esac +} + +fetch_latest_tag() { + curl -fsSL "https://api.github.com/repos/${REPO}/releases/latest" | \ + sed -n 's/.*"tag_name":[[:space:]]*"\([^"]*\)".*/\1/p' | head -n 1 +} + +checksum_verify() { + archive_path="$1" + checksums_path="$2" + expected_line="$(grep " $(basename "$archive_path")\$" "$checksums_path" || true)" + if [ -z "$expected_line" ]; then + log "checksum entry not found for $(basename "$archive_path")" + exit 1 + fi + expected_sum="$(printf '%s' "$expected_line" | awk '{print $1}')" + if command -v sha256sum >/dev/null 2>&1; then + actual_sum="$(sha256sum "$archive_path" | awk '{print $1}')" + else + actual_sum="$(shasum -a 256 "$archive_path" | awk '{print $1}')" + fi + if [ "$expected_sum" != "$actual_sum" ]; then + log "checksum verification failed for $(basename "$archive_path")" + exit 1 + fi +} + +install_binary() { + src="$1" + dest="$2" + dest_dir="$(dirname "$dest")" + install_cmd="install" + if [ ! -w "$dest_dir" ]; then + if command -v sudo >/dev/null 2>&1; then + install_cmd="sudo install" + else + log "write access required for $dest_dir and sudo is not available" + exit 1 + fi + fi + # shellcheck disable=SC2086 + $install_cmd -d "$dest_dir" + # shellcheck disable=SC2086 + $install_cmd "$src" "$dest" +} + +require_cmd curl +require_cmd tar +require_cmd grep +require_cmd sed +require_cmd awk +if ! command -v sha256sum >/dev/null 2>&1 && ! command -v shasum >/dev/null 2>&1; then + log "missing required checksum tool: sha256sum or shasum" + exit 1 +fi + +OS="$(detect_os)" +ARCH="$(detect_arch)" +if [ -z "$RELEASE_TAG" ]; then + RELEASE_TAG="$(fetch_latest_tag)" +fi +if [ -z "$RELEASE_TAG" ]; then + log "failed to resolve latest release tag" + exit 1 +fi + +ARCHIVE_NAME="${CLI_ASSET_PREFIX}_${OS}_${ARCH}.tar.gz" +CHECKSUM_NAME="checksums.txt" +BASE_URL="https://github.com/${REPO}/releases/download/${RELEASE_TAG}" +ARCHIVE_PATH="${TMP_DIR}/${ARCHIVE_NAME}" +CHECKSUM_PATH="${TMP_DIR}/${CHECKSUM_NAME}" + +log "downloading ${ARCHIVE_NAME} from ${RELEASE_TAG}" +curl -fsSL "${BASE_URL}/${ARCHIVE_NAME}" -o "$ARCHIVE_PATH" +curl -fsSL "${BASE_URL}/${CHECKSUM_NAME}" -o "$CHECKSUM_PATH" + +checksum_verify "$ARCHIVE_PATH" "$CHECKSUM_PATH" +tar -xzf "$ARCHIVE_PATH" -C "$TMP_DIR" + +if [ ! -f "${TMP_DIR}/${CLI_ASSET_PREFIX}" ]; then + log "archive did not contain ${CLI_ASSET_PREFIX}" + exit 1 +fi + +install_binary "${TMP_DIR}/${CLI_ASSET_PREFIX}" "$INSTALL_PATH" +log "installed relayfile to ${INSTALL_PATH}" diff --git a/sdk/relayfile-sdk/package-lock.json b/sdk/relayfile-sdk/package-lock.json index 99610dee..92a15660 100644 --- a/sdk/relayfile-sdk/package-lock.json +++ b/sdk/relayfile-sdk/package-lock.json @@ -7,6 +7,7 @@ "": { "name": "@relayfile/sdk", "version": "0.1.0", + "license": "MIT", "devDependencies": { "typescript": "^5.7.3" } diff --git a/sdk/relayfile-sdk/package.json b/sdk/relayfile-sdk/package.json index ffbc8c67..9aa65423 100644 --- a/sdk/relayfile-sdk/package.json +++ b/sdk/relayfile-sdk/package.json @@ -1,11 +1,11 @@ { "name": "@relayfile/sdk", - "version": "0.2.0", - "description": "TypeScript SDK for the RelayFile filesystem-over-REST API.", - "license": "Apache-2.0", + "version": "0.1.0", + "description": "Official TypeScript SDK for RelayFile.", + "license": "MIT", "type": "module", - "main": "./dist/index.js", - "types": "./dist/index.d.ts", + "main": "dist/index.js", + "types": "dist/index.d.ts", "exports": { ".": { "types": "./dist/index.d.ts", @@ -16,8 +16,7 @@ "dist" ], "scripts": { - "build": "tsc -p tsconfig.json", - "clean": "rm -rf dist", + "build": "tsc", "prepublishOnly": "npm run build" }, "publishConfig": { diff --git a/sdk/relayfile-sdk/tsconfig.json b/sdk/relayfile-sdk/tsconfig.json index 4f95f6b2..b2cf52d8 100644 --- a/sdk/relayfile-sdk/tsconfig.json +++ b/sdk/relayfile-sdk/tsconfig.json @@ -1,19 +1,13 @@ { "compilerOptions": { "target": "ES2022", - "module": "ES2022", - "moduleResolution": "Bundler", + "module": "ESNext", + "moduleResolution": "bundler", "declaration": true, - "declarationMap": true, - "sourceMap": true, "outDir": "dist", - "rootDir": "src", - "strict": true, - "noUncheckedIndexedAccess": true, - "exactOptionalPropertyTypes": false, - "skipLibCheck": true + "rootDir": "src" }, "include": [ - "src/**/*.ts" + "src" ] } From d90792bf2a013f83c47f4e83adb8f0447a1ef570 Mon Sep 17 00:00:00 2001 From: Khaliq Date: Tue, 24 Mar 2026 15:27:03 +0100 Subject: [PATCH 03/10] Add npm trusted publishing job to release workflow Adds OIDC-based npm publish with provenance for the TypeScript SDK. Includes npm update step per prpm trusted publishing guidance to avoid outdated npm versions on runners. Co-Authored-By: Claude Opus 4.6 (1M context) --- .github/workflows/release.yml | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 78e4eed0..253d716e 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -8,6 +8,7 @@ on: permissions: contents: write packages: write + id-token: write jobs: release: @@ -37,6 +38,34 @@ jobs: dist/*.tar.gz dist/checksums.txt + npm-publish: + runs-on: ubuntu-latest + needs: release + defaults: + run: + working-directory: sdk/relayfile-sdk + steps: + - name: Check out source + uses: actions/checkout@v4 + + - name: Set up Node + uses: actions/setup-node@v4 + with: + node-version: "20" + registry-url: "https://registry.npmjs.org" + + - name: Update npm to latest + run: npm install -g npm@latest + + - name: Install dependencies + run: npm ci + + - name: Build + run: npm run build + + - name: Publish with provenance + run: npm publish --access public --provenance + docker: runs-on: ubuntu-latest needs: release From e00895d6d8dd5fb02b9f0e69bc2e03122616c099 Mon Sep 17 00:00:00 2001 From: Khaliq Date: Tue, 24 Mar 2026 15:27:29 +0100 Subject: [PATCH 04/10] Add CI/CD workflows, SDK build config updates, and CI design doc Adds dedicated GitHub Actions workflows for CI (tests + typecheck), npm publishing, and Go binary releases. Updates SDK package.json, tsconfig, and README. Adds CI/CD design doc. Co-Authored-By: Claude Opus 4.6 (1M context) --- .github/workflows/ci.yml | 145 ++++++++++ .github/workflows/publish-npm.yml | 141 +++++++++ .github/workflows/release-binaries.yml | 127 ++++++++ docs/ci-cd-design.md | 384 +++++++++++++++++++++++++ sdk/relayfile-sdk/README.md | 114 ++------ sdk/relayfile-sdk/package-lock.json | 3 + sdk/relayfile-sdk/package.json | 33 ++- sdk/relayfile-sdk/tsconfig.json | 7 +- 8 files changed, 847 insertions(+), 107 deletions(-) create mode 100644 .github/workflows/ci.yml create mode 100644 .github/workflows/publish-npm.yml create mode 100644 .github/workflows/release-binaries.yml create mode 100644 docs/ci-cd-design.md diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 00000000..2cde704c --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,145 @@ +name: CI + +on: + pull_request: + push: + branches: + - main + +concurrency: + group: ci-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + go-test: + name: Go Test + runs-on: ubuntu-latest + strategy: + matrix: + go-version: + - "1.22" + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup Go + uses: actions/setup-go@v5 + with: + go-version: ${{ matrix.go-version }} + cache: true + + - name: Run tests + run: go test ./... + + go-build: + name: Go Build + runs-on: ubuntu-latest + needs: go-test + strategy: + matrix: + go-version: + - "1.22" + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup Go + uses: actions/setup-go@v5 + with: + go-version: ${{ matrix.go-version }} + cache: true + + - name: Build all binaries + run: make build + + - name: Upload binaries + uses: actions/upload-artifact@v4 + with: + name: relayfile-binaries + path: bin/ + if-no-files-found: error + + sdk-typecheck: + name: SDK Typecheck + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup Node + uses: actions/setup-node@v4 + with: + node-version: "22" + cache: npm + cache-dependency-path: sdk/relayfile-sdk/package-lock.json + + - name: Install SDK dependencies + working-directory: sdk/relayfile-sdk + run: npm ci + + - name: Build SDK + working-directory: sdk/relayfile-sdk + run: npm run build + + - name: Typecheck SDK + working-directory: sdk/relayfile-sdk + run: npx tsc --noEmit + + e2e: + name: E2E + runs-on: ubuntu-latest + needs: + - go-build + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup Go + uses: actions/setup-go@v5 + with: + go-version: "1.22" + cache: true + + - name: Setup Node + uses: actions/setup-node@v4 + with: + node-version: "22" + cache: npm + cache-dependency-path: sdk/relayfile-sdk/package-lock.json + + - name: Download binaries + uses: actions/download-artifact@v4 + with: + name: relayfile-binaries + path: bin + + - name: Make binaries executable + run: chmod +x bin/* + + - name: Run E2E suite + env: + CI: "true" + run: npx --yes tsx scripts/e2e.ts --ci + + workers-typecheck: + name: Workers Typecheck + if: ${{ hashFiles('packages/server/tsconfig.json') != '' }} + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup Node + uses: actions/setup-node@v4 + with: + node-version: "22" + cache: npm + cache-dependency-path: packages/server/package-lock.json + + - name: Install worker dependencies + working-directory: packages/server + run: npm ci + + - name: Typecheck workers + working-directory: packages/server + run: npx tsc --noEmit diff --git a/.github/workflows/publish-npm.yml b/.github/workflows/publish-npm.yml new file mode 100644 index 00000000..1170111a --- /dev/null +++ b/.github/workflows/publish-npm.yml @@ -0,0 +1,141 @@ +name: Publish NPM Package + +on: + workflow_dispatch: + inputs: + version: + description: "Version bump type" + required: true + type: choice + options: + - patch + - minor + - major + - prerelease + default: patch + custom_version: + description: "Custom version (optional, overrides version type)" + required: false + type: string + dry_run: + description: "Dry run (do not actually publish)" + required: false + type: boolean + default: false + tag: + description: "NPM dist-tag" + required: false + type: choice + options: + - latest + - next + - beta + - alpha + default: latest + +concurrency: + group: publish-npm + cancel-in-progress: false + +permissions: + contents: write + id-token: write + +jobs: + publish-sdk: + name: Publish @relayfile/sdk + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + token: ${{ secrets.GITHUB_TOKEN }} + + - name: Setup Node 22 + uses: actions/setup-node@v4 + with: + node-version: "22" + cache: npm + cache-dependency-path: sdk/relayfile-sdk/package-lock.json + registry-url: "https://registry.npmjs.org" + + - name: Install deps + working-directory: sdk/relayfile-sdk + run: npm ci + + - name: Version bump + id: version + working-directory: sdk/relayfile-sdk + run: | + CUSTOM_VERSION="${{ github.event.inputs.custom_version }}" + VERSION_TYPE="${{ github.event.inputs.version }}" + + if [ -n "$CUSTOM_VERSION" ]; then + npm version "$CUSTOM_VERSION" --no-git-tag-version --allow-same-version + else + npm version "$VERSION_TYPE" --no-git-tag-version + fi + + NEW_VERSION="$(node -p "require('./package.json').version")" + echo "new_version=$NEW_VERSION" >> "$GITHUB_OUTPUT" + echo "tag_name=sdk-v$NEW_VERSION" >> "$GITHUB_OUTPUT" + + - name: Build + working-directory: sdk/relayfile-sdk + run: npm run build + + - name: Test + working-directory: sdk/relayfile-sdk + run: npx tsc --noEmit + + - name: Dry run check + if: github.event.inputs.dry_run == 'true' + working-directory: sdk/relayfile-sdk + env: + NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} + run: npm publish --dry-run --access public --tag "${{ github.event.inputs.tag }}" --ignore-scripts + + - name: Publish + if: github.event.inputs.dry_run != 'true' + working-directory: sdk/relayfile-sdk + env: + NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} + run: npm publish --access public --provenance --tag "${{ github.event.inputs.tag }}" --ignore-scripts + + - name: Commit version bump and create git tag + if: github.event.inputs.dry_run != 'true' + run: | + NEW_VERSION="${{ steps.version.outputs.new_version }}" + TAG_NAME="${{ steps.version.outputs.tag_name }}" + + git config user.name "GitHub Actions" + git config user.email "actions@github.com" + + git add sdk/relayfile-sdk/package.json sdk/relayfile-sdk/package-lock.json + git commit -m "chore(sdk): release v${NEW_VERSION}" + git tag -a "${TAG_NAME}" -m "SDK ${NEW_VERSION}" + + git push origin HEAD:main + git push origin "${TAG_NAME}" + + - name: Create GitHub Release + if: github.event.inputs.dry_run != 'true' + uses: softprops/action-gh-release@v2 + with: + tag_name: ${{ steps.version.outputs.tag_name }} + name: ${{ steps.version.outputs.tag_name }} + body: | + ## @relayfile/sdk ${{ steps.version.outputs.new_version }} + + Install: + ```bash + npm install @relayfile/sdk@${{ steps.version.outputs.new_version }} + ``` + + Dist-tag: `${{ github.event.inputs.tag }}` + Provenance: enabled via `npm publish --provenance` + generate_release_notes: true + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/release-binaries.yml b/.github/workflows/release-binaries.yml new file mode 100644 index 00000000..f51433c9 --- /dev/null +++ b/.github/workflows/release-binaries.yml @@ -0,0 +1,127 @@ +name: Release Binaries + +on: + push: + tags: + - "v*" + +permissions: + contents: write + packages: write + +concurrency: + group: release-binaries-${{ github.ref }} + cancel-in-progress: false + +env: + GO_VERSION: "1.22" + REGISTRY_IMAGE: ghcr.io/agentworkforce/relayfile + +jobs: + build: + name: Build ${{ matrix.goos }}-${{ matrix.goarch }} + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + - goos: linux + goarch: amd64 + - goos: linux + goarch: arm64 + - goos: darwin + goarch: amd64 + - goos: darwin + goarch: arm64 + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup Go 1.22 + uses: actions/setup-go@v5 + with: + go-version: ${{ env.GO_VERSION }} + cache: true + + - name: Cross-compile binaries + env: + CGO_ENABLED: "0" + GOOS: ${{ matrix.goos }} + GOARCH: ${{ matrix.goarch }} + run: | + mkdir -p dist + go build -o "dist/relayfile-${GOOS}-${GOARCH}" ./cmd/relayfile + go build -o "dist/relayfile-mount-${GOOS}-${GOARCH}" ./cmd/relayfile-mount + go build -o "dist/relayfile-cli-${GOOS}-${GOARCH}" ./cmd/relayfile-cli + + - name: Generate SHA256 checksums + working-directory: dist + run: | + shasum -a 256 \ + "relayfile-${{ matrix.goos }}-${{ matrix.goarch }}" \ + "relayfile-mount-${{ matrix.goos }}-${{ matrix.goarch }}" \ + "relayfile-cli-${{ matrix.goos }}-${{ matrix.goarch }}" \ + > "checksums-${{ matrix.goos }}-${{ matrix.goarch }}.txt" + + - name: Upload build artifacts + uses: actions/upload-artifact@v4 + with: + name: release-${{ matrix.goos }}-${{ matrix.goarch }} + path: | + dist/relayfile-${{ matrix.goos }}-${{ matrix.goarch }} + dist/relayfile-mount-${{ matrix.goos }}-${{ matrix.goarch }} + dist/relayfile-cli-${{ matrix.goos }}-${{ matrix.goarch }} + dist/checksums-${{ matrix.goos }}-${{ matrix.goarch }}.txt + + release: + name: Publish GitHub Release + runs-on: ubuntu-latest + needs: build + + steps: + - name: Download release artifacts + uses: actions/download-artifact@v4 + with: + path: release-assets + merge-multiple: true + + - name: Show release assets + run: ls -lah release-assets + + - name: Upload to GitHub Release + uses: softprops/action-gh-release@v2 + with: + tag_name: ${{ github.ref_name }} + files: release-assets/* + generate_release_notes: true + + docker: + name: Build and Push Docker Image + runs-on: ubuntu-latest + needs: release + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build and push Docker image + uses: docker/build-push-action@v6 + with: + context: . + file: ./Dockerfile + platforms: linux/amd64,linux/arm64 + push: true + tags: | + ${{ env.REGISTRY_IMAGE }}:latest + ${{ env.REGISTRY_IMAGE }}:${{ github.ref_name }} diff --git a/docs/ci-cd-design.md b/docs/ci-cd-design.md new file mode 100644 index 00000000..0fbfefe2 --- /dev/null +++ b/docs/ci-cd-design.md @@ -0,0 +1,384 @@ +# RelayFile CI/CD Pipeline Design + +## Overview + +Four GitHub Actions workflows covering continuous integration, SDK publishing, binary releases, and Cloudflare Workers deployment. + +--- + +## 1. `ci.yml` — Continuous Integration + +**Trigger:** Pull requests targeting `main`, pushes to `main`, manual dispatch. + +```yaml +name: CI + +on: + pull_request: + branches: [main] + push: + branches: [main] + workflow_dispatch: + +concurrency: + group: ci-${{ github.ref }} + cancel-in-progress: true +``` + +### Jobs + +#### `go-test` — Go Tests & Build +- **Runner:** `ubuntu-latest` +- **Steps:** + 1. `actions/checkout@v4` + 2. `actions/setup-go@v5` with `go-version-file: go.mod`, `cache: true` + 3. `go test ./...` — run all Go package tests + 4. `go build ./cmd/relayfile ./cmd/relayfile-mount ./cmd/relayfile-cli` — verify all three binaries compile + +#### `sdk` — TypeScript SDK +- **Runner:** `ubuntu-latest` +- **Steps:** + 1. `actions/checkout@v4` + 2. `actions/setup-node@v4` with `node-version: "20"` + 3. `cd sdk/relayfile-sdk && npm ci` + 4. `npm run build` — compile TypeScript + 5. `npx tsc --noEmit` — strict type-check (catches errors not surfaced by build) + +#### `e2e` — End-to-End Tests +- **Runner:** `ubuntu-latest` +- **Needs:** `go-test`, `sdk` +- **Steps:** + 1. `actions/checkout@v4` + 2. Setup Go + Node (same as above) + 3. `go build -o bin/relayfile-server ./cmd/relayfile` — build the server binary + 4. Start server in background: `bin/relayfile-server &` + 5. Wait for server ready (curl health check with retry) + 6. `npx tsx scripts/e2e.ts --ci` — run E2E test suite + 7. Kill server, collect exit code + +#### `workers-typecheck` — CF Workers Type Check (conditional) +- **Runner:** `ubuntu-latest` +- **Condition:** `hashFiles('packages/server/tsconfig.json') != ''` +- **Steps:** + 1. `actions/checkout@v4` + 2. `actions/setup-node@v4` with `node-version: "20"` + 3. `cd packages/server && npm ci && npx tsc --noEmit` + +### Secrets & Permissions +- **Permissions:** default (read) +- **Secrets:** none + +--- + +## 2. `publish-npm.yml` — SDK Publishing + +**Trigger:** `workflow_dispatch` (manual only). + +```yaml +name: Publish SDK + +on: + workflow_dispatch: + inputs: + version: + description: "Version bump type" + required: true + type: choice + options: [patch, minor, major, prepatch, preminor, premajor, prerelease] + custom_version: + description: "Custom version (optional, overrides version type)" + required: false + type: string + preid: + description: "Prerelease identifier (used with pre* version types)" + required: false + type: choice + options: [beta, alpha, rc] + default: "beta" + dry_run: + description: "Dry run (do not actually publish)" + required: false + type: boolean + default: false + tag: + description: "NPM dist-tag" + required: false + type: choice + options: [latest, next, beta, alpha] + default: "latest" + +concurrency: + group: publish-sdk + cancel-in-progress: false + +permissions: + contents: write + id-token: write # REQUIRED for npm provenance (OIDC) +``` + +### Jobs + +#### `build` — Build & Version +- **Runner:** `ubuntu-latest` +- **Outputs:** `new_version`, `is_prerelease` +- **Steps:** + 1. `actions/checkout@v4` with `token: ${{ secrets.GITHUB_TOKEN }}` + 2. `actions/setup-node@v4` with `node-version: "20"`, `registry-url: "https://registry.npmjs.org"` **(CRITICAL)** + 3. `npm ci` in `sdk/relayfile-sdk` + 4. Version bump logic: + - If `custom_version` is set, use `npm version "$CUSTOM_VERSION" --no-git-tag-version --allow-same-version` + - Otherwise, `npm version "$VERSION_TYPE" --no-git-tag-version --preid="$PREID"` + 5. Read new version from `package.json`, detect prerelease (`*-*` pattern) + 6. `npm run build` — compile SDK + 7. `npm test` (if tests exist) — verify before publish + 8. `actions/upload-artifact@v4` — upload `package.json` + `dist/` + +#### `publish` — Publish to NPM +- **Runner:** `ubuntu-latest` +- **Needs:** `build` +- **Steps:** + 1. `actions/checkout@v4` + 2. `actions/setup-node@v4` with `node-version: "20"`, `registry-url: "https://registry.npmjs.org"` + 3. `actions/download-artifact@v4` — restore built artifacts + 4. `npm install -g npm@latest` — ensure OIDC/provenance support + 5. Verify `dist/` contents (file count, dry-run pack) + 6. **Dry run path:** `npm publish --dry-run --access public --tag $TAG --ignore-scripts` + 7. **Publish path:** `npm publish --access public --provenance --tag $TAG --ignore-scripts` + +#### `create-tag` — Git Tag & Release +- **Runner:** `ubuntu-latest` +- **Needs:** `build`, `publish` +- **Condition:** `github.event.inputs.dry_run != 'true'` +- **Steps:** + 1. `actions/checkout@v4` with `fetch-depth: 0` + 2. Download artifacts (for updated `package.json`) + 3. Commit bumped `package.json` to `main` + 4. Create annotated tag `sdk-v${NEW_VERSION}` + 5. Push tag + +#### `summary` — Report +- **Condition:** `always()` +- Writes job summary table with version, tag, dry-run status, per-stage results + +### Secrets & Permissions +| Secret | Purpose | +|--------|---------| +| `GITHUB_TOKEN` (automatic) | Push commits, create tags/releases | +| `NODE_AUTH_TOKEN` | NPM automation token **(CRITICAL — must be set as repo secret)** | + +| Permission | Purpose | +|------------|---------| +| `contents: write` | Push version commits, create tags | +| `id-token: write` | OIDC token for npm provenance attestation | + +### NPM Token Setup +1. Generate an **Automation** token at npmjs.com (not Granular — Automation tokens bypass 2FA) +2. Add as repository secret: `Settings > Secrets > Actions > NODE_AUTH_TOKEN` +3. `actions/setup-node` with `registry-url` automatically wires `NODE_AUTH_TOKEN` into `.npmrc` + +--- + +## 3. `release-binaries.yml` — Binary Releases + +**Trigger:** Tag push matching `v*`. + +```yaml +name: Release Binaries + +on: + push: + tags: + - "v*" + +permissions: + contents: write + packages: write # For GHCR Docker push + +concurrency: + group: release-binaries + cancel-in-progress: false +``` + +### Jobs + +#### `test` — Gate +- **Runner:** `ubuntu-latest` +- **Steps:** + 1. Checkout + setup Go + 2. `make test` — full test suite must pass before release + +#### `build` — Cross-Compile (matrix) +- **Runner:** `${{ matrix.os }}` +- **Needs:** `test` +- **Matrix:** + +| os | GOOS | GOARCH | suffix | +|----|------|--------|--------| +| `ubuntu-latest` | linux | amd64 | linux-amd64 | +| `ubuntu-latest` | linux | arm64 | linux-arm64 | +| `macos-latest` | darwin | amd64 | darwin-amd64 | +| `macos-latest` | darwin | arm64 | darwin-arm64 | + +- **Steps:** + 1. Checkout + setup Go + 2. `make build-all VERSION=${{ github.ref_name }}` or direct `go build` with ldflags + 3. Produces three binaries per platform: `relayfile`, `relayfile-server`, `relayfile-mount` + 4. Upload each platform's binaries as artifact + +#### `release` — Create GitHub Release +- **Runner:** `ubuntu-latest` +- **Needs:** `build` +- **Steps:** + 1. Download all artifacts + 2. Create tarballs: `{binary}_{goos}_{goarch}.tar.gz` + 3. Generate checksums: `sha256sum *.tar.gz > checksums.txt` + 4. `softprops/action-gh-release@v2` with: + - `tag_name: ${{ github.ref_name }}` + - `files: dist/*.tar.gz, dist/checksums.txt` + - `generate_release_notes: true` + +Alternatively, delegate to `make release VERSION=${{ github.ref_name }}` which already produces tarballs + checksums in `dist/`. + +#### `docker` — Docker Image +- **Runner:** `ubuntu-latest` +- **Needs:** `release` +- **Steps:** + 1. Checkout + 2. `docker/setup-buildx-action@v3` + 3. `docker/login-action@v3` — login to `ghcr.io` with `GITHUB_TOKEN` + 4. `docker/metadata-action@v5` — extract tags (`type=ref,event=tag` + `type=raw,value=latest`) + 5. `docker/build-push-action@v6`: + - `platforms: linux/amd64,linux/arm64` + - `push: true` + - `context: .`, `file: ./Dockerfile` + +### Secrets & Permissions +| Secret | Purpose | +|--------|---------| +| `GITHUB_TOKEN` (automatic) | Create release, push Docker image to GHCR | + +| Permission | Purpose | +|------------|---------| +| `contents: write` | Create GitHub releases | +| `packages: write` | Push to GitHub Container Registry | + +### Release Flow +``` +git tag v1.2.0 && git push origin v1.2.0 + -> test -> build (4 platforms x 3 binaries) -> release (tarballs + checksums) -> docker (multi-arch) +``` + +--- + +## 4. `deploy-workers.yml` — Cloudflare Workers Deployment + +**Trigger:** Push to `main` with changes in `packages/server/`. + +```yaml +name: Deploy Workers + +on: + push: + branches: [main] + paths: + - "packages/server/**" + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: deploy-workers + cancel-in-progress: false +``` + +### Jobs + +#### `test` — Pre-Deploy Gate +- **Runner:** `ubuntu-latest` +- **Steps:** + 1. Checkout + setup Node + 2. `cd packages/server && npm ci` + 3. `npx tsc --noEmit` — type-check + 4. Run tests if present + +#### `migrate` — D1 Database Migrations +- **Runner:** `ubuntu-latest` +- **Needs:** `test` +- **Steps:** + 1. Checkout + setup Node + 2. `npm ci` in `packages/server` + 3. `npx wrangler d1 migrations apply $D1_DATABASE_NAME --remote` + +#### `deploy` — Deploy to Production +- **Runner:** `ubuntu-latest` +- **Needs:** `migrate` +- **Steps:** + 1. Checkout + setup Node + 2. `npm ci` in `packages/server` + 3. `npx wrangler deploy` (uses `wrangler.toml` in `packages/server/`) + +### Secrets & Permissions +| Secret | Purpose | +|--------|---------| +| `CLOUDFLARE_API_TOKEN` | Wrangler authentication | +| `CLOUDFLARE_ACCOUNT_ID` | Target Cloudflare account | + +| Variable | Purpose | +|----------|---------| +| `D1_DATABASE_NAME` | D1 database name for migrations | +| `D1_DATABASE_ID` | D1 database ID (if needed by wrangler config) | + +| Permission | Purpose | +|------------|---------| +| `contents: read` | Checkout only | + +### Wrangler Auth +Wrangler reads `CLOUDFLARE_API_TOKEN` and `CLOUDFLARE_ACCOUNT_ID` from environment automatically. Set both as repository secrets. + +--- + +## Secrets Summary + +| Secret | Workflow(s) | How to Obtain | +|--------|------------|---------------| +| `GITHUB_TOKEN` | all | Automatic — provided by Actions | +| `NODE_AUTH_TOKEN` | `publish-npm` | npmjs.com > Access Tokens > Automation | +| `CLOUDFLARE_API_TOKEN` | `deploy-workers` | Cloudflare dashboard > API Tokens > Create (Workers edit) | +| `CLOUDFLARE_ACCOUNT_ID` | `deploy-workers` | Cloudflare dashboard > Account ID | + +## Workflow Interaction Diagram + +``` +PR opened / push to main + | + v +ci.yml -----> go-test + sdk + e2e + workers-typecheck + (gate for merge) + +push to main + packages/server/** changed + | + v +deploy-workers.yml -----> test -> migrate -> deploy + +manual trigger + | + v +publish-npm.yml -----> build -> publish (@relayfile/sdk) -> tag + release + +git tag v* + | + v +release-binaries.yml -----> test -> build (4 platforms) -> release (tarballs) -> docker (GHCR) +``` + +## Migration from Existing Workflows + +The four new workflows replace the current set: + +| Current | Replaced By | Notes | +|---------|-------------|-------| +| `contract.yml` | `ci.yml` | Expanded: adds E2E, Go build check, workers typecheck | +| `publish-sdk.yml` | `publish-npm.yml` | Same pattern, kept aligned with relaycast reference | +| `release-binary.yml` | `release-binaries.yml` | Now triggered on `v*` tags (not every push to main), adds checksums, all 3 binaries, Docker | +| `release.yml` | `release-binaries.yml` | Consolidated — Docker job folded into binary release workflow | diff --git a/sdk/relayfile-sdk/README.md b/sdk/relayfile-sdk/README.md index 8379c429..5427de48 100644 --- a/sdk/relayfile-sdk/README.md +++ b/sdk/relayfile-sdk/README.md @@ -1,6 +1,6 @@ # @relayfile/sdk -TypeScript SDK for the RelayFile API contract in `openapi/relayfile-v1.openapi.yaml`. +TypeScript SDK for relayfile — real-time filesystem for humans and agents. ## Install @@ -8,107 +8,33 @@ TypeScript SDK for the RelayFile API contract in `openapi/relayfile-v1.openapi.y npm install @relayfile/sdk ``` -## Usage +## Quick Example ```ts -import { InvalidStateError, QueueFullError, RelayFileClient, RevisionConflictError } from "@relayfile/sdk"; +import { RelayFileClient } from "@relayfile/sdk"; const client = new RelayFileClient({ - baseUrl: "https://relayfile.agent-relay.com", - token: () => process.env.RELAYFILE_TOKEN ?? "", - retry: { - maxRetries: 3, - baseDelayMs: 100, - maxDelayMs: 2000, - jitterRatio: 0.2 - } + baseUrl: "https://api.relayfile.com", + token: process.env.RELAYFILE_TOKEN ?? "" }); -const workspaceId = "ws_123"; +const workspaceId = "workspace_123"; -const controller = new AbortController(); -const tree = await client.listTree(workspaceId, { - path: "/", - depth: 2, - signal: controller.signal -}); -const file = await client.readFile(workspaceId, "/documents/page.md"); -const query = await client.queryFiles(workspaceId, { - path: "/documents", - provider: "salesforce", // Any provider - relation: "account:123", - properties: { status: "active" }, - limit: 25 -}); -const events = await client.getEvents(workspaceId, { provider: "salesforce", limit: 50 }); -const ops = await client.listOps(workspaceId, { status: "dead_lettered", action: "file_upsert", provider: "salesforce", limit: 20 }); -const sync = await client.getSyncStatus(workspaceId, { provider: "salesforce" }); -const ingress = await client.getSyncIngressStatus(workspaceId, { provider: "salesforce" }); -const adminIngress = await client.getAdminIngressStatus({ provider: "salesforce", alertProfile: "balanced", deadLetterThreshold: 2, nonZeroOnly: true, maxAlerts: 50, includeWorkspaces: true, includeAlerts: true }); -const adminSync = await client.getAdminSyncStatus({ provider: "salesforce", nonZeroOnly: true, includeWorkspaces: true, limit: 100, lagSecondsThreshold: 45, maxAlerts: 50, includeAlerts: true }); -const deadLetters = await client.getSyncDeadLetters(workspaceId, { provider: "salesforce", limit: 20 }); -console.log(events.events.length); -console.log(query.items.length); -console.log(ops.items.length); -console.log(sync.providers[0]?.status, sync.providers[0]?.failureCodes, sync.providers[0]?.deadLetteredEnvelopes, sync.providers[0]?.deadLetteredOps); -console.log(ingress.queueDepth, ingress.droppedTotal); -console.log(ingress.queueUtilization, ingress.oldestPendingAgeSeconds); -console.log(ingress.coalescedTotal, ingress.suppressedTotal, ingress.staleTotal); -console.log(ingress.dedupeRate, ingress.coalesceRate); -console.log(ingress.deadLetterByProvider); -console.log(ingress.ingressByProvider["salesforce"]?.pendingTotal, ingress.ingressByProvider["salesforce"]?.oldestPendingAgeSeconds); -console.log(adminIngress.alertProfile, adminIngress.effectiveAlertProfile, adminIngress.workspaceCount, adminIngress.returnedWorkspaceCount, adminIngress.nextCursor, adminIngress.pendingTotal, adminIngress.thresholds.deadLetter, adminIngress.alertTotals.critical, adminIngress.alertsTruncated, adminIngress.alerts.length, Object.keys(adminIngress.workspaces)); -console.log(adminSync.workspaceCount, adminSync.returnedWorkspaceCount, adminSync.nextCursor, adminSync.providerStatusCount, adminSync.errorCount, adminSync.failureCodes, adminSync.thresholds.lagSeconds, adminSync.alertTotals.critical, adminSync.alertsTruncated, adminSync.alerts.length); -console.log(deadLetters.items.length); -if (deadLetters.items.length > 0) { - const detail = await client.getSyncDeadLetter(workspaceId, deadLetters.items[0].envelopeId); - console.log(detail.lastError); - await client.replaySyncDeadLetter(workspaceId, deadLetters.items[0].envelopeId); - await client.ackSyncDeadLetter(workspaceId, deadLetters.items[0].envelopeId); -} -if (ops.items.length > 0) { - await client.replayOp(workspaceId, ops.items[0].opId); - await client.replayAdminOp(ops.items[0].opId); -} -await client.replayAdminEnvelope("env_123"); +const tree = await client.listTree(workspaceId, { path: "/", depth: 2 }); +console.log(tree.entries.map((entry) => entry.path)); -try { - const write = await client.writeFile({ - workspaceId, - path: file.path, - baseRevision: file.revision, - content: file.content + "\n\nUpdated by agent.", - contentType: "text/markdown", - semantics: { - properties: { stage: "active" }, - relations: ["db:investments"], - permissions: ["scope:fs:read"], - comments: ["comment_123"] - } - }); - console.log(write.opId); -} catch (err) { - if (err instanceof RevisionConflictError) { - console.error("conflict", err.currentRevision); - } - if (err instanceof QueueFullError) { - console.error("ingress saturated, retry in", err.retryAfterSeconds ?? 1, "seconds"); - } - if (err instanceof InvalidStateError) { - console.error("cannot replay in current state"); - } - throw err; -} +const file = await client.readFile(workspaceId, "/notes/todo.md"); +console.log(file.content); + +await client.writeFile({ + workspaceId, + path: "/notes/todo.md", + baseRevision: file.revision, + content: `${file.content}\n- Follow up with SDK publish`, + contentType: "text/markdown" +}); ``` -## Notes +## Full Docs -- All requests send `X-Correlation-Id` automatically if not provided. -- Requests retry transient `429/5xx` and network errors with jittered exponential backoff. -- Most option/input shapes accept `signal?: AbortSignal` for request cancellation. -- `writeFile` and `deleteFile` require optimistic concurrency via revision preconditions. -- `RevisionConflictError` is thrown for HTTP `409` conflict responses. -- `QueueFullError` is thrown for HTTP `429` with `code=queue_full` and surfaces `retryAfterSeconds`. -- `InvalidStateError` is thrown for HTTP `409` with `code=invalid_state` (for replay preconditions). -- `replayAdminEnvelope` and `replayAdminOp` call admin replay endpoints and require a token with `admin:replay`. -- `getBackendStatus`, `getAdminIngressStatus`, and `getAdminSyncStatus` are admin APIs and require `admin:read` (or `admin:replay`). +Full documentation is available at https://github.com/AgentWorkforce/relayfile diff --git a/sdk/relayfile-sdk/package-lock.json b/sdk/relayfile-sdk/package-lock.json index 92a15660..eec28ca3 100644 --- a/sdk/relayfile-sdk/package-lock.json +++ b/sdk/relayfile-sdk/package-lock.json @@ -10,6 +10,9 @@ "license": "MIT", "devDependencies": { "typescript": "^5.7.3" + }, + "engines": { + "node": ">=18" } }, "node_modules/typescript": { diff --git a/sdk/relayfile-sdk/package.json b/sdk/relayfile-sdk/package.json index 9aa65423..08152ce6 100644 --- a/sdk/relayfile-sdk/package.json +++ b/sdk/relayfile-sdk/package.json @@ -1,32 +1,41 @@ { "name": "@relayfile/sdk", "version": "0.1.0", - "description": "Official TypeScript SDK for RelayFile.", - "license": "MIT", - "type": "module", + "description": "TypeScript SDK for relayfile — real-time filesystem for humans and agents", "main": "dist/index.js", + "module": "dist/index.js", "types": "dist/index.d.ts", - "exports": { - ".": { - "types": "./dist/index.d.ts", - "import": "./dist/index.js" - } - }, + "type": "module", "files": [ - "dist" + "dist", + "README.md", + "LICENSE" ], "scripts": { "build": "tsc", + "typecheck": "tsc --noEmit", "prepublishOnly": "npm run build" }, "publishConfig": { - "access": "public" + "access": "public", + "provenance": true }, "repository": { "type": "git", - "url": "https://github.com/AgentWorkforce/relayfile.git", + "url": "https://github.com/AgentWorkforce/relayfile", "directory": "sdk/relayfile-sdk" }, + "license": "MIT", + "keywords": [ + "relayfile", + "filesystem", + "sync", + "agent", + "collaboration" + ], + "engines": { + "node": ">=18" + }, "devDependencies": { "typescript": "^5.7.3" } diff --git a/sdk/relayfile-sdk/tsconfig.json b/sdk/relayfile-sdk/tsconfig.json index b2cf52d8..7ea6cb81 100644 --- a/sdk/relayfile-sdk/tsconfig.json +++ b/sdk/relayfile-sdk/tsconfig.json @@ -4,8 +4,13 @@ "module": "ESNext", "moduleResolution": "bundler", "declaration": true, + "declarationMap": true, + "sourceMap": true, "outDir": "dist", - "rootDir": "src" + "rootDir": "src", + "strict": true, + "esModuleInterop": true, + "skipLibCheck": true }, "include": [ "src" From 795fae17b836497272b0081361f964990cfda5f4 Mon Sep 17 00:00:00 2001 From: Khaliq Date: Tue, 24 Mar 2026 15:29:57 +0100 Subject: [PATCH 05/10] Add relayfile CLI npm package and update publish workflow Adds sdk/relayfile package that downloads the correct platform binary on install, so users can `npx relayfile` or `npm install -g relayfile`. Updates publish-npm workflow to use OIDC (no NPM_TOKEN), adds npm update step, and publishes both @relayfile/sdk and relayfile packages with version sync. Co-Authored-By: Claude Opus 4.6 (1M context) --- .github/workflows/publish-npm.yml | 41 ++++++++++++++-- sdk/relayfile/package.json | 32 +++++++++++++ sdk/relayfile/scripts/install.js | 79 +++++++++++++++++++++++++++++++ 3 files changed, 148 insertions(+), 4 deletions(-) create mode 100644 sdk/relayfile/package.json create mode 100644 sdk/relayfile/scripts/install.js diff --git a/.github/workflows/publish-npm.yml b/.github/workflows/publish-npm.yml index 1170111a..fdd42d74 100644 --- a/.github/workflows/publish-npm.yml +++ b/.github/workflows/publish-npm.yml @@ -61,6 +61,9 @@ jobs: cache-dependency-path: sdk/relayfile-sdk/package-lock.json registry-url: "https://registry.npmjs.org" + - name: Update npm to latest + run: npm install -g npm@latest + - name: Install deps working-directory: sdk/relayfile-sdk run: npm ci @@ -93,15 +96,11 @@ jobs: - name: Dry run check if: github.event.inputs.dry_run == 'true' working-directory: sdk/relayfile-sdk - env: - NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} run: npm publish --dry-run --access public --tag "${{ github.event.inputs.tag }}" --ignore-scripts - name: Publish if: github.event.inputs.dry_run != 'true' working-directory: sdk/relayfile-sdk - env: - NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} run: npm publish --access public --provenance --tag "${{ github.event.inputs.tag }}" --ignore-scripts - name: Commit version bump and create git tag @@ -139,3 +138,37 @@ jobs: generate_release_notes: true env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + publish-cli: + name: Publish relayfile (CLI) + runs-on: ubuntu-latest + needs: publish-sdk + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup Node 22 + uses: actions/setup-node@v4 + with: + node-version: "22" + registry-url: "https://registry.npmjs.org" + + - name: Update npm to latest + run: npm install -g npm@latest + + - name: Sync version with SDK + working-directory: sdk/relayfile + run: | + SDK_VERSION="$(node -p "require('../relayfile-sdk/package.json').version")" + npm version "$SDK_VERSION" --no-git-tag-version --allow-same-version + + - name: Dry run check + if: github.event.inputs.dry_run == 'true' + working-directory: sdk/relayfile + run: npm publish --dry-run --access public --tag "${{ github.event.inputs.tag }}" + + - name: Publish + if: github.event.inputs.dry_run != 'true' + working-directory: sdk/relayfile + run: npm publish --access public --provenance --tag "${{ github.event.inputs.tag }}" diff --git a/sdk/relayfile/package.json b/sdk/relayfile/package.json new file mode 100644 index 00000000..54da6081 --- /dev/null +++ b/sdk/relayfile/package.json @@ -0,0 +1,32 @@ +{ + "name": "relayfile", + "version": "0.1.0", + "description": "CLI for relayfile — real-time filesystem for humans and agents", + "bin": { + "relayfile": "bin/relayfile" + }, + "scripts": { + "postinstall": "node scripts/install.js" + }, + "publishConfig": { + "access": "public", + "provenance": true + }, + "repository": { + "type": "git", + "url": "https://github.com/AgentWorkforce/relayfile", + "directory": "sdk/relayfile" + }, + "license": "MIT", + "keywords": [ + "relayfile", + "filesystem", + "sync", + "agent", + "collaboration", + "cli" + ], + "engines": { + "node": ">=18" + } +} diff --git a/sdk/relayfile/scripts/install.js b/sdk/relayfile/scripts/install.js new file mode 100644 index 00000000..7d0d5b1e --- /dev/null +++ b/sdk/relayfile/scripts/install.js @@ -0,0 +1,79 @@ +#!/usr/bin/env node + +const { execSync } = require("child_process"); +const fs = require("fs"); +const path = require("path"); +const os = require("os"); +const https = require("https"); + +const VERSION = require("../package.json").version; +const BIN_DIR = path.join(__dirname, "..", "bin"); +const BIN_PATH = path.join(BIN_DIR, "relayfile"); + +const PLATFORM_MAP = { + darwin: "darwin", + linux: "linux", + win32: "windows", +}; + +const ARCH_MAP = { + x64: "amd64", + arm64: "arm64", +}; + +function getDownloadUrl() { + const platform = PLATFORM_MAP[os.platform()]; + const arch = ARCH_MAP[os.arch()]; + + if (!platform || !arch) { + console.error( + `Unsupported platform: ${os.platform()} ${os.arch()}` + ); + process.exit(1); + } + + const ext = platform === "windows" ? ".exe" : ""; + return `https://github.com/AgentWorkforce/relayfile/releases/download/v${VERSION}/relayfile-${platform}-${arch}${ext}`; +} + +function download(url, dest) { + return new Promise((resolve, reject) => { + const follow = (url) => { + https.get(url, (res) => { + if (res.statusCode >= 300 && res.statusCode < 400 && res.headers.location) { + follow(res.headers.location); + return; + } + if (res.statusCode !== 200) { + reject(new Error(`Download failed: HTTP ${res.statusCode} from ${url}`)); + return; + } + const file = fs.createWriteStream(dest); + res.pipe(file); + file.on("finish", () => { + file.close(resolve); + }); + }).on("error", reject); + }; + follow(url); + }); +} + +async function main() { + const url = getDownloadUrl(); + + fs.mkdirSync(BIN_DIR, { recursive: true }); + + console.log(`Downloading relayfile v${VERSION}...`); + try { + await download(url, BIN_PATH); + fs.chmodSync(BIN_PATH, 0o755); + console.log("relayfile installed successfully."); + } catch (err) { + console.error(`Failed to download relayfile: ${err.message}`); + console.error("You can install manually from https://github.com/AgentWorkforce/relayfile/releases"); + process.exit(1); + } +} + +main(); From 469936c9213f16f7dd3e21d391d9b7c48a9703e8 Mon Sep 17 00:00:00 2001 From: Khaliq Date: Tue, 24 Mar 2026 15:32:10 +0100 Subject: [PATCH 06/10] Set up npm workspaces for SDK and CLI packages Configures root package.json with workspaces pointing to sdk/relayfile-sdk and sdk/relayfile. Makes CLI postinstall non-fatal so installs work before a release binary exists. Co-Authored-By: Claude Opus 4.6 (1M context) --- package-lock.json | 48 ++++++++++++++++++++++++++++++++++++++ package.json | 8 ++++++- sdk/relayfile/package.json | 2 +- 3 files changed, 56 insertions(+), 2 deletions(-) diff --git a/package-lock.json b/package-lock.json index b62db861..53795d07 100644 --- a/package-lock.json +++ b/package-lock.json @@ -4,6 +4,10 @@ "requires": true, "packages": { "": { + "workspaces": [ + "sdk/relayfile-sdk", + "sdk/relayfile" + ], "dependencies": { "@agent-relay/sdk": "latest" } @@ -98,6 +102,10 @@ "url": "https://github.com/sponsors/colinhacks" } }, + "node_modules/@relayfile/sdk": { + "resolved": "sdk/relayfile-sdk", + "link": true + }, "node_modules/@sinclair/typebox": { "version": "0.34.48", "resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.34.48.tgz", @@ -395,6 +403,10 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/relayfile": { + "resolved": "sdk/relayfile", + "link": true + }, "node_modules/restore-cursor": { "version": "5.1.0", "resolved": "https://registry.npmjs.org/restore-cursor/-/restore-cursor-5.1.0.tgz", @@ -500,6 +512,20 @@ "node": ">=8" } }, + "node_modules/typescript": { + "version": "5.9.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", + "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, "node_modules/wrap-ansi": { "version": "10.0.0", "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-10.0.0.tgz", @@ -582,6 +608,28 @@ "peerDependencies": { "zod": "^3.25 || ^4" } + }, + "sdk/relayfile": { + "version": "0.1.0", + "hasInstallScript": true, + "license": "MIT", + "bin": { + "relayfile": "bin/relayfile" + }, + "engines": { + "node": ">=18" + } + }, + "sdk/relayfile-sdk": { + "name": "@relayfile/sdk", + "version": "0.1.0", + "license": "MIT", + "devDependencies": { + "typescript": "^5.7.3" + }, + "engines": { + "node": ">=18" + } } } } diff --git a/package.json b/package.json index 0c34ed82..ea9f165f 100644 --- a/package.json +++ b/package.json @@ -1,8 +1,14 @@ { "private": true, "description": "relayfile — real-time filesystem for humans and agents", + "workspaces": [ + "sdk/relayfile-sdk", + "sdk/relayfile" + ], "scripts": { - "workflow": "agent-relay run" + "workflow": "agent-relay run", + "build": "npm run build --workspace=sdk/relayfile-sdk", + "typecheck": "npm run typecheck --workspace=sdk/relayfile-sdk" }, "dependencies": { "@agent-relay/sdk": "latest" diff --git a/sdk/relayfile/package.json b/sdk/relayfile/package.json index 54da6081..dfec41f7 100644 --- a/sdk/relayfile/package.json +++ b/sdk/relayfile/package.json @@ -6,7 +6,7 @@ "relayfile": "bin/relayfile" }, "scripts": { - "postinstall": "node scripts/install.js" + "postinstall": "node scripts/install.js || true" }, "publishConfig": { "access": "public", From 589426fec286260cc87d96f83d67a1b99b276d64 Mon Sep 17 00:00:00 2001 From: Khaliq Date: Tue, 24 Mar 2026 15:34:46 +0100 Subject: [PATCH 07/10] Move sdk/ to packages/ for conventional npm workspace layout Renames sdk/relayfile-sdk and sdk/relayfile to packages/. Updates all workflow files, GitHub Actions, and package.json references to use the new paths. Co-Authored-By: Claude Opus 4.6 (1M context) --- .github/workflows/ci.yml | 10 +- .github/workflows/contract.yml | 4 +- .github/workflows/publish-npm.yml | 22 ++--- .github/workflows/publish-sdk.yml | 22 ++--- .github/workflows/release.yml | 2 +- package-lock.json | 98 ++----------------- package.json | 8 +- {sdk => packages}/relayfile-sdk/README.md | 0 .../relayfile-sdk/package-lock.json | 0 {sdk => packages}/relayfile-sdk/package.json | 2 +- {sdk => packages}/relayfile-sdk/src/client.ts | 0 {sdk => packages}/relayfile-sdk/src/errors.ts | 0 {sdk => packages}/relayfile-sdk/src/index.ts | 0 {sdk => packages}/relayfile-sdk/src/types.ts | 0 {sdk => packages}/relayfile-sdk/tsconfig.json | 0 {sdk => packages}/relayfile/package.json | 2 +- .../relayfile/scripts/install.js | 0 workflows/relayfile-bulk-and-export.ts | 12 +-- workflows/relayfile-ci-and-publish.ts | 20 ++-- workflows/relayfile-cloud-server.ts | 2 +- workflows/relayfile-developer-experience.ts | 10 +- 21 files changed, 64 insertions(+), 150 deletions(-) rename {sdk => packages}/relayfile-sdk/README.md (100%) rename {sdk => packages}/relayfile-sdk/package-lock.json (100%) rename {sdk => packages}/relayfile-sdk/package.json (95%) rename {sdk => packages}/relayfile-sdk/src/client.ts (100%) rename {sdk => packages}/relayfile-sdk/src/errors.ts (100%) rename {sdk => packages}/relayfile-sdk/src/index.ts (100%) rename {sdk => packages}/relayfile-sdk/src/types.ts (100%) rename {sdk => packages}/relayfile-sdk/tsconfig.json (100%) rename {sdk => packages}/relayfile/package.json (94%) rename {sdk => packages}/relayfile/scripts/install.js (100%) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2cde704c..fed32a2b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -71,18 +71,18 @@ jobs: with: node-version: "22" cache: npm - cache-dependency-path: sdk/relayfile-sdk/package-lock.json + cache-dependency-path: packages/relayfile-sdk/package-lock.json - name: Install SDK dependencies - working-directory: sdk/relayfile-sdk + working-directory: packages/relayfile-sdk run: npm ci - name: Build SDK - working-directory: sdk/relayfile-sdk + working-directory: packages/relayfile-sdk run: npm run build - name: Typecheck SDK - working-directory: sdk/relayfile-sdk + working-directory: packages/relayfile-sdk run: npx tsc --noEmit e2e: @@ -105,7 +105,7 @@ jobs: with: node-version: "22" cache: npm - cache-dependency-path: sdk/relayfile-sdk/package-lock.json + cache-dependency-path: packages/relayfile-sdk/package-lock.json - name: Download binaries uses: actions/download-artifact@v4 diff --git a/.github/workflows/contract.yml b/.github/workflows/contract.yml index a7243261..68631616 100644 --- a/.github/workflows/contract.yml +++ b/.github/workflows/contract.yml @@ -28,11 +28,11 @@ jobs: node-version: "20" - name: Install SDK deps - working-directory: sdk/relayfile-sdk + working-directory: packages/relayfile-sdk run: npm ci - name: Build SDK - working-directory: sdk/relayfile-sdk + working-directory: packages/relayfile-sdk run: npm run build - name: Validate contract surface diff --git a/.github/workflows/publish-npm.yml b/.github/workflows/publish-npm.yml index fdd42d74..32bab2e6 100644 --- a/.github/workflows/publish-npm.yml +++ b/.github/workflows/publish-npm.yml @@ -58,19 +58,19 @@ jobs: with: node-version: "22" cache: npm - cache-dependency-path: sdk/relayfile-sdk/package-lock.json + cache-dependency-path: packages/relayfile-sdk/package-lock.json registry-url: "https://registry.npmjs.org" - name: Update npm to latest run: npm install -g npm@latest - name: Install deps - working-directory: sdk/relayfile-sdk + working-directory: packages/relayfile-sdk run: npm ci - name: Version bump id: version - working-directory: sdk/relayfile-sdk + working-directory: packages/relayfile-sdk run: | CUSTOM_VERSION="${{ github.event.inputs.custom_version }}" VERSION_TYPE="${{ github.event.inputs.version }}" @@ -86,21 +86,21 @@ jobs: echo "tag_name=sdk-v$NEW_VERSION" >> "$GITHUB_OUTPUT" - name: Build - working-directory: sdk/relayfile-sdk + working-directory: packages/relayfile-sdk run: npm run build - name: Test - working-directory: sdk/relayfile-sdk + working-directory: packages/relayfile-sdk run: npx tsc --noEmit - name: Dry run check if: github.event.inputs.dry_run == 'true' - working-directory: sdk/relayfile-sdk + working-directory: packages/relayfile-sdk run: npm publish --dry-run --access public --tag "${{ github.event.inputs.tag }}" --ignore-scripts - name: Publish if: github.event.inputs.dry_run != 'true' - working-directory: sdk/relayfile-sdk + working-directory: packages/relayfile-sdk run: npm publish --access public --provenance --tag "${{ github.event.inputs.tag }}" --ignore-scripts - name: Commit version bump and create git tag @@ -112,7 +112,7 @@ jobs: git config user.name "GitHub Actions" git config user.email "actions@github.com" - git add sdk/relayfile-sdk/package.json sdk/relayfile-sdk/package-lock.json + git add packages/relayfile-sdk/package.json packages/relayfile-sdk/package-lock.json git commit -m "chore(sdk): release v${NEW_VERSION}" git tag -a "${TAG_NAME}" -m "SDK ${NEW_VERSION}" @@ -158,17 +158,17 @@ jobs: run: npm install -g npm@latest - name: Sync version with SDK - working-directory: sdk/relayfile + working-directory: packages/relayfile run: | SDK_VERSION="$(node -p "require('../relayfile-sdk/package.json').version")" npm version "$SDK_VERSION" --no-git-tag-version --allow-same-version - name: Dry run check if: github.event.inputs.dry_run == 'true' - working-directory: sdk/relayfile + working-directory: packages/relayfile run: npm publish --dry-run --access public --tag "${{ github.event.inputs.tag }}" - name: Publish if: github.event.inputs.dry_run != 'true' - working-directory: sdk/relayfile + working-directory: packages/relayfile run: npm publish --access public --provenance --tag "${{ github.event.inputs.tag }}" diff --git a/.github/workflows/publish-sdk.yml b/.github/workflows/publish-sdk.yml index 2409c6c6..eafbe75c 100644 --- a/.github/workflows/publish-sdk.yml +++ b/.github/workflows/publish-sdk.yml @@ -73,12 +73,12 @@ jobs: registry-url: "https://registry.npmjs.org" - name: Install deps - working-directory: sdk/relayfile-sdk + working-directory: packages/relayfile-sdk run: npm ci - name: Version bump id: bump - working-directory: sdk/relayfile-sdk + working-directory: packages/relayfile-sdk run: | CUSTOM_VERSION="${{ github.event.inputs.custom_version }}" VERSION_TYPE="${{ github.event.inputs.version }}" @@ -105,7 +105,7 @@ jobs: fi - name: Build - working-directory: sdk/relayfile-sdk + working-directory: packages/relayfile-sdk run: npm run build - name: Upload build artifacts @@ -113,8 +113,8 @@ jobs: with: name: sdk-build path: | - sdk/relayfile-sdk/package.json - sdk/relayfile-sdk/dist/ + packages/relayfile-sdk/package.json + packages/relayfile-sdk/dist/ retention-days: 1 publish: @@ -136,13 +136,13 @@ jobs: uses: actions/download-artifact@v4 with: name: sdk-build - path: sdk/relayfile-sdk + path: packages/relayfile-sdk - name: Update npm for OIDC support run: npm install -g npm@latest - name: Verify build artifacts - working-directory: sdk/relayfile-sdk + working-directory: packages/relayfile-sdk run: | echo "=== Verifying dist/ contents ===" if [ ! -d "dist" ]; then @@ -163,14 +163,14 @@ jobs: - name: Dry run check if: github.event.inputs.dry_run == 'true' - working-directory: sdk/relayfile-sdk + working-directory: packages/relayfile-sdk run: | echo "Dry run - would publish @relayfile/sdk@${{ needs.build.outputs.new_version }}" npm publish --dry-run --access public --tag ${{ github.event.inputs.tag }} --ignore-scripts - name: Publish with provenance if: github.event.inputs.dry_run != 'true' - working-directory: sdk/relayfile-sdk + working-directory: packages/relayfile-sdk run: npm publish --access public --provenance --tag ${{ github.event.inputs.tag }} --ignore-scripts create-tag: @@ -190,7 +190,7 @@ jobs: uses: actions/download-artifact@v4 with: name: sdk-build - path: sdk/relayfile-sdk + path: packages/relayfile-sdk - name: Commit and tag run: | @@ -199,7 +199,7 @@ jobs: NEW_VERSION="${{ needs.build.outputs.new_version }}" - git add sdk/relayfile-sdk/package.json + git add packages/relayfile-sdk/package.json if ! git diff --staged --quiet; then git commit -m "chore(sdk): v${NEW_VERSION}" git push origin HEAD:main diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 253d716e..a4cf3f2c 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -43,7 +43,7 @@ jobs: needs: release defaults: run: - working-directory: sdk/relayfile-sdk + working-directory: packages/relayfile-sdk steps: - name: Check out source uses: actions/checkout@v4 diff --git a/package-lock.json b/package-lock.json index 53795d07..e54a3f19 100644 --- a/package-lock.json +++ b/package-lock.json @@ -5,8 +5,8 @@ "packages": { "": { "workspaces": [ - "sdk/relayfile-sdk", - "sdk/relayfile" + "packages/relayfile-sdk", + "packages/relayfile" ], "dependencies": { "@agent-relay/sdk": "latest" @@ -14,8 +14,6 @@ }, "node_modules/@agent-relay/config": { "version": "3.2.15", - "resolved": "https://registry.npmjs.org/@agent-relay/config/-/config-3.2.15.tgz", - "integrity": "sha512-oIgspBgO2T9ulauilHmYm5ATk30Jd1YyCGAy67CbELOznF27XuAKfaGHuMHjMIRMhj/PI7VG89DhYaGyHk7O4w==", "dependencies": { "zod": "^3.23.8", "zod-to-json-schema": "^3.23.1" @@ -69,8 +67,6 @@ }, "node_modules/@relaycast/sdk": { "version": "1.0.0", - "resolved": "https://registry.npmjs.org/@relaycast/sdk/-/sdk-1.0.0.tgz", - "integrity": "sha512-s01xslec5xyDXxxkVDTJyHpRhzqlXC2gVoglvhu+HK1h5JeOKq13AFlhe2MszkxjJAQ0HJ36MItWXuGogbRdOg==", "dependencies": { "@relaycast/types": "1.0.0", "zod": "^4.3.6" @@ -78,8 +74,6 @@ }, "node_modules/@relaycast/sdk/node_modules/zod": { "version": "4.3.6", - "resolved": "https://registry.npmjs.org/zod/-/zod-4.3.6.tgz", - "integrity": "sha512-rftlrkhHZOcjDwkGlnUtZZkvaPHCsDATp4pGpuOOMDaTdDDXF91wuVDJoWoPsKX/3YPQ5fHuF3STjcYyKr+Qhg==", "license": "MIT", "funding": { "url": "https://github.com/sponsors/colinhacks" @@ -87,35 +81,27 @@ }, "node_modules/@relaycast/types": { "version": "1.0.0", - "resolved": "https://registry.npmjs.org/@relaycast/types/-/types-1.0.0.tgz", - "integrity": "sha512-f9DnZ91jro+NX3CypIPhTyBzGpmA2ZRE3zu/3aPAAe7JGVrHTcOq0Or9s/pzxSc49m5y6p7vXi9+TYgDeL/xWA==", "dependencies": { "zod": "^4.3.6" } }, "node_modules/@relaycast/types/node_modules/zod": { "version": "4.3.6", - "resolved": "https://registry.npmjs.org/zod/-/zod-4.3.6.tgz", - "integrity": "sha512-rftlrkhHZOcjDwkGlnUtZZkvaPHCsDATp4pGpuOOMDaTdDDXF91wuVDJoWoPsKX/3YPQ5fHuF3STjcYyKr+Qhg==", "license": "MIT", "funding": { "url": "https://github.com/sponsors/colinhacks" } }, "node_modules/@relayfile/sdk": { - "resolved": "sdk/relayfile-sdk", + "resolved": "packages/relayfile-sdk", "link": true }, "node_modules/@sinclair/typebox": { "version": "0.34.48", - "resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.34.48.tgz", - "integrity": "sha512-kKJTNuK3AQOrgjjotVxMrCn1sUJwM76wMszfq1kdU4uYVJjvEWuFQ6HgvLt4Xz3fSmZlTOxJ/Ie13KnIcWQXFA==", "license": "MIT" }, "node_modules/ansi-escapes": { "version": "7.3.0", - "resolved": "https://registry.npmjs.org/ansi-escapes/-/ansi-escapes-7.3.0.tgz", - "integrity": "sha512-BvU8nYgGQBxcmMuEeUEmNTvrMVjJNSH7RgW24vXexN4Ven6qCvy4TntnvlnwnMLTVlcRQQdbRY8NKnaIoeWDNg==", "license": "MIT", "dependencies": { "environment": "^1.0.0" @@ -129,8 +115,6 @@ }, "node_modules/ansi-regex": { "version": "6.2.2", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.2.2.tgz", - "integrity": "sha512-Bq3SmSpyFHaWjPk8If9yc6svM8c56dB5BAtW4Qbw5jHTwwXXcTLoRMkpDJp6VL0XzlWaCHTXrkFURMYmD0sLqg==", "license": "MIT", "engines": { "node": ">=12" @@ -141,8 +125,6 @@ }, "node_modules/ansi-styles": { "version": "4.3.0", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", - "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", "license": "MIT", "dependencies": { "color-convert": "^2.0.1" @@ -156,8 +138,6 @@ }, "node_modules/chalk": { "version": "4.1.2", - "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", - "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", "license": "MIT", "dependencies": { "ansi-styles": "^4.1.0", @@ -172,8 +152,6 @@ }, "node_modules/cli-cursor": { "version": "5.0.0", - "resolved": "https://registry.npmjs.org/cli-cursor/-/cli-cursor-5.0.0.tgz", - "integrity": "sha512-aCj4O5wKyszjMmDT4tZj93kxyydN/K5zPWSCe6/0AV/AA1pqe5ZBIw0a2ZfPQV7lL5/yb5HsUreJ6UFAF1tEQw==", "license": "MIT", "dependencies": { "restore-cursor": "^5.0.0" @@ -187,8 +165,6 @@ }, "node_modules/cli-truncate": { "version": "5.2.0", - "resolved": "https://registry.npmjs.org/cli-truncate/-/cli-truncate-5.2.0.tgz", - "integrity": "sha512-xRwvIOMGrfOAnM1JYtqQImuaNtDEv9v6oIYAs4LIHwTiKee8uwvIi363igssOC0O5U04i4AlENs79LQLu9tEMw==", "license": "MIT", "dependencies": { "slice-ansi": "^8.0.0", @@ -203,8 +179,6 @@ }, "node_modules/color-convert": { "version": "2.0.1", - "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", - "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", "license": "MIT", "dependencies": { "color-name": "~1.1.4" @@ -215,20 +189,14 @@ }, "node_modules/color-name": { "version": "1.1.4", - "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", - "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", "license": "MIT" }, "node_modules/emoji-regex": { "version": "10.6.0", - "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-10.6.0.tgz", - "integrity": "sha512-toUI84YS5YmxW219erniWD0CIVOo46xGKColeNQRgOzDorgBi1v4D71/OFzgD9GO2UGKIv1C3Sp8DAn0+j5w7A==", "license": "MIT" }, "node_modules/environment": { "version": "1.1.0", - "resolved": "https://registry.npmjs.org/environment/-/environment-1.1.0.tgz", - "integrity": "sha512-xUtoPkMggbz0MPyPiIWr1Kp4aeWJjDZ6SMvURhimjdZgsRuDplF5/s9hcgGhyXMhs+6vpnuoiZ2kFiu3FMnS8Q==", "license": "MIT", "engines": { "node": ">=18" @@ -239,14 +207,10 @@ }, "node_modules/eventemitter3": { "version": "5.0.4", - "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-5.0.4.tgz", - "integrity": "sha512-mlsTRyGaPBjPedk6Bvw+aqbsXDtoAyAzm5MO7JgU+yVRyMQ5O8bD4Kcci7BS85f93veegeCPkL8R4GLClnjLFw==", "license": "MIT" }, "node_modules/get-east-asian-width": { "version": "1.5.0", - "resolved": "https://registry.npmjs.org/get-east-asian-width/-/get-east-asian-width-1.5.0.tgz", - "integrity": "sha512-CQ+bEO+Tva/qlmw24dCejulK5pMzVnUOFOijVogd3KQs07HnRIgp8TGipvCCRT06xeYEbpbgwaCxglFyiuIcmA==", "license": "MIT", "engines": { "node": ">=18" @@ -257,8 +221,6 @@ }, "node_modules/has-flag": { "version": "4.0.0", - "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", - "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", "license": "MIT", "engines": { "node": ">=8" @@ -266,8 +228,6 @@ }, "node_modules/is-fullwidth-code-point": { "version": "5.1.0", - "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-5.1.0.tgz", - "integrity": "sha512-5XHYaSyiqADb4RnZ1Bdad6cPp8Toise4TzEjcOYDHZkTCbKgiUl7WTUCpNWHuxmDt91wnsZBc9xinNzopv3JMQ==", "license": "MIT", "dependencies": { "get-east-asian-width": "^1.3.1" @@ -281,8 +241,6 @@ }, "node_modules/listr2": { "version": "10.2.1", - "resolved": "https://registry.npmjs.org/listr2/-/listr2-10.2.1.tgz", - "integrity": "sha512-7I5knELsJKTUjXG+A6BkKAiGkW1i25fNa/xlUl9hFtk15WbE9jndA89xu5FzQKrY5llajE1hfZZFMILXkDHk/Q==", "license": "MIT", "dependencies": { "cli-truncate": "^5.2.0", @@ -297,8 +255,6 @@ }, "node_modules/log-update": { "version": "6.1.0", - "resolved": "https://registry.npmjs.org/log-update/-/log-update-6.1.0.tgz", - "integrity": "sha512-9ie8ItPR6tjY5uYJh8K/Zrv/RMZ5VOlOWvtZdEHYSTFKZfIBPQa9tOAEeAWhd+AnIneLJ22w5fjOYtoutpWq5w==", "license": "MIT", "dependencies": { "ansi-escapes": "^7.0.0", @@ -316,8 +272,6 @@ }, "node_modules/log-update/node_modules/ansi-styles": { "version": "6.2.3", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-6.2.3.tgz", - "integrity": "sha512-4Dj6M28JB+oAH8kFkTLUo+a2jwOFkuqb3yucU0CANcRRUbxS0cP0nZYCGjcc3BNXwRIsUVmDGgzawme7zvJHvg==", "license": "MIT", "engines": { "node": ">=12" @@ -328,8 +282,6 @@ }, "node_modules/log-update/node_modules/slice-ansi": { "version": "7.1.2", - "resolved": "https://registry.npmjs.org/slice-ansi/-/slice-ansi-7.1.2.tgz", - "integrity": "sha512-iOBWFgUX7caIZiuutICxVgX1SdxwAVFFKwt1EvMYYec/NWO5meOJ6K5uQxhrYBdQJne4KxiqZc+KptFOWFSI9w==", "license": "MIT", "dependencies": { "ansi-styles": "^6.2.1", @@ -344,8 +296,6 @@ }, "node_modules/log-update/node_modules/string-width": { "version": "7.2.0", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-7.2.0.tgz", - "integrity": "sha512-tsaTIkKW9b4N+AEj+SVA+WhJzV7/zMhcSu78mLKWSk7cXMOSHsBKFWUs0fWwq8QyK3MgJBQRX6Gbi4kYbdvGkQ==", "license": "MIT", "dependencies": { "emoji-regex": "^10.3.0", @@ -361,8 +311,6 @@ }, "node_modules/log-update/node_modules/wrap-ansi": { "version": "9.0.2", - "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-9.0.2.tgz", - "integrity": "sha512-42AtmgqjV+X1VpdOfyTGOYRi0/zsoLqtXQckTmqTeybT+BDIbM/Guxo7x3pE2vtpr1ok6xRqM9OpBe+Jyoqyww==", "license": "MIT", "dependencies": { "ansi-styles": "^6.2.1", @@ -378,8 +326,6 @@ }, "node_modules/mimic-function": { "version": "5.0.1", - "resolved": "https://registry.npmjs.org/mimic-function/-/mimic-function-5.0.1.tgz", - "integrity": "sha512-VP79XUPxV2CigYP3jWwAUFSku2aKqBH7uTAapFWCBqutsbmDo96KY5o8uh6U+/YSIn5OxJnXp73beVkpqMIGhA==", "license": "MIT", "engines": { "node": ">=18" @@ -390,8 +336,6 @@ }, "node_modules/onetime": { "version": "7.0.0", - "resolved": "https://registry.npmjs.org/onetime/-/onetime-7.0.0.tgz", - "integrity": "sha512-VXJjc87FScF88uafS3JllDgvAm+c/Slfz06lorj2uAY34rlUu0Nt+v8wreiImcrgAjjIHp1rXpTDlLOGw29WwQ==", "license": "MIT", "dependencies": { "mimic-function": "^5.0.0" @@ -404,13 +348,11 @@ } }, "node_modules/relayfile": { - "resolved": "sdk/relayfile", + "resolved": "packages/relayfile", "link": true }, "node_modules/restore-cursor": { "version": "5.1.0", - "resolved": "https://registry.npmjs.org/restore-cursor/-/restore-cursor-5.1.0.tgz", - "integrity": "sha512-oMA2dcrw6u0YfxJQXm342bFKX/E4sG9rbTzO9ptUcR/e8A33cHuvStiYOwH7fszkZlZ1z/ta9AAoPk2F4qIOHA==", "license": "MIT", "dependencies": { "onetime": "^7.0.0", @@ -425,14 +367,10 @@ }, "node_modules/rfdc": { "version": "1.4.1", - "resolved": "https://registry.npmjs.org/rfdc/-/rfdc-1.4.1.tgz", - "integrity": "sha512-q1b3N5QkRUWUl7iyylaaj3kOpIT0N2i9MqIEQXP73GVsN9cw3fdx8X63cEmWhJGi2PPCF23Ijp7ktmd39rawIA==", "license": "MIT" }, "node_modules/signal-exit": { "version": "4.1.0", - "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-4.1.0.tgz", - "integrity": "sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==", "license": "ISC", "engines": { "node": ">=14" @@ -443,8 +381,6 @@ }, "node_modules/slice-ansi": { "version": "8.0.0", - "resolved": "https://registry.npmjs.org/slice-ansi/-/slice-ansi-8.0.0.tgz", - "integrity": "sha512-stxByr12oeeOyY2BlviTNQlYV5xOj47GirPr4yA1hE9JCtxfQN0+tVbkxwCtYDQWhEKWFHsEK48ORg5jrouCAg==", "license": "MIT", "dependencies": { "ansi-styles": "^6.2.3", @@ -459,8 +395,6 @@ }, "node_modules/slice-ansi/node_modules/ansi-styles": { "version": "6.2.3", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-6.2.3.tgz", - "integrity": "sha512-4Dj6M28JB+oAH8kFkTLUo+a2jwOFkuqb3yucU0CANcRRUbxS0cP0nZYCGjcc3BNXwRIsUVmDGgzawme7zvJHvg==", "license": "MIT", "engines": { "node": ">=12" @@ -471,8 +405,6 @@ }, "node_modules/string-width": { "version": "8.2.0", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-8.2.0.tgz", - "integrity": "sha512-6hJPQ8N0V0P3SNmP6h2J99RLuzrWz2gvT7VnK5tKvrNqJoyS9W4/Fb8mo31UiPvy00z7DQXkP2hnKBVav76thw==", "license": "MIT", "dependencies": { "get-east-asian-width": "^1.5.0", @@ -487,8 +419,6 @@ }, "node_modules/strip-ansi": { "version": "7.2.0", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.2.0.tgz", - "integrity": "sha512-yDPMNjp4WyfYBkHnjIRLfca1i6KMyGCtsVgoKe/z1+6vukgaENdgGBZt+ZmKPc4gavvEZ5OgHfHdrazhgNyG7w==", "license": "MIT", "dependencies": { "ansi-regex": "^6.2.2" @@ -502,8 +432,6 @@ }, "node_modules/supports-color": { "version": "7.2.0", - "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", - "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", "license": "MIT", "dependencies": { "has-flag": "^4.0.0" @@ -514,8 +442,6 @@ }, "node_modules/typescript": { "version": "5.9.3", - "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", - "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", "dev": true, "license": "Apache-2.0", "bin": { @@ -528,8 +454,6 @@ }, "node_modules/wrap-ansi": { "version": "10.0.0", - "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-10.0.0.tgz", - "integrity": "sha512-SGcvg80f0wUy2/fXES19feHMz8E0JoXv2uNgHOu4Dgi2OrCy1lqwFYEJz1BLbDI0exjPMe/ZdzZ/YpGECBG/aQ==", "license": "MIT", "dependencies": { "ansi-styles": "^6.2.3", @@ -545,8 +469,6 @@ }, "node_modules/wrap-ansi/node_modules/ansi-styles": { "version": "6.2.3", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-6.2.3.tgz", - "integrity": "sha512-4Dj6M28JB+oAH8kFkTLUo+a2jwOFkuqb3yucU0CANcRRUbxS0cP0nZYCGjcc3BNXwRIsUVmDGgzawme7zvJHvg==", "license": "MIT", "engines": { "node": ">=12" @@ -557,8 +479,6 @@ }, "node_modules/ws": { "version": "8.20.0", - "resolved": "https://registry.npmjs.org/ws/-/ws-8.20.0.tgz", - "integrity": "sha512-sAt8BhgNbzCtgGbt2OxmpuryO63ZoDk/sqaB/znQm94T4fCEsy/yV+7CdC1kJhOU9lboAEU7R3kquuycDoibVA==", "license": "MIT", "engines": { "node": ">=10.0.0" @@ -578,8 +498,6 @@ }, "node_modules/yaml": { "version": "2.8.3", - "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.8.3.tgz", - "integrity": "sha512-AvbaCLOO2Otw/lW5bmh9d/WEdcDFdQp2Z2ZUH3pX9U2ihyUY0nvLv7J6TrWowklRGPYbB/IuIMfYgxaCPg5Bpg==", "license": "ISC", "bin": { "yaml": "bin.mjs" @@ -593,8 +511,6 @@ }, "node_modules/zod": { "version": "3.25.76", - "resolved": "https://registry.npmjs.org/zod/-/zod-3.25.76.tgz", - "integrity": "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ==", "license": "MIT", "funding": { "url": "https://github.com/sponsors/colinhacks" @@ -602,14 +518,12 @@ }, "node_modules/zod-to-json-schema": { "version": "3.25.1", - "resolved": "https://registry.npmjs.org/zod-to-json-schema/-/zod-to-json-schema-3.25.1.tgz", - "integrity": "sha512-pM/SU9d3YAggzi6MtR4h7ruuQlqKtad8e9S0fmxcMi+ueAK5Korys/aWcV9LIIHTVbj01NdzxcnXSN+O74ZIVA==", "license": "ISC", "peerDependencies": { "zod": "^3.25 || ^4" } }, - "sdk/relayfile": { + "packages/relayfile": { "version": "0.1.0", "hasInstallScript": true, "license": "MIT", @@ -620,7 +534,7 @@ "node": ">=18" } }, - "sdk/relayfile-sdk": { + "packages/relayfile-sdk": { "name": "@relayfile/sdk", "version": "0.1.0", "license": "MIT", diff --git a/package.json b/package.json index ea9f165f..fa913222 100644 --- a/package.json +++ b/package.json @@ -2,13 +2,13 @@ "private": true, "description": "relayfile — real-time filesystem for humans and agents", "workspaces": [ - "sdk/relayfile-sdk", - "sdk/relayfile" + "packages/relayfile-sdk", + "packages/relayfile" ], "scripts": { "workflow": "agent-relay run", - "build": "npm run build --workspace=sdk/relayfile-sdk", - "typecheck": "npm run typecheck --workspace=sdk/relayfile-sdk" + "build": "npm run build --workspace=packages/relayfile-sdk", + "typecheck": "npm run typecheck --workspace=packages/relayfile-sdk" }, "dependencies": { "@agent-relay/sdk": "latest" diff --git a/sdk/relayfile-sdk/README.md b/packages/relayfile-sdk/README.md similarity index 100% rename from sdk/relayfile-sdk/README.md rename to packages/relayfile-sdk/README.md diff --git a/sdk/relayfile-sdk/package-lock.json b/packages/relayfile-sdk/package-lock.json similarity index 100% rename from sdk/relayfile-sdk/package-lock.json rename to packages/relayfile-sdk/package-lock.json diff --git a/sdk/relayfile-sdk/package.json b/packages/relayfile-sdk/package.json similarity index 95% rename from sdk/relayfile-sdk/package.json rename to packages/relayfile-sdk/package.json index 08152ce6..13bae942 100644 --- a/sdk/relayfile-sdk/package.json +++ b/packages/relayfile-sdk/package.json @@ -23,7 +23,7 @@ "repository": { "type": "git", "url": "https://github.com/AgentWorkforce/relayfile", - "directory": "sdk/relayfile-sdk" + "directory": "packages/relayfile-sdk" }, "license": "MIT", "keywords": [ diff --git a/sdk/relayfile-sdk/src/client.ts b/packages/relayfile-sdk/src/client.ts similarity index 100% rename from sdk/relayfile-sdk/src/client.ts rename to packages/relayfile-sdk/src/client.ts diff --git a/sdk/relayfile-sdk/src/errors.ts b/packages/relayfile-sdk/src/errors.ts similarity index 100% rename from sdk/relayfile-sdk/src/errors.ts rename to packages/relayfile-sdk/src/errors.ts diff --git a/sdk/relayfile-sdk/src/index.ts b/packages/relayfile-sdk/src/index.ts similarity index 100% rename from sdk/relayfile-sdk/src/index.ts rename to packages/relayfile-sdk/src/index.ts diff --git a/sdk/relayfile-sdk/src/types.ts b/packages/relayfile-sdk/src/types.ts similarity index 100% rename from sdk/relayfile-sdk/src/types.ts rename to packages/relayfile-sdk/src/types.ts diff --git a/sdk/relayfile-sdk/tsconfig.json b/packages/relayfile-sdk/tsconfig.json similarity index 100% rename from sdk/relayfile-sdk/tsconfig.json rename to packages/relayfile-sdk/tsconfig.json diff --git a/sdk/relayfile/package.json b/packages/relayfile/package.json similarity index 94% rename from sdk/relayfile/package.json rename to packages/relayfile/package.json index dfec41f7..5a77b7ed 100644 --- a/sdk/relayfile/package.json +++ b/packages/relayfile/package.json @@ -15,7 +15,7 @@ "repository": { "type": "git", "url": "https://github.com/AgentWorkforce/relayfile", - "directory": "sdk/relayfile" + "directory": "packages/relayfile" }, "license": "MIT", "keywords": [ diff --git a/sdk/relayfile/scripts/install.js b/packages/relayfile/scripts/install.js similarity index 100% rename from sdk/relayfile/scripts/install.js rename to packages/relayfile/scripts/install.js diff --git a/workflows/relayfile-bulk-and-export.ts b/workflows/relayfile-bulk-and-export.ts index 62bf02dc..becf702d 100644 --- a/workflows/relayfile-bulk-and-export.ts +++ b/workflows/relayfile-bulk-and-export.ts @@ -77,13 +77,13 @@ const result = await workflow('relayfile-bulk-and-export') .step('read-ts-sdk', { type: 'deterministic', - command: `cat ${RELAYFILE}/sdk/relayfile-sdk/src/client.ts`, + command: `cat ${RELAYFILE}/packages/relayfile-sdk/src/client.ts`, captureOutput: true, }) .step('read-ts-types', { type: 'deterministic', - command: `cat ${RELAYFILE}/sdk/relayfile-sdk/src/types.ts`, + command: `cat ${RELAYFILE}/packages/relayfile-sdk/src/types.ts`, captureOutput: true, }) @@ -249,7 +249,7 @@ Current SDK types: Changes: -1. Edit ${RELAYFILE}/sdk/relayfile-sdk/src/types.ts — add: +1. Edit ${RELAYFILE}/packages/relayfile-sdk/src/types.ts — add: - BulkWriteFile: { path: string; contentType?: string; content: string; encoding?: "utf-8" | "base64" } - BulkWriteInput: { workspaceId: string; files: BulkWriteFile[]; correlationId?: string; signal?: AbortSignal } - BulkWriteResponse: { imported: number; errors: { path: string; error: string }[] } @@ -259,7 +259,7 @@ Changes: - FilesystemEvent (already exists? add if missing): { eventId, type, path, revision, timestamp } - Add encoding?: "utf-8" | "base64" to WriteFileInput and FileReadResponse -2. Edit ${RELAYFILE}/sdk/relayfile-sdk/src/client.ts — add methods: +2. Edit ${RELAYFILE}/packages/relayfile-sdk/src/client.ts — add methods: - bulkWrite(input: BulkWriteInput): Promise POST /v1/workspaces/{workspaceId}/fs/bulk with JSON body - exportWorkspace(options: ExportOptions): Promise @@ -269,7 +269,7 @@ Changes: Connect to ws:// or wss:// endpoint Return object with { close(), on(event, handler) } -3. Re-export new types from ${RELAYFILE}/sdk/relayfile-sdk/src/index.ts +3. Re-export new types from ${RELAYFILE}/packages/relayfile-sdk/src/index.ts Write all changes to disk.`, verification: { type: 'exit_code' }, @@ -347,7 +347,7 @@ Write all test files to disk.`, command: `cd ${RELAYFILE} && echo "=== New/modified files ===" && \ grep -rl "BulkWrite\|ExportWorkspace\|handleBulkWrite\|handleExport\|websocket\|WebSocket" internal/ --include="*.go" | sort && \ echo "" && echo "=== SDK updates ===" && \ -grep -c "bulkWrite\|exportWorkspace\|connectWebSocket" sdk/relayfile-sdk/src/client.ts && \ +grep -c "bulkWrite\|exportWorkspace\|connectWebSocket" packages/relayfile-sdk/src/client.ts && \ echo "" && echo "=== Build check ===" && \ go build ./... 2>&1 | tail -10; echo "EXIT: $?"`, captureOutput: true, diff --git a/workflows/relayfile-ci-and-publish.ts b/workflows/relayfile-ci-and-publish.ts index d419d0f8..75b33fcd 100644 --- a/workflows/relayfile-ci-and-publish.ts +++ b/workflows/relayfile-ci-and-publish.ts @@ -72,7 +72,7 @@ const result = await workflow('relayfile-ci-and-publish') .step('read-sdk-package', { type: 'deterministic', - command: `cat ${RELAYFILE}/sdk/relayfile-sdk/package.json 2>/dev/null || echo "no package.json"`, + command: `cat ${RELAYFILE}/packages/relayfile-sdk/package.json 2>/dev/null || echo "no package.json"`, captureOutput: true, }) @@ -109,7 +109,7 @@ Write a design doc at ${RELAYFILE}/docs/ci-cd-design.md covering: 1. **ci.yml** — runs on every PR and push to main: - Go tests: go test ./... (all packages) - Go build: go build ./cmd/relayfile ./cmd/relayfile-mount ./cmd/relayfile-cli - - TS SDK: cd sdk/relayfile-sdk && npm ci && npm run build && npx tsc --noEmit + - TS SDK: cd packages/relayfile-sdk && npm ci && npm run build && npx tsc --noEmit - E2E test: start Go server, run scripts/e2e.ts --ci - CF Workers typecheck: cd packages/server && npx tsc --noEmit (if it exists) @@ -159,7 +159,7 @@ Create ${RELAYFILE}/.github/workflows/ci.yml: - Jobs: 1. go-test: setup Go 1.22, run go test ./... 2. go-build: build all 3 binaries - 3. sdk-typecheck: setup Node 22, cd sdk/relayfile-sdk, npm ci, npm run build, tsc --noEmit + 3. sdk-typecheck: setup Node 22, cd packages/relayfile-sdk, npm ci, npm run build, tsc --noEmit 4. e2e: depends on go-build, start server, run e2e.ts --ci 5. workers-typecheck (conditional): if packages/server exists, tsc --noEmit @@ -189,7 +189,7 @@ Key requirements: - Steps: 1. Checkout 2. Setup Node 22 with registry-url: "https://registry.npmjs.org" - 3. Install deps: cd sdk/relayfile-sdk && npm ci + 3. Install deps: cd packages/relayfile-sdk && npm ci 4. Version bump: npm version {type} --no-git-tag-version 5. Build: npm run build 6. Test: npx tsc --noEmit @@ -249,7 +249,7 @@ Write to disk.`, Current package.json: {{steps.read-sdk-package.output}} -Update ${RELAYFILE}/sdk/relayfile-sdk/package.json: +Update ${RELAYFILE}/packages/relayfile-sdk/package.json: { "name": "@relayfile/sdk", @@ -272,14 +272,14 @@ Update ${RELAYFILE}/sdk/relayfile-sdk/package.json: "repository": { "type": "git", "url": "https://github.com/AgentWorkforce/relayfile", - "directory": "sdk/relayfile-sdk" + "directory": "packages/relayfile-sdk" }, "license": "MIT", "keywords": ["relayfile", "filesystem", "sync", "agent", "collaboration"], "engines": { "node": ">=18" } } -Also create/update ${RELAYFILE}/sdk/relayfile-sdk/tsconfig.json: +Also create/update ${RELAYFILE}/packages/relayfile-sdk/tsconfig.json: { "compilerOptions": { "target": "ES2022", @@ -297,13 +297,13 @@ Also create/update ${RELAYFILE}/sdk/relayfile-sdk/tsconfig.json: "include": ["src"] } -Create ${RELAYFILE}/sdk/relayfile-sdk/README.md with: +Create ${RELAYFILE}/packages/relayfile-sdk/README.md with: - Package name + description - Install: npm install @relayfile/sdk - Quick example: create client, list tree, read/write file - Link to full docs -Verify it builds: cd sdk/relayfile-sdk && npm run build +Verify it builds: cd packages/relayfile-sdk && npm run build Write all files to disk.`, verification: { type: 'exit_code' }, @@ -323,7 +323,7 @@ echo "" && echo "=== Provenance check ===" && \ grep -c "provenance" .github/workflows/publish-npm.yml && \ grep -c "id-token: write" .github/workflows/publish-npm.yml && \ echo "" && echo "=== SDK builds ===" && \ -cd sdk/relayfile-sdk && npm install 2>&1 | tail -1 && npm run build 2>&1 | tail -3; echo "EXIT: $?"`, +cd packages/relayfile-sdk && npm install 2>&1 | tail -1 && npm run build 2>&1 | tail -3; echo "EXIT: $?"`, captureOutput: true, failOnError: false, }) diff --git a/workflows/relayfile-cloud-server.ts b/workflows/relayfile-cloud-server.ts index 9a5cf315..03310c1d 100644 --- a/workflows/relayfile-cloud-server.ts +++ b/workflows/relayfile-cloud-server.ts @@ -118,7 +118,7 @@ const result = await workflow('relayfile-cloud-server') .step('read-sdk-types', { type: 'deterministic', - command: `cat ${RELAYFILE}/sdk/relayfile-sdk/src/types.ts`, + command: `cat ${RELAYFILE}/packages/relayfile-sdk/src/types.ts`, captureOutput: true, }) diff --git a/workflows/relayfile-developer-experience.ts b/workflows/relayfile-developer-experience.ts index 682d2860..932ecdb7 100644 --- a/workflows/relayfile-developer-experience.ts +++ b/workflows/relayfile-developer-experience.ts @@ -329,9 +329,9 @@ Write all docs to disk.`, dependsOn: ['implement-cli'], task: `Prepare the TypeScript SDK for npm publishing. -Read the current SDK: cat ${RELAYFILE}/sdk/relayfile-sdk/package.json +Read the current SDK: cat ${RELAYFILE}/packages/relayfile-sdk/package.json -Update ${RELAYFILE}/sdk/relayfile-sdk/package.json: +Update ${RELAYFILE}/packages/relayfile-sdk/package.json: - name: "@relayfile/sdk" (or "relayfile-sdk") - version: "0.1.0" - main: "dist/index.js" @@ -342,7 +342,7 @@ Update ${RELAYFILE}/sdk/relayfile-sdk/package.json: prepublishOnly: "npm run build" - repository, license, description fields -Create ${RELAYFILE}/sdk/relayfile-sdk/tsconfig.json: +Create ${RELAYFILE}/packages/relayfile-sdk/tsconfig.json: - target: ES2022 - module: ESNext - moduleResolution: bundler @@ -350,7 +350,7 @@ Create ${RELAYFILE}/sdk/relayfile-sdk/tsconfig.json: - outDir: dist - rootDir: src -Verify it builds: cd sdk/relayfile-sdk && npm run build +Verify it builds: cd packages/relayfile-sdk && npm run build Write changes to disk.`, verification: { type: 'exit_code' }, @@ -370,7 +370,7 @@ done && \ echo "" && echo "=== Install script ===" && \ [ -f scripts/install.sh ] && echo "install.sh: OK" || echo "install.sh: MISSING" && \ echo "" && echo "=== SDK builds ===" && \ -cd sdk/relayfile-sdk && npm install 2>&1 | tail -1 && npx tsc --noEmit 2>&1 | tail -3; echo "EXIT: $?"`, +cd packages/relayfile-sdk && npm install 2>&1 | tail -1 && npx tsc --noEmit 2>&1 | tail -3; echo "EXIT: $?"`, captureOutput: true, failOnError: false, }) From cb814f27ed0a5aba02598f5ceeb7dd32921ec89d Mon Sep 17 00:00:00 2001 From: Khaliq Date: Tue, 24 Mar 2026 15:36:17 +0100 Subject: [PATCH 08/10] Remove provenance from publishConfig, fix repo URLs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Provenance only works in CI with OIDC — removing from package.json so local publishes work. CI workflows already pass --provenance explicitly. Also normalizes repository URLs to avoid npm publish warnings. Co-Authored-By: Claude Opus 4.6 (1M context) --- packages/relayfile-sdk/package.json | 5 ++--- packages/relayfile/package.json | 5 ++--- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/packages/relayfile-sdk/package.json b/packages/relayfile-sdk/package.json index 13bae942..5b4d7957 100644 --- a/packages/relayfile-sdk/package.json +++ b/packages/relayfile-sdk/package.json @@ -17,12 +17,11 @@ "prepublishOnly": "npm run build" }, "publishConfig": { - "access": "public", - "provenance": true + "access": "public" }, "repository": { "type": "git", - "url": "https://github.com/AgentWorkforce/relayfile", + "url": "git+https://github.com/AgentWorkforce/relayfile.git", "directory": "packages/relayfile-sdk" }, "license": "MIT", diff --git a/packages/relayfile/package.json b/packages/relayfile/package.json index 5a77b7ed..3d1db477 100644 --- a/packages/relayfile/package.json +++ b/packages/relayfile/package.json @@ -9,12 +9,11 @@ "postinstall": "node scripts/install.js || true" }, "publishConfig": { - "access": "public", - "provenance": true + "access": "public" }, "repository": { "type": "git", - "url": "https://github.com/AgentWorkforce/relayfile", + "url": "git+https://github.com/AgentWorkforce/relayfile.git", "directory": "packages/relayfile" }, "license": "MIT", From 86430fb63f4717680fbd0b8845b87cbb821363e7 Mon Sep 17 00:00:00 2001 From: Khaliq Date: Tue, 24 Mar 2026 16:14:29 +0100 Subject: [PATCH 09/10] feat: add relayauth integration workflow Workflow to add relayauth JWT verification to the Go server and mount daemon. Path-scoped access (relayfile:fs:write:/src/api/*), JWKS caching, backwards compat with existing relayfile JWTs. Depends on: @relayauth/sdk (for types reference), Go JWKS verification Co-Authored-By: Claude Opus 4.6 (1M context) --- workflows/integrate-relayauth.ts | 285 +++++++++++++++++++++++++++++++ 1 file changed, 285 insertions(+) create mode 100644 workflows/integrate-relayauth.ts diff --git a/workflows/integrate-relayauth.ts b/workflows/integrate-relayauth.ts new file mode 100644 index 00000000..56ffff24 --- /dev/null +++ b/workflows/integrate-relayauth.ts @@ -0,0 +1,285 @@ +/** + * integrate-relayauth.ts + * + * Integrates relayauth into relayfile: verify relayauth JWTs for all + * filesystem operations. Scoped access to paths, revision-controlled + * writes, and audit trail for every file operation. + * + * Depends on: @relayauth/sdk (TokenVerifier, ScopeChecker) + * + * Changes: + * - Go server: verify relayauth JWTs (via JWKS) alongside existing JWT auth + * - Go mount daemon: accept relayauth tokens for authentication + * - TS SDK: pass relayauth tokens in requests + * - Scope enforcement: relayfile:fs:read:/path, relayfile:fs:write:/path + * - Path-scoped access: agent can only read/write files matching their scope paths + * + * Run: agent-relay run workflows/integrate-relayauth.ts + */ + +import { workflow } from '@agent-relay/sdk/workflows'; + +const RELAYFILE = '/Users/khaliqgant/Projects/AgentWorkforce/relayfile'; +const RELAYAUTH = '/Users/khaliqgant/Projects/AgentWorkforce/relayauth'; + +async function main() { +const result = await workflow('integrate-relayauth-relayfile') + .description('Add relayauth JWT verification to relayfile server and mount daemon') + .pattern('dag') + .channel('wf-relayfile-relayauth') + .maxConcurrency(4) + .timeout(3_600_000) + + .agent('architect', { + cli: 'claude', + preset: 'lead', + role: 'Design the integration, review code, fix issues', + cwd: RELAYFILE, + }) + .agent('go-dev', { + cli: 'codex', + preset: 'worker', + role: 'Implement Go server and mount daemon auth changes', + cwd: RELAYFILE, + }) + .agent('sdk-dev', { + cli: 'codex', + preset: 'worker', + role: 'Update TS SDK to pass relayauth tokens', + cwd: RELAYFILE, + }) + .agent('test-writer', { + cli: 'codex', + preset: 'worker', + role: 'Write tests for auth integration', + cwd: RELAYFILE, + }) + .agent('reviewer', { + cli: 'claude', + preset: 'reviewer', + role: 'Review for security, path-scoping correctness, backwards compat', + cwd: RELAYFILE, + }) + + // ── Phase 1: Read existing code ──────────────────────────────────── + + .step('read-go-auth', { + type: 'deterministic', + command: `cat ${RELAYFILE}/internal/httpapi/auth.go`, + captureOutput: true, + }) + + .step('read-go-server', { + type: 'deterministic', + command: `head -100 ${RELAYFILE}/internal/httpapi/server.go`, + captureOutput: true, + }) + + .step('read-mount-daemon', { + type: 'deterministic', + command: `cat ${RELAYFILE}/cmd/relayfile-mount/main.go`, + captureOutput: true, + }) + + .step('read-ts-sdk-client', { + type: 'deterministic', + command: `head -60 ${RELAYFILE}/sdk/relayfile-sdk/src/client.ts`, + captureOutput: true, + }) + + .step('read-relayauth-types', { + type: 'deterministic', + command: `cat ${RELAYAUTH}/packages/types/src/token.ts && echo "=== SCOPE ===" && cat ${RELAYAUTH}/packages/types/src/scope.ts`, + captureOutput: true, + }) + + .step('read-relayauth-sdk', { + type: 'deterministic', + command: `cat ${RELAYAUTH}/packages/sdk/src/verify.ts`, + captureOutput: true, + }) + + // ── Phase 2: Write tests + Implement ────────────────────────────── + + .step('write-tests', { + agent: 'test-writer', + dependsOn: ['read-go-auth', 'read-relayauth-types'], + task: `Write tests for relayfile + relayauth integration. + +Current Go auth: +{{steps.read-go-auth.output}} + +RelayAuth token format: +{{steps.read-relayauth-types.output}} + +Create ${RELAYFILE}/internal/httpapi/relayauth_test.go: + +Tests (Go testing package): +1. Valid relayauth JWT with relayfile:fs:read:* → can read any file +2. Valid relayauth JWT with relayfile:fs:read:/src/* → can read /src/foo.ts, cannot read /config/secret.yaml +3. Valid relayauth JWT with relayfile:fs:write:/src/api/* → can write /src/api/route.ts, cannot write /src/ui/page.tsx +4. Expired JWT → 401 +5. JWT with wrong audience (not "relayfile") → 401 +6. Legacy relayfile JWT (existing format) → still works +7. No token → 401 +8. Path-scoped write: agent writes to allowed path → 200 +9. Path-scoped write: agent writes to disallowed path → 403 +10. SponsorChain present in auth context + +Mock JWKS by serving a test public key on a local HTTP server in the test. +Write to disk.`, + verification: { type: 'exit_code' }, + }) + + .step('implement-go-auth', { + agent: 'go-dev', + dependsOn: ['read-go-auth', 'read-go-server', 'read-relayauth-sdk', 'write-tests'], + task: `Add relayauth JWT verification to the Go server. + +Current Go auth: +{{steps.read-go-auth.output}} + +Current server: +{{steps.read-go-server.output}} + +RelayAuth SDK (reference for token format): +{{steps.read-relayauth-sdk.output}} + +Changes to ${RELAYFILE}/internal/httpapi/auth.go: + +1. Add JWKS fetching: fetch public keys from RELAYAUTH_JWKS_URL env var + - Cache JWKS with 5-minute TTL + - Use crypto/rsa + encoding/json to parse JWK → public key + +2. Add JWT verification function: verifyRelayAuthToken(tokenString) → (claims, error) + - Parse JWT header to get kid (key ID) + - Look up public key from cached JWKS + - Verify RS256 signature + - Check exp, iss, aud ("relayfile" must be in aud array) + - Return parsed claims including scopes, sponsorId, sponsorChain + +3. Update the auth middleware chain: + - Try relayauth JWT first (if RELAYAUTH_JWKS_URL is configured) + - If valid: extract scopes, attach to request context + - If not valid (not a JWT, wrong format): fall back to existing auth + - Keep existing auth fully functional as fallback + +4. Add path-scoped access enforcement: + - New function: checkPathScope(scopes []string, action string, path string) bool + - For each relayfile:fs:{action}:{pathPattern} scope, check if the request path matches + - Wildcard matching: /src/* matches /src/foo.ts and /src/api/route.ts + - Apply on read and write operations + +5. Add env var: RELAYAUTH_JWKS_URL (optional — if not set, relayauth is disabled) + +Write changes to disk. Use standard library only (no external JWT packages).`, + verification: { type: 'exit_code' }, + }) + + .step('implement-mount-auth', { + agent: 'go-dev', + dependsOn: ['read-mount-daemon', 'implement-go-auth'], + task: `Update mount daemon to accept relayauth tokens. + +Current mount daemon: +{{steps.read-mount-daemon.output}} + +The mount daemon already accepts a --token flag. The change is minimal: +relayauth tokens are JWTs — they work with the existing --token flag. +The server-side auth (implemented in the previous step) handles verification. + +Changes to ${RELAYFILE}/cmd/relayfile-mount/main.go: +1. Add --relayauth-url flag (optional) for documentation/logging purposes +2. Log at startup: "Authenticating via relayauth" if token looks like a JWT (has 3 dot-separated parts) +3. No functional change needed — the token is passed as Bearer header, server verifies + +Write changes to disk. Keep it minimal.`, + verification: { type: 'exit_code' }, + }) + + .step('implement-ts-sdk', { + agent: 'sdk-dev', + dependsOn: ['read-ts-sdk-client'], + task: `Update the TS SDK to support relayauth tokens. + +Current SDK client: +{{steps.read-ts-sdk-client.output}} + +The SDK already accepts a token in RelayFileClientOptions. The change is: + +1. Add optional relayauthToken to RelayFileClientOptions: + relayauthToken?: string | (() => string | Promise); + +2. When relayauthToken is set, use it as the Bearer token instead of the regular token. + This allows the SDK to work with either relayfile-native tokens or relayauth tokens. + +3. Add a helper: RelayFileClient.fromRelayAuth(baseUrl, relayauthToken) + Convenience factory that creates a client authenticated via relayauth. + +Edit ${RELAYFILE}/sdk/relayfile-sdk/src/client.ts +Write to disk.`, + verification: { type: 'exit_code' }, + }) + + .step('verify-files', { + type: 'deterministic', + dependsOn: ['implement-go-auth', 'implement-mount-auth', 'implement-ts-sdk', 'write-tests'], + command: `cd ${RELAYFILE} && echo "=== Go files ===" && ls internal/httpapi/relayauth_test.go 2>&1 && echo "=== Go build ===" && go build ./... 2>&1 | tail -5; echo "EXIT: $?"`, + captureOutput: true, + failOnError: false, + }) + + // ── Phase 3: Review + Fix ───────────────────────────────────────── + + .step('run-tests', { + type: 'deterministic', + dependsOn: ['verify-files'], + command: `cd ${RELAYFILE} && go test ./internal/httpapi/... 2>&1 | tail -20; echo "EXIT: $?"`, + captureOutput: true, + failOnError: false, + }) + + .step('review', { + agent: 'reviewer', + dependsOn: ['run-tests'], + task: `Review the relayauth integration. + +Test results: +{{steps.run-tests.output}} + +Read changed files: +- cat ${RELAYFILE}/internal/httpapi/auth.go +- cat ${RELAYFILE}/internal/httpapi/relayauth_test.go + +Verify: +1. JWKS caching has a TTL (not fetched on every request) +2. Path-scoped access: /src/* correctly matches /src/foo.ts but not /config/x +3. Backwards compat: existing JWT auth still works when RELAYAUTH_JWKS_URL not set +4. No hardcoded keys or URLs +5. RS256 verification uses standard library correctly +6. SponsorChain is passed through to request context`, + verification: { type: 'exit_code' }, + }) + + .step('fix', { + agent: 'architect', + dependsOn: ['review'], + task: `Fix issues from review and tests. + +Tests: {{steps.run-tests.output}} +Review: {{steps.review.output}} + +Fix all issues. Run go test ./... and verify clean.`, + verification: { type: 'exit_code' }, + }) + + .onError('retry', { maxRetries: 1, retryDelayMs: 10_000 }) + .run({ + cwd: RELAYFILE, + onEvent: (e: any) => console.log(`[${e.type}] ${e.stepName ?? e.step ?? ''} ${e.error ?? ''}`.trim()), + }); + +console.log(`\nRelayfile + RelayAuth integration: ${result.status}`); +} + +main().catch(console.error); From c6e0e33e5f7713e0dfa15311d01158bedc50a848 Mon Sep 17 00:00:00 2001 From: Khaliq Date: Thu, 26 Mar 2026 07:41:56 +0100 Subject: [PATCH 10/10] fix: correct relayauth workflow sdk path and verification --- workflows/integrate-relayauth.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/workflows/integrate-relayauth.ts b/workflows/integrate-relayauth.ts index 56ffff24..ac9b7a4e 100644 --- a/workflows/integrate-relayauth.ts +++ b/workflows/integrate-relayauth.ts @@ -83,7 +83,7 @@ const result = await workflow('integrate-relayauth-relayfile') .step('read-ts-sdk-client', { type: 'deterministic', - command: `head -60 ${RELAYFILE}/sdk/relayfile-sdk/src/client.ts`, + command: `head -60 ${RELAYFILE}/packages/relayfile-sdk/src/client.ts`, captureOutput: true, }) @@ -216,7 +216,7 @@ The SDK already accepts a token in RelayFileClientOptions. The change is: 3. Add a helper: RelayFileClient.fromRelayAuth(baseUrl, relayauthToken) Convenience factory that creates a client authenticated via relayauth. -Edit ${RELAYFILE}/sdk/relayfile-sdk/src/client.ts +Edit ${RELAYFILE}/packages/relayfile-sdk/src/client.ts Write to disk.`, verification: { type: 'exit_code' }, }) @@ -224,7 +224,7 @@ Write to disk.`, .step('verify-files', { type: 'deterministic', dependsOn: ['implement-go-auth', 'implement-mount-auth', 'implement-ts-sdk', 'write-tests'], - command: `cd ${RELAYFILE} && echo "=== Go files ===" && ls internal/httpapi/relayauth_test.go 2>&1 && echo "=== Go build ===" && go build ./... 2>&1 | tail -5; echo "EXIT: $?"`, + command: `bash -lc 'cd ${RELAYFILE} && echo "=== Go files ===" && ls internal/httpapi/relayauth_test.go 2>&1 && echo "=== Go build ===" && set -o pipefail && go build ./... 2>&1 | tail -5'`, captureOutput: true, failOnError: false, }) @@ -234,7 +234,7 @@ Write to disk.`, .step('run-tests', { type: 'deterministic', dependsOn: ['verify-files'], - command: `cd ${RELAYFILE} && go test ./internal/httpapi/... 2>&1 | tail -20; echo "EXIT: $?"`, + command: `bash -lc 'cd ${RELAYFILE} && set -o pipefail && go test ./internal/httpapi/... 2>&1 | tail -20'`, captureOutput: true, failOnError: false, })