diff --git a/.github/workflows/proto-check.yml b/.github/workflows/proto-check.yml index b0ec5fe..4c6685a 100644 --- a/.github/workflows/proto-check.yml +++ b/.github/workflows/proto-check.yml @@ -17,7 +17,7 @@ jobs: - uses: actions/setup-go@v5 with: - go-version: '1.25.10' + go-version: '1.25.12' - name: Install protoc uses: arduino/setup-protoc@v3 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 2867b0d..e7ba70e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -18,7 +18,7 @@ jobs: - uses: actions/setup-go@v5 with: - go-version: '1.25.10' + go-version: '1.25.12' cache-dependency-path: server/go.sum - name: Run go vet @@ -45,7 +45,7 @@ jobs: - uses: actions/setup-go@v5 with: - go-version: '1.25.10' + go-version: '1.25.12' cache-dependency-path: server/go.sum - name: golangci-lint @@ -128,12 +128,16 @@ jobs: - uses: actions/setup-go@v5 with: - go-version: '1.25.10' + go-version: '1.25.12' cache-dependency-path: server/go.sum - name: Run integration tests working-directory: server - run: go test -race -count=1 -tags=integration ./... + # Serialize with -p 1 -parallel 1: these integration tests spin up + # embedded NATS clusters and real sidecar runners/tunnels. Running them + # concurrently on the 2-core runner starves timing-sensitive setup + # (JetStream elections, tunnel handshakes) and causes contention flakes. + run: go test -race -count=1 -p 1 -parallel 1 -tags=integration ./... e2e: runs-on: ubuntu-latest @@ -142,7 +146,7 @@ jobs: - uses: actions/setup-go@v5 with: - go-version: '1.25.10' + go-version: '1.25.12' cache-dependency-path: server/go.sum # The e2e suite spawns aetherlite as a subprocess and exercises the @@ -155,7 +159,10 @@ jobs: working-directory: server env: AETHER_ALLOW_DEV_MODE: "true" - run: go test -tags=e2e -count=1 -timeout 360s ./internal/proxysidecar/integration_e2e/... + # -parallel 1: the e2e tests share one aetherlite subprocess; running the + # t.Parallel tunnel tests concurrently against the shared gateway causes + # tunnel-lifecycle races (PEER_RESET) under 2-core-runner load. Serialize. + run: go test -tags=e2e -count=1 -p 1 -parallel 1 -timeout 360s ./internal/proxysidecar/integration_e2e/... security: runs-on: ubuntu-latest @@ -164,7 +171,7 @@ jobs: - uses: actions/setup-go@v5 with: - go-version: '1.25.10' + go-version: '1.25.12' cache-dependency-path: server/go.sum - name: Install govulncheck @@ -173,3 +180,13 @@ jobs: - name: Run govulncheck working-directory: server run: govulncheck ./... + + # Informational scan of the published Go SDK, which reaches the Docker + # orchestrator (sdk/go/orchestrators/docker) and thus CALLS the docker/docker + # vulnerabilities tracked in SECURITY.md's Known Issues. continue-on-error so + # the build stays green on those already-tracked, no-upstream-fix advisories, + # while surfacing them (and any genuinely new SDK vuln) in the CI log. + - name: Run govulncheck (SDK — informational, non-blocking) + working-directory: sdk/go + continue-on-error: true + run: govulncheck ./... diff --git a/.github/workflows/version-check.yml b/.github/workflows/version-check.yml index 0954898..2ab4143 100644 --- a/.github/workflows/version-check.yml +++ b/.github/workflows/version-check.yml @@ -23,7 +23,7 @@ jobs: python-version: '3.12' - name: Install dependencies - run: pip install pyyaml + run: pip install pyyaml uv - name: Check version consistency run: python scripts/update-versions.py --check --verbose diff --git a/.gitignore b/.gitignore index 113a657..30f254a 100644 --- a/.gitignore +++ b/.gitignore @@ -32,3 +32,4 @@ build/ /RELEASING.md /server/coverage.out /.run-ci-logs/ +/bin/ diff --git a/CHANGELOG.md b/CHANGELOG.md index 7e7ad17..0c78667 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,9 +6,42 @@ The format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). Aet --- -## [Unreleased] +## [0.2.2] - Unreleased -Internal builds carry a **v0.2.1** gateway/SDK version stamp; no v0.2.1 tag is published yet. +Work landed since the **v0.2.1** release (2026-05-22); not yet tagged. + +### Added + +- **[GATEWAY / SDK] `uu::` user-broadcast topic** for cross-window user targeting (`uu::{user_id}`). A platform→user channel carrying ordinary `MessageEnvelope` protos; only Service, WorkflowEngine, and Bridge principals may publish (`routing.go` enforces the sender-type gate). SDKs gain user-broadcast send helpers and the channel is documented. +- **[GATEWAY / AETHERLITE] OpenTelemetry OTLP metrics + Go runtime metrics** (`internal/tracing/meter.go`), gated on the same `OTEL_EXPORTER_OTLP_ENDPOINT` env var as tracing — when unset, metrics are a no-op. Both the gateway and aetherlite binaries initialize the OTLP gRPC meter alongside the existing tracer. +- **[GATEWAY] `ServiceIdentity.no_pool_consumer`** proto field lets a Service principal opt OUT of pool-task consumer routing (serve-only), so a service can connect without being enrolled as a POOL task consumer. +- **[AUTHPROXY] Internal-only `/auth/verify-optional` endpoint** — an optional-auth variant of `/auth/verify` for `ext_authz` optional-auth flows. With no credential present it passes through anonymously and emits EMPTY identity headers; a present-but-invalid credential still fails closed. +- **[PROXY SIDECAR] Tenant-relay and aggregator service-tunnel modes** for relaying sandbox tunnels through a tenant boundary, plus a `WatchTenants` `snapshot_complete` sentinel and support for dialing `unix://` terminator backends. +- **[GATEWAY / CLEANUP] Stale-task TTL sweeps.** A stale `agent_startup` pool-task reaper and a stale-interactive-task sweep (`InteractiveTaskTTL` / `InteractiveTaskCancelInterval`) auto-cancel non-terminal tasks left behind by disconnected clients; sweeps run single-node-direct so aetherlite (no leader election) reaps reliably. +- **[GATEWAY] Task COMPLETE/FAIL outcome logging.** The gateway now logs every task-op outcome — success and each rejection reason (not found, unauthorized, completeFn error) — for the COMPLETE/FAIL branches. +- **[GO SDK] `ServiceTopic` helper** for workspace-less service principals. + +### Changed + +- **[AETHERLITE / BADGER ROUTER] Badger message + offset retention TTLs.** The single-node Badger router now expires stored messages (default 24h, override via `AETHER_MESSAGE_RETENTION_TTL`) and orphaned per-consumer offset keys (default 7d, override via `AETHER_OFFSET_RETENTION_TTL`). Active consumers keep re-saving their offset so the TTL refreshes and the key never expires while in use; an offset for a consumer that never returns (e.g. a closed browser tab) is eventually reclaimed instead of growing without bound. +- **[GATEWAY] Broadcast/progress subscriptions use resume-or-tail semantics.** Global-user (`gu::`), user-workspace (`uw::`), user-broadcast (`uu::`), and per-user progress (`pg::`) lanes now subscribe as named resume-or-tail (exclusive) consumers, so a reconnect replays only the missed gap instead of re-delivering the whole lane; the task-message lane does a full replay on (re)connect. +- **[GATEWAY / SDK] Resolved OBO subject propagated on delivered messages** so downstream consumers can attribute a delivered message to the on-behalf-of subject rather than only the system actor. +- **[AUDIT] High-volume proxy-route event coalescing.** A bounded, sharded coalescer collapses repetitive `proxy_http_routed`-class events (e.g. per-token chat streaming) while never coalescing failures, denials, or auth/task/kv/control events, which are always audited individually. Complements the new scheduled audit-retention sweep. + +### Fixed + +- **[GO SDK] Metric/event topic helpers now use the `::` separator.** `MetricTopic`/`MetricWildcardTopic` and `EventTopic`/`EventWildcardTopic` emitted the legacy dot form (`metric.*` / `event.*`), which the gateway's `validateTopicFormat` rejects as `ERR_INVALID_TOPIC` — so every `SendMetric`/`SendEvent` publish was silently dropped. They now emit the `::` identity separator. +- **[GATEWAY] Agent principals can publish their own metrics without workspace WRITE.** `checkMessageSend` gated metric publishes on a per-workspace WRITE grant, so intentionally READ-only agents (e.g. sandbox agents seeded READ on their home workspace) could never emit their own token/usage metrics. An agent targeting the metric fan-in plane (`metric::*`) is now additively granted ReadWrite, realizing the "same-workspace publish is implicit" contract; cross-workspace, topic-eligibility, and negative-delta checks are unaffected. +- **[WORKFLOW] Event dispatch moved off the receive loop (`AsyncMessageHandler`).** Handling events inline on the receive loop let a synchronous KV-coordination call self-deadlock (the response needed the blocked loop), surfacing as `coalesce gate: DEADLINE_EXCEEDED`; event handling now runs asynchronously. +- **[BADGER ROUTER] Consumer-offset durability.** The router persists a consumer's offset after a replay and retries `saveOffset` on an optimistic-concurrency conflict, so offsets survive reconnects instead of being lost under contention. +- **[AETHERLITE] Connection-storm write-contention hardening** for the Badger and SQLite lite stores, reducing write contention when many clients connect at once. +- **[ORCHESTRATION / CLEANUP] `orchestrated_task_queue` retirement.** Queue rows whose task is terminal or missing are now retired reliably and orphans reconciled on a schedule (`QueueReconcileInterval`, default 5m; 0 disables). + +--- + +## [0.2.1] - 2026-05-22 + +The accumulated work that shipped in the **v0.2.1** SDK/gateway version bump (Go / Python / TypeScript). ### Added diff --git a/CLAUDE.md b/CLAUDE.md index 667cbce..af57227 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -14,7 +14,7 @@ This project is made by scitrera.ai. Therefore, the naming involves "scitrera"; - api/ - Protobuf definitions and generated code (own Go module: `github.com/scitrera/aether/api`) - sdk/ - Client SDKs (Go, Python, TypeScript) - server/ - Go server (module: `github.com/scitrera/aether`) - - server/cmd/ - Main entry points (gateway, auth-proxy, migrate, cleanup, loadtest) + - server/cmd/ - Main entry points (gateway, aetherlite, auth-proxy, proxy-sidecar, workflow, migrate, cleanup, init-secrets, readiness-check, loadtest) - server/configs/ - YAML configuration files (e.g., dev.yaml) - server/deployments/ - Kubernetes manifests and Docker Compose files - server/docs/ - Server documentation (quickstart, scaling, monitoring, admin API, error codes, etc.) @@ -24,7 +24,7 @@ This project is made by scitrera.ai. Therefore, the naming involves "scitrera"; - server/scripts/ - Server dev scripts (infra, test, certs, load test) - server/specification.md - Full system specification (v4.0, reflects actual implementation) - server/go.mod, server/go.sum - Go module files - - server/Dockerfile - Multi-stage build for gateway, cleanup, and migrate binaries + - server/Dockerfile - Multi-stage build for gateway, cleanup, migrate, auth-proxy, init-secrets, workflow, aetherlite, and proxy-sidecar binaries - server/Makefile - Build/test/run targets (run from server/ directory) - scripts/ - Repo-wide scripts (compile_protos.sh) - refs/ - Reference materials (external open source code; exclude from general scans) @@ -111,7 +111,7 @@ cd server | **Orchestration** | `server/internal/orchestration/` | Task dispatch via AMQP, claim-based delivery, profile management | | **Admin Server** | `server/internal/admin/server.go` | REST API + embedded UI; ops server for health probes + Prometheus metrics | | **Auth Proxy** | `server/cmd/auth-proxy/` + `server/internal/authproxy/` | Standalone auth gateway for external services (e.g., MemoryLayer) | -| **Identity Model** | `server/pkg/models/identity.go` | Seven principal types (Agent, Task, User, Orchestrator, WorkflowEngine, MetricsBridge, Bridge), topic address derivation via `ToTopic()` | +| **Identity Model** | `server/pkg/models/identity.go` | Eight principal types (Agent, Task, User, Service, Orchestrator, WorkflowEngine, MetricsBridge, Bridge), topic address derivation via `ToTopic()` | ### Topic Schema and Routing @@ -123,11 +123,13 @@ cd server | `tb` | `tb::{workspace}::{impl}` | Task broadcast (load-balancing) | | `us` | `us::{user_id}::{window_id}` | User window-specific | | `uw` | `uw::{user_id}::{workspace}` | User workspace-scoped | +| `uu` | `uu::{user_id}` | User broadcast — reaches all of a user's windows regardless of active workspace; workspace-agnostic, ordinary (non-progress) messages. Platform-principal senders only (see permission matrix). | | `ga` | `ga::{workspace}` | Global agent broadcast | | `gu` | `gu::{workspace}` | Global user broadcast | | `pg` | `pg::{workspace}` | Progress updates (server-side recipient filtering) | -| `event.*` | Write: `event.{workspace}` (gateway rewrites to `event::receiver{shard}`); Subscribe (WE): `event::receiver0` | Workflow Engine fan-in; today 1 shard (`event::receiver0`) | -| `metric.*` | Write: `metric.{workspace}` (gateway rewrites to `metric::receiver{shard}`); Subscribe (MB): `metric::receiver0` | Metrics Bridge fan-in; today 1 shard (`metric::receiver0`) | +| `event::` | Write: `event::{workspace}` (gateway rewrites to `event::receiver{shard}`); Subscribe (WE): `event::receiver0` | Workflow Engine fan-in; today 1 shard (`event::receiver0`). A legacy `event.*` (dot) form is rejected as an invalid topic prefix. | +| `metric::` | Write: `metric::{workspace}` (gateway rewrites to `metric::receiver{shard}`); Subscribe (MB): `metric::receiver0` | Metrics Bridge fan-in; today 1 shard (`metric::receiver0`) | +| `tk` | `tk::{workspace}::{task_id}::events` (TaskEvent stream) and `tk::{workspace}::{task_id}::msg` (per-task chat) | Per-task lanes; SUBJECT sessions auto-subscribe for the task's lifetime. Task-message lane uses full replay; user broadcast lanes (`gu`/`uw`/`uu`) and `pg` use resume-or-tail. | | `br` | `br::{impl}::{spec}` | Bridge (cross-workspace messaging integration) | ### Connection Flow @@ -185,7 +187,9 @@ An active gRPC stream connection represents both the distributed lock for that i Cross-workspace sends are blocked: workspace-scoped principals cannot target topics in other workspaces. Bridges are cross-workspace by design (no workspace component) and check ACL per-message against the target workspace. -**Cross-workspace event/metric broadcast:** Sending to `event.*` or `metric.*` in another workspace requires `capability/event_broadcast` or `capability/metric_broadcast` ACL permission. Sending to the sender's own native workspace is implicitly permitted. +**User-broadcast (`uu::{user_id}`):** A workspace-agnostic channel that reaches every one of a user's windows regardless of which workspace each window is viewing (the non-progress complement to `pg::us::{user}`). Because the topic carries no workspace segment, the workspace ACL cannot gate it, so authorization is by principal type: **only Service, WorkflowEngine, and Bridge principals may publish** (`enforceTopicPermissions`). Users, Agents, Tasks, and Orchestrators are denied and must reach a user via `us::`/`uw::`/progress or task ownership. Users subscribe to their own `uu::` topic on connect. + +**Cross-workspace event/metric broadcast:** Sending to `event::` or `metric::` topics in another workspace requires `capability/event_broadcast` or `capability/metric_broadcast` ACL permission. Sending to the sender's own native workspace is implicitly permitted. **Metric payloads are structured:** `METRIC` messages must carry a `Metric` proto payload (fields: `trace_id`, `entries` [{`name`, `kind`, `qty`}], `metadata`, `client_timestamp_ms`). All entries are additive deltas; negative `qty` requires the `capability/metric_credit` ACL permission. See spec Section 4.5 for details and error codes. @@ -224,6 +228,12 @@ PostgreSQL is used for persistent features that gracefully degrade when unavaila Schema is managed by embedded migrations in `server/migrations/` that auto-run on startup. +### Observability +Both `gateway` and `aetherlite` export OpenTelemetry traces and metrics via OTLP gRPC (`internal/tracing/`: `InitTracer`, `InitMeter`, `NewLogBridge`), gated on the standard `OTEL_EXPORTER_OTLP_ENDPOINT` env var (no-op when unset; OTLP gRPC default port 4317). This is independent of the admin ops server's Prometheus `/metrics` endpoint (port 9090). + +### Background Cleanup / Reconcile +The cleanup service (`internal/cleanup/`, wired in both `gateway` and `aetherlite`) runs periodic sweeps: orphaned `orchestrated_task_queue` reconciliation (`QueueReconcileInterval`, default 5m), audit-log retention sweep, stale `agent_startup` / interactive-task TTL reapers, and a stale regular pool-task sweep (`CancelStalePoolTasks`). On single-node AetherLite these run leader-gated/single-node-direct. + ## Horizontal Scaling - **Stateless Design:** All state in Redis and PostgreSQL. Gateway instances share nothing. @@ -235,7 +245,7 @@ Schema is managed by embedded migrations in `server/migrations/` that auto-run o ### Deployment Options - **Kubernetes:** `server/deployments/k8s/gateway/` — Deployment, Service, Ingress, ConfigMap, cert-manager - **Docker Compose:** `server/deployments/docker-compose/multi-instance.yaml` — 3 instances + nginx -- **Dockerfile:** `server/Dockerfile` — Multi-stage build, non-root user, ports 50051/9090/31880 (build from repo root: `docker build -f server/Dockerfile .`) +- **Dockerfile:** `server/Dockerfile` — Multi-stage build, non-root user, ports 50051/9090/31880/8080 (build from repo root: `docker build -f server/Dockerfile .`) ## Client SDKs diff --git a/README.md b/README.md index ac11f1c..1e2a1e7 100644 --- a/README.md +++ b/README.md @@ -125,6 +125,9 @@ AETHER_ALLOW_DEV_MODE=true ./aetherlite --dev --insecure-admin State is persisted in `./aether-lite-data/`. gRPC on `:50051`, admin UI on `:31880`. See [`./docs/aetherlite.md`](./docs/aetherlite.md) for details. +Badger-backed messages and consumer offsets are retention-capped (env `AETHER_MESSAGE_RETENTION_TTL`, +default `24h`; `AETHER_OFFSET_RETENTION_TTL`, default `168h`) so the embedded log does not grow without bound. + > Production-ready for single-node deployments. No horizontal scaling, no cross-node messaging — data loss on > hardware failure unless S3 backups are configured. @@ -179,6 +182,8 @@ RabbitMQ Streams. #### Start Development Dependencies ```bash +cd server # scripts, configs, and cmd/ all live under server/ + # RabbitMQ Streams (ports 55552 stream, 55672 AMQP, management UI on 15672) ./scripts/docker_rmq_test.sh @@ -189,14 +194,16 @@ RabbitMQ Streams. #### Build and Run ```bash +cd server # if not already there + # Build go build -o gateway ./cmd/gateway -# Run with the default dev config +# Run with the dev config ./gateway --config configs/dev.yaml -# Or run directly -go run ./cmd/gateway +# Or run directly with dev defaults (no config file required) +go run ./cmd/gateway --dev ``` #### Run Tests @@ -257,11 +264,12 @@ log_level: "info" | Flag | Description | |---------------------------------------------------------------------|--------------------------------------------------------| -| `--config ` | Path to YAML config file (default: `configs/dev.yaml`) | +| `--config ` | Path to YAML config file (required unless `--dev`) | +| `--dev` | Start with hardcoded dev defaults when no config file | | `--port ` | gRPC server port (overrides config) | | `--tls` | Enable mTLS | | `--cert-file`, `--key-file`, `--ca-file` | mTLS certificate paths | -| `--db-host`, `--db-port`, `--db-user`, `--db-password`, `--db-name` | PostgreSQL overrides | +| `--db-host`, `--db-port`, `--db-user`, `--db-name` | PostgreSQL overrides | | `--redis ` | Redis address override | | `--stream-url` | RabbitMQ Stream URL override | | `--amqp-url` | RabbitMQ AMQP URL override | @@ -277,8 +285,8 @@ Every connection authenticates as exactly one of eight principal types. | **Unique Task** | One connection per identity | `workspace` + `implementation` + `unique_specifier` | Named finite unit of work | | **Non-Unique Task** | Many connections allowed | `workspace` + `implementation` (server assigns ID) | Workers competing for tasks on a shared broadcast topic | | **User** | One connection per window | `user_id` + `window_id` | Multiple browser tabs allowed | -| **Workflow Engine** | One active connection | N/A (Future: sharding) | Sole subscriber to `event.*` topics | -| **Metrics Bridge** | One active connection | N/A (Future: sharding) | Sole subscriber to `metric.*` topics; receive-only | +| **Workflow Engine** | One active connection | N/A (Future: sharding) | Sole subscriber to `event::` topics | +| **Metrics Bridge** | One active connection | N/A (Future: sharding) | Sole subscriber to `metric::` topics; receive-only | | **Orchestrator** | One per specifier | `implementation` + `specifier` | Receives `TaskAssignment` messages to spin up compute | | **Service** | One per specifier | `implementation` + `specifier` | Cross-workspace HTTP-over-Aether proxy; addressable via `sv::{impl}::{spec}` | | **Bridge** | One per specifier | `implementation` + `specifier` | Cross-workspace messaging integration; sends to any workspace subject to ACL | @@ -295,11 +303,12 @@ Messages are routed by a structured topic prefix. | `tb` | Task Broadcast | `tb::{workspace}::{impl}` | Load-balancing topic; all workers of a type compete | | `us` | User (Window) | `us::{user_id}::{window_id}` | Specific browser window | | `uw` | User (Workspace) | `uw::{user_id}::{workspace}` | User scoped to a workspace | +| `uu` | User (Broadcast) | `uu::{user_id}` | Reaches all of a user's windows regardless of active workspace; platform-principal senders only | | `ga` | Global Agents | `ga::{workspace}` | Broadcast to all agents in a workspace | | `gu` | Global Users | `gu::{workspace}` | Broadcast to all users in a workspace | | `pg` | Progress | `pg::{workspace}` | Progress updates with server-side recipient filtering | -| `event.*` | Workflow Engine | `event.{workspace}` | Workflow Engine is the sole subscriber | -| `metric.*` | Metrics Bridge | `metric.{workspace}` | Metrics Bridge is the sole subscriber | +| `event::` | Workflow Engine | `event::{workspace}` | Workflow Engine is the sole subscriber | +| `metric::` | Metrics Bridge | `metric::{workspace}` | Metrics Bridge is the sole subscriber | | `sv` | Service | `sv::{impl}::{spec}` | Cross-workspace service proxy endpoint | | `br` | Bridge | `br::{impl}::{spec}` | Cross-workspace messaging bridge endpoint | diff --git a/SECURITY.md b/SECURITY.md index e554f40..f8c96e2 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -55,6 +55,8 @@ The following vulnerabilities are tracked but unresolved at the time of the curr |---|---|---| | [GO-2026-4887](https://pkg.go.dev/vuln/GO-2026-4887) | `github.com/docker/docker` ≤ v28.5.2 | No upstream fix released. Tracking. | | [GO-2026-4883](https://pkg.go.dev/vuln/GO-2026-4883) | `github.com/docker/docker` ≤ v28.5.2 | No upstream fix released. Tracking. | +| [GO-2026-5617](https://pkg.go.dev/vuln/GO-2026-5617) | `github.com/docker/docker` ≤ v28.5.2 | `docker cp` bind-mount redirection race. No upstream fix released. Tracking. | +| [GO-2026-5668](https://pkg.go.dev/vuln/GO-2026-5668) | `github.com/docker/docker` ≤ v28.5.2 | `docker cp` symlink-swap arbitrary-empty-file race. No upstream fix released. Tracking. | Mitigation: callers that don't need the Docker orchestrator can build their applications without importing `sdk/go/orchestrators/docker`. We will bump the dependency immediately when upstream ships fixed releases. diff --git a/api/go.mod b/api/go.mod index a4f30a4..c5ffbf4 100644 --- a/api/go.mod +++ b/api/go.mod @@ -1,9 +1,9 @@ module github.com/scitrera/aether/api -go 1.25.10 +go 1.25.12 require ( - google.golang.org/grpc v1.80.0 + google.golang.org/grpc v1.81.1 google.golang.org/protobuf v1.36.11 ) diff --git a/api/go.sum b/api/go.sum index b64affa..1f74e93 100644 --- a/api/go.sum +++ b/api/go.sum @@ -12,16 +12,16 @@ github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= -go.opentelemetry.io/otel v1.39.0 h1:8yPrr/S0ND9QEfTfdP9V+SiwT4E0G7Y5MO7p85nis48= -go.opentelemetry.io/otel v1.39.0/go.mod h1:kLlFTywNWrFyEdH0oj2xK0bFYZtHRYUdv1NklR/tgc8= -go.opentelemetry.io/otel/metric v1.39.0 h1:d1UzonvEZriVfpNKEVmHXbdf909uGTOQjA0HF0Ls5Q0= -go.opentelemetry.io/otel/metric v1.39.0/go.mod h1:jrZSWL33sD7bBxg1xjrqyDjnuzTUB0x1nBERXd7Ftcs= -go.opentelemetry.io/otel/sdk v1.39.0 h1:nMLYcjVsvdui1B/4FRkwjzoRVsMK8uL/cj0OyhKzt18= -go.opentelemetry.io/otel/sdk v1.39.0/go.mod h1:vDojkC4/jsTJsE+kh+LXYQlbL8CgrEcwmt1ENZszdJE= -go.opentelemetry.io/otel/sdk/metric v1.39.0 h1:cXMVVFVgsIf2YL6QkRF4Urbr/aMInf+2WKg+sEJTtB8= -go.opentelemetry.io/otel/sdk/metric v1.39.0/go.mod h1:xq9HEVH7qeX69/JnwEfp6fVq5wosJsY1mt4lLfYdVew= -go.opentelemetry.io/otel/trace v1.39.0 h1:2d2vfpEDmCJ5zVYz7ijaJdOF59xLomrvj7bjt6/qCJI= -go.opentelemetry.io/otel/trace v1.39.0/go.mod h1:88w4/PnZSazkGzz/w84VHpQafiU4EtqqlVdxWy+rNOA= +go.opentelemetry.io/otel v1.43.0 h1:mYIM03dnh5zfN7HautFE4ieIig9amkNANT+xcVxAj9I= +go.opentelemetry.io/otel v1.43.0/go.mod h1:JuG+u74mvjvcm8vj8pI5XiHy1zDeoCS2LB1spIq7Ay0= +go.opentelemetry.io/otel/metric v1.43.0 h1:d7638QeInOnuwOONPp4JAOGfbCEpYb+K6DVWvdxGzgM= +go.opentelemetry.io/otel/metric v1.43.0/go.mod h1:RDnPtIxvqlgO8GRW18W6Z/4P462ldprJtfxHxyKd2PY= +go.opentelemetry.io/otel/sdk v1.43.0 h1:pi5mE86i5rTeLXqoF/hhiBtUNcrAGHLKQdhg4h4V9Dg= +go.opentelemetry.io/otel/sdk v1.43.0/go.mod h1:P+IkVU3iWukmiit/Yf9AWvpyRDlUeBaRg6Y+C58QHzg= +go.opentelemetry.io/otel/sdk/metric v1.43.0 h1:S88dyqXjJkuBNLeMcVPRFXpRw2fuwdvfCGLEo89fDkw= +go.opentelemetry.io/otel/sdk/metric v1.43.0/go.mod h1:C/RJtwSEJ5hzTiUz5pXF1kILHStzb9zFlIEe85bhj6A= +go.opentelemetry.io/otel/trace v1.43.0 h1:BkNrHpup+4k4w+ZZ86CZoHHEkohws8AY+WTX09nk+3A= +go.opentelemetry.io/otel/trace v1.43.0/go.mod h1:/QJhyVBUUswCphDVxq+8mld+AvhXZLhe+8WVFxiFff0= golang.org/x/net v0.54.0 h1:2zJIZAxAHV/OHCDTCOHAYehQzLfSXuf/5SoL/Dv6w/w= golang.org/x/net v0.54.0/go.mod h1:Sj4oj8jK6XmHpBZU/zWHw3BV3abl4Kvi+Ut7cQcY+cQ= golang.org/x/sys v0.44.0 h1:ildZl3J4uzeKP07r2F++Op7E9B29JRUy+a27EibtBTQ= @@ -32,7 +32,7 @@ gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= google.golang.org/genproto/googleapis/rpc v0.0.0-20260226221140-a57be14db171 h1:ggcbiqK8WWh6l1dnltU4BgWGIGo+EVYxCaAPih/zQXQ= google.golang.org/genproto/googleapis/rpc v0.0.0-20260226221140-a57be14db171/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= -google.golang.org/grpc v1.80.0 h1:Xr6m2WmWZLETvUNvIUmeD5OAagMw3FiKmMlTdViWsHM= -google.golang.org/grpc v1.80.0/go.mod h1:ho/dLnxwi3EDJA4Zghp7k2Ec1+c2jqup0bFkw07bwF4= +google.golang.org/grpc v1.81.1 h1:VnnIIZ88UzOOKLukQi+ImGz8O1Wdp8nAGGnvOfEIWQQ= +google.golang.org/grpc v1.81.1/go.mod h1:xGH9GfzOyMTGIOXBJmXt+BX/V0kcdQbdcuwQ/zNw42I= google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= diff --git a/api/proto/aether.pb.go b/api/proto/aether.pb.go index 315f8a0..8dc246a 100644 --- a/api/proto/aether.pb.go +++ b/api/proto/aether.pb.go @@ -507,6 +507,124 @@ func (TaskClass) EnumDescriptor() ([]byte, []int) { return file_aether_proto_rawDescGZIP(), []int{7} } +// TaskPriority orders task dispatch. Unlike TaskClass (a UI hint), the server +// USES priority for scheduling: among pending tasks eligible for delivery, +// higher priority is dispatched first; ties break FIFO (oldest created_at). +// Underlying values are intentionally spaced so new levels can be inserted +// later, and the numeric value is used directly as the descending sort key. +type TaskPriority int32 + +const ( + TaskPriority_TASK_PRIORITY_UNSPECIFIED TaskPriority = 0 // Treated as NORMAL for back-compat. + TaskPriority_TASK_PRIORITY_XLOW TaskPriority = 10 // Lowest; best-effort, yields to everything. + TaskPriority_TASK_PRIORITY_LOW TaskPriority = 20 // Below normal. + TaskPriority_TASK_PRIORITY_NORMAL TaskPriority = 30 // Default. + TaskPriority_TASK_PRIORITY_HIGH TaskPriority = 40 // Above normal; jumps ahead of normal/low. + TaskPriority_TASK_PRIORITY_PREEMPT TaskPriority = 50 // Highest; reserved for future true preemption. +) + +// Enum value maps for TaskPriority. +var ( + TaskPriority_name = map[int32]string{ + 0: "TASK_PRIORITY_UNSPECIFIED", + 10: "TASK_PRIORITY_XLOW", + 20: "TASK_PRIORITY_LOW", + 30: "TASK_PRIORITY_NORMAL", + 40: "TASK_PRIORITY_HIGH", + 50: "TASK_PRIORITY_PREEMPT", + } + TaskPriority_value = map[string]int32{ + "TASK_PRIORITY_UNSPECIFIED": 0, + "TASK_PRIORITY_XLOW": 10, + "TASK_PRIORITY_LOW": 20, + "TASK_PRIORITY_NORMAL": 30, + "TASK_PRIORITY_HIGH": 40, + "TASK_PRIORITY_PREEMPT": 50, + } +) + +func (x TaskPriority) Enum() *TaskPriority { + p := new(TaskPriority) + *p = x + return p +} + +func (x TaskPriority) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (TaskPriority) Descriptor() protoreflect.EnumDescriptor { + return file_aether_proto_enumTypes[8].Descriptor() +} + +func (TaskPriority) Type() protoreflect.EnumType { + return &file_aether_proto_enumTypes[8] +} + +func (x TaskPriority) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use TaskPriority.Descriptor instead. +func (TaskPriority) EnumDescriptor() ([]byte, []int) { + return file_aether_proto_rawDescGZIP(), []int{8} +} + +// BackoffStrategy describes how the task store scales delays across retry +// attempts when a RetryPolicy is attached to a task. Defaults to +// EXPONENTIAL when unspecified (preserves current behavior). +type BackoffStrategy int32 + +const ( + BackoffStrategy_BACKOFF_STRATEGY_UNSPECIFIED BackoffStrategy = 0 + BackoffStrategy_BACKOFF_STRATEGY_FIXED BackoffStrategy = 1 // Same delay every attempt. + BackoffStrategy_BACKOFF_STRATEGY_EXPONENTIAL BackoffStrategy = 2 // delay = initial * 2^(n-1), capped at max_delay_ms. + BackoffStrategy_BACKOFF_STRATEGY_EXPLICIT_SCHEDULE BackoffStrategy = 3 // Use schedule_ms[attempt-1]; clamp the last entry. +) + +// Enum value maps for BackoffStrategy. +var ( + BackoffStrategy_name = map[int32]string{ + 0: "BACKOFF_STRATEGY_UNSPECIFIED", + 1: "BACKOFF_STRATEGY_FIXED", + 2: "BACKOFF_STRATEGY_EXPONENTIAL", + 3: "BACKOFF_STRATEGY_EXPLICIT_SCHEDULE", + } + BackoffStrategy_value = map[string]int32{ + "BACKOFF_STRATEGY_UNSPECIFIED": 0, + "BACKOFF_STRATEGY_FIXED": 1, + "BACKOFF_STRATEGY_EXPONENTIAL": 2, + "BACKOFF_STRATEGY_EXPLICIT_SCHEDULE": 3, + } +) + +func (x BackoffStrategy) Enum() *BackoffStrategy { + p := new(BackoffStrategy) + *p = x + return p +} + +func (x BackoffStrategy) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (BackoffStrategy) Descriptor() protoreflect.EnumDescriptor { + return file_aether_proto_enumTypes[9].Descriptor() +} + +func (BackoffStrategy) Type() protoreflect.EnumType { + return &file_aether_proto_enumTypes[9] +} + +func (x BackoffStrategy) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use BackoffStrategy.Descriptor instead. +func (BackoffStrategy) EnumDescriptor() ([]byte, []int) { + return file_aether_proto_rawDescGZIP(), []int{9} +} + // WaitReason enumerates why a task is in a WAITING_* state. Each value pairs // with a specific TaskStatus: WAIT_REASON_INPUT <-> TASK_STATUS_WAITING_INPUT, // WAIT_REASON_AUTHORITY <-> TASK_STATUS_WAITING_AUTHORITY, @@ -551,11 +669,11 @@ func (x WaitReason) String() string { } func (WaitReason) Descriptor() protoreflect.EnumDescriptor { - return file_aether_proto_enumTypes[8].Descriptor() + return file_aether_proto_enumTypes[10].Descriptor() } func (WaitReason) Type() protoreflect.EnumType { - return &file_aether_proto_enumTypes[8] + return &file_aether_proto_enumTypes[10] } func (x WaitReason) Number() protoreflect.EnumNumber { @@ -564,7 +682,7 @@ func (x WaitReason) Number() protoreflect.EnumNumber { // Deprecated: Use WaitReason.Descriptor instead. func (WaitReason) EnumDescriptor() ([]byte, []int) { - return file_aether_proto_rawDescGZIP(), []int{8} + return file_aether_proto_rawDescGZIP(), []int{10} } // AuthorityRequestStatus tracks the lifecycle of an AuthorityRequest. @@ -610,11 +728,11 @@ func (x AuthorityRequestStatus) String() string { } func (AuthorityRequestStatus) Descriptor() protoreflect.EnumDescriptor { - return file_aether_proto_enumTypes[9].Descriptor() + return file_aether_proto_enumTypes[11].Descriptor() } func (AuthorityRequestStatus) Type() protoreflect.EnumType { - return &file_aether_proto_enumTypes[9] + return &file_aether_proto_enumTypes[11] } func (x AuthorityRequestStatus) Number() protoreflect.EnumNumber { @@ -623,7 +741,7 @@ func (x AuthorityRequestStatus) Number() protoreflect.EnumNumber { // Deprecated: Use AuthorityRequestStatus.Descriptor instead. func (AuthorityRequestStatus) EnumDescriptor() ([]byte, []int) { - return file_aether_proto_rawDescGZIP(), []int{9} + return file_aether_proto_rawDescGZIP(), []int{11} } // ProgressKind classifies a progress update by its intended UI surface or @@ -675,11 +793,11 @@ func (x ProgressKind) String() string { } func (ProgressKind) Descriptor() protoreflect.EnumDescriptor { - return file_aether_proto_enumTypes[10].Descriptor() + return file_aether_proto_enumTypes[12].Descriptor() } func (ProgressKind) Type() protoreflect.EnumType { - return &file_aether_proto_enumTypes[10] + return &file_aether_proto_enumTypes[12] } func (x ProgressKind) Number() protoreflect.EnumNumber { @@ -688,7 +806,7 @@ func (x ProgressKind) Number() protoreflect.EnumNumber { // Deprecated: Use ProgressKind.Descriptor instead. func (ProgressKind) EnumDescriptor() ([]byte, []int) { - return file_aether_proto_rawDescGZIP(), []int{10} + return file_aether_proto_rawDescGZIP(), []int{12} } type KVOperation_OpType int32 @@ -702,29 +820,66 @@ const ( KVOperation_DECREMENT KVOperation_OpType = 5 KVOperation_INCREMENT_IF KVOperation_OpType = 6 // Atomic increment that succeeds only if result <= guard_value KVOperation_DECREMENT_IF KVOperation_OpType = 7 // Atomic decrement that succeeds only if result >= guard_value + // Atomic conditional writes — the building blocks for distributed + // coordination primitives (mutex, leader election, run-once). All three + // are backed across Redis / Badger / NATS-JetStream. KVResponse.applied + // reports whether the conditional mutation took effect. + KVOperation_SET_NX KVOperation_OpType = 8 // Set `value` only if the key is absent. applied=true iff written. + KVOperation_COMPARE_AND_SET KVOperation_OpType = 9 // Set `value` only if current == expected_value. applied=true iff swapped. + KVOperation_COMPARE_AND_DELETE KVOperation_OpType = 10 // Delete only if current == expected_value. applied=true iff deleted. + // REMOVAL-ONLY purge of ANOTHER principal's KV namespace, for lifecycle + // managers reaping ephemeral principals (e.g. sandbox-provider clearing a + // destroyed sidecar's private state). Deletes every key in + // (target_identity, scope) optionally filtered by `key` as a prefix. + // Gated by the capability/kv_purge_identity ACL grant. By design it + // returns ONLY KVResponse.counter_value (count deleted) — never keys or + // values — so the grant cannot be used to exfiltrate another principal's + // data. Maintains component separation: the gateway has no sandbox-specific + // knowledge; the caller decides what to purge and when. + KVOperation_PURGE_IDENTITY KVOperation_OpType = 11 + // Atomic set primitives backing fan-in joins (set-completeness) and + // at-most-once dedup ledgers, native across Redis / Badger / NATS-JetStream. + // SET_ADD adds `value` (the member) to the set at `key`, (re)setting `ttl` + // on a newly-added member: KVResponse.applied=true iff newly added, + // counter_value=cardinality after the add. SET_CARD returns + // counter_value=cardinality (0 if absent). + KVOperation_SET_ADD KVOperation_OpType = 12 + KVOperation_SET_CARD KVOperation_OpType = 13 ) // Enum value maps for KVOperation_OpType. var ( KVOperation_OpType_name = map[int32]string{ - 0: "GET", - 1: "PUT", - 2: "LIST", - 3: "DELETE", - 4: "INCREMENT", - 5: "DECREMENT", - 6: "INCREMENT_IF", - 7: "DECREMENT_IF", + 0: "GET", + 1: "PUT", + 2: "LIST", + 3: "DELETE", + 4: "INCREMENT", + 5: "DECREMENT", + 6: "INCREMENT_IF", + 7: "DECREMENT_IF", + 8: "SET_NX", + 9: "COMPARE_AND_SET", + 10: "COMPARE_AND_DELETE", + 11: "PURGE_IDENTITY", + 12: "SET_ADD", + 13: "SET_CARD", } KVOperation_OpType_value = map[string]int32{ - "GET": 0, - "PUT": 1, - "LIST": 2, - "DELETE": 3, - "INCREMENT": 4, - "DECREMENT": 5, - "INCREMENT_IF": 6, - "DECREMENT_IF": 7, + "GET": 0, + "PUT": 1, + "LIST": 2, + "DELETE": 3, + "INCREMENT": 4, + "DECREMENT": 5, + "INCREMENT_IF": 6, + "DECREMENT_IF": 7, + "SET_NX": 8, + "COMPARE_AND_SET": 9, + "COMPARE_AND_DELETE": 10, + "PURGE_IDENTITY": 11, + "SET_ADD": 12, + "SET_CARD": 13, } ) @@ -739,11 +894,11 @@ func (x KVOperation_OpType) String() string { } func (KVOperation_OpType) Descriptor() protoreflect.EnumDescriptor { - return file_aether_proto_enumTypes[11].Descriptor() + return file_aether_proto_enumTypes[13].Descriptor() } func (KVOperation_OpType) Type() protoreflect.EnumType { - return &file_aether_proto_enumTypes[11] + return &file_aether_proto_enumTypes[13] } func (x KVOperation_OpType) Number() protoreflect.EnumNumber { @@ -820,11 +975,11 @@ func (x KVOperation_Scope) String() string { } func (KVOperation_Scope) Descriptor() protoreflect.EnumDescriptor { - return file_aether_proto_enumTypes[12].Descriptor() + return file_aether_proto_enumTypes[14].Descriptor() } func (KVOperation_Scope) Type() protoreflect.EnumType { - return &file_aether_proto_enumTypes[12] + return &file_aether_proto_enumTypes[14] } func (x KVOperation_Scope) Number() protoreflect.EnumNumber { @@ -866,11 +1021,11 @@ func (x Signal_SignalType) String() string { } func (Signal_SignalType) Descriptor() protoreflect.EnumDescriptor { - return file_aether_proto_enumTypes[13].Descriptor() + return file_aether_proto_enumTypes[15].Descriptor() } func (Signal_SignalType) Type() protoreflect.EnumType { - return &file_aether_proto_enumTypes[13] + return &file_aether_proto_enumTypes[15] } func (x Signal_SignalType) Number() protoreflect.EnumNumber { @@ -918,11 +1073,11 @@ func (x CheckpointOperation_OpType) String() string { } func (CheckpointOperation_OpType) Descriptor() protoreflect.EnumDescriptor { - return file_aether_proto_enumTypes[14].Descriptor() + return file_aether_proto_enumTypes[16].Descriptor() } func (CheckpointOperation_OpType) Type() protoreflect.EnumType { - return &file_aether_proto_enumTypes[14] + return &file_aether_proto_enumTypes[16] } func (x CheckpointOperation_OpType) Number() protoreflect.EnumNumber { @@ -931,7 +1086,7 @@ func (x CheckpointOperation_OpType) Number() protoreflect.EnumNumber { // Deprecated: Use CheckpointOperation_OpType.Descriptor instead. func (CheckpointOperation_OpType) EnumDescriptor() ([]byte, []int) { - return file_aether_proto_rawDescGZIP(), []int{32, 0} + return file_aether_proto_rawDescGZIP(), []int{34, 0} } type AdminQuery_OpType int32 @@ -973,11 +1128,11 @@ func (x AdminQuery_OpType) String() string { } func (AdminQuery_OpType) Descriptor() protoreflect.EnumDescriptor { - return file_aether_proto_enumTypes[15].Descriptor() + return file_aether_proto_enumTypes[17].Descriptor() } func (AdminQuery_OpType) Type() protoreflect.EnumType { - return &file_aether_proto_enumTypes[15] + return &file_aether_proto_enumTypes[17] } func (x AdminQuery_OpType) Number() protoreflect.EnumNumber { @@ -986,7 +1141,7 @@ func (x AdminQuery_OpType) Number() protoreflect.EnumNumber { // Deprecated: Use AdminQuery_OpType.Descriptor instead. func (AdminQuery_OpType) EnumDescriptor() ([]byte, []int) { - return file_aether_proto_rawDescGZIP(), []int{34, 0} + return file_aether_proto_rawDescGZIP(), []int{36, 0} } type SessionOperation_OpType int32 @@ -1022,11 +1177,11 @@ func (x SessionOperation_OpType) String() string { } func (SessionOperation_OpType) Descriptor() protoreflect.EnumDescriptor { - return file_aether_proto_enumTypes[16].Descriptor() + return file_aether_proto_enumTypes[18].Descriptor() } func (SessionOperation_OpType) Type() protoreflect.EnumType { - return &file_aether_proto_enumTypes[16] + return &file_aether_proto_enumTypes[18] } func (x SessionOperation_OpType) Number() protoreflect.EnumNumber { @@ -1035,7 +1190,7 @@ func (x SessionOperation_OpType) Number() protoreflect.EnumNumber { // Deprecated: Use SessionOperation_OpType.Descriptor instead. func (SessionOperation_OpType) EnumDescriptor() ([]byte, []int) { - return file_aether_proto_rawDescGZIP(), []int{42, 0} + return file_aether_proto_rawDescGZIP(), []int{44, 0} } type TaskQuery_OpType int32 @@ -1068,11 +1223,11 @@ func (x TaskQuery_OpType) String() string { } func (TaskQuery_OpType) Descriptor() protoreflect.EnumDescriptor { - return file_aether_proto_enumTypes[17].Descriptor() + return file_aether_proto_enumTypes[19].Descriptor() } func (TaskQuery_OpType) Type() protoreflect.EnumType { - return &file_aether_proto_enumTypes[17] + return &file_aether_proto_enumTypes[19] } func (x TaskQuery_OpType) Number() protoreflect.EnumNumber { @@ -1081,7 +1236,7 @@ func (x TaskQuery_OpType) Number() protoreflect.EnumNumber { // Deprecated: Use TaskQuery_OpType.Descriptor instead. func (TaskQuery_OpType) EnumDescriptor() ([]byte, []int) { - return file_aether_proto_rawDescGZIP(), []int{44, 0} + return file_aether_proto_rawDescGZIP(), []int{46, 0} } type TaskOperation_OpType int32 @@ -1095,6 +1250,7 @@ const ( TaskOperation_WAIT_FOR TaskOperation_OpType = 5 // Transition running -> WAITING_DEPENDENCY on specific task ids. Requires wait_spec. TaskOperation_RESUME TaskOperation_OpType = 6 // Force-resume a paused task (admin/manual; normally task_waker fires). TaskOperation_REJECT TaskOperation_OpType = 7 // Agent declines before processing -> REJECTED terminal state. + TaskOperation_CLAIM TaskOperation_OpType = 8 // Assignee transitions an assigned/pending task to RUNNING. ) // Enum value maps for TaskOperation_OpType. @@ -1108,6 +1264,7 @@ var ( 5: "WAIT_FOR", 6: "RESUME", 7: "REJECT", + 8: "CLAIM", } TaskOperation_OpType_value = map[string]int32{ "RETRY": 0, @@ -1118,6 +1275,7 @@ var ( "WAIT_FOR": 5, "RESUME": 6, "REJECT": 7, + "CLAIM": 8, } ) @@ -1132,11 +1290,11 @@ func (x TaskOperation_OpType) String() string { } func (TaskOperation_OpType) Descriptor() protoreflect.EnumDescriptor { - return file_aether_proto_enumTypes[18].Descriptor() + return file_aether_proto_enumTypes[20].Descriptor() } func (TaskOperation_OpType) Type() protoreflect.EnumType { - return &file_aether_proto_enumTypes[18] + return &file_aether_proto_enumTypes[20] } func (x TaskOperation_OpType) Number() protoreflect.EnumNumber { @@ -1145,7 +1303,7 @@ func (x TaskOperation_OpType) Number() protoreflect.EnumNumber { // Deprecated: Use TaskOperation_OpType.Descriptor instead. func (TaskOperation_OpType) EnumDescriptor() ([]byte, []int) { - return file_aether_proto_rawDescGZIP(), []int{48, 0} + return file_aether_proto_rawDescGZIP(), []int{50, 0} } type WorkspaceOperation_OpType int32 @@ -1190,11 +1348,11 @@ func (x WorkspaceOperation_OpType) String() string { } func (WorkspaceOperation_OpType) Descriptor() protoreflect.EnumDescriptor { - return file_aether_proto_enumTypes[19].Descriptor() + return file_aether_proto_enumTypes[21].Descriptor() } func (WorkspaceOperation_OpType) Type() protoreflect.EnumType { - return &file_aether_proto_enumTypes[19] + return &file_aether_proto_enumTypes[21] } func (x WorkspaceOperation_OpType) Number() protoreflect.EnumNumber { @@ -1203,7 +1361,7 @@ func (x WorkspaceOperation_OpType) Number() protoreflect.EnumNumber { // Deprecated: Use WorkspaceOperation_OpType.Descriptor instead. func (WorkspaceOperation_OpType) EnumDescriptor() ([]byte, []int) { - return file_aether_proto_rawDescGZIP(), []int{52, 0} + return file_aether_proto_rawDescGZIP(), []int{54, 0} } type AgentOperation_OpType int32 @@ -1251,11 +1409,11 @@ func (x AgentOperation_OpType) String() string { } func (AgentOperation_OpType) Descriptor() protoreflect.EnumDescriptor { - return file_aether_proto_enumTypes[20].Descriptor() + return file_aether_proto_enumTypes[22].Descriptor() } func (AgentOperation_OpType) Type() protoreflect.EnumType { - return &file_aether_proto_enumTypes[20] + return &file_aether_proto_enumTypes[22] } func (x AgentOperation_OpType) Number() protoreflect.EnumNumber { @@ -1264,7 +1422,7 @@ func (x AgentOperation_OpType) Number() protoreflect.EnumNumber { // Deprecated: Use AgentOperation_OpType.Descriptor instead. func (AgentOperation_OpType) EnumDescriptor() ([]byte, []int) { - return file_aether_proto_rawDescGZIP(), []int{59, 0} + return file_aether_proto_rawDescGZIP(), []int{61, 0} } type ACLOperation_OpType int32 @@ -1279,6 +1437,25 @@ const ( ACLOperation_SET_FALLBACK_POLICY ACLOperation_OpType = 9 // PUT /api/acl/fallback-policy - Set/update a fallback policy ACLOperation_CLEANUP_EXPIRED ACLOperation_OpType = 10 // POST /api/acl/cleanup/expired-rules - Remove expired ACL rules ACLOperation_CLEANUP_AUDIT_LOGS ACLOperation_OpType = 11 // POST /api/acl/cleanup/audit-logs - Remove old audit log entries + // Role/group authorization. REST equivalents under /api/acl/groups, + // /api/acl/roles, and /api/acl/principals/{type}/{id}/{groups,roles}. + ACLOperation_CREATE_GROUP ACLOperation_OpType = 17 // Create a group (group_request) + ACLOperation_DELETE_GROUP ACLOperation_OpType = 18 // Delete a group (name) + ACLOperation_GET_GROUP ACLOperation_OpType = 19 // Get a group (name) + ACLOperation_LIST_GROUPS ACLOperation_OpType = 20 // List all groups + ACLOperation_ADD_GROUP_MEMBER ACLOperation_OpType = 21 // Add a member to a group (name + member_request) + ACLOperation_REMOVE_GROUP_MEMBER ACLOperation_OpType = 22 // Remove a member from a group (name + principal) + ACLOperation_LIST_GROUP_MEMBERS ACLOperation_OpType = 23 // List members of a group (name) + ACLOperation_CREATE_ROLE ACLOperation_OpType = 24 // Create a role (role_request) + ACLOperation_DELETE_ROLE ACLOperation_OpType = 25 // Delete a role (name) + ACLOperation_GET_ROLE ACLOperation_OpType = 26 // Get a role (name) + ACLOperation_LIST_ROLES ACLOperation_OpType = 27 // List all roles + ACLOperation_ASSIGN_ROLE ACLOperation_OpType = 28 // Assign a role (name + assignment_request) + ACLOperation_UNASSIGN_ROLE ACLOperation_OpType = 29 // Unassign a role (name + principal) + ACLOperation_LIST_ROLE_ASSIGNMENTS ACLOperation_OpType = 30 // List assignees of a role (name) + ACLOperation_LIST_PRINCIPAL_GROUPS ACLOperation_OpType = 31 // List groups a principal belongs to (principal) + ACLOperation_LIST_PRINCIPAL_ROLES ACLOperation_OpType = 32 // List roles assigned to a principal (principal) + ACLOperation_EXPLAIN_ACCESS ACLOperation_OpType = 33 // Explain effective access (principal + resource_type + resource_id [+ required_level]) ) // Enum value maps for ACLOperation_OpType. @@ -1293,17 +1470,51 @@ var ( 9: "SET_FALLBACK_POLICY", 10: "CLEANUP_EXPIRED", 11: "CLEANUP_AUDIT_LOGS", + 17: "CREATE_GROUP", + 18: "DELETE_GROUP", + 19: "GET_GROUP", + 20: "LIST_GROUPS", + 21: "ADD_GROUP_MEMBER", + 22: "REMOVE_GROUP_MEMBER", + 23: "LIST_GROUP_MEMBERS", + 24: "CREATE_ROLE", + 25: "DELETE_ROLE", + 26: "GET_ROLE", + 27: "LIST_ROLES", + 28: "ASSIGN_ROLE", + 29: "UNASSIGN_ROLE", + 30: "LIST_ROLE_ASSIGNMENTS", + 31: "LIST_PRINCIPAL_GROUPS", + 32: "LIST_PRINCIPAL_ROLES", + 33: "EXPLAIN_ACCESS", } ACLOperation_OpType_value = map[string]int32{ - "LIST_RULES": 0, - "GET_RULE": 1, - "GRANT": 2, - "REVOKE": 3, - "QUERY_AUDIT": 4, - "GET_FALLBACK_POLICY": 8, - "SET_FALLBACK_POLICY": 9, - "CLEANUP_EXPIRED": 10, - "CLEANUP_AUDIT_LOGS": 11, + "LIST_RULES": 0, + "GET_RULE": 1, + "GRANT": 2, + "REVOKE": 3, + "QUERY_AUDIT": 4, + "GET_FALLBACK_POLICY": 8, + "SET_FALLBACK_POLICY": 9, + "CLEANUP_EXPIRED": 10, + "CLEANUP_AUDIT_LOGS": 11, + "CREATE_GROUP": 17, + "DELETE_GROUP": 18, + "GET_GROUP": 19, + "LIST_GROUPS": 20, + "ADD_GROUP_MEMBER": 21, + "REMOVE_GROUP_MEMBER": 22, + "LIST_GROUP_MEMBERS": 23, + "CREATE_ROLE": 24, + "DELETE_ROLE": 25, + "GET_ROLE": 26, + "LIST_ROLES": 27, + "ASSIGN_ROLE": 28, + "UNASSIGN_ROLE": 29, + "LIST_ROLE_ASSIGNMENTS": 30, + "LIST_PRINCIPAL_GROUPS": 31, + "LIST_PRINCIPAL_ROLES": 32, + "EXPLAIN_ACCESS": 33, } ) @@ -1318,11 +1529,11 @@ func (x ACLOperation_OpType) String() string { } func (ACLOperation_OpType) Descriptor() protoreflect.EnumDescriptor { - return file_aether_proto_enumTypes[21].Descriptor() + return file_aether_proto_enumTypes[23].Descriptor() } func (ACLOperation_OpType) Type() protoreflect.EnumType { - return &file_aether_proto_enumTypes[21] + return &file_aether_proto_enumTypes[23] } func (x ACLOperation_OpType) Number() protoreflect.EnumNumber { @@ -1331,7 +1542,7 @@ func (x ACLOperation_OpType) Number() protoreflect.EnumNumber { // Deprecated: Use ACLOperation_OpType.Descriptor instead. func (ACLOperation_OpType) EnumDescriptor() ([]byte, []int) { - return file_aether_proto_rawDescGZIP(), []int{67, 0} + return file_aether_proto_rawDescGZIP(), []int{69, 0} } type AuthorityGrantOperation_OpType int32 @@ -1385,11 +1596,11 @@ func (x AuthorityGrantOperation_OpType) String() string { } func (AuthorityGrantOperation_OpType) Descriptor() protoreflect.EnumDescriptor { - return file_aether_proto_enumTypes[22].Descriptor() + return file_aether_proto_enumTypes[24].Descriptor() } func (AuthorityGrantOperation_OpType) Type() protoreflect.EnumType { - return &file_aether_proto_enumTypes[22] + return &file_aether_proto_enumTypes[24] } func (x AuthorityGrantOperation_OpType) Number() protoreflect.EnumNumber { @@ -1398,7 +1609,7 @@ func (x AuthorityGrantOperation_OpType) Number() protoreflect.EnumNumber { // Deprecated: Use AuthorityGrantOperation_OpType.Descriptor instead. func (AuthorityGrantOperation_OpType) EnumDescriptor() ([]byte, []int) { - return file_aether_proto_rawDescGZIP(), []int{82, 0} + return file_aether_proto_rawDescGZIP(), []int{94, 0} } type ResolveAuthorityRequestPayload_Decision int32 @@ -1434,11 +1645,11 @@ func (x ResolveAuthorityRequestPayload_Decision) String() string { } func (ResolveAuthorityRequestPayload_Decision) Descriptor() protoreflect.EnumDescriptor { - return file_aether_proto_enumTypes[23].Descriptor() + return file_aether_proto_enumTypes[25].Descriptor() } func (ResolveAuthorityRequestPayload_Decision) Type() protoreflect.EnumType { - return &file_aether_proto_enumTypes[23] + return &file_aether_proto_enumTypes[25] } func (x ResolveAuthorityRequestPayload_Decision) Number() protoreflect.EnumNumber { @@ -1447,7 +1658,7 @@ func (x ResolveAuthorityRequestPayload_Decision) Number() protoreflect.EnumNumbe // Deprecated: Use ResolveAuthorityRequestPayload_Decision.Descriptor instead. func (ResolveAuthorityRequestPayload_Decision) EnumDescriptor() ([]byte, []int) { - return file_aether_proto_rawDescGZIP(), []int{96, 0} + return file_aether_proto_rawDescGZIP(), []int{108, 0} } type AuthorityRequestOperation_OpType int32 @@ -1492,11 +1703,11 @@ func (x AuthorityRequestOperation_OpType) String() string { } func (AuthorityRequestOperation_OpType) Descriptor() protoreflect.EnumDescriptor { - return file_aether_proto_enumTypes[24].Descriptor() + return file_aether_proto_enumTypes[26].Descriptor() } func (AuthorityRequestOperation_OpType) Type() protoreflect.EnumType { - return &file_aether_proto_enumTypes[24] + return &file_aether_proto_enumTypes[26] } func (x AuthorityRequestOperation_OpType) Number() protoreflect.EnumNumber { @@ -1505,7 +1716,7 @@ func (x AuthorityRequestOperation_OpType) Number() protoreflect.EnumNumber { // Deprecated: Use AuthorityRequestOperation_OpType.Descriptor instead. func (AuthorityRequestOperation_OpType) EnumDescriptor() ([]byte, []int) { - return file_aether_proto_rawDescGZIP(), []int{98, 0} + return file_aether_proto_rawDescGZIP(), []int{110, 0} } type AuthorityRequestEvent_EventType int32 @@ -1550,11 +1761,11 @@ func (x AuthorityRequestEvent_EventType) String() string { } func (AuthorityRequestEvent_EventType) Descriptor() protoreflect.EnumDescriptor { - return file_aether_proto_enumTypes[25].Descriptor() + return file_aether_proto_enumTypes[27].Descriptor() } func (AuthorityRequestEvent_EventType) Type() protoreflect.EnumType { - return &file_aether_proto_enumTypes[25] + return &file_aether_proto_enumTypes[27] } func (x AuthorityRequestEvent_EventType) Number() protoreflect.EnumNumber { @@ -1563,7 +1774,7 @@ func (x AuthorityRequestEvent_EventType) Number() protoreflect.EnumNumber { // Deprecated: Use AuthorityRequestEvent_EventType.Descriptor instead. func (AuthorityRequestEvent_EventType) EnumDescriptor() ([]byte, []int) { - return file_aether_proto_rawDescGZIP(), []int{100, 0} + return file_aether_proto_rawDescGZIP(), []int{112, 0} } type TokenOperation_OpType int32 @@ -1605,11 +1816,11 @@ func (x TokenOperation_OpType) String() string { } func (TokenOperation_OpType) Descriptor() protoreflect.EnumDescriptor { - return file_aether_proto_enumTypes[26].Descriptor() + return file_aether_proto_enumTypes[28].Descriptor() } func (TokenOperation_OpType) Type() protoreflect.EnumType { - return &file_aether_proto_enumTypes[26] + return &file_aether_proto_enumTypes[28] } func (x TokenOperation_OpType) Number() protoreflect.EnumNumber { @@ -1618,7 +1829,7 @@ func (x TokenOperation_OpType) Number() protoreflect.EnumNumber { // Deprecated: Use TokenOperation_OpType.Descriptor instead. func (TokenOperation_OpType) EnumDescriptor() ([]byte, []int) { - return file_aether_proto_rawDescGZIP(), []int{101, 0} + return file_aether_proto_rawDescGZIP(), []int{113, 0} } type WorkflowOperation_OpType int32 @@ -1655,6 +1866,10 @@ const ( WorkflowOperation_SEND_SM_EVENT WorkflowOperation_OpType = 22 // Schedule upsert (idempotent create-or-update) WorkflowOperation_UPSERT_SCHEDULE WorkflowOperation_OpType = 23 + // Fan-in / barrier / coalesce join observability + operator control. + WorkflowOperation_LIST_JOINS WorkflowOperation_OpType = 24 // in-flight + recently-terminal join instances (workspace filter) + WorkflowOperation_GET_JOIN WorkflowOperation_OpType = 25 // inspect one instance: id=join_name, secondary_id=correlation_key + WorkflowOperation_CANCEL_JOIN WorkflowOperation_OpType = 26 // operator GC of a wedged instance ) // Enum value maps for WorkflowOperation_OpType. @@ -1684,6 +1899,9 @@ var ( 21: "CREATE_SM_INSTANCE", 22: "SEND_SM_EVENT", 23: "UPSERT_SCHEDULE", + 24: "LIST_JOINS", + 25: "GET_JOIN", + 26: "CANCEL_JOIN", } WorkflowOperation_OpType_value = map[string]int32{ "LIST_RULES": 0, @@ -1710,6 +1928,9 @@ var ( "CREATE_SM_INSTANCE": 21, "SEND_SM_EVENT": 22, "UPSERT_SCHEDULE": 23, + "LIST_JOINS": 24, + "GET_JOIN": 25, + "CANCEL_JOIN": 26, } ) @@ -1724,11 +1945,11 @@ func (x WorkflowOperation_OpType) String() string { } func (WorkflowOperation_OpType) Descriptor() protoreflect.EnumDescriptor { - return file_aether_proto_enumTypes[27].Descriptor() + return file_aether_proto_enumTypes[29].Descriptor() } func (WorkflowOperation_OpType) Type() protoreflect.EnumType { - return &file_aether_proto_enumTypes[27] + return &file_aether_proto_enumTypes[29] } func (x WorkflowOperation_OpType) Number() protoreflect.EnumNumber { @@ -1737,7 +1958,7 @@ func (x WorkflowOperation_OpType) Number() protoreflect.EnumNumber { // Deprecated: Use WorkflowOperation_OpType.Descriptor instead. func (WorkflowOperation_OpType) EnumDescriptor() ([]byte, []int) { - return file_aether_proto_rawDescGZIP(), []int{109, 0} + return file_aether_proto_rawDescGZIP(), []int{121, 0} } type ProxyError_Kind int32 @@ -1788,11 +2009,11 @@ func (x ProxyError_Kind) String() string { } func (ProxyError_Kind) Descriptor() protoreflect.EnumDescriptor { - return file_aether_proto_enumTypes[28].Descriptor() + return file_aether_proto_enumTypes[30].Descriptor() } func (ProxyError_Kind) Type() protoreflect.EnumType { - return &file_aether_proto_enumTypes[28] + return &file_aether_proto_enumTypes[30] } func (x ProxyError_Kind) Number() protoreflect.EnumNumber { @@ -1801,7 +2022,7 @@ func (x ProxyError_Kind) Number() protoreflect.EnumNumber { // Deprecated: Use ProxyError_Kind.Descriptor instead. func (ProxyError_Kind) EnumDescriptor() ([]byte, []int) { - return file_aether_proto_rawDescGZIP(), []int{120, 0} + return file_aether_proto_rawDescGZIP(), []int{132, 0} } type TunnelOpen_Protocol int32 @@ -1837,11 +2058,11 @@ func (x TunnelOpen_Protocol) String() string { } func (TunnelOpen_Protocol) Descriptor() protoreflect.EnumDescriptor { - return file_aether_proto_enumTypes[29].Descriptor() + return file_aether_proto_enumTypes[31].Descriptor() } func (TunnelOpen_Protocol) Type() protoreflect.EnumType { - return &file_aether_proto_enumTypes[29] + return &file_aether_proto_enumTypes[31] } func (x TunnelOpen_Protocol) Number() protoreflect.EnumNumber { @@ -1850,7 +2071,7 @@ func (x TunnelOpen_Protocol) Number() protoreflect.EnumNumber { // Deprecated: Use TunnelOpen_Protocol.Descriptor instead. func (TunnelOpen_Protocol) EnumDescriptor() ([]byte, []int) { - return file_aether_proto_rawDescGZIP(), []int{121, 0} + return file_aether_proto_rawDescGZIP(), []int{133, 0} } type TunnelClose_Reason int32 @@ -1892,11 +2113,11 @@ func (x TunnelClose_Reason) String() string { } func (TunnelClose_Reason) Descriptor() protoreflect.EnumDescriptor { - return file_aether_proto_enumTypes[30].Descriptor() + return file_aether_proto_enumTypes[32].Descriptor() } func (TunnelClose_Reason) Type() protoreflect.EnumType { - return &file_aether_proto_enumTypes[30] + return &file_aether_proto_enumTypes[32] } func (x TunnelClose_Reason) Number() protoreflect.EnumNumber { @@ -1905,7 +2126,7 @@ func (x TunnelClose_Reason) Number() protoreflect.EnumNumber { // Deprecated: Use TunnelClose_Reason.Descriptor instead. func (TunnelClose_Reason) EnumDescriptor() ([]byte, []int) { - return file_aether_proto_rawDescGZIP(), []int{123, 0} + return file_aether_proto_rawDescGZIP(), []int{135, 0} } type TaskSubscriptionOperation_OpType int32 @@ -1941,11 +2162,11 @@ func (x TaskSubscriptionOperation_OpType) String() string { } func (TaskSubscriptionOperation_OpType) Descriptor() protoreflect.EnumDescriptor { - return file_aether_proto_enumTypes[31].Descriptor() + return file_aether_proto_enumTypes[33].Descriptor() } func (TaskSubscriptionOperation_OpType) Type() protoreflect.EnumType { - return &file_aether_proto_enumTypes[31] + return &file_aether_proto_enumTypes[33] } func (x TaskSubscriptionOperation_OpType) Number() protoreflect.EnumNumber { @@ -1954,7 +2175,7 @@ func (x TaskSubscriptionOperation_OpType) Number() protoreflect.EnumNumber { // Deprecated: Use TaskSubscriptionOperation_OpType.Descriptor instead. func (TaskSubscriptionOperation_OpType) EnumDescriptor() ([]byte, []int) { - return file_aether_proto_rawDescGZIP(), []int{131, 0} + return file_aether_proto_rawDescGZIP(), []int{143, 0} } type UpstreamMessage struct { @@ -4051,6 +4272,15 @@ type ServiceIdentity struct { state protoimpl.MessageState `protogen:"open.v1"` Implementation string `protobuf:"bytes,1,opt,name=implementation,proto3" json:"implementation,omitempty"` // e.g., "frontend-api", "platform-backend" Specifier string `protobuf:"bytes,2,opt,name=specifier,proto3" json:"specifier,omitempty"` // Instance identifier for uniqueness (e.g., "pod-1", "default") + // When true, the gateway does NOT add this service connection to the + // pool-task worker index, so it is never targeted for POOL-mode task + // assignments for its implementation. Default false = pool consumer + // (back-compat: existing services keep receiving pool tasks). Serve-only + // instances that register no task handlers (e.g. a MemoryLayer server with + // its in-process worker disabled) set this so pool tasks are routed only to + // real workers instead of being claimed-and-dropped (and stuck until + // reconcile). Agents are always pool consumers regardless of this field. + NoPoolConsumer bool `protobuf:"varint,3,opt,name=no_pool_consumer,json=noPoolConsumer,proto3" json:"no_pool_consumer,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -4099,6 +4329,13 @@ func (x *ServiceIdentity) GetSpecifier() string { return "" } +func (x *ServiceIdentity) GetNoPoolConsumer() bool { + if x != nil { + return x.NoPoolConsumer + } + return false +} + type AgentIdentity struct { state protoimpl.MessageState `protogen:"open.v1"` Workspace string `protobuf:"bytes,1,opt,name=workspace,proto3" json:"workspace,omitempty"` @@ -4784,9 +5021,26 @@ type KVOperation struct { // Step size for INCREMENT_IF / DECREMENT_IF. When 0 the server uses a // default of 1, matching unguarded INCREMENT/DECREMENT. Must be >= 0; // negative deltas are rejected by the server. - DeltaValue int64 `protobuf:"varint,11,opt,name=delta_value,json=deltaValue,proto3" json:"delta_value,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + DeltaValue int64 `protobuf:"varint,11,opt,name=delta_value,json=deltaValue,proto3" json:"delta_value,omitempty"` + // Expected current value for COMPARE_AND_SET / COMPARE_AND_DELETE. The + // mutation applies only when the stored value equals these bytes. Unused by + // other ops. For an empty/absent comparison use SET_NX instead. + ExpectedValue []byte `protobuf:"bytes,12,opt,name=expected_value,json=expectedValue,proto3" json:"expected_value,omitempty"` + // LIST pagination. limit caps the keys returned in one LIST response (<=0 → + // server default). cursor pages through results: pass the previous + // KVResponse.next_cursor to fetch the next page; empty starts from the + // beginning. For LIST, `key` (field 3) is the key prefix filter, applied + // server-side BEFORE the limit. Unused by non-LIST ops. + Limit int32 `protobuf:"varint,13,opt,name=limit,proto3" json:"limit,omitempty"` + Cursor string `protobuf:"bytes,14,opt,name=cursor,proto3" json:"cursor,omitempty"` + // PURGE_IDENTITY only: the principal whose KV namespace to purge (e.g. + // "sv::sandbox-sidecar::"). The caller's OWN identity authorizes + // the op via capability/kv_purge_identity; this names the TARGET namespace. + // `key` (field 3) acts as an optional prefix filter; `scope` selects which + // scope to purge. Ignored by all other ops. + TargetIdentity string `protobuf:"bytes,15,opt,name=target_identity,json=targetIdentity,proto3" json:"target_identity,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *KVOperation) Reset() { @@ -4896,6 +5150,34 @@ func (x *KVOperation) GetDeltaValue() int64 { return 0 } +func (x *KVOperation) GetExpectedValue() []byte { + if x != nil { + return x.ExpectedValue + } + return nil +} + +func (x *KVOperation) GetLimit() int32 { + if x != nil { + return x.Limit + } + return 0 +} + +func (x *KVOperation) GetCursor() string { + if x != nil { + return x.Cursor + } + return "" +} + +func (x *KVOperation) GetTargetIdentity() string { + if x != nil { + return x.TargetIdentity + } + return "" +} + type KVResponse struct { state protoimpl.MessageState `protogen:"open.v1"` Success bool `protobuf:"varint,1,opt,name=success,proto3" json:"success,omitempty"` @@ -4907,10 +5189,20 @@ type KVResponse struct { // transparently, but Python proto deserializes a `string` field as // UTF-8 `str` — destroying or silently mangling non-UTF-8 binary data. // The single-value `bytes value` field above already gets this right. - KvMap map[string][]byte `protobuf:"bytes,4,rep,name=kv_map,json=kvMap,proto3" json:"kv_map,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` - RequestId string `protobuf:"bytes,5,opt,name=request_id,json=requestId,proto3" json:"request_id,omitempty"` // Echoed from the originating KVOperation for correlation - CounterValue int64 `protobuf:"varint,6,opt,name=counter_value,json=counterValue,proto3" json:"counter_value,omitempty"` // Result of INCREMENT/DECREMENT (and INCREMENT_IF/DECREMENT_IF) - Applied bool `protobuf:"varint,7,opt,name=applied,proto3" json:"applied,omitempty"` // True iff INCREMENT_IF/DECREMENT_IF mutation was applied; for unguarded ops always true on success + KvMap map[string][]byte `protobuf:"bytes,4,rep,name=kv_map,json=kvMap,proto3" json:"kv_map,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` + RequestId string `protobuf:"bytes,5,opt,name=request_id,json=requestId,proto3" json:"request_id,omitempty"` // Echoed from the originating KVOperation for correlation + CounterValue int64 `protobuf:"varint,6,opt,name=counter_value,json=counterValue,proto3" json:"counter_value,omitempty"` // Result of INCREMENT/DECREMENT (and INCREMENT_IF/DECREMENT_IF) + // True iff a conditional mutation was applied. For INCREMENT_IF/DECREMENT_IF + // this is the guard result; for SET_NX/COMPARE_AND_SET/COMPARE_AND_DELETE it + // reports whether the write/delete took effect. For unguarded ops always + // true on success. On a failed COMPARE_AND_SET/COMPARE_AND_DELETE, `value` + // carries the live stored value so callers can observe the current holder. + Applied bool `protobuf:"varint,7,opt,name=applied,proto3" json:"applied,omitempty"` + // LIST pagination. next_cursor is an opaque token to pass as + // KVOperation.cursor for the next page; empty when iteration is complete. + // has_more is true when more matching keys remain beyond this page. + NextCursor string `protobuf:"bytes,8,opt,name=next_cursor,json=nextCursor,proto3" json:"next_cursor,omitempty"` + HasMore bool `protobuf:"varint,9,opt,name=has_more,json=hasMore,proto3" json:"has_more,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -4994,6 +5286,20 @@ func (x *KVResponse) GetApplied() bool { return false } +func (x *KVResponse) GetNextCursor() string { + if x != nil { + return x.NextCursor + } + return "" +} + +func (x *KVResponse) GetHasMore() bool { + if x != nil { + return x.HasMore + } + return false +} + type IncomingMessage struct { state protoimpl.MessageState `protogen:"open.v1"` SourceTopic string `protobuf:"bytes,1,opt,name=source_topic,json=sourceTopic,proto3" json:"source_topic,omitempty"` @@ -5008,9 +5314,16 @@ type IncomingMessage struct { // recover the originating workspace from this field rather than from // source_topic (which carries the sender's identity-topic, not the // declared event/metric workspace). - Workspace string `protobuf:"bytes,4,opt,name=workspace,proto3" json:"workspace,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + Workspace string `protobuf:"bytes,4,opt,name=workspace,proto3" json:"workspace,omitempty"` + // Gateway-set, spoof-proof resolved on-behalf-of subject, mirrored from + // MessageEnvelope.on_behalf_subject at delivery time. Populated ONLY when the + // sender's SendMessage carried an AuthorizationContext the gateway resolved + // to an OBO subject. Lets a recipient identify the *user* a message was sent + // for, distinct from the sending identity in source_topic. Empty for direct + // (non-OBO) sends. See MessageEnvelope.on_behalf_subject. + OnBehalfSubject *PrincipalRef `protobuf:"bytes,5,opt,name=on_behalf_subject,json=onBehalfSubject,proto3" json:"on_behalf_subject,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *IncomingMessage) Reset() { @@ -5071,6 +5384,13 @@ func (x *IncomingMessage) GetWorkspace() string { return "" } +func (x *IncomingMessage) GetOnBehalfSubject() *PrincipalRef { + if x != nil { + return x.OnBehalfSubject + } + return nil +} + type ConfigSnapshot struct { state protoimpl.MessageState `protogen:"open.v1"` // Legacy fields. The server stops auto-populating these as part of the @@ -5289,60 +5609,51 @@ func (x *ErrorResponse) GetRequestId() string { return "" } -type CreateTaskRequest struct { - state protoimpl.MessageState `protogen:"open.v1"` - TaskType string `protobuf:"bytes,1,opt,name=task_type,json=taskType,proto3" json:"task_type,omitempty"` - Workspace string `protobuf:"bytes,2,opt,name=workspace,proto3" json:"workspace,omitempty"` - AssignmentMode TaskAssignmentMode `protobuf:"varint,3,opt,name=assignment_mode,json=assignmentMode,proto3,enum=aether.v1.TaskAssignmentMode" json:"assignment_mode,omitempty"` - // For TARGETED mode only - TargetAgentId string `protobuf:"bytes,4,opt,name=target_agent_id,json=targetAgentId,proto3" json:"target_agent_id,omitempty"` // Full identity string (e.g., "ag.prod.worker.inst-5") - // For targeted tasks that trigger orchestration - LaunchParamOverrides map[string]string `protobuf:"bytes,5,rep,name=launch_param_overrides,json=launchParamOverrides,proto3" json:"launch_param_overrides,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` - Metadata map[string]string `protobuf:"bytes,6,rep,name=metadata,proto3" json:"metadata,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` - // Optional binary payload for task input data (e.g., serialized configs, protobuf work items). - // Subject to server-enforced size limit (default 512KB). - Payload []byte `protobuf:"bytes,7,opt,name=payload,proto3" json:"payload,omitempty"` - // For POOL mode: target agent implementation type (e.g., "data-processor") - TargetImplementation string `protobuf:"bytes,8,opt,name=target_implementation,json=targetImplementation,proto3" json:"target_implementation,omitempty"` - Authorization *AuthorizationContext `protobuf:"bytes,9,opt,name=authorization,proto3" json:"authorization,omitempty"` - // Optional correlation ID. If non-empty, the server will send a - // CreateTaskResponse with this request_id echoed so the creator can - // learn the resulting task_id. Leave empty for fire-and-forget semantics. - RequestId string `protobuf:"bytes,10,opt,name=request_id,json=requestId,proto3" json:"request_id,omitempty"` - // Optional. Identity string ("::::") - // that the task's spawned worker will register as. When non-empty AND the - // server can mint a task token (creator has Manage on the workspace, the - // identity's workspace matches `workspace`, and the workspace is not on - // the platform-blocklist), the gateway returns a fresh single-use task - // token in CreateTaskResponse.task_token. The worker presents that token - // at connection init to authenticate AS this identity. Leave empty when - // the worker uses some other auth mechanism (mTLS-only, pre-issued API - // key, etc.) and no per-task token is wanted. - TargetIdentity string `protobuf:"bytes,11,opt,name=target_identity,json=targetIdentity,proto3" json:"target_identity,omitempty"` - // Optional UI hint; defaults to UNSPECIFIED ⇒ INTERACTIVE. - TaskClass TaskClass `protobuf:"varint,12,opt,name=task_class,json=taskClass,proto3,enum=aether.v1.TaskClass" json:"task_class,omitempty"` - // Client-minted opaque session identifier (A2A contextId, Phase 1). - // Persisted on Task.context_id and queryable via TaskFilter.context_id. - // Empty = no session grouping. - ContextId string `protobuf:"bytes,13,opt,name=context_id,json=contextId,proto3" json:"context_id,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache +// RetryPolicy lifts retry scheduling out of individual workers and into the +// task store. When a task carries a RetryPolicy and a worker calls FailTask +// without an explicit reschedule, the store computes next_retry_at from +// this policy and re-pends the task. The waker then picks it up at the +// scheduled time. Workers can still call RescheduleTaskAt to override +// (e.g., to honor a Retry-After header). +type RetryPolicy struct { + state protoimpl.MessageState `protogen:"open.v1"` + // Total attempts allowed (1 = no retries). 0 means use server default (3). + MaxAttempts int32 `protobuf:"varint,1,opt,name=max_attempts,json=maxAttempts,proto3" json:"max_attempts,omitempty"` + Backoff BackoffStrategy `protobuf:"varint,2,opt,name=backoff,proto3,enum=aether.v1.BackoffStrategy" json:"backoff,omitempty"` + InitialDelayMs int64 `protobuf:"varint,3,opt,name=initial_delay_ms,json=initialDelayMs,proto3" json:"initial_delay_ms,omitempty"` + MaxDelayMs int64 `protobuf:"varint,4,opt,name=max_delay_ms,json=maxDelayMs,proto3" json:"max_delay_ms,omitempty"` + // Multiplicative random jitter in [0,1]. Final delay = computed * + // (1 + uniform(-jitter, +jitter)). + JitterFactor float64 `protobuf:"fixed64,5,opt,name=jitter_factor,json=jitterFactor,proto3" json:"jitter_factor,omitempty"` + // For BACKOFF_STRATEGY_EXPLICIT_SCHEDULE. Indexed by 0-based attempt + // number; the final entry is used for any subsequent attempt. + ScheduleMs []int64 `protobuf:"varint,6,rep,packed,name=schedule_ms,json=scheduleMs,proto3" json:"schedule_ms,omitempty"` + // Optional worker convention: HTTP-style status codes that should + // trigger a retry. The task store does not inspect failure context; + // workers use this to decide whether to FailTask or CompleteTask with + // permanent-failure semantics. + RetryableStatusCodes []int32 `protobuf:"varint,7,rep,packed,name=retryable_status_codes,json=retryableStatusCodes,proto3" json:"retryable_status_codes,omitempty"` + // Worker hint: prefer Retry-After (or analogous) over computed delay + // when set on the failure response. + HonorRetryAfter bool `protobuf:"varint,8,opt,name=honor_retry_after,json=honorRetryAfter,proto3" json:"honor_retry_after,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } -func (x *CreateTaskRequest) Reset() { - *x = CreateTaskRequest{} +func (x *RetryPolicy) Reset() { + *x = RetryPolicy{} mi := &file_aether_proto_msgTypes[29] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } -func (x *CreateTaskRequest) String() string { +func (x *RetryPolicy) String() string { return protoimpl.X.MessageStringOf(x) } -func (*CreateTaskRequest) ProtoMessage() {} +func (*RetryPolicy) ProtoMessage() {} -func (x *CreateTaskRequest) ProtoReflect() protoreflect.Message { +func (x *RetryPolicy) ProtoReflect() protoreflect.Message { mi := &file_aether_proto_msgTypes[29] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) @@ -5354,11 +5665,233 @@ func (x *CreateTaskRequest) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use CreateTaskRequest.ProtoReflect.Descriptor instead. -func (*CreateTaskRequest) Descriptor() ([]byte, []int) { +// Deprecated: Use RetryPolicy.ProtoReflect.Descriptor instead. +func (*RetryPolicy) Descriptor() ([]byte, []int) { return file_aether_proto_rawDescGZIP(), []int{29} } +func (x *RetryPolicy) GetMaxAttempts() int32 { + if x != nil { + return x.MaxAttempts + } + return 0 +} + +func (x *RetryPolicy) GetBackoff() BackoffStrategy { + if x != nil { + return x.Backoff + } + return BackoffStrategy_BACKOFF_STRATEGY_UNSPECIFIED +} + +func (x *RetryPolicy) GetInitialDelayMs() int64 { + if x != nil { + return x.InitialDelayMs + } + return 0 +} + +func (x *RetryPolicy) GetMaxDelayMs() int64 { + if x != nil { + return x.MaxDelayMs + } + return 0 +} + +func (x *RetryPolicy) GetJitterFactor() float64 { + if x != nil { + return x.JitterFactor + } + return 0 +} + +func (x *RetryPolicy) GetScheduleMs() []int64 { + if x != nil { + return x.ScheduleMs + } + return nil +} + +func (x *RetryPolicy) GetRetryableStatusCodes() []int32 { + if x != nil { + return x.RetryableStatusCodes + } + return nil +} + +func (x *RetryPolicy) GetHonorRetryAfter() bool { + if x != nil { + return x.HonorRetryAfter + } + return false +} + +// TaskCompletionEvent opts a task into "feed B": when it reaches a terminal +// status the server publishes a domain event onto the event plane (event::*) +// so a workflow join (or any rule) can gather over task completions without the +// worker emitting its own event. The emitted payload carries +// {task_id, status, workspace, correlation_id, metadata}. +type TaskCompletionEvent struct { + state protoimpl.MessageState `protogen:"open.v1"` + // enabled turns the feature on. When false (default) no completion event is + // emitted (today's behavior). + Enabled bool `protobuf:"varint,1,opt,name=enabled,proto3" json:"enabled,omitempty"` + // event_name is the event name to publish. Empty ⇒ derived as + // "task.completed" / "task.failed" / "task.cancelled" from the terminal status. + EventName string `protobuf:"bytes,2,opt,name=event_name,json=eventName,proto3" json:"event_name,omitempty"` + // on_statuses restricts emission to these terminal statuses. Empty ⇒ all + // terminal statuses (completed, failed, cancelled). + OnStatuses []TaskStatus `protobuf:"varint,3,rep,packed,name=on_statuses,json=onStatuses,proto3,enum=aether.v1.TaskStatus" json:"on_statuses,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *TaskCompletionEvent) Reset() { + *x = TaskCompletionEvent{} + mi := &file_aether_proto_msgTypes[30] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *TaskCompletionEvent) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*TaskCompletionEvent) ProtoMessage() {} + +func (x *TaskCompletionEvent) ProtoReflect() protoreflect.Message { + mi := &file_aether_proto_msgTypes[30] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use TaskCompletionEvent.ProtoReflect.Descriptor instead. +func (*TaskCompletionEvent) Descriptor() ([]byte, []int) { + return file_aether_proto_rawDescGZIP(), []int{30} +} + +func (x *TaskCompletionEvent) GetEnabled() bool { + if x != nil { + return x.Enabled + } + return false +} + +func (x *TaskCompletionEvent) GetEventName() string { + if x != nil { + return x.EventName + } + return "" +} + +func (x *TaskCompletionEvent) GetOnStatuses() []TaskStatus { + if x != nil { + return x.OnStatuses + } + return nil +} + +type CreateTaskRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + TaskType string `protobuf:"bytes,1,opt,name=task_type,json=taskType,proto3" json:"task_type,omitempty"` + Workspace string `protobuf:"bytes,2,opt,name=workspace,proto3" json:"workspace,omitempty"` + AssignmentMode TaskAssignmentMode `protobuf:"varint,3,opt,name=assignment_mode,json=assignmentMode,proto3,enum=aether.v1.TaskAssignmentMode" json:"assignment_mode,omitempty"` + // For TARGETED mode only + TargetAgentId string `protobuf:"bytes,4,opt,name=target_agent_id,json=targetAgentId,proto3" json:"target_agent_id,omitempty"` // Full identity string (e.g., "ag.prod.worker.inst-5") + // For targeted tasks that trigger orchestration + LaunchParamOverrides map[string]string `protobuf:"bytes,5,rep,name=launch_param_overrides,json=launchParamOverrides,proto3" json:"launch_param_overrides,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` + Metadata map[string]string `protobuf:"bytes,6,rep,name=metadata,proto3" json:"metadata,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` + // Optional binary payload for task input data (e.g., serialized configs, protobuf work items). + // Subject to server-enforced size limit (default 512KB). + Payload []byte `protobuf:"bytes,7,opt,name=payload,proto3" json:"payload,omitempty"` + // For POOL mode: target agent implementation type (e.g., "data-processor") + TargetImplementation string `protobuf:"bytes,8,opt,name=target_implementation,json=targetImplementation,proto3" json:"target_implementation,omitempty"` + Authorization *AuthorizationContext `protobuf:"bytes,9,opt,name=authorization,proto3" json:"authorization,omitempty"` + // Optional correlation ID. If non-empty, the server will send a + // CreateTaskResponse with this request_id echoed so the creator can + // learn the resulting task_id. Leave empty for fire-and-forget semantics. + RequestId string `protobuf:"bytes,10,opt,name=request_id,json=requestId,proto3" json:"request_id,omitempty"` + // Optional. Identity string ("::::") + // that the task's spawned worker will register as. When non-empty AND the + // server can mint a task token (creator has Manage on the workspace, the + // identity's workspace matches `workspace`, and the workspace is not on + // the platform-blocklist), the gateway returns a fresh single-use task + // token in CreateTaskResponse.task_token. The worker presents that token + // at connection init to authenticate AS this identity. Leave empty when + // the worker uses some other auth mechanism (mTLS-only, pre-issued API + // key, etc.) and no per-task token is wanted. + TargetIdentity string `protobuf:"bytes,11,opt,name=target_identity,json=targetIdentity,proto3" json:"target_identity,omitempty"` + // Optional UI hint; defaults to UNSPECIFIED ⇒ INTERACTIVE. + TaskClass TaskClass `protobuf:"varint,12,opt,name=task_class,json=taskClass,proto3,enum=aether.v1.TaskClass" json:"task_class,omitempty"` + // Client-minted opaque session identifier (A2A contextId, Phase 1). + // Persisted on Task.context_id and queryable via TaskFilter.context_id. + // Empty = no session grouping. + ContextId string `protobuf:"bytes,13,opt,name=context_id,json=contextId,proto3" json:"context_id,omitempty"` + // Optional. When set, the task store computes next_retry_at on FailTask + // according to this policy and re-pends the task automatically (up to + // max_attempts). Absent = legacy behavior (immediate re-pend, hardcoded + // max_retries=3). + RetryPolicy *RetryPolicy `protobuf:"bytes,14,opt,name=retry_policy,json=retryPolicy,proto3" json:"retry_policy,omitempty"` + // Optional dispatch priority. Defaults to UNSPECIFIED ⇒ NORMAL. Higher + // priority pending tasks are delivered before lower ones (ties break FIFO). + Priority TaskPriority `protobuf:"varint,15,opt,name=priority,proto3,enum=aether.v1.TaskPriority" json:"priority,omitempty"` + // Optional idempotency key for exactly-once task creation. When non-empty the + // gateway dedupes creation on this key: a duplicate request returns the + // existing task identity instead of creating a second task. Workflow joins + // set it to make an on_complete create_task fire exactly once under retries + // or engine restart. + IdempotencyKey string `protobuf:"bytes,16,opt,name=idempotency_key,json=idempotencyKey,proto3" json:"idempotency_key,omitempty"` + // Optional fan-out/fan-in correlation identity, distinct from task_id. When + // non-empty it is persisted on the task and propagated to child spawns, and is + // queryable via TaskFilter.correlation_id. The barrier/group id a join matches. + CorrelationId string `protobuf:"bytes,17,opt,name=correlation_id,json=correlationId,proto3" json:"correlation_id,omitempty"` + // Optional top-of-fan-out-tree identity (the flow / run id). Defaults to the + // task's own id when it is a root; inherited by descendants on spawn. Becomes + // the DAG run id when the join engine grows into full fan-out/fan-in. + RootTaskId string `protobuf:"bytes,18,opt,name=root_task_id,json=rootTaskId,proto3" json:"root_task_id,omitempty"` + // Optional "feed B" config: emit a domain event onto event::* when this task + // reaches a (selected) terminal status. Absent/disabled = no emission. + CompletionEvent *TaskCompletionEvent `protobuf:"bytes,19,opt,name=completion_event,json=completionEvent,proto3" json:"completion_event,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *CreateTaskRequest) Reset() { + *x = CreateTaskRequest{} + mi := &file_aether_proto_msgTypes[31] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *CreateTaskRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*CreateTaskRequest) ProtoMessage() {} + +func (x *CreateTaskRequest) ProtoReflect() protoreflect.Message { + mi := &file_aether_proto_msgTypes[31] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use CreateTaskRequest.ProtoReflect.Descriptor instead. +func (*CreateTaskRequest) Descriptor() ([]byte, []int) { + return file_aether_proto_rawDescGZIP(), []int{31} +} + func (x *CreateTaskRequest) GetTaskType() string { if x != nil { return x.TaskType @@ -5450,6 +5983,48 @@ func (x *CreateTaskRequest) GetContextId() string { return "" } +func (x *CreateTaskRequest) GetRetryPolicy() *RetryPolicy { + if x != nil { + return x.RetryPolicy + } + return nil +} + +func (x *CreateTaskRequest) GetPriority() TaskPriority { + if x != nil { + return x.Priority + } + return TaskPriority_TASK_PRIORITY_UNSPECIFIED +} + +func (x *CreateTaskRequest) GetIdempotencyKey() string { + if x != nil { + return x.IdempotencyKey + } + return "" +} + +func (x *CreateTaskRequest) GetCorrelationId() string { + if x != nil { + return x.CorrelationId + } + return "" +} + +func (x *CreateTaskRequest) GetRootTaskId() string { + if x != nil { + return x.RootTaskId + } + return "" +} + +func (x *CreateTaskRequest) GetCompletionEvent() *TaskCompletionEvent { + if x != nil { + return x.CompletionEvent + } + return nil +} + // CreateTaskResponse is sent in response to CreateTaskRequest when the // request carries a non-empty request_id. Gives the creator the server- // assigned task_id so it can later COMPLETE/FAIL/CANCEL the task. @@ -5494,7 +6069,7 @@ type CreateTaskResponse struct { func (x *CreateTaskResponse) Reset() { *x = CreateTaskResponse{} - mi := &file_aether_proto_msgTypes[30] + mi := &file_aether_proto_msgTypes[32] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -5506,7 +6081,7 @@ func (x *CreateTaskResponse) String() string { func (*CreateTaskResponse) ProtoMessage() {} func (x *CreateTaskResponse) ProtoReflect() protoreflect.Message { - mi := &file_aether_proto_msgTypes[30] + mi := &file_aether_proto_msgTypes[32] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -5519,7 +6094,7 @@ func (x *CreateTaskResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use CreateTaskResponse.ProtoReflect.Descriptor instead. func (*CreateTaskResponse) Descriptor() ([]byte, []int) { - return file_aether_proto_rawDescGZIP(), []int{30} + return file_aether_proto_rawDescGZIP(), []int{32} } func (x *CreateTaskResponse) GetSuccess() bool { @@ -5614,7 +6189,7 @@ type TaskAssignment struct { func (x *TaskAssignment) Reset() { *x = TaskAssignment{} - mi := &file_aether_proto_msgTypes[31] + mi := &file_aether_proto_msgTypes[33] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -5626,7 +6201,7 @@ func (x *TaskAssignment) String() string { func (*TaskAssignment) ProtoMessage() {} func (x *TaskAssignment) ProtoReflect() protoreflect.Message { - mi := &file_aether_proto_msgTypes[31] + mi := &file_aether_proto_msgTypes[33] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -5639,7 +6214,7 @@ func (x *TaskAssignment) ProtoReflect() protoreflect.Message { // Deprecated: Use TaskAssignment.ProtoReflect.Descriptor instead. func (*TaskAssignment) Descriptor() ([]byte, []int) { - return file_aether_proto_rawDescGZIP(), []int{31} + return file_aether_proto_rawDescGZIP(), []int{33} } func (x *TaskAssignment) GetTaskId() string { @@ -5765,7 +6340,7 @@ type CheckpointOperation struct { func (x *CheckpointOperation) Reset() { *x = CheckpointOperation{} - mi := &file_aether_proto_msgTypes[32] + mi := &file_aether_proto_msgTypes[34] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -5777,7 +6352,7 @@ func (x *CheckpointOperation) String() string { func (*CheckpointOperation) ProtoMessage() {} func (x *CheckpointOperation) ProtoReflect() protoreflect.Message { - mi := &file_aether_proto_msgTypes[32] + mi := &file_aether_proto_msgTypes[34] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -5790,7 +6365,7 @@ func (x *CheckpointOperation) ProtoReflect() protoreflect.Message { // Deprecated: Use CheckpointOperation.ProtoReflect.Descriptor instead. func (*CheckpointOperation) Descriptor() ([]byte, []int) { - return file_aether_proto_rawDescGZIP(), []int{32} + return file_aether_proto_rawDescGZIP(), []int{34} } func (x *CheckpointOperation) GetOp() CheckpointOperation_OpType { @@ -5848,7 +6423,7 @@ type CheckpointResponse struct { func (x *CheckpointResponse) Reset() { *x = CheckpointResponse{} - mi := &file_aether_proto_msgTypes[33] + mi := &file_aether_proto_msgTypes[35] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -5860,7 +6435,7 @@ func (x *CheckpointResponse) String() string { func (*CheckpointResponse) ProtoMessage() {} func (x *CheckpointResponse) ProtoReflect() protoreflect.Message { - mi := &file_aether_proto_msgTypes[33] + mi := &file_aether_proto_msgTypes[35] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -5873,7 +6448,7 @@ func (x *CheckpointResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use CheckpointResponse.ProtoReflect.Descriptor instead. func (*CheckpointResponse) Descriptor() ([]byte, []int) { - return file_aether_proto_rawDescGZIP(), []int{33} + return file_aether_proto_rawDescGZIP(), []int{35} } func (x *CheckpointResponse) GetSuccess() bool { @@ -5936,7 +6511,7 @@ type AdminQuery struct { func (x *AdminQuery) Reset() { *x = AdminQuery{} - mi := &file_aether_proto_msgTypes[34] + mi := &file_aether_proto_msgTypes[36] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -5948,7 +6523,7 @@ func (x *AdminQuery) String() string { func (*AdminQuery) ProtoMessage() {} func (x *AdminQuery) ProtoReflect() protoreflect.Message { - mi := &file_aether_proto_msgTypes[34] + mi := &file_aether_proto_msgTypes[36] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -5961,7 +6536,7 @@ func (x *AdminQuery) ProtoReflect() protoreflect.Message { // Deprecated: Use AdminQuery.ProtoReflect.Descriptor instead. func (*AdminQuery) Descriptor() ([]byte, []int) { - return file_aether_proto_rawDescGZIP(), []int{34} + return file_aether_proto_rawDescGZIP(), []int{36} } func (x *AdminQuery) GetOp() AdminQuery_OpType { @@ -6006,7 +6581,7 @@ type ConnectionFilter struct { func (x *ConnectionFilter) Reset() { *x = ConnectionFilter{} - mi := &file_aether_proto_msgTypes[35] + mi := &file_aether_proto_msgTypes[37] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -6018,7 +6593,7 @@ func (x *ConnectionFilter) String() string { func (*ConnectionFilter) ProtoMessage() {} func (x *ConnectionFilter) ProtoReflect() protoreflect.Message { - mi := &file_aether_proto_msgTypes[35] + mi := &file_aether_proto_msgTypes[37] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -6031,7 +6606,7 @@ func (x *ConnectionFilter) ProtoReflect() protoreflect.Message { // Deprecated: Use ConnectionFilter.ProtoReflect.Descriptor instead. func (*ConnectionFilter) Descriptor() ([]byte, []int) { - return file_aether_proto_rawDescGZIP(), []int{35} + return file_aether_proto_rawDescGZIP(), []int{37} } func (x *ConnectionFilter) GetType() PrincipalType { @@ -6082,7 +6657,7 @@ type ConnectionInfo struct { func (x *ConnectionInfo) Reset() { *x = ConnectionInfo{} - mi := &file_aether_proto_msgTypes[36] + mi := &file_aether_proto_msgTypes[38] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -6094,7 +6669,7 @@ func (x *ConnectionInfo) String() string { func (*ConnectionInfo) ProtoMessage() {} func (x *ConnectionInfo) ProtoReflect() protoreflect.Message { - mi := &file_aether_proto_msgTypes[36] + mi := &file_aether_proto_msgTypes[38] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -6107,7 +6682,7 @@ func (x *ConnectionInfo) ProtoReflect() protoreflect.Message { // Deprecated: Use ConnectionInfo.ProtoReflect.Descriptor instead. func (*ConnectionInfo) Descriptor() ([]byte, []int) { - return file_aether_proto_rawDescGZIP(), []int{36} + return file_aether_proto_rawDescGZIP(), []int{38} } func (x *ConnectionInfo) GetSessionId() string { @@ -6208,7 +6783,7 @@ type AdminResponse struct { func (x *AdminResponse) Reset() { *x = AdminResponse{} - mi := &file_aether_proto_msgTypes[37] + mi := &file_aether_proto_msgTypes[39] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -6220,7 +6795,7 @@ func (x *AdminResponse) String() string { func (*AdminResponse) ProtoMessage() {} func (x *AdminResponse) ProtoReflect() protoreflect.Message { - mi := &file_aether_proto_msgTypes[37] + mi := &file_aether_proto_msgTypes[39] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -6233,7 +6808,7 @@ func (x *AdminResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use AdminResponse.ProtoReflect.Descriptor instead. func (*AdminResponse) Descriptor() ([]byte, []int) { - return file_aether_proto_rawDescGZIP(), []int{37} + return file_aether_proto_rawDescGZIP(), []int{39} } func (x *AdminResponse) GetSuccess() bool { @@ -6313,7 +6888,7 @@ type HealthInfo struct { func (x *HealthInfo) Reset() { *x = HealthInfo{} - mi := &file_aether_proto_msgTypes[38] + mi := &file_aether_proto_msgTypes[40] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -6325,7 +6900,7 @@ func (x *HealthInfo) String() string { func (*HealthInfo) ProtoMessage() {} func (x *HealthInfo) ProtoReflect() protoreflect.Message { - mi := &file_aether_proto_msgTypes[38] + mi := &file_aether_proto_msgTypes[40] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -6338,7 +6913,7 @@ func (x *HealthInfo) ProtoReflect() protoreflect.Message { // Deprecated: Use HealthInfo.ProtoReflect.Descriptor instead. func (*HealthInfo) Descriptor() ([]byte, []int) { - return file_aether_proto_rawDescGZIP(), []int{38} + return file_aether_proto_rawDescGZIP(), []int{40} } func (x *HealthInfo) GetStatus() HealthStatus { @@ -6382,7 +6957,7 @@ type HealthCheck struct { func (x *HealthCheck) Reset() { *x = HealthCheck{} - mi := &file_aether_proto_msgTypes[39] + mi := &file_aether_proto_msgTypes[41] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -6394,7 +6969,7 @@ func (x *HealthCheck) String() string { func (*HealthCheck) ProtoMessage() {} func (x *HealthCheck) ProtoReflect() protoreflect.Message { - mi := &file_aether_proto_msgTypes[39] + mi := &file_aether_proto_msgTypes[41] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -6407,7 +6982,7 @@ func (x *HealthCheck) ProtoReflect() protoreflect.Message { // Deprecated: Use HealthCheck.ProtoReflect.Descriptor instead. func (*HealthCheck) Descriptor() ([]byte, []int) { - return file_aether_proto_rawDescGZIP(), []int{39} + return file_aether_proto_rawDescGZIP(), []int{41} } func (x *HealthCheck) GetStatus() HealthCheckStatus { @@ -6449,7 +7024,7 @@ type GatewayInfo struct { func (x *GatewayInfo) Reset() { *x = GatewayInfo{} - mi := &file_aether_proto_msgTypes[40] + mi := &file_aether_proto_msgTypes[42] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -6461,7 +7036,7 @@ func (x *GatewayInfo) String() string { func (*GatewayInfo) ProtoMessage() {} func (x *GatewayInfo) ProtoReflect() protoreflect.Message { - mi := &file_aether_proto_msgTypes[40] + mi := &file_aether_proto_msgTypes[42] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -6474,7 +7049,7 @@ func (x *GatewayInfo) ProtoReflect() protoreflect.Message { // Deprecated: Use GatewayInfo.ProtoReflect.Descriptor instead. func (*GatewayInfo) Descriptor() ([]byte, []int) { - return file_aether_proto_rawDescGZIP(), []int{40} + return file_aether_proto_rawDescGZIP(), []int{42} } func (x *GatewayInfo) GetGatewayId() string { @@ -6562,7 +7137,7 @@ type GatewayStats struct { func (x *GatewayStats) Reset() { *x = GatewayStats{} - mi := &file_aether_proto_msgTypes[41] + mi := &file_aether_proto_msgTypes[43] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -6574,7 +7149,7 @@ func (x *GatewayStats) String() string { func (*GatewayStats) ProtoMessage() {} func (x *GatewayStats) ProtoReflect() protoreflect.Message { - mi := &file_aether_proto_msgTypes[41] + mi := &file_aether_proto_msgTypes[43] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -6587,7 +7162,7 @@ func (x *GatewayStats) ProtoReflect() protoreflect.Message { // Deprecated: Use GatewayStats.ProtoReflect.Descriptor instead. func (*GatewayStats) Descriptor() ([]byte, []int) { - return file_aether_proto_rawDescGZIP(), []int{41} + return file_aether_proto_rawDescGZIP(), []int{43} } func (x *GatewayStats) GetAgentConnections() int32 { @@ -6724,7 +7299,7 @@ type SessionOperation struct { func (x *SessionOperation) Reset() { *x = SessionOperation{} - mi := &file_aether_proto_msgTypes[42] + mi := &file_aether_proto_msgTypes[44] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -6736,7 +7311,7 @@ func (x *SessionOperation) String() string { func (*SessionOperation) ProtoMessage() {} func (x *SessionOperation) ProtoReflect() protoreflect.Message { - mi := &file_aether_proto_msgTypes[42] + mi := &file_aether_proto_msgTypes[44] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -6749,7 +7324,7 @@ func (x *SessionOperation) ProtoReflect() protoreflect.Message { // Deprecated: Use SessionOperation.ProtoReflect.Descriptor instead. func (*SessionOperation) Descriptor() ([]byte, []int) { - return file_aether_proto_rawDescGZIP(), []int{42} + return file_aether_proto_rawDescGZIP(), []int{44} } func (x *SessionOperation) GetOp() SessionOperation_OpType { @@ -6816,7 +7391,7 @@ type SessionOperationResponse struct { func (x *SessionOperationResponse) Reset() { *x = SessionOperationResponse{} - mi := &file_aether_proto_msgTypes[43] + mi := &file_aether_proto_msgTypes[45] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -6828,7 +7403,7 @@ func (x *SessionOperationResponse) String() string { func (*SessionOperationResponse) ProtoMessage() {} func (x *SessionOperationResponse) ProtoReflect() protoreflect.Message { - mi := &file_aether_proto_msgTypes[43] + mi := &file_aether_proto_msgTypes[45] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -6841,7 +7416,7 @@ func (x *SessionOperationResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use SessionOperationResponse.ProtoReflect.Descriptor instead. func (*SessionOperationResponse) Descriptor() ([]byte, []int) { - return file_aether_proto_rawDescGZIP(), []int{43} + return file_aether_proto_rawDescGZIP(), []int{45} } func (x *SessionOperationResponse) GetSuccess() bool { @@ -6912,7 +7487,7 @@ type TaskQuery struct { func (x *TaskQuery) Reset() { *x = TaskQuery{} - mi := &file_aether_proto_msgTypes[44] + mi := &file_aether_proto_msgTypes[46] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -6924,7 +7499,7 @@ func (x *TaskQuery) String() string { func (*TaskQuery) ProtoMessage() {} func (x *TaskQuery) ProtoReflect() protoreflect.Message { - mi := &file_aether_proto_msgTypes[44] + mi := &file_aether_proto_msgTypes[46] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -6937,7 +7512,7 @@ func (x *TaskQuery) ProtoReflect() protoreflect.Message { // Deprecated: Use TaskQuery.ProtoReflect.Descriptor instead. func (*TaskQuery) Descriptor() ([]byte, []int) { - return file_aether_proto_rawDescGZIP(), []int{44} + return file_aether_proto_rawDescGZIP(), []int{46} } func (x *TaskQuery) GetOp() TaskQuery_OpType { @@ -7019,13 +7594,23 @@ type TaskFilter struct { // not just direct children. Default false: direct children only, preserving // the existing parent_task_id behavior. IncludeDescendants bool `protobuf:"varint,20,opt,name=include_descendants,json=includeDescendants,proto3" json:"include_descendants,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + // Exact dispatch-priority filter. UNSPECIFIED (0) = no filter; otherwise + // returns only tasks whose stored priority equals this level. + Priority TaskPriority `protobuf:"varint,21,opt,name=priority,proto3,enum=aether.v1.TaskPriority" json:"priority,omitempty"` + // Minimum dispatch-priority threshold. UNSPECIFIED (0) = no filter; + // otherwise returns tasks whose priority is >= this level (e.g. pass HIGH + // to get everything HIGH and PREEMPT). Combinable with other filters. + MinPriority TaskPriority `protobuf:"varint,22,opt,name=min_priority,json=minPriority,proto3,enum=aether.v1.TaskPriority" json:"min_priority,omitempty"` + // Filter by fan-out/fan-in correlation identity / flow id. Empty = no filter. + CorrelationId string `protobuf:"bytes,23,opt,name=correlation_id,json=correlationId,proto3" json:"correlation_id,omitempty"` + RootTaskId string `protobuf:"bytes,24,opt,name=root_task_id,json=rootTaskId,proto3" json:"root_task_id,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *TaskFilter) Reset() { *x = TaskFilter{} - mi := &file_aether_proto_msgTypes[45] + mi := &file_aether_proto_msgTypes[47] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -7037,7 +7622,7 @@ func (x *TaskFilter) String() string { func (*TaskFilter) ProtoMessage() {} func (x *TaskFilter) ProtoReflect() protoreflect.Message { - mi := &file_aether_proto_msgTypes[45] + mi := &file_aether_proto_msgTypes[47] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -7050,7 +7635,7 @@ func (x *TaskFilter) ProtoReflect() protoreflect.Message { // Deprecated: Use TaskFilter.ProtoReflect.Descriptor instead. func (*TaskFilter) Descriptor() ([]byte, []int) { - return file_aether_proto_rawDescGZIP(), []int{45} + return file_aether_proto_rawDescGZIP(), []int{47} } func (x *TaskFilter) GetStatus() TaskStatus { @@ -7193,6 +7778,34 @@ func (x *TaskFilter) GetIncludeDescendants() bool { return false } +func (x *TaskFilter) GetPriority() TaskPriority { + if x != nil { + return x.Priority + } + return TaskPriority_TASK_PRIORITY_UNSPECIFIED +} + +func (x *TaskFilter) GetMinPriority() TaskPriority { + if x != nil { + return x.MinPriority + } + return TaskPriority_TASK_PRIORITY_UNSPECIFIED +} + +func (x *TaskFilter) GetCorrelationId() string { + if x != nil { + return x.CorrelationId + } + return "" +} + +func (x *TaskFilter) GetRootTaskId() string { + if x != nil { + return x.RootTaskId + } + return "" +} + // TaskInfo represents a task. // Matches the TaskInfo struct in state_provider.go. type TaskInfo struct { @@ -7243,14 +7856,26 @@ type TaskInfo struct { ContextId string `protobuf:"bytes,29,opt,name=context_id,json=contextId,proto3" json:"context_id,omitempty"` // Unix seconds when the task entered a WAITING_* / HIBERNATED state. // 0 means the task has never been paused. - PausedAt int64 `protobuf:"varint,30,opt,name=paused_at,json=pausedAt,proto3" json:"paused_at,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + PausedAt int64 `protobuf:"varint,30,opt,name=paused_at,json=pausedAt,proto3" json:"paused_at,omitempty"` + // Dispatch priority. UNSPECIFIED is normalized to NORMAL on creation, so + // persisted tasks always report a concrete level. + Priority TaskPriority `protobuf:"varint,31,opt,name=priority,proto3,enum=aether.v1.TaskPriority" json:"priority,omitempty"` + // Fan-out/fan-in correlation identity (the barrier/group id a join matches), + // distinct from task_id and propagated to child spawns. Empty = none. + CorrelationId string `protobuf:"bytes,32,opt,name=correlation_id,json=correlationId,proto3" json:"correlation_id,omitempty"` + // Top-of-fan-out-tree identity (flow / run id); defaults to the task's own id + // for a root, inherited by descendants. Empty = none. + RootTaskId string `protobuf:"bytes,33,opt,name=root_task_id,json=rootTaskId,proto3" json:"root_task_id,omitempty"` + // "Feed B" config persisted on the task; read at the terminal transition to + // decide whether/what to emit onto event::*. + CompletionEvent *TaskCompletionEvent `protobuf:"bytes,34,opt,name=completion_event,json=completionEvent,proto3" json:"completion_event,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *TaskInfo) Reset() { *x = TaskInfo{} - mi := &file_aether_proto_msgTypes[46] + mi := &file_aether_proto_msgTypes[48] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -7262,7 +7887,7 @@ func (x *TaskInfo) String() string { func (*TaskInfo) ProtoMessage() {} func (x *TaskInfo) ProtoReflect() protoreflect.Message { - mi := &file_aether_proto_msgTypes[46] + mi := &file_aether_proto_msgTypes[48] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -7275,7 +7900,7 @@ func (x *TaskInfo) ProtoReflect() protoreflect.Message { // Deprecated: Use TaskInfo.ProtoReflect.Descriptor instead. func (*TaskInfo) Descriptor() ([]byte, []int) { - return file_aether_proto_rawDescGZIP(), []int{46} + return file_aether_proto_rawDescGZIP(), []int{48} } func (x *TaskInfo) GetTaskId() string { @@ -7488,6 +8113,34 @@ func (x *TaskInfo) GetPausedAt() int64 { return 0 } +func (x *TaskInfo) GetPriority() TaskPriority { + if x != nil { + return x.Priority + } + return TaskPriority_TASK_PRIORITY_UNSPECIFIED +} + +func (x *TaskInfo) GetCorrelationId() string { + if x != nil { + return x.CorrelationId + } + return "" +} + +func (x *TaskInfo) GetRootTaskId() string { + if x != nil { + return x.RootTaskId + } + return "" +} + +func (x *TaskInfo) GetCompletionEvent() *TaskCompletionEvent { + if x != nil { + return x.CompletionEvent + } + return nil +} + // TaskQueryResponse is sent in response to TaskQuery operations. // Contains task data for either GET (single task) or LIST (multiple tasks). type TaskQueryResponse struct { @@ -7513,7 +8166,7 @@ type TaskQueryResponse struct { func (x *TaskQueryResponse) Reset() { *x = TaskQueryResponse{} - mi := &file_aether_proto_msgTypes[47] + mi := &file_aether_proto_msgTypes[49] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -7525,7 +8178,7 @@ func (x *TaskQueryResponse) String() string { func (*TaskQueryResponse) ProtoMessage() {} func (x *TaskQueryResponse) ProtoReflect() protoreflect.Message { - mi := &file_aether_proto_msgTypes[47] + mi := &file_aether_proto_msgTypes[49] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -7538,7 +8191,7 @@ func (x *TaskQueryResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use TaskQueryResponse.ProtoReflect.Descriptor instead. func (*TaskQueryResponse) Descriptor() ([]byte, []int) { - return file_aether_proto_rawDescGZIP(), []int{47} + return file_aether_proto_rawDescGZIP(), []int{49} } func (x *TaskQueryResponse) GetSuccess() bool { @@ -7616,7 +8269,7 @@ type TaskOperation struct { func (x *TaskOperation) Reset() { *x = TaskOperation{} - mi := &file_aether_proto_msgTypes[48] + mi := &file_aether_proto_msgTypes[50] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -7628,7 +8281,7 @@ func (x *TaskOperation) String() string { func (*TaskOperation) ProtoMessage() {} func (x *TaskOperation) ProtoReflect() protoreflect.Message { - mi := &file_aether_proto_msgTypes[48] + mi := &file_aether_proto_msgTypes[50] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -7641,7 +8294,7 @@ func (x *TaskOperation) ProtoReflect() protoreflect.Message { // Deprecated: Use TaskOperation.ProtoReflect.Descriptor instead. func (*TaskOperation) Descriptor() ([]byte, []int) { - return file_aether_proto_rawDescGZIP(), []int{48} + return file_aether_proto_rawDescGZIP(), []int{50} } func (x *TaskOperation) GetOp() TaskOperation_OpType { @@ -7720,7 +8373,7 @@ type WaitSpec struct { func (x *WaitSpec) Reset() { *x = WaitSpec{} - mi := &file_aether_proto_msgTypes[49] + mi := &file_aether_proto_msgTypes[51] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -7732,7 +8385,7 @@ func (x *WaitSpec) String() string { func (*WaitSpec) ProtoMessage() {} func (x *WaitSpec) ProtoReflect() protoreflect.Message { - mi := &file_aether_proto_msgTypes[49] + mi := &file_aether_proto_msgTypes[51] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -7745,7 +8398,7 @@ func (x *WaitSpec) ProtoReflect() protoreflect.Message { // Deprecated: Use WaitSpec.ProtoReflect.Descriptor instead. func (*WaitSpec) Descriptor() ([]byte, []int) { - return file_aether_proto_rawDescGZIP(), []int{49} + return file_aether_proto_rawDescGZIP(), []int{51} } func (x *WaitSpec) GetReason() WaitReason { @@ -7837,7 +8490,7 @@ type HibernationDescriptor struct { func (x *HibernationDescriptor) Reset() { *x = HibernationDescriptor{} - mi := &file_aether_proto_msgTypes[50] + mi := &file_aether_proto_msgTypes[52] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -7849,7 +8502,7 @@ func (x *HibernationDescriptor) String() string { func (*HibernationDescriptor) ProtoMessage() {} func (x *HibernationDescriptor) ProtoReflect() protoreflect.Message { - mi := &file_aether_proto_msgTypes[50] + mi := &file_aether_proto_msgTypes[52] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -7862,7 +8515,7 @@ func (x *HibernationDescriptor) ProtoReflect() protoreflect.Message { // Deprecated: Use HibernationDescriptor.ProtoReflect.Descriptor instead. func (*HibernationDescriptor) Descriptor() ([]byte, []int) { - return file_aether_proto_rawDescGZIP(), []int{50} + return file_aether_proto_rawDescGZIP(), []int{52} } func (x *HibernationDescriptor) GetCheckpointKey() string { @@ -7911,7 +8564,7 @@ type TaskOperationResponse struct { func (x *TaskOperationResponse) Reset() { *x = TaskOperationResponse{} - mi := &file_aether_proto_msgTypes[51] + mi := &file_aether_proto_msgTypes[53] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -7923,7 +8576,7 @@ func (x *TaskOperationResponse) String() string { func (*TaskOperationResponse) ProtoMessage() {} func (x *TaskOperationResponse) ProtoReflect() protoreflect.Message { - mi := &file_aether_proto_msgTypes[51] + mi := &file_aether_proto_msgTypes[53] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -7936,7 +8589,7 @@ func (x *TaskOperationResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use TaskOperationResponse.ProtoReflect.Descriptor instead. func (*TaskOperationResponse) Descriptor() ([]byte, []int) { - return file_aether_proto_rawDescGZIP(), []int{51} + return file_aether_proto_rawDescGZIP(), []int{53} } func (x *TaskOperationResponse) GetSuccess() bool { @@ -8001,7 +8654,7 @@ type WorkspaceOperation struct { func (x *WorkspaceOperation) Reset() { *x = WorkspaceOperation{} - mi := &file_aether_proto_msgTypes[52] + mi := &file_aether_proto_msgTypes[54] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -8013,7 +8666,7 @@ func (x *WorkspaceOperation) String() string { func (*WorkspaceOperation) ProtoMessage() {} func (x *WorkspaceOperation) ProtoReflect() protoreflect.Message { - mi := &file_aether_proto_msgTypes[52] + mi := &file_aether_proto_msgTypes[54] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -8026,7 +8679,7 @@ func (x *WorkspaceOperation) ProtoReflect() protoreflect.Message { // Deprecated: Use WorkspaceOperation.ProtoReflect.Descriptor instead. func (*WorkspaceOperation) Descriptor() ([]byte, []int) { - return file_aether_proto_rawDescGZIP(), []int{52} + return file_aether_proto_rawDescGZIP(), []int{54} } func (x *WorkspaceOperation) GetOp() WorkspaceOperation_OpType { @@ -8077,7 +8730,7 @@ type WorkspaceFilter struct { func (x *WorkspaceFilter) Reset() { *x = WorkspaceFilter{} - mi := &file_aether_proto_msgTypes[53] + mi := &file_aether_proto_msgTypes[55] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -8089,7 +8742,7 @@ func (x *WorkspaceFilter) String() string { func (*WorkspaceFilter) ProtoMessage() {} func (x *WorkspaceFilter) ProtoReflect() protoreflect.Message { - mi := &file_aether_proto_msgTypes[53] + mi := &file_aether_proto_msgTypes[55] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -8102,7 +8755,7 @@ func (x *WorkspaceFilter) ProtoReflect() protoreflect.Message { // Deprecated: Use WorkspaceFilter.ProtoReflect.Descriptor instead. func (*WorkspaceFilter) Descriptor() ([]byte, []int) { - return file_aether_proto_rawDescGZIP(), []int{53} + return file_aether_proto_rawDescGZIP(), []int{55} } func (x *WorkspaceFilter) GetTenantId() string { @@ -8148,7 +8801,7 @@ type WorkspaceInfo struct { func (x *WorkspaceInfo) Reset() { *x = WorkspaceInfo{} - mi := &file_aether_proto_msgTypes[54] + mi := &file_aether_proto_msgTypes[56] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -8160,7 +8813,7 @@ func (x *WorkspaceInfo) String() string { func (*WorkspaceInfo) ProtoMessage() {} func (x *WorkspaceInfo) ProtoReflect() protoreflect.Message { - mi := &file_aether_proto_msgTypes[54] + mi := &file_aether_proto_msgTypes[56] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -8173,7 +8826,7 @@ func (x *WorkspaceInfo) ProtoReflect() protoreflect.Message { // Deprecated: Use WorkspaceInfo.ProtoReflect.Descriptor instead. func (*WorkspaceInfo) Descriptor() ([]byte, []int) { - return file_aether_proto_rawDescGZIP(), []int{54} + return file_aether_proto_rawDescGZIP(), []int{56} } func (x *WorkspaceInfo) GetWorkspaceId() string { @@ -8278,7 +8931,7 @@ type WorkspaceResponse struct { func (x *WorkspaceResponse) Reset() { *x = WorkspaceResponse{} - mi := &file_aether_proto_msgTypes[55] + mi := &file_aether_proto_msgTypes[57] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -8290,7 +8943,7 @@ func (x *WorkspaceResponse) String() string { func (*WorkspaceResponse) ProtoMessage() {} func (x *WorkspaceResponse) ProtoReflect() protoreflect.Message { - mi := &file_aether_proto_msgTypes[55] + mi := &file_aether_proto_msgTypes[57] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -8303,7 +8956,7 @@ func (x *WorkspaceResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use WorkspaceResponse.ProtoReflect.Descriptor instead. func (*WorkspaceResponse) Descriptor() ([]byte, []int) { - return file_aether_proto_rawDescGZIP(), []int{55} + return file_aether_proto_rawDescGZIP(), []int{57} } func (x *WorkspaceResponse) GetSuccess() bool { @@ -8377,7 +9030,7 @@ type MessageFlowInfo struct { func (x *MessageFlowInfo) Reset() { *x = MessageFlowInfo{} - mi := &file_aether_proto_msgTypes[56] + mi := &file_aether_proto_msgTypes[58] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -8389,7 +9042,7 @@ func (x *MessageFlowInfo) String() string { func (*MessageFlowInfo) ProtoMessage() {} func (x *MessageFlowInfo) ProtoReflect() protoreflect.Message { - mi := &file_aether_proto_msgTypes[56] + mi := &file_aether_proto_msgTypes[58] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -8402,7 +9055,7 @@ func (x *MessageFlowInfo) ProtoReflect() protoreflect.Message { // Deprecated: Use MessageFlowInfo.ProtoReflect.Descriptor instead. func (*MessageFlowInfo) Descriptor() ([]byte, []int) { - return file_aether_proto_rawDescGZIP(), []int{56} + return file_aether_proto_rawDescGZIP(), []int{58} } func (x *MessageFlowInfo) GetWorkspaceId() string { @@ -8450,7 +9103,7 @@ type FlowNode struct { func (x *FlowNode) Reset() { *x = FlowNode{} - mi := &file_aether_proto_msgTypes[57] + mi := &file_aether_proto_msgTypes[59] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -8462,7 +9115,7 @@ func (x *FlowNode) String() string { func (*FlowNode) ProtoMessage() {} func (x *FlowNode) ProtoReflect() protoreflect.Message { - mi := &file_aether_proto_msgTypes[57] + mi := &file_aether_proto_msgTypes[59] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -8475,7 +9128,7 @@ func (x *FlowNode) ProtoReflect() protoreflect.Message { // Deprecated: Use FlowNode.ProtoReflect.Descriptor instead. func (*FlowNode) Descriptor() ([]byte, []int) { - return file_aether_proto_rawDescGZIP(), []int{57} + return file_aether_proto_rawDescGZIP(), []int{59} } func (x *FlowNode) GetId() string { @@ -8541,7 +9194,7 @@ type FlowEdge struct { func (x *FlowEdge) Reset() { *x = FlowEdge{} - mi := &file_aether_proto_msgTypes[58] + mi := &file_aether_proto_msgTypes[60] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -8553,7 +9206,7 @@ func (x *FlowEdge) String() string { func (*FlowEdge) ProtoMessage() {} func (x *FlowEdge) ProtoReflect() protoreflect.Message { - mi := &file_aether_proto_msgTypes[58] + mi := &file_aether_proto_msgTypes[60] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -8566,7 +9219,7 @@ func (x *FlowEdge) ProtoReflect() protoreflect.Message { // Deprecated: Use FlowEdge.ProtoReflect.Descriptor instead. func (*FlowEdge) Descriptor() ([]byte, []int) { - return file_aether_proto_rawDescGZIP(), []int{58} + return file_aether_proto_rawDescGZIP(), []int{60} } func (x *FlowEdge) GetFrom() string { @@ -8627,7 +9280,7 @@ type AgentOperation struct { func (x *AgentOperation) Reset() { *x = AgentOperation{} - mi := &file_aether_proto_msgTypes[59] + mi := &file_aether_proto_msgTypes[61] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -8639,7 +9292,7 @@ func (x *AgentOperation) String() string { func (*AgentOperation) ProtoMessage() {} func (x *AgentOperation) ProtoReflect() protoreflect.Message { - mi := &file_aether_proto_msgTypes[59] + mi := &file_aether_proto_msgTypes[61] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -8652,7 +9305,7 @@ func (x *AgentOperation) ProtoReflect() protoreflect.Message { // Deprecated: Use AgentOperation.ProtoReflect.Descriptor instead. func (*AgentOperation) Descriptor() ([]byte, []int) { - return file_aether_proto_rawDescGZIP(), []int{59} + return file_aether_proto_rawDescGZIP(), []int{61} } func (x *AgentOperation) GetOp() AgentOperation_OpType { @@ -8709,7 +9362,7 @@ type AgentFilter struct { func (x *AgentFilter) Reset() { *x = AgentFilter{} - mi := &file_aether_proto_msgTypes[60] + mi := &file_aether_proto_msgTypes[62] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -8721,7 +9374,7 @@ func (x *AgentFilter) String() string { func (*AgentFilter) ProtoMessage() {} func (x *AgentFilter) ProtoReflect() protoreflect.Message { - mi := &file_aether_proto_msgTypes[60] + mi := &file_aether_proto_msgTypes[62] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -8734,7 +9387,7 @@ func (x *AgentFilter) ProtoReflect() protoreflect.Message { // Deprecated: Use AgentFilter.ProtoReflect.Descriptor instead. func (*AgentFilter) Descriptor() ([]byte, []int) { - return file_aether_proto_rawDescGZIP(), []int{60} + return file_aether_proto_rawDescGZIP(), []int{62} } func (x *AgentFilter) GetOrchestratorProfile() string { @@ -8790,7 +9443,7 @@ type AgentRegistrationInfo struct { func (x *AgentRegistrationInfo) Reset() { *x = AgentRegistrationInfo{} - mi := &file_aether_proto_msgTypes[61] + mi := &file_aether_proto_msgTypes[63] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -8802,7 +9455,7 @@ func (x *AgentRegistrationInfo) String() string { func (*AgentRegistrationInfo) ProtoMessage() {} func (x *AgentRegistrationInfo) ProtoReflect() protoreflect.Message { - mi := &file_aether_proto_msgTypes[61] + mi := &file_aether_proto_msgTypes[63] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -8815,7 +9468,7 @@ func (x *AgentRegistrationInfo) ProtoReflect() protoreflect.Message { // Deprecated: Use AgentRegistrationInfo.ProtoReflect.Descriptor instead. func (*AgentRegistrationInfo) Descriptor() ([]byte, []int) { - return file_aether_proto_rawDescGZIP(), []int{61} + return file_aether_proto_rawDescGZIP(), []int{63} } func (x *AgentRegistrationInfo) GetImplementation() string { @@ -8902,7 +9555,7 @@ type AgentResourceSchemaEntry struct { func (x *AgentResourceSchemaEntry) Reset() { *x = AgentResourceSchemaEntry{} - mi := &file_aether_proto_msgTypes[62] + mi := &file_aether_proto_msgTypes[64] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -8914,7 +9567,7 @@ func (x *AgentResourceSchemaEntry) String() string { func (*AgentResourceSchemaEntry) ProtoMessage() {} func (x *AgentResourceSchemaEntry) ProtoReflect() protoreflect.Message { - mi := &file_aether_proto_msgTypes[62] + mi := &file_aether_proto_msgTypes[64] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -8927,7 +9580,7 @@ func (x *AgentResourceSchemaEntry) ProtoReflect() protoreflect.Message { // Deprecated: Use AgentResourceSchemaEntry.ProtoReflect.Descriptor instead. func (*AgentResourceSchemaEntry) Descriptor() ([]byte, []int) { - return file_aether_proto_rawDescGZIP(), []int{62} + return file_aether_proto_rawDescGZIP(), []int{64} } func (x *AgentResourceSchemaEntry) GetResourceTypePrefix() string { @@ -8964,7 +9617,7 @@ type AgentLaunchParams struct { func (x *AgentLaunchParams) Reset() { *x = AgentLaunchParams{} - mi := &file_aether_proto_msgTypes[63] + mi := &file_aether_proto_msgTypes[65] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -8976,7 +9629,7 @@ func (x *AgentLaunchParams) String() string { func (*AgentLaunchParams) ProtoMessage() {} func (x *AgentLaunchParams) ProtoReflect() protoreflect.Message { - mi := &file_aether_proto_msgTypes[63] + mi := &file_aether_proto_msgTypes[65] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -8989,7 +9642,7 @@ func (x *AgentLaunchParams) ProtoReflect() protoreflect.Message { // Deprecated: Use AgentLaunchParams.ProtoReflect.Descriptor instead. func (*AgentLaunchParams) Descriptor() ([]byte, []int) { - return file_aether_proto_rawDescGZIP(), []int{63} + return file_aether_proto_rawDescGZIP(), []int{65} } func (x *AgentLaunchParams) GetSpecifier() string { @@ -9026,7 +9679,7 @@ type OrchestratorInfo struct { func (x *OrchestratorInfo) Reset() { *x = OrchestratorInfo{} - mi := &file_aether_proto_msgTypes[64] + mi := &file_aether_proto_msgTypes[66] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -9038,7 +9691,7 @@ func (x *OrchestratorInfo) String() string { func (*OrchestratorInfo) ProtoMessage() {} func (x *OrchestratorInfo) ProtoReflect() protoreflect.Message { - mi := &file_aether_proto_msgTypes[64] + mi := &file_aether_proto_msgTypes[66] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -9051,7 +9704,7 @@ func (x *OrchestratorInfo) ProtoReflect() protoreflect.Message { // Deprecated: Use OrchestratorInfo.ProtoReflect.Descriptor instead. func (*OrchestratorInfo) Descriptor() ([]byte, []int) { - return file_aether_proto_rawDescGZIP(), []int{64} + return file_aether_proto_rawDescGZIP(), []int{66} } func (x *OrchestratorInfo) GetOrchestratorId() string { @@ -9087,7 +9740,7 @@ type AgentLaunchResult struct { func (x *AgentLaunchResult) Reset() { *x = AgentLaunchResult{} - mi := &file_aether_proto_msgTypes[65] + mi := &file_aether_proto_msgTypes[67] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -9099,7 +9752,7 @@ func (x *AgentLaunchResult) String() string { func (*AgentLaunchResult) ProtoMessage() {} func (x *AgentLaunchResult) ProtoReflect() protoreflect.Message { - mi := &file_aether_proto_msgTypes[65] + mi := &file_aether_proto_msgTypes[67] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -9112,7 +9765,7 @@ func (x *AgentLaunchResult) ProtoReflect() protoreflect.Message { // Deprecated: Use AgentLaunchResult.ProtoReflect.Descriptor instead. func (*AgentLaunchResult) Descriptor() ([]byte, []int) { - return file_aether_proto_rawDescGZIP(), []int{65} + return file_aether_proto_rawDescGZIP(), []int{67} } func (x *AgentLaunchResult) GetTaskId() string { @@ -9156,7 +9809,7 @@ type AgentResponse struct { func (x *AgentResponse) Reset() { *x = AgentResponse{} - mi := &file_aether_proto_msgTypes[66] + mi := &file_aether_proto_msgTypes[68] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -9168,7 +9821,7 @@ func (x *AgentResponse) String() string { func (*AgentResponse) ProtoMessage() {} func (x *AgentResponse) ProtoReflect() protoreflect.Message { - mi := &file_aether_proto_msgTypes[66] + mi := &file_aether_proto_msgTypes[68] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -9181,7 +9834,7 @@ func (x *AgentResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use AgentResponse.ProtoReflect.Descriptor instead. func (*AgentResponse) Descriptor() ([]byte, []int) { - return file_aether_proto_rawDescGZIP(), []int{66} + return file_aether_proto_rawDescGZIP(), []int{68} } func (x *AgentResponse) GetSuccess() bool { @@ -9283,14 +9936,34 @@ type ACLOperation struct { // For SET_FALLBACK_POLICY: fallback policy update data (added in subtask-6-2) FallbackRequest *ACLSetFallbackRequest `protobuf:"bytes,14,opt,name=fallback_request,json=fallbackRequest,proto3" json:"fallback_request,omitempty"` // Client-generated correlation ID for matching responses to requests - RequestId string `protobuf:"bytes,19,opt,name=request_id,json=requestId,proto3" json:"request_id,omitempty"` + RequestId string `protobuf:"bytes,19,opt,name=request_id,json=requestId,proto3" json:"request_id,omitempty"` + // Role/group fields. + // name: group/role name for GET/DELETE/LIST_*_MEMBERS/ASSIGNMENTS/ADD/ASSIGN. + Name string `protobuf:"bytes,20,opt,name=name,proto3" json:"name,omitempty"` + // principal: member/assignee for REMOVE_GROUP_MEMBER, UNASSIGN_ROLE, and + // the subject for LIST_PRINCIPAL_GROUPS / LIST_PRINCIPAL_ROLES. + Principal *PrincipalRef `protobuf:"bytes,21,opt,name=principal,proto3" json:"principal,omitempty"` + GroupRequest *ACLGroupRequest `protobuf:"bytes,22,opt,name=group_request,json=groupRequest,proto3" json:"group_request,omitempty"` // CREATE_GROUP + RoleRequest *ACLRoleRequest `protobuf:"bytes,23,opt,name=role_request,json=roleRequest,proto3" json:"role_request,omitempty"` // CREATE_ROLE + MemberRequest *ACLGroupMemberRequest `protobuf:"bytes,24,opt,name=member_request,json=memberRequest,proto3" json:"member_request,omitempty"` // ADD_GROUP_MEMBER + AssignmentRequest *ACLRoleAssignmentRequest `protobuf:"bytes,25,opt,name=assignment_request,json=assignmentRequest,proto3" json:"assignment_request,omitempty"` // ASSIGN_ROLE + // EXPLAIN_ACCESS: the resource to explain against (principal carries the + // subject; required_level is the threshold the decision is compared to). + ResourceType string `protobuf:"bytes,26,opt,name=resource_type,json=resourceType,proto3" json:"resource_type,omitempty"` + ResourceId string `protobuf:"bytes,27,opt,name=resource_id,json=resourceId,proto3" json:"resource_id,omitempty"` + RequiredLevel int32 `protobuf:"varint,28,opt,name=required_level,json=requiredLevel,proto3" json:"required_level,omitempty"` + // Optional on-behalf-of authority context. When set, the gateway runs the + // admin ACL check against the subject (the user) rather than the actor + // (the platform-server). Mirrors SessionOperation.authorization (field 6) + // and AuditQuery.authorization (field 18). + Authorization *AuthorizationContext `protobuf:"bytes,29,opt,name=authorization,proto3" json:"authorization,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } func (x *ACLOperation) Reset() { *x = ACLOperation{} - mi := &file_aether_proto_msgTypes[67] + mi := &file_aether_proto_msgTypes[69] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -9302,7 +9975,7 @@ func (x *ACLOperation) String() string { func (*ACLOperation) ProtoMessage() {} func (x *ACLOperation) ProtoReflect() protoreflect.Message { - mi := &file_aether_proto_msgTypes[67] + mi := &file_aether_proto_msgTypes[69] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -9315,7 +9988,7 @@ func (x *ACLOperation) ProtoReflect() protoreflect.Message { // Deprecated: Use ACLOperation.ProtoReflect.Descriptor instead. func (*ACLOperation) Descriptor() ([]byte, []int) { - return file_aether_proto_rawDescGZIP(), []int{67} + return file_aether_proto_rawDescGZIP(), []int{69} } func (x *ACLOperation) GetOp() ACLOperation_OpType { @@ -9381,81 +10054,151 @@ func (x *ACLOperation) GetRequestId() string { return "" } -// ACLRuleFilter specifies filtering parameters for listing ACL rules. -// Matches RuleFilter struct in internal/acl/service.go. -type ACLRuleFilter struct { - state protoimpl.MessageState `protogen:"open.v1"` - PrincipalType string `protobuf:"bytes,1,opt,name=principal_type,json=principalType,proto3" json:"principal_type,omitempty"` // Filter by principal type (e.g., "agent", "task", "user") - PrincipalId string `protobuf:"bytes,2,opt,name=principal_id,json=principalId,proto3" json:"principal_id,omitempty"` // Filter by principal ID - ResourceType string `protobuf:"bytes,3,opt,name=resource_type,json=resourceType,proto3" json:"resource_type,omitempty"` // Filter by resource type (e.g., "workspace", "agent") - ResourceId string `protobuf:"bytes,4,opt,name=resource_id,json=resourceId,proto3" json:"resource_id,omitempty"` // Filter by resource ID - Limit int32 `protobuf:"varint,5,opt,name=limit,proto3" json:"limit,omitempty"` // Maximum number of results (0 = no limit) - Offset int32 `protobuf:"varint,6,opt,name=offset,proto3" json:"offset,omitempty"` // Offset for pagination - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache -} - -func (x *ACLRuleFilter) Reset() { - *x = ACLRuleFilter{} - mi := &file_aether_proto_msgTypes[68] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) +func (x *ACLOperation) GetName() string { + if x != nil { + return x.Name + } + return "" } -func (x *ACLRuleFilter) String() string { - return protoimpl.X.MessageStringOf(x) +func (x *ACLOperation) GetPrincipal() *PrincipalRef { + if x != nil { + return x.Principal + } + return nil } -func (*ACLRuleFilter) ProtoMessage() {} - -func (x *ACLRuleFilter) ProtoReflect() protoreflect.Message { - mi := &file_aether_proto_msgTypes[68] +func (x *ACLOperation) GetGroupRequest() *ACLGroupRequest { if x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms + return x.GroupRequest } - return mi.MessageOf(x) + return nil } -// Deprecated: Use ACLRuleFilter.ProtoReflect.Descriptor instead. -func (*ACLRuleFilter) Descriptor() ([]byte, []int) { - return file_aether_proto_rawDescGZIP(), []int{68} +func (x *ACLOperation) GetRoleRequest() *ACLRoleRequest { + if x != nil { + return x.RoleRequest + } + return nil } -func (x *ACLRuleFilter) GetPrincipalType() string { +func (x *ACLOperation) GetMemberRequest() *ACLGroupMemberRequest { if x != nil { - return x.PrincipalType + return x.MemberRequest } - return "" + return nil } -func (x *ACLRuleFilter) GetPrincipalId() string { +func (x *ACLOperation) GetAssignmentRequest() *ACLRoleAssignmentRequest { if x != nil { - return x.PrincipalId + return x.AssignmentRequest } - return "" + return nil } -func (x *ACLRuleFilter) GetResourceType() string { +func (x *ACLOperation) GetResourceType() string { if x != nil { return x.ResourceType } return "" } -func (x *ACLRuleFilter) GetResourceId() string { +func (x *ACLOperation) GetResourceId() string { if x != nil { return x.ResourceId } return "" } -func (x *ACLRuleFilter) GetLimit() int32 { +func (x *ACLOperation) GetRequiredLevel() int32 { if x != nil { - return x.Limit + return x.RequiredLevel + } + return 0 +} + +func (x *ACLOperation) GetAuthorization() *AuthorizationContext { + if x != nil { + return x.Authorization + } + return nil +} + +// ACLRuleFilter specifies filtering parameters for listing ACL rules. +// Matches RuleFilter struct in internal/acl/service.go. +type ACLRuleFilter struct { + state protoimpl.MessageState `protogen:"open.v1"` + PrincipalType string `protobuf:"bytes,1,opt,name=principal_type,json=principalType,proto3" json:"principal_type,omitempty"` // Filter by principal type (e.g., "agent", "task", "user") + PrincipalId string `protobuf:"bytes,2,opt,name=principal_id,json=principalId,proto3" json:"principal_id,omitempty"` // Filter by principal ID + ResourceType string `protobuf:"bytes,3,opt,name=resource_type,json=resourceType,proto3" json:"resource_type,omitempty"` // Filter by resource type (e.g., "workspace", "agent") + ResourceId string `protobuf:"bytes,4,opt,name=resource_id,json=resourceId,proto3" json:"resource_id,omitempty"` // Filter by resource ID + Limit int32 `protobuf:"varint,5,opt,name=limit,proto3" json:"limit,omitempty"` // Maximum number of results (0 = no limit) + Offset int32 `protobuf:"varint,6,opt,name=offset,proto3" json:"offset,omitempty"` // Offset for pagination + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ACLRuleFilter) Reset() { + *x = ACLRuleFilter{} + mi := &file_aether_proto_msgTypes[70] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ACLRuleFilter) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ACLRuleFilter) ProtoMessage() {} + +func (x *ACLRuleFilter) ProtoReflect() protoreflect.Message { + mi := &file_aether_proto_msgTypes[70] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ACLRuleFilter.ProtoReflect.Descriptor instead. +func (*ACLRuleFilter) Descriptor() ([]byte, []int) { + return file_aether_proto_rawDescGZIP(), []int{70} +} + +func (x *ACLRuleFilter) GetPrincipalType() string { + if x != nil { + return x.PrincipalType + } + return "" +} + +func (x *ACLRuleFilter) GetPrincipalId() string { + if x != nil { + return x.PrincipalId + } + return "" +} + +func (x *ACLRuleFilter) GetResourceType() string { + if x != nil { + return x.ResourceType + } + return "" +} + +func (x *ACLRuleFilter) GetResourceId() string { + if x != nil { + return x.ResourceId + } + return "" +} + +func (x *ACLRuleFilter) GetLimit() int32 { + if x != nil { + return x.Limit } return 0 } @@ -9487,7 +10230,7 @@ type ACLAuditFilter struct { func (x *ACLAuditFilter) Reset() { *x = ACLAuditFilter{} - mi := &file_aether_proto_msgTypes[69] + mi := &file_aether_proto_msgTypes[71] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -9499,7 +10242,7 @@ func (x *ACLAuditFilter) String() string { func (*ACLAuditFilter) ProtoMessage() {} func (x *ACLAuditFilter) ProtoReflect() protoreflect.Message { - mi := &file_aether_proto_msgTypes[69] + mi := &file_aether_proto_msgTypes[71] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -9512,7 +10255,7 @@ func (x *ACLAuditFilter) ProtoReflect() protoreflect.Message { // Deprecated: Use ACLAuditFilter.ProtoReflect.Descriptor instead. func (*ACLAuditFilter) Descriptor() ([]byte, []int) { - return file_aether_proto_rawDescGZIP(), []int{69} + return file_aether_proto_rawDescGZIP(), []int{71} } func (x *ACLAuditFilter) GetStartTime() int64 { @@ -9603,7 +10346,7 @@ type ACLGrantRequest struct { func (x *ACLGrantRequest) Reset() { *x = ACLGrantRequest{} - mi := &file_aether_proto_msgTypes[70] + mi := &file_aether_proto_msgTypes[72] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -9615,7 +10358,7 @@ func (x *ACLGrantRequest) String() string { func (*ACLGrantRequest) ProtoMessage() {} func (x *ACLGrantRequest) ProtoReflect() protoreflect.Message { - mi := &file_aether_proto_msgTypes[70] + mi := &file_aether_proto_msgTypes[72] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -9628,7 +10371,7 @@ func (x *ACLGrantRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use ACLGrantRequest.ProtoReflect.Descriptor instead. func (*ACLGrantRequest) Descriptor() ([]byte, []int) { - return file_aether_proto_rawDescGZIP(), []int{70} + return file_aether_proto_rawDescGZIP(), []int{72} } func (x *ACLGrantRequest) GetPrincipalType() string { @@ -9700,7 +10443,7 @@ type ACLSetFallbackRequest struct { func (x *ACLSetFallbackRequest) Reset() { *x = ACLSetFallbackRequest{} - mi := &file_aether_proto_msgTypes[71] + mi := &file_aether_proto_msgTypes[73] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -9712,7 +10455,7 @@ func (x *ACLSetFallbackRequest) String() string { func (*ACLSetFallbackRequest) ProtoMessage() {} func (x *ACLSetFallbackRequest) ProtoReflect() protoreflect.Message { - mi := &file_aether_proto_msgTypes[71] + mi := &file_aether_proto_msgTypes[73] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -9725,7 +10468,7 @@ func (x *ACLSetFallbackRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use ACLSetFallbackRequest.ProtoReflect.Descriptor instead. func (*ACLSetFallbackRequest) Descriptor() ([]byte, []int) { - return file_aether_proto_rawDescGZIP(), []int{71} + return file_aether_proto_rawDescGZIP(), []int{73} } func (x *ACLSetFallbackRequest) GetRuleCategory() string { @@ -9768,7 +10511,7 @@ type ACLAuthorityGrantFilter struct { func (x *ACLAuthorityGrantFilter) Reset() { *x = ACLAuthorityGrantFilter{} - mi := &file_aether_proto_msgTypes[72] + mi := &file_aether_proto_msgTypes[74] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -9780,7 +10523,7 @@ func (x *ACLAuthorityGrantFilter) String() string { func (*ACLAuthorityGrantFilter) ProtoMessage() {} func (x *ACLAuthorityGrantFilter) ProtoReflect() protoreflect.Message { - mi := &file_aether_proto_msgTypes[72] + mi := &file_aether_proto_msgTypes[74] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -9793,7 +10536,7 @@ func (x *ACLAuthorityGrantFilter) ProtoReflect() protoreflect.Message { // Deprecated: Use ACLAuthorityGrantFilter.ProtoReflect.Descriptor instead. func (*ACLAuthorityGrantFilter) Descriptor() ([]byte, []int) { - return file_aether_proto_rawDescGZIP(), []int{72} + return file_aether_proto_rawDescGZIP(), []int{74} } func (x *ACLAuthorityGrantFilter) GetRootGrantId() string { @@ -9883,7 +10626,7 @@ type ACLAuthorityGrantResourceScopeEntry struct { func (x *ACLAuthorityGrantResourceScopeEntry) Reset() { *x = ACLAuthorityGrantResourceScopeEntry{} - mi := &file_aether_proto_msgTypes[73] + mi := &file_aether_proto_msgTypes[75] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -9895,7 +10638,7 @@ func (x *ACLAuthorityGrantResourceScopeEntry) String() string { func (*ACLAuthorityGrantResourceScopeEntry) ProtoMessage() {} func (x *ACLAuthorityGrantResourceScopeEntry) ProtoReflect() protoreflect.Message { - mi := &file_aether_proto_msgTypes[73] + mi := &file_aether_proto_msgTypes[75] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -9908,7 +10651,7 @@ func (x *ACLAuthorityGrantResourceScopeEntry) ProtoReflect() protoreflect.Messag // Deprecated: Use ACLAuthorityGrantResourceScopeEntry.ProtoReflect.Descriptor instead. func (*ACLAuthorityGrantResourceScopeEntry) Descriptor() ([]byte, []int) { - return file_aether_proto_rawDescGZIP(), []int{73} + return file_aether_proto_rawDescGZIP(), []int{75} } func (x *ACLAuthorityGrantResourceScopeEntry) GetResourceType() string { @@ -9951,7 +10694,7 @@ type ACLAuthorityGrantRequest struct { func (x *ACLAuthorityGrantRequest) Reset() { *x = ACLAuthorityGrantRequest{} - mi := &file_aether_proto_msgTypes[74] + mi := &file_aether_proto_msgTypes[76] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -9963,7 +10706,7 @@ func (x *ACLAuthorityGrantRequest) String() string { func (*ACLAuthorityGrantRequest) ProtoMessage() {} func (x *ACLAuthorityGrantRequest) ProtoReflect() protoreflect.Message { - mi := &file_aether_proto_msgTypes[74] + mi := &file_aether_proto_msgTypes[76] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -9976,7 +10719,7 @@ func (x *ACLAuthorityGrantRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use ACLAuthorityGrantRequest.ProtoReflect.Descriptor instead. func (*ACLAuthorityGrantRequest) Descriptor() ([]byte, []int) { - return file_aether_proto_rawDescGZIP(), []int{74} + return file_aether_proto_rawDescGZIP(), []int{76} } func (x *ACLAuthorityGrantRequest) GetSubject() *PrincipalRef { @@ -10120,7 +10863,7 @@ type ACLRenewAuthorityGrantRequest struct { func (x *ACLRenewAuthorityGrantRequest) Reset() { *x = ACLRenewAuthorityGrantRequest{} - mi := &file_aether_proto_msgTypes[75] + mi := &file_aether_proto_msgTypes[77] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -10132,7 +10875,7 @@ func (x *ACLRenewAuthorityGrantRequest) String() string { func (*ACLRenewAuthorityGrantRequest) ProtoMessage() {} func (x *ACLRenewAuthorityGrantRequest) ProtoReflect() protoreflect.Message { - mi := &file_aether_proto_msgTypes[75] + mi := &file_aether_proto_msgTypes[77] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -10145,7 +10888,7 @@ func (x *ACLRenewAuthorityGrantRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use ACLRenewAuthorityGrantRequest.ProtoReflect.Descriptor instead. func (*ACLRenewAuthorityGrantRequest) Descriptor() ([]byte, []int) { - return file_aether_proto_rawDescGZIP(), []int{75} + return file_aether_proto_rawDescGZIP(), []int{77} } func (x *ACLRenewAuthorityGrantRequest) GetGrantId() string { @@ -10190,7 +10933,7 @@ type ACLRuleInfo struct { func (x *ACLRuleInfo) Reset() { *x = ACLRuleInfo{} - mi := &file_aether_proto_msgTypes[76] + mi := &file_aether_proto_msgTypes[78] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -10202,7 +10945,7 @@ func (x *ACLRuleInfo) String() string { func (*ACLRuleInfo) ProtoMessage() {} func (x *ACLRuleInfo) ProtoReflect() protoreflect.Message { - mi := &file_aether_proto_msgTypes[76] + mi := &file_aether_proto_msgTypes[78] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -10215,7 +10958,7 @@ func (x *ACLRuleInfo) ProtoReflect() protoreflect.Message { // Deprecated: Use ACLRuleInfo.ProtoReflect.Descriptor instead. func (*ACLRuleInfo) Descriptor() ([]byte, []int) { - return file_aether_proto_rawDescGZIP(), []int{76} + return file_aether_proto_rawDescGZIP(), []int{78} } func (x *ACLRuleInfo) GetRuleId() string { @@ -10312,7 +11055,7 @@ type ACLFallbackPolicyInfo struct { func (x *ACLFallbackPolicyInfo) Reset() { *x = ACLFallbackPolicyInfo{} - mi := &file_aether_proto_msgTypes[77] + mi := &file_aether_proto_msgTypes[79] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -10324,7 +11067,7 @@ func (x *ACLFallbackPolicyInfo) String() string { func (*ACLFallbackPolicyInfo) ProtoMessage() {} func (x *ACLFallbackPolicyInfo) ProtoReflect() protoreflect.Message { - mi := &file_aether_proto_msgTypes[77] + mi := &file_aether_proto_msgTypes[79] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -10337,7 +11080,7 @@ func (x *ACLFallbackPolicyInfo) ProtoReflect() protoreflect.Message { // Deprecated: Use ACLFallbackPolicyInfo.ProtoReflect.Descriptor instead. func (*ACLFallbackPolicyInfo) Descriptor() ([]byte, []int) { - return file_aether_proto_rawDescGZIP(), []int{77} + return file_aether_proto_rawDescGZIP(), []int{79} } func (x *ACLFallbackPolicyInfo) GetPolicyId() string { @@ -10408,7 +11151,7 @@ type ACLAuditEntryInfo struct { func (x *ACLAuditEntryInfo) Reset() { *x = ACLAuditEntryInfo{} - mi := &file_aether_proto_msgTypes[78] + mi := &file_aether_proto_msgTypes[80] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -10420,7 +11163,7 @@ func (x *ACLAuditEntryInfo) String() string { func (*ACLAuditEntryInfo) ProtoMessage() {} func (x *ACLAuditEntryInfo) ProtoReflect() protoreflect.Message { - mi := &file_aether_proto_msgTypes[78] + mi := &file_aether_proto_msgTypes[80] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -10433,7 +11176,7 @@ func (x *ACLAuditEntryInfo) ProtoReflect() protoreflect.Message { // Deprecated: Use ACLAuditEntryInfo.ProtoReflect.Descriptor instead. func (*ACLAuditEntryInfo) Descriptor() ([]byte, []int) { - return file_aether_proto_rawDescGZIP(), []int{78} + return file_aether_proto_rawDescGZIP(), []int{80} } func (x *ACLAuditEntryInfo) GetAuditId() int64 { @@ -10581,7 +11324,7 @@ type ACLAuthorityGrantInfo struct { func (x *ACLAuthorityGrantInfo) Reset() { *x = ACLAuthorityGrantInfo{} - mi := &file_aether_proto_msgTypes[79] + mi := &file_aether_proto_msgTypes[81] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -10593,7 +11336,7 @@ func (x *ACLAuthorityGrantInfo) String() string { func (*ACLAuthorityGrantInfo) ProtoMessage() {} func (x *ACLAuthorityGrantInfo) ProtoReflect() protoreflect.Message { - mi := &file_aether_proto_msgTypes[79] + mi := &file_aether_proto_msgTypes[81] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -10606,7 +11349,7 @@ func (x *ACLAuthorityGrantInfo) ProtoReflect() protoreflect.Message { // Deprecated: Use ACLAuthorityGrantInfo.ProtoReflect.Descriptor instead. func (*ACLAuthorityGrantInfo) Descriptor() ([]byte, []int) { - return file_aether_proto_rawDescGZIP(), []int{79} + return file_aether_proto_rawDescGZIP(), []int{81} } func (x *ACLAuthorityGrantInfo) GetGrantId() string { @@ -10702,113 +11445,868 @@ func (x *ACLAuthorityGrantInfo) GetMaxAccessLevel() int32 { func (x *ACLAuthorityGrantInfo) GetAccessLevelName() string { if x != nil { - return x.AccessLevelName + return x.AccessLevelName + } + return "" +} + +func (x *ACLAuthorityGrantInfo) GetAudienceType() string { + if x != nil { + return x.AudienceType + } + return "" +} + +func (x *ACLAuthorityGrantInfo) GetAudienceId() string { + if x != nil { + return x.AudienceId + } + return "" +} + +func (x *ACLAuthorityGrantInfo) GetValidWhileAudienceActive() bool { + if x != nil { + return x.ValidWhileAudienceActive + } + return false +} + +func (x *ACLAuthorityGrantInfo) GetExpiresAt() int64 { + if x != nil { + return x.ExpiresAt + } + return 0 +} + +func (x *ACLAuthorityGrantInfo) GetRenewableUntil() int64 { + if x != nil { + return x.RenewableUntil + } + return 0 +} + +func (x *ACLAuthorityGrantInfo) GetRenewedAt() int64 { + if x != nil { + return x.RenewedAt + } + return 0 +} + +func (x *ACLAuthorityGrantInfo) GetRevoked() bool { + if x != nil { + return x.Revoked + } + return false +} + +func (x *ACLAuthorityGrantInfo) GetRevokedAt() int64 { + if x != nil { + return x.RevokedAt + } + return 0 +} + +func (x *ACLAuthorityGrantInfo) GetReason() string { + if x != nil { + return x.Reason + } + return "" +} + +func (x *ACLAuthorityGrantInfo) GetMetadata() map[string]string { + if x != nil { + return x.Metadata + } + return nil +} + +func (x *ACLAuthorityGrantInfo) GetCreatedAt() int64 { + if x != nil { + return x.CreatedAt + } + return 0 +} + +// ACLCleanupResult represents the result of a cleanup operation. +// Used for CLEANUP_EXPIRED and CLEANUP_AUDIT_LOGS responses. +type ACLCleanupResult struct { + state protoimpl.MessageState `protogen:"open.v1"` + DeletedCount int64 `protobuf:"varint,1,opt,name=deleted_count,json=deletedCount,proto3" json:"deleted_count,omitempty"` // Number of items deleted + Message string `protobuf:"bytes,2,opt,name=message,proto3" json:"message,omitempty"` // Human-readable result message + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ACLCleanupResult) Reset() { + *x = ACLCleanupResult{} + mi := &file_aether_proto_msgTypes[82] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ACLCleanupResult) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ACLCleanupResult) ProtoMessage() {} + +func (x *ACLCleanupResult) ProtoReflect() protoreflect.Message { + mi := &file_aether_proto_msgTypes[82] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ACLCleanupResult.ProtoReflect.Descriptor instead. +func (*ACLCleanupResult) Descriptor() ([]byte, []int) { + return file_aether_proto_rawDescGZIP(), []int{82} +} + +func (x *ACLCleanupResult) GetDeletedCount() int64 { + if x != nil { + return x.DeletedCount + } + return 0 +} + +func (x *ACLCleanupResult) GetMessage() string { + if x != nil { + return x.Message + } + return "" +} + +// ACLGroupRequest contains data for creating a group. +type ACLGroupRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` + Description string `protobuf:"bytes,2,opt,name=description,proto3" json:"description,omitempty"` + CreatedBy string `protobuf:"bytes,3,opt,name=created_by,json=createdBy,proto3" json:"created_by,omitempty"` + Metadata map[string]string `protobuf:"bytes,4,rep,name=metadata,proto3" json:"metadata,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ACLGroupRequest) Reset() { + *x = ACLGroupRequest{} + mi := &file_aether_proto_msgTypes[83] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ACLGroupRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ACLGroupRequest) ProtoMessage() {} + +func (x *ACLGroupRequest) ProtoReflect() protoreflect.Message { + mi := &file_aether_proto_msgTypes[83] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ACLGroupRequest.ProtoReflect.Descriptor instead. +func (*ACLGroupRequest) Descriptor() ([]byte, []int) { + return file_aether_proto_rawDescGZIP(), []int{83} +} + +func (x *ACLGroupRequest) GetName() string { + if x != nil { + return x.Name + } + return "" +} + +func (x *ACLGroupRequest) GetDescription() string { + if x != nil { + return x.Description + } + return "" +} + +func (x *ACLGroupRequest) GetCreatedBy() string { + if x != nil { + return x.CreatedBy + } + return "" +} + +func (x *ACLGroupRequest) GetMetadata() map[string]string { + if x != nil { + return x.Metadata + } + return nil +} + +// ACLRoleRequest contains data for creating a role. +type ACLRoleRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` + Description string `protobuf:"bytes,2,opt,name=description,proto3" json:"description,omitempty"` + CreatedBy string `protobuf:"bytes,3,opt,name=created_by,json=createdBy,proto3" json:"created_by,omitempty"` + Metadata map[string]string `protobuf:"bytes,4,rep,name=metadata,proto3" json:"metadata,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ACLRoleRequest) Reset() { + *x = ACLRoleRequest{} + mi := &file_aether_proto_msgTypes[84] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ACLRoleRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ACLRoleRequest) ProtoMessage() {} + +func (x *ACLRoleRequest) ProtoReflect() protoreflect.Message { + mi := &file_aether_proto_msgTypes[84] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ACLRoleRequest.ProtoReflect.Descriptor instead. +func (*ACLRoleRequest) Descriptor() ([]byte, []int) { + return file_aether_proto_rawDescGZIP(), []int{84} +} + +func (x *ACLRoleRequest) GetName() string { + if x != nil { + return x.Name + } + return "" +} + +func (x *ACLRoleRequest) GetDescription() string { + if x != nil { + return x.Description + } + return "" +} + +func (x *ACLRoleRequest) GetCreatedBy() string { + if x != nil { + return x.CreatedBy + } + return "" +} + +func (x *ACLRoleRequest) GetMetadata() map[string]string { + if x != nil { + return x.Metadata + } + return nil +} + +// ACLGroupMemberRequest contains data for adding a member to a group. +type ACLGroupMemberRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + MemberType string `protobuf:"bytes,1,opt,name=member_type,json=memberType,proto3" json:"member_type,omitempty"` // principal type or "group" (nesting) + MemberId string `protobuf:"bytes,2,opt,name=member_id,json=memberId,proto3" json:"member_id,omitempty"` + GrantedBy string `protobuf:"bytes,3,opt,name=granted_by,json=grantedBy,proto3" json:"granted_by,omitempty"` + ExpiresAt int64 `protobuf:"varint,4,opt,name=expires_at,json=expiresAt,proto3" json:"expires_at,omitempty"` // Unix timestamp, 0 = no expiration + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ACLGroupMemberRequest) Reset() { + *x = ACLGroupMemberRequest{} + mi := &file_aether_proto_msgTypes[85] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ACLGroupMemberRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ACLGroupMemberRequest) ProtoMessage() {} + +func (x *ACLGroupMemberRequest) ProtoReflect() protoreflect.Message { + mi := &file_aether_proto_msgTypes[85] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ACLGroupMemberRequest.ProtoReflect.Descriptor instead. +func (*ACLGroupMemberRequest) Descriptor() ([]byte, []int) { + return file_aether_proto_rawDescGZIP(), []int{85} +} + +func (x *ACLGroupMemberRequest) GetMemberType() string { + if x != nil { + return x.MemberType + } + return "" +} + +func (x *ACLGroupMemberRequest) GetMemberId() string { + if x != nil { + return x.MemberId + } + return "" +} + +func (x *ACLGroupMemberRequest) GetGrantedBy() string { + if x != nil { + return x.GrantedBy + } + return "" +} + +func (x *ACLGroupMemberRequest) GetExpiresAt() int64 { + if x != nil { + return x.ExpiresAt + } + return 0 +} + +// ACLRoleAssignmentRequest contains data for assigning a role. +type ACLRoleAssignmentRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + AssigneeType string `protobuf:"bytes,1,opt,name=assignee_type,json=assigneeType,proto3" json:"assignee_type,omitempty"` // principal type or "group" + AssigneeId string `protobuf:"bytes,2,opt,name=assignee_id,json=assigneeId,proto3" json:"assignee_id,omitempty"` + GrantedBy string `protobuf:"bytes,3,opt,name=granted_by,json=grantedBy,proto3" json:"granted_by,omitempty"` + ExpiresAt int64 `protobuf:"varint,4,opt,name=expires_at,json=expiresAt,proto3" json:"expires_at,omitempty"` // Unix timestamp, 0 = no expiration + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ACLRoleAssignmentRequest) Reset() { + *x = ACLRoleAssignmentRequest{} + mi := &file_aether_proto_msgTypes[86] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ACLRoleAssignmentRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ACLRoleAssignmentRequest) ProtoMessage() {} + +func (x *ACLRoleAssignmentRequest) ProtoReflect() protoreflect.Message { + mi := &file_aether_proto_msgTypes[86] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ACLRoleAssignmentRequest.ProtoReflect.Descriptor instead. +func (*ACLRoleAssignmentRequest) Descriptor() ([]byte, []int) { + return file_aether_proto_rawDescGZIP(), []int{86} +} + +func (x *ACLRoleAssignmentRequest) GetAssigneeType() string { + if x != nil { + return x.AssigneeType + } + return "" +} + +func (x *ACLRoleAssignmentRequest) GetAssigneeId() string { + if x != nil { + return x.AssigneeId + } + return "" +} + +func (x *ACLRoleAssignmentRequest) GetGrantedBy() string { + if x != nil { + return x.GrantedBy + } + return "" +} + +func (x *ACLRoleAssignmentRequest) GetExpiresAt() int64 { + if x != nil { + return x.ExpiresAt + } + return 0 +} + +// ACLGroupInfo represents a group definition. +type ACLGroupInfo struct { + state protoimpl.MessageState `protogen:"open.v1"` + GroupId string `protobuf:"bytes,1,opt,name=group_id,json=groupId,proto3" json:"group_id,omitempty"` + GroupName string `protobuf:"bytes,2,opt,name=group_name,json=groupName,proto3" json:"group_name,omitempty"` + Description string `protobuf:"bytes,3,opt,name=description,proto3" json:"description,omitempty"` + CreatedBy string `protobuf:"bytes,4,opt,name=created_by,json=createdBy,proto3" json:"created_by,omitempty"` + CreatedAt int64 `protobuf:"varint,5,opt,name=created_at,json=createdAt,proto3" json:"created_at,omitempty"` + Metadata map[string]string `protobuf:"bytes,6,rep,name=metadata,proto3" json:"metadata,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ACLGroupInfo) Reset() { + *x = ACLGroupInfo{} + mi := &file_aether_proto_msgTypes[87] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ACLGroupInfo) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ACLGroupInfo) ProtoMessage() {} + +func (x *ACLGroupInfo) ProtoReflect() protoreflect.Message { + mi := &file_aether_proto_msgTypes[87] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ACLGroupInfo.ProtoReflect.Descriptor instead. +func (*ACLGroupInfo) Descriptor() ([]byte, []int) { + return file_aether_proto_rawDescGZIP(), []int{87} +} + +func (x *ACLGroupInfo) GetGroupId() string { + if x != nil { + return x.GroupId + } + return "" +} + +func (x *ACLGroupInfo) GetGroupName() string { + if x != nil { + return x.GroupName + } + return "" +} + +func (x *ACLGroupInfo) GetDescription() string { + if x != nil { + return x.Description + } + return "" +} + +func (x *ACLGroupInfo) GetCreatedBy() string { + if x != nil { + return x.CreatedBy + } + return "" +} + +func (x *ACLGroupInfo) GetCreatedAt() int64 { + if x != nil { + return x.CreatedAt + } + return 0 +} + +func (x *ACLGroupInfo) GetMetadata() map[string]string { + if x != nil { + return x.Metadata + } + return nil +} + +// ACLRoleInfo represents a role definition. +type ACLRoleInfo struct { + state protoimpl.MessageState `protogen:"open.v1"` + RoleId string `protobuf:"bytes,1,opt,name=role_id,json=roleId,proto3" json:"role_id,omitempty"` + RoleName string `protobuf:"bytes,2,opt,name=role_name,json=roleName,proto3" json:"role_name,omitempty"` + Description string `protobuf:"bytes,3,opt,name=description,proto3" json:"description,omitempty"` + CreatedBy string `protobuf:"bytes,4,opt,name=created_by,json=createdBy,proto3" json:"created_by,omitempty"` + CreatedAt int64 `protobuf:"varint,5,opt,name=created_at,json=createdAt,proto3" json:"created_at,omitempty"` + Metadata map[string]string `protobuf:"bytes,6,rep,name=metadata,proto3" json:"metadata,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ACLRoleInfo) Reset() { + *x = ACLRoleInfo{} + mi := &file_aether_proto_msgTypes[88] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ACLRoleInfo) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ACLRoleInfo) ProtoMessage() {} + +func (x *ACLRoleInfo) ProtoReflect() protoreflect.Message { + mi := &file_aether_proto_msgTypes[88] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ACLRoleInfo.ProtoReflect.Descriptor instead. +func (*ACLRoleInfo) Descriptor() ([]byte, []int) { + return file_aether_proto_rawDescGZIP(), []int{88} +} + +func (x *ACLRoleInfo) GetRoleId() string { + if x != nil { + return x.RoleId + } + return "" +} + +func (x *ACLRoleInfo) GetRoleName() string { + if x != nil { + return x.RoleName + } + return "" +} + +func (x *ACLRoleInfo) GetDescription() string { + if x != nil { + return x.Description + } + return "" +} + +func (x *ACLRoleInfo) GetCreatedBy() string { + if x != nil { + return x.CreatedBy + } + return "" +} + +func (x *ACLRoleInfo) GetCreatedAt() int64 { + if x != nil { + return x.CreatedAt + } + return 0 +} + +func (x *ACLRoleInfo) GetMetadata() map[string]string { + if x != nil { + return x.Metadata + } + return nil +} + +// ACLGroupMemberInfo represents a single group-membership edge. +type ACLGroupMemberInfo struct { + state protoimpl.MessageState `protogen:"open.v1"` + GroupName string `protobuf:"bytes,1,opt,name=group_name,json=groupName,proto3" json:"group_name,omitempty"` + MemberType string `protobuf:"bytes,2,opt,name=member_type,json=memberType,proto3" json:"member_type,omitempty"` + MemberId string `protobuf:"bytes,3,opt,name=member_id,json=memberId,proto3" json:"member_id,omitempty"` + GrantedBy string `protobuf:"bytes,4,opt,name=granted_by,json=grantedBy,proto3" json:"granted_by,omitempty"` + GrantedAt int64 `protobuf:"varint,5,opt,name=granted_at,json=grantedAt,proto3" json:"granted_at,omitempty"` + ExpiresAt int64 `protobuf:"varint,6,opt,name=expires_at,json=expiresAt,proto3" json:"expires_at,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ACLGroupMemberInfo) Reset() { + *x = ACLGroupMemberInfo{} + mi := &file_aether_proto_msgTypes[89] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ACLGroupMemberInfo) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ACLGroupMemberInfo) ProtoMessage() {} + +func (x *ACLGroupMemberInfo) ProtoReflect() protoreflect.Message { + mi := &file_aether_proto_msgTypes[89] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ACLGroupMemberInfo.ProtoReflect.Descriptor instead. +func (*ACLGroupMemberInfo) Descriptor() ([]byte, []int) { + return file_aether_proto_rawDescGZIP(), []int{89} +} + +func (x *ACLGroupMemberInfo) GetGroupName() string { + if x != nil { + return x.GroupName + } + return "" +} + +func (x *ACLGroupMemberInfo) GetMemberType() string { + if x != nil { + return x.MemberType + } + return "" +} + +func (x *ACLGroupMemberInfo) GetMemberId() string { + if x != nil { + return x.MemberId + } + return "" +} + +func (x *ACLGroupMemberInfo) GetGrantedBy() string { + if x != nil { + return x.GrantedBy + } + return "" +} + +func (x *ACLGroupMemberInfo) GetGrantedAt() int64 { + if x != nil { + return x.GrantedAt + } + return 0 +} + +func (x *ACLGroupMemberInfo) GetExpiresAt() int64 { + if x != nil { + return x.ExpiresAt + } + return 0 +} + +// ACLRoleAssignmentInfo represents a single role-assignment edge. +type ACLRoleAssignmentInfo struct { + state protoimpl.MessageState `protogen:"open.v1"` + RoleName string `protobuf:"bytes,1,opt,name=role_name,json=roleName,proto3" json:"role_name,omitempty"` + AssigneeType string `protobuf:"bytes,2,opt,name=assignee_type,json=assigneeType,proto3" json:"assignee_type,omitempty"` + AssigneeId string `protobuf:"bytes,3,opt,name=assignee_id,json=assigneeId,proto3" json:"assignee_id,omitempty"` + GrantedBy string `protobuf:"bytes,4,opt,name=granted_by,json=grantedBy,proto3" json:"granted_by,omitempty"` + GrantedAt int64 `protobuf:"varint,5,opt,name=granted_at,json=grantedAt,proto3" json:"granted_at,omitempty"` + ExpiresAt int64 `protobuf:"varint,6,opt,name=expires_at,json=expiresAt,proto3" json:"expires_at,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ACLRoleAssignmentInfo) Reset() { + *x = ACLRoleAssignmentInfo{} + mi := &file_aether_proto_msgTypes[90] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ACLRoleAssignmentInfo) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ACLRoleAssignmentInfo) ProtoMessage() {} + +func (x *ACLRoleAssignmentInfo) ProtoReflect() protoreflect.Message { + mi := &file_aether_proto_msgTypes[90] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ACLRoleAssignmentInfo.ProtoReflect.Descriptor instead. +func (*ACLRoleAssignmentInfo) Descriptor() ([]byte, []int) { + return file_aether_proto_rawDescGZIP(), []int{90} +} + +func (x *ACLRoleAssignmentInfo) GetRoleName() string { + if x != nil { + return x.RoleName } return "" } -func (x *ACLAuthorityGrantInfo) GetAudienceType() string { +func (x *ACLRoleAssignmentInfo) GetAssigneeType() string { if x != nil { - return x.AudienceType + return x.AssigneeType } return "" } -func (x *ACLAuthorityGrantInfo) GetAudienceId() string { +func (x *ACLRoleAssignmentInfo) GetAssigneeId() string { if x != nil { - return x.AudienceId + return x.AssigneeId } return "" } -func (x *ACLAuthorityGrantInfo) GetValidWhileAudienceActive() bool { +func (x *ACLRoleAssignmentInfo) GetGrantedBy() string { if x != nil { - return x.ValidWhileAudienceActive + return x.GrantedBy } - return false + return "" } -func (x *ACLAuthorityGrantInfo) GetExpiresAt() int64 { +func (x *ACLRoleAssignmentInfo) GetGrantedAt() int64 { if x != nil { - return x.ExpiresAt + return x.GrantedAt } return 0 } -func (x *ACLAuthorityGrantInfo) GetRenewableUntil() int64 { +func (x *ACLRoleAssignmentInfo) GetExpiresAt() int64 { if x != nil { - return x.RenewableUntil + return x.ExpiresAt } return 0 } -func (x *ACLAuthorityGrantInfo) GetRenewedAt() int64 { +// ACLAccessContributionInfo is one rule that matched a principal or one of its +// groups/roles for an explained resource. +type ACLAccessContributionInfo struct { + state protoimpl.MessageState `protogen:"open.v1"` + Subject string `protobuf:"bytes,1,opt,name=subject,proto3" json:"subject,omitempty"` // granting subject, e.g. "role:wsadmin" + RuleId string `protobuf:"bytes,2,opt,name=rule_id,json=ruleId,proto3" json:"rule_id,omitempty"` + AccessLevel int32 `protobuf:"varint,3,opt,name=access_level,json=accessLevel,proto3" json:"access_level,omitempty"` + Resource string `protobuf:"bytes,4,opt,name=resource,proto3" json:"resource,omitempty"` // matched object pattern, e.g. "workspace:prod" or "workspace:*" + Expired bool `protobuf:"varint,5,opt,name=expired,proto3" json:"expired,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ACLAccessContributionInfo) Reset() { + *x = ACLAccessContributionInfo{} + mi := &file_aether_proto_msgTypes[91] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ACLAccessContributionInfo) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ACLAccessContributionInfo) ProtoMessage() {} + +func (x *ACLAccessContributionInfo) ProtoReflect() protoreflect.Message { + mi := &file_aether_proto_msgTypes[91] if x != nil { - return x.RenewedAt + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms } - return 0 + return mi.MessageOf(x) } -func (x *ACLAuthorityGrantInfo) GetRevoked() bool { +// Deprecated: Use ACLAccessContributionInfo.ProtoReflect.Descriptor instead. +func (*ACLAccessContributionInfo) Descriptor() ([]byte, []int) { + return file_aether_proto_rawDescGZIP(), []int{91} +} + +func (x *ACLAccessContributionInfo) GetSubject() string { if x != nil { - return x.Revoked + return x.Subject } - return false + return "" } -func (x *ACLAuthorityGrantInfo) GetRevokedAt() int64 { +func (x *ACLAccessContributionInfo) GetRuleId() string { if x != nil { - return x.RevokedAt + return x.RuleId } - return 0 + return "" } -func (x *ACLAuthorityGrantInfo) GetReason() string { +func (x *ACLAccessContributionInfo) GetAccessLevel() int32 { if x != nil { - return x.Reason + return x.AccessLevel } - return "" + return 0 } -func (x *ACLAuthorityGrantInfo) GetMetadata() map[string]string { +func (x *ACLAccessContributionInfo) GetResource() string { if x != nil { - return x.Metadata + return x.Resource } - return nil + return "" } -func (x *ACLAuthorityGrantInfo) GetCreatedAt() int64 { +func (x *ACLAccessContributionInfo) GetExpired() bool { if x != nil { - return x.CreatedAt + return x.Expired } - return 0 + return false } -// ACLCleanupResult represents the result of a cleanup operation. -// Used for CLEANUP_EXPIRED and CLEANUP_AUDIT_LOGS responses. -type ACLCleanupResult struct { - state protoimpl.MessageState `protogen:"open.v1"` - DeletedCount int64 `protobuf:"varint,1,opt,name=deleted_count,json=deletedCount,proto3" json:"deleted_count,omitempty"` // Number of items deleted - Message string `protobuf:"bytes,2,opt,name=message,proto3" json:"message,omitempty"` // Human-readable result message - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache +// ACLAccessExplanationInfo explains how a principal's effective access to a +// resource is decided (EXPLAIN_ACCESS). The gateway records an "explain_access" +// audit event attributing the call to the connected principal. +type ACLAccessExplanationInfo struct { + state protoimpl.MessageState `protogen:"open.v1"` + Principal string `protobuf:"bytes,1,opt,name=principal,proto3" json:"principal,omitempty"` // self subject "type:id" + Subjects []string `protobuf:"bytes,2,rep,name=subjects,proto3" json:"subjects,omitempty"` // self + transitive groups/roles + Contributions []*ACLAccessContributionInfo `protobuf:"bytes,3,rep,name=contributions,proto3" json:"contributions,omitempty"` + Allowed bool `protobuf:"varint,4,opt,name=allowed,proto3" json:"allowed,omitempty"` + Decision string `protobuf:"bytes,5,opt,name=decision,proto3" json:"decision,omitempty"` // "ALLOW" / "DENY" + EffectiveAccessLevel int32 `protobuf:"varint,6,opt,name=effective_access_level,json=effectiveAccessLevel,proto3" json:"effective_access_level,omitempty"` + FallbackApplied bool `protobuf:"varint,7,opt,name=fallback_applied,json=fallbackApplied,proto3" json:"fallback_applied,omitempty"` + Reason string `protobuf:"bytes,8,opt,name=reason,proto3" json:"reason,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } -func (x *ACLCleanupResult) Reset() { - *x = ACLCleanupResult{} - mi := &file_aether_proto_msgTypes[80] +func (x *ACLAccessExplanationInfo) Reset() { + *x = ACLAccessExplanationInfo{} + mi := &file_aether_proto_msgTypes[92] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } -func (x *ACLCleanupResult) String() string { +func (x *ACLAccessExplanationInfo) String() string { return protoimpl.X.MessageStringOf(x) } -func (*ACLCleanupResult) ProtoMessage() {} +func (*ACLAccessExplanationInfo) ProtoMessage() {} -func (x *ACLCleanupResult) ProtoReflect() protoreflect.Message { - mi := &file_aether_proto_msgTypes[80] +func (x *ACLAccessExplanationInfo) ProtoReflect() protoreflect.Message { + mi := &file_aether_proto_msgTypes[92] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -10819,21 +12317,63 @@ func (x *ACLCleanupResult) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use ACLCleanupResult.ProtoReflect.Descriptor instead. -func (*ACLCleanupResult) Descriptor() ([]byte, []int) { - return file_aether_proto_rawDescGZIP(), []int{80} +// Deprecated: Use ACLAccessExplanationInfo.ProtoReflect.Descriptor instead. +func (*ACLAccessExplanationInfo) Descriptor() ([]byte, []int) { + return file_aether_proto_rawDescGZIP(), []int{92} } -func (x *ACLCleanupResult) GetDeletedCount() int64 { +func (x *ACLAccessExplanationInfo) GetPrincipal() string { if x != nil { - return x.DeletedCount + return x.Principal + } + return "" +} + +func (x *ACLAccessExplanationInfo) GetSubjects() []string { + if x != nil { + return x.Subjects + } + return nil +} + +func (x *ACLAccessExplanationInfo) GetContributions() []*ACLAccessContributionInfo { + if x != nil { + return x.Contributions + } + return nil +} + +func (x *ACLAccessExplanationInfo) GetAllowed() bool { + if x != nil { + return x.Allowed + } + return false +} + +func (x *ACLAccessExplanationInfo) GetDecision() string { + if x != nil { + return x.Decision + } + return "" +} + +func (x *ACLAccessExplanationInfo) GetEffectiveAccessLevel() int32 { + if x != nil { + return x.EffectiveAccessLevel } return 0 } -func (x *ACLCleanupResult) GetMessage() string { +func (x *ACLAccessExplanationInfo) GetFallbackApplied() bool { if x != nil { - return x.Message + return x.FallbackApplied + } + return false +} + +func (x *ACLAccessExplanationInfo) GetReason() string { + if x != nil { + return x.Reason } return "" } @@ -10867,14 +12407,22 @@ type ACLResponse struct { AuthorityGrants []*ACLAuthorityGrantInfo `protobuf:"bytes,14,rep,name=authority_grants,json=authorityGrants,proto3" json:"authority_grants,omitempty"` TotalAuthorityGrants int32 `protobuf:"varint,15,opt,name=total_authority_grants,json=totalAuthorityGrants,proto3" json:"total_authority_grants,omitempty"` // Echoed from the originating ACLOperation for correlation - RequestId string `protobuf:"bytes,12,opt,name=request_id,json=requestId,proto3" json:"request_id,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + RequestId string `protobuf:"bytes,12,opt,name=request_id,json=requestId,proto3" json:"request_id,omitempty"` + // Role/group results. + Group *ACLGroupInfo `protobuf:"bytes,16,opt,name=group,proto3" json:"group,omitempty"` // GET_GROUP / CREATE_GROUP + Groups []*ACLGroupInfo `protobuf:"bytes,17,rep,name=groups,proto3" json:"groups,omitempty"` // LIST_GROUPS + Role *ACLRoleInfo `protobuf:"bytes,18,opt,name=role,proto3" json:"role,omitempty"` // GET_ROLE / CREATE_ROLE + Roles []*ACLRoleInfo `protobuf:"bytes,19,rep,name=roles,proto3" json:"roles,omitempty"` // LIST_ROLES + GroupMembers []*ACLGroupMemberInfo `protobuf:"bytes,20,rep,name=group_members,json=groupMembers,proto3" json:"group_members,omitempty"` // LIST_GROUP_MEMBERS / LIST_PRINCIPAL_GROUPS + RoleAssignments []*ACLRoleAssignmentInfo `protobuf:"bytes,21,rep,name=role_assignments,json=roleAssignments,proto3" json:"role_assignments,omitempty"` // LIST_ROLE_ASSIGNMENTS / LIST_PRINCIPAL_ROLES + Explanation *ACLAccessExplanationInfo `protobuf:"bytes,22,opt,name=explanation,proto3" json:"explanation,omitempty"` // EXPLAIN_ACCESS + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *ACLResponse) Reset() { *x = ACLResponse{} - mi := &file_aether_proto_msgTypes[81] + mi := &file_aether_proto_msgTypes[93] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -10886,7 +12434,7 @@ func (x *ACLResponse) String() string { func (*ACLResponse) ProtoMessage() {} func (x *ACLResponse) ProtoReflect() protoreflect.Message { - mi := &file_aether_proto_msgTypes[81] + mi := &file_aether_proto_msgTypes[93] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -10899,7 +12447,7 @@ func (x *ACLResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use ACLResponse.ProtoReflect.Descriptor instead. func (*ACLResponse) Descriptor() ([]byte, []int) { - return file_aether_proto_rawDescGZIP(), []int{81} + return file_aether_proto_rawDescGZIP(), []int{93} } func (x *ACLResponse) GetSuccess() bool { @@ -11000,6 +12548,55 @@ func (x *ACLResponse) GetRequestId() string { return "" } +func (x *ACLResponse) GetGroup() *ACLGroupInfo { + if x != nil { + return x.Group + } + return nil +} + +func (x *ACLResponse) GetGroups() []*ACLGroupInfo { + if x != nil { + return x.Groups + } + return nil +} + +func (x *ACLResponse) GetRole() *ACLRoleInfo { + if x != nil { + return x.Role + } + return nil +} + +func (x *ACLResponse) GetRoles() []*ACLRoleInfo { + if x != nil { + return x.Roles + } + return nil +} + +func (x *ACLResponse) GetGroupMembers() []*ACLGroupMemberInfo { + if x != nil { + return x.GroupMembers + } + return nil +} + +func (x *ACLResponse) GetRoleAssignments() []*ACLRoleAssignmentInfo { + if x != nil { + return x.RoleAssignments + } + return nil +} + +func (x *ACLResponse) GetExplanation() *ACLAccessExplanationInfo { + if x != nil { + return x.Explanation + } + return nil +} + // AuthorityGrantOperation allows ordinary streaming clients to exchange, // derive, inspect, renew, and revoke authority grants without going through // the admin ACL surface. @@ -11028,7 +12625,7 @@ type AuthorityGrantOperation struct { func (x *AuthorityGrantOperation) Reset() { *x = AuthorityGrantOperation{} - mi := &file_aether_proto_msgTypes[82] + mi := &file_aether_proto_msgTypes[94] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -11040,7 +12637,7 @@ func (x *AuthorityGrantOperation) String() string { func (*AuthorityGrantOperation) ProtoMessage() {} func (x *AuthorityGrantOperation) ProtoReflect() protoreflect.Message { - mi := &file_aether_proto_msgTypes[82] + mi := &file_aether_proto_msgTypes[94] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -11053,7 +12650,7 @@ func (x *AuthorityGrantOperation) ProtoReflect() protoreflect.Message { // Deprecated: Use AuthorityGrantOperation.ProtoReflect.Descriptor instead. func (*AuthorityGrantOperation) Descriptor() ([]byte, []int) { - return file_aether_proto_rawDescGZIP(), []int{82} + return file_aether_proto_rawDescGZIP(), []int{94} } func (x *AuthorityGrantOperation) GetOp() AuthorityGrantOperation_OpType { @@ -11151,7 +12748,7 @@ type AuthorityGrantExchangeRequest struct { func (x *AuthorityGrantExchangeRequest) Reset() { *x = AuthorityGrantExchangeRequest{} - mi := &file_aether_proto_msgTypes[83] + mi := &file_aether_proto_msgTypes[95] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -11163,7 +12760,7 @@ func (x *AuthorityGrantExchangeRequest) String() string { func (*AuthorityGrantExchangeRequest) ProtoMessage() {} func (x *AuthorityGrantExchangeRequest) ProtoReflect() protoreflect.Message { - mi := &file_aether_proto_msgTypes[83] + mi := &file_aether_proto_msgTypes[95] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -11176,7 +12773,7 @@ func (x *AuthorityGrantExchangeRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use AuthorityGrantExchangeRequest.ProtoReflect.Descriptor instead. func (*AuthorityGrantExchangeRequest) Descriptor() ([]byte, []int) { - return file_aether_proto_rawDescGZIP(), []int{83} + return file_aether_proto_rawDescGZIP(), []int{95} } func (x *AuthorityGrantExchangeRequest) GetSourceSessionId() string { @@ -11301,7 +12898,7 @@ type AuthorityGrantDeriveRequest struct { func (x *AuthorityGrantDeriveRequest) Reset() { *x = AuthorityGrantDeriveRequest{} - mi := &file_aether_proto_msgTypes[84] + mi := &file_aether_proto_msgTypes[96] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -11313,7 +12910,7 @@ func (x *AuthorityGrantDeriveRequest) String() string { func (*AuthorityGrantDeriveRequest) ProtoMessage() {} func (x *AuthorityGrantDeriveRequest) ProtoReflect() protoreflect.Message { - mi := &file_aether_proto_msgTypes[84] + mi := &file_aether_proto_msgTypes[96] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -11326,7 +12923,7 @@ func (x *AuthorityGrantDeriveRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use AuthorityGrantDeriveRequest.ProtoReflect.Descriptor instead. func (*AuthorityGrantDeriveRequest) Descriptor() ([]byte, []int) { - return file_aether_proto_rawDescGZIP(), []int{84} + return file_aether_proto_rawDescGZIP(), []int{96} } func (x *AuthorityGrantDeriveRequest) GetParentGrantId() string { @@ -11456,7 +13053,7 @@ type AuthorityGrantResponse struct { func (x *AuthorityGrantResponse) Reset() { *x = AuthorityGrantResponse{} - mi := &file_aether_proto_msgTypes[85] + mi := &file_aether_proto_msgTypes[97] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -11468,7 +13065,7 @@ func (x *AuthorityGrantResponse) String() string { func (*AuthorityGrantResponse) ProtoMessage() {} func (x *AuthorityGrantResponse) ProtoReflect() protoreflect.Message { - mi := &file_aether_proto_msgTypes[85] + mi := &file_aether_proto_msgTypes[97] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -11481,7 +13078,7 @@ func (x *AuthorityGrantResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use AuthorityGrantResponse.ProtoReflect.Descriptor instead. func (*AuthorityGrantResponse) Descriptor() ([]byte, []int) { - return file_aether_proto_rawDescGZIP(), []int{85} + return file_aether_proto_rawDescGZIP(), []int{97} } func (x *AuthorityGrantResponse) GetSuccess() bool { @@ -11554,7 +13151,7 @@ type AuthorityGrantListRequest struct { func (x *AuthorityGrantListRequest) Reset() { *x = AuthorityGrantListRequest{} - mi := &file_aether_proto_msgTypes[86] + mi := &file_aether_proto_msgTypes[98] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -11566,7 +13163,7 @@ func (x *AuthorityGrantListRequest) String() string { func (*AuthorityGrantListRequest) ProtoMessage() {} func (x *AuthorityGrantListRequest) ProtoReflect() protoreflect.Message { - mi := &file_aether_proto_msgTypes[86] + mi := &file_aether_proto_msgTypes[98] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -11579,7 +13176,7 @@ func (x *AuthorityGrantListRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use AuthorityGrantListRequest.ProtoReflect.Descriptor instead. func (*AuthorityGrantListRequest) Descriptor() ([]byte, []int) { - return file_aether_proto_rawDescGZIP(), []int{86} + return file_aether_proto_rawDescGZIP(), []int{98} } func (x *AuthorityGrantListRequest) GetAudienceType() string { @@ -11630,7 +13227,7 @@ type AuthorityGrantBatchExchangeRequest struct { func (x *AuthorityGrantBatchExchangeRequest) Reset() { *x = AuthorityGrantBatchExchangeRequest{} - mi := &file_aether_proto_msgTypes[87] + mi := &file_aether_proto_msgTypes[99] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -11642,7 +13239,7 @@ func (x *AuthorityGrantBatchExchangeRequest) String() string { func (*AuthorityGrantBatchExchangeRequest) ProtoMessage() {} func (x *AuthorityGrantBatchExchangeRequest) ProtoReflect() protoreflect.Message { - mi := &file_aether_proto_msgTypes[87] + mi := &file_aether_proto_msgTypes[99] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -11655,7 +13252,7 @@ func (x *AuthorityGrantBatchExchangeRequest) ProtoReflect() protoreflect.Message // Deprecated: Use AuthorityGrantBatchExchangeRequest.ProtoReflect.Descriptor instead. func (*AuthorityGrantBatchExchangeRequest) Descriptor() ([]byte, []int) { - return file_aether_proto_rawDescGZIP(), []int{87} + return file_aether_proto_rawDescGZIP(), []int{99} } func (x *AuthorityGrantBatchExchangeRequest) GetRequests() []*AuthorityGrantExchangeRequest { @@ -11696,7 +13293,7 @@ type AuthorityGrantDeriveForTargetRequest struct { func (x *AuthorityGrantDeriveForTargetRequest) Reset() { *x = AuthorityGrantDeriveForTargetRequest{} - mi := &file_aether_proto_msgTypes[88] + mi := &file_aether_proto_msgTypes[100] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -11708,7 +13305,7 @@ func (x *AuthorityGrantDeriveForTargetRequest) String() string { func (*AuthorityGrantDeriveForTargetRequest) ProtoMessage() {} func (x *AuthorityGrantDeriveForTargetRequest) ProtoReflect() protoreflect.Message { - mi := &file_aether_proto_msgTypes[88] + mi := &file_aether_proto_msgTypes[100] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -11721,7 +13318,7 @@ func (x *AuthorityGrantDeriveForTargetRequest) ProtoReflect() protoreflect.Messa // Deprecated: Use AuthorityGrantDeriveForTargetRequest.ProtoReflect.Descriptor instead. func (*AuthorityGrantDeriveForTargetRequest) Descriptor() ([]byte, []int) { - return file_aether_proto_rawDescGZIP(), []int{88} + return file_aether_proto_rawDescGZIP(), []int{100} } func (x *AuthorityGrantDeriveForTargetRequest) GetParentGrantId() string { @@ -11818,7 +13415,7 @@ type AuthorityIdentity struct { func (x *AuthorityIdentity) Reset() { *x = AuthorityIdentity{} - mi := &file_aether_proto_msgTypes[89] + mi := &file_aether_proto_msgTypes[101] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -11830,7 +13427,7 @@ func (x *AuthorityIdentity) String() string { func (*AuthorityIdentity) ProtoMessage() {} func (x *AuthorityIdentity) ProtoReflect() protoreflect.Message { - mi := &file_aether_proto_msgTypes[89] + mi := &file_aether_proto_msgTypes[101] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -11843,7 +13440,7 @@ func (x *AuthorityIdentity) ProtoReflect() protoreflect.Message { // Deprecated: Use AuthorityIdentity.ProtoReflect.Descriptor instead. func (*AuthorityIdentity) Descriptor() ([]byte, []int) { - return file_aether_proto_rawDescGZIP(), []int{89} + return file_aether_proto_rawDescGZIP(), []int{101} } func (x *AuthorityIdentity) GetSubject() *PrincipalRef { @@ -11893,7 +13490,7 @@ type AuthoritySpan struct { func (x *AuthoritySpan) Reset() { *x = AuthoritySpan{} - mi := &file_aether_proto_msgTypes[90] + mi := &file_aether_proto_msgTypes[102] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -11905,7 +13502,7 @@ func (x *AuthoritySpan) String() string { func (*AuthoritySpan) ProtoMessage() {} func (x *AuthoritySpan) ProtoReflect() protoreflect.Message { - mi := &file_aether_proto_msgTypes[90] + mi := &file_aether_proto_msgTypes[102] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -11918,7 +13515,7 @@ func (x *AuthoritySpan) ProtoReflect() protoreflect.Message { // Deprecated: Use AuthoritySpan.ProtoReflect.Descriptor instead. func (*AuthoritySpan) Descriptor() ([]byte, []int) { - return file_aether_proto_rawDescGZIP(), []int{90} + return file_aether_proto_rawDescGZIP(), []int{102} } func (x *AuthoritySpan) GetWorkspaceScope() []string { @@ -11995,7 +13592,7 @@ type AuthorityGrantRevocation struct { func (x *AuthorityGrantRevocation) Reset() { *x = AuthorityGrantRevocation{} - mi := &file_aether_proto_msgTypes[91] + mi := &file_aether_proto_msgTypes[103] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -12007,7 +13604,7 @@ func (x *AuthorityGrantRevocation) String() string { func (*AuthorityGrantRevocation) ProtoMessage() {} func (x *AuthorityGrantRevocation) ProtoReflect() protoreflect.Message { - mi := &file_aether_proto_msgTypes[91] + mi := &file_aether_proto_msgTypes[103] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -12020,7 +13617,7 @@ func (x *AuthorityGrantRevocation) ProtoReflect() protoreflect.Message { // Deprecated: Use AuthorityGrantRevocation.ProtoReflect.Descriptor instead. func (*AuthorityGrantRevocation) Descriptor() ([]byte, []int) { - return file_aether_proto_rawDescGZIP(), []int{91} + return file_aether_proto_rawDescGZIP(), []int{103} } func (x *AuthorityGrantRevocation) GetGrantId() string { @@ -12074,7 +13671,7 @@ type AuthorityRequestRoutingTarget struct { func (x *AuthorityRequestRoutingTarget) Reset() { *x = AuthorityRequestRoutingTarget{} - mi := &file_aether_proto_msgTypes[92] + mi := &file_aether_proto_msgTypes[104] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -12086,7 +13683,7 @@ func (x *AuthorityRequestRoutingTarget) String() string { func (*AuthorityRequestRoutingTarget) ProtoMessage() {} func (x *AuthorityRequestRoutingTarget) ProtoReflect() protoreflect.Message { - mi := &file_aether_proto_msgTypes[92] + mi := &file_aether_proto_msgTypes[104] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -12099,7 +13696,7 @@ func (x *AuthorityRequestRoutingTarget) ProtoReflect() protoreflect.Message { // Deprecated: Use AuthorityRequestRoutingTarget.ProtoReflect.Descriptor instead. func (*AuthorityRequestRoutingTarget) Descriptor() ([]byte, []int) { - return file_aether_proto_rawDescGZIP(), []int{92} + return file_aether_proto_rawDescGZIP(), []int{104} } func (x *AuthorityRequestRoutingTarget) GetPrincipal() *PrincipalRef { @@ -12128,7 +13725,7 @@ type AuthorityRequestResourceScopeEntry struct { func (x *AuthorityRequestResourceScopeEntry) Reset() { *x = AuthorityRequestResourceScopeEntry{} - mi := &file_aether_proto_msgTypes[93] + mi := &file_aether_proto_msgTypes[105] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -12140,7 +13737,7 @@ func (x *AuthorityRequestResourceScopeEntry) String() string { func (*AuthorityRequestResourceScopeEntry) ProtoMessage() {} func (x *AuthorityRequestResourceScopeEntry) ProtoReflect() protoreflect.Message { - mi := &file_aether_proto_msgTypes[93] + mi := &file_aether_proto_msgTypes[105] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -12153,7 +13750,7 @@ func (x *AuthorityRequestResourceScopeEntry) ProtoReflect() protoreflect.Message // Deprecated: Use AuthorityRequestResourceScopeEntry.ProtoReflect.Descriptor instead. func (*AuthorityRequestResourceScopeEntry) Descriptor() ([]byte, []int) { - return file_aether_proto_rawDescGZIP(), []int{93} + return file_aether_proto_rawDescGZIP(), []int{105} } func (x *AuthorityRequestResourceScopeEntry) GetResourceType() string { @@ -12211,7 +13808,7 @@ type AuthorityRequest struct { func (x *AuthorityRequest) Reset() { *x = AuthorityRequest{} - mi := &file_aether_proto_msgTypes[94] + mi := &file_aether_proto_msgTypes[106] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -12223,7 +13820,7 @@ func (x *AuthorityRequest) String() string { func (*AuthorityRequest) ProtoMessage() {} func (x *AuthorityRequest) ProtoReflect() protoreflect.Message { - mi := &file_aether_proto_msgTypes[94] + mi := &file_aether_proto_msgTypes[106] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -12236,7 +13833,7 @@ func (x *AuthorityRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use AuthorityRequest.ProtoReflect.Descriptor instead. func (*AuthorityRequest) Descriptor() ([]byte, []int) { - return file_aether_proto_rawDescGZIP(), []int{94} + return file_aether_proto_rawDescGZIP(), []int{106} } func (x *AuthorityRequest) GetRequestId() string { @@ -12409,7 +14006,7 @@ type CreateAuthorityRequestPayload struct { func (x *CreateAuthorityRequestPayload) Reset() { *x = CreateAuthorityRequestPayload{} - mi := &file_aether_proto_msgTypes[95] + mi := &file_aether_proto_msgTypes[107] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -12421,7 +14018,7 @@ func (x *CreateAuthorityRequestPayload) String() string { func (*CreateAuthorityRequestPayload) ProtoMessage() {} func (x *CreateAuthorityRequestPayload) ProtoReflect() protoreflect.Message { - mi := &file_aether_proto_msgTypes[95] + mi := &file_aether_proto_msgTypes[107] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -12434,7 +14031,7 @@ func (x *CreateAuthorityRequestPayload) ProtoReflect() protoreflect.Message { // Deprecated: Use CreateAuthorityRequestPayload.ProtoReflect.Descriptor instead. func (*CreateAuthorityRequestPayload) Descriptor() ([]byte, []int) { - return file_aether_proto_rawDescGZIP(), []int{95} + return file_aether_proto_rawDescGZIP(), []int{107} } func (x *CreateAuthorityRequestPayload) GetRequestingActor() *PrincipalRef { @@ -12551,7 +14148,7 @@ type ResolveAuthorityRequestPayload struct { func (x *ResolveAuthorityRequestPayload) Reset() { *x = ResolveAuthorityRequestPayload{} - mi := &file_aether_proto_msgTypes[96] + mi := &file_aether_proto_msgTypes[108] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -12563,7 +14160,7 @@ func (x *ResolveAuthorityRequestPayload) String() string { func (*ResolveAuthorityRequestPayload) ProtoMessage() {} func (x *ResolveAuthorityRequestPayload) ProtoReflect() protoreflect.Message { - mi := &file_aether_proto_msgTypes[96] + mi := &file_aether_proto_msgTypes[108] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -12576,7 +14173,7 @@ func (x *ResolveAuthorityRequestPayload) ProtoReflect() protoreflect.Message { // Deprecated: Use ResolveAuthorityRequestPayload.ProtoReflect.Descriptor instead. func (*ResolveAuthorityRequestPayload) Descriptor() ([]byte, []int) { - return file_aether_proto_rawDescGZIP(), []int{96} + return file_aether_proto_rawDescGZIP(), []int{108} } func (x *ResolveAuthorityRequestPayload) GetDecision() ResolveAuthorityRequestPayload_Decision { @@ -12660,7 +14257,7 @@ type AuthorityRequestListFilter struct { func (x *AuthorityRequestListFilter) Reset() { *x = AuthorityRequestListFilter{} - mi := &file_aether_proto_msgTypes[97] + mi := &file_aether_proto_msgTypes[109] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -12672,7 +14269,7 @@ func (x *AuthorityRequestListFilter) String() string { func (*AuthorityRequestListFilter) ProtoMessage() {} func (x *AuthorityRequestListFilter) ProtoReflect() protoreflect.Message { - mi := &file_aether_proto_msgTypes[97] + mi := &file_aether_proto_msgTypes[109] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -12685,7 +14282,7 @@ func (x *AuthorityRequestListFilter) ProtoReflect() protoreflect.Message { // Deprecated: Use AuthorityRequestListFilter.ProtoReflect.Descriptor instead. func (*AuthorityRequestListFilter) Descriptor() ([]byte, []int) { - return file_aether_proto_rawDescGZIP(), []int{97} + return file_aether_proto_rawDescGZIP(), []int{109} } func (x *AuthorityRequestListFilter) GetStatus() AuthorityRequestStatus { @@ -12742,7 +14339,7 @@ type AuthorityRequestOperation struct { func (x *AuthorityRequestOperation) Reset() { *x = AuthorityRequestOperation{} - mi := &file_aether_proto_msgTypes[98] + mi := &file_aether_proto_msgTypes[110] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -12754,7 +14351,7 @@ func (x *AuthorityRequestOperation) String() string { func (*AuthorityRequestOperation) ProtoMessage() {} func (x *AuthorityRequestOperation) ProtoReflect() protoreflect.Message { - mi := &file_aether_proto_msgTypes[98] + mi := &file_aether_proto_msgTypes[110] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -12767,7 +14364,7 @@ func (x *AuthorityRequestOperation) ProtoReflect() protoreflect.Message { // Deprecated: Use AuthorityRequestOperation.ProtoReflect.Descriptor instead. func (*AuthorityRequestOperation) Descriptor() ([]byte, []int) { - return file_aether_proto_rawDescGZIP(), []int{98} + return file_aether_proto_rawDescGZIP(), []int{110} } func (x *AuthorityRequestOperation) GetOp() AuthorityRequestOperation_OpType { @@ -12835,7 +14432,7 @@ type AuthorityRequestOperationResponse struct { func (x *AuthorityRequestOperationResponse) Reset() { *x = AuthorityRequestOperationResponse{} - mi := &file_aether_proto_msgTypes[99] + mi := &file_aether_proto_msgTypes[111] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -12847,7 +14444,7 @@ func (x *AuthorityRequestOperationResponse) String() string { func (*AuthorityRequestOperationResponse) ProtoMessage() {} func (x *AuthorityRequestOperationResponse) ProtoReflect() protoreflect.Message { - mi := &file_aether_proto_msgTypes[99] + mi := &file_aether_proto_msgTypes[111] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -12860,7 +14457,7 @@ func (x *AuthorityRequestOperationResponse) ProtoReflect() protoreflect.Message // Deprecated: Use AuthorityRequestOperationResponse.ProtoReflect.Descriptor instead. func (*AuthorityRequestOperationResponse) Descriptor() ([]byte, []int) { - return file_aether_proto_rawDescGZIP(), []int{99} + return file_aether_proto_rawDescGZIP(), []int{111} } func (x *AuthorityRequestOperationResponse) GetSuccess() bool { @@ -12918,7 +14515,7 @@ type AuthorityRequestEvent struct { func (x *AuthorityRequestEvent) Reset() { *x = AuthorityRequestEvent{} - mi := &file_aether_proto_msgTypes[100] + mi := &file_aether_proto_msgTypes[112] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -12930,7 +14527,7 @@ func (x *AuthorityRequestEvent) String() string { func (*AuthorityRequestEvent) ProtoMessage() {} func (x *AuthorityRequestEvent) ProtoReflect() protoreflect.Message { - mi := &file_aether_proto_msgTypes[100] + mi := &file_aether_proto_msgTypes[112] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -12943,7 +14540,7 @@ func (x *AuthorityRequestEvent) ProtoReflect() protoreflect.Message { // Deprecated: Use AuthorityRequestEvent.ProtoReflect.Descriptor instead. func (*AuthorityRequestEvent) Descriptor() ([]byte, []int) { - return file_aether_proto_rawDescGZIP(), []int{100} + return file_aether_proto_rawDescGZIP(), []int{112} } func (x *AuthorityRequestEvent) GetEventType() AuthorityRequestEvent_EventType { @@ -12992,7 +14589,7 @@ type TokenOperation struct { func (x *TokenOperation) Reset() { *x = TokenOperation{} - mi := &file_aether_proto_msgTypes[101] + mi := &file_aether_proto_msgTypes[113] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -13004,7 +14601,7 @@ func (x *TokenOperation) String() string { func (*TokenOperation) ProtoMessage() {} func (x *TokenOperation) ProtoReflect() protoreflect.Message { - mi := &file_aether_proto_msgTypes[101] + mi := &file_aether_proto_msgTypes[113] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -13017,7 +14614,7 @@ func (x *TokenOperation) ProtoReflect() protoreflect.Message { // Deprecated: Use TokenOperation.ProtoReflect.Descriptor instead. func (*TokenOperation) Descriptor() ([]byte, []int) { - return file_aether_proto_rawDescGZIP(), []int{101} + return file_aether_proto_rawDescGZIP(), []int{113} } func (x *TokenOperation) GetOp() TokenOperation_OpType { @@ -13070,7 +14667,7 @@ type TokenCreateRequest struct { func (x *TokenCreateRequest) Reset() { *x = TokenCreateRequest{} - mi := &file_aether_proto_msgTypes[102] + mi := &file_aether_proto_msgTypes[114] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -13082,7 +14679,7 @@ func (x *TokenCreateRequest) String() string { func (*TokenCreateRequest) ProtoMessage() {} func (x *TokenCreateRequest) ProtoReflect() protoreflect.Message { - mi := &file_aether_proto_msgTypes[102] + mi := &file_aether_proto_msgTypes[114] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -13095,7 +14692,7 @@ func (x *TokenCreateRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use TokenCreateRequest.ProtoReflect.Descriptor instead. func (*TokenCreateRequest) Descriptor() ([]byte, []int) { - return file_aether_proto_rawDescGZIP(), []int{102} + return file_aether_proto_rawDescGZIP(), []int{114} } func (x *TokenCreateRequest) GetName() string { @@ -13152,7 +14749,7 @@ type TokenFilter struct { func (x *TokenFilter) Reset() { *x = TokenFilter{} - mi := &file_aether_proto_msgTypes[103] + mi := &file_aether_proto_msgTypes[115] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -13164,7 +14761,7 @@ func (x *TokenFilter) String() string { func (*TokenFilter) ProtoMessage() {} func (x *TokenFilter) ProtoReflect() protoreflect.Message { - mi := &file_aether_proto_msgTypes[103] + mi := &file_aether_proto_msgTypes[115] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -13177,7 +14774,7 @@ func (x *TokenFilter) ProtoReflect() protoreflect.Message { // Deprecated: Use TokenFilter.ProtoReflect.Descriptor instead. func (*TokenFilter) Descriptor() ([]byte, []int) { - return file_aether_proto_rawDescGZIP(), []int{103} + return file_aether_proto_rawDescGZIP(), []int{115} } func (x *TokenFilter) GetLimit() int32 { @@ -13222,7 +14819,7 @@ type TokenInfo struct { func (x *TokenInfo) Reset() { *x = TokenInfo{} - mi := &file_aether_proto_msgTypes[104] + mi := &file_aether_proto_msgTypes[116] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -13234,7 +14831,7 @@ func (x *TokenInfo) String() string { func (*TokenInfo) ProtoMessage() {} func (x *TokenInfo) ProtoReflect() protoreflect.Message { - mi := &file_aether_proto_msgTypes[104] + mi := &file_aether_proto_msgTypes[116] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -13247,7 +14844,7 @@ func (x *TokenInfo) ProtoReflect() protoreflect.Message { // Deprecated: Use TokenInfo.ProtoReflect.Descriptor instead. func (*TokenInfo) Descriptor() ([]byte, []int) { - return file_aether_proto_rawDescGZIP(), []int{104} + return file_aether_proto_rawDescGZIP(), []int{116} } func (x *TokenInfo) GetId() string { @@ -13360,7 +14957,7 @@ type TokenResponse struct { func (x *TokenResponse) Reset() { *x = TokenResponse{} - mi := &file_aether_proto_msgTypes[105] + mi := &file_aether_proto_msgTypes[117] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -13372,7 +14969,7 @@ func (x *TokenResponse) String() string { func (*TokenResponse) ProtoMessage() {} func (x *TokenResponse) ProtoReflect() protoreflect.Message { - mi := &file_aether_proto_msgTypes[105] + mi := &file_aether_proto_msgTypes[117] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -13385,7 +14982,7 @@ func (x *TokenResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use TokenResponse.ProtoReflect.Descriptor instead. func (*TokenResponse) Descriptor() ([]byte, []int) { - return file_aether_proto_rawDescGZIP(), []int{105} + return file_aether_proto_rawDescGZIP(), []int{117} } func (x *TokenResponse) GetSuccess() bool { @@ -13496,7 +15093,7 @@ type ProgressReport struct { func (x *ProgressReport) Reset() { *x = ProgressReport{} - mi := &file_aether_proto_msgTypes[106] + mi := &file_aether_proto_msgTypes[118] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -13508,7 +15105,7 @@ func (x *ProgressReport) String() string { func (*ProgressReport) ProtoMessage() {} func (x *ProgressReport) ProtoReflect() protoreflect.Message { - mi := &file_aether_proto_msgTypes[106] + mi := &file_aether_proto_msgTypes[118] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -13521,7 +15118,7 @@ func (x *ProgressReport) ProtoReflect() protoreflect.Message { // Deprecated: Use ProgressReport.ProtoReflect.Descriptor instead. func (*ProgressReport) Descriptor() ([]byte, []int) { - return file_aether_proto_rawDescGZIP(), []int{106} + return file_aether_proto_rawDescGZIP(), []int{118} } func (x *ProgressReport) GetTaskId() string { @@ -13606,7 +15203,7 @@ type ProgressStep struct { func (x *ProgressStep) Reset() { *x = ProgressStep{} - mi := &file_aether_proto_msgTypes[107] + mi := &file_aether_proto_msgTypes[119] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -13618,7 +15215,7 @@ func (x *ProgressStep) String() string { func (*ProgressStep) ProtoMessage() {} func (x *ProgressStep) ProtoReflect() protoreflect.Message { - mi := &file_aether_proto_msgTypes[107] + mi := &file_aether_proto_msgTypes[119] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -13631,7 +15228,7 @@ func (x *ProgressStep) ProtoReflect() protoreflect.Message { // Deprecated: Use ProgressStep.ProtoReflect.Descriptor instead. func (*ProgressStep) Descriptor() ([]byte, []int) { - return file_aether_proto_rawDescGZIP(), []int{107} + return file_aether_proto_rawDescGZIP(), []int{119} } func (x *ProgressStep) GetName() string { @@ -13708,7 +15305,7 @@ type ProgressUpdate struct { func (x *ProgressUpdate) Reset() { *x = ProgressUpdate{} - mi := &file_aether_proto_msgTypes[108] + mi := &file_aether_proto_msgTypes[120] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -13720,7 +15317,7 @@ func (x *ProgressUpdate) String() string { func (*ProgressUpdate) ProtoMessage() {} func (x *ProgressUpdate) ProtoReflect() protoreflect.Message { - mi := &file_aether_proto_msgTypes[108] + mi := &file_aether_proto_msgTypes[120] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -13733,7 +15330,7 @@ func (x *ProgressUpdate) ProtoReflect() protoreflect.Message { // Deprecated: Use ProgressUpdate.ProtoReflect.Descriptor instead. func (*ProgressUpdate) Descriptor() ([]byte, []int) { - return file_aether_proto_rawDescGZIP(), []int{108} + return file_aether_proto_rawDescGZIP(), []int{120} } func (x *ProgressUpdate) GetSource() string { @@ -13839,7 +15436,7 @@ type WorkflowOperation struct { func (x *WorkflowOperation) Reset() { *x = WorkflowOperation{} - mi := &file_aether_proto_msgTypes[109] + mi := &file_aether_proto_msgTypes[121] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -13851,7 +15448,7 @@ func (x *WorkflowOperation) String() string { func (*WorkflowOperation) ProtoMessage() {} func (x *WorkflowOperation) ProtoReflect() protoreflect.Message { - mi := &file_aether_proto_msgTypes[109] + mi := &file_aether_proto_msgTypes[121] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -13864,7 +15461,7 @@ func (x *WorkflowOperation) ProtoReflect() protoreflect.Message { // Deprecated: Use WorkflowOperation.ProtoReflect.Descriptor instead. func (*WorkflowOperation) Descriptor() ([]byte, []int) { - return file_aether_proto_rawDescGZIP(), []int{109} + return file_aether_proto_rawDescGZIP(), []int{121} } func (x *WorkflowOperation) GetOp() WorkflowOperation_OpType { @@ -13933,7 +15530,7 @@ type WorkflowResponse struct { func (x *WorkflowResponse) Reset() { *x = WorkflowResponse{} - mi := &file_aether_proto_msgTypes[110] + mi := &file_aether_proto_msgTypes[122] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -13945,7 +15542,7 @@ func (x *WorkflowResponse) String() string { func (*WorkflowResponse) ProtoMessage() {} func (x *WorkflowResponse) ProtoReflect() protoreflect.Message { - mi := &file_aether_proto_msgTypes[110] + mi := &file_aether_proto_msgTypes[122] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -13958,7 +15555,7 @@ func (x *WorkflowResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use WorkflowResponse.ProtoReflect.Descriptor instead. func (*WorkflowResponse) Descriptor() ([]byte, []int) { - return file_aether_proto_rawDescGZIP(), []int{110} + return file_aether_proto_rawDescGZIP(), []int{122} } func (x *WorkflowResponse) GetSuccess() bool { @@ -14031,14 +15628,28 @@ type MessageEnvelope struct { // IncomingMessage.workspace at delivery time so workflow engines and // metrics bridges (which subscribe to a workspace-agnostic fan-in shard) // can recover the originating workspace. - Workspace string `protobuf:"bytes,6,opt,name=workspace,proto3" json:"workspace,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + Workspace string `protobuf:"bytes,6,opt,name=workspace,proto3" json:"workspace,omitempty"` + // Gateway-set, spoof-proof resolved on-behalf-of subject. Populated ONLY + // when the sender's SendMessage carried an AuthorizationContext that the + // gateway resolved to an OBO subject (valid grant, delegate + audience + // match) — set from the resolved authority in routeMessage, the same way + // `source` is set from the authenticated sender and ProxyHttp mints X-Auth-* + // identity headers. Empty for direct (non-OBO) sends and older gateways. + // + // Lets a recipient identify the *user* a message was sent for, distinct from + // the sending *identity* in `source`. Example: the agent-harness sends from + // its agent identity but acts for a user; platform-bridge reads this to scope + // the tool registry to that user. Subject identity only — recipients that + // need to *act for* the subject use the task authority-grant path + // (CreateTaskResponse.authority_grant_id), not this field. + OnBehalfSubject *PrincipalRef `protobuf:"bytes,7,opt,name=on_behalf_subject,json=onBehalfSubject,proto3" json:"on_behalf_subject,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *MessageEnvelope) Reset() { *x = MessageEnvelope{} - mi := &file_aether_proto_msgTypes[111] + mi := &file_aether_proto_msgTypes[123] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -14050,7 +15661,7 @@ func (x *MessageEnvelope) String() string { func (*MessageEnvelope) ProtoMessage() {} func (x *MessageEnvelope) ProtoReflect() protoreflect.Message { - mi := &file_aether_proto_msgTypes[111] + mi := &file_aether_proto_msgTypes[123] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -14063,7 +15674,7 @@ func (x *MessageEnvelope) ProtoReflect() protoreflect.Message { // Deprecated: Use MessageEnvelope.ProtoReflect.Descriptor instead. func (*MessageEnvelope) Descriptor() ([]byte, []int) { - return file_aether_proto_rawDescGZIP(), []int{111} + return file_aether_proto_rawDescGZIP(), []int{123} } func (x *MessageEnvelope) GetSource() string { @@ -14108,6 +15719,13 @@ func (x *MessageEnvelope) GetWorkspace() string { return "" } +func (x *MessageEnvelope) GetOnBehalfSubject() *PrincipalRef { + if x != nil { + return x.OnBehalfSubject + } + return nil +} + // AuditQuery requests entries from the comprehensive audit log. // Requires system-level admin access or workspace-scoped read access. type AuditQuery struct { @@ -14140,7 +15758,7 @@ type AuditQuery struct { func (x *AuditQuery) Reset() { *x = AuditQuery{} - mi := &file_aether_proto_msgTypes[112] + mi := &file_aether_proto_msgTypes[124] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -14152,7 +15770,7 @@ func (x *AuditQuery) String() string { func (*AuditQuery) ProtoMessage() {} func (x *AuditQuery) ProtoReflect() protoreflect.Message { - mi := &file_aether_proto_msgTypes[112] + mi := &file_aether_proto_msgTypes[124] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -14165,7 +15783,7 @@ func (x *AuditQuery) ProtoReflect() protoreflect.Message { // Deprecated: Use AuditQuery.ProtoReflect.Descriptor instead. func (*AuditQuery) Descriptor() ([]byte, []int) { - return file_aether_proto_rawDescGZIP(), []int{112} + return file_aether_proto_rawDescGZIP(), []int{124} } func (x *AuditQuery) GetRequestId() string { @@ -14329,7 +15947,7 @@ type AuditQueryResponse struct { func (x *AuditQueryResponse) Reset() { *x = AuditQueryResponse{} - mi := &file_aether_proto_msgTypes[113] + mi := &file_aether_proto_msgTypes[125] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -14341,7 +15959,7 @@ func (x *AuditQueryResponse) String() string { func (*AuditQueryResponse) ProtoMessage() {} func (x *AuditQueryResponse) ProtoReflect() protoreflect.Message { - mi := &file_aether_proto_msgTypes[113] + mi := &file_aether_proto_msgTypes[125] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -14354,7 +15972,7 @@ func (x *AuditQueryResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use AuditQueryResponse.ProtoReflect.Descriptor instead. func (*AuditQueryResponse) Descriptor() ([]byte, []int) { - return file_aether_proto_rawDescGZIP(), []int{113} + return file_aether_proto_rawDescGZIP(), []int{125} } func (x *AuditQueryResponse) GetRequestId() string { @@ -14424,7 +16042,7 @@ type AuditEntry struct { func (x *AuditEntry) Reset() { *x = AuditEntry{} - mi := &file_aether_proto_msgTypes[114] + mi := &file_aether_proto_msgTypes[126] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -14436,7 +16054,7 @@ func (x *AuditEntry) String() string { func (*AuditEntry) ProtoMessage() {} func (x *AuditEntry) ProtoReflect() protoreflect.Message { - mi := &file_aether_proto_msgTypes[114] + mi := &file_aether_proto_msgTypes[126] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -14449,7 +16067,7 @@ func (x *AuditEntry) ProtoReflect() protoreflect.Message { // Deprecated: Use AuditEntry.ProtoReflect.Descriptor instead. func (*AuditEntry) Descriptor() ([]byte, []int) { - return file_aether_proto_rawDescGZIP(), []int{114} + return file_aether_proto_rawDescGZIP(), []int{126} } func (x *AuditEntry) GetAuditId() int64 { @@ -14636,7 +16254,7 @@ type SubmitAuditEventRequest struct { func (x *SubmitAuditEventRequest) Reset() { *x = SubmitAuditEventRequest{} - mi := &file_aether_proto_msgTypes[115] + mi := &file_aether_proto_msgTypes[127] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -14648,7 +16266,7 @@ func (x *SubmitAuditEventRequest) String() string { func (*SubmitAuditEventRequest) ProtoMessage() {} func (x *SubmitAuditEventRequest) ProtoReflect() protoreflect.Message { - mi := &file_aether_proto_msgTypes[115] + mi := &file_aether_proto_msgTypes[127] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -14661,7 +16279,7 @@ func (x *SubmitAuditEventRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use SubmitAuditEventRequest.ProtoReflect.Descriptor instead. func (*SubmitAuditEventRequest) Descriptor() ([]byte, []int) { - return file_aether_proto_rawDescGZIP(), []int{115} + return file_aether_proto_rawDescGZIP(), []int{127} } func (x *SubmitAuditEventRequest) GetEventType() string { @@ -14742,7 +16360,7 @@ type SubmitAuditEventResponse struct { func (x *SubmitAuditEventResponse) Reset() { *x = SubmitAuditEventResponse{} - mi := &file_aether_proto_msgTypes[116] + mi := &file_aether_proto_msgTypes[128] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -14754,7 +16372,7 @@ func (x *SubmitAuditEventResponse) String() string { func (*SubmitAuditEventResponse) ProtoMessage() {} func (x *SubmitAuditEventResponse) ProtoReflect() protoreflect.Message { - mi := &file_aether_proto_msgTypes[116] + mi := &file_aether_proto_msgTypes[128] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -14767,7 +16385,7 @@ func (x *SubmitAuditEventResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use SubmitAuditEventResponse.ProtoReflect.Descriptor instead. func (*SubmitAuditEventResponse) Descriptor() ([]byte, []int) { - return file_aether_proto_rawDescGZIP(), []int{116} + return file_aether_proto_rawDescGZIP(), []int{128} } func (x *SubmitAuditEventResponse) GetClientRequestId() string { @@ -14847,7 +16465,7 @@ type ProxyHttpRequest struct { func (x *ProxyHttpRequest) Reset() { *x = ProxyHttpRequest{} - mi := &file_aether_proto_msgTypes[117] + mi := &file_aether_proto_msgTypes[129] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -14859,7 +16477,7 @@ func (x *ProxyHttpRequest) String() string { func (*ProxyHttpRequest) ProtoMessage() {} func (x *ProxyHttpRequest) ProtoReflect() protoreflect.Message { - mi := &file_aether_proto_msgTypes[117] + mi := &file_aether_proto_msgTypes[129] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -14872,7 +16490,7 @@ func (x *ProxyHttpRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use ProxyHttpRequest.ProtoReflect.Descriptor instead. func (*ProxyHttpRequest) Descriptor() ([]byte, []int) { - return file_aether_proto_rawDescGZIP(), []int{117} + return file_aether_proto_rawDescGZIP(), []int{129} } func (x *ProxyHttpRequest) GetRequestId() string { @@ -15004,7 +16622,7 @@ type ProxyHttpResponse struct { func (x *ProxyHttpResponse) Reset() { *x = ProxyHttpResponse{} - mi := &file_aether_proto_msgTypes[118] + mi := &file_aether_proto_msgTypes[130] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -15016,7 +16634,7 @@ func (x *ProxyHttpResponse) String() string { func (*ProxyHttpResponse) ProtoMessage() {} func (x *ProxyHttpResponse) ProtoReflect() protoreflect.Message { - mi := &file_aether_proto_msgTypes[118] + mi := &file_aether_proto_msgTypes[130] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -15029,7 +16647,7 @@ func (x *ProxyHttpResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use ProxyHttpResponse.ProtoReflect.Descriptor instead. func (*ProxyHttpResponse) Descriptor() ([]byte, []int) { - return file_aether_proto_rawDescGZIP(), []int{118} + return file_aether_proto_rawDescGZIP(), []int{130} } func (x *ProxyHttpResponse) GetRequestId() string { @@ -15089,7 +16707,7 @@ type ProxyHttpBodyChunk struct { func (x *ProxyHttpBodyChunk) Reset() { *x = ProxyHttpBodyChunk{} - mi := &file_aether_proto_msgTypes[119] + mi := &file_aether_proto_msgTypes[131] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -15101,7 +16719,7 @@ func (x *ProxyHttpBodyChunk) String() string { func (*ProxyHttpBodyChunk) ProtoMessage() {} func (x *ProxyHttpBodyChunk) ProtoReflect() protoreflect.Message { - mi := &file_aether_proto_msgTypes[119] + mi := &file_aether_proto_msgTypes[131] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -15114,7 +16732,7 @@ func (x *ProxyHttpBodyChunk) ProtoReflect() protoreflect.Message { // Deprecated: Use ProxyHttpBodyChunk.ProtoReflect.Descriptor instead. func (*ProxyHttpBodyChunk) Descriptor() ([]byte, []int) { - return file_aether_proto_rawDescGZIP(), []int{119} + return file_aether_proto_rawDescGZIP(), []int{131} } func (x *ProxyHttpBodyChunk) GetRequestId() string { @@ -15164,7 +16782,7 @@ type ProxyError struct { func (x *ProxyError) Reset() { *x = ProxyError{} - mi := &file_aether_proto_msgTypes[120] + mi := &file_aether_proto_msgTypes[132] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -15176,7 +16794,7 @@ func (x *ProxyError) String() string { func (*ProxyError) ProtoMessage() {} func (x *ProxyError) ProtoReflect() protoreflect.Message { - mi := &file_aether_proto_msgTypes[120] + mi := &file_aether_proto_msgTypes[132] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -15189,7 +16807,7 @@ func (x *ProxyError) ProtoReflect() protoreflect.Message { // Deprecated: Use ProxyError.ProtoReflect.Descriptor instead. func (*ProxyError) Descriptor() ([]byte, []int) { - return file_aether_proto_rawDescGZIP(), []int{120} + return file_aether_proto_rawDescGZIP(), []int{132} } func (x *ProxyError) GetKind() ProxyError_Kind { @@ -15232,7 +16850,7 @@ type TunnelOpen struct { func (x *TunnelOpen) Reset() { *x = TunnelOpen{} - mi := &file_aether_proto_msgTypes[121] + mi := &file_aether_proto_msgTypes[133] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -15244,7 +16862,7 @@ func (x *TunnelOpen) String() string { func (*TunnelOpen) ProtoMessage() {} func (x *TunnelOpen) ProtoReflect() protoreflect.Message { - mi := &file_aether_proto_msgTypes[121] + mi := &file_aether_proto_msgTypes[133] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -15257,7 +16875,7 @@ func (x *TunnelOpen) ProtoReflect() protoreflect.Message { // Deprecated: Use TunnelOpen.ProtoReflect.Descriptor instead. func (*TunnelOpen) Descriptor() ([]byte, []int) { - return file_aether_proto_rawDescGZIP(), []int{121} + return file_aether_proto_rawDescGZIP(), []int{133} } func (x *TunnelOpen) GetTunnelId() string { @@ -15349,7 +16967,7 @@ type TunnelData struct { func (x *TunnelData) Reset() { *x = TunnelData{} - mi := &file_aether_proto_msgTypes[122] + mi := &file_aether_proto_msgTypes[134] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -15361,7 +16979,7 @@ func (x *TunnelData) String() string { func (*TunnelData) ProtoMessage() {} func (x *TunnelData) ProtoReflect() protoreflect.Message { - mi := &file_aether_proto_msgTypes[122] + mi := &file_aether_proto_msgTypes[134] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -15374,7 +16992,7 @@ func (x *TunnelData) ProtoReflect() protoreflect.Message { // Deprecated: Use TunnelData.ProtoReflect.Descriptor instead. func (*TunnelData) Descriptor() ([]byte, []int) { - return file_aether_proto_rawDescGZIP(), []int{122} + return file_aether_proto_rawDescGZIP(), []int{134} } func (x *TunnelData) GetTunnelId() string { @@ -15416,7 +17034,7 @@ type TunnelClose struct { func (x *TunnelClose) Reset() { *x = TunnelClose{} - mi := &file_aether_proto_msgTypes[123] + mi := &file_aether_proto_msgTypes[135] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -15428,7 +17046,7 @@ func (x *TunnelClose) String() string { func (*TunnelClose) ProtoMessage() {} func (x *TunnelClose) ProtoReflect() protoreflect.Message { - mi := &file_aether_proto_msgTypes[123] + mi := &file_aether_proto_msgTypes[135] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -15441,7 +17059,7 @@ func (x *TunnelClose) ProtoReflect() protoreflect.Message { // Deprecated: Use TunnelClose.ProtoReflect.Descriptor instead. func (*TunnelClose) Descriptor() ([]byte, []int) { - return file_aether_proto_rawDescGZIP(), []int{123} + return file_aether_proto_rawDescGZIP(), []int{135} } func (x *TunnelClose) GetTunnelId() string { @@ -15476,7 +17094,7 @@ type TunnelAck struct { func (x *TunnelAck) Reset() { *x = TunnelAck{} - mi := &file_aether_proto_msgTypes[124] + mi := &file_aether_proto_msgTypes[136] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -15488,7 +17106,7 @@ func (x *TunnelAck) String() string { func (*TunnelAck) ProtoMessage() {} func (x *TunnelAck) ProtoReflect() protoreflect.Message { - mi := &file_aether_proto_msgTypes[124] + mi := &file_aether_proto_msgTypes[136] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -15501,7 +17119,7 @@ func (x *TunnelAck) ProtoReflect() protoreflect.Message { // Deprecated: Use TunnelAck.ProtoReflect.Descriptor instead. func (*TunnelAck) Descriptor() ([]byte, []int) { - return file_aether_proto_rawDescGZIP(), []int{124} + return file_aether_proto_rawDescGZIP(), []int{136} } func (x *TunnelAck) GetTunnelId() string { @@ -15548,7 +17166,7 @@ type ResolveAuthorityRequest struct { func (x *ResolveAuthorityRequest) Reset() { *x = ResolveAuthorityRequest{} - mi := &file_aether_proto_msgTypes[125] + mi := &file_aether_proto_msgTypes[137] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -15560,7 +17178,7 @@ func (x *ResolveAuthorityRequest) String() string { func (*ResolveAuthorityRequest) ProtoMessage() {} func (x *ResolveAuthorityRequest) ProtoReflect() protoreflect.Message { - mi := &file_aether_proto_msgTypes[125] + mi := &file_aether_proto_msgTypes[137] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -15573,7 +17191,7 @@ func (x *ResolveAuthorityRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use ResolveAuthorityRequest.ProtoReflect.Descriptor instead. func (*ResolveAuthorityRequest) Descriptor() ([]byte, []int) { - return file_aether_proto_rawDescGZIP(), []int{125} + return file_aether_proto_rawDescGZIP(), []int{137} } func (x *ResolveAuthorityRequest) GetRequestId() string { @@ -15634,7 +17252,7 @@ type ResolveAuthorityResponse struct { func (x *ResolveAuthorityResponse) Reset() { *x = ResolveAuthorityResponse{} - mi := &file_aether_proto_msgTypes[126] + mi := &file_aether_proto_msgTypes[138] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -15646,7 +17264,7 @@ func (x *ResolveAuthorityResponse) String() string { func (*ResolveAuthorityResponse) ProtoMessage() {} func (x *ResolveAuthorityResponse) ProtoReflect() protoreflect.Message { - mi := &file_aether_proto_msgTypes[126] + mi := &file_aether_proto_msgTypes[138] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -15659,7 +17277,7 @@ func (x *ResolveAuthorityResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use ResolveAuthorityResponse.ProtoReflect.Descriptor instead. func (*ResolveAuthorityResponse) Descriptor() ([]byte, []int) { - return file_aether_proto_rawDescGZIP(), []int{126} + return file_aether_proto_rawDescGZIP(), []int{138} } func (x *ResolveAuthorityResponse) GetRequestId() string { @@ -15706,7 +17324,7 @@ type ResolvedAuthority struct { func (x *ResolvedAuthority) Reset() { *x = ResolvedAuthority{} - mi := &file_aether_proto_msgTypes[127] + mi := &file_aether_proto_msgTypes[139] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -15718,7 +17336,7 @@ func (x *ResolvedAuthority) String() string { func (*ResolvedAuthority) ProtoMessage() {} func (x *ResolvedAuthority) ProtoReflect() protoreflect.Message { - mi := &file_aether_proto_msgTypes[127] + mi := &file_aether_proto_msgTypes[139] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -15731,7 +17349,7 @@ func (x *ResolvedAuthority) ProtoReflect() protoreflect.Message { // Deprecated: Use ResolvedAuthority.ProtoReflect.Descriptor instead. func (*ResolvedAuthority) Descriptor() ([]byte, []int) { - return file_aether_proto_rawDescGZIP(), []int{127} + return file_aether_proto_rawDescGZIP(), []int{139} } func (x *ResolvedAuthority) GetActor() *PrincipalRef { @@ -15778,7 +17396,7 @@ type AuthorityGrantInfo struct { func (x *AuthorityGrantInfo) Reset() { *x = AuthorityGrantInfo{} - mi := &file_aether_proto_msgTypes[128] + mi := &file_aether_proto_msgTypes[140] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -15790,7 +17408,7 @@ func (x *AuthorityGrantInfo) String() string { func (*AuthorityGrantInfo) ProtoMessage() {} func (x *AuthorityGrantInfo) ProtoReflect() protoreflect.Message { - mi := &file_aether_proto_msgTypes[128] + mi := &file_aether_proto_msgTypes[140] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -15803,7 +17421,7 @@ func (x *AuthorityGrantInfo) ProtoReflect() protoreflect.Message { // Deprecated: Use AuthorityGrantInfo.ProtoReflect.Descriptor instead. func (*AuthorityGrantInfo) Descriptor() ([]byte, []int) { - return file_aether_proto_rawDescGZIP(), []int{128} + return file_aether_proto_rawDescGZIP(), []int{140} } func (x *AuthorityGrantInfo) GetGrantId() string { @@ -15896,7 +17514,7 @@ type ConnectionStatusRequest struct { func (x *ConnectionStatusRequest) Reset() { *x = ConnectionStatusRequest{} - mi := &file_aether_proto_msgTypes[129] + mi := &file_aether_proto_msgTypes[141] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -15908,7 +17526,7 @@ func (x *ConnectionStatusRequest) String() string { func (*ConnectionStatusRequest) ProtoMessage() {} func (x *ConnectionStatusRequest) ProtoReflect() protoreflect.Message { - mi := &file_aether_proto_msgTypes[129] + mi := &file_aether_proto_msgTypes[141] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -15921,7 +17539,7 @@ func (x *ConnectionStatusRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use ConnectionStatusRequest.ProtoReflect.Descriptor instead. func (*ConnectionStatusRequest) Descriptor() ([]byte, []int) { - return file_aether_proto_rawDescGZIP(), []int{129} + return file_aether_proto_rawDescGZIP(), []int{141} } func (x *ConnectionStatusRequest) GetRequestId() string { @@ -15954,7 +17572,7 @@ type ConnectionStatusResponse struct { func (x *ConnectionStatusResponse) Reset() { *x = ConnectionStatusResponse{} - mi := &file_aether_proto_msgTypes[130] + mi := &file_aether_proto_msgTypes[142] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -15966,7 +17584,7 @@ func (x *ConnectionStatusResponse) String() string { func (*ConnectionStatusResponse) ProtoMessage() {} func (x *ConnectionStatusResponse) ProtoReflect() protoreflect.Message { - mi := &file_aether_proto_msgTypes[130] + mi := &file_aether_proto_msgTypes[142] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -15979,7 +17597,7 @@ func (x *ConnectionStatusResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use ConnectionStatusResponse.ProtoReflect.Descriptor instead. func (*ConnectionStatusResponse) Descriptor() ([]byte, []int) { - return file_aether_proto_rawDescGZIP(), []int{130} + return file_aether_proto_rawDescGZIP(), []int{142} } func (x *ConnectionStatusResponse) GetRequestId() string { @@ -16045,7 +17663,7 @@ type TaskSubscriptionOperation struct { func (x *TaskSubscriptionOperation) Reset() { *x = TaskSubscriptionOperation{} - mi := &file_aether_proto_msgTypes[131] + mi := &file_aether_proto_msgTypes[143] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -16057,7 +17675,7 @@ func (x *TaskSubscriptionOperation) String() string { func (*TaskSubscriptionOperation) ProtoMessage() {} func (x *TaskSubscriptionOperation) ProtoReflect() protoreflect.Message { - mi := &file_aether_proto_msgTypes[131] + mi := &file_aether_proto_msgTypes[143] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -16070,7 +17688,7 @@ func (x *TaskSubscriptionOperation) ProtoReflect() protoreflect.Message { // Deprecated: Use TaskSubscriptionOperation.ProtoReflect.Descriptor instead. func (*TaskSubscriptionOperation) Descriptor() ([]byte, []int) { - return file_aether_proto_rawDescGZIP(), []int{131} + return file_aether_proto_rawDescGZIP(), []int{143} } func (x *TaskSubscriptionOperation) GetOp() TaskSubscriptionOperation_OpType { @@ -16131,7 +17749,7 @@ type TaskSubscriptionOperationResponse struct { func (x *TaskSubscriptionOperationResponse) Reset() { *x = TaskSubscriptionOperationResponse{} - mi := &file_aether_proto_msgTypes[132] + mi := &file_aether_proto_msgTypes[144] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -16143,7 +17761,7 @@ func (x *TaskSubscriptionOperationResponse) String() string { func (*TaskSubscriptionOperationResponse) ProtoMessage() {} func (x *TaskSubscriptionOperationResponse) ProtoReflect() protoreflect.Message { - mi := &file_aether_proto_msgTypes[132] + mi := &file_aether_proto_msgTypes[144] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -16156,7 +17774,7 @@ func (x *TaskSubscriptionOperationResponse) ProtoReflect() protoreflect.Message // Deprecated: Use TaskSubscriptionOperationResponse.ProtoReflect.Descriptor instead. func (*TaskSubscriptionOperationResponse) Descriptor() ([]byte, []int) { - return file_aether_proto_rawDescGZIP(), []int{132} + return file_aether_proto_rawDescGZIP(), []int{144} } func (x *TaskSubscriptionOperationResponse) GetSuccess() bool { @@ -16218,7 +17836,7 @@ type TaskEvent struct { func (x *TaskEvent) Reset() { *x = TaskEvent{} - mi := &file_aether_proto_msgTypes[133] + mi := &file_aether_proto_msgTypes[145] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -16230,7 +17848,7 @@ func (x *TaskEvent) String() string { func (*TaskEvent) ProtoMessage() {} func (x *TaskEvent) ProtoReflect() protoreflect.Message { - mi := &file_aether_proto_msgTypes[133] + mi := &file_aether_proto_msgTypes[145] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -16243,7 +17861,7 @@ func (x *TaskEvent) ProtoReflect() protoreflect.Message { // Deprecated: Use TaskEvent.ProtoReflect.Descriptor instead. func (*TaskEvent) Descriptor() ([]byte, []int) { - return file_aether_proto_rawDescGZIP(), []int{133} + return file_aether_proto_rawDescGZIP(), []int{145} } func (x *TaskEvent) GetTaskId() string { @@ -16365,7 +17983,7 @@ type TaskStatusChangedEvent struct { func (x *TaskStatusChangedEvent) Reset() { *x = TaskStatusChangedEvent{} - mi := &file_aether_proto_msgTypes[134] + mi := &file_aether_proto_msgTypes[146] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -16377,7 +17995,7 @@ func (x *TaskStatusChangedEvent) String() string { func (*TaskStatusChangedEvent) ProtoMessage() {} func (x *TaskStatusChangedEvent) ProtoReflect() protoreflect.Message { - mi := &file_aether_proto_msgTypes[134] + mi := &file_aether_proto_msgTypes[146] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -16390,7 +18008,7 @@ func (x *TaskStatusChangedEvent) ProtoReflect() protoreflect.Message { // Deprecated: Use TaskStatusChangedEvent.ProtoReflect.Descriptor instead. func (*TaskStatusChangedEvent) Descriptor() ([]byte, []int) { - return file_aether_proto_rawDescGZIP(), []int{134} + return file_aether_proto_rawDescGZIP(), []int{146} } func (x *TaskStatusChangedEvent) GetFromStatus() TaskStatus { @@ -16428,7 +18046,7 @@ type TaskProgressEvent struct { func (x *TaskProgressEvent) Reset() { *x = TaskProgressEvent{} - mi := &file_aether_proto_msgTypes[135] + mi := &file_aether_proto_msgTypes[147] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -16440,7 +18058,7 @@ func (x *TaskProgressEvent) String() string { func (*TaskProgressEvent) ProtoMessage() {} func (x *TaskProgressEvent) ProtoReflect() protoreflect.Message { - mi := &file_aether_proto_msgTypes[135] + mi := &file_aether_proto_msgTypes[147] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -16453,7 +18071,7 @@ func (x *TaskProgressEvent) ProtoReflect() protoreflect.Message { // Deprecated: Use TaskProgressEvent.ProtoReflect.Descriptor instead. func (*TaskProgressEvent) Descriptor() ([]byte, []int) { - return file_aether_proto_rawDescGZIP(), []int{135} + return file_aether_proto_rawDescGZIP(), []int{147} } func (x *TaskProgressEvent) GetState() string { @@ -16498,7 +18116,7 @@ type TaskChildLifecycleEvent struct { func (x *TaskChildLifecycleEvent) Reset() { *x = TaskChildLifecycleEvent{} - mi := &file_aether_proto_msgTypes[136] + mi := &file_aether_proto_msgTypes[148] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -16510,7 +18128,7 @@ func (x *TaskChildLifecycleEvent) String() string { func (*TaskChildLifecycleEvent) ProtoMessage() {} func (x *TaskChildLifecycleEvent) ProtoReflect() protoreflect.Message { - mi := &file_aether_proto_msgTypes[136] + mi := &file_aether_proto_msgTypes[148] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -16523,7 +18141,7 @@ func (x *TaskChildLifecycleEvent) ProtoReflect() protoreflect.Message { // Deprecated: Use TaskChildLifecycleEvent.ProtoReflect.Descriptor instead. func (*TaskChildLifecycleEvent) Descriptor() ([]byte, []int) { - return file_aether_proto_rawDescGZIP(), []int{136} + return file_aether_proto_rawDescGZIP(), []int{148} } func (x *TaskChildLifecycleEvent) GetChildTaskId() string { @@ -16559,7 +18177,7 @@ type TaskAuthorityRequestEventRelay struct { func (x *TaskAuthorityRequestEventRelay) Reset() { *x = TaskAuthorityRequestEventRelay{} - mi := &file_aether_proto_msgTypes[137] + mi := &file_aether_proto_msgTypes[149] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -16571,7 +18189,7 @@ func (x *TaskAuthorityRequestEventRelay) String() string { func (*TaskAuthorityRequestEventRelay) ProtoMessage() {} func (x *TaskAuthorityRequestEventRelay) ProtoReflect() protoreflect.Message { - mi := &file_aether_proto_msgTypes[137] + mi := &file_aether_proto_msgTypes[149] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -16584,7 +18202,7 @@ func (x *TaskAuthorityRequestEventRelay) ProtoReflect() protoreflect.Message { // Deprecated: Use TaskAuthorityRequestEventRelay.ProtoReflect.Descriptor instead. func (*TaskAuthorityRequestEventRelay) Descriptor() ([]byte, []int) { - return file_aether_proto_rawDescGZIP(), []int{137} + return file_aether_proto_rawDescGZIP(), []int{149} } func (x *TaskAuthorityRequestEventRelay) GetEvent() *AuthorityRequestEvent { @@ -16760,10 +18378,11 @@ const file_aether_proto_rawDesc = "" + "\x12supported_profiles\x18\x03 \x03(\tR\x11supportedProfiles\"V\n" + "\x0eBridgeIdentity\x12&\n" + "\x0eimplementation\x18\x01 \x01(\tR\x0eimplementation\x12\x1c\n" + - "\tspecifier\x18\x02 \x01(\tR\tspecifier\"W\n" + + "\tspecifier\x18\x02 \x01(\tR\tspecifier\"\x81\x01\n" + "\x0fServiceIdentity\x12&\n" + "\x0eimplementation\x18\x01 \x01(\tR\x0eimplementation\x12\x1c\n" + - "\tspecifier\x18\x02 \x01(\tR\tspecifier\"s\n" + + "\tspecifier\x18\x02 \x01(\tR\tspecifier\x12(\n" + + "\x10no_pool_consumer\x18\x03 \x01(\bR\x0enoPoolConsumer\"s\n" + "\rAgentIdentity\x12\x1c\n" + "\tworkspace\x18\x01 \x01(\tR\tworkspace\x12&\n" + "\x0eimplementation\x18\x02 \x01(\tR\x0eimplementation\x12\x1c\n" + @@ -16811,7 +18430,7 @@ const file_aether_proto_rawDesc = "" + "\x04kind\x18\x02 \x01(\tR\x04kind\x12\x10\n" + "\x03qty\x18\x03 \x01(\x01R\x03qty\";\n" + "\x0fSwitchWorkspace\x12(\n" + - "\x10new_workspace_id\x18\x01 \x01(\tR\x0enewWorkspaceId\"\xb2\x05\n" + + "\x10new_workspace_id\x18\x01 \x01(\tR\x0enewWorkspaceId\"\x99\a\n" + "\vKVOperation\x12-\n" + "\x02op\x18\x01 \x01(\x0e2\x1d.aether.v1.KVOperation.OpTypeR\x02op\x122\n" + "\x05scope\x18\x02 \x01(\x0e2\x1c.aether.v1.KVOperation.ScopeR\x05scope\x12\x10\n" + @@ -16827,7 +18446,11 @@ const file_aether_proto_rawDesc = "" + " \x01(\x03R\n" + "guardValue\x12\x1f\n" + "\vdelta_value\x18\v \x01(\x03R\n" + - "deltaValue\"r\n" + + "deltaValue\x12%\n" + + "\x0eexpected_value\x18\f \x01(\fR\rexpectedValue\x12\x14\n" + + "\x05limit\x18\r \x01(\x05R\x05limit\x12\x16\n" + + "\x06cursor\x18\x0e \x01(\tR\x06cursor\x12'\n" + + "\x0ftarget_identity\x18\x0f \x01(\tR\x0etargetIdentity\"\xda\x01\n" + "\x06OpType\x12\a\n" + "\x03GET\x10\x00\x12\a\n" + "\x03PUT\x10\x01\x12\b\n" + @@ -16837,7 +18460,15 @@ const file_aether_proto_rawDesc = "" + "\tINCREMENT\x10\x04\x12\r\n" + "\tDECREMENT\x10\x05\x12\x10\n" + "\fINCREMENT_IF\x10\x06\x12\x10\n" + - "\fDECREMENT_IF\x10\a\"\xb2\x01\n" + + "\fDECREMENT_IF\x10\a\x12\n" + + "\n" + + "\x06SET_NX\x10\b\x12\x13\n" + + "\x0fCOMPARE_AND_SET\x10\t\x12\x16\n" + + "\x12COMPARE_AND_DELETE\x10\n" + + "\x12\x12\n" + + "\x0ePURGE_IDENTITY\x10\v\x12\v\n" + + "\aSET_ADD\x10\f\x12\f\n" + + "\bSET_CARD\x10\r\"\xb2\x01\n" + "\x05Scope\x12\x15\n" + "\x11SCOPE_UNSPECIFIED\x10\x00\x12\n" + "\n" + @@ -16848,7 +18479,7 @@ const file_aether_proto_rawDesc = "" + "\x10GLOBAL_EXCLUSIVE\x10\x05\x12\x17\n" + "\x13WORKSPACE_EXCLUSIVE\x10\x06\x12\x0f\n" + "\vUSER_SHARED\x10\a\x12\x19\n" + - "\x15USER_WORKSPACE_SHARED\x10\b\"\xa1\x02\n" + + "\x15USER_WORKSPACE_SHARED\x10\b\"\xdd\x02\n" + "\n" + "KVResponse\x12\x18\n" + "\asuccess\x18\x01 \x01(\bR\asuccess\x12\x14\n" + @@ -16858,16 +18489,20 @@ const file_aether_proto_rawDesc = "" + "\n" + "request_id\x18\x05 \x01(\tR\trequestId\x12#\n" + "\rcounter_value\x18\x06 \x01(\x03R\fcounterValue\x12\x18\n" + - "\aapplied\x18\a \x01(\bR\aapplied\x1a8\n" + + "\aapplied\x18\a \x01(\bR\aapplied\x12\x1f\n" + + "\vnext_cursor\x18\b \x01(\tR\n" + + "nextCursor\x12\x19\n" + + "\bhas_more\x18\t \x01(\bR\ahasMore\x1a8\n" + "\n" + "KvMapEntry\x12\x10\n" + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + - "\x05value\x18\x02 \x01(\fR\x05value:\x028\x01\"\xa7\x01\n" + + "\x05value\x18\x02 \x01(\fR\x05value:\x028\x01\"\xec\x01\n" + "\x0fIncomingMessage\x12!\n" + "\fsource_topic\x18\x01 \x01(\tR\vsourceTopic\x12\x18\n" + "\apayload\x18\x02 \x01(\fR\apayload\x129\n" + "\fmessage_type\x18\x03 \x01(\x0e2\x16.aether.v1.MessageTypeR\vmessageType\x12\x1c\n" + - "\tworkspace\x18\x04 \x01(\tR\tworkspace\"\xf0\x05\n" + + "\tworkspace\x18\x04 \x01(\tR\tworkspace\x12C\n" + + "\x11on_behalf_subject\x18\x05 \x01(\v2\x17.aether.v1.PrincipalRefR\x0fonBehalfSubject\"\xf0\x05\n" + "\x0eConfigSnapshot\x125\n" + "\x02kv\x18\x01 \x03(\v2!.aether.v1.ConfigSnapshot.KvEntryB\x02\x18\x01R\x02kv\x12H\n" + "\tglobal_kv\x18\x02 \x03(\v2'.aether.v1.ConfigSnapshot.GlobalKvEntryB\x02\x18\x01R\bglobalKv\x12M\n" + @@ -16902,7 +18537,24 @@ const file_aether_proto_rawDesc = "" + "\tretryable\x18\x03 \x01(\bR\tretryable\x12$\n" + "\x0eretry_after_ms\x18\x04 \x01(\x03R\fretryAfterMs\x12\x1d\n" + "\n" + - "request_id\x18\x05 \x01(\tR\trequestId\"\xac\x06\n" + + "request_id\x18\x05 \x01(\tR\trequestId\"\xda\x02\n" + + "\vRetryPolicy\x12!\n" + + "\fmax_attempts\x18\x01 \x01(\x05R\vmaxAttempts\x124\n" + + "\abackoff\x18\x02 \x01(\x0e2\x1a.aether.v1.BackoffStrategyR\abackoff\x12(\n" + + "\x10initial_delay_ms\x18\x03 \x01(\x03R\x0einitialDelayMs\x12 \n" + + "\fmax_delay_ms\x18\x04 \x01(\x03R\n" + + "maxDelayMs\x12#\n" + + "\rjitter_factor\x18\x05 \x01(\x01R\fjitterFactor\x12\x1f\n" + + "\vschedule_ms\x18\x06 \x03(\x03R\n" + + "scheduleMs\x124\n" + + "\x16retryable_status_codes\x18\a \x03(\x05R\x14retryableStatusCodes\x12*\n" + + "\x11honor_retry_after\x18\b \x01(\bR\x0fhonorRetryAfter\"\x86\x01\n" + + "\x13TaskCompletionEvent\x12\x18\n" + + "\aenabled\x18\x01 \x01(\bR\aenabled\x12\x1d\n" + + "\n" + + "event_name\x18\x02 \x01(\tR\teventName\x126\n" + + "\von_statuses\x18\x03 \x03(\x0e2\x15.aether.v1.TaskStatusR\n" + + "onStatuses\"\xd9\b\n" + "\x11CreateTaskRequest\x12\x1b\n" + "\ttask_type\x18\x01 \x01(\tR\btaskType\x12\x1c\n" + "\tworkspace\x18\x02 \x01(\tR\tworkspace\x12F\n" + @@ -16920,7 +18572,14 @@ const file_aether_proto_rawDesc = "" + "\n" + "task_class\x18\f \x01(\x0e2\x14.aether.v1.TaskClassR\ttaskClass\x12\x1d\n" + "\n" + - "context_id\x18\r \x01(\tR\tcontextId\x1aG\n" + + "context_id\x18\r \x01(\tR\tcontextId\x129\n" + + "\fretry_policy\x18\x0e \x01(\v2\x16.aether.v1.RetryPolicyR\vretryPolicy\x123\n" + + "\bpriority\x18\x0f \x01(\x0e2\x17.aether.v1.TaskPriorityR\bpriority\x12'\n" + + "\x0fidempotency_key\x18\x10 \x01(\tR\x0eidempotencyKey\x12%\n" + + "\x0ecorrelation_id\x18\x11 \x01(\tR\rcorrelationId\x12 \n" + + "\froot_task_id\x18\x12 \x01(\tR\n" + + "rootTaskId\x12I\n" + + "\x10completion_event\x18\x13 \x01(\v2\x1e.aether.v1.TaskCompletionEventR\x0fcompletionEvent\x1aG\n" + "\x19LaunchParamOverridesEntry\x12\x10\n" + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\x1a;\n" + @@ -17112,7 +18771,7 @@ const file_aether_proto_rawDesc = "" + "request_id\x18\x04 \x01(\tR\trequestId\"\x1b\n" + "\x06OpType\x12\b\n" + "\x04LIST\x10\x00\x12\a\n" + - "\x03GET\x10\x01\"\xfb\x06\n" + + "\x03GET\x10\x01\"\xb5\b\n" + "\n" + "TaskFilter\x12-\n" + "\x06status\x18\x01 \x01(\x0e2\x15.aether.v1.TaskStatusR\x06status\x12\x1c\n" + @@ -17139,7 +18798,12 @@ const file_aether_proto_rawDesc = "" + "\x1estatus_timestamp_after_unix_ms\x18\x12 \x01(\x03R\x1astatusTimestampAfterUnixMs\x12\x1d\n" + "\n" + "page_token\x18\x13 \x01(\tR\tpageToken\x12/\n" + - "\x13include_descendants\x18\x14 \x01(\bR\x12includeDescendants\"\xc1\t\n" + + "\x13include_descendants\x18\x14 \x01(\bR\x12includeDescendants\x123\n" + + "\bpriority\x18\x15 \x01(\x0e2\x17.aether.v1.TaskPriorityR\bpriority\x12:\n" + + "\fmin_priority\x18\x16 \x01(\x0e2\x17.aether.v1.TaskPriorityR\vminPriority\x12%\n" + + "\x0ecorrelation_id\x18\x17 \x01(\tR\rcorrelationId\x12 \n" + + "\froot_task_id\x18\x18 \x01(\tR\n" + + "rootTaskId\"\x8a\v\n" + "\bTaskInfo\x12\x17\n" + "\atask_id\x18\x01 \x01(\tR\x06taskId\x12\x1b\n" + "\ttask_type\x18\x02 \x01(\tR\btaskType\x12-\n" + @@ -17178,7 +18842,12 @@ const file_aether_proto_rawDesc = "" + "depends_on\x18\x1c \x03(\tR\tdependsOn\x12\x1d\n" + "\n" + "context_id\x18\x1d \x01(\tR\tcontextId\x12\x1b\n" + - "\tpaused_at\x18\x1e \x01(\x03R\bpausedAt\x1a;\n" + + "\tpaused_at\x18\x1e \x01(\x03R\bpausedAt\x123\n" + + "\bpriority\x18\x1f \x01(\x0e2\x17.aether.v1.TaskPriorityR\bpriority\x12%\n" + + "\x0ecorrelation_id\x18 \x01(\tR\rcorrelationId\x12 \n" + + "\froot_task_id\x18! \x01(\tR\n" + + "rootTaskId\x12I\n" + + "\x10completion_event\x18\" \x01(\v2\x1e.aether.v1.TaskCompletionEventR\x0fcompletionEvent\x1a;\n" + "\rMetadataEntry\x12\x10\n" + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"\xff\x01\n" + @@ -17191,14 +18860,14 @@ const file_aether_proto_rawDesc = "" + "totalCount\x12\x1d\n" + "\n" + "request_id\x18\x06 \x01(\tR\trequestId\x12&\n" + - "\x0fnext_page_token\x18\a \x01(\tR\rnextPageToken\"\xac\x02\n" + + "\x0fnext_page_token\x18\a \x01(\tR\rnextPageToken\"\xb7\x02\n" + "\rTaskOperation\x12/\n" + "\x02op\x18\x01 \x01(\x0e2\x1f.aether.v1.TaskOperation.OpTypeR\x02op\x12\x17\n" + "\atask_id\x18\x02 \x01(\tR\x06taskId\x12\x16\n" + "\x06reason\x18\x03 \x01(\tR\x06reason\x12\x1d\n" + "\n" + "request_id\x18\x04 \x01(\tR\trequestId\x120\n" + - "\twait_spec\x18\x05 \x01(\v2\x13.aether.v1.WaitSpecR\bwaitSpec\"h\n" + + "\twait_spec\x18\x05 \x01(\v2\x13.aether.v1.WaitSpecR\bwaitSpec\"s\n" + "\x06OpType\x12\t\n" + "\x05RETRY\x10\x00\x12\n" + "\n" + @@ -17210,7 +18879,8 @@ const file_aether_proto_rawDesc = "" + "\n" + "\x06RESUME\x10\x06\x12\n" + "\n" + - "\x06REJECT\x10\a\"\xf6\x03\n" + + "\x06REJECT\x10\a\x12\t\n" + + "\x05CLAIM\x10\b\"\xf6\x03\n" + "\bWaitSpec\x12-\n" + "\x06reason\x18\x01 \x01(\x0e2\x15.aether.v1.WaitReasonR\x06reason\x12-\n" + "\x12expected_principal\x18\x02 \x01(\tR\x11expectedPrincipal\x12D\n" + @@ -17381,7 +19051,7 @@ const file_aether_proto_rawDesc = "" + "\rorchestrators\x18\a \x03(\v2\x1b.aether.v1.OrchestratorInfoR\rorchestrators\x12A\n" + "\rlaunch_result\x18\b \x01(\v2\x1c.aether.v1.AgentLaunchResultR\flaunchResult\x12\x1d\n" + "\n" + - "request_id\x18\t \x01(\tR\trequestId\"\xb0\a\n" + + "request_id\x18\t \x01(\tR\trequestId\"\x9e\x0e\n" + "\fACLOperation\x12.\n" + "\x02op\x18\x01 \x01(\x0e2\x1e.aether.v1.ACLOperation.OpTypeR\x02op\x12\x17\n" + "\arule_id\x18\x02 \x01(\tR\x06ruleId\x12#\n" + @@ -17394,7 +19064,18 @@ const file_aether_proto_rawDesc = "" + "\rgrant_request\x18\f \x01(\v2\x1a.aether.v1.ACLGrantRequestR\fgrantRequest\x12K\n" + "\x10fallback_request\x18\x0e \x01(\v2 .aether.v1.ACLSetFallbackRequestR\x0ffallbackRequest\x12\x1d\n" + "\n" + - "request_id\x18\x13 \x01(\tR\trequestId\"\xd0\x02\n" + + "request_id\x18\x13 \x01(\tR\trequestId\x12\x12\n" + + "\x04name\x18\x14 \x01(\tR\x04name\x125\n" + + "\tprincipal\x18\x15 \x01(\v2\x17.aether.v1.PrincipalRefR\tprincipal\x12?\n" + + "\rgroup_request\x18\x16 \x01(\v2\x1a.aether.v1.ACLGroupRequestR\fgroupRequest\x12<\n" + + "\frole_request\x18\x17 \x01(\v2\x19.aether.v1.ACLRoleRequestR\vroleRequest\x12G\n" + + "\x0emember_request\x18\x18 \x01(\v2 .aether.v1.ACLGroupMemberRequestR\rmemberRequest\x12R\n" + + "\x12assignment_request\x18\x19 \x01(\v2#.aether.v1.ACLRoleAssignmentRequestR\x11assignmentRequest\x12#\n" + + "\rresource_type\x18\x1a \x01(\tR\fresourceType\x12\x1f\n" + + "\vresource_id\x18\x1b \x01(\tR\n" + + "resourceId\x12%\n" + + "\x0erequired_level\x18\x1c \x01(\x05R\rrequiredLevel\x12E\n" + + "\rauthorization\x18\x1d \x01(\v2\x1f.aether.v1.AuthorizationContextR\rauthorization\"\xa3\x05\n" + "\x06OpType\x12\x0e\n" + "\n" + "LIST_RULES\x10\x00\x12\f\n" + @@ -17407,7 +19088,25 @@ const file_aether_proto_rawDesc = "" + "\x13SET_FALLBACK_POLICY\x10\t\x12\x13\n" + "\x0fCLEANUP_EXPIRED\x10\n" + "\x12\x16\n" + - "\x12CLEANUP_AUDIT_LOGS\x10\v\"\x04\b\x05\x10\x05\"\x04\b\x06\x10\x06\"\x04\b\a\x10\a\"\x04\b\f\x10\f\"\x04\b\r\x10\r\"\x04\b\x0e\x10\x0e\"\x04\b\x0f\x10\x0f\"\x04\b\x10\x10\x10*\x15LIST_AUTHORITY_GRANTS*\x13GET_AUTHORITY_GRANT*\x16CREATE_AUTHORITY_GRANT*\x15RENEW_AUTHORITY_GRANT*\x16REVOKE_AUTHORITY_GRANTJ\x04\b\x03\x10\x04J\x04\b\x06\x10\aJ\x04\b\a\x10\bJ\x04\b\r\x10\x0eJ\x04\b\b\x10\tJ\x04\b\x10\x10\x11J\x04\b\x11\x10\x12J\x04\b\x12\x10\x13R\x12authority_grant_idR\x16authority_grant_filterR\x17authority_grant_requestR\x1dauthority_grant_renew_request\"\xcd\x01\n" + + "\x12CLEANUP_AUDIT_LOGS\x10\v\x12\x10\n" + + "\fCREATE_GROUP\x10\x11\x12\x10\n" + + "\fDELETE_GROUP\x10\x12\x12\r\n" + + "\tGET_GROUP\x10\x13\x12\x0f\n" + + "\vLIST_GROUPS\x10\x14\x12\x14\n" + + "\x10ADD_GROUP_MEMBER\x10\x15\x12\x17\n" + + "\x13REMOVE_GROUP_MEMBER\x10\x16\x12\x16\n" + + "\x12LIST_GROUP_MEMBERS\x10\x17\x12\x0f\n" + + "\vCREATE_ROLE\x10\x18\x12\x0f\n" + + "\vDELETE_ROLE\x10\x19\x12\f\n" + + "\bGET_ROLE\x10\x1a\x12\x0e\n" + + "\n" + + "LIST_ROLES\x10\x1b\x12\x0f\n" + + "\vASSIGN_ROLE\x10\x1c\x12\x11\n" + + "\rUNASSIGN_ROLE\x10\x1d\x12\x19\n" + + "\x15LIST_ROLE_ASSIGNMENTS\x10\x1e\x12\x19\n" + + "\x15LIST_PRINCIPAL_GROUPS\x10\x1f\x12\x18\n" + + "\x14LIST_PRINCIPAL_ROLES\x10 \x12\x12\n" + + "\x0eEXPLAIN_ACCESS\x10!\"\x04\b\x05\x10\x05\"\x04\b\x06\x10\x06\"\x04\b\a\x10\a\"\x04\b\f\x10\f\"\x04\b\r\x10\r\"\x04\b\x0e\x10\x0e\"\x04\b\x0f\x10\x0f\"\x04\b\x10\x10\x10*\x15LIST_AUTHORITY_GRANTS*\x13GET_AUTHORITY_GRANT*\x16CREATE_AUTHORITY_GRANT*\x15RENEW_AUTHORITY_GRANT*\x16REVOKE_AUTHORITY_GRANTJ\x04\b\x03\x10\x04J\x04\b\x06\x10\aJ\x04\b\a\x10\bJ\x04\b\r\x10\x0eJ\x04\b\b\x10\tJ\x04\b\x10\x10\x11J\x04\b\x11\x10\x12J\x04\b\x12\x10\x13R\x12authority_grant_idR\x16authority_grant_filterR\x17authority_grant_requestR\x1dauthority_grant_renew_request\"\xcd\x01\n" + "\rACLRuleFilter\x12%\n" + "\x0eprincipal_type\x18\x01 \x01(\tR\rprincipalType\x12!\n" + "\fprincipal_id\x18\x02 \x01(\tR\vprincipalId\x12#\n" + @@ -17584,7 +19283,104 @@ const file_aether_proto_rawDesc = "" + "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"Q\n" + "\x10ACLCleanupResult\x12#\n" + "\rdeleted_count\x18\x01 \x01(\x03R\fdeletedCount\x12\x18\n" + - "\amessage\x18\x02 \x01(\tR\amessage\"\xc7\x05\n" + + "\amessage\x18\x02 \x01(\tR\amessage\"\xe9\x01\n" + + "\x0fACLGroupRequest\x12\x12\n" + + "\x04name\x18\x01 \x01(\tR\x04name\x12 \n" + + "\vdescription\x18\x02 \x01(\tR\vdescription\x12\x1d\n" + + "\n" + + "created_by\x18\x03 \x01(\tR\tcreatedBy\x12D\n" + + "\bmetadata\x18\x04 \x03(\v2(.aether.v1.ACLGroupRequest.MetadataEntryR\bmetadata\x1a;\n" + + "\rMetadataEntry\x12\x10\n" + + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + + "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"\xe7\x01\n" + + "\x0eACLRoleRequest\x12\x12\n" + + "\x04name\x18\x01 \x01(\tR\x04name\x12 \n" + + "\vdescription\x18\x02 \x01(\tR\vdescription\x12\x1d\n" + + "\n" + + "created_by\x18\x03 \x01(\tR\tcreatedBy\x12C\n" + + "\bmetadata\x18\x04 \x03(\v2'.aether.v1.ACLRoleRequest.MetadataEntryR\bmetadata\x1a;\n" + + "\rMetadataEntry\x12\x10\n" + + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + + "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"\x93\x01\n" + + "\x15ACLGroupMemberRequest\x12\x1f\n" + + "\vmember_type\x18\x01 \x01(\tR\n" + + "memberType\x12\x1b\n" + + "\tmember_id\x18\x02 \x01(\tR\bmemberId\x12\x1d\n" + + "\n" + + "granted_by\x18\x03 \x01(\tR\tgrantedBy\x12\x1d\n" + + "\n" + + "expires_at\x18\x04 \x01(\x03R\texpiresAt\"\x9e\x01\n" + + "\x18ACLRoleAssignmentRequest\x12#\n" + + "\rassignee_type\x18\x01 \x01(\tR\fassigneeType\x12\x1f\n" + + "\vassignee_id\x18\x02 \x01(\tR\n" + + "assigneeId\x12\x1d\n" + + "\n" + + "granted_by\x18\x03 \x01(\tR\tgrantedBy\x12\x1d\n" + + "\n" + + "expires_at\x18\x04 \x01(\x03R\texpiresAt\"\xa8\x02\n" + + "\fACLGroupInfo\x12\x19\n" + + "\bgroup_id\x18\x01 \x01(\tR\agroupId\x12\x1d\n" + + "\n" + + "group_name\x18\x02 \x01(\tR\tgroupName\x12 \n" + + "\vdescription\x18\x03 \x01(\tR\vdescription\x12\x1d\n" + + "\n" + + "created_by\x18\x04 \x01(\tR\tcreatedBy\x12\x1d\n" + + "\n" + + "created_at\x18\x05 \x01(\x03R\tcreatedAt\x12A\n" + + "\bmetadata\x18\x06 \x03(\v2%.aether.v1.ACLGroupInfo.MetadataEntryR\bmetadata\x1a;\n" + + "\rMetadataEntry\x12\x10\n" + + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + + "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"\xa2\x02\n" + + "\vACLRoleInfo\x12\x17\n" + + "\arole_id\x18\x01 \x01(\tR\x06roleId\x12\x1b\n" + + "\trole_name\x18\x02 \x01(\tR\broleName\x12 \n" + + "\vdescription\x18\x03 \x01(\tR\vdescription\x12\x1d\n" + + "\n" + + "created_by\x18\x04 \x01(\tR\tcreatedBy\x12\x1d\n" + + "\n" + + "created_at\x18\x05 \x01(\x03R\tcreatedAt\x12@\n" + + "\bmetadata\x18\x06 \x03(\v2$.aether.v1.ACLRoleInfo.MetadataEntryR\bmetadata\x1a;\n" + + "\rMetadataEntry\x12\x10\n" + + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + + "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"\xce\x01\n" + + "\x12ACLGroupMemberInfo\x12\x1d\n" + + "\n" + + "group_name\x18\x01 \x01(\tR\tgroupName\x12\x1f\n" + + "\vmember_type\x18\x02 \x01(\tR\n" + + "memberType\x12\x1b\n" + + "\tmember_id\x18\x03 \x01(\tR\bmemberId\x12\x1d\n" + + "\n" + + "granted_by\x18\x04 \x01(\tR\tgrantedBy\x12\x1d\n" + + "\n" + + "granted_at\x18\x05 \x01(\x03R\tgrantedAt\x12\x1d\n" + + "\n" + + "expires_at\x18\x06 \x01(\x03R\texpiresAt\"\xd7\x01\n" + + "\x15ACLRoleAssignmentInfo\x12\x1b\n" + + "\trole_name\x18\x01 \x01(\tR\broleName\x12#\n" + + "\rassignee_type\x18\x02 \x01(\tR\fassigneeType\x12\x1f\n" + + "\vassignee_id\x18\x03 \x01(\tR\n" + + "assigneeId\x12\x1d\n" + + "\n" + + "granted_by\x18\x04 \x01(\tR\tgrantedBy\x12\x1d\n" + + "\n" + + "granted_at\x18\x05 \x01(\x03R\tgrantedAt\x12\x1d\n" + + "\n" + + "expires_at\x18\x06 \x01(\x03R\texpiresAt\"\xa7\x01\n" + + "\x19ACLAccessContributionInfo\x12\x18\n" + + "\asubject\x18\x01 \x01(\tR\asubject\x12\x17\n" + + "\arule_id\x18\x02 \x01(\tR\x06ruleId\x12!\n" + + "\faccess_level\x18\x03 \x01(\x05R\vaccessLevel\x12\x1a\n" + + "\bresource\x18\x04 \x01(\tR\bresource\x12\x18\n" + + "\aexpired\x18\x05 \x01(\bR\aexpired\"\xcf\x02\n" + + "\x18ACLAccessExplanationInfo\x12\x1c\n" + + "\tprincipal\x18\x01 \x01(\tR\tprincipal\x12\x1a\n" + + "\bsubjects\x18\x02 \x03(\tR\bsubjects\x12J\n" + + "\rcontributions\x18\x03 \x03(\v2$.aether.v1.ACLAccessContributionInfoR\rcontributions\x12\x18\n" + + "\aallowed\x18\x04 \x01(\bR\aallowed\x12\x1a\n" + + "\bdecision\x18\x05 \x01(\tR\bdecision\x124\n" + + "\x16effective_access_level\x18\x06 \x01(\x05R\x14effectiveAccessLevel\x12)\n" + + "\x10fallback_applied\x18\a \x01(\bR\x0ffallbackApplied\x12\x16\n" + + "\x06reason\x18\b \x01(\tR\x06reason\"\xd9\b\n" + "\vACLResponse\x12\x18\n" + "\asuccess\x18\x01 \x01(\bR\asuccess\x12\x14\n" + "\x05error\x18\x02 \x01(\tR\x05error\x12\x18\n" + @@ -17602,7 +19398,14 @@ const file_aether_proto_rawDesc = "" + "\x10authority_grants\x18\x0e \x03(\v2 .aether.v1.ACLAuthorityGrantInfoR\x0fauthorityGrants\x124\n" + "\x16total_authority_grants\x18\x0f \x01(\x05R\x14totalAuthorityGrants\x12\x1d\n" + "\n" + - "request_id\x18\f \x01(\tR\trequestIdJ\x04\b\a\x10\b\"\xb6\x06\n" + + "request_id\x18\f \x01(\tR\trequestId\x12-\n" + + "\x05group\x18\x10 \x01(\v2\x17.aether.v1.ACLGroupInfoR\x05group\x12/\n" + + "\x06groups\x18\x11 \x03(\v2\x17.aether.v1.ACLGroupInfoR\x06groups\x12*\n" + + "\x04role\x18\x12 \x01(\v2\x16.aether.v1.ACLRoleInfoR\x04role\x12,\n" + + "\x05roles\x18\x13 \x03(\v2\x16.aether.v1.ACLRoleInfoR\x05roles\x12B\n" + + "\rgroup_members\x18\x14 \x03(\v2\x1d.aether.v1.ACLGroupMemberInfoR\fgroupMembers\x12K\n" + + "\x10role_assignments\x18\x15 \x03(\v2 .aether.v1.ACLRoleAssignmentInfoR\x0froleAssignments\x12E\n" + + "\vexplanation\x18\x16 \x01(\v2#.aether.v1.ACLAccessExplanationInfoR\vexplanationJ\x04\b\a\x10\b\"\xb6\x06\n" + "\x17AuthorityGrantOperation\x129\n" + "\x02op\x18\x01 \x01(\x0e2).aether.v1.AuthorityGrantOperation.OpTypeR\x02op\x12\x19\n" + "\bgrant_id\x18\x02 \x01(\tR\agrantId\x12S\n" + @@ -17948,7 +19751,7 @@ const file_aether_proto_rawDesc = "" + "\x04kind\x18\f \x01(\x0e2\x17.aether.v1.ProgressKindR\x04kind\x1a;\n" + "\rMetadataEntry\x12\x10\n" + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + - "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"\xe9\x05\n" + + "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"\x98\x06\n" + "\x11WorkflowOperation\x123\n" + "\x02op\x18\x01 \x01(\x0e2#.aether.v1.WorkflowOperation.OpTypeR\x02op\x12\x0e\n" + "\x02id\x18\x02 \x01(\tR\x02id\x12!\n" + @@ -17957,7 +19760,7 @@ const file_aether_proto_rawDesc = "" + "\x04data\x18\x05 \x01(\fR\x04data\x12\x1d\n" + "\n" + "request_id\x18\x06 \x01(\tR\trequestId\x12#\n" + - "\rstatus_filter\x18\a \x01(\tR\fstatusFilter\"\xf5\x03\n" + + "\rstatus_filter\x18\a \x01(\tR\fstatusFilter\"\xa4\x04\n" + "\x06OpType\x12\x0e\n" + "\n" + "LIST_RULES\x10\x00\x12\f\n" + @@ -17984,7 +19787,11 @@ const file_aether_proto_rawDesc = "" + "\x0fGET_SM_INSTANCE\x10\x14\x12\x16\n" + "\x12CREATE_SM_INSTANCE\x10\x15\x12\x11\n" + "\rSEND_SM_EVENT\x10\x16\x12\x13\n" + - "\x0fUPSERT_SCHEDULE\x10\x17\"\xb0\x01\n" + + "\x0fUPSERT_SCHEDULE\x10\x17\x12\x0e\n" + + "\n" + + "LIST_JOINS\x10\x18\x12\f\n" + + "\bGET_JOIN\x10\x19\x12\x0f\n" + + "\vCANCEL_JOIN\x10\x1a\"\xb0\x01\n" + "\x10WorkflowResponse\x12\x18\n" + "\asuccess\x18\x01 \x01(\bR\asuccess\x12\x14\n" + "\x05error\x18\x02 \x01(\tR\x05error\x12\x18\n" + @@ -17993,14 +19800,15 @@ const file_aether_proto_rawDesc = "" + "\vtotal_count\x18\x05 \x01(\x05R\n" + "totalCount\x12\x1d\n" + "\n" + - "request_id\x18\x06 \x01(\tR\trequestId\"\xc2\x02\n" + + "request_id\x18\x06 \x01(\tR\trequestId\"\x87\x03\n" + "\x0fMessageEnvelope\x12\x16\n" + "\x06source\x18\x01 \x01(\tR\x06source\x12\x18\n" + "\apayload\x18\x02 \x01(\fR\apayload\x129\n" + "\fmessage_type\x18\x03 \x01(\x0e2\x16.aether.v1.MessageTypeR\vmessageType\x12!\n" + "\ftimestamp_ms\x18\x04 \x01(\x03R\vtimestampMs\x12D\n" + "\bmetadata\x18\x05 \x03(\v2(.aether.v1.MessageEnvelope.MetadataEntryR\bmetadata\x12\x1c\n" + - "\tworkspace\x18\x06 \x01(\tR\tworkspace\x1a;\n" + + "\tworkspace\x18\x06 \x01(\tR\tworkspace\x12C\n" + + "\x11on_behalf_subject\x18\a \x01(\v2\x17.aether.v1.PrincipalRefR\x0fonBehalfSubject\x1a;\n" + "\rMetadataEntry\x12\x10\n" + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"\x86\x06\n" + @@ -18350,7 +20158,20 @@ const file_aether_proto_rawDesc = "" + "\x16TASK_CLASS_UNSPECIFIED\x10\x00\x12\x1a\n" + "\x16TASK_CLASS_INTERACTIVE\x10\x01\x12\x19\n" + "\x15TASK_CLASS_BACKGROUND\x10\x02\x12\x14\n" + - "\x10TASK_CLASS_BATCH\x10\x03*\x94\x01\n" + + "\x10TASK_CLASS_BATCH\x10\x03*\xa9\x01\n" + + "\fTaskPriority\x12\x1d\n" + + "\x19TASK_PRIORITY_UNSPECIFIED\x10\x00\x12\x16\n" + + "\x12TASK_PRIORITY_XLOW\x10\n" + + "\x12\x15\n" + + "\x11TASK_PRIORITY_LOW\x10\x14\x12\x18\n" + + "\x14TASK_PRIORITY_NORMAL\x10\x1e\x12\x16\n" + + "\x12TASK_PRIORITY_HIGH\x10(\x12\x19\n" + + "\x15TASK_PRIORITY_PREEMPT\x102*\x99\x01\n" + + "\x0fBackoffStrategy\x12 \n" + + "\x1cBACKOFF_STRATEGY_UNSPECIFIED\x10\x00\x12\x1a\n" + + "\x16BACKOFF_STRATEGY_FIXED\x10\x01\x12 \n" + + "\x1cBACKOFF_STRATEGY_EXPONENTIAL\x10\x02\x12&\n" + + "\"BACKOFF_STRATEGY_EXPLICIT_SCHEDULE\x10\x03*\x94\x01\n" + "\n" + "WaitReason\x12\x1b\n" + "\x17WAIT_REASON_UNSPECIFIED\x10\x00\x12\x15\n" + @@ -18385,8 +20206,8 @@ func file_aether_proto_rawDescGZIP() []byte { return file_aether_proto_rawDescData } -var file_aether_proto_enumTypes = make([]protoimpl.EnumInfo, 32) -var file_aether_proto_msgTypes = make([]protoimpl.MessageInfo, 172) +var file_aether_proto_enumTypes = make([]protoimpl.EnumInfo, 34) +var file_aether_proto_msgTypes = make([]protoimpl.MessageInfo, 188) var file_aether_proto_goTypes = []any{ (MessageType)(0), // 0: aether.v1.MessageType (PrincipalType)(0), // 1: aether.v1.PrincipalType @@ -18396,497 +20217,544 @@ var file_aether_proto_goTypes = []any{ (AccessLevel)(0), // 5: aether.v1.AccessLevel (TaskAssignmentMode)(0), // 6: aether.v1.TaskAssignmentMode (TaskClass)(0), // 7: aether.v1.TaskClass - (WaitReason)(0), // 8: aether.v1.WaitReason - (AuthorityRequestStatus)(0), // 9: aether.v1.AuthorityRequestStatus - (ProgressKind)(0), // 10: aether.v1.ProgressKind - (KVOperation_OpType)(0), // 11: aether.v1.KVOperation.OpType - (KVOperation_Scope)(0), // 12: aether.v1.KVOperation.Scope - (Signal_SignalType)(0), // 13: aether.v1.Signal.SignalType - (CheckpointOperation_OpType)(0), // 14: aether.v1.CheckpointOperation.OpType - (AdminQuery_OpType)(0), // 15: aether.v1.AdminQuery.OpType - (SessionOperation_OpType)(0), // 16: aether.v1.SessionOperation.OpType - (TaskQuery_OpType)(0), // 17: aether.v1.TaskQuery.OpType - (TaskOperation_OpType)(0), // 18: aether.v1.TaskOperation.OpType - (WorkspaceOperation_OpType)(0), // 19: aether.v1.WorkspaceOperation.OpType - (AgentOperation_OpType)(0), // 20: aether.v1.AgentOperation.OpType - (ACLOperation_OpType)(0), // 21: aether.v1.ACLOperation.OpType - (AuthorityGrantOperation_OpType)(0), // 22: aether.v1.AuthorityGrantOperation.OpType - (ResolveAuthorityRequestPayload_Decision)(0), // 23: aether.v1.ResolveAuthorityRequestPayload.Decision - (AuthorityRequestOperation_OpType)(0), // 24: aether.v1.AuthorityRequestOperation.OpType - (AuthorityRequestEvent_EventType)(0), // 25: aether.v1.AuthorityRequestEvent.EventType - (TokenOperation_OpType)(0), // 26: aether.v1.TokenOperation.OpType - (WorkflowOperation_OpType)(0), // 27: aether.v1.WorkflowOperation.OpType - (ProxyError_Kind)(0), // 28: aether.v1.ProxyError.Kind - (TunnelOpen_Protocol)(0), // 29: aether.v1.TunnelOpen.Protocol - (TunnelClose_Reason)(0), // 30: aether.v1.TunnelClose.Reason - (TaskSubscriptionOperation_OpType)(0), // 31: aether.v1.TaskSubscriptionOperation.OpType - (*UpstreamMessage)(nil), // 32: aether.v1.UpstreamMessage - (*DownstreamMessage)(nil), // 33: aether.v1.DownstreamMessage - (*TaskHibernated)(nil), // 34: aether.v1.TaskHibernated - (*ConnectionAck)(nil), // 35: aether.v1.ConnectionAck - (*InitConnection)(nil), // 36: aether.v1.InitConnection - (*BuildInfo)(nil), // 37: aether.v1.BuildInfo - (*ExtensionDeclaration)(nil), // 38: aether.v1.ExtensionDeclaration - (*NegotiatedExtension)(nil), // 39: aether.v1.NegotiatedExtension - (*WorkflowEngineIdentity)(nil), // 40: aether.v1.WorkflowEngineIdentity - (*MetricsBridgeIdentity)(nil), // 41: aether.v1.MetricsBridgeIdentity - (*OrchestratorIdentity)(nil), // 42: aether.v1.OrchestratorIdentity - (*BridgeIdentity)(nil), // 43: aether.v1.BridgeIdentity - (*ServiceIdentity)(nil), // 44: aether.v1.ServiceIdentity - (*AgentIdentity)(nil), // 45: aether.v1.AgentIdentity - (*TaskIdentity)(nil), // 46: aether.v1.TaskIdentity - (*UserIdentity)(nil), // 47: aether.v1.UserIdentity - (*PrincipalRef)(nil), // 48: aether.v1.PrincipalRef - (*AuthorizationContext)(nil), // 49: aether.v1.AuthorizationContext - (*ResolvedAuthorityInfo)(nil), // 50: aether.v1.ResolvedAuthorityInfo - (*SendMessage)(nil), // 51: aether.v1.SendMessage - (*Metric)(nil), // 52: aether.v1.Metric - (*MetricEntry)(nil), // 53: aether.v1.MetricEntry - (*SwitchWorkspace)(nil), // 54: aether.v1.SwitchWorkspace - (*KVOperation)(nil), // 55: aether.v1.KVOperation - (*KVResponse)(nil), // 56: aether.v1.KVResponse - (*IncomingMessage)(nil), // 57: aether.v1.IncomingMessage - (*ConfigSnapshot)(nil), // 58: aether.v1.ConfigSnapshot - (*Signal)(nil), // 59: aether.v1.Signal - (*ErrorResponse)(nil), // 60: aether.v1.ErrorResponse - (*CreateTaskRequest)(nil), // 61: aether.v1.CreateTaskRequest - (*CreateTaskResponse)(nil), // 62: aether.v1.CreateTaskResponse - (*TaskAssignment)(nil), // 63: aether.v1.TaskAssignment - (*CheckpointOperation)(nil), // 64: aether.v1.CheckpointOperation - (*CheckpointResponse)(nil), // 65: aether.v1.CheckpointResponse - (*AdminQuery)(nil), // 66: aether.v1.AdminQuery - (*ConnectionFilter)(nil), // 67: aether.v1.ConnectionFilter - (*ConnectionInfo)(nil), // 68: aether.v1.ConnectionInfo - (*AdminResponse)(nil), // 69: aether.v1.AdminResponse - (*HealthInfo)(nil), // 70: aether.v1.HealthInfo - (*HealthCheck)(nil), // 71: aether.v1.HealthCheck - (*GatewayInfo)(nil), // 72: aether.v1.GatewayInfo - (*GatewayStats)(nil), // 73: aether.v1.GatewayStats - (*SessionOperation)(nil), // 74: aether.v1.SessionOperation - (*SessionOperationResponse)(nil), // 75: aether.v1.SessionOperationResponse - (*TaskQuery)(nil), // 76: aether.v1.TaskQuery - (*TaskFilter)(nil), // 77: aether.v1.TaskFilter - (*TaskInfo)(nil), // 78: aether.v1.TaskInfo - (*TaskQueryResponse)(nil), // 79: aether.v1.TaskQueryResponse - (*TaskOperation)(nil), // 80: aether.v1.TaskOperation - (*WaitSpec)(nil), // 81: aether.v1.WaitSpec - (*HibernationDescriptor)(nil), // 82: aether.v1.HibernationDescriptor - (*TaskOperationResponse)(nil), // 83: aether.v1.TaskOperationResponse - (*WorkspaceOperation)(nil), // 84: aether.v1.WorkspaceOperation - (*WorkspaceFilter)(nil), // 85: aether.v1.WorkspaceFilter - (*WorkspaceInfo)(nil), // 86: aether.v1.WorkspaceInfo - (*WorkspaceResponse)(nil), // 87: aether.v1.WorkspaceResponse - (*MessageFlowInfo)(nil), // 88: aether.v1.MessageFlowInfo - (*FlowNode)(nil), // 89: aether.v1.FlowNode - (*FlowEdge)(nil), // 90: aether.v1.FlowEdge - (*AgentOperation)(nil), // 91: aether.v1.AgentOperation - (*AgentFilter)(nil), // 92: aether.v1.AgentFilter - (*AgentRegistrationInfo)(nil), // 93: aether.v1.AgentRegistrationInfo - (*AgentResourceSchemaEntry)(nil), // 94: aether.v1.AgentResourceSchemaEntry - (*AgentLaunchParams)(nil), // 95: aether.v1.AgentLaunchParams - (*OrchestratorInfo)(nil), // 96: aether.v1.OrchestratorInfo - (*AgentLaunchResult)(nil), // 97: aether.v1.AgentLaunchResult - (*AgentResponse)(nil), // 98: aether.v1.AgentResponse - (*ACLOperation)(nil), // 99: aether.v1.ACLOperation - (*ACLRuleFilter)(nil), // 100: aether.v1.ACLRuleFilter - (*ACLAuditFilter)(nil), // 101: aether.v1.ACLAuditFilter - (*ACLGrantRequest)(nil), // 102: aether.v1.ACLGrantRequest - (*ACLSetFallbackRequest)(nil), // 103: aether.v1.ACLSetFallbackRequest - (*ACLAuthorityGrantFilter)(nil), // 104: aether.v1.ACLAuthorityGrantFilter - (*ACLAuthorityGrantResourceScopeEntry)(nil), // 105: aether.v1.ACLAuthorityGrantResourceScopeEntry - (*ACLAuthorityGrantRequest)(nil), // 106: aether.v1.ACLAuthorityGrantRequest - (*ACLRenewAuthorityGrantRequest)(nil), // 107: aether.v1.ACLRenewAuthorityGrantRequest - (*ACLRuleInfo)(nil), // 108: aether.v1.ACLRuleInfo - (*ACLFallbackPolicyInfo)(nil), // 109: aether.v1.ACLFallbackPolicyInfo - (*ACLAuditEntryInfo)(nil), // 110: aether.v1.ACLAuditEntryInfo - (*ACLAuthorityGrantInfo)(nil), // 111: aether.v1.ACLAuthorityGrantInfo - (*ACLCleanupResult)(nil), // 112: aether.v1.ACLCleanupResult - (*ACLResponse)(nil), // 113: aether.v1.ACLResponse - (*AuthorityGrantOperation)(nil), // 114: aether.v1.AuthorityGrantOperation - (*AuthorityGrantExchangeRequest)(nil), // 115: aether.v1.AuthorityGrantExchangeRequest - (*AuthorityGrantDeriveRequest)(nil), // 116: aether.v1.AuthorityGrantDeriveRequest - (*AuthorityGrantResponse)(nil), // 117: aether.v1.AuthorityGrantResponse - (*AuthorityGrantListRequest)(nil), // 118: aether.v1.AuthorityGrantListRequest - (*AuthorityGrantBatchExchangeRequest)(nil), // 119: aether.v1.AuthorityGrantBatchExchangeRequest - (*AuthorityGrantDeriveForTargetRequest)(nil), // 120: aether.v1.AuthorityGrantDeriveForTargetRequest - (*AuthorityIdentity)(nil), // 121: aether.v1.AuthorityIdentity - (*AuthoritySpan)(nil), // 122: aether.v1.AuthoritySpan - (*AuthorityGrantRevocation)(nil), // 123: aether.v1.AuthorityGrantRevocation - (*AuthorityRequestRoutingTarget)(nil), // 124: aether.v1.AuthorityRequestRoutingTarget - (*AuthorityRequestResourceScopeEntry)(nil), // 125: aether.v1.AuthorityRequestResourceScopeEntry - (*AuthorityRequest)(nil), // 126: aether.v1.AuthorityRequest - (*CreateAuthorityRequestPayload)(nil), // 127: aether.v1.CreateAuthorityRequestPayload - (*ResolveAuthorityRequestPayload)(nil), // 128: aether.v1.ResolveAuthorityRequestPayload - (*AuthorityRequestListFilter)(nil), // 129: aether.v1.AuthorityRequestListFilter - (*AuthorityRequestOperation)(nil), // 130: aether.v1.AuthorityRequestOperation - (*AuthorityRequestOperationResponse)(nil), // 131: aether.v1.AuthorityRequestOperationResponse - (*AuthorityRequestEvent)(nil), // 132: aether.v1.AuthorityRequestEvent - (*TokenOperation)(nil), // 133: aether.v1.TokenOperation - (*TokenCreateRequest)(nil), // 134: aether.v1.TokenCreateRequest - (*TokenFilter)(nil), // 135: aether.v1.TokenFilter - (*TokenInfo)(nil), // 136: aether.v1.TokenInfo - (*TokenResponse)(nil), // 137: aether.v1.TokenResponse - (*ProgressReport)(nil), // 138: aether.v1.ProgressReport - (*ProgressStep)(nil), // 139: aether.v1.ProgressStep - (*ProgressUpdate)(nil), // 140: aether.v1.ProgressUpdate - (*WorkflowOperation)(nil), // 141: aether.v1.WorkflowOperation - (*WorkflowResponse)(nil), // 142: aether.v1.WorkflowResponse - (*MessageEnvelope)(nil), // 143: aether.v1.MessageEnvelope - (*AuditQuery)(nil), // 144: aether.v1.AuditQuery - (*AuditQueryResponse)(nil), // 145: aether.v1.AuditQueryResponse - (*AuditEntry)(nil), // 146: aether.v1.AuditEntry - (*SubmitAuditEventRequest)(nil), // 147: aether.v1.SubmitAuditEventRequest - (*SubmitAuditEventResponse)(nil), // 148: aether.v1.SubmitAuditEventResponse - (*ProxyHttpRequest)(nil), // 149: aether.v1.ProxyHttpRequest - (*ProxyHttpResponse)(nil), // 150: aether.v1.ProxyHttpResponse - (*ProxyHttpBodyChunk)(nil), // 151: aether.v1.ProxyHttpBodyChunk - (*ProxyError)(nil), // 152: aether.v1.ProxyError - (*TunnelOpen)(nil), // 153: aether.v1.TunnelOpen - (*TunnelData)(nil), // 154: aether.v1.TunnelData - (*TunnelClose)(nil), // 155: aether.v1.TunnelClose - (*TunnelAck)(nil), // 156: aether.v1.TunnelAck - (*ResolveAuthorityRequest)(nil), // 157: aether.v1.ResolveAuthorityRequest - (*ResolveAuthorityResponse)(nil), // 158: aether.v1.ResolveAuthorityResponse - (*ResolvedAuthority)(nil), // 159: aether.v1.ResolvedAuthority - (*AuthorityGrantInfo)(nil), // 160: aether.v1.AuthorityGrantInfo - (*ConnectionStatusRequest)(nil), // 161: aether.v1.ConnectionStatusRequest - (*ConnectionStatusResponse)(nil), // 162: aether.v1.ConnectionStatusResponse - (*TaskSubscriptionOperation)(nil), // 163: aether.v1.TaskSubscriptionOperation - (*TaskSubscriptionOperationResponse)(nil), // 164: aether.v1.TaskSubscriptionOperationResponse - (*TaskEvent)(nil), // 165: aether.v1.TaskEvent - (*TaskStatusChangedEvent)(nil), // 166: aether.v1.TaskStatusChangedEvent - (*TaskProgressEvent)(nil), // 167: aether.v1.TaskProgressEvent - (*TaskChildLifecycleEvent)(nil), // 168: aether.v1.TaskChildLifecycleEvent - (*TaskAuthorityRequestEventRelay)(nil), // 169: aether.v1.TaskAuthorityRequestEventRelay - nil, // 170: aether.v1.InitConnection.CredentialsEntry - nil, // 171: aether.v1.Metric.MetadataEntry - nil, // 172: aether.v1.KVResponse.KvMapEntry - nil, // 173: aether.v1.ConfigSnapshot.KvEntry - nil, // 174: aether.v1.ConfigSnapshot.GlobalKvEntry - nil, // 175: aether.v1.ConfigSnapshot.TaskContextEntry - nil, // 176: aether.v1.ConfigSnapshot.WorkspaceExclusiveKvEntry - nil, // 177: aether.v1.ConfigSnapshot.GlobalExclusiveKvEntry - nil, // 178: aether.v1.CreateTaskRequest.LaunchParamOverridesEntry - nil, // 179: aether.v1.CreateTaskRequest.MetadataEntry - nil, // 180: aether.v1.TaskAssignment.MetadataEntry - nil, // 181: aether.v1.TaskAssignment.LaunchParamsEntry - nil, // 182: aether.v1.HealthInfo.ChecksEntry - nil, // 183: aether.v1.TaskInfo.MetadataEntry - nil, // 184: aether.v1.WaitSpec.InputMatchEntry - nil, // 185: aether.v1.WorkspaceInfo.MetadataEntry - nil, // 186: aether.v1.AgentRegistrationInfo.LaunchParamsEntry - nil, // 187: aether.v1.AgentRegistrationInfo.CapabilitiesEntry - nil, // 188: aether.v1.AgentLaunchParams.ParamOverridesEntry - nil, // 189: aether.v1.ACLAuthorityGrantRequest.MetadataEntry - nil, // 190: aether.v1.ACLAuditEntryInfo.MetadataEntry - nil, // 191: aether.v1.ACLAuthorityGrantInfo.MetadataEntry - nil, // 192: aether.v1.AuthorityGrantExchangeRequest.MetadataEntry - nil, // 193: aether.v1.AuthorityGrantDeriveRequest.MetadataEntry - nil, // 194: aether.v1.AuthorityRequest.MetadataEntry - nil, // 195: aether.v1.CreateAuthorityRequestPayload.MetadataEntry - nil, // 196: aether.v1.ProgressReport.MetadataEntry - nil, // 197: aether.v1.ProgressUpdate.MetadataEntry - nil, // 198: aether.v1.MessageEnvelope.MetadataEntry - nil, // 199: aether.v1.SubmitAuditEventRequest.MetadataEntry - nil, // 200: aether.v1.ProxyHttpRequest.HeadersEntry - nil, // 201: aether.v1.ProxyHttpResponse.HeadersEntry - nil, // 202: aether.v1.TunnelOpen.MetadataEntry - nil, // 203: aether.v1.TaskProgressEvent.MetadataEntry + (TaskPriority)(0), // 8: aether.v1.TaskPriority + (BackoffStrategy)(0), // 9: aether.v1.BackoffStrategy + (WaitReason)(0), // 10: aether.v1.WaitReason + (AuthorityRequestStatus)(0), // 11: aether.v1.AuthorityRequestStatus + (ProgressKind)(0), // 12: aether.v1.ProgressKind + (KVOperation_OpType)(0), // 13: aether.v1.KVOperation.OpType + (KVOperation_Scope)(0), // 14: aether.v1.KVOperation.Scope + (Signal_SignalType)(0), // 15: aether.v1.Signal.SignalType + (CheckpointOperation_OpType)(0), // 16: aether.v1.CheckpointOperation.OpType + (AdminQuery_OpType)(0), // 17: aether.v1.AdminQuery.OpType + (SessionOperation_OpType)(0), // 18: aether.v1.SessionOperation.OpType + (TaskQuery_OpType)(0), // 19: aether.v1.TaskQuery.OpType + (TaskOperation_OpType)(0), // 20: aether.v1.TaskOperation.OpType + (WorkspaceOperation_OpType)(0), // 21: aether.v1.WorkspaceOperation.OpType + (AgentOperation_OpType)(0), // 22: aether.v1.AgentOperation.OpType + (ACLOperation_OpType)(0), // 23: aether.v1.ACLOperation.OpType + (AuthorityGrantOperation_OpType)(0), // 24: aether.v1.AuthorityGrantOperation.OpType + (ResolveAuthorityRequestPayload_Decision)(0), // 25: aether.v1.ResolveAuthorityRequestPayload.Decision + (AuthorityRequestOperation_OpType)(0), // 26: aether.v1.AuthorityRequestOperation.OpType + (AuthorityRequestEvent_EventType)(0), // 27: aether.v1.AuthorityRequestEvent.EventType + (TokenOperation_OpType)(0), // 28: aether.v1.TokenOperation.OpType + (WorkflowOperation_OpType)(0), // 29: aether.v1.WorkflowOperation.OpType + (ProxyError_Kind)(0), // 30: aether.v1.ProxyError.Kind + (TunnelOpen_Protocol)(0), // 31: aether.v1.TunnelOpen.Protocol + (TunnelClose_Reason)(0), // 32: aether.v1.TunnelClose.Reason + (TaskSubscriptionOperation_OpType)(0), // 33: aether.v1.TaskSubscriptionOperation.OpType + (*UpstreamMessage)(nil), // 34: aether.v1.UpstreamMessage + (*DownstreamMessage)(nil), // 35: aether.v1.DownstreamMessage + (*TaskHibernated)(nil), // 36: aether.v1.TaskHibernated + (*ConnectionAck)(nil), // 37: aether.v1.ConnectionAck + (*InitConnection)(nil), // 38: aether.v1.InitConnection + (*BuildInfo)(nil), // 39: aether.v1.BuildInfo + (*ExtensionDeclaration)(nil), // 40: aether.v1.ExtensionDeclaration + (*NegotiatedExtension)(nil), // 41: aether.v1.NegotiatedExtension + (*WorkflowEngineIdentity)(nil), // 42: aether.v1.WorkflowEngineIdentity + (*MetricsBridgeIdentity)(nil), // 43: aether.v1.MetricsBridgeIdentity + (*OrchestratorIdentity)(nil), // 44: aether.v1.OrchestratorIdentity + (*BridgeIdentity)(nil), // 45: aether.v1.BridgeIdentity + (*ServiceIdentity)(nil), // 46: aether.v1.ServiceIdentity + (*AgentIdentity)(nil), // 47: aether.v1.AgentIdentity + (*TaskIdentity)(nil), // 48: aether.v1.TaskIdentity + (*UserIdentity)(nil), // 49: aether.v1.UserIdentity + (*PrincipalRef)(nil), // 50: aether.v1.PrincipalRef + (*AuthorizationContext)(nil), // 51: aether.v1.AuthorizationContext + (*ResolvedAuthorityInfo)(nil), // 52: aether.v1.ResolvedAuthorityInfo + (*SendMessage)(nil), // 53: aether.v1.SendMessage + (*Metric)(nil), // 54: aether.v1.Metric + (*MetricEntry)(nil), // 55: aether.v1.MetricEntry + (*SwitchWorkspace)(nil), // 56: aether.v1.SwitchWorkspace + (*KVOperation)(nil), // 57: aether.v1.KVOperation + (*KVResponse)(nil), // 58: aether.v1.KVResponse + (*IncomingMessage)(nil), // 59: aether.v1.IncomingMessage + (*ConfigSnapshot)(nil), // 60: aether.v1.ConfigSnapshot + (*Signal)(nil), // 61: aether.v1.Signal + (*ErrorResponse)(nil), // 62: aether.v1.ErrorResponse + (*RetryPolicy)(nil), // 63: aether.v1.RetryPolicy + (*TaskCompletionEvent)(nil), // 64: aether.v1.TaskCompletionEvent + (*CreateTaskRequest)(nil), // 65: aether.v1.CreateTaskRequest + (*CreateTaskResponse)(nil), // 66: aether.v1.CreateTaskResponse + (*TaskAssignment)(nil), // 67: aether.v1.TaskAssignment + (*CheckpointOperation)(nil), // 68: aether.v1.CheckpointOperation + (*CheckpointResponse)(nil), // 69: aether.v1.CheckpointResponse + (*AdminQuery)(nil), // 70: aether.v1.AdminQuery + (*ConnectionFilter)(nil), // 71: aether.v1.ConnectionFilter + (*ConnectionInfo)(nil), // 72: aether.v1.ConnectionInfo + (*AdminResponse)(nil), // 73: aether.v1.AdminResponse + (*HealthInfo)(nil), // 74: aether.v1.HealthInfo + (*HealthCheck)(nil), // 75: aether.v1.HealthCheck + (*GatewayInfo)(nil), // 76: aether.v1.GatewayInfo + (*GatewayStats)(nil), // 77: aether.v1.GatewayStats + (*SessionOperation)(nil), // 78: aether.v1.SessionOperation + (*SessionOperationResponse)(nil), // 79: aether.v1.SessionOperationResponse + (*TaskQuery)(nil), // 80: aether.v1.TaskQuery + (*TaskFilter)(nil), // 81: aether.v1.TaskFilter + (*TaskInfo)(nil), // 82: aether.v1.TaskInfo + (*TaskQueryResponse)(nil), // 83: aether.v1.TaskQueryResponse + (*TaskOperation)(nil), // 84: aether.v1.TaskOperation + (*WaitSpec)(nil), // 85: aether.v1.WaitSpec + (*HibernationDescriptor)(nil), // 86: aether.v1.HibernationDescriptor + (*TaskOperationResponse)(nil), // 87: aether.v1.TaskOperationResponse + (*WorkspaceOperation)(nil), // 88: aether.v1.WorkspaceOperation + (*WorkspaceFilter)(nil), // 89: aether.v1.WorkspaceFilter + (*WorkspaceInfo)(nil), // 90: aether.v1.WorkspaceInfo + (*WorkspaceResponse)(nil), // 91: aether.v1.WorkspaceResponse + (*MessageFlowInfo)(nil), // 92: aether.v1.MessageFlowInfo + (*FlowNode)(nil), // 93: aether.v1.FlowNode + (*FlowEdge)(nil), // 94: aether.v1.FlowEdge + (*AgentOperation)(nil), // 95: aether.v1.AgentOperation + (*AgentFilter)(nil), // 96: aether.v1.AgentFilter + (*AgentRegistrationInfo)(nil), // 97: aether.v1.AgentRegistrationInfo + (*AgentResourceSchemaEntry)(nil), // 98: aether.v1.AgentResourceSchemaEntry + (*AgentLaunchParams)(nil), // 99: aether.v1.AgentLaunchParams + (*OrchestratorInfo)(nil), // 100: aether.v1.OrchestratorInfo + (*AgentLaunchResult)(nil), // 101: aether.v1.AgentLaunchResult + (*AgentResponse)(nil), // 102: aether.v1.AgentResponse + (*ACLOperation)(nil), // 103: aether.v1.ACLOperation + (*ACLRuleFilter)(nil), // 104: aether.v1.ACLRuleFilter + (*ACLAuditFilter)(nil), // 105: aether.v1.ACLAuditFilter + (*ACLGrantRequest)(nil), // 106: aether.v1.ACLGrantRequest + (*ACLSetFallbackRequest)(nil), // 107: aether.v1.ACLSetFallbackRequest + (*ACLAuthorityGrantFilter)(nil), // 108: aether.v1.ACLAuthorityGrantFilter + (*ACLAuthorityGrantResourceScopeEntry)(nil), // 109: aether.v1.ACLAuthorityGrantResourceScopeEntry + (*ACLAuthorityGrantRequest)(nil), // 110: aether.v1.ACLAuthorityGrantRequest + (*ACLRenewAuthorityGrantRequest)(nil), // 111: aether.v1.ACLRenewAuthorityGrantRequest + (*ACLRuleInfo)(nil), // 112: aether.v1.ACLRuleInfo + (*ACLFallbackPolicyInfo)(nil), // 113: aether.v1.ACLFallbackPolicyInfo + (*ACLAuditEntryInfo)(nil), // 114: aether.v1.ACLAuditEntryInfo + (*ACLAuthorityGrantInfo)(nil), // 115: aether.v1.ACLAuthorityGrantInfo + (*ACLCleanupResult)(nil), // 116: aether.v1.ACLCleanupResult + (*ACLGroupRequest)(nil), // 117: aether.v1.ACLGroupRequest + (*ACLRoleRequest)(nil), // 118: aether.v1.ACLRoleRequest + (*ACLGroupMemberRequest)(nil), // 119: aether.v1.ACLGroupMemberRequest + (*ACLRoleAssignmentRequest)(nil), // 120: aether.v1.ACLRoleAssignmentRequest + (*ACLGroupInfo)(nil), // 121: aether.v1.ACLGroupInfo + (*ACLRoleInfo)(nil), // 122: aether.v1.ACLRoleInfo + (*ACLGroupMemberInfo)(nil), // 123: aether.v1.ACLGroupMemberInfo + (*ACLRoleAssignmentInfo)(nil), // 124: aether.v1.ACLRoleAssignmentInfo + (*ACLAccessContributionInfo)(nil), // 125: aether.v1.ACLAccessContributionInfo + (*ACLAccessExplanationInfo)(nil), // 126: aether.v1.ACLAccessExplanationInfo + (*ACLResponse)(nil), // 127: aether.v1.ACLResponse + (*AuthorityGrantOperation)(nil), // 128: aether.v1.AuthorityGrantOperation + (*AuthorityGrantExchangeRequest)(nil), // 129: aether.v1.AuthorityGrantExchangeRequest + (*AuthorityGrantDeriveRequest)(nil), // 130: aether.v1.AuthorityGrantDeriveRequest + (*AuthorityGrantResponse)(nil), // 131: aether.v1.AuthorityGrantResponse + (*AuthorityGrantListRequest)(nil), // 132: aether.v1.AuthorityGrantListRequest + (*AuthorityGrantBatchExchangeRequest)(nil), // 133: aether.v1.AuthorityGrantBatchExchangeRequest + (*AuthorityGrantDeriveForTargetRequest)(nil), // 134: aether.v1.AuthorityGrantDeriveForTargetRequest + (*AuthorityIdentity)(nil), // 135: aether.v1.AuthorityIdentity + (*AuthoritySpan)(nil), // 136: aether.v1.AuthoritySpan + (*AuthorityGrantRevocation)(nil), // 137: aether.v1.AuthorityGrantRevocation + (*AuthorityRequestRoutingTarget)(nil), // 138: aether.v1.AuthorityRequestRoutingTarget + (*AuthorityRequestResourceScopeEntry)(nil), // 139: aether.v1.AuthorityRequestResourceScopeEntry + (*AuthorityRequest)(nil), // 140: aether.v1.AuthorityRequest + (*CreateAuthorityRequestPayload)(nil), // 141: aether.v1.CreateAuthorityRequestPayload + (*ResolveAuthorityRequestPayload)(nil), // 142: aether.v1.ResolveAuthorityRequestPayload + (*AuthorityRequestListFilter)(nil), // 143: aether.v1.AuthorityRequestListFilter + (*AuthorityRequestOperation)(nil), // 144: aether.v1.AuthorityRequestOperation + (*AuthorityRequestOperationResponse)(nil), // 145: aether.v1.AuthorityRequestOperationResponse + (*AuthorityRequestEvent)(nil), // 146: aether.v1.AuthorityRequestEvent + (*TokenOperation)(nil), // 147: aether.v1.TokenOperation + (*TokenCreateRequest)(nil), // 148: aether.v1.TokenCreateRequest + (*TokenFilter)(nil), // 149: aether.v1.TokenFilter + (*TokenInfo)(nil), // 150: aether.v1.TokenInfo + (*TokenResponse)(nil), // 151: aether.v1.TokenResponse + (*ProgressReport)(nil), // 152: aether.v1.ProgressReport + (*ProgressStep)(nil), // 153: aether.v1.ProgressStep + (*ProgressUpdate)(nil), // 154: aether.v1.ProgressUpdate + (*WorkflowOperation)(nil), // 155: aether.v1.WorkflowOperation + (*WorkflowResponse)(nil), // 156: aether.v1.WorkflowResponse + (*MessageEnvelope)(nil), // 157: aether.v1.MessageEnvelope + (*AuditQuery)(nil), // 158: aether.v1.AuditQuery + (*AuditQueryResponse)(nil), // 159: aether.v1.AuditQueryResponse + (*AuditEntry)(nil), // 160: aether.v1.AuditEntry + (*SubmitAuditEventRequest)(nil), // 161: aether.v1.SubmitAuditEventRequest + (*SubmitAuditEventResponse)(nil), // 162: aether.v1.SubmitAuditEventResponse + (*ProxyHttpRequest)(nil), // 163: aether.v1.ProxyHttpRequest + (*ProxyHttpResponse)(nil), // 164: aether.v1.ProxyHttpResponse + (*ProxyHttpBodyChunk)(nil), // 165: aether.v1.ProxyHttpBodyChunk + (*ProxyError)(nil), // 166: aether.v1.ProxyError + (*TunnelOpen)(nil), // 167: aether.v1.TunnelOpen + (*TunnelData)(nil), // 168: aether.v1.TunnelData + (*TunnelClose)(nil), // 169: aether.v1.TunnelClose + (*TunnelAck)(nil), // 170: aether.v1.TunnelAck + (*ResolveAuthorityRequest)(nil), // 171: aether.v1.ResolveAuthorityRequest + (*ResolveAuthorityResponse)(nil), // 172: aether.v1.ResolveAuthorityResponse + (*ResolvedAuthority)(nil), // 173: aether.v1.ResolvedAuthority + (*AuthorityGrantInfo)(nil), // 174: aether.v1.AuthorityGrantInfo + (*ConnectionStatusRequest)(nil), // 175: aether.v1.ConnectionStatusRequest + (*ConnectionStatusResponse)(nil), // 176: aether.v1.ConnectionStatusResponse + (*TaskSubscriptionOperation)(nil), // 177: aether.v1.TaskSubscriptionOperation + (*TaskSubscriptionOperationResponse)(nil), // 178: aether.v1.TaskSubscriptionOperationResponse + (*TaskEvent)(nil), // 179: aether.v1.TaskEvent + (*TaskStatusChangedEvent)(nil), // 180: aether.v1.TaskStatusChangedEvent + (*TaskProgressEvent)(nil), // 181: aether.v1.TaskProgressEvent + (*TaskChildLifecycleEvent)(nil), // 182: aether.v1.TaskChildLifecycleEvent + (*TaskAuthorityRequestEventRelay)(nil), // 183: aether.v1.TaskAuthorityRequestEventRelay + nil, // 184: aether.v1.InitConnection.CredentialsEntry + nil, // 185: aether.v1.Metric.MetadataEntry + nil, // 186: aether.v1.KVResponse.KvMapEntry + nil, // 187: aether.v1.ConfigSnapshot.KvEntry + nil, // 188: aether.v1.ConfigSnapshot.GlobalKvEntry + nil, // 189: aether.v1.ConfigSnapshot.TaskContextEntry + nil, // 190: aether.v1.ConfigSnapshot.WorkspaceExclusiveKvEntry + nil, // 191: aether.v1.ConfigSnapshot.GlobalExclusiveKvEntry + nil, // 192: aether.v1.CreateTaskRequest.LaunchParamOverridesEntry + nil, // 193: aether.v1.CreateTaskRequest.MetadataEntry + nil, // 194: aether.v1.TaskAssignment.MetadataEntry + nil, // 195: aether.v1.TaskAssignment.LaunchParamsEntry + nil, // 196: aether.v1.HealthInfo.ChecksEntry + nil, // 197: aether.v1.TaskInfo.MetadataEntry + nil, // 198: aether.v1.WaitSpec.InputMatchEntry + nil, // 199: aether.v1.WorkspaceInfo.MetadataEntry + nil, // 200: aether.v1.AgentRegistrationInfo.LaunchParamsEntry + nil, // 201: aether.v1.AgentRegistrationInfo.CapabilitiesEntry + nil, // 202: aether.v1.AgentLaunchParams.ParamOverridesEntry + nil, // 203: aether.v1.ACLAuthorityGrantRequest.MetadataEntry + nil, // 204: aether.v1.ACLAuditEntryInfo.MetadataEntry + nil, // 205: aether.v1.ACLAuthorityGrantInfo.MetadataEntry + nil, // 206: aether.v1.ACLGroupRequest.MetadataEntry + nil, // 207: aether.v1.ACLRoleRequest.MetadataEntry + nil, // 208: aether.v1.ACLGroupInfo.MetadataEntry + nil, // 209: aether.v1.ACLRoleInfo.MetadataEntry + nil, // 210: aether.v1.AuthorityGrantExchangeRequest.MetadataEntry + nil, // 211: aether.v1.AuthorityGrantDeriveRequest.MetadataEntry + nil, // 212: aether.v1.AuthorityRequest.MetadataEntry + nil, // 213: aether.v1.CreateAuthorityRequestPayload.MetadataEntry + nil, // 214: aether.v1.ProgressReport.MetadataEntry + nil, // 215: aether.v1.ProgressUpdate.MetadataEntry + nil, // 216: aether.v1.MessageEnvelope.MetadataEntry + nil, // 217: aether.v1.SubmitAuditEventRequest.MetadataEntry + nil, // 218: aether.v1.ProxyHttpRequest.HeadersEntry + nil, // 219: aether.v1.ProxyHttpResponse.HeadersEntry + nil, // 220: aether.v1.TunnelOpen.MetadataEntry + nil, // 221: aether.v1.TaskProgressEvent.MetadataEntry } var file_aether_proto_depIdxs = []int32{ - 36, // 0: aether.v1.UpstreamMessage.init:type_name -> aether.v1.InitConnection - 51, // 1: aether.v1.UpstreamMessage.send:type_name -> aether.v1.SendMessage - 54, // 2: aether.v1.UpstreamMessage.switch_workspace:type_name -> aether.v1.SwitchWorkspace - 55, // 3: aether.v1.UpstreamMessage.kv_op:type_name -> aether.v1.KVOperation - 61, // 4: aether.v1.UpstreamMessage.create_task:type_name -> aether.v1.CreateTaskRequest - 64, // 5: aether.v1.UpstreamMessage.checkpoint_op:type_name -> aether.v1.CheckpointOperation - 66, // 6: aether.v1.UpstreamMessage.admin_query:type_name -> aether.v1.AdminQuery - 74, // 7: aether.v1.UpstreamMessage.session_op:type_name -> aether.v1.SessionOperation - 76, // 8: aether.v1.UpstreamMessage.task_query:type_name -> aether.v1.TaskQuery - 80, // 9: aether.v1.UpstreamMessage.task_op:type_name -> aether.v1.TaskOperation - 84, // 10: aether.v1.UpstreamMessage.workspace_op:type_name -> aether.v1.WorkspaceOperation - 91, // 11: aether.v1.UpstreamMessage.agent_op:type_name -> aether.v1.AgentOperation - 99, // 12: aether.v1.UpstreamMessage.acl_op:type_name -> aether.v1.ACLOperation - 138, // 13: aether.v1.UpstreamMessage.progress:type_name -> aether.v1.ProgressReport - 141, // 14: aether.v1.UpstreamMessage.workflow_op:type_name -> aether.v1.WorkflowOperation - 142, // 15: aether.v1.UpstreamMessage.workflow_response:type_name -> aether.v1.WorkflowResponse - 133, // 16: aether.v1.UpstreamMessage.token_op:type_name -> aether.v1.TokenOperation - 144, // 17: aether.v1.UpstreamMessage.audit_query:type_name -> aether.v1.AuditQuery - 114, // 18: aether.v1.UpstreamMessage.authority_grant_op:type_name -> aether.v1.AuthorityGrantOperation - 149, // 19: aether.v1.UpstreamMessage.proxy_http_request:type_name -> aether.v1.ProxyHttpRequest - 151, // 20: aether.v1.UpstreamMessage.proxy_http_body_chunk:type_name -> aether.v1.ProxyHttpBodyChunk - 153, // 21: aether.v1.UpstreamMessage.tunnel_open:type_name -> aether.v1.TunnelOpen - 154, // 22: aether.v1.UpstreamMessage.tunnel_data:type_name -> aether.v1.TunnelData - 155, // 23: aether.v1.UpstreamMessage.tunnel_close:type_name -> aether.v1.TunnelClose - 150, // 24: aether.v1.UpstreamMessage.proxy_http_response:type_name -> aether.v1.ProxyHttpResponse - 156, // 25: aether.v1.UpstreamMessage.tunnel_ack:type_name -> aether.v1.TunnelAck - 157, // 26: aether.v1.UpstreamMessage.resolve_authority_request:type_name -> aether.v1.ResolveAuthorityRequest - 161, // 27: aether.v1.UpstreamMessage.connection_status_request:type_name -> aether.v1.ConnectionStatusRequest - 147, // 28: aether.v1.UpstreamMessage.submit_audit_event:type_name -> aether.v1.SubmitAuditEventRequest - 130, // 29: aether.v1.UpstreamMessage.authority_request_op:type_name -> aether.v1.AuthorityRequestOperation - 163, // 30: aether.v1.UpstreamMessage.task_subscription_op:type_name -> aether.v1.TaskSubscriptionOperation - 57, // 31: aether.v1.DownstreamMessage.msg:type_name -> aether.v1.IncomingMessage - 58, // 32: aether.v1.DownstreamMessage.config:type_name -> aether.v1.ConfigSnapshot - 59, // 33: aether.v1.DownstreamMessage.signal:type_name -> aether.v1.Signal - 60, // 34: aether.v1.DownstreamMessage.error:type_name -> aether.v1.ErrorResponse - 56, // 35: aether.v1.DownstreamMessage.kv:type_name -> aether.v1.KVResponse - 63, // 36: aether.v1.DownstreamMessage.task_assignment:type_name -> aether.v1.TaskAssignment - 35, // 37: aether.v1.DownstreamMessage.connection_ack:type_name -> aether.v1.ConnectionAck - 65, // 38: aether.v1.DownstreamMessage.checkpoint:type_name -> aether.v1.CheckpointResponse - 69, // 39: aether.v1.DownstreamMessage.admin:type_name -> aether.v1.AdminResponse - 75, // 40: aether.v1.DownstreamMessage.session_response:type_name -> aether.v1.SessionOperationResponse - 79, // 41: aether.v1.DownstreamMessage.task_query:type_name -> aether.v1.TaskQueryResponse - 83, // 42: aether.v1.DownstreamMessage.task_op:type_name -> aether.v1.TaskOperationResponse - 87, // 43: aether.v1.DownstreamMessage.workspace:type_name -> aether.v1.WorkspaceResponse - 98, // 44: aether.v1.DownstreamMessage.agent:type_name -> aether.v1.AgentResponse - 113, // 45: aether.v1.DownstreamMessage.acl:type_name -> aether.v1.ACLResponse - 140, // 46: aether.v1.DownstreamMessage.progress_update:type_name -> aether.v1.ProgressUpdate - 142, // 47: aether.v1.DownstreamMessage.workflow_response:type_name -> aether.v1.WorkflowResponse - 141, // 48: aether.v1.DownstreamMessage.workflow_op:type_name -> aether.v1.WorkflowOperation - 137, // 49: aether.v1.DownstreamMessage.token:type_name -> aether.v1.TokenResponse - 145, // 50: aether.v1.DownstreamMessage.audit_response:type_name -> aether.v1.AuditQueryResponse - 117, // 51: aether.v1.DownstreamMessage.authority_grant:type_name -> aether.v1.AuthorityGrantResponse - 62, // 52: aether.v1.DownstreamMessage.create_task:type_name -> aether.v1.CreateTaskResponse - 150, // 53: aether.v1.DownstreamMessage.proxy_http_response:type_name -> aether.v1.ProxyHttpResponse - 151, // 54: aether.v1.DownstreamMessage.proxy_http_body_chunk:type_name -> aether.v1.ProxyHttpBodyChunk - 156, // 55: aether.v1.DownstreamMessage.tunnel_ack:type_name -> aether.v1.TunnelAck - 155, // 56: aether.v1.DownstreamMessage.tunnel_close:type_name -> aether.v1.TunnelClose - 154, // 57: aether.v1.DownstreamMessage.tunnel_data:type_name -> aether.v1.TunnelData - 149, // 58: aether.v1.DownstreamMessage.proxy_http_request:type_name -> aether.v1.ProxyHttpRequest - 158, // 59: aether.v1.DownstreamMessage.resolve_authority_response:type_name -> aether.v1.ResolveAuthorityResponse - 162, // 60: aether.v1.DownstreamMessage.connection_status_response:type_name -> aether.v1.ConnectionStatusResponse - 123, // 61: aether.v1.DownstreamMessage.authority_grant_revocation:type_name -> aether.v1.AuthorityGrantRevocation - 148, // 62: aether.v1.DownstreamMessage.submit_audit_event_response:type_name -> aether.v1.SubmitAuditEventResponse - 131, // 63: aether.v1.DownstreamMessage.authority_request_response:type_name -> aether.v1.AuthorityRequestOperationResponse - 132, // 64: aether.v1.DownstreamMessage.authority_request_event:type_name -> aether.v1.AuthorityRequestEvent - 34, // 65: aether.v1.DownstreamMessage.task_hibernated:type_name -> aether.v1.TaskHibernated - 164, // 66: aether.v1.DownstreamMessage.task_subscription_response:type_name -> aether.v1.TaskSubscriptionOperationResponse - 165, // 67: aether.v1.DownstreamMessage.task_event:type_name -> aether.v1.TaskEvent - 82, // 68: aether.v1.TaskHibernated.descriptor:type_name -> aether.v1.HibernationDescriptor - 39, // 69: aether.v1.ConnectionAck.negotiated_extensions:type_name -> aether.v1.NegotiatedExtension - 37, // 70: aether.v1.ConnectionAck.server_build_info:type_name -> aether.v1.BuildInfo - 45, // 71: aether.v1.InitConnection.agent:type_name -> aether.v1.AgentIdentity - 46, // 72: aether.v1.InitConnection.task:type_name -> aether.v1.TaskIdentity - 47, // 73: aether.v1.InitConnection.user:type_name -> aether.v1.UserIdentity - 42, // 74: aether.v1.InitConnection.orchestrator:type_name -> aether.v1.OrchestratorIdentity - 40, // 75: aether.v1.InitConnection.workflow_engine:type_name -> aether.v1.WorkflowEngineIdentity - 41, // 76: aether.v1.InitConnection.metrics_bridge:type_name -> aether.v1.MetricsBridgeIdentity - 43, // 77: aether.v1.InitConnection.bridge:type_name -> aether.v1.BridgeIdentity - 44, // 78: aether.v1.InitConnection.service:type_name -> aether.v1.ServiceIdentity - 170, // 79: aether.v1.InitConnection.credentials:type_name -> aether.v1.InitConnection.CredentialsEntry - 38, // 80: aether.v1.InitConnection.extensions:type_name -> aether.v1.ExtensionDeclaration - 37, // 81: aether.v1.InitConnection.client_build_info:type_name -> aether.v1.BuildInfo - 48, // 82: aether.v1.AuthorizationContext.subject:type_name -> aether.v1.PrincipalRef - 50, // 83: aether.v1.AuthorizationContext.resolved:type_name -> aether.v1.ResolvedAuthorityInfo - 48, // 84: aether.v1.ResolvedAuthorityInfo.root_subject:type_name -> aether.v1.PrincipalRef + 38, // 0: aether.v1.UpstreamMessage.init:type_name -> aether.v1.InitConnection + 53, // 1: aether.v1.UpstreamMessage.send:type_name -> aether.v1.SendMessage + 56, // 2: aether.v1.UpstreamMessage.switch_workspace:type_name -> aether.v1.SwitchWorkspace + 57, // 3: aether.v1.UpstreamMessage.kv_op:type_name -> aether.v1.KVOperation + 65, // 4: aether.v1.UpstreamMessage.create_task:type_name -> aether.v1.CreateTaskRequest + 68, // 5: aether.v1.UpstreamMessage.checkpoint_op:type_name -> aether.v1.CheckpointOperation + 70, // 6: aether.v1.UpstreamMessage.admin_query:type_name -> aether.v1.AdminQuery + 78, // 7: aether.v1.UpstreamMessage.session_op:type_name -> aether.v1.SessionOperation + 80, // 8: aether.v1.UpstreamMessage.task_query:type_name -> aether.v1.TaskQuery + 84, // 9: aether.v1.UpstreamMessage.task_op:type_name -> aether.v1.TaskOperation + 88, // 10: aether.v1.UpstreamMessage.workspace_op:type_name -> aether.v1.WorkspaceOperation + 95, // 11: aether.v1.UpstreamMessage.agent_op:type_name -> aether.v1.AgentOperation + 103, // 12: aether.v1.UpstreamMessage.acl_op:type_name -> aether.v1.ACLOperation + 152, // 13: aether.v1.UpstreamMessage.progress:type_name -> aether.v1.ProgressReport + 155, // 14: aether.v1.UpstreamMessage.workflow_op:type_name -> aether.v1.WorkflowOperation + 156, // 15: aether.v1.UpstreamMessage.workflow_response:type_name -> aether.v1.WorkflowResponse + 147, // 16: aether.v1.UpstreamMessage.token_op:type_name -> aether.v1.TokenOperation + 158, // 17: aether.v1.UpstreamMessage.audit_query:type_name -> aether.v1.AuditQuery + 128, // 18: aether.v1.UpstreamMessage.authority_grant_op:type_name -> aether.v1.AuthorityGrantOperation + 163, // 19: aether.v1.UpstreamMessage.proxy_http_request:type_name -> aether.v1.ProxyHttpRequest + 165, // 20: aether.v1.UpstreamMessage.proxy_http_body_chunk:type_name -> aether.v1.ProxyHttpBodyChunk + 167, // 21: aether.v1.UpstreamMessage.tunnel_open:type_name -> aether.v1.TunnelOpen + 168, // 22: aether.v1.UpstreamMessage.tunnel_data:type_name -> aether.v1.TunnelData + 169, // 23: aether.v1.UpstreamMessage.tunnel_close:type_name -> aether.v1.TunnelClose + 164, // 24: aether.v1.UpstreamMessage.proxy_http_response:type_name -> aether.v1.ProxyHttpResponse + 170, // 25: aether.v1.UpstreamMessage.tunnel_ack:type_name -> aether.v1.TunnelAck + 171, // 26: aether.v1.UpstreamMessage.resolve_authority_request:type_name -> aether.v1.ResolveAuthorityRequest + 175, // 27: aether.v1.UpstreamMessage.connection_status_request:type_name -> aether.v1.ConnectionStatusRequest + 161, // 28: aether.v1.UpstreamMessage.submit_audit_event:type_name -> aether.v1.SubmitAuditEventRequest + 144, // 29: aether.v1.UpstreamMessage.authority_request_op:type_name -> aether.v1.AuthorityRequestOperation + 177, // 30: aether.v1.UpstreamMessage.task_subscription_op:type_name -> aether.v1.TaskSubscriptionOperation + 59, // 31: aether.v1.DownstreamMessage.msg:type_name -> aether.v1.IncomingMessage + 60, // 32: aether.v1.DownstreamMessage.config:type_name -> aether.v1.ConfigSnapshot + 61, // 33: aether.v1.DownstreamMessage.signal:type_name -> aether.v1.Signal + 62, // 34: aether.v1.DownstreamMessage.error:type_name -> aether.v1.ErrorResponse + 58, // 35: aether.v1.DownstreamMessage.kv:type_name -> aether.v1.KVResponse + 67, // 36: aether.v1.DownstreamMessage.task_assignment:type_name -> aether.v1.TaskAssignment + 37, // 37: aether.v1.DownstreamMessage.connection_ack:type_name -> aether.v1.ConnectionAck + 69, // 38: aether.v1.DownstreamMessage.checkpoint:type_name -> aether.v1.CheckpointResponse + 73, // 39: aether.v1.DownstreamMessage.admin:type_name -> aether.v1.AdminResponse + 79, // 40: aether.v1.DownstreamMessage.session_response:type_name -> aether.v1.SessionOperationResponse + 83, // 41: aether.v1.DownstreamMessage.task_query:type_name -> aether.v1.TaskQueryResponse + 87, // 42: aether.v1.DownstreamMessage.task_op:type_name -> aether.v1.TaskOperationResponse + 91, // 43: aether.v1.DownstreamMessage.workspace:type_name -> aether.v1.WorkspaceResponse + 102, // 44: aether.v1.DownstreamMessage.agent:type_name -> aether.v1.AgentResponse + 127, // 45: aether.v1.DownstreamMessage.acl:type_name -> aether.v1.ACLResponse + 154, // 46: aether.v1.DownstreamMessage.progress_update:type_name -> aether.v1.ProgressUpdate + 156, // 47: aether.v1.DownstreamMessage.workflow_response:type_name -> aether.v1.WorkflowResponse + 155, // 48: aether.v1.DownstreamMessage.workflow_op:type_name -> aether.v1.WorkflowOperation + 151, // 49: aether.v1.DownstreamMessage.token:type_name -> aether.v1.TokenResponse + 159, // 50: aether.v1.DownstreamMessage.audit_response:type_name -> aether.v1.AuditQueryResponse + 131, // 51: aether.v1.DownstreamMessage.authority_grant:type_name -> aether.v1.AuthorityGrantResponse + 66, // 52: aether.v1.DownstreamMessage.create_task:type_name -> aether.v1.CreateTaskResponse + 164, // 53: aether.v1.DownstreamMessage.proxy_http_response:type_name -> aether.v1.ProxyHttpResponse + 165, // 54: aether.v1.DownstreamMessage.proxy_http_body_chunk:type_name -> aether.v1.ProxyHttpBodyChunk + 170, // 55: aether.v1.DownstreamMessage.tunnel_ack:type_name -> aether.v1.TunnelAck + 169, // 56: aether.v1.DownstreamMessage.tunnel_close:type_name -> aether.v1.TunnelClose + 168, // 57: aether.v1.DownstreamMessage.tunnel_data:type_name -> aether.v1.TunnelData + 163, // 58: aether.v1.DownstreamMessage.proxy_http_request:type_name -> aether.v1.ProxyHttpRequest + 172, // 59: aether.v1.DownstreamMessage.resolve_authority_response:type_name -> aether.v1.ResolveAuthorityResponse + 176, // 60: aether.v1.DownstreamMessage.connection_status_response:type_name -> aether.v1.ConnectionStatusResponse + 137, // 61: aether.v1.DownstreamMessage.authority_grant_revocation:type_name -> aether.v1.AuthorityGrantRevocation + 162, // 62: aether.v1.DownstreamMessage.submit_audit_event_response:type_name -> aether.v1.SubmitAuditEventResponse + 145, // 63: aether.v1.DownstreamMessage.authority_request_response:type_name -> aether.v1.AuthorityRequestOperationResponse + 146, // 64: aether.v1.DownstreamMessage.authority_request_event:type_name -> aether.v1.AuthorityRequestEvent + 36, // 65: aether.v1.DownstreamMessage.task_hibernated:type_name -> aether.v1.TaskHibernated + 178, // 66: aether.v1.DownstreamMessage.task_subscription_response:type_name -> aether.v1.TaskSubscriptionOperationResponse + 179, // 67: aether.v1.DownstreamMessage.task_event:type_name -> aether.v1.TaskEvent + 86, // 68: aether.v1.TaskHibernated.descriptor:type_name -> aether.v1.HibernationDescriptor + 41, // 69: aether.v1.ConnectionAck.negotiated_extensions:type_name -> aether.v1.NegotiatedExtension + 39, // 70: aether.v1.ConnectionAck.server_build_info:type_name -> aether.v1.BuildInfo + 47, // 71: aether.v1.InitConnection.agent:type_name -> aether.v1.AgentIdentity + 48, // 72: aether.v1.InitConnection.task:type_name -> aether.v1.TaskIdentity + 49, // 73: aether.v1.InitConnection.user:type_name -> aether.v1.UserIdentity + 44, // 74: aether.v1.InitConnection.orchestrator:type_name -> aether.v1.OrchestratorIdentity + 42, // 75: aether.v1.InitConnection.workflow_engine:type_name -> aether.v1.WorkflowEngineIdentity + 43, // 76: aether.v1.InitConnection.metrics_bridge:type_name -> aether.v1.MetricsBridgeIdentity + 45, // 77: aether.v1.InitConnection.bridge:type_name -> aether.v1.BridgeIdentity + 46, // 78: aether.v1.InitConnection.service:type_name -> aether.v1.ServiceIdentity + 184, // 79: aether.v1.InitConnection.credentials:type_name -> aether.v1.InitConnection.CredentialsEntry + 40, // 80: aether.v1.InitConnection.extensions:type_name -> aether.v1.ExtensionDeclaration + 39, // 81: aether.v1.InitConnection.client_build_info:type_name -> aether.v1.BuildInfo + 50, // 82: aether.v1.AuthorizationContext.subject:type_name -> aether.v1.PrincipalRef + 52, // 83: aether.v1.AuthorizationContext.resolved:type_name -> aether.v1.ResolvedAuthorityInfo + 50, // 84: aether.v1.ResolvedAuthorityInfo.root_subject:type_name -> aether.v1.PrincipalRef 0, // 85: aether.v1.SendMessage.message_type:type_name -> aether.v1.MessageType - 49, // 86: aether.v1.SendMessage.authorization:type_name -> aether.v1.AuthorizationContext - 53, // 87: aether.v1.Metric.entries:type_name -> aether.v1.MetricEntry - 171, // 88: aether.v1.Metric.metadata:type_name -> aether.v1.Metric.MetadataEntry - 11, // 89: aether.v1.KVOperation.op:type_name -> aether.v1.KVOperation.OpType - 12, // 90: aether.v1.KVOperation.scope:type_name -> aether.v1.KVOperation.Scope - 49, // 91: aether.v1.KVOperation.authorization:type_name -> aether.v1.AuthorizationContext - 172, // 92: aether.v1.KVResponse.kv_map:type_name -> aether.v1.KVResponse.KvMapEntry + 51, // 86: aether.v1.SendMessage.authorization:type_name -> aether.v1.AuthorizationContext + 55, // 87: aether.v1.Metric.entries:type_name -> aether.v1.MetricEntry + 185, // 88: aether.v1.Metric.metadata:type_name -> aether.v1.Metric.MetadataEntry + 13, // 89: aether.v1.KVOperation.op:type_name -> aether.v1.KVOperation.OpType + 14, // 90: aether.v1.KVOperation.scope:type_name -> aether.v1.KVOperation.Scope + 51, // 91: aether.v1.KVOperation.authorization:type_name -> aether.v1.AuthorizationContext + 186, // 92: aether.v1.KVResponse.kv_map:type_name -> aether.v1.KVResponse.KvMapEntry 0, // 93: aether.v1.IncomingMessage.message_type:type_name -> aether.v1.MessageType - 173, // 94: aether.v1.ConfigSnapshot.kv:type_name -> aether.v1.ConfigSnapshot.KvEntry - 174, // 95: aether.v1.ConfigSnapshot.global_kv:type_name -> aether.v1.ConfigSnapshot.GlobalKvEntry - 175, // 96: aether.v1.ConfigSnapshot.task_context:type_name -> aether.v1.ConfigSnapshot.TaskContextEntry - 176, // 97: aether.v1.ConfigSnapshot.workspace_exclusive_kv:type_name -> aether.v1.ConfigSnapshot.WorkspaceExclusiveKvEntry - 177, // 98: aether.v1.ConfigSnapshot.global_exclusive_kv:type_name -> aether.v1.ConfigSnapshot.GlobalExclusiveKvEntry - 13, // 99: aether.v1.Signal.type:type_name -> aether.v1.Signal.SignalType - 6, // 100: aether.v1.CreateTaskRequest.assignment_mode:type_name -> aether.v1.TaskAssignmentMode - 178, // 101: aether.v1.CreateTaskRequest.launch_param_overrides:type_name -> aether.v1.CreateTaskRequest.LaunchParamOverridesEntry - 179, // 102: aether.v1.CreateTaskRequest.metadata:type_name -> aether.v1.CreateTaskRequest.MetadataEntry - 49, // 103: aether.v1.CreateTaskRequest.authorization:type_name -> aether.v1.AuthorizationContext - 7, // 104: aether.v1.CreateTaskRequest.task_class:type_name -> aether.v1.TaskClass - 180, // 105: aether.v1.TaskAssignment.metadata:type_name -> aether.v1.TaskAssignment.MetadataEntry - 181, // 106: aether.v1.TaskAssignment.launch_params:type_name -> aether.v1.TaskAssignment.LaunchParamsEntry - 7, // 107: aether.v1.TaskAssignment.task_class:type_name -> aether.v1.TaskClass - 14, // 108: aether.v1.CheckpointOperation.op:type_name -> aether.v1.CheckpointOperation.OpType - 15, // 109: aether.v1.AdminQuery.op:type_name -> aether.v1.AdminQuery.OpType - 67, // 110: aether.v1.AdminQuery.filter:type_name -> aether.v1.ConnectionFilter - 1, // 111: aether.v1.ConnectionFilter.type:type_name -> aether.v1.PrincipalType - 1, // 112: aether.v1.ConnectionInfo.type:type_name -> aether.v1.PrincipalType - 70, // 113: aether.v1.AdminResponse.health:type_name -> aether.v1.HealthInfo - 72, // 114: aether.v1.AdminResponse.info:type_name -> aether.v1.GatewayInfo - 73, // 115: aether.v1.AdminResponse.stats:type_name -> aether.v1.GatewayStats - 68, // 116: aether.v1.AdminResponse.connection:type_name -> aether.v1.ConnectionInfo - 68, // 117: aether.v1.AdminResponse.connections:type_name -> aether.v1.ConnectionInfo - 3, // 118: aether.v1.HealthInfo.status:type_name -> aether.v1.HealthStatus - 182, // 119: aether.v1.HealthInfo.checks:type_name -> aether.v1.HealthInfo.ChecksEntry - 73, // 120: aether.v1.HealthInfo.stats:type_name -> aether.v1.GatewayStats - 4, // 121: aether.v1.HealthCheck.status:type_name -> aether.v1.HealthCheckStatus - 16, // 122: aether.v1.SessionOperation.op:type_name -> aether.v1.SessionOperation.OpType - 67, // 123: aether.v1.SessionOperation.filter:type_name -> aether.v1.ConnectionFilter - 49, // 124: aether.v1.SessionOperation.authorization:type_name -> aether.v1.AuthorizationContext - 68, // 125: aether.v1.SessionOperationResponse.connection:type_name -> aether.v1.ConnectionInfo - 68, // 126: aether.v1.SessionOperationResponse.connections:type_name -> aether.v1.ConnectionInfo - 17, // 127: aether.v1.TaskQuery.op:type_name -> aether.v1.TaskQuery.OpType - 77, // 128: aether.v1.TaskQuery.filter:type_name -> aether.v1.TaskFilter - 2, // 129: aether.v1.TaskFilter.status:type_name -> aether.v1.TaskStatus - 2, // 130: aether.v1.TaskFilter.statuses:type_name -> aether.v1.TaskStatus - 7, // 131: aether.v1.TaskFilter.task_class:type_name -> aether.v1.TaskClass - 7, // 132: aether.v1.TaskFilter.exclude_task_classes:type_name -> aether.v1.TaskClass - 2, // 133: aether.v1.TaskFilter.exclude_statuses:type_name -> aether.v1.TaskStatus - 48, // 134: aether.v1.TaskFilter.creator_actor:type_name -> aether.v1.PrincipalRef - 2, // 135: aether.v1.TaskInfo.status:type_name -> aether.v1.TaskStatus - 183, // 136: aether.v1.TaskInfo.metadata:type_name -> aether.v1.TaskInfo.MetadataEntry - 7, // 137: aether.v1.TaskInfo.task_class:type_name -> aether.v1.TaskClass - 81, // 138: aether.v1.TaskInfo.wait_spec:type_name -> aether.v1.WaitSpec - 78, // 139: aether.v1.TaskQueryResponse.task:type_name -> aether.v1.TaskInfo - 78, // 140: aether.v1.TaskQueryResponse.tasks:type_name -> aether.v1.TaskInfo - 18, // 141: aether.v1.TaskOperation.op:type_name -> aether.v1.TaskOperation.OpType - 81, // 142: aether.v1.TaskOperation.wait_spec:type_name -> aether.v1.WaitSpec - 8, // 143: aether.v1.WaitSpec.reason:type_name -> aether.v1.WaitReason - 184, // 144: aether.v1.WaitSpec.input_match:type_name -> aether.v1.WaitSpec.InputMatchEntry - 82, // 145: aether.v1.WaitSpec.hibernation:type_name -> aether.v1.HibernationDescriptor - 78, // 146: aether.v1.TaskOperationResponse.task:type_name -> aether.v1.TaskInfo - 19, // 147: aether.v1.WorkspaceOperation.op:type_name -> aether.v1.WorkspaceOperation.OpType - 85, // 148: aether.v1.WorkspaceOperation.filter:type_name -> aether.v1.WorkspaceFilter - 86, // 149: aether.v1.WorkspaceOperation.workspace:type_name -> aether.v1.WorkspaceInfo - 185, // 150: aether.v1.WorkspaceInfo.metadata:type_name -> aether.v1.WorkspaceInfo.MetadataEntry - 86, // 151: aether.v1.WorkspaceResponse.workspace:type_name -> aether.v1.WorkspaceInfo - 86, // 152: aether.v1.WorkspaceResponse.workspaces:type_name -> aether.v1.WorkspaceInfo - 88, // 153: aether.v1.WorkspaceResponse.message_flow:type_name -> aether.v1.MessageFlowInfo - 89, // 154: aether.v1.MessageFlowInfo.nodes:type_name -> aether.v1.FlowNode - 90, // 155: aether.v1.MessageFlowInfo.edges:type_name -> aether.v1.FlowEdge - 1, // 156: aether.v1.FlowNode.type:type_name -> aether.v1.PrincipalType - 20, // 157: aether.v1.AgentOperation.op:type_name -> aether.v1.AgentOperation.OpType - 92, // 158: aether.v1.AgentOperation.filter:type_name -> aether.v1.AgentFilter - 93, // 159: aether.v1.AgentOperation.agent:type_name -> aether.v1.AgentRegistrationInfo - 95, // 160: aether.v1.AgentOperation.launch_params:type_name -> aether.v1.AgentLaunchParams - 186, // 161: aether.v1.AgentRegistrationInfo.launch_params:type_name -> aether.v1.AgentRegistrationInfo.LaunchParamsEntry - 94, // 162: aether.v1.AgentRegistrationInfo.resource_schema:type_name -> aether.v1.AgentResourceSchemaEntry - 187, // 163: aether.v1.AgentRegistrationInfo.capabilities:type_name -> aether.v1.AgentRegistrationInfo.CapabilitiesEntry - 188, // 164: aether.v1.AgentLaunchParams.param_overrides:type_name -> aether.v1.AgentLaunchParams.ParamOverridesEntry - 93, // 165: aether.v1.AgentResponse.agent:type_name -> aether.v1.AgentRegistrationInfo - 93, // 166: aether.v1.AgentResponse.agents:type_name -> aether.v1.AgentRegistrationInfo - 96, // 167: aether.v1.AgentResponse.orchestrators:type_name -> aether.v1.OrchestratorInfo - 97, // 168: aether.v1.AgentResponse.launch_result:type_name -> aether.v1.AgentLaunchResult - 21, // 169: aether.v1.ACLOperation.op:type_name -> aether.v1.ACLOperation.OpType - 100, // 170: aether.v1.ACLOperation.rule_filter:type_name -> aether.v1.ACLRuleFilter - 101, // 171: aether.v1.ACLOperation.audit_filter:type_name -> aether.v1.ACLAuditFilter - 102, // 172: aether.v1.ACLOperation.grant_request:type_name -> aether.v1.ACLGrantRequest - 103, // 173: aether.v1.ACLOperation.fallback_request:type_name -> aether.v1.ACLSetFallbackRequest - 48, // 174: aether.v1.ACLAuthorityGrantRequest.subject:type_name -> aether.v1.PrincipalRef - 48, // 175: aether.v1.ACLAuthorityGrantRequest.delegate:type_name -> aether.v1.PrincipalRef - 48, // 176: aether.v1.ACLAuthorityGrantRequest.issued_by:type_name -> aether.v1.PrincipalRef - 48, // 177: aether.v1.ACLAuthorityGrantRequest.root_subject:type_name -> aether.v1.PrincipalRef - 105, // 178: aether.v1.ACLAuthorityGrantRequest.resource_scope:type_name -> aether.v1.ACLAuthorityGrantResourceScopeEntry - 189, // 179: aether.v1.ACLAuthorityGrantRequest.metadata:type_name -> aether.v1.ACLAuthorityGrantRequest.MetadataEntry - 190, // 180: aether.v1.ACLAuditEntryInfo.metadata:type_name -> aether.v1.ACLAuditEntryInfo.MetadataEntry - 48, // 181: aether.v1.ACLAuthorityGrantInfo.subject:type_name -> aether.v1.PrincipalRef - 48, // 182: aether.v1.ACLAuthorityGrantInfo.delegate:type_name -> aether.v1.PrincipalRef - 48, // 183: aether.v1.ACLAuthorityGrantInfo.issued_by:type_name -> aether.v1.PrincipalRef - 48, // 184: aether.v1.ACLAuthorityGrantInfo.root_subject:type_name -> aether.v1.PrincipalRef - 105, // 185: aether.v1.ACLAuthorityGrantInfo.resource_scope:type_name -> aether.v1.ACLAuthorityGrantResourceScopeEntry - 191, // 186: aether.v1.ACLAuthorityGrantInfo.metadata:type_name -> aether.v1.ACLAuthorityGrantInfo.MetadataEntry - 108, // 187: aether.v1.ACLResponse.rule:type_name -> aether.v1.ACLRuleInfo - 108, // 188: aether.v1.ACLResponse.rules:type_name -> aether.v1.ACLRuleInfo - 109, // 189: aether.v1.ACLResponse.fallback_policy:type_name -> aether.v1.ACLFallbackPolicyInfo - 110, // 190: aether.v1.ACLResponse.audit_entries:type_name -> aether.v1.ACLAuditEntryInfo - 112, // 191: aether.v1.ACLResponse.cleanup_result:type_name -> aether.v1.ACLCleanupResult - 111, // 192: aether.v1.ACLResponse.authority_grant:type_name -> aether.v1.ACLAuthorityGrantInfo - 111, // 193: aether.v1.ACLResponse.authority_grants:type_name -> aether.v1.ACLAuthorityGrantInfo - 22, // 194: aether.v1.AuthorityGrantOperation.op:type_name -> aether.v1.AuthorityGrantOperation.OpType - 115, // 195: aether.v1.AuthorityGrantOperation.exchange_request:type_name -> aether.v1.AuthorityGrantExchangeRequest - 116, // 196: aether.v1.AuthorityGrantOperation.derive_request:type_name -> aether.v1.AuthorityGrantDeriveRequest - 107, // 197: aether.v1.AuthorityGrantOperation.renew_request:type_name -> aether.v1.ACLRenewAuthorityGrantRequest - 118, // 198: aether.v1.AuthorityGrantOperation.list_request:type_name -> aether.v1.AuthorityGrantListRequest - 119, // 199: aether.v1.AuthorityGrantOperation.batch_exchange_request:type_name -> aether.v1.AuthorityGrantBatchExchangeRequest - 120, // 200: aether.v1.AuthorityGrantOperation.derive_for_target_request:type_name -> aether.v1.AuthorityGrantDeriveForTargetRequest - 105, // 201: aether.v1.AuthorityGrantExchangeRequest.resource_scope:type_name -> aether.v1.ACLAuthorityGrantResourceScopeEntry - 192, // 202: aether.v1.AuthorityGrantExchangeRequest.metadata:type_name -> aether.v1.AuthorityGrantExchangeRequest.MetadataEntry - 48, // 203: aether.v1.AuthorityGrantDeriveRequest.delegate:type_name -> aether.v1.PrincipalRef - 105, // 204: aether.v1.AuthorityGrantDeriveRequest.resource_scope:type_name -> aether.v1.ACLAuthorityGrantResourceScopeEntry - 193, // 205: aether.v1.AuthorityGrantDeriveRequest.metadata:type_name -> aether.v1.AuthorityGrantDeriveRequest.MetadataEntry - 111, // 206: aether.v1.AuthorityGrantResponse.grant:type_name -> aether.v1.ACLAuthorityGrantInfo - 111, // 207: aether.v1.AuthorityGrantResponse.grants:type_name -> aether.v1.ACLAuthorityGrantInfo - 115, // 208: aether.v1.AuthorityGrantBatchExchangeRequest.requests:type_name -> aether.v1.AuthorityGrantExchangeRequest - 48, // 209: aether.v1.AuthorityGrantDeriveForTargetRequest.target:type_name -> aether.v1.PrincipalRef - 48, // 210: aether.v1.AuthorityIdentity.subject:type_name -> aether.v1.PrincipalRef - 48, // 211: aether.v1.AuthorityIdentity.root_subject:type_name -> aether.v1.PrincipalRef - 48, // 212: aether.v1.AuthorityIdentity.delegate:type_name -> aether.v1.PrincipalRef - 48, // 213: aether.v1.AuthorityIdentity.issued_by:type_name -> aether.v1.PrincipalRef - 48, // 214: aether.v1.AuthorityRequestRoutingTarget.principal:type_name -> aether.v1.PrincipalRef - 9, // 215: aether.v1.AuthorityRequest.status:type_name -> aether.v1.AuthorityRequestStatus - 48, // 216: aether.v1.AuthorityRequest.requesting_actor:type_name -> aether.v1.PrincipalRef - 48, // 217: aether.v1.AuthorityRequest.target_subject:type_name -> aether.v1.PrincipalRef - 125, // 218: aether.v1.AuthorityRequest.desired_resource_scope:type_name -> aether.v1.AuthorityRequestResourceScopeEntry - 5, // 219: aether.v1.AuthorityRequest.requested_access_level:type_name -> aether.v1.AccessLevel - 124, // 220: aether.v1.AuthorityRequest.routing_target:type_name -> aether.v1.AuthorityRequestRoutingTarget - 194, // 221: aether.v1.AuthorityRequest.metadata:type_name -> aether.v1.AuthorityRequest.MetadataEntry - 48, // 222: aether.v1.AuthorityRequest.resolved_by:type_name -> aether.v1.PrincipalRef - 48, // 223: aether.v1.CreateAuthorityRequestPayload.requesting_actor:type_name -> aether.v1.PrincipalRef - 48, // 224: aether.v1.CreateAuthorityRequestPayload.target_subject:type_name -> aether.v1.PrincipalRef - 125, // 225: aether.v1.CreateAuthorityRequestPayload.desired_resource_scope:type_name -> aether.v1.AuthorityRequestResourceScopeEntry - 5, // 226: aether.v1.CreateAuthorityRequestPayload.requested_access_level:type_name -> aether.v1.AccessLevel - 124, // 227: aether.v1.CreateAuthorityRequestPayload.routing_target:type_name -> aether.v1.AuthorityRequestRoutingTarget - 195, // 228: aether.v1.CreateAuthorityRequestPayload.metadata:type_name -> aether.v1.CreateAuthorityRequestPayload.MetadataEntry - 23, // 229: aether.v1.ResolveAuthorityRequestPayload.decision:type_name -> aether.v1.ResolveAuthorityRequestPayload.Decision - 125, // 230: aether.v1.ResolveAuthorityRequestPayload.granted_resource_scope:type_name -> aether.v1.AuthorityRequestResourceScopeEntry - 5, // 231: aether.v1.ResolveAuthorityRequestPayload.granted_access_level:type_name -> aether.v1.AccessLevel - 9, // 232: aether.v1.AuthorityRequestListFilter.status:type_name -> aether.v1.AuthorityRequestStatus - 24, // 233: aether.v1.AuthorityRequestOperation.op:type_name -> aether.v1.AuthorityRequestOperation.OpType - 127, // 234: aether.v1.AuthorityRequestOperation.create:type_name -> aether.v1.CreateAuthorityRequestPayload - 128, // 235: aether.v1.AuthorityRequestOperation.resolve:type_name -> aether.v1.ResolveAuthorityRequestPayload - 129, // 236: aether.v1.AuthorityRequestOperation.list_filter:type_name -> aether.v1.AuthorityRequestListFilter - 126, // 237: aether.v1.AuthorityRequestOperationResponse.request:type_name -> aether.v1.AuthorityRequest - 126, // 238: aether.v1.AuthorityRequestOperationResponse.requests:type_name -> aether.v1.AuthorityRequest - 25, // 239: aether.v1.AuthorityRequestEvent.event_type:type_name -> aether.v1.AuthorityRequestEvent.EventType - 126, // 240: aether.v1.AuthorityRequestEvent.request:type_name -> aether.v1.AuthorityRequest - 26, // 241: aether.v1.TokenOperation.op:type_name -> aether.v1.TokenOperation.OpType - 134, // 242: aether.v1.TokenOperation.create_request:type_name -> aether.v1.TokenCreateRequest - 135, // 243: aether.v1.TokenOperation.filter:type_name -> aether.v1.TokenFilter - 136, // 244: aether.v1.TokenResponse.token:type_name -> aether.v1.TokenInfo - 136, // 245: aether.v1.TokenResponse.tokens:type_name -> aether.v1.TokenInfo - 136, // 246: aether.v1.TokenResponse.created_token:type_name -> aether.v1.TokenInfo - 139, // 247: aether.v1.ProgressReport.step:type_name -> aether.v1.ProgressStep - 196, // 248: aether.v1.ProgressReport.metadata:type_name -> aether.v1.ProgressReport.MetadataEntry - 10, // 249: aether.v1.ProgressReport.kind:type_name -> aether.v1.ProgressKind - 139, // 250: aether.v1.ProgressUpdate.step:type_name -> aether.v1.ProgressStep - 197, // 251: aether.v1.ProgressUpdate.metadata:type_name -> aether.v1.ProgressUpdate.MetadataEntry - 10, // 252: aether.v1.ProgressUpdate.kind:type_name -> aether.v1.ProgressKind - 27, // 253: aether.v1.WorkflowOperation.op:type_name -> aether.v1.WorkflowOperation.OpType - 0, // 254: aether.v1.MessageEnvelope.message_type:type_name -> aether.v1.MessageType - 198, // 255: aether.v1.MessageEnvelope.metadata:type_name -> aether.v1.MessageEnvelope.MetadataEntry - 49, // 256: aether.v1.AuditQuery.authorization:type_name -> aether.v1.AuthorizationContext - 146, // 257: aether.v1.AuditQueryResponse.entries:type_name -> aether.v1.AuditEntry - 199, // 258: aether.v1.SubmitAuditEventRequest.metadata:type_name -> aether.v1.SubmitAuditEventRequest.MetadataEntry - 200, // 259: aether.v1.ProxyHttpRequest.headers:type_name -> aether.v1.ProxyHttpRequest.HeadersEntry - 49, // 260: aether.v1.ProxyHttpRequest.authorization:type_name -> aether.v1.AuthorizationContext - 201, // 261: aether.v1.ProxyHttpResponse.headers:type_name -> aether.v1.ProxyHttpResponse.HeadersEntry - 152, // 262: aether.v1.ProxyHttpResponse.error:type_name -> aether.v1.ProxyError - 28, // 263: aether.v1.ProxyError.kind:type_name -> aether.v1.ProxyError.Kind - 29, // 264: aether.v1.TunnelOpen.protocol:type_name -> aether.v1.TunnelOpen.Protocol - 202, // 265: aether.v1.TunnelOpen.metadata:type_name -> aether.v1.TunnelOpen.MetadataEntry - 49, // 266: aether.v1.TunnelOpen.authorization:type_name -> aether.v1.AuthorizationContext - 30, // 267: aether.v1.TunnelClose.reason:type_name -> aether.v1.TunnelClose.Reason - 48, // 268: aether.v1.ResolveAuthorityRequest.actor:type_name -> aether.v1.PrincipalRef - 48, // 269: aether.v1.ResolveAuthorityRequest.subject:type_name -> aether.v1.PrincipalRef - 159, // 270: aether.v1.ResolveAuthorityResponse.authority:type_name -> aether.v1.ResolvedAuthority - 48, // 271: aether.v1.ResolvedAuthority.actor:type_name -> aether.v1.PrincipalRef - 48, // 272: aether.v1.ResolvedAuthority.subject:type_name -> aether.v1.PrincipalRef - 160, // 273: aether.v1.ResolvedAuthority.grant:type_name -> aether.v1.AuthorityGrantInfo - 48, // 274: aether.v1.ConnectionStatusRequest.principal:type_name -> aether.v1.PrincipalRef - 31, // 275: aether.v1.TaskSubscriptionOperation.op:type_name -> aether.v1.TaskSubscriptionOperation.OpType - 166, // 276: aether.v1.TaskEvent.status_changed:type_name -> aether.v1.TaskStatusChangedEvent - 167, // 277: aether.v1.TaskEvent.progress:type_name -> aether.v1.TaskProgressEvent - 168, // 278: aether.v1.TaskEvent.child_lifecycle:type_name -> aether.v1.TaskChildLifecycleEvent - 169, // 279: aether.v1.TaskEvent.authority_request:type_name -> aether.v1.TaskAuthorityRequestEventRelay - 2, // 280: aether.v1.TaskStatusChangedEvent.from_status:type_name -> aether.v1.TaskStatus - 2, // 281: aether.v1.TaskStatusChangedEvent.to_status:type_name -> aether.v1.TaskStatus - 203, // 282: aether.v1.TaskProgressEvent.metadata:type_name -> aether.v1.TaskProgressEvent.MetadataEntry - 2, // 283: aether.v1.TaskChildLifecycleEvent.child_status:type_name -> aether.v1.TaskStatus - 132, // 284: aether.v1.TaskAuthorityRequestEventRelay.event:type_name -> aether.v1.AuthorityRequestEvent - 71, // 285: aether.v1.HealthInfo.ChecksEntry.value:type_name -> aether.v1.HealthCheck - 32, // 286: aether.v1.AetherGateway.Connect:input_type -> aether.v1.UpstreamMessage - 33, // 287: aether.v1.AetherGateway.Connect:output_type -> aether.v1.DownstreamMessage - 287, // [287:288] is the sub-list for method output_type - 286, // [286:287] is the sub-list for method input_type - 286, // [286:286] is the sub-list for extension type_name - 286, // [286:286] is the sub-list for extension extendee - 0, // [0:286] is the sub-list for field type_name + 50, // 94: aether.v1.IncomingMessage.on_behalf_subject:type_name -> aether.v1.PrincipalRef + 187, // 95: aether.v1.ConfigSnapshot.kv:type_name -> aether.v1.ConfigSnapshot.KvEntry + 188, // 96: aether.v1.ConfigSnapshot.global_kv:type_name -> aether.v1.ConfigSnapshot.GlobalKvEntry + 189, // 97: aether.v1.ConfigSnapshot.task_context:type_name -> aether.v1.ConfigSnapshot.TaskContextEntry + 190, // 98: aether.v1.ConfigSnapshot.workspace_exclusive_kv:type_name -> aether.v1.ConfigSnapshot.WorkspaceExclusiveKvEntry + 191, // 99: aether.v1.ConfigSnapshot.global_exclusive_kv:type_name -> aether.v1.ConfigSnapshot.GlobalExclusiveKvEntry + 15, // 100: aether.v1.Signal.type:type_name -> aether.v1.Signal.SignalType + 9, // 101: aether.v1.RetryPolicy.backoff:type_name -> aether.v1.BackoffStrategy + 2, // 102: aether.v1.TaskCompletionEvent.on_statuses:type_name -> aether.v1.TaskStatus + 6, // 103: aether.v1.CreateTaskRequest.assignment_mode:type_name -> aether.v1.TaskAssignmentMode + 192, // 104: aether.v1.CreateTaskRequest.launch_param_overrides:type_name -> aether.v1.CreateTaskRequest.LaunchParamOverridesEntry + 193, // 105: aether.v1.CreateTaskRequest.metadata:type_name -> aether.v1.CreateTaskRequest.MetadataEntry + 51, // 106: aether.v1.CreateTaskRequest.authorization:type_name -> aether.v1.AuthorizationContext + 7, // 107: aether.v1.CreateTaskRequest.task_class:type_name -> aether.v1.TaskClass + 63, // 108: aether.v1.CreateTaskRequest.retry_policy:type_name -> aether.v1.RetryPolicy + 8, // 109: aether.v1.CreateTaskRequest.priority:type_name -> aether.v1.TaskPriority + 64, // 110: aether.v1.CreateTaskRequest.completion_event:type_name -> aether.v1.TaskCompletionEvent + 194, // 111: aether.v1.TaskAssignment.metadata:type_name -> aether.v1.TaskAssignment.MetadataEntry + 195, // 112: aether.v1.TaskAssignment.launch_params:type_name -> aether.v1.TaskAssignment.LaunchParamsEntry + 7, // 113: aether.v1.TaskAssignment.task_class:type_name -> aether.v1.TaskClass + 16, // 114: aether.v1.CheckpointOperation.op:type_name -> aether.v1.CheckpointOperation.OpType + 17, // 115: aether.v1.AdminQuery.op:type_name -> aether.v1.AdminQuery.OpType + 71, // 116: aether.v1.AdminQuery.filter:type_name -> aether.v1.ConnectionFilter + 1, // 117: aether.v1.ConnectionFilter.type:type_name -> aether.v1.PrincipalType + 1, // 118: aether.v1.ConnectionInfo.type:type_name -> aether.v1.PrincipalType + 74, // 119: aether.v1.AdminResponse.health:type_name -> aether.v1.HealthInfo + 76, // 120: aether.v1.AdminResponse.info:type_name -> aether.v1.GatewayInfo + 77, // 121: aether.v1.AdminResponse.stats:type_name -> aether.v1.GatewayStats + 72, // 122: aether.v1.AdminResponse.connection:type_name -> aether.v1.ConnectionInfo + 72, // 123: aether.v1.AdminResponse.connections:type_name -> aether.v1.ConnectionInfo + 3, // 124: aether.v1.HealthInfo.status:type_name -> aether.v1.HealthStatus + 196, // 125: aether.v1.HealthInfo.checks:type_name -> aether.v1.HealthInfo.ChecksEntry + 77, // 126: aether.v1.HealthInfo.stats:type_name -> aether.v1.GatewayStats + 4, // 127: aether.v1.HealthCheck.status:type_name -> aether.v1.HealthCheckStatus + 18, // 128: aether.v1.SessionOperation.op:type_name -> aether.v1.SessionOperation.OpType + 71, // 129: aether.v1.SessionOperation.filter:type_name -> aether.v1.ConnectionFilter + 51, // 130: aether.v1.SessionOperation.authorization:type_name -> aether.v1.AuthorizationContext + 72, // 131: aether.v1.SessionOperationResponse.connection:type_name -> aether.v1.ConnectionInfo + 72, // 132: aether.v1.SessionOperationResponse.connections:type_name -> aether.v1.ConnectionInfo + 19, // 133: aether.v1.TaskQuery.op:type_name -> aether.v1.TaskQuery.OpType + 81, // 134: aether.v1.TaskQuery.filter:type_name -> aether.v1.TaskFilter + 2, // 135: aether.v1.TaskFilter.status:type_name -> aether.v1.TaskStatus + 2, // 136: aether.v1.TaskFilter.statuses:type_name -> aether.v1.TaskStatus + 7, // 137: aether.v1.TaskFilter.task_class:type_name -> aether.v1.TaskClass + 7, // 138: aether.v1.TaskFilter.exclude_task_classes:type_name -> aether.v1.TaskClass + 2, // 139: aether.v1.TaskFilter.exclude_statuses:type_name -> aether.v1.TaskStatus + 50, // 140: aether.v1.TaskFilter.creator_actor:type_name -> aether.v1.PrincipalRef + 8, // 141: aether.v1.TaskFilter.priority:type_name -> aether.v1.TaskPriority + 8, // 142: aether.v1.TaskFilter.min_priority:type_name -> aether.v1.TaskPriority + 2, // 143: aether.v1.TaskInfo.status:type_name -> aether.v1.TaskStatus + 197, // 144: aether.v1.TaskInfo.metadata:type_name -> aether.v1.TaskInfo.MetadataEntry + 7, // 145: aether.v1.TaskInfo.task_class:type_name -> aether.v1.TaskClass + 85, // 146: aether.v1.TaskInfo.wait_spec:type_name -> aether.v1.WaitSpec + 8, // 147: aether.v1.TaskInfo.priority:type_name -> aether.v1.TaskPriority + 64, // 148: aether.v1.TaskInfo.completion_event:type_name -> aether.v1.TaskCompletionEvent + 82, // 149: aether.v1.TaskQueryResponse.task:type_name -> aether.v1.TaskInfo + 82, // 150: aether.v1.TaskQueryResponse.tasks:type_name -> aether.v1.TaskInfo + 20, // 151: aether.v1.TaskOperation.op:type_name -> aether.v1.TaskOperation.OpType + 85, // 152: aether.v1.TaskOperation.wait_spec:type_name -> aether.v1.WaitSpec + 10, // 153: aether.v1.WaitSpec.reason:type_name -> aether.v1.WaitReason + 198, // 154: aether.v1.WaitSpec.input_match:type_name -> aether.v1.WaitSpec.InputMatchEntry + 86, // 155: aether.v1.WaitSpec.hibernation:type_name -> aether.v1.HibernationDescriptor + 82, // 156: aether.v1.TaskOperationResponse.task:type_name -> aether.v1.TaskInfo + 21, // 157: aether.v1.WorkspaceOperation.op:type_name -> aether.v1.WorkspaceOperation.OpType + 89, // 158: aether.v1.WorkspaceOperation.filter:type_name -> aether.v1.WorkspaceFilter + 90, // 159: aether.v1.WorkspaceOperation.workspace:type_name -> aether.v1.WorkspaceInfo + 199, // 160: aether.v1.WorkspaceInfo.metadata:type_name -> aether.v1.WorkspaceInfo.MetadataEntry + 90, // 161: aether.v1.WorkspaceResponse.workspace:type_name -> aether.v1.WorkspaceInfo + 90, // 162: aether.v1.WorkspaceResponse.workspaces:type_name -> aether.v1.WorkspaceInfo + 92, // 163: aether.v1.WorkspaceResponse.message_flow:type_name -> aether.v1.MessageFlowInfo + 93, // 164: aether.v1.MessageFlowInfo.nodes:type_name -> aether.v1.FlowNode + 94, // 165: aether.v1.MessageFlowInfo.edges:type_name -> aether.v1.FlowEdge + 1, // 166: aether.v1.FlowNode.type:type_name -> aether.v1.PrincipalType + 22, // 167: aether.v1.AgentOperation.op:type_name -> aether.v1.AgentOperation.OpType + 96, // 168: aether.v1.AgentOperation.filter:type_name -> aether.v1.AgentFilter + 97, // 169: aether.v1.AgentOperation.agent:type_name -> aether.v1.AgentRegistrationInfo + 99, // 170: aether.v1.AgentOperation.launch_params:type_name -> aether.v1.AgentLaunchParams + 200, // 171: aether.v1.AgentRegistrationInfo.launch_params:type_name -> aether.v1.AgentRegistrationInfo.LaunchParamsEntry + 98, // 172: aether.v1.AgentRegistrationInfo.resource_schema:type_name -> aether.v1.AgentResourceSchemaEntry + 201, // 173: aether.v1.AgentRegistrationInfo.capabilities:type_name -> aether.v1.AgentRegistrationInfo.CapabilitiesEntry + 202, // 174: aether.v1.AgentLaunchParams.param_overrides:type_name -> aether.v1.AgentLaunchParams.ParamOverridesEntry + 97, // 175: aether.v1.AgentResponse.agent:type_name -> aether.v1.AgentRegistrationInfo + 97, // 176: aether.v1.AgentResponse.agents:type_name -> aether.v1.AgentRegistrationInfo + 100, // 177: aether.v1.AgentResponse.orchestrators:type_name -> aether.v1.OrchestratorInfo + 101, // 178: aether.v1.AgentResponse.launch_result:type_name -> aether.v1.AgentLaunchResult + 23, // 179: aether.v1.ACLOperation.op:type_name -> aether.v1.ACLOperation.OpType + 104, // 180: aether.v1.ACLOperation.rule_filter:type_name -> aether.v1.ACLRuleFilter + 105, // 181: aether.v1.ACLOperation.audit_filter:type_name -> aether.v1.ACLAuditFilter + 106, // 182: aether.v1.ACLOperation.grant_request:type_name -> aether.v1.ACLGrantRequest + 107, // 183: aether.v1.ACLOperation.fallback_request:type_name -> aether.v1.ACLSetFallbackRequest + 50, // 184: aether.v1.ACLOperation.principal:type_name -> aether.v1.PrincipalRef + 117, // 185: aether.v1.ACLOperation.group_request:type_name -> aether.v1.ACLGroupRequest + 118, // 186: aether.v1.ACLOperation.role_request:type_name -> aether.v1.ACLRoleRequest + 119, // 187: aether.v1.ACLOperation.member_request:type_name -> aether.v1.ACLGroupMemberRequest + 120, // 188: aether.v1.ACLOperation.assignment_request:type_name -> aether.v1.ACLRoleAssignmentRequest + 51, // 189: aether.v1.ACLOperation.authorization:type_name -> aether.v1.AuthorizationContext + 50, // 190: aether.v1.ACLAuthorityGrantRequest.subject:type_name -> aether.v1.PrincipalRef + 50, // 191: aether.v1.ACLAuthorityGrantRequest.delegate:type_name -> aether.v1.PrincipalRef + 50, // 192: aether.v1.ACLAuthorityGrantRequest.issued_by:type_name -> aether.v1.PrincipalRef + 50, // 193: aether.v1.ACLAuthorityGrantRequest.root_subject:type_name -> aether.v1.PrincipalRef + 109, // 194: aether.v1.ACLAuthorityGrantRequest.resource_scope:type_name -> aether.v1.ACLAuthorityGrantResourceScopeEntry + 203, // 195: aether.v1.ACLAuthorityGrantRequest.metadata:type_name -> aether.v1.ACLAuthorityGrantRequest.MetadataEntry + 204, // 196: aether.v1.ACLAuditEntryInfo.metadata:type_name -> aether.v1.ACLAuditEntryInfo.MetadataEntry + 50, // 197: aether.v1.ACLAuthorityGrantInfo.subject:type_name -> aether.v1.PrincipalRef + 50, // 198: aether.v1.ACLAuthorityGrantInfo.delegate:type_name -> aether.v1.PrincipalRef + 50, // 199: aether.v1.ACLAuthorityGrantInfo.issued_by:type_name -> aether.v1.PrincipalRef + 50, // 200: aether.v1.ACLAuthorityGrantInfo.root_subject:type_name -> aether.v1.PrincipalRef + 109, // 201: aether.v1.ACLAuthorityGrantInfo.resource_scope:type_name -> aether.v1.ACLAuthorityGrantResourceScopeEntry + 205, // 202: aether.v1.ACLAuthorityGrantInfo.metadata:type_name -> aether.v1.ACLAuthorityGrantInfo.MetadataEntry + 206, // 203: aether.v1.ACLGroupRequest.metadata:type_name -> aether.v1.ACLGroupRequest.MetadataEntry + 207, // 204: aether.v1.ACLRoleRequest.metadata:type_name -> aether.v1.ACLRoleRequest.MetadataEntry + 208, // 205: aether.v1.ACLGroupInfo.metadata:type_name -> aether.v1.ACLGroupInfo.MetadataEntry + 209, // 206: aether.v1.ACLRoleInfo.metadata:type_name -> aether.v1.ACLRoleInfo.MetadataEntry + 125, // 207: aether.v1.ACLAccessExplanationInfo.contributions:type_name -> aether.v1.ACLAccessContributionInfo + 112, // 208: aether.v1.ACLResponse.rule:type_name -> aether.v1.ACLRuleInfo + 112, // 209: aether.v1.ACLResponse.rules:type_name -> aether.v1.ACLRuleInfo + 113, // 210: aether.v1.ACLResponse.fallback_policy:type_name -> aether.v1.ACLFallbackPolicyInfo + 114, // 211: aether.v1.ACLResponse.audit_entries:type_name -> aether.v1.ACLAuditEntryInfo + 116, // 212: aether.v1.ACLResponse.cleanup_result:type_name -> aether.v1.ACLCleanupResult + 115, // 213: aether.v1.ACLResponse.authority_grant:type_name -> aether.v1.ACLAuthorityGrantInfo + 115, // 214: aether.v1.ACLResponse.authority_grants:type_name -> aether.v1.ACLAuthorityGrantInfo + 121, // 215: aether.v1.ACLResponse.group:type_name -> aether.v1.ACLGroupInfo + 121, // 216: aether.v1.ACLResponse.groups:type_name -> aether.v1.ACLGroupInfo + 122, // 217: aether.v1.ACLResponse.role:type_name -> aether.v1.ACLRoleInfo + 122, // 218: aether.v1.ACLResponse.roles:type_name -> aether.v1.ACLRoleInfo + 123, // 219: aether.v1.ACLResponse.group_members:type_name -> aether.v1.ACLGroupMemberInfo + 124, // 220: aether.v1.ACLResponse.role_assignments:type_name -> aether.v1.ACLRoleAssignmentInfo + 126, // 221: aether.v1.ACLResponse.explanation:type_name -> aether.v1.ACLAccessExplanationInfo + 24, // 222: aether.v1.AuthorityGrantOperation.op:type_name -> aether.v1.AuthorityGrantOperation.OpType + 129, // 223: aether.v1.AuthorityGrantOperation.exchange_request:type_name -> aether.v1.AuthorityGrantExchangeRequest + 130, // 224: aether.v1.AuthorityGrantOperation.derive_request:type_name -> aether.v1.AuthorityGrantDeriveRequest + 111, // 225: aether.v1.AuthorityGrantOperation.renew_request:type_name -> aether.v1.ACLRenewAuthorityGrantRequest + 132, // 226: aether.v1.AuthorityGrantOperation.list_request:type_name -> aether.v1.AuthorityGrantListRequest + 133, // 227: aether.v1.AuthorityGrantOperation.batch_exchange_request:type_name -> aether.v1.AuthorityGrantBatchExchangeRequest + 134, // 228: aether.v1.AuthorityGrantOperation.derive_for_target_request:type_name -> aether.v1.AuthorityGrantDeriveForTargetRequest + 109, // 229: aether.v1.AuthorityGrantExchangeRequest.resource_scope:type_name -> aether.v1.ACLAuthorityGrantResourceScopeEntry + 210, // 230: aether.v1.AuthorityGrantExchangeRequest.metadata:type_name -> aether.v1.AuthorityGrantExchangeRequest.MetadataEntry + 50, // 231: aether.v1.AuthorityGrantDeriveRequest.delegate:type_name -> aether.v1.PrincipalRef + 109, // 232: aether.v1.AuthorityGrantDeriveRequest.resource_scope:type_name -> aether.v1.ACLAuthorityGrantResourceScopeEntry + 211, // 233: aether.v1.AuthorityGrantDeriveRequest.metadata:type_name -> aether.v1.AuthorityGrantDeriveRequest.MetadataEntry + 115, // 234: aether.v1.AuthorityGrantResponse.grant:type_name -> aether.v1.ACLAuthorityGrantInfo + 115, // 235: aether.v1.AuthorityGrantResponse.grants:type_name -> aether.v1.ACLAuthorityGrantInfo + 129, // 236: aether.v1.AuthorityGrantBatchExchangeRequest.requests:type_name -> aether.v1.AuthorityGrantExchangeRequest + 50, // 237: aether.v1.AuthorityGrantDeriveForTargetRequest.target:type_name -> aether.v1.PrincipalRef + 50, // 238: aether.v1.AuthorityIdentity.subject:type_name -> aether.v1.PrincipalRef + 50, // 239: aether.v1.AuthorityIdentity.root_subject:type_name -> aether.v1.PrincipalRef + 50, // 240: aether.v1.AuthorityIdentity.delegate:type_name -> aether.v1.PrincipalRef + 50, // 241: aether.v1.AuthorityIdentity.issued_by:type_name -> aether.v1.PrincipalRef + 50, // 242: aether.v1.AuthorityRequestRoutingTarget.principal:type_name -> aether.v1.PrincipalRef + 11, // 243: aether.v1.AuthorityRequest.status:type_name -> aether.v1.AuthorityRequestStatus + 50, // 244: aether.v1.AuthorityRequest.requesting_actor:type_name -> aether.v1.PrincipalRef + 50, // 245: aether.v1.AuthorityRequest.target_subject:type_name -> aether.v1.PrincipalRef + 139, // 246: aether.v1.AuthorityRequest.desired_resource_scope:type_name -> aether.v1.AuthorityRequestResourceScopeEntry + 5, // 247: aether.v1.AuthorityRequest.requested_access_level:type_name -> aether.v1.AccessLevel + 138, // 248: aether.v1.AuthorityRequest.routing_target:type_name -> aether.v1.AuthorityRequestRoutingTarget + 212, // 249: aether.v1.AuthorityRequest.metadata:type_name -> aether.v1.AuthorityRequest.MetadataEntry + 50, // 250: aether.v1.AuthorityRequest.resolved_by:type_name -> aether.v1.PrincipalRef + 50, // 251: aether.v1.CreateAuthorityRequestPayload.requesting_actor:type_name -> aether.v1.PrincipalRef + 50, // 252: aether.v1.CreateAuthorityRequestPayload.target_subject:type_name -> aether.v1.PrincipalRef + 139, // 253: aether.v1.CreateAuthorityRequestPayload.desired_resource_scope:type_name -> aether.v1.AuthorityRequestResourceScopeEntry + 5, // 254: aether.v1.CreateAuthorityRequestPayload.requested_access_level:type_name -> aether.v1.AccessLevel + 138, // 255: aether.v1.CreateAuthorityRequestPayload.routing_target:type_name -> aether.v1.AuthorityRequestRoutingTarget + 213, // 256: aether.v1.CreateAuthorityRequestPayload.metadata:type_name -> aether.v1.CreateAuthorityRequestPayload.MetadataEntry + 25, // 257: aether.v1.ResolveAuthorityRequestPayload.decision:type_name -> aether.v1.ResolveAuthorityRequestPayload.Decision + 139, // 258: aether.v1.ResolveAuthorityRequestPayload.granted_resource_scope:type_name -> aether.v1.AuthorityRequestResourceScopeEntry + 5, // 259: aether.v1.ResolveAuthorityRequestPayload.granted_access_level:type_name -> aether.v1.AccessLevel + 11, // 260: aether.v1.AuthorityRequestListFilter.status:type_name -> aether.v1.AuthorityRequestStatus + 26, // 261: aether.v1.AuthorityRequestOperation.op:type_name -> aether.v1.AuthorityRequestOperation.OpType + 141, // 262: aether.v1.AuthorityRequestOperation.create:type_name -> aether.v1.CreateAuthorityRequestPayload + 142, // 263: aether.v1.AuthorityRequestOperation.resolve:type_name -> aether.v1.ResolveAuthorityRequestPayload + 143, // 264: aether.v1.AuthorityRequestOperation.list_filter:type_name -> aether.v1.AuthorityRequestListFilter + 140, // 265: aether.v1.AuthorityRequestOperationResponse.request:type_name -> aether.v1.AuthorityRequest + 140, // 266: aether.v1.AuthorityRequestOperationResponse.requests:type_name -> aether.v1.AuthorityRequest + 27, // 267: aether.v1.AuthorityRequestEvent.event_type:type_name -> aether.v1.AuthorityRequestEvent.EventType + 140, // 268: aether.v1.AuthorityRequestEvent.request:type_name -> aether.v1.AuthorityRequest + 28, // 269: aether.v1.TokenOperation.op:type_name -> aether.v1.TokenOperation.OpType + 148, // 270: aether.v1.TokenOperation.create_request:type_name -> aether.v1.TokenCreateRequest + 149, // 271: aether.v1.TokenOperation.filter:type_name -> aether.v1.TokenFilter + 150, // 272: aether.v1.TokenResponse.token:type_name -> aether.v1.TokenInfo + 150, // 273: aether.v1.TokenResponse.tokens:type_name -> aether.v1.TokenInfo + 150, // 274: aether.v1.TokenResponse.created_token:type_name -> aether.v1.TokenInfo + 153, // 275: aether.v1.ProgressReport.step:type_name -> aether.v1.ProgressStep + 214, // 276: aether.v1.ProgressReport.metadata:type_name -> aether.v1.ProgressReport.MetadataEntry + 12, // 277: aether.v1.ProgressReport.kind:type_name -> aether.v1.ProgressKind + 153, // 278: aether.v1.ProgressUpdate.step:type_name -> aether.v1.ProgressStep + 215, // 279: aether.v1.ProgressUpdate.metadata:type_name -> aether.v1.ProgressUpdate.MetadataEntry + 12, // 280: aether.v1.ProgressUpdate.kind:type_name -> aether.v1.ProgressKind + 29, // 281: aether.v1.WorkflowOperation.op:type_name -> aether.v1.WorkflowOperation.OpType + 0, // 282: aether.v1.MessageEnvelope.message_type:type_name -> aether.v1.MessageType + 216, // 283: aether.v1.MessageEnvelope.metadata:type_name -> aether.v1.MessageEnvelope.MetadataEntry + 50, // 284: aether.v1.MessageEnvelope.on_behalf_subject:type_name -> aether.v1.PrincipalRef + 51, // 285: aether.v1.AuditQuery.authorization:type_name -> aether.v1.AuthorizationContext + 160, // 286: aether.v1.AuditQueryResponse.entries:type_name -> aether.v1.AuditEntry + 217, // 287: aether.v1.SubmitAuditEventRequest.metadata:type_name -> aether.v1.SubmitAuditEventRequest.MetadataEntry + 218, // 288: aether.v1.ProxyHttpRequest.headers:type_name -> aether.v1.ProxyHttpRequest.HeadersEntry + 51, // 289: aether.v1.ProxyHttpRequest.authorization:type_name -> aether.v1.AuthorizationContext + 219, // 290: aether.v1.ProxyHttpResponse.headers:type_name -> aether.v1.ProxyHttpResponse.HeadersEntry + 166, // 291: aether.v1.ProxyHttpResponse.error:type_name -> aether.v1.ProxyError + 30, // 292: aether.v1.ProxyError.kind:type_name -> aether.v1.ProxyError.Kind + 31, // 293: aether.v1.TunnelOpen.protocol:type_name -> aether.v1.TunnelOpen.Protocol + 220, // 294: aether.v1.TunnelOpen.metadata:type_name -> aether.v1.TunnelOpen.MetadataEntry + 51, // 295: aether.v1.TunnelOpen.authorization:type_name -> aether.v1.AuthorizationContext + 32, // 296: aether.v1.TunnelClose.reason:type_name -> aether.v1.TunnelClose.Reason + 50, // 297: aether.v1.ResolveAuthorityRequest.actor:type_name -> aether.v1.PrincipalRef + 50, // 298: aether.v1.ResolveAuthorityRequest.subject:type_name -> aether.v1.PrincipalRef + 173, // 299: aether.v1.ResolveAuthorityResponse.authority:type_name -> aether.v1.ResolvedAuthority + 50, // 300: aether.v1.ResolvedAuthority.actor:type_name -> aether.v1.PrincipalRef + 50, // 301: aether.v1.ResolvedAuthority.subject:type_name -> aether.v1.PrincipalRef + 174, // 302: aether.v1.ResolvedAuthority.grant:type_name -> aether.v1.AuthorityGrantInfo + 50, // 303: aether.v1.ConnectionStatusRequest.principal:type_name -> aether.v1.PrincipalRef + 33, // 304: aether.v1.TaskSubscriptionOperation.op:type_name -> aether.v1.TaskSubscriptionOperation.OpType + 180, // 305: aether.v1.TaskEvent.status_changed:type_name -> aether.v1.TaskStatusChangedEvent + 181, // 306: aether.v1.TaskEvent.progress:type_name -> aether.v1.TaskProgressEvent + 182, // 307: aether.v1.TaskEvent.child_lifecycle:type_name -> aether.v1.TaskChildLifecycleEvent + 183, // 308: aether.v1.TaskEvent.authority_request:type_name -> aether.v1.TaskAuthorityRequestEventRelay + 2, // 309: aether.v1.TaskStatusChangedEvent.from_status:type_name -> aether.v1.TaskStatus + 2, // 310: aether.v1.TaskStatusChangedEvent.to_status:type_name -> aether.v1.TaskStatus + 221, // 311: aether.v1.TaskProgressEvent.metadata:type_name -> aether.v1.TaskProgressEvent.MetadataEntry + 2, // 312: aether.v1.TaskChildLifecycleEvent.child_status:type_name -> aether.v1.TaskStatus + 146, // 313: aether.v1.TaskAuthorityRequestEventRelay.event:type_name -> aether.v1.AuthorityRequestEvent + 75, // 314: aether.v1.HealthInfo.ChecksEntry.value:type_name -> aether.v1.HealthCheck + 34, // 315: aether.v1.AetherGateway.Connect:input_type -> aether.v1.UpstreamMessage + 35, // 316: aether.v1.AetherGateway.Connect:output_type -> aether.v1.DownstreamMessage + 316, // [316:317] is the sub-list for method output_type + 315, // [315:316] is the sub-list for method input_type + 315, // [315:315] is the sub-list for extension type_name + 315, // [315:315] is the sub-list for extension extendee + 0, // [0:315] is the sub-list for field type_name } func init() { file_aether_proto_init() } @@ -18976,7 +20844,7 @@ func file_aether_proto_init() { (*InitConnection_Bridge)(nil), (*InitConnection_Service)(nil), } - file_aether_proto_msgTypes[133].OneofWrappers = []any{ + file_aether_proto_msgTypes[145].OneofWrappers = []any{ (*TaskEvent_StatusChanged)(nil), (*TaskEvent_Progress)(nil), (*TaskEvent_ChildLifecycle)(nil), @@ -18987,8 +20855,8 @@ func file_aether_proto_init() { File: protoimpl.DescBuilder{ GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_aether_proto_rawDesc), len(file_aether_proto_rawDesc)), - NumEnums: 32, - NumMessages: 172, + NumEnums: 34, + NumMessages: 188, NumExtensions: 0, NumServices: 1, }, diff --git a/api/proto/aether.proto b/api/proto/aether.proto index 3c14886..efd4ac0 100644 --- a/api/proto/aether.proto +++ b/api/proto/aether.proto @@ -269,6 +269,15 @@ message BridgeIdentity { message ServiceIdentity { string implementation = 1; // e.g., "frontend-api", "platform-backend" string specifier = 2; // Instance identifier for uniqueness (e.g., "pod-1", "default") + // When true, the gateway does NOT add this service connection to the + // pool-task worker index, so it is never targeted for POOL-mode task + // assignments for its implementation. Default false = pool consumer + // (back-compat: existing services keep receiving pool tasks). Serve-only + // instances that register no task handlers (e.g. a MemoryLayer server with + // its in-process worker disabled) set this so pool tasks are routed only to + // real workers instead of being claimed-and-dropped (and stuck until + // reconcile). Agents are always pool consumers regardless of this field. + bool no_pool_consumer = 3; } message AgentIdentity { @@ -466,6 +475,31 @@ message KVOperation { DECREMENT = 5; INCREMENT_IF = 6; // Atomic increment that succeeds only if result <= guard_value DECREMENT_IF = 7; // Atomic decrement that succeeds only if result >= guard_value + // Atomic conditional writes — the building blocks for distributed + // coordination primitives (mutex, leader election, run-once). All three + // are backed across Redis / Badger / NATS-JetStream. KVResponse.applied + // reports whether the conditional mutation took effect. + SET_NX = 8; // Set `value` only if the key is absent. applied=true iff written. + COMPARE_AND_SET = 9; // Set `value` only if current == expected_value. applied=true iff swapped. + COMPARE_AND_DELETE = 10; // Delete only if current == expected_value. applied=true iff deleted. + // REMOVAL-ONLY purge of ANOTHER principal's KV namespace, for lifecycle + // managers reaping ephemeral principals (e.g. sandbox-provider clearing a + // destroyed sidecar's private state). Deletes every key in + // (target_identity, scope) optionally filtered by `key` as a prefix. + // Gated by the capability/kv_purge_identity ACL grant. By design it + // returns ONLY KVResponse.counter_value (count deleted) — never keys or + // values — so the grant cannot be used to exfiltrate another principal's + // data. Maintains component separation: the gateway has no sandbox-specific + // knowledge; the caller decides what to purge and when. + PURGE_IDENTITY = 11; + // Atomic set primitives backing fan-in joins (set-completeness) and + // at-most-once dedup ledgers, native across Redis / Badger / NATS-JetStream. + // SET_ADD adds `value` (the member) to the set at `key`, (re)setting `ttl` + // on a newly-added member: KVResponse.applied=true iff newly added, + // counter_value=cardinality after the add. SET_CARD returns + // counter_value=cardinality (0 if absent). + SET_ADD = 12; + SET_CARD = 13; } // Scope identifies the (identity x sharing) cell of the KV matrix. // @@ -504,6 +538,23 @@ message KVOperation { // default of 1, matching unguarded INCREMENT/DECREMENT. Must be >= 0; // negative deltas are rejected by the server. int64 delta_value = 11; + // Expected current value for COMPARE_AND_SET / COMPARE_AND_DELETE. The + // mutation applies only when the stored value equals these bytes. Unused by + // other ops. For an empty/absent comparison use SET_NX instead. + bytes expected_value = 12; + // LIST pagination. limit caps the keys returned in one LIST response (<=0 → + // server default). cursor pages through results: pass the previous + // KVResponse.next_cursor to fetch the next page; empty starts from the + // beginning. For LIST, `key` (field 3) is the key prefix filter, applied + // server-side BEFORE the limit. Unused by non-LIST ops. + int32 limit = 13; + string cursor = 14; + // PURGE_IDENTITY only: the principal whose KV namespace to purge (e.g. + // "sv::sandbox-sidecar::"). The caller's OWN identity authorizes + // the op via capability/kv_purge_identity; this names the TARGET namespace. + // `key` (field 3) acts as an optional prefix filter; `scope` selects which + // scope to purge. Ignored by all other ops. + string target_identity = 15; } message KVResponse { @@ -519,7 +570,17 @@ message KVResponse { map kv_map = 4; string request_id = 5; // Echoed from the originating KVOperation for correlation int64 counter_value = 6; // Result of INCREMENT/DECREMENT (and INCREMENT_IF/DECREMENT_IF) - bool applied = 7; // True iff INCREMENT_IF/DECREMENT_IF mutation was applied; for unguarded ops always true on success + // True iff a conditional mutation was applied. For INCREMENT_IF/DECREMENT_IF + // this is the guard result; for SET_NX/COMPARE_AND_SET/COMPARE_AND_DELETE it + // reports whether the write/delete took effect. For unguarded ops always + // true on success. On a failed COMPARE_AND_SET/COMPARE_AND_DELETE, `value` + // carries the live stored value so callers can observe the current holder. + bool applied = 7; + // LIST pagination. next_cursor is an opaque token to pass as + // KVOperation.cursor for the next page; empty when iteration is complete. + // has_more is true when more matching keys remain beyond this page. + string next_cursor = 8; + bool has_more = 9; } message IncomingMessage { @@ -536,6 +597,14 @@ message IncomingMessage { // source_topic (which carries the sender's identity-topic, not the // declared event/metric workspace). string workspace = 4; + + // Gateway-set, spoof-proof resolved on-behalf-of subject, mirrored from + // MessageEnvelope.on_behalf_subject at delivery time. Populated ONLY when the + // sender's SendMessage carried an AuthorizationContext the gateway resolved + // to an OBO subject. Lets a recipient identify the *user* a message was sent + // for, distinct from the sending identity in source_topic. Empty for direct + // (non-OBO) sends. See MessageEnvelope.on_behalf_subject. + PrincipalRef on_behalf_subject = 5; } message ConfigSnapshot { @@ -601,6 +670,75 @@ enum TaskClass { // terminal state (completed / failed). } +// TaskPriority orders task dispatch. Unlike TaskClass (a UI hint), the server +// USES priority for scheduling: among pending tasks eligible for delivery, +// higher priority is dispatched first; ties break FIFO (oldest created_at). +// Underlying values are intentionally spaced so new levels can be inserted +// later, and the numeric value is used directly as the descending sort key. +enum TaskPriority { + TASK_PRIORITY_UNSPECIFIED = 0; // Treated as NORMAL for back-compat. + TASK_PRIORITY_XLOW = 10; // Lowest; best-effort, yields to everything. + TASK_PRIORITY_LOW = 20; // Below normal. + TASK_PRIORITY_NORMAL = 30; // Default. + TASK_PRIORITY_HIGH = 40; // Above normal; jumps ahead of normal/low. + TASK_PRIORITY_PREEMPT = 50; // Highest; reserved for future true preemption. +} + +// BackoffStrategy describes how the task store scales delays across retry +// attempts when a RetryPolicy is attached to a task. Defaults to +// EXPONENTIAL when unspecified (preserves current behavior). +enum BackoffStrategy { + BACKOFF_STRATEGY_UNSPECIFIED = 0; + BACKOFF_STRATEGY_FIXED = 1; // Same delay every attempt. + BACKOFF_STRATEGY_EXPONENTIAL = 2; // delay = initial * 2^(n-1), capped at max_delay_ms. + BACKOFF_STRATEGY_EXPLICIT_SCHEDULE = 3; // Use schedule_ms[attempt-1]; clamp the last entry. +} + +// RetryPolicy lifts retry scheduling out of individual workers and into the +// task store. When a task carries a RetryPolicy and a worker calls FailTask +// without an explicit reschedule, the store computes next_retry_at from +// this policy and re-pends the task. The waker then picks it up at the +// scheduled time. Workers can still call RescheduleTaskAt to override +// (e.g., to honor a Retry-After header). +message RetryPolicy { + // Total attempts allowed (1 = no retries). 0 means use server default (3). + int32 max_attempts = 1; + BackoffStrategy backoff = 2; + int64 initial_delay_ms = 3; + int64 max_delay_ms = 4; + // Multiplicative random jitter in [0,1]. Final delay = computed * + // (1 + uniform(-jitter, +jitter)). + double jitter_factor = 5; + // For BACKOFF_STRATEGY_EXPLICIT_SCHEDULE. Indexed by 0-based attempt + // number; the final entry is used for any subsequent attempt. + repeated int64 schedule_ms = 6; + // Optional worker convention: HTTP-style status codes that should + // trigger a retry. The task store does not inspect failure context; + // workers use this to decide whether to FailTask or CompleteTask with + // permanent-failure semantics. + repeated int32 retryable_status_codes = 7; + // Worker hint: prefer Retry-After (or analogous) over computed delay + // when set on the failure response. + bool honor_retry_after = 8; +} + +// TaskCompletionEvent opts a task into "feed B": when it reaches a terminal +// status the server publishes a domain event onto the event plane (event::*) +// so a workflow join (or any rule) can gather over task completions without the +// worker emitting its own event. The emitted payload carries +// {task_id, status, workspace, correlation_id, metadata}. +message TaskCompletionEvent { + // enabled turns the feature on. When false (default) no completion event is + // emitted (today's behavior). + bool enabled = 1; + // event_name is the event name to publish. Empty ⇒ derived as + // "task.completed" / "task.failed" / "task.cancelled" from the terminal status. + string event_name = 2; + // on_statuses restricts emission to these terminal statuses. Empty ⇒ all + // terminal statuses (completed, failed, cancelled). + repeated TaskStatus on_statuses = 3; +} + message CreateTaskRequest { string task_type = 1; string workspace = 2; @@ -647,6 +785,37 @@ message CreateTaskRequest { // Persisted on Task.context_id and queryable via TaskFilter.context_id. // Empty = no session grouping. string context_id = 13; + + // Optional. When set, the task store computes next_retry_at on FailTask + // according to this policy and re-pends the task automatically (up to + // max_attempts). Absent = legacy behavior (immediate re-pend, hardcoded + // max_retries=3). + RetryPolicy retry_policy = 14; + + // Optional dispatch priority. Defaults to UNSPECIFIED ⇒ NORMAL. Higher + // priority pending tasks are delivered before lower ones (ties break FIFO). + TaskPriority priority = 15; + + // Optional idempotency key for exactly-once task creation. When non-empty the + // gateway dedupes creation on this key: a duplicate request returns the + // existing task identity instead of creating a second task. Workflow joins + // set it to make an on_complete create_task fire exactly once under retries + // or engine restart. + string idempotency_key = 16; + + // Optional fan-out/fan-in correlation identity, distinct from task_id. When + // non-empty it is persisted on the task and propagated to child spawns, and is + // queryable via TaskFilter.correlation_id. The barrier/group id a join matches. + string correlation_id = 17; + + // Optional top-of-fan-out-tree identity (the flow / run id). Defaults to the + // task's own id when it is a root; inherited by descendants on spawn. Becomes + // the DAG run id when the join engine grows into full fan-out/fan-in. + string root_task_id = 18; + + // Optional "feed B" config: emit a domain event onto event::* when this task + // reaches a (selected) terminal status. Absent/disabled = no emission. + TaskCompletionEvent completion_event = 19; } // CreateTaskResponse is sent in response to CreateTaskRequest when the @@ -1050,6 +1219,18 @@ message TaskFilter { // not just direct children. Default false: direct children only, preserving // the existing parent_task_id behavior. bool include_descendants = 20; + + // Exact dispatch-priority filter. UNSPECIFIED (0) = no filter; otherwise + // returns only tasks whose stored priority equals this level. + TaskPriority priority = 21; + // Minimum dispatch-priority threshold. UNSPECIFIED (0) = no filter; + // otherwise returns tasks whose priority is >= this level (e.g. pass HIGH + // to get everything HIGH and PREEMPT). Combinable with other filters. + TaskPriority min_priority = 22; + + // Filter by fan-out/fan-in correlation identity / flow id. Empty = no filter. + string correlation_id = 23; + string root_task_id = 24; } // TaskInfo represents a task. @@ -1105,6 +1286,20 @@ message TaskInfo { // Unix seconds when the task entered a WAITING_* / HIBERNATED state. // 0 means the task has never been paused. int64 paused_at = 30; + + // Dispatch priority. UNSPECIFIED is normalized to NORMAL on creation, so + // persisted tasks always report a concrete level. + TaskPriority priority = 31; + + // Fan-out/fan-in correlation identity (the barrier/group id a join matches), + // distinct from task_id and propagated to child spawns. Empty = none. + string correlation_id = 32; + // Top-of-fan-out-tree identity (flow / run id); defaults to the task's own id + // for a root, inherited by descendants. Empty = none. + string root_task_id = 33; + // "Feed B" config persisted on the task; read at the terminal transition to + // decide whether/what to emit onto event::*. + TaskCompletionEvent completion_event = 34; } // TaskQueryResponse is sent in response to TaskQuery operations. @@ -1148,6 +1343,7 @@ message TaskOperation { WAIT_FOR = 5; // Transition running -> WAITING_DEPENDENCY on specific task ids. Requires wait_spec. RESUME = 6; // Force-resume a paused task (admin/manual; normally task_waker fires). REJECT = 7; // Agent declines before processing -> REJECTED terminal state. + CLAIM = 8; // Assignee transitions an assigned/pending task to RUNNING. } OpType op = 1; @@ -1567,6 +1763,25 @@ message ACLOperation { // admin endpoints under /api/acl/authority-grants are unaffected. reserved 12, 13, 14, 15, 16; reserved "LIST_AUTHORITY_GRANTS", "GET_AUTHORITY_GRANT", "CREATE_AUTHORITY_GRANT", "RENEW_AUTHORITY_GRANT", "REVOKE_AUTHORITY_GRANT"; + // Role/group authorization. REST equivalents under /api/acl/groups, + // /api/acl/roles, and /api/acl/principals/{type}/{id}/{groups,roles}. + CREATE_GROUP = 17; // Create a group (group_request) + DELETE_GROUP = 18; // Delete a group (name) + GET_GROUP = 19; // Get a group (name) + LIST_GROUPS = 20; // List all groups + ADD_GROUP_MEMBER = 21; // Add a member to a group (name + member_request) + REMOVE_GROUP_MEMBER = 22; // Remove a member from a group (name + principal) + LIST_GROUP_MEMBERS = 23; // List members of a group (name) + CREATE_ROLE = 24; // Create a role (role_request) + DELETE_ROLE = 25; // Delete a role (name) + GET_ROLE = 26; // Get a role (name) + LIST_ROLES = 27; // List all roles + ASSIGN_ROLE = 28; // Assign a role (name + assignment_request) + UNASSIGN_ROLE = 29; // Unassign a role (name + principal) + LIST_ROLE_ASSIGNMENTS = 30; // List assignees of a role (name) + LIST_PRINCIPAL_GROUPS = 31; // List groups a principal belongs to (principal) + LIST_PRINCIPAL_ROLES = 32; // List roles assigned to a principal (principal) + EXPLAIN_ACCESS = 33; // Explain effective access (principal + resource_type + resource_id [+ required_level]) } OpType op = 1; @@ -1605,6 +1820,29 @@ message ACLOperation { // Client-generated correlation ID for matching responses to requests string request_id = 19; + + // Role/group fields. + // name: group/role name for GET/DELETE/LIST_*_MEMBERS/ASSIGNMENTS/ADD/ASSIGN. + string name = 20; + // principal: member/assignee for REMOVE_GROUP_MEMBER, UNASSIGN_ROLE, and + // the subject for LIST_PRINCIPAL_GROUPS / LIST_PRINCIPAL_ROLES. + PrincipalRef principal = 21; + ACLGroupRequest group_request = 22; // CREATE_GROUP + ACLRoleRequest role_request = 23; // CREATE_ROLE + ACLGroupMemberRequest member_request = 24; // ADD_GROUP_MEMBER + ACLRoleAssignmentRequest assignment_request = 25; // ASSIGN_ROLE + + // EXPLAIN_ACCESS: the resource to explain against (principal carries the + // subject; required_level is the threshold the decision is compared to). + string resource_type = 26; + string resource_id = 27; + int32 required_level = 28; + + // Optional on-behalf-of authority context. When set, the gateway runs the + // admin ACL check against the subject (the user) rather than the actor + // (the platform-server). Mirrors SessionOperation.authorization (field 6) + // and AuditQuery.authorization (field 18). + AuthorizationContext authorization = 29; } // ACLRuleFilter specifies filtering parameters for listing ACL rules. @@ -1789,6 +2027,102 @@ message ACLCleanupResult { string message = 2; // Human-readable result message } +// ACLGroupRequest contains data for creating a group. +message ACLGroupRequest { + string name = 1; + string description = 2; + string created_by = 3; + map metadata = 4; +} + +// ACLRoleRequest contains data for creating a role. +message ACLRoleRequest { + string name = 1; + string description = 2; + string created_by = 3; + map metadata = 4; +} + +// ACLGroupMemberRequest contains data for adding a member to a group. +message ACLGroupMemberRequest { + string member_type = 1; // principal type or "group" (nesting) + string member_id = 2; + string granted_by = 3; + int64 expires_at = 4; // Unix timestamp, 0 = no expiration +} + +// ACLRoleAssignmentRequest contains data for assigning a role. +message ACLRoleAssignmentRequest { + string assignee_type = 1; // principal type or "group" + string assignee_id = 2; + string granted_by = 3; + int64 expires_at = 4; // Unix timestamp, 0 = no expiration +} + +// ACLGroupInfo represents a group definition. +message ACLGroupInfo { + string group_id = 1; + string group_name = 2; + string description = 3; + string created_by = 4; + int64 created_at = 5; + map metadata = 6; +} + +// ACLRoleInfo represents a role definition. +message ACLRoleInfo { + string role_id = 1; + string role_name = 2; + string description = 3; + string created_by = 4; + int64 created_at = 5; + map metadata = 6; +} + +// ACLGroupMemberInfo represents a single group-membership edge. +message ACLGroupMemberInfo { + string group_name = 1; + string member_type = 2; + string member_id = 3; + string granted_by = 4; + int64 granted_at = 5; + int64 expires_at = 6; +} + +// ACLRoleAssignmentInfo represents a single role-assignment edge. +message ACLRoleAssignmentInfo { + string role_name = 1; + string assignee_type = 2; + string assignee_id = 3; + string granted_by = 4; + int64 granted_at = 5; + int64 expires_at = 6; +} + +// ACLAccessContributionInfo is one rule that matched a principal or one of its +// groups/roles for an explained resource. +message ACLAccessContributionInfo { + string subject = 1; // granting subject, e.g. "role:wsadmin" + string rule_id = 2; + int32 access_level = 3; + string resource = 4; // matched object pattern, e.g. "workspace:prod" or "workspace:*" + bool expired = 5; +} + +// ACLAccessExplanationInfo explains how a principal's effective access to a +// resource is decided (EXPLAIN_ACCESS). The gateway records an "explain_access" +// audit event attributing the call to the connected principal. +message ACLAccessExplanationInfo { + string principal = 1; // self subject "type:id" + repeated string subjects = 2; // self + transitive groups/roles + repeated ACLAccessContributionInfo contributions = 3; + bool allowed = 4; + string decision = 5; // "ALLOW" / "DENY" + int32 effective_access_level = 6; + bool fallback_applied = 7; + string reason = 8; +} + // ACLResponse is sent in response to ACLOperation. // Contains data based on the operation type that was requested. message ACLResponse { @@ -1836,6 +2170,15 @@ message ACLResponse { // Echoed from the originating ACLOperation for correlation string request_id = 12; + + // Role/group results. + ACLGroupInfo group = 16; // GET_GROUP / CREATE_GROUP + repeated ACLGroupInfo groups = 17; // LIST_GROUPS + ACLRoleInfo role = 18; // GET_ROLE / CREATE_ROLE + repeated ACLRoleInfo roles = 19; // LIST_ROLES + repeated ACLGroupMemberInfo group_members = 20; // LIST_GROUP_MEMBERS / LIST_PRINCIPAL_GROUPS + repeated ACLRoleAssignmentInfo role_assignments = 21; // LIST_ROLE_ASSIGNMENTS / LIST_PRINCIPAL_ROLES + ACLAccessExplanationInfo explanation = 22; // EXPLAIN_ACCESS } // ============================================================================ @@ -2475,6 +2818,10 @@ message WorkflowOperation { SEND_SM_EVENT = 22; // Schedule upsert (idempotent create-or-update) UPSERT_SCHEDULE = 23; + // Fan-in / barrier / coalesce join observability + operator control. + LIST_JOINS = 24; // in-flight + recently-terminal join instances (workspace filter) + GET_JOIN = 25; // inspect one instance: id=join_name, secondary_id=correlation_key + CANCEL_JOIN = 26; // operator GC of a wedged instance } OpType op = 1; string id = 2; // Entity ID for GET/UPDATE/DELETE @@ -2534,6 +2881,21 @@ message MessageEnvelope { // metrics bridges (which subscribe to a workspace-agnostic fan-in shard) // can recover the originating workspace. string workspace = 6; + + // Gateway-set, spoof-proof resolved on-behalf-of subject. Populated ONLY + // when the sender's SendMessage carried an AuthorizationContext that the + // gateway resolved to an OBO subject (valid grant, delegate + audience + // match) — set from the resolved authority in routeMessage, the same way + // `source` is set from the authenticated sender and ProxyHttp mints X-Auth-* + // identity headers. Empty for direct (non-OBO) sends and older gateways. + // + // Lets a recipient identify the *user* a message was sent for, distinct from + // the sending *identity* in `source`. Example: the agent-harness sends from + // its agent identity but acts for a user; platform-bridge reads this to scope + // the tool registry to that user. Subject identity only — recipients that + // need to *act for* the subject use the task authority-grant path + // (CreateTaskResponse.authority_grant_id), not this field. + PrincipalRef on_behalf_subject = 7; } // ========================================================================= diff --git a/api/proto/sandbox_relay_tunnel.pb.go b/api/proto/sandbox_relay_tunnel.pb.go new file mode 100644 index 0000000..ad02d84 --- /dev/null +++ b/api/proto/sandbox_relay_tunnel.pb.go @@ -0,0 +1,364 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.11 +// protoc v3.21.12 +// source: sandbox_relay_tunnel.proto + +package aether + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" + unsafe "unsafe" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +// TunnelFrame wraps either direction of a relayed gateway session, or the +// relay's opening hello. Provider->relay carries UpstreamMessage (up); +// relay->provider carries DownstreamMessage (down). +type TunnelFrame struct { + state protoimpl.MessageState `protogen:"open.v1"` + // Types that are valid to be assigned to F: + // + // *TunnelFrame_Up + // *TunnelFrame_Down + // *TunnelFrame_Hello + F isTunnelFrame_F `protobuf_oneof:"f"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *TunnelFrame) Reset() { + *x = TunnelFrame{} + mi := &file_sandbox_relay_tunnel_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *TunnelFrame) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*TunnelFrame) ProtoMessage() {} + +func (x *TunnelFrame) ProtoReflect() protoreflect.Message { + mi := &file_sandbox_relay_tunnel_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use TunnelFrame.ProtoReflect.Descriptor instead. +func (*TunnelFrame) Descriptor() ([]byte, []int) { + return file_sandbox_relay_tunnel_proto_rawDescGZIP(), []int{0} +} + +func (x *TunnelFrame) GetF() isTunnelFrame_F { + if x != nil { + return x.F + } + return nil +} + +func (x *TunnelFrame) GetUp() *UpstreamMessage { + if x != nil { + if x, ok := x.F.(*TunnelFrame_Up); ok { + return x.Up + } + } + return nil +} + +func (x *TunnelFrame) GetDown() *DownstreamMessage { + if x != nil { + if x, ok := x.F.(*TunnelFrame_Down); ok { + return x.Down + } + } + return nil +} + +func (x *TunnelFrame) GetHello() *TunnelHello { + if x != nil { + if x, ok := x.F.(*TunnelFrame_Hello); ok { + return x.Hello + } + } + return nil +} + +type isTunnelFrame_F interface { + isTunnelFrame_F() +} + +type TunnelFrame_Up struct { + Up *UpstreamMessage `protobuf:"bytes,1,opt,name=up,proto3,oneof"` +} + +type TunnelFrame_Down struct { + Down *DownstreamMessage `protobuf:"bytes,2,opt,name=down,proto3,oneof"` +} + +type TunnelFrame_Hello struct { + Hello *TunnelHello `protobuf:"bytes,3,opt,name=hello,proto3,oneof"` +} + +func (*TunnelFrame_Up) isTunnelFrame_F() {} + +func (*TunnelFrame_Down) isTunnelFrame_F() {} + +func (*TunnelFrame_Hello) isTunnelFrame_F() {} + +// TunnelHello is the relay's opening frame, announcing the tenant it serves. +// The tenant string is a hint that MUST be validated against the relay's +// peer-certificate CN before any pairing. +type TunnelHello struct { + state protoimpl.MessageState `protogen:"open.v1"` + Tenant string `protobuf:"bytes,1,opt,name=tenant,proto3" json:"tenant,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *TunnelHello) Reset() { + *x = TunnelHello{} + mi := &file_sandbox_relay_tunnel_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *TunnelHello) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*TunnelHello) ProtoMessage() {} + +func (x *TunnelHello) ProtoReflect() protoreflect.Message { + mi := &file_sandbox_relay_tunnel_proto_msgTypes[1] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use TunnelHello.ProtoReflect.Descriptor instead. +func (*TunnelHello) Descriptor() ([]byte, []int) { + return file_sandbox_relay_tunnel_proto_rawDescGZIP(), []int{1} +} + +func (x *TunnelHello) GetTenant() string { + if x != nil { + return x.Tenant + } + return "" +} + +// WatchTenantsRequest carries no parameters; the aggregator streams every +// tenant transition for the lifetime of the call. +type WatchTenantsRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *WatchTenantsRequest) Reset() { + *x = WatchTenantsRequest{} + mi := &file_sandbox_relay_tunnel_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *WatchTenantsRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*WatchTenantsRequest) ProtoMessage() {} + +func (x *WatchTenantsRequest) ProtoReflect() protoreflect.Message { + mi := &file_sandbox_relay_tunnel_proto_msgTypes[2] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use WatchTenantsRequest.ProtoReflect.Descriptor instead. +func (*WatchTenantsRequest) Descriptor() ([]byte, []int) { + return file_sandbox_relay_tunnel_proto_rawDescGZIP(), []int{2} +} + +// TenantEvent reports a tenant's relay coming online (online=true) or going +// offline (online=false). +// +// snapshot_complete is a terminal sentinel emitted exactly once per +// (re)subscribe, after the burst of currently-online tenants is replayed and +// before any live transitions stream. The provider uses it to deterministically +// prune tenants that left during a watch disconnect (they are absent from the +// replay set). Live events always carry snapshot_complete=false (the zero +// value); the sentinel carries an empty tenant and online is irrelevant. +type TenantEvent struct { + state protoimpl.MessageState `protogen:"open.v1"` + Tenant string `protobuf:"bytes,1,opt,name=tenant,proto3" json:"tenant,omitempty"` + Online bool `protobuf:"varint,2,opt,name=online,proto3" json:"online,omitempty"` + SnapshotComplete bool `protobuf:"varint,3,opt,name=snapshot_complete,json=snapshotComplete,proto3" json:"snapshot_complete,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *TenantEvent) Reset() { + *x = TenantEvent{} + mi := &file_sandbox_relay_tunnel_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *TenantEvent) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*TenantEvent) ProtoMessage() {} + +func (x *TenantEvent) ProtoReflect() protoreflect.Message { + mi := &file_sandbox_relay_tunnel_proto_msgTypes[3] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use TenantEvent.ProtoReflect.Descriptor instead. +func (*TenantEvent) Descriptor() ([]byte, []int) { + return file_sandbox_relay_tunnel_proto_rawDescGZIP(), []int{3} +} + +func (x *TenantEvent) GetTenant() string { + if x != nil { + return x.Tenant + } + return "" +} + +func (x *TenantEvent) GetOnline() bool { + if x != nil { + return x.Online + } + return false +} + +func (x *TenantEvent) GetSnapshotComplete() bool { + if x != nil { + return x.SnapshotComplete + } + return false +} + +var File_sandbox_relay_tunnel_proto protoreflect.FileDescriptor + +const file_sandbox_relay_tunnel_proto_rawDesc = "" + + "\n" + + "\x1asandbox_relay_tunnel.proto\x12\taether.v1\x1a\faether.proto\"\xa4\x01\n" + + "\vTunnelFrame\x12,\n" + + "\x02up\x18\x01 \x01(\v2\x1a.aether.v1.UpstreamMessageH\x00R\x02up\x122\n" + + "\x04down\x18\x02 \x01(\v2\x1c.aether.v1.DownstreamMessageH\x00R\x04down\x12.\n" + + "\x05hello\x18\x03 \x01(\v2\x16.aether.v1.TunnelHelloH\x00R\x05helloB\x03\n" + + "\x01f\"%\n" + + "\vTunnelHello\x12\x16\n" + + "\x06tenant\x18\x01 \x01(\tR\x06tenant\"\x15\n" + + "\x13WatchTenantsRequest\"j\n" + + "\vTenantEvent\x12\x16\n" + + "\x06tenant\x18\x01 \x01(\tR\x06tenant\x12\x16\n" + + "\x06online\x18\x02 \x01(\bR\x06online\x12+\n" + + "\x11snapshot_complete\x18\x03 \x01(\bR\x10snapshotComplete2\x9c\x01\n" + + "\x12SandboxRelayTunnel\x12<\n" + + "\x06Tunnel\x12\x16.aether.v1.TunnelFrame\x1a\x16.aether.v1.TunnelFrame(\x010\x01\x12H\n" + + "\fWatchTenants\x12\x1e.aether.v1.WatchTenantsRequest\x1a\x16.aether.v1.TenantEvent0\x01B-Z+github.com/scitrera/aether/api/proto;aetherb\x06proto3" + +var ( + file_sandbox_relay_tunnel_proto_rawDescOnce sync.Once + file_sandbox_relay_tunnel_proto_rawDescData []byte +) + +func file_sandbox_relay_tunnel_proto_rawDescGZIP() []byte { + file_sandbox_relay_tunnel_proto_rawDescOnce.Do(func() { + file_sandbox_relay_tunnel_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_sandbox_relay_tunnel_proto_rawDesc), len(file_sandbox_relay_tunnel_proto_rawDesc))) + }) + return file_sandbox_relay_tunnel_proto_rawDescData +} + +var file_sandbox_relay_tunnel_proto_msgTypes = make([]protoimpl.MessageInfo, 4) +var file_sandbox_relay_tunnel_proto_goTypes = []any{ + (*TunnelFrame)(nil), // 0: aether.v1.TunnelFrame + (*TunnelHello)(nil), // 1: aether.v1.TunnelHello + (*WatchTenantsRequest)(nil), // 2: aether.v1.WatchTenantsRequest + (*TenantEvent)(nil), // 3: aether.v1.TenantEvent + (*UpstreamMessage)(nil), // 4: aether.v1.UpstreamMessage + (*DownstreamMessage)(nil), // 5: aether.v1.DownstreamMessage +} +var file_sandbox_relay_tunnel_proto_depIdxs = []int32{ + 4, // 0: aether.v1.TunnelFrame.up:type_name -> aether.v1.UpstreamMessage + 5, // 1: aether.v1.TunnelFrame.down:type_name -> aether.v1.DownstreamMessage + 1, // 2: aether.v1.TunnelFrame.hello:type_name -> aether.v1.TunnelHello + 0, // 3: aether.v1.SandboxRelayTunnel.Tunnel:input_type -> aether.v1.TunnelFrame + 2, // 4: aether.v1.SandboxRelayTunnel.WatchTenants:input_type -> aether.v1.WatchTenantsRequest + 0, // 5: aether.v1.SandboxRelayTunnel.Tunnel:output_type -> aether.v1.TunnelFrame + 3, // 6: aether.v1.SandboxRelayTunnel.WatchTenants:output_type -> aether.v1.TenantEvent + 5, // [5:7] is the sub-list for method output_type + 3, // [3:5] is the sub-list for method input_type + 3, // [3:3] is the sub-list for extension type_name + 3, // [3:3] is the sub-list for extension extendee + 0, // [0:3] is the sub-list for field type_name +} + +func init() { file_sandbox_relay_tunnel_proto_init() } +func file_sandbox_relay_tunnel_proto_init() { + if File_sandbox_relay_tunnel_proto != nil { + return + } + file_aether_proto_init() + file_sandbox_relay_tunnel_proto_msgTypes[0].OneofWrappers = []any{ + (*TunnelFrame_Up)(nil), + (*TunnelFrame_Down)(nil), + (*TunnelFrame_Hello)(nil), + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_sandbox_relay_tunnel_proto_rawDesc), len(file_sandbox_relay_tunnel_proto_rawDesc)), + NumEnums: 0, + NumMessages: 4, + NumExtensions: 0, + NumServices: 1, + }, + GoTypes: file_sandbox_relay_tunnel_proto_goTypes, + DependencyIndexes: file_sandbox_relay_tunnel_proto_depIdxs, + MessageInfos: file_sandbox_relay_tunnel_proto_msgTypes, + }.Build() + File_sandbox_relay_tunnel_proto = out.File + file_sandbox_relay_tunnel_proto_goTypes = nil + file_sandbox_relay_tunnel_proto_depIdxs = nil +} diff --git a/api/proto/sandbox_relay_tunnel.proto b/api/proto/sandbox_relay_tunnel.proto new file mode 100644 index 0000000..0c9ef58 --- /dev/null +++ b/api/proto/sandbox_relay_tunnel.proto @@ -0,0 +1,57 @@ +syntax = "proto3"; + +package aether.v1; + +option go_package = "github.com/scitrera/aether/api/proto;aether"; + +import "aether.proto"; + +// SandboxRelayTunnel is the aggregator's relay-facing surface. A tenant-relay +// sidecar dials Tunnel() and announces its tenant via TunnelHello; the +// aggregator pairs that relay with a provider's AetherGateway.Connect stream +// for the same tenant and splices the two 1:1 (NOT a mux — each pair owns its +// own gateway session, lock, and session id). WatchTenants lets a provider +// learn which tenants currently have an online relay so it can dial in. +service SandboxRelayTunnel { + // Tunnel carries one provider<->gateway session, framed in both directions. + // The relay's first frame MUST be a TunnelHello announcing its tenant. + rpc Tunnel(stream TunnelFrame) returns (stream TunnelFrame); + + // WatchTenants streams tenant online/offline transitions to a provider. + rpc WatchTenants(WatchTenantsRequest) returns (stream TenantEvent); +} + +// TunnelFrame wraps either direction of a relayed gateway session, or the +// relay's opening hello. Provider->relay carries UpstreamMessage (up); +// relay->provider carries DownstreamMessage (down). +message TunnelFrame { + oneof f { + UpstreamMessage up = 1; + DownstreamMessage down = 2; + TunnelHello hello = 3; + } +} + +// TunnelHello is the relay's opening frame, announcing the tenant it serves. +// The tenant string is a hint that MUST be validated against the relay's +// peer-certificate CN before any pairing. +message TunnelHello { string tenant = 1; } + +// WatchTenantsRequest carries no parameters; the aggregator streams every +// tenant transition for the lifetime of the call. +message WatchTenantsRequest {} + +// TenantEvent reports a tenant's relay coming online (online=true) or going +// offline (online=false). +// +// snapshot_complete is a terminal sentinel emitted exactly once per +// (re)subscribe, after the burst of currently-online tenants is replayed and +// before any live transitions stream. The provider uses it to deterministically +// prune tenants that left during a watch disconnect (they are absent from the +// replay set). Live events always carry snapshot_complete=false (the zero +// value); the sentinel carries an empty tenant and online is irrelevant. +message TenantEvent { + string tenant = 1; + bool online = 2; + bool snapshot_complete = 3; +} diff --git a/api/proto/sandbox_relay_tunnel_grpc.pb.go b/api/proto/sandbox_relay_tunnel_grpc.pb.go new file mode 100644 index 0000000..35b290b --- /dev/null +++ b/api/proto/sandbox_relay_tunnel_grpc.pb.go @@ -0,0 +1,176 @@ +// Code generated by protoc-gen-go-grpc. DO NOT EDIT. +// versions: +// - protoc-gen-go-grpc v1.6.2 +// - protoc v3.21.12 +// source: sandbox_relay_tunnel.proto + +package aether + +import ( + context "context" + grpc "google.golang.org/grpc" + codes "google.golang.org/grpc/codes" + status "google.golang.org/grpc/status" +) + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the grpc package it is being compiled against. +// Requires gRPC-Go v1.64.0 or later. +const _ = grpc.SupportPackageIsVersion9 + +const ( + SandboxRelayTunnel_Tunnel_FullMethodName = "/aether.v1.SandboxRelayTunnel/Tunnel" + SandboxRelayTunnel_WatchTenants_FullMethodName = "/aether.v1.SandboxRelayTunnel/WatchTenants" +) + +// SandboxRelayTunnelClient is the client API for SandboxRelayTunnel service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. +// +// SandboxRelayTunnel is the aggregator's relay-facing surface. A tenant-relay +// sidecar dials Tunnel() and announces its tenant via TunnelHello; the +// aggregator pairs that relay with a provider's AetherGateway.Connect stream +// for the same tenant and splices the two 1:1 (NOT a mux — each pair owns its +// own gateway session, lock, and session id). WatchTenants lets a provider +// learn which tenants currently have an online relay so it can dial in. +type SandboxRelayTunnelClient interface { + // Tunnel carries one provider<->gateway session, framed in both directions. + // The relay's first frame MUST be a TunnelHello announcing its tenant. + Tunnel(ctx context.Context, opts ...grpc.CallOption) (grpc.BidiStreamingClient[TunnelFrame, TunnelFrame], error) + // WatchTenants streams tenant online/offline transitions to a provider. + WatchTenants(ctx context.Context, in *WatchTenantsRequest, opts ...grpc.CallOption) (grpc.ServerStreamingClient[TenantEvent], error) +} + +type sandboxRelayTunnelClient struct { + cc grpc.ClientConnInterface +} + +func NewSandboxRelayTunnelClient(cc grpc.ClientConnInterface) SandboxRelayTunnelClient { + return &sandboxRelayTunnelClient{cc} +} + +func (c *sandboxRelayTunnelClient) Tunnel(ctx context.Context, opts ...grpc.CallOption) (grpc.BidiStreamingClient[TunnelFrame, TunnelFrame], error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + stream, err := c.cc.NewStream(ctx, &SandboxRelayTunnel_ServiceDesc.Streams[0], SandboxRelayTunnel_Tunnel_FullMethodName, cOpts...) + if err != nil { + return nil, err + } + x := &grpc.GenericClientStream[TunnelFrame, TunnelFrame]{ClientStream: stream} + return x, nil +} + +// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. +type SandboxRelayTunnel_TunnelClient = grpc.BidiStreamingClient[TunnelFrame, TunnelFrame] + +func (c *sandboxRelayTunnelClient) WatchTenants(ctx context.Context, in *WatchTenantsRequest, opts ...grpc.CallOption) (grpc.ServerStreamingClient[TenantEvent], error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + stream, err := c.cc.NewStream(ctx, &SandboxRelayTunnel_ServiceDesc.Streams[1], SandboxRelayTunnel_WatchTenants_FullMethodName, cOpts...) + if err != nil { + return nil, err + } + x := &grpc.GenericClientStream[WatchTenantsRequest, TenantEvent]{ClientStream: stream} + if err := x.ClientStream.SendMsg(in); err != nil { + return nil, err + } + if err := x.ClientStream.CloseSend(); err != nil { + return nil, err + } + return x, nil +} + +// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. +type SandboxRelayTunnel_WatchTenantsClient = grpc.ServerStreamingClient[TenantEvent] + +// SandboxRelayTunnelServer is the server API for SandboxRelayTunnel service. +// All implementations must embed UnimplementedSandboxRelayTunnelServer +// for forward compatibility. +// +// SandboxRelayTunnel is the aggregator's relay-facing surface. A tenant-relay +// sidecar dials Tunnel() and announces its tenant via TunnelHello; the +// aggregator pairs that relay with a provider's AetherGateway.Connect stream +// for the same tenant and splices the two 1:1 (NOT a mux — each pair owns its +// own gateway session, lock, and session id). WatchTenants lets a provider +// learn which tenants currently have an online relay so it can dial in. +type SandboxRelayTunnelServer interface { + // Tunnel carries one provider<->gateway session, framed in both directions. + // The relay's first frame MUST be a TunnelHello announcing its tenant. + Tunnel(grpc.BidiStreamingServer[TunnelFrame, TunnelFrame]) error + // WatchTenants streams tenant online/offline transitions to a provider. + WatchTenants(*WatchTenantsRequest, grpc.ServerStreamingServer[TenantEvent]) error + mustEmbedUnimplementedSandboxRelayTunnelServer() +} + +// UnimplementedSandboxRelayTunnelServer must be embedded to have +// forward compatible implementations. +// +// NOTE: this should be embedded by value instead of pointer to avoid a nil +// pointer dereference when methods are called. +type UnimplementedSandboxRelayTunnelServer struct{} + +func (UnimplementedSandboxRelayTunnelServer) Tunnel(grpc.BidiStreamingServer[TunnelFrame, TunnelFrame]) error { + return status.Error(codes.Unimplemented, "method Tunnel not implemented") +} +func (UnimplementedSandboxRelayTunnelServer) WatchTenants(*WatchTenantsRequest, grpc.ServerStreamingServer[TenantEvent]) error { + return status.Error(codes.Unimplemented, "method WatchTenants not implemented") +} +func (UnimplementedSandboxRelayTunnelServer) mustEmbedUnimplementedSandboxRelayTunnelServer() {} +func (UnimplementedSandboxRelayTunnelServer) testEmbeddedByValue() {} + +// UnsafeSandboxRelayTunnelServer may be embedded to opt out of forward compatibility for this service. +// Use of this interface is not recommended, as added methods to SandboxRelayTunnelServer will +// result in compilation errors. +type UnsafeSandboxRelayTunnelServer interface { + mustEmbedUnimplementedSandboxRelayTunnelServer() +} + +func RegisterSandboxRelayTunnelServer(s grpc.ServiceRegistrar, srv SandboxRelayTunnelServer) { + // If the following call panics, it indicates UnimplementedSandboxRelayTunnelServer was + // embedded by pointer and is nil. This will cause panics if an + // unimplemented method is ever invoked, so we test this at initialization + // time to prevent it from happening at runtime later due to I/O. + if t, ok := srv.(interface{ testEmbeddedByValue() }); ok { + t.testEmbeddedByValue() + } + s.RegisterService(&SandboxRelayTunnel_ServiceDesc, srv) +} + +func _SandboxRelayTunnel_Tunnel_Handler(srv interface{}, stream grpc.ServerStream) error { + return srv.(SandboxRelayTunnelServer).Tunnel(&grpc.GenericServerStream[TunnelFrame, TunnelFrame]{ServerStream: stream}) +} + +// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. +type SandboxRelayTunnel_TunnelServer = grpc.BidiStreamingServer[TunnelFrame, TunnelFrame] + +func _SandboxRelayTunnel_WatchTenants_Handler(srv interface{}, stream grpc.ServerStream) error { + m := new(WatchTenantsRequest) + if err := stream.RecvMsg(m); err != nil { + return err + } + return srv.(SandboxRelayTunnelServer).WatchTenants(m, &grpc.GenericServerStream[WatchTenantsRequest, TenantEvent]{ServerStream: stream}) +} + +// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. +type SandboxRelayTunnel_WatchTenantsServer = grpc.ServerStreamingServer[TenantEvent] + +// SandboxRelayTunnel_ServiceDesc is the grpc.ServiceDesc for SandboxRelayTunnel service. +// It's only intended for direct use with grpc.RegisterService, +// and not to be introspected or modified (even as a copy) +var SandboxRelayTunnel_ServiceDesc = grpc.ServiceDesc{ + ServiceName: "aether.v1.SandboxRelayTunnel", + HandlerType: (*SandboxRelayTunnelServer)(nil), + Methods: []grpc.MethodDesc{}, + Streams: []grpc.StreamDesc{ + { + StreamName: "Tunnel", + Handler: _SandboxRelayTunnel_Tunnel_Handler, + ServerStreams: true, + ClientStreams: true, + }, + { + StreamName: "WatchTenants", + Handler: _SandboxRelayTunnel_WatchTenants_Handler, + ServerStreams: true, + }, + }, + Metadata: "sandbox_relay_tunnel.proto", +} diff --git a/docs/admin-ui.md b/docs/admin-ui.md index 75bcf61..f40e094 100644 --- a/docs/admin-ui.md +++ b/docs/admin-ui.md @@ -12,7 +12,7 @@ http://localhost:31880 The UI is a single-page application embedded in the gateway binary. It loads from the embedded filesystem automatically; no separate deployment step is required. -The admin API is served under the `/api` path prefix. All API endpoints return JSON. +The admin API is served under the `/api/v1` path prefix (versioned). Two additional endpoints, `/health` and `/info`, are registered unversioned directly on the server root and require no authentication. All API endpoints return JSON. ### Default Port @@ -24,10 +24,10 @@ admin: port: 31880 ``` -It can be overridden at startup with the `--admin-port` flag: +It can be overridden at startup with the `--admin-port` flag (or the `AETHER_ADMIN_PORT` environment variable). Note this is distinct from the ops/metrics port (default 9090, see below) — don't confuse the two: ```bash -./gateway --config configs/dev.yaml --admin-port 9090 +./gateway --config configs/dev.yaml --admin-port 8888 ``` ## Authentication @@ -43,7 +43,7 @@ admin: api_key: "your-secret-key" ``` -All requests to `/api/**` (except `/api/health`) must include: +All requests to `/api/v1/**` must include: ``` Authorization: Bearer your-secret-key @@ -56,9 +56,11 @@ If `api_key` is set but TLS is not enabled, the gateway logs a warning that the If no API key is configured, the gateway will refuse to start unless the insecure flag is explicitly set: ```bash -./gateway --dev --insecure-admin +AETHER_ALLOW_DEV_MODE=1 ./gateway --dev --insecure-admin ``` +**Note:** `--insecure-admin` additionally requires the `AETHER_ALLOW_DEV_MODE` environment variable to be set to any non-empty value; without it the gateway process exits immediately at startup with `--insecure-admin requires AETHER_ALLOW_DEV_MODE env var to be set; this flag is NOT for production use`, even if `--dev` is also passed. + This is equivalent to setting `InsecureNoAuth: true` in the server configuration. A warning is logged: ``` @@ -81,10 +83,10 @@ When both files are set, the admin server starts with TLS and the API key is tra ### WebSocket Authentication -The WebSocket endpoint at `/api/ws/events` is under the `/api` subrouter and inherits the API key middleware. Clients that cannot set the `Authorization` header in the WebSocket upgrade request (browser `WebSocket` API) may pass the token via the `Sec-WebSocket-Protocol` subprotocol: +The WebSocket endpoint at `/api/v1/ws/events` is under the `/api/v1` subrouter and inherits the API key middleware. Clients that cannot set the `Authorization` header in the WebSocket upgrade request (browser `WebSocket` API) may pass the token via the `Sec-WebSocket-Protocol` subprotocol: ```javascript -const ws = new WebSocket("ws://localhost:31880/api/ws/events", ["auth", "Bearer your-secret-key"]); +const ws = new WebSocket("ws://localhost:31880/api/v1/ws/events", ["auth", "Bearer your-secret-key"]); ``` The server echoes back the `auth` subprotocol in the upgrade response to confirm the method was accepted. @@ -112,77 +114,74 @@ admin: Setting `cors_origin: "*"` is permitted but disables WebSocket connections in production mode (to prevent cross-site WebSocket hijacking). Omitting `cors_origin` restricts the UI to same-origin requests. -## Health Probes +## Health Probes and Prometheus Metrics (Separate Ops Server) + +`/health/live`, `/health/ready`, `/health/startup`, and `GET /metrics` are **not** served by the admin server described above — they run on a dedicated ops server, on its own port, unauthenticated and not subject to rate limiting. This isolates health/metrics scraping from the admin API's auth and rate-limit policy. -The following endpoints are unauthenticated and not subject to rate limiting. They are intended for Kubernetes liveness/readiness/startup probes. +The ops port defaults to **9090** and is configured independently of the admin port via `gateway.ops_port` in the YAML config, the `--ops-port` flag, or the `AETHER_OPS_PORT` environment variable. See the [Monitoring Guide](monitoring.md) for full metric details. | Endpoint | Method | Description | |---|---|---| | `/health/live` | GET | Always returns `200 OK` while the process is running. Use as liveness probe. | -| `/health/ready` | GET | Returns `200` when Redis and RabbitMQ are reachable; `503` otherwise. Use as readiness probe. | -| `/health/startup` | GET | Returns `200` once routes are registered and the server is accepting connections; `503` before that. Use as startup probe. | +| `/health/ready` | GET | Returns `200` when dependencies (Redis, RabbitMQ) are reachable; `503` otherwise. Use as readiness probe. | +| `/health/startup` | GET | Returns `200` once the ops server has been marked ready by the gateway; `503` before that. Use as startup probe. | +| `/metrics` | GET | Prometheus scrape endpoint, unauthenticated by convention. | -## Prometheus Metrics +In production, network-isolate the ops port so that only the Prometheus scraper and orchestrator (Kubernetes) can reach it. Do not expose it through the public load balancer. -The Prometheus scrape endpoint is unauthenticated by convention: - -``` -GET /metrics -``` - -In production, isolate the admin port so that only the Prometheus scraper can reach it. Do not expose it through the public load balancer. +Note that the admin server also exposes an unversioned, unauthenticated `GET /health` (dependency health as JSON, not the K8s-probe format above) and `GET /info` on the admin port — see [Health and Info](#health-and-info) below. ## REST API Reference -All endpoints below are prefixed with `/api` and require the `Authorization: Bearer ` header unless otherwise noted. +All endpoints below are prefixed with `/api/v1` and require the `Authorization: Bearer ` header unless otherwise noted. ### Health and Info | Method | Path | Description | |---|---|---| -| GET | `/api/health` | Gateway health status including dependency checks. Exempt from API key auth. | -| GET | `/api/info` | Gateway identity, version, and build information. | -| GET | `/api/stats` | Active connection counts and message routing statistics. | +| GET | `/health` | Gateway health status including dependency checks. Unversioned (no `/api/v1` prefix) and exempt from API key auth. | +| GET | `/info` | Gateway identity, version, and build information. Unversioned (no `/api/v1` prefix) and exempt from API key auth. | +| GET | `/api/v1/stats` | Active connection counts and message routing statistics. | ### Connections | Method | Path | Description | |---|---|---| -| GET | `/api/connections` | List all active connections. Filter by `?type=agent` or `?workspace=my-workspace`. | -| GET | `/api/connections/{session_id}` | Get details for a specific session. | -| DELETE | `/api/connections/{session_id}` | Force-disconnect a session. Releases the distributed lock and terminates the gRPC stream. | +| GET | `/api/v1/connections` | List all active connections. Filter by `?type=agent` or `?workspace=my-workspace`. | +| GET | `/api/v1/connections/{session_id}` | Get details for a specific session. | +| DELETE | `/api/v1/connections/{session_id}` | Force-disconnect a session. Releases the distributed lock and terminates the gRPC stream. | ### Tasks | Method | Path | Description | |---|---|---| -| GET | `/api/tasks` | List tasks. Filter by `?status=pending`, `?workspace=ws`, `?type=unique`. | -| GET | `/api/tasks/{task_id}` | Get a specific task record. | -| POST | `/api/tasks/{task_id}/retry` | Schedule a failed task for retry. | -| POST | `/api/tasks/{task_id}/cancel` | Cancel a pending or running task. | +| GET | `/api/v1/tasks` | List tasks. Filter by `?status=pending`, `?workspace=ws`, `?type=unique`. | +| GET | `/api/v1/tasks/{task_id}` | Get a specific task record. | +| POST | `/api/v1/tasks/{task_id}/retry` | Schedule a failed task for retry. | +| POST | `/api/v1/tasks/{task_id}/cancel` | Cancel a pending or running task. | ### Workspaces | Method | Path | Body fields | Description | |---|---|---|---| -| GET | `/api/workspaces` | — | List all workspaces. | -| POST | `/api/workspaces` | `workspace_id`, `display_name`, `description`, `tenant_id`, `metadata` | Create a workspace. | -| GET | `/api/workspaces/{workspace_id}` | — | Get workspace details. | -| PUT | `/api/workspaces/{workspace_id}` | `display_name`, `description`, `tenant_id`, `metadata` | Update workspace metadata. | -| DELETE | `/api/workspaces/{workspace_id}` | — | Delete a workspace. | -| GET | `/api/workspaces/{workspace_id}/message-flow` | — | Get recent message flow graph for the workspace. | +| GET | `/api/v1/workspaces` | — | List all workspaces. | +| POST | `/api/v1/workspaces` | `workspace_id`, `display_name`, `description`, `tenant_id`, `metadata` | Create a workspace. | +| GET | `/api/v1/workspaces/{workspace_id}` | — | Get workspace details. | +| PUT | `/api/v1/workspaces/{workspace_id}` | `display_name`, `description`, `tenant_id`, `metadata` | Update workspace metadata. | +| DELETE | `/api/v1/workspaces/{workspace_id}` | — | Delete a workspace. | +| GET | `/api/v1/workspaces/{workspace_id}/message-flow` | — | Get recent message flow graph for the workspace. | ### Agents and Orchestration | Method | Path | Body fields | Description | |---|---|---|---| -| GET | `/api/agents` | — | List registered agent implementations. | -| POST | `/api/agents` | `implementation`, `description`, `launch_params` | Register an agent implementation. | -| GET | `/api/agents/{implementation}` | — | Get agent registration details. | -| PUT | `/api/agents/{implementation}` | `description`, `launch_params` | Update agent registration. | -| DELETE | `/api/agents/{implementation}` | — | Remove an agent registration. | -| POST | `/api/agents/{implementation}/launch` | `specifier`, `workspace` | Manually trigger an agent launch via the registered Orchestrator. Defaults: `specifier="default"`, `workspace="default"`. | -| GET | `/api/orchestrators` | — | List registered Orchestrator profiles. | +| GET | `/api/v1/agents` | — | List registered agent implementations. | +| POST | `/api/v1/agents` | `implementation`, `description`, `launch_params` | Register an agent implementation. | +| GET | `/api/v1/agents/{implementation}` | — | Get agent registration details. | +| PUT | `/api/v1/agents/{implementation}` | `description`, `launch_params` | Update agent registration. | +| DELETE | `/api/v1/agents/{implementation}` | — | Remove an agent registration. | +| POST | `/api/v1/agents/{implementation}/launch` | `specifier`, `workspace` | Manually trigger an agent launch via the registered Orchestrator. Defaults: `specifier="default"`, `workspace="default"`. | +| GET | `/api/v1/orchestrators` | — | List registered Orchestrator profiles. | ### KV Store @@ -190,10 +189,10 @@ The KV store is organized into scopes. Common scopes are `global` and workspace | Method | Path | Body fields | Description | |---|---|---|---| -| GET | `/api/kv` | — | List keys. Filter by `?scope=global&prefix=my/`. Defaults to `scope=global`. | -| GET | `/api/kv/{scope}/{key}` | — | Get a KV entry. The key supports slashes (e.g., `/api/kv/global/demo/setting`). | -| PUT | `/api/kv/{scope}/{key}` | `value`, `ttl` (seconds, optional) | Set a KV entry. A `ttl` of `0` means no expiration. | -| DELETE | `/api/kv/{scope}/{key}` | — | Delete a KV entry. | +| GET | `/api/v1/kv` | — | List keys. Filter by `?scope=global&prefix=my/`. Defaults to `scope=global`. | +| GET | `/api/v1/kv/{scope}/{key}` | — | Get a KV entry. The key supports slashes (e.g., `/api/v1/kv/global/demo/setting`). | +| PUT | `/api/v1/kv/{scope}/{key}` | `value`, `ttl` (seconds, optional) | Set a KV entry. A `ttl` of `0` means no expiration. | +| DELETE | `/api/v1/kv/{scope}/{key}` | — | Delete a KV entry. | ### ACL Management @@ -203,30 +202,74 @@ Access control rules govern which principals can perform which operations on whi | Method | Path | Description | |---|---|---| -| GET | `/api/acl/rules` | List ACL rules. Filter by `?principal_type=agent&principal_id=...&resource_type=...&resource_id=...`. | -| POST | `/api/acl/rules` | Grant access. Body: `principal_type`, `principal_id`, `resource_type`, `resource_id`, `granted_by`. All fields required. | -| GET | `/api/acl/rules/{rule_id}` | Get a specific rule. Query params: `principal_type`, `principal_id`, `resource_type`, `resource_id`. | -| DELETE | `/api/acl/rules/{rule_id}` | Revoke access. Query params: `principal_type`, `principal_id`, `resource_type`, `resource_id`. | +| GET | `/api/v1/acl/rules` | List ACL rules. Filter by `?principal_type=agent&principal_id=...&resource_type=...&resource_id=...`. | +| POST | `/api/v1/acl/rules` | Grant access. Body: `principal_type`, `principal_id`, `resource_type`, `resource_id`, `granted_by`. All fields required. | +| GET | `/api/v1/acl/rules/{rule_id}` | Get a specific rule. Query params: `principal_type`, `principal_id`, `resource_type`, `resource_id`. | +| DELETE | `/api/v1/acl/rules/{rule_id}` | Revoke access. Query params: `principal_type`, `principal_id`, `resource_type`, `resource_id`. | **Audit Log** | Method | Path | Description | |---|---|---| -| GET | `/api/acl/audit` | Query the ACL decision audit log. Filter by `principal_type`, `principal_id`, `resource_type`, `resource_id`, `decision`, `workspace`, `limit`. | +| GET | `/api/v1/acl/audit` | Query the ACL decision audit log. Filter by `principal_type`, `principal_id`, `resource_type`, `resource_id`, `decision`, `workspace`, `limit`. | **Fallback Policy** | Method | Path | Description | |---|---|---| -| GET | `/api/acl/fallback-policy` | Get the fallback policy for a rule category. Query param: `rule_category`. | -| PUT | `/api/acl/fallback-policy` | Set the fallback policy. Body: `rule_category`, `updated_by`. | +| GET | `/api/v1/acl/fallback-policy` | Get the fallback policy for a rule category. Query param: `rule_category`. | +| PUT | `/api/v1/acl/fallback-policy` | Set the fallback policy. Body: `rule_category`, `updated_by`. | **Maintenance** | Method | Path | Description | |---|---|---| -| POST | `/api/acl/cleanup/expired-rules` | Delete all expired ACL rules. Returns `count` of deleted rules. | -| POST | `/api/acl/cleanup/audit-logs` | Delete ACL audit log entries older than the retention window. Query param: `?retention_days=90` (default 90). Returns `count` of deleted entries. | +| POST | `/api/v1/acl/cleanup/expired-rules` | Delete all expired ACL rules. Returns `count` of deleted rules. | +| POST | `/api/v1/acl/cleanup/audit-logs` | Delete ACL audit log entries older than the retention window. Query param: `?retention_days=90` (default 90). Returns `count` of deleted entries. | + +**Authority Grants** + +Authority grants model delegated authority (subject → delegate, issued by a principal, optionally chained via `parent_grant_id`) with expiration and hop-count limits. + +| Method | Path | Body fields | Description | +|---|---|---|---| +| GET | `/api/v1/acl/authority-grants` | — | List grants. Filter by `root_grant_id`, `subject_type`, `subject_id`, `delegate_type`, `delegate_id`, `audience_type`, `audience_id`, `include_revoked`, `active_only`, `limit`, `offset`. | +| POST | `/api/v1/acl/authority-grants` | `subject`, `delegate`, `issued_by` (each `{principal_type, principal_id}`), `root_subject`, `parent_grant_id`, `may_delegate`, `remaining_hops`, `workspace_scope`, `expires_at` | Create a grant. `subject`, `delegate`, `issued_by`, and `expires_at` are required. | +| GET | `/api/v1/acl/authority-grants/{grant_id}` | — | Get a specific grant. | +| POST | `/api/v1/acl/authority-grants/{grant_id}/renew` | `expires_at` | Extend a grant's expiration. `expires_at` is required. | +| POST | `/api/v1/acl/authority-grants/{grant_id}/revoke` | — | Revoke a grant. | + +**Groups** + +| Method | Path | Body fields | Description | +|---|---|---|---| +| GET | `/api/v1/acl/groups` | — | List ACL groups. | +| POST | `/api/v1/acl/groups` | `name`, `description`, `created_by`, `metadata` | Create a group. `name` is required. | +| GET | `/api/v1/acl/groups/{name}` | — | Get a group. | +| DELETE | `/api/v1/acl/groups/{name}` | — | Delete a group. | +| GET | `/api/v1/acl/groups/{name}/members` | — | List group members. | +| POST | `/api/v1/acl/groups/{name}/members` | `member_type`, `member_id`, `granted_by`, `expires_at` | Add a member. `member_type` and `member_id` are required. | +| DELETE | `/api/v1/acl/groups/{name}/members` | — | Remove a member. | + +**Roles** + +| Method | Path | Body fields | Description | +|---|---|---|---| +| GET | `/api/v1/acl/roles` | — | List ACL roles. | +| POST | `/api/v1/acl/roles` | `name`, `description`, `created_by`, `metadata` | Create a role. `name` is required. | +| GET | `/api/v1/acl/roles/{name}` | — | Get a role. | +| DELETE | `/api/v1/acl/roles/{name}` | — | Delete a role. | +| GET | `/api/v1/acl/roles/{name}/assignments` | — | List role assignments. | +| POST | `/api/v1/acl/roles/{name}/assignments` | `assignee_type`, `assignee_id`, `granted_by`, `expires_at` | Assign the role. `assignee_type` and `assignee_id` are required. | +| DELETE | `/api/v1/acl/roles/{name}/assignments` | — | Unassign the role. | + +**Principals** + +| Method | Path | Description | +|---|---|---| +| GET | `/api/v1/acl/principals/{type}/{id}/groups` | List the groups a principal belongs to. | +| GET | `/api/v1/acl/principals/{type}/{id}/roles` | List the roles assigned to a principal. | +| GET | `/api/v1/acl/principals/{type}/{id}/effective` | Explain the principal's effective access (resolved rules, group/role membership). | ### API Token Management @@ -234,23 +277,34 @@ API tokens are used by clients to authenticate gRPC connections via the API key | Method | Path | Body fields | Description | |---|---|---|---| -| GET | `/api/tokens` | — | List all tokens (metadata only, no secret values). | -| POST | `/api/tokens` | `name`, `principal_type`, `workspace_patterns`, `scopes`, `expires_in_hours`, `created_by` | Create a token. Returns the plaintext token value once. Defaults: `workspace_patterns=["*"]`, `scopes=["connect"]`, `created_by="admin"`. | -| GET | `/api/tokens/{token_id}` | — | Get token metadata. | -| POST | `/api/tokens/{token_id}/revoke` | — | Revoke a token without deleting its record. | -| DELETE | `/api/tokens/{token_id}` | — | Permanently delete a token record. | +| GET | `/api/v1/tokens` | — | List all tokens (metadata only, no secret values). | +| POST | `/api/v1/tokens` | `name`, `principal_type`, `workspace_patterns`, `scopes`, `expires_in_hours`, `created_by` | Create a token. Returns the plaintext token value once. Defaults: `workspace_patterns=["*"]`, `scopes=["connect"]`, `created_by="admin"`. | +| GET | `/api/v1/tokens/{token_id}` | — | Get token metadata. | +| POST | `/api/v1/tokens/{token_id}/revoke` | — | Revoke a token without deleting its record. | +| DELETE | `/api/v1/tokens/{token_id}` | — | Permanently delete a token record. | ### Messaging | Method | Path | Body fields | Description | |---|---|---|---| -| POST | `/api/messages/send` | `target_topic`, `payload`, `message_type`, `source_topic` | Inject a message directly into the routing layer. `message_type` must be one of `CHAT`, `CONTROL`, `TOOL_CALL`, `EVENT`, `METRIC`, or empty (defaults to `CHAT`). | +| POST | `/api/v1/messages/send` | `target_topic`, `payload`, `message_type` | Inject a message directly into the routing layer. `message_type` must be one of `CHAT`, `CONTROL`, `TOOL_CALL`, `EVENT`, `METRIC`, or empty (defaults to `CHAT`). `target_topic` is required. Note: there is no `source_topic` body field — the request does not accept a spoofed source. | + +### Workspace Rate Limits + +Per-workspace message rate limits (messages/second), separate from the admin API's own IP-based rate limiting described above. + +| Method | Path | Body fields | Description | +|---|---|---|---| +| GET | `/api/v1/rate-limits` | — | List all configured workspace rate limits. | +| GET | `/api/v1/workspaces/{workspace_id}/rate-limit` | — | Get the rate limit for a workspace. | +| PUT | `/api/v1/workspaces/{workspace_id}/rate-limit` | `messages_per_second` | Set the rate limit for a workspace. Must be `>= 0`. | +| DELETE | `/api/v1/workspaces/{workspace_id}/rate-limit` | — | Remove the workspace-specific rate limit (falls back to the default). | ## WebSocket Monitoring The WebSocket endpoint provides a real-time stream of gateway events and per-topic message monitoring. -**Endpoint:** `ws://localhost:31880/api/ws/events` +**Endpoint:** `ws://localhost:31880/api/v1/ws/events` Authentication follows the same API key rules as the REST API. See the WebSocket Authentication section above for browser compatibility. @@ -323,7 +377,7 @@ Multiple topic subscriptions can be active on a single WebSocket connection. Eac ```bash # Install websocat: https://github.com/vi/websocat websocat -H "Authorization: Bearer your-secret-key" \ - "ws://localhost:31880/api/ws/events" + "ws://localhost:31880/api/v1/ws/events" # After connecting, send: {"action":"subscribe_monitor","topic":"ag.hello.greeter.agent-a"} @@ -333,5 +387,6 @@ websocat -H "Authorization: Bearer your-secret-key" \ - [Quickstart](quickstart.md) — start the gateway and connect your first agents - [Horizontal Scaling](horizontal-scaling.md) — deploy multiple gateway instances +- [Monitoring](monitoring.md) — Prometheus metrics, OTLP export, and the ops server's health/metrics endpoints - [Error Codes](error-codes.md) — protocol-level error reference - [Specification](specification.md) — full system specification diff --git a/docs/aetherlite.md b/docs/aetherlite.md index 09e0aa5..326b03b 100644 --- a/docs/aetherlite.md +++ b/docs/aetherlite.md @@ -10,7 +10,7 @@ AetherLite is a deployment mode for Aether that replaces all external services w | Single-node production | Supported | Overkill unless you need Redis/RabbitMQ for other reasons | | Edge or embedded deployments | Recommended | Usually impractical | | Prototyping / demos | Recommended | — | -| Multi-node horizontal scaling | **Not supported** | Required | +| Multi-node horizontal scaling | Experimental (see [Cluster Mode](#cluster-mode-experimental)) | Required | | High-throughput (thousands of msg/s) | Evaluate carefully | Preferred | AetherLite is production-ready for single-node workloads. It is **not** a development-only toy — data is durable on disk and message replay works correctly. @@ -37,25 +37,23 @@ AETHER_ALLOW_DEV_MODE=true ./aetherlite --data-dir /var/lib/aether-lite --insecu | Flag | Default | Description | |---|---|---| | `--config ` | — | Optional YAML config file | +| `--secrets-file ` | — | Optional generated-secrets.yaml merged into config (HMAC, admin key, TLS paths) | | `--data-dir ` | `./aether-lite-data` | Directory for SQLite and Badger storage | | `--port ` | `50051` | gRPC server port | | `--admin-port ` | `31880` | Admin UI port | +| `--ops-port ` | `0` (use config default `9090`) | Override the ops/metrics server port | | `--dev` | `false` | Development mode (relaxed security) | | `--insecure-admin` | `false` | Allow admin API without auth key (requires `AETHER_ALLOW_DEV_MODE`) | | `--workflow` | `true` | Enable embedded workflow server | +| `--workflow-config ` | — | Optional workflow config file (overrides auto-config) | | `--workflow-admin-port ` | `31881` | Workflow admin API port | | `--version` | — | Print version and exit | +| `--help` | — | Print usage and exit | -### Option 2: Individual binaries with `--lite` flag - -The `--lite` flag (or `mode: lite` in config) switches any standard binary to lite mode: - -```bash -go build -o gateway ./cmd/gateway -./gateway --lite --data-dir ./aether-lite-data --insecure-admin -``` - -This is useful when you want only the gateway component without the workflow server. +> Lite mode is only available via the dedicated `cmd/aetherlite` binary. The +> `cmd/gateway` binary no longer supports lite mode — it exits at startup if +> `--lite` (or `mode: lite`) is set. Use `cmd/aetherlite` for embedded +> single-binary deployments. ## Data Directory Layout @@ -63,16 +61,26 @@ AetherLite stores all persistent state under a single directory: ``` aether-lite-data/ - aether.db — SQLite: tasks, ACL rules, audit log, agent registry, orchestration profiles - badger/ — Badger: sessions, KV store, checkpoints, tokens, message log, consumer offsets + audit.db — SQLite: comprehensive audit log (dedicated WAL writer lock) + acl.db — SQLite: ACL rules, fallback policies, authority grants + registry.db — SQLite: agent registry, orchestrator profiles + tasks.db — SQLite: tasks, task audit events, timers, checkpoints, assignments, DLQ, orchestrated queue + tokens.db — SQLite: API tokens workflow.db — SQLite: workflow rules, definitions, executions, schedules, state machines + badger/ — Badger: sessions, KV store, checkpoints, tokens (Badger-backed), message log, consumer offsets ``` +Each domain owns its own SQLite file and runs its own migration set on +construction — the historical single-file `aether.db` was retired (each +store used to share one file via a compat layer). SQLite connections are +pinned to a single connection per file (WAL mode, `busy_timeout=5000`) to +serialize writes. + The data directory is created automatically on first run. Back it up like any other database directory. ## Configuration -AetherLite uses the same YAML configuration format as the full gateway. When using `./aetherlite`, `mode: lite` is always forced. When using `./gateway --lite`, add `mode: lite` to your YAML. +AetherLite uses the same YAML configuration format as the full gateway. When using `./aetherlite`, `mode: lite` is always forced. Minimal production config example: @@ -145,8 +153,8 @@ AetherLite uses the same gateway code as full Aether. The only difference is whi | Redis checkpoint store | `BadgerCheckpointStore` | `CheckpointManager` | | Redis token store | `BadgerTokenStore` | `state.TokenStore` | | RabbitMQ Streams router | `BadgerRouter` | `MessageRouter` | -| PostgreSQL task store | SQLite task store | `*tasks.TaskStore` | -| AMQP task dispatcher | `MemoryTaskDispatcher` | `TaskDispatcher` | +| PostgreSQL task store | SQLite task store (`tasks.db`) | `tasks.Store` | +| AMQP task dispatcher | `PollingTaskDispatcher` (`JetStreamTaskDispatcher` in cluster mode) | `TaskDispatcher` | | Redis quota counters | `MemoryQuotaManager` | `QuotaChecker` | The gateway server (`internal/gateway/server.go`) is identical in both modes. @@ -156,7 +164,7 @@ The gateway server (`internal/gateway/server.go`) is identical in both modes. | Property | AetherLite | Full Aether | |---|---|---| | External dependencies | None | Redis, RabbitMQ, PostgreSQL | -| Horizontal scaling | Single-node only | Multiple instances supported | +| Horizontal scaling | Single-node by default; experimental multi-node via `AETHERLITE_CLUSTER_MODE` (see [Cluster Mode](#cluster-mode-experimental)) | Multiple instances supported | | Message durability | Durable (Badger on disk) | Durable (RabbitMQ Streams) | | Message replay on reconnect | Supported | Supported | | Quota persistence across restarts | No (in-memory) | Yes (Redis) | @@ -164,6 +172,28 @@ The gateway server (`internal/gateway/server.go`) is identical in both modes. | Throughput ceiling | Limited by local disk I/O | Higher (distributed) | | PostgreSQL features | Via SQLite (same schema) | Native PostgreSQL | +## Cluster Mode (Experimental) + +Setting `AETHERLITE_CLUSTER_MODE=true` flips AetherLite from single-node +Badger backends to an embedded-NATS/JetStream-backed cluster: the message +router, session manager, KV store, and checkpoint store all become +JetStream-backed, with an optional S3-or-local backup coordinator for cold- +start restore. Peers are named via `AETHERLITE_CLUSTER_PEERS`, and +`AETHERLITE_HA_MODE` selects `async`/`sync`/`auto` replication. + +The per-domain SQLite stores (`tasks.db`, `audit.db`, `acl.db`, +`registry.db`, `tokens.db`) remain node-local even in cluster mode — they +are not themselves replicated, though several of the stores they back +(registry, ACL rules, authority-request lifecycle, API tokens) are mirrored +to peers via JetStream KV watches on top of the local SQLite writes. The +task dispatcher switches from `PollingTaskDispatcher` to +`JetStreamTaskDispatcher` when `AETHER_DISPATCHER=jetstream` (the default +once cluster mode is enabled). + +For the full list of `AETHERLITE_CLUSTER_*` / `AETHERLITE_NATS_*` / +`AETHERLITE_S3_*` variables and their defaults, see +[environment.md](environment.md#aetherlite-cluster-mode-jetstream). + ## Upgrading from AetherLite to Full Mode If you start with AetherLite and later need to scale horizontally, you can migrate: @@ -175,7 +205,7 @@ If you start with AetherLite and later need to scale horizontally, you can migra ## Limitations and Known Issues -- **Single-node only.** Badger and SQLite are local files; they cannot be shared across multiple gateway processes. +- **Single-node by default.** Badger and the per-domain SQLite files are local; they cannot be shared across multiple gateway processes unless `AETHERLITE_CLUSTER_MODE=true` is set (see [Cluster Mode](#cluster-mode-experimental)), which replicates the router/session/KV/checkpoint layer via embedded NATS/JetStream — the SQLite-backed stores themselves stay node-local. - **Quota counts reset on restart.** The in-memory quota manager does not persist connection counters across restarts. Workspace connection limits are re-enforced from zero after each restart. - **Back-pressure on message channels.** Under extreme back-pressure, the live subscriber delivery channel may drop messages. Those messages remain durably stored in Badger and are replayed on the subscriber's next reconnect. - **SQLite concurrency.** SQLite is configured in WAL mode with a 5-second busy timeout. Very high write concurrency may cause transient `SQLITE_BUSY` errors on the audit log or task store. If you hit this consistently, consider the full PostgreSQL deployment. diff --git a/docs/audit_testing_guide.md b/docs/audit_testing_guide.md index 1a70cb3..27c513d 100644 --- a/docs/audit_testing_guide.md +++ b/docs/audit_testing_guide.md @@ -23,7 +23,10 @@ The comprehensive audit logging system records all security-relevant events incl 2. **Database Migration:** ```bash - # Ensure migration 005 has been applied + # Ensure at least migration 008 (comprehensive_audit_schema) has been + # applied. Migrations 010 (consolidate_audit), 011 (audit_partitioning), + # and 021 (audit_log_source) also alter comprehensive_audit_log, so in + # practice just run migrate up to head. migrate -path migrations -database "postgres://aether:aether_dev@localhost:5432/aether?sslmode=disable" up ``` @@ -33,18 +36,11 @@ The comprehensive audit logging system records all security-relevant events incl ## Automated Test -Run the automated integration test: - -```bash -./scripts/test_connection_audit.sh -``` - -This script will: -1. Check prerequisites -2. Clear old test audit logs -3. Create and run a test client -4. Verify audit events were logged -5. Display results +> **Note:** `scripts/test_connection_audit.sh` (and the other `scripts/test_*_audit.sh` +> / `scripts/test_retention_cleanup.sh` scripts referenced by this guide) do not +> exist in `server/scripts` in this repo. `server/scripts` currently only ships +> `docker_rmq_test.sh`, `docker_valkey_test.sh`, and the `proxy_load` load-test +> harness. Use the manual testing steps below until an automated script lands. ## Manual Testing @@ -62,18 +58,16 @@ export AETHER_AUDIT_FLUSH_PERIOD=2s go run ./cmd/gateway ``` -You should see: -``` -Audit logger configuration: - Enabled: true - Event types: [connection auth message kv admin acl] - Verbosity: high - Batch size: 10 - Flush period: 2s - Retention: 90 days - Channel buffer: 1000 -Audit logger initialized (migration: 005_comprehensive_audit_schema.sql) -``` +There is no dedicated "Audit logger configuration" startup banner printed by +the gateway — `audit.NewAuditLogger` initializes silently. To confirm the +config landed, check the effective values in code/config instead (or add a +temporary log line). With `AETHER_AUDIT_EVENT_TYPES=all`, the enabled event +types resolve to `[connection auth message kv task admin acl authorization +authority_request]` (see `EventType*` constants in +`internal/audit/types.go` and the `all` expansion in +`internal/audit/config.go`) — note `custom` is excluded from `all` since it is +reserved for principal-submitted events via `SubmitAuditEvent`, not +gateway-emitted ones. ### 2. Connect a Test Client @@ -447,8 +441,8 @@ SELECT success, metadata->>'scope' as scope, metadata->>'key' as key, - metadata->>'old_value' as old_value, - metadata->>'new_value' as new_value, + metadata->>'old_value_size' as old_value_size, + metadata->>'new_value_size' as new_value_size, metadata->>'result_count' as result_count FROM comprehensive_audit_log WHERE event_type = 'kv' @@ -467,12 +461,12 @@ For the test client above, you should see: metadata: { "scope": "address", "key": "test-key", - "old_value": "", - "new_value": "value-1", "new_value_size": 7, "is_update": false } ``` + Note: `old_value_size`/`new_value_size` (byte counts) are logged, never the + literal `old_value`/`new_value` strings — see the security note above. 2. **KV GET (success)** ```sql @@ -492,9 +486,7 @@ For the test client above, you should see: metadata: { "scope": "address", "key": "test-key", - "old_value": "value-1", "old_value_size": 7, - "new_value": "value-2", "new_value_size": 7, "is_update": true } @@ -507,7 +499,7 @@ For the test client above, you should see: metadata: { "scope": "address", "key": "ttl-key", - "new_value": "expires", + "new_value_size": 7, "ttl_seconds": 300, "is_update": false } @@ -541,7 +533,6 @@ For the test client above, you should see: metadata: { "scope": "address", "key": "test-key", - "old_value": "value-2", "old_value_size": 7 } ``` @@ -551,11 +542,12 @@ For the test client above, you should see: Check that KV operations include all expected metadata: ```sql --- Check PUT operations include before/after values +-- Check PUT operations include before/after value sizes (not the values +-- themselves — see the security note above) SELECT operation, - metadata->>'old_value' as old_value, - metadata->>'new_value' as new_value, + metadata->>'old_value_size' as old_value_size, + metadata->>'new_value_size' as new_value_size, metadata->>'is_update' as is_update FROM comprehensive_audit_log WHERE operation = 'kv_put' @@ -563,11 +555,11 @@ ORDER BY timestamp DESC; ``` ```sql --- Check DELETE operations include old_value +-- Check DELETE operations include old_value_size SELECT operation, metadata->>'key' as key, - metadata->>'old_value' as old_value + metadata->>'old_value_size' as old_value_size FROM comprehensive_audit_log WHERE operation = 'kv_delete' ORDER BY timestamp DESC; @@ -628,9 +620,9 @@ ORDER BY timestamp DESC; #### Scenario 1: PUT Create vs Update 1. PUT a new key -2. Verify: is_update=false, old_value="" +2. Verify: is_update=false, old_value_size absent/0 3. PUT the same key with new value -4. Verify: is_update=true, old_value contains previous value +4. Verify: is_update=true, old_value_size reflects the previous value's byte length (the previous value itself is not logged) #### Scenario 2: Different Scopes Test KV operations with different scopes: @@ -673,9 +665,9 @@ LIMIT 10; ### KV Audit Success Criteria ✅ All KV operations logged (GET, PUT, DELETE, LIST) -✅ PUT operations include before/after values -✅ PUT updates set is_update=true and include old_value -✅ DELETE operations include old_value +✅ PUT operations include before/after value sizes (old_value_size/new_value_size), not the values themselves +✅ PUT updates set is_update=true and include old_value_size +✅ DELETE operations include old_value_size ✅ LIST operations include result_count ✅ PUT with TTL includes ttl_seconds ✅ Failed operations have success=false @@ -837,7 +829,19 @@ ORDER BY timestamp; #### 4. Verify Expected Events -For each message sent, you should see TWO audit events: +> **Coalescing note:** Successful `message_received`/`message_routed` events +> (and `proxy_http_routed`) for the same (sender, target-topic, operation) +> tuple are coalesced within a rolling window — only the first occurrence is +> written, later repeats within the window are suppressed — to keep +> high-volume chat token streaming from flooding `comprehensive_audit_log`. +> Failures and every other event type are never coalesced. The window +> defaults to 60s and is configurable via `AETHER_AUDIT_COALESCE_WINDOW` +> (`0` disables coalescing). If you send the same message repeatedly to the +> same target within the window, expect fewer rows than sends. Send to a +> fresh target topic (or wait out the window) between test messages if you +> need one row per send. See `internal/gateway/audit_coalesce.go`. + +For each message sent (outside the coalescing window, or to a distinct target), you should see TWO audit events: 1. **message_received** - Logged after ACL check - operation: `message_received` @@ -967,7 +971,6 @@ ORDER BY timestamp DESC; ✅ High verbosity: Includes message_content (truncated to 1KB) ✅ Failed operations have success=false with error_message ✅ All operations include session_id for correlation -✅ task_id included in metadata for correlation with task system ### Verbosity Level Comparison @@ -1121,6 +1124,19 @@ WHERE workspace = 'test' AND operation = 'test'; ## 4. Testing Admin Action Audit Events +> **STALE / NOT CURRENTLY WIRED UP:** `internal/audit/types.go` still defines +> `OpAdminStateQuery` ("admin_state_query"), `OpAdminSessionDisconnect` +> ("admin_session_disconnect"), and `OpAdminConfigChange` +> ("admin_config_change"), but as of this writing nothing in the codebase +> calls into the audit logger from the admin HTTP handlers +> (`internal/admin/server.go`'s `/api/connections`, `/api/agents`, +> `/api/kv/...`, `/api/tasks/.../retry|cancel`, etc. — verified via `grep -rn +> "audit\." internal/admin`, zero hits outside tests). Following the steps +> below will exercise the admin REST API but will **not** produce any +> `event_type = 'admin'` rows in `comprehensive_audit_log`. Treat this section +> as a spec for the intended behavior once admin-action auditing is +> (re)implemented, not as a description of current behavior. + ### Automated Test Run the automated admin action audit test: @@ -1777,29 +1793,35 @@ Expected: Only `test_5d` and `test_2d` should remain (< 7 days old) ### Test 7: Background Cleanup Job -The gateway runs a background cleanup job daily. To test it manually: - -1. **Start Gateway:** - ```bash - export AETHER_AUDIT_RETENTION_DAYS=30 - go run ./cmd/gateway +The scheduled audit-log retention sweep is run by `internal/cleanup.Service` +(`CleanupOldAuditLogs`, `internal/cleanup/service.go`), the same periodic-job +runner that handles task purge/reconciliation — it is a `cleanup.Config` +field, not a standalone ticker in `cmd/gateway/main.go`. + +- Config keys (YAML only — there is no `AETHER_AUDIT_RETENTION_DAYS`-style env + override for these two; that env var only affects `AuditConfig.RetentionDays`, + a separate, unused-by-the-sweep knob consumed by the manual + `CleanupOldACLAuditLogs` admin RPC path): + - `cleanup.audit_retention_days` — default `90` + - `cleanup.audit_cleanup_interval` — default `24h`; set to `"0"` to disable + the scheduled sweep entirely + +1. **Start Gateway with a short interval for testing** (edit `configs/dev.yaml` + or your config file): + ```yaml + cleanup: + audit_retention_days: 30 + audit_cleanup_interval: "1m" ``` - -2. **Check Logs:** - Look for log messages about cleanup job: - ``` - [INFO] Audit log cleanup job started (runs daily) - [INFO] [Audit] Cleanup: purged N old audit log entries + ```bash + go run ./cmd/gateway -config configs/dev.yaml ``` -3. **Verify in Database:** - After 24 hours, the cleanup job should have run automatically and deleted old records. - -For immediate testing, you can modify the ticker interval in `cmd/gateway/main.go`: -```go -// Change from 24 hours to 1 minute for testing -ticker := time.NewTicker(1 * time.Minute) -``` +2. **Verify in Database:** + After the configured interval, the cleanup job should have run + automatically and deleted rows older than `audit_retention_days`. The job + logs its result via `JobResult` (job name `audit_log_cleanup`); check + gateway logs for that job name and its `ItemCount`/`Details`. ### Retention Policy Verification @@ -1841,17 +1863,13 @@ Different compliance requirements may need different retention periods: - **PCI-DSS**: 1 year minimum - **General Security**: 90 days -Configure retention via environment variable: - -```bash -# 90-day retention (default) -export AETHER_AUDIT_RETENTION_DAYS=90 - -# 1-year retention (SOC2, PCI-DSS) -export AETHER_AUDIT_RETENTION_DAYS=365 +Configure the scheduled sweep's retention window via the gateway YAML config +(`cleanup.audit_retention_days`; there is no environment-variable override for +this field — see Test 7 above): -# 6-year retention (HIPAA) -export AETHER_AUDIT_RETENTION_DAYS=2190 +```yaml +cleanup: + audit_retention_days: 90 # default (SOC2/PCI-DSS: 365, HIPAA: 2190) ``` ### Cleanup Performance Testing @@ -1898,13 +1916,13 @@ Expected: - Records older than the boundary are deleted ✅ **Background Job:** -- Gateway starts background cleanup job on startup -- Job runs every 24 hours -- Job uses retention period from audit configuration -- Job logs cleanup results (number of records purged) +- Gateway's `cleanup.Service` starts the `audit_log_cleanup` periodic job on startup (when `cleanup.audit_cleanup_interval` is nonzero) +- Job runs every 24 hours by default (`cleanup.audit_cleanup_interval`) +- Job uses the retention period from `cleanup.audit_retention_days` +- Job logs cleanup results (number of records purged) via its `JobResult` ✅ **Configuration:** -- Retention period configurable via `AETHER_AUDIT_RETENTION_DAYS` environment variable +- Retention period configurable via `cleanup.audit_retention_days` in the gateway YAML config (no environment-variable override) - Default retention is 90 days - Custom retention periods work correctly (7, 30, 90, 365, etc.) diff --git a/docs/environment.md b/docs/environment.md index 2f0dc67..235ecd4 100644 --- a/docs/environment.md +++ b/docs/environment.md @@ -61,6 +61,8 @@ For setup walkthroughs, see `docs/quickstart.md`, | [Gateway — Tracing & Logging](#gateway-tracing--logging) | `gateway` / `all` | OpenTelemetry, log level/format | | [Gateway — Proxy data plane](#gateway-proxy-data-plane) | `gateway` | Proxy bypass kill switch | | [AetherLite](#aetherlite-cmdaetherlite) | `aetherlite` | Embedded single-binary flag overrides | +| [AetherLite — Badger retention](#aetherlite-badger-retention-single-node-mode) | `aetherlite` | Message / consumer-offset TTL (single-node Badger) | +| [AetherLite — Cluster mode](#aetherlite-cluster-mode-jetstream) | `aetherlite` | Embedded NATS/JetStream clustering + S3 backup | | [Migrate](#migrate-cmdmigrate) | `migrate` | Standalone DB migration tool | | [Auth Proxy](#auth-proxy-cmdauth-proxy) | `auth-proxy` | Reverse-proxy auth gateway | | [Auth Proxy — Login flow](#auth-proxy-browser-login-flow) | `auth-proxy` | Browser OIDC login + session cookies | @@ -87,7 +89,8 @@ section. | `PORT` | int | `50051` | gRPC listen port. Unprefixed by design for PaaS compatibility. | | `AETHER_DEV_MODE` | bool | `false` | Gates several "fail-closed" validations: allows missing token HMAC key, allows `verify_signature:false`, allows plaintext gRPC with credential auth. **NOT FOR PRODUCTION.** | | `AETHER_ALLOW_DEV_MODE` | bool | `false` | Required opt-in for `--insecure-admin` and for `admin.cors_origin="*"` to be accepted by the config validator. **NOT FOR PRODUCTION.** | -| `AETHER_AUTO_TLS` | bool | `false` | When `true`, the gateway auto-generates a self-signed CA + server cert into `secrets-dir/tls/` if no TLS files are configured. Dev convenience; **NOT FOR PRODUCTION** unless the generated CA is trust-rooted externally. | +| `AETHER_AUTO_TLS` | bool | `false` | When `true`, the gateway auto-generates a self-signed CA + server cert into `secrets-dir/tls/` if no TLS files are configured. Dev convenience; **NOT FOR PRODUCTION** unless the generated CA is trust-rooted externally. Read directly by `cmd/gateway`'s startup flow, not by `ApplyEnvOverrides()` — **not honoured by `cmd/aetherlite`**. | +| `AETHER_TENANT_ID` | string | (none) | Names this gateway instance's own tenant scope; minted into `X-Auth-Tenant-ID` on proxied requests. Read directly by both `cmd/gateway` and `cmd/aetherlite`. When unset, the proxy path falls back to the sender's workspace as the tenant scope. Also read by `proxy-sidecar` (see [Proxy Sidecar](#proxy-sidecar-cmdproxy-sidecar)) to propagate the same identifier in identity headers. | ### Gateway — Admin API @@ -131,6 +134,8 @@ come from `internal/audit/types.go`. | `AETHER_AUDIT_FLUSH_PERIOD` | duration | `5s` | Maximum interval between flushes when the batch is not full. Renamed from `AUDIT_FLUSH_PERIOD`. | | `AETHER_AUDIT_RETENTION_DAYS` | int > 0 | `90` | Retention window for batched writes. Renamed from `AUDIT_RETENTION_DAYS`. | | `AETHER_AUDIT_CHANNEL_BUFFER` | int > 0 | `1000` | Size of the async event channel buffer. Renamed from `AUDIT_CHANNEL_BUFFER`. | +| `AETHER_AUDIT_COALESCE_WINDOW` | duration | `60s` | Window over which a burst of identical successful audit events (e.g. chat-streaming chatter) is coalesced into one record. Read by the unified config loader only (no legacy unprefixed name). | +| `AETHER_AUDIT_FOREIGN_RATE_PER_SEC` | float | `100` | Per-principal rate limit (events/sec) for foreign `SubmitAuditEvent` submissions. Read directly by both `cmd/gateway` and `cmd/aetherlite` (not via `ApplyEnvOverrides()`). | ### Gateway — Storage (PostgreSQL, Redis, RabbitMQ) @@ -149,7 +154,7 @@ gateway image can drop straight into PaaS targets that inject | `REDIS_PASSWORD` | string | (none) | Redis password. | | `STREAM_URL` | URL (`rabbitmq-stream://`) | `rabbitmq-stream://guest:guest@localhost:55552` (dev) | RabbitMQ Streams endpoint. | | `AMQP_URL` | URL (`amqp://` or `amqps://`) | `amqp://guest:guest@localhost:55672/` (dev) | RabbitMQ AMQP endpoint (used for delayed-exchange orchestration). | -| `AETHER_FANIN_SHARDS` | int ≥ 1 | `1` | Number of fan-in shards for `event::receiver{N}` and `metric::receiver{N}` topics. Today only shard 0 is used (stub always returns 0). Future releases will use fnv64 hashing to distribute workspaces across N shards, enabling N parallel Workflow Engine and Metrics Bridge instances. | +| `AETHER_FANIN_SHARDS` | int ≥ 1 | `1` | **Reserved, not yet read from the environment.** `pkg/sharding.TotalShards()` is a stub that always returns `1` regardless of any env var; only `event::receiver0` / `metric::receiver0` are used today. This name is reserved for the future fnv64-hashing fan-in sharding scheme (`pkg/sharding`) that will distribute workspaces across N shards for parallel Workflow Engine / Metrics Bridge instances. | ### Gateway — Tracing & Logging @@ -190,6 +195,44 @@ Precedence at the call site is: | `AETHERLITE_WORKFLOW_CONFIG` | path | (empty) | `--workflow-config` | lite-only | | `AETHERLITE_WORKFLOW_ADMIN_PORT` | int | `31881` | `--workflow-admin-port` | lite-only | +### AetherLite — Badger retention (single-node mode) + +These apply only when AetherLite is running single-node (Badger) backends — +i.e. `AETHERLITE_CLUSTER_MODE` is not enabled. They are read directly via +`os.Getenv` (not via `config.EnvStr`/`EnvInt`, so there is no matching CLI +flag) and applied to the `BadgerRouter` at startup. + +| Variable | Type | Default | Description | +|---|---|---|---| +| `AETHER_MESSAGE_RETENTION_TTL` | Go duration string | `24h` | Badger per-message retention (native per-entry expiry) to bound topic log growth. Set to `"0"` to retain forever. An invalid duration string is ignored with a warning (falls back to the 24h default). | +| `AETHER_OFFSET_RETENTION_TTL` | Go duration string | `168h` (7d) | Badger per-consumer-offset retention, decoupled from message retention. Set to `"0"` to retain forever. An invalid duration string is ignored with a warning (falls back to the 7d default). | + +### AetherLite — Cluster mode (JetStream) + +`AETHERLITE_CLUSTER_MODE=true` flips AetherLite from single-node Badger +backends to an embedded-NATS/JetStream-backed cluster (router, session, KV, +checkpoint, dispatcher, and an S3-or-local backup coordinator). These +variables are parsed once at startup by `readClusterEnv()` in +`cmd/aetherlite/cluster_wiring.go`; all are `AETHERLITE_*`-prefixed except +the shared `AETHER_DISPATCHER` toggle. + +| Variable | Type | Default | Description | +|---|---|---|---| +| `AETHERLITE_CLUSTER_MODE` | bool | `false` | Master switch. Enables the embedded NATS server + JetStream-backed router/session/KV/checkpoint/dispatcher and the backup coordinator. | +| `AETHERLITE_CLUSTER_PEERS` | comma-list | (empty) | NATS route URLs of peer nodes. Empty means single-node JetStream (no cluster route listener). | +| `AETHERLITE_HA_MODE` | enum (`auto`/`async`/`sync`) | `auto` | Replication mode once peers are configured. Unknown values fall back to `auto` with a warning. | +| `AETHERLITE_NATS_CLIENT_PORT` | int | `0` (ephemeral) | Embedded NATS client port. | +| `AETHERLITE_NATS_CLUSTER_PORT` | int | `6222` | Embedded NATS peer-routing port. | +| `AETHERLITE_S3_BUCKET` | string | (empty) | S3 bucket for the backup coordinator. Empty falls back to `LocalFileStorage` rooted under `{data-dir}/backups`. | +| `AETHERLITE_S3_PREFIX` | string | `aetherlite/` | Key prefix for S3-stored backups. | +| `AETHERLITE_S3_REGION` | string | `us-east-1` | AWS region for the S3 backup bucket. | +| `AETHERLITE_S3_ENDPOINT` | string | (empty) | Optional custom S3 endpoint (MinIO, R2, etc.). | +| `AETHERLITE_S3_ACCESS_KEY` | string | (empty) | Optional static S3 access key (falls back to the default AWS credential chain when unset). | +| `AETHERLITE_S3_SECRET_KEY` | string | (empty) | Optional static S3 secret key. | +| `AETHERLITE_S3_FORCE_PATH_STYLE` | bool | `false` | Forces path-style S3 URLs (required by most MinIO deployments). | +| `AETHERLITE_RESTORE_FROM_S3` | bool | `false` | On startup, restores JetStream state from the configured backup storage before constructing cluster-mode backends. Destructive of any current state for the restored domains — only runs when explicitly requested. | +| `AETHER_DISPATCHER` | enum (`polling`/`jetstream`) | `jetstream` when `AETHERLITE_CLUSTER_MODE=true`, otherwise `polling` | Task dispatcher backend. Shared name with no `AETHERLITE_*` prefix because the concept (polling vs. event-driven dispatch) isn't cluster-specific, even though only `cmd/aetherlite` reads it today. | + ## Migrate (`cmd/migrate`) The standalone migration runner. Use it to apply embedded SQL migrations diff --git a/docs/horizontal-scaling.md b/docs/horizontal-scaling.md index 05ff421..069eba8 100644 --- a/docs/horizontal-scaling.md +++ b/docs/horizontal-scaling.md @@ -1,6 +1,6 @@ # Horizontal Scaling Architecture -> **Note: AetherLite is single-node only.** If you are running `./aetherlite` or `./gateway --lite`, horizontal scaling is not supported — all state is held in local Badger and SQLite databases that cannot be shared across processes. To scale horizontally, use the full Aether stack with Redis, RabbitMQ, and PostgreSQL as described in this document. See [aetherlite.md](aetherlite.md) for AetherLite limitations. +> **Note: AetherLite is single-node only.** If you are running `./aetherlite`, horizontal scaling is not supported — all state is held in local Badger and SQLite databases that cannot be shared across processes. To scale horizontally, use the full Aether stack with Redis, RabbitMQ, and PostgreSQL as described in this document. See [aetherlite.md](aetherlite.md) for AetherLite limitations. This document describes how to deploy Aether Gateway in a horizontally scaled configuration with session affinity for high availability and load distribution. @@ -223,7 +223,7 @@ When a gateway instance crashes or becomes unavailable: - If session affinity expired, distribute via round-robin or least-connections 4. **New Gateway Acquires Lock**: - Old lock may still exist (if crash prevented cleanup) - - Old lock has TTL and expires within 60 seconds + - Old lock has TTL and expires within 30 seconds - New gateway either acquires immediately (if old lock expired) or rejects with `DuplicateIdentityError` 5. **Client Retries**: If rejected, client waits and retries exponentially (up to old lock TTL expiration) @@ -268,6 +268,10 @@ spec: **Client Responsibility**: Implement idempotent message handling or track `message_id` for deduplication. +### Task Reassignment After Worker Disconnect + +A background `DisconnectReaper` periodically scans tasks whose assigned worker has been disconnected longer than the task's grace window and fails them so they can be retried or reclaimed. It is decoupled from session-lock cleanup — a worker can reconnect to the same gateway or a different one in the fleet without losing its task — and is multi-gateway safe, since task state transitions are idempotent no-ops once a task is already terminal. + ## Production Deployment Checklist ### Prerequisites @@ -280,12 +284,12 @@ spec: - [ ] Set `AETHER_GATEWAY_ID` to unique value per instance (e.g., Pod name) - [ ] Set `REDIS_ADDR` to Redis cluster endpoint - [ ] Set `STREAM_URL` to RabbitMQ Streams cluster URL -- [ ] Configure `PORT` (default 50051 for gRPC) -- [ ] Set lock TTL via `LOCK_TTL_SECONDS` (default: 60) +- [ ] Configure `AETHER_PORT` (default 50051 for gRPC) +- [ ] Session lock TTL is a fixed 30-second constant (not configurable); locks are refreshed every 10 seconds while the connection is alive ### Load Balancer Configuration - [ ] Enable session affinity (ClientIP or Cookie) -- [ ] Configure health checks on port 50051 (gRPC) or 8080 (HTTP health endpoint) +- [ ] Configure health checks on port 50051 (gRPC) or 9090 (HTTP health endpoint — the ops/metrics port, see [Monitoring](monitoring.md)) - [ ] Set health check interval: 10 seconds - [ ] Set health check timeout: 5 seconds - [ ] Set unhealthy threshold: 2 consecutive failures @@ -369,7 +373,7 @@ redis-cli GET "session:ag.workspace1.impl.spec" redis-cli DEL "session:ag.workspace1.impl.spec" ``` -**Prevention**: Ensure `LOCK_TTL_SECONDS` is set (default 60). Locks auto-expire even if not explicitly released. +**Prevention**: Session lock TTL is a fixed 30-second constant (`LockTTL` in `internal/state/session.go`) — it is not configurable via environment variable. Locks auto-expire even if not explicitly released, and connected gateways refresh the lock every 10 seconds (`LockRefreshInterval`) to keep it alive. ### Problem: Uneven Load Distribution diff --git a/docs/monitoring.md b/docs/monitoring.md index 68de150..22f747e 100644 --- a/docs/monitoring.md +++ b/docs/monitoring.md @@ -10,7 +10,7 @@ Aether Gateway exposes Prometheus metrics at: GET http://localhost:9090/metrics ``` -The metrics port and endpoint are configured in the admin server. In production, this port should be network-isolated (internal only) to prevent metrics exposure on the public load balancer. +Metrics are served by a dedicated ops server that is separate from the admin UI/API server (see [Admin UI Guide](admin-ui.md)) — it exists purely for Kubernetes health probes and the Prometheus scrape endpoint, with no authentication. The port defaults to 9090 and is configured via `gateway.ops_port` in the YAML config, the `--ops-port` flag, or the `AETHER_OPS_PORT` environment variable. In production, this port should be network-isolated (internal only) to prevent metrics exposure on the public load balancer. **Important:** The `/metrics` endpoint is intentionally unauthenticated per Prometheus convention. Restrict network access at the infrastructure level (firewall, VPC, etc.). @@ -25,7 +25,7 @@ These metrics track client connection lifecycle and multiplicity. | `aether_active_connections` | Gauge | `workspace`, `principal_type` | Currently active client connections broken down by workspace and principal type (agent, task, user, orchestrator, workflow_engine, metrics_bridge) | | `aether_admin_active_connections_total` | Gauge | | Total number of active gRPC connections across all workspaces and principal types (updated every 5 seconds) | | `aether_connection_duration_seconds` | Histogram | `workspace`, `principal_type` | Duration of individual client connections in seconds. Buckets: 1s to ~4.5h (exponential, base 2) | -| `aether_connection_attempts_total` | Counter | `workspace`, `principal_type`, `status` | Total connection attempts by outcome (success, failure, duplicate, auth_failed) | +| `aether_connection_attempts_total` | Counter | `workspace`, `principal_type`, `status` | Total connection attempts by outcome (`success`, `lock_failed`, `quota_exceeded`, `acl_denied`) | **Example queries:** ```promql @@ -47,7 +47,7 @@ These metrics monitor message throughput, errors, and latency. | Metric Name | Type | Labels | Description | |-------------|------|--------|-------------| | `aether_messages_routed_total` | Counter | `workspace`, `message_type` | Total messages successfully routed (CHAT, CONTROL, TOOL_CALL, EVENT, METRIC) | -| `aether_message_errors_total` | Counter | `workspace`, `error_type` | Total message routing errors by type (invalid_topic, unauthorized, offline_target, etc.) | +| `aether_message_errors_total` | Counter | `workspace`, `error_type` | Total message routing errors by type (`payload_too_large`, `rate_limited`, `workspace_rate_limited`, `permission_denied`, `cross_workspace_broadcast_denied`, `sv_wildcard_unavailable`, `metric_validation`, `publish_failed`) | | `aether_message_routing_latency_seconds` | Histogram | `workspace` | Latency of message routing from receive to publish (time spent in gateway). Buckets: 0.1ms to ~3.2s (exponential, base 2) | **Example queries:** @@ -69,24 +69,42 @@ These metrics track configuration store operations and performance. | Metric Name | Type | Labels | Description | |-------------|------|--------|-------------| -| `aether_kv_operations_total` | Counter | `operation`, `scope`, `status` | Total KV operations (get, set, delete, list) by scope (tenant, workspace, address) and outcome (success, failure) | +| `aether_kv_operations_total` | Counter | `operation`, `scope`, `status` | Total KV operations by proto `OpType` name (`GET`, `PUT`, `LIST`, `DELETE`, `INCREMENT`, `DECREMENT`, `COMPARE_AND_SET`, `COMPARE_AND_DELETE`, etc.), scope, and outcome (`ok`, `error`) | | `aether_kv_operation_latency_seconds` | Histogram | `operation`, `scope` | Latency of KV store operations. Buckets: 0.1ms to ~800ms (exponential, base 2) | **Example queries:** ```promql # KV operation success rate -sum(rate(aether_kv_operations_total{status="success"}[5m])) / +sum(rate(aether_kv_operations_total{status="ok"}[5m])) / sum(rate(aether_kv_operations_total[5m])) # KV get latency p95 -histogram_quantile(0.95, rate(aether_kv_operation_latency_seconds_bucket{operation="get"}[5m])) +histogram_quantile(0.95, rate(aether_kv_operation_latency_seconds_bucket{operation="GET"}[5m])) ``` -### Authentication & Authorization Metrics +### Redis & RabbitMQ Backend Metrics + +These metrics track the health of the two stateful backends the gateway depends on (not exposed by AetherLite, which uses Badger/JetStream instead). | Metric Name | Type | Labels | Description | |-------------|------|--------|-------------| -| `aether_auth_attempts_total` | Counter | `method`, `status` | Authentication attempts by method (api_key, token, mTLS) and outcome (success, failure, invalid) | +| `aether_redis_operations_total` | Counter | `operation`, `status` | Total Redis session/lock operations (`lock_acquire`, `lock_refresh`, `lock_release`, `session_register`, `session_unregister`) by outcome (`success`, `failure`) | +| `aether_session_lock_duration_seconds` | Histogram | | Duration of session lock acquisition. Buckets: 0.1ms to ~3.2s (exponential, base 2) | +| `aether_circuit_breaker_state` | Gauge | `subsystem` | Circuit breaker state: 0=closed, 1=open, 2=half-open | +| `aether_rabbitmq_publish_total` | Counter | `status` | Total RabbitMQ stream publish attempts by outcome | +| `aether_rabbitmq_publish_duration_seconds` | Histogram | | Duration of RabbitMQ stream publish operations. Buckets: 0.1ms to ~3.2s (exponential, base 2) | + +**Example queries:** +```promql +# Redis operation failure rate +sum(rate(aether_redis_operations_total{status="failure"}[5m])) / +sum(rate(aether_redis_operations_total[5m])) + +# Circuit breaker currently open (1) for any subsystem +aether_circuit_breaker_state == 1 +``` + +**Note:** There is currently no dedicated authentication-attempt metric (`aether_auth_attempts_total` does not exist in the codebase). Credential authentication (mTLS/API key/OAuth) failures short-circuit the connection before any Prometheus counter is incremented, so they are only visible in gateway logs, not in `aether_connection_attempts_total` — that metric only covers post-authentication outcomes (lock, quota, ACL). ### Orchestration Metrics @@ -273,7 +291,7 @@ Place these rules in your Prometheus alerting configuration: - alert: AetherKVOperationFailureRate expr: | ( - sum(rate(aether_kv_operations_total{status="failure"}[5m])) by (scope) + sum(rate(aether_kv_operations_total{status="error"}[5m])) by (scope) / sum(rate(aether_kv_operations_total[5m])) by (scope) ) > 0.05 @@ -300,23 +318,25 @@ Place these rules in your Prometheus alerting configuration: description: "Gateway has zero active connections. Check if gateway is running and if clients can reach it." ``` -### Authentication Failures +### Redis Lock Failures + +There is no dedicated authentication-attempt metric (see the note in [Redis & RabbitMQ Backend Metrics](#redis--rabbitmq-backend-metrics)). The closest actionable signal for identity/session problems is a spike in Redis lock failures, which surfaces both stale-lock contention and Redis connectivity issues: ```yaml -- alert: AetherHighAuthFailureRate +- alert: AetherHighRedisLockFailureRate expr: | ( - sum(rate(aether_auth_attempts_total{status!="success"}[5m])) + sum(rate(aether_redis_operations_total{operation="lock_acquire", status="failure"}[5m])) / - sum(rate(aether_auth_attempts_total[5m])) - ) > 0.25 + sum(rate(aether_redis_operations_total{operation="lock_acquire"}[5m])) + ) > 0.10 for: 5m labels: severity: warning component: aether annotations: - summary: "High authentication failure rate" - description: "Authentication failure rate is {{ humanizePercentage $value }}. Check API keys and token configuration." + summary: "High Redis lock acquisition failure rate" + description: "Lock acquisition failure rate is {{ humanizePercentage $value }}. Check for stale locks (DuplicateIdentityError) and Redis connectivity/latency." ``` --- @@ -745,7 +765,7 @@ Import this dashboard into Grafana for a complete monitoring view: "pluginVersion": "8.0.0", "targets": [ { - "expr": "sum(rate(aether_kv_operations_total{status=\"success\"}[1m])) by (operation)", + "expr": "sum(rate(aether_kv_operations_total{status=\"ok\"}[1m])) by (operation)", "legendFormat": "{{ operation }}", "refId": "A" } @@ -830,7 +850,7 @@ Import this dashboard into Grafana for a complete monitoring view: "pluginVersion": "8.0.0", "targets": [ { - "expr": "sum(rate(aether_kv_operations_total{status=\"failure\"}[5m])) by (scope) / sum(rate(aether_kv_operations_total[5m])) by (scope)", + "expr": "sum(rate(aether_kv_operations_total{status=\"error\"}[5m])) by (scope) / sum(rate(aether_kv_operations_total[5m])) by (scope)", "legendFormat": "{{ scope }}", "refId": "A" } @@ -871,7 +891,7 @@ Add this job to your Prometheus `prometheus.yml`: scrape_configs: - job_name: 'aether-gateway' static_configs: - - targets: ['localhost:9090'] # Adjust port to match admin server port + - targets: ['localhost:9090'] # Adjust port to match the ops server port (gateway.ops_port / AETHER_OPS_PORT) scrape_interval: 30s scrape_timeout: 10s # Optional: authentication if you expose metrics on a secure endpoint @@ -889,6 +909,25 @@ scrape_configs: --- +## OTLP Traces & Metrics Export + +In addition to the Prometheus `/metrics` endpoint, both `gateway` and `aetherlite` can export traces and metrics directly via OTLP/gRPC to an OpenTelemetry Collector. + +Both exporters are gated on the same environment variable, `OTEL_EXPORTER_OTLP_ENDPOINT`. When it is unset, tracing and metrics export are both disabled and the SDKs fall back to no-op providers — there is no separate on/off flag for each signal. + +```bash +# Typical OTLP/gRPC collector endpoint (insecure/plaintext for http:// scheme) +export OTEL_EXPORTER_OTLP_ENDPOINT="http://otel-collector:4317" +``` + +- **Traces**: exported via `otlptracegrpc`, batched, and tagged with `service.name` (`aether-gateway` or `aether-lite`). +- **Metrics**: exported via `otlpmetricgrpc` with a periodic reader (~60s interval by default), tagged with `service.name`, `service.namespace=scitrera`, and `service.version`. +- **Go runtime metrics**: `InitMeter` also starts the OpenTelemetry Go runtime instrumentation, so `go_*`/process metrics (GC, goroutines, memory) are included in the OTLP metrics stream alongside the application-level counters/histograms. +- The OTLP metrics exporter is a separate signal path from the Prometheus `aether_*` metrics described above — both can be enabled simultaneously and carry overlapping but not identical data (OTLP export includes Go runtime metrics that are not registered with the Prometheus registry). +- Both `otlptracegrpc` and `otlpmetricgrpc` connect to the **same** endpoint over gRPC (typically port 4317); do not point `OTEL_EXPORTER_OTLP_ENDPOINT` at an OTLP/HTTP port (4318) — the gRPC exporters will fail against it. + +--- + ## Health Check Endpoints Aether Gateway exposes Kubernetes-compatible health probes on the admin HTTP server: @@ -953,28 +992,28 @@ Group panels by concern: 4. **Orchestration** (trigger rates, resource usage) ### 5. Metering Integration -Usage metering (billable connections, message byte counts, KV operations by scope) is planned but not yet integrated. Use Prometheus metrics as the current source of truth for capacity planning. +A dedicated `aether_metering_*` metric namespace is already exposed for usage-based billing: `aether_metering_messages_routed_total`, `aether_metering_bytes_routed_total`, `aether_metering_kv_operations_total`, `aether_metering_checkpoint_operations_total`, and `aether_metering_task_operations_total` are actively incremented in the routing and orchestration paths (labeled by `workspace` and operation type). `aether_metering_active_connections` is defined but not currently updated by any code path. A billing pipeline that consumes these counters is not yet built — use them as the current source of truth for capacity planning and manual usage analysis in the meantime. --- ## Troubleshooting Guide ### High Message Error Rate -1. Check `message_errors_total` by `error_type` to identify failure category +1. Check `aether_message_errors_total` by `error_type` to identify failure category 2. Examine gateway logs for specific error messages -3. Verify RabbitMQ Streams health (producer/consumer status) -4. Check Redis for lock contention issues +3. Verify RabbitMQ Streams health via `aether_rabbitmq_publish_total{status="failure"}` and producer/consumer status +4. Check Redis for lock contention issues via `aether_redis_operations_total{status="failure"}` ### Latency Spikes 1. Compare message routing latency with RabbitMQ broker latency -2. Check gateway CPU/memory utilization (OS metrics) +2. Check gateway CPU/memory utilization (OS metrics) and Go runtime metrics (`go_memstats_*`, `go_goroutines`) exported via OTLP — see [OTLP Traces & Metrics Export](#otlp-traces--metrics-export) 3. Monitor KV operation latency separately (Redis latency) 4. Correlate with burst in connection attempts or message volume ### Connection Failures -1. Check `connection_attempts_total{status!="success"}` by principal type -2. Verify auth configuration (API keys, tokens) -3. Monitor Redis lock contention (`connection_attempts_total{status="duplicate"}`) +1. Check `aether_connection_attempts_total{status!="success"}` by principal type +2. Verify auth configuration (API keys, tokens) — credential authentication failures are not counted in this metric, so check gateway logs directly +3. Monitor Redis lock contention (`aether_connection_attempts_total{status="lock_failed"}`, `aether_redis_operations_total{operation="lock_acquire", status="failure"}`) 4. Check network connectivity between clients and gateway ### No Active Connections diff --git a/docs/on-behalf-of-acl-design.md b/docs/on-behalf-of-acl-design.md index 5195e91..81edc74 100644 --- a/docs/on-behalf-of-acl-design.md +++ b/docs/on-behalf-of-acl-design.md @@ -17,6 +17,7 @@ Implemented phase 1 enforcement: - request-time `on_behalf_of` resolution against persisted authority grants - subject ACL evaluation intersected with grant constraints - actor/subject/root-subject/grant lineage recorded in comprehensive audit events +- gateway-set, spoof-proof propagation of the resolved OBO subject onto delivered messages (`MessageEnvelope.on_behalf_subject` / `IncomingMessage.on_behalf_subject`) so a recipient can identify the user a message was sent for, distinct from the sending identity Implemented admin lifecycle surface: @@ -402,6 +403,15 @@ Suggested effective access formula: The actor's own resource ACL is not intersected into the subject decision. The actor's authority is represented by being the named delegate on a valid grant. This keeps the model understandable and prevents service ACLs from unexpectedly shrinking subject rights after a grant has already been issued. +### Delivery-Time Subject Propagation + +When a `SendMessage` resolves through an `AuthorizationContext` to an OBO subject, the gateway stamps that resolved subject onto the delivered message so the recipient can distinguish "who sent this" from "who it was sent for" — the message-bus analog of the `X-Auth-*` identity headers `ProxyHttp` mints from the same resolved authority. + +- `MessageEnvelope.on_behalf_subject` (`PrincipalRef`) is set by the gateway, never by the sender — spoof-proof, like `source`. +- Mirrored to the recipient as `IncomingMessage.on_behalf_subject`. +- Populated only when the send carried an `AuthorizationContext` that resolved to a grant with a non-empty `subject_type`/`subject_id`; empty for direct (non-OBO) sends. +- Subject identity only — it lets a recipient (e.g. a platform bridge) scope behavior to the acting-for user, but it does not carry authority. A recipient that needs to *act for* the subject still goes through the task authority-grant path (`CreateTaskResponse.authority_grant_id`), not this field. + ### Casbin Split Casbin remains the base ACL engine. Authority grants are enforced in Aether's service layer. diff --git a/docs/quickstart.md b/docs/quickstart.md index 21e6812..e628a19 100644 --- a/docs/quickstart.md +++ b/docs/quickstart.md @@ -46,7 +46,7 @@ AetherLite creates a `./aether-lite-data/` directory on first run and stores all You should see output like: ``` -AetherLite v0.1.0 — embedded single-binary server +AetherLite v0.2.2 — embedded single-binary server INF AetherLite gRPC gateway listening addr=:50051 INF admin UI listening port=31880 INF AetherLite is ready @@ -68,17 +68,19 @@ The full deployment uses Docker-managed RabbitMQ and Redis and exposes the compl ## Step 1: Start Dependencies -Aether requires RabbitMQ Streams for messaging and Redis (Valkey) for session state. The repository includes Docker scripts for both. +Aether requires RabbitMQ Streams for messaging and Redis (Valkey) for session state. The repository includes Docker scripts for both, under `server/scripts/`. -Open two terminal windows (or run both in the background): +Open two terminal windows (or run both in the background), both starting from the `server/` directory: ```bash # Terminal 1 — RabbitMQ with Streams plugin # Starts on: stream port 55552, AMQP port 55672, management UI at http://localhost:15672 +cd server ./scripts/docker_rmq_test.sh # Terminal 2 — Redis (Valkey) # Starts on: port 56379 (cluster ports 56379-56381) +cd server ./scripts/docker_valkey_test.sh ``` @@ -87,7 +89,8 @@ Wait a few seconds until both containers report that they are ready. You can ver ## Step 2: Build and Start the Gateway ```bash -# Build the gateway binary +# Build the gateway binary (from the server/ directory) +cd server go build -o gateway ./cmd/gateway # Start with development defaults (no config file required) @@ -109,7 +112,7 @@ The `--insecure-admin` flag permits the admin UI to run without an API key. This You should see output similar to: ``` -Aether Gateway v0.1.0 +Aether Gateway v0.2.2 Aether Gateway starting... ... INF Aether Gateway listening addr=:50051 diff --git a/docs/specification.md b/docs/specification.md index aa3aace..cbd71da 100644 --- a/docs/specification.md +++ b/docs/specification.md @@ -56,13 +56,28 @@ All messages are wrapped in a server-stamped `MessageEnvelope` before publishing ```protobuf message MessageEnvelope { - string source = 1; // Server-verified sender topic (cannot be spoofed) + string source = 1; // Server-verified sender topic (cannot be spoofed) bytes payload = 2; MessageType message_type = 3; - int64 timestamp_ms = 4; // Server-assigned timestamp + int64 timestamp_ms = 4; // Server-assigned timestamp + map metadata = 5; // Optional extensibility metadata + string workspace = 6; // Sender-declared, gateway-verified workspace context + PrincipalRef on_behalf_subject = 7; // Gateway-resolved OBO subject (see below) } ``` +**On-behalf-of (OBO) subject propagation.** When a sender's `SendMessage` +carries an `AuthorizationContext` that the gateway resolves to a valid OBO +subject, the gateway stamps the resolved subject onto +`MessageEnvelope.on_behalf_subject` — set from the resolved authority the same +spoof-proof way `source` is set from the authenticated sender. It is mirrored +onto `IncomingMessage.on_behalf_subject` at delivery time so a recipient can +identify the *user* a message was sent for, distinct from the sending +*identity* in `source`. Empty for direct (non-OBO) sends. `workspace` is +likewise propagated onto `IncomingMessage.workspace` so consumers of the +workspace-agnostic event/metric fan-in shards can recover the originating +workspace. + --- ## 2. System Architecture @@ -95,7 +110,7 @@ message MessageEnvelope { ### 2.3a AetherLite Mode -AetherLite is an alternative deployment mode that eliminates all external service dependencies by substituting in-process backends. It is activated via the `--lite` flag on any binary, or via `mode: lite` in the YAML config. The `cmd/aetherlite` binary always runs in lite mode. +AetherLite is an alternative deployment mode that eliminates all external service dependencies by substituting in-process backends. It runs as the dedicated `cmd/aetherlite` binary, which always operates in lite mode. The `cmd/gateway` binary no longer supports lite mode (it exits at startup if `--lite` or `mode: lite` is set). **Backend substitutions:** @@ -191,11 +206,12 @@ Authentication methods are configured via `auth.modes` in the YAML config. Multi | `tb` | Task Broadcast | `tb::{workspace}::{impl}` | Load-balancing topic; workers compete/round-robin | | `us` | User (Window) | `us::{user_id}::{window_id}` | Specific browser window | | `uw` | User (Workspace) | `uw::{user_id}::{workspace}` | User scoped to a workspace | +| `uu` | User (Broadcast) | `uu::{user_id}` | All of a user's windows regardless of active workspace; workspace-agnostic, ordinary (non-progress) messages. Platform-principal senders only. | | `ga` | Global Agents | `ga::{workspace}` | Broadcast to all agents in a workspace | | `gu` | Global Users | `gu::{workspace}` | Broadcast to all users in a workspace | | `pg` | Progress | `pg::{workspace}` | Progress updates with server-side recipient filtering | -| `event.*` | Workflow Engine | **Write (senders):** `event.{workspace}` or `event::*` — gateway rewrites to `event::receiver{shard}`. **Subscribe (WE):** `event::receiver{shard}` (today: always `event::receiver0`). Workspace attribution surfaced via `IncomingMessage.workspace`. | Fan-in event routing; Workflow Engine is the sole subscriber per shard. | -| `metric.*` | Metrics Bridge | **Write (senders):** `metric.{workspace}` or `metric::*` — gateway rewrites to `metric::receiver{shard}`. **Subscribe (MB):** `metric::receiver{shard}` (today: always `metric::receiver0`). Workspace attribution surfaced via `IncomingMessage.workspace`. | Fan-in metric routing; Metrics Bridge is the sole subscriber per shard. | +| `event::*` | Workflow Engine | **Write (senders):** `event::{workspace}` or `event::*` — gateway rewrites to `event::receiver{shard}`. **Subscribe (WE):** `event::receiver{shard}` (today: always `event::receiver0`). Workspace attribution surfaced via `IncomingMessage.workspace`. | Fan-in event routing; Workflow Engine is the sole subscriber per shard. | +| `metric::*` | Metrics Bridge | **Write (senders):** `metric::{workspace}` or `metric::*` — gateway rewrites to `metric::receiver{shard}`. **Subscribe (MB):** `metric::receiver{shard}` (today: always `metric::receiver0`). Workspace attribution surfaced via `IncomingMessage.workspace`. | Fan-in metric routing; Metrics Bridge is the sole subscriber per shard. | | `sv` | Service | `sv::{impl}::{spec}` | Cross-workspace service proxy endpoint (no workspace component) | | `br` | Bridge | `br::{impl}::{spec}` | Cross-workspace messaging bridge endpoint (no workspace component) | @@ -205,36 +221,44 @@ Topic names are validated against allowed prefixes and must be 1-256 characters. Each principal type automatically subscribes to specific topics on connection: -| Principal | Exclusive (offset-tracked) | Shared (no offset) | -|---|---|---| -| **Agent** | `ag::{ws}::{impl}::{spec}` | `ga::{ws}`, `pg::{ws}` | -| **Task (Unique)** | `tu::{ws}::{impl}::{spec}` | — | -| **Task (Non-Unique)** | `ta::{ws}::{impl}::{id}` | `tb::{ws}::{impl}` | -| **User** | `us::{uid}::{wid}` | `gu::{ws}`, `uw::{uid}::{ws}`, `pg::{ws}` | -| **Workflow Engine** | `event::receiver{shard}` (today: `event::receiver0`) | — | -| **Metrics Bridge** | `metric::receiver{shard}` (today: `metric::receiver0`) | — | -| **Orchestrator** | — | — (receives tasks via direct gRPC stream) | +| Principal | Exclusive durable (offset-tracked) | Resume-or-tail (durable) | Anonymous tail (no offset) | +|---|---|---|---| +| **Agent** | `ag::{ws}::{impl}::{spec}` | `pg::{ws}` | `ga::{ws}` | +| **Task (Unique)** | `tu::{ws}::{impl}::{spec}` | — | — | +| **Task (Non-Unique)** | `ta::{ws}::{impl}::{id}` | — | `tb::{ws}::{impl}` | +| **User** | `us::{uid}::{wid}` | `uu::{uid}`, `pg::us::{uid}`, `gu::{ws}`, `uw::{uid}::{ws}`, `pg::{ws}` | — | +| **Workflow Engine** | `event::receiver{shard}` (today: `event::receiver0`) | — | — | +| **Metrics Bridge** | `metric::receiver{shard}` (today: `metric::receiver0`) | — | — | +| **Orchestrator** | — | — | — (receives tasks via direct gRPC stream) | -**Exclusive subscriptions** use RabbitMQ consumer offset tracking so messages are replayed from the last committed position on reconnection. **Shared subscriptions** fan out locally from a single RabbitMQ consumer. +Three subscription modes back these lanes (`internal/gateway/interfaces.go`, `internal/router`): + +- **Exclusive durable (offset-tracked)** — identity topics use a named durable consumer that resumes from its last committed offset, so messages published while the client was away are replayed on reconnect. Agent/Task identity topics additionally accept a timestamp hint (`SubscribeExclusiveFromTimestamp`) so a cold-started, pool-dispatched worker can replay from the message that triggered it. +- **Resume-or-tail** (`SubscribeExclusiveResumeOrTail`) — the user broadcast / progress lanes (`gu`, `uw`, `uu`, `pg::{ws}`, `pg::us::{uid}`) use a per-window named durable that starts at the *tail* on a first-ever subscribe (no full-history dump) and replays only the gap on reconnect. +- **Anonymous tail** (`Subscribe`) — `ga::{ws}` and `tb::{ws}::{impl}` use an ephemeral consumer delivered from the current tail with no offset persistence. + +The per-task chat lane `tk::{ws}::{task_id}::msg` uses an anonymous tail for the live path plus a cold named consumer at connect time that replays the in-flight turn **in full** (the reloaded window needs every already-emitted token to re-render). All lanes fan out locally from a single shared consumer per gateway. ### 4.3 Permission Matrix | Sender | Can Send To | Cannot Send To | |---|---|---| -| **Agent** | Agents, Tasks, Users, Events, Metrics | Orchestrators, Progress | -| **Task** | Agents, Tasks, Users, Events, Metrics | Orchestrators, Progress | -| **User** | Agents, Tasks, Users | Events, Metrics, Progress | -| **Workflow Engine** | Agents, Tasks, Users, Events, Metrics | — | +| **Agent** | Agents, Tasks, Users, Events, Metrics | Orchestrators, Progress, User-Broadcast | +| **Task** | Agents, Tasks, Users, Events, Metrics | Orchestrators, Progress, User-Broadcast | +| **User** | Agents, Tasks, Users | Events, Metrics, Progress, User-Broadcast | +| **Workflow Engine** | Agents, Tasks, Users, Events, Metrics, User-Broadcast | — | | **Metrics Bridge** | *None (receive only)* | All | -| **Orchestrator** | Agent/Task topics only (status updates) | Events, Metrics | -| **Service** | Any topic (cross-workspace); per-message ACL checked against target workspace | — | -| **Bridge** | Any topic in any workspace; per-message ACL checked against target workspace | — | +| **Orchestrator** | Agent/Task topics only (status updates) | Events, Metrics, User-Broadcast | +| **Service** | Any topic (cross-workspace), incl. User-Broadcast; per-message ACL checked against target workspace | — | +| **Bridge** | Any topic in any workspace, incl. User-Broadcast; per-message ACL checked against target workspace | — | **Cross-workspace enforcement:** Workspace-scoped principals cannot send to topics in other workspaces. The workspace component of the target topic must match the sender's workspace. -**Cross-workspace event/metric broadcast:** Sending to `event.*` or `metric.*` topics in a workspace other than the sender's own native workspace requires the `capability/event_broadcast` or `capability/metric_broadcast` ACL permission respectively. Sending to the sender's own workspace is implicitly permitted. +**Cross-workspace event/metric broadcast:** Sending to `event::*` or `metric::*` topics in a workspace other than the sender's own native workspace requires the `capability/event_broadcast` or `capability/metric_broadcast` ACL permission respectively. Sending to the sender's own workspace is implicitly permitted. + +**User-broadcast (`uu::{user_id}`):** A workspace-agnostic channel that reaches every one of a user's open windows regardless of which workspace each window is viewing — the non-progress complement to the per-user progress topic `pg::us::{user_id}`. It carries ordinary `MessageEnvelope` protos (delivered as `IncomingMessage`), so it participates fully in the message-type system, audit, and metrics. Because the topic has no workspace component, the per-message workspace ACL cannot gate it; authorization is instead by principal type: **only Service, WorkflowEngine, and Bridge principals may publish.** All workspace-scoped principals (Users, Agents, Tasks, Orchestrators) are denied and must reach a user via `us::`/`uw::`/progress or task ownership. Users subscribe to their own `uu::{user_id}` topic on connect. -**Metric payload enforcement:** Messages with type `METRIC` sent to `metric.*` topics must carry a valid `Metric` proto as their payload (see [Section 4.5](#45-metric-payload-schema)). The gateway validates and rejects malformed metric messages before they reach the Metrics Bridge. +**Metric payload enforcement:** Messages with type `METRIC` sent to `metric::*` topics must carry a valid `Metric` proto as their payload (see [Section 4.5](#45-metric-payload-schema)). The gateway validates and rejects malformed metric messages before they reach the Metrics Bridge. ### 4.4 Progress Reports @@ -307,7 +331,7 @@ Events and metrics are routed through a **fan-in** layer that decouples workspac #### Design -- **Sender write API is unchanged.** Agents, tasks, and workflow engines send to `event.{workspace}` or `metric.{workspace}` exactly as before. The gateway rewrites the destination to the appropriate receiver shard before publishing to RabbitMQ Streams. +- **Sender write API is unchanged.** Agents, tasks, and workflow engines send to `event::{workspace}` or `metric::{workspace}` (or the legacy wildcard `event::*` / `metric::*`) exactly as before. The gateway rewrites the destination to the appropriate receiver shard before publishing to the message log. - **Receiver topics** follow the pattern `event::receiver{N}` and `metric::receiver{N}`. Today only shard 0 exists. - **Shard assignment** (`ShardForWorkspace`) is a stub that always returns 0. A future release will use fnv64 hashing to distribute workspaces across N shards, enabling N parallel Workflow Engine and Metrics Bridge instances. - **Exclusive offset-tracked subscriptions** are used for all receiver topics. Both the Workflow Engine and Metrics Bridge reconnect to the last committed offset, providing replay-on-reconnect and at-least-once delivery semantics. @@ -386,7 +410,7 @@ The `ProgressUpdate` fields are populated as follows: | `task_id` | The task that changed state | | `state` | New status: `running`, `completed`, `failed`, `cancelled`, `pending` | | `summary` | Human-readable description (includes error message for failures) | -| `recipient` | Parent agent's topic (e.g., `ag.default.orchestrator.main`) | +| `recipient` | Parent agent's topic (e.g., `ag::default::orchestrator::main`) | | `workspace` | Task's workspace | Notifications are sent at these transition points: @@ -587,7 +611,7 @@ service AetherGateway { |---|---| | `ConnectionAck` | Session ID for reconnection; `resumed` flag | | `ConfigSnapshot` | Workspace KV + global KV + task context on connect | -| `IncomingMessage` | Routed message with server-verified source topic | +| `IncomingMessage` | Routed message with server-verified source topic, declared `workspace`, and (for OBO sends) the gateway-resolved `on_behalf_subject` | | `KVResponse` | KV operation result with request ID correlation | | `CheckpointResponse` | Checkpoint operation result | | `TaskAssignment` | Orchestrator: task to execute with launch params and auth token | @@ -673,8 +697,8 @@ Configuration is loaded from a YAML file with environment variable overrides and ### 14.1 Go SDK (`sdk/go/`) -Full-featured SDK with typed clients for all six principal types. Features: -- `AgentClient`, `TaskClient`, `UserClient`, `OrchestratorClient`, `WorkflowEngineClient`, `MetricsBridgeClient` +Full-featured SDK with typed clients for all eight principal types. Features: +- `AgentClient`, `TaskClient`, `UserClient`, `OrchestratorClient`, `WorkflowEngineClient`, `MetricsBridgeClient`, `ServiceClient`, `BridgeClient` - Synchronous and asynchronous KV and checkpoint operations - Auto-reconnection with exponential backoff and jitter - Task creation with targeted, broadcast, and auto assignment modes @@ -701,39 +725,39 @@ Agent and User clients with gRPC transport: ### 15.1 Cold Start Agent -1. User sends message to `ag.default.gpu-worker.01`. +1. User sends message to `ag::default::gpu-worker::01`. 2. Gateway checks `identityIndex` — not locally connected. 3. Gateway checks Redis — no active lock. -4. Gateway publishes message to `ag.default.gpu-worker.01` stream (persisted). +4. Gateway publishes message to `ag::default::gpu-worker::01` stream (persisted). 5. Gateway creates orchestration task for `gpu-worker` implementation. 6. Dispatcher publishes task notification to AMQP queue. 7. A gateway with a connected orchestrator claims the task. 8. `TaskAssignment` + auth token sent to orchestrator via gRPC stream. 9. Orchestrator spins up a container with the token. -10. Container connects as `ag.default.gpu-worker.01` with the token. +10. Container connects as `ag::default::gpu-worker::01` with the token. 11. Gateway validates token, acquires lock, subscribes to topic. 12. Agent receives the persisted message from RabbitMQ stream (offset replay). ### 15.2 User Workspace Switch -1. Alice is in `Finance` workspace, receiving on `gu.finance`, `uw.alice.finance`, `pg.finance`. +1. Alice is in `Finance` workspace, receiving on `gu::finance`, `uw::alice::finance`, `pg::finance`. 2. Alice clicks "HR" in the UI. 3. Client sends `SwitchWorkspace("HR")`. 4. Gateway checks ACL for Alice's access to workspace "HR". 5. If denied, sends `ERR_PERMISSION_DENIED` to client. -6. If allowed, unsubscribes from `gu.finance`, `uw.alice.finance`, `pg.finance`. +6. If allowed, unsubscribes from `gu::finance`, `uw::alice::finance`, `pg::finance`. 7. Updates Alice's identity workspace under write lock. -8. Subscribes to `gu.hr`, `uw.alice.hr`, `pg.hr`. -9. Alice's `us.alice.browser1` subscription remains active (window-scoped, independent of workspace). +8. Subscribes to `gu::hr`, `uw::alice::hr`, `pg::hr`. +9. Alice's `us::alice::browser1` subscription remains active (window-scoped, independent of workspace). ### 15.3 Workflow Trigger -1. `ag.finance.payroll.01` finishes processing. -2. Agent sends a message with type `EVENT` to `event.finance`. +1. `ag::finance::payroll::01` finishes processing. +2. Agent sends a message with type `EVENT` to `event::finance`. 3. Gateway verifies the sender is allowed to publish events (agents can). -4. Message is published to the `event.finance` RabbitMQ stream. -5. The Workflow Engine (sole subscriber to `event.finance`) receives it. -6. Workflow Engine evaluates rules and sends a command to `ag.default.email-sender.01`. +4. Message is published to the `event::finance` RabbitMQ stream. +5. The Workflow Engine (sole subscriber to `event::finance`) receives it. +6. Workflow Engine evaluates rules and sends a command to `ag::default::email-sender::01`. --- diff --git a/scripts/compile_protos.sh b/scripts/compile_protos.sh index a5efcdf..176e099 100755 --- a/scripts/compile_protos.sh +++ b/scripts/compile_protos.sh @@ -45,7 +45,7 @@ cd "$PROTO_DIR" protoc \ --go_out=. --go_opt=paths=source_relative \ --go-grpc_out=. --go-grpc_opt=paths=source_relative \ - aether.proto + aether.proto sandbox_relay_tunnel.proto if [ $? -eq 0 ]; then echo "✓ Go code generated successfully" @@ -76,7 +76,7 @@ else --python_out="$OUTPUT_PYTHON" \ --pyi_out="$OUTPUT_PYTHON" \ --grpc_python_out="$OUTPUT_PYTHON" \ - aether.proto + aether.proto sandbox_relay_tunnel.proto if [ $? -eq 0 ]; then echo "✓ Python code generated successfully" @@ -86,6 +86,16 @@ else sed -i 's/^import aether_pb2 as aether__pb2$/from . import aether_pb2 as aether__pb2/' "$OUTPUT_PYTHON/aether_pb2_grpc.py" echo "✓ Fixed relative imports in gRPC file" fi + # The sandbox_relay_tunnel proto imports aether.proto and has its own + # grpc stub; both generated modules reference sibling pb2 modules by + # bare import, which breaks under the package's relative layout. + if [ -f "$OUTPUT_PYTHON/sandbox_relay_tunnel_pb2.py" ]; then + sed -i 's/^import aether_pb2 as aether__pb2$/from . import aether_pb2 as aether__pb2/' "$OUTPUT_PYTHON/sandbox_relay_tunnel_pb2.py" + fi + if [ -f "$OUTPUT_PYTHON/sandbox_relay_tunnel_pb2_grpc.py" ]; then + sed -i 's/^import sandbox_relay_tunnel_pb2 as sandbox__relay__tunnel__pb2$/from . import sandbox_relay_tunnel_pb2 as sandbox__relay__tunnel__pb2/' "$OUTPUT_PYTHON/sandbox_relay_tunnel_pb2_grpc.py" + echo "✓ Fixed relative imports in sandbox_relay_tunnel gRPC file" + fi else echo "✗ Python code generation failed" exit 1 @@ -114,7 +124,7 @@ if [ -f "$PROTO_LOADER_GEN" ]; then --grpcLib=@grpc/grpc-js \ --outDir="$OUTPUT_TS" \ --includeComments \ - aether.proto + aether.proto sandbox_relay_tunnel.proto if [ $? -eq 0 ]; then echo "✓ TypeScript types generated successfully (proto-loader-gen-types)" diff --git a/sdk/go/aether/admin_roles.go b/sdk/go/aether/admin_roles.go new file mode 100644 index 0000000..f5b3447 --- /dev/null +++ b/sdk/go/aether/admin_roles.go @@ -0,0 +1,224 @@ +package aether + +import ( + "context" + + pb "github.com/scitrera/aether/api/proto" +) + +// ============================================================================= +// Role/group authorization admin operations. +// +// These mirror the REST endpoints under /api/acl/groups, /api/acl/roles, and +// /api/acl/principals/{type}/{id}/{groups,roles}. A group is a named collection +// of principals; a role is a named permission bundle (its permissions are +// granted with CreateACLRule using PrincipalType="role", PrincipalID=). +// Membership/assignment edges are resolved transitively and combined additively +// at evaluation time. +// ============================================================================= + +// CreateGroupOptions configures CreateGroup. +type CreateGroupOptions struct { + AdminTimeoutOption + Name string // Required. + Description string + CreatedBy string + Metadata map[string]string +} + +// CreateGroup creates a named group. +func (a *AdminClient) CreateGroup(ctx context.Context, opts CreateGroupOptions) (*ACLResponse, error) { + return a.base.ACL().SendOpSync(ctx, &pb.ACLOperation{ + Op: pb.ACLOperation_CREATE_GROUP, + GroupRequest: &pb.ACLGroupRequest{ + Name: opts.Name, + Description: opts.Description, + CreatedBy: opts.CreatedBy, + Metadata: opts.Metadata, + }, + }, opts.Timeout) +} + +// NameOptions configures operations that take a single group/role name. +type NameOptions struct { + AdminTimeoutOption + Name string // Required. +} + +// GetGroup fetches a group by name. +func (a *AdminClient) GetGroup(ctx context.Context, opts NameOptions) (*ACLResponse, error) { + return a.base.ACL().SendOpSync(ctx, &pb.ACLOperation{Op: pb.ACLOperation_GET_GROUP, Name: opts.Name}, opts.Timeout) +} + +// DeleteGroup deletes a group by name. +func (a *AdminClient) DeleteGroup(ctx context.Context, opts NameOptions) (*ACLResponse, error) { + return a.base.ACL().SendOpSync(ctx, &pb.ACLOperation{Op: pb.ACLOperation_DELETE_GROUP, Name: opts.Name}, opts.Timeout) +} + +// ListGroups lists all groups. +func (a *AdminClient) ListGroups(ctx context.Context, opts AdminTimeoutOption) (*ACLResponse, error) { + return a.base.ACL().SendOpSync(ctx, &pb.ACLOperation{Op: pb.ACLOperation_LIST_GROUPS}, opts.Timeout) +} + +// AddGroupMemberOptions configures AddGroupMember. +type AddGroupMemberOptions struct { + AdminTimeoutOption + GroupName string // Required. + MemberType string // principal type or "group" (nesting). Required. + MemberID string // Required. + GrantedBy string + ExpiresAt int64 // Unix seconds, 0 = no expiry. +} + +// AddGroupMember adds (or refreshes) a member of a group. +func (a *AdminClient) AddGroupMember(ctx context.Context, opts AddGroupMemberOptions) (*ACLResponse, error) { + return a.base.ACL().SendOpSync(ctx, &pb.ACLOperation{ + Op: pb.ACLOperation_ADD_GROUP_MEMBER, + Name: opts.GroupName, + MemberRequest: &pb.ACLGroupMemberRequest{ + MemberType: opts.MemberType, + MemberId: opts.MemberID, + GrantedBy: opts.GrantedBy, + ExpiresAt: opts.ExpiresAt, + }, + }, opts.Timeout) +} + +// PrincipalEdgeOptions configures member/assignee removal and principal lookups. +type PrincipalEdgeOptions struct { + AdminTimeoutOption + Name string // group/role name (for remove/unassign). Empty for principal lookups. + PrincipalType string // Required. + PrincipalID string // Required. +} + +// RemoveGroupMember removes a member from a group. +func (a *AdminClient) RemoveGroupMember(ctx context.Context, opts PrincipalEdgeOptions) (*ACLResponse, error) { + return a.base.ACL().SendOpSync(ctx, &pb.ACLOperation{ + Op: pb.ACLOperation_REMOVE_GROUP_MEMBER, + Name: opts.Name, + Principal: &pb.PrincipalRef{PrincipalType: opts.PrincipalType, PrincipalId: opts.PrincipalID}, + }, opts.Timeout) +} + +// ListGroupMembers lists the direct members of a group. +func (a *AdminClient) ListGroupMembers(ctx context.Context, opts NameOptions) (*ACLResponse, error) { + return a.base.ACL().SendOpSync(ctx, &pb.ACLOperation{Op: pb.ACLOperation_LIST_GROUP_MEMBERS, Name: opts.Name}, opts.Timeout) +} + +// CreateRoleOptions configures CreateRole. +type CreateRoleOptions struct { + AdminTimeoutOption + Name string // Required. + Description string + CreatedBy string + Metadata map[string]string +} + +// CreateRole creates a named role. Grant its permissions with CreateACLRule +// using PrincipalType="role", PrincipalID=. +func (a *AdminClient) CreateRole(ctx context.Context, opts CreateRoleOptions) (*ACLResponse, error) { + return a.base.ACL().SendOpSync(ctx, &pb.ACLOperation{ + Op: pb.ACLOperation_CREATE_ROLE, + RoleRequest: &pb.ACLRoleRequest{ + Name: opts.Name, + Description: opts.Description, + CreatedBy: opts.CreatedBy, + Metadata: opts.Metadata, + }, + }, opts.Timeout) +} + +// GetRole fetches a role by name. +func (a *AdminClient) GetRole(ctx context.Context, opts NameOptions) (*ACLResponse, error) { + return a.base.ACL().SendOpSync(ctx, &pb.ACLOperation{Op: pb.ACLOperation_GET_ROLE, Name: opts.Name}, opts.Timeout) +} + +// DeleteRole deletes a role (and its permission rules) by name. +func (a *AdminClient) DeleteRole(ctx context.Context, opts NameOptions) (*ACLResponse, error) { + return a.base.ACL().SendOpSync(ctx, &pb.ACLOperation{Op: pb.ACLOperation_DELETE_ROLE, Name: opts.Name}, opts.Timeout) +} + +// ListRoles lists all roles. +func (a *AdminClient) ListRoles(ctx context.Context, opts AdminTimeoutOption) (*ACLResponse, error) { + return a.base.ACL().SendOpSync(ctx, &pb.ACLOperation{Op: pb.ACLOperation_LIST_ROLES}, opts.Timeout) +} + +// AssignRoleOptions configures AssignRole. +type AssignRoleOptions struct { + AdminTimeoutOption + RoleName string // Required. + AssigneeType string // principal type or "group". Required. + AssigneeID string // Required. + GrantedBy string + ExpiresAt int64 // Unix seconds, 0 = no expiry. +} + +// AssignRole assigns (or refreshes) a role to a principal or group. +func (a *AdminClient) AssignRole(ctx context.Context, opts AssignRoleOptions) (*ACLResponse, error) { + return a.base.ACL().SendOpSync(ctx, &pb.ACLOperation{ + Op: pb.ACLOperation_ASSIGN_ROLE, + Name: opts.RoleName, + AssignmentRequest: &pb.ACLRoleAssignmentRequest{ + AssigneeType: opts.AssigneeType, + AssigneeId: opts.AssigneeID, + GrantedBy: opts.GrantedBy, + ExpiresAt: opts.ExpiresAt, + }, + }, opts.Timeout) +} + +// UnassignRole removes a role assignment. +func (a *AdminClient) UnassignRole(ctx context.Context, opts PrincipalEdgeOptions) (*ACLResponse, error) { + return a.base.ACL().SendOpSync(ctx, &pb.ACLOperation{ + Op: pb.ACLOperation_UNASSIGN_ROLE, + Name: opts.Name, + Principal: &pb.PrincipalRef{PrincipalType: opts.PrincipalType, PrincipalId: opts.PrincipalID}, + }, opts.Timeout) +} + +// ListRoleAssignments lists the direct assignees of a role. +func (a *AdminClient) ListRoleAssignments(ctx context.Context, opts NameOptions) (*ACLResponse, error) { + return a.base.ACL().SendOpSync(ctx, &pb.ACLOperation{Op: pb.ACLOperation_LIST_ROLE_ASSIGNMENTS, Name: opts.Name}, opts.Timeout) +} + +// ListPrincipalGroups lists the groups a principal is a direct member of. +func (a *AdminClient) ListPrincipalGroups(ctx context.Context, opts PrincipalEdgeOptions) (*ACLResponse, error) { + return a.base.ACL().SendOpSync(ctx, &pb.ACLOperation{ + Op: pb.ACLOperation_LIST_PRINCIPAL_GROUPS, + Principal: &pb.PrincipalRef{PrincipalType: opts.PrincipalType, PrincipalId: opts.PrincipalID}, + }, opts.Timeout) +} + +// ListPrincipalRoles lists the roles directly assigned to a principal. +func (a *AdminClient) ListPrincipalRoles(ctx context.Context, opts PrincipalEdgeOptions) (*ACLResponse, error) { + return a.base.ACL().SendOpSync(ctx, &pb.ACLOperation{ + Op: pb.ACLOperation_LIST_PRINCIPAL_ROLES, + Principal: &pb.PrincipalRef{PrincipalType: opts.PrincipalType, PrincipalId: opts.PrincipalID}, + }, opts.Timeout) +} + +// ExplainAccessOptions configures ExplainAccess. +type ExplainAccessOptions struct { + AdminTimeoutOption + PrincipalType string // canonical principal_type, e.g. "user". Required. + PrincipalID string // canonical principal_id. Required. + ResourceType string // Required. + ResourceID string // Required. + RequiredLevel int32 // threshold the decision is compared against (0 = NONE). +} + +// ExplainAccess explains how a principal's effective access to a resource is +// decided: the resolved subject set (self + transitive groups/roles), every +// rule that matched, and the resulting decision. It does not gate access, but +// the gateway records an "explain_access" audit event attributing the call to +// the connected principal. Read the result from ACLResponse.Explanation. +func (a *AdminClient) ExplainAccess(ctx context.Context, opts ExplainAccessOptions) (*ACLResponse, error) { + return a.base.ACL().SendOpSync(ctx, &pb.ACLOperation{ + Op: pb.ACLOperation_EXPLAIN_ACCESS, + Principal: &pb.PrincipalRef{PrincipalType: opts.PrincipalType, PrincipalId: opts.PrincipalID}, + ResourceType: opts.ResourceType, + ResourceId: opts.ResourceID, + RequiredLevel: opts.RequiredLevel, + }, opts.Timeout) +} diff --git a/sdk/go/aether/agent.go b/sdk/go/aether/agent.go index 0cd662a..749d715 100644 --- a/sdk/go/aether/agent.go +++ b/sdk/go/aether/agent.go @@ -436,6 +436,7 @@ func (c *AgentClient) CreateTask(opts CreateTaskOptions) error { LaunchParamOverrides: opts.LaunchParamOverrides, Metadata: opts.Metadata, Payload: opts.Payload, + Priority: opts.Priority, }, }, } diff --git a/sdk/go/aether/bridge.go b/sdk/go/aether/bridge.go index 6b04525..a13d273 100644 --- a/sdk/go/aether/bridge.go +++ b/sdk/go/aether/bridge.go @@ -227,6 +227,22 @@ func (c *BridgeClient) SendToUserWorkspaceWithType(userID, workspace string, pay return c.sendMessage(topic, payload, msgType) } +// SendToUserBroadcast sends a message to every one of a user's windows, +// regardless of which workspace each window is viewing (topic uu::{user_id}). +// +// This is the workspace-agnostic, non-progress channel for platform→user +// notifications. Uses OPAQUE message type by default; use +// SendToUserBroadcastWithType for other types. +func (c *BridgeClient) SendToUserBroadcast(userID string, payload []byte) error { + return c.SendToUserBroadcastWithType(userID, payload, pb.MessageType_OPAQUE) +} + +// SendToUserBroadcastWithType sends a user-broadcast message with a custom message type. +func (c *BridgeClient) SendToUserBroadcastWithType(userID string, payload []byte, msgType pb.MessageType) error { + topic := UserBroadcastTopic(userID) + return c.sendMessage(topic, payload, msgType) +} + // ============================================================================= // Broadcast Helpers // ============================================================================= diff --git a/sdk/go/aether/client.go b/sdk/go/aether/client.go index 99fa711..c321c1e 100644 --- a/sdk/go/aether/client.go +++ b/sdk/go/aether/client.go @@ -25,6 +25,7 @@ import ( "google.golang.org/grpc/credentials" "google.golang.org/grpc/credentials/insecure" "google.golang.org/grpc/keepalive" + "google.golang.org/grpc/metadata" ) // ============================================================================= @@ -45,6 +46,13 @@ type BaseClient struct { tlsConfig *TLSConfig creds map[string]string + // streamMetadata, when non-empty, is attached to the Connect stream's + // outgoing gRPC context as transport-level headers (via + // metadata.NewOutgoingContext) on every establishStream — including after + // a reconnect. Readable server-side via metadata.FromIncomingContext. + // nil/empty preserves the pre-existing behavior (no extra headers). + streamMetadata map[string]string + // preDialedConn, when non-nil, takes precedence over serverAddr/tlsConfig. // Used by embedded callers (e.g. AetherLite's workflow engine) that // already have a *grpc.ClientConn pointing at an in-process bufconn- @@ -98,22 +106,35 @@ type BaseClient struct { checkpointResponseQueue chan *CheckpointResponse // Pending request maps for request_id-based correlation - pendingKVRequests pendingRequests[*KVResponse] - pendingCheckpointRequests pendingRequests[*CheckpointResponse] - pendingCreateTaskRequests pendingRequests[*CreateTaskResponse] - pendingTaskQueryRequests pendingRequests[*TaskQueryResponse] - pendingTaskOpRequests pendingRequests[*TaskOperationResponse] - pendingWorkflowRequests pendingRequests[*WorkflowResponse] - pendingWorkspaceRequests pendingRequests[*WorkspaceResponse] - pendingAgentRequests pendingRequests[*AgentResponse] - pendingACLRequests pendingRequests[*ACLResponse] - pendingTokenRequests pendingRequests[*TokenResponse] - pendingAuthorityGrantRequests pendingRequests[*pb.AuthorityGrantResponse] - pendingAdminRequests pendingRequests[*AdminResponse] - pendingSessionRequests pendingRequests[*SessionOperationResponse] - pendingAuditSubmitRequests pendingRequests[*pb.SubmitAuditEventResponse] - pendingAuditQueryRequests pendingRequests[*pb.AuditQueryResponse] - requestIDCounter atomic.Uint64 + pendingKVRequests pendingRequests[*KVResponse] + pendingCheckpointRequests pendingRequests[*CheckpointResponse] + pendingCreateTaskRequests pendingRequests[*CreateTaskResponse] + pendingTaskQueryRequests pendingRequests[*TaskQueryResponse] + pendingTaskOpRequests pendingRequests[*TaskOperationResponse] + pendingWorkflowRequests pendingRequests[*WorkflowResponse] + pendingWorkspaceRequests pendingRequests[*WorkspaceResponse] + pendingAgentRequests pendingRequests[*AgentResponse] + pendingACLRequests pendingRequests[*ACLResponse] + pendingTokenRequests pendingRequests[*TokenResponse] + pendingAuthorityGrantRequests pendingRequests[*pb.AuthorityGrantResponse] + pendingConnectionStatusRequests pendingRequests[*pb.ConnectionStatusResponse] + pendingAdminRequests pendingRequests[*AdminResponse] + pendingSessionRequests pendingRequests[*SessionOperationResponse] + pendingAuditSubmitRequests pendingRequests[*pb.SubmitAuditEventResponse] + pendingAuditQueryRequests pendingRequests[*pb.AuditQueryResponse] + requestIDCounter atomic.Uint64 + + // rawDownstreamTap, when non-nil, is invoked for every downstream + // message before typed dispatch. Returning true means the tap claimed + // the message and the SDK skips its own typed handling. This exists for + // the proxy-sidecar shared-runtime relay: a sandbox session's KV / task / + // workspace ops are relayed upstream as raw envelopes through this + // client, so the gateway's correlated responses arrive here with no + // pending request to resolve (the in-sandbox SDK owns the correlation). + // The tap lets the relay route those responses back to the originating + // session by request_id. Set once before the connection loop starts; not + // mutated concurrently. + rawDownstreamTap func(*pb.DownstreamMessage) bool // Per-client inflight registries for proxy requests and tunnels. These // must be per-BaseClient (not package-globals) because each BaseClient's @@ -130,26 +151,28 @@ type BaseClient struct { authorityCaches []*AuthorityGrantCache // Cached KV, Checkpoint, and Workflow helpers (for sync mutex to work across calls) - kvOnce sync.Once - kvInstance *KV - cpOnce sync.Once - cpInstance *Checkpoint - workflowOnce sync.Once - workflowInstance *WorkflowOps - workspaceOnce sync.Once - workspaceInstance *WorkspaceOps - agentOnce sync.Once - agentInstance *AgentOps - aclOnce sync.Once - aclInstance *ACLOps - tokenOnce sync.Once - tokenInstance *TokenOps - authorityOnce sync.Once - authorityInstance *AuthorityGrantOps - adminOnce sync.Once - adminInstance *AdminOps - sessionOnce sync.Once - sessionInstance *SessionOps + kvOnce sync.Once + kvInstance *KV + cpOnce sync.Once + cpInstance *Checkpoint + workflowOnce sync.Once + workflowInstance *WorkflowOps + workspaceOnce sync.Once + workspaceInstance *WorkspaceOps + agentOnce sync.Once + agentInstance *AgentOps + aclOnce sync.Once + aclInstance *ACLOps + tokenOnce sync.Once + tokenInstance *TokenOps + authorityOnce sync.Once + authorityInstance *AuthorityGrantOps + adminOnce sync.Once + adminInstance *AdminOps + sessionOnce sync.Once + sessionInstance *SessionOps + connectionOnce sync.Once + connectionInstance *ConnectionOps // InitConnection message builder (set by specific client types) initMsgBuilder func() *pb.InitConnection @@ -180,6 +203,11 @@ type BaseClientConfig struct { // Keys and values are passed to the server as metadata. Credentials map[string]string + // Metadata, when non-empty, is attached to the Connect stream's outgoing + // gRPC context as transport-level headers. Readable server-side via + // metadata.FromIncomingContext. nil/empty = no extra headers. + Metadata map[string]string + // QueueSize is the size of the outgoing message queue. // Default: 100. QueueSize int @@ -255,6 +283,7 @@ func NewBaseClient(cfg BaseClientConfig) (*BaseClient, error) { options: connOpts, tlsConfig: cfg.TLS, creds: cfg.Credentials, + streamMetadata: cfg.Metadata, queueSize: cfg.QueueSize, sendSem: sendSem, sendCh: make(chan *pb.UpstreamMessage, cfg.QueueSize), @@ -462,8 +491,17 @@ func (c *BaseClient) establishStream(ctx context.Context) error { // Create a context for the stream c.streamCtx, c.streamCancel = context.WithCancel(ctx) + // Attach any caller-supplied stream metadata as transport-level headers. + // These are readable server-side via metadata.FromIncomingContext before + // the first frame is processed (e.g. an aggregator pairing by tenant). When + // streamMetadata is empty this is a no-op, preserving prior behavior. + streamCtx := context.Context(c.streamCtx) + if len(c.streamMetadata) > 0 { + streamCtx = metadata.NewOutgoingContext(streamCtx, metadata.New(c.streamMetadata)) + } + // Create the stream - stream, err := c.client.Connect(c.streamCtx) + stream, err := c.client.Connect(streamCtx) if err != nil { return FromGRPCError(err) } @@ -616,6 +654,18 @@ func (c *BaseClient) OnMessage(handler MessageHandler) { c.handlers.OnMessage = handler } +// SetRawDownstreamTap installs a tap invoked for every downstream message +// before typed dispatch. Returning true claims the message — the SDK then +// skips its own typed handling for it. Passing nil clears the tap. +// +// Used by the proxy-sidecar shared-runtime relay to route responses for ops +// a sandbox session issued (which this client has no pending correlation +// state for) back to that session. Must be set before the connection loop +// starts; it is read on the receive loop without locking. +func (c *BaseClient) SetRawDownstreamTap(tap func(*pb.DownstreamMessage) bool) { + c.rawDownstreamTap = tap +} + // OnConfig registers a handler for configuration snapshots. func (c *BaseClient) OnConfig(handler ConfigHandler) { c.handlers.OnConfig = handler @@ -921,8 +971,20 @@ func messageTypeToProto(mt MessageType) pb.MessageType { // SendWithOptions sends a message using the provided SendMessageOptions. // This is the options-based counterpart to the typed Send methods on each client type. func (c *BaseClient) SendWithOptions(opts SendMessageOptions) error { - msgType := messageTypeToProto(opts.MessageType) - return c.sendMessage(opts.TargetTopic, opts.Payload, msgType) + send := &pb.SendMessage{ + TargetTopic: opts.TargetTopic, + Payload: opts.Payload, + MessageType: messageTypeToProto(opts.MessageType), + } + // Explicit OBO: when set, the gateway resolves it and stamps the subject + // onto the delivered envelope (MessageEnvelope.on_behalf_subject). A bare + // send never assumes an OBO context. + if opts.Authorization != nil { + send.Authorization = opts.Authorization + } + return c.Send(&pb.UpstreamMessage{ + Payload: &pb.UpstreamMessage_Send{Send: send}, + }) } // sendMessage sends a message to a target topic. @@ -1765,6 +1827,13 @@ func (c *BaseClient) dispatchResponse(ctx context.Context, response *pb.Downstre c.ConfirmConnection() } + // Raw downstream tap: lets a relay claim responses for ops it forwarded + // on behalf of a sandbox session. When the tap claims the message we skip + // typed dispatch — this client has no pending correlation state for it. + if tap := c.rawDownstreamTap; tap != nil && tap(response) { + return nil + } + // Dispatch based on payload type switch payload := response.GetPayload().(type) { case *pb.DownstreamMessage_Msg: @@ -1824,6 +1893,9 @@ func (c *BaseClient) dispatchResponse(ctx context.Context, response *pb.Downstre case *pb.DownstreamMessage_SessionResponse: return c.handleSessionResponse(ctx, payload.SessionResponse) + case *pb.DownstreamMessage_ConnectionStatusResponse: + return c.handleConnectionStatusResponse(ctx, payload.ConnectionStatusResponse) + case *pb.DownstreamMessage_AuthorityGrant: return c.handleAuthorityGrantResponse(ctx, payload.AuthorityGrant) @@ -1907,10 +1979,11 @@ func (c *BaseClient) dispatchResponse(ctx context.Context, response *pb.Downstre func (c *BaseClient) handleIncomingMessage(ctx context.Context, msg *pb.IncomingMessage) error { // Convert to high-level Message type message := &Message{ - SourceTopic: msg.GetSourceTopic(), - Payload: msg.GetPayload(), - MessageType: msg.GetMessageType(), - ReceivedAt: time.Now(), + SourceTopic: msg.GetSourceTopic(), + Payload: msg.GetPayload(), + MessageType: msg.GetMessageType(), + OnBehalfSubject: msg.GetOnBehalfSubject(), + ReceivedAt: time.Now(), } // Dispatch to generic message handler @@ -2059,6 +2132,8 @@ func (c *BaseClient) handleKVResponse(ctx context.Context, kv *pb.KVResponse) er RequestId: kv.GetRequestId(), CounterValue: kv.GetCounterValue(), Applied: kv.GetApplied(), + NextCursor: kv.GetNextCursor(), + HasMore: kv.GetHasMore(), } // If response has a request_id, try to resolve a pending correlated request first @@ -2088,6 +2163,18 @@ func (c *BaseClient) handleKVResponse(ctx context.Context, kv *pb.KVResponse) er } // handleTaskAssignment processes a task assignment from the server. +// +// The user-registered OnTaskAssignment handler is invoked in its own +// goroutine rather than synchronously on the receive loop. Task workers +// almost always need to make gateway round-trips during delivery +// (CompleteTask/FailTask at a minimum, and frequently KVGetSync to +// resolve resources); the response to those round-trips can ONLY arrive +// via this same receive loop, so a synchronous dispatch would deadlock +// the worker on its own KVResponse until the per-op timeout fires. +// +// Errors from the handler are logged via OnError if registered; panics +// are recovered and logged. This call returns nil immediately so the +// receive loop can continue processing inbound frames. func (c *BaseClient) handleTaskAssignment(ctx context.Context, ta *pb.TaskAssignment) error { if c.handlers.OnTaskAssignment == nil { return nil @@ -2114,7 +2201,25 @@ func (c *BaseClient) handleTaskAssignment(ctx context.Context, ta *pb.TaskAssign assignment.AssignedAt = time.Now() } - return c.handlers.OnTaskAssignment(ctx, assignment) + go func() { + defer func() { + if r := recover(); r != nil { + if c.handlers.OnError != nil { + _ = c.handlers.OnError(ctx, &ErrorInfo{ + Code: "TASK_ASSIGNMENT_HANDLER_PANIC", + Message: fmt.Sprintf("OnTaskAssignment handler panic: %v", r), + }) + } + } + }() + if err := c.handlers.OnTaskAssignment(ctx, assignment); err != nil && c.handlers.OnError != nil { + _ = c.handlers.OnError(ctx, &ErrorInfo{ + Code: "TASK_ASSIGNMENT_HANDLER_ERROR", + Message: err.Error(), + }) + } + }() + return nil } // handleConnectionAck processes the connection acknowledgment from the server. @@ -2298,6 +2403,9 @@ func (c *BaseClient) CreateTask(taskType, workspace string, opts CreateTaskOptio LaunchParamOverrides: opts.LaunchParamOverrides, Metadata: opts.Metadata, Payload: opts.Payload, + RetryPolicy: opts.RetryPolicy, + Priority: opts.Priority, + Authorization: opts.Authorization, } return c.Send(&pb.UpstreamMessage{ Payload: &pb.UpstreamMessage_CreateTask{CreateTask: req}, @@ -2327,6 +2435,9 @@ func (c *BaseClient) CreateTaskSync(ctx context.Context, taskType, workspace str LaunchParamOverrides: opts.LaunchParamOverrides, Metadata: opts.Metadata, Payload: opts.Payload, + RetryPolicy: opts.RetryPolicy, + Priority: opts.Priority, + Authorization: opts.Authorization, RequestId: requestID, } if err := c.Send(&pb.UpstreamMessage{ @@ -2750,6 +2861,17 @@ func (c *BaseClient) FailTask(ctx context.Context, taskID, reason string, timeou return c.doTaskOperation(ctx, pb.TaskOperation_FAIL, taskID, reason, timeout) } +// ClaimTask sends a task CLAIM operation and returns the response synchronously. +// CLAIM transitions an assigned/pending task to RUNNING; the caller must be the +// task's assignee, creator, OBO subject, or a workspace admin (gateway +// authorizeTaskOp). For a SELF_ASSIGN task the creator is also the assignee, so +// the creating client may CLAIM its own task — which is what drives the +// gateway's "running" lifecycle notice and (for subject-participating tasks) +// the per-user subject auto-subscribe. +func (c *BaseClient) ClaimTask(ctx context.Context, taskID string, timeout time.Duration) (*TaskOperationResponse, error) { + return c.doTaskOperation(ctx, pb.TaskOperation_CLAIM, taskID, "", timeout) +} + // ============================================================================= // Proto Conversion Helpers // ============================================================================= @@ -2881,9 +3003,86 @@ func protoACLResponseToSDK(resp *pb.ACLResponse) *ACLResponse { Message: cr.GetMessage(), } } + if g := resp.GetGroup(); g != nil { + r.Group = protoACLGroupInfoToSDK(g) + } + for _, g := range resp.GetGroups() { + r.Groups = append(r.Groups, protoACLGroupInfoToSDK(g)) + } + if role := resp.GetRole(); role != nil { + r.Role = protoACLRoleInfoToSDK(role) + } + for _, role := range resp.GetRoles() { + r.Roles = append(r.Roles, protoACLRoleInfoToSDK(role)) + } + for _, m := range resp.GetGroupMembers() { + r.GroupMembers = append(r.GroupMembers, &ACLGroupMemberInfo{ + GroupName: m.GetGroupName(), + MemberType: m.GetMemberType(), + MemberID: m.GetMemberId(), + GrantedBy: m.GetGrantedBy(), + GrantedAt: m.GetGrantedAt(), + ExpiresAt: m.GetExpiresAt(), + }) + } + for _, a := range resp.GetRoleAssignments() { + r.RoleAssignments = append(r.RoleAssignments, &ACLRoleAssignmentInfo{ + RoleName: a.GetRoleName(), + AssigneeType: a.GetAssigneeType(), + AssigneeID: a.GetAssigneeId(), + GrantedBy: a.GetGrantedBy(), + GrantedAt: a.GetGrantedAt(), + ExpiresAt: a.GetExpiresAt(), + }) + } + if e := resp.GetExplanation(); e != nil { + exp := &ACLAccessExplanationInfo{ + Principal: e.GetPrincipal(), + Subjects: e.GetSubjects(), + Allowed: e.GetAllowed(), + Decision: e.GetDecision(), + EffectiveLevel: e.GetEffectiveAccessLevel(), + FallbackApplied: e.GetFallbackApplied(), + Reason: e.GetReason(), + } + for _, c := range e.GetContributions() { + exp.Contributions = append(exp.Contributions, &ACLAccessContributionInfo{ + Subject: c.GetSubject(), + RuleID: c.GetRuleId(), + AccessLevel: c.GetAccessLevel(), + Resource: c.GetResource(), + Expired: c.GetExpired(), + }) + } + r.Explanation = exp + } return r } +// protoACLGroupInfoToSDK converts a protobuf ACLGroupInfo to the SDK type. +func protoACLGroupInfoToSDK(g *pb.ACLGroupInfo) *ACLGroupInfo { + return &ACLGroupInfo{ + GroupID: g.GetGroupId(), + GroupName: g.GetGroupName(), + Description: g.GetDescription(), + CreatedBy: g.GetCreatedBy(), + CreatedAt: g.GetCreatedAt(), + Metadata: g.GetMetadata(), + } +} + +// protoACLRoleInfoToSDK converts a protobuf ACLRoleInfo to the SDK type. +func protoACLRoleInfoToSDK(role *pb.ACLRoleInfo) *ACLRoleInfo { + return &ACLRoleInfo{ + RoleID: role.GetRoleId(), + RoleName: role.GetRoleName(), + Description: role.GetDescription(), + CreatedBy: role.GetCreatedBy(), + CreatedAt: role.GetCreatedAt(), + Metadata: role.GetMetadata(), + } +} + // protoACLRuleInfoToSDK converts a protobuf ACLRuleInfo to the SDK type. func protoACLRuleInfoToSDK(rule *pb.ACLRuleInfo) *ACLRuleInfo { return &ACLRuleInfo{ diff --git a/sdk/go/aether/client_test.go b/sdk/go/aether/client_test.go index 8749bab..c5d2f31 100644 --- a/sdk/go/aether/client_test.go +++ b/sdk/go/aether/client_test.go @@ -451,6 +451,66 @@ func TestBaseClient_Send_Success(t *testing.T) { } } +// TestSendWithOptions_ThreadsAuthorization verifies that an explicit OBO +// AuthorizationContext on SendMessageOptions is threaded onto the wire +// SendMessage, and that a bare send carries no authorization (explicit by +// design — a send never assumes an OBO context). +func TestSendWithOptions_ThreadsAuthorization(t *testing.T) { + newRunningClient := func() *BaseClient { + c, err := NewBaseClient(BaseClientConfig{ServerAddr: TestServerAddr}) + if err != nil { + t.Fatalf("NewBaseClient() error = %v", err) + } + c.running.Store(true) + return c + } + dequeueSend := func(c *BaseClient) *pb.SendMessage { + select { + case m := <-c.RequestQueue(): + return m.GetSend() + default: + t.Fatal("message should be in the queue") + return nil + } + } + + // Explicit authorization is forwarded. + authz := &pb.AuthorizationContext{ + AuthorityMode: "on_behalf_of", + Subject: &pb.PrincipalRef{PrincipalType: "user", PrincipalId: "alice@example.com"}, + GrantId: "grant-123", + } + c := newRunningClient() + if err := c.SendWithOptions(SendMessageOptions{ + TargetTopic: "test.topic", + Payload: []byte("hi"), + MessageType: MessageTypeChat, + Authorization: authz, + }); err != nil { + t.Fatalf("SendWithOptions() error = %v", err) + } + send := dequeueSend(c) + if send.GetAuthorization() == nil { + t.Fatal("expected authorization to be threaded onto SendMessage") + } + if got := send.GetAuthorization().GetSubject().GetPrincipalId(); got != "alice@example.com" { + t.Errorf("authorization subject = %q, want alice@example.com", got) + } + + // Bare send (no authorization) stays nil. + c2 := newRunningClient() + if err := c2.SendWithOptions(SendMessageOptions{ + TargetTopic: "test.topic", + Payload: []byte("hi"), + MessageType: MessageTypeChat, + }); err != nil { + t.Fatalf("SendWithOptions() error = %v", err) + } + if send := dequeueSend(c2); send.GetAuthorization() != nil { + t.Error("bare send must not assume an OBO authorization") + } +} + // ============================================================================= // Connection Lifecycle Tests // ============================================================================= @@ -937,8 +997,26 @@ func TestBaseClient_DispatchResponse_TaskAssignment(t *testing.T) { t.Errorf("dispatchResponse() error = %v", err) } - if len(tracker.tasks) != 1 { - t.Errorf("Task assignment handler called %d times, want 1", len(tracker.tasks)) + // OnTaskAssignment is dispatched on its own goroutine — workers + // typically need to round-trip the gateway (CompleteTask, KVGetSync) + // during delivery and would deadlock the receive loop if invoked + // synchronously. Poll for handler completion under the tracker's + // mutex. + deadline := time.Now().Add(2 * time.Second) + for { + tracker.mu.Lock() + n := len(tracker.tasks) + tracker.mu.Unlock() + if n >= 1 || time.Now().After(deadline) { + break + } + time.Sleep(5 * time.Millisecond) + } + tracker.mu.Lock() + got := len(tracker.tasks) + tracker.mu.Unlock() + if got != 1 { + t.Errorf("Task assignment handler called %d times, want 1", got) } } diff --git a/sdk/go/aether/connection_ops.go b/sdk/go/aether/connection_ops.go new file mode 100644 index 0000000..daaa687 --- /dev/null +++ b/sdk/go/aether/connection_ops.go @@ -0,0 +1,134 @@ +// Package aether connection-status query op for the Go SDK. +// +// This file adds a thin BaseClient helper that mirrors the Python async +// client's connection_status() method. It asks the gateway whether a given +// principal currently holds a live session lock. +// +// Self-checks (principal == caller's identity) are trivially allowed by the +// gateway; cross-principal checks require the caller to hold a +// `capability/query_connections` READ grant. +// +// The op follows the same request_id-correlated SendOpSync pattern used by +// AuthorityGrantOps: the raw protobuf ConnectionStatusResponse is returned +// regardless of `ok`, so callers interpret `ok=false` (unknown principal, +// permission denied) and the embedded `error` themselves. `connected` is the +// authoritative presence boolean; `last_seen_at` is best-effort and may be 0. + +package aether + +import ( + "context" + "sync" + "time" + + pb "github.com/scitrera/aether/api/proto" +) + +// DefaultConnectionStatusTimeout is the default timeout for synchronous +// connection-status queries. Mirrors the Python client default (10s). +const DefaultConnectionStatusTimeout = 10 * time.Second + +// ConnectionOps provides connection-status queries on a BaseClient. +type ConnectionOps struct { + client *BaseClient + syncMu sync.Mutex // serializes synchronous connection-status queries +} + +// newConnectionOps creates a new ConnectionOps helper for a client. +func newConnectionOps(client *BaseClient) *ConnectionOps { + return &ConnectionOps{client: client} +} + +// Status queries the gateway for the live-connection status of a principal +// and waits for the correlated response. A zero timeout uses +// DefaultConnectionStatusTimeout. +// +// principalType is the REST-form principal kind (e.g. "service", "agent", +// "task"); principalID is the principal's ID. The raw +// ConnectionStatusResponse is returned regardless of `ok`. +func (o *ConnectionOps) Status(ctx context.Context, principalType, principalID string, timeout time.Duration) (*pb.ConnectionStatusResponse, error) { + o.syncMu.Lock() + defer o.syncMu.Unlock() + + if timeout == 0 { + timeout = DefaultConnectionStatusTimeout + } + + requestID := o.client.NextRequestID() + req := &pb.ConnectionStatusRequest{ + RequestId: requestID, + Principal: &pb.PrincipalRef{ + PrincipalType: principalType, + PrincipalId: principalID, + }, + } + + ch := o.client.RegisterPendingConnectionStatusRequest(requestID) + defer o.client.pendingConnectionStatusRequests.Delete(requestID) + + if err := o.client.Send(&pb.UpstreamMessage{ + Payload: &pb.UpstreamMessage_ConnectionStatusRequest{ConnectionStatusRequest: req}, + }); err != nil { + return nil, err + } + + timer := time.NewTimer(timeout) + defer timer.Stop() + + select { + case <-ctx.Done(): + return nil, NewTimeoutError("context canceled", timeout.Seconds()) + case <-timer.C: + return nil, NewTimeoutError("connection status query timed out", timeout.Seconds()) + case resp := <-ch: + return resp, nil + } +} + +// Connection returns the ConnectionOps helper for this client. +func (c *BaseClient) Connection() *ConnectionOps { + c.connectionOnce.Do(func() { + c.connectionInstance = newConnectionOps(c) + }) + return c.connectionInstance +} + +// ConnectionStatus is a convenience wrapper that queries the live-connection +// status of a principal using the default timeout. It mirrors the Python +// async client's connection_status(). The raw ConnectionStatusResponse is +// returned regardless of `ok`; callers must check `resp.GetOk()` and +// `resp.GetError()`. The authoritative presence signal is `resp.GetConnected()`. +func (c *BaseClient) ConnectionStatus(ctx context.Context, principalType, principalID string) (*pb.ConnectionStatusResponse, error) { + return c.Connection().Status(ctx, principalType, principalID, 0) +} + +// ============================================================================= +// Pending-request plumbing (called from client.go dispatchResponse) +// ============================================================================= + +// RegisterPendingConnectionStatusRequest registers a pending +// connection-status request channel keyed by request ID. +func (c *BaseClient) RegisterPendingConnectionStatusRequest(requestID string) chan *pb.ConnectionStatusResponse { + return c.pendingConnectionStatusRequests.Register(requestID) +} + +// ResolvePendingConnectionStatusRequest delivers a connection-status response +// to the pending channel keyed by request ID. Returns true if a pending +// request was found. +func (c *BaseClient) ResolvePendingConnectionStatusRequest(requestID string, resp *pb.ConnectionStatusResponse) bool { + return c.pendingConnectionStatusRequests.Resolve(requestID, resp) +} + +// handleConnectionStatusResponse processes a connection-status response from +// the server. Called from BaseClient.dispatchResponse for +// DownstreamMessage_ConnectionStatusResponse. +func (c *BaseClient) handleConnectionStatusResponse(_ context.Context, resp *pb.ConnectionStatusResponse) error { + if reqID := resp.GetRequestId(); reqID != "" { + if c.ResolvePendingConnectionStatusRequest(reqID, resp) { + return nil + } + } + // Fallback: deliver to first pending request (servers that omit request_id). + c.pendingConnectionStatusRequests.ResolveFirst(resp) + return nil +} diff --git a/sdk/go/aether/connection_ops_test.go b/sdk/go/aether/connection_ops_test.go new file mode 100644 index 0000000..38270b1 --- /dev/null +++ b/sdk/go/aether/connection_ops_test.go @@ -0,0 +1,127 @@ +package aether + +import ( + "context" + "testing" + "time" + + pb "github.com/scitrera/aether/api/proto" +) + +// newTestConnBaseClient builds a BaseClient put into the "running" state so +// Send queues messages instead of failing. +func newTestConnBaseClient(t *testing.T) *BaseClient { + t.Helper() + bc, err := NewBaseClient(BaseClientConfig{ServerAddr: TestServerAddr}) + if err != nil { + t.Fatalf("NewBaseClient() error = %v", err) + } + bc.running.Store(true) + return bc +} + +func TestBaseClient_ConnectionStatus_RoundTrip(t *testing.T) { + bc := newTestConnBaseClient(t) + + go func() { + time.Sleep(10 * time.Millisecond) + msg := <-bc.RequestQueue() + req := msg.GetConnectionStatusRequest() + if req == nil { + t.Errorf("expected ConnectionStatusRequest, got %T", msg.GetPayload()) + return + } + if req.GetPrincipal().GetPrincipalType() != "service" || + req.GetPrincipal().GetPrincipalId() != "sandbox-sidecar::sb-1" { + t.Errorf("unexpected principal: %+v", req.GetPrincipal()) + return + } + bc.pendingConnectionStatusRequests.Range(func(key, val any) bool { + ch := val.(chan *pb.ConnectionStatusResponse) + bc.pendingConnectionStatusRequests.Delete(key) + ch <- &pb.ConnectionStatusResponse{ + RequestId: req.GetRequestId(), + Ok: true, + Connected: true, + } + return false + }) + }() + + resp, err := bc.ConnectionStatus(context.Background(), "service", "sandbox-sidecar::sb-1") + if err != nil { + t.Fatalf("ConnectionStatus() error = %v", err) + } + if !resp.GetOk() || !resp.GetConnected() { + t.Errorf("got %+v, want Ok=true Connected=true", resp) + } +} + +func TestBaseClient_ConnectionStatus_Disconnected(t *testing.T) { + bc := newTestConnBaseClient(t) + + go func() { + time.Sleep(10 * time.Millisecond) + msg := <-bc.RequestQueue() + req := msg.GetConnectionStatusRequest() + if req == nil { + return + } + bc.pendingConnectionStatusRequests.Range(func(key, val any) bool { + ch := val.(chan *pb.ConnectionStatusResponse) + bc.pendingConnectionStatusRequests.Delete(key) + ch <- &pb.ConnectionStatusResponse{RequestId: req.GetRequestId(), Ok: true, Connected: false} + return false + }) + }() + + resp, err := bc.ConnectionStatus(context.Background(), "service", "sandbox-sidecar::sb-2") + if err != nil { + t.Fatalf("ConnectionStatus() error = %v", err) + } + if !resp.GetOk() || resp.GetConnected() { + t.Errorf("got %+v, want Ok=true Connected=false", resp) + } +} + +func TestBaseClient_ConnectionStatus_Timeout(t *testing.T) { + bc := newTestConnBaseClient(t) + + // No responder; expect a timeout error. + _, err := bc.Connection().Status(context.Background(), "service", "sandbox-sidecar::sb-3", 50*time.Millisecond) + if err == nil { + t.Fatal("ConnectionStatus should time out when no response arrives") + } +} + +// TestBaseClient_DispatchResponse_ConnectionStatus verifies dispatch wiring: +// an inbound ConnectionStatusResponse with a known request_id resolves the +// pending sync caller. +func TestBaseClient_DispatchResponse_ConnectionStatus(t *testing.T) { + bc := newTestConnBaseClient(t) + + requestID := bc.NextRequestID() + ch := bc.RegisterPendingConnectionStatusRequest(requestID) + + protoResp := &pb.DownstreamMessage{ + Payload: &pb.DownstreamMessage_ConnectionStatusResponse{ + ConnectionStatusResponse: &pb.ConnectionStatusResponse{ + RequestId: requestID, + Ok: true, + Connected: true, + }, + }, + } + if err := bc.dispatchResponse(context.Background(), protoResp); err != nil { + t.Errorf("dispatchResponse() error = %v", err) + } + + select { + case got := <-ch: + if !got.GetOk() || !got.GetConnected() { + t.Errorf("got %+v, want Ok=true Connected=true", got) + } + case <-time.After(time.Second): + t.Fatal("dispatchResponse did not resolve pending connection-status request") + } +} diff --git a/sdk/go/aether/coordkv.go b/sdk/go/aether/coordkv.go new file mode 100644 index 0000000..3d6d1ef --- /dev/null +++ b/sdk/go/aether/coordkv.go @@ -0,0 +1,117 @@ +package aether + +import ( + "context" + "errors" + "time" + + "github.com/scitrera/aether/sdk/go/coord" +) + +// CoordScope binds the coordination primitives to a KV scope and (where the +// scope requires them) a user/workspace, plus an optional per-operation gRPC +// timeout. The zero value targets the global scope with the default KV timeout. +// +// Coordination keys are ordinary KV keys: pick a scope that all participating +// replicas share. Cluster-wide locks typically use KVScopeWorkspace with a +// fixed workspace (or KVScopeGlobal for tenant-wide coordination). +type CoordScope struct { + Scope KVScope + UserID string + Workspace string + // OpTimeout bounds each underlying synchronous KV round-trip. Zero uses + // DefaultKVTimeout. + OpTimeout time.Duration +} + +func (cs CoordScope) timeout() time.Duration { + if cs.OpTimeout <= 0 { + return DefaultKVTimeout + } + return cs.OpTimeout +} + +// kvCoordBackend adapts *KV to coord.Locker and coord.Counter, binding a fixed +// scope/user/workspace. Each method maps 1:1 to a synchronous KV op. +type kvCoordBackend struct { + kv *KV + scope CoordScope +} + +var _ coord.Locker = (*kvCoordBackend)(nil) +var _ coord.Counter = (*kvCoordBackend)(nil) + +func (b *kvCoordBackend) TryAcquire(ctx context.Context, key, owner string, ttl time.Duration) (bool, error) { + return b.kv.SetNXSync(ctx, key, []byte(owner), b.scope.Scope, b.scope.UserID, b.scope.Workspace, ttl, b.scope.timeout()) +} + +func (b *kvCoordBackend) Refresh(ctx context.Context, key, owner string, ttl time.Duration) (bool, error) { + // Re-assert ownership and extend the lease: compare-and-set owner→owner. + return b.kv.CompareAndSetSync(ctx, key, []byte(owner), []byte(owner), b.scope.Scope, b.scope.UserID, b.scope.Workspace, ttl, b.scope.timeout()) +} + +func (b *kvCoordBackend) Release(ctx context.Context, key, owner string) (bool, error) { + return b.kv.CompareAndDeleteSync(ctx, key, []byte(owner), b.scope.Scope, b.scope.UserID, b.scope.Workspace, b.scope.timeout()) +} + +func (b *kvCoordBackend) Peek(ctx context.Context, key string) (string, error) { + resp, err := b.kv.GetSync(ctx, KVGetOptions{ + Key: key, + Scope: b.scope.Scope, + UserID: b.scope.UserID, + Workspace: b.scope.Workspace, + Timeout: b.scope.timeout(), + }) + if err != nil { + return "", err + } + if resp == nil || !resp.Success { + return "", nil + } + return string(resp.Value), nil +} + +func (b *kvCoordBackend) IncrementIf(ctx context.Context, key string, delta, ceiling int64) (int64, bool, error) { + return b.kv.IncrementIfSync(ctx, key, b.scope.Scope, b.scope.UserID, b.scope.Workspace, delta, ceiling, b.scope.timeout()) +} + +func (b *kvCoordBackend) DecrementIf(ctx context.Context, key string, delta, floor int64) (int64, bool, error) { + return b.kv.DecrementIfSync(ctx, key, b.scope.Scope, b.scope.UserID, b.scope.Workspace, delta, floor, b.scope.timeout()) +} + +// Locker returns a coord.Locker bound to scope, for use with coord.NewMutex / +// coord.NewLeaderElection / coord.NewOnce when finer control than the +// convenience constructors below is needed. +func (kv *KV) Locker(scope CoordScope) coord.Locker { + return &kvCoordBackend{kv: kv, scope: scope} +} + +// Counter returns a coord.Counter bound to scope, for use with +// coord.NewSemaphore. +func (kv *KV) Counter(scope CoordScope) coord.Counter { + return &kvCoordBackend{kv: kv, scope: scope} +} + +// NewMutex returns a distributed Mutex over key in the given scope. +func (kv *KV) NewMutex(key string, scope CoordScope, opts ...coord.MutexOption) *coord.Mutex { + return coord.NewMutex(kv.Locker(scope), key, opts...) +} + +// NewLeaderElection returns a LeaderElection over key in the given scope. Wire +// OnAcquire/OnLose then call Start. +func (kv *KV) NewLeaderElection(key string, scope CoordScope, opts ...coord.LeaderOption) *coord.LeaderElection { + return coord.NewLeaderElection(kv.Locker(scope), key, opts...) +} + +// NewSemaphore returns an N-permit Semaphore over key in the given scope. +func (kv *KV) NewSemaphore(key string, limit int64, scope CoordScope) (*coord.Semaphore, error) { + if kv == nil { + return nil, errors.New("aether: nil KV") + } + return coord.NewSemaphore(kv.Counter(scope), key, limit) +} + +// NewOnce returns a cluster-wide run-once guard over key in the given scope. +func (kv *KV) NewOnce(key string, scope CoordScope, opts ...coord.OnceOption) *coord.Once { + return coord.NewOnce(kv.Locker(scope), key, opts...) +} diff --git a/sdk/go/aether/handlers.go b/sdk/go/aether/handlers.go index ac8595e..5bf3b78 100644 --- a/sdk/go/aether/handlers.go +++ b/sdk/go/aether/handlers.go @@ -33,6 +33,13 @@ type Message struct { // MessageType is the type of the message (CHAT, CONTROL, TOOL_CALL, EVENT, METRIC). MessageType pb.MessageType + // OnBehalfSubject is the gateway-resolved on-behalf-of subject the message + // was sent for, when the sender supplied an OBO AuthorizationContext. + // Gateway-set and spoof-proof (like SourceTopic). Nil for direct (non-OBO) + // sends. Lets a recipient identify the user a message is sent for, distinct + // from the sending identity in SourceTopic. + OnBehalfSubject *pb.PrincipalRef + // ReceivedAt is the local time when the message was received. ReceivedAt time.Time } @@ -106,6 +113,13 @@ type KVResponse struct { // Applied is true iff an INCREMENT_IF/DECREMENT_IF mutation was applied. Applied bool + + // NextCursor (LIST) is an opaque token to pass as KVListOptions.Cursor for + // the next page; empty when iteration is complete. + NextCursor string + + // HasMore (LIST) is true when more matching keys remain beyond this page. + HasMore bool } // TaskAssignment represents a task assignment from the gateway. @@ -418,6 +432,78 @@ type ACLResponse struct { AuditEntries []*ACLAuditEntryInfo TotalAuditEntries int32 CleanupResult *ACLCleanupResult + + // Role/group results. + Group *ACLGroupInfo // GET_GROUP / CREATE_GROUP + Groups []*ACLGroupInfo // LIST_GROUPS + Role *ACLRoleInfo // GET_ROLE / CREATE_ROLE + Roles []*ACLRoleInfo // LIST_ROLES + GroupMembers []*ACLGroupMemberInfo // LIST_GROUP_MEMBERS / LIST_PRINCIPAL_GROUPS / ADD_GROUP_MEMBER + RoleAssignments []*ACLRoleAssignmentInfo // LIST_ROLE_ASSIGNMENTS / LIST_PRINCIPAL_ROLES / ASSIGN_ROLE + Explanation *ACLAccessExplanationInfo // EXPLAIN_ACCESS +} + +// ACLAccessContributionInfo is one rule that matched a principal or one of its +// groups/roles for an explained resource. +type ACLAccessContributionInfo struct { + Subject string + RuleID string + AccessLevel int32 + Resource string + Expired bool +} + +// ACLAccessExplanationInfo explains how a principal's effective access to a +// resource is decided: resolved subjects, matching rules, and the decision. +type ACLAccessExplanationInfo struct { + Principal string + Subjects []string + Contributions []*ACLAccessContributionInfo + Allowed bool + Decision string + EffectiveLevel int32 + FallbackApplied bool + Reason string +} + +// ACLGroupInfo describes a group definition. +type ACLGroupInfo struct { + GroupID string + GroupName string + Description string + CreatedBy string + CreatedAt int64 + Metadata map[string]string +} + +// ACLRoleInfo describes a role definition. +type ACLRoleInfo struct { + RoleID string + RoleName string + Description string + CreatedBy string + CreatedAt int64 + Metadata map[string]string +} + +// ACLGroupMemberInfo describes a single group-membership edge. +type ACLGroupMemberInfo struct { + GroupName string + MemberType string + MemberID string + GrantedBy string + GrantedAt int64 + ExpiresAt int64 +} + +// ACLRoleAssignmentInfo describes a single role-assignment edge. +type ACLRoleAssignmentInfo struct { + RoleName string + AssigneeType string + AssigneeID string + GrantedBy string + GrantedAt int64 + ExpiresAt int64 } // TokenInfo represents an API token. @@ -698,14 +784,33 @@ type ProgressHandler func(ctx context.Context, update *ProgressUpdate) error // TaskAssignmentHandler is called when a task assignment is received. // -// This is used by Orchestrators to receive task assignments that require -// starting new agent or task instances. +// Orchestrators use this to receive assignments that require starting +// new agent/task instances; service workers (e.g. webhookservice) use +// it to receive pool-dispatched work items and report the outcome via +// CompleteTask / FailTask. +// +// Dispatch model: the SDK invokes this handler on a dedicated goroutine +// rather than synchronously on the receive loop. That matters because +// worker handlers almost always need to round-trip the gateway during +// delivery — at minimum CompleteTask / FailTask, frequently KVGetSync +// to resolve state — and the responses can only arrive via the same +// receive loop. A synchronous handler that performed those round-trips +// would deadlock on its own response until the per-op timeout fires. +// +// Implications: +// - Handler return values are not awaited by the receive loop; a +// non-nil error is delivered to OnError (if registered) and +// otherwise dropped. +// - Panics inside the handler are recovered; OnError sees a synthetic +// ErrorInfo with code "TASK_ASSIGNMENT_HANDLER_PANIC". +// - Multiple assignments may be in-flight concurrently. Handlers must +// be safe to invoke from multiple goroutines. // // Example: // // orchestrator.OnTaskAssignment(func(ctx context.Context, task *aether.TaskAssignment) error { // log.Printf("Starting %s for task %s", task.TargetImplementation, task.TaskID) -// // Launch container/process based on task.LaunchParams +// // Launch container/process; safe to call CompleteTask / KVGetSync here. // return nil // }) type TaskAssignmentHandler func(ctx context.Context, task *TaskAssignment) error diff --git a/sdk/go/aether/kv.go b/sdk/go/aether/kv.go index 731ba5f..83f705a 100644 --- a/sdk/go/aether/kv.go +++ b/sdk/go/aether/kv.go @@ -105,17 +105,25 @@ func (kv *KV) Get(key string, scope KVScope, userID, workspace string) error { // GetWithRequestID retrieves a value from the KV store with a specific request ID for correlation. func (kv *KV) GetWithRequestID(key string, scope KVScope, userID, workspace, requestID string) error { + return kv.getWithReqIDAuth(key, scope, userID, workspace, requestID, nil) +} + +// getWithReqIDAuth is the internal GET builder carrying an optional +// on-behalf-of AuthorizationContext (proto field 9). GetWithRequestID passes +// nil (direct mode); GetSync forwards KVGetOptions.Authorization. +func (kv *KV) getWithReqIDAuth(key string, scope KVScope, userID, workspace, requestID string, auth *pb.AuthorizationContext) error { if scope == "" { scope = KVScopeGlobal } op := &pb.KVOperation{ - Op: pb.KVOperation_GET, - Scope: kvScopeToProto(scope), - Key: key, - UserId: userID, - Workspace: workspace, - RequestId: requestID, + Op: pb.KVOperation_GET, + Scope: kvScopeToProto(scope), + Key: key, + UserId: userID, + Workspace: workspace, + RequestId: requestID, + Authorization: auth, } return kv.client.Send(&pb.UpstreamMessage{ @@ -143,19 +151,27 @@ func (kv *KV) Put(key string, value []byte, scope KVScope, userID, workspace str // PutWithRequestID stores a value in the KV store with a specific request ID for correlation. func (kv *KV) PutWithRequestID(key string, value []byte, scope KVScope, userID, workspace string, ttl int64, requestID string) error { + return kv.putWithReqIDAuth(key, value, scope, userID, workspace, ttl, requestID, nil) +} + +// putWithReqIDAuth is the internal PUT builder carrying an optional +// on-behalf-of AuthorizationContext (proto field 9). PutWithRequestID passes +// nil (direct mode); PutSync forwards KVPutOptions.Authorization. +func (kv *KV) putWithReqIDAuth(key string, value []byte, scope KVScope, userID, workspace string, ttl int64, requestID string, auth *pb.AuthorizationContext) error { if scope == "" { scope = KVScopeGlobal } op := &pb.KVOperation{ - Op: pb.KVOperation_PUT, - Scope: kvScopeToProto(scope), - Key: key, - Value: value, - UserId: userID, - Workspace: workspace, - Ttl: ttl, - RequestId: requestID, + Op: pb.KVOperation_PUT, + Scope: kvScopeToProto(scope), + Key: key, + Value: value, + UserId: userID, + Workspace: workspace, + Ttl: ttl, + RequestId: requestID, + Authorization: auth, } return kv.client.Send(&pb.UpstreamMessage{ @@ -181,17 +197,34 @@ func (kv *KV) List(keyPrefix string, scope KVScope, userID, workspace string) er // ListWithRequestID lists keys with a specific request ID for correlation. func (kv *KV) ListWithRequestID(keyPrefix string, scope KVScope, userID, workspace, requestID string) error { + return kv.listWithOpts(keyPrefix, scope, userID, workspace, requestID, 0, "") +} + +// listWithOpts is the internal LIST builder carrying pagination (limit/cursor). +// ListWithRequestID passes the zero values (server default, first page); +// ListSync forwards KVListOptions.Limit/Cursor. +func (kv *KV) listWithOpts(keyPrefix string, scope KVScope, userID, workspace, requestID string, limit int32, cursor string) error { + return kv.listWithOptsAuth(keyPrefix, scope, userID, workspace, requestID, limit, cursor, nil) +} + +// listWithOptsAuth is the internal LIST builder carrying pagination plus an +// optional on-behalf-of AuthorizationContext (proto field 9). listWithOpts +// passes nil (direct mode); ListSync forwards KVListOptions.Authorization. +func (kv *KV) listWithOptsAuth(keyPrefix string, scope KVScope, userID, workspace, requestID string, limit int32, cursor string, auth *pb.AuthorizationContext) error { if scope == "" { scope = KVScopeGlobal } op := &pb.KVOperation{ - Op: pb.KVOperation_LIST, - Scope: kvScopeToProto(scope), - Key: keyPrefix, - UserId: userID, - Workspace: workspace, - RequestId: requestID, + Op: pb.KVOperation_LIST, + Scope: kvScopeToProto(scope), + Key: keyPrefix, + UserId: userID, + Workspace: workspace, + RequestId: requestID, + Limit: limit, + Cursor: cursor, + Authorization: auth, } return kv.client.Send(&pb.UpstreamMessage{ @@ -289,6 +322,46 @@ func (kv *KV) DecrementWithRequestID(key string, scope KVScope, userID, workspac }) } +// PurgeIdentitySync REMOVAL-ONLY purges every key in another principal's KV +// namespace (opts.TargetIdentity, opts.Scope), optionally filtered by +// opts.KeyPrefix, and waits for the response. Requires the +// capability/kv_purge_identity grant on THIS client's identity. The response +// carries only the deleted count (KVResponse.CounterValue) — never keys or +// values — so the capability cannot be used to read another principal's data. +func (kv *KV) PurgeIdentitySync(ctx context.Context, opts KVPurgeOptions) (*KVResponse, error) { + kv.syncMu.Lock() + defer kv.syncMu.Unlock() + + timeout := opts.Timeout + if timeout == 0 { + timeout = DefaultKVTimeout + } + + requestID := kv.client.NextRequestID() + ch := kv.client.RegisterPendingKVRequest(requestID) + defer kv.client.pendingKVRequests.Delete(requestID) + + scope := opts.Scope + if scope == "" { + scope = KVScopeGlobal + } + + op := &pb.KVOperation{ + Op: pb.KVOperation_PURGE_IDENTITY, + Scope: kvScopeToProto(scope), + Key: opts.KeyPrefix, + TargetIdentity: opts.TargetIdentity, + RequestId: requestID, + } + if err := kv.client.Send(&pb.UpstreamMessage{ + Payload: &pb.UpstreamMessage_KvOp{KvOp: op}, + }); err != nil { + return nil, err + } + + return kv.waitForCorrelatedResponse(ctx, ch, timeout) +} + // IncrementSync atomically increments a counter and waits for the response. func (kv *KV) IncrementSync(ctx context.Context, key string, scope KVScope, userID, workspace string, timeout time.Duration) (*KVResponse, error) { kv.syncMu.Lock() @@ -457,6 +530,201 @@ func (kv *KV) DecrementIfSync(ctx context.Context, key string, scope KVScope, us return resp.CounterValue, resp.Applied, nil } +// ============================================================================= +// Conditional-write Operations (coordination primitives) +// ============================================================================= +// +// SetNX / CompareAndSet / CompareAndDelete are the atomic building blocks for +// distributed coordination (see the coord package for Mutex / LeaderElection / +// Once helpers built on them). The lease TTL is carried over the wire in whole +// seconds, so sub-second lock leases are not representable — use seconds-scale +// TTLs (the coord defaults are 30s lease / 10s renew). + +// SetNXSync sets key=value only if the key is absent. Returns applied=true iff +// the value was written. ttl is the lease lifetime (truncated to whole +// seconds; 0 means no expiry). +func (kv *KV) SetNXSync(ctx context.Context, key string, value []byte, scope KVScope, userID, workspace string, ttl, timeout time.Duration) (bool, error) { + kv.syncMu.Lock() + defer kv.syncMu.Unlock() + + if timeout == 0 { + timeout = DefaultKVTimeout + } + if scope == "" { + scope = KVScopeGlobal + } + + requestID := kv.client.NextRequestID() + ch := kv.client.RegisterPendingKVRequest(requestID) + defer kv.client.pendingKVRequests.Delete(requestID) + + op := &pb.KVOperation{ + Op: pb.KVOperation_SET_NX, + Scope: kvScopeToProto(scope), + Key: key, + Value: value, + UserId: userID, + Workspace: workspace, + Ttl: int64(ttl.Seconds()), + RequestId: requestID, + } + if err := kv.client.Send(&pb.UpstreamMessage{Payload: &pb.UpstreamMessage_KvOp{KvOp: op}}); err != nil { + return false, err + } + resp, err := kv.waitForCorrelatedResponse(ctx, ch, timeout) + if err != nil { + return false, err + } + return resp.Applied, nil +} + +// CompareAndSetSync sets key=value only if the current stored value equals +// expected. Returns applied=true iff the swap was applied. ttl is the new lease +// lifetime (whole seconds; 0 means no expiry). +func (kv *KV) CompareAndSetSync(ctx context.Context, key string, expected, value []byte, scope KVScope, userID, workspace string, ttl, timeout time.Duration) (bool, error) { + kv.syncMu.Lock() + defer kv.syncMu.Unlock() + + if timeout == 0 { + timeout = DefaultKVTimeout + } + if scope == "" { + scope = KVScopeGlobal + } + + requestID := kv.client.NextRequestID() + ch := kv.client.RegisterPendingKVRequest(requestID) + defer kv.client.pendingKVRequests.Delete(requestID) + + op := &pb.KVOperation{ + Op: pb.KVOperation_COMPARE_AND_SET, + Scope: kvScopeToProto(scope), + Key: key, + ExpectedValue: expected, + Value: value, + UserId: userID, + Workspace: workspace, + Ttl: int64(ttl.Seconds()), + RequestId: requestID, + } + if err := kv.client.Send(&pb.UpstreamMessage{Payload: &pb.UpstreamMessage_KvOp{KvOp: op}}); err != nil { + return false, err + } + resp, err := kv.waitForCorrelatedResponse(ctx, ch, timeout) + if err != nil { + return false, err + } + return resp.Applied, nil +} + +// CompareAndDeleteSync deletes key only if the current stored value equals +// expected. Returns applied=true iff the delete was applied. +func (kv *KV) CompareAndDeleteSync(ctx context.Context, key string, expected []byte, scope KVScope, userID, workspace string, timeout time.Duration) (bool, error) { + kv.syncMu.Lock() + defer kv.syncMu.Unlock() + + if timeout == 0 { + timeout = DefaultKVTimeout + } + if scope == "" { + scope = KVScopeGlobal + } + + requestID := kv.client.NextRequestID() + ch := kv.client.RegisterPendingKVRequest(requestID) + defer kv.client.pendingKVRequests.Delete(requestID) + + op := &pb.KVOperation{ + Op: pb.KVOperation_COMPARE_AND_DELETE, + Scope: kvScopeToProto(scope), + Key: key, + ExpectedValue: expected, + UserId: userID, + Workspace: workspace, + RequestId: requestID, + } + if err := kv.client.Send(&pb.UpstreamMessage{Payload: &pb.UpstreamMessage_KvOp{KvOp: op}}); err != nil { + return false, err + } + resp, err := kv.waitForCorrelatedResponse(ctx, ch, timeout) + if err != nil { + return false, err + } + return resp.Applied, nil +} + +// SetAddSync adds member to the set at key, returning whether it was newly +// added and the set's cardinality after the add. ttl (re)sets expiry on a new +// member. Mirrors the SetAdd KV primitive. +func (kv *KV) SetAddSync(ctx context.Context, key string, member []byte, scope KVScope, userID, workspace string, ttl, timeout time.Duration) (bool, int64, error) { + kv.syncMu.Lock() + defer kv.syncMu.Unlock() + + if timeout == 0 { + timeout = DefaultKVTimeout + } + if scope == "" { + scope = KVScopeGlobal + } + + requestID := kv.client.NextRequestID() + ch := kv.client.RegisterPendingKVRequest(requestID) + defer kv.client.pendingKVRequests.Delete(requestID) + + op := &pb.KVOperation{ + Op: pb.KVOperation_SET_ADD, + Scope: kvScopeToProto(scope), + Key: key, + Value: member, + UserId: userID, + Workspace: workspace, + Ttl: int64(ttl.Seconds()), + RequestId: requestID, + } + if err := kv.client.Send(&pb.UpstreamMessage{Payload: &pb.UpstreamMessage_KvOp{KvOp: op}}); err != nil { + return false, 0, err + } + resp, err := kv.waitForCorrelatedResponse(ctx, ch, timeout) + if err != nil { + return false, 0, err + } + return resp.Applied, resp.CounterValue, nil +} + +// SetCardSync returns the cardinality of the set at key (0 if absent). +func (kv *KV) SetCardSync(ctx context.Context, key string, scope KVScope, userID, workspace string, timeout time.Duration) (int64, error) { + kv.syncMu.Lock() + defer kv.syncMu.Unlock() + + if timeout == 0 { + timeout = DefaultKVTimeout + } + if scope == "" { + scope = KVScopeGlobal + } + + requestID := kv.client.NextRequestID() + ch := kv.client.RegisterPendingKVRequest(requestID) + defer kv.client.pendingKVRequests.Delete(requestID) + + op := &pb.KVOperation{ + Op: pb.KVOperation_SET_CARD, + Scope: kvScopeToProto(scope), + Key: key, + UserId: userID, + Workspace: workspace, + RequestId: requestID, + } + if err := kv.client.Send(&pb.UpstreamMessage{Payload: &pb.UpstreamMessage_KvOp{KvOp: op}}); err != nil { + return 0, err + } + resp, err := kv.waitForCorrelatedResponse(ctx, ch, timeout) + if err != nil { + return 0, err + } + return resp.CounterValue, nil +} + // ============================================================================= // Synchronous KV Operations // ============================================================================= @@ -494,7 +762,7 @@ func (kv *KV) GetSync(ctx context.Context, opts KVGetOptions) (*KVResponse, erro scope = KVScopeGlobal } - if err := kv.GetWithRequestID(opts.Key, scope, opts.UserID, opts.Workspace, requestID); err != nil { + if err := kv.getWithReqIDAuth(opts.Key, scope, opts.UserID, opts.Workspace, requestID, opts.Authorization); err != nil { return nil, err } @@ -541,7 +809,7 @@ func (kv *KV) PutSync(ctx context.Context, opts KVPutOptions) (*KVResponse, erro ttlSeconds = int64(opts.TTL.Seconds()) } - if err := kv.PutWithRequestID(opts.Key, opts.Value, scope, opts.UserID, opts.Workspace, ttlSeconds, requestID); err != nil { + if err := kv.putWithReqIDAuth(opts.Key, opts.Value, scope, opts.UserID, opts.Workspace, ttlSeconds, requestID, opts.Authorization); err != nil { return nil, err } @@ -582,7 +850,7 @@ func (kv *KV) ListSync(ctx context.Context, opts KVListOptions) (*KVResponse, er scope = KVScopeGlobal } - if err := kv.ListWithRequestID(opts.KeyPrefix, scope, opts.UserID, opts.Workspace, requestID); err != nil { + if err := kv.listWithOptsAuth(opts.KeyPrefix, scope, opts.UserID, opts.Workspace, requestID, opts.Limit, opts.Cursor, opts.Authorization); err != nil { return nil, err } diff --git a/sdk/go/aether/options.go b/sdk/go/aether/options.go index d5c5c65..0f7fbb9 100644 --- a/sdk/go/aether/options.go +++ b/sdk/go/aether/options.go @@ -8,6 +8,8 @@ package aether import ( "crypto/tls" "time" + + pb "github.com/scitrera/aether/api/proto" ) // ============================================================================= @@ -170,7 +172,13 @@ type ClientOptions struct { // Keys and values are passed to the server as metadata. Credentials map[string]string - // Metadata contains additional metadata to send with the connection. + // Metadata contains additional gRPC metadata (headers) attached to the + // Connect stream's outgoing context, readable server-side via + // metadata.FromIncomingContext. Unlike Credentials (carried in the + // InitConnection payload), these ride as transport-level headers and are + // available before the first frame is read — e.g. an aggregator that pairs + // an inbound Connect by an "x-aether-tenant" header. Empty/nil = no extra + // headers (the pre-existing behavior), so this is fully backward-compatible. Metadata map[string]string } @@ -481,6 +489,14 @@ type KVGetOptions struct { // If empty, uses the client's current workspace. Workspace string + // Authorization carries an on-behalf-of AuthorizationContext stamped onto + // the KVOperation (proto field 9). When set, the synchronous GetSync path + // forwards it so the gateway resolves the operation under the named subject + // (e.g. a user OBO grant) instead of the connecting principal. nil leaves + // the operation as direct (today's behavior). Only the *Sync paths honor + // this; the async builders always send nil for backward compatibility. + Authorization *pb.AuthorizationContext + // Timeout for synchronous operations. // If 0, uses a default timeout. Timeout time.Duration @@ -511,6 +527,13 @@ type KVPutOptions struct { // 0 means no expiration. TTL time.Duration + // Authorization carries an on-behalf-of AuthorizationContext stamped onto + // the KVOperation (proto field 9). When set, the synchronous PutSync path + // forwards it so the gateway resolves the write under the named subject + // (e.g. a user OBO grant). nil leaves the write as direct (today's + // behavior). Only the *Sync paths honor this. + Authorization *pb.AuthorizationContext + // Timeout for synchronous operations. // If 0, uses a default timeout. Timeout time.Duration @@ -533,6 +556,23 @@ type KVListOptions struct { // If empty, uses the client's current workspace. Workspace string + // Limit caps the number of keys returned in a single LIST response. + // 0 = server default. The KeyPrefix filter is applied server-side BEFORE + // the limit, so the cap bounds matching keys, not all keys in the scope. + Limit int32 + + // Cursor pages through LIST results larger than Limit: pass the previous + // KVResponse.NextCursor to fetch the next page; empty starts from the + // beginning. Iterate until KVResponse.HasMore is false. + Cursor string + + // Authorization carries an on-behalf-of AuthorizationContext stamped onto + // the KVOperation (proto field 9). When set, the synchronous ListSync path + // forwards it so the gateway resolves the list under the named subject + // (e.g. a user OBO grant). nil leaves the list as direct (today's + // behavior). Only the *Sync paths honor this. + Authorization *pb.AuthorizationContext + // Timeout for synchronous operations. // If 0, uses a default timeout. Timeout time.Duration @@ -560,6 +600,26 @@ type KVDeleteOptions struct { Timeout time.Duration } +// KVPurgeOptions configures a REMOVAL-ONLY PURGE_IDENTITY operation: delete +// every key in ANOTHER principal's KV namespace. Requires the +// capability/kv_purge_identity grant on the calling client's identity. The +// response carries only the deleted count — never keys or values. +type KVPurgeOptions struct { + // TargetIdentity is the principal whose namespace to purge, e.g. + // "sv::sandbox-sidecar::". Required. + TargetIdentity string + + // Scope selects which scope to purge. Default: KVScopeGlobal. + Scope KVScope + + // KeyPrefix, when set, restricts the purge to keys beginning with it. + // Empty purges the entire (target, scope) namespace. + KeyPrefix string + + // Timeout for the synchronous operation. 0 → default. + Timeout time.Duration +} + // ============================================================================= // Message Type // ============================================================================= @@ -672,6 +732,31 @@ type CreateTaskOptions struct { // flows (sandbox lease, etc.). Use TargetAgentID for the Agent-typed // equivalent; the two are mutually exclusive. TargetIdentity string + + // RetryPolicy, when non-nil, is persisted on the created task. The + // task store consults it on FailTask to compute next_retry_at, + // re-pending the task up to the policy's max_attempts. Use this to + // lift retry scheduling out of the worker (e.g., svix-style schedules + // for webhook delivery). Absent = the server's legacy default + // (immediate re-pend, max_retries=3). + RetryPolicy *pb.RetryPolicy + + // Priority is the optional dispatch priority for the task. Higher + // priority pending tasks are delivered before lower ones (ties break + // FIFO). Defaults to UNSPECIFIED, which the server normalizes to NORMAL. + // Use the pb.TaskPriority_TASK_PRIORITY_* constants. + Priority pb.TaskPriority + + // Authorization, when non-nil, is forwarded verbatim as the created + // task's AuthorizationContext. The gateway runs it through + // resolveAuthorizationContext and seeds the task's Authority subject from + // it, so an on-behalf-of context here makes the task act on behalf of the + // named subject (e.g. seeding task.Authority.Subject* for subject- + // participation notifications). When nil the gateway falls back to its + // own inheritance chain (nested-task authority or none). Typically the + // caller forwards an inbound ProxyHttpRequest.GetAuthorization() to make a + // service-created notification task carry the originating user's subject. + Authorization *pb.AuthorizationContext } // ============================================================================= @@ -742,6 +827,16 @@ type SendMessageOptions struct { // Metadata is optional message metadata. Metadata map[string]string + + // Authorization optionally carries an on-behalf-of AuthorizationContext for + // this send. The gateway validates it (grant + delegate + audience) and, on + // success, stamps the resolved subject onto the delivered + // MessageEnvelope.on_behalf_subject so the recipient can identify the user + // the message is sent for. EXPLICIT by design: a bare send never assumes an + // OBO context. Use OBOAuthorizationFromContext(ctx) to populate this from a + // context set via WithOBOAuthorization, or build the *pb.AuthorizationContext + // directly. Nil ⇒ direct (non-OBO) send. + Authorization *pb.AuthorizationContext } // ============================================================================= diff --git a/sdk/go/aether/proxy.go b/sdk/go/aether/proxy.go index 6714c80..e1e88db 100644 --- a/sdk/go/aether/proxy.go +++ b/sdk/go/aether/proxy.go @@ -608,6 +608,15 @@ func WithOBOAuthorization(ctx context.Context, auth *pb.AuthorizationContext) co return context.WithValue(ctx, oboContextKey{}, auth) } +// OBOAuthorizationFromContext returns the OBO AuthorizationContext carried by +// ctx (set via WithOBOAuthorization), or nil. It is the public, opt-in helper +// for populating SendMessageOptions.Authorization from a context, pairing with +// WithOBOAuthorization. Explicit by design: callers choose to forward the OBO +// onto a send rather than having every send under an OBO context assume it. +func OBOAuthorizationFromContext(ctx context.Context) *pb.AuthorizationContext { + return oboFromContext(ctx) +} + // oboFromContext retrieves an OBO AuthorizationContext from the context, or nil. func oboFromContext(ctx context.Context) *pb.AuthorizationContext { if ctx == nil { diff --git a/sdk/go/aether/service.go b/sdk/go/aether/service.go index 65d9ad3..c0028b4 100644 --- a/sdk/go/aether/service.go +++ b/sdk/go/aether/service.go @@ -65,6 +65,7 @@ func NewServiceClient(opts ServiceOptions) (*ServiceClient, error) { Connection: opts.Connection, TLS: opts.TLS, Credentials: opts.Credentials, + Metadata: opts.Metadata, } base, err := NewBaseClient(cfg) @@ -105,3 +106,20 @@ func (c *ServiceClient) Implementation() string { func (c *ServiceClient) Specifier() string { return c.specifier } + +// SendToUserBroadcast sends a message to every one of a user's windows, +// regardless of which workspace each window is viewing (topic uu::{user_id}). +// +// This is the workspace-agnostic, non-progress channel for platform→user +// notifications — the intended way for a service principal to push an opaque +// update to a user without abusing the progress channel. Uses OPAQUE message +// type by default; use SendToUserBroadcastWithType for other types. +func (c *ServiceClient) SendToUserBroadcast(userID string, payload []byte) error { + return c.SendToUserBroadcastWithType(userID, payload, pb.MessageType_OPAQUE) +} + +// SendToUserBroadcastWithType sends a user-broadcast message with a custom message type. +func (c *ServiceClient) SendToUserBroadcastWithType(userID string, payload []byte, msgType pb.MessageType) error { + topic := UserBroadcastTopic(userID) + return c.sendMessage(topic, payload, msgType) +} diff --git a/sdk/go/aether/stream_metadata_test.go b/sdk/go/aether/stream_metadata_test.go new file mode 100644 index 0000000..af02bcf --- /dev/null +++ b/sdk/go/aether/stream_metadata_test.go @@ -0,0 +1,135 @@ +package aether + +import ( + "context" + "net" + "sync" + "testing" + "time" + + pb "github.com/scitrera/aether/api/proto" + + "google.golang.org/grpc" + "google.golang.org/grpc/metadata" +) + +// metadataCapturingGateway is a minimal in-process AetherGateway server that +// records the gRPC metadata seen on the inbound Connect stream. It exists to +// prove ClientOptions.Metadata is emitted as transport-level headers (e.g. the +// aggregator's x-aether-tenant pairing hint) before any frame is processed. +type metadataCapturingGateway struct { + pb.UnimplementedAetherGatewayServer + + mu sync.Mutex + md metadata.MD + seen chan struct{} +} + +func (g *metadataCapturingGateway) Connect(stream grpc.BidiStreamingServer[pb.UpstreamMessage, pb.DownstreamMessage]) error { + md, _ := metadata.FromIncomingContext(stream.Context()) + g.mu.Lock() + g.md = md.Copy() + g.mu.Unlock() + close(g.seen) + + // Drain inbound frames until the client tears the stream down. We never + // send a ConnectionAck — the test only needs the init frame to have been + // flushed, which Connect's establishStream does synchronously. + for { + if _, err := stream.Recv(); err != nil { + return nil + } + } +} + +func TestServiceClient_ConnectEmitsStreamMetadata(t *testing.T) { + lis, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("listen: %v", err) + } + defer lis.Close() + + gw := &metadataCapturingGateway{seen: make(chan struct{})} + srv := grpc.NewServer() + pb.RegisterAetherGatewayServer(srv, gw) + go func() { _ = srv.Serve(lis) }() + defer srv.Stop() + + client, err := NewServiceClient(ServiceOptions{ + ClientOptions: ClientOptions{ + ServerAddr: lis.Addr().String(), + Metadata: map[string]string{"x-aether-tenant": "acme"}, + }, + Implementation: "sandbox-provider", + Specifier: "default", + }) + if err != nil { + t.Fatalf("NewServiceClient: %v", err) + } + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + if err := client.Connect(ctx); err != nil { + t.Fatalf("Connect: %v", err) + } + defer client.Close() + + select { + case <-gw.seen: + case <-ctx.Done(): + t.Fatal("timed out waiting for server to observe Connect metadata") + } + + gw.mu.Lock() + got := gw.md.Get("x-aether-tenant") + gw.mu.Unlock() + if len(got) != 1 || got[0] != "acme" { + t.Fatalf("x-aether-tenant header = %v, want [acme]", got) + } +} + +// TestServiceClient_ConnectNoMetadataIsBackwardCompatible asserts that when no +// Metadata is supplied, Connect still succeeds and emits no x-aether-tenant +// header (zero-value behavior is unchanged). +func TestServiceClient_ConnectNoMetadataIsBackwardCompatible(t *testing.T) { + lis, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("listen: %v", err) + } + defer lis.Close() + + gw := &metadataCapturingGateway{seen: make(chan struct{})} + srv := grpc.NewServer() + pb.RegisterAetherGatewayServer(srv, gw) + go func() { _ = srv.Serve(lis) }() + defer srv.Stop() + + client, err := NewServiceClient(ServiceOptions{ + ClientOptions: ClientOptions{ServerAddr: lis.Addr().String()}, + Implementation: "sandbox-provider", + Specifier: "default", + }) + if err != nil { + t.Fatalf("NewServiceClient: %v", err) + } + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + if err := client.Connect(ctx); err != nil { + t.Fatalf("Connect: %v", err) + } + defer client.Close() + + select { + case <-gw.seen: + case <-ctx.Done(): + t.Fatal("timed out waiting for server to observe Connect") + } + + gw.mu.Lock() + got := gw.md.Get("x-aether-tenant") + gw.mu.Unlock() + if len(got) != 0 { + t.Fatalf("x-aether-tenant header = %v, want none", got) + } +} diff --git a/sdk/go/aether/topics.go b/sdk/go/aether/topics.go index cd7cd8a..3a41726 100644 --- a/sdk/go/aether/topics.go +++ b/sdk/go/aether/topics.go @@ -44,6 +44,11 @@ const ( // TopicPrefixUserWorkspace is the prefix for user workspace topics. TopicPrefixUserWorkspace = "uw" + // TopicPrefixUserBroadcast is the prefix for per-user broadcast topics + // (workspace-agnostic; reaches all of a user's windows). Publishing is + // restricted to platform principals (service / workflow-engine / bridge). + TopicPrefixUserBroadcast = "uu" + // TopicPrefixGlobalAgents is the prefix for global agent broadcast topics. TopicPrefixGlobalAgents = "ga" @@ -61,6 +66,9 @@ const ( // TopicPrefixBridge is the prefix for bridge topics. TopicPrefixBridge = "br" + + // TopicPrefixService is the prefix for workspace-less service-principal topics. + TopicPrefixService = "sv" ) // ============================================================================= @@ -79,6 +87,20 @@ func BridgeTopic(implementation, specifier string) string { return fmt.Sprintf("%s::%s::%s", TopicPrefixBridge, implementation, specifier) } +// ServiceTopic creates a topic string for a workspace-less service principal. +// +// Format: sv::{implementation} — no specifier; the gateway routes to an active +// instance of the service. Use this to address shared services like the +// platform-bridge. +// +// Example: +// +// topic := aether.ServiceTopic("platform-bridge") +// // Returns: "sv::platform-bridge" +func ServiceTopic(implementation string) string { + return fmt.Sprintf("%s::%s", TopicPrefixService, implementation) +} + // ============================================================================= // Topic Construction Helpers - Agents // ============================================================================= @@ -192,6 +214,24 @@ func UserWorkspaceTopic(userID, workspace string) string { return fmt.Sprintf("%s::%s::%s", TopicPrefixUserWorkspace, userID, workspace) } +// UserBroadcastTopic creates a per-user broadcast topic string. +// +// Format: uu::{user_id} +// +// Messages sent to this topic reach every one of a user's open windows +// regardless of which workspace each window is currently viewing — the +// workspace-agnostic, non-progress complement to the per-user progress topic. +// Only platform principals (service / workflow-engine / bridge) may publish; +// the gateway rejects sends from workspace-scoped principals. +// +// Example: +// +// topic := aether.UserBroadcastTopic("alice") +// // Returns: "uu::alice" +func UserBroadcastTopic(userID string) string { + return fmt.Sprintf("%s::%s", TopicPrefixUserBroadcast, userID) +} + // GlobalUsersTopic creates a broadcast topic for all users in a workspace. // // Format: gu::{workspace} @@ -212,7 +252,7 @@ func GlobalUsersTopic(workspace string) string { // EventTopic creates a topic string for broadcast events. // -// Format: event.{event_type} +// Format: event::{event_type} // // Event topics are used by the Workflow Engine to receive and process // broadcast events. Only workflow engines can subscribe to event topics. @@ -220,23 +260,26 @@ func GlobalUsersTopic(workspace string) string { // Example: // // topic := aether.EventTopic("task.completed") -// // Returns: "event.task.completed" +// // Returns: "event::task.completed" func EventTopic(eventType string) string { - return fmt.Sprintf("%s.%s", TopicPrefixEvent, eventType) + return fmt.Sprintf("%s::%s", TopicPrefixEvent, eventType) } // EventWildcardTopic returns the wildcard pattern for all events. // -// Format: event.* +// Format: event::* // -// This is the topic that Workflow Engines subscribe to for receiving all events. +// This is the topic publishers target to broadcast to the Workflow Engine +// fan-in. The gateway rewrites event::* / event:: onto the sharded receiver +// topic (validateTopicFormat requires the "::" identity separator — a legacy +// "event.*" form is rejected as an invalid topic prefix). func EventWildcardTopic() string { - return TopicPrefixEvent + ".*" + return TopicPrefixEvent + "::*" } // MetricTopic creates a topic string for telemetry/metrics. // -// Format: metric.{metric_type} +// Format: metric::{metric_type} // // Metric topics are used by the Metrics Bridge to receive telemetry data. // Only metrics bridges can subscribe to metric topics. @@ -244,18 +287,21 @@ func EventWildcardTopic() string { // Example: // // topic := aether.MetricTopic("performance") -// // Returns: "metric.performance" +// // Returns: "metric::performance" func MetricTopic(metricType string) string { - return fmt.Sprintf("%s.%s", TopicPrefixMetric, metricType) + return fmt.Sprintf("%s::%s", TopicPrefixMetric, metricType) } // MetricWildcardTopic returns the wildcard pattern for all metrics. // -// Format: metric.* +// Format: metric::* // -// This is the topic that Metrics Bridges subscribe to for receiving all metrics. +// This is the topic publishers target to fan a metric into the Metrics Bridge. +// The gateway rewrites metric::* / metric:: onto the sharded receiver topic +// (validateTopicFormat requires the "::" identity separator — a legacy "metric.*" +// form is rejected as an invalid topic prefix, so metrics never route). func MetricWildcardTopic() string { - return TopicPrefixMetric + ".*" + return TopicPrefixMetric + "::*" } // ProgressTopic creates a topic string for workspace progress updates. diff --git a/sdk/go/aether/topics_test.go b/sdk/go/aether/topics_test.go index 90ae37e..6e9c1ea 100644 --- a/sdk/go/aether/topics_test.go +++ b/sdk/go/aether/topics_test.go @@ -306,12 +306,12 @@ func TestEventTopic(t *testing.T) { { name: "simple event type", eventType: "task.completed", - want: "event.task.completed", + want: "event::task.completed", }, { name: "workflow event", eventType: "workflow.started", - want: "event.workflow.started", + want: "event::workflow.started", }, } @@ -326,7 +326,7 @@ func TestEventTopic(t *testing.T) { } func TestEventWildcardTopic(t *testing.T) { - want := "event.*" + want := "event::*" got := EventWildcardTopic() if got != want { t.Errorf("EventWildcardTopic() = %v, want %v", got, want) @@ -342,12 +342,12 @@ func TestMetricTopic(t *testing.T) { { name: "performance metric", metricType: "performance", - want: "metric.performance", + want: "metric::performance", }, { name: "latency metric", metricType: "latency", - want: "metric.latency", + want: "metric::latency", }, } @@ -362,7 +362,7 @@ func TestMetricTopic(t *testing.T) { } func TestMetricWildcardTopic(t *testing.T) { - want := "metric.*" + want := "metric::*" got := MetricWildcardTopic() if got != want { t.Errorf("MetricWildcardTopic() = %v, want %v", got, want) diff --git a/sdk/go/aether/version.go b/sdk/go/aether/version.go index ffaf617..932e27f 100644 --- a/sdk/go/aether/version.go +++ b/sdk/go/aether/version.go @@ -2,4 +2,4 @@ package aether // Version is the current release version of the Aether Go SDK. // This is updated automatically by scripts/update-versions.py. -const Version = "0.2.1" +const Version = "0.2.2" diff --git a/sdk/go/aether/workflow.go b/sdk/go/aether/workflow.go index 9d4da70..6826f69 100644 --- a/sdk/go/aether/workflow.go +++ b/sdk/go/aether/workflow.go @@ -311,6 +311,22 @@ func (c *WorkflowEngineClient) SendToUserWorkspaceWithType(userID, workspace str return c.sendMessage(topic, payload, msgType) } +// SendToUserBroadcast sends a message to every one of a user's windows, +// regardless of which workspace each window is viewing (topic uu::{user_id}). +// +// This is the workspace-agnostic, non-progress channel for platform→user +// notifications. Uses OPAQUE message type by default; use +// SendToUserBroadcastWithType for other types. +func (c *WorkflowEngineClient) SendToUserBroadcast(userID string, payload []byte) error { + return c.SendToUserBroadcastWithType(userID, payload, pb.MessageType_OPAQUE) +} + +// SendToUserBroadcastWithType sends a user-broadcast message with a custom message type. +func (c *WorkflowEngineClient) SendToUserBroadcastWithType(userID string, payload []byte, msgType pb.MessageType) error { + topic := UserBroadcastTopic(userID) + return c.sendMessage(topic, payload, msgType) +} + // ============================================================================= // Metric Publishing // ============================================================================= diff --git a/sdk/go/coord/coord.go b/sdk/go/coord/coord.go new file mode 100644 index 0000000..8342281 --- /dev/null +++ b/sdk/go/coord/coord.go @@ -0,0 +1,91 @@ +// Package coord provides backend-agnostic distributed coordination primitives +// — Mutex, LeaderElection, Semaphore, and Once — built on a small set of +// atomic conditional-write operations. +// +// The same algorithms run server-side (in-process, over an Aether KV store) +// and client-side (over the gRPC KV protocol) by depending only on the Locker +// and Counter interfaces below. Adapters in the server and SDK bind these +// interfaces to a concrete backend (Redis / Badger / NATS-JetStream) and a KV +// scope; this package contains no backend, network, or scope knowledge. +// +// Lock model: a lock is a KV key holding an opaque owner token with a lease +// TTL. acquire = TryAcquire (set-if-absent), refresh = Refresh +// (compare-and-set owner→owner, extending the TTL), release = Release +// (compare-and-delete). If the holder dies without releasing, the TTL expires +// and the lock becomes acquirable again. Fencing tokens are per-instance +// random IDs (see NewOwnerID). +package coord + +import ( + "context" + "crypto/rand" + "encoding/hex" + "os" + "time" +) + +// Locker is the minimal atomic-conditional-write surface a backend must +// provide for Mutex / LeaderElection / Once. Each method maps 1:1 to an Aether +// KV primitive; all composition (reentrancy, renewal, blocking) lives in this +// package, keeping adapters trivial. +type Locker interface { + // TryAcquire sets key=owner with lease ttl only if key is currently + // absent. Returns true iff the caller now holds the lock. (Maps to SetNX.) + TryAcquire(ctx context.Context, key, owner string, ttl time.Duration) (bool, error) + // Refresh extends the lease (and re-asserts ownership) iff the current + // holder is owner. Returns false when the lock has been lost. (Maps to + // CompareAndSet owner→owner.) + Refresh(ctx context.Context, key, owner string, ttl time.Duration) (bool, error) + // Release frees key iff the current holder is owner. Returns true iff the + // delete was applied. (Maps to CompareAndDelete.) + Release(ctx context.Context, key, owner string) (bool, error) + // Peek returns the current owner of key, or "" when the key is unheld. + Peek(ctx context.Context, key string) (string, error) +} + +// Counter is the guarded-counter surface backing Semaphore. Maps 1:1 to the +// KV IncrementIf / DecrementIf operations. +type Counter interface { + // IncrementIf adds delta iff the result would not exceed ceiling. Returns + // the (possibly unchanged) value and whether the mutation applied. + IncrementIf(ctx context.Context, key string, delta, ceiling int64) (int64, bool, error) + // DecrementIf subtracts delta iff the result would not drop below floor. + DecrementIf(ctx context.Context, key string, delta, floor int64) (int64, bool, error) +} + +const ( + // DefaultLeaseTTL is the default lock/lease lifetime when none is supplied. + DefaultLeaseTTL = 30 * time.Second + // DefaultRenewInterval is the default lease-renewal cadence. It must be + // comfortably below the lease TTL so a transient backend blip does not lose + // an otherwise-healthy lock. + DefaultRenewInterval = 10 * time.Second +) + +// NewOwnerID returns a process-unique fencing token of the form +// ":<16 hex chars>", stable for the lifetime of the returned string +// and distinct across restarts. Use one owner ID per logical lock holder. +func NewOwnerID() string { + hostname, _ := os.Hostname() + if hostname == "" { + hostname = "unknown" + } + var b [8]byte + _, _ = rand.Read(b[:]) + return hostname + ":" + hex.EncodeToString(b[:]) +} + +// sleepCtx sleeps for d, returning false if ctx is cancelled first. +func sleepCtx(ctx context.Context, d time.Duration) bool { + if d <= 0 { + return ctx.Err() == nil + } + t := time.NewTimer(d) + defer t.Stop() + select { + case <-ctx.Done(): + return false + case <-t.C: + return true + } +} diff --git a/sdk/go/coord/coord_test.go b/sdk/go/coord/coord_test.go new file mode 100644 index 0000000..ec82b49 --- /dev/null +++ b/sdk/go/coord/coord_test.go @@ -0,0 +1,274 @@ +package coord + +import ( + "context" + "sync" + "sync/atomic" + "testing" + "time" +) + +func TestMutex_MutualExclusion(t *testing.T) { + lk := NewMemoryLocker() + ctx := context.Background() + + const goroutines = 20 + const incrsPer = 50 + var counter int + var mu sync.Mutex // guards counter only to detect races via -race; the + // distributed Mutex is what actually serializes critical sections. + + var wg sync.WaitGroup + var maxConcurrent int64 + var inCrit int64 + for i := 0; i < goroutines; i++ { + wg.Add(1) + go func() { + defer wg.Done() + m := NewMutex(lk, "crit", WithLeaseTTL(2*time.Second)) + for j := 0; j < incrsPer; j++ { + if err := m.Lock(ctx); err != nil { + t.Errorf("Lock: %v", err) + return + } + n := atomic.AddInt64(&inCrit, 1) + if n > atomic.LoadInt64(&maxConcurrent) { + atomic.StoreInt64(&maxConcurrent, n) + } + mu.Lock() + counter++ + mu.Unlock() + atomic.AddInt64(&inCrit, -1) + if err := m.Unlock(ctx); err != nil { + t.Errorf("Unlock: %v", err) + return + } + } + }() + } + wg.Wait() + + if counter != goroutines*incrsPer { + t.Errorf("counter = %d, want %d", counter, goroutines*incrsPer) + } + if maxConcurrent > 1 { + t.Errorf("observed %d concurrent critical-section holders, want 1", maxConcurrent) + } +} + +func TestMutex_TryLockContended(t *testing.T) { + lk := NewMemoryLocker() + ctx := context.Background() + + a := NewMutex(lk, "k", WithLeaseTTL(time.Minute)) + b := NewMutex(lk, "k", WithLeaseTTL(time.Minute)) + + ok, err := a.TryLock(ctx) + if err != nil || !ok { + t.Fatalf("a.TryLock ok=%v err=%v", ok, err) + } + ok, err = b.TryLock(ctx) + if err != nil { + t.Fatalf("b.TryLock err=%v", err) + } + if ok { + t.Fatal("b should not acquire while a holds") + } + if err := a.Unlock(ctx); err != nil { + t.Fatalf("a.Unlock: %v", err) + } + ok, err = b.TryLock(ctx) + if err != nil || !ok { + t.Fatalf("b.TryLock after release ok=%v err=%v", ok, err) + } +} + +func TestLeaderElection_ExactlyOneLeader(t *testing.T) { + lk := NewMemoryLocker() + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + const n = 5 + var acquireCount int64 + var current int64 // number currently believing they're leader + var maxLeaders int64 + + elections := make([]*LeaderElection, n) + for i := 0; i < n; i++ { + le := NewLeaderElection(lk, "leader", + WithLeaderLeaseTTL(300*time.Millisecond), + WithLeaderRenewInterval(60*time.Millisecond)) + le.OnAcquire(func(_ context.Context) { + atomic.AddInt64(&acquireCount, 1) + n := atomic.AddInt64(¤t, 1) + for { + m := atomic.LoadInt64(&maxLeaders) + if n <= m || atomic.CompareAndSwapInt64(&maxLeaders, m, n) { + break + } + } + }) + le.OnLose(func() { atomic.AddInt64(¤t, -1) }) + elections[i] = le + le.Start(ctx) + } + + // Let the cluster settle on a leader. + time.Sleep(400 * time.Millisecond) + + leaders := 0 + for _, le := range elections { + if le.IsLeader() { + leaders++ + } + } + if leaders != 1 { + t.Fatalf("IsLeader() true for %d replicas, want 1", leaders) + } + if atomic.LoadInt64(&maxLeaders) > 1 { + t.Fatalf("max concurrent OnAcquire-active leaders = %d, want 1", maxLeaders) + } + + for _, le := range elections { + _ = le.Shutdown(context.Background()) + } +} + +func TestLeaderElection_FailoverOnShutdown(t *testing.T) { + lk := NewMemoryLocker() + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + mk := func() *LeaderElection { + return NewLeaderElection(lk, "leader", + WithLeaderLeaseTTL(300*time.Millisecond), + WithLeaderRenewInterval(50*time.Millisecond)) + } + a, b := mk(), mk() + a.Start(ctx) + time.Sleep(150 * time.Millisecond) + b.Start(ctx) + time.Sleep(150 * time.Millisecond) + + // Identify the current leader and shut it down; the other must take over. + var leader, follower *LeaderElection + switch { + case a.IsLeader() && !b.IsLeader(): + leader, follower = a, b + case b.IsLeader() && !a.IsLeader(): + leader, follower = b, a + default: + t.Fatalf("expected exactly one leader before failover (a=%v b=%v)", a.IsLeader(), b.IsLeader()) + } + + if err := leader.Shutdown(context.Background()); err != nil { + t.Fatalf("shutdown leader: %v", err) + } + // Follower should acquire within a couple renew intervals (shutdown + // releases the key eagerly). + deadline := time.Now().Add(2 * time.Second) + for time.Now().Before(deadline) && !follower.IsLeader() { + time.Sleep(20 * time.Millisecond) + } + if !follower.IsLeader() { + t.Fatal("follower did not take over leadership after leader shutdown") + } + _ = follower.Shutdown(context.Background()) +} + +func TestSemaphore_Limit(t *testing.T) { + lk := NewMemoryLocker() + ctx := context.Background() + sem, err := NewSemaphore(lk, "sem", 3) + if err != nil { + t.Fatalf("NewSemaphore: %v", err) + } + + // Acquire up to the limit. + for i := 0; i < 3; i++ { + ok, err := sem.TryAcquire(ctx) + if err != nil || !ok { + t.Fatalf("acquire %d ok=%v err=%v", i, ok, err) + } + } + // 4th must be refused. + ok, err := sem.TryAcquire(ctx) + if err != nil { + t.Fatalf("4th acquire err=%v", err) + } + if ok { + t.Fatal("4th acquire should be refused at limit 3") + } + // Release one, then a new acquire should succeed. + if released, err := sem.Release(ctx); err != nil || !released { + t.Fatalf("release released=%v err=%v", released, err) + } + ok, err = sem.TryAcquire(ctx) + if err != nil || !ok { + t.Fatalf("acquire after release ok=%v err=%v", ok, err) + } + // Release floor: releasing below 0 must not apply. + for i := 0; i < 3; i++ { + _, _ = sem.Release(ctx) + } + released, err := sem.Release(ctx) + if err != nil { + t.Fatalf("over-release err=%v", err) + } + if released { + t.Fatal("release below floor 0 should not apply") + } +} + +func TestOnce_SingleWinner(t *testing.T) { + lk := NewMemoryLocker() + ctx := context.Background() + + const n = 30 + var runs int64 + var wg sync.WaitGroup + start := make(chan struct{}) + for i := 0; i < n; i++ { + wg.Add(1) + go func() { + defer wg.Done() + o := NewOnce(lk, "init") + <-start + ran, err := o.Do(ctx, func(_ context.Context) error { + atomic.AddInt64(&runs, 1) + return nil + }) + if err != nil { + t.Errorf("Do: %v", err) + } + _ = ran + }() + } + close(start) + wg.Wait() + + if runs != 1 { + t.Fatalf("fn ran %d times, want exactly 1", runs) + } +} + +func TestOnce_RetryAfterFailure(t *testing.T) { + lk := NewMemoryLocker() + ctx := context.Background() + + o := NewOnce(lk, "init") + ran, err := o.Do(ctx, func(_ context.Context) error { + return context.Canceled // simulate failure + }) + if !ran || err == nil { + t.Fatalf("first Do ran=%v err=%v, want ran=true err!=nil", ran, err) + } + + // Marker should have been released, so a second attempt can run. + o2 := NewOnce(lk, "init") + var ran2 bool + ran2, err = o2.Do(ctx, func(_ context.Context) error { return nil }) + if !ran2 || err != nil { + t.Fatalf("retry Do ran=%v err=%v, want ran=true err=nil", ran2, err) + } +} diff --git a/sdk/go/coord/leader.go b/sdk/go/coord/leader.go new file mode 100644 index 0000000..2386772 --- /dev/null +++ b/sdk/go/coord/leader.go @@ -0,0 +1,222 @@ +package coord + +import ( + "context" + "sync" + "time" +) + +// LeaderElection races for and holds leadership on a single KV key. Exactly +// one holder leads at a time: leadership is the lock, and the lease TTL ensures +// a crashed leader is replaced after at most ttl. Unlike best-effort +// read-back schemes, acquisition uses an atomic set-if-absent (TryAcquire), so +// two replicas cannot both believe they are leader. +// +// API mirrors the common OnAcquire/OnLose shape so existing callers can adopt +// it without restructuring: +// +// le := coord.NewLeaderElection(locker, "service::leader") +// le.OnAcquire(func(ctx context.Context) { startLeaderWork(ctx) }) +// le.OnLose(func() { stopLeaderWork() }) +// le.Start(ctx) +// defer le.Shutdown(context.Background()) +type LeaderElection struct { + locker Locker + key string + owner string + ttl time.Duration + renewInterval time.Duration + + mu sync.Mutex + isLeader bool + onAcquire []func(ctx context.Context) + onLose []func() + + loopCancel context.CancelFunc + done chan struct{} +} + +// LeaderOption customizes a LeaderElection. +type LeaderOption func(*LeaderElection) + +// WithLeaderLeaseTTL sets the leadership lease TTL. +func WithLeaderLeaseTTL(ttl time.Duration) LeaderOption { + return func(l *LeaderElection) { l.ttl = ttl } +} + +// WithLeaderRenewInterval sets the renewal cadence (must be < lease TTL). +func WithLeaderRenewInterval(d time.Duration) LeaderOption { + return func(l *LeaderElection) { l.renewInterval = d } +} + +// WithLeaderOwnerID sets an explicit fencing token (defaults to NewOwnerID). +func WithLeaderOwnerID(owner string) LeaderOption { + return func(l *LeaderElection) { l.owner = owner } +} + +// NewLeaderElection creates a LeaderElection for key backed by locker. +func NewLeaderElection(locker Locker, key string, opts ...LeaderOption) *LeaderElection { + l := &LeaderElection{ + locker: locker, + key: key, + owner: NewOwnerID(), + ttl: DefaultLeaseTTL, + } + for _, o := range opts { + o(l) + } + if l.renewInterval <= 0 { + l.renewInterval = l.ttl / 3 + if l.renewInterval <= 0 { + l.renewInterval = DefaultRenewInterval + } + } + return l +} + +// OwnerID returns this replica's fencing token. +func (l *LeaderElection) OwnerID() string { return l.owner } + +// OnAcquire registers a callback invoked (in its own goroutine) when this +// replica becomes leader. The callback receives a context cancelled when +// leadership is lost or the loop shuts down. Must be called before Start. +func (l *LeaderElection) OnAcquire(fn func(ctx context.Context)) { + l.mu.Lock() + defer l.mu.Unlock() + l.onAcquire = append(l.onAcquire, fn) +} + +// OnLose registers a callback invoked synchronously when leadership is lost. +// Must be called before Start. +func (l *LeaderElection) OnLose(fn func()) { + l.mu.Lock() + defer l.mu.Unlock() + l.onLose = append(l.onLose, fn) +} + +// IsLeader reports the current leadership status. Thread-safe. +func (l *LeaderElection) IsLeader() bool { + l.mu.Lock() + defer l.mu.Unlock() + return l.isLeader +} + +// Start launches the election loop in a background goroutine. Idempotent. +func (l *LeaderElection) Start(ctx context.Context) { + l.mu.Lock() + defer l.mu.Unlock() + if l.done != nil { + return + } + lctx, cancel := context.WithCancel(ctx) + l.loopCancel = cancel + l.done = make(chan struct{}) + go l.electionLoop(lctx) +} + +// Shutdown cancels the election loop, fires OnLose if currently leader, and +// best-effort releases the key so the next leader need not wait for TTL +// expiry. Blocks until the loop exits or ctx is cancelled. +func (l *LeaderElection) Shutdown(ctx context.Context) error { + l.mu.Lock() + cancel := l.loopCancel + done := l.done + l.mu.Unlock() + + if cancel != nil { + cancel() + } + if done != nil { + select { + case <-done: + case <-ctx.Done(): + return ctx.Err() + } + } + + // Best-effort release so a successor can acquire immediately. + relCtx, relCancel := context.WithTimeout(context.Background(), 5*time.Second) + defer relCancel() + _, _ = l.locker.Release(relCtx, l.key, l.owner) + return nil +} + +func (l *LeaderElection) electionLoop(ctx context.Context) { + defer func() { + l.mu.Lock() + wasLeader := l.isLeader + l.isLeader = false + onLose := l.onLose + l.mu.Unlock() + if wasLeader { + fireOnLose(onLose) + } + close(l.done) + }() + + for ctx.Err() == nil { + acquired, err := l.locker.TryAcquire(ctx, l.key, l.owner, l.ttl) + if err != nil || !acquired { + // Not leader: wait a renew interval (jittered) and retry. Jitter + // spreads contender retries so they don't poll in lock-step. + if !sleepCtx(ctx, jitter(l.renewInterval)) { + return + } + continue + } + + leaderCtx, leaderCancel := context.WithCancel(ctx) + l.mu.Lock() + l.isLeader = true + onAcquire := l.onAcquire + l.mu.Unlock() + fireOnAcquire(leaderCtx, onAcquire) + + // Hold leadership until the lease is lost or ctx is cancelled. + l.holdLoop(ctx) + + l.mu.Lock() + l.isLeader = false + onLose := l.onLose + l.mu.Unlock() + leaderCancel() + fireOnLose(onLose) + + if ctx.Err() != nil { + return + } + } +} + +// holdLoop renews leadership every renewInterval until renewal fails or ctx is +// cancelled. +func (l *LeaderElection) holdLoop(ctx context.Context) { + ticker := time.NewTicker(l.renewInterval) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + ok, err := l.locker.Refresh(ctx, l.key, l.owner, l.ttl) + if ctx.Err() != nil { + return + } + if err != nil || !ok { + return // leadership lost + } + } + } +} + +func fireOnAcquire(ctx context.Context, callbacks []func(ctx context.Context)) { + for _, fn := range callbacks { + go fn(ctx) + } +} + +func fireOnLose(callbacks []func()) { + for _, fn := range callbacks { + fn() + } +} diff --git a/sdk/go/coord/memory_locker.go b/sdk/go/coord/memory_locker.go new file mode 100644 index 0000000..12d1a17 --- /dev/null +++ b/sdk/go/coord/memory_locker.go @@ -0,0 +1,138 @@ +package coord + +import ( + "context" + "strconv" + "sync" + "time" +) + +// MemoryLocker is an in-process implementation of both Locker and Counter, +// honoring TTL at access time. It is intended for tests and for single-process +// coordination where a shared external backend is unnecessary. It is +// goroutine-safe. +type MemoryLocker struct { + mu sync.Mutex + m map[string]memEntry +} + +type memEntry struct { + value string + expiresAt time.Time // zero = no expiry +} + +// NewMemoryLocker returns an empty in-memory locker/counter. +func NewMemoryLocker() *MemoryLocker { + return &MemoryLocker{m: make(map[string]memEntry)} +} + +// liveLocked returns the live value at key (honoring TTL), deleting it if +// expired. Caller must hold m.mu. +func (l *MemoryLocker) liveLocked(key string, now time.Time) (string, bool) { + e, ok := l.m[key] + if !ok { + return "", false + } + if !e.expiresAt.IsZero() && now.After(e.expiresAt) { + delete(l.m, key) + return "", false + } + return e.value, true +} + +func (l *MemoryLocker) setLocked(key, value string, ttl time.Duration) { + e := memEntry{value: value} + if ttl > 0 { + e.expiresAt = time.Now().Add(ttl) + } + l.m[key] = e +} + +// TryAcquire implements Locker. +func (l *MemoryLocker) TryAcquire(_ context.Context, key, owner string, ttl time.Duration) (bool, error) { + l.mu.Lock() + defer l.mu.Unlock() + if _, ok := l.liveLocked(key, time.Now()); ok { + return false, nil + } + l.setLocked(key, owner, ttl) + return true, nil +} + +// Refresh implements Locker. +func (l *MemoryLocker) Refresh(_ context.Context, key, owner string, ttl time.Duration) (bool, error) { + l.mu.Lock() + defer l.mu.Unlock() + cur, ok := l.liveLocked(key, time.Now()) + if !ok || cur != owner { + return false, nil + } + l.setLocked(key, owner, ttl) + return true, nil +} + +// Release implements Locker. +func (l *MemoryLocker) Release(_ context.Context, key, owner string) (bool, error) { + l.mu.Lock() + defer l.mu.Unlock() + cur, ok := l.liveLocked(key, time.Now()) + if !ok || cur != owner { + return false, nil + } + delete(l.m, key) + return true, nil +} + +// Peek implements Locker. +func (l *MemoryLocker) Peek(_ context.Context, key string) (string, error) { + l.mu.Lock() + defer l.mu.Unlock() + cur, _ := l.liveLocked(key, time.Now()) + return cur, nil +} + +// IncrementIf implements Counter. +func (l *MemoryLocker) IncrementIf(_ context.Context, key string, delta, ceiling int64) (int64, bool, error) { + l.mu.Lock() + defer l.mu.Unlock() + cur := l.counterLocked(key) + proposed := cur + delta + if proposed > ceiling { + return cur, false, nil + } + l.m[key] = memEntry{value: strconv.FormatInt(proposed, 10)} + return proposed, true, nil +} + +// DecrementIf implements Counter. +func (l *MemoryLocker) DecrementIf(_ context.Context, key string, delta, floor int64) (int64, bool, error) { + l.mu.Lock() + defer l.mu.Unlock() + cur := l.counterLocked(key) + proposed := cur - delta + if proposed < floor { + return cur, false, nil + } + l.m[key] = memEntry{value: strconv.FormatInt(proposed, 10)} + return proposed, true, nil +} + +func (l *MemoryLocker) counterLocked(key string) int64 { + cur, ok := l.liveLocked(key, time.Now()) + if !ok || cur == "" { + return 0 + } + n, _ := strconv.ParseInt(cur, 10, 64) + return n +} + +// ForceExpire immediately expires key, simulating TTL elapse or holder death. +// Test helper. +func (l *MemoryLocker) ForceExpire(key string) { + l.mu.Lock() + defer l.mu.Unlock() + if e, ok := l.m[key]; ok { + e.expiresAt = time.Now().Add(-time.Second) + l.m[key] = e + } +} diff --git a/sdk/go/coord/mutex.go b/sdk/go/coord/mutex.go new file mode 100644 index 0000000..186575f --- /dev/null +++ b/sdk/go/coord/mutex.go @@ -0,0 +1,193 @@ +package coord + +import ( + "context" + "errors" + "math/rand/v2" + "sync" + "time" +) + +// ErrNotHeld is returned by Unlock when the caller does not (or no longer) +// holds the lock. +var ErrNotHeld = errors.New("coord: lock not held") + +// Mutex is a distributed mutual-exclusion lock over a single KV key. A held +// Mutex auto-renews its lease in the background until Unlock or until renewal +// fails (e.g. the backend lost the key); a renewal failure is surfaced via the +// channel returned by Lost. +// +// A Mutex is single-holder and not safe for concurrent Lock/Unlock from +// multiple goroutines on the same instance; create one Mutex per goroutine +// that needs the lock (they coordinate through the shared backend key). +type Mutex struct { + locker Locker + key string + owner string + ttl time.Duration + renewInterval time.Duration + + mu sync.Mutex + held bool + cancelHold context.CancelFunc + lost chan struct{} +} + +// MutexOption customizes a Mutex. +type MutexOption func(*Mutex) + +// WithLeaseTTL sets the lock lease TTL and (proportionally) the renew interval +// when the latter is left at its default. +func WithLeaseTTL(ttl time.Duration) MutexOption { + return func(m *Mutex) { m.ttl = ttl } +} + +// WithRenewInterval overrides the lease-renewal cadence. Must be < lease TTL. +func WithRenewInterval(d time.Duration) MutexOption { + return func(m *Mutex) { m.renewInterval = d } +} + +// WithOwnerID sets an explicit fencing token instead of a generated one. +func WithOwnerID(owner string) MutexOption { + return func(m *Mutex) { m.owner = owner } +} + +// NewMutex creates a Mutex for key backed by locker. +func NewMutex(locker Locker, key string, opts ...MutexOption) *Mutex { + m := &Mutex{ + locker: locker, + key: key, + owner: NewOwnerID(), + ttl: DefaultLeaseTTL, + renewInterval: 0, + } + for _, o := range opts { + o(m) + } + if m.renewInterval <= 0 { + m.renewInterval = m.ttl / 3 + if m.renewInterval <= 0 { + m.renewInterval = DefaultRenewInterval + } + } + return m +} + +// OwnerID returns this Mutex's fencing token. +func (m *Mutex) OwnerID() string { return m.owner } + +// TryLock attempts to acquire the lock once without blocking. Returns true iff +// acquired. On success a background renewer keeps the lease alive until Unlock. +func (m *Mutex) TryLock(ctx context.Context) (bool, error) { + m.mu.Lock() + defer m.mu.Unlock() + if m.held { + return true, nil + } + ok, err := m.locker.TryAcquire(ctx, m.key, m.owner, m.ttl) + if err != nil { + return false, err + } + if !ok { + return false, nil + } + m.startHoldLocked() + return true, nil +} + +// Lock blocks until the lock is acquired or ctx is cancelled. Acquisition +// attempts are spaced by renewInterval with ±25% jitter to avoid lock-step +// thundering herds across contenders. +func (m *Mutex) Lock(ctx context.Context) error { + for { + ok, err := m.TryLock(ctx) + if err != nil { + return err + } + if ok { + return nil + } + if !sleepCtx(ctx, jitter(m.renewInterval)) { + return ctx.Err() + } + } +} + +// Unlock releases the lock and stops the background renewer. Returns ErrNotHeld +// if the lock was not held (or was already lost). +func (m *Mutex) Unlock(ctx context.Context) error { + m.mu.Lock() + if !m.held { + m.mu.Unlock() + return ErrNotHeld + } + m.stopHoldLocked() + m.mu.Unlock() + + _, err := m.locker.Release(ctx, m.key, m.owner) + return err +} + +// Lost returns a channel closed when the background renewer determines the +// lock has been lost (renewal failed or the key was taken over). The channel +// is recreated on each successful acquire; capture it after Lock/TryLock. +func (m *Mutex) Lost() <-chan struct{} { + m.mu.Lock() + defer m.mu.Unlock() + return m.lost +} + +// startHoldLocked launches the renewer. Caller must hold m.mu. +func (m *Mutex) startHoldLocked() { + m.held = true + m.lost = make(chan struct{}) + holdCtx, cancel := context.WithCancel(context.Background()) + m.cancelHold = cancel + lost := m.lost + go m.renewLoop(holdCtx, lost) +} + +// stopHoldLocked tears down the renewer. Caller must hold m.mu. +func (m *Mutex) stopHoldLocked() { + m.held = false + if m.cancelHold != nil { + m.cancelHold() + m.cancelHold = nil + } +} + +// renewLoop refreshes the lease until the hold context is cancelled or a +// refresh fails; on failure it marks the lock lost. +func (m *Mutex) renewLoop(ctx context.Context, lost chan struct{}) { + ticker := time.NewTicker(m.renewInterval) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + ok, err := m.locker.Refresh(ctx, m.key, m.owner, m.ttl) + if ctx.Err() != nil { + return + } + if err != nil || !ok { + m.mu.Lock() + if m.held { + m.held = false + close(lost) + } + m.cancelHold = nil + m.mu.Unlock() + return + } + } + } +} + +// jitter returns d scaled by a random factor in [0.75, 1.25). +func jitter(d time.Duration) time.Duration { + if d <= 0 { + return 0 + } + return time.Duration(float64(d) * (0.75 + rand.Float64()*0.5)) +} diff --git a/sdk/go/coord/once.go b/sdk/go/coord/once.go new file mode 100644 index 0000000..91cc97e --- /dev/null +++ b/sdk/go/coord/once.go @@ -0,0 +1,62 @@ +package coord + +import ( + "context" + "time" +) + +// Once provides cluster-wide run-exactly-once semantics for a named action, +// backed by an atomic set-if-absent on a marker key. The first caller to claim +// the key runs fn; concurrent and later callers observe the claim and skip. +// +// On fn error the marker is released so the action can be retried by a +// subsequent caller (the failed run does not "consume" the once). On success +// the marker persists (subject to the optional TTL) so the action is not +// repeated. +type Once struct { + locker Locker + key string + owner string + ttl time.Duration // 0 = marker never expires +} + +// OnceOption customizes a Once. +type OnceOption func(*Once) + +// WithOnceTTL bounds how long the completion marker persists. After it expires +// the action becomes runnable again. Zero (default) means the marker is +// permanent. Use a TTL for periodic "once per window" semantics. +func WithOnceTTL(ttl time.Duration) OnceOption { + return func(o *Once) { o.ttl = ttl } +} + +// NewOnce creates a Once guarding the action identified by key. +func NewOnce(locker Locker, key string, opts ...OnceOption) *Once { + o := &Once{locker: locker, key: key, owner: NewOwnerID()} + for _, opt := range opts { + opt(o) + } + return o +} + +// Do runs fn exactly once cluster-wide for this key. It returns ran=true iff +// this caller won the claim and executed fn (in which case err is fn's error). +// ran=false means another caller already claimed the action; err is then any +// backend error from attempting the claim (nil on a normal skip). +func (o *Once) Do(ctx context.Context, fn func(ctx context.Context) error) (ran bool, err error) { + claimed, err := o.locker.TryAcquire(ctx, o.key, o.owner, o.ttl) + if err != nil { + return false, err + } + if !claimed { + return false, nil + } + if fnErr := fn(ctx); fnErr != nil { + // Release the marker so the action can be retried by someone else. + // Best-effort: if release fails the marker's TTL (when set) still + // eventually frees it. + _, _ = o.locker.Release(context.WithoutCancel(ctx), o.key, o.owner) + return true, fnErr + } + return true, nil +} diff --git a/sdk/go/coord/semaphore.go b/sdk/go/coord/semaphore.go new file mode 100644 index 0000000..eb29b7c --- /dev/null +++ b/sdk/go/coord/semaphore.go @@ -0,0 +1,70 @@ +package coord + +import ( + "context" + "fmt" + "time" +) + +// Semaphore is a distributed counting semaphore over a single KV counter key. +// It admits up to N concurrent permit holders cluster-wide, backed by the +// atomic guarded-counter ops (IncrementIf ceiling=N / DecrementIf floor=0). +// +// Caveat: permits are not leased. A holder that crashes without calling +// Release leaks its permit until an operator resets the counter. For +// crash-safe single-holder exclusion use Mutex (TTL-leased) instead; Semaphore +// fits cooperative concurrency limiting where holders reliably release. +type Semaphore struct { + counter Counter + key string + limit int64 +} + +// NewSemaphore creates an N-permit Semaphore over key. limit must be >= 1. +func NewSemaphore(counter Counter, key string, limit int64) (*Semaphore, error) { + if limit < 1 { + return nil, fmt.Errorf("coord: semaphore limit must be >= 1, got %d", limit) + } + return &Semaphore{counter: counter, key: key, limit: limit}, nil +} + +// TryAcquire takes one permit without blocking. Returns true iff a permit was +// granted (i.e. fewer than N were outstanding). +func (s *Semaphore) TryAcquire(ctx context.Context) (bool, error) { + _, applied, err := s.counter.IncrementIf(ctx, s.key, 1, s.limit) + if err != nil { + return false, err + } + return applied, nil +} + +// Acquire blocks until a permit is granted or ctx is cancelled. Failed +// attempts are retried after pollInterval (jittered); a non-positive +// pollInterval uses DefaultRenewInterval. +func (s *Semaphore) Acquire(ctx context.Context, pollInterval time.Duration) error { + if pollInterval <= 0 { + pollInterval = DefaultRenewInterval + } + for { + ok, err := s.TryAcquire(ctx) + if err != nil { + return err + } + if ok { + return nil + } + if !sleepCtx(ctx, jitter(pollInterval)) { + return ctx.Err() + } + } +} + +// Release returns one permit. Returns true iff a permit was released (the +// counter was above its floor of 0). +func (s *Semaphore) Release(ctx context.Context) (bool, error) { + _, applied, err := s.counter.DecrementIf(ctx, s.key, 1, 0) + if err != nil { + return false, err + } + return applied, nil +} diff --git a/sdk/go/go.mod b/sdk/go/go.mod index badd47d..987114c 100644 --- a/sdk/go/go.mod +++ b/sdk/go/go.mod @@ -1,11 +1,11 @@ module github.com/scitrera/aether/sdk/go -go 1.25.10 +go 1.25.12 require ( github.com/docker/docker v28.5.2+incompatible - github.com/scitrera/aether/api v0.2.1 - github.com/scitrera/go-backpressure v0.1.0 + github.com/scitrera/aether/api v0.2.2 + github.com/scitrera/go-backpressure v0.1.1 google.golang.org/grpc v1.81.1 google.golang.org/protobuf v1.36.11 ) diff --git a/sdk/go/go.sum b/sdk/go/go.sum index a6589b9..c69d363 100644 --- a/sdk/go/go.sum +++ b/sdk/go/go.sum @@ -57,8 +57,8 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/scitrera/go-backpressure v0.1.0 h1:I55SB+PUqLB92FYueKgLRLwECsYGJje6K6NaAeSdAxM= -github.com/scitrera/go-backpressure v0.1.0/go.mod h1:2bK7pLzy0LE3BPQ029FflI06VvjHQNAevkS4pTUGYmQ= +github.com/scitrera/go-backpressure v0.1.1 h1:MhLWsg0qAVjF43/yQ6JHlrvT6sWpRIIPSu/58XjMbo0= +github.com/scitrera/go-backpressure v0.1.1/go.mod h1:2bK7pLzy0LE3BPQ029FflI06VvjHQNAevkS4pTUGYmQ= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= diff --git a/sdk/python-ag2/pyproject.toml b/sdk/python-ag2/pyproject.toml index d654107..da2cd74 100644 --- a/sdk/python-ag2/pyproject.toml +++ b/sdk/python-ag2/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "scitrera-aether-ag2" -version = "0.0.1" +version = "0.0.2" description = "ag2 (AutoGen 2) adapter for the Scitrera Aether distributed control plane" readme = "README.md" license = "Apache-2.0" diff --git a/sdk/python-ag2/scitrera_aether_ag2/__init__.py b/sdk/python-ag2/scitrera_aether_ag2/__init__.py index c9744ce..9053df5 100644 --- a/sdk/python-ag2/scitrera_aether_ag2/__init__.py +++ b/sdk/python-ag2/scitrera_aether_ag2/__init__.py @@ -1,6 +1,6 @@ """ag2 ↔ Aether adapter — public API re-exports.""" -__version__ = "0.0.1" +__version__ = "0.0.2" from .identity import AetherIdentity from .remote import ( diff --git a/sdk/python-client/pyproject.toml b/sdk/python-client/pyproject.toml index 9666c7a..967c8a4 100644 --- a/sdk/python-client/pyproject.toml +++ b/sdk/python-client/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "scitrera-aether-client" -version = "0.2.1" +version = "0.2.2" description = "Python client SDK for Scitrera Aether distributed control plane" readme = "README.md" license = "Apache-2.0" @@ -36,13 +36,13 @@ classifiers = [ "Framework :: AsyncIO", ] dependencies = [ - "grpcio==1.80.0", + "grpcio==1.81.1", "protobuf>6", ] [project.optional-dependencies] dev = [ - "grpcio==1.80.0", + "grpcio==1.81.1", "pytest==9.0.3", "pytest==9.0.3", "mypy>=1.8.0", diff --git a/sdk/python-client/scitrera_aether_client/__init__.py b/sdk/python-client/scitrera_aether_client/__init__.py index 273fc40..c3840dc 100644 --- a/sdk/python-client/scitrera_aether_client/__init__.py +++ b/sdk/python-client/scitrera_aether_client/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.2.1" +__version__ = "0.2.2" # Import the proxy module for its side effect: installs the # ``ProxyHttpResponse`` / ``ProxyHttpBodyChunk`` dispatcher hook on @@ -41,9 +41,15 @@ KV_SCOPE_USER_SHARED, KV_SCOPE_USER_WORKSPACE_SHARED, + # Canonical principal-type strings (mirror server models.PrincipalType) + PrincipalType, + # Auth credentials builder Credentials, + # Service-addressing wildcard sentinel + SERVICE_WILDCARD, + # internal _logging_lite_hook, ) @@ -394,6 +400,9 @@ # Types - Other 'Credentials', + # Service-addressing wildcard sentinel + 'SERVICE_WILDCARD', + # Metric builder 'MetricBuilder', 'new_metric', diff --git a/sdk/python-client/scitrera_aether_client/_common.py b/sdk/python-client/scitrera_aether_client/_common.py index efdc7b0..5ee2b3d 100644 --- a/sdk/python-client/scitrera_aether_client/_common.py +++ b/sdk/python-client/scitrera_aether_client/_common.py @@ -197,6 +197,7 @@ def create_service_init(implementation: str, specifier: str, credentials: Optional[Dict[str, str]] = None, resume_session_id: str = "", extensions: Optional[List["aether_pb2.ExtensionDeclaration"]] = None, + no_pool_consumer: bool = False, ) -> aether_pb2.InitConnection: """Create an InitConnection message for a Service principal (workspace-less). @@ -205,11 +206,17 @@ def create_service_init(implementation: str, specifier: str, privileged work on behalf of users via ``AuthorizationContext``. Canonical identity string: ``sv::{implementation}::{specifier}``. + + ``no_pool_consumer`` opts this connection OUT of pool-task routing — the + gateway will not add it to the implementation worker index, so it is never + targeted for POOL-mode task assignments. Set it for serve-only instances + that register no task handlers (default False = consumer, back-compat). """ return _apply_client_version_meta(aether_pb2.InitConnection( service=aether_pb2.ServiceIdentity( implementation=implementation, specifier=specifier, + no_pool_consumer=no_pool_consumer, ), credentials=credentials or {}, resume_session_id=resume_session_id, @@ -230,18 +237,57 @@ def create_service_init(implementation: str, specifier: str, IDENTITY_SEP = "::" +class _ServiceWildcard: + """Sentinel type for the service-addressing wildcard specifier. + + A single module-level instance (:data:`SERVICE_WILDCARD`) is the only + value that should ever exist; identity comparison (``is``) is the + intended check. ``repr`` is descriptive so it reads clearly in logs + and assertion failures. + """ + + __slots__ = () + + def __repr__(self) -> str: # pragma: no cover - trivial + return "SERVICE_WILDCARD" + + +# Explicit, opt-in wildcard for service addressing. Pass this as the +# ``specifier`` to :func:`create_topic_service` (or to a client's +# ``send_message_to_service``) to emit the bare ``sv::{impl}`` wildcard +# target. The gateway (``models.topics.ParseSendTarget``) resolves a bare +# 2-segment ``sv::{impl}`` to any registered ``sv::{impl}::{spec}`` instance. +# +# This MUST be opted into deliberately: a normal non-empty specifier emits +# the canonical 3-segment ``sv::{impl}::{spec}`` and an empty string ``""`` +# STAYS 3-segment (``sv::{impl}::``) — only this sentinel produces the +# 2-segment wildcard, so "" can never silently become a wildcard. +SERVICE_WILDCARD = _ServiceWildcard() + + def create_topic_agent(workspace: str, implementation: str, specifier: str) -> str: """Create a topic string for a specific agent.""" return f"ag{IDENTITY_SEP}{workspace}{IDENTITY_SEP}{implementation}{IDENTITY_SEP}{specifier}" -def create_topic_service(implementation: str, specifier: str) -> str: - """Create a topic string for a specific service principal. +def create_topic_service(implementation, specifier) -> str: + """Create a topic string for a service principal. + + Two shapes, selected by ``specifier``: - Service principals are workspace-less (canonical identity is - ``sv::{implementation}::{specifier}``); the topic mirrors that shape. - Mirrors :func:`models.topics.ServiceTopic` on the gateway side. + * A concrete ``specifier`` (any ``str``, including the empty string) + emits the canonical 3-segment ``sv::{implementation}::{specifier}``. + An empty string therefore yields ``sv::{implementation}::`` (a + concrete, trailing-``::`` target) — it is NOT treated as a wildcard. + * The :data:`SERVICE_WILDCARD` sentinel emits the bare 2-segment + ``sv::{implementation}`` wildcard, which the gateway resolves to any + registered ``sv::{implementation}::{spec}`` instance. + + Mirrors :func:`models.topics.ServiceTopic` (concrete) and + :func:`models.topics.ParseSendTarget` (wildcard) on the gateway side. """ + if specifier is SERVICE_WILDCARD: + return f"sv{IDENTITY_SEP}{implementation}" return f"sv{IDENTITY_SEP}{implementation}{IDENTITY_SEP}{specifier}" @@ -267,6 +313,18 @@ def create_topic_user_workspace(user_id: str, workspace: str) -> str: return f"uw{IDENTITY_SEP}{user_id}{IDENTITY_SEP}{workspace}" +def create_topic_user_broadcast(user_id: str) -> str: + """Create a per-user broadcast topic (uu::{user_id}). + + Reaches every one of a user's open windows regardless of which workspace + each window is currently viewing -- the workspace-agnostic, non-progress + complement to the per-user progress topic. Only platform principals + (service / workflow-engine / bridge) may publish; the gateway rejects + sends from workspace-scoped principals. + """ + return f"uu{IDENTITY_SEP}{user_id}" + + def create_topic_global_agents(workspace: str) -> str: """Create a global broadcast topic for all agents in a workspace.""" return f"ga{IDENTITY_SEP}{workspace}" @@ -311,6 +369,26 @@ def create_topic_global_users(workspace: str) -> str: KV_SCOPE_USER_SHARED = "user-shared" KV_SCOPE_USER_WORKSPACE_SHARED = "user-workspace-shared" + +class PrincipalType: + """Canonical Aether principal-type strings. + + These MUST match the server's ``models.PrincipalType`` constants verbatim + (server/pkg/models/identity.go) — the gateway compares them exactly (e.g. + api-key ``parsePrincipalType``), so a case/spelling mismatch like ``"user"`` + vs ``"User"`` is rejected. Always reference these constants instead of + hardcoding the string when minting tokens / asserting an identity type. + """ + + AGENT = "Agent" + TASK = "Task" + USER = "User" + SERVICE = "Service" + WORKFLOW_ENGINE = "WorkflowEngine" + METRICS_BRIDGE = "MetricsBridge" + ORCHESTRATOR = "Orchestrator" + BRIDGE = "Bridge" + _SCOPE_MAP = { "global": aether_pb2.KVOperation.GLOBAL, "workspace": aether_pb2.KVOperation.WORKSPACE, diff --git a/sdk/python-client/scitrera_aether_client/admin.py b/sdk/python-client/scitrera_aether_client/admin.py index dbabf31..a006a9f 100644 --- a/sdk/python-client/scitrera_aether_client/admin.py +++ b/sdk/python-client/scitrera_aether_client/admin.py @@ -141,12 +141,17 @@ def create_acl_rule(self, granted_by: str = "", reason: str = "", expires_at: int = 0, + authorization: Optional[aether_pb2.AuthorizationContext] = None, timeout: float = 10.0): """Create an ACL rule granting access. ``access_level`` uses the numeric tiers from ``internal/acl/types.go``: ``0=NONE, 10=READ, 20=READWRITE, 30=MANAGE, 40=ADMIN, 50=SUPERADMIN``. + ``authorization`` is an optional on-behalf-of authority context. When + set, the gateway runs the admin ACL check against the subject (the + user) rather than the actor (the platform-server). + Note: The TS surface uses a string ``permission`` and a ``metadata`` map. The Python proto uses ``access_level`` (int) and ``granted_by`` @@ -165,14 +170,55 @@ def create_acl_rule(self, reason=reason, expires_at=expires_at, ), + authorization=authorization, ) return self._client.acl_op(op, timeout=timeout) - def delete_acl_rule(self, rule_id: str, timeout: float = 10.0): - """Delete (revoke) an ACL rule by ID.""" + def delete_acl_rule(self, rule_id: str, + authorization: Optional[aether_pb2.AuthorizationContext] = None, + timeout: float = 10.0): + """Delete (revoke) an ACL rule by its UUID rule_id. + + The gateway resolves the rule's principal+resource from the UUID and + then deletes by composite key. Returns an error if the rule_id is not + found. + + ``authorization`` is an optional on-behalf-of authority context (see + :meth:`create_acl_rule`). + """ op = aether_pb2.ACLOperation( op=aether_pb2.ACLOperation.REVOKE, rule_id=rule_id, + authorization=authorization, + ) + return self._client.acl_op(op, timeout=timeout) + + def revoke_acl_rule(self, + principal_type: str, + principal_id: str, + resource_type: str, + resource_id: str, + authorization: Optional[aether_pb2.AuthorizationContext] = None, + timeout: float = 10.0): + """Revoke an ACL rule by its composite (principal + resource) key. + + This is the direct revoke path: it sends ``ACLOperation(op=REVOKE, + rule_filter=ACLRuleFilter(...))`` which the gateway maps to a delete + by composite key without any UUID lookup. Prefer this method when you + already have the principal and resource identifiers. + + ``authorization`` is an optional on-behalf-of authority context (see + :meth:`create_acl_rule`). + """ + op = aether_pb2.ACLOperation( + op=aether_pb2.ACLOperation.REVOKE, + rule_filter=aether_pb2.ACLRuleFilter( + principal_type=principal_type, + principal_id=principal_id, + resource_type=resource_type, + resource_id=resource_id, + ), + authorization=authorization, ) return self._client.acl_op(op, timeout=timeout) @@ -183,8 +229,13 @@ def list_acl_rules(self, resource_id: str = "", limit: int = 0, offset: int = 0, + authorization: Optional[aether_pb2.AuthorizationContext] = None, timeout: float = 10.0): - """List ACL rules with optional filters.""" + """List ACL rules with optional filters. + + ``authorization`` is an optional on-behalf-of authority context (see + :meth:`create_acl_rule`). + """ op = aether_pb2.ACLOperation( op=aether_pb2.ACLOperation.LIST_RULES, rule_filter=aether_pb2.ACLRuleFilter( @@ -195,6 +246,7 @@ def list_acl_rules(self, limit=limit, offset=offset, ), + authorization=authorization, ) return self._client.acl_op(op, timeout=timeout) @@ -229,6 +281,269 @@ def set_fallback_policy(self, ) return self._client.acl_op(op, timeout=timeout) + # ------------------------------------------------------------------ + # Role / Group Operations + # ------------------------------------------------------------------ + # + # A group is a named collection of principals; a role is a named + # permission bundle (its permissions are granted with + # :meth:`create_acl_rule` using ``principal_type="role"``, + # ``principal_id=``). Membership/assignment edges are resolved + # transitively and combined additively at evaluation time. These mirror + # the Go SDK helpers in ``sdk/go/aether/admin_roles.go``. + + def create_role(self, + name: str, + description: str = "", + created_by: str = "", + metadata: Optional[Dict[str, str]] = None, + timeout: float = 10.0): + """Create a named role. + + Grant its permissions with :meth:`create_acl_rule` using + ``principal_type="role"``, ``principal_id=``. + + Note: + Creating a role that already exists returns a response with + ``success=False`` and an ``error`` like ``ErrRoleExists`` rather + than raising; callers decide how to treat that. + """ + op = aether_pb2.ACLOperation( + op=aether_pb2.ACLOperation.CREATE_ROLE, + role_request=aether_pb2.ACLRoleRequest( + name=name, + description=description, + created_by=created_by, + metadata=metadata or {}, + ), + ) + return self._client.acl_op(op, timeout=timeout) + + def delete_role(self, name: str, timeout: float = 10.0): + """Delete a role (and its permission rules) by name.""" + op = aether_pb2.ACLOperation( + op=aether_pb2.ACLOperation.DELETE_ROLE, + name=name, + ) + return self._client.acl_op(op, timeout=timeout) + + def get_role(self, name: str, timeout: float = 10.0): + """Fetch a role by name.""" + op = aether_pb2.ACLOperation( + op=aether_pb2.ACLOperation.GET_ROLE, + name=name, + ) + return self._client.acl_op(op, timeout=timeout) + + def list_roles(self, timeout: float = 10.0): + """List all roles.""" + op = aether_pb2.ACLOperation( + op=aether_pb2.ACLOperation.LIST_ROLES, + ) + return self._client.acl_op(op, timeout=timeout) + + def assign_role(self, + name: str, + assignee_type: str, + assignee_id: str, + granted_by: str = "", + expires_at: int = 0, + timeout: float = 10.0): + """Assign (or refresh) a role to a principal or group. + + ``assignee_type`` is a principal type or ``"group"``. ``expires_at`` + is Unix seconds; ``0`` means no expiry. + """ + op = aether_pb2.ACLOperation( + op=aether_pb2.ACLOperation.ASSIGN_ROLE, + name=name, + assignment_request=aether_pb2.ACLRoleAssignmentRequest( + assignee_type=assignee_type, + assignee_id=assignee_id, + granted_by=granted_by, + expires_at=expires_at, + ), + ) + return self._client.acl_op(op, timeout=timeout) + + def unassign_role(self, + name: str, + assignee_type: str, + assignee_id: str, + timeout: float = 10.0): + """Remove a role assignment from a principal or group.""" + op = aether_pb2.ACLOperation( + op=aether_pb2.ACLOperation.UNASSIGN_ROLE, + name=name, + principal=aether_pb2.PrincipalRef( + principal_type=assignee_type, + principal_id=assignee_id, + ), + ) + return self._client.acl_op(op, timeout=timeout) + + def list_role_assignments(self, name: str, timeout: float = 10.0): + """List the direct assignees of a role.""" + op = aether_pb2.ACLOperation( + op=aether_pb2.ACLOperation.LIST_ROLE_ASSIGNMENTS, + name=name, + ) + return self._client.acl_op(op, timeout=timeout) + + def list_principal_roles(self, + principal_type: str, + principal_id: str, + timeout: float = 10.0): + """List the roles directly assigned to a principal.""" + op = aether_pb2.ACLOperation( + op=aether_pb2.ACLOperation.LIST_PRINCIPAL_ROLES, + principal=aether_pb2.PrincipalRef( + principal_type=principal_type, + principal_id=principal_id, + ), + ) + return self._client.acl_op(op, timeout=timeout) + + def create_group(self, + name: str, + description: str = "", + created_by: str = "", + metadata: Optional[Dict[str, str]] = None, + timeout: float = 10.0): + """Create a named group. + + Note: + Creating a group that already exists returns a response with + ``success=False`` and an ``error`` rather than raising; callers + decide how to treat that. + """ + op = aether_pb2.ACLOperation( + op=aether_pb2.ACLOperation.CREATE_GROUP, + group_request=aether_pb2.ACLGroupRequest( + name=name, + description=description, + created_by=created_by, + metadata=metadata or {}, + ), + ) + return self._client.acl_op(op, timeout=timeout) + + def delete_group(self, name: str, timeout: float = 10.0): + """Delete a group by name.""" + op = aether_pb2.ACLOperation( + op=aether_pb2.ACLOperation.DELETE_GROUP, + name=name, + ) + return self._client.acl_op(op, timeout=timeout) + + def get_group(self, name: str, timeout: float = 10.0): + """Fetch a group by name.""" + op = aether_pb2.ACLOperation( + op=aether_pb2.ACLOperation.GET_GROUP, + name=name, + ) + return self._client.acl_op(op, timeout=timeout) + + def list_groups(self, timeout: float = 10.0): + """List all groups.""" + op = aether_pb2.ACLOperation( + op=aether_pb2.ACLOperation.LIST_GROUPS, + ) + return self._client.acl_op(op, timeout=timeout) + + def add_group_member(self, + name: str, + member_type: str, + member_id: str, + granted_by: str = "", + expires_at: int = 0, + timeout: float = 10.0): + """Add (or refresh) a member of a group. + + ``member_type`` is a principal type or ``"group"`` (for nesting). + ``expires_at`` is Unix seconds; ``0`` means no expiry. + """ + op = aether_pb2.ACLOperation( + op=aether_pb2.ACLOperation.ADD_GROUP_MEMBER, + name=name, + member_request=aether_pb2.ACLGroupMemberRequest( + member_type=member_type, + member_id=member_id, + granted_by=granted_by, + expires_at=expires_at, + ), + ) + return self._client.acl_op(op, timeout=timeout) + + def remove_group_member(self, + name: str, + member_type: str, + member_id: str, + timeout: float = 10.0): + """Remove a member from a group.""" + op = aether_pb2.ACLOperation( + op=aether_pb2.ACLOperation.REMOVE_GROUP_MEMBER, + name=name, + principal=aether_pb2.PrincipalRef( + principal_type=member_type, + principal_id=member_id, + ), + ) + return self._client.acl_op(op, timeout=timeout) + + def list_group_members(self, name: str, timeout: float = 10.0): + """List the direct members of a group.""" + op = aether_pb2.ACLOperation( + op=aether_pb2.ACLOperation.LIST_GROUP_MEMBERS, + name=name, + ) + return self._client.acl_op(op, timeout=timeout) + + def list_principal_groups(self, + principal_type: str, + principal_id: str, + timeout: float = 10.0): + """List the groups a principal is a direct member of.""" + op = aether_pb2.ACLOperation( + op=aether_pb2.ACLOperation.LIST_PRINCIPAL_GROUPS, + principal=aether_pb2.PrincipalRef( + principal_type=principal_type, + principal_id=principal_id, + ), + ) + return self._client.acl_op(op, timeout=timeout) + + def explain_access(self, + principal_type: str, + principal_id: str, + resource_type: str, + resource_id: str, + required_level: int = 0, + timeout: float = 10.0): + """Explain how a principal's effective access to a resource is decided. + + Resolves the subject set (self + transitive groups/roles), every rule + that matched, and the resulting decision. This does not gate access, + but the gateway records an ``explain_access`` audit event attributing + the call to the connected principal. ``required_level`` is the + threshold the decision is compared against (``0`` = NONE). + + Read the result from the response's ``explanation`` field + (:class:`~aether_pb2.ACLAccessExplanationInfo`), whose ``allowed`` and + ``effective_access_level`` fields carry the resolved decision. + """ + op = aether_pb2.ACLOperation( + op=aether_pb2.ACLOperation.EXPLAIN_ACCESS, + principal=aether_pb2.PrincipalRef( + principal_type=principal_type, + principal_id=principal_id, + ), + resource_type=resource_type, + resource_id=resource_id, + required_level=required_level, + ) + return self._client.acl_op(op, timeout=timeout) + # ------------------------------------------------------------------ # Workspace Operations # ------------------------------------------------------------------ diff --git a/sdk/python-client/scitrera_aether_client/admin_async.py b/sdk/python-client/scitrera_aether_client/admin_async.py index ccf109e..916eca3 100644 --- a/sdk/python-client/scitrera_aether_client/admin_async.py +++ b/sdk/python-client/scitrera_aether_client/admin_async.py @@ -116,8 +116,14 @@ async def create_acl_rule(self, granted_by: str = "", reason: str = "", expires_at: int = 0, + authorization: Optional[aether_pb2.AuthorizationContext] = None, timeout: float = 10.0): - """Create an ACL rule granting access. See :meth:`AdminClient.create_acl_rule`.""" + """Create an ACL rule granting access. See :meth:`AdminClient.create_acl_rule`. + + ``authorization`` is an optional on-behalf-of authority context. When + set, the gateway runs the admin ACL check against the subject (the + user) rather than the actor (the platform-server). + """ op = aether_pb2.ACLOperation( op=aether_pb2.ACLOperation.GRANT, grant_request=aether_pb2.ACLGrantRequest( @@ -130,14 +136,55 @@ async def create_acl_rule(self, reason=reason, expires_at=expires_at, ), + authorization=authorization, ) return await self._client.acl_op(op, timeout=timeout) - async def delete_acl_rule(self, rule_id: str, timeout: float = 10.0): - """Delete (revoke) an ACL rule by ID.""" + async def delete_acl_rule(self, rule_id: str, + authorization: Optional[aether_pb2.AuthorizationContext] = None, + timeout: float = 10.0): + """Delete (revoke) an ACL rule by its UUID rule_id. + + The gateway resolves the rule's principal+resource from the UUID and + then deletes by composite key. Returns an error if the rule_id is not + found. + + ``authorization`` is an optional on-behalf-of authority context (see + :meth:`create_acl_rule`). + """ op = aether_pb2.ACLOperation( op=aether_pb2.ACLOperation.REVOKE, rule_id=rule_id, + authorization=authorization, + ) + return await self._client.acl_op(op, timeout=timeout) + + async def revoke_acl_rule(self, + principal_type: str, + principal_id: str, + resource_type: str, + resource_id: str, + authorization: Optional[aether_pb2.AuthorizationContext] = None, + timeout: float = 10.0): + """Revoke an ACL rule by its composite (principal + resource) key. + + This is the direct revoke path: it sends ``ACLOperation(op=REVOKE, + rule_filter=ACLRuleFilter(...))`` which the gateway maps to a delete + by composite key without any UUID lookup. Prefer this method when you + already have the principal and resource identifiers. + + ``authorization`` is an optional on-behalf-of authority context (see + :meth:`create_acl_rule`). + """ + op = aether_pb2.ACLOperation( + op=aether_pb2.ACLOperation.REVOKE, + rule_filter=aether_pb2.ACLRuleFilter( + principal_type=principal_type, + principal_id=principal_id, + resource_type=resource_type, + resource_id=resource_id, + ), + authorization=authorization, ) return await self._client.acl_op(op, timeout=timeout) @@ -148,8 +195,13 @@ async def list_acl_rules(self, resource_id: str = "", limit: int = 0, offset: int = 0, + authorization: Optional[aether_pb2.AuthorizationContext] = None, timeout: float = 10.0): - """List ACL rules with optional filters.""" + """List ACL rules with optional filters. + + ``authorization`` is an optional on-behalf-of authority context (see + :meth:`create_acl_rule`). + """ op = aether_pb2.ACLOperation( op=aether_pb2.ACLOperation.LIST_RULES, rule_filter=aether_pb2.ACLRuleFilter( @@ -160,6 +212,7 @@ async def list_acl_rules(self, limit=limit, offset=offset, ), + authorization=authorization, ) return await self._client.acl_op(op, timeout=timeout) @@ -187,6 +240,269 @@ async def set_fallback_policy(self, ) return await self._client.acl_op(op, timeout=timeout) + # ------------------------------------------------------------------ + # Role / Group Operations + # ------------------------------------------------------------------ + # + # A group is a named collection of principals; a role is a named + # permission bundle (its permissions are granted with + # :meth:`create_acl_rule` using ``principal_type="role"``, + # ``principal_id=``). Membership/assignment edges are resolved + # transitively and combined additively at evaluation time. These mirror + # the Go SDK helpers in ``sdk/go/aether/admin_roles.go``. + + async def create_role(self, + name: str, + description: str = "", + created_by: str = "", + metadata: Optional[Dict[str, str]] = None, + timeout: float = 10.0): + """Create a named role. + + Grant its permissions with :meth:`create_acl_rule` using + ``principal_type="role"``, ``principal_id=``. + + Note: + Creating a role that already exists returns a response with + ``success=False`` and an ``error`` like ``ErrRoleExists`` rather + than raising; callers decide how to treat that. + """ + op = aether_pb2.ACLOperation( + op=aether_pb2.ACLOperation.CREATE_ROLE, + role_request=aether_pb2.ACLRoleRequest( + name=name, + description=description, + created_by=created_by, + metadata=metadata or {}, + ), + ) + return await self._client.acl_op(op, timeout=timeout) + + async def delete_role(self, name: str, timeout: float = 10.0): + """Delete a role (and its permission rules) by name.""" + op = aether_pb2.ACLOperation( + op=aether_pb2.ACLOperation.DELETE_ROLE, + name=name, + ) + return await self._client.acl_op(op, timeout=timeout) + + async def get_role(self, name: str, timeout: float = 10.0): + """Fetch a role by name.""" + op = aether_pb2.ACLOperation( + op=aether_pb2.ACLOperation.GET_ROLE, + name=name, + ) + return await self._client.acl_op(op, timeout=timeout) + + async def list_roles(self, timeout: float = 10.0): + """List all roles.""" + op = aether_pb2.ACLOperation( + op=aether_pb2.ACLOperation.LIST_ROLES, + ) + return await self._client.acl_op(op, timeout=timeout) + + async def assign_role(self, + name: str, + assignee_type: str, + assignee_id: str, + granted_by: str = "", + expires_at: int = 0, + timeout: float = 10.0): + """Assign (or refresh) a role to a principal or group. + + ``assignee_type`` is a principal type or ``"group"``. ``expires_at`` + is Unix seconds; ``0`` means no expiry. + """ + op = aether_pb2.ACLOperation( + op=aether_pb2.ACLOperation.ASSIGN_ROLE, + name=name, + assignment_request=aether_pb2.ACLRoleAssignmentRequest( + assignee_type=assignee_type, + assignee_id=assignee_id, + granted_by=granted_by, + expires_at=expires_at, + ), + ) + return await self._client.acl_op(op, timeout=timeout) + + async def unassign_role(self, + name: str, + assignee_type: str, + assignee_id: str, + timeout: float = 10.0): + """Remove a role assignment from a principal or group.""" + op = aether_pb2.ACLOperation( + op=aether_pb2.ACLOperation.UNASSIGN_ROLE, + name=name, + principal=aether_pb2.PrincipalRef( + principal_type=assignee_type, + principal_id=assignee_id, + ), + ) + return await self._client.acl_op(op, timeout=timeout) + + async def list_role_assignments(self, name: str, timeout: float = 10.0): + """List the direct assignees of a role.""" + op = aether_pb2.ACLOperation( + op=aether_pb2.ACLOperation.LIST_ROLE_ASSIGNMENTS, + name=name, + ) + return await self._client.acl_op(op, timeout=timeout) + + async def list_principal_roles(self, + principal_type: str, + principal_id: str, + timeout: float = 10.0): + """List the roles directly assigned to a principal.""" + op = aether_pb2.ACLOperation( + op=aether_pb2.ACLOperation.LIST_PRINCIPAL_ROLES, + principal=aether_pb2.PrincipalRef( + principal_type=principal_type, + principal_id=principal_id, + ), + ) + return await self._client.acl_op(op, timeout=timeout) + + async def create_group(self, + name: str, + description: str = "", + created_by: str = "", + metadata: Optional[Dict[str, str]] = None, + timeout: float = 10.0): + """Create a named group. + + Note: + Creating a group that already exists returns a response with + ``success=False`` and an ``error`` rather than raising; callers + decide how to treat that. + """ + op = aether_pb2.ACLOperation( + op=aether_pb2.ACLOperation.CREATE_GROUP, + group_request=aether_pb2.ACLGroupRequest( + name=name, + description=description, + created_by=created_by, + metadata=metadata or {}, + ), + ) + return await self._client.acl_op(op, timeout=timeout) + + async def delete_group(self, name: str, timeout: float = 10.0): + """Delete a group by name.""" + op = aether_pb2.ACLOperation( + op=aether_pb2.ACLOperation.DELETE_GROUP, + name=name, + ) + return await self._client.acl_op(op, timeout=timeout) + + async def get_group(self, name: str, timeout: float = 10.0): + """Fetch a group by name.""" + op = aether_pb2.ACLOperation( + op=aether_pb2.ACLOperation.GET_GROUP, + name=name, + ) + return await self._client.acl_op(op, timeout=timeout) + + async def list_groups(self, timeout: float = 10.0): + """List all groups.""" + op = aether_pb2.ACLOperation( + op=aether_pb2.ACLOperation.LIST_GROUPS, + ) + return await self._client.acl_op(op, timeout=timeout) + + async def add_group_member(self, + name: str, + member_type: str, + member_id: str, + granted_by: str = "", + expires_at: int = 0, + timeout: float = 10.0): + """Add (or refresh) a member of a group. + + ``member_type`` is a principal type or ``"group"`` (for nesting). + ``expires_at`` is Unix seconds; ``0`` means no expiry. + """ + op = aether_pb2.ACLOperation( + op=aether_pb2.ACLOperation.ADD_GROUP_MEMBER, + name=name, + member_request=aether_pb2.ACLGroupMemberRequest( + member_type=member_type, + member_id=member_id, + granted_by=granted_by, + expires_at=expires_at, + ), + ) + return await self._client.acl_op(op, timeout=timeout) + + async def remove_group_member(self, + name: str, + member_type: str, + member_id: str, + timeout: float = 10.0): + """Remove a member from a group.""" + op = aether_pb2.ACLOperation( + op=aether_pb2.ACLOperation.REMOVE_GROUP_MEMBER, + name=name, + principal=aether_pb2.PrincipalRef( + principal_type=member_type, + principal_id=member_id, + ), + ) + return await self._client.acl_op(op, timeout=timeout) + + async def list_group_members(self, name: str, timeout: float = 10.0): + """List the direct members of a group.""" + op = aether_pb2.ACLOperation( + op=aether_pb2.ACLOperation.LIST_GROUP_MEMBERS, + name=name, + ) + return await self._client.acl_op(op, timeout=timeout) + + async def list_principal_groups(self, + principal_type: str, + principal_id: str, + timeout: float = 10.0): + """List the groups a principal is a direct member of.""" + op = aether_pb2.ACLOperation( + op=aether_pb2.ACLOperation.LIST_PRINCIPAL_GROUPS, + principal=aether_pb2.PrincipalRef( + principal_type=principal_type, + principal_id=principal_id, + ), + ) + return await self._client.acl_op(op, timeout=timeout) + + async def explain_access(self, + principal_type: str, + principal_id: str, + resource_type: str, + resource_id: str, + required_level: int = 0, + timeout: float = 10.0): + """Explain how a principal's effective access to a resource is decided. + + Resolves the subject set (self + transitive groups/roles), every rule + that matched, and the resulting decision. This does not gate access, + but the gateway records an ``explain_access`` audit event attributing + the call to the connected principal. ``required_level`` is the + threshold the decision is compared against (``0`` = NONE). + + Read the result from the response's ``explanation`` field + (:class:`~aether_pb2.ACLAccessExplanationInfo`), whose ``allowed`` and + ``effective_access_level`` fields carry the resolved decision. + """ + op = aether_pb2.ACLOperation( + op=aether_pb2.ACLOperation.EXPLAIN_ACCESS, + principal=aether_pb2.PrincipalRef( + principal_type=principal_type, + principal_id=principal_id, + ), + resource_type=resource_type, + resource_id=resource_id, + required_level=required_level, + ) + return await self._client.acl_op(op, timeout=timeout) + # ------------------------------------------------------------------ # Workspace Operations # ------------------------------------------------------------------ diff --git a/sdk/python-client/scitrera_aether_client/client.py b/sdk/python-client/scitrera_aether_client/client.py index 95e7f2e..c232434 100644 --- a/sdk/python-client/scitrera_aether_client/client.py +++ b/sdk/python-client/scitrera_aether_client/client.py @@ -25,6 +25,7 @@ create_topic_global_agents, create_topic_global_users, create_topic_user_workspace, + create_topic_user_broadcast, SELF_ASSIGN, TARGETED, POOL, @@ -1162,7 +1163,6 @@ def kv_increment_if(self, key: str, delta: int = 1, ceiling: int = 0, op=aether_pb2.KVOperation.INCREMENT_IF, scope=_scope_to_proto(scope), key=key, - int_value=delta, delta_value=int(delta), guard_value=ceiling, user_id=user_id, @@ -1189,7 +1189,6 @@ def kv_decrement_if(self, key: str, delta: int = 1, floor: int = 0, op=aether_pb2.KVOperation.DECREMENT_IF, scope=_scope_to_proto(scope), key=key, - int_value=delta, delta_value=int(delta), guard_value=floor, user_id=user_id, @@ -1208,7 +1207,9 @@ def create_task(self, task_type: str, workspace: str, metadata: Optional[Dict[str, str]] = None, payload: Optional[bytes] = None, assignment_mode: int = SELF_ASSIGN, - context_id: str = "") -> None: + context_id: str = "", + priority: int = 0, + retry_policy: Optional[aether_pb2.RetryPolicy] = None) -> None: """ Create a new task. @@ -1223,6 +1224,9 @@ def create_task(self, task_type: str, workspace: str, assignment_mode: SELF_ASSIGN (0), TARGETED (1), or POOL (2) [automatically handled in most cases] context_id: Optional client-minted session identifier (A2A contextId). Tasks sharing a context_id are groupable via TaskFilter.context_id. + priority: Optional dispatch priority (TaskPriority enum value). Higher + priority pending tasks are delivered before lower ones. 0 (UNSPECIFIED) + is normalized to NORMAL by the server. """ if target_agent_id and assignment_mode == SELF_ASSIGN: assignment_mode = TARGETED @@ -1238,6 +1242,8 @@ def create_task(self, task_type: str, workspace: str, metadata=metadata or {}, payload=payload or b"", context_id=context_id, + priority=priority, # type: ignore[arg-type] + retry_policy=retry_policy, ) self.request_queue.put(aether_pb2.UpstreamMessage(create_task=req)) @@ -1250,6 +1256,8 @@ def create_task_sync(self, task_type: str, workspace: str, assignment_mode: int = SELF_ASSIGN, authorization: Optional[aether_pb2.AuthorizationContext] = None, context_id: str = "", + priority: int = 0, + retry_policy: Optional[aether_pb2.RetryPolicy] = None, timeout: float = 10.0) -> Optional[aether_pb2.CreateTaskResponse]: """ Create a new task and wait for the server's response containing the task_id. @@ -1295,6 +1303,8 @@ def create_task_sync(self, task_type: str, workspace: str, authorization=authorization, context_id=context_id, request_id=request_id, + priority=priority, # type: ignore[arg-type] + retry_policy=retry_policy, ) return self._send_sync_op( aether_pb2.UpstreamMessage(create_task=req), request_id, timeout, @@ -3712,6 +3722,16 @@ def send_message_to_user_workspace(self, user_id: str, workspace: str, payload: """Send a message to a user's workspace-scoped topic.""" return self._send_message(create_topic_user_workspace(user_id, workspace), payload, message_type=message_type) + def send_message_to_user_broadcast(self, user_id: str, payload: bytes, + message_type: int = aether_pb2.OPAQUE): + """Send a message to every one of a user's windows (uu::{user_id}). + + Workspace-agnostic, non-progress channel for platform->user + notifications; reaches all of the user's windows regardless of the + workspace each is viewing. + """ + return self._send_message(create_topic_user_broadcast(user_id), payload, message_type=message_type) + def send_metric(self, metric): """Send a metric to the metrics bridge. @@ -3816,7 +3836,8 @@ def __init__(self, implementation: str, specifier: str, tls_client_cert_path: Optional[str] = None, tls_client_key: Optional[bytes] = None, tls_client_key_path: Optional[str] = None, - extensions: Optional[List[aether_pb2.ExtensionDeclaration]] = None): + extensions: Optional[List[aether_pb2.ExtensionDeclaration]] = None, + consumes_pool_tasks: bool = True): if not implementation: raise InvalidArgumentError( message="Service principal requires an implementation identifier", @@ -3840,7 +3861,8 @@ def __init__(self, implementation: str, specifier: str, self.implementation = implementation self.specifier = specifier self.init = create_service_init(implementation, specifier, credentials, - extensions=extensions) + extensions=extensions, + no_pool_consumer=not consumes_pool_tasks) def connect(self, target: str = "localhost:50051"): super()._connect(self.init, target) @@ -3881,3 +3903,16 @@ def send_message_to_user_workspace(self, user_id: str, workspace: str, payload: create_topic_user_workspace(user_id, workspace), payload, message_type=message_type, ) + + def send_message_to_user_broadcast(self, user_id: str, payload: bytes, + message_type: int = aether_pb2.OPAQUE): + """Send a message to every one of a user's windows (uu::{user_id}). + + Workspace-agnostic, non-progress channel for platform->user + notifications; the intended way for a service principal to push an + opaque update to a user without abusing the progress channel. + """ + self._send_message( + create_topic_user_broadcast(user_id), + payload, message_type=message_type, + ) diff --git a/sdk/python-client/scitrera_aether_client/client_async.py b/sdk/python-client/scitrera_aether_client/client_async.py index cea2d9b..07a4d2b 100644 --- a/sdk/python-client/scitrera_aether_client/client_async.py +++ b/sdk/python-client/scitrera_aether_client/client_async.py @@ -5,7 +5,9 @@ """ import asyncio import logging +import os import random +import time import uuid from dataclasses import dataclass from typing import Callable, Optional, Dict, List, Awaitable, Union @@ -25,11 +27,13 @@ create_service_init, create_topic_agent, create_topic_service, + SERVICE_WILDCARD, create_topic_task, create_topic_user, create_topic_global_agents, create_topic_global_users, create_topic_user_workspace, + create_topic_user_broadcast, SELF_ASSIGN, TARGETED, POOL, @@ -75,6 +79,88 @@ DisconnectCallback = Union[Callable[[str], None], Callable[[str], Awaitable[None]]] +# --------------------------------------------------------------------------- +# gRPC keepalive (silent-drop detection) +# --------------------------------------------------------------------------- +# Without keepalive, a SILENT TCP drop (e.g. a hard gateway pod kill that never +# sends RST/FIN, or a stateful firewall/LB that forgets the connection) is never +# observed by the client: the bidi receive loop blocks forever on a half-open +# socket and ``auto_reconnect`` never fires because no error is ever raised. +# HTTP/2 PING keepalive forces the transport to probe the peer on an interval; +# if no PING ACK arrives within the timeout, grpc fails the call with +# UNAVAILABLE, which surfaces as a recoverable error and lets the reconnect +# loop re-establish the stream. +# +# Server policy (gateway/aetherlite KeepaliveEnforcementPolicy): MinTime=10s, +# PermitWithoutStream=false. We MUST NOT ping faster than the server's MinTime +# or it will GOAWAY us with "too_many_pings". The defaults below match the +# server's own ServerParameters cadence (Time=30s / Timeout=10s) with a 30s +# minimum ping floor, leaving comfortable headroom above the 10s MinTime. +# +# All values are env-overridable so they can be tuned per-deployment without a +# code change (e.g. tighten for flaky networks). Env vars take precedence over +# the constructor default only when the constructor kwarg is left at its +# sentinel; see ``_resolve_keepalive_options``. +_DEFAULT_KEEPALIVE_TIME_MS = 30000 +_DEFAULT_KEEPALIVE_TIMEOUT_MS = 10000 +_DEFAULT_KEEPALIVE_PERMIT_WITHOUT_CALLS = 1 +_DEFAULT_HTTP2_MAX_PINGS_WITHOUT_DATA = 0 # 0 = unlimited (long-lived idle stream) +_DEFAULT_HTTP2_MIN_PING_INTERVAL_WITHOUT_DATA_MS = 30000 + + +def _env_int(name: str, default: int) -> int: + """Read an int from the environment, falling back to ``default``.""" + import os + + raw = os.environ.get(name) + if raw is None or raw == "": + return default + try: + return int(raw) + except (TypeError, ValueError): + logger.warning("Invalid int for %s=%r; using default %d", name, raw, default) + return default + + +def _resolve_keepalive_options( + keepalive_time_ms: Optional[int] = None, + keepalive_timeout_ms: Optional[int] = None, + keepalive_permit_without_calls: Optional[int] = None, + http2_max_pings_without_data: Optional[int] = None, + http2_min_ping_interval_without_data_ms: Optional[int] = None, +) -> list: + """Build the gRPC channel ``options`` list for keepalive. + + Resolution order per option: explicit constructor arg (non-None) > + environment variable (``AETHER_GRPC_KEEPALIVE_*``) > module default. + + Returns a list of (key, value) tuples suitable for ``options=`` on both + ``grpc.aio.secure_channel`` and ``grpc.aio.insecure_channel``. + """ + time_ms = keepalive_time_ms if keepalive_time_ms is not None else _env_int( + "AETHER_GRPC_KEEPALIVE_TIME_MS", _DEFAULT_KEEPALIVE_TIME_MS) + timeout_ms = keepalive_timeout_ms if keepalive_timeout_ms is not None else _env_int( + "AETHER_GRPC_KEEPALIVE_TIMEOUT_MS", _DEFAULT_KEEPALIVE_TIMEOUT_MS) + permit = keepalive_permit_without_calls if keepalive_permit_without_calls is not None else _env_int( + "AETHER_GRPC_KEEPALIVE_PERMIT_WITHOUT_CALLS", _DEFAULT_KEEPALIVE_PERMIT_WITHOUT_CALLS) + max_pings = http2_max_pings_without_data if http2_max_pings_without_data is not None else _env_int( + "AETHER_GRPC_HTTP2_MAX_PINGS_WITHOUT_DATA", _DEFAULT_HTTP2_MAX_PINGS_WITHOUT_DATA) + min_ping_ms = (http2_min_ping_interval_without_data_ms + if http2_min_ping_interval_without_data_ms is not None + else _env_int("AETHER_GRPC_HTTP2_MIN_PING_INTERVAL_WITHOUT_DATA_MS", + _DEFAULT_HTTP2_MIN_PING_INTERVAL_WITHOUT_DATA_MS)) + return [ + ("grpc.keepalive_time_ms", time_ms), + ("grpc.keepalive_timeout_ms", timeout_ms), + ("grpc.keepalive_permit_without_calls", permit), + ("grpc.http2.max_pings_without_data", max_pings), + # Client-side floor between pings when the stream has no in-flight data. + # Kept >= the server's MinTime (10s) so we never trip "too_many_pings". + ("grpc.http2.min_time_between_pings_ms", min_ping_ms), + ("grpc.http2.min_ping_interval_without_data_ms", min_ping_ms), + ] + + def _principal_ref(principal_type: str, principal_id: str) -> aether_pb2.PrincipalRef: return aether_pb2.PrincipalRef( principal_type=principal_type, @@ -119,10 +205,11 @@ class ResolvedIdentity: def _principal_type_from_string(t: str) -> "aether_pb2.PrincipalType": return _PRINCIPAL_TYPE_FROM_STRING.get((t or "").lower(), - aether_pb2.PrincipalType.PRINCIPAL_TYPE_UNSPECIFIED) + aether_pb2.PrincipalType.PRINCIPAL_TYPE_UNSPECIFIED) -def _authority_resource_scope_entries(resource_scope: Optional[Dict[str, List[str]]]) -> List[aether_pb2.ACLAuthorityGrantResourceScopeEntry]: +def _authority_resource_scope_entries(resource_scope: Optional[Dict[str, List[str]]]) -> List[ + aether_pb2.ACLAuthorityGrantResourceScopeEntry]: if not resource_scope: return [] @@ -173,7 +260,12 @@ def __init__(self, tls_client_cert: Optional[bytes] = None, tls_client_cert_path: Optional[str] = None, tls_client_key: Optional[bytes] = None, - tls_client_key_path: Optional[str] = None): + tls_client_key_path: Optional[str] = None, + keepalive_time_ms: Optional[int] = None, + keepalive_timeout_ms: Optional[int] = None, + keepalive_permit_without_calls: Optional[int] = None, + http2_max_pings_without_data: Optional[int] = None, + http2_min_ping_interval_without_data_ms: Optional[int] = None): """ Initialize the base async client. @@ -190,6 +282,17 @@ def __init__(self, tls_client_cert_path: Path to client certificate file (alternative to tls_client_cert) tls_client_key: Client private key bytes for mTLS (optional) tls_client_key_path: Path to client private key file (alternative to tls_client_key) + keepalive_time_ms: HTTP/2 PING interval in ms (default env/30000). Drives + silent-drop detection — see ``_resolve_keepalive_options``. + keepalive_timeout_ms: Time to wait for a PING ACK before failing the + connection (default env/10000). + keepalive_permit_without_calls: 1 to keep pinging when no RPC is in + flight, 0 to only ping during active calls (default env/1). + http2_max_pings_without_data: Max pings allowed without sending data; + 0 = unlimited (default env/0). + http2_min_ping_interval_without_data_ms: Client-side floor between + pings when no data is in flight; kept >= the gateway's 10s MinTime + to avoid "too_many_pings" GOAWAY (default env/30000). """ self.target: Optional[str] = None self.channel: Optional[grpc_aio.Channel] = None @@ -216,11 +319,36 @@ def __init__(self, # Retry configuration self.max_retries = max_retries + # AETHER_MAX_RECONNECT_ATTEMPTS overrides the reconnect cap fleet-wide via + # env (gitops-friendly, no per-call-site change): 0 = retry forever so a + # long-lived service survives an Aether-gateway roll without a manual + # rollout-restart. Capped exponential backoff (max_backoff) still applies, + # and the initial connect() is unaffected (it fails fast at startup). + _env_max_retries = os.environ.get("AETHER_MAX_RECONNECT_ATTEMPTS") + if _env_max_retries: + try: + self.max_retries = int(_env_max_retries) + except ValueError: + logger.warning( + "ignoring non-integer AETHER_MAX_RECONNECT_ATTEMPTS=%r", + _env_max_retries, + ) self.initial_backoff = initial_backoff self.max_backoff = max_backoff self.backoff_multiplier = backoff_multiplier self.auto_reconnect = auto_reconnect + # gRPC keepalive options (silent-drop detection). Resolved once here so + # every (re)connect — secure or insecure — uses the same channel + # options. See _resolve_keepalive_options for the resolution order. + self._channel_options = _resolve_keepalive_options( + keepalive_time_ms=keepalive_time_ms, + keepalive_timeout_ms=keepalive_timeout_ms, + keepalive_permit_without_calls=keepalive_permit_without_calls, + http2_max_pings_without_data=http2_max_pings_without_data, + http2_min_ping_interval_without_data_ms=http2_min_ping_interval_without_data_ms, + ) + # Message callbacks (can be sync or async functions) self.on_message: Optional[MessageCallback] = None self.on_config: Optional[ConfigCallback] = None @@ -257,6 +385,10 @@ def __init__(self, self._reconnecting = False self._reconnect_attempt = 0 self._connection_confirmed = False + # Monotonic timestamp of the last CONFIRMED connect (set on every + # (re)connect). Powers connection_healthy() for a lenient, connection- + # gated liveness probe: live now, or down-but-recently-up within grace. + self._last_connected_at: Optional[float] = None self._connection_generation: int = 0 self._session_id: Optional[str] = None self._init_msg: Optional[aether_pb2.InitConnection] = None @@ -311,6 +443,29 @@ def __init__(self, def is_running(self) -> bool: return not self._stop_event.is_set() or self._reconnecting + @property + def is_connected(self) -> bool: + """True while a CONFIRMED gateway stream is live (False during the gap + between a drop and a successful reconnect, and after close()).""" + return self._connection_confirmed + + def connection_healthy(self, grace_seconds: float = 90.0) -> bool: + """Lenient, connection-gated health signal for a k8s liveness probe. + + Returns True when the gateway stream is live now, OR when it dropped but + was confirmed-up within ``grace_seconds`` (a reconnect is in flight — + max_retries=0 retries forever with capped backoff, so a normal gateway + roll heals well inside the grace and never trips the probe). Returns + False only when the connection has been down longer than the grace + (a genuinely stuck/unreachable gateway), so the probe restarts the pod. + Before the first successful connect it returns True (startup grace; the + initial connect()'s own failure path governs boot).""" + if self._connection_confirmed: + return True + if self._last_connected_at is None: + return True + return (time.monotonic() - self._last_connected_at) < grace_seconds + def negotiated_extensions(self) -> List[str]: """ Return the list of extension URIs the server confirmed support for @@ -483,6 +638,7 @@ async def _listen_loop(self): # Connection confirmed when we receive first response from server if not self._connection_confirmed: self._connection_confirmed = True + self._last_connected_at = time.monotonic() if self._reconnecting: logger.info("Reconnected to %s", self.target) else: @@ -923,11 +1079,14 @@ def _build_tls_credentials(self) -> grpc.ChannelCredentials: async def _do_connect(self, init_msg: aether_pb2.InitConnection, target: str): """Internal method to establish connection (no retry logic).""" + # Pass keepalive options to BOTH channel builders so silent TCP drops + # are detected regardless of TLS. self._channel_options is resolved in + # __init__ (env-overridable). if self.tls_enabled: credentials = self._build_tls_credentials() - self.channel = grpc_aio.secure_channel(target, credentials) + self.channel = grpc_aio.secure_channel(target, credentials, options=self._channel_options) else: - self.channel = grpc_aio.insecure_channel(target) + self.channel = grpc_aio.insecure_channel(target, options=self._channel_options) self.stub = aether_pb2_grpc.AetherGatewayStub(self.channel) # Fresh event for this connection's request generator. Must be @@ -1388,7 +1547,6 @@ async def kv_increment_if(self, key: str, delta: int = 1, ceiling: int = 0, op=aether_pb2.KVOperation.INCREMENT_IF, scope=_scope_to_proto(scope), key=key, - int_value=delta, delta_value=int(delta), guard_value=ceiling, user_id=user_id, @@ -1431,7 +1589,6 @@ async def kv_decrement_if(self, key: str, delta: int = 1, floor: int = 0, op=aether_pb2.KVOperation.DECREMENT_IF, scope=_scope_to_proto(scope), key=key, - int_value=delta, delta_value=int(delta), guard_value=floor, user_id=user_id, @@ -1460,7 +1617,9 @@ async def create_task(self, task_type: str, workspace: str, assignment_mode: int = SELF_ASSIGN, authorization: Optional[aether_pb2.AuthorizationContext] = None, task_class: int = 0, - context_id: str = "") -> None: + context_id: str = "", + priority: int = 0, + retry_policy: Optional[aether_pb2.RetryPolicy] = None) -> None: """ Create a new task. @@ -1495,6 +1654,8 @@ async def create_task(self, task_type: str, workspace: str, authorization=authorization, task_class=task_class, # type: ignore[arg-type] context_id=context_id, + priority=priority, # type: ignore[arg-type] + retry_policy=retry_policy, ) await self._request_queue.put(aether_pb2.UpstreamMessage(create_task=req)) @@ -1509,6 +1670,8 @@ async def create_task_sync(self, task_type: str, workspace: str, target_identity: str = "", task_class: int = 0, context_id: str = "", + priority: int = 0, + retry_policy: Optional[aether_pb2.RetryPolicy] = None, timeout: float = 10.0) -> Optional[aether_pb2.CreateTaskResponse]: """ Create a new task and wait for the server's response containing the task_id. @@ -1558,6 +1721,8 @@ async def create_task_sync(self, task_type: str, workspace: str, task_class=task_class, # type: ignore[arg-type] context_id=context_id, request_id=request_id, + priority=priority, # type: ignore[arg-type] + retry_policy=retry_policy, ) return await self._send_sync_op( aether_pb2.UpstreamMessage(create_task=req), request_id, timeout, @@ -2003,6 +2168,33 @@ async def cancel_task(self, task_id: str, reason: str = "", request_id, timeout, ) + async def claim_task(self, task_id: str, *, + timeout: float = 5.0) -> Optional[aether_pb2.TaskOperationResponse]: + """ + Claim a pending/assigned task, transitioning it to RUNNING. + + Used by the assignee (or the task's on-behalf-of subject) to start a + task — e.g. a per-turn chat_message task claimed straight out of the + queue. Idempotent: re-claiming a running task succeeds without error. + + Args: + task_id: The task ID to claim + timeout: Timeout in seconds + + Returns: + TaskOperationResponse or None if timeout + """ + request_id = str(uuid.uuid4()) + op = aether_pb2.TaskOperation( + op=aether_pb2.TaskOperation.CLAIM, + task_id=task_id, + request_id=request_id, + ) + return await self._send_sync_op( + aether_pb2.UpstreamMessage(task_op=op), + request_id, timeout, + ) + async def retry_task(self, task_id: str, timeout: float = 10.0) -> Optional[aether_pb2.TaskOperationResponse]: """ @@ -3209,17 +3401,17 @@ def make_authority_cache(self, **kwargs): # ========================================================================= async def resolve_authority( - self, - grant_id: str, - subject_type: str, - subject_id: str, - *, - actor_type: Optional[str] = None, - actor_id: Optional[str] = None, - audience_type: str = "", - audience_id: str = "", - request_id: Optional[str] = None, - timeout: Optional[float] = None, + self, + grant_id: str, + subject_type: str, + subject_id: str, + *, + actor_type: Optional[str] = None, + actor_id: Optional[str] = None, + audience_type: str = "", + audience_id: str = "", + request_id: Optional[str] = None, + timeout: Optional[float] = None, ) -> Optional[aether_pb2.ResolveAuthorityResponse]: """Resolve a runtime authority grant for a (subject, actor) pair. @@ -3262,12 +3454,12 @@ async def resolve_authority( ) async def connection_status( - self, - principal_type: str, - principal_id: str, - *, - request_id: Optional[str] = None, - timeout: Optional[float] = None, + self, + principal_type: str, + principal_id: str, + *, + request_id: Optional[str] = None, + timeout: Optional[float] = None, ) -> Optional[aether_pb2.ConnectionStatusResponse]: """Query connection status for a principal. @@ -3851,7 +4043,7 @@ async def send_message_to_agent(self, workspace: str, implementation: str, speci payload, message_type=message_type, authorization=authorization, ) - async def send_message_to_service(self, implementation: str, specifier: str, + async def send_message_to_service(self, implementation: str, specifier, payload: bytes, message_type: int = aether_pb2.OPAQUE, authorization: Optional[aether_pb2.AuthorizationContext] = None): """Send a message to a service principal (``sv::{impl}::{specifier}``). @@ -3860,6 +4052,12 @@ async def send_message_to_service(self, implementation: str, specifier: str, traffic on the service topic to the registered service connection (typically a proxy-sidecar relay+terminator). Useful when an agent replies to an RPC originated from a service identity. + + ``specifier`` accepts a concrete ``str`` (canonical 3-segment + target; an empty string stays concrete as ``sv::{impl}::``) or the + :data:`SERVICE_WILDCARD` sentinel for the bare 2-segment + ``sv::{impl}`` wildcard the gateway resolves to any registered + instance of that implementation. """ await self._send_message( create_topic_service(implementation, specifier), @@ -3977,7 +4175,8 @@ def __init__(self, implementation: str, specifier: str, tls_client_cert_path: Optional[str] = None, tls_client_key: Optional[bytes] = None, tls_client_key_path: Optional[str] = None, - extensions: Optional[List[aether_pb2.ExtensionDeclaration]] = None): + extensions: Optional[List[aether_pb2.ExtensionDeclaration]] = None, + consumes_pool_tasks: bool = True): if not implementation: raise InvalidArgumentError( message="Service principal requires an implementation identifier", @@ -4001,7 +4200,8 @@ def __init__(self, implementation: str, specifier: str, self.implementation = implementation self.specifier = specifier self.init = create_service_init(implementation, specifier, credentials, - extensions=extensions) + extensions=extensions, + no_pool_consumer=not consumes_pool_tasks) async def connect(self, target: str = "localhost:50051"): await self._connect(self.init, target) @@ -4019,10 +4219,17 @@ async def send_message_to_agent(self, workspace: str, implementation: str, speci payload, message_type=message_type, authorization=authorization, ) - async def send_message_to_service(self, implementation: str, specifier: str, + async def send_message_to_service(self, implementation: str, specifier, payload: bytes, message_type: int = aether_pb2.OPAQUE, authorization: Optional[aether_pb2.AuthorizationContext] = None): - """Send a message to another service principal.""" + """Send a message to another service principal. + + ``specifier`` accepts a concrete ``str`` (canonical 3-segment + target; an empty string stays concrete as ``sv::{impl}::``) or the + :data:`SERVICE_WILDCARD` sentinel for the bare 2-segment + ``sv::{impl}`` wildcard the gateway resolves to any registered + instance of that implementation. + """ await self._send_message( create_topic_service(implementation, specifier), payload, message_type=message_type, authorization=authorization, @@ -4052,6 +4259,24 @@ async def send_message_to_user_workspace(self, user_id: str, workspace: str, pay payload, message_type=message_type, authorization=authorization, ) + async def send_message_to_user_broadcast(self, user_id: str, payload: bytes, + message_type: int = aether_pb2.OPAQUE, + authorization: Optional[aether_pb2.AuthorizationContext] = None): + """Send a message to every one of a user's windows (uu::{user_id}). + + Workspace-agnostic, non-progress channel for platform->user + notifications; the intended way for a service principal to push an + opaque update to a user without abusing the progress channel. + """ + await self._send_message( + create_topic_user_broadcast(user_id), + payload, message_type=message_type, authorization=authorization, + ) + + async def send_event(self, payload: bytes): + """Send an event to the workflow engine (``event::*`` plane).""" + await self._send_message("event::*", payload, message_type=aether_pb2.EVENT) + async def send_metric(self, metric): """Send a metric to the metrics bridge. @@ -4435,6 +4660,20 @@ async def send_message_to_user_workspace(self, user_id: str, workspace: str, pay payload, message_type=message_type, authorization=authorization, ) + async def send_message_to_user_broadcast(self, user_id: str, payload: bytes, + message_type: int = aether_pb2.OPAQUE, + authorization: Optional[aether_pb2.AuthorizationContext] = None): + """Send a message to every one of a user's windows (uu::{user_id}). + + Workspace-agnostic, non-progress channel for platform->user + notifications; reaches all of the user's windows regardless of the + workspace each is viewing. + """ + await self._send_message( + create_topic_user_broadcast(user_id), + payload, message_type=message_type, authorization=authorization, + ) + async def send_metric(self, metric): """Send a metric to the metrics bridge. diff --git a/sdk/python-client/scitrera_aether_client/proto/aether_pb2.py b/sdk/python-client/scitrera_aether_client/proto/aether_pb2.py index 2096a66..f28053e 100644 --- a/sdk/python-client/scitrera_aether_client/proto/aether_pb2.py +++ b/sdk/python-client/scitrera_aether_client/proto/aether_pb2.py @@ -24,7 +24,7 @@ -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x0c\x61\x65ther.proto\x12\taether.v1\"\xb1\r\n\x0fUpstreamMessage\x12)\n\x04init\x18\x01 \x01(\x0b\x32\x19.aether.v1.InitConnectionH\x00\x12&\n\x04send\x18\x02 \x01(\x0b\x32\x16.aether.v1.SendMessageH\x00\x12\x36\n\x10switch_workspace\x18\x03 \x01(\x0b\x32\x1a.aether.v1.SwitchWorkspaceH\x00\x12\'\n\x05kv_op\x18\x04 \x01(\x0b\x32\x16.aether.v1.KVOperationH\x00\x12\x33\n\x0b\x63reate_task\x18\x05 \x01(\x0b\x32\x1c.aether.v1.CreateTaskRequestH\x00\x12\x37\n\rcheckpoint_op\x18\x06 \x01(\x0b\x32\x1e.aether.v1.CheckpointOperationH\x00\x12,\n\x0b\x61\x64min_query\x18\x07 \x01(\x0b\x32\x15.aether.v1.AdminQueryH\x00\x12\x31\n\nsession_op\x18\x08 \x01(\x0b\x32\x1b.aether.v1.SessionOperationH\x00\x12*\n\ntask_query\x18\t \x01(\x0b\x32\x14.aether.v1.TaskQueryH\x00\x12+\n\x07task_op\x18\n \x01(\x0b\x32\x18.aether.v1.TaskOperationH\x00\x12\x35\n\x0cworkspace_op\x18\x0b \x01(\x0b\x32\x1d.aether.v1.WorkspaceOperationH\x00\x12-\n\x08\x61gent_op\x18\x0c \x01(\x0b\x32\x19.aether.v1.AgentOperationH\x00\x12)\n\x06\x61\x63l_op\x18\r \x01(\x0b\x32\x17.aether.v1.ACLOperationH\x00\x12-\n\x08progress\x18\x0e \x01(\x0b\x32\x19.aether.v1.ProgressReportH\x00\x12\x33\n\x0bworkflow_op\x18\x0f \x01(\x0b\x32\x1c.aether.v1.WorkflowOperationH\x00\x12\x38\n\x11workflow_response\x18\x10 \x01(\x0b\x32\x1b.aether.v1.WorkflowResponseH\x00\x12-\n\x08token_op\x18\x11 \x01(\x0b\x32\x19.aether.v1.TokenOperationH\x00\x12,\n\x0b\x61udit_query\x18\x12 \x01(\x0b\x32\x15.aether.v1.AuditQueryH\x00\x12@\n\x12\x61uthority_grant_op\x18\x13 \x01(\x0b\x32\".aether.v1.AuthorityGrantOperationH\x00\x12\x39\n\x12proxy_http_request\x18\x14 \x01(\x0b\x32\x1b.aether.v1.ProxyHttpRequestH\x00\x12>\n\x15proxy_http_body_chunk\x18\x15 \x01(\x0b\x32\x1d.aether.v1.ProxyHttpBodyChunkH\x00\x12,\n\x0btunnel_open\x18\x16 \x01(\x0b\x32\x15.aether.v1.TunnelOpenH\x00\x12,\n\x0btunnel_data\x18\x17 \x01(\x0b\x32\x15.aether.v1.TunnelDataH\x00\x12.\n\x0ctunnel_close\x18\x18 \x01(\x0b\x32\x16.aether.v1.TunnelCloseH\x00\x12;\n\x13proxy_http_response\x18\x19 \x01(\x0b\x32\x1c.aether.v1.ProxyHttpResponseH\x00\x12*\n\ntunnel_ack\x18\x1a \x01(\x0b\x32\x14.aether.v1.TunnelAckH\x00\x12G\n\x19resolve_authority_request\x18\x1b \x01(\x0b\x32\".aether.v1.ResolveAuthorityRequestH\x00\x12G\n\x19\x63onnection_status_request\x18\x1c \x01(\x0b\x32\".aether.v1.ConnectionStatusRequestH\x00\x12@\n\x12submit_audit_event\x18\x1d \x01(\x0b\x32\".aether.v1.SubmitAuditEventRequestH\x00\x12\x44\n\x14\x61uthority_request_op\x18\x1e \x01(\x0b\x32$.aether.v1.AuthorityRequestOperationH\x00\x12\x44\n\x14task_subscription_op\x18\x1f \x01(\x0b\x32$.aether.v1.TaskSubscriptionOperationH\x00\x12\x19\n\x11\x61\x63tive_extensions\x18 \x03(\tB\t\n\x07payload\"\xba\x10\n\x11\x44ownstreamMessage\x12)\n\x03msg\x18\x01 \x01(\x0b\x32\x1a.aether.v1.IncomingMessageH\x00\x12+\n\x06\x63onfig\x18\x02 \x01(\x0b\x32\x19.aether.v1.ConfigSnapshotH\x00\x12#\n\x06signal\x18\x03 \x01(\x0b\x32\x11.aether.v1.SignalH\x00\x12)\n\x05\x65rror\x18\x04 \x01(\x0b\x32\x18.aether.v1.ErrorResponseH\x00\x12#\n\x02kv\x18\x05 \x01(\x0b\x32\x15.aether.v1.KVResponseH\x00\x12\x34\n\x0ftask_assignment\x18\x06 \x01(\x0b\x32\x19.aether.v1.TaskAssignmentH\x00\x12\x32\n\x0e\x63onnection_ack\x18\x07 \x01(\x0b\x32\x18.aether.v1.ConnectionAckH\x00\x12\x33\n\ncheckpoint\x18\x08 \x01(\x0b\x32\x1d.aether.v1.CheckpointResponseH\x00\x12)\n\x05\x61\x64min\x18\t \x01(\x0b\x32\x18.aether.v1.AdminResponseH\x00\x12?\n\x10session_response\x18\n \x01(\x0b\x32#.aether.v1.SessionOperationResponseH\x00\x12\x32\n\ntask_query\x18\x0b \x01(\x0b\x32\x1c.aether.v1.TaskQueryResponseH\x00\x12\x33\n\x07task_op\x18\x0c \x01(\x0b\x32 .aether.v1.TaskOperationResponseH\x00\x12\x31\n\tworkspace\x18\r \x01(\x0b\x32\x1c.aether.v1.WorkspaceResponseH\x00\x12)\n\x05\x61gent\x18\x0e \x01(\x0b\x32\x18.aether.v1.AgentResponseH\x00\x12%\n\x03\x61\x63l\x18\x0f \x01(\x0b\x32\x16.aether.v1.ACLResponseH\x00\x12\x34\n\x0fprogress_update\x18\x10 \x01(\x0b\x32\x19.aether.v1.ProgressUpdateH\x00\x12\x38\n\x11workflow_response\x18\x11 \x01(\x0b\x32\x1b.aether.v1.WorkflowResponseH\x00\x12\x33\n\x0bworkflow_op\x18\x12 \x01(\x0b\x32\x1c.aether.v1.WorkflowOperationH\x00\x12)\n\x05token\x18\x13 \x01(\x0b\x32\x18.aether.v1.TokenResponseH\x00\x12\x37\n\x0e\x61udit_response\x18\x14 \x01(\x0b\x32\x1d.aether.v1.AuditQueryResponseH\x00\x12<\n\x0f\x61uthority_grant\x18\x15 \x01(\x0b\x32!.aether.v1.AuthorityGrantResponseH\x00\x12\x34\n\x0b\x63reate_task\x18\x16 \x01(\x0b\x32\x1d.aether.v1.CreateTaskResponseH\x00\x12;\n\x13proxy_http_response\x18\x17 \x01(\x0b\x32\x1c.aether.v1.ProxyHttpResponseH\x00\x12>\n\x15proxy_http_body_chunk\x18\x18 \x01(\x0b\x32\x1d.aether.v1.ProxyHttpBodyChunkH\x00\x12*\n\ntunnel_ack\x18\x19 \x01(\x0b\x32\x14.aether.v1.TunnelAckH\x00\x12.\n\x0ctunnel_close\x18\x1a \x01(\x0b\x32\x16.aether.v1.TunnelCloseH\x00\x12,\n\x0btunnel_data\x18\x1b \x01(\x0b\x32\x15.aether.v1.TunnelDataH\x00\x12\x39\n\x12proxy_http_request\x18\x1c \x01(\x0b\x32\x1b.aether.v1.ProxyHttpRequestH\x00\x12I\n\x1aresolve_authority_response\x18\x1d \x01(\x0b\x32#.aether.v1.ResolveAuthorityResponseH\x00\x12I\n\x1a\x63onnection_status_response\x18\x1e \x01(\x0b\x32#.aether.v1.ConnectionStatusResponseH\x00\x12I\n\x1a\x61uthority_grant_revocation\x18\x1f \x01(\x0b\x32#.aether.v1.AuthorityGrantRevocationH\x00\x12J\n\x1bsubmit_audit_event_response\x18 \x01(\x0b\x32#.aether.v1.SubmitAuditEventResponseH\x00\x12R\n\x1a\x61uthority_request_response\x18! \x01(\x0b\x32,.aether.v1.AuthorityRequestOperationResponseH\x00\x12\x43\n\x17\x61uthority_request_event\x18\" \x01(\x0b\x32 .aether.v1.AuthorityRequestEventH\x00\x12\x34\n\x0ftask_hibernated\x18# \x01(\x0b\x32\x19.aether.v1.TaskHibernatedH\x00\x12R\n\x1atask_subscription_response\x18$ \x01(\x0b\x32,.aether.v1.TaskSubscriptionOperationResponseH\x00\x12*\n\ntask_event\x18% \x01(\x0b\x32\x14.aether.v1.TaskEventH\x00\x12\x19\n\x11\x61\x63tive_extensions\x18& \x03(\tB\t\n\x07payload\"W\n\x0eTaskHibernated\x12\x0f\n\x07task_id\x18\x01 \x01(\t\x12\x34\n\ndescriptor\x18\x02 \x01(\x0b\x32 .aether.v1.HibernationDescriptor\"\xb6\x02\n\rConnectionAck\x12\x12\n\nsession_id\x18\x01 \x01(\t\x12\x0f\n\x07resumed\x18\x02 \x01(\x08\x12\x13\n\x0b\x61ssigned_id\x18\x03 \x01(\t\x12=\n\x15negotiated_extensions\x18\x04 \x03(\x0b\x32\x1e.aether.v1.NegotiatedExtension\x12#\n\x1bserver_supported_extensions\x18\x05 \x03(\t\x12\x16\n\x0eserver_version\x18\x32 \x01(\t\x12/\n\x11server_build_info\x18\x33 \x01(\x0b\x32\x14.aether.v1.BuildInfo\x12\"\n\x1ainitial_connection_unix_ms\x18< \x01(\x03\x12\x1a\n\x12reconnection_count\x18= \x01(\x05\"\xcd\x05\n\x0eInitConnection\x12)\n\x05\x61gent\x18\x01 \x01(\x0b\x32\x18.aether.v1.AgentIdentityH\x00\x12\'\n\x04task\x18\x02 \x01(\x0b\x32\x17.aether.v1.TaskIdentityH\x00\x12\'\n\x04user\x18\x03 \x01(\x0b\x32\x17.aether.v1.UserIdentityH\x00\x12\x37\n\x0corchestrator\x18\x04 \x01(\x0b\x32\x1f.aether.v1.OrchestratorIdentityH\x00\x12<\n\x0fworkflow_engine\x18\x05 \x01(\x0b\x32!.aether.v1.WorkflowEngineIdentityH\x00\x12:\n\x0emetrics_bridge\x18\x06 \x01(\x0b\x32 .aether.v1.MetricsBridgeIdentityH\x00\x12+\n\x06\x62ridge\x18\x07 \x01(\x0b\x32\x19.aether.v1.BridgeIdentityH\x00\x12-\n\x07service\x18\x08 \x01(\x0b\x32\x1a.aether.v1.ServiceIdentityH\x00\x12?\n\x0b\x63redentials\x18\n \x03(\x0b\x32*.aether.v1.InitConnection.CredentialsEntry\x12\x19\n\x11resume_session_id\x18\x0b \x01(\t\x12\x33\n\nextensions\x18\x0c \x03(\x0b\x32\x1f.aether.v1.ExtensionDeclaration\x12\x16\n\x0e\x63lient_version\x18\x32 \x01(\t\x12\x12\n\nclient_sdk\x18\x33 \x01(\t\x12/\n\x11\x63lient_build_info\x18\x34 \x01(\x0b\x32\x14.aether.v1.BuildInfo\x1a\x32\n\x10\x43redentialsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x42\r\n\x0b\x63lient_type\"J\n\tBuildInfo\x12\x0e\n\x06\x63ommit\x18\x01 \x01(\t\x12\x10\n\x08\x62uilt_at\x18\x02 \x01(\t\x12\x0f\n\x07runtime\x18\x03 \x01(\t\x12\n\n\x02os\x18\x04 \x01(\t\"[\n\x14\x45xtensionDeclaration\x12\x0b\n\x03uri\x18\x01 \x01(\t\x12\x0f\n\x07version\x18\x02 \x01(\t\x12\x10\n\x08required\x18\x03 \x01(\x08\x12\x13\n\x0bjson_schema\x18\x04 \x01(\t\"`\n\x13NegotiatedExtension\x12\x0b\n\x03uri\x18\x01 \x01(\t\x12\x0f\n\x07version\x18\x02 \x01(\t\x12\x11\n\tsupported\x18\x03 \x01(\x08\x12\x18\n\x10rejection_reason\x18\x04 \x01(\t\"-\n\x16WorkflowEngineIdentity\x12\x13\n\x0binstance_id\x18\x01 \x01(\t\",\n\x15MetricsBridgeIdentity\x12\x13\n\x0binstance_id\x18\x01 \x01(\t\"]\n\x14OrchestratorIdentity\x12\x16\n\x0eimplementation\x18\x01 \x01(\t\x12\x11\n\tspecifier\x18\x02 \x01(\t\x12\x1a\n\x12supported_profiles\x18\x03 \x03(\t\";\n\x0e\x42ridgeIdentity\x12\x16\n\x0eimplementation\x18\x01 \x01(\t\x12\x11\n\tspecifier\x18\x02 \x01(\t\"<\n\x0fServiceIdentity\x12\x16\n\x0eimplementation\x18\x01 \x01(\t\x12\x11\n\tspecifier\x18\x02 \x01(\t\"M\n\rAgentIdentity\x12\x11\n\tworkspace\x18\x01 \x01(\t\x12\x16\n\x0eimplementation\x18\x02 \x01(\t\x12\x11\n\tspecifier\x18\x03 \x01(\t\"S\n\x0cTaskIdentity\x12\x11\n\tworkspace\x18\x01 \x01(\t\x12\x16\n\x0eimplementation\x18\x02 \x01(\t\x12\x18\n\x10unique_specifier\x18\x03 \x01(\t\"2\n\x0cUserIdentity\x12\x0f\n\x07user_id\x18\x01 \x01(\t\x12\x11\n\twindow_id\x18\x02 \x01(\t\"<\n\x0cPrincipalRef\x12\x16\n\x0eprincipal_type\x18\x01 \x01(\t\x12\x14\n\x0cprincipal_id\x18\x02 \x01(\t\"\x9e\x01\n\x14\x41uthorizationContext\x12\x16\n\x0e\x61uthority_mode\x18\x01 \x01(\t\x12(\n\x07subject\x18\x02 \x01(\x0b\x32\x17.aether.v1.PrincipalRef\x12\x10\n\x08grant_id\x18\x03 \x01(\t\x12\x32\n\x08resolved\x18\n \x01(\x0b\x32 .aether.v1.ResolvedAuthorityInfo\"\xbc\x01\n\x15ResolvedAuthorityInfo\x12-\n\x0croot_subject\x18\x01 \x01(\x0b\x32\x17.aether.v1.PrincipalRef\x12\x15\n\raudience_type\x18\x02 \x01(\t\x12\x13\n\x0b\x61udience_id\x18\x03 \x01(\t\x12\x18\n\x10max_access_level\x18\x04 \x01(\x05\x12\x17\n\x0fworkspace_scope\x18\x05 \x03(\t\x12\x15\n\rexpires_at_ms\x18\x06 \x01(\x03\"\xb1\x01\n\x0bSendMessage\x12\x14\n\x0ctarget_topic\x18\x01 \x01(\t\x12\x0f\n\x07payload\x18\x02 \x01(\x0c\x12,\n\x0cmessage_type\x18\x03 \x01(\x0e\x32\x16.aether.v1.MessageType\x12\x36\n\rauthorization\x18\x04 \x01(\x0b\x32\x1f.aether.v1.AuthorizationContext\x12\x15\n\rapp_workspace\x18\x05 \x01(\t\"\xc4\x01\n\x06Metric\x12\x10\n\x08trace_id\x18\x01 \x01(\t\x12\'\n\x07\x65ntries\x18\x02 \x03(\x0b\x32\x16.aether.v1.MetricEntry\x12\x31\n\x08metadata\x18\x03 \x03(\x0b\x32\x1f.aether.v1.Metric.MetadataEntry\x12\x1b\n\x13\x63lient_timestamp_ms\x18\x04 \x01(\x03\x1a/\n\rMetadataEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"6\n\x0bMetricEntry\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0c\n\x04kind\x18\x02 \x01(\t\x12\x0b\n\x03qty\x18\x03 \x01(\x01\"+\n\x0fSwitchWorkspace\x12\x18\n\x10new_workspace_id\x18\x01 \x01(\t\"\xd1\x04\n\x0bKVOperation\x12)\n\x02op\x18\x01 \x01(\x0e\x32\x1d.aether.v1.KVOperation.OpType\x12+\n\x05scope\x18\x02 \x01(\x0e\x32\x1c.aether.v1.KVOperation.Scope\x12\x0b\n\x03key\x18\x03 \x01(\t\x12\r\n\x05value\x18\x04 \x01(\x0c\x12\x0f\n\x07user_id\x18\x05 \x01(\t\x12\x11\n\tworkspace\x18\x06 \x01(\t\x12\x0b\n\x03ttl\x18\x07 \x01(\x03\x12\x12\n\nrequest_id\x18\x08 \x01(\t\x12\x36\n\rauthorization\x18\t \x01(\x0b\x32\x1f.aether.v1.AuthorizationContext\x12\x13\n\x0bguard_value\x18\n \x01(\x03\x12\x13\n\x0b\x64\x65lta_value\x18\x0b \x01(\x03\"r\n\x06OpType\x12\x07\n\x03GET\x10\x00\x12\x07\n\x03PUT\x10\x01\x12\x08\n\x04LIST\x10\x02\x12\n\n\x06\x44\x45LETE\x10\x03\x12\r\n\tINCREMENT\x10\x04\x12\r\n\tDECREMENT\x10\x05\x12\x10\n\x0cINCREMENT_IF\x10\x06\x12\x10\n\x0c\x44\x45\x43REMENT_IF\x10\x07\"\xb2\x01\n\x05Scope\x12\x15\n\x11SCOPE_UNSPECIFIED\x10\x00\x12\n\n\x06GLOBAL\x10\x01\x12\r\n\tWORKSPACE\x10\x02\x12\x08\n\x04USER\x10\x03\x12\x12\n\x0eUSER_WORKSPACE\x10\x04\x12\x14\n\x10GLOBAL_EXCLUSIVE\x10\x05\x12\x17\n\x13WORKSPACE_EXCLUSIVE\x10\x06\x12\x0f\n\x0bUSER_SHARED\x10\x07\x12\x19\n\x15USER_WORKSPACE_SHARED\x10\x08\"\xd6\x01\n\nKVResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\r\n\x05value\x18\x02 \x01(\x0c\x12\x0c\n\x04keys\x18\x03 \x03(\t\x12\x30\n\x06kv_map\x18\x04 \x03(\x0b\x32 .aether.v1.KVResponse.KvMapEntry\x12\x12\n\nrequest_id\x18\x05 \x01(\t\x12\x15\n\rcounter_value\x18\x06 \x01(\x03\x12\x0f\n\x07\x61pplied\x18\x07 \x01(\x08\x1a,\n\nKvMapEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\x0c:\x02\x38\x01\"y\n\x0fIncomingMessage\x12\x14\n\x0csource_topic\x18\x01 \x01(\t\x12\x0f\n\x07payload\x18\x02 \x01(\x0c\x12,\n\x0cmessage_type\x18\x03 \x01(\x0e\x32\x16.aether.v1.MessageType\x12\x11\n\tworkspace\x18\x04 \x01(\t\"\xf0\x04\n\x0e\x43onfigSnapshot\x12\x31\n\x02kv\x18\x01 \x03(\x0b\x32!.aether.v1.ConfigSnapshot.KvEntryB\x02\x18\x01\x12>\n\tglobal_kv\x18\x02 \x03(\x0b\x32\'.aether.v1.ConfigSnapshot.GlobalKvEntryB\x02\x18\x01\x12@\n\x0ctask_context\x18\x03 \x03(\x0b\x32*.aether.v1.ConfigSnapshot.TaskContextEntry\x12S\n\x16workspace_exclusive_kv\x18\x04 \x03(\x0b\x32\x33.aether.v1.ConfigSnapshot.WorkspaceExclusiveKvEntry\x12M\n\x13global_exclusive_kv\x18\x05 \x03(\x0b\x32\x30.aether.v1.ConfigSnapshot.GlobalExclusiveKvEntry\x1a)\n\x07KvEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\x0c:\x02\x38\x01\x1a/\n\rGlobalKvEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\x0c:\x02\x38\x01\x1a\x32\n\x10TaskContextEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x1a;\n\x19WorkspaceExclusiveKvEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\x0c:\x02\x38\x01\x1a\x38\n\x16GlobalExclusiveKvEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\x0c:\x02\x38\x01\"\x81\x01\n\x06Signal\x12*\n\x04type\x18\x01 \x01(\x0e\x32\x1c.aether.v1.Signal.SignalType\x12\x0e\n\x06reason\x18\x02 \x01(\t\";\n\nSignalType\x12\x14\n\x10\x46ORCE_DISCONNECT\x10\x00\x12\x17\n\x13GRACEFUL_DISCONNECT\x10\x01\"m\n\rErrorResponse\x12\x0c\n\x04\x63ode\x18\x01 \x01(\t\x12\x0f\n\x07message\x18\x02 \x01(\t\x12\x11\n\tretryable\x18\x03 \x01(\x08\x12\x16\n\x0eretry_after_ms\x18\x04 \x01(\x03\x12\x12\n\nrequest_id\x18\x05 \x01(\t\"\xe1\x04\n\x11\x43reateTaskRequest\x12\x11\n\ttask_type\x18\x01 \x01(\t\x12\x11\n\tworkspace\x18\x02 \x01(\t\x12\x36\n\x0f\x61ssignment_mode\x18\x03 \x01(\x0e\x32\x1d.aether.v1.TaskAssignmentMode\x12\x17\n\x0ftarget_agent_id\x18\x04 \x01(\t\x12V\n\x16launch_param_overrides\x18\x05 \x03(\x0b\x32\x36.aether.v1.CreateTaskRequest.LaunchParamOverridesEntry\x12<\n\x08metadata\x18\x06 \x03(\x0b\x32*.aether.v1.CreateTaskRequest.MetadataEntry\x12\x0f\n\x07payload\x18\x07 \x01(\x0c\x12\x1d\n\x15target_implementation\x18\x08 \x01(\t\x12\x36\n\rauthorization\x18\t \x01(\x0b\x32\x1f.aether.v1.AuthorizationContext\x12\x12\n\nrequest_id\x18\n \x01(\t\x12\x17\n\x0ftarget_identity\x18\x0b \x01(\t\x12(\n\ntask_class\x18\x0c \x01(\x0e\x32\x14.aether.v1.TaskClass\x12\x12\n\ncontext_id\x18\r \x01(\t\x1a;\n\x19LaunchParamOverridesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x1a/\n\rMetadataEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\xca\x01\n\x12\x43reateTaskResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x0f\n\x07task_id\x18\x02 \x01(\t\x12\x0e\n\x06status\x18\x03 \x01(\t\x12\x12\n\nerror_code\x18\x04 \x01(\t\x12\x15\n\rerror_message\x18\x05 \x01(\t\x12\x12\n\nrequest_id\x18\x06 \x01(\t\x12\x13\n\x0b\x61ssigned_to\x18\x07 \x01(\t\x12\x12\n\ntask_token\x18\x08 \x01(\t\x12\x1a\n\x12\x61uthority_grant_id\x18\t \x01(\t\"\x87\x04\n\x0eTaskAssignment\x12\x0f\n\x07task_id\x18\x01 \x01(\t\x12\x11\n\ttask_type\x18\x02 \x01(\t\x12\x13\n\x0b\x61ssigned_to\x18\x03 \x01(\t\x12\x39\n\x08metadata\x18\x04 \x03(\x0b\x32\'.aether.v1.TaskAssignment.MetadataEntry\x12\x13\n\x0b\x61ssigned_at\x18\x05 \x01(\x03\x12\x0f\n\x07profile\x18\x06 \x01(\t\x12\x42\n\rlaunch_params\x18\x07 \x03(\x0b\x32+.aether.v1.TaskAssignment.LaunchParamsEntry\x12\x1d\n\x15target_implementation\x18\x08 \x01(\t\x12\x11\n\tworkspace\x18\t \x01(\t\x12\x11\n\tspecifier\x18\n \x01(\t\x12\x0f\n\x07payload\x18\x0b \x01(\x0c\x12(\n\ntask_class\x18\x0c \x01(\x0e\x32\x14.aether.v1.TaskClass\x12\x16\n\x0e\x63heckpoint_key\x18\r \x01(\t\x12\x19\n\x11resume_session_id\x18\x0e \x01(\t\x1a/\n\rMetadataEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x1a\x33\n\x11LaunchParamsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\xb8\x01\n\x13\x43heckpointOperation\x12\x31\n\x02op\x18\x01 \x01(\x0e\x32%.aether.v1.CheckpointOperation.OpType\x12\x0b\n\x03key\x18\x02 \x01(\t\x12\x0c\n\x04\x64\x61ta\x18\x03 \x01(\x0c\x12\x0b\n\x03ttl\x18\x04 \x01(\x03\x12\x12\n\nrequest_id\x18\x05 \x01(\t\"2\n\x06OpType\x12\x08\n\x04SAVE\x10\x00\x12\x08\n\x04LOAD\x10\x01\x12\n\n\x06\x44\x45LETE\x10\x02\x12\x08\n\x04LIST\x10\x03\"v\n\x12\x43heckpointResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x0c\n\x04\x64\x61ta\x18\x02 \x01(\x0c\x12\x0c\n\x04keys\x18\x03 \x03(\t\x12\r\n\x05\x65rror\x18\x04 \x01(\t\x12\x10\n\x08saved_at\x18\x05 \x01(\x03\x12\x12\n\nrequest_id\x18\x06 \x01(\t\"\xec\x01\n\nAdminQuery\x12(\n\x02op\x18\x01 \x01(\x0e\x32\x1c.aether.v1.AdminQuery.OpType\x12\x12\n\nsession_id\x18\x02 \x01(\t\x12+\n\x06\x66ilter\x18\x03 \x01(\x0b\x32\x1b.aether.v1.ConnectionFilter\x12\x12\n\nrequest_id\x18\x04 \x01(\t\"_\n\x06OpType\x12\x0e\n\nGET_HEALTH\x10\x00\x12\x0c\n\x08GET_INFO\x10\x01\x12\r\n\tGET_STATS\x10\x02\x12\x14\n\x10LIST_CONNECTIONS\x10\x03\x12\x12\n\x0eGET_CONNECTION\x10\x04\"l\n\x10\x43onnectionFilter\x12&\n\x04type\x18\x01 \x01(\x0e\x32\x18.aether.v1.PrincipalType\x12\x11\n\tworkspace\x18\x02 \x01(\t\x12\r\n\x05limit\x18\x03 \x01(\x05\x12\x0e\n\x06offset\x18\x04 \x01(\x05\"\xf0\x01\n\x0e\x43onnectionInfo\x12\x12\n\nsession_id\x18\x01 \x01(\t\x12&\n\x04type\x18\x02 \x01(\x0e\x32\x18.aether.v1.PrincipalType\x12\x10\n\x08identity\x18\x03 \x01(\t\x12\x11\n\tworkspace\x18\x04 \x01(\t\x12\x16\n\x0eimplementation\x18\x05 \x01(\t\x12\x11\n\tspecifier\x18\x06 \x01(\t\x12\x14\n\x0c\x63onnected_at\x18\x07 \x01(\x03\x12\x10\n\x08\x64uration\x18\x08 \x01(\t\x12\x13\n\x0bremote_addr\x18\t \x01(\t\x12\x15\n\rlast_activity\x18\n \x01(\x03\"\xac\x02\n\rAdminResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\r\n\x05\x65rror\x18\x02 \x01(\t\x12%\n\x06health\x18\x03 \x01(\x0b\x32\x15.aether.v1.HealthInfo\x12$\n\x04info\x18\x04 \x01(\x0b\x32\x16.aether.v1.GatewayInfo\x12&\n\x05stats\x18\x05 \x01(\x0b\x32\x17.aether.v1.GatewayStats\x12-\n\nconnection\x18\x06 \x01(\x0b\x32\x19.aether.v1.ConnectionInfo\x12.\n\x0b\x63onnections\x18\x07 \x03(\x0b\x32\x19.aether.v1.ConnectionInfo\x12\x13\n\x0btotal_count\x18\x08 \x01(\x05\x12\x12\n\nrequest_id\x18\t \x01(\t\"\xea\x01\n\nHealthInfo\x12\'\n\x06status\x18\x01 \x01(\x0e\x32\x17.aether.v1.HealthStatus\x12\x11\n\ttimestamp\x18\x02 \x01(\x03\x12\x31\n\x06\x63hecks\x18\x03 \x03(\x0b\x32!.aether.v1.HealthInfo.ChecksEntry\x12&\n\x05stats\x18\x04 \x01(\x0b\x32\x17.aether.v1.GatewayStats\x1a\x45\n\x0b\x43hecksEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12%\n\x05value\x18\x02 \x01(\x0b\x32\x16.aether.v1.HealthCheck:\x02\x38\x01\"[\n\x0bHealthCheck\x12,\n\x06status\x18\x01 \x01(\x0e\x32\x1c.aether.v1.HealthCheckStatus\x12\x0f\n\x07latency\x18\x02 \x01(\t\x12\r\n\x05\x65rror\x18\x03 \x01(\t\"\xb4\x01\n\x0bGatewayInfo\x12\x12\n\ngateway_id\x18\x01 \x01(\t\x12\x0f\n\x07version\x18\x02 \x01(\t\x12\x12\n\nstarted_at\x18\x03 \x01(\x03\x12\x0e\n\x06uptime\x18\x04 \x01(\t\x12\x12\n\ngo_version\x18\x05 \x01(\t\x12\x16\n\x0enum_goroutines\x18\x06 \x01(\x05\x12\x17\n\x0fmemory_alloc_mb\x18\x07 \x01(\x01\x12\x17\n\x0fnum_connections\x18\x08 \x01(\x05\"\x9a\x03\n\x0cGatewayStats\x12\x19\n\x11\x61gent_connections\x18\x01 \x01(\x05\x12\x18\n\x10task_connections\x18\x02 \x01(\x05\x12\x18\n\x10user_connections\x18\x03 \x01(\x05\x12 \n\x18orchestrator_connections\x18\x04 \x01(\x05\x12!\n\x19workflow_engine_connected\x18\x05 \x01(\x08\x12 \n\x18metrics_bridge_connected\x18\x06 \x01(\x08\x12\x13\n\x0btotal_tasks\x18\x07 \x01(\x05\x12\x15\n\rpending_tasks\x18\x08 \x01(\x05\x12\x15\n\rrunning_tasks\x18\t \x01(\x05\x12\x17\n\x0f\x63ompleted_tasks\x18\n \x01(\x05\x12\x14\n\x0c\x66\x61iled_tasks\x18\x0b \x01(\x05\x12\x1b\n\x13messages_per_second\x18\x0c \x01(\x01\x12\x16\n\x0etotal_messages\x18\r \x01(\x03\x12\x15\n\ractive_timers\x18\x0e \x01(\x05\x12\x16\n\x0epending_timers\x18\x0f \x01(\x05\"\x8c\x02\n\x10SessionOperation\x12.\n\x02op\x18\x01 \x01(\x0e\x32\".aether.v1.SessionOperation.OpType\x12\x12\n\nsession_id\x18\x02 \x01(\t\x12\x0e\n\x06reason\x18\x03 \x01(\t\x12\x12\n\nrequest_id\x18\x04 \x01(\t\x12+\n\x06\x66ilter\x18\x05 \x01(\x0b\x32\x1b.aether.v1.ConnectionFilter\x12\x36\n\rauthorization\x18\x06 \x01(\x0b\x32\x1f.aether.v1.AuthorizationContext\"+\n\x06OpType\x12\x0e\n\nDISCONNECT\x10\x00\x12\x08\n\x04LIST\x10\x01\x12\x07\n\x03GET\x10\x02\"\xd3\x01\n\x18SessionOperationResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x0f\n\x07message\x18\x02 \x01(\t\x12\r\n\x05\x65rror\x18\x03 \x01(\t\x12\x12\n\nrequest_id\x18\x04 \x01(\t\x12-\n\nconnection\x18\x05 \x01(\x0b\x32\x19.aether.v1.ConnectionInfo\x12.\n\x0b\x63onnections\x18\x06 \x03(\x0b\x32\x19.aether.v1.ConnectionInfo\x12\x13\n\x0btotal_count\x18\x07 \x01(\x05\"\x9d\x01\n\tTaskQuery\x12\'\n\x02op\x18\x01 \x01(\x0e\x32\x1b.aether.v1.TaskQuery.OpType\x12\x0f\n\x07task_id\x18\x02 \x01(\t\x12%\n\x06\x66ilter\x18\x03 \x01(\x0b\x32\x15.aether.v1.TaskFilter\x12\x12\n\nrequest_id\x18\x04 \x01(\t\"\x1b\n\x06OpType\x12\x08\n\x04LIST\x10\x00\x12\x07\n\x03GET\x10\x01\"\xe4\x04\n\nTaskFilter\x12%\n\x06status\x18\x01 \x01(\x0e\x32\x15.aether.v1.TaskStatus\x12\x11\n\tworkspace\x18\x02 \x01(\t\x12\x11\n\ttask_type\x18\x03 \x01(\t\x12\r\n\x05limit\x18\x04 \x01(\x05\x12\x0e\n\x06offset\x18\x05 \x01(\x05\x12\'\n\x08statuses\x18\x06 \x03(\x0e\x32\x15.aether.v1.TaskStatus\x12\x14\n\x0csubject_type\x18\x07 \x01(\t\x12\x12\n\nsubject_id\x18\x08 \x01(\t\x12\x16\n\x0e\x61uthority_mode\x18\t \x01(\t\x12\x1a\n\x12\x61uthority_grant_id\x18\n \x01(\t\x12\x1f\n\x17root_authority_grant_id\x18\x0b \x01(\t\x12\x16\n\x0eparent_task_id\x18\x0c \x01(\t\x12(\n\ntask_class\x18\r \x01(\x0e\x32\x14.aether.v1.TaskClass\x12\x32\n\x14\x65xclude_task_classes\x18\x0e \x03(\x0e\x32\x14.aether.v1.TaskClass\x12\x12\n\ncontext_id\x18\x0f \x01(\t\x12/\n\x10\x65xclude_statuses\x18\x10 \x03(\x0e\x32\x15.aether.v1.TaskStatus\x12.\n\rcreator_actor\x18\x11 \x01(\x0b\x32\x17.aether.v1.PrincipalRef\x12&\n\x1estatus_timestamp_after_unix_ms\x18\x12 \x01(\x03\x12\x12\n\npage_token\x18\x13 \x01(\t\x12\x1b\n\x13include_descendants\x18\x14 \x01(\x08\"\xb4\x06\n\x08TaskInfo\x12\x0f\n\x07task_id\x18\x01 \x01(\t\x12\x11\n\ttask_type\x18\x02 \x01(\t\x12%\n\x06status\x18\x03 \x01(\x0e\x32\x15.aether.v1.TaskStatus\x12\x11\n\tworkspace\x18\x04 \x01(\t\x12\x14\n\x0ctarget_topic\x18\x05 \x01(\t\x12\x13\n\x0b\x61ssigned_to\x18\x06 \x01(\t\x12\x12\n\ncreated_at\x18\x07 \x01(\x03\x12\x12\n\nstarted_at\x18\x08 \x01(\x03\x12\x14\n\x0c\x63ompleted_at\x18\t \x01(\x03\x12\x0f\n\x07\x61ttempt\x18\n \x01(\x05\x12\x14\n\x0cmax_attempts\x18\x0b \x01(\x05\x12\r\n\x05\x65rror\x18\x0c \x01(\t\x12\x33\n\x08metadata\x18\r \x03(\x0b\x32!.aether.v1.TaskInfo.MetadataEntry\x12\x16\n\x0e\x61uthority_mode\x18\x0e \x01(\t\x12\x14\n\x0csubject_type\x18\x0f \x01(\t\x12\x12\n\nsubject_id\x18\x10 \x01(\t\x12\x19\n\x11root_subject_type\x18\x11 \x01(\t\x12\x17\n\x0froot_subject_id\x18\x12 \x01(\t\x12\x1a\n\x12\x61uthority_grant_id\x18\x13 \x01(\t\x12\x1f\n\x17root_authority_grant_id\x18\x14 \x01(\t\x12!\n\x19parent_authority_grant_id\x18\x15 \x01(\t\x12\x18\n\x10\x63reator_actor_id\x18\x16 \x01(\t\x12\x16\n\x0eparent_task_id\x18\x17 \x01(\t\x12(\n\ntask_class\x18\x18 \x01(\x0e\x32\x14.aether.v1.TaskClass\x12\x17\n\x0f\x64isconnected_at\x18\x19 \x01(\x03\x12\x17\n\x0fgrace_window_ms\x18\x1a \x01(\x03\x12&\n\twait_spec\x18\x1b \x01(\x0b\x32\x13.aether.v1.WaitSpec\x12\x12\n\ndepends_on\x18\x1c \x03(\t\x12\x12\n\ncontext_id\x18\x1d \x01(\t\x12\x11\n\tpaused_at\x18\x1e \x01(\x03\x1a/\n\rMetadataEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\xbc\x01\n\x11TaskQueryResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\r\n\x05\x65rror\x18\x02 \x01(\t\x12!\n\x04task\x18\x03 \x01(\x0b\x32\x13.aether.v1.TaskInfo\x12\"\n\x05tasks\x18\x04 \x03(\x0b\x32\x13.aether.v1.TaskInfo\x12\x13\n\x0btotal_count\x18\x05 \x01(\x05\x12\x12\n\nrequest_id\x18\x06 \x01(\t\x12\x17\n\x0fnext_page_token\x18\x07 \x01(\t\"\x83\x02\n\rTaskOperation\x12+\n\x02op\x18\x01 \x01(\x0e\x32\x1f.aether.v1.TaskOperation.OpType\x12\x0f\n\x07task_id\x18\x02 \x01(\t\x12\x0e\n\x06reason\x18\x03 \x01(\t\x12\x12\n\nrequest_id\x18\x04 \x01(\t\x12&\n\twait_spec\x18\x05 \x01(\x0b\x32\x13.aether.v1.WaitSpec\"h\n\x06OpType\x12\t\n\x05RETRY\x10\x00\x12\n\n\x06\x43\x41NCEL\x10\x01\x12\x0c\n\x08\x43OMPLETE\x10\x02\x12\x08\n\x04\x46\x41IL\x10\x03\x12\t\n\x05PAUSE\x10\x04\x12\x0c\n\x08WAIT_FOR\x10\x05\x12\n\n\x06RESUME\x10\x06\x12\n\n\x06REJECT\x10\x07\"\xec\x02\n\x08WaitSpec\x12%\n\x06reason\x18\x01 \x01(\x0e\x32\x15.aether.v1.WaitReason\x12\x1a\n\x12\x65xpected_principal\x18\x02 \x01(\t\x12\x38\n\x0binput_match\x18\x03 \x03(\x0b\x32#.aether.v1.WaitSpec.InputMatchEntry\x12\x1c\n\x14\x61uthority_request_id\x18\x04 \x01(\t\x12\x12\n\ndepends_on\x18\x05 \x03(\t\x12\x13\n\x0bwake_on_any\x18\x06 \x01(\x08\x12\x12\n\ntimeout_ms\x18\x07 \x01(\x03\x12\x1e\n\x16scheduled_wake_unix_ms\x18\x08 \x01(\x03\x12\x35\n\x0bhibernation\x18\t \x01(\x0b\x32 .aether.v1.HibernationDescriptor\x1a\x31\n\x0fInputMatchEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\x7f\n\x15HibernationDescriptor\x12\x16\n\x0e\x63heckpoint_key\x18\x01 \x01(\t\x12\x19\n\x11resume_session_id\x18\x02 \x01(\t\x12\x18\n\x10wake_event_types\x18\x03 \x03(\t\x12\x19\n\x11\x65scalation_policy\x18\x04 \x01(\t\"\x7f\n\x15TaskOperationResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x0f\n\x07message\x18\x02 \x01(\t\x12\r\n\x05\x65rror\x18\x03 \x01(\t\x12!\n\x04task\x18\x04 \x01(\x0b\x32\x13.aether.v1.TaskInfo\x12\x12\n\nrequest_id\x18\x05 \x01(\t\"\xa0\x02\n\x12WorkspaceOperation\x12\x30\n\x02op\x18\x01 \x01(\x0e\x32$.aether.v1.WorkspaceOperation.OpType\x12\x14\n\x0cworkspace_id\x18\x02 \x01(\t\x12*\n\x06\x66ilter\x18\x03 \x01(\x0b\x32\x1a.aether.v1.WorkspaceFilter\x12+\n\tworkspace\x18\x04 \x01(\x0b\x32\x18.aether.v1.WorkspaceInfo\x12\x12\n\nrequest_id\x18\x05 \x01(\t\"U\n\x06OpType\x12\x08\n\x04LIST\x10\x00\x12\x07\n\x03GET\x10\x01\x12\n\n\x06\x43REATE\x10\x02\x12\n\n\x06UPDATE\x10\x03\x12\n\n\x06\x44\x45LETE\x10\x04\x12\x14\n\x10GET_MESSAGE_FLOW\x10\x05\"C\n\x0fWorkspaceFilter\x12\x11\n\ttenant_id\x18\x01 \x01(\t\x12\r\n\x05limit\x18\x02 \x01(\x05\x12\x0e\n\x06offset\x18\x03 \x01(\x05\"\xd1\x02\n\rWorkspaceInfo\x12\x14\n\x0cworkspace_id\x18\x01 \x01(\t\x12\x14\n\x0c\x64isplay_name\x18\x02 \x01(\t\x12\x13\n\x0b\x64\x65scription\x18\x03 \x01(\t\x12\x11\n\ttenant_id\x18\x04 \x01(\t\x12\x12\n\ncreated_at\x18\x05 \x01(\x03\x12\x12\n\nupdated_at\x18\x06 \x01(\x03\x12\x38\n\x08metadata\x18\x07 \x03(\x0b\x32&.aether.v1.WorkspaceInfo.MetadataEntry\x12\x15\n\ractive_agents\x18\x08 \x01(\x05\x12\x14\n\x0c\x61\x63tive_tasks\x18\t \x01(\x05\x12\x14\n\x0c\x61\x63tive_users\x18\n \x01(\x05\x12\x16\n\x0etotal_messages\x18\x0b \x01(\x03\x1a/\n\rMetadataEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\xfa\x01\n\x11WorkspaceResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\r\n\x05\x65rror\x18\x02 \x01(\t\x12\x0f\n\x07message\x18\x03 \x01(\t\x12+\n\tworkspace\x18\x04 \x01(\x0b\x32\x18.aether.v1.WorkspaceInfo\x12,\n\nworkspaces\x18\x05 \x03(\x0b\x32\x18.aether.v1.WorkspaceInfo\x12\x13\n\x0btotal_count\x18\x06 \x01(\x05\x12\x30\n\x0cmessage_flow\x18\x07 \x01(\x0b\x32\x1a.aether.v1.MessageFlowInfo\x12\x12\n\nrequest_id\x18\x08 \x01(\t\"\x83\x01\n\x0fMessageFlowInfo\x12\x14\n\x0cworkspace_id\x18\x01 \x01(\t\x12\"\n\x05nodes\x18\x02 \x03(\x0b\x32\x13.aether.v1.FlowNode\x12\"\n\x05\x65\x64ges\x18\x03 \x03(\x0b\x32\x13.aether.v1.FlowEdge\x12\x12\n\nupdated_at\x18\x04 \x01(\x03\"\x97\x01\n\x08\x46lowNode\x12\n\n\x02id\x18\x01 \x01(\t\x12\r\n\x05label\x18\x02 \x01(\t\x12&\n\x04type\x18\x03 \x01(\x0e\x32\x18.aether.v1.PrincipalType\x12\x0e\n\x06status\x18\x04 \x01(\t\x12\x16\n\x0eimplementation\x18\x05 \x01(\t\x12\x11\n\tspecifier\x18\x06 \x01(\t\x12\r\n\x05topic\x18\x07 \x01(\t\"B\n\x08\x46lowEdge\x12\x0c\n\x04\x66rom\x18\x01 \x01(\t\x12\n\n\x02to\x18\x02 \x01(\t\x12\r\n\x05label\x18\x03 \x01(\t\x12\r\n\x05\x63ount\x18\x04 \x01(\x03\"\xdf\x02\n\x0e\x41gentOperation\x12,\n\x02op\x18\x01 \x01(\x0e\x32 .aether.v1.AgentOperation.OpType\x12\x16\n\x0eimplementation\x18\x02 \x01(\t\x12&\n\x06\x66ilter\x18\x03 \x01(\x0b\x32\x16.aether.v1.AgentFilter\x12/\n\x05\x61gent\x18\x04 \x01(\x0b\x32 .aether.v1.AgentRegistrationInfo\x12\x33\n\rlaunch_params\x18\x05 \x01(\x0b\x32\x1c.aether.v1.AgentLaunchParams\x12\x12\n\nrequest_id\x18\x06 \x01(\t\"e\n\x06OpType\x12\x08\n\x04LIST\x10\x00\x12\x07\n\x03GET\x10\x01\x12\x0c\n\x08REGISTER\x10\x02\x12\n\n\x06UPDATE\x10\x03\x12\n\n\x06\x44\x45LETE\x10\x04\x12\n\n\x06LAUNCH\x10\x05\x12\x16\n\x12LIST_ORCHESTRATORS\x10\x06\"J\n\x0b\x41gentFilter\x12\x1c\n\x14orchestrator_profile\x18\x01 \x01(\t\x12\r\n\x05limit\x18\x02 \x01(\x05\x12\x0e\n\x06offset\x18\x03 \x01(\x05\"\xde\x03\n\x15\x41gentRegistrationInfo\x12\x16\n\x0eimplementation\x18\x01 \x01(\t\x12\x1c\n\x14orchestrator_profile\x18\x02 \x01(\t\x12\x13\n\x0b\x64\x65scription\x18\x03 \x01(\t\x12I\n\rlaunch_params\x18\x04 \x03(\x0b\x32\x32.aether.v1.AgentRegistrationInfo.LaunchParamsEntry\x12\x15\n\rregistered_at\x18\x05 \x01(\x03\x12\x12\n\nupdated_at\x18\x06 \x01(\x03\x12<\n\x0fresource_schema\x18\x07 \x03(\x0b\x32#.aether.v1.AgentResourceSchemaEntry\x12H\n\x0c\x63\x61pabilities\x18\x08 \x03(\x0b\x32\x32.aether.v1.AgentRegistrationInfo.CapabilitiesEntry\x12\x12\n\nextensions\x18\t \x03(\t\x1a\x33\n\x11LaunchParamsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x1a\x33\n\x11\x43\x61pabilitiesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\x08:\x02\x38\x01\"n\n\x18\x41gentResourceSchemaEntry\x12\x1c\n\x14resource_type_prefix\x18\x01 \x01(\t\x12\x18\n\x10permission_verbs\x18\x02 \x03(\t\x12\x1a\n\x12resource_id_schema\x18\x03 \x01(\t\"\xbb\x01\n\x11\x41gentLaunchParams\x12\x11\n\tspecifier\x18\x01 \x01(\t\x12\x11\n\tworkspace\x18\x02 \x01(\t\x12I\n\x0fparam_overrides\x18\x03 \x03(\x0b\x32\x30.aether.v1.AgentLaunchParams.ParamOverridesEntry\x1a\x35\n\x13ParamOverridesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"S\n\x10OrchestratorInfo\x12\x17\n\x0forchestrator_id\x18\x01 \x01(\t\x12\x10\n\x08profiles\x18\x02 \x03(\t\x12\x14\n\x0c\x63onnected_at\x18\x03 \x01(\x03\"5\n\x11\x41gentLaunchResult\x12\x0f\n\x07task_id\x18\x01 \x01(\t\x12\x0f\n\x07message\x18\x02 \x01(\t\"\xb5\x02\n\rAgentResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\r\n\x05\x65rror\x18\x02 \x01(\t\x12\x0f\n\x07message\x18\x03 \x01(\t\x12/\n\x05\x61gent\x18\x04 \x01(\x0b\x32 .aether.v1.AgentRegistrationInfo\x12\x30\n\x06\x61gents\x18\x05 \x03(\x0b\x32 .aether.v1.AgentRegistrationInfo\x12\x13\n\x0btotal_count\x18\x06 \x01(\x05\x12\x32\n\rorchestrators\x18\x07 \x03(\x0b\x32\x1b.aether.v1.OrchestratorInfo\x12\x33\n\rlaunch_result\x18\x08 \x01(\x0b\x32\x1c.aether.v1.AgentLaunchResult\x12\x12\n\nrequest_id\x18\t \x01(\t\"\xc4\x06\n\x0c\x41\x43LOperation\x12*\n\x02op\x18\x01 \x01(\x0e\x32\x1e.aether.v1.ACLOperation.OpType\x12\x0f\n\x07rule_id\x18\x02 \x01(\t\x12\x15\n\rrule_category\x18\x04 \x01(\t\x12\x16\n\x0eretention_days\x18\x05 \x01(\x05\x12-\n\x0brule_filter\x18\n \x01(\x0b\x32\x18.aether.v1.ACLRuleFilter\x12/\n\x0c\x61udit_filter\x18\x0b \x01(\x0b\x32\x19.aether.v1.ACLAuditFilter\x12\x31\n\rgrant_request\x18\x0c \x01(\x0b\x32\x1a.aether.v1.ACLGrantRequest\x12:\n\x10\x66\x61llback_request\x18\x0e \x01(\x0b\x32 .aether.v1.ACLSetFallbackRequest\x12\x12\n\nrequest_id\x18\x13 \x01(\t\"\xd0\x02\n\x06OpType\x12\x0e\n\nLIST_RULES\x10\x00\x12\x0c\n\x08GET_RULE\x10\x01\x12\t\n\x05GRANT\x10\x02\x12\n\n\x06REVOKE\x10\x03\x12\x0f\n\x0bQUERY_AUDIT\x10\x04\x12\x17\n\x13GET_FALLBACK_POLICY\x10\x08\x12\x17\n\x13SET_FALLBACK_POLICY\x10\t\x12\x13\n\x0f\x43LEANUP_EXPIRED\x10\n\x12\x16\n\x12\x43LEANUP_AUDIT_LOGS\x10\x0b\"\x04\x08\x05\x10\x05\"\x04\x08\x06\x10\x06\"\x04\x08\x07\x10\x07\"\x04\x08\x0c\x10\x0c\"\x04\x08\r\x10\r\"\x04\x08\x0e\x10\x0e\"\x04\x08\x0f\x10\x0f\"\x04\x08\x10\x10\x10*\x15LIST_AUTHORITY_GRANTS*\x13GET_AUTHORITY_GRANT*\x16\x43REATE_AUTHORITY_GRANT*\x15RENEW_AUTHORITY_GRANT*\x16REVOKE_AUTHORITY_GRANTJ\x04\x08\x03\x10\x04J\x04\x08\x06\x10\x07J\x04\x08\x07\x10\x08J\x04\x08\r\x10\x0eJ\x04\x08\x08\x10\tJ\x04\x08\x10\x10\x11J\x04\x08\x11\x10\x12J\x04\x08\x12\x10\x13R\x12\x61uthority_grant_idR\x16\x61uthority_grant_filterR\x17\x61uthority_grant_requestR\x1d\x61uthority_grant_renew_request\"\x88\x01\n\rACLRuleFilter\x12\x16\n\x0eprincipal_type\x18\x01 \x01(\t\x12\x14\n\x0cprincipal_id\x18\x02 \x01(\t\x12\x15\n\rresource_type\x18\x03 \x01(\t\x12\x13\n\x0bresource_id\x18\x04 \x01(\t\x12\r\n\x05limit\x18\x05 \x01(\x05\x12\x0e\n\x06offset\x18\x06 \x01(\x05\"\xd4\x01\n\x0e\x41\x43LAuditFilter\x12\x12\n\nstart_time\x18\x01 \x01(\x03\x12\x10\n\x08\x65nd_time\x18\x02 \x01(\x03\x12\x16\n\x0eprincipal_type\x18\x03 \x01(\t\x12\x14\n\x0cprincipal_id\x18\x04 \x01(\t\x12\x15\n\rresource_type\x18\x05 \x01(\t\x12\x13\n\x0bresource_id\x18\x06 \x01(\t\x12\x10\n\x08\x64\x65\x63ision\x18\x07 \x01(\t\x12\x11\n\tworkspace\x18\x08 \x01(\t\x12\r\n\x05limit\x18\t \x01(\x05\x12\x0e\n\x06offset\x18\n \x01(\x05\"\xb9\x01\n\x0f\x41\x43LGrantRequest\x12\x16\n\x0eprincipal_type\x18\x01 \x01(\t\x12\x14\n\x0cprincipal_id\x18\x02 \x01(\t\x12\x15\n\rresource_type\x18\x03 \x01(\t\x12\x13\n\x0bresource_id\x18\x04 \x01(\t\x12\x14\n\x0c\x61\x63\x63\x65ss_level\x18\x05 \x01(\x05\x12\x12\n\ngranted_by\x18\x06 \x01(\t\x12\x0e\n\x06reason\x18\x07 \x01(\t\x12\x12\n\nexpires_at\x18\x08 \x01(\x03\"a\n\x15\x41\x43LSetFallbackRequest\x12\x15\n\rrule_category\x18\x01 \x01(\t\x12\x1d\n\x15\x66\x61llback_access_level\x18\x02 \x01(\x05\x12\x12\n\nupdated_by\x18\x03 \x01(\t\"\xff\x01\n\x17\x41\x43LAuthorityGrantFilter\x12\x15\n\rroot_grant_id\x18\x01 \x01(\t\x12\x14\n\x0csubject_type\x18\x02 \x01(\t\x12\x12\n\nsubject_id\x18\x03 \x01(\t\x12\x15\n\rdelegate_type\x18\x04 \x01(\t\x12\x13\n\x0b\x64\x65legate_id\x18\x05 \x01(\t\x12\x15\n\raudience_type\x18\x06 \x01(\t\x12\x13\n\x0b\x61udience_id\x18\x07 \x01(\t\x12\x17\n\x0finclude_revoked\x18\x08 \x01(\x08\x12\x13\n\x0b\x61\x63tive_only\x18\t \x01(\x08\x12\r\n\x05limit\x18\n \x01(\x05\x12\x0e\n\x06offset\x18\x0b \x01(\x05\"N\n#ACLAuthorityGrantResourceScopeEntry\x12\x15\n\rresource_type\x18\x01 \x01(\t\x12\x10\n\x08patterns\x18\x02 \x03(\t\"\xa9\x05\n\x18\x41\x43LAuthorityGrantRequest\x12(\n\x07subject\x18\x01 \x01(\x0b\x32\x17.aether.v1.PrincipalRef\x12)\n\x08\x64\x65legate\x18\x02 \x01(\x0b\x32\x17.aether.v1.PrincipalRef\x12*\n\tissued_by\x18\x03 \x01(\x0b\x32\x17.aether.v1.PrincipalRef\x12-\n\x0croot_subject\x18\x04 \x01(\x0b\x32\x17.aether.v1.PrincipalRef\x12\x17\n\x0fparent_grant_id\x18\x05 \x01(\t\x12\x14\n\x0cmay_delegate\x18\x06 \x01(\x08\x12\x16\n\x0eremaining_hops\x18\x07 \x01(\x05\x12\x17\n\x0fworkspace_scope\x18\x08 \x03(\t\x12\x46\n\x0eresource_scope\x18\t \x03(\x0b\x32..aether.v1.ACLAuthorityGrantResourceScopeEntry\x12\x17\n\x0foperation_scope\x18\n \x03(\t\x12\x18\n\x10max_access_level\x18\x0b \x01(\x05\x12\x15\n\raudience_type\x18\x0c \x01(\t\x12\x13\n\x0b\x61udience_id\x18\r \x01(\t\x12#\n\x1bvalid_while_audience_active\x18\x0e \x01(\x08\x12\x12\n\nexpires_at\x18\x0f \x01(\x03\x12\x17\n\x0frenewable_until\x18\x10 \x01(\x03\x12\x0e\n\x06reason\x18\x11 \x01(\t\x12\x43\n\x08metadata\x18\x12 \x03(\x0b\x32\x31.aether.v1.ACLAuthorityGrantRequest.MetadataEntry\x1a/\n\rMetadataEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"]\n\x1d\x41\x43LRenewAuthorityGrantRequest\x12\x10\n\x08grant_id\x18\x01 \x01(\t\x12\x12\n\nexpires_at\x18\x02 \x01(\x03\x12\x16\n\x0e\x65xtend_seconds\x18\x03 \x01(\x05\"\xf5\x01\n\x0b\x41\x43LRuleInfo\x12\x0f\n\x07rule_id\x18\x01 \x01(\t\x12\x16\n\x0eprincipal_type\x18\x02 \x01(\t\x12\x14\n\x0cprincipal_id\x18\x03 \x01(\t\x12\x15\n\rresource_type\x18\x04 \x01(\t\x12\x13\n\x0bresource_id\x18\x05 \x01(\t\x12\x14\n\x0c\x61\x63\x63\x65ss_level\x18\x06 \x01(\x05\x12\x19\n\x11\x61\x63\x63\x65ss_level_name\x18\x07 \x01(\t\x12\x12\n\ngranted_by\x18\x08 \x01(\t\x12\x12\n\ngranted_at\x18\t \x01(\x03\x12\x12\n\nexpires_at\x18\n \x01(\x03\x12\x0e\n\x06reason\x18\x0b \x01(\t\"\xac\x01\n\x15\x41\x43LFallbackPolicyInfo\x12\x11\n\tpolicy_id\x18\x01 \x01(\t\x12\x15\n\rrule_category\x18\x02 \x01(\t\x12\x1d\n\x15\x66\x61llback_access_level\x18\x03 \x01(\x05\x12\"\n\x1a\x66\x61llback_access_level_name\x18\x04 \x01(\t\x12\x12\n\nupdated_by\x18\x05 \x01(\t\x12\x12\n\nupdated_at\x18\x06 \x01(\x03\"\xc3\x03\n\x11\x41\x43LAuditEntryInfo\x12\x10\n\x08\x61udit_id\x18\x01 \x01(\x03\x12\x11\n\ttimestamp\x18\x02 \x01(\x03\x12\x10\n\x08\x64\x65\x63ision\x18\x03 \x01(\t\x12\x14\n\x0c\x61\x63\x63\x65ss_level\x18\x04 \x01(\x05\x12\x19\n\x11\x61\x63\x63\x65ss_level_name\x18\x05 \x01(\t\x12\x16\n\x0eprincipal_type\x18\x06 \x01(\t\x12\x14\n\x0cprincipal_id\x18\x07 \x01(\t\x12\x15\n\rresource_type\x18\x08 \x01(\t\x12\x13\n\x0bresource_id\x18\t \x01(\t\x12\x11\n\toperation\x18\n \x01(\t\x12\x11\n\tworkspace\x18\x0b \x01(\t\x12\x0f\n\x07rule_id\x18\r \x01(\t\x12\x18\n\x10\x66\x61llback_applied\x18\x0e \x01(\x08\x12\x12\n\ngateway_id\x18\x0f \x01(\t\x12\x12\n\nsession_id\x18\x10 \x01(\t\x12<\n\x08metadata\x18\x11 \x03(\x0b\x32*.aether.v1.ACLAuditEntryInfo.MetadataEntry\x1a/\n\rMetadataEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01J\x04\x08\x0c\x10\r\"\xb4\x06\n\x15\x41\x43LAuthorityGrantInfo\x12\x10\n\x08grant_id\x18\x01 \x01(\t\x12\x15\n\rroot_grant_id\x18\x02 \x01(\t\x12(\n\x07subject\x18\x03 \x01(\x0b\x32\x17.aether.v1.PrincipalRef\x12)\n\x08\x64\x65legate\x18\x04 \x01(\x0b\x32\x17.aether.v1.PrincipalRef\x12*\n\tissued_by\x18\x05 \x01(\x0b\x32\x17.aether.v1.PrincipalRef\x12-\n\x0croot_subject\x18\x06 \x01(\x0b\x32\x17.aether.v1.PrincipalRef\x12\x17\n\x0fparent_grant_id\x18\x07 \x01(\t\x12\x14\n\x0cmay_delegate\x18\x08 \x01(\x08\x12\x16\n\x0eremaining_hops\x18\t \x01(\x05\x12\x17\n\x0fworkspace_scope\x18\n \x03(\t\x12\x46\n\x0eresource_scope\x18\x0b \x03(\x0b\x32..aether.v1.ACLAuthorityGrantResourceScopeEntry\x12\x17\n\x0foperation_scope\x18\x0c \x03(\t\x12\x18\n\x10max_access_level\x18\r \x01(\x05\x12\x19\n\x11\x61\x63\x63\x65ss_level_name\x18\x0e \x01(\t\x12\x15\n\raudience_type\x18\x0f \x01(\t\x12\x13\n\x0b\x61udience_id\x18\x10 \x01(\t\x12#\n\x1bvalid_while_audience_active\x18\x11 \x01(\x08\x12\x12\n\nexpires_at\x18\x12 \x01(\x03\x12\x17\n\x0frenewable_until\x18\x13 \x01(\x03\x12\x12\n\nrenewed_at\x18\x14 \x01(\x03\x12\x0f\n\x07revoked\x18\x15 \x01(\x08\x12\x12\n\nrevoked_at\x18\x16 \x01(\x03\x12\x0e\n\x06reason\x18\x17 \x01(\t\x12@\n\x08metadata\x18\x18 \x03(\x0b\x32..aether.v1.ACLAuthorityGrantInfo.MetadataEntry\x12\x12\n\ncreated_at\x18\x19 \x01(\x03\x1a/\n\rMetadataEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\":\n\x10\x41\x43LCleanupResult\x12\x15\n\rdeleted_count\x18\x01 \x01(\x03\x12\x0f\n\x07message\x18\x02 \x01(\t\"\x93\x04\n\x0b\x41\x43LResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\r\n\x05\x65rror\x18\x02 \x01(\t\x12\x0f\n\x07message\x18\x03 \x01(\t\x12$\n\x04rule\x18\x04 \x01(\x0b\x32\x16.aether.v1.ACLRuleInfo\x12%\n\x05rules\x18\x05 \x03(\x0b\x32\x16.aether.v1.ACLRuleInfo\x12\x13\n\x0btotal_rules\x18\x06 \x01(\x05\x12\x39\n\x0f\x66\x61llback_policy\x18\x08 \x01(\x0b\x32 .aether.v1.ACLFallbackPolicyInfo\x12\x33\n\raudit_entries\x18\t \x03(\x0b\x32\x1c.aether.v1.ACLAuditEntryInfo\x12\x1b\n\x13total_audit_entries\x18\n \x01(\x05\x12\x33\n\x0e\x63leanup_result\x18\x0b \x01(\x0b\x32\x1b.aether.v1.ACLCleanupResult\x12\x39\n\x0f\x61uthority_grant\x18\r \x01(\x0b\x32 .aether.v1.ACLAuthorityGrantInfo\x12:\n\x10\x61uthority_grants\x18\x0e \x03(\x0b\x32 .aether.v1.ACLAuthorityGrantInfo\x12\x1e\n\x16total_authority_grants\x18\x0f \x01(\x05\x12\x12\n\nrequest_id\x18\x0c \x01(\tJ\x04\x08\x07\x10\x08\"\xb5\x05\n\x17\x41uthorityGrantOperation\x12\x35\n\x02op\x18\x01 \x01(\x0e\x32).aether.v1.AuthorityGrantOperation.OpType\x12\x10\n\x08grant_id\x18\x02 \x01(\t\x12\x42\n\x10\x65xchange_request\x18\x03 \x01(\x0b\x32(.aether.v1.AuthorityGrantExchangeRequest\x12>\n\x0e\x64\x65rive_request\x18\x04 \x01(\x0b\x32&.aether.v1.AuthorityGrantDeriveRequest\x12?\n\rrenew_request\x18\x05 \x01(\x0b\x32(.aether.v1.ACLRenewAuthorityGrantRequest\x12\x12\n\nrequest_id\x18\x06 \x01(\t\x12:\n\x0clist_request\x18\x07 \x01(\x0b\x32$.aether.v1.AuthorityGrantListRequest\x12M\n\x16\x62\x61tch_exchange_request\x18\x08 \x01(\x0b\x32-.aether.v1.AuthorityGrantBatchExchangeRequest\x12R\n\x19\x64\x65rive_for_target_request\x18\t \x01(\x0b\x32/.aether.v1.AuthorityGrantDeriveForTargetRequest\"\x98\x01\n\x06OpType\x12\x0c\n\x08\x45XCHANGE\x10\x00\x12\n\n\x06\x44\x45RIVE\x10\x01\x12\x07\n\x03GET\x10\x02\x12\t\n\x05RENEW\x10\x03\x12\n\n\x06REVOKE\x10\x04\x12\x12\n\x0eLIST_MY_GRANTS\x10\x05\x12\x15\n\x11LIST_GRANTS_ON_ME\x10\x06\x12\x12\n\x0e\x42\x41TCH_EXCHANGE\x10\x07\x12\x15\n\x11\x44\x45RIVE_FOR_TARGET\x10\x08\"\x85\x04\n\x1d\x41uthorityGrantExchangeRequest\x12\x19\n\x11source_session_id\x18\x01 \x01(\t\x12\x17\n\x0fworkspace_scope\x18\x02 \x03(\t\x12\x46\n\x0eresource_scope\x18\x03 \x03(\x0b\x32..aether.v1.ACLAuthorityGrantResourceScopeEntry\x12\x17\n\x0foperation_scope\x18\x04 \x03(\t\x12\x18\n\x10max_access_level\x18\x05 \x01(\x05\x12\x15\n\raudience_type\x18\x06 \x01(\t\x12\x13\n\x0b\x61udience_id\x18\x07 \x01(\t\x12#\n\x1bvalid_while_audience_active\x18\x08 \x01(\x08\x12\x12\n\nexpires_at\x18\t \x01(\x03\x12\x17\n\x0frenewable_until\x18\n \x01(\x03\x12\x14\n\x0cmay_delegate\x18\x0b \x01(\x08\x12\x16\n\x0eremaining_hops\x18\x0c \x01(\x05\x12\x0e\n\x06reason\x18\r \x01(\t\x12H\n\x08metadata\x18\x0e \x03(\x0b\x32\x36.aether.v1.AuthorityGrantExchangeRequest.MetadataEntry\x1a/\n\rMetadataEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\xaa\x04\n\x1b\x41uthorityGrantDeriveRequest\x12\x17\n\x0fparent_grant_id\x18\x01 \x01(\t\x12)\n\x08\x64\x65legate\x18\x02 \x01(\x0b\x32\x17.aether.v1.PrincipalRef\x12\x17\n\x0fworkspace_scope\x18\x03 \x03(\t\x12\x46\n\x0eresource_scope\x18\x04 \x03(\x0b\x32..aether.v1.ACLAuthorityGrantResourceScopeEntry\x12\x17\n\x0foperation_scope\x18\x05 \x03(\t\x12\x18\n\x10max_access_level\x18\x06 \x01(\x05\x12\x15\n\raudience_type\x18\x07 \x01(\t\x12\x13\n\x0b\x61udience_id\x18\x08 \x01(\t\x12#\n\x1bvalid_while_audience_active\x18\t \x01(\x08\x12\x12\n\nexpires_at\x18\n \x01(\x03\x12\x17\n\x0frenewable_until\x18\x0b \x01(\x03\x12\x14\n\x0cmay_delegate\x18\x0c \x01(\x08\x12\x16\n\x0eremaining_hops\x18\r \x01(\x05\x12\x0e\n\x06reason\x18\x0e \x01(\t\x12\x46\n\x08metadata\x18\x0f \x03(\x0b\x32\x34.aether.v1.AuthorityGrantDeriveRequest.MetadataEntry\x1a/\n\rMetadataEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\xef\x01\n\x16\x41uthorityGrantResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\r\n\x05\x65rror\x18\x02 \x01(\t\x12\x0f\n\x07message\x18\x03 \x01(\t\x12/\n\x05grant\x18\x04 \x01(\x0b\x32 .aether.v1.ACLAuthorityGrantInfo\x12\x12\n\nrequest_id\x18\x05 \x01(\t\x12\x30\n\x06grants\x18\x06 \x03(\x0b\x32 .aether.v1.ACLAuthorityGrantInfo\x12\r\n\x05total\x18\x07 \x01(\x05\x12\x1e\n\x16\x63\x61\x63he_hint_ttl_seconds\x18\x08 \x01(\x05\"\x7f\n\x19\x41uthorityGrantListRequest\x12\x15\n\raudience_type\x18\x01 \x01(\t\x12\x13\n\x0b\x61udience_id\x18\x02 \x01(\t\x12\x17\n\x0finclude_revoked\x18\x03 \x01(\x08\x12\r\n\x05limit\x18\x04 \x01(\x05\x12\x0e\n\x06offset\x18\x05 \x01(\x05\"}\n\"AuthorityGrantBatchExchangeRequest\x12:\n\x08requests\x18\x01 \x03(\x0b\x32(.aether.v1.AuthorityGrantExchangeRequest\x12\x1b\n\x13stop_on_first_error\x18\x02 \x01(\x08\"\xb2\x02\n$AuthorityGrantDeriveForTargetRequest\x12\x17\n\x0fparent_grant_id\x18\x01 \x01(\t\x12\'\n\x06target\x18\x02 \x01(\x0b\x32\x17.aether.v1.PrincipalRef\x12\x15\n\raudience_type\x18\x03 \x01(\t\x12\x13\n\x0b\x61udience_id\x18\x04 \x01(\t\x12\x17\n\x0foperation_scope\x18\x05 \x03(\t\x12\x18\n\x10max_access_level\x18\x06 \x01(\x05\x12\x12\n\nexpires_at\x18\x07 \x01(\x03\x12\x17\n\x0frenewable_until\x18\x08 \x01(\x03\x12\x14\n\x0cmay_delegate\x18\t \x01(\x08\x12\x16\n\x0eremaining_hops\x18\n \x01(\x05\x12\x0e\n\x06reason\x18\x0b \x01(\t\"\xc3\x01\n\x11\x41uthorityIdentity\x12(\n\x07subject\x18\x01 \x01(\x0b\x32\x17.aether.v1.PrincipalRef\x12-\n\x0croot_subject\x18\x02 \x01(\x0b\x32\x17.aether.v1.PrincipalRef\x12)\n\x08\x64\x65legate\x18\x03 \x01(\x0b\x32\x17.aether.v1.PrincipalRef\x12*\n\tissued_by\x18\x04 \x01(\x0b\x32\x17.aether.v1.PrincipalRef\"\xd1\x01\n\rAuthoritySpan\x12\x17\n\x0fworkspace_scope\x18\x01 \x03(\t\x12\x18\n\x10max_access_level\x18\x02 \x01(\x05\x12\x15\n\raudience_type\x18\x03 \x01(\t\x12\x13\n\x0b\x61udience_id\x18\x04 \x01(\t\x12#\n\x1bvalid_while_audience_active\x18\x05 \x01(\x08\x12\x12\n\nexpires_at\x18\x06 \x01(\x03\x12\x17\n\x0frenewable_until\x18\x07 \x01(\x03\x12\x0f\n\x07revoked\x18\x08 \x01(\x08\"x\n\x18\x41uthorityGrantRevocation\x12\x10\n\x08grant_id\x18\x01 \x01(\t\x12\x15\n\rroot_grant_id\x18\x02 \x01(\t\x12\x0e\n\x06reason\x18\x03 \x01(\t\x12\x12\n\nrevoked_at\x18\x04 \x01(\x03\x12\x0f\n\x07\x63\x61scade\x18\x05 \x01(\x08\"_\n\x1d\x41uthorityRequestRoutingTarget\x12*\n\tprincipal\x18\x01 \x01(\x0b\x32\x17.aether.v1.PrincipalRef\x12\x12\n\ncapability\x18\x02 \x01(\t\"M\n\"AuthorityRequestResourceScopeEntry\x12\x15\n\rresource_type\x18\x01 \x01(\t\x12\x10\n\x08patterns\x18\x02 \x03(\t\"\xc7\x06\n\x10\x41uthorityRequest\x12\x12\n\nrequest_id\x18\x01 \x01(\t\x12\x31\n\x06status\x18\x02 \x01(\x0e\x32!.aether.v1.AuthorityRequestStatus\x12\x31\n\x10requesting_actor\x18\x03 \x01(\x0b\x32\x17.aether.v1.PrincipalRef\x12/\n\x0etarget_subject\x18\x04 \x01(\x0b\x32\x17.aether.v1.PrincipalRef\x12\x1f\n\x17\x64\x65sired_workspace_scope\x18\x05 \x03(\t\x12M\n\x16\x64\x65sired_resource_scope\x18\x06 \x03(\x0b\x32-.aether.v1.AuthorityRequestResourceScopeEntry\x12\x1f\n\x17\x64\x65sired_operation_scope\x18\x07 \x03(\t\x12\x36\n\x16requested_access_level\x18\x08 \x01(\x0e\x32\x16.aether.v1.AccessLevel\x12\"\n\x1arequested_duration_seconds\x18\t \x01(\x03\x12\x15\n\raudience_type\x18\n \x01(\t\x12\x13\n\x0b\x61udience_id\x18\x0b \x01(\t\x12@\n\x0erouting_target\x18\x0c \x01(\x0b\x32(.aether.v1.AuthorityRequestRoutingTarget\x12\x0e\n\x06reason\x18\r \x01(\t\x12\x0f\n\x07task_id\x18\x0e \x01(\t\x12;\n\x08metadata\x18\x0f \x03(\x0b\x32).aether.v1.AuthorityRequest.MetadataEntry\x12\x12\n\ncreated_at\x18\x10 \x01(\x03\x12\x12\n\nexpires_at\x18\x11 \x01(\x03\x12\x13\n\x0bresolved_at\x18\x12 \x01(\x03\x12\x18\n\x10granted_grant_id\x18\x13 \x01(\t\x12,\n\x0bresolved_by\x18\x14 \x01(\x0b\x32\x17.aether.v1.PrincipalRef\x12\x19\n\x11resolution_reason\x18\x15 \x01(\t\x1a/\n\rMetadataEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\xfa\x04\n\x1d\x43reateAuthorityRequestPayload\x12\x31\n\x10requesting_actor\x18\x01 \x01(\x0b\x32\x17.aether.v1.PrincipalRef\x12/\n\x0etarget_subject\x18\x02 \x01(\x0b\x32\x17.aether.v1.PrincipalRef\x12\x1f\n\x17\x64\x65sired_workspace_scope\x18\x03 \x03(\t\x12M\n\x16\x64\x65sired_resource_scope\x18\x04 \x03(\x0b\x32-.aether.v1.AuthorityRequestResourceScopeEntry\x12\x1f\n\x17\x64\x65sired_operation_scope\x18\x05 \x03(\t\x12\x36\n\x16requested_access_level\x18\x06 \x01(\x0e\x32\x16.aether.v1.AccessLevel\x12\"\n\x1arequested_duration_seconds\x18\x07 \x01(\x03\x12\x15\n\raudience_type\x18\x08 \x01(\t\x12\x13\n\x0b\x61udience_id\x18\t \x01(\t\x12@\n\x0erouting_target\x18\n \x01(\x0b\x32(.aether.v1.AuthorityRequestRoutingTarget\x12\x0e\n\x06reason\x18\x0b \x01(\t\x12\x0f\n\x07task_id\x18\x0c \x01(\t\x12H\n\x08metadata\x18\r \x03(\x0b\x32\x36.aether.v1.CreateAuthorityRequestPayload.MetadataEntry\x1a/\n\rMetadataEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\xca\x03\n\x1eResolveAuthorityRequestPayload\x12\x44\n\x08\x64\x65\x63ision\x18\x01 \x01(\x0e\x32\x32.aether.v1.ResolveAuthorityRequestPayload.Decision\x12\x1f\n\x17granted_workspace_scope\x18\x02 \x03(\t\x12M\n\x16granted_resource_scope\x18\x03 \x03(\x0b\x32-.aether.v1.AuthorityRequestResourceScopeEntry\x12\x1f\n\x17granted_operation_scope\x18\x04 \x03(\t\x12\x34\n\x14granted_access_level\x18\x05 \x01(\x0e\x32\x16.aether.v1.AccessLevel\x12 \n\x18granted_duration_seconds\x18\x06 \x01(\x03\x12\x0e\n\x06reason\x18\x07 \x01(\t\x12\x14\n\x0cmay_delegate\x18\x08 \x01(\x08\x12\x16\n\x0eremaining_hops\x18\t \x01(\x05\";\n\x08\x44\x65\x63ision\x12\x18\n\x14\x44\x45\x43ISION_UNSPECIFIED\x10\x00\x12\x0b\n\x07\x41PPROVE\x10\x01\x12\x08\n\x04\x44\x45NY\x10\x02\"\xa0\x01\n\x1a\x41uthorityRequestListFilter\x12\x31\n\x06status\x18\x01 \x01(\x0e\x32!.aether.v1.AuthorityRequestStatus\x12\x11\n\tworkspace\x18\x02 \x01(\t\x12\r\n\x05limit\x18\x03 \x01(\x05\x12\x0e\n\x06offset\x18\x04 \x01(\x05\x12\x1d\n\x15matching_capabilities\x18\x05 \x03(\t\"\xb5\x03\n\x19\x41uthorityRequestOperation\x12\x37\n\x02op\x18\x01 \x01(\x0e\x32+.aether.v1.AuthorityRequestOperation.OpType\x12\x12\n\nrequest_id\x18\x02 \x01(\t\x12\x38\n\x06\x63reate\x18\x03 \x01(\x0b\x32(.aether.v1.CreateAuthorityRequestPayload\x12:\n\x07resolve\x18\x04 \x01(\x0b\x32).aether.v1.ResolveAuthorityRequestPayload\x12:\n\x0blist_filter\x18\x05 \x01(\x0b\x32%.aether.v1.AuthorityRequestListFilter\x12\x19\n\x11\x63lient_request_id\x18\x06 \x01(\t\x12\x0e\n\x06reason\x18\x07 \x01(\t\"n\n\x06OpType\x12$\n AUTHORITY_REQUEST_OP_UNSPECIFIED\x10\x00\x12\n\n\x06\x43REATE\x10\x01\x12\x07\n\x03GET\x10\x02\x12\x10\n\x0cLIST_PENDING\x10\x03\x12\x0b\n\x07RESOLVE\x10\x04\x12\n\n\x06\x43\x41NCEL\x10\x05\"\xd0\x01\n!AuthorityRequestOperationResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\r\n\x05\x65rror\x18\x02 \x01(\t\x12\x19\n\x11\x63lient_request_id\x18\x03 \x01(\t\x12,\n\x07request\x18\x04 \x01(\x0b\x32\x1b.aether.v1.AuthorityRequest\x12-\n\x08requests\x18\x05 \x03(\x0b\x32\x1b.aether.v1.AuthorityRequest\x12\x13\n\x0btotal_count\x18\x06 \x01(\x05\"\x8b\x03\n\x15\x41uthorityRequestEvent\x12>\n\nevent_type\x18\x01 \x01(\x0e\x32*.aether.v1.AuthorityRequestEvent.EventType\x12,\n\x07request\x18\x02 \x01(\x0b\x32\x1b.aether.v1.AuthorityRequest\x12\x12\n\nemitted_at\x18\x03 \x01(\x03\"\xef\x01\n\tEventType\x12\'\n#AUTHORITY_REQUEST_EVENT_UNSPECIFIED\x10\x00\x12#\n\x1f\x41UTHORITY_REQUEST_EVENT_CREATED\x10\x01\x12$\n AUTHORITY_REQUEST_EVENT_APPROVED\x10\x02\x12\"\n\x1e\x41UTHORITY_REQUEST_EVENT_DENIED\x10\x03\x12#\n\x1f\x41UTHORITY_REQUEST_EVENT_EXPIRED\x10\x04\x12%\n!AUTHORITY_REQUEST_EVENT_CANCELLED\x10\x05\"\x84\x02\n\x0eTokenOperation\x12,\n\x02op\x18\x01 \x01(\x0e\x32 .aether.v1.TokenOperation.OpType\x12\x10\n\x08token_id\x18\x02 \x01(\t\x12\x35\n\x0e\x63reate_request\x18\x03 \x01(\x0b\x32\x1d.aether.v1.TokenCreateRequest\x12&\n\x06\x66ilter\x18\x04 \x01(\x0b\x32\x16.aether.v1.TokenFilter\x12\x12\n\nrequest_id\x18\x05 \x01(\t\"?\n\x06OpType\x12\x08\n\x04LIST\x10\x00\x12\x07\n\x03GET\x10\x01\x12\n\n\x06\x43REATE\x10\x02\x12\n\n\x06\x44\x45LETE\x10\x03\x12\n\n\x06REVOKE\x10\x04\"\x94\x01\n\x12TokenCreateRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x16\n\x0eprincipal_type\x18\x02 \x01(\t\x12\x1a\n\x12workspace_patterns\x18\x03 \x03(\t\x12\x0e\n\x06scopes\x18\x04 \x03(\t\x12\x18\n\x10\x65xpires_in_hours\x18\x05 \x01(\x05\x12\x12\n\ncreated_by\x18\x06 \x01(\t\"E\n\x0bTokenFilter\x12\r\n\x05limit\x18\x01 \x01(\x05\x12\x0e\n\x06offset\x18\x02 \x01(\x05\x12\x17\n\x0finclude_revoked\x18\x03 \x01(\x08\"\xf4\x01\n\tTokenInfo\x12\n\n\x02id\x18\x01 \x01(\t\x12\x0c\n\x04name\x18\x02 \x01(\t\x12\x16\n\x0eprincipal_type\x18\x03 \x01(\t\x12\x1a\n\x12workspace_patterns\x18\x04 \x03(\t\x12\x0e\n\x06scopes\x18\x05 \x03(\t\x12\x12\n\ncreated_by\x18\x06 \x01(\t\x12\x12\n\nexpires_at\x18\x07 \x01(\x03\x12\x14\n\x0clast_used_at\x18\x08 \x01(\x03\x12\x0f\n\x07revoked\x18\t \x01(\x08\x12\x12\n\nrevoked_at\x18\n \x01(\x03\x12\x12\n\ncreated_at\x18\x0b \x01(\x03\x12\x12\n\nupdated_at\x18\x0c \x01(\x03\"\xfa\x01\n\rTokenResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\r\n\x05\x65rror\x18\x02 \x01(\t\x12\x0f\n\x07message\x18\x03 \x01(\t\x12#\n\x05token\x18\x04 \x01(\x0b\x32\x14.aether.v1.TokenInfo\x12$\n\x06tokens\x18\x05 \x03(\x0b\x32\x14.aether.v1.TokenInfo\x12\x13\n\x0btotal_count\x18\x06 \x01(\x05\x12\x17\n\x0fplaintext_token\x18\x07 \x01(\t\x12+\n\rcreated_token\x18\x08 \x01(\x0b\x32\x14.aether.v1.TokenInfo\x12\x12\n\nrequest_id\x18\t \x01(\t\"\xb6\x02\n\x0eProgressReport\x12\x0f\n\x07task_id\x18\x01 \x01(\t\x12\r\n\x05state\x18\x02 \x01(\t\x12\x12\n\ncompletion\x18\x03 \x01(\x01\x12\x0f\n\x07summary\x18\x04 \x01(\t\x12%\n\x04step\x18\x05 \x01(\x0b\x32\x17.aether.v1.ProgressStep\x12\x11\n\trecipient\x18\x06 \x01(\t\x12\x12\n\nrequest_id\x18\x07 \x01(\t\x12\x39\n\x08metadata\x18\x08 \x03(\x0b\x32\'.aether.v1.ProgressReport.MetadataEntry\x12%\n\x04kind\x18\t \x01(\x0e\x32\x17.aether.v1.ProgressKind\x1a/\n\rMetadataEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"f\n\x0cProgressStep\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0e\n\x06\x64\x65tail\x18\x02 \x01(\t\x12\x10\n\x08sequence\x18\x03 \x01(\x05\x12\x13\n\x0btotal_steps\x18\x04 \x01(\x05\x12\x11\n\tstep_type\x18\x05 \x01(\t\"\xef\x02\n\x0eProgressUpdate\x12\x0e\n\x06source\x18\x01 \x01(\t\x12\x0f\n\x07task_id\x18\x02 \x01(\t\x12\r\n\x05state\x18\x03 \x01(\t\x12\x12\n\ncompletion\x18\x04 \x01(\x01\x12\x0f\n\x07summary\x18\x05 \x01(\t\x12%\n\x04step\x18\x06 \x01(\x0b\x32\x17.aether.v1.ProgressStep\x12\x14\n\x0ctimestamp_ms\x18\x07 \x01(\x03\x12\x11\n\tworkspace\x18\x08 \x01(\t\x12\x12\n\nrequest_id\x18\t \x01(\t\x12\x39\n\x08metadata\x18\n \x03(\x0b\x32\'.aether.v1.ProgressUpdate.MetadataEntry\x12\x11\n\trecipient\x18\x0b \x01(\t\x12%\n\x04kind\x18\x0c \x01(\x0e\x32\x17.aether.v1.ProgressKind\x1a/\n\rMetadataEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\xaa\x05\n\x11WorkflowOperation\x12/\n\x02op\x18\x01 \x01(\x0e\x32#.aether.v1.WorkflowOperation.OpType\x12\n\n\x02id\x18\x02 \x01(\t\x12\x14\n\x0csecondary_id\x18\x03 \x01(\t\x12\x11\n\tworkspace\x18\x04 \x01(\t\x12\x0c\n\x04\x64\x61ta\x18\x05 \x01(\x0c\x12\x12\n\nrequest_id\x18\x06 \x01(\t\x12\x15\n\rstatus_filter\x18\x07 \x01(\t\"\xf5\x03\n\x06OpType\x12\x0e\n\nLIST_RULES\x10\x00\x12\x0c\n\x08GET_RULE\x10\x01\x12\x0f\n\x0b\x43REATE_RULE\x10\x02\x12\x0f\n\x0bUPDATE_RULE\x10\x03\x12\x0f\n\x0b\x44\x45LETE_RULE\x10\x04\x12\x12\n\x0eLIST_WORKFLOWS\x10\x05\x12\x10\n\x0cGET_WORKFLOW\x10\x06\x12\x13\n\x0f\x43REATE_WORKFLOW\x10\x07\x12\x13\n\x0f\x44\x45LETE_WORKFLOW\x10\x08\x12\x12\n\x0eLIST_SCHEDULES\x10\t\x12\x13\n\x0f\x43REATE_SCHEDULE\x10\n\x12\x13\n\x0f\x44\x45LETE_SCHEDULE\x10\x0b\x12\x13\n\x0fLIST_EXECUTIONS\x10\x0c\x12\x11\n\rGET_EXECUTION\x10\r\x12\x14\n\x10\x43\x41NCEL_EXECUTION\x10\x0e\x12\x17\n\x13LIST_STATE_MACHINES\x10\x0f\x12\x15\n\x11GET_STATE_MACHINE\x10\x10\x12\x18\n\x14\x43REATE_STATE_MACHINE\x10\x11\x12\x18\n\x14\x44\x45LETE_STATE_MACHINE\x10\x12\x12\x15\n\x11LIST_SM_INSTANCES\x10\x13\x12\x13\n\x0fGET_SM_INSTANCE\x10\x14\x12\x16\n\x12\x43REATE_SM_INSTANCE\x10\x15\x12\x11\n\rSEND_SM_EVENT\x10\x16\x12\x13\n\x0fUPSERT_SCHEDULE\x10\x17\"z\n\x10WorkflowResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\r\n\x05\x65rror\x18\x02 \x01(\t\x12\x0f\n\x07message\x18\x03 \x01(\t\x12\x0c\n\x04\x64\x61ta\x18\x04 \x01(\x0c\x12\x13\n\x0btotal_count\x18\x05 \x01(\x05\x12\x12\n\nrequest_id\x18\x06 \x01(\t\"\xf6\x01\n\x0fMessageEnvelope\x12\x0e\n\x06source\x18\x01 \x01(\t\x12\x0f\n\x07payload\x18\x02 \x01(\x0c\x12,\n\x0cmessage_type\x18\x03 \x01(\x0e\x32\x16.aether.v1.MessageType\x12\x14\n\x0ctimestamp_ms\x18\x04 \x01(\x03\x12:\n\x08metadata\x18\x05 \x03(\x0b\x32(.aether.v1.MessageEnvelope.MetadataEntry\x12\x11\n\tworkspace\x18\x06 \x01(\t\x1a/\n\rMetadataEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\xf7\x03\n\nAuditQuery\x12\x12\n\nrequest_id\x18\x01 \x01(\t\x12\x12\n\nstart_time\x18\x02 \x01(\x03\x12\x10\n\x08\x65nd_time\x18\x03 \x01(\x03\x12\x12\n\nevent_type\x18\x04 \x01(\t\x12\x12\n\nactor_type\x18\x05 \x01(\t\x12\x10\n\x08\x61\x63tor_id\x18\x06 \x01(\t\x12\x15\n\rresource_type\x18\x07 \x01(\t\x12\x13\n\x0bresource_id\x18\x08 \x01(\t\x12\x11\n\toperation\x18\t \x01(\t\x12\x11\n\tworkspace\x18\n \x01(\t\x12\x15\n\ronly_failures\x18\x0b \x01(\x08\x12\r\n\x05limit\x18\x0c \x01(\x05\x12\x0e\n\x06offset\x18\r \x01(\x05\x12\x14\n\x0csubject_type\x18\x0e \x01(\t\x12\x12\n\nsubject_id\x18\x0f \x01(\t\x12\x16\n\x0e\x61uthority_mode\x18\x10 \x01(\t\x12\x1a\n\x12\x61uthority_grant_id\x18\x11 \x01(\t\x12\x36\n\rauthorization\x18\x12 \x01(\x0b\x32\x1f.aether.v1.AuthorizationContext\x12\x1b\n\x13\x65xclude_actor_types\x18\x13 \x03(\t\x12\x1a\n\x12\x65xclude_workspaces\x18\x14 \x03(\t\x12\x1e\n\x16\x65xclude_service_direct\x18\x15 \x01(\x08\"\x85\x01\n\x12\x41uditQueryResponse\x12\x12\n\nrequest_id\x18\x01 \x01(\t\x12\x0f\n\x07success\x18\x02 \x01(\x08\x12\r\n\x05\x65rror\x18\x03 \x01(\t\x12&\n\x07\x65ntries\x18\x04 \x03(\x0b\x32\x15.aether.v1.AuditEntry\x12\x13\n\x0btotal_count\x18\x05 \x01(\x05\"\x8a\x04\n\nAuditEntry\x12\x10\n\x08\x61udit_id\x18\x01 \x01(\x03\x12\x11\n\ttimestamp\x18\x02 \x01(\x03\x12\x12\n\nevent_type\x18\x03 \x01(\t\x12\x12\n\nactor_type\x18\x04 \x01(\t\x12\x10\n\x08\x61\x63tor_id\x18\x05 \x01(\t\x12\x15\n\rresource_type\x18\x06 \x01(\t\x12\x13\n\x0bresource_id\x18\x07 \x01(\t\x12\x11\n\toperation\x18\x08 \x01(\t\x12\x11\n\tworkspace\x18\t \x01(\t\x12\x12\n\nsession_id\x18\n \x01(\t\x12\x12\n\ngateway_id\x18\x0b \x01(\t\x12\x0f\n\x07success\x18\x0c \x01(\x08\x12\x15\n\rerror_message\x18\r \x01(\t\x12\x15\n\rmetadata_json\x18\x0e \x01(\t\x12\x14\n\x0csubject_type\x18\x0f \x01(\t\x12\x12\n\nsubject_id\x18\x10 \x01(\t\x12\x19\n\x11root_subject_type\x18\x11 \x01(\t\x12\x17\n\x0froot_subject_id\x18\x12 \x01(\t\x12\x16\n\x0e\x61uthority_mode\x18\x13 \x01(\t\x12\x1f\n\x17root_authority_grant_id\x18\x14 \x01(\t\x12\x1a\n\x12\x61uthority_grant_id\x18\x15 \x01(\t\x12!\n\x19parent_authority_grant_id\x18\x16 \x01(\t\x12\x0e\n\x06source\x18\x17 \x01(\t\"\xb7\x02\n\x17SubmitAuditEventRequest\x12\x12\n\nevent_type\x18\x01 \x01(\t\x12\x11\n\toperation\x18\x02 \x01(\t\x12\x15\n\rresource_type\x18\x03 \x01(\t\x12\x13\n\x0bresource_id\x18\x04 \x01(\t\x12\x11\n\tworkspace\x18\x05 \x01(\t\x12\x0f\n\x07success\x18\x06 \x01(\x08\x12\x15\n\rerror_message\x18\x07 \x01(\t\x12\x42\n\x08metadata\x18\x08 \x03(\x0b\x32\x30.aether.v1.SubmitAuditEventRequest.MetadataEntry\x12\x19\n\x11\x63lient_request_id\x18\t \x01(\t\x1a/\n\rMetadataEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"q\n\x18SubmitAuditEventResponse\x12\x19\n\x11\x63lient_request_id\x18\x01 \x01(\t\x12\x0f\n\x07success\x18\x02 \x01(\x08\x12\x12\n\nerror_code\x18\x03 \x01(\t\x12\x15\n\rerror_message\x18\x04 \x01(\t\"\xfe\x03\n\x10ProxyHttpRequest\x12\x12\n\nrequest_id\x18\x01 \x01(\t\x12\x14\n\x0ctarget_topic\x18\x02 \x01(\t\x12\x0e\n\x06method\x18\x03 \x01(\t\x12\x0c\n\x04path\x18\x04 \x01(\t\x12\x39\n\x07headers\x18\x05 \x03(\x0b\x32(.aether.v1.ProxyHttpRequest.HeadersEntry\x12\x0c\n\x04\x62ody\x18\x06 \x01(\x0c\x12\x14\n\x0c\x62ody_chunked\x18\x07 \x01(\x08\x12\x36\n\rauthorization\x18\x08 \x01(\x0b\x32\x1f.aether.v1.AuthorizationContext\x12\x15\n\rapp_workspace\x18\t \x01(\t\x12\x12\n\ntimeout_ms\x18\n \x01(\x03\x12\x18\n\x10\x66ollow_redirects\x18\x0b \x01(\x08\x12\x14\n\x0c\x62\x61\x63kend_name\x18\x0c \x01(\t\x12$\n\x1cstream_response_indefinitely\x18\r \x01(\x08\x12\x1e\n\x16stream_idle_timeout_ms\x18\x0e \x01(\x03\x12\x1f\n\x17max_response_body_bytes\x18\x0f \x01(\x03\x12\x19\n\x11proxy_chain_depth\x18\x10 \x01(\r\x1a.\n\x0cHeadersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\xf2\x01\n\x11ProxyHttpResponse\x12\x12\n\nrequest_id\x18\x01 \x01(\t\x12\x13\n\x0bstatus_code\x18\x02 \x01(\x05\x12:\n\x07headers\x18\x03 \x03(\x0b\x32).aether.v1.ProxyHttpResponse.HeadersEntry\x12\x0c\n\x04\x62ody\x18\x04 \x01(\x0c\x12\x14\n\x0c\x62ody_chunked\x18\x05 \x01(\x08\x12$\n\x05\x65rror\x18\x06 \x01(\x0b\x32\x15.aether.v1.ProxyError\x1a.\n\x0cHeadersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"d\n\x12ProxyHttpBodyChunk\x12\x12\n\nrequest_id\x18\x01 \x01(\t\x12\x12\n\nis_request\x18\x02 \x01(\x08\x12\x0b\n\x03seq\x18\x03 \x01(\r\x12\x0c\n\x04\x64\x61ta\x18\x04 \x01(\x0c\x12\x0b\n\x03\x66in\x18\x05 \x01(\x08\"\xe2\x01\n\nProxyError\x12(\n\x04kind\x18\x01 \x01(\x0e\x32\x1a.aether.v1.ProxyError.Kind\x12\x0f\n\x07message\x18\x02 \x01(\t\"\x98\x01\n\x04Kind\x12\x0b\n\x07UNKNOWN\x10\x00\x12\x0f\n\x0b\x44IAL_FAILED\x10\x01\x12\x0b\n\x07TIMEOUT\x10\x02\x12\x12\n\x0eUPSTREAM_RESET\x10\x03\x12\x0e\n\nACL_DENIED\x10\x04\x12\x17\n\x13SIDECAR_UNAVAILABLE\x10\x05\x12\x15\n\x11PAYLOAD_TOO_LARGE\x10\x06\x12\x11\n\rDECODE_FAILED\x10\x07\"\xbd\x03\n\nTunnelOpen\x12\x11\n\ttunnel_id\x18\x01 \x01(\t\x12\x14\n\x0ctarget_topic\x18\x02 \x01(\t\x12\x30\n\x08protocol\x18\x03 \x01(\x0e\x32\x1e.aether.v1.TunnelOpen.Protocol\x12\x13\n\x0bremote_hint\x18\x04 \x01(\t\x12\x35\n\x08metadata\x18\x05 \x03(\x0b\x32#.aether.v1.TunnelOpen.MetadataEntry\x12\x36\n\rauthorization\x18\x06 \x01(\x0b\x32\x1f.aether.v1.AuthorizationContext\x12\x17\n\x0fidle_timeout_ms\x18\x07 \x01(\x03\x12\x11\n\tmax_bytes\x18\x08 \x01(\x03\x12\x15\n\rsession_token\x18\t \x01(\t\x12\x14\n\x0c\x62\x61\x63kend_name\x18\n \x01(\t\x12\x19\n\x11proxy_chain_depth\x18\x0b \x01(\r\x1a/\n\rMetadataEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"+\n\x08Protocol\x12\x07\n\x03TCP\x10\x00\x12\x07\n\x03UDP\x10\x01\x12\r\n\tWEBSOCKET\x10\x02\"G\n\nTunnelData\x12\x11\n\ttunnel_id\x18\x01 \x01(\t\x12\x0b\n\x03seq\x18\x02 \x01(\r\x12\x0c\n\x04\x64\x61ta\x18\x03 \x01(\x0c\x12\x0b\n\x03\x66in\x18\x04 \x01(\x08\"\xad\x01\n\x0bTunnelClose\x12\x11\n\ttunnel_id\x18\x01 \x01(\t\x12-\n\x06reason\x18\x02 \x01(\x0e\x32\x1d.aether.v1.TunnelClose.Reason\x12\x0e\n\x06\x64\x65tail\x18\x03 \x01(\t\"L\n\x06Reason\x12\n\n\x06NORMAL\x10\x00\x12\x0e\n\nPEER_RESET\x10\x01\x12\x10\n\x0cIDLE_TIMEOUT\x10\x02\x12\t\n\x05QUOTA\x10\x03\x12\t\n\x05\x45RROR\x10\x04\"@\n\tTunnelAck\x12\x11\n\ttunnel_id\x18\x01 \x01(\t\x12\x0f\n\x07\x61\x63k_seq\x18\x02 \x01(\r\x12\x0f\n\x07\x63redits\x18\x03 \x01(\r\"\xbd\x01\n\x17ResolveAuthorityRequest\x12\x12\n\nrequest_id\x18\x01 \x01(\t\x12&\n\x05\x61\x63tor\x18\x02 \x01(\x0b\x32\x17.aether.v1.PrincipalRef\x12\x10\n\x08grant_id\x18\x03 \x01(\t\x12(\n\x07subject\x18\x04 \x01(\x0b\x32\x17.aether.v1.PrincipalRef\x12\x15\n\raudience_type\x18\x05 \x01(\t\x12\x13\n\x0b\x61udience_id\x18\x06 \x01(\t\"z\n\x18ResolveAuthorityResponse\x12\x12\n\nrequest_id\x18\x01 \x01(\t\x12\n\n\x02ok\x18\x02 \x01(\x08\x12\r\n\x05\x65rror\x18\x03 \x01(\t\x12/\n\tauthority\x18\x04 \x01(\x0b\x32\x1c.aether.v1.ResolvedAuthority\"\x93\x01\n\x11ResolvedAuthority\x12&\n\x05\x61\x63tor\x18\x01 \x01(\x0b\x32\x17.aether.v1.PrincipalRef\x12(\n\x07subject\x18\x02 \x01(\x0b\x32\x17.aether.v1.PrincipalRef\x12,\n\x05grant\x18\x03 \x01(\x0b\x32\x1d.aether.v1.AuthorityGrantInfo\"\x88\x02\n\x12\x41uthorityGrantInfo\x12\x10\n\x08grant_id\x18\x01 \x01(\t\x12\x14\n\x0csubject_type\x18\x02 \x01(\t\x12\x12\n\nsubject_id\x18\x03 \x01(\t\x12\x19\n\x11root_subject_type\x18\x04 \x01(\t\x12\x17\n\x0froot_subject_id\x18\x05 \x01(\t\x12\x15\n\raudience_type\x18\x06 \x01(\t\x12\x13\n\x0b\x61udience_id\x18\x07 \x01(\t\x12\x18\n\x10max_access_level\x18\x08 \x01(\x05\x12\x17\n\x0fworkspace_scope\x18\t \x03(\t\x12\x12\n\nexpires_at\x18\n \x01(\x03\x12\x0f\n\x07revoked\x18\x0b \x01(\x08\"Y\n\x17\x43onnectionStatusRequest\x12\x12\n\nrequest_id\x18\x01 \x01(\t\x12*\n\tprincipal\x18\x02 \x01(\x0b\x32\x17.aether.v1.PrincipalRef\"r\n\x18\x43onnectionStatusResponse\x12\x12\n\nrequest_id\x18\x01 \x01(\t\x12\n\n\x02ok\x18\x02 \x01(\x08\x12\r\n\x05\x65rror\x18\x03 \x01(\t\x12\x11\n\tconnected\x18\x04 \x01(\x08\x12\x14\n\x0clast_seen_at\x18\x05 \x01(\x03\"\x9d\x02\n\x19TaskSubscriptionOperation\x12\x37\n\x02op\x18\x01 \x01(\x0e\x32+.aether.v1.TaskSubscriptionOperation.OpType\x12\x0f\n\x07task_id\x18\x02 \x01(\t\x12\x11\n\trecursive\x18\x03 \x01(\x08\x12\x19\n\x11\x63lient_request_id\x18\x04 \x01(\t\x12\x1f\n\x17start_timestamp_unix_ms\x18\x05 \x01(\x03\x12\x17\n\x0fsubscription_id\x18\x06 \x01(\t\"N\n\x06OpType\x12$\n TASK_SUBSCRIPTION_OP_UNSPECIFIED\x10\x00\x12\r\n\tSUBSCRIBE\x10\x01\x12\x0f\n\x0bUNSUBSCRIBE\x10\x02\"\x88\x01\n!TaskSubscriptionOperationResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\r\n\x05\x65rror\x18\x02 \x01(\t\x12\x19\n\x11\x63lient_request_id\x18\x03 \x01(\t\x12\x0f\n\x07task_id\x18\x04 \x01(\t\x12\x17\n\x0fsubscription_id\x18\x05 \x01(\t\"\xfb\x02\n\tTaskEvent\x12\x0f\n\x07task_id\x18\x01 \x01(\t\x12\x1a\n\x12\x65mitted_at_unix_ms\x18\x02 \x01(\x03\x12\x11\n\tworkspace\x18\x03 \x01(\t\x12\x16\n\x0eparent_task_id\x18\x04 \x01(\t\x12\x17\n\x0fsubscription_id\x18\x05 \x01(\t\x12;\n\x0estatus_changed\x18\n \x01(\x0b\x32!.aether.v1.TaskStatusChangedEventH\x00\x12\x30\n\x08progress\x18\x0b \x01(\x0b\x32\x1c.aether.v1.TaskProgressEventH\x00\x12=\n\x0f\x63hild_lifecycle\x18\x0c \x01(\x0b\x32\".aether.v1.TaskChildLifecycleEventH\x00\x12\x46\n\x11\x61uthority_request\x18\r \x01(\x0b\x32).aether.v1.TaskAuthorityRequestEventRelayH\x00\x42\x07\n\x05\x65vent\"~\n\x16TaskStatusChangedEvent\x12*\n\x0b\x66rom_status\x18\x01 \x01(\x0e\x32\x15.aether.v1.TaskStatus\x12(\n\tto_status\x18\x02 \x01(\x0e\x32\x15.aether.v1.TaskStatus\x12\x0e\n\x06reason\x18\x03 \x01(\t\"\xb4\x01\n\x11TaskProgressEvent\x12\r\n\x05state\x18\x01 \x01(\t\x12\x10\n\x08progress\x18\x02 \x01(\x01\x12\x0f\n\x07message\x18\x03 \x01(\t\x12<\n\x08metadata\x18\x04 \x03(\x0b\x32*.aether.v1.TaskProgressEvent.MetadataEntry\x1a/\n\rMetadataEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"p\n\x17TaskChildLifecycleEvent\x12\x15\n\rchild_task_id\x18\x01 \x01(\t\x12+\n\x0c\x63hild_status\x18\x02 \x01(\x0e\x32\x15.aether.v1.TaskStatus\x12\x11\n\tlifecycle\x18\x03 \x01(\t\"Q\n\x1eTaskAuthorityRequestEventRelay\x12/\n\x05\x65vent\x18\x01 \x01(\x0b\x32 .aether.v1.AuthorityRequestEvent*t\n\x0bMessageType\x12\x1c\n\x18MESSAGE_TYPE_UNSPECIFIED\x10\x00\x12\x08\n\x04\x43HAT\x10\x01\x12\x0b\n\x07\x43ONTROL\x10\x02\x12\r\n\tTOOL_CALL\x10\x03\x12\t\n\x05\x45VENT\x10\x04\x12\n\n\x06METRIC\x10\x05\x12\n\n\x06OPAQUE\x10\x06*\xf2\x01\n\rPrincipalType\x12\x1e\n\x1aPRINCIPAL_TYPE_UNSPECIFIED\x10\x00\x12\x13\n\x0fPRINCIPAL_AGENT\x10\x01\x12\x12\n\x0ePRINCIPAL_TASK\x10\x02\x12\x12\n\x0ePRINCIPAL_USER\x10\x03\x12\x1a\n\x16PRINCIPAL_ORCHESTRATOR\x10\x04\x12\x1d\n\x19PRINCIPAL_WORKFLOW_ENGINE\x10\x05\x12\x1c\n\x18PRINCIPAL_METRICS_BRIDGE\x10\x06\x12\x14\n\x10PRINCIPAL_BRIDGE\x10\x07\x12\x15\n\x11PRINCIPAL_SERVICE\x10\x08*\xc4\x02\n\nTaskStatus\x12\x1b\n\x17TASK_STATUS_UNSPECIFIED\x10\x00\x12\x16\n\x12TASK_STATUS_QUEUED\x10\x01\x12\x17\n\x13TASK_STATUS_RUNNING\x10\x02\x12\x19\n\x15TASK_STATUS_COMPLETED\x10\x03\x12\x16\n\x12TASK_STATUS_FAILED\x10\x04\x12\x19\n\x15TASK_STATUS_CANCELLED\x10\x05\x12\x1d\n\x19TASK_STATUS_WAITING_INPUT\x10\x06\x12!\n\x1dTASK_STATUS_WAITING_AUTHORITY\x10\x07\x12\"\n\x1eTASK_STATUS_WAITING_DEPENDENCY\x10\x08\x12\x1a\n\x16TASK_STATUS_HIBERNATED\x10\t\x12\x18\n\x14TASK_STATUS_REJECTED\x10\n*\x81\x01\n\x0cHealthStatus\x12\x1d\n\x19HEALTH_STATUS_UNSPECIFIED\x10\x00\x12\x19\n\x15HEALTH_STATUS_HEALTHY\x10\x01\x12\x1a\n\x16HEALTH_STATUS_DEGRADED\x10\x02\x12\x1b\n\x17HEALTH_STATUS_UNHEALTHY\x10\x03*s\n\x11HealthCheckStatus\x12#\n\x1fHEALTH_CHECK_STATUS_UNSPECIFIED\x10\x00\x12\x1a\n\x16HEALTH_CHECK_STATUS_OK\x10\x01\x12\x1d\n\x19HEALTH_CHECK_STATUS_ERROR\x10\x02*\xc3\x01\n\x0b\x41\x63\x63\x65ssLevel\x12\x1c\n\x18\x41\x43\x43\x45SS_LEVEL_UNSPECIFIED\x10\x00\x12\x15\n\x11\x41\x43\x43\x45SS_LEVEL_NONE\x10\x01\x12\x15\n\x11\x41\x43\x43\x45SS_LEVEL_READ\x10\x02\x12\x1a\n\x16\x41\x43\x43\x45SS_LEVEL_READWRITE\x10\x03\x12\x17\n\x13\x41\x43\x43\x45SS_LEVEL_MANAGE\x10\x04\x12\x16\n\x12\x41\x43\x43\x45SS_LEVEL_ADMIN\x10\x05\x12\x1b\n\x17\x41\x43\x43\x45SS_LEVEL_SUPERADMIN\x10\x06*=\n\x12TaskAssignmentMode\x12\x0f\n\x0bSELF_ASSIGN\x10\x00\x12\x0c\n\x08TARGETED\x10\x01\x12\x08\n\x04POOL\x10\x02*t\n\tTaskClass\x12\x1a\n\x16TASK_CLASS_UNSPECIFIED\x10\x00\x12\x1a\n\x16TASK_CLASS_INTERACTIVE\x10\x01\x12\x19\n\x15TASK_CLASS_BACKGROUND\x10\x02\x12\x14\n\x10TASK_CLASS_BATCH\x10\x03*\x94\x01\n\nWaitReason\x12\x1b\n\x17WAIT_REASON_UNSPECIFIED\x10\x00\x12\x15\n\x11WAIT_REASON_INPUT\x10\x01\x12\x19\n\x15WAIT_REASON_AUTHORITY\x10\x02\x12\x1a\n\x16WAIT_REASON_DEPENDENCY\x10\x03\x12\x1b\n\x17WAIT_REASON_HIBERNATION\x10\x04*\x82\x02\n\x16\x41uthorityRequestStatus\x12(\n$AUTHORITY_REQUEST_STATUS_UNSPECIFIED\x10\x00\x12$\n AUTHORITY_REQUEST_STATUS_PENDING\x10\x01\x12%\n!AUTHORITY_REQUEST_STATUS_APPROVED\x10\x02\x12#\n\x1f\x41UTHORITY_REQUEST_STATUS_DENIED\x10\x03\x12$\n AUTHORITY_REQUEST_STATUS_EXPIRED\x10\x04\x12&\n\"AUTHORITY_REQUEST_STATUS_CANCELLED\x10\x05*t\n\x0cProgressKind\x12\x1d\n\x19PROGRESS_KIND_UNSPECIFIED\x10\x00\x12\x16\n\x12PROGRESS_KIND_CHAT\x10\x01\x12\x15\n\x11PROGRESS_KIND_APP\x10\x02\x12\x16\n\x12PROGRESS_KIND_TASK\x10\x03\x32X\n\rAetherGateway\x12G\n\x07\x43onnect\x12\x1a.aether.v1.UpstreamMessage\x1a\x1c.aether.v1.DownstreamMessage(\x01\x30\x01\x42-Z+github.com/scitrera/aether/api/proto;aetherb\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x0c\x61\x65ther.proto\x12\taether.v1\"\xb1\r\n\x0fUpstreamMessage\x12)\n\x04init\x18\x01 \x01(\x0b\x32\x19.aether.v1.InitConnectionH\x00\x12&\n\x04send\x18\x02 \x01(\x0b\x32\x16.aether.v1.SendMessageH\x00\x12\x36\n\x10switch_workspace\x18\x03 \x01(\x0b\x32\x1a.aether.v1.SwitchWorkspaceH\x00\x12\'\n\x05kv_op\x18\x04 \x01(\x0b\x32\x16.aether.v1.KVOperationH\x00\x12\x33\n\x0b\x63reate_task\x18\x05 \x01(\x0b\x32\x1c.aether.v1.CreateTaskRequestH\x00\x12\x37\n\rcheckpoint_op\x18\x06 \x01(\x0b\x32\x1e.aether.v1.CheckpointOperationH\x00\x12,\n\x0b\x61\x64min_query\x18\x07 \x01(\x0b\x32\x15.aether.v1.AdminQueryH\x00\x12\x31\n\nsession_op\x18\x08 \x01(\x0b\x32\x1b.aether.v1.SessionOperationH\x00\x12*\n\ntask_query\x18\t \x01(\x0b\x32\x14.aether.v1.TaskQueryH\x00\x12+\n\x07task_op\x18\n \x01(\x0b\x32\x18.aether.v1.TaskOperationH\x00\x12\x35\n\x0cworkspace_op\x18\x0b \x01(\x0b\x32\x1d.aether.v1.WorkspaceOperationH\x00\x12-\n\x08\x61gent_op\x18\x0c \x01(\x0b\x32\x19.aether.v1.AgentOperationH\x00\x12)\n\x06\x61\x63l_op\x18\r \x01(\x0b\x32\x17.aether.v1.ACLOperationH\x00\x12-\n\x08progress\x18\x0e \x01(\x0b\x32\x19.aether.v1.ProgressReportH\x00\x12\x33\n\x0bworkflow_op\x18\x0f \x01(\x0b\x32\x1c.aether.v1.WorkflowOperationH\x00\x12\x38\n\x11workflow_response\x18\x10 \x01(\x0b\x32\x1b.aether.v1.WorkflowResponseH\x00\x12-\n\x08token_op\x18\x11 \x01(\x0b\x32\x19.aether.v1.TokenOperationH\x00\x12,\n\x0b\x61udit_query\x18\x12 \x01(\x0b\x32\x15.aether.v1.AuditQueryH\x00\x12@\n\x12\x61uthority_grant_op\x18\x13 \x01(\x0b\x32\".aether.v1.AuthorityGrantOperationH\x00\x12\x39\n\x12proxy_http_request\x18\x14 \x01(\x0b\x32\x1b.aether.v1.ProxyHttpRequestH\x00\x12>\n\x15proxy_http_body_chunk\x18\x15 \x01(\x0b\x32\x1d.aether.v1.ProxyHttpBodyChunkH\x00\x12,\n\x0btunnel_open\x18\x16 \x01(\x0b\x32\x15.aether.v1.TunnelOpenH\x00\x12,\n\x0btunnel_data\x18\x17 \x01(\x0b\x32\x15.aether.v1.TunnelDataH\x00\x12.\n\x0ctunnel_close\x18\x18 \x01(\x0b\x32\x16.aether.v1.TunnelCloseH\x00\x12;\n\x13proxy_http_response\x18\x19 \x01(\x0b\x32\x1c.aether.v1.ProxyHttpResponseH\x00\x12*\n\ntunnel_ack\x18\x1a \x01(\x0b\x32\x14.aether.v1.TunnelAckH\x00\x12G\n\x19resolve_authority_request\x18\x1b \x01(\x0b\x32\".aether.v1.ResolveAuthorityRequestH\x00\x12G\n\x19\x63onnection_status_request\x18\x1c \x01(\x0b\x32\".aether.v1.ConnectionStatusRequestH\x00\x12@\n\x12submit_audit_event\x18\x1d \x01(\x0b\x32\".aether.v1.SubmitAuditEventRequestH\x00\x12\x44\n\x14\x61uthority_request_op\x18\x1e \x01(\x0b\x32$.aether.v1.AuthorityRequestOperationH\x00\x12\x44\n\x14task_subscription_op\x18\x1f \x01(\x0b\x32$.aether.v1.TaskSubscriptionOperationH\x00\x12\x19\n\x11\x61\x63tive_extensions\x18 \x03(\tB\t\n\x07payload\"\xba\x10\n\x11\x44ownstreamMessage\x12)\n\x03msg\x18\x01 \x01(\x0b\x32\x1a.aether.v1.IncomingMessageH\x00\x12+\n\x06\x63onfig\x18\x02 \x01(\x0b\x32\x19.aether.v1.ConfigSnapshotH\x00\x12#\n\x06signal\x18\x03 \x01(\x0b\x32\x11.aether.v1.SignalH\x00\x12)\n\x05\x65rror\x18\x04 \x01(\x0b\x32\x18.aether.v1.ErrorResponseH\x00\x12#\n\x02kv\x18\x05 \x01(\x0b\x32\x15.aether.v1.KVResponseH\x00\x12\x34\n\x0ftask_assignment\x18\x06 \x01(\x0b\x32\x19.aether.v1.TaskAssignmentH\x00\x12\x32\n\x0e\x63onnection_ack\x18\x07 \x01(\x0b\x32\x18.aether.v1.ConnectionAckH\x00\x12\x33\n\ncheckpoint\x18\x08 \x01(\x0b\x32\x1d.aether.v1.CheckpointResponseH\x00\x12)\n\x05\x61\x64min\x18\t \x01(\x0b\x32\x18.aether.v1.AdminResponseH\x00\x12?\n\x10session_response\x18\n \x01(\x0b\x32#.aether.v1.SessionOperationResponseH\x00\x12\x32\n\ntask_query\x18\x0b \x01(\x0b\x32\x1c.aether.v1.TaskQueryResponseH\x00\x12\x33\n\x07task_op\x18\x0c \x01(\x0b\x32 .aether.v1.TaskOperationResponseH\x00\x12\x31\n\tworkspace\x18\r \x01(\x0b\x32\x1c.aether.v1.WorkspaceResponseH\x00\x12)\n\x05\x61gent\x18\x0e \x01(\x0b\x32\x18.aether.v1.AgentResponseH\x00\x12%\n\x03\x61\x63l\x18\x0f \x01(\x0b\x32\x16.aether.v1.ACLResponseH\x00\x12\x34\n\x0fprogress_update\x18\x10 \x01(\x0b\x32\x19.aether.v1.ProgressUpdateH\x00\x12\x38\n\x11workflow_response\x18\x11 \x01(\x0b\x32\x1b.aether.v1.WorkflowResponseH\x00\x12\x33\n\x0bworkflow_op\x18\x12 \x01(\x0b\x32\x1c.aether.v1.WorkflowOperationH\x00\x12)\n\x05token\x18\x13 \x01(\x0b\x32\x18.aether.v1.TokenResponseH\x00\x12\x37\n\x0e\x61udit_response\x18\x14 \x01(\x0b\x32\x1d.aether.v1.AuditQueryResponseH\x00\x12<\n\x0f\x61uthority_grant\x18\x15 \x01(\x0b\x32!.aether.v1.AuthorityGrantResponseH\x00\x12\x34\n\x0b\x63reate_task\x18\x16 \x01(\x0b\x32\x1d.aether.v1.CreateTaskResponseH\x00\x12;\n\x13proxy_http_response\x18\x17 \x01(\x0b\x32\x1c.aether.v1.ProxyHttpResponseH\x00\x12>\n\x15proxy_http_body_chunk\x18\x18 \x01(\x0b\x32\x1d.aether.v1.ProxyHttpBodyChunkH\x00\x12*\n\ntunnel_ack\x18\x19 \x01(\x0b\x32\x14.aether.v1.TunnelAckH\x00\x12.\n\x0ctunnel_close\x18\x1a \x01(\x0b\x32\x16.aether.v1.TunnelCloseH\x00\x12,\n\x0btunnel_data\x18\x1b \x01(\x0b\x32\x15.aether.v1.TunnelDataH\x00\x12\x39\n\x12proxy_http_request\x18\x1c \x01(\x0b\x32\x1b.aether.v1.ProxyHttpRequestH\x00\x12I\n\x1aresolve_authority_response\x18\x1d \x01(\x0b\x32#.aether.v1.ResolveAuthorityResponseH\x00\x12I\n\x1a\x63onnection_status_response\x18\x1e \x01(\x0b\x32#.aether.v1.ConnectionStatusResponseH\x00\x12I\n\x1a\x61uthority_grant_revocation\x18\x1f \x01(\x0b\x32#.aether.v1.AuthorityGrantRevocationH\x00\x12J\n\x1bsubmit_audit_event_response\x18 \x01(\x0b\x32#.aether.v1.SubmitAuditEventResponseH\x00\x12R\n\x1a\x61uthority_request_response\x18! \x01(\x0b\x32,.aether.v1.AuthorityRequestOperationResponseH\x00\x12\x43\n\x17\x61uthority_request_event\x18\" \x01(\x0b\x32 .aether.v1.AuthorityRequestEventH\x00\x12\x34\n\x0ftask_hibernated\x18# \x01(\x0b\x32\x19.aether.v1.TaskHibernatedH\x00\x12R\n\x1atask_subscription_response\x18$ \x01(\x0b\x32,.aether.v1.TaskSubscriptionOperationResponseH\x00\x12*\n\ntask_event\x18% \x01(\x0b\x32\x14.aether.v1.TaskEventH\x00\x12\x19\n\x11\x61\x63tive_extensions\x18& \x03(\tB\t\n\x07payload\"W\n\x0eTaskHibernated\x12\x0f\n\x07task_id\x18\x01 \x01(\t\x12\x34\n\ndescriptor\x18\x02 \x01(\x0b\x32 .aether.v1.HibernationDescriptor\"\xb6\x02\n\rConnectionAck\x12\x12\n\nsession_id\x18\x01 \x01(\t\x12\x0f\n\x07resumed\x18\x02 \x01(\x08\x12\x13\n\x0b\x61ssigned_id\x18\x03 \x01(\t\x12=\n\x15negotiated_extensions\x18\x04 \x03(\x0b\x32\x1e.aether.v1.NegotiatedExtension\x12#\n\x1bserver_supported_extensions\x18\x05 \x03(\t\x12\x16\n\x0eserver_version\x18\x32 \x01(\t\x12/\n\x11server_build_info\x18\x33 \x01(\x0b\x32\x14.aether.v1.BuildInfo\x12\"\n\x1ainitial_connection_unix_ms\x18< \x01(\x03\x12\x1a\n\x12reconnection_count\x18= \x01(\x05\"\xcd\x05\n\x0eInitConnection\x12)\n\x05\x61gent\x18\x01 \x01(\x0b\x32\x18.aether.v1.AgentIdentityH\x00\x12\'\n\x04task\x18\x02 \x01(\x0b\x32\x17.aether.v1.TaskIdentityH\x00\x12\'\n\x04user\x18\x03 \x01(\x0b\x32\x17.aether.v1.UserIdentityH\x00\x12\x37\n\x0corchestrator\x18\x04 \x01(\x0b\x32\x1f.aether.v1.OrchestratorIdentityH\x00\x12<\n\x0fworkflow_engine\x18\x05 \x01(\x0b\x32!.aether.v1.WorkflowEngineIdentityH\x00\x12:\n\x0emetrics_bridge\x18\x06 \x01(\x0b\x32 .aether.v1.MetricsBridgeIdentityH\x00\x12+\n\x06\x62ridge\x18\x07 \x01(\x0b\x32\x19.aether.v1.BridgeIdentityH\x00\x12-\n\x07service\x18\x08 \x01(\x0b\x32\x1a.aether.v1.ServiceIdentityH\x00\x12?\n\x0b\x63redentials\x18\n \x03(\x0b\x32*.aether.v1.InitConnection.CredentialsEntry\x12\x19\n\x11resume_session_id\x18\x0b \x01(\t\x12\x33\n\nextensions\x18\x0c \x03(\x0b\x32\x1f.aether.v1.ExtensionDeclaration\x12\x16\n\x0e\x63lient_version\x18\x32 \x01(\t\x12\x12\n\nclient_sdk\x18\x33 \x01(\t\x12/\n\x11\x63lient_build_info\x18\x34 \x01(\x0b\x32\x14.aether.v1.BuildInfo\x1a\x32\n\x10\x43redentialsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x42\r\n\x0b\x63lient_type\"J\n\tBuildInfo\x12\x0e\n\x06\x63ommit\x18\x01 \x01(\t\x12\x10\n\x08\x62uilt_at\x18\x02 \x01(\t\x12\x0f\n\x07runtime\x18\x03 \x01(\t\x12\n\n\x02os\x18\x04 \x01(\t\"[\n\x14\x45xtensionDeclaration\x12\x0b\n\x03uri\x18\x01 \x01(\t\x12\x0f\n\x07version\x18\x02 \x01(\t\x12\x10\n\x08required\x18\x03 \x01(\x08\x12\x13\n\x0bjson_schema\x18\x04 \x01(\t\"`\n\x13NegotiatedExtension\x12\x0b\n\x03uri\x18\x01 \x01(\t\x12\x0f\n\x07version\x18\x02 \x01(\t\x12\x11\n\tsupported\x18\x03 \x01(\x08\x12\x18\n\x10rejection_reason\x18\x04 \x01(\t\"-\n\x16WorkflowEngineIdentity\x12\x13\n\x0binstance_id\x18\x01 \x01(\t\",\n\x15MetricsBridgeIdentity\x12\x13\n\x0binstance_id\x18\x01 \x01(\t\"]\n\x14OrchestratorIdentity\x12\x16\n\x0eimplementation\x18\x01 \x01(\t\x12\x11\n\tspecifier\x18\x02 \x01(\t\x12\x1a\n\x12supported_profiles\x18\x03 \x03(\t\";\n\x0e\x42ridgeIdentity\x12\x16\n\x0eimplementation\x18\x01 \x01(\t\x12\x11\n\tspecifier\x18\x02 \x01(\t\"V\n\x0fServiceIdentity\x12\x16\n\x0eimplementation\x18\x01 \x01(\t\x12\x11\n\tspecifier\x18\x02 \x01(\t\x12\x18\n\x10no_pool_consumer\x18\x03 \x01(\x08\"M\n\rAgentIdentity\x12\x11\n\tworkspace\x18\x01 \x01(\t\x12\x16\n\x0eimplementation\x18\x02 \x01(\t\x12\x11\n\tspecifier\x18\x03 \x01(\t\"S\n\x0cTaskIdentity\x12\x11\n\tworkspace\x18\x01 \x01(\t\x12\x16\n\x0eimplementation\x18\x02 \x01(\t\x12\x18\n\x10unique_specifier\x18\x03 \x01(\t\"2\n\x0cUserIdentity\x12\x0f\n\x07user_id\x18\x01 \x01(\t\x12\x11\n\twindow_id\x18\x02 \x01(\t\"<\n\x0cPrincipalRef\x12\x16\n\x0eprincipal_type\x18\x01 \x01(\t\x12\x14\n\x0cprincipal_id\x18\x02 \x01(\t\"\x9e\x01\n\x14\x41uthorizationContext\x12\x16\n\x0e\x61uthority_mode\x18\x01 \x01(\t\x12(\n\x07subject\x18\x02 \x01(\x0b\x32\x17.aether.v1.PrincipalRef\x12\x10\n\x08grant_id\x18\x03 \x01(\t\x12\x32\n\x08resolved\x18\n \x01(\x0b\x32 .aether.v1.ResolvedAuthorityInfo\"\xbc\x01\n\x15ResolvedAuthorityInfo\x12-\n\x0croot_subject\x18\x01 \x01(\x0b\x32\x17.aether.v1.PrincipalRef\x12\x15\n\raudience_type\x18\x02 \x01(\t\x12\x13\n\x0b\x61udience_id\x18\x03 \x01(\t\x12\x18\n\x10max_access_level\x18\x04 \x01(\x05\x12\x17\n\x0fworkspace_scope\x18\x05 \x03(\t\x12\x15\n\rexpires_at_ms\x18\x06 \x01(\x03\"\xb1\x01\n\x0bSendMessage\x12\x14\n\x0ctarget_topic\x18\x01 \x01(\t\x12\x0f\n\x07payload\x18\x02 \x01(\x0c\x12,\n\x0cmessage_type\x18\x03 \x01(\x0e\x32\x16.aether.v1.MessageType\x12\x36\n\rauthorization\x18\x04 \x01(\x0b\x32\x1f.aether.v1.AuthorizationContext\x12\x15\n\rapp_workspace\x18\x05 \x01(\t\"\xc4\x01\n\x06Metric\x12\x10\n\x08trace_id\x18\x01 \x01(\t\x12\'\n\x07\x65ntries\x18\x02 \x03(\x0b\x32\x16.aether.v1.MetricEntry\x12\x31\n\x08metadata\x18\x03 \x03(\x0b\x32\x1f.aether.v1.Metric.MetadataEntry\x12\x1b\n\x13\x63lient_timestamp_ms\x18\x04 \x01(\x03\x1a/\n\rMetadataEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"6\n\x0bMetricEntry\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0c\n\x04kind\x18\x02 \x01(\t\x12\x0b\n\x03qty\x18\x03 \x01(\x01\"+\n\x0fSwitchWorkspace\x12\x18\n\x10new_workspace_id\x18\x01 \x01(\t\"\x8a\x06\n\x0bKVOperation\x12)\n\x02op\x18\x01 \x01(\x0e\x32\x1d.aether.v1.KVOperation.OpType\x12+\n\x05scope\x18\x02 \x01(\x0e\x32\x1c.aether.v1.KVOperation.Scope\x12\x0b\n\x03key\x18\x03 \x01(\t\x12\r\n\x05value\x18\x04 \x01(\x0c\x12\x0f\n\x07user_id\x18\x05 \x01(\t\x12\x11\n\tworkspace\x18\x06 \x01(\t\x12\x0b\n\x03ttl\x18\x07 \x01(\x03\x12\x12\n\nrequest_id\x18\x08 \x01(\t\x12\x36\n\rauthorization\x18\t \x01(\x0b\x32\x1f.aether.v1.AuthorizationContext\x12\x13\n\x0bguard_value\x18\n \x01(\x03\x12\x13\n\x0b\x64\x65lta_value\x18\x0b \x01(\x03\x12\x16\n\x0e\x65xpected_value\x18\x0c \x01(\x0c\x12\r\n\x05limit\x18\r \x01(\x05\x12\x0e\n\x06\x63ursor\x18\x0e \x01(\t\x12\x17\n\x0ftarget_identity\x18\x0f \x01(\t\"\xda\x01\n\x06OpType\x12\x07\n\x03GET\x10\x00\x12\x07\n\x03PUT\x10\x01\x12\x08\n\x04LIST\x10\x02\x12\n\n\x06\x44\x45LETE\x10\x03\x12\r\n\tINCREMENT\x10\x04\x12\r\n\tDECREMENT\x10\x05\x12\x10\n\x0cINCREMENT_IF\x10\x06\x12\x10\n\x0c\x44\x45\x43REMENT_IF\x10\x07\x12\n\n\x06SET_NX\x10\x08\x12\x13\n\x0f\x43OMPARE_AND_SET\x10\t\x12\x16\n\x12\x43OMPARE_AND_DELETE\x10\n\x12\x12\n\x0ePURGE_IDENTITY\x10\x0b\x12\x0b\n\x07SET_ADD\x10\x0c\x12\x0c\n\x08SET_CARD\x10\r\"\xb2\x01\n\x05Scope\x12\x15\n\x11SCOPE_UNSPECIFIED\x10\x00\x12\n\n\x06GLOBAL\x10\x01\x12\r\n\tWORKSPACE\x10\x02\x12\x08\n\x04USER\x10\x03\x12\x12\n\x0eUSER_WORKSPACE\x10\x04\x12\x14\n\x10GLOBAL_EXCLUSIVE\x10\x05\x12\x17\n\x13WORKSPACE_EXCLUSIVE\x10\x06\x12\x0f\n\x0bUSER_SHARED\x10\x07\x12\x19\n\x15USER_WORKSPACE_SHARED\x10\x08\"\xfd\x01\n\nKVResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\r\n\x05value\x18\x02 \x01(\x0c\x12\x0c\n\x04keys\x18\x03 \x03(\t\x12\x30\n\x06kv_map\x18\x04 \x03(\x0b\x32 .aether.v1.KVResponse.KvMapEntry\x12\x12\n\nrequest_id\x18\x05 \x01(\t\x12\x15\n\rcounter_value\x18\x06 \x01(\x03\x12\x0f\n\x07\x61pplied\x18\x07 \x01(\x08\x12\x13\n\x0bnext_cursor\x18\x08 \x01(\t\x12\x10\n\x08has_more\x18\t \x01(\x08\x1a,\n\nKvMapEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\x0c:\x02\x38\x01\"\xad\x01\n\x0fIncomingMessage\x12\x14\n\x0csource_topic\x18\x01 \x01(\t\x12\x0f\n\x07payload\x18\x02 \x01(\x0c\x12,\n\x0cmessage_type\x18\x03 \x01(\x0e\x32\x16.aether.v1.MessageType\x12\x11\n\tworkspace\x18\x04 \x01(\t\x12\x32\n\x11on_behalf_subject\x18\x05 \x01(\x0b\x32\x17.aether.v1.PrincipalRef\"\xf0\x04\n\x0e\x43onfigSnapshot\x12\x31\n\x02kv\x18\x01 \x03(\x0b\x32!.aether.v1.ConfigSnapshot.KvEntryB\x02\x18\x01\x12>\n\tglobal_kv\x18\x02 \x03(\x0b\x32\'.aether.v1.ConfigSnapshot.GlobalKvEntryB\x02\x18\x01\x12@\n\x0ctask_context\x18\x03 \x03(\x0b\x32*.aether.v1.ConfigSnapshot.TaskContextEntry\x12S\n\x16workspace_exclusive_kv\x18\x04 \x03(\x0b\x32\x33.aether.v1.ConfigSnapshot.WorkspaceExclusiveKvEntry\x12M\n\x13global_exclusive_kv\x18\x05 \x03(\x0b\x32\x30.aether.v1.ConfigSnapshot.GlobalExclusiveKvEntry\x1a)\n\x07KvEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\x0c:\x02\x38\x01\x1a/\n\rGlobalKvEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\x0c:\x02\x38\x01\x1a\x32\n\x10TaskContextEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x1a;\n\x19WorkspaceExclusiveKvEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\x0c:\x02\x38\x01\x1a\x38\n\x16GlobalExclusiveKvEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\x0c:\x02\x38\x01\"\x81\x01\n\x06Signal\x12*\n\x04type\x18\x01 \x01(\x0e\x32\x1c.aether.v1.Signal.SignalType\x12\x0e\n\x06reason\x18\x02 \x01(\t\";\n\nSignalType\x12\x14\n\x10\x46ORCE_DISCONNECT\x10\x00\x12\x17\n\x13GRACEFUL_DISCONNECT\x10\x01\"m\n\rErrorResponse\x12\x0c\n\x04\x63ode\x18\x01 \x01(\t\x12\x0f\n\x07message\x18\x02 \x01(\t\x12\x11\n\tretryable\x18\x03 \x01(\x08\x12\x16\n\x0eretry_after_ms\x18\x04 \x01(\x03\x12\x12\n\nrequest_id\x18\x05 \x01(\t\"\xe7\x01\n\x0bRetryPolicy\x12\x14\n\x0cmax_attempts\x18\x01 \x01(\x05\x12+\n\x07\x62\x61\x63koff\x18\x02 \x01(\x0e\x32\x1a.aether.v1.BackoffStrategy\x12\x18\n\x10initial_delay_ms\x18\x03 \x01(\x03\x12\x14\n\x0cmax_delay_ms\x18\x04 \x01(\x03\x12\x15\n\rjitter_factor\x18\x05 \x01(\x01\x12\x13\n\x0bschedule_ms\x18\x06 \x03(\x03\x12\x1e\n\x16retryable_status_codes\x18\x07 \x03(\x05\x12\x19\n\x11honor_retry_after\x18\x08 \x01(\x08\"f\n\x13TaskCompletionEvent\x12\x0f\n\x07\x65nabled\x18\x01 \x01(\x08\x12\x12\n\nevent_name\x18\x02 \x01(\t\x12*\n\x0bon_statuses\x18\x03 \x03(\x0e\x32\x15.aether.v1.TaskStatus\"\xbb\x06\n\x11\x43reateTaskRequest\x12\x11\n\ttask_type\x18\x01 \x01(\t\x12\x11\n\tworkspace\x18\x02 \x01(\t\x12\x36\n\x0f\x61ssignment_mode\x18\x03 \x01(\x0e\x32\x1d.aether.v1.TaskAssignmentMode\x12\x17\n\x0ftarget_agent_id\x18\x04 \x01(\t\x12V\n\x16launch_param_overrides\x18\x05 \x03(\x0b\x32\x36.aether.v1.CreateTaskRequest.LaunchParamOverridesEntry\x12<\n\x08metadata\x18\x06 \x03(\x0b\x32*.aether.v1.CreateTaskRequest.MetadataEntry\x12\x0f\n\x07payload\x18\x07 \x01(\x0c\x12\x1d\n\x15target_implementation\x18\x08 \x01(\t\x12\x36\n\rauthorization\x18\t \x01(\x0b\x32\x1f.aether.v1.AuthorizationContext\x12\x12\n\nrequest_id\x18\n \x01(\t\x12\x17\n\x0ftarget_identity\x18\x0b \x01(\t\x12(\n\ntask_class\x18\x0c \x01(\x0e\x32\x14.aether.v1.TaskClass\x12\x12\n\ncontext_id\x18\r \x01(\t\x12,\n\x0cretry_policy\x18\x0e \x01(\x0b\x32\x16.aether.v1.RetryPolicy\x12)\n\x08priority\x18\x0f \x01(\x0e\x32\x17.aether.v1.TaskPriority\x12\x17\n\x0fidempotency_key\x18\x10 \x01(\t\x12\x16\n\x0e\x63orrelation_id\x18\x11 \x01(\t\x12\x14\n\x0croot_task_id\x18\x12 \x01(\t\x12\x38\n\x10\x63ompletion_event\x18\x13 \x01(\x0b\x32\x1e.aether.v1.TaskCompletionEvent\x1a;\n\x19LaunchParamOverridesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x1a/\n\rMetadataEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\xca\x01\n\x12\x43reateTaskResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x0f\n\x07task_id\x18\x02 \x01(\t\x12\x0e\n\x06status\x18\x03 \x01(\t\x12\x12\n\nerror_code\x18\x04 \x01(\t\x12\x15\n\rerror_message\x18\x05 \x01(\t\x12\x12\n\nrequest_id\x18\x06 \x01(\t\x12\x13\n\x0b\x61ssigned_to\x18\x07 \x01(\t\x12\x12\n\ntask_token\x18\x08 \x01(\t\x12\x1a\n\x12\x61uthority_grant_id\x18\t \x01(\t\"\x87\x04\n\x0eTaskAssignment\x12\x0f\n\x07task_id\x18\x01 \x01(\t\x12\x11\n\ttask_type\x18\x02 \x01(\t\x12\x13\n\x0b\x61ssigned_to\x18\x03 \x01(\t\x12\x39\n\x08metadata\x18\x04 \x03(\x0b\x32\'.aether.v1.TaskAssignment.MetadataEntry\x12\x13\n\x0b\x61ssigned_at\x18\x05 \x01(\x03\x12\x0f\n\x07profile\x18\x06 \x01(\t\x12\x42\n\rlaunch_params\x18\x07 \x03(\x0b\x32+.aether.v1.TaskAssignment.LaunchParamsEntry\x12\x1d\n\x15target_implementation\x18\x08 \x01(\t\x12\x11\n\tworkspace\x18\t \x01(\t\x12\x11\n\tspecifier\x18\n \x01(\t\x12\x0f\n\x07payload\x18\x0b \x01(\x0c\x12(\n\ntask_class\x18\x0c \x01(\x0e\x32\x14.aether.v1.TaskClass\x12\x16\n\x0e\x63heckpoint_key\x18\r \x01(\t\x12\x19\n\x11resume_session_id\x18\x0e \x01(\t\x1a/\n\rMetadataEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x1a\x33\n\x11LaunchParamsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\xb8\x01\n\x13\x43heckpointOperation\x12\x31\n\x02op\x18\x01 \x01(\x0e\x32%.aether.v1.CheckpointOperation.OpType\x12\x0b\n\x03key\x18\x02 \x01(\t\x12\x0c\n\x04\x64\x61ta\x18\x03 \x01(\x0c\x12\x0b\n\x03ttl\x18\x04 \x01(\x03\x12\x12\n\nrequest_id\x18\x05 \x01(\t\"2\n\x06OpType\x12\x08\n\x04SAVE\x10\x00\x12\x08\n\x04LOAD\x10\x01\x12\n\n\x06\x44\x45LETE\x10\x02\x12\x08\n\x04LIST\x10\x03\"v\n\x12\x43heckpointResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x0c\n\x04\x64\x61ta\x18\x02 \x01(\x0c\x12\x0c\n\x04keys\x18\x03 \x03(\t\x12\r\n\x05\x65rror\x18\x04 \x01(\t\x12\x10\n\x08saved_at\x18\x05 \x01(\x03\x12\x12\n\nrequest_id\x18\x06 \x01(\t\"\xec\x01\n\nAdminQuery\x12(\n\x02op\x18\x01 \x01(\x0e\x32\x1c.aether.v1.AdminQuery.OpType\x12\x12\n\nsession_id\x18\x02 \x01(\t\x12+\n\x06\x66ilter\x18\x03 \x01(\x0b\x32\x1b.aether.v1.ConnectionFilter\x12\x12\n\nrequest_id\x18\x04 \x01(\t\"_\n\x06OpType\x12\x0e\n\nGET_HEALTH\x10\x00\x12\x0c\n\x08GET_INFO\x10\x01\x12\r\n\tGET_STATS\x10\x02\x12\x14\n\x10LIST_CONNECTIONS\x10\x03\x12\x12\n\x0eGET_CONNECTION\x10\x04\"l\n\x10\x43onnectionFilter\x12&\n\x04type\x18\x01 \x01(\x0e\x32\x18.aether.v1.PrincipalType\x12\x11\n\tworkspace\x18\x02 \x01(\t\x12\r\n\x05limit\x18\x03 \x01(\x05\x12\x0e\n\x06offset\x18\x04 \x01(\x05\"\xf0\x01\n\x0e\x43onnectionInfo\x12\x12\n\nsession_id\x18\x01 \x01(\t\x12&\n\x04type\x18\x02 \x01(\x0e\x32\x18.aether.v1.PrincipalType\x12\x10\n\x08identity\x18\x03 \x01(\t\x12\x11\n\tworkspace\x18\x04 \x01(\t\x12\x16\n\x0eimplementation\x18\x05 \x01(\t\x12\x11\n\tspecifier\x18\x06 \x01(\t\x12\x14\n\x0c\x63onnected_at\x18\x07 \x01(\x03\x12\x10\n\x08\x64uration\x18\x08 \x01(\t\x12\x13\n\x0bremote_addr\x18\t \x01(\t\x12\x15\n\rlast_activity\x18\n \x01(\x03\"\xac\x02\n\rAdminResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\r\n\x05\x65rror\x18\x02 \x01(\t\x12%\n\x06health\x18\x03 \x01(\x0b\x32\x15.aether.v1.HealthInfo\x12$\n\x04info\x18\x04 \x01(\x0b\x32\x16.aether.v1.GatewayInfo\x12&\n\x05stats\x18\x05 \x01(\x0b\x32\x17.aether.v1.GatewayStats\x12-\n\nconnection\x18\x06 \x01(\x0b\x32\x19.aether.v1.ConnectionInfo\x12.\n\x0b\x63onnections\x18\x07 \x03(\x0b\x32\x19.aether.v1.ConnectionInfo\x12\x13\n\x0btotal_count\x18\x08 \x01(\x05\x12\x12\n\nrequest_id\x18\t \x01(\t\"\xea\x01\n\nHealthInfo\x12\'\n\x06status\x18\x01 \x01(\x0e\x32\x17.aether.v1.HealthStatus\x12\x11\n\ttimestamp\x18\x02 \x01(\x03\x12\x31\n\x06\x63hecks\x18\x03 \x03(\x0b\x32!.aether.v1.HealthInfo.ChecksEntry\x12&\n\x05stats\x18\x04 \x01(\x0b\x32\x17.aether.v1.GatewayStats\x1a\x45\n\x0b\x43hecksEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12%\n\x05value\x18\x02 \x01(\x0b\x32\x16.aether.v1.HealthCheck:\x02\x38\x01\"[\n\x0bHealthCheck\x12,\n\x06status\x18\x01 \x01(\x0e\x32\x1c.aether.v1.HealthCheckStatus\x12\x0f\n\x07latency\x18\x02 \x01(\t\x12\r\n\x05\x65rror\x18\x03 \x01(\t\"\xb4\x01\n\x0bGatewayInfo\x12\x12\n\ngateway_id\x18\x01 \x01(\t\x12\x0f\n\x07version\x18\x02 \x01(\t\x12\x12\n\nstarted_at\x18\x03 \x01(\x03\x12\x0e\n\x06uptime\x18\x04 \x01(\t\x12\x12\n\ngo_version\x18\x05 \x01(\t\x12\x16\n\x0enum_goroutines\x18\x06 \x01(\x05\x12\x17\n\x0fmemory_alloc_mb\x18\x07 \x01(\x01\x12\x17\n\x0fnum_connections\x18\x08 \x01(\x05\"\x9a\x03\n\x0cGatewayStats\x12\x19\n\x11\x61gent_connections\x18\x01 \x01(\x05\x12\x18\n\x10task_connections\x18\x02 \x01(\x05\x12\x18\n\x10user_connections\x18\x03 \x01(\x05\x12 \n\x18orchestrator_connections\x18\x04 \x01(\x05\x12!\n\x19workflow_engine_connected\x18\x05 \x01(\x08\x12 \n\x18metrics_bridge_connected\x18\x06 \x01(\x08\x12\x13\n\x0btotal_tasks\x18\x07 \x01(\x05\x12\x15\n\rpending_tasks\x18\x08 \x01(\x05\x12\x15\n\rrunning_tasks\x18\t \x01(\x05\x12\x17\n\x0f\x63ompleted_tasks\x18\n \x01(\x05\x12\x14\n\x0c\x66\x61iled_tasks\x18\x0b \x01(\x05\x12\x1b\n\x13messages_per_second\x18\x0c \x01(\x01\x12\x16\n\x0etotal_messages\x18\r \x01(\x03\x12\x15\n\ractive_timers\x18\x0e \x01(\x05\x12\x16\n\x0epending_timers\x18\x0f \x01(\x05\"\x8c\x02\n\x10SessionOperation\x12.\n\x02op\x18\x01 \x01(\x0e\x32\".aether.v1.SessionOperation.OpType\x12\x12\n\nsession_id\x18\x02 \x01(\t\x12\x0e\n\x06reason\x18\x03 \x01(\t\x12\x12\n\nrequest_id\x18\x04 \x01(\t\x12+\n\x06\x66ilter\x18\x05 \x01(\x0b\x32\x1b.aether.v1.ConnectionFilter\x12\x36\n\rauthorization\x18\x06 \x01(\x0b\x32\x1f.aether.v1.AuthorizationContext\"+\n\x06OpType\x12\x0e\n\nDISCONNECT\x10\x00\x12\x08\n\x04LIST\x10\x01\x12\x07\n\x03GET\x10\x02\"\xd3\x01\n\x18SessionOperationResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x0f\n\x07message\x18\x02 \x01(\t\x12\r\n\x05\x65rror\x18\x03 \x01(\t\x12\x12\n\nrequest_id\x18\x04 \x01(\t\x12-\n\nconnection\x18\x05 \x01(\x0b\x32\x19.aether.v1.ConnectionInfo\x12.\n\x0b\x63onnections\x18\x06 \x03(\x0b\x32\x19.aether.v1.ConnectionInfo\x12\x13\n\x0btotal_count\x18\x07 \x01(\x05\"\x9d\x01\n\tTaskQuery\x12\'\n\x02op\x18\x01 \x01(\x0e\x32\x1b.aether.v1.TaskQuery.OpType\x12\x0f\n\x07task_id\x18\x02 \x01(\t\x12%\n\x06\x66ilter\x18\x03 \x01(\x0b\x32\x15.aether.v1.TaskFilter\x12\x12\n\nrequest_id\x18\x04 \x01(\t\"\x1b\n\x06OpType\x12\x08\n\x04LIST\x10\x00\x12\x07\n\x03GET\x10\x01\"\xec\x05\n\nTaskFilter\x12%\n\x06status\x18\x01 \x01(\x0e\x32\x15.aether.v1.TaskStatus\x12\x11\n\tworkspace\x18\x02 \x01(\t\x12\x11\n\ttask_type\x18\x03 \x01(\t\x12\r\n\x05limit\x18\x04 \x01(\x05\x12\x0e\n\x06offset\x18\x05 \x01(\x05\x12\'\n\x08statuses\x18\x06 \x03(\x0e\x32\x15.aether.v1.TaskStatus\x12\x14\n\x0csubject_type\x18\x07 \x01(\t\x12\x12\n\nsubject_id\x18\x08 \x01(\t\x12\x16\n\x0e\x61uthority_mode\x18\t \x01(\t\x12\x1a\n\x12\x61uthority_grant_id\x18\n \x01(\t\x12\x1f\n\x17root_authority_grant_id\x18\x0b \x01(\t\x12\x16\n\x0eparent_task_id\x18\x0c \x01(\t\x12(\n\ntask_class\x18\r \x01(\x0e\x32\x14.aether.v1.TaskClass\x12\x32\n\x14\x65xclude_task_classes\x18\x0e \x03(\x0e\x32\x14.aether.v1.TaskClass\x12\x12\n\ncontext_id\x18\x0f \x01(\t\x12/\n\x10\x65xclude_statuses\x18\x10 \x03(\x0e\x32\x15.aether.v1.TaskStatus\x12.\n\rcreator_actor\x18\x11 \x01(\x0b\x32\x17.aether.v1.PrincipalRef\x12&\n\x1estatus_timestamp_after_unix_ms\x18\x12 \x01(\x03\x12\x12\n\npage_token\x18\x13 \x01(\t\x12\x1b\n\x13include_descendants\x18\x14 \x01(\x08\x12)\n\x08priority\x18\x15 \x01(\x0e\x32\x17.aether.v1.TaskPriority\x12-\n\x0cmin_priority\x18\x16 \x01(\x0e\x32\x17.aether.v1.TaskPriority\x12\x16\n\x0e\x63orrelation_id\x18\x17 \x01(\t\x12\x14\n\x0croot_task_id\x18\x18 \x01(\t\"\xc7\x07\n\x08TaskInfo\x12\x0f\n\x07task_id\x18\x01 \x01(\t\x12\x11\n\ttask_type\x18\x02 \x01(\t\x12%\n\x06status\x18\x03 \x01(\x0e\x32\x15.aether.v1.TaskStatus\x12\x11\n\tworkspace\x18\x04 \x01(\t\x12\x14\n\x0ctarget_topic\x18\x05 \x01(\t\x12\x13\n\x0b\x61ssigned_to\x18\x06 \x01(\t\x12\x12\n\ncreated_at\x18\x07 \x01(\x03\x12\x12\n\nstarted_at\x18\x08 \x01(\x03\x12\x14\n\x0c\x63ompleted_at\x18\t \x01(\x03\x12\x0f\n\x07\x61ttempt\x18\n \x01(\x05\x12\x14\n\x0cmax_attempts\x18\x0b \x01(\x05\x12\r\n\x05\x65rror\x18\x0c \x01(\t\x12\x33\n\x08metadata\x18\r \x03(\x0b\x32!.aether.v1.TaskInfo.MetadataEntry\x12\x16\n\x0e\x61uthority_mode\x18\x0e \x01(\t\x12\x14\n\x0csubject_type\x18\x0f \x01(\t\x12\x12\n\nsubject_id\x18\x10 \x01(\t\x12\x19\n\x11root_subject_type\x18\x11 \x01(\t\x12\x17\n\x0froot_subject_id\x18\x12 \x01(\t\x12\x1a\n\x12\x61uthority_grant_id\x18\x13 \x01(\t\x12\x1f\n\x17root_authority_grant_id\x18\x14 \x01(\t\x12!\n\x19parent_authority_grant_id\x18\x15 \x01(\t\x12\x18\n\x10\x63reator_actor_id\x18\x16 \x01(\t\x12\x16\n\x0eparent_task_id\x18\x17 \x01(\t\x12(\n\ntask_class\x18\x18 \x01(\x0e\x32\x14.aether.v1.TaskClass\x12\x17\n\x0f\x64isconnected_at\x18\x19 \x01(\x03\x12\x17\n\x0fgrace_window_ms\x18\x1a \x01(\x03\x12&\n\twait_spec\x18\x1b \x01(\x0b\x32\x13.aether.v1.WaitSpec\x12\x12\n\ndepends_on\x18\x1c \x03(\t\x12\x12\n\ncontext_id\x18\x1d \x01(\t\x12\x11\n\tpaused_at\x18\x1e \x01(\x03\x12)\n\x08priority\x18\x1f \x01(\x0e\x32\x17.aether.v1.TaskPriority\x12\x16\n\x0e\x63orrelation_id\x18 \x01(\t\x12\x14\n\x0croot_task_id\x18! \x01(\t\x12\x38\n\x10\x63ompletion_event\x18\" \x01(\x0b\x32\x1e.aether.v1.TaskCompletionEvent\x1a/\n\rMetadataEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\xbc\x01\n\x11TaskQueryResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\r\n\x05\x65rror\x18\x02 \x01(\t\x12!\n\x04task\x18\x03 \x01(\x0b\x32\x13.aether.v1.TaskInfo\x12\"\n\x05tasks\x18\x04 \x03(\x0b\x32\x13.aether.v1.TaskInfo\x12\x13\n\x0btotal_count\x18\x05 \x01(\x05\x12\x12\n\nrequest_id\x18\x06 \x01(\t\x12\x17\n\x0fnext_page_token\x18\x07 \x01(\t\"\x8e\x02\n\rTaskOperation\x12+\n\x02op\x18\x01 \x01(\x0e\x32\x1f.aether.v1.TaskOperation.OpType\x12\x0f\n\x07task_id\x18\x02 \x01(\t\x12\x0e\n\x06reason\x18\x03 \x01(\t\x12\x12\n\nrequest_id\x18\x04 \x01(\t\x12&\n\twait_spec\x18\x05 \x01(\x0b\x32\x13.aether.v1.WaitSpec\"s\n\x06OpType\x12\t\n\x05RETRY\x10\x00\x12\n\n\x06\x43\x41NCEL\x10\x01\x12\x0c\n\x08\x43OMPLETE\x10\x02\x12\x08\n\x04\x46\x41IL\x10\x03\x12\t\n\x05PAUSE\x10\x04\x12\x0c\n\x08WAIT_FOR\x10\x05\x12\n\n\x06RESUME\x10\x06\x12\n\n\x06REJECT\x10\x07\x12\t\n\x05\x43LAIM\x10\x08\"\xec\x02\n\x08WaitSpec\x12%\n\x06reason\x18\x01 \x01(\x0e\x32\x15.aether.v1.WaitReason\x12\x1a\n\x12\x65xpected_principal\x18\x02 \x01(\t\x12\x38\n\x0binput_match\x18\x03 \x03(\x0b\x32#.aether.v1.WaitSpec.InputMatchEntry\x12\x1c\n\x14\x61uthority_request_id\x18\x04 \x01(\t\x12\x12\n\ndepends_on\x18\x05 \x03(\t\x12\x13\n\x0bwake_on_any\x18\x06 \x01(\x08\x12\x12\n\ntimeout_ms\x18\x07 \x01(\x03\x12\x1e\n\x16scheduled_wake_unix_ms\x18\x08 \x01(\x03\x12\x35\n\x0bhibernation\x18\t \x01(\x0b\x32 .aether.v1.HibernationDescriptor\x1a\x31\n\x0fInputMatchEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\x7f\n\x15HibernationDescriptor\x12\x16\n\x0e\x63heckpoint_key\x18\x01 \x01(\t\x12\x19\n\x11resume_session_id\x18\x02 \x01(\t\x12\x18\n\x10wake_event_types\x18\x03 \x03(\t\x12\x19\n\x11\x65scalation_policy\x18\x04 \x01(\t\"\x7f\n\x15TaskOperationResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x0f\n\x07message\x18\x02 \x01(\t\x12\r\n\x05\x65rror\x18\x03 \x01(\t\x12!\n\x04task\x18\x04 \x01(\x0b\x32\x13.aether.v1.TaskInfo\x12\x12\n\nrequest_id\x18\x05 \x01(\t\"\xa0\x02\n\x12WorkspaceOperation\x12\x30\n\x02op\x18\x01 \x01(\x0e\x32$.aether.v1.WorkspaceOperation.OpType\x12\x14\n\x0cworkspace_id\x18\x02 \x01(\t\x12*\n\x06\x66ilter\x18\x03 \x01(\x0b\x32\x1a.aether.v1.WorkspaceFilter\x12+\n\tworkspace\x18\x04 \x01(\x0b\x32\x18.aether.v1.WorkspaceInfo\x12\x12\n\nrequest_id\x18\x05 \x01(\t\"U\n\x06OpType\x12\x08\n\x04LIST\x10\x00\x12\x07\n\x03GET\x10\x01\x12\n\n\x06\x43REATE\x10\x02\x12\n\n\x06UPDATE\x10\x03\x12\n\n\x06\x44\x45LETE\x10\x04\x12\x14\n\x10GET_MESSAGE_FLOW\x10\x05\"C\n\x0fWorkspaceFilter\x12\x11\n\ttenant_id\x18\x01 \x01(\t\x12\r\n\x05limit\x18\x02 \x01(\x05\x12\x0e\n\x06offset\x18\x03 \x01(\x05\"\xd1\x02\n\rWorkspaceInfo\x12\x14\n\x0cworkspace_id\x18\x01 \x01(\t\x12\x14\n\x0c\x64isplay_name\x18\x02 \x01(\t\x12\x13\n\x0b\x64\x65scription\x18\x03 \x01(\t\x12\x11\n\ttenant_id\x18\x04 \x01(\t\x12\x12\n\ncreated_at\x18\x05 \x01(\x03\x12\x12\n\nupdated_at\x18\x06 \x01(\x03\x12\x38\n\x08metadata\x18\x07 \x03(\x0b\x32&.aether.v1.WorkspaceInfo.MetadataEntry\x12\x15\n\ractive_agents\x18\x08 \x01(\x05\x12\x14\n\x0c\x61\x63tive_tasks\x18\t \x01(\x05\x12\x14\n\x0c\x61\x63tive_users\x18\n \x01(\x05\x12\x16\n\x0etotal_messages\x18\x0b \x01(\x03\x1a/\n\rMetadataEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\xfa\x01\n\x11WorkspaceResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\r\n\x05\x65rror\x18\x02 \x01(\t\x12\x0f\n\x07message\x18\x03 \x01(\t\x12+\n\tworkspace\x18\x04 \x01(\x0b\x32\x18.aether.v1.WorkspaceInfo\x12,\n\nworkspaces\x18\x05 \x03(\x0b\x32\x18.aether.v1.WorkspaceInfo\x12\x13\n\x0btotal_count\x18\x06 \x01(\x05\x12\x30\n\x0cmessage_flow\x18\x07 \x01(\x0b\x32\x1a.aether.v1.MessageFlowInfo\x12\x12\n\nrequest_id\x18\x08 \x01(\t\"\x83\x01\n\x0fMessageFlowInfo\x12\x14\n\x0cworkspace_id\x18\x01 \x01(\t\x12\"\n\x05nodes\x18\x02 \x03(\x0b\x32\x13.aether.v1.FlowNode\x12\"\n\x05\x65\x64ges\x18\x03 \x03(\x0b\x32\x13.aether.v1.FlowEdge\x12\x12\n\nupdated_at\x18\x04 \x01(\x03\"\x97\x01\n\x08\x46lowNode\x12\n\n\x02id\x18\x01 \x01(\t\x12\r\n\x05label\x18\x02 \x01(\t\x12&\n\x04type\x18\x03 \x01(\x0e\x32\x18.aether.v1.PrincipalType\x12\x0e\n\x06status\x18\x04 \x01(\t\x12\x16\n\x0eimplementation\x18\x05 \x01(\t\x12\x11\n\tspecifier\x18\x06 \x01(\t\x12\r\n\x05topic\x18\x07 \x01(\t\"B\n\x08\x46lowEdge\x12\x0c\n\x04\x66rom\x18\x01 \x01(\t\x12\n\n\x02to\x18\x02 \x01(\t\x12\r\n\x05label\x18\x03 \x01(\t\x12\r\n\x05\x63ount\x18\x04 \x01(\x03\"\xdf\x02\n\x0e\x41gentOperation\x12,\n\x02op\x18\x01 \x01(\x0e\x32 .aether.v1.AgentOperation.OpType\x12\x16\n\x0eimplementation\x18\x02 \x01(\t\x12&\n\x06\x66ilter\x18\x03 \x01(\x0b\x32\x16.aether.v1.AgentFilter\x12/\n\x05\x61gent\x18\x04 \x01(\x0b\x32 .aether.v1.AgentRegistrationInfo\x12\x33\n\rlaunch_params\x18\x05 \x01(\x0b\x32\x1c.aether.v1.AgentLaunchParams\x12\x12\n\nrequest_id\x18\x06 \x01(\t\"e\n\x06OpType\x12\x08\n\x04LIST\x10\x00\x12\x07\n\x03GET\x10\x01\x12\x0c\n\x08REGISTER\x10\x02\x12\n\n\x06UPDATE\x10\x03\x12\n\n\x06\x44\x45LETE\x10\x04\x12\n\n\x06LAUNCH\x10\x05\x12\x16\n\x12LIST_ORCHESTRATORS\x10\x06\"J\n\x0b\x41gentFilter\x12\x1c\n\x14orchestrator_profile\x18\x01 \x01(\t\x12\r\n\x05limit\x18\x02 \x01(\x05\x12\x0e\n\x06offset\x18\x03 \x01(\x05\"\xde\x03\n\x15\x41gentRegistrationInfo\x12\x16\n\x0eimplementation\x18\x01 \x01(\t\x12\x1c\n\x14orchestrator_profile\x18\x02 \x01(\t\x12\x13\n\x0b\x64\x65scription\x18\x03 \x01(\t\x12I\n\rlaunch_params\x18\x04 \x03(\x0b\x32\x32.aether.v1.AgentRegistrationInfo.LaunchParamsEntry\x12\x15\n\rregistered_at\x18\x05 \x01(\x03\x12\x12\n\nupdated_at\x18\x06 \x01(\x03\x12<\n\x0fresource_schema\x18\x07 \x03(\x0b\x32#.aether.v1.AgentResourceSchemaEntry\x12H\n\x0c\x63\x61pabilities\x18\x08 \x03(\x0b\x32\x32.aether.v1.AgentRegistrationInfo.CapabilitiesEntry\x12\x12\n\nextensions\x18\t \x03(\t\x1a\x33\n\x11LaunchParamsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x1a\x33\n\x11\x43\x61pabilitiesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\x08:\x02\x38\x01\"n\n\x18\x41gentResourceSchemaEntry\x12\x1c\n\x14resource_type_prefix\x18\x01 \x01(\t\x12\x18\n\x10permission_verbs\x18\x02 \x03(\t\x12\x1a\n\x12resource_id_schema\x18\x03 \x01(\t\"\xbb\x01\n\x11\x41gentLaunchParams\x12\x11\n\tspecifier\x18\x01 \x01(\t\x12\x11\n\tworkspace\x18\x02 \x01(\t\x12I\n\x0fparam_overrides\x18\x03 \x03(\x0b\x32\x30.aether.v1.AgentLaunchParams.ParamOverridesEntry\x1a\x35\n\x13ParamOverridesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"S\n\x10OrchestratorInfo\x12\x17\n\x0forchestrator_id\x18\x01 \x01(\t\x12\x10\n\x08profiles\x18\x02 \x03(\t\x12\x14\n\x0c\x63onnected_at\x18\x03 \x01(\x03\"5\n\x11\x41gentLaunchResult\x12\x0f\n\x07task_id\x18\x01 \x01(\t\x12\x0f\n\x07message\x18\x02 \x01(\t\"\xb5\x02\n\rAgentResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\r\n\x05\x65rror\x18\x02 \x01(\t\x12\x0f\n\x07message\x18\x03 \x01(\t\x12/\n\x05\x61gent\x18\x04 \x01(\x0b\x32 .aether.v1.AgentRegistrationInfo\x12\x30\n\x06\x61gents\x18\x05 \x03(\x0b\x32 .aether.v1.AgentRegistrationInfo\x12\x13\n\x0btotal_count\x18\x06 \x01(\x05\x12\x32\n\rorchestrators\x18\x07 \x03(\x0b\x32\x1b.aether.v1.OrchestratorInfo\x12\x33\n\rlaunch_result\x18\x08 \x01(\x0b\x32\x1c.aether.v1.AgentLaunchResult\x12\x12\n\nrequest_id\x18\t \x01(\t\"\xac\x0c\n\x0c\x41\x43LOperation\x12*\n\x02op\x18\x01 \x01(\x0e\x32\x1e.aether.v1.ACLOperation.OpType\x12\x0f\n\x07rule_id\x18\x02 \x01(\t\x12\x15\n\rrule_category\x18\x04 \x01(\t\x12\x16\n\x0eretention_days\x18\x05 \x01(\x05\x12-\n\x0brule_filter\x18\n \x01(\x0b\x32\x18.aether.v1.ACLRuleFilter\x12/\n\x0c\x61udit_filter\x18\x0b \x01(\x0b\x32\x19.aether.v1.ACLAuditFilter\x12\x31\n\rgrant_request\x18\x0c \x01(\x0b\x32\x1a.aether.v1.ACLGrantRequest\x12:\n\x10\x66\x61llback_request\x18\x0e \x01(\x0b\x32 .aether.v1.ACLSetFallbackRequest\x12\x12\n\nrequest_id\x18\x13 \x01(\t\x12\x0c\n\x04name\x18\x14 \x01(\t\x12*\n\tprincipal\x18\x15 \x01(\x0b\x32\x17.aether.v1.PrincipalRef\x12\x31\n\rgroup_request\x18\x16 \x01(\x0b\x32\x1a.aether.v1.ACLGroupRequest\x12/\n\x0crole_request\x18\x17 \x01(\x0b\x32\x19.aether.v1.ACLRoleRequest\x12\x38\n\x0emember_request\x18\x18 \x01(\x0b\x32 .aether.v1.ACLGroupMemberRequest\x12?\n\x12\x61ssignment_request\x18\x19 \x01(\x0b\x32#.aether.v1.ACLRoleAssignmentRequest\x12\x15\n\rresource_type\x18\x1a \x01(\t\x12\x13\n\x0bresource_id\x18\x1b \x01(\t\x12\x16\n\x0erequired_level\x18\x1c \x01(\x05\x12\x36\n\rauthorization\x18\x1d \x01(\x0b\x32\x1f.aether.v1.AuthorizationContext\"\xa3\x05\n\x06OpType\x12\x0e\n\nLIST_RULES\x10\x00\x12\x0c\n\x08GET_RULE\x10\x01\x12\t\n\x05GRANT\x10\x02\x12\n\n\x06REVOKE\x10\x03\x12\x0f\n\x0bQUERY_AUDIT\x10\x04\x12\x17\n\x13GET_FALLBACK_POLICY\x10\x08\x12\x17\n\x13SET_FALLBACK_POLICY\x10\t\x12\x13\n\x0f\x43LEANUP_EXPIRED\x10\n\x12\x16\n\x12\x43LEANUP_AUDIT_LOGS\x10\x0b\x12\x10\n\x0c\x43REATE_GROUP\x10\x11\x12\x10\n\x0c\x44\x45LETE_GROUP\x10\x12\x12\r\n\tGET_GROUP\x10\x13\x12\x0f\n\x0bLIST_GROUPS\x10\x14\x12\x14\n\x10\x41\x44\x44_GROUP_MEMBER\x10\x15\x12\x17\n\x13REMOVE_GROUP_MEMBER\x10\x16\x12\x16\n\x12LIST_GROUP_MEMBERS\x10\x17\x12\x0f\n\x0b\x43REATE_ROLE\x10\x18\x12\x0f\n\x0b\x44\x45LETE_ROLE\x10\x19\x12\x0c\n\x08GET_ROLE\x10\x1a\x12\x0e\n\nLIST_ROLES\x10\x1b\x12\x0f\n\x0b\x41SSIGN_ROLE\x10\x1c\x12\x11\n\rUNASSIGN_ROLE\x10\x1d\x12\x19\n\x15LIST_ROLE_ASSIGNMENTS\x10\x1e\x12\x19\n\x15LIST_PRINCIPAL_GROUPS\x10\x1f\x12\x18\n\x14LIST_PRINCIPAL_ROLES\x10 \x12\x12\n\x0e\x45XPLAIN_ACCESS\x10!\"\x04\x08\x05\x10\x05\"\x04\x08\x06\x10\x06\"\x04\x08\x07\x10\x07\"\x04\x08\x0c\x10\x0c\"\x04\x08\r\x10\r\"\x04\x08\x0e\x10\x0e\"\x04\x08\x0f\x10\x0f\"\x04\x08\x10\x10\x10*\x15LIST_AUTHORITY_GRANTS*\x13GET_AUTHORITY_GRANT*\x16\x43REATE_AUTHORITY_GRANT*\x15RENEW_AUTHORITY_GRANT*\x16REVOKE_AUTHORITY_GRANTJ\x04\x08\x03\x10\x04J\x04\x08\x06\x10\x07J\x04\x08\x07\x10\x08J\x04\x08\r\x10\x0eJ\x04\x08\x08\x10\tJ\x04\x08\x10\x10\x11J\x04\x08\x11\x10\x12J\x04\x08\x12\x10\x13R\x12\x61uthority_grant_idR\x16\x61uthority_grant_filterR\x17\x61uthority_grant_requestR\x1d\x61uthority_grant_renew_request\"\x88\x01\n\rACLRuleFilter\x12\x16\n\x0eprincipal_type\x18\x01 \x01(\t\x12\x14\n\x0cprincipal_id\x18\x02 \x01(\t\x12\x15\n\rresource_type\x18\x03 \x01(\t\x12\x13\n\x0bresource_id\x18\x04 \x01(\t\x12\r\n\x05limit\x18\x05 \x01(\x05\x12\x0e\n\x06offset\x18\x06 \x01(\x05\"\xd4\x01\n\x0e\x41\x43LAuditFilter\x12\x12\n\nstart_time\x18\x01 \x01(\x03\x12\x10\n\x08\x65nd_time\x18\x02 \x01(\x03\x12\x16\n\x0eprincipal_type\x18\x03 \x01(\t\x12\x14\n\x0cprincipal_id\x18\x04 \x01(\t\x12\x15\n\rresource_type\x18\x05 \x01(\t\x12\x13\n\x0bresource_id\x18\x06 \x01(\t\x12\x10\n\x08\x64\x65\x63ision\x18\x07 \x01(\t\x12\x11\n\tworkspace\x18\x08 \x01(\t\x12\r\n\x05limit\x18\t \x01(\x05\x12\x0e\n\x06offset\x18\n \x01(\x05\"\xb9\x01\n\x0f\x41\x43LGrantRequest\x12\x16\n\x0eprincipal_type\x18\x01 \x01(\t\x12\x14\n\x0cprincipal_id\x18\x02 \x01(\t\x12\x15\n\rresource_type\x18\x03 \x01(\t\x12\x13\n\x0bresource_id\x18\x04 \x01(\t\x12\x14\n\x0c\x61\x63\x63\x65ss_level\x18\x05 \x01(\x05\x12\x12\n\ngranted_by\x18\x06 \x01(\t\x12\x0e\n\x06reason\x18\x07 \x01(\t\x12\x12\n\nexpires_at\x18\x08 \x01(\x03\"a\n\x15\x41\x43LSetFallbackRequest\x12\x15\n\rrule_category\x18\x01 \x01(\t\x12\x1d\n\x15\x66\x61llback_access_level\x18\x02 \x01(\x05\x12\x12\n\nupdated_by\x18\x03 \x01(\t\"\xff\x01\n\x17\x41\x43LAuthorityGrantFilter\x12\x15\n\rroot_grant_id\x18\x01 \x01(\t\x12\x14\n\x0csubject_type\x18\x02 \x01(\t\x12\x12\n\nsubject_id\x18\x03 \x01(\t\x12\x15\n\rdelegate_type\x18\x04 \x01(\t\x12\x13\n\x0b\x64\x65legate_id\x18\x05 \x01(\t\x12\x15\n\raudience_type\x18\x06 \x01(\t\x12\x13\n\x0b\x61udience_id\x18\x07 \x01(\t\x12\x17\n\x0finclude_revoked\x18\x08 \x01(\x08\x12\x13\n\x0b\x61\x63tive_only\x18\t \x01(\x08\x12\r\n\x05limit\x18\n \x01(\x05\x12\x0e\n\x06offset\x18\x0b \x01(\x05\"N\n#ACLAuthorityGrantResourceScopeEntry\x12\x15\n\rresource_type\x18\x01 \x01(\t\x12\x10\n\x08patterns\x18\x02 \x03(\t\"\xa9\x05\n\x18\x41\x43LAuthorityGrantRequest\x12(\n\x07subject\x18\x01 \x01(\x0b\x32\x17.aether.v1.PrincipalRef\x12)\n\x08\x64\x65legate\x18\x02 \x01(\x0b\x32\x17.aether.v1.PrincipalRef\x12*\n\tissued_by\x18\x03 \x01(\x0b\x32\x17.aether.v1.PrincipalRef\x12-\n\x0croot_subject\x18\x04 \x01(\x0b\x32\x17.aether.v1.PrincipalRef\x12\x17\n\x0fparent_grant_id\x18\x05 \x01(\t\x12\x14\n\x0cmay_delegate\x18\x06 \x01(\x08\x12\x16\n\x0eremaining_hops\x18\x07 \x01(\x05\x12\x17\n\x0fworkspace_scope\x18\x08 \x03(\t\x12\x46\n\x0eresource_scope\x18\t \x03(\x0b\x32..aether.v1.ACLAuthorityGrantResourceScopeEntry\x12\x17\n\x0foperation_scope\x18\n \x03(\t\x12\x18\n\x10max_access_level\x18\x0b \x01(\x05\x12\x15\n\raudience_type\x18\x0c \x01(\t\x12\x13\n\x0b\x61udience_id\x18\r \x01(\t\x12#\n\x1bvalid_while_audience_active\x18\x0e \x01(\x08\x12\x12\n\nexpires_at\x18\x0f \x01(\x03\x12\x17\n\x0frenewable_until\x18\x10 \x01(\x03\x12\x0e\n\x06reason\x18\x11 \x01(\t\x12\x43\n\x08metadata\x18\x12 \x03(\x0b\x32\x31.aether.v1.ACLAuthorityGrantRequest.MetadataEntry\x1a/\n\rMetadataEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"]\n\x1d\x41\x43LRenewAuthorityGrantRequest\x12\x10\n\x08grant_id\x18\x01 \x01(\t\x12\x12\n\nexpires_at\x18\x02 \x01(\x03\x12\x16\n\x0e\x65xtend_seconds\x18\x03 \x01(\x05\"\xf5\x01\n\x0b\x41\x43LRuleInfo\x12\x0f\n\x07rule_id\x18\x01 \x01(\t\x12\x16\n\x0eprincipal_type\x18\x02 \x01(\t\x12\x14\n\x0cprincipal_id\x18\x03 \x01(\t\x12\x15\n\rresource_type\x18\x04 \x01(\t\x12\x13\n\x0bresource_id\x18\x05 \x01(\t\x12\x14\n\x0c\x61\x63\x63\x65ss_level\x18\x06 \x01(\x05\x12\x19\n\x11\x61\x63\x63\x65ss_level_name\x18\x07 \x01(\t\x12\x12\n\ngranted_by\x18\x08 \x01(\t\x12\x12\n\ngranted_at\x18\t \x01(\x03\x12\x12\n\nexpires_at\x18\n \x01(\x03\x12\x0e\n\x06reason\x18\x0b \x01(\t\"\xac\x01\n\x15\x41\x43LFallbackPolicyInfo\x12\x11\n\tpolicy_id\x18\x01 \x01(\t\x12\x15\n\rrule_category\x18\x02 \x01(\t\x12\x1d\n\x15\x66\x61llback_access_level\x18\x03 \x01(\x05\x12\"\n\x1a\x66\x61llback_access_level_name\x18\x04 \x01(\t\x12\x12\n\nupdated_by\x18\x05 \x01(\t\x12\x12\n\nupdated_at\x18\x06 \x01(\x03\"\xc3\x03\n\x11\x41\x43LAuditEntryInfo\x12\x10\n\x08\x61udit_id\x18\x01 \x01(\x03\x12\x11\n\ttimestamp\x18\x02 \x01(\x03\x12\x10\n\x08\x64\x65\x63ision\x18\x03 \x01(\t\x12\x14\n\x0c\x61\x63\x63\x65ss_level\x18\x04 \x01(\x05\x12\x19\n\x11\x61\x63\x63\x65ss_level_name\x18\x05 \x01(\t\x12\x16\n\x0eprincipal_type\x18\x06 \x01(\t\x12\x14\n\x0cprincipal_id\x18\x07 \x01(\t\x12\x15\n\rresource_type\x18\x08 \x01(\t\x12\x13\n\x0bresource_id\x18\t \x01(\t\x12\x11\n\toperation\x18\n \x01(\t\x12\x11\n\tworkspace\x18\x0b \x01(\t\x12\x0f\n\x07rule_id\x18\r \x01(\t\x12\x18\n\x10\x66\x61llback_applied\x18\x0e \x01(\x08\x12\x12\n\ngateway_id\x18\x0f \x01(\t\x12\x12\n\nsession_id\x18\x10 \x01(\t\x12<\n\x08metadata\x18\x11 \x03(\x0b\x32*.aether.v1.ACLAuditEntryInfo.MetadataEntry\x1a/\n\rMetadataEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01J\x04\x08\x0c\x10\r\"\xb4\x06\n\x15\x41\x43LAuthorityGrantInfo\x12\x10\n\x08grant_id\x18\x01 \x01(\t\x12\x15\n\rroot_grant_id\x18\x02 \x01(\t\x12(\n\x07subject\x18\x03 \x01(\x0b\x32\x17.aether.v1.PrincipalRef\x12)\n\x08\x64\x65legate\x18\x04 \x01(\x0b\x32\x17.aether.v1.PrincipalRef\x12*\n\tissued_by\x18\x05 \x01(\x0b\x32\x17.aether.v1.PrincipalRef\x12-\n\x0croot_subject\x18\x06 \x01(\x0b\x32\x17.aether.v1.PrincipalRef\x12\x17\n\x0fparent_grant_id\x18\x07 \x01(\t\x12\x14\n\x0cmay_delegate\x18\x08 \x01(\x08\x12\x16\n\x0eremaining_hops\x18\t \x01(\x05\x12\x17\n\x0fworkspace_scope\x18\n \x03(\t\x12\x46\n\x0eresource_scope\x18\x0b \x03(\x0b\x32..aether.v1.ACLAuthorityGrantResourceScopeEntry\x12\x17\n\x0foperation_scope\x18\x0c \x03(\t\x12\x18\n\x10max_access_level\x18\r \x01(\x05\x12\x19\n\x11\x61\x63\x63\x65ss_level_name\x18\x0e \x01(\t\x12\x15\n\raudience_type\x18\x0f \x01(\t\x12\x13\n\x0b\x61udience_id\x18\x10 \x01(\t\x12#\n\x1bvalid_while_audience_active\x18\x11 \x01(\x08\x12\x12\n\nexpires_at\x18\x12 \x01(\x03\x12\x17\n\x0frenewable_until\x18\x13 \x01(\x03\x12\x12\n\nrenewed_at\x18\x14 \x01(\x03\x12\x0f\n\x07revoked\x18\x15 \x01(\x08\x12\x12\n\nrevoked_at\x18\x16 \x01(\x03\x12\x0e\n\x06reason\x18\x17 \x01(\t\x12@\n\x08metadata\x18\x18 \x03(\x0b\x32..aether.v1.ACLAuthorityGrantInfo.MetadataEntry\x12\x12\n\ncreated_at\x18\x19 \x01(\x03\x1a/\n\rMetadataEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\":\n\x10\x41\x43LCleanupResult\x12\x15\n\rdeleted_count\x18\x01 \x01(\x03\x12\x0f\n\x07message\x18\x02 \x01(\t\"\xb5\x01\n\x0f\x41\x43LGroupRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x13\n\x0b\x64\x65scription\x18\x02 \x01(\t\x12\x12\n\ncreated_by\x18\x03 \x01(\t\x12:\n\x08metadata\x18\x04 \x03(\x0b\x32(.aether.v1.ACLGroupRequest.MetadataEntry\x1a/\n\rMetadataEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\xb3\x01\n\x0e\x41\x43LRoleRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x13\n\x0b\x64\x65scription\x18\x02 \x01(\t\x12\x12\n\ncreated_by\x18\x03 \x01(\t\x12\x39\n\x08metadata\x18\x04 \x03(\x0b\x32\'.aether.v1.ACLRoleRequest.MetadataEntry\x1a/\n\rMetadataEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"g\n\x15\x41\x43LGroupMemberRequest\x12\x13\n\x0bmember_type\x18\x01 \x01(\t\x12\x11\n\tmember_id\x18\x02 \x01(\t\x12\x12\n\ngranted_by\x18\x03 \x01(\t\x12\x12\n\nexpires_at\x18\x04 \x01(\x03\"n\n\x18\x41\x43LRoleAssignmentRequest\x12\x15\n\rassignee_type\x18\x01 \x01(\t\x12\x13\n\x0b\x61ssignee_id\x18\x02 \x01(\t\x12\x12\n\ngranted_by\x18\x03 \x01(\t\x12\x12\n\nexpires_at\x18\x04 \x01(\x03\"\xdb\x01\n\x0c\x41\x43LGroupInfo\x12\x10\n\x08group_id\x18\x01 \x01(\t\x12\x12\n\ngroup_name\x18\x02 \x01(\t\x12\x13\n\x0b\x64\x65scription\x18\x03 \x01(\t\x12\x12\n\ncreated_by\x18\x04 \x01(\t\x12\x12\n\ncreated_at\x18\x05 \x01(\x03\x12\x37\n\x08metadata\x18\x06 \x03(\x0b\x32%.aether.v1.ACLGroupInfo.MetadataEntry\x1a/\n\rMetadataEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\xd7\x01\n\x0b\x41\x43LRoleInfo\x12\x0f\n\x07role_id\x18\x01 \x01(\t\x12\x11\n\trole_name\x18\x02 \x01(\t\x12\x13\n\x0b\x64\x65scription\x18\x03 \x01(\t\x12\x12\n\ncreated_by\x18\x04 \x01(\t\x12\x12\n\ncreated_at\x18\x05 \x01(\x03\x12\x36\n\x08metadata\x18\x06 \x03(\x0b\x32$.aether.v1.ACLRoleInfo.MetadataEntry\x1a/\n\rMetadataEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\x8c\x01\n\x12\x41\x43LGroupMemberInfo\x12\x12\n\ngroup_name\x18\x01 \x01(\t\x12\x13\n\x0bmember_type\x18\x02 \x01(\t\x12\x11\n\tmember_id\x18\x03 \x01(\t\x12\x12\n\ngranted_by\x18\x04 \x01(\t\x12\x12\n\ngranted_at\x18\x05 \x01(\x03\x12\x12\n\nexpires_at\x18\x06 \x01(\x03\"\x92\x01\n\x15\x41\x43LRoleAssignmentInfo\x12\x11\n\trole_name\x18\x01 \x01(\t\x12\x15\n\rassignee_type\x18\x02 \x01(\t\x12\x13\n\x0b\x61ssignee_id\x18\x03 \x01(\t\x12\x12\n\ngranted_by\x18\x04 \x01(\t\x12\x12\n\ngranted_at\x18\x05 \x01(\x03\x12\x12\n\nexpires_at\x18\x06 \x01(\x03\"v\n\x19\x41\x43LAccessContributionInfo\x12\x0f\n\x07subject\x18\x01 \x01(\t\x12\x0f\n\x07rule_id\x18\x02 \x01(\t\x12\x14\n\x0c\x61\x63\x63\x65ss_level\x18\x03 \x01(\x05\x12\x10\n\x08resource\x18\x04 \x01(\t\x12\x0f\n\x07\x65xpired\x18\x05 \x01(\x08\"\xe9\x01\n\x18\x41\x43LAccessExplanationInfo\x12\x11\n\tprincipal\x18\x01 \x01(\t\x12\x10\n\x08subjects\x18\x02 \x03(\t\x12;\n\rcontributions\x18\x03 \x03(\x0b\x32$.aether.v1.ACLAccessContributionInfo\x12\x0f\n\x07\x61llowed\x18\x04 \x01(\x08\x12\x10\n\x08\x64\x65\x63ision\x18\x05 \x01(\t\x12\x1e\n\x16\x65\x66\x66\x65\x63tive_access_level\x18\x06 \x01(\x05\x12\x18\n\x10\x66\x61llback_applied\x18\x07 \x01(\x08\x12\x0e\n\x06reason\x18\x08 \x01(\t\"\xdd\x06\n\x0b\x41\x43LResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\r\n\x05\x65rror\x18\x02 \x01(\t\x12\x0f\n\x07message\x18\x03 \x01(\t\x12$\n\x04rule\x18\x04 \x01(\x0b\x32\x16.aether.v1.ACLRuleInfo\x12%\n\x05rules\x18\x05 \x03(\x0b\x32\x16.aether.v1.ACLRuleInfo\x12\x13\n\x0btotal_rules\x18\x06 \x01(\x05\x12\x39\n\x0f\x66\x61llback_policy\x18\x08 \x01(\x0b\x32 .aether.v1.ACLFallbackPolicyInfo\x12\x33\n\raudit_entries\x18\t \x03(\x0b\x32\x1c.aether.v1.ACLAuditEntryInfo\x12\x1b\n\x13total_audit_entries\x18\n \x01(\x05\x12\x33\n\x0e\x63leanup_result\x18\x0b \x01(\x0b\x32\x1b.aether.v1.ACLCleanupResult\x12\x39\n\x0f\x61uthority_grant\x18\r \x01(\x0b\x32 .aether.v1.ACLAuthorityGrantInfo\x12:\n\x10\x61uthority_grants\x18\x0e \x03(\x0b\x32 .aether.v1.ACLAuthorityGrantInfo\x12\x1e\n\x16total_authority_grants\x18\x0f \x01(\x05\x12\x12\n\nrequest_id\x18\x0c \x01(\t\x12&\n\x05group\x18\x10 \x01(\x0b\x32\x17.aether.v1.ACLGroupInfo\x12\'\n\x06groups\x18\x11 \x03(\x0b\x32\x17.aether.v1.ACLGroupInfo\x12$\n\x04role\x18\x12 \x01(\x0b\x32\x16.aether.v1.ACLRoleInfo\x12%\n\x05roles\x18\x13 \x03(\x0b\x32\x16.aether.v1.ACLRoleInfo\x12\x34\n\rgroup_members\x18\x14 \x03(\x0b\x32\x1d.aether.v1.ACLGroupMemberInfo\x12:\n\x10role_assignments\x18\x15 \x03(\x0b\x32 .aether.v1.ACLRoleAssignmentInfo\x12\x38\n\x0b\x65xplanation\x18\x16 \x01(\x0b\x32#.aether.v1.ACLAccessExplanationInfoJ\x04\x08\x07\x10\x08\"\xb5\x05\n\x17\x41uthorityGrantOperation\x12\x35\n\x02op\x18\x01 \x01(\x0e\x32).aether.v1.AuthorityGrantOperation.OpType\x12\x10\n\x08grant_id\x18\x02 \x01(\t\x12\x42\n\x10\x65xchange_request\x18\x03 \x01(\x0b\x32(.aether.v1.AuthorityGrantExchangeRequest\x12>\n\x0e\x64\x65rive_request\x18\x04 \x01(\x0b\x32&.aether.v1.AuthorityGrantDeriveRequest\x12?\n\rrenew_request\x18\x05 \x01(\x0b\x32(.aether.v1.ACLRenewAuthorityGrantRequest\x12\x12\n\nrequest_id\x18\x06 \x01(\t\x12:\n\x0clist_request\x18\x07 \x01(\x0b\x32$.aether.v1.AuthorityGrantListRequest\x12M\n\x16\x62\x61tch_exchange_request\x18\x08 \x01(\x0b\x32-.aether.v1.AuthorityGrantBatchExchangeRequest\x12R\n\x19\x64\x65rive_for_target_request\x18\t \x01(\x0b\x32/.aether.v1.AuthorityGrantDeriveForTargetRequest\"\x98\x01\n\x06OpType\x12\x0c\n\x08\x45XCHANGE\x10\x00\x12\n\n\x06\x44\x45RIVE\x10\x01\x12\x07\n\x03GET\x10\x02\x12\t\n\x05RENEW\x10\x03\x12\n\n\x06REVOKE\x10\x04\x12\x12\n\x0eLIST_MY_GRANTS\x10\x05\x12\x15\n\x11LIST_GRANTS_ON_ME\x10\x06\x12\x12\n\x0e\x42\x41TCH_EXCHANGE\x10\x07\x12\x15\n\x11\x44\x45RIVE_FOR_TARGET\x10\x08\"\x85\x04\n\x1d\x41uthorityGrantExchangeRequest\x12\x19\n\x11source_session_id\x18\x01 \x01(\t\x12\x17\n\x0fworkspace_scope\x18\x02 \x03(\t\x12\x46\n\x0eresource_scope\x18\x03 \x03(\x0b\x32..aether.v1.ACLAuthorityGrantResourceScopeEntry\x12\x17\n\x0foperation_scope\x18\x04 \x03(\t\x12\x18\n\x10max_access_level\x18\x05 \x01(\x05\x12\x15\n\raudience_type\x18\x06 \x01(\t\x12\x13\n\x0b\x61udience_id\x18\x07 \x01(\t\x12#\n\x1bvalid_while_audience_active\x18\x08 \x01(\x08\x12\x12\n\nexpires_at\x18\t \x01(\x03\x12\x17\n\x0frenewable_until\x18\n \x01(\x03\x12\x14\n\x0cmay_delegate\x18\x0b \x01(\x08\x12\x16\n\x0eremaining_hops\x18\x0c \x01(\x05\x12\x0e\n\x06reason\x18\r \x01(\t\x12H\n\x08metadata\x18\x0e \x03(\x0b\x32\x36.aether.v1.AuthorityGrantExchangeRequest.MetadataEntry\x1a/\n\rMetadataEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\xaa\x04\n\x1b\x41uthorityGrantDeriveRequest\x12\x17\n\x0fparent_grant_id\x18\x01 \x01(\t\x12)\n\x08\x64\x65legate\x18\x02 \x01(\x0b\x32\x17.aether.v1.PrincipalRef\x12\x17\n\x0fworkspace_scope\x18\x03 \x03(\t\x12\x46\n\x0eresource_scope\x18\x04 \x03(\x0b\x32..aether.v1.ACLAuthorityGrantResourceScopeEntry\x12\x17\n\x0foperation_scope\x18\x05 \x03(\t\x12\x18\n\x10max_access_level\x18\x06 \x01(\x05\x12\x15\n\raudience_type\x18\x07 \x01(\t\x12\x13\n\x0b\x61udience_id\x18\x08 \x01(\t\x12#\n\x1bvalid_while_audience_active\x18\t \x01(\x08\x12\x12\n\nexpires_at\x18\n \x01(\x03\x12\x17\n\x0frenewable_until\x18\x0b \x01(\x03\x12\x14\n\x0cmay_delegate\x18\x0c \x01(\x08\x12\x16\n\x0eremaining_hops\x18\r \x01(\x05\x12\x0e\n\x06reason\x18\x0e \x01(\t\x12\x46\n\x08metadata\x18\x0f \x03(\x0b\x32\x34.aether.v1.AuthorityGrantDeriveRequest.MetadataEntry\x1a/\n\rMetadataEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\xef\x01\n\x16\x41uthorityGrantResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\r\n\x05\x65rror\x18\x02 \x01(\t\x12\x0f\n\x07message\x18\x03 \x01(\t\x12/\n\x05grant\x18\x04 \x01(\x0b\x32 .aether.v1.ACLAuthorityGrantInfo\x12\x12\n\nrequest_id\x18\x05 \x01(\t\x12\x30\n\x06grants\x18\x06 \x03(\x0b\x32 .aether.v1.ACLAuthorityGrantInfo\x12\r\n\x05total\x18\x07 \x01(\x05\x12\x1e\n\x16\x63\x61\x63he_hint_ttl_seconds\x18\x08 \x01(\x05\"\x7f\n\x19\x41uthorityGrantListRequest\x12\x15\n\raudience_type\x18\x01 \x01(\t\x12\x13\n\x0b\x61udience_id\x18\x02 \x01(\t\x12\x17\n\x0finclude_revoked\x18\x03 \x01(\x08\x12\r\n\x05limit\x18\x04 \x01(\x05\x12\x0e\n\x06offset\x18\x05 \x01(\x05\"}\n\"AuthorityGrantBatchExchangeRequest\x12:\n\x08requests\x18\x01 \x03(\x0b\x32(.aether.v1.AuthorityGrantExchangeRequest\x12\x1b\n\x13stop_on_first_error\x18\x02 \x01(\x08\"\xb2\x02\n$AuthorityGrantDeriveForTargetRequest\x12\x17\n\x0fparent_grant_id\x18\x01 \x01(\t\x12\'\n\x06target\x18\x02 \x01(\x0b\x32\x17.aether.v1.PrincipalRef\x12\x15\n\raudience_type\x18\x03 \x01(\t\x12\x13\n\x0b\x61udience_id\x18\x04 \x01(\t\x12\x17\n\x0foperation_scope\x18\x05 \x03(\t\x12\x18\n\x10max_access_level\x18\x06 \x01(\x05\x12\x12\n\nexpires_at\x18\x07 \x01(\x03\x12\x17\n\x0frenewable_until\x18\x08 \x01(\x03\x12\x14\n\x0cmay_delegate\x18\t \x01(\x08\x12\x16\n\x0eremaining_hops\x18\n \x01(\x05\x12\x0e\n\x06reason\x18\x0b \x01(\t\"\xc3\x01\n\x11\x41uthorityIdentity\x12(\n\x07subject\x18\x01 \x01(\x0b\x32\x17.aether.v1.PrincipalRef\x12-\n\x0croot_subject\x18\x02 \x01(\x0b\x32\x17.aether.v1.PrincipalRef\x12)\n\x08\x64\x65legate\x18\x03 \x01(\x0b\x32\x17.aether.v1.PrincipalRef\x12*\n\tissued_by\x18\x04 \x01(\x0b\x32\x17.aether.v1.PrincipalRef\"\xd1\x01\n\rAuthoritySpan\x12\x17\n\x0fworkspace_scope\x18\x01 \x03(\t\x12\x18\n\x10max_access_level\x18\x02 \x01(\x05\x12\x15\n\raudience_type\x18\x03 \x01(\t\x12\x13\n\x0b\x61udience_id\x18\x04 \x01(\t\x12#\n\x1bvalid_while_audience_active\x18\x05 \x01(\x08\x12\x12\n\nexpires_at\x18\x06 \x01(\x03\x12\x17\n\x0frenewable_until\x18\x07 \x01(\x03\x12\x0f\n\x07revoked\x18\x08 \x01(\x08\"x\n\x18\x41uthorityGrantRevocation\x12\x10\n\x08grant_id\x18\x01 \x01(\t\x12\x15\n\rroot_grant_id\x18\x02 \x01(\t\x12\x0e\n\x06reason\x18\x03 \x01(\t\x12\x12\n\nrevoked_at\x18\x04 \x01(\x03\x12\x0f\n\x07\x63\x61scade\x18\x05 \x01(\x08\"_\n\x1d\x41uthorityRequestRoutingTarget\x12*\n\tprincipal\x18\x01 \x01(\x0b\x32\x17.aether.v1.PrincipalRef\x12\x12\n\ncapability\x18\x02 \x01(\t\"M\n\"AuthorityRequestResourceScopeEntry\x12\x15\n\rresource_type\x18\x01 \x01(\t\x12\x10\n\x08patterns\x18\x02 \x03(\t\"\xc7\x06\n\x10\x41uthorityRequest\x12\x12\n\nrequest_id\x18\x01 \x01(\t\x12\x31\n\x06status\x18\x02 \x01(\x0e\x32!.aether.v1.AuthorityRequestStatus\x12\x31\n\x10requesting_actor\x18\x03 \x01(\x0b\x32\x17.aether.v1.PrincipalRef\x12/\n\x0etarget_subject\x18\x04 \x01(\x0b\x32\x17.aether.v1.PrincipalRef\x12\x1f\n\x17\x64\x65sired_workspace_scope\x18\x05 \x03(\t\x12M\n\x16\x64\x65sired_resource_scope\x18\x06 \x03(\x0b\x32-.aether.v1.AuthorityRequestResourceScopeEntry\x12\x1f\n\x17\x64\x65sired_operation_scope\x18\x07 \x03(\t\x12\x36\n\x16requested_access_level\x18\x08 \x01(\x0e\x32\x16.aether.v1.AccessLevel\x12\"\n\x1arequested_duration_seconds\x18\t \x01(\x03\x12\x15\n\raudience_type\x18\n \x01(\t\x12\x13\n\x0b\x61udience_id\x18\x0b \x01(\t\x12@\n\x0erouting_target\x18\x0c \x01(\x0b\x32(.aether.v1.AuthorityRequestRoutingTarget\x12\x0e\n\x06reason\x18\r \x01(\t\x12\x0f\n\x07task_id\x18\x0e \x01(\t\x12;\n\x08metadata\x18\x0f \x03(\x0b\x32).aether.v1.AuthorityRequest.MetadataEntry\x12\x12\n\ncreated_at\x18\x10 \x01(\x03\x12\x12\n\nexpires_at\x18\x11 \x01(\x03\x12\x13\n\x0bresolved_at\x18\x12 \x01(\x03\x12\x18\n\x10granted_grant_id\x18\x13 \x01(\t\x12,\n\x0bresolved_by\x18\x14 \x01(\x0b\x32\x17.aether.v1.PrincipalRef\x12\x19\n\x11resolution_reason\x18\x15 \x01(\t\x1a/\n\rMetadataEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\xfa\x04\n\x1d\x43reateAuthorityRequestPayload\x12\x31\n\x10requesting_actor\x18\x01 \x01(\x0b\x32\x17.aether.v1.PrincipalRef\x12/\n\x0etarget_subject\x18\x02 \x01(\x0b\x32\x17.aether.v1.PrincipalRef\x12\x1f\n\x17\x64\x65sired_workspace_scope\x18\x03 \x03(\t\x12M\n\x16\x64\x65sired_resource_scope\x18\x04 \x03(\x0b\x32-.aether.v1.AuthorityRequestResourceScopeEntry\x12\x1f\n\x17\x64\x65sired_operation_scope\x18\x05 \x03(\t\x12\x36\n\x16requested_access_level\x18\x06 \x01(\x0e\x32\x16.aether.v1.AccessLevel\x12\"\n\x1arequested_duration_seconds\x18\x07 \x01(\x03\x12\x15\n\raudience_type\x18\x08 \x01(\t\x12\x13\n\x0b\x61udience_id\x18\t \x01(\t\x12@\n\x0erouting_target\x18\n \x01(\x0b\x32(.aether.v1.AuthorityRequestRoutingTarget\x12\x0e\n\x06reason\x18\x0b \x01(\t\x12\x0f\n\x07task_id\x18\x0c \x01(\t\x12H\n\x08metadata\x18\r \x03(\x0b\x32\x36.aether.v1.CreateAuthorityRequestPayload.MetadataEntry\x1a/\n\rMetadataEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\xca\x03\n\x1eResolveAuthorityRequestPayload\x12\x44\n\x08\x64\x65\x63ision\x18\x01 \x01(\x0e\x32\x32.aether.v1.ResolveAuthorityRequestPayload.Decision\x12\x1f\n\x17granted_workspace_scope\x18\x02 \x03(\t\x12M\n\x16granted_resource_scope\x18\x03 \x03(\x0b\x32-.aether.v1.AuthorityRequestResourceScopeEntry\x12\x1f\n\x17granted_operation_scope\x18\x04 \x03(\t\x12\x34\n\x14granted_access_level\x18\x05 \x01(\x0e\x32\x16.aether.v1.AccessLevel\x12 \n\x18granted_duration_seconds\x18\x06 \x01(\x03\x12\x0e\n\x06reason\x18\x07 \x01(\t\x12\x14\n\x0cmay_delegate\x18\x08 \x01(\x08\x12\x16\n\x0eremaining_hops\x18\t \x01(\x05\";\n\x08\x44\x65\x63ision\x12\x18\n\x14\x44\x45\x43ISION_UNSPECIFIED\x10\x00\x12\x0b\n\x07\x41PPROVE\x10\x01\x12\x08\n\x04\x44\x45NY\x10\x02\"\xa0\x01\n\x1a\x41uthorityRequestListFilter\x12\x31\n\x06status\x18\x01 \x01(\x0e\x32!.aether.v1.AuthorityRequestStatus\x12\x11\n\tworkspace\x18\x02 \x01(\t\x12\r\n\x05limit\x18\x03 \x01(\x05\x12\x0e\n\x06offset\x18\x04 \x01(\x05\x12\x1d\n\x15matching_capabilities\x18\x05 \x03(\t\"\xb5\x03\n\x19\x41uthorityRequestOperation\x12\x37\n\x02op\x18\x01 \x01(\x0e\x32+.aether.v1.AuthorityRequestOperation.OpType\x12\x12\n\nrequest_id\x18\x02 \x01(\t\x12\x38\n\x06\x63reate\x18\x03 \x01(\x0b\x32(.aether.v1.CreateAuthorityRequestPayload\x12:\n\x07resolve\x18\x04 \x01(\x0b\x32).aether.v1.ResolveAuthorityRequestPayload\x12:\n\x0blist_filter\x18\x05 \x01(\x0b\x32%.aether.v1.AuthorityRequestListFilter\x12\x19\n\x11\x63lient_request_id\x18\x06 \x01(\t\x12\x0e\n\x06reason\x18\x07 \x01(\t\"n\n\x06OpType\x12$\n AUTHORITY_REQUEST_OP_UNSPECIFIED\x10\x00\x12\n\n\x06\x43REATE\x10\x01\x12\x07\n\x03GET\x10\x02\x12\x10\n\x0cLIST_PENDING\x10\x03\x12\x0b\n\x07RESOLVE\x10\x04\x12\n\n\x06\x43\x41NCEL\x10\x05\"\xd0\x01\n!AuthorityRequestOperationResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\r\n\x05\x65rror\x18\x02 \x01(\t\x12\x19\n\x11\x63lient_request_id\x18\x03 \x01(\t\x12,\n\x07request\x18\x04 \x01(\x0b\x32\x1b.aether.v1.AuthorityRequest\x12-\n\x08requests\x18\x05 \x03(\x0b\x32\x1b.aether.v1.AuthorityRequest\x12\x13\n\x0btotal_count\x18\x06 \x01(\x05\"\x8b\x03\n\x15\x41uthorityRequestEvent\x12>\n\nevent_type\x18\x01 \x01(\x0e\x32*.aether.v1.AuthorityRequestEvent.EventType\x12,\n\x07request\x18\x02 \x01(\x0b\x32\x1b.aether.v1.AuthorityRequest\x12\x12\n\nemitted_at\x18\x03 \x01(\x03\"\xef\x01\n\tEventType\x12\'\n#AUTHORITY_REQUEST_EVENT_UNSPECIFIED\x10\x00\x12#\n\x1f\x41UTHORITY_REQUEST_EVENT_CREATED\x10\x01\x12$\n AUTHORITY_REQUEST_EVENT_APPROVED\x10\x02\x12\"\n\x1e\x41UTHORITY_REQUEST_EVENT_DENIED\x10\x03\x12#\n\x1f\x41UTHORITY_REQUEST_EVENT_EXPIRED\x10\x04\x12%\n!AUTHORITY_REQUEST_EVENT_CANCELLED\x10\x05\"\x84\x02\n\x0eTokenOperation\x12,\n\x02op\x18\x01 \x01(\x0e\x32 .aether.v1.TokenOperation.OpType\x12\x10\n\x08token_id\x18\x02 \x01(\t\x12\x35\n\x0e\x63reate_request\x18\x03 \x01(\x0b\x32\x1d.aether.v1.TokenCreateRequest\x12&\n\x06\x66ilter\x18\x04 \x01(\x0b\x32\x16.aether.v1.TokenFilter\x12\x12\n\nrequest_id\x18\x05 \x01(\t\"?\n\x06OpType\x12\x08\n\x04LIST\x10\x00\x12\x07\n\x03GET\x10\x01\x12\n\n\x06\x43REATE\x10\x02\x12\n\n\x06\x44\x45LETE\x10\x03\x12\n\n\x06REVOKE\x10\x04\"\x94\x01\n\x12TokenCreateRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x16\n\x0eprincipal_type\x18\x02 \x01(\t\x12\x1a\n\x12workspace_patterns\x18\x03 \x03(\t\x12\x0e\n\x06scopes\x18\x04 \x03(\t\x12\x18\n\x10\x65xpires_in_hours\x18\x05 \x01(\x05\x12\x12\n\ncreated_by\x18\x06 \x01(\t\"E\n\x0bTokenFilter\x12\r\n\x05limit\x18\x01 \x01(\x05\x12\x0e\n\x06offset\x18\x02 \x01(\x05\x12\x17\n\x0finclude_revoked\x18\x03 \x01(\x08\"\xf4\x01\n\tTokenInfo\x12\n\n\x02id\x18\x01 \x01(\t\x12\x0c\n\x04name\x18\x02 \x01(\t\x12\x16\n\x0eprincipal_type\x18\x03 \x01(\t\x12\x1a\n\x12workspace_patterns\x18\x04 \x03(\t\x12\x0e\n\x06scopes\x18\x05 \x03(\t\x12\x12\n\ncreated_by\x18\x06 \x01(\t\x12\x12\n\nexpires_at\x18\x07 \x01(\x03\x12\x14\n\x0clast_used_at\x18\x08 \x01(\x03\x12\x0f\n\x07revoked\x18\t \x01(\x08\x12\x12\n\nrevoked_at\x18\n \x01(\x03\x12\x12\n\ncreated_at\x18\x0b \x01(\x03\x12\x12\n\nupdated_at\x18\x0c \x01(\x03\"\xfa\x01\n\rTokenResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\r\n\x05\x65rror\x18\x02 \x01(\t\x12\x0f\n\x07message\x18\x03 \x01(\t\x12#\n\x05token\x18\x04 \x01(\x0b\x32\x14.aether.v1.TokenInfo\x12$\n\x06tokens\x18\x05 \x03(\x0b\x32\x14.aether.v1.TokenInfo\x12\x13\n\x0btotal_count\x18\x06 \x01(\x05\x12\x17\n\x0fplaintext_token\x18\x07 \x01(\t\x12+\n\rcreated_token\x18\x08 \x01(\x0b\x32\x14.aether.v1.TokenInfo\x12\x12\n\nrequest_id\x18\t \x01(\t\"\xb6\x02\n\x0eProgressReport\x12\x0f\n\x07task_id\x18\x01 \x01(\t\x12\r\n\x05state\x18\x02 \x01(\t\x12\x12\n\ncompletion\x18\x03 \x01(\x01\x12\x0f\n\x07summary\x18\x04 \x01(\t\x12%\n\x04step\x18\x05 \x01(\x0b\x32\x17.aether.v1.ProgressStep\x12\x11\n\trecipient\x18\x06 \x01(\t\x12\x12\n\nrequest_id\x18\x07 \x01(\t\x12\x39\n\x08metadata\x18\x08 \x03(\x0b\x32\'.aether.v1.ProgressReport.MetadataEntry\x12%\n\x04kind\x18\t \x01(\x0e\x32\x17.aether.v1.ProgressKind\x1a/\n\rMetadataEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"f\n\x0cProgressStep\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0e\n\x06\x64\x65tail\x18\x02 \x01(\t\x12\x10\n\x08sequence\x18\x03 \x01(\x05\x12\x13\n\x0btotal_steps\x18\x04 \x01(\x05\x12\x11\n\tstep_type\x18\x05 \x01(\t\"\xef\x02\n\x0eProgressUpdate\x12\x0e\n\x06source\x18\x01 \x01(\t\x12\x0f\n\x07task_id\x18\x02 \x01(\t\x12\r\n\x05state\x18\x03 \x01(\t\x12\x12\n\ncompletion\x18\x04 \x01(\x01\x12\x0f\n\x07summary\x18\x05 \x01(\t\x12%\n\x04step\x18\x06 \x01(\x0b\x32\x17.aether.v1.ProgressStep\x12\x14\n\x0ctimestamp_ms\x18\x07 \x01(\x03\x12\x11\n\tworkspace\x18\x08 \x01(\t\x12\x12\n\nrequest_id\x18\t \x01(\t\x12\x39\n\x08metadata\x18\n \x03(\x0b\x32\'.aether.v1.ProgressUpdate.MetadataEntry\x12\x11\n\trecipient\x18\x0b \x01(\t\x12%\n\x04kind\x18\x0c \x01(\x0e\x32\x17.aether.v1.ProgressKind\x1a/\n\rMetadataEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\xd9\x05\n\x11WorkflowOperation\x12/\n\x02op\x18\x01 \x01(\x0e\x32#.aether.v1.WorkflowOperation.OpType\x12\n\n\x02id\x18\x02 \x01(\t\x12\x14\n\x0csecondary_id\x18\x03 \x01(\t\x12\x11\n\tworkspace\x18\x04 \x01(\t\x12\x0c\n\x04\x64\x61ta\x18\x05 \x01(\x0c\x12\x12\n\nrequest_id\x18\x06 \x01(\t\x12\x15\n\rstatus_filter\x18\x07 \x01(\t\"\xa4\x04\n\x06OpType\x12\x0e\n\nLIST_RULES\x10\x00\x12\x0c\n\x08GET_RULE\x10\x01\x12\x0f\n\x0b\x43REATE_RULE\x10\x02\x12\x0f\n\x0bUPDATE_RULE\x10\x03\x12\x0f\n\x0b\x44\x45LETE_RULE\x10\x04\x12\x12\n\x0eLIST_WORKFLOWS\x10\x05\x12\x10\n\x0cGET_WORKFLOW\x10\x06\x12\x13\n\x0f\x43REATE_WORKFLOW\x10\x07\x12\x13\n\x0f\x44\x45LETE_WORKFLOW\x10\x08\x12\x12\n\x0eLIST_SCHEDULES\x10\t\x12\x13\n\x0f\x43REATE_SCHEDULE\x10\n\x12\x13\n\x0f\x44\x45LETE_SCHEDULE\x10\x0b\x12\x13\n\x0fLIST_EXECUTIONS\x10\x0c\x12\x11\n\rGET_EXECUTION\x10\r\x12\x14\n\x10\x43\x41NCEL_EXECUTION\x10\x0e\x12\x17\n\x13LIST_STATE_MACHINES\x10\x0f\x12\x15\n\x11GET_STATE_MACHINE\x10\x10\x12\x18\n\x14\x43REATE_STATE_MACHINE\x10\x11\x12\x18\n\x14\x44\x45LETE_STATE_MACHINE\x10\x12\x12\x15\n\x11LIST_SM_INSTANCES\x10\x13\x12\x13\n\x0fGET_SM_INSTANCE\x10\x14\x12\x16\n\x12\x43REATE_SM_INSTANCE\x10\x15\x12\x11\n\rSEND_SM_EVENT\x10\x16\x12\x13\n\x0fUPSERT_SCHEDULE\x10\x17\x12\x0e\n\nLIST_JOINS\x10\x18\x12\x0c\n\x08GET_JOIN\x10\x19\x12\x0f\n\x0b\x43\x41NCEL_JOIN\x10\x1a\"z\n\x10WorkflowResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\r\n\x05\x65rror\x18\x02 \x01(\t\x12\x0f\n\x07message\x18\x03 \x01(\t\x12\x0c\n\x04\x64\x61ta\x18\x04 \x01(\x0c\x12\x13\n\x0btotal_count\x18\x05 \x01(\x05\x12\x12\n\nrequest_id\x18\x06 \x01(\t\"\xaa\x02\n\x0fMessageEnvelope\x12\x0e\n\x06source\x18\x01 \x01(\t\x12\x0f\n\x07payload\x18\x02 \x01(\x0c\x12,\n\x0cmessage_type\x18\x03 \x01(\x0e\x32\x16.aether.v1.MessageType\x12\x14\n\x0ctimestamp_ms\x18\x04 \x01(\x03\x12:\n\x08metadata\x18\x05 \x03(\x0b\x32(.aether.v1.MessageEnvelope.MetadataEntry\x12\x11\n\tworkspace\x18\x06 \x01(\t\x12\x32\n\x11on_behalf_subject\x18\x07 \x01(\x0b\x32\x17.aether.v1.PrincipalRef\x1a/\n\rMetadataEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\xf7\x03\n\nAuditQuery\x12\x12\n\nrequest_id\x18\x01 \x01(\t\x12\x12\n\nstart_time\x18\x02 \x01(\x03\x12\x10\n\x08\x65nd_time\x18\x03 \x01(\x03\x12\x12\n\nevent_type\x18\x04 \x01(\t\x12\x12\n\nactor_type\x18\x05 \x01(\t\x12\x10\n\x08\x61\x63tor_id\x18\x06 \x01(\t\x12\x15\n\rresource_type\x18\x07 \x01(\t\x12\x13\n\x0bresource_id\x18\x08 \x01(\t\x12\x11\n\toperation\x18\t \x01(\t\x12\x11\n\tworkspace\x18\n \x01(\t\x12\x15\n\ronly_failures\x18\x0b \x01(\x08\x12\r\n\x05limit\x18\x0c \x01(\x05\x12\x0e\n\x06offset\x18\r \x01(\x05\x12\x14\n\x0csubject_type\x18\x0e \x01(\t\x12\x12\n\nsubject_id\x18\x0f \x01(\t\x12\x16\n\x0e\x61uthority_mode\x18\x10 \x01(\t\x12\x1a\n\x12\x61uthority_grant_id\x18\x11 \x01(\t\x12\x36\n\rauthorization\x18\x12 \x01(\x0b\x32\x1f.aether.v1.AuthorizationContext\x12\x1b\n\x13\x65xclude_actor_types\x18\x13 \x03(\t\x12\x1a\n\x12\x65xclude_workspaces\x18\x14 \x03(\t\x12\x1e\n\x16\x65xclude_service_direct\x18\x15 \x01(\x08\"\x85\x01\n\x12\x41uditQueryResponse\x12\x12\n\nrequest_id\x18\x01 \x01(\t\x12\x0f\n\x07success\x18\x02 \x01(\x08\x12\r\n\x05\x65rror\x18\x03 \x01(\t\x12&\n\x07\x65ntries\x18\x04 \x03(\x0b\x32\x15.aether.v1.AuditEntry\x12\x13\n\x0btotal_count\x18\x05 \x01(\x05\"\x8a\x04\n\nAuditEntry\x12\x10\n\x08\x61udit_id\x18\x01 \x01(\x03\x12\x11\n\ttimestamp\x18\x02 \x01(\x03\x12\x12\n\nevent_type\x18\x03 \x01(\t\x12\x12\n\nactor_type\x18\x04 \x01(\t\x12\x10\n\x08\x61\x63tor_id\x18\x05 \x01(\t\x12\x15\n\rresource_type\x18\x06 \x01(\t\x12\x13\n\x0bresource_id\x18\x07 \x01(\t\x12\x11\n\toperation\x18\x08 \x01(\t\x12\x11\n\tworkspace\x18\t \x01(\t\x12\x12\n\nsession_id\x18\n \x01(\t\x12\x12\n\ngateway_id\x18\x0b \x01(\t\x12\x0f\n\x07success\x18\x0c \x01(\x08\x12\x15\n\rerror_message\x18\r \x01(\t\x12\x15\n\rmetadata_json\x18\x0e \x01(\t\x12\x14\n\x0csubject_type\x18\x0f \x01(\t\x12\x12\n\nsubject_id\x18\x10 \x01(\t\x12\x19\n\x11root_subject_type\x18\x11 \x01(\t\x12\x17\n\x0froot_subject_id\x18\x12 \x01(\t\x12\x16\n\x0e\x61uthority_mode\x18\x13 \x01(\t\x12\x1f\n\x17root_authority_grant_id\x18\x14 \x01(\t\x12\x1a\n\x12\x61uthority_grant_id\x18\x15 \x01(\t\x12!\n\x19parent_authority_grant_id\x18\x16 \x01(\t\x12\x0e\n\x06source\x18\x17 \x01(\t\"\xb7\x02\n\x17SubmitAuditEventRequest\x12\x12\n\nevent_type\x18\x01 \x01(\t\x12\x11\n\toperation\x18\x02 \x01(\t\x12\x15\n\rresource_type\x18\x03 \x01(\t\x12\x13\n\x0bresource_id\x18\x04 \x01(\t\x12\x11\n\tworkspace\x18\x05 \x01(\t\x12\x0f\n\x07success\x18\x06 \x01(\x08\x12\x15\n\rerror_message\x18\x07 \x01(\t\x12\x42\n\x08metadata\x18\x08 \x03(\x0b\x32\x30.aether.v1.SubmitAuditEventRequest.MetadataEntry\x12\x19\n\x11\x63lient_request_id\x18\t \x01(\t\x1a/\n\rMetadataEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"q\n\x18SubmitAuditEventResponse\x12\x19\n\x11\x63lient_request_id\x18\x01 \x01(\t\x12\x0f\n\x07success\x18\x02 \x01(\x08\x12\x12\n\nerror_code\x18\x03 \x01(\t\x12\x15\n\rerror_message\x18\x04 \x01(\t\"\xfe\x03\n\x10ProxyHttpRequest\x12\x12\n\nrequest_id\x18\x01 \x01(\t\x12\x14\n\x0ctarget_topic\x18\x02 \x01(\t\x12\x0e\n\x06method\x18\x03 \x01(\t\x12\x0c\n\x04path\x18\x04 \x01(\t\x12\x39\n\x07headers\x18\x05 \x03(\x0b\x32(.aether.v1.ProxyHttpRequest.HeadersEntry\x12\x0c\n\x04\x62ody\x18\x06 \x01(\x0c\x12\x14\n\x0c\x62ody_chunked\x18\x07 \x01(\x08\x12\x36\n\rauthorization\x18\x08 \x01(\x0b\x32\x1f.aether.v1.AuthorizationContext\x12\x15\n\rapp_workspace\x18\t \x01(\t\x12\x12\n\ntimeout_ms\x18\n \x01(\x03\x12\x18\n\x10\x66ollow_redirects\x18\x0b \x01(\x08\x12\x14\n\x0c\x62\x61\x63kend_name\x18\x0c \x01(\t\x12$\n\x1cstream_response_indefinitely\x18\r \x01(\x08\x12\x1e\n\x16stream_idle_timeout_ms\x18\x0e \x01(\x03\x12\x1f\n\x17max_response_body_bytes\x18\x0f \x01(\x03\x12\x19\n\x11proxy_chain_depth\x18\x10 \x01(\r\x1a.\n\x0cHeadersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\xf2\x01\n\x11ProxyHttpResponse\x12\x12\n\nrequest_id\x18\x01 \x01(\t\x12\x13\n\x0bstatus_code\x18\x02 \x01(\x05\x12:\n\x07headers\x18\x03 \x03(\x0b\x32).aether.v1.ProxyHttpResponse.HeadersEntry\x12\x0c\n\x04\x62ody\x18\x04 \x01(\x0c\x12\x14\n\x0c\x62ody_chunked\x18\x05 \x01(\x08\x12$\n\x05\x65rror\x18\x06 \x01(\x0b\x32\x15.aether.v1.ProxyError\x1a.\n\x0cHeadersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"d\n\x12ProxyHttpBodyChunk\x12\x12\n\nrequest_id\x18\x01 \x01(\t\x12\x12\n\nis_request\x18\x02 \x01(\x08\x12\x0b\n\x03seq\x18\x03 \x01(\r\x12\x0c\n\x04\x64\x61ta\x18\x04 \x01(\x0c\x12\x0b\n\x03\x66in\x18\x05 \x01(\x08\"\xe2\x01\n\nProxyError\x12(\n\x04kind\x18\x01 \x01(\x0e\x32\x1a.aether.v1.ProxyError.Kind\x12\x0f\n\x07message\x18\x02 \x01(\t\"\x98\x01\n\x04Kind\x12\x0b\n\x07UNKNOWN\x10\x00\x12\x0f\n\x0b\x44IAL_FAILED\x10\x01\x12\x0b\n\x07TIMEOUT\x10\x02\x12\x12\n\x0eUPSTREAM_RESET\x10\x03\x12\x0e\n\nACL_DENIED\x10\x04\x12\x17\n\x13SIDECAR_UNAVAILABLE\x10\x05\x12\x15\n\x11PAYLOAD_TOO_LARGE\x10\x06\x12\x11\n\rDECODE_FAILED\x10\x07\"\xbd\x03\n\nTunnelOpen\x12\x11\n\ttunnel_id\x18\x01 \x01(\t\x12\x14\n\x0ctarget_topic\x18\x02 \x01(\t\x12\x30\n\x08protocol\x18\x03 \x01(\x0e\x32\x1e.aether.v1.TunnelOpen.Protocol\x12\x13\n\x0bremote_hint\x18\x04 \x01(\t\x12\x35\n\x08metadata\x18\x05 \x03(\x0b\x32#.aether.v1.TunnelOpen.MetadataEntry\x12\x36\n\rauthorization\x18\x06 \x01(\x0b\x32\x1f.aether.v1.AuthorizationContext\x12\x17\n\x0fidle_timeout_ms\x18\x07 \x01(\x03\x12\x11\n\tmax_bytes\x18\x08 \x01(\x03\x12\x15\n\rsession_token\x18\t \x01(\t\x12\x14\n\x0c\x62\x61\x63kend_name\x18\n \x01(\t\x12\x19\n\x11proxy_chain_depth\x18\x0b \x01(\r\x1a/\n\rMetadataEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"+\n\x08Protocol\x12\x07\n\x03TCP\x10\x00\x12\x07\n\x03UDP\x10\x01\x12\r\n\tWEBSOCKET\x10\x02\"G\n\nTunnelData\x12\x11\n\ttunnel_id\x18\x01 \x01(\t\x12\x0b\n\x03seq\x18\x02 \x01(\r\x12\x0c\n\x04\x64\x61ta\x18\x03 \x01(\x0c\x12\x0b\n\x03\x66in\x18\x04 \x01(\x08\"\xad\x01\n\x0bTunnelClose\x12\x11\n\ttunnel_id\x18\x01 \x01(\t\x12-\n\x06reason\x18\x02 \x01(\x0e\x32\x1d.aether.v1.TunnelClose.Reason\x12\x0e\n\x06\x64\x65tail\x18\x03 \x01(\t\"L\n\x06Reason\x12\n\n\x06NORMAL\x10\x00\x12\x0e\n\nPEER_RESET\x10\x01\x12\x10\n\x0cIDLE_TIMEOUT\x10\x02\x12\t\n\x05QUOTA\x10\x03\x12\t\n\x05\x45RROR\x10\x04\"@\n\tTunnelAck\x12\x11\n\ttunnel_id\x18\x01 \x01(\t\x12\x0f\n\x07\x61\x63k_seq\x18\x02 \x01(\r\x12\x0f\n\x07\x63redits\x18\x03 \x01(\r\"\xbd\x01\n\x17ResolveAuthorityRequest\x12\x12\n\nrequest_id\x18\x01 \x01(\t\x12&\n\x05\x61\x63tor\x18\x02 \x01(\x0b\x32\x17.aether.v1.PrincipalRef\x12\x10\n\x08grant_id\x18\x03 \x01(\t\x12(\n\x07subject\x18\x04 \x01(\x0b\x32\x17.aether.v1.PrincipalRef\x12\x15\n\raudience_type\x18\x05 \x01(\t\x12\x13\n\x0b\x61udience_id\x18\x06 \x01(\t\"z\n\x18ResolveAuthorityResponse\x12\x12\n\nrequest_id\x18\x01 \x01(\t\x12\n\n\x02ok\x18\x02 \x01(\x08\x12\r\n\x05\x65rror\x18\x03 \x01(\t\x12/\n\tauthority\x18\x04 \x01(\x0b\x32\x1c.aether.v1.ResolvedAuthority\"\x93\x01\n\x11ResolvedAuthority\x12&\n\x05\x61\x63tor\x18\x01 \x01(\x0b\x32\x17.aether.v1.PrincipalRef\x12(\n\x07subject\x18\x02 \x01(\x0b\x32\x17.aether.v1.PrincipalRef\x12,\n\x05grant\x18\x03 \x01(\x0b\x32\x1d.aether.v1.AuthorityGrantInfo\"\x88\x02\n\x12\x41uthorityGrantInfo\x12\x10\n\x08grant_id\x18\x01 \x01(\t\x12\x14\n\x0csubject_type\x18\x02 \x01(\t\x12\x12\n\nsubject_id\x18\x03 \x01(\t\x12\x19\n\x11root_subject_type\x18\x04 \x01(\t\x12\x17\n\x0froot_subject_id\x18\x05 \x01(\t\x12\x15\n\raudience_type\x18\x06 \x01(\t\x12\x13\n\x0b\x61udience_id\x18\x07 \x01(\t\x12\x18\n\x10max_access_level\x18\x08 \x01(\x05\x12\x17\n\x0fworkspace_scope\x18\t \x03(\t\x12\x12\n\nexpires_at\x18\n \x01(\x03\x12\x0f\n\x07revoked\x18\x0b \x01(\x08\"Y\n\x17\x43onnectionStatusRequest\x12\x12\n\nrequest_id\x18\x01 \x01(\t\x12*\n\tprincipal\x18\x02 \x01(\x0b\x32\x17.aether.v1.PrincipalRef\"r\n\x18\x43onnectionStatusResponse\x12\x12\n\nrequest_id\x18\x01 \x01(\t\x12\n\n\x02ok\x18\x02 \x01(\x08\x12\r\n\x05\x65rror\x18\x03 \x01(\t\x12\x11\n\tconnected\x18\x04 \x01(\x08\x12\x14\n\x0clast_seen_at\x18\x05 \x01(\x03\"\x9d\x02\n\x19TaskSubscriptionOperation\x12\x37\n\x02op\x18\x01 \x01(\x0e\x32+.aether.v1.TaskSubscriptionOperation.OpType\x12\x0f\n\x07task_id\x18\x02 \x01(\t\x12\x11\n\trecursive\x18\x03 \x01(\x08\x12\x19\n\x11\x63lient_request_id\x18\x04 \x01(\t\x12\x1f\n\x17start_timestamp_unix_ms\x18\x05 \x01(\x03\x12\x17\n\x0fsubscription_id\x18\x06 \x01(\t\"N\n\x06OpType\x12$\n TASK_SUBSCRIPTION_OP_UNSPECIFIED\x10\x00\x12\r\n\tSUBSCRIBE\x10\x01\x12\x0f\n\x0bUNSUBSCRIBE\x10\x02\"\x88\x01\n!TaskSubscriptionOperationResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\r\n\x05\x65rror\x18\x02 \x01(\t\x12\x19\n\x11\x63lient_request_id\x18\x03 \x01(\t\x12\x0f\n\x07task_id\x18\x04 \x01(\t\x12\x17\n\x0fsubscription_id\x18\x05 \x01(\t\"\xfb\x02\n\tTaskEvent\x12\x0f\n\x07task_id\x18\x01 \x01(\t\x12\x1a\n\x12\x65mitted_at_unix_ms\x18\x02 \x01(\x03\x12\x11\n\tworkspace\x18\x03 \x01(\t\x12\x16\n\x0eparent_task_id\x18\x04 \x01(\t\x12\x17\n\x0fsubscription_id\x18\x05 \x01(\t\x12;\n\x0estatus_changed\x18\n \x01(\x0b\x32!.aether.v1.TaskStatusChangedEventH\x00\x12\x30\n\x08progress\x18\x0b \x01(\x0b\x32\x1c.aether.v1.TaskProgressEventH\x00\x12=\n\x0f\x63hild_lifecycle\x18\x0c \x01(\x0b\x32\".aether.v1.TaskChildLifecycleEventH\x00\x12\x46\n\x11\x61uthority_request\x18\r \x01(\x0b\x32).aether.v1.TaskAuthorityRequestEventRelayH\x00\x42\x07\n\x05\x65vent\"~\n\x16TaskStatusChangedEvent\x12*\n\x0b\x66rom_status\x18\x01 \x01(\x0e\x32\x15.aether.v1.TaskStatus\x12(\n\tto_status\x18\x02 \x01(\x0e\x32\x15.aether.v1.TaskStatus\x12\x0e\n\x06reason\x18\x03 \x01(\t\"\xb4\x01\n\x11TaskProgressEvent\x12\r\n\x05state\x18\x01 \x01(\t\x12\x10\n\x08progress\x18\x02 \x01(\x01\x12\x0f\n\x07message\x18\x03 \x01(\t\x12<\n\x08metadata\x18\x04 \x03(\x0b\x32*.aether.v1.TaskProgressEvent.MetadataEntry\x1a/\n\rMetadataEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"p\n\x17TaskChildLifecycleEvent\x12\x15\n\rchild_task_id\x18\x01 \x01(\t\x12+\n\x0c\x63hild_status\x18\x02 \x01(\x0e\x32\x15.aether.v1.TaskStatus\x12\x11\n\tlifecycle\x18\x03 \x01(\t\"Q\n\x1eTaskAuthorityRequestEventRelay\x12/\n\x05\x65vent\x18\x01 \x01(\x0b\x32 .aether.v1.AuthorityRequestEvent*t\n\x0bMessageType\x12\x1c\n\x18MESSAGE_TYPE_UNSPECIFIED\x10\x00\x12\x08\n\x04\x43HAT\x10\x01\x12\x0b\n\x07\x43ONTROL\x10\x02\x12\r\n\tTOOL_CALL\x10\x03\x12\t\n\x05\x45VENT\x10\x04\x12\n\n\x06METRIC\x10\x05\x12\n\n\x06OPAQUE\x10\x06*\xf2\x01\n\rPrincipalType\x12\x1e\n\x1aPRINCIPAL_TYPE_UNSPECIFIED\x10\x00\x12\x13\n\x0fPRINCIPAL_AGENT\x10\x01\x12\x12\n\x0ePRINCIPAL_TASK\x10\x02\x12\x12\n\x0ePRINCIPAL_USER\x10\x03\x12\x1a\n\x16PRINCIPAL_ORCHESTRATOR\x10\x04\x12\x1d\n\x19PRINCIPAL_WORKFLOW_ENGINE\x10\x05\x12\x1c\n\x18PRINCIPAL_METRICS_BRIDGE\x10\x06\x12\x14\n\x10PRINCIPAL_BRIDGE\x10\x07\x12\x15\n\x11PRINCIPAL_SERVICE\x10\x08*\xc4\x02\n\nTaskStatus\x12\x1b\n\x17TASK_STATUS_UNSPECIFIED\x10\x00\x12\x16\n\x12TASK_STATUS_QUEUED\x10\x01\x12\x17\n\x13TASK_STATUS_RUNNING\x10\x02\x12\x19\n\x15TASK_STATUS_COMPLETED\x10\x03\x12\x16\n\x12TASK_STATUS_FAILED\x10\x04\x12\x19\n\x15TASK_STATUS_CANCELLED\x10\x05\x12\x1d\n\x19TASK_STATUS_WAITING_INPUT\x10\x06\x12!\n\x1dTASK_STATUS_WAITING_AUTHORITY\x10\x07\x12\"\n\x1eTASK_STATUS_WAITING_DEPENDENCY\x10\x08\x12\x1a\n\x16TASK_STATUS_HIBERNATED\x10\t\x12\x18\n\x14TASK_STATUS_REJECTED\x10\n*\x81\x01\n\x0cHealthStatus\x12\x1d\n\x19HEALTH_STATUS_UNSPECIFIED\x10\x00\x12\x19\n\x15HEALTH_STATUS_HEALTHY\x10\x01\x12\x1a\n\x16HEALTH_STATUS_DEGRADED\x10\x02\x12\x1b\n\x17HEALTH_STATUS_UNHEALTHY\x10\x03*s\n\x11HealthCheckStatus\x12#\n\x1fHEALTH_CHECK_STATUS_UNSPECIFIED\x10\x00\x12\x1a\n\x16HEALTH_CHECK_STATUS_OK\x10\x01\x12\x1d\n\x19HEALTH_CHECK_STATUS_ERROR\x10\x02*\xc3\x01\n\x0b\x41\x63\x63\x65ssLevel\x12\x1c\n\x18\x41\x43\x43\x45SS_LEVEL_UNSPECIFIED\x10\x00\x12\x15\n\x11\x41\x43\x43\x45SS_LEVEL_NONE\x10\x01\x12\x15\n\x11\x41\x43\x43\x45SS_LEVEL_READ\x10\x02\x12\x1a\n\x16\x41\x43\x43\x45SS_LEVEL_READWRITE\x10\x03\x12\x17\n\x13\x41\x43\x43\x45SS_LEVEL_MANAGE\x10\x04\x12\x16\n\x12\x41\x43\x43\x45SS_LEVEL_ADMIN\x10\x05\x12\x1b\n\x17\x41\x43\x43\x45SS_LEVEL_SUPERADMIN\x10\x06*=\n\x12TaskAssignmentMode\x12\x0f\n\x0bSELF_ASSIGN\x10\x00\x12\x0c\n\x08TARGETED\x10\x01\x12\x08\n\x04POOL\x10\x02*t\n\tTaskClass\x12\x1a\n\x16TASK_CLASS_UNSPECIFIED\x10\x00\x12\x1a\n\x16TASK_CLASS_INTERACTIVE\x10\x01\x12\x19\n\x15TASK_CLASS_BACKGROUND\x10\x02\x12\x14\n\x10TASK_CLASS_BATCH\x10\x03*\xa9\x01\n\x0cTaskPriority\x12\x1d\n\x19TASK_PRIORITY_UNSPECIFIED\x10\x00\x12\x16\n\x12TASK_PRIORITY_XLOW\x10\n\x12\x15\n\x11TASK_PRIORITY_LOW\x10\x14\x12\x18\n\x14TASK_PRIORITY_NORMAL\x10\x1e\x12\x16\n\x12TASK_PRIORITY_HIGH\x10(\x12\x19\n\x15TASK_PRIORITY_PREEMPT\x10\x32*\x99\x01\n\x0f\x42\x61\x63koffStrategy\x12 \n\x1c\x42\x41\x43KOFF_STRATEGY_UNSPECIFIED\x10\x00\x12\x1a\n\x16\x42\x41\x43KOFF_STRATEGY_FIXED\x10\x01\x12 \n\x1c\x42\x41\x43KOFF_STRATEGY_EXPONENTIAL\x10\x02\x12&\n\"BACKOFF_STRATEGY_EXPLICIT_SCHEDULE\x10\x03*\x94\x01\n\nWaitReason\x12\x1b\n\x17WAIT_REASON_UNSPECIFIED\x10\x00\x12\x15\n\x11WAIT_REASON_INPUT\x10\x01\x12\x19\n\x15WAIT_REASON_AUTHORITY\x10\x02\x12\x1a\n\x16WAIT_REASON_DEPENDENCY\x10\x03\x12\x1b\n\x17WAIT_REASON_HIBERNATION\x10\x04*\x82\x02\n\x16\x41uthorityRequestStatus\x12(\n$AUTHORITY_REQUEST_STATUS_UNSPECIFIED\x10\x00\x12$\n AUTHORITY_REQUEST_STATUS_PENDING\x10\x01\x12%\n!AUTHORITY_REQUEST_STATUS_APPROVED\x10\x02\x12#\n\x1f\x41UTHORITY_REQUEST_STATUS_DENIED\x10\x03\x12$\n AUTHORITY_REQUEST_STATUS_EXPIRED\x10\x04\x12&\n\"AUTHORITY_REQUEST_STATUS_CANCELLED\x10\x05*t\n\x0cProgressKind\x12\x1d\n\x19PROGRESS_KIND_UNSPECIFIED\x10\x00\x12\x16\n\x12PROGRESS_KIND_CHAT\x10\x01\x12\x15\n\x11PROGRESS_KIND_APP\x10\x02\x12\x16\n\x12PROGRESS_KIND_TASK\x10\x03\x32X\n\rAetherGateway\x12G\n\x07\x43onnect\x12\x1a.aether.v1.UpstreamMessage\x1a\x1c.aether.v1.DownstreamMessage(\x01\x30\x01\x42-Z+github.com/scitrera/aether/api/proto;aetherb\x06proto3') _globals = globals() _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) @@ -80,6 +80,14 @@ _globals['_ACLAUDITENTRYINFO_METADATAENTRY']._serialized_options = b'8\001' _globals['_ACLAUTHORITYGRANTINFO_METADATAENTRY']._loaded_options = None _globals['_ACLAUTHORITYGRANTINFO_METADATAENTRY']._serialized_options = b'8\001' + _globals['_ACLGROUPREQUEST_METADATAENTRY']._loaded_options = None + _globals['_ACLGROUPREQUEST_METADATAENTRY']._serialized_options = b'8\001' + _globals['_ACLROLEREQUEST_METADATAENTRY']._loaded_options = None + _globals['_ACLROLEREQUEST_METADATAENTRY']._serialized_options = b'8\001' + _globals['_ACLGROUPINFO_METADATAENTRY']._loaded_options = None + _globals['_ACLGROUPINFO_METADATAENTRY']._serialized_options = b'8\001' + _globals['_ACLROLEINFO_METADATAENTRY']._loaded_options = None + _globals['_ACLROLEINFO_METADATAENTRY']._serialized_options = b'8\001' _globals['_AUTHORITYGRANTEXCHANGEREQUEST_METADATAENTRY']._loaded_options = None _globals['_AUTHORITYGRANTEXCHANGEREQUEST_METADATAENTRY']._serialized_options = b'8\001' _globals['_AUTHORITYGRANTDERIVEREQUEST_METADATAENTRY']._loaded_options = None @@ -104,28 +112,32 @@ _globals['_TUNNELOPEN_METADATAENTRY']._serialized_options = b'8\001' _globals['_TASKPROGRESSEVENT_METADATAENTRY']._loaded_options = None _globals['_TASKPROGRESSEVENT_METADATAENTRY']._serialized_options = b'8\001' - _globals['_MESSAGETYPE']._serialized_start=37936 - _globals['_MESSAGETYPE']._serialized_end=38052 - _globals['_PRINCIPALTYPE']._serialized_start=38055 - _globals['_PRINCIPALTYPE']._serialized_end=38297 - _globals['_TASKSTATUS']._serialized_start=38300 - _globals['_TASKSTATUS']._serialized_end=38624 - _globals['_HEALTHSTATUS']._serialized_start=38627 - _globals['_HEALTHSTATUS']._serialized_end=38756 - _globals['_HEALTHCHECKSTATUS']._serialized_start=38758 - _globals['_HEALTHCHECKSTATUS']._serialized_end=38873 - _globals['_ACCESSLEVEL']._serialized_start=38876 - _globals['_ACCESSLEVEL']._serialized_end=39071 - _globals['_TASKASSIGNMENTMODE']._serialized_start=39073 - _globals['_TASKASSIGNMENTMODE']._serialized_end=39134 - _globals['_TASKCLASS']._serialized_start=39136 - _globals['_TASKCLASS']._serialized_end=39252 - _globals['_WAITREASON']._serialized_start=39255 - _globals['_WAITREASON']._serialized_end=39403 - _globals['_AUTHORITYREQUESTSTATUS']._serialized_start=39406 - _globals['_AUTHORITYREQUESTSTATUS']._serialized_end=39664 - _globals['_PROGRESSKIND']._serialized_start=39666 - _globals['_PROGRESSKIND']._serialized_end=39782 + _globals['_MESSAGETYPE']._serialized_start=41933 + _globals['_MESSAGETYPE']._serialized_end=42049 + _globals['_PRINCIPALTYPE']._serialized_start=42052 + _globals['_PRINCIPALTYPE']._serialized_end=42294 + _globals['_TASKSTATUS']._serialized_start=42297 + _globals['_TASKSTATUS']._serialized_end=42621 + _globals['_HEALTHSTATUS']._serialized_start=42624 + _globals['_HEALTHSTATUS']._serialized_end=42753 + _globals['_HEALTHCHECKSTATUS']._serialized_start=42755 + _globals['_HEALTHCHECKSTATUS']._serialized_end=42870 + _globals['_ACCESSLEVEL']._serialized_start=42873 + _globals['_ACCESSLEVEL']._serialized_end=43068 + _globals['_TASKASSIGNMENTMODE']._serialized_start=43070 + _globals['_TASKASSIGNMENTMODE']._serialized_end=43131 + _globals['_TASKCLASS']._serialized_start=43133 + _globals['_TASKCLASS']._serialized_end=43249 + _globals['_TASKPRIORITY']._serialized_start=43252 + _globals['_TASKPRIORITY']._serialized_end=43421 + _globals['_BACKOFFSTRATEGY']._serialized_start=43424 + _globals['_BACKOFFSTRATEGY']._serialized_end=43577 + _globals['_WAITREASON']._serialized_start=43580 + _globals['_WAITREASON']._serialized_end=43728 + _globals['_AUTHORITYREQUESTSTATUS']._serialized_start=43731 + _globals['_AUTHORITYREQUESTSTATUS']._serialized_end=43989 + _globals['_PROGRESSKIND']._serialized_start=43991 + _globals['_PROGRESSKIND']._serialized_end=44107 _globals['_UPSTREAMMESSAGE']._serialized_start=28 _globals['_UPSTREAMMESSAGE']._serialized_end=1741 _globals['_DOWNSTREAMMESSAGE']._serialized_start=1744 @@ -153,365 +165,397 @@ _globals['_BRIDGEIDENTITY']._serialized_start=5429 _globals['_BRIDGEIDENTITY']._serialized_end=5488 _globals['_SERVICEIDENTITY']._serialized_start=5490 - _globals['_SERVICEIDENTITY']._serialized_end=5550 - _globals['_AGENTIDENTITY']._serialized_start=5552 - _globals['_AGENTIDENTITY']._serialized_end=5629 - _globals['_TASKIDENTITY']._serialized_start=5631 - _globals['_TASKIDENTITY']._serialized_end=5714 - _globals['_USERIDENTITY']._serialized_start=5716 - _globals['_USERIDENTITY']._serialized_end=5766 - _globals['_PRINCIPALREF']._serialized_start=5768 - _globals['_PRINCIPALREF']._serialized_end=5828 - _globals['_AUTHORIZATIONCONTEXT']._serialized_start=5831 - _globals['_AUTHORIZATIONCONTEXT']._serialized_end=5989 - _globals['_RESOLVEDAUTHORITYINFO']._serialized_start=5992 - _globals['_RESOLVEDAUTHORITYINFO']._serialized_end=6180 - _globals['_SENDMESSAGE']._serialized_start=6183 - _globals['_SENDMESSAGE']._serialized_end=6360 - _globals['_METRIC']._serialized_start=6363 - _globals['_METRIC']._serialized_end=6559 - _globals['_METRIC_METADATAENTRY']._serialized_start=6512 - _globals['_METRIC_METADATAENTRY']._serialized_end=6559 - _globals['_METRICENTRY']._serialized_start=6561 - _globals['_METRICENTRY']._serialized_end=6615 - _globals['_SWITCHWORKSPACE']._serialized_start=6617 - _globals['_SWITCHWORKSPACE']._serialized_end=6660 - _globals['_KVOPERATION']._serialized_start=6663 - _globals['_KVOPERATION']._serialized_end=7256 - _globals['_KVOPERATION_OPTYPE']._serialized_start=6961 - _globals['_KVOPERATION_OPTYPE']._serialized_end=7075 - _globals['_KVOPERATION_SCOPE']._serialized_start=7078 - _globals['_KVOPERATION_SCOPE']._serialized_end=7256 - _globals['_KVRESPONSE']._serialized_start=7259 - _globals['_KVRESPONSE']._serialized_end=7473 - _globals['_KVRESPONSE_KVMAPENTRY']._serialized_start=7429 - _globals['_KVRESPONSE_KVMAPENTRY']._serialized_end=7473 - _globals['_INCOMINGMESSAGE']._serialized_start=7475 - _globals['_INCOMINGMESSAGE']._serialized_end=7596 - _globals['_CONFIGSNAPSHOT']._serialized_start=7599 - _globals['_CONFIGSNAPSHOT']._serialized_end=8223 - _globals['_CONFIGSNAPSHOT_KVENTRY']._serialized_start=7962 - _globals['_CONFIGSNAPSHOT_KVENTRY']._serialized_end=8003 - _globals['_CONFIGSNAPSHOT_GLOBALKVENTRY']._serialized_start=8005 - _globals['_CONFIGSNAPSHOT_GLOBALKVENTRY']._serialized_end=8052 - _globals['_CONFIGSNAPSHOT_TASKCONTEXTENTRY']._serialized_start=8054 - _globals['_CONFIGSNAPSHOT_TASKCONTEXTENTRY']._serialized_end=8104 - _globals['_CONFIGSNAPSHOT_WORKSPACEEXCLUSIVEKVENTRY']._serialized_start=8106 - _globals['_CONFIGSNAPSHOT_WORKSPACEEXCLUSIVEKVENTRY']._serialized_end=8165 - _globals['_CONFIGSNAPSHOT_GLOBALEXCLUSIVEKVENTRY']._serialized_start=8167 - _globals['_CONFIGSNAPSHOT_GLOBALEXCLUSIVEKVENTRY']._serialized_end=8223 - _globals['_SIGNAL']._serialized_start=8226 - _globals['_SIGNAL']._serialized_end=8355 - _globals['_SIGNAL_SIGNALTYPE']._serialized_start=8296 - _globals['_SIGNAL_SIGNALTYPE']._serialized_end=8355 - _globals['_ERRORRESPONSE']._serialized_start=8357 - _globals['_ERRORRESPONSE']._serialized_end=8466 - _globals['_CREATETASKREQUEST']._serialized_start=8469 - _globals['_CREATETASKREQUEST']._serialized_end=9078 - _globals['_CREATETASKREQUEST_LAUNCHPARAMOVERRIDESENTRY']._serialized_start=8970 - _globals['_CREATETASKREQUEST_LAUNCHPARAMOVERRIDESENTRY']._serialized_end=9029 - _globals['_CREATETASKREQUEST_METADATAENTRY']._serialized_start=6512 - _globals['_CREATETASKREQUEST_METADATAENTRY']._serialized_end=6559 - _globals['_CREATETASKRESPONSE']._serialized_start=9081 - _globals['_CREATETASKRESPONSE']._serialized_end=9283 - _globals['_TASKASSIGNMENT']._serialized_start=9286 - _globals['_TASKASSIGNMENT']._serialized_end=9805 - _globals['_TASKASSIGNMENT_METADATAENTRY']._serialized_start=6512 - _globals['_TASKASSIGNMENT_METADATAENTRY']._serialized_end=6559 - _globals['_TASKASSIGNMENT_LAUNCHPARAMSENTRY']._serialized_start=9754 - _globals['_TASKASSIGNMENT_LAUNCHPARAMSENTRY']._serialized_end=9805 - _globals['_CHECKPOINTOPERATION']._serialized_start=9808 - _globals['_CHECKPOINTOPERATION']._serialized_end=9992 - _globals['_CHECKPOINTOPERATION_OPTYPE']._serialized_start=9942 - _globals['_CHECKPOINTOPERATION_OPTYPE']._serialized_end=9992 - _globals['_CHECKPOINTRESPONSE']._serialized_start=9994 - _globals['_CHECKPOINTRESPONSE']._serialized_end=10112 - _globals['_ADMINQUERY']._serialized_start=10115 - _globals['_ADMINQUERY']._serialized_end=10351 - _globals['_ADMINQUERY_OPTYPE']._serialized_start=10256 - _globals['_ADMINQUERY_OPTYPE']._serialized_end=10351 - _globals['_CONNECTIONFILTER']._serialized_start=10353 - _globals['_CONNECTIONFILTER']._serialized_end=10461 - _globals['_CONNECTIONINFO']._serialized_start=10464 - _globals['_CONNECTIONINFO']._serialized_end=10704 - _globals['_ADMINRESPONSE']._serialized_start=10707 - _globals['_ADMINRESPONSE']._serialized_end=11007 - _globals['_HEALTHINFO']._serialized_start=11010 - _globals['_HEALTHINFO']._serialized_end=11244 - _globals['_HEALTHINFO_CHECKSENTRY']._serialized_start=11175 - _globals['_HEALTHINFO_CHECKSENTRY']._serialized_end=11244 - _globals['_HEALTHCHECK']._serialized_start=11246 - _globals['_HEALTHCHECK']._serialized_end=11337 - _globals['_GATEWAYINFO']._serialized_start=11340 - _globals['_GATEWAYINFO']._serialized_end=11520 - _globals['_GATEWAYSTATS']._serialized_start=11523 - _globals['_GATEWAYSTATS']._serialized_end=11933 - _globals['_SESSIONOPERATION']._serialized_start=11936 - _globals['_SESSIONOPERATION']._serialized_end=12204 - _globals['_SESSIONOPERATION_OPTYPE']._serialized_start=12161 - _globals['_SESSIONOPERATION_OPTYPE']._serialized_end=12204 - _globals['_SESSIONOPERATIONRESPONSE']._serialized_start=12207 - _globals['_SESSIONOPERATIONRESPONSE']._serialized_end=12418 - _globals['_TASKQUERY']._serialized_start=12421 - _globals['_TASKQUERY']._serialized_end=12578 - _globals['_TASKQUERY_OPTYPE']._serialized_start=12551 - _globals['_TASKQUERY_OPTYPE']._serialized_end=12578 - _globals['_TASKFILTER']._serialized_start=12581 - _globals['_TASKFILTER']._serialized_end=13193 - _globals['_TASKINFO']._serialized_start=13196 - _globals['_TASKINFO']._serialized_end=14016 - _globals['_TASKINFO_METADATAENTRY']._serialized_start=6512 - _globals['_TASKINFO_METADATAENTRY']._serialized_end=6559 - _globals['_TASKQUERYRESPONSE']._serialized_start=14019 - _globals['_TASKQUERYRESPONSE']._serialized_end=14207 - _globals['_TASKOPERATION']._serialized_start=14210 - _globals['_TASKOPERATION']._serialized_end=14469 - _globals['_TASKOPERATION_OPTYPE']._serialized_start=14365 - _globals['_TASKOPERATION_OPTYPE']._serialized_end=14469 - _globals['_WAITSPEC']._serialized_start=14472 - _globals['_WAITSPEC']._serialized_end=14836 - _globals['_WAITSPEC_INPUTMATCHENTRY']._serialized_start=14787 - _globals['_WAITSPEC_INPUTMATCHENTRY']._serialized_end=14836 - _globals['_HIBERNATIONDESCRIPTOR']._serialized_start=14838 - _globals['_HIBERNATIONDESCRIPTOR']._serialized_end=14965 - _globals['_TASKOPERATIONRESPONSE']._serialized_start=14967 - _globals['_TASKOPERATIONRESPONSE']._serialized_end=15094 - _globals['_WORKSPACEOPERATION']._serialized_start=15097 - _globals['_WORKSPACEOPERATION']._serialized_end=15385 - _globals['_WORKSPACEOPERATION_OPTYPE']._serialized_start=15300 - _globals['_WORKSPACEOPERATION_OPTYPE']._serialized_end=15385 - _globals['_WORKSPACEFILTER']._serialized_start=15387 - _globals['_WORKSPACEFILTER']._serialized_end=15454 - _globals['_WORKSPACEINFO']._serialized_start=15457 - _globals['_WORKSPACEINFO']._serialized_end=15794 - _globals['_WORKSPACEINFO_METADATAENTRY']._serialized_start=6512 - _globals['_WORKSPACEINFO_METADATAENTRY']._serialized_end=6559 - _globals['_WORKSPACERESPONSE']._serialized_start=15797 - _globals['_WORKSPACERESPONSE']._serialized_end=16047 - _globals['_MESSAGEFLOWINFO']._serialized_start=16050 - _globals['_MESSAGEFLOWINFO']._serialized_end=16181 - _globals['_FLOWNODE']._serialized_start=16184 - _globals['_FLOWNODE']._serialized_end=16335 - _globals['_FLOWEDGE']._serialized_start=16337 - _globals['_FLOWEDGE']._serialized_end=16403 - _globals['_AGENTOPERATION']._serialized_start=16406 - _globals['_AGENTOPERATION']._serialized_end=16757 - _globals['_AGENTOPERATION_OPTYPE']._serialized_start=16656 - _globals['_AGENTOPERATION_OPTYPE']._serialized_end=16757 - _globals['_AGENTFILTER']._serialized_start=16759 - _globals['_AGENTFILTER']._serialized_end=16833 - _globals['_AGENTREGISTRATIONINFO']._serialized_start=16836 - _globals['_AGENTREGISTRATIONINFO']._serialized_end=17314 - _globals['_AGENTREGISTRATIONINFO_LAUNCHPARAMSENTRY']._serialized_start=9754 - _globals['_AGENTREGISTRATIONINFO_LAUNCHPARAMSENTRY']._serialized_end=9805 - _globals['_AGENTREGISTRATIONINFO_CAPABILITIESENTRY']._serialized_start=17263 - _globals['_AGENTREGISTRATIONINFO_CAPABILITIESENTRY']._serialized_end=17314 - _globals['_AGENTRESOURCESCHEMAENTRY']._serialized_start=17316 - _globals['_AGENTRESOURCESCHEMAENTRY']._serialized_end=17426 - _globals['_AGENTLAUNCHPARAMS']._serialized_start=17429 - _globals['_AGENTLAUNCHPARAMS']._serialized_end=17616 - _globals['_AGENTLAUNCHPARAMS_PARAMOVERRIDESENTRY']._serialized_start=17563 - _globals['_AGENTLAUNCHPARAMS_PARAMOVERRIDESENTRY']._serialized_end=17616 - _globals['_ORCHESTRATORINFO']._serialized_start=17618 - _globals['_ORCHESTRATORINFO']._serialized_end=17701 - _globals['_AGENTLAUNCHRESULT']._serialized_start=17703 - _globals['_AGENTLAUNCHRESULT']._serialized_end=17756 - _globals['_AGENTRESPONSE']._serialized_start=17759 - _globals['_AGENTRESPONSE']._serialized_end=18068 - _globals['_ACLOPERATION']._serialized_start=18071 - _globals['_ACLOPERATION']._serialized_end=18907 - _globals['_ACLOPERATION_OPTYPE']._serialized_start=18423 - _globals['_ACLOPERATION_OPTYPE']._serialized_end=18759 - _globals['_ACLRULEFILTER']._serialized_start=18910 - _globals['_ACLRULEFILTER']._serialized_end=19046 - _globals['_ACLAUDITFILTER']._serialized_start=19049 - _globals['_ACLAUDITFILTER']._serialized_end=19261 - _globals['_ACLGRANTREQUEST']._serialized_start=19264 - _globals['_ACLGRANTREQUEST']._serialized_end=19449 - _globals['_ACLSETFALLBACKREQUEST']._serialized_start=19451 - _globals['_ACLSETFALLBACKREQUEST']._serialized_end=19548 - _globals['_ACLAUTHORITYGRANTFILTER']._serialized_start=19551 - _globals['_ACLAUTHORITYGRANTFILTER']._serialized_end=19806 - _globals['_ACLAUTHORITYGRANTRESOURCESCOPEENTRY']._serialized_start=19808 - _globals['_ACLAUTHORITYGRANTRESOURCESCOPEENTRY']._serialized_end=19886 - _globals['_ACLAUTHORITYGRANTREQUEST']._serialized_start=19889 - _globals['_ACLAUTHORITYGRANTREQUEST']._serialized_end=20570 - _globals['_ACLAUTHORITYGRANTREQUEST_METADATAENTRY']._serialized_start=6512 - _globals['_ACLAUTHORITYGRANTREQUEST_METADATAENTRY']._serialized_end=6559 - _globals['_ACLRENEWAUTHORITYGRANTREQUEST']._serialized_start=20572 - _globals['_ACLRENEWAUTHORITYGRANTREQUEST']._serialized_end=20665 - _globals['_ACLRULEINFO']._serialized_start=20668 - _globals['_ACLRULEINFO']._serialized_end=20913 - _globals['_ACLFALLBACKPOLICYINFO']._serialized_start=20916 - _globals['_ACLFALLBACKPOLICYINFO']._serialized_end=21088 - _globals['_ACLAUDITENTRYINFO']._serialized_start=21091 - _globals['_ACLAUDITENTRYINFO']._serialized_end=21542 - _globals['_ACLAUDITENTRYINFO_METADATAENTRY']._serialized_start=6512 - _globals['_ACLAUDITENTRYINFO_METADATAENTRY']._serialized_end=6559 - _globals['_ACLAUTHORITYGRANTINFO']._serialized_start=21545 - _globals['_ACLAUTHORITYGRANTINFO']._serialized_end=22365 - _globals['_ACLAUTHORITYGRANTINFO_METADATAENTRY']._serialized_start=6512 - _globals['_ACLAUTHORITYGRANTINFO_METADATAENTRY']._serialized_end=6559 - _globals['_ACLCLEANUPRESULT']._serialized_start=22367 - _globals['_ACLCLEANUPRESULT']._serialized_end=22425 - _globals['_ACLRESPONSE']._serialized_start=22428 - _globals['_ACLRESPONSE']._serialized_end=22959 - _globals['_AUTHORITYGRANTOPERATION']._serialized_start=22962 - _globals['_AUTHORITYGRANTOPERATION']._serialized_end=23655 - _globals['_AUTHORITYGRANTOPERATION_OPTYPE']._serialized_start=23503 - _globals['_AUTHORITYGRANTOPERATION_OPTYPE']._serialized_end=23655 - _globals['_AUTHORITYGRANTEXCHANGEREQUEST']._serialized_start=23658 - _globals['_AUTHORITYGRANTEXCHANGEREQUEST']._serialized_end=24175 - _globals['_AUTHORITYGRANTEXCHANGEREQUEST_METADATAENTRY']._serialized_start=6512 - _globals['_AUTHORITYGRANTEXCHANGEREQUEST_METADATAENTRY']._serialized_end=6559 - _globals['_AUTHORITYGRANTDERIVEREQUEST']._serialized_start=24178 - _globals['_AUTHORITYGRANTDERIVEREQUEST']._serialized_end=24732 - _globals['_AUTHORITYGRANTDERIVEREQUEST_METADATAENTRY']._serialized_start=6512 - _globals['_AUTHORITYGRANTDERIVEREQUEST_METADATAENTRY']._serialized_end=6559 - _globals['_AUTHORITYGRANTRESPONSE']._serialized_start=24735 - _globals['_AUTHORITYGRANTRESPONSE']._serialized_end=24974 - _globals['_AUTHORITYGRANTLISTREQUEST']._serialized_start=24976 - _globals['_AUTHORITYGRANTLISTREQUEST']._serialized_end=25103 - _globals['_AUTHORITYGRANTBATCHEXCHANGEREQUEST']._serialized_start=25105 - _globals['_AUTHORITYGRANTBATCHEXCHANGEREQUEST']._serialized_end=25230 - _globals['_AUTHORITYGRANTDERIVEFORTARGETREQUEST']._serialized_start=25233 - _globals['_AUTHORITYGRANTDERIVEFORTARGETREQUEST']._serialized_end=25539 - _globals['_AUTHORITYIDENTITY']._serialized_start=25542 - _globals['_AUTHORITYIDENTITY']._serialized_end=25737 - _globals['_AUTHORITYSPAN']._serialized_start=25740 - _globals['_AUTHORITYSPAN']._serialized_end=25949 - _globals['_AUTHORITYGRANTREVOCATION']._serialized_start=25951 - _globals['_AUTHORITYGRANTREVOCATION']._serialized_end=26071 - _globals['_AUTHORITYREQUESTROUTINGTARGET']._serialized_start=26073 - _globals['_AUTHORITYREQUESTROUTINGTARGET']._serialized_end=26168 - _globals['_AUTHORITYREQUESTRESOURCESCOPEENTRY']._serialized_start=26170 - _globals['_AUTHORITYREQUESTRESOURCESCOPEENTRY']._serialized_end=26247 - _globals['_AUTHORITYREQUEST']._serialized_start=26250 - _globals['_AUTHORITYREQUEST']._serialized_end=27089 - _globals['_AUTHORITYREQUEST_METADATAENTRY']._serialized_start=6512 - _globals['_AUTHORITYREQUEST_METADATAENTRY']._serialized_end=6559 - _globals['_CREATEAUTHORITYREQUESTPAYLOAD']._serialized_start=27092 - _globals['_CREATEAUTHORITYREQUESTPAYLOAD']._serialized_end=27726 - _globals['_CREATEAUTHORITYREQUESTPAYLOAD_METADATAENTRY']._serialized_start=6512 - _globals['_CREATEAUTHORITYREQUESTPAYLOAD_METADATAENTRY']._serialized_end=6559 - _globals['_RESOLVEAUTHORITYREQUESTPAYLOAD']._serialized_start=27729 - _globals['_RESOLVEAUTHORITYREQUESTPAYLOAD']._serialized_end=28187 - _globals['_RESOLVEAUTHORITYREQUESTPAYLOAD_DECISION']._serialized_start=28128 - _globals['_RESOLVEAUTHORITYREQUESTPAYLOAD_DECISION']._serialized_end=28187 - _globals['_AUTHORITYREQUESTLISTFILTER']._serialized_start=28190 - _globals['_AUTHORITYREQUESTLISTFILTER']._serialized_end=28350 - _globals['_AUTHORITYREQUESTOPERATION']._serialized_start=28353 - _globals['_AUTHORITYREQUESTOPERATION']._serialized_end=28790 - _globals['_AUTHORITYREQUESTOPERATION_OPTYPE']._serialized_start=28680 - _globals['_AUTHORITYREQUESTOPERATION_OPTYPE']._serialized_end=28790 - _globals['_AUTHORITYREQUESTOPERATIONRESPONSE']._serialized_start=28793 - _globals['_AUTHORITYREQUESTOPERATIONRESPONSE']._serialized_end=29001 - _globals['_AUTHORITYREQUESTEVENT']._serialized_start=29004 - _globals['_AUTHORITYREQUESTEVENT']._serialized_end=29399 - _globals['_AUTHORITYREQUESTEVENT_EVENTTYPE']._serialized_start=29160 - _globals['_AUTHORITYREQUESTEVENT_EVENTTYPE']._serialized_end=29399 - _globals['_TOKENOPERATION']._serialized_start=29402 - _globals['_TOKENOPERATION']._serialized_end=29662 - _globals['_TOKENOPERATION_OPTYPE']._serialized_start=29599 - _globals['_TOKENOPERATION_OPTYPE']._serialized_end=29662 - _globals['_TOKENCREATEREQUEST']._serialized_start=29665 - _globals['_TOKENCREATEREQUEST']._serialized_end=29813 - _globals['_TOKENFILTER']._serialized_start=29815 - _globals['_TOKENFILTER']._serialized_end=29884 - _globals['_TOKENINFO']._serialized_start=29887 - _globals['_TOKENINFO']._serialized_end=30131 - _globals['_TOKENRESPONSE']._serialized_start=30134 - _globals['_TOKENRESPONSE']._serialized_end=30384 - _globals['_PROGRESSREPORT']._serialized_start=30387 - _globals['_PROGRESSREPORT']._serialized_end=30697 - _globals['_PROGRESSREPORT_METADATAENTRY']._serialized_start=6512 - _globals['_PROGRESSREPORT_METADATAENTRY']._serialized_end=6559 - _globals['_PROGRESSSTEP']._serialized_start=30699 - _globals['_PROGRESSSTEP']._serialized_end=30801 - _globals['_PROGRESSUPDATE']._serialized_start=30804 - _globals['_PROGRESSUPDATE']._serialized_end=31171 - _globals['_PROGRESSUPDATE_METADATAENTRY']._serialized_start=6512 - _globals['_PROGRESSUPDATE_METADATAENTRY']._serialized_end=6559 - _globals['_WORKFLOWOPERATION']._serialized_start=31174 - _globals['_WORKFLOWOPERATION']._serialized_end=31856 - _globals['_WORKFLOWOPERATION_OPTYPE']._serialized_start=31355 - _globals['_WORKFLOWOPERATION_OPTYPE']._serialized_end=31856 - _globals['_WORKFLOWRESPONSE']._serialized_start=31858 - _globals['_WORKFLOWRESPONSE']._serialized_end=31980 - _globals['_MESSAGEENVELOPE']._serialized_start=31983 - _globals['_MESSAGEENVELOPE']._serialized_end=32229 - _globals['_MESSAGEENVELOPE_METADATAENTRY']._serialized_start=6512 - _globals['_MESSAGEENVELOPE_METADATAENTRY']._serialized_end=6559 - _globals['_AUDITQUERY']._serialized_start=32232 - _globals['_AUDITQUERY']._serialized_end=32735 - _globals['_AUDITQUERYRESPONSE']._serialized_start=32738 - _globals['_AUDITQUERYRESPONSE']._serialized_end=32871 - _globals['_AUDITENTRY']._serialized_start=32874 - _globals['_AUDITENTRY']._serialized_end=33396 - _globals['_SUBMITAUDITEVENTREQUEST']._serialized_start=33399 - _globals['_SUBMITAUDITEVENTREQUEST']._serialized_end=33710 - _globals['_SUBMITAUDITEVENTREQUEST_METADATAENTRY']._serialized_start=6512 - _globals['_SUBMITAUDITEVENTREQUEST_METADATAENTRY']._serialized_end=6559 - _globals['_SUBMITAUDITEVENTRESPONSE']._serialized_start=33712 - _globals['_SUBMITAUDITEVENTRESPONSE']._serialized_end=33825 - _globals['_PROXYHTTPREQUEST']._serialized_start=33828 - _globals['_PROXYHTTPREQUEST']._serialized_end=34338 - _globals['_PROXYHTTPREQUEST_HEADERSENTRY']._serialized_start=34292 - _globals['_PROXYHTTPREQUEST_HEADERSENTRY']._serialized_end=34338 - _globals['_PROXYHTTPRESPONSE']._serialized_start=34341 - _globals['_PROXYHTTPRESPONSE']._serialized_end=34583 - _globals['_PROXYHTTPRESPONSE_HEADERSENTRY']._serialized_start=34292 - _globals['_PROXYHTTPRESPONSE_HEADERSENTRY']._serialized_end=34338 - _globals['_PROXYHTTPBODYCHUNK']._serialized_start=34585 - _globals['_PROXYHTTPBODYCHUNK']._serialized_end=34685 - _globals['_PROXYERROR']._serialized_start=34688 - _globals['_PROXYERROR']._serialized_end=34914 - _globals['_PROXYERROR_KIND']._serialized_start=34762 - _globals['_PROXYERROR_KIND']._serialized_end=34914 - _globals['_TUNNELOPEN']._serialized_start=34917 - _globals['_TUNNELOPEN']._serialized_end=35362 - _globals['_TUNNELOPEN_METADATAENTRY']._serialized_start=6512 - _globals['_TUNNELOPEN_METADATAENTRY']._serialized_end=6559 - _globals['_TUNNELOPEN_PROTOCOL']._serialized_start=35319 - _globals['_TUNNELOPEN_PROTOCOL']._serialized_end=35362 - _globals['_TUNNELDATA']._serialized_start=35364 - _globals['_TUNNELDATA']._serialized_end=35435 - _globals['_TUNNELCLOSE']._serialized_start=35438 - _globals['_TUNNELCLOSE']._serialized_end=35611 - _globals['_TUNNELCLOSE_REASON']._serialized_start=35535 - _globals['_TUNNELCLOSE_REASON']._serialized_end=35611 - _globals['_TUNNELACK']._serialized_start=35613 - _globals['_TUNNELACK']._serialized_end=35677 - _globals['_RESOLVEAUTHORITYREQUEST']._serialized_start=35680 - _globals['_RESOLVEAUTHORITYREQUEST']._serialized_end=35869 - _globals['_RESOLVEAUTHORITYRESPONSE']._serialized_start=35871 - _globals['_RESOLVEAUTHORITYRESPONSE']._serialized_end=35993 - _globals['_RESOLVEDAUTHORITY']._serialized_start=35996 - _globals['_RESOLVEDAUTHORITY']._serialized_end=36143 - _globals['_AUTHORITYGRANTINFO']._serialized_start=36146 - _globals['_AUTHORITYGRANTINFO']._serialized_end=36410 - _globals['_CONNECTIONSTATUSREQUEST']._serialized_start=36412 - _globals['_CONNECTIONSTATUSREQUEST']._serialized_end=36501 - _globals['_CONNECTIONSTATUSRESPONSE']._serialized_start=36503 - _globals['_CONNECTIONSTATUSRESPONSE']._serialized_end=36617 - _globals['_TASKSUBSCRIPTIONOPERATION']._serialized_start=36620 - _globals['_TASKSUBSCRIPTIONOPERATION']._serialized_end=36905 - _globals['_TASKSUBSCRIPTIONOPERATION_OPTYPE']._serialized_start=36827 - _globals['_TASKSUBSCRIPTIONOPERATION_OPTYPE']._serialized_end=36905 - _globals['_TASKSUBSCRIPTIONOPERATIONRESPONSE']._serialized_start=36908 - _globals['_TASKSUBSCRIPTIONOPERATIONRESPONSE']._serialized_end=37044 - _globals['_TASKEVENT']._serialized_start=37047 - _globals['_TASKEVENT']._serialized_end=37426 - _globals['_TASKSTATUSCHANGEDEVENT']._serialized_start=37428 - _globals['_TASKSTATUSCHANGEDEVENT']._serialized_end=37554 - _globals['_TASKPROGRESSEVENT']._serialized_start=37557 - _globals['_TASKPROGRESSEVENT']._serialized_end=37737 - _globals['_TASKPROGRESSEVENT_METADATAENTRY']._serialized_start=6512 - _globals['_TASKPROGRESSEVENT_METADATAENTRY']._serialized_end=6559 - _globals['_TASKCHILDLIFECYCLEEVENT']._serialized_start=37739 - _globals['_TASKCHILDLIFECYCLEEVENT']._serialized_end=37851 - _globals['_TASKAUTHORITYREQUESTEVENTRELAY']._serialized_start=37853 - _globals['_TASKAUTHORITYREQUESTEVENTRELAY']._serialized_end=37934 - _globals['_AETHERGATEWAY']._serialized_start=39784 - _globals['_AETHERGATEWAY']._serialized_end=39872 + _globals['_SERVICEIDENTITY']._serialized_end=5576 + _globals['_AGENTIDENTITY']._serialized_start=5578 + _globals['_AGENTIDENTITY']._serialized_end=5655 + _globals['_TASKIDENTITY']._serialized_start=5657 + _globals['_TASKIDENTITY']._serialized_end=5740 + _globals['_USERIDENTITY']._serialized_start=5742 + _globals['_USERIDENTITY']._serialized_end=5792 + _globals['_PRINCIPALREF']._serialized_start=5794 + _globals['_PRINCIPALREF']._serialized_end=5854 + _globals['_AUTHORIZATIONCONTEXT']._serialized_start=5857 + _globals['_AUTHORIZATIONCONTEXT']._serialized_end=6015 + _globals['_RESOLVEDAUTHORITYINFO']._serialized_start=6018 + _globals['_RESOLVEDAUTHORITYINFO']._serialized_end=6206 + _globals['_SENDMESSAGE']._serialized_start=6209 + _globals['_SENDMESSAGE']._serialized_end=6386 + _globals['_METRIC']._serialized_start=6389 + _globals['_METRIC']._serialized_end=6585 + _globals['_METRIC_METADATAENTRY']._serialized_start=6538 + _globals['_METRIC_METADATAENTRY']._serialized_end=6585 + _globals['_METRICENTRY']._serialized_start=6587 + _globals['_METRICENTRY']._serialized_end=6641 + _globals['_SWITCHWORKSPACE']._serialized_start=6643 + _globals['_SWITCHWORKSPACE']._serialized_end=6686 + _globals['_KVOPERATION']._serialized_start=6689 + _globals['_KVOPERATION']._serialized_end=7467 + _globals['_KVOPERATION_OPTYPE']._serialized_start=7068 + _globals['_KVOPERATION_OPTYPE']._serialized_end=7286 + _globals['_KVOPERATION_SCOPE']._serialized_start=7289 + _globals['_KVOPERATION_SCOPE']._serialized_end=7467 + _globals['_KVRESPONSE']._serialized_start=7470 + _globals['_KVRESPONSE']._serialized_end=7723 + _globals['_KVRESPONSE_KVMAPENTRY']._serialized_start=7679 + _globals['_KVRESPONSE_KVMAPENTRY']._serialized_end=7723 + _globals['_INCOMINGMESSAGE']._serialized_start=7726 + _globals['_INCOMINGMESSAGE']._serialized_end=7899 + _globals['_CONFIGSNAPSHOT']._serialized_start=7902 + _globals['_CONFIGSNAPSHOT']._serialized_end=8526 + _globals['_CONFIGSNAPSHOT_KVENTRY']._serialized_start=8265 + _globals['_CONFIGSNAPSHOT_KVENTRY']._serialized_end=8306 + _globals['_CONFIGSNAPSHOT_GLOBALKVENTRY']._serialized_start=8308 + _globals['_CONFIGSNAPSHOT_GLOBALKVENTRY']._serialized_end=8355 + _globals['_CONFIGSNAPSHOT_TASKCONTEXTENTRY']._serialized_start=8357 + _globals['_CONFIGSNAPSHOT_TASKCONTEXTENTRY']._serialized_end=8407 + _globals['_CONFIGSNAPSHOT_WORKSPACEEXCLUSIVEKVENTRY']._serialized_start=8409 + _globals['_CONFIGSNAPSHOT_WORKSPACEEXCLUSIVEKVENTRY']._serialized_end=8468 + _globals['_CONFIGSNAPSHOT_GLOBALEXCLUSIVEKVENTRY']._serialized_start=8470 + _globals['_CONFIGSNAPSHOT_GLOBALEXCLUSIVEKVENTRY']._serialized_end=8526 + _globals['_SIGNAL']._serialized_start=8529 + _globals['_SIGNAL']._serialized_end=8658 + _globals['_SIGNAL_SIGNALTYPE']._serialized_start=8599 + _globals['_SIGNAL_SIGNALTYPE']._serialized_end=8658 + _globals['_ERRORRESPONSE']._serialized_start=8660 + _globals['_ERRORRESPONSE']._serialized_end=8769 + _globals['_RETRYPOLICY']._serialized_start=8772 + _globals['_RETRYPOLICY']._serialized_end=9003 + _globals['_TASKCOMPLETIONEVENT']._serialized_start=9005 + _globals['_TASKCOMPLETIONEVENT']._serialized_end=9107 + _globals['_CREATETASKREQUEST']._serialized_start=9110 + _globals['_CREATETASKREQUEST']._serialized_end=9937 + _globals['_CREATETASKREQUEST_LAUNCHPARAMOVERRIDESENTRY']._serialized_start=9829 + _globals['_CREATETASKREQUEST_LAUNCHPARAMOVERRIDESENTRY']._serialized_end=9888 + _globals['_CREATETASKREQUEST_METADATAENTRY']._serialized_start=6538 + _globals['_CREATETASKREQUEST_METADATAENTRY']._serialized_end=6585 + _globals['_CREATETASKRESPONSE']._serialized_start=9940 + _globals['_CREATETASKRESPONSE']._serialized_end=10142 + _globals['_TASKASSIGNMENT']._serialized_start=10145 + _globals['_TASKASSIGNMENT']._serialized_end=10664 + _globals['_TASKASSIGNMENT_METADATAENTRY']._serialized_start=6538 + _globals['_TASKASSIGNMENT_METADATAENTRY']._serialized_end=6585 + _globals['_TASKASSIGNMENT_LAUNCHPARAMSENTRY']._serialized_start=10613 + _globals['_TASKASSIGNMENT_LAUNCHPARAMSENTRY']._serialized_end=10664 + _globals['_CHECKPOINTOPERATION']._serialized_start=10667 + _globals['_CHECKPOINTOPERATION']._serialized_end=10851 + _globals['_CHECKPOINTOPERATION_OPTYPE']._serialized_start=10801 + _globals['_CHECKPOINTOPERATION_OPTYPE']._serialized_end=10851 + _globals['_CHECKPOINTRESPONSE']._serialized_start=10853 + _globals['_CHECKPOINTRESPONSE']._serialized_end=10971 + _globals['_ADMINQUERY']._serialized_start=10974 + _globals['_ADMINQUERY']._serialized_end=11210 + _globals['_ADMINQUERY_OPTYPE']._serialized_start=11115 + _globals['_ADMINQUERY_OPTYPE']._serialized_end=11210 + _globals['_CONNECTIONFILTER']._serialized_start=11212 + _globals['_CONNECTIONFILTER']._serialized_end=11320 + _globals['_CONNECTIONINFO']._serialized_start=11323 + _globals['_CONNECTIONINFO']._serialized_end=11563 + _globals['_ADMINRESPONSE']._serialized_start=11566 + _globals['_ADMINRESPONSE']._serialized_end=11866 + _globals['_HEALTHINFO']._serialized_start=11869 + _globals['_HEALTHINFO']._serialized_end=12103 + _globals['_HEALTHINFO_CHECKSENTRY']._serialized_start=12034 + _globals['_HEALTHINFO_CHECKSENTRY']._serialized_end=12103 + _globals['_HEALTHCHECK']._serialized_start=12105 + _globals['_HEALTHCHECK']._serialized_end=12196 + _globals['_GATEWAYINFO']._serialized_start=12199 + _globals['_GATEWAYINFO']._serialized_end=12379 + _globals['_GATEWAYSTATS']._serialized_start=12382 + _globals['_GATEWAYSTATS']._serialized_end=12792 + _globals['_SESSIONOPERATION']._serialized_start=12795 + _globals['_SESSIONOPERATION']._serialized_end=13063 + _globals['_SESSIONOPERATION_OPTYPE']._serialized_start=13020 + _globals['_SESSIONOPERATION_OPTYPE']._serialized_end=13063 + _globals['_SESSIONOPERATIONRESPONSE']._serialized_start=13066 + _globals['_SESSIONOPERATIONRESPONSE']._serialized_end=13277 + _globals['_TASKQUERY']._serialized_start=13280 + _globals['_TASKQUERY']._serialized_end=13437 + _globals['_TASKQUERY_OPTYPE']._serialized_start=13410 + _globals['_TASKQUERY_OPTYPE']._serialized_end=13437 + _globals['_TASKFILTER']._serialized_start=13440 + _globals['_TASKFILTER']._serialized_end=14188 + _globals['_TASKINFO']._serialized_start=14191 + _globals['_TASKINFO']._serialized_end=15158 + _globals['_TASKINFO_METADATAENTRY']._serialized_start=6538 + _globals['_TASKINFO_METADATAENTRY']._serialized_end=6585 + _globals['_TASKQUERYRESPONSE']._serialized_start=15161 + _globals['_TASKQUERYRESPONSE']._serialized_end=15349 + _globals['_TASKOPERATION']._serialized_start=15352 + _globals['_TASKOPERATION']._serialized_end=15622 + _globals['_TASKOPERATION_OPTYPE']._serialized_start=15507 + _globals['_TASKOPERATION_OPTYPE']._serialized_end=15622 + _globals['_WAITSPEC']._serialized_start=15625 + _globals['_WAITSPEC']._serialized_end=15989 + _globals['_WAITSPEC_INPUTMATCHENTRY']._serialized_start=15940 + _globals['_WAITSPEC_INPUTMATCHENTRY']._serialized_end=15989 + _globals['_HIBERNATIONDESCRIPTOR']._serialized_start=15991 + _globals['_HIBERNATIONDESCRIPTOR']._serialized_end=16118 + _globals['_TASKOPERATIONRESPONSE']._serialized_start=16120 + _globals['_TASKOPERATIONRESPONSE']._serialized_end=16247 + _globals['_WORKSPACEOPERATION']._serialized_start=16250 + _globals['_WORKSPACEOPERATION']._serialized_end=16538 + _globals['_WORKSPACEOPERATION_OPTYPE']._serialized_start=16453 + _globals['_WORKSPACEOPERATION_OPTYPE']._serialized_end=16538 + _globals['_WORKSPACEFILTER']._serialized_start=16540 + _globals['_WORKSPACEFILTER']._serialized_end=16607 + _globals['_WORKSPACEINFO']._serialized_start=16610 + _globals['_WORKSPACEINFO']._serialized_end=16947 + _globals['_WORKSPACEINFO_METADATAENTRY']._serialized_start=6538 + _globals['_WORKSPACEINFO_METADATAENTRY']._serialized_end=6585 + _globals['_WORKSPACERESPONSE']._serialized_start=16950 + _globals['_WORKSPACERESPONSE']._serialized_end=17200 + _globals['_MESSAGEFLOWINFO']._serialized_start=17203 + _globals['_MESSAGEFLOWINFO']._serialized_end=17334 + _globals['_FLOWNODE']._serialized_start=17337 + _globals['_FLOWNODE']._serialized_end=17488 + _globals['_FLOWEDGE']._serialized_start=17490 + _globals['_FLOWEDGE']._serialized_end=17556 + _globals['_AGENTOPERATION']._serialized_start=17559 + _globals['_AGENTOPERATION']._serialized_end=17910 + _globals['_AGENTOPERATION_OPTYPE']._serialized_start=17809 + _globals['_AGENTOPERATION_OPTYPE']._serialized_end=17910 + _globals['_AGENTFILTER']._serialized_start=17912 + _globals['_AGENTFILTER']._serialized_end=17986 + _globals['_AGENTREGISTRATIONINFO']._serialized_start=17989 + _globals['_AGENTREGISTRATIONINFO']._serialized_end=18467 + _globals['_AGENTREGISTRATIONINFO_LAUNCHPARAMSENTRY']._serialized_start=10613 + _globals['_AGENTREGISTRATIONINFO_LAUNCHPARAMSENTRY']._serialized_end=10664 + _globals['_AGENTREGISTRATIONINFO_CAPABILITIESENTRY']._serialized_start=18416 + _globals['_AGENTREGISTRATIONINFO_CAPABILITIESENTRY']._serialized_end=18467 + _globals['_AGENTRESOURCESCHEMAENTRY']._serialized_start=18469 + _globals['_AGENTRESOURCESCHEMAENTRY']._serialized_end=18579 + _globals['_AGENTLAUNCHPARAMS']._serialized_start=18582 + _globals['_AGENTLAUNCHPARAMS']._serialized_end=18769 + _globals['_AGENTLAUNCHPARAMS_PARAMOVERRIDESENTRY']._serialized_start=18716 + _globals['_AGENTLAUNCHPARAMS_PARAMOVERRIDESENTRY']._serialized_end=18769 + _globals['_ORCHESTRATORINFO']._serialized_start=18771 + _globals['_ORCHESTRATORINFO']._serialized_end=18854 + _globals['_AGENTLAUNCHRESULT']._serialized_start=18856 + _globals['_AGENTLAUNCHRESULT']._serialized_end=18909 + _globals['_AGENTRESPONSE']._serialized_start=18912 + _globals['_AGENTRESPONSE']._serialized_end=19221 + _globals['_ACLOPERATION']._serialized_start=19224 + _globals['_ACLOPERATION']._serialized_end=20804 + _globals['_ACLOPERATION_OPTYPE']._serialized_start=19981 + _globals['_ACLOPERATION_OPTYPE']._serialized_end=20656 + _globals['_ACLRULEFILTER']._serialized_start=20807 + _globals['_ACLRULEFILTER']._serialized_end=20943 + _globals['_ACLAUDITFILTER']._serialized_start=20946 + _globals['_ACLAUDITFILTER']._serialized_end=21158 + _globals['_ACLGRANTREQUEST']._serialized_start=21161 + _globals['_ACLGRANTREQUEST']._serialized_end=21346 + _globals['_ACLSETFALLBACKREQUEST']._serialized_start=21348 + _globals['_ACLSETFALLBACKREQUEST']._serialized_end=21445 + _globals['_ACLAUTHORITYGRANTFILTER']._serialized_start=21448 + _globals['_ACLAUTHORITYGRANTFILTER']._serialized_end=21703 + _globals['_ACLAUTHORITYGRANTRESOURCESCOPEENTRY']._serialized_start=21705 + _globals['_ACLAUTHORITYGRANTRESOURCESCOPEENTRY']._serialized_end=21783 + _globals['_ACLAUTHORITYGRANTREQUEST']._serialized_start=21786 + _globals['_ACLAUTHORITYGRANTREQUEST']._serialized_end=22467 + _globals['_ACLAUTHORITYGRANTREQUEST_METADATAENTRY']._serialized_start=6538 + _globals['_ACLAUTHORITYGRANTREQUEST_METADATAENTRY']._serialized_end=6585 + _globals['_ACLRENEWAUTHORITYGRANTREQUEST']._serialized_start=22469 + _globals['_ACLRENEWAUTHORITYGRANTREQUEST']._serialized_end=22562 + _globals['_ACLRULEINFO']._serialized_start=22565 + _globals['_ACLRULEINFO']._serialized_end=22810 + _globals['_ACLFALLBACKPOLICYINFO']._serialized_start=22813 + _globals['_ACLFALLBACKPOLICYINFO']._serialized_end=22985 + _globals['_ACLAUDITENTRYINFO']._serialized_start=22988 + _globals['_ACLAUDITENTRYINFO']._serialized_end=23439 + _globals['_ACLAUDITENTRYINFO_METADATAENTRY']._serialized_start=6538 + _globals['_ACLAUDITENTRYINFO_METADATAENTRY']._serialized_end=6585 + _globals['_ACLAUTHORITYGRANTINFO']._serialized_start=23442 + _globals['_ACLAUTHORITYGRANTINFO']._serialized_end=24262 + _globals['_ACLAUTHORITYGRANTINFO_METADATAENTRY']._serialized_start=6538 + _globals['_ACLAUTHORITYGRANTINFO_METADATAENTRY']._serialized_end=6585 + _globals['_ACLCLEANUPRESULT']._serialized_start=24264 + _globals['_ACLCLEANUPRESULT']._serialized_end=24322 + _globals['_ACLGROUPREQUEST']._serialized_start=24325 + _globals['_ACLGROUPREQUEST']._serialized_end=24506 + _globals['_ACLGROUPREQUEST_METADATAENTRY']._serialized_start=6538 + _globals['_ACLGROUPREQUEST_METADATAENTRY']._serialized_end=6585 + _globals['_ACLROLEREQUEST']._serialized_start=24509 + _globals['_ACLROLEREQUEST']._serialized_end=24688 + _globals['_ACLROLEREQUEST_METADATAENTRY']._serialized_start=6538 + _globals['_ACLROLEREQUEST_METADATAENTRY']._serialized_end=6585 + _globals['_ACLGROUPMEMBERREQUEST']._serialized_start=24690 + _globals['_ACLGROUPMEMBERREQUEST']._serialized_end=24793 + _globals['_ACLROLEASSIGNMENTREQUEST']._serialized_start=24795 + _globals['_ACLROLEASSIGNMENTREQUEST']._serialized_end=24905 + _globals['_ACLGROUPINFO']._serialized_start=24908 + _globals['_ACLGROUPINFO']._serialized_end=25127 + _globals['_ACLGROUPINFO_METADATAENTRY']._serialized_start=6538 + _globals['_ACLGROUPINFO_METADATAENTRY']._serialized_end=6585 + _globals['_ACLROLEINFO']._serialized_start=25130 + _globals['_ACLROLEINFO']._serialized_end=25345 + _globals['_ACLROLEINFO_METADATAENTRY']._serialized_start=6538 + _globals['_ACLROLEINFO_METADATAENTRY']._serialized_end=6585 + _globals['_ACLGROUPMEMBERINFO']._serialized_start=25348 + _globals['_ACLGROUPMEMBERINFO']._serialized_end=25488 + _globals['_ACLROLEASSIGNMENTINFO']._serialized_start=25491 + _globals['_ACLROLEASSIGNMENTINFO']._serialized_end=25637 + _globals['_ACLACCESSCONTRIBUTIONINFO']._serialized_start=25639 + _globals['_ACLACCESSCONTRIBUTIONINFO']._serialized_end=25757 + _globals['_ACLACCESSEXPLANATIONINFO']._serialized_start=25760 + _globals['_ACLACCESSEXPLANATIONINFO']._serialized_end=25993 + _globals['_ACLRESPONSE']._serialized_start=25996 + _globals['_ACLRESPONSE']._serialized_end=26857 + _globals['_AUTHORITYGRANTOPERATION']._serialized_start=26860 + _globals['_AUTHORITYGRANTOPERATION']._serialized_end=27553 + _globals['_AUTHORITYGRANTOPERATION_OPTYPE']._serialized_start=27401 + _globals['_AUTHORITYGRANTOPERATION_OPTYPE']._serialized_end=27553 + _globals['_AUTHORITYGRANTEXCHANGEREQUEST']._serialized_start=27556 + _globals['_AUTHORITYGRANTEXCHANGEREQUEST']._serialized_end=28073 + _globals['_AUTHORITYGRANTEXCHANGEREQUEST_METADATAENTRY']._serialized_start=6538 + _globals['_AUTHORITYGRANTEXCHANGEREQUEST_METADATAENTRY']._serialized_end=6585 + _globals['_AUTHORITYGRANTDERIVEREQUEST']._serialized_start=28076 + _globals['_AUTHORITYGRANTDERIVEREQUEST']._serialized_end=28630 + _globals['_AUTHORITYGRANTDERIVEREQUEST_METADATAENTRY']._serialized_start=6538 + _globals['_AUTHORITYGRANTDERIVEREQUEST_METADATAENTRY']._serialized_end=6585 + _globals['_AUTHORITYGRANTRESPONSE']._serialized_start=28633 + _globals['_AUTHORITYGRANTRESPONSE']._serialized_end=28872 + _globals['_AUTHORITYGRANTLISTREQUEST']._serialized_start=28874 + _globals['_AUTHORITYGRANTLISTREQUEST']._serialized_end=29001 + _globals['_AUTHORITYGRANTBATCHEXCHANGEREQUEST']._serialized_start=29003 + _globals['_AUTHORITYGRANTBATCHEXCHANGEREQUEST']._serialized_end=29128 + _globals['_AUTHORITYGRANTDERIVEFORTARGETREQUEST']._serialized_start=29131 + _globals['_AUTHORITYGRANTDERIVEFORTARGETREQUEST']._serialized_end=29437 + _globals['_AUTHORITYIDENTITY']._serialized_start=29440 + _globals['_AUTHORITYIDENTITY']._serialized_end=29635 + _globals['_AUTHORITYSPAN']._serialized_start=29638 + _globals['_AUTHORITYSPAN']._serialized_end=29847 + _globals['_AUTHORITYGRANTREVOCATION']._serialized_start=29849 + _globals['_AUTHORITYGRANTREVOCATION']._serialized_end=29969 + _globals['_AUTHORITYREQUESTROUTINGTARGET']._serialized_start=29971 + _globals['_AUTHORITYREQUESTROUTINGTARGET']._serialized_end=30066 + _globals['_AUTHORITYREQUESTRESOURCESCOPEENTRY']._serialized_start=30068 + _globals['_AUTHORITYREQUESTRESOURCESCOPEENTRY']._serialized_end=30145 + _globals['_AUTHORITYREQUEST']._serialized_start=30148 + _globals['_AUTHORITYREQUEST']._serialized_end=30987 + _globals['_AUTHORITYREQUEST_METADATAENTRY']._serialized_start=6538 + _globals['_AUTHORITYREQUEST_METADATAENTRY']._serialized_end=6585 + _globals['_CREATEAUTHORITYREQUESTPAYLOAD']._serialized_start=30990 + _globals['_CREATEAUTHORITYREQUESTPAYLOAD']._serialized_end=31624 + _globals['_CREATEAUTHORITYREQUESTPAYLOAD_METADATAENTRY']._serialized_start=6538 + _globals['_CREATEAUTHORITYREQUESTPAYLOAD_METADATAENTRY']._serialized_end=6585 + _globals['_RESOLVEAUTHORITYREQUESTPAYLOAD']._serialized_start=31627 + _globals['_RESOLVEAUTHORITYREQUESTPAYLOAD']._serialized_end=32085 + _globals['_RESOLVEAUTHORITYREQUESTPAYLOAD_DECISION']._serialized_start=32026 + _globals['_RESOLVEAUTHORITYREQUESTPAYLOAD_DECISION']._serialized_end=32085 + _globals['_AUTHORITYREQUESTLISTFILTER']._serialized_start=32088 + _globals['_AUTHORITYREQUESTLISTFILTER']._serialized_end=32248 + _globals['_AUTHORITYREQUESTOPERATION']._serialized_start=32251 + _globals['_AUTHORITYREQUESTOPERATION']._serialized_end=32688 + _globals['_AUTHORITYREQUESTOPERATION_OPTYPE']._serialized_start=32578 + _globals['_AUTHORITYREQUESTOPERATION_OPTYPE']._serialized_end=32688 + _globals['_AUTHORITYREQUESTOPERATIONRESPONSE']._serialized_start=32691 + _globals['_AUTHORITYREQUESTOPERATIONRESPONSE']._serialized_end=32899 + _globals['_AUTHORITYREQUESTEVENT']._serialized_start=32902 + _globals['_AUTHORITYREQUESTEVENT']._serialized_end=33297 + _globals['_AUTHORITYREQUESTEVENT_EVENTTYPE']._serialized_start=33058 + _globals['_AUTHORITYREQUESTEVENT_EVENTTYPE']._serialized_end=33297 + _globals['_TOKENOPERATION']._serialized_start=33300 + _globals['_TOKENOPERATION']._serialized_end=33560 + _globals['_TOKENOPERATION_OPTYPE']._serialized_start=33497 + _globals['_TOKENOPERATION_OPTYPE']._serialized_end=33560 + _globals['_TOKENCREATEREQUEST']._serialized_start=33563 + _globals['_TOKENCREATEREQUEST']._serialized_end=33711 + _globals['_TOKENFILTER']._serialized_start=33713 + _globals['_TOKENFILTER']._serialized_end=33782 + _globals['_TOKENINFO']._serialized_start=33785 + _globals['_TOKENINFO']._serialized_end=34029 + _globals['_TOKENRESPONSE']._serialized_start=34032 + _globals['_TOKENRESPONSE']._serialized_end=34282 + _globals['_PROGRESSREPORT']._serialized_start=34285 + _globals['_PROGRESSREPORT']._serialized_end=34595 + _globals['_PROGRESSREPORT_METADATAENTRY']._serialized_start=6538 + _globals['_PROGRESSREPORT_METADATAENTRY']._serialized_end=6585 + _globals['_PROGRESSSTEP']._serialized_start=34597 + _globals['_PROGRESSSTEP']._serialized_end=34699 + _globals['_PROGRESSUPDATE']._serialized_start=34702 + _globals['_PROGRESSUPDATE']._serialized_end=35069 + _globals['_PROGRESSUPDATE_METADATAENTRY']._serialized_start=6538 + _globals['_PROGRESSUPDATE_METADATAENTRY']._serialized_end=6585 + _globals['_WORKFLOWOPERATION']._serialized_start=35072 + _globals['_WORKFLOWOPERATION']._serialized_end=35801 + _globals['_WORKFLOWOPERATION_OPTYPE']._serialized_start=35253 + _globals['_WORKFLOWOPERATION_OPTYPE']._serialized_end=35801 + _globals['_WORKFLOWRESPONSE']._serialized_start=35803 + _globals['_WORKFLOWRESPONSE']._serialized_end=35925 + _globals['_MESSAGEENVELOPE']._serialized_start=35928 + _globals['_MESSAGEENVELOPE']._serialized_end=36226 + _globals['_MESSAGEENVELOPE_METADATAENTRY']._serialized_start=6538 + _globals['_MESSAGEENVELOPE_METADATAENTRY']._serialized_end=6585 + _globals['_AUDITQUERY']._serialized_start=36229 + _globals['_AUDITQUERY']._serialized_end=36732 + _globals['_AUDITQUERYRESPONSE']._serialized_start=36735 + _globals['_AUDITQUERYRESPONSE']._serialized_end=36868 + _globals['_AUDITENTRY']._serialized_start=36871 + _globals['_AUDITENTRY']._serialized_end=37393 + _globals['_SUBMITAUDITEVENTREQUEST']._serialized_start=37396 + _globals['_SUBMITAUDITEVENTREQUEST']._serialized_end=37707 + _globals['_SUBMITAUDITEVENTREQUEST_METADATAENTRY']._serialized_start=6538 + _globals['_SUBMITAUDITEVENTREQUEST_METADATAENTRY']._serialized_end=6585 + _globals['_SUBMITAUDITEVENTRESPONSE']._serialized_start=37709 + _globals['_SUBMITAUDITEVENTRESPONSE']._serialized_end=37822 + _globals['_PROXYHTTPREQUEST']._serialized_start=37825 + _globals['_PROXYHTTPREQUEST']._serialized_end=38335 + _globals['_PROXYHTTPREQUEST_HEADERSENTRY']._serialized_start=38289 + _globals['_PROXYHTTPREQUEST_HEADERSENTRY']._serialized_end=38335 + _globals['_PROXYHTTPRESPONSE']._serialized_start=38338 + _globals['_PROXYHTTPRESPONSE']._serialized_end=38580 + _globals['_PROXYHTTPRESPONSE_HEADERSENTRY']._serialized_start=38289 + _globals['_PROXYHTTPRESPONSE_HEADERSENTRY']._serialized_end=38335 + _globals['_PROXYHTTPBODYCHUNK']._serialized_start=38582 + _globals['_PROXYHTTPBODYCHUNK']._serialized_end=38682 + _globals['_PROXYERROR']._serialized_start=38685 + _globals['_PROXYERROR']._serialized_end=38911 + _globals['_PROXYERROR_KIND']._serialized_start=38759 + _globals['_PROXYERROR_KIND']._serialized_end=38911 + _globals['_TUNNELOPEN']._serialized_start=38914 + _globals['_TUNNELOPEN']._serialized_end=39359 + _globals['_TUNNELOPEN_METADATAENTRY']._serialized_start=6538 + _globals['_TUNNELOPEN_METADATAENTRY']._serialized_end=6585 + _globals['_TUNNELOPEN_PROTOCOL']._serialized_start=39316 + _globals['_TUNNELOPEN_PROTOCOL']._serialized_end=39359 + _globals['_TUNNELDATA']._serialized_start=39361 + _globals['_TUNNELDATA']._serialized_end=39432 + _globals['_TUNNELCLOSE']._serialized_start=39435 + _globals['_TUNNELCLOSE']._serialized_end=39608 + _globals['_TUNNELCLOSE_REASON']._serialized_start=39532 + _globals['_TUNNELCLOSE_REASON']._serialized_end=39608 + _globals['_TUNNELACK']._serialized_start=39610 + _globals['_TUNNELACK']._serialized_end=39674 + _globals['_RESOLVEAUTHORITYREQUEST']._serialized_start=39677 + _globals['_RESOLVEAUTHORITYREQUEST']._serialized_end=39866 + _globals['_RESOLVEAUTHORITYRESPONSE']._serialized_start=39868 + _globals['_RESOLVEAUTHORITYRESPONSE']._serialized_end=39990 + _globals['_RESOLVEDAUTHORITY']._serialized_start=39993 + _globals['_RESOLVEDAUTHORITY']._serialized_end=40140 + _globals['_AUTHORITYGRANTINFO']._serialized_start=40143 + _globals['_AUTHORITYGRANTINFO']._serialized_end=40407 + _globals['_CONNECTIONSTATUSREQUEST']._serialized_start=40409 + _globals['_CONNECTIONSTATUSREQUEST']._serialized_end=40498 + _globals['_CONNECTIONSTATUSRESPONSE']._serialized_start=40500 + _globals['_CONNECTIONSTATUSRESPONSE']._serialized_end=40614 + _globals['_TASKSUBSCRIPTIONOPERATION']._serialized_start=40617 + _globals['_TASKSUBSCRIPTIONOPERATION']._serialized_end=40902 + _globals['_TASKSUBSCRIPTIONOPERATION_OPTYPE']._serialized_start=40824 + _globals['_TASKSUBSCRIPTIONOPERATION_OPTYPE']._serialized_end=40902 + _globals['_TASKSUBSCRIPTIONOPERATIONRESPONSE']._serialized_start=40905 + _globals['_TASKSUBSCRIPTIONOPERATIONRESPONSE']._serialized_end=41041 + _globals['_TASKEVENT']._serialized_start=41044 + _globals['_TASKEVENT']._serialized_end=41423 + _globals['_TASKSTATUSCHANGEDEVENT']._serialized_start=41425 + _globals['_TASKSTATUSCHANGEDEVENT']._serialized_end=41551 + _globals['_TASKPROGRESSEVENT']._serialized_start=41554 + _globals['_TASKPROGRESSEVENT']._serialized_end=41734 + _globals['_TASKPROGRESSEVENT_METADATAENTRY']._serialized_start=6538 + _globals['_TASKPROGRESSEVENT_METADATAENTRY']._serialized_end=6585 + _globals['_TASKCHILDLIFECYCLEEVENT']._serialized_start=41736 + _globals['_TASKCHILDLIFECYCLEEVENT']._serialized_end=41848 + _globals['_TASKAUTHORITYREQUESTEVENTRELAY']._serialized_start=41850 + _globals['_TASKAUTHORITYREQUESTEVENTRELAY']._serialized_end=41931 + _globals['_AETHERGATEWAY']._serialized_start=44109 + _globals['_AETHERGATEWAY']._serialized_end=44197 # @@protoc_insertion_point(module_scope) diff --git a/sdk/python-client/scitrera_aether_client/proto/aether_pb2.pyi b/sdk/python-client/scitrera_aether_client/proto/aether_pb2.pyi index a89fb0f..1ca82b9 100644 --- a/sdk/python-client/scitrera_aether_client/proto/aether_pb2.pyi +++ b/sdk/python-client/scitrera_aether_client/proto/aether_pb2.pyi @@ -79,6 +79,22 @@ class TaskClass(int, metaclass=_enum_type_wrapper.EnumTypeWrapper): TASK_CLASS_BACKGROUND: _ClassVar[TaskClass] TASK_CLASS_BATCH: _ClassVar[TaskClass] +class TaskPriority(int, metaclass=_enum_type_wrapper.EnumTypeWrapper): + __slots__ = () + TASK_PRIORITY_UNSPECIFIED: _ClassVar[TaskPriority] + TASK_PRIORITY_XLOW: _ClassVar[TaskPriority] + TASK_PRIORITY_LOW: _ClassVar[TaskPriority] + TASK_PRIORITY_NORMAL: _ClassVar[TaskPriority] + TASK_PRIORITY_HIGH: _ClassVar[TaskPriority] + TASK_PRIORITY_PREEMPT: _ClassVar[TaskPriority] + +class BackoffStrategy(int, metaclass=_enum_type_wrapper.EnumTypeWrapper): + __slots__ = () + BACKOFF_STRATEGY_UNSPECIFIED: _ClassVar[BackoffStrategy] + BACKOFF_STRATEGY_FIXED: _ClassVar[BackoffStrategy] + BACKOFF_STRATEGY_EXPONENTIAL: _ClassVar[BackoffStrategy] + BACKOFF_STRATEGY_EXPLICIT_SCHEDULE: _ClassVar[BackoffStrategy] + class WaitReason(int, metaclass=_enum_type_wrapper.EnumTypeWrapper): __slots__ = () WAIT_REASON_UNSPECIFIED: _ClassVar[WaitReason] @@ -150,6 +166,16 @@ TASK_CLASS_UNSPECIFIED: TaskClass TASK_CLASS_INTERACTIVE: TaskClass TASK_CLASS_BACKGROUND: TaskClass TASK_CLASS_BATCH: TaskClass +TASK_PRIORITY_UNSPECIFIED: TaskPriority +TASK_PRIORITY_XLOW: TaskPriority +TASK_PRIORITY_LOW: TaskPriority +TASK_PRIORITY_NORMAL: TaskPriority +TASK_PRIORITY_HIGH: TaskPriority +TASK_PRIORITY_PREEMPT: TaskPriority +BACKOFF_STRATEGY_UNSPECIFIED: BackoffStrategy +BACKOFF_STRATEGY_FIXED: BackoffStrategy +BACKOFF_STRATEGY_EXPONENTIAL: BackoffStrategy +BACKOFF_STRATEGY_EXPLICIT_SCHEDULE: BackoffStrategy WAIT_REASON_UNSPECIFIED: WaitReason WAIT_REASON_INPUT: WaitReason WAIT_REASON_AUTHORITY: WaitReason @@ -450,12 +476,14 @@ class BridgeIdentity(_message.Message): def __init__(self, implementation: _Optional[str] = ..., specifier: _Optional[str] = ...) -> None: ... class ServiceIdentity(_message.Message): - __slots__ = ("implementation", "specifier") + __slots__ = ("implementation", "specifier", "no_pool_consumer") IMPLEMENTATION_FIELD_NUMBER: _ClassVar[int] SPECIFIER_FIELD_NUMBER: _ClassVar[int] + NO_POOL_CONSUMER_FIELD_NUMBER: _ClassVar[int] implementation: str specifier: str - def __init__(self, implementation: _Optional[str] = ..., specifier: _Optional[str] = ...) -> None: ... + no_pool_consumer: bool + def __init__(self, implementation: _Optional[str] = ..., specifier: _Optional[str] = ..., no_pool_consumer: bool = ...) -> None: ... class AgentIdentity(_message.Message): __slots__ = ("workspace", "implementation", "specifier") @@ -571,7 +599,7 @@ class SwitchWorkspace(_message.Message): def __init__(self, new_workspace_id: _Optional[str] = ...) -> None: ... class KVOperation(_message.Message): - __slots__ = ("op", "scope", "key", "value", "user_id", "workspace", "ttl", "request_id", "authorization", "guard_value", "delta_value") + __slots__ = ("op", "scope", "key", "value", "user_id", "workspace", "ttl", "request_id", "authorization", "guard_value", "delta_value", "expected_value", "limit", "cursor", "target_identity") class OpType(int, metaclass=_enum_type_wrapper.EnumTypeWrapper): __slots__ = () GET: _ClassVar[KVOperation.OpType] @@ -582,6 +610,12 @@ class KVOperation(_message.Message): DECREMENT: _ClassVar[KVOperation.OpType] INCREMENT_IF: _ClassVar[KVOperation.OpType] DECREMENT_IF: _ClassVar[KVOperation.OpType] + SET_NX: _ClassVar[KVOperation.OpType] + COMPARE_AND_SET: _ClassVar[KVOperation.OpType] + COMPARE_AND_DELETE: _ClassVar[KVOperation.OpType] + PURGE_IDENTITY: _ClassVar[KVOperation.OpType] + SET_ADD: _ClassVar[KVOperation.OpType] + SET_CARD: _ClassVar[KVOperation.OpType] GET: KVOperation.OpType PUT: KVOperation.OpType LIST: KVOperation.OpType @@ -590,6 +624,12 @@ class KVOperation(_message.Message): DECREMENT: KVOperation.OpType INCREMENT_IF: KVOperation.OpType DECREMENT_IF: KVOperation.OpType + SET_NX: KVOperation.OpType + COMPARE_AND_SET: KVOperation.OpType + COMPARE_AND_DELETE: KVOperation.OpType + PURGE_IDENTITY: KVOperation.OpType + SET_ADD: KVOperation.OpType + SET_CARD: KVOperation.OpType class Scope(int, metaclass=_enum_type_wrapper.EnumTypeWrapper): __slots__ = () SCOPE_UNSPECIFIED: _ClassVar[KVOperation.Scope] @@ -621,6 +661,10 @@ class KVOperation(_message.Message): AUTHORIZATION_FIELD_NUMBER: _ClassVar[int] GUARD_VALUE_FIELD_NUMBER: _ClassVar[int] DELTA_VALUE_FIELD_NUMBER: _ClassVar[int] + EXPECTED_VALUE_FIELD_NUMBER: _ClassVar[int] + LIMIT_FIELD_NUMBER: _ClassVar[int] + CURSOR_FIELD_NUMBER: _ClassVar[int] + TARGET_IDENTITY_FIELD_NUMBER: _ClassVar[int] op: KVOperation.OpType scope: KVOperation.Scope key: str @@ -632,10 +676,14 @@ class KVOperation(_message.Message): authorization: AuthorizationContext guard_value: int delta_value: int - def __init__(self, op: _Optional[_Union[KVOperation.OpType, str]] = ..., scope: _Optional[_Union[KVOperation.Scope, str]] = ..., key: _Optional[str] = ..., value: _Optional[bytes] = ..., user_id: _Optional[str] = ..., workspace: _Optional[str] = ..., ttl: _Optional[int] = ..., request_id: _Optional[str] = ..., authorization: _Optional[_Union[AuthorizationContext, _Mapping]] = ..., guard_value: _Optional[int] = ..., delta_value: _Optional[int] = ...) -> None: ... + expected_value: bytes + limit: int + cursor: str + target_identity: str + def __init__(self, op: _Optional[_Union[KVOperation.OpType, str]] = ..., scope: _Optional[_Union[KVOperation.Scope, str]] = ..., key: _Optional[str] = ..., value: _Optional[bytes] = ..., user_id: _Optional[str] = ..., workspace: _Optional[str] = ..., ttl: _Optional[int] = ..., request_id: _Optional[str] = ..., authorization: _Optional[_Union[AuthorizationContext, _Mapping]] = ..., guard_value: _Optional[int] = ..., delta_value: _Optional[int] = ..., expected_value: _Optional[bytes] = ..., limit: _Optional[int] = ..., cursor: _Optional[str] = ..., target_identity: _Optional[str] = ...) -> None: ... class KVResponse(_message.Message): - __slots__ = ("success", "value", "keys", "kv_map", "request_id", "counter_value", "applied") + __slots__ = ("success", "value", "keys", "kv_map", "request_id", "counter_value", "applied", "next_cursor", "has_more") class KvMapEntry(_message.Message): __slots__ = ("key", "value") KEY_FIELD_NUMBER: _ClassVar[int] @@ -650,6 +698,8 @@ class KVResponse(_message.Message): REQUEST_ID_FIELD_NUMBER: _ClassVar[int] COUNTER_VALUE_FIELD_NUMBER: _ClassVar[int] APPLIED_FIELD_NUMBER: _ClassVar[int] + NEXT_CURSOR_FIELD_NUMBER: _ClassVar[int] + HAS_MORE_FIELD_NUMBER: _ClassVar[int] success: bool value: bytes keys: _containers.RepeatedScalarFieldContainer[str] @@ -657,19 +707,23 @@ class KVResponse(_message.Message): request_id: str counter_value: int applied: bool - def __init__(self, success: bool = ..., value: _Optional[bytes] = ..., keys: _Optional[_Iterable[str]] = ..., kv_map: _Optional[_Mapping[str, bytes]] = ..., request_id: _Optional[str] = ..., counter_value: _Optional[int] = ..., applied: bool = ...) -> None: ... + next_cursor: str + has_more: bool + def __init__(self, success: bool = ..., value: _Optional[bytes] = ..., keys: _Optional[_Iterable[str]] = ..., kv_map: _Optional[_Mapping[str, bytes]] = ..., request_id: _Optional[str] = ..., counter_value: _Optional[int] = ..., applied: bool = ..., next_cursor: _Optional[str] = ..., has_more: bool = ...) -> None: ... class IncomingMessage(_message.Message): - __slots__ = ("source_topic", "payload", "message_type", "workspace") + __slots__ = ("source_topic", "payload", "message_type", "workspace", "on_behalf_subject") SOURCE_TOPIC_FIELD_NUMBER: _ClassVar[int] PAYLOAD_FIELD_NUMBER: _ClassVar[int] MESSAGE_TYPE_FIELD_NUMBER: _ClassVar[int] WORKSPACE_FIELD_NUMBER: _ClassVar[int] + ON_BEHALF_SUBJECT_FIELD_NUMBER: _ClassVar[int] source_topic: str payload: bytes message_type: MessageType workspace: str - def __init__(self, source_topic: _Optional[str] = ..., payload: _Optional[bytes] = ..., message_type: _Optional[_Union[MessageType, str]] = ..., workspace: _Optional[str] = ...) -> None: ... + on_behalf_subject: PrincipalRef + def __init__(self, source_topic: _Optional[str] = ..., payload: _Optional[bytes] = ..., message_type: _Optional[_Union[MessageType, str]] = ..., workspace: _Optional[str] = ..., on_behalf_subject: _Optional[_Union[PrincipalRef, _Mapping]] = ...) -> None: ... class ConfigSnapshot(_message.Message): __slots__ = ("kv", "global_kv", "task_context", "workspace_exclusive_kv", "global_exclusive_kv") @@ -748,8 +802,38 @@ class ErrorResponse(_message.Message): request_id: str def __init__(self, code: _Optional[str] = ..., message: _Optional[str] = ..., retryable: bool = ..., retry_after_ms: _Optional[int] = ..., request_id: _Optional[str] = ...) -> None: ... +class RetryPolicy(_message.Message): + __slots__ = ("max_attempts", "backoff", "initial_delay_ms", "max_delay_ms", "jitter_factor", "schedule_ms", "retryable_status_codes", "honor_retry_after") + MAX_ATTEMPTS_FIELD_NUMBER: _ClassVar[int] + BACKOFF_FIELD_NUMBER: _ClassVar[int] + INITIAL_DELAY_MS_FIELD_NUMBER: _ClassVar[int] + MAX_DELAY_MS_FIELD_NUMBER: _ClassVar[int] + JITTER_FACTOR_FIELD_NUMBER: _ClassVar[int] + SCHEDULE_MS_FIELD_NUMBER: _ClassVar[int] + RETRYABLE_STATUS_CODES_FIELD_NUMBER: _ClassVar[int] + HONOR_RETRY_AFTER_FIELD_NUMBER: _ClassVar[int] + max_attempts: int + backoff: BackoffStrategy + initial_delay_ms: int + max_delay_ms: int + jitter_factor: float + schedule_ms: _containers.RepeatedScalarFieldContainer[int] + retryable_status_codes: _containers.RepeatedScalarFieldContainer[int] + honor_retry_after: bool + def __init__(self, max_attempts: _Optional[int] = ..., backoff: _Optional[_Union[BackoffStrategy, str]] = ..., initial_delay_ms: _Optional[int] = ..., max_delay_ms: _Optional[int] = ..., jitter_factor: _Optional[float] = ..., schedule_ms: _Optional[_Iterable[int]] = ..., retryable_status_codes: _Optional[_Iterable[int]] = ..., honor_retry_after: bool = ...) -> None: ... + +class TaskCompletionEvent(_message.Message): + __slots__ = ("enabled", "event_name", "on_statuses") + ENABLED_FIELD_NUMBER: _ClassVar[int] + EVENT_NAME_FIELD_NUMBER: _ClassVar[int] + ON_STATUSES_FIELD_NUMBER: _ClassVar[int] + enabled: bool + event_name: str + on_statuses: _containers.RepeatedScalarFieldContainer[TaskStatus] + def __init__(self, enabled: bool = ..., event_name: _Optional[str] = ..., on_statuses: _Optional[_Iterable[_Union[TaskStatus, str]]] = ...) -> None: ... + class CreateTaskRequest(_message.Message): - __slots__ = ("task_type", "workspace", "assignment_mode", "target_agent_id", "launch_param_overrides", "metadata", "payload", "target_implementation", "authorization", "request_id", "target_identity", "task_class", "context_id") + __slots__ = ("task_type", "workspace", "assignment_mode", "target_agent_id", "launch_param_overrides", "metadata", "payload", "target_implementation", "authorization", "request_id", "target_identity", "task_class", "context_id", "retry_policy", "priority", "idempotency_key", "correlation_id", "root_task_id", "completion_event") class LaunchParamOverridesEntry(_message.Message): __slots__ = ("key", "value") KEY_FIELD_NUMBER: _ClassVar[int] @@ -777,6 +861,12 @@ class CreateTaskRequest(_message.Message): TARGET_IDENTITY_FIELD_NUMBER: _ClassVar[int] TASK_CLASS_FIELD_NUMBER: _ClassVar[int] CONTEXT_ID_FIELD_NUMBER: _ClassVar[int] + RETRY_POLICY_FIELD_NUMBER: _ClassVar[int] + PRIORITY_FIELD_NUMBER: _ClassVar[int] + IDEMPOTENCY_KEY_FIELD_NUMBER: _ClassVar[int] + CORRELATION_ID_FIELD_NUMBER: _ClassVar[int] + ROOT_TASK_ID_FIELD_NUMBER: _ClassVar[int] + COMPLETION_EVENT_FIELD_NUMBER: _ClassVar[int] task_type: str workspace: str assignment_mode: TaskAssignmentMode @@ -790,7 +880,13 @@ class CreateTaskRequest(_message.Message): target_identity: str task_class: TaskClass context_id: str - def __init__(self, task_type: _Optional[str] = ..., workspace: _Optional[str] = ..., assignment_mode: _Optional[_Union[TaskAssignmentMode, str]] = ..., target_agent_id: _Optional[str] = ..., launch_param_overrides: _Optional[_Mapping[str, str]] = ..., metadata: _Optional[_Mapping[str, str]] = ..., payload: _Optional[bytes] = ..., target_implementation: _Optional[str] = ..., authorization: _Optional[_Union[AuthorizationContext, _Mapping]] = ..., request_id: _Optional[str] = ..., target_identity: _Optional[str] = ..., task_class: _Optional[_Union[TaskClass, str]] = ..., context_id: _Optional[str] = ...) -> None: ... + retry_policy: RetryPolicy + priority: TaskPriority + idempotency_key: str + correlation_id: str + root_task_id: str + completion_event: TaskCompletionEvent + def __init__(self, task_type: _Optional[str] = ..., workspace: _Optional[str] = ..., assignment_mode: _Optional[_Union[TaskAssignmentMode, str]] = ..., target_agent_id: _Optional[str] = ..., launch_param_overrides: _Optional[_Mapping[str, str]] = ..., metadata: _Optional[_Mapping[str, str]] = ..., payload: _Optional[bytes] = ..., target_implementation: _Optional[str] = ..., authorization: _Optional[_Union[AuthorizationContext, _Mapping]] = ..., request_id: _Optional[str] = ..., target_identity: _Optional[str] = ..., task_class: _Optional[_Union[TaskClass, str]] = ..., context_id: _Optional[str] = ..., retry_policy: _Optional[_Union[RetryPolicy, _Mapping]] = ..., priority: _Optional[_Union[TaskPriority, str]] = ..., idempotency_key: _Optional[str] = ..., correlation_id: _Optional[str] = ..., root_task_id: _Optional[str] = ..., completion_event: _Optional[_Union[TaskCompletionEvent, _Mapping]] = ...) -> None: ... class CreateTaskResponse(_message.Message): __slots__ = ("success", "task_id", "status", "error_code", "error_message", "request_id", "assigned_to", "task_token", "authority_grant_id") @@ -1126,7 +1222,7 @@ class TaskQuery(_message.Message): def __init__(self, op: _Optional[_Union[TaskQuery.OpType, str]] = ..., task_id: _Optional[str] = ..., filter: _Optional[_Union[TaskFilter, _Mapping]] = ..., request_id: _Optional[str] = ...) -> None: ... class TaskFilter(_message.Message): - __slots__ = ("status", "workspace", "task_type", "limit", "offset", "statuses", "subject_type", "subject_id", "authority_mode", "authority_grant_id", "root_authority_grant_id", "parent_task_id", "task_class", "exclude_task_classes", "context_id", "exclude_statuses", "creator_actor", "status_timestamp_after_unix_ms", "page_token", "include_descendants") + __slots__ = ("status", "workspace", "task_type", "limit", "offset", "statuses", "subject_type", "subject_id", "authority_mode", "authority_grant_id", "root_authority_grant_id", "parent_task_id", "task_class", "exclude_task_classes", "context_id", "exclude_statuses", "creator_actor", "status_timestamp_after_unix_ms", "page_token", "include_descendants", "priority", "min_priority", "correlation_id", "root_task_id") STATUS_FIELD_NUMBER: _ClassVar[int] WORKSPACE_FIELD_NUMBER: _ClassVar[int] TASK_TYPE_FIELD_NUMBER: _ClassVar[int] @@ -1147,6 +1243,10 @@ class TaskFilter(_message.Message): STATUS_TIMESTAMP_AFTER_UNIX_MS_FIELD_NUMBER: _ClassVar[int] PAGE_TOKEN_FIELD_NUMBER: _ClassVar[int] INCLUDE_DESCENDANTS_FIELD_NUMBER: _ClassVar[int] + PRIORITY_FIELD_NUMBER: _ClassVar[int] + MIN_PRIORITY_FIELD_NUMBER: _ClassVar[int] + CORRELATION_ID_FIELD_NUMBER: _ClassVar[int] + ROOT_TASK_ID_FIELD_NUMBER: _ClassVar[int] status: TaskStatus workspace: str task_type: str @@ -1167,10 +1267,14 @@ class TaskFilter(_message.Message): status_timestamp_after_unix_ms: int page_token: str include_descendants: bool - def __init__(self, status: _Optional[_Union[TaskStatus, str]] = ..., workspace: _Optional[str] = ..., task_type: _Optional[str] = ..., limit: _Optional[int] = ..., offset: _Optional[int] = ..., statuses: _Optional[_Iterable[_Union[TaskStatus, str]]] = ..., subject_type: _Optional[str] = ..., subject_id: _Optional[str] = ..., authority_mode: _Optional[str] = ..., authority_grant_id: _Optional[str] = ..., root_authority_grant_id: _Optional[str] = ..., parent_task_id: _Optional[str] = ..., task_class: _Optional[_Union[TaskClass, str]] = ..., exclude_task_classes: _Optional[_Iterable[_Union[TaskClass, str]]] = ..., context_id: _Optional[str] = ..., exclude_statuses: _Optional[_Iterable[_Union[TaskStatus, str]]] = ..., creator_actor: _Optional[_Union[PrincipalRef, _Mapping]] = ..., status_timestamp_after_unix_ms: _Optional[int] = ..., page_token: _Optional[str] = ..., include_descendants: bool = ...) -> None: ... + priority: TaskPriority + min_priority: TaskPriority + correlation_id: str + root_task_id: str + def __init__(self, status: _Optional[_Union[TaskStatus, str]] = ..., workspace: _Optional[str] = ..., task_type: _Optional[str] = ..., limit: _Optional[int] = ..., offset: _Optional[int] = ..., statuses: _Optional[_Iterable[_Union[TaskStatus, str]]] = ..., subject_type: _Optional[str] = ..., subject_id: _Optional[str] = ..., authority_mode: _Optional[str] = ..., authority_grant_id: _Optional[str] = ..., root_authority_grant_id: _Optional[str] = ..., parent_task_id: _Optional[str] = ..., task_class: _Optional[_Union[TaskClass, str]] = ..., exclude_task_classes: _Optional[_Iterable[_Union[TaskClass, str]]] = ..., context_id: _Optional[str] = ..., exclude_statuses: _Optional[_Iterable[_Union[TaskStatus, str]]] = ..., creator_actor: _Optional[_Union[PrincipalRef, _Mapping]] = ..., status_timestamp_after_unix_ms: _Optional[int] = ..., page_token: _Optional[str] = ..., include_descendants: bool = ..., priority: _Optional[_Union[TaskPriority, str]] = ..., min_priority: _Optional[_Union[TaskPriority, str]] = ..., correlation_id: _Optional[str] = ..., root_task_id: _Optional[str] = ...) -> None: ... class TaskInfo(_message.Message): - __slots__ = ("task_id", "task_type", "status", "workspace", "target_topic", "assigned_to", "created_at", "started_at", "completed_at", "attempt", "max_attempts", "error", "metadata", "authority_mode", "subject_type", "subject_id", "root_subject_type", "root_subject_id", "authority_grant_id", "root_authority_grant_id", "parent_authority_grant_id", "creator_actor_id", "parent_task_id", "task_class", "disconnected_at", "grace_window_ms", "wait_spec", "depends_on", "context_id", "paused_at") + __slots__ = ("task_id", "task_type", "status", "workspace", "target_topic", "assigned_to", "created_at", "started_at", "completed_at", "attempt", "max_attempts", "error", "metadata", "authority_mode", "subject_type", "subject_id", "root_subject_type", "root_subject_id", "authority_grant_id", "root_authority_grant_id", "parent_authority_grant_id", "creator_actor_id", "parent_task_id", "task_class", "disconnected_at", "grace_window_ms", "wait_spec", "depends_on", "context_id", "paused_at", "priority", "correlation_id", "root_task_id", "completion_event") class MetadataEntry(_message.Message): __slots__ = ("key", "value") KEY_FIELD_NUMBER: _ClassVar[int] @@ -1208,6 +1312,10 @@ class TaskInfo(_message.Message): DEPENDS_ON_FIELD_NUMBER: _ClassVar[int] CONTEXT_ID_FIELD_NUMBER: _ClassVar[int] PAUSED_AT_FIELD_NUMBER: _ClassVar[int] + PRIORITY_FIELD_NUMBER: _ClassVar[int] + CORRELATION_ID_FIELD_NUMBER: _ClassVar[int] + ROOT_TASK_ID_FIELD_NUMBER: _ClassVar[int] + COMPLETION_EVENT_FIELD_NUMBER: _ClassVar[int] task_id: str task_type: str status: TaskStatus @@ -1238,7 +1346,11 @@ class TaskInfo(_message.Message): depends_on: _containers.RepeatedScalarFieldContainer[str] context_id: str paused_at: int - def __init__(self, task_id: _Optional[str] = ..., task_type: _Optional[str] = ..., status: _Optional[_Union[TaskStatus, str]] = ..., workspace: _Optional[str] = ..., target_topic: _Optional[str] = ..., assigned_to: _Optional[str] = ..., created_at: _Optional[int] = ..., started_at: _Optional[int] = ..., completed_at: _Optional[int] = ..., attempt: _Optional[int] = ..., max_attempts: _Optional[int] = ..., error: _Optional[str] = ..., metadata: _Optional[_Mapping[str, str]] = ..., authority_mode: _Optional[str] = ..., subject_type: _Optional[str] = ..., subject_id: _Optional[str] = ..., root_subject_type: _Optional[str] = ..., root_subject_id: _Optional[str] = ..., authority_grant_id: _Optional[str] = ..., root_authority_grant_id: _Optional[str] = ..., parent_authority_grant_id: _Optional[str] = ..., creator_actor_id: _Optional[str] = ..., parent_task_id: _Optional[str] = ..., task_class: _Optional[_Union[TaskClass, str]] = ..., disconnected_at: _Optional[int] = ..., grace_window_ms: _Optional[int] = ..., wait_spec: _Optional[_Union[WaitSpec, _Mapping]] = ..., depends_on: _Optional[_Iterable[str]] = ..., context_id: _Optional[str] = ..., paused_at: _Optional[int] = ...) -> None: ... + priority: TaskPriority + correlation_id: str + root_task_id: str + completion_event: TaskCompletionEvent + def __init__(self, task_id: _Optional[str] = ..., task_type: _Optional[str] = ..., status: _Optional[_Union[TaskStatus, str]] = ..., workspace: _Optional[str] = ..., target_topic: _Optional[str] = ..., assigned_to: _Optional[str] = ..., created_at: _Optional[int] = ..., started_at: _Optional[int] = ..., completed_at: _Optional[int] = ..., attempt: _Optional[int] = ..., max_attempts: _Optional[int] = ..., error: _Optional[str] = ..., metadata: _Optional[_Mapping[str, str]] = ..., authority_mode: _Optional[str] = ..., subject_type: _Optional[str] = ..., subject_id: _Optional[str] = ..., root_subject_type: _Optional[str] = ..., root_subject_id: _Optional[str] = ..., authority_grant_id: _Optional[str] = ..., root_authority_grant_id: _Optional[str] = ..., parent_authority_grant_id: _Optional[str] = ..., creator_actor_id: _Optional[str] = ..., parent_task_id: _Optional[str] = ..., task_class: _Optional[_Union[TaskClass, str]] = ..., disconnected_at: _Optional[int] = ..., grace_window_ms: _Optional[int] = ..., wait_spec: _Optional[_Union[WaitSpec, _Mapping]] = ..., depends_on: _Optional[_Iterable[str]] = ..., context_id: _Optional[str] = ..., paused_at: _Optional[int] = ..., priority: _Optional[_Union[TaskPriority, str]] = ..., correlation_id: _Optional[str] = ..., root_task_id: _Optional[str] = ..., completion_event: _Optional[_Union[TaskCompletionEvent, _Mapping]] = ...) -> None: ... class TaskQueryResponse(_message.Message): __slots__ = ("success", "error", "task", "tasks", "total_count", "request_id", "next_page_token") @@ -1270,6 +1382,7 @@ class TaskOperation(_message.Message): WAIT_FOR: _ClassVar[TaskOperation.OpType] RESUME: _ClassVar[TaskOperation.OpType] REJECT: _ClassVar[TaskOperation.OpType] + CLAIM: _ClassVar[TaskOperation.OpType] RETRY: TaskOperation.OpType CANCEL: TaskOperation.OpType COMPLETE: TaskOperation.OpType @@ -1278,6 +1391,7 @@ class TaskOperation(_message.Message): WAIT_FOR: TaskOperation.OpType RESUME: TaskOperation.OpType REJECT: TaskOperation.OpType + CLAIM: TaskOperation.OpType OP_FIELD_NUMBER: _ClassVar[int] TASK_ID_FIELD_NUMBER: _ClassVar[int] REASON_FIELD_NUMBER: _ClassVar[int] @@ -1623,7 +1737,7 @@ class AgentResponse(_message.Message): def __init__(self, success: bool = ..., error: _Optional[str] = ..., message: _Optional[str] = ..., agent: _Optional[_Union[AgentRegistrationInfo, _Mapping]] = ..., agents: _Optional[_Iterable[_Union[AgentRegistrationInfo, _Mapping]]] = ..., total_count: _Optional[int] = ..., orchestrators: _Optional[_Iterable[_Union[OrchestratorInfo, _Mapping]]] = ..., launch_result: _Optional[_Union[AgentLaunchResult, _Mapping]] = ..., request_id: _Optional[str] = ...) -> None: ... class ACLOperation(_message.Message): - __slots__ = ("op", "rule_id", "rule_category", "retention_days", "rule_filter", "audit_filter", "grant_request", "fallback_request", "request_id") + __slots__ = ("op", "rule_id", "rule_category", "retention_days", "rule_filter", "audit_filter", "grant_request", "fallback_request", "request_id", "name", "principal", "group_request", "role_request", "member_request", "assignment_request", "resource_type", "resource_id", "required_level", "authorization") class OpType(int, metaclass=_enum_type_wrapper.EnumTypeWrapper): __slots__ = () LIST_RULES: _ClassVar[ACLOperation.OpType] @@ -1635,6 +1749,23 @@ class ACLOperation(_message.Message): SET_FALLBACK_POLICY: _ClassVar[ACLOperation.OpType] CLEANUP_EXPIRED: _ClassVar[ACLOperation.OpType] CLEANUP_AUDIT_LOGS: _ClassVar[ACLOperation.OpType] + CREATE_GROUP: _ClassVar[ACLOperation.OpType] + DELETE_GROUP: _ClassVar[ACLOperation.OpType] + GET_GROUP: _ClassVar[ACLOperation.OpType] + LIST_GROUPS: _ClassVar[ACLOperation.OpType] + ADD_GROUP_MEMBER: _ClassVar[ACLOperation.OpType] + REMOVE_GROUP_MEMBER: _ClassVar[ACLOperation.OpType] + LIST_GROUP_MEMBERS: _ClassVar[ACLOperation.OpType] + CREATE_ROLE: _ClassVar[ACLOperation.OpType] + DELETE_ROLE: _ClassVar[ACLOperation.OpType] + GET_ROLE: _ClassVar[ACLOperation.OpType] + LIST_ROLES: _ClassVar[ACLOperation.OpType] + ASSIGN_ROLE: _ClassVar[ACLOperation.OpType] + UNASSIGN_ROLE: _ClassVar[ACLOperation.OpType] + LIST_ROLE_ASSIGNMENTS: _ClassVar[ACLOperation.OpType] + LIST_PRINCIPAL_GROUPS: _ClassVar[ACLOperation.OpType] + LIST_PRINCIPAL_ROLES: _ClassVar[ACLOperation.OpType] + EXPLAIN_ACCESS: _ClassVar[ACLOperation.OpType] LIST_RULES: ACLOperation.OpType GET_RULE: ACLOperation.OpType GRANT: ACLOperation.OpType @@ -1644,6 +1775,23 @@ class ACLOperation(_message.Message): SET_FALLBACK_POLICY: ACLOperation.OpType CLEANUP_EXPIRED: ACLOperation.OpType CLEANUP_AUDIT_LOGS: ACLOperation.OpType + CREATE_GROUP: ACLOperation.OpType + DELETE_GROUP: ACLOperation.OpType + GET_GROUP: ACLOperation.OpType + LIST_GROUPS: ACLOperation.OpType + ADD_GROUP_MEMBER: ACLOperation.OpType + REMOVE_GROUP_MEMBER: ACLOperation.OpType + LIST_GROUP_MEMBERS: ACLOperation.OpType + CREATE_ROLE: ACLOperation.OpType + DELETE_ROLE: ACLOperation.OpType + GET_ROLE: ACLOperation.OpType + LIST_ROLES: ACLOperation.OpType + ASSIGN_ROLE: ACLOperation.OpType + UNASSIGN_ROLE: ACLOperation.OpType + LIST_ROLE_ASSIGNMENTS: ACLOperation.OpType + LIST_PRINCIPAL_GROUPS: ACLOperation.OpType + LIST_PRINCIPAL_ROLES: ACLOperation.OpType + EXPLAIN_ACCESS: ACLOperation.OpType OP_FIELD_NUMBER: _ClassVar[int] RULE_ID_FIELD_NUMBER: _ClassVar[int] RULE_CATEGORY_FIELD_NUMBER: _ClassVar[int] @@ -1653,6 +1801,16 @@ class ACLOperation(_message.Message): GRANT_REQUEST_FIELD_NUMBER: _ClassVar[int] FALLBACK_REQUEST_FIELD_NUMBER: _ClassVar[int] REQUEST_ID_FIELD_NUMBER: _ClassVar[int] + NAME_FIELD_NUMBER: _ClassVar[int] + PRINCIPAL_FIELD_NUMBER: _ClassVar[int] + GROUP_REQUEST_FIELD_NUMBER: _ClassVar[int] + ROLE_REQUEST_FIELD_NUMBER: _ClassVar[int] + MEMBER_REQUEST_FIELD_NUMBER: _ClassVar[int] + ASSIGNMENT_REQUEST_FIELD_NUMBER: _ClassVar[int] + RESOURCE_TYPE_FIELD_NUMBER: _ClassVar[int] + RESOURCE_ID_FIELD_NUMBER: _ClassVar[int] + REQUIRED_LEVEL_FIELD_NUMBER: _ClassVar[int] + AUTHORIZATION_FIELD_NUMBER: _ClassVar[int] op: ACLOperation.OpType rule_id: str rule_category: str @@ -1662,7 +1820,17 @@ class ACLOperation(_message.Message): grant_request: ACLGrantRequest fallback_request: ACLSetFallbackRequest request_id: str - def __init__(self, op: _Optional[_Union[ACLOperation.OpType, str]] = ..., rule_id: _Optional[str] = ..., rule_category: _Optional[str] = ..., retention_days: _Optional[int] = ..., rule_filter: _Optional[_Union[ACLRuleFilter, _Mapping]] = ..., audit_filter: _Optional[_Union[ACLAuditFilter, _Mapping]] = ..., grant_request: _Optional[_Union[ACLGrantRequest, _Mapping]] = ..., fallback_request: _Optional[_Union[ACLSetFallbackRequest, _Mapping]] = ..., request_id: _Optional[str] = ...) -> None: ... + name: str + principal: PrincipalRef + group_request: ACLGroupRequest + role_request: ACLRoleRequest + member_request: ACLGroupMemberRequest + assignment_request: ACLRoleAssignmentRequest + resource_type: str + resource_id: str + required_level: int + authorization: AuthorizationContext + def __init__(self, op: _Optional[_Union[ACLOperation.OpType, str]] = ..., rule_id: _Optional[str] = ..., rule_category: _Optional[str] = ..., retention_days: _Optional[int] = ..., rule_filter: _Optional[_Union[ACLRuleFilter, _Mapping]] = ..., audit_filter: _Optional[_Union[ACLAuditFilter, _Mapping]] = ..., grant_request: _Optional[_Union[ACLGrantRequest, _Mapping]] = ..., fallback_request: _Optional[_Union[ACLSetFallbackRequest, _Mapping]] = ..., request_id: _Optional[str] = ..., name: _Optional[str] = ..., principal: _Optional[_Union[PrincipalRef, _Mapping]] = ..., group_request: _Optional[_Union[ACLGroupRequest, _Mapping]] = ..., role_request: _Optional[_Union[ACLRoleRequest, _Mapping]] = ..., member_request: _Optional[_Union[ACLGroupMemberRequest, _Mapping]] = ..., assignment_request: _Optional[_Union[ACLRoleAssignmentRequest, _Mapping]] = ..., resource_type: _Optional[str] = ..., resource_id: _Optional[str] = ..., required_level: _Optional[int] = ..., authorization: _Optional[_Union[AuthorizationContext, _Mapping]] = ...) -> None: ... class ACLRuleFilter(_message.Message): __slots__ = ("principal_type", "principal_id", "resource_type", "resource_id", "limit", "offset") @@ -1979,8 +2147,182 @@ class ACLCleanupResult(_message.Message): message: str def __init__(self, deleted_count: _Optional[int] = ..., message: _Optional[str] = ...) -> None: ... +class ACLGroupRequest(_message.Message): + __slots__ = ("name", "description", "created_by", "metadata") + class MetadataEntry(_message.Message): + __slots__ = ("key", "value") + KEY_FIELD_NUMBER: _ClassVar[int] + VALUE_FIELD_NUMBER: _ClassVar[int] + key: str + value: str + def __init__(self, key: _Optional[str] = ..., value: _Optional[str] = ...) -> None: ... + NAME_FIELD_NUMBER: _ClassVar[int] + DESCRIPTION_FIELD_NUMBER: _ClassVar[int] + CREATED_BY_FIELD_NUMBER: _ClassVar[int] + METADATA_FIELD_NUMBER: _ClassVar[int] + name: str + description: str + created_by: str + metadata: _containers.ScalarMap[str, str] + def __init__(self, name: _Optional[str] = ..., description: _Optional[str] = ..., created_by: _Optional[str] = ..., metadata: _Optional[_Mapping[str, str]] = ...) -> None: ... + +class ACLRoleRequest(_message.Message): + __slots__ = ("name", "description", "created_by", "metadata") + class MetadataEntry(_message.Message): + __slots__ = ("key", "value") + KEY_FIELD_NUMBER: _ClassVar[int] + VALUE_FIELD_NUMBER: _ClassVar[int] + key: str + value: str + def __init__(self, key: _Optional[str] = ..., value: _Optional[str] = ...) -> None: ... + NAME_FIELD_NUMBER: _ClassVar[int] + DESCRIPTION_FIELD_NUMBER: _ClassVar[int] + CREATED_BY_FIELD_NUMBER: _ClassVar[int] + METADATA_FIELD_NUMBER: _ClassVar[int] + name: str + description: str + created_by: str + metadata: _containers.ScalarMap[str, str] + def __init__(self, name: _Optional[str] = ..., description: _Optional[str] = ..., created_by: _Optional[str] = ..., metadata: _Optional[_Mapping[str, str]] = ...) -> None: ... + +class ACLGroupMemberRequest(_message.Message): + __slots__ = ("member_type", "member_id", "granted_by", "expires_at") + MEMBER_TYPE_FIELD_NUMBER: _ClassVar[int] + MEMBER_ID_FIELD_NUMBER: _ClassVar[int] + GRANTED_BY_FIELD_NUMBER: _ClassVar[int] + EXPIRES_AT_FIELD_NUMBER: _ClassVar[int] + member_type: str + member_id: str + granted_by: str + expires_at: int + def __init__(self, member_type: _Optional[str] = ..., member_id: _Optional[str] = ..., granted_by: _Optional[str] = ..., expires_at: _Optional[int] = ...) -> None: ... + +class ACLRoleAssignmentRequest(_message.Message): + __slots__ = ("assignee_type", "assignee_id", "granted_by", "expires_at") + ASSIGNEE_TYPE_FIELD_NUMBER: _ClassVar[int] + ASSIGNEE_ID_FIELD_NUMBER: _ClassVar[int] + GRANTED_BY_FIELD_NUMBER: _ClassVar[int] + EXPIRES_AT_FIELD_NUMBER: _ClassVar[int] + assignee_type: str + assignee_id: str + granted_by: str + expires_at: int + def __init__(self, assignee_type: _Optional[str] = ..., assignee_id: _Optional[str] = ..., granted_by: _Optional[str] = ..., expires_at: _Optional[int] = ...) -> None: ... + +class ACLGroupInfo(_message.Message): + __slots__ = ("group_id", "group_name", "description", "created_by", "created_at", "metadata") + class MetadataEntry(_message.Message): + __slots__ = ("key", "value") + KEY_FIELD_NUMBER: _ClassVar[int] + VALUE_FIELD_NUMBER: _ClassVar[int] + key: str + value: str + def __init__(self, key: _Optional[str] = ..., value: _Optional[str] = ...) -> None: ... + GROUP_ID_FIELD_NUMBER: _ClassVar[int] + GROUP_NAME_FIELD_NUMBER: _ClassVar[int] + DESCRIPTION_FIELD_NUMBER: _ClassVar[int] + CREATED_BY_FIELD_NUMBER: _ClassVar[int] + CREATED_AT_FIELD_NUMBER: _ClassVar[int] + METADATA_FIELD_NUMBER: _ClassVar[int] + group_id: str + group_name: str + description: str + created_by: str + created_at: int + metadata: _containers.ScalarMap[str, str] + def __init__(self, group_id: _Optional[str] = ..., group_name: _Optional[str] = ..., description: _Optional[str] = ..., created_by: _Optional[str] = ..., created_at: _Optional[int] = ..., metadata: _Optional[_Mapping[str, str]] = ...) -> None: ... + +class ACLRoleInfo(_message.Message): + __slots__ = ("role_id", "role_name", "description", "created_by", "created_at", "metadata") + class MetadataEntry(_message.Message): + __slots__ = ("key", "value") + KEY_FIELD_NUMBER: _ClassVar[int] + VALUE_FIELD_NUMBER: _ClassVar[int] + key: str + value: str + def __init__(self, key: _Optional[str] = ..., value: _Optional[str] = ...) -> None: ... + ROLE_ID_FIELD_NUMBER: _ClassVar[int] + ROLE_NAME_FIELD_NUMBER: _ClassVar[int] + DESCRIPTION_FIELD_NUMBER: _ClassVar[int] + CREATED_BY_FIELD_NUMBER: _ClassVar[int] + CREATED_AT_FIELD_NUMBER: _ClassVar[int] + METADATA_FIELD_NUMBER: _ClassVar[int] + role_id: str + role_name: str + description: str + created_by: str + created_at: int + metadata: _containers.ScalarMap[str, str] + def __init__(self, role_id: _Optional[str] = ..., role_name: _Optional[str] = ..., description: _Optional[str] = ..., created_by: _Optional[str] = ..., created_at: _Optional[int] = ..., metadata: _Optional[_Mapping[str, str]] = ...) -> None: ... + +class ACLGroupMemberInfo(_message.Message): + __slots__ = ("group_name", "member_type", "member_id", "granted_by", "granted_at", "expires_at") + GROUP_NAME_FIELD_NUMBER: _ClassVar[int] + MEMBER_TYPE_FIELD_NUMBER: _ClassVar[int] + MEMBER_ID_FIELD_NUMBER: _ClassVar[int] + GRANTED_BY_FIELD_NUMBER: _ClassVar[int] + GRANTED_AT_FIELD_NUMBER: _ClassVar[int] + EXPIRES_AT_FIELD_NUMBER: _ClassVar[int] + group_name: str + member_type: str + member_id: str + granted_by: str + granted_at: int + expires_at: int + def __init__(self, group_name: _Optional[str] = ..., member_type: _Optional[str] = ..., member_id: _Optional[str] = ..., granted_by: _Optional[str] = ..., granted_at: _Optional[int] = ..., expires_at: _Optional[int] = ...) -> None: ... + +class ACLRoleAssignmentInfo(_message.Message): + __slots__ = ("role_name", "assignee_type", "assignee_id", "granted_by", "granted_at", "expires_at") + ROLE_NAME_FIELD_NUMBER: _ClassVar[int] + ASSIGNEE_TYPE_FIELD_NUMBER: _ClassVar[int] + ASSIGNEE_ID_FIELD_NUMBER: _ClassVar[int] + GRANTED_BY_FIELD_NUMBER: _ClassVar[int] + GRANTED_AT_FIELD_NUMBER: _ClassVar[int] + EXPIRES_AT_FIELD_NUMBER: _ClassVar[int] + role_name: str + assignee_type: str + assignee_id: str + granted_by: str + granted_at: int + expires_at: int + def __init__(self, role_name: _Optional[str] = ..., assignee_type: _Optional[str] = ..., assignee_id: _Optional[str] = ..., granted_by: _Optional[str] = ..., granted_at: _Optional[int] = ..., expires_at: _Optional[int] = ...) -> None: ... + +class ACLAccessContributionInfo(_message.Message): + __slots__ = ("subject", "rule_id", "access_level", "resource", "expired") + SUBJECT_FIELD_NUMBER: _ClassVar[int] + RULE_ID_FIELD_NUMBER: _ClassVar[int] + ACCESS_LEVEL_FIELD_NUMBER: _ClassVar[int] + RESOURCE_FIELD_NUMBER: _ClassVar[int] + EXPIRED_FIELD_NUMBER: _ClassVar[int] + subject: str + rule_id: str + access_level: int + resource: str + expired: bool + def __init__(self, subject: _Optional[str] = ..., rule_id: _Optional[str] = ..., access_level: _Optional[int] = ..., resource: _Optional[str] = ..., expired: bool = ...) -> None: ... + +class ACLAccessExplanationInfo(_message.Message): + __slots__ = ("principal", "subjects", "contributions", "allowed", "decision", "effective_access_level", "fallback_applied", "reason") + PRINCIPAL_FIELD_NUMBER: _ClassVar[int] + SUBJECTS_FIELD_NUMBER: _ClassVar[int] + CONTRIBUTIONS_FIELD_NUMBER: _ClassVar[int] + ALLOWED_FIELD_NUMBER: _ClassVar[int] + DECISION_FIELD_NUMBER: _ClassVar[int] + EFFECTIVE_ACCESS_LEVEL_FIELD_NUMBER: _ClassVar[int] + FALLBACK_APPLIED_FIELD_NUMBER: _ClassVar[int] + REASON_FIELD_NUMBER: _ClassVar[int] + principal: str + subjects: _containers.RepeatedScalarFieldContainer[str] + contributions: _containers.RepeatedCompositeFieldContainer[ACLAccessContributionInfo] + allowed: bool + decision: str + effective_access_level: int + fallback_applied: bool + reason: str + def __init__(self, principal: _Optional[str] = ..., subjects: _Optional[_Iterable[str]] = ..., contributions: _Optional[_Iterable[_Union[ACLAccessContributionInfo, _Mapping]]] = ..., allowed: bool = ..., decision: _Optional[str] = ..., effective_access_level: _Optional[int] = ..., fallback_applied: bool = ..., reason: _Optional[str] = ...) -> None: ... + class ACLResponse(_message.Message): - __slots__ = ("success", "error", "message", "rule", "rules", "total_rules", "fallback_policy", "audit_entries", "total_audit_entries", "cleanup_result", "authority_grant", "authority_grants", "total_authority_grants", "request_id") + __slots__ = ("success", "error", "message", "rule", "rules", "total_rules", "fallback_policy", "audit_entries", "total_audit_entries", "cleanup_result", "authority_grant", "authority_grants", "total_authority_grants", "request_id", "group", "groups", "role", "roles", "group_members", "role_assignments", "explanation") SUCCESS_FIELD_NUMBER: _ClassVar[int] ERROR_FIELD_NUMBER: _ClassVar[int] MESSAGE_FIELD_NUMBER: _ClassVar[int] @@ -1995,6 +2337,13 @@ class ACLResponse(_message.Message): AUTHORITY_GRANTS_FIELD_NUMBER: _ClassVar[int] TOTAL_AUTHORITY_GRANTS_FIELD_NUMBER: _ClassVar[int] REQUEST_ID_FIELD_NUMBER: _ClassVar[int] + GROUP_FIELD_NUMBER: _ClassVar[int] + GROUPS_FIELD_NUMBER: _ClassVar[int] + ROLE_FIELD_NUMBER: _ClassVar[int] + ROLES_FIELD_NUMBER: _ClassVar[int] + GROUP_MEMBERS_FIELD_NUMBER: _ClassVar[int] + ROLE_ASSIGNMENTS_FIELD_NUMBER: _ClassVar[int] + EXPLANATION_FIELD_NUMBER: _ClassVar[int] success: bool error: str message: str @@ -2009,7 +2358,14 @@ class ACLResponse(_message.Message): authority_grants: _containers.RepeatedCompositeFieldContainer[ACLAuthorityGrantInfo] total_authority_grants: int request_id: str - def __init__(self, success: bool = ..., error: _Optional[str] = ..., message: _Optional[str] = ..., rule: _Optional[_Union[ACLRuleInfo, _Mapping]] = ..., rules: _Optional[_Iterable[_Union[ACLRuleInfo, _Mapping]]] = ..., total_rules: _Optional[int] = ..., fallback_policy: _Optional[_Union[ACLFallbackPolicyInfo, _Mapping]] = ..., audit_entries: _Optional[_Iterable[_Union[ACLAuditEntryInfo, _Mapping]]] = ..., total_audit_entries: _Optional[int] = ..., cleanup_result: _Optional[_Union[ACLCleanupResult, _Mapping]] = ..., authority_grant: _Optional[_Union[ACLAuthorityGrantInfo, _Mapping]] = ..., authority_grants: _Optional[_Iterable[_Union[ACLAuthorityGrantInfo, _Mapping]]] = ..., total_authority_grants: _Optional[int] = ..., request_id: _Optional[str] = ...) -> None: ... + group: ACLGroupInfo + groups: _containers.RepeatedCompositeFieldContainer[ACLGroupInfo] + role: ACLRoleInfo + roles: _containers.RepeatedCompositeFieldContainer[ACLRoleInfo] + group_members: _containers.RepeatedCompositeFieldContainer[ACLGroupMemberInfo] + role_assignments: _containers.RepeatedCompositeFieldContainer[ACLRoleAssignmentInfo] + explanation: ACLAccessExplanationInfo + def __init__(self, success: bool = ..., error: _Optional[str] = ..., message: _Optional[str] = ..., rule: _Optional[_Union[ACLRuleInfo, _Mapping]] = ..., rules: _Optional[_Iterable[_Union[ACLRuleInfo, _Mapping]]] = ..., total_rules: _Optional[int] = ..., fallback_policy: _Optional[_Union[ACLFallbackPolicyInfo, _Mapping]] = ..., audit_entries: _Optional[_Iterable[_Union[ACLAuditEntryInfo, _Mapping]]] = ..., total_audit_entries: _Optional[int] = ..., cleanup_result: _Optional[_Union[ACLCleanupResult, _Mapping]] = ..., authority_grant: _Optional[_Union[ACLAuthorityGrantInfo, _Mapping]] = ..., authority_grants: _Optional[_Iterable[_Union[ACLAuthorityGrantInfo, _Mapping]]] = ..., total_authority_grants: _Optional[int] = ..., request_id: _Optional[str] = ..., group: _Optional[_Union[ACLGroupInfo, _Mapping]] = ..., groups: _Optional[_Iterable[_Union[ACLGroupInfo, _Mapping]]] = ..., role: _Optional[_Union[ACLRoleInfo, _Mapping]] = ..., roles: _Optional[_Iterable[_Union[ACLRoleInfo, _Mapping]]] = ..., group_members: _Optional[_Iterable[_Union[ACLGroupMemberInfo, _Mapping]]] = ..., role_assignments: _Optional[_Iterable[_Union[ACLRoleAssignmentInfo, _Mapping]]] = ..., explanation: _Optional[_Union[ACLAccessExplanationInfo, _Mapping]] = ...) -> None: ... class AuthorityGrantOperation(_message.Message): __slots__ = ("op", "grant_id", "exchange_request", "derive_request", "renew_request", "request_id", "list_request", "batch_exchange_request", "derive_for_target_request") @@ -2677,6 +3033,9 @@ class WorkflowOperation(_message.Message): CREATE_SM_INSTANCE: _ClassVar[WorkflowOperation.OpType] SEND_SM_EVENT: _ClassVar[WorkflowOperation.OpType] UPSERT_SCHEDULE: _ClassVar[WorkflowOperation.OpType] + LIST_JOINS: _ClassVar[WorkflowOperation.OpType] + GET_JOIN: _ClassVar[WorkflowOperation.OpType] + CANCEL_JOIN: _ClassVar[WorkflowOperation.OpType] LIST_RULES: WorkflowOperation.OpType GET_RULE: WorkflowOperation.OpType CREATE_RULE: WorkflowOperation.OpType @@ -2701,6 +3060,9 @@ class WorkflowOperation(_message.Message): CREATE_SM_INSTANCE: WorkflowOperation.OpType SEND_SM_EVENT: WorkflowOperation.OpType UPSERT_SCHEDULE: WorkflowOperation.OpType + LIST_JOINS: WorkflowOperation.OpType + GET_JOIN: WorkflowOperation.OpType + CANCEL_JOIN: WorkflowOperation.OpType OP_FIELD_NUMBER: _ClassVar[int] ID_FIELD_NUMBER: _ClassVar[int] SECONDARY_ID_FIELD_NUMBER: _ClassVar[int] @@ -2734,7 +3096,7 @@ class WorkflowResponse(_message.Message): def __init__(self, success: bool = ..., error: _Optional[str] = ..., message: _Optional[str] = ..., data: _Optional[bytes] = ..., total_count: _Optional[int] = ..., request_id: _Optional[str] = ...) -> None: ... class MessageEnvelope(_message.Message): - __slots__ = ("source", "payload", "message_type", "timestamp_ms", "metadata", "workspace") + __slots__ = ("source", "payload", "message_type", "timestamp_ms", "metadata", "workspace", "on_behalf_subject") class MetadataEntry(_message.Message): __slots__ = ("key", "value") KEY_FIELD_NUMBER: _ClassVar[int] @@ -2748,13 +3110,15 @@ class MessageEnvelope(_message.Message): TIMESTAMP_MS_FIELD_NUMBER: _ClassVar[int] METADATA_FIELD_NUMBER: _ClassVar[int] WORKSPACE_FIELD_NUMBER: _ClassVar[int] + ON_BEHALF_SUBJECT_FIELD_NUMBER: _ClassVar[int] source: str payload: bytes message_type: MessageType timestamp_ms: int metadata: _containers.ScalarMap[str, str] workspace: str - def __init__(self, source: _Optional[str] = ..., payload: _Optional[bytes] = ..., message_type: _Optional[_Union[MessageType, str]] = ..., timestamp_ms: _Optional[int] = ..., metadata: _Optional[_Mapping[str, str]] = ..., workspace: _Optional[str] = ...) -> None: ... + on_behalf_subject: PrincipalRef + def __init__(self, source: _Optional[str] = ..., payload: _Optional[bytes] = ..., message_type: _Optional[_Union[MessageType, str]] = ..., timestamp_ms: _Optional[int] = ..., metadata: _Optional[_Mapping[str, str]] = ..., workspace: _Optional[str] = ..., on_behalf_subject: _Optional[_Union[PrincipalRef, _Mapping]] = ...) -> None: ... class AuditQuery(_message.Message): __slots__ = ("request_id", "start_time", "end_time", "event_type", "actor_type", "actor_id", "resource_type", "resource_id", "operation", "workspace", "only_failures", "limit", "offset", "subject_type", "subject_id", "authority_mode", "authority_grant_id", "authorization", "exclude_actor_types", "exclude_workspaces", "exclude_service_direct") diff --git a/sdk/python-client/scitrera_aether_client/proxy.py b/sdk/python-client/scitrera_aether_client/proxy.py index cbf91c3..d0f5d22 100644 --- a/sdk/python-client/scitrera_aether_client/proxy.py +++ b/sdk/python-client/scitrera_aether_client/proxy.py @@ -415,11 +415,16 @@ def _patched_do_connect(self, init_msg, target): # type: ignore[no-untyped-def] from .proto import aether_pb2_grpc as _grpc_mod import grpc + # Preserve gRPC keepalive options (silent-drop detection) when the + # proxy hook takes over channel creation. _channel_options is set in + # the client __init__; getattr keeps this safe for any base that + # predates the keepalive support. + _opts = getattr(self, "_channel_options", None) if self.tls_enabled: credentials = self._build_tls_credentials() - self.channel = grpc.secure_channel(target, credentials) + self.channel = grpc.secure_channel(target, credentials, options=_opts) else: - self.channel = grpc.insecure_channel(target) + self.channel = grpc.insecure_channel(target, options=_opts) self.stub = _grpc_mod.AetherGatewayStub(self.channel) if self._session_id: @@ -452,11 +457,16 @@ async def _patched_do_connect(self, init_msg, target): # type: ignore[no-untype from .proto import aether_pb2_grpc as _grpc_mod import grpc.aio as _grpc_aio # type: ignore + # Preserve gRPC keepalive options (silent-drop detection) when the + # proxy hook takes over channel creation — without this, proxied + # connections would silently lose the keepalive that the unpatched + # _do_connect installs. _channel_options is set in the client __init__. + _opts = getattr(self, "_channel_options", None) if self.tls_enabled: credentials = self._build_tls_credentials() - self.channel = _grpc_aio.secure_channel(target, credentials) + self.channel = _grpc_aio.secure_channel(target, credentials, options=_opts) else: - self.channel = _grpc_aio.insecure_channel(target) + self.channel = _grpc_aio.insecure_channel(target, options=_opts) self.stub = _grpc_mod.AetherGatewayStub(self.channel) raw_stream = self.stub.Connect(self._request_generator()) @@ -619,6 +629,11 @@ def proxy_http( """ _ensure_hooks_installed() + # Defensive: callers may pass body=None explicitly even though the default + # is b"". Normalise here so len(body) never raises TypeError. + if body is None: + body = b"" + if request_id is None: request_id = str(uuid.uuid4()) @@ -738,6 +753,11 @@ async def proxy_http_async( ) -> Any: _ensure_hooks_installed() + # Defensive: callers may pass body=None explicitly even though the default + # is b"". Normalise here so len(body) never raises TypeError. + if body is None: + body = b"" + if request_id is None: request_id = str(uuid.uuid4()) diff --git a/sdk/python-client/scitrera_aether_client/proxy_terminator.py b/sdk/python-client/scitrera_aether_client/proxy_terminator.py index 2e2b027..31d862d 100644 --- a/sdk/python-client/scitrera_aether_client/proxy_terminator.py +++ b/sdk/python-client/scitrera_aether_client/proxy_terminator.py @@ -157,6 +157,12 @@ class _PendingRequest: # Backward-compat user/principal headers that downstream services read. _HDR_USER_ID = "X-Auth-User-ID" _HDR_PRINCIPAL_TYPE = "X-Auth-Principal-Type" +# Deployment tenant. Unlike the actor/subject headers (derived from the +# AuthorizationContext / gateway identity), the tenant is a per-deployment value +# the terminator's owner supplies via ``ProxyHttpTerminator(tenant_id=...)``. The +# Go sidecar mints this from its configured ``cfg.TenantID`` (identityheaders.go +# ::MintInto); the Python terminator left it to "upstream layers" until now. +_HDR_TENANT_ID = "X-Auth-Tenant-ID" # OBO policy: how strict-mode dispatch handles on_behalf_of requests. # - "require_resolver": OBO requires a configured resolver AND a non-None @@ -631,10 +637,21 @@ def __init__( body_chunk_size: int = PROXY_BODY_CHUNK_SIZE, resolver: Optional[AuthorityResolverProtocol] = None, obo_policy: OBOPolicy = "require_resolver", + tenant_id: Optional[str] = None, ) -> None: """Construct a terminator. Args: + tenant_id: Optional deployment tenant. When set AND + ``header_mode="strict"``, every minted request is stamped with + ``X-Auth-Tenant-ID=`` (direct AND OBO). This is the + per-deployment value the Go sidecar reads from ``cfg.TenantID``; + the Python terminator's ``_mint_auth_headers`` does not carry it + (it is not on the wire envelope), so a per-tenant service + (MemoryLayer, data-connectors) must supply it here — otherwise a + fail-closed downstream that requires ``X-Auth-Tenant-ID`` rejects + every request. Inbound ``x-auth-*`` headers are stripped first, so + this value is authoritative and cannot be spoofed by the caller. resolver: Optional :class:`AuthorityResolverProtocol`. When set AND ``header_mode="strict"``, OBO requests have their minted ``X-Auth-*`` set extended with grant-derived fields @@ -672,6 +689,7 @@ def __init__( self._body_chunk_size = body_chunk_size self._resolver = resolver self._obo_policy: OBOPolicy = obo_policy + self._tenant_id = tenant_id self._dispatcher = _get_terminator_dispatcher(client) self._started = False @@ -725,6 +743,14 @@ async def _dispatch( sanitized = _strip_inbound_identity_headers(inbound_headers) authz = req.authorization if req.HasField("authorization") else None sanitized.update(_mint_auth_headers(authz)) + # Stamp the deployment tenant (the one header _mint_auth_headers can + # NOT derive from the wire envelope — see _HDR_TENANT_ID). Applies to + # both direct and OBO requests; inbound x-auth-* were already stripped + # so this is authoritative. A fail-closed downstream (e.g. MemoryLayer's + # AetherAuthenticationService) requires X-Auth-Tenant-ID, so a per-tenant + # terminator MUST set tenant_id or every request is rejected. + if self._tenant_id: + sanitized[_HDR_TENANT_ID] = self._tenant_id # Overlay actor headers from the client's own gateway identity. # The actor reflects the authenticated principal that opened the # gateway connection — equivalent to Go's terminator deriving diff --git a/sdk/python-client/tests/test_admin.py b/sdk/python-client/tests/test_admin.py index f5de725..078783d 100644 --- a/sdk/python-client/tests/test_admin.py +++ b/sdk/python-client/tests/test_admin.py @@ -162,6 +162,75 @@ def call(): assert msg.workflow_op.workspace == "ws-1" assert msg.workflow_op.request_id != "" + def test_create_role_queues_upstream(self, agent_client: AgentClient): + agent_client.request_queue = _queue.Queue() + agent_client._acl_response_queue.put(object()) + + admin = AdminClient(agent_client) + admin.create_role( + name="bid-reviewer", + description="Reviews bids", + created_by="ops", + metadata={"team": "jgl"}, + timeout=1.0, + ) + + msg = agent_client.request_queue.get_nowait() + assert msg.HasField("acl_op") + assert msg.acl_op.op == aether_pb2.ACLOperation.CREATE_ROLE + rr = msg.acl_op.role_request + assert rr.name == "bid-reviewer" + assert rr.description == "Reviews bids" + assert rr.created_by == "ops" + assert dict(rr.metadata) == {"team": "jgl"} + + def test_assign_role_queues_upstream(self, agent_client: AgentClient): + agent_client.request_queue = _queue.Queue() + agent_client._acl_response_queue.put(object()) + + admin = AdminClient(agent_client) + admin.assign_role( + name="bid-reviewer", + assignee_type="user", + assignee_id="alice", + granted_by="ops", + expires_at=1700000000, + timeout=1.0, + ) + + msg = agent_client.request_queue.get_nowait() + assert msg.HasField("acl_op") + assert msg.acl_op.op == aether_pb2.ACLOperation.ASSIGN_ROLE + assert msg.acl_op.name == "bid-reviewer" + ar = msg.acl_op.assignment_request + assert ar.assignee_type == "user" + assert ar.assignee_id == "alice" + assert ar.granted_by == "ops" + assert ar.expires_at == 1700000000 + + def test_explain_access_queues_upstream(self, agent_client: AgentClient): + agent_client.request_queue = _queue.Queue() + agent_client._acl_response_queue.put(object()) + + admin = AdminClient(agent_client) + admin.explain_access( + principal_type="user", + principal_id="alice", + resource_type="workspace", + resource_id="ws-1", + required_level=40, + timeout=1.0, + ) + + msg = agent_client.request_queue.get_nowait() + assert msg.HasField("acl_op") + assert msg.acl_op.op == aether_pb2.ACLOperation.EXPLAIN_ACCESS + assert msg.acl_op.principal.principal_type == "user" + assert msg.acl_op.principal.principal_id == "alice" + assert msg.acl_op.resource_type == "workspace" + assert msg.acl_op.resource_id == "ws-1" + assert msg.acl_op.required_level == 40 + # ============================================================================= # Async happy-path: one per category @@ -239,3 +308,156 @@ async def test_get_workflow_rule_queues_upstream(self, async_agent_client: Async assert msg.workflow_op.op == aether_pb2.WorkflowOperation.GET_RULE assert msg.workflow_op.id == "rule-1" assert msg.workflow_op.workspace == "ws-1" + + @pytest.mark.asyncio + async def test_create_role_queues_upstream(self, async_agent_client: AsyncAgentClient): + await async_agent_client._acl_response_queue.put(object()) + + admin = AsyncAdminClient(async_agent_client) + await admin.create_role( + name="bid-reviewer", description="Reviews bids", timeout=1.0, + ) + + msg = async_agent_client._request_queue.get_nowait() + assert msg.HasField("acl_op") + assert msg.acl_op.op == aether_pb2.ACLOperation.CREATE_ROLE + assert msg.acl_op.role_request.name == "bid-reviewer" + assert msg.acl_op.role_request.description == "Reviews bids" + + @pytest.mark.asyncio + async def test_add_group_member_queues_upstream(self, async_agent_client: AsyncAgentClient): + await async_agent_client._acl_response_queue.put(object()) + + admin = AsyncAdminClient(async_agent_client) + await admin.add_group_member( + name="reviewers", + member_type="user", + member_id="alice", + timeout=1.0, + ) + + msg = async_agent_client._request_queue.get_nowait() + assert msg.HasField("acl_op") + assert msg.acl_op.op == aether_pb2.ACLOperation.ADD_GROUP_MEMBER + assert msg.acl_op.name == "reviewers" + assert msg.acl_op.member_request.member_type == "user" + assert msg.acl_op.member_request.member_id == "alice" + + @pytest.mark.asyncio + async def test_explain_access_queues_upstream(self, async_agent_client: AsyncAgentClient): + await async_agent_client._acl_response_queue.put(object()) + + admin = AsyncAdminClient(async_agent_client) + await admin.explain_access( + principal_type="user", + principal_id="alice", + resource_type="workspace", + resource_id="ws-1", + required_level=40, + timeout=1.0, + ) + + msg = async_agent_client._request_queue.get_nowait() + assert msg.HasField("acl_op") + assert msg.acl_op.op == aether_pb2.ACLOperation.EXPLAIN_ACCESS + assert msg.acl_op.principal.principal_type == "user" + assert msg.acl_op.principal.principal_id == "alice" + assert msg.acl_op.required_level == 40 + + @pytest.mark.asyncio + async def test_revoke_acl_rule_queues_upstream(self, async_agent_client: AsyncAgentClient): + """revoke_acl_rule sends REVOKE with rule_filter (composite-key path).""" + await async_agent_client._acl_response_queue.put(object()) + + admin = AsyncAdminClient(async_agent_client) + await admin.revoke_acl_rule( + principal_type="user", + principal_id="alice", + resource_type="workspace", + resource_id="ws-1", + timeout=1.0, + ) + + msg = async_agent_client._request_queue.get_nowait() + assert msg.HasField("acl_op") + assert msg.acl_op.op == aether_pb2.ACLOperation.REVOKE + rf = msg.acl_op.rule_filter + assert rf.principal_type == "user" + assert rf.principal_id == "alice" + assert rf.resource_type == "workspace" + assert rf.resource_id == "ws-1" + # rule_id must be empty — this path does NOT send a UUID + assert msg.acl_op.rule_id == "" + + @pytest.mark.asyncio + async def test_delete_acl_rule_queues_upstream(self, async_agent_client: AsyncAgentClient): + """delete_acl_rule sends REVOKE with rule_id (UUID path, gateway resolves to composite key).""" + await async_agent_client._acl_response_queue.put(object()) + + admin = AsyncAdminClient(async_agent_client) + await admin.delete_acl_rule( + rule_id="aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee", + timeout=1.0, + ) + + msg = async_agent_client._request_queue.get_nowait() + assert msg.HasField("acl_op") + assert msg.acl_op.op == aether_pb2.ACLOperation.REVOKE + assert msg.acl_op.rule_id == "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee" + # rule_filter must be empty — this path sends only the UUID + assert msg.acl_op.rule_filter is None or ( + msg.acl_op.rule_filter.principal_type == "" + and msg.acl_op.rule_filter.resource_type == "" + ) + + +# ============================================================================= +# Sync: revoke_acl_rule and delete_acl_rule +# ============================================================================= + +class TestSyncAdminClientRevokeACLRule: + """Verify the sync revoke_acl_rule and delete_acl_rule wire the correct proto fields.""" + + def test_revoke_acl_rule_queues_upstream(self, agent_client: AgentClient): + """revoke_acl_rule sends REVOKE with rule_filter (composite-key path).""" + agent_client.request_queue = _queue.Queue() + agent_client._acl_response_queue.put(object()) + + admin = AdminClient(agent_client) + admin.revoke_acl_rule( + principal_type="agent", + principal_id="ag.my-agent", + resource_type="workspace", + resource_id="ws-42", + timeout=1.0, + ) + + msg = agent_client.request_queue.get_nowait() + assert msg.HasField("acl_op") + assert msg.acl_op.op == aether_pb2.ACLOperation.REVOKE + rf = msg.acl_op.rule_filter + assert rf.principal_type == "agent" + assert rf.principal_id == "ag.my-agent" + assert rf.resource_type == "workspace" + assert rf.resource_id == "ws-42" + assert msg.acl_op.rule_id == "" + + def test_delete_acl_rule_queues_upstream(self, agent_client: AgentClient): + """delete_acl_rule sends REVOKE with rule_id (UUID path, gateway resolves to composite key).""" + agent_client.request_queue = _queue.Queue() + agent_client._acl_response_queue.put(object()) + + admin = AdminClient(agent_client) + admin.delete_acl_rule( + rule_id="12345678-1234-1234-1234-123456789abc", + timeout=1.0, + ) + + msg = agent_client.request_queue.get_nowait() + assert msg.HasField("acl_op") + assert msg.acl_op.op == aether_pb2.ACLOperation.REVOKE + assert msg.acl_op.rule_id == "12345678-1234-1234-1234-123456789abc" + assert msg.acl_op.rule_filter is None or ( + msg.acl_op.rule_filter.principal_type == "" + and msg.acl_op.rule_filter.resource_type == "" + ) diff --git a/sdk/python-client/tests/test_client.py b/sdk/python-client/tests/test_client.py index 5eeb027..34f302c 100644 --- a/sdk/python-client/tests/test_client.py +++ b/sdk/python-client/tests/test_client.py @@ -39,10 +39,13 @@ SELF_ASSIGN, TARGETED, create_topic_agent, + create_topic_service, create_topic_task, create_topic_user, + create_topic_user_broadcast, create_topic_global_agents, create_topic_global_users, + SERVICE_WILDCARD, ) from scitrera_aether_client.exceptions import ( AuthenticationError, @@ -1398,6 +1401,11 @@ def test_create_topic_user(self): topic = create_topic_user("user-123", "window-456") assert topic == "us::user-123::window-456" + def test_create_topic_user_broadcast(self): + """Test per-user broadcast topic creation.""" + topic = create_topic_user_broadcast("user-123") + assert topic == "uu::user-123" + def test_create_topic_global_agents(self): """Test global agents broadcast topic creation.""" topic = create_topic_global_agents("workspace") @@ -1408,6 +1416,36 @@ def test_create_topic_global_users(self): topic = create_topic_global_users("workspace") assert topic == "gu::workspace" + def test_create_topic_service_concrete(self): + """A concrete specifier emits the canonical 3-segment service topic.""" + topic = create_topic_service("platform-bridge", "bridge-1") + assert topic == "sv::platform-bridge::bridge-1" + + def test_create_topic_service_empty_specifier_stays_concrete(self): + """An empty-string specifier stays 3-segment (NOT a wildcard). + + ``sv::platform-bridge::`` (trailing ``::``) is a concrete target the + gateway treats as non-wildcard — guarding against the trailing-``::`` + footgun where "" silently became a wildcard. + """ + topic = create_topic_service("platform-bridge", "") + assert topic == "sv::platform-bridge::" + + def test_create_topic_service_wildcard_sentinel(self): + """The SERVICE_WILDCARD sentinel emits the bare 2-segment wildcard. + + ``sv::platform-bridge`` is what the gateway's ParseSendTarget resolves + to any registered ``sv::platform-bridge::`` instance. + """ + topic = create_topic_service("platform-bridge", SERVICE_WILDCARD) + assert topic == "sv::platform-bridge" + + def test_service_wildcard_is_singleton(self): + """SERVICE_WILDCARD is a stable, repr-friendly identity sentinel.""" + assert repr(SERVICE_WILDCARD) == "SERVICE_WILDCARD" + # Identity comparison is the intended check (used by create_topic_service). + assert SERVICE_WILDCARD is SERVICE_WILDCARD + # ============================================================================= # Message Type Tests @@ -2069,6 +2107,29 @@ def test_init_creates_init_proto(self): assert client.init.service.implementation == "my-svc" assert client.init.service.specifier == "pod-1" + def test_init_default_is_pool_consumer(self): + """Default ServiceClient is a pool consumer (no_pool_consumer=False, back-compat).""" + from scitrera_aether_client.client import ServiceClient + client = ServiceClient(implementation="my-svc", specifier="pod-1") + assert client.init.service.no_pool_consumer is False + + def test_init_consumes_pool_tasks_false_sets_no_pool_consumer(self): + """consumes_pool_tasks=False opts the service out of pool-task routing.""" + from scitrera_aether_client.client import ServiceClient + client = ServiceClient(implementation="my-svc", specifier="pod-1", + consumes_pool_tasks=False) + assert client.init.service.no_pool_consumer is True + + def test_async_init_consumes_pool_tasks_maps_no_pool_consumer(self): + """AsyncServiceClient mirrors the mapping: consume=True->False, False->True.""" + from scitrera_aether_client.client_async import AsyncServiceClient + worker = AsyncServiceClient(implementation="memorylayer", specifier="wkr", + consumes_pool_tasks=True) + server = AsyncServiceClient(implementation="memorylayer", specifier="srv", + consumes_pool_tasks=False) + assert worker.init.service.no_pool_consumer is False + assert server.init.service.no_pool_consumer is True + def test_init_empty_implementation_raises(self): """ServiceClient raises InvalidArgumentError for empty implementation.""" from scitrera_aether_client.client import ServiceClient @@ -2133,6 +2194,15 @@ def test_send_message_to_user_workspace_queues_message(self): assert msg.HasField("send") assert msg.send.target_topic == "uw::user-1::workspace-a" + def test_send_message_to_user_broadcast_queues_message(self): + """send_message_to_user_broadcast puts correct UpstreamMessage on request_queue.""" + from scitrera_aether_client.client import ServiceClient + client = ServiceClient(implementation="my-svc", specifier="pod-1") + client.send_message_to_user_broadcast("user-1", b"payload") + msg = client.request_queue.get_nowait() + assert msg.HasField("send") + assert msg.send.target_topic == "uu::user-1" + def test_exported_from_package(self): """ServiceClient is importable from the top-level package.""" import scitrera_aether_client diff --git a/sdk/python-client/tests/test_proxy.py b/sdk/python-client/tests/test_proxy.py index 5c7f93b..f23b25a 100644 --- a/sdk/python-client/tests/test_proxy.py +++ b/sdk/python-client/tests/test_proxy.py @@ -577,4 +577,73 @@ async def _producer(): r = await http.get("/v1/hello") await producer_task assert r.status_code == 200 - assert r.content == b"hi" + + +# --------------------------------------------------------------------------- +# Defensive body=None normalisation (Task #26) +# --------------------------------------------------------------------------- + +def test_proxy_http_body_none_does_not_raise(): + """proxy_http must not raise TypeError when body=None is passed explicitly. + + The signature default is b"" but callers may pass None and rely on the + defensive normalisation added at the top of the function body. + """ + client = _SyncClientStub() + client._proxy_dispatcher = proxy_mod._ProxyDispatcher() + + request_id = "test-none-body-sync" + + def _responder(): + import time as _t + _t.sleep(0.05) + client._proxy_dispatcher.handle_response( + _build_response(request_id, status=200, body=b"ok") + ) + + import threading + t = threading.Thread(target=_responder, daemon=True) + t.start() + + # Should not raise TypeError: object of type 'NoneType' has no len() + result = proxy_http( + client, + target_topic="sv::test::default", + method="GET", + path="/healthz", + body=None, # type: ignore[arg-type] -- intentional None + request_id=request_id, + timeout=5.0, + ) + t.join(timeout=2.0) + assert result.status_code == 200 + + +@pytest.mark.asyncio +async def test_proxy_http_async_body_none_does_not_raise(): + """proxy_http_async must not raise TypeError when body=None is passed explicitly.""" + client = _AsyncClientStub() + client._proxy_dispatcher = proxy_mod._ProxyDispatcher() + + request_id = "test-none-body-async" + + async def _responder(): + await asyncio.sleep(0.05) + client._proxy_dispatcher.handle_response( + _build_response(request_id, status=200, body=b"ok") + ) + + responder_task = asyncio.create_task(_responder()) + + # Should not raise TypeError: object of type 'NoneType' has no len() + result = await proxy_http_async( + client, + target_topic="sv::test::default", + method="GET", + path="/healthz", + body=None, # type: ignore[arg-type] -- intentional None + request_id=request_id, + timeout=5.0, + ) + await responder_task + assert result.status_code == 200 diff --git a/sdk/python-client/tests/test_proxy_terminator.py b/sdk/python-client/tests/test_proxy_terminator.py index 6f3e8c8..34b4dba 100644 --- a/sdk/python-client/tests/test_proxy_terminator.py +++ b/sdk/python-client/tests/test_proxy_terminator.py @@ -224,6 +224,55 @@ async def handler(req: MintedRequest) -> aether_pb2.ProxyHttpResponse: assert not resp.body_chunked +@pytest.mark.asyncio +async def test_strict_mode_stamps_configured_tenant_and_overrides_spoof(): + """tenant_id is minted as X-Auth-Tenant-ID and a caller value cannot spoof it.""" + client = _AsyncClientStub() + received: List[MintedRequest] = [] + + async def handler(req: MintedRequest) -> aether_pb2.ProxyHttpResponse: + received.append(req) + return aether_pb2.ProxyHttpResponse(request_id=req.request_id, status_code=200, body=b"ok") + + term = ProxyHttpTerminator( + client=client, handler=handler, allow_paths=["/v1/*"], + header_mode="strict", tenant_id="botwinick", + ) + await term.start() + + # Caller tries to spoof a different tenant; strict mode strips inbound + # x-auth-* then the terminator stamps the configured tenant authoritatively. + req = _build_request("req-t", path="/v1/echo", headers={"X-Auth-Tenant-ID": "attacker"}) + dispatcher = _get_terminator_dispatcher(client) + await dispatcher.handle_request(req) + await dispatcher.wait_idle() + + assert len(received) == 1 + assert received[0].headers["X-Auth-Tenant-ID"] == "botwinick" + + +@pytest.mark.asyncio +async def test_strict_mode_without_tenant_id_stamps_no_tenant(): + """Default (tenant_id=None) preserves the prior behaviour: no tenant header.""" + client = _AsyncClientStub() + received: List[MintedRequest] = [] + + async def handler(req: MintedRequest) -> aether_pb2.ProxyHttpResponse: + received.append(req) + return aether_pb2.ProxyHttpResponse(request_id=req.request_id, status_code=200, body=b"ok") + + term = ProxyHttpTerminator(client=client, handler=handler, allow_paths=["/v1/*"], header_mode="strict") + await term.start() + + req = _build_request("req-nt", path="/v1/echo") + dispatcher = _get_terminator_dispatcher(client) + await dispatcher.handle_request(req) + await dispatcher.wait_idle() + + assert len(received) == 1 + assert "X-Auth-Tenant-ID" not in received[0].headers + + @pytest.mark.asyncio async def test_chunked_request_reassembled_in_seq_order(): client = _AsyncClientStub() diff --git a/sdk/typescript/package.json b/sdk/typescript/package.json index 114522b..739e16c 100644 --- a/sdk/typescript/package.json +++ b/sdk/typescript/package.json @@ -1,6 +1,6 @@ { "name": "@scitrera/aether-client", - "version": "0.2.1", + "version": "0.2.2", "description": "TypeScript/JavaScript SDK for the Aether distributed control plane", "license": "Apache-2.0", "author": "scitrera.ai", diff --git a/sdk/typescript/src/__tests__/client.test.ts b/sdk/typescript/src/__tests__/client.test.ts index e63b882..2e854ee 100644 --- a/sdk/typescript/src/__tests__/client.test.ts +++ b/sdk/typescript/src/__tests__/client.test.ts @@ -14,6 +14,7 @@ import { taskBroadcastTopic, userTopic, userWorkspaceTopic, + userBroadcastTopic, globalUsersTopic, eventTopic, eventWildcardTopic, @@ -25,6 +26,7 @@ import { TOPIC_PREFIX_TASK_BROADCAST, TOPIC_PREFIX_USER, TOPIC_PREFIX_USER_WORKSPACE, + TOPIC_PREFIX_USER_BROADCAST, TOPIC_PREFIX_GLOBAL_AGENTS, TOPIC_PREFIX_GLOBAL_USERS, TOPIC_PREFIX_EVENT, @@ -144,6 +146,7 @@ describe("Topic Construction", () => { expect(TOPIC_PREFIX_TASK_BROADCAST).toBe("tb"); expect(TOPIC_PREFIX_USER).toBe("us"); expect(TOPIC_PREFIX_USER_WORKSPACE).toBe("uw"); + expect(TOPIC_PREFIX_USER_BROADCAST).toBe("uu"); expect(TOPIC_PREFIX_GLOBAL_AGENTS).toBe("ga"); expect(TOPIC_PREFIX_GLOBAL_USERS).toBe("gu"); expect(TOPIC_PREFIX_EVENT).toBe("event"); @@ -200,6 +203,12 @@ describe("Topic Construction", () => { }); }); + describe("userBroadcastTopic", () => { + it("creates correct user broadcast topic", () => { + expect(userBroadcastTopic("alice")).toBe("uu::alice"); + }); + }); + describe("globalUsersTopic", () => { it("creates correct global users topic", () => { expect(globalUsersTopic("prod")).toBe("gu::prod"); diff --git a/sdk/typescript/src/agents.ts b/sdk/typescript/src/agents.ts index 26d9472..add6381 100644 --- a/sdk/typescript/src/agents.ts +++ b/sdk/typescript/src/agents.ts @@ -11,7 +11,7 @@ import { AetherClient } from "./client.js"; import type { AetherClientOptions } from "./client.js"; -import { MessageType, TaskAssignmentMode } from "./types.js"; +import { MessageType, TaskAssignmentMode, TaskPriority } from "./types.js"; import type { MessageHandler } from "./types.js"; import { InvalidArgumentError } from "./errors.js"; import { @@ -65,6 +65,12 @@ export interface CreateTaskOptions { metadata?: Record; /** Assignment mode. Default: SelfAssign. */ assignmentMode?: TaskAssignmentMode; + /** + * Optional dispatch priority. Higher priority pending tasks are delivered + * before lower ones. Defaults to Unspecified, which the server normalizes + * to Normal. + */ + priority?: TaskPriority; } // ============================================================================= @@ -387,6 +393,7 @@ export class AgentClient extends AetherClient { targetImplementation: opts.targetImplementation ?? "", launchParamOverrides: opts.launchParamOverrides ?? {}, metadata: opts.metadata ?? {}, + priority: opts.priority ?? TaskPriority.Unspecified, }, }); } diff --git a/sdk/typescript/src/bridge.ts b/sdk/typescript/src/bridge.ts index d5d20f3..7b5cae0 100644 --- a/sdk/typescript/src/bridge.ts +++ b/sdk/typescript/src/bridge.ts @@ -24,6 +24,7 @@ import { taskBroadcastTopic, userTopic, userWorkspaceTopic, + userBroadcastTopic, globalAgentsTopic, globalUsersTopic, } from "./topics.js"; @@ -228,6 +229,25 @@ export class BridgeClient extends AetherClient { this._sendMessage(userWorkspaceTopic(userId, workspace), payload, messageType); } + /** + * Sends a message to every one of a user's windows, regardless of which + * workspace each window is viewing (topic uu::{userId}). + * + * This is the workspace-agnostic, non-progress channel for platform→user + * notifications. + * + * @param userId - Target user's ID + * @param payload - Message payload (bytes) + * @param messageType - Message type. Default: Opaque + */ + sendToUserBroadcast( + userId: string, + payload: Uint8Array, + messageType: MessageType = MessageType.Opaque, + ): void { + this._sendMessage(userBroadcastTopic(userId), payload, messageType); + } + // =========================================================================== // Broadcast Helpers // =========================================================================== diff --git a/sdk/typescript/src/index.ts b/sdk/typescript/src/index.ts index c12f0a6..285d30b 100644 --- a/sdk/typescript/src/index.ts +++ b/sdk/typescript/src/index.ts @@ -234,6 +234,7 @@ export { taskBroadcastTopic, userTopic, userWorkspaceTopic, + userBroadcastTopic, globalUsersTopic, eventTopic, eventWildcardTopic, @@ -248,6 +249,7 @@ export { TOPIC_PREFIX_TASK_BROADCAST, TOPIC_PREFIX_USER, TOPIC_PREFIX_USER_WORKSPACE, + TOPIC_PREFIX_USER_BROADCAST, TOPIC_PREFIX_GLOBAL_AGENTS, TOPIC_PREFIX_GLOBAL_USERS, TOPIC_PREFIX_EVENT, diff --git a/sdk/typescript/src/proto/aether.ts b/sdk/typescript/src/proto/aether.ts index 9a54be8..7b56f53 100644 --- a/sdk/typescript/src/proto/aether.ts +++ b/sdk/typescript/src/proto/aether.ts @@ -10,6 +10,8 @@ type SubtypeConstructor any, Subtype> export interface ProtoGrpcType { aether: { v1: { + ACLAccessContributionInfo: MessageTypeDefinition + ACLAccessExplanationInfo: MessageTypeDefinition ACLAuditEntryInfo: MessageTypeDefinition ACLAuditFilter: MessageTypeDefinition ACLAuthorityGrantFilter: MessageTypeDefinition @@ -19,9 +21,17 @@ export interface ProtoGrpcType { ACLCleanupResult: MessageTypeDefinition ACLFallbackPolicyInfo: MessageTypeDefinition ACLGrantRequest: MessageTypeDefinition + ACLGroupInfo: MessageTypeDefinition + ACLGroupMemberInfo: MessageTypeDefinition + ACLGroupMemberRequest: MessageTypeDefinition + ACLGroupRequest: MessageTypeDefinition ACLOperation: MessageTypeDefinition ACLRenewAuthorityGrantRequest: MessageTypeDefinition ACLResponse: MessageTypeDefinition + ACLRoleAssignmentInfo: MessageTypeDefinition + ACLRoleAssignmentRequest: MessageTypeDefinition + ACLRoleInfo: MessageTypeDefinition + ACLRoleRequest: MessageTypeDefinition ACLRuleFilter: MessageTypeDefinition ACLRuleInfo: MessageTypeDefinition ACLSetFallbackRequest: MessageTypeDefinition @@ -60,6 +70,7 @@ export interface ProtoGrpcType { AuthorityRequestStatus: EnumTypeDefinition AuthoritySpan: MessageTypeDefinition AuthorizationContext: MessageTypeDefinition + BackoffStrategy: EnumTypeDefinition BridgeIdentity: MessageTypeDefinition BuildInfo: MessageTypeDefinition CheckpointOperation: MessageTypeDefinition @@ -113,6 +124,7 @@ export interface ProtoGrpcType { ResolveAuthorityResponse: MessageTypeDefinition ResolvedAuthority: MessageTypeDefinition ResolvedAuthorityInfo: MessageTypeDefinition + RetryPolicy: MessageTypeDefinition SendMessage: MessageTypeDefinition ServiceIdentity: MessageTypeDefinition SessionOperation: MessageTypeDefinition @@ -126,6 +138,7 @@ export interface ProtoGrpcType { TaskAuthorityRequestEventRelay: MessageTypeDefinition TaskChildLifecycleEvent: MessageTypeDefinition TaskClass: EnumTypeDefinition + TaskCompletionEvent: MessageTypeDefinition TaskEvent: MessageTypeDefinition TaskFilter: MessageTypeDefinition TaskHibernated: MessageTypeDefinition @@ -133,6 +146,7 @@ export interface ProtoGrpcType { TaskInfo: MessageTypeDefinition TaskOperation: MessageTypeDefinition TaskOperationResponse: MessageTypeDefinition + TaskPriority: EnumTypeDefinition TaskProgressEvent: MessageTypeDefinition TaskQuery: MessageTypeDefinition TaskQueryResponse: MessageTypeDefinition diff --git a/sdk/typescript/src/proto/aether/v1/ACLAccessContributionInfo.ts b/sdk/typescript/src/proto/aether/v1/ACLAccessContributionInfo.ts new file mode 100644 index 0000000..8aab591 --- /dev/null +++ b/sdk/typescript/src/proto/aether/v1/ACLAccessContributionInfo.ts @@ -0,0 +1,38 @@ +// Original file: aether.proto + + +/** + * ACLAccessContributionInfo is one rule that matched a principal or one of its + * groups/roles for an explained resource. + */ +export interface ACLAccessContributionInfo { + /** + * granting subject, e.g. "role:wsadmin" + */ + 'subject'?: (string); + 'ruleId'?: (string); + 'accessLevel'?: (number); + /** + * matched object pattern, e.g. "workspace:prod" or "workspace:*" + */ + 'resource'?: (string); + 'expired'?: (boolean); +} + +/** + * ACLAccessContributionInfo is one rule that matched a principal or one of its + * groups/roles for an explained resource. + */ +export interface ACLAccessContributionInfo__Output { + /** + * granting subject, e.g. "role:wsadmin" + */ + 'subject': (string); + 'ruleId': (string); + 'accessLevel': (number); + /** + * matched object pattern, e.g. "workspace:prod" or "workspace:*" + */ + 'resource': (string); + 'expired': (boolean); +} diff --git a/sdk/typescript/src/proto/aether/v1/ACLAccessExplanationInfo.ts b/sdk/typescript/src/proto/aether/v1/ACLAccessExplanationInfo.ts new file mode 100644 index 0000000..c04bf4d --- /dev/null +++ b/sdk/typescript/src/proto/aether/v1/ACLAccessExplanationInfo.ts @@ -0,0 +1,53 @@ +// Original file: aether.proto + +import type { ACLAccessContributionInfo as _aether_v1_ACLAccessContributionInfo, ACLAccessContributionInfo__Output as _aether_v1_ACLAccessContributionInfo__Output } from '../../aether/v1/ACLAccessContributionInfo'; + +/** + * ACLAccessExplanationInfo explains how a principal's effective access to a + * resource is decided (EXPLAIN_ACCESS). The gateway records an "explain_access" + * audit event attributing the call to the connected principal. + */ +export interface ACLAccessExplanationInfo { + /** + * self subject "type:id" + */ + 'principal'?: (string); + /** + * self + transitive groups/roles + */ + 'subjects'?: (string)[]; + 'contributions'?: (_aether_v1_ACLAccessContributionInfo)[]; + 'allowed'?: (boolean); + /** + * "ALLOW" / "DENY" + */ + 'decision'?: (string); + 'effectiveAccessLevel'?: (number); + 'fallbackApplied'?: (boolean); + 'reason'?: (string); +} + +/** + * ACLAccessExplanationInfo explains how a principal's effective access to a + * resource is decided (EXPLAIN_ACCESS). The gateway records an "explain_access" + * audit event attributing the call to the connected principal. + */ +export interface ACLAccessExplanationInfo__Output { + /** + * self subject "type:id" + */ + 'principal': (string); + /** + * self + transitive groups/roles + */ + 'subjects': (string)[]; + 'contributions': (_aether_v1_ACLAccessContributionInfo__Output)[]; + 'allowed': (boolean); + /** + * "ALLOW" / "DENY" + */ + 'decision': (string); + 'effectiveAccessLevel': (number); + 'fallbackApplied': (boolean); + 'reason': (string); +} diff --git a/sdk/typescript/src/proto/aether/v1/ACLGroupInfo.ts b/sdk/typescript/src/proto/aether/v1/ACLGroupInfo.ts new file mode 100644 index 0000000..f420244 --- /dev/null +++ b/sdk/typescript/src/proto/aether/v1/ACLGroupInfo.ts @@ -0,0 +1,27 @@ +// Original file: aether.proto + +import type { Long } from '@grpc/proto-loader'; + +/** + * ACLGroupInfo represents a group definition. + */ +export interface ACLGroupInfo { + 'groupId'?: (string); + 'groupName'?: (string); + 'description'?: (string); + 'createdBy'?: (string); + 'createdAt'?: (number | string | Long); + 'metadata'?: ({[key: string]: string}); +} + +/** + * ACLGroupInfo represents a group definition. + */ +export interface ACLGroupInfo__Output { + 'groupId': (string); + 'groupName': (string); + 'description': (string); + 'createdBy': (string); + 'createdAt': (string); + 'metadata': ({[key: string]: string}); +} diff --git a/sdk/typescript/src/proto/aether/v1/ACLGroupMemberInfo.ts b/sdk/typescript/src/proto/aether/v1/ACLGroupMemberInfo.ts new file mode 100644 index 0000000..69ab2f5 --- /dev/null +++ b/sdk/typescript/src/proto/aether/v1/ACLGroupMemberInfo.ts @@ -0,0 +1,27 @@ +// Original file: aether.proto + +import type { Long } from '@grpc/proto-loader'; + +/** + * ACLGroupMemberInfo represents a single group-membership edge. + */ +export interface ACLGroupMemberInfo { + 'groupName'?: (string); + 'memberType'?: (string); + 'memberId'?: (string); + 'grantedBy'?: (string); + 'grantedAt'?: (number | string | Long); + 'expiresAt'?: (number | string | Long); +} + +/** + * ACLGroupMemberInfo represents a single group-membership edge. + */ +export interface ACLGroupMemberInfo__Output { + 'groupName': (string); + 'memberType': (string); + 'memberId': (string); + 'grantedBy': (string); + 'grantedAt': (string); + 'expiresAt': (string); +} diff --git a/sdk/typescript/src/proto/aether/v1/ACLGroupMemberRequest.ts b/sdk/typescript/src/proto/aether/v1/ACLGroupMemberRequest.ts new file mode 100644 index 0000000..7b25164 --- /dev/null +++ b/sdk/typescript/src/proto/aether/v1/ACLGroupMemberRequest.ts @@ -0,0 +1,35 @@ +// Original file: aether.proto + +import type { Long } from '@grpc/proto-loader'; + +/** + * ACLGroupMemberRequest contains data for adding a member to a group. + */ +export interface ACLGroupMemberRequest { + /** + * principal type or "group" (nesting) + */ + 'memberType'?: (string); + 'memberId'?: (string); + 'grantedBy'?: (string); + /** + * Unix timestamp, 0 = no expiration + */ + 'expiresAt'?: (number | string | Long); +} + +/** + * ACLGroupMemberRequest contains data for adding a member to a group. + */ +export interface ACLGroupMemberRequest__Output { + /** + * principal type or "group" (nesting) + */ + 'memberType': (string); + 'memberId': (string); + 'grantedBy': (string); + /** + * Unix timestamp, 0 = no expiration + */ + 'expiresAt': (string); +} diff --git a/sdk/typescript/src/proto/aether/v1/ACLGroupRequest.ts b/sdk/typescript/src/proto/aether/v1/ACLGroupRequest.ts new file mode 100644 index 0000000..9e87e7f --- /dev/null +++ b/sdk/typescript/src/proto/aether/v1/ACLGroupRequest.ts @@ -0,0 +1,22 @@ +// Original file: aether.proto + + +/** + * ACLGroupRequest contains data for creating a group. + */ +export interface ACLGroupRequest { + 'name'?: (string); + 'description'?: (string); + 'createdBy'?: (string); + 'metadata'?: ({[key: string]: string}); +} + +/** + * ACLGroupRequest contains data for creating a group. + */ +export interface ACLGroupRequest__Output { + 'name': (string); + 'description': (string); + 'createdBy': (string); + 'metadata': ({[key: string]: string}); +} diff --git a/sdk/typescript/src/proto/aether/v1/ACLOperation.ts b/sdk/typescript/src/proto/aether/v1/ACLOperation.ts index a8b139b..a671f5d 100644 --- a/sdk/typescript/src/proto/aether/v1/ACLOperation.ts +++ b/sdk/typescript/src/proto/aether/v1/ACLOperation.ts @@ -4,6 +4,12 @@ import type { ACLRuleFilter as _aether_v1_ACLRuleFilter, ACLRuleFilter__Output a import type { ACLAuditFilter as _aether_v1_ACLAuditFilter, ACLAuditFilter__Output as _aether_v1_ACLAuditFilter__Output } from '../../aether/v1/ACLAuditFilter'; import type { ACLGrantRequest as _aether_v1_ACLGrantRequest, ACLGrantRequest__Output as _aether_v1_ACLGrantRequest__Output } from '../../aether/v1/ACLGrantRequest'; import type { ACLSetFallbackRequest as _aether_v1_ACLSetFallbackRequest, ACLSetFallbackRequest__Output as _aether_v1_ACLSetFallbackRequest__Output } from '../../aether/v1/ACLSetFallbackRequest'; +import type { PrincipalRef as _aether_v1_PrincipalRef, PrincipalRef__Output as _aether_v1_PrincipalRef__Output } from '../../aether/v1/PrincipalRef'; +import type { ACLGroupRequest as _aether_v1_ACLGroupRequest, ACLGroupRequest__Output as _aether_v1_ACLGroupRequest__Output } from '../../aether/v1/ACLGroupRequest'; +import type { ACLRoleRequest as _aether_v1_ACLRoleRequest, ACLRoleRequest__Output as _aether_v1_ACLRoleRequest__Output } from '../../aether/v1/ACLRoleRequest'; +import type { ACLGroupMemberRequest as _aether_v1_ACLGroupMemberRequest, ACLGroupMemberRequest__Output as _aether_v1_ACLGroupMemberRequest__Output } from '../../aether/v1/ACLGroupMemberRequest'; +import type { ACLRoleAssignmentRequest as _aether_v1_ACLRoleAssignmentRequest, ACLRoleAssignmentRequest__Output as _aether_v1_ACLRoleAssignmentRequest__Output } from '../../aether/v1/ACLRoleAssignmentRequest'; +import type { AuthorizationContext as _aether_v1_AuthorizationContext, AuthorizationContext__Output as _aether_v1_AuthorizationContext__Output } from '../../aether/v1/AuthorizationContext'; // Original file: aether.proto @@ -44,6 +50,75 @@ export const _aether_v1_ACLOperation_OpType = { * POST /api/acl/cleanup/audit-logs - Remove old audit log entries */ CLEANUP_AUDIT_LOGS: 'CLEANUP_AUDIT_LOGS', + /** + * Role/group authorization. REST equivalents under /api/acl/groups, + * /api/acl/roles, and /api/acl/principals/{type}/{id}/{groups,roles}. + */ + CREATE_GROUP: 'CREATE_GROUP', + /** + * Delete a group (name) + */ + DELETE_GROUP: 'DELETE_GROUP', + /** + * Get a group (name) + */ + GET_GROUP: 'GET_GROUP', + /** + * List all groups + */ + LIST_GROUPS: 'LIST_GROUPS', + /** + * Add a member to a group (name + member_request) + */ + ADD_GROUP_MEMBER: 'ADD_GROUP_MEMBER', + /** + * Remove a member from a group (name + principal) + */ + REMOVE_GROUP_MEMBER: 'REMOVE_GROUP_MEMBER', + /** + * List members of a group (name) + */ + LIST_GROUP_MEMBERS: 'LIST_GROUP_MEMBERS', + /** + * Create a role (role_request) + */ + CREATE_ROLE: 'CREATE_ROLE', + /** + * Delete a role (name) + */ + DELETE_ROLE: 'DELETE_ROLE', + /** + * Get a role (name) + */ + GET_ROLE: 'GET_ROLE', + /** + * List all roles + */ + LIST_ROLES: 'LIST_ROLES', + /** + * Assign a role (name + assignment_request) + */ + ASSIGN_ROLE: 'ASSIGN_ROLE', + /** + * Unassign a role (name + principal) + */ + UNASSIGN_ROLE: 'UNASSIGN_ROLE', + /** + * List assignees of a role (name) + */ + LIST_ROLE_ASSIGNMENTS: 'LIST_ROLE_ASSIGNMENTS', + /** + * List groups a principal belongs to (principal) + */ + LIST_PRINCIPAL_GROUPS: 'LIST_PRINCIPAL_GROUPS', + /** + * List roles assigned to a principal (principal) + */ + LIST_PRINCIPAL_ROLES: 'LIST_PRINCIPAL_ROLES', + /** + * Explain effective access (principal + resource_type + resource_id [+ required_level]) + */ + EXPLAIN_ACCESS: 'EXPLAIN_ACCESS', } as const; export type _aether_v1_ACLOperation_OpType = @@ -92,6 +167,92 @@ export type _aether_v1_ACLOperation_OpType = */ | 'CLEANUP_AUDIT_LOGS' | 11 + /** + * Role/group authorization. REST equivalents under /api/acl/groups, + * /api/acl/roles, and /api/acl/principals/{type}/{id}/{groups,roles}. + */ + | 'CREATE_GROUP' + | 17 + /** + * Delete a group (name) + */ + | 'DELETE_GROUP' + | 18 + /** + * Get a group (name) + */ + | 'GET_GROUP' + | 19 + /** + * List all groups + */ + | 'LIST_GROUPS' + | 20 + /** + * Add a member to a group (name + member_request) + */ + | 'ADD_GROUP_MEMBER' + | 21 + /** + * Remove a member from a group (name + principal) + */ + | 'REMOVE_GROUP_MEMBER' + | 22 + /** + * List members of a group (name) + */ + | 'LIST_GROUP_MEMBERS' + | 23 + /** + * Create a role (role_request) + */ + | 'CREATE_ROLE' + | 24 + /** + * Delete a role (name) + */ + | 'DELETE_ROLE' + | 25 + /** + * Get a role (name) + */ + | 'GET_ROLE' + | 26 + /** + * List all roles + */ + | 'LIST_ROLES' + | 27 + /** + * Assign a role (name + assignment_request) + */ + | 'ASSIGN_ROLE' + | 28 + /** + * Unassign a role (name + principal) + */ + | 'UNASSIGN_ROLE' + | 29 + /** + * List assignees of a role (name) + */ + | 'LIST_ROLE_ASSIGNMENTS' + | 30 + /** + * List groups a principal belongs to (principal) + */ + | 'LIST_PRINCIPAL_GROUPS' + | 31 + /** + * List roles assigned to a principal (principal) + */ + | 'LIST_PRINCIPAL_ROLES' + | 32 + /** + * Explain effective access (principal + resource_type + resource_id [+ required_level]) + */ + | 'EXPLAIN_ACCESS' + | 33 export type _aether_v1_ACLOperation_OpType__Output = typeof _aether_v1_ACLOperation_OpType[keyof typeof _aether_v1_ACLOperation_OpType] @@ -149,6 +310,46 @@ export interface ACLOperation { * Client-generated correlation ID for matching responses to requests */ 'requestId'?: (string); + /** + * Role/group fields. + * name: group/role name for GET/DELETE/LIST_*_MEMBERS/ASSIGNMENTS/ADD/ASSIGN. + */ + 'name'?: (string); + /** + * principal: member/assignee for REMOVE_GROUP_MEMBER, UNASSIGN_ROLE, and + * the subject for LIST_PRINCIPAL_GROUPS / LIST_PRINCIPAL_ROLES. + */ + 'principal'?: (_aether_v1_PrincipalRef | null); + /** + * CREATE_GROUP + */ + 'groupRequest'?: (_aether_v1_ACLGroupRequest | null); + /** + * CREATE_ROLE + */ + 'roleRequest'?: (_aether_v1_ACLRoleRequest | null); + /** + * ADD_GROUP_MEMBER + */ + 'memberRequest'?: (_aether_v1_ACLGroupMemberRequest | null); + /** + * ASSIGN_ROLE + */ + 'assignmentRequest'?: (_aether_v1_ACLRoleAssignmentRequest | null); + /** + * EXPLAIN_ACCESS: the resource to explain against (principal carries the + * subject; required_level is the threshold the decision is compared to). + */ + 'resourceType'?: (string); + 'resourceId'?: (string); + 'requiredLevel'?: (number); + /** + * Optional on-behalf-of authority context. When set, the gateway runs the + * admin ACL check against the subject (the user) rather than the actor + * (the platform-server). Mirrors SessionOperation.authorization (field 6) + * and AuditQuery.authorization (field 18). + */ + 'authorization'?: (_aether_v1_AuthorizationContext | null); } /** @@ -205,4 +406,44 @@ export interface ACLOperation__Output { * Client-generated correlation ID for matching responses to requests */ 'requestId': (string); + /** + * Role/group fields. + * name: group/role name for GET/DELETE/LIST_*_MEMBERS/ASSIGNMENTS/ADD/ASSIGN. + */ + 'name': (string); + /** + * principal: member/assignee for REMOVE_GROUP_MEMBER, UNASSIGN_ROLE, and + * the subject for LIST_PRINCIPAL_GROUPS / LIST_PRINCIPAL_ROLES. + */ + 'principal': (_aether_v1_PrincipalRef__Output | null); + /** + * CREATE_GROUP + */ + 'groupRequest': (_aether_v1_ACLGroupRequest__Output | null); + /** + * CREATE_ROLE + */ + 'roleRequest': (_aether_v1_ACLRoleRequest__Output | null); + /** + * ADD_GROUP_MEMBER + */ + 'memberRequest': (_aether_v1_ACLGroupMemberRequest__Output | null); + /** + * ASSIGN_ROLE + */ + 'assignmentRequest': (_aether_v1_ACLRoleAssignmentRequest__Output | null); + /** + * EXPLAIN_ACCESS: the resource to explain against (principal carries the + * subject; required_level is the threshold the decision is compared to). + */ + 'resourceType': (string); + 'resourceId': (string); + 'requiredLevel': (number); + /** + * Optional on-behalf-of authority context. When set, the gateway runs the + * admin ACL check against the subject (the user) rather than the actor + * (the platform-server). Mirrors SessionOperation.authorization (field 6) + * and AuditQuery.authorization (field 18). + */ + 'authorization': (_aether_v1_AuthorizationContext__Output | null); } diff --git a/sdk/typescript/src/proto/aether/v1/ACLResponse.ts b/sdk/typescript/src/proto/aether/v1/ACLResponse.ts index 4f72310..1282af1 100644 --- a/sdk/typescript/src/proto/aether/v1/ACLResponse.ts +++ b/sdk/typescript/src/proto/aether/v1/ACLResponse.ts @@ -5,6 +5,11 @@ import type { ACLFallbackPolicyInfo as _aether_v1_ACLFallbackPolicyInfo, ACLFall import type { ACLAuditEntryInfo as _aether_v1_ACLAuditEntryInfo, ACLAuditEntryInfo__Output as _aether_v1_ACLAuditEntryInfo__Output } from '../../aether/v1/ACLAuditEntryInfo'; import type { ACLCleanupResult as _aether_v1_ACLCleanupResult, ACLCleanupResult__Output as _aether_v1_ACLCleanupResult__Output } from '../../aether/v1/ACLCleanupResult'; import type { ACLAuthorityGrantInfo as _aether_v1_ACLAuthorityGrantInfo, ACLAuthorityGrantInfo__Output as _aether_v1_ACLAuthorityGrantInfo__Output } from '../../aether/v1/ACLAuthorityGrantInfo'; +import type { ACLGroupInfo as _aether_v1_ACLGroupInfo, ACLGroupInfo__Output as _aether_v1_ACLGroupInfo__Output } from '../../aether/v1/ACLGroupInfo'; +import type { ACLRoleInfo as _aether_v1_ACLRoleInfo, ACLRoleInfo__Output as _aether_v1_ACLRoleInfo__Output } from '../../aether/v1/ACLRoleInfo'; +import type { ACLGroupMemberInfo as _aether_v1_ACLGroupMemberInfo, ACLGroupMemberInfo__Output as _aether_v1_ACLGroupMemberInfo__Output } from '../../aether/v1/ACLGroupMemberInfo'; +import type { ACLRoleAssignmentInfo as _aether_v1_ACLRoleAssignmentInfo, ACLRoleAssignmentInfo__Output as _aether_v1_ACLRoleAssignmentInfo__Output } from '../../aether/v1/ACLRoleAssignmentInfo'; +import type { ACLAccessExplanationInfo as _aether_v1_ACLAccessExplanationInfo, ACLAccessExplanationInfo__Output as _aether_v1_ACLAccessExplanationInfo__Output } from '../../aether/v1/ACLAccessExplanationInfo'; /** * ACLResponse is sent in response to ACLOperation. @@ -61,6 +66,34 @@ export interface ACLResponse { */ 'authorityGrants'?: (_aether_v1_ACLAuthorityGrantInfo)[]; 'totalAuthorityGrants'?: (number); + /** + * Role/group results. + */ + 'group'?: (_aether_v1_ACLGroupInfo | null); + /** + * LIST_GROUPS + */ + 'groups'?: (_aether_v1_ACLGroupInfo)[]; + /** + * GET_ROLE / CREATE_ROLE + */ + 'role'?: (_aether_v1_ACLRoleInfo | null); + /** + * LIST_ROLES + */ + 'roles'?: (_aether_v1_ACLRoleInfo)[]; + /** + * LIST_GROUP_MEMBERS / LIST_PRINCIPAL_GROUPS + */ + 'groupMembers'?: (_aether_v1_ACLGroupMemberInfo)[]; + /** + * LIST_ROLE_ASSIGNMENTS / LIST_PRINCIPAL_ROLES + */ + 'roleAssignments'?: (_aether_v1_ACLRoleAssignmentInfo)[]; + /** + * EXPLAIN_ACCESS + */ + 'explanation'?: (_aether_v1_ACLAccessExplanationInfo | null); } /** @@ -118,4 +151,32 @@ export interface ACLResponse__Output { */ 'authorityGrants': (_aether_v1_ACLAuthorityGrantInfo__Output)[]; 'totalAuthorityGrants': (number); + /** + * Role/group results. + */ + 'group': (_aether_v1_ACLGroupInfo__Output | null); + /** + * LIST_GROUPS + */ + 'groups': (_aether_v1_ACLGroupInfo__Output)[]; + /** + * GET_ROLE / CREATE_ROLE + */ + 'role': (_aether_v1_ACLRoleInfo__Output | null); + /** + * LIST_ROLES + */ + 'roles': (_aether_v1_ACLRoleInfo__Output)[]; + /** + * LIST_GROUP_MEMBERS / LIST_PRINCIPAL_GROUPS + */ + 'groupMembers': (_aether_v1_ACLGroupMemberInfo__Output)[]; + /** + * LIST_ROLE_ASSIGNMENTS / LIST_PRINCIPAL_ROLES + */ + 'roleAssignments': (_aether_v1_ACLRoleAssignmentInfo__Output)[]; + /** + * EXPLAIN_ACCESS + */ + 'explanation': (_aether_v1_ACLAccessExplanationInfo__Output | null); } diff --git a/sdk/typescript/src/proto/aether/v1/ACLRoleAssignmentInfo.ts b/sdk/typescript/src/proto/aether/v1/ACLRoleAssignmentInfo.ts new file mode 100644 index 0000000..2555862 --- /dev/null +++ b/sdk/typescript/src/proto/aether/v1/ACLRoleAssignmentInfo.ts @@ -0,0 +1,27 @@ +// Original file: aether.proto + +import type { Long } from '@grpc/proto-loader'; + +/** + * ACLRoleAssignmentInfo represents a single role-assignment edge. + */ +export interface ACLRoleAssignmentInfo { + 'roleName'?: (string); + 'assigneeType'?: (string); + 'assigneeId'?: (string); + 'grantedBy'?: (string); + 'grantedAt'?: (number | string | Long); + 'expiresAt'?: (number | string | Long); +} + +/** + * ACLRoleAssignmentInfo represents a single role-assignment edge. + */ +export interface ACLRoleAssignmentInfo__Output { + 'roleName': (string); + 'assigneeType': (string); + 'assigneeId': (string); + 'grantedBy': (string); + 'grantedAt': (string); + 'expiresAt': (string); +} diff --git a/sdk/typescript/src/proto/aether/v1/ACLRoleAssignmentRequest.ts b/sdk/typescript/src/proto/aether/v1/ACLRoleAssignmentRequest.ts new file mode 100644 index 0000000..5f7d111 --- /dev/null +++ b/sdk/typescript/src/proto/aether/v1/ACLRoleAssignmentRequest.ts @@ -0,0 +1,35 @@ +// Original file: aether.proto + +import type { Long } from '@grpc/proto-loader'; + +/** + * ACLRoleAssignmentRequest contains data for assigning a role. + */ +export interface ACLRoleAssignmentRequest { + /** + * principal type or "group" + */ + 'assigneeType'?: (string); + 'assigneeId'?: (string); + 'grantedBy'?: (string); + /** + * Unix timestamp, 0 = no expiration + */ + 'expiresAt'?: (number | string | Long); +} + +/** + * ACLRoleAssignmentRequest contains data for assigning a role. + */ +export interface ACLRoleAssignmentRequest__Output { + /** + * principal type or "group" + */ + 'assigneeType': (string); + 'assigneeId': (string); + 'grantedBy': (string); + /** + * Unix timestamp, 0 = no expiration + */ + 'expiresAt': (string); +} diff --git a/sdk/typescript/src/proto/aether/v1/ACLRoleInfo.ts b/sdk/typescript/src/proto/aether/v1/ACLRoleInfo.ts new file mode 100644 index 0000000..350abda --- /dev/null +++ b/sdk/typescript/src/proto/aether/v1/ACLRoleInfo.ts @@ -0,0 +1,27 @@ +// Original file: aether.proto + +import type { Long } from '@grpc/proto-loader'; + +/** + * ACLRoleInfo represents a role definition. + */ +export interface ACLRoleInfo { + 'roleId'?: (string); + 'roleName'?: (string); + 'description'?: (string); + 'createdBy'?: (string); + 'createdAt'?: (number | string | Long); + 'metadata'?: ({[key: string]: string}); +} + +/** + * ACLRoleInfo represents a role definition. + */ +export interface ACLRoleInfo__Output { + 'roleId': (string); + 'roleName': (string); + 'description': (string); + 'createdBy': (string); + 'createdAt': (string); + 'metadata': ({[key: string]: string}); +} diff --git a/sdk/typescript/src/proto/aether/v1/ACLRoleRequest.ts b/sdk/typescript/src/proto/aether/v1/ACLRoleRequest.ts new file mode 100644 index 0000000..03ed56e --- /dev/null +++ b/sdk/typescript/src/proto/aether/v1/ACLRoleRequest.ts @@ -0,0 +1,22 @@ +// Original file: aether.proto + + +/** + * ACLRoleRequest contains data for creating a role. + */ +export interface ACLRoleRequest { + 'name'?: (string); + 'description'?: (string); + 'createdBy'?: (string); + 'metadata'?: ({[key: string]: string}); +} + +/** + * ACLRoleRequest contains data for creating a role. + */ +export interface ACLRoleRequest__Output { + 'name': (string); + 'description': (string); + 'createdBy': (string); + 'metadata': ({[key: string]: string}); +} diff --git a/sdk/typescript/src/proto/aether/v1/BackoffStrategy.ts b/sdk/typescript/src/proto/aether/v1/BackoffStrategy.ts new file mode 100644 index 0000000..63690e9 --- /dev/null +++ b/sdk/typescript/src/proto/aether/v1/BackoffStrategy.ts @@ -0,0 +1,53 @@ +// Original file: aether.proto + +/** + * BackoffStrategy describes how the task store scales delays across retry + * attempts when a RetryPolicy is attached to a task. Defaults to + * EXPONENTIAL when unspecified (preserves current behavior). + */ +export const BackoffStrategy = { + BACKOFF_STRATEGY_UNSPECIFIED: 'BACKOFF_STRATEGY_UNSPECIFIED', + /** + * Same delay every attempt. + */ + BACKOFF_STRATEGY_FIXED: 'BACKOFF_STRATEGY_FIXED', + /** + * delay = initial * 2^(n-1), capped at max_delay_ms. + */ + BACKOFF_STRATEGY_EXPONENTIAL: 'BACKOFF_STRATEGY_EXPONENTIAL', + /** + * Use schedule_ms[attempt-1]; clamp the last entry. + */ + BACKOFF_STRATEGY_EXPLICIT_SCHEDULE: 'BACKOFF_STRATEGY_EXPLICIT_SCHEDULE', +} as const; + +/** + * BackoffStrategy describes how the task store scales delays across retry + * attempts when a RetryPolicy is attached to a task. Defaults to + * EXPONENTIAL when unspecified (preserves current behavior). + */ +export type BackoffStrategy = + | 'BACKOFF_STRATEGY_UNSPECIFIED' + | 0 + /** + * Same delay every attempt. + */ + | 'BACKOFF_STRATEGY_FIXED' + | 1 + /** + * delay = initial * 2^(n-1), capped at max_delay_ms. + */ + | 'BACKOFF_STRATEGY_EXPONENTIAL' + | 2 + /** + * Use schedule_ms[attempt-1]; clamp the last entry. + */ + | 'BACKOFF_STRATEGY_EXPLICIT_SCHEDULE' + | 3 + +/** + * BackoffStrategy describes how the task store scales delays across retry + * attempts when a RetryPolicy is attached to a task. Defaults to + * EXPONENTIAL when unspecified (preserves current behavior). + */ +export type BackoffStrategy__Output = typeof BackoffStrategy[keyof typeof BackoffStrategy] diff --git a/sdk/typescript/src/proto/aether/v1/BridgeIdentity.ts b/sdk/typescript/src/proto/aether/v1/BridgeIdentity.ts index ad80409..e206651 100644 --- a/sdk/typescript/src/proto/aether/v1/BridgeIdentity.ts +++ b/sdk/typescript/src/proto/aether/v1/BridgeIdentity.ts @@ -3,7 +3,7 @@ export interface BridgeIdentity { /** - * e.g., "aether-msgbridge", "webhook-bridge" + * e.g., "example-bridge", "webhook-bridge" */ 'implementation'?: (string); /** @@ -14,7 +14,7 @@ export interface BridgeIdentity { export interface BridgeIdentity__Output { /** - * e.g., "aether-msgbridge", "webhook-bridge" + * e.g., "example-bridge", "webhook-bridge" */ 'implementation': (string); /** diff --git a/sdk/typescript/src/proto/aether/v1/CreateTaskRequest.ts b/sdk/typescript/src/proto/aether/v1/CreateTaskRequest.ts index 7afd5f3..6bc10c3 100644 --- a/sdk/typescript/src/proto/aether/v1/CreateTaskRequest.ts +++ b/sdk/typescript/src/proto/aether/v1/CreateTaskRequest.ts @@ -3,6 +3,9 @@ import type { TaskAssignmentMode as _aether_v1_TaskAssignmentMode, TaskAssignmentMode__Output as _aether_v1_TaskAssignmentMode__Output } from '../../aether/v1/TaskAssignmentMode'; import type { AuthorizationContext as _aether_v1_AuthorizationContext, AuthorizationContext__Output as _aether_v1_AuthorizationContext__Output } from '../../aether/v1/AuthorizationContext'; import type { TaskClass as _aether_v1_TaskClass, TaskClass__Output as _aether_v1_TaskClass__Output } from '../../aether/v1/TaskClass'; +import type { RetryPolicy as _aether_v1_RetryPolicy, RetryPolicy__Output as _aether_v1_RetryPolicy__Output } from '../../aether/v1/RetryPolicy'; +import type { TaskPriority as _aether_v1_TaskPriority, TaskPriority__Output as _aether_v1_TaskPriority__Output } from '../../aether/v1/TaskPriority'; +import type { TaskCompletionEvent as _aether_v1_TaskCompletionEvent, TaskCompletionEvent__Output as _aether_v1_TaskCompletionEvent__Output } from '../../aether/v1/TaskCompletionEvent'; export interface CreateTaskRequest { 'taskType'?: (string); @@ -55,6 +58,43 @@ export interface CreateTaskRequest { * Empty = no session grouping. */ 'contextId'?: (string); + /** + * Optional. When set, the task store computes next_retry_at on FailTask + * according to this policy and re-pends the task automatically (up to + * max_attempts). Absent = legacy behavior (immediate re-pend, hardcoded + * max_retries=3). + */ + 'retryPolicy'?: (_aether_v1_RetryPolicy | null); + /** + * Optional dispatch priority. Defaults to UNSPECIFIED ⇒ NORMAL. Higher + * priority pending tasks are delivered before lower ones (ties break FIFO). + */ + 'priority'?: (_aether_v1_TaskPriority); + /** + * Optional idempotency key for exactly-once task creation. When non-empty the + * gateway dedupes creation on this key: a duplicate request returns the + * existing task identity instead of creating a second task. Workflow joins + * set it to make an on_complete create_task fire exactly once under retries + * or engine restart. + */ + 'idempotencyKey'?: (string); + /** + * Optional fan-out/fan-in correlation identity, distinct from task_id. When + * non-empty it is persisted on the task and propagated to child spawns, and is + * queryable via TaskFilter.correlation_id. The barrier/group id a join matches. + */ + 'correlationId'?: (string); + /** + * Optional top-of-fan-out-tree identity (the flow / run id). Defaults to the + * task's own id when it is a root; inherited by descendants on spawn. Becomes + * the DAG run id when the join engine grows into full fan-out/fan-in. + */ + 'rootTaskId'?: (string); + /** + * Optional "feed B" config: emit a domain event onto event::* when this task + * reaches a (selected) terminal status. Absent/disabled = no emission. + */ + 'completionEvent'?: (_aether_v1_TaskCompletionEvent | null); } export interface CreateTaskRequest__Output { @@ -108,4 +148,41 @@ export interface CreateTaskRequest__Output { * Empty = no session grouping. */ 'contextId': (string); + /** + * Optional. When set, the task store computes next_retry_at on FailTask + * according to this policy and re-pends the task automatically (up to + * max_attempts). Absent = legacy behavior (immediate re-pend, hardcoded + * max_retries=3). + */ + 'retryPolicy': (_aether_v1_RetryPolicy__Output | null); + /** + * Optional dispatch priority. Defaults to UNSPECIFIED ⇒ NORMAL. Higher + * priority pending tasks are delivered before lower ones (ties break FIFO). + */ + 'priority': (_aether_v1_TaskPriority__Output); + /** + * Optional idempotency key for exactly-once task creation. When non-empty the + * gateway dedupes creation on this key: a duplicate request returns the + * existing task identity instead of creating a second task. Workflow joins + * set it to make an on_complete create_task fire exactly once under retries + * or engine restart. + */ + 'idempotencyKey': (string); + /** + * Optional fan-out/fan-in correlation identity, distinct from task_id. When + * non-empty it is persisted on the task and propagated to child spawns, and is + * queryable via TaskFilter.correlation_id. The barrier/group id a join matches. + */ + 'correlationId': (string); + /** + * Optional top-of-fan-out-tree identity (the flow / run id). Defaults to the + * task's own id when it is a root; inherited by descendants on spawn. Becomes + * the DAG run id when the join engine grows into full fan-out/fan-in. + */ + 'rootTaskId': (string); + /** + * Optional "feed B" config: emit a domain event onto event::* when this task + * reaches a (selected) terminal status. Absent/disabled = no emission. + */ + 'completionEvent': (_aether_v1_TaskCompletionEvent__Output | null); } diff --git a/sdk/typescript/src/proto/aether/v1/IncomingMessage.ts b/sdk/typescript/src/proto/aether/v1/IncomingMessage.ts index fad5c82..82bb53f 100644 --- a/sdk/typescript/src/proto/aether/v1/IncomingMessage.ts +++ b/sdk/typescript/src/proto/aether/v1/IncomingMessage.ts @@ -1,6 +1,7 @@ // Original file: aether.proto import type { MessageType as _aether_v1_MessageType, MessageType__Output as _aether_v1_MessageType__Output } from '../../aether/v1/MessageType'; +import type { PrincipalRef as _aether_v1_PrincipalRef, PrincipalRef__Output as _aether_v1_PrincipalRef__Output } from '../../aether/v1/PrincipalRef'; export interface IncomingMessage { 'sourceTopic'?: (string); @@ -18,6 +19,15 @@ export interface IncomingMessage { * declared event/metric workspace). */ 'workspace'?: (string); + /** + * Gateway-set, spoof-proof resolved on-behalf-of subject, mirrored from + * MessageEnvelope.on_behalf_subject at delivery time. Populated ONLY when the + * sender's SendMessage carried an AuthorizationContext the gateway resolved + * to an OBO subject. Lets a recipient identify the *user* a message was sent + * for, distinct from the sending identity in source_topic. Empty for direct + * (non-OBO) sends. See MessageEnvelope.on_behalf_subject. + */ + 'onBehalfSubject'?: (_aether_v1_PrincipalRef | null); } export interface IncomingMessage__Output { @@ -36,4 +46,13 @@ export interface IncomingMessage__Output { * declared event/metric workspace). */ 'workspace': (string); + /** + * Gateway-set, spoof-proof resolved on-behalf-of subject, mirrored from + * MessageEnvelope.on_behalf_subject at delivery time. Populated ONLY when the + * sender's SendMessage carried an AuthorizationContext the gateway resolved + * to an OBO subject. Lets a recipient identify the *user* a message was sent + * for, distinct from the sending identity in source_topic. Empty for direct + * (non-OBO) sends. See MessageEnvelope.on_behalf_subject. + */ + 'onBehalfSubject': (_aether_v1_PrincipalRef__Output | null); } diff --git a/sdk/typescript/src/proto/aether/v1/KVOperation.ts b/sdk/typescript/src/proto/aether/v1/KVOperation.ts index 8580ce0..6bb9699 100644 --- a/sdk/typescript/src/proto/aether/v1/KVOperation.ts +++ b/sdk/typescript/src/proto/aether/v1/KVOperation.ts @@ -20,6 +20,43 @@ export const _aether_v1_KVOperation_OpType = { * Atomic decrement that succeeds only if result >= guard_value */ DECREMENT_IF: 'DECREMENT_IF', + /** + * Atomic conditional writes — the building blocks for distributed + * coordination primitives (mutex, leader election, run-once). All three + * are backed across Redis / Badger / NATS-JetStream. KVResponse.applied + * reports whether the conditional mutation took effect. + */ + SET_NX: 'SET_NX', + /** + * Set `value` only if current == expected_value. applied=true iff swapped. + */ + COMPARE_AND_SET: 'COMPARE_AND_SET', + /** + * Delete only if current == expected_value. applied=true iff deleted. + */ + COMPARE_AND_DELETE: 'COMPARE_AND_DELETE', + /** + * REMOVAL-ONLY purge of ANOTHER principal's KV namespace, for lifecycle + * managers reaping ephemeral principals (e.g. sandbox-provider clearing a + * destroyed sidecar's private state). Deletes every key in + * (target_identity, scope) optionally filtered by `key` as a prefix. + * Gated by the capability/kv_purge_identity ACL grant. By design it + * returns ONLY KVResponse.counter_value (count deleted) — never keys or + * values — so the grant cannot be used to exfiltrate another principal's + * data. Maintains component separation: the gateway has no sandbox-specific + * knowledge; the caller decides what to purge and when. + */ + PURGE_IDENTITY: 'PURGE_IDENTITY', + /** + * Atomic set primitives backing fan-in joins (set-completeness) and + * at-most-once dedup ledgers, native across Redis / Badger / NATS-JetStream. + * SET_ADD adds `value` (the member) to the set at `key`, (re)setting `ttl` + * on a newly-added member: KVResponse.applied=true iff newly added, + * counter_value=cardinality after the add. SET_CARD returns + * counter_value=cardinality (0 if absent). + */ + SET_ADD: 'SET_ADD', + SET_CARD: 'SET_CARD', } as const; export type _aether_v1_KVOperation_OpType = @@ -45,6 +82,49 @@ export type _aether_v1_KVOperation_OpType = */ | 'DECREMENT_IF' | 7 + /** + * Atomic conditional writes — the building blocks for distributed + * coordination primitives (mutex, leader election, run-once). All three + * are backed across Redis / Badger / NATS-JetStream. KVResponse.applied + * reports whether the conditional mutation took effect. + */ + | 'SET_NX' + | 8 + /** + * Set `value` only if current == expected_value. applied=true iff swapped. + */ + | 'COMPARE_AND_SET' + | 9 + /** + * Delete only if current == expected_value. applied=true iff deleted. + */ + | 'COMPARE_AND_DELETE' + | 10 + /** + * REMOVAL-ONLY purge of ANOTHER principal's KV namespace, for lifecycle + * managers reaping ephemeral principals (e.g. sandbox-provider clearing a + * destroyed sidecar's private state). Deletes every key in + * (target_identity, scope) optionally filtered by `key` as a prefix. + * Gated by the capability/kv_purge_identity ACL grant. By design it + * returns ONLY KVResponse.counter_value (count deleted) — never keys or + * values — so the grant cannot be used to exfiltrate another principal's + * data. Maintains component separation: the gateway has no sandbox-specific + * knowledge; the caller decides what to purge and when. + */ + | 'PURGE_IDENTITY' + | 11 + /** + * Atomic set primitives backing fan-in joins (set-completeness) and + * at-most-once dedup ledgers, native across Redis / Badger / NATS-JetStream. + * SET_ADD adds `value` (the member) to the set at `key`, (re)setting `ttl` + * on a newly-added member: KVResponse.applied=true iff newly added, + * counter_value=cardinality after the add. SET_CARD returns + * counter_value=cardinality (0 if absent). + */ + | 'SET_ADD' + | 12 + | 'SET_CARD' + | 13 export type _aether_v1_KVOperation_OpType__Output = typeof _aether_v1_KVOperation_OpType[keyof typeof _aether_v1_KVOperation_OpType] @@ -210,6 +290,29 @@ export interface KVOperation { * negative deltas are rejected by the server. */ 'deltaValue'?: (number | string | Long); + /** + * Expected current value for COMPARE_AND_SET / COMPARE_AND_DELETE. The + * mutation applies only when the stored value equals these bytes. Unused by + * other ops. For an empty/absent comparison use SET_NX instead. + */ + 'expectedValue'?: (Buffer | Uint8Array | string); + /** + * LIST pagination. limit caps the keys returned in one LIST response (<=0 → + * server default). cursor pages through results: pass the previous + * KVResponse.next_cursor to fetch the next page; empty starts from the + * beginning. For LIST, `key` (field 3) is the key prefix filter, applied + * server-side BEFORE the limit. Unused by non-LIST ops. + */ + 'limit'?: (number); + 'cursor'?: (string); + /** + * PURGE_IDENTITY only: the principal whose KV namespace to purge (e.g. + * "sv::sandbox-sidecar::"). The caller's OWN identity authorizes + * the op via capability/kv_purge_identity; this names the TARGET namespace. + * `key` (field 3) acts as an optional prefix filter; `scope` selects which + * scope to purge. Ignored by all other ops. + */ + 'targetIdentity'?: (string); } /** @@ -248,4 +351,27 @@ export interface KVOperation__Output { * negative deltas are rejected by the server. */ 'deltaValue': (string); + /** + * Expected current value for COMPARE_AND_SET / COMPARE_AND_DELETE. The + * mutation applies only when the stored value equals these bytes. Unused by + * other ops. For an empty/absent comparison use SET_NX instead. + */ + 'expectedValue': (Buffer); + /** + * LIST pagination. limit caps the keys returned in one LIST response (<=0 → + * server default). cursor pages through results: pass the previous + * KVResponse.next_cursor to fetch the next page; empty starts from the + * beginning. For LIST, `key` (field 3) is the key prefix filter, applied + * server-side BEFORE the limit. Unused by non-LIST ops. + */ + 'limit': (number); + 'cursor': (string); + /** + * PURGE_IDENTITY only: the principal whose KV namespace to purge (e.g. + * "sv::sandbox-sidecar::"). The caller's OWN identity authorizes + * the op via capability/kv_purge_identity; this names the TARGET namespace. + * `key` (field 3) acts as an optional prefix filter; `scope` selects which + * scope to purge. Ignored by all other ops. + */ + 'targetIdentity': (string); } diff --git a/sdk/typescript/src/proto/aether/v1/KVResponse.ts b/sdk/typescript/src/proto/aether/v1/KVResponse.ts index 2052ce4..c5bcd3d 100644 --- a/sdk/typescript/src/proto/aether/v1/KVResponse.ts +++ b/sdk/typescript/src/proto/aether/v1/KVResponse.ts @@ -30,9 +30,20 @@ export interface KVResponse { */ 'counterValue'?: (number | string | Long); /** - * True iff INCREMENT_IF/DECREMENT_IF mutation was applied; for unguarded ops always true on success + * True iff a conditional mutation was applied. For INCREMENT_IF/DECREMENT_IF + * this is the guard result; for SET_NX/COMPARE_AND_SET/COMPARE_AND_DELETE it + * reports whether the write/delete took effect. For unguarded ops always + * true on success. On a failed COMPARE_AND_SET/COMPARE_AND_DELETE, `value` + * carries the live stored value so callers can observe the current holder. */ 'applied'?: (boolean); + /** + * LIST pagination. next_cursor is an opaque token to pass as + * KVOperation.cursor for the next page; empty when iteration is complete. + * has_more is true when more matching keys remain beyond this page. + */ + 'nextCursor'?: (string); + 'hasMore'?: (boolean); } export interface KVResponse__Output { @@ -63,7 +74,18 @@ export interface KVResponse__Output { */ 'counterValue': (string); /** - * True iff INCREMENT_IF/DECREMENT_IF mutation was applied; for unguarded ops always true on success + * True iff a conditional mutation was applied. For INCREMENT_IF/DECREMENT_IF + * this is the guard result; for SET_NX/COMPARE_AND_SET/COMPARE_AND_DELETE it + * reports whether the write/delete took effect. For unguarded ops always + * true on success. On a failed COMPARE_AND_SET/COMPARE_AND_DELETE, `value` + * carries the live stored value so callers can observe the current holder. */ 'applied': (boolean); + /** + * LIST pagination. next_cursor is an opaque token to pass as + * KVOperation.cursor for the next page; empty when iteration is complete. + * has_more is true when more matching keys remain beyond this page. + */ + 'nextCursor': (string); + 'hasMore': (boolean); } diff --git a/sdk/typescript/src/proto/aether/v1/MessageEnvelope.ts b/sdk/typescript/src/proto/aether/v1/MessageEnvelope.ts index ec7c04b..47f6a0a 100644 --- a/sdk/typescript/src/proto/aether/v1/MessageEnvelope.ts +++ b/sdk/typescript/src/proto/aether/v1/MessageEnvelope.ts @@ -1,6 +1,7 @@ // Original file: aether.proto import type { MessageType as _aether_v1_MessageType, MessageType__Output as _aether_v1_MessageType__Output } from '../../aether/v1/MessageType'; +import type { PrincipalRef as _aether_v1_PrincipalRef, PrincipalRef__Output as _aether_v1_PrincipalRef__Output } from '../../aether/v1/PrincipalRef'; import type { Long } from '@grpc/proto-loader'; /** @@ -45,6 +46,22 @@ export interface MessageEnvelope { * can recover the originating workspace. */ 'workspace'?: (string); + /** + * Gateway-set, spoof-proof resolved on-behalf-of subject. Populated ONLY + * when the sender's SendMessage carried an AuthorizationContext that the + * gateway resolved to an OBO subject (valid grant, delegate + audience + * match) — set from the resolved authority in routeMessage, the same way + * `source` is set from the authenticated sender and ProxyHttp mints X-Auth-* + * identity headers. Empty for direct (non-OBO) sends and older gateways. + * + * Lets a recipient identify the *user* a message was sent for, distinct from + * the sending *identity* in `source`. Example: the agent-harness sends from + * its agent identity but acts for a user; platform-bridge reads this to scope + * the tool registry to that user. Subject identity only — recipients that + * need to *act for* the subject use the task authority-grant path + * (CreateTaskResponse.authority_grant_id), not this field. + */ + 'onBehalfSubject'?: (_aether_v1_PrincipalRef | null); } /** @@ -89,4 +106,20 @@ export interface MessageEnvelope__Output { * can recover the originating workspace. */ 'workspace': (string); + /** + * Gateway-set, spoof-proof resolved on-behalf-of subject. Populated ONLY + * when the sender's SendMessage carried an AuthorizationContext that the + * gateway resolved to an OBO subject (valid grant, delegate + audience + * match) — set from the resolved authority in routeMessage, the same way + * `source` is set from the authenticated sender and ProxyHttp mints X-Auth-* + * identity headers. Empty for direct (non-OBO) sends and older gateways. + * + * Lets a recipient identify the *user* a message was sent for, distinct from + * the sending *identity* in `source`. Example: the agent-harness sends from + * its agent identity but acts for a user; platform-bridge reads this to scope + * the tool registry to that user. Subject identity only — recipients that + * need to *act for* the subject use the task authority-grant path + * (CreateTaskResponse.authority_grant_id), not this field. + */ + 'onBehalfSubject': (_aether_v1_PrincipalRef__Output | null); } diff --git a/sdk/typescript/src/proto/aether/v1/RetryPolicy.ts b/sdk/typescript/src/proto/aether/v1/RetryPolicy.ts new file mode 100644 index 0000000..1ce6138 --- /dev/null +++ b/sdk/typescript/src/proto/aether/v1/RetryPolicy.ts @@ -0,0 +1,84 @@ +// Original file: aether.proto + +import type { BackoffStrategy as _aether_v1_BackoffStrategy, BackoffStrategy__Output as _aether_v1_BackoffStrategy__Output } from '../../aether/v1/BackoffStrategy'; +import type { Long } from '@grpc/proto-loader'; + +/** + * RetryPolicy lifts retry scheduling out of individual workers and into the + * task store. When a task carries a RetryPolicy and a worker calls FailTask + * without an explicit reschedule, the store computes next_retry_at from + * this policy and re-pends the task. The waker then picks it up at the + * scheduled time. Workers can still call RescheduleTaskAt to override + * (e.g., to honor a Retry-After header). + */ +export interface RetryPolicy { + /** + * Total attempts allowed (1 = no retries). 0 means use server default (3). + */ + 'maxAttempts'?: (number); + 'backoff'?: (_aether_v1_BackoffStrategy); + 'initialDelayMs'?: (number | string | Long); + 'maxDelayMs'?: (number | string | Long); + /** + * Multiplicative random jitter in [0,1]. Final delay = computed * + * (1 + uniform(-jitter, +jitter)). + */ + 'jitterFactor'?: (number | string); + /** + * For BACKOFF_STRATEGY_EXPLICIT_SCHEDULE. Indexed by 0-based attempt + * number; the final entry is used for any subsequent attempt. + */ + 'scheduleMs'?: (number | string | Long)[]; + /** + * Optional worker convention: HTTP-style status codes that should + * trigger a retry. The task store does not inspect failure context; + * workers use this to decide whether to FailTask or CompleteTask with + * permanent-failure semantics. + */ + 'retryableStatusCodes'?: (number)[]; + /** + * Worker hint: prefer Retry-After (or analogous) over computed delay + * when set on the failure response. + */ + 'honorRetryAfter'?: (boolean); +} + +/** + * RetryPolicy lifts retry scheduling out of individual workers and into the + * task store. When a task carries a RetryPolicy and a worker calls FailTask + * without an explicit reschedule, the store computes next_retry_at from + * this policy and re-pends the task. The waker then picks it up at the + * scheduled time. Workers can still call RescheduleTaskAt to override + * (e.g., to honor a Retry-After header). + */ +export interface RetryPolicy__Output { + /** + * Total attempts allowed (1 = no retries). 0 means use server default (3). + */ + 'maxAttempts': (number); + 'backoff': (_aether_v1_BackoffStrategy__Output); + 'initialDelayMs': (string); + 'maxDelayMs': (string); + /** + * Multiplicative random jitter in [0,1]. Final delay = computed * + * (1 + uniform(-jitter, +jitter)). + */ + 'jitterFactor': (number); + /** + * For BACKOFF_STRATEGY_EXPLICIT_SCHEDULE. Indexed by 0-based attempt + * number; the final entry is used for any subsequent attempt. + */ + 'scheduleMs': (string)[]; + /** + * Optional worker convention: HTTP-style status codes that should + * trigger a retry. The task store does not inspect failure context; + * workers use this to decide whether to FailTask or CompleteTask with + * permanent-failure semantics. + */ + 'retryableStatusCodes': (number)[]; + /** + * Worker hint: prefer Retry-After (or analogous) over computed delay + * when set on the failure response. + */ + 'honorRetryAfter': (boolean); +} diff --git a/sdk/typescript/src/proto/aether/v1/ServiceIdentity.ts b/sdk/typescript/src/proto/aether/v1/ServiceIdentity.ts index 553e41d..ae3106c 100644 --- a/sdk/typescript/src/proto/aether/v1/ServiceIdentity.ts +++ b/sdk/typescript/src/proto/aether/v1/ServiceIdentity.ts @@ -10,6 +10,17 @@ export interface ServiceIdentity { * Instance identifier for uniqueness (e.g., "pod-1", "default") */ 'specifier'?: (string); + /** + * When true, the gateway does NOT add this service connection to the + * pool-task worker index, so it is never targeted for POOL-mode task + * assignments for its implementation. Default false = pool consumer + * (back-compat: existing services keep receiving pool tasks). Serve-only + * instances that register no task handlers (e.g. a MemoryLayer server with + * its in-process worker disabled) set this so pool tasks are routed only to + * real workers instead of being claimed-and-dropped (and stuck until + * reconcile). Agents are always pool consumers regardless of this field. + */ + 'noPoolConsumer'?: (boolean); } export interface ServiceIdentity__Output { @@ -21,4 +32,15 @@ export interface ServiceIdentity__Output { * Instance identifier for uniqueness (e.g., "pod-1", "default") */ 'specifier': (string); + /** + * When true, the gateway does NOT add this service connection to the + * pool-task worker index, so it is never targeted for POOL-mode task + * assignments for its implementation. Default false = pool consumer + * (back-compat: existing services keep receiving pool tasks). Serve-only + * instances that register no task handlers (e.g. a MemoryLayer server with + * its in-process worker disabled) set this so pool tasks are routed only to + * real workers instead of being claimed-and-dropped (and stuck until + * reconcile). Agents are always pool consumers regardless of this field. + */ + 'noPoolConsumer': (boolean); } diff --git a/sdk/typescript/src/proto/aether/v1/TaskCompletionEvent.ts b/sdk/typescript/src/proto/aether/v1/TaskCompletionEvent.ts new file mode 100644 index 0000000..e6ab3f8 --- /dev/null +++ b/sdk/typescript/src/proto/aether/v1/TaskCompletionEvent.ts @@ -0,0 +1,53 @@ +// Original file: aether.proto + +import type { TaskStatus as _aether_v1_TaskStatus, TaskStatus__Output as _aether_v1_TaskStatus__Output } from '../../aether/v1/TaskStatus'; + +/** + * TaskCompletionEvent opts a task into "feed B": when it reaches a terminal + * status the server publishes a domain event onto the event plane (event::*) + * so a workflow join (or any rule) can gather over task completions without the + * worker emitting its own event. The emitted payload carries + * {task_id, status, workspace, correlation_id, metadata}. + */ +export interface TaskCompletionEvent { + /** + * enabled turns the feature on. When false (default) no completion event is + * emitted (today's behavior). + */ + 'enabled'?: (boolean); + /** + * event_name is the event name to publish. Empty ⇒ derived as + * "task.completed" / "task.failed" / "task.cancelled" from the terminal status. + */ + 'eventName'?: (string); + /** + * on_statuses restricts emission to these terminal statuses. Empty ⇒ all + * terminal statuses (completed, failed, cancelled). + */ + 'onStatuses'?: (_aether_v1_TaskStatus)[]; +} + +/** + * TaskCompletionEvent opts a task into "feed B": when it reaches a terminal + * status the server publishes a domain event onto the event plane (event::*) + * so a workflow join (or any rule) can gather over task completions without the + * worker emitting its own event. The emitted payload carries + * {task_id, status, workspace, correlation_id, metadata}. + */ +export interface TaskCompletionEvent__Output { + /** + * enabled turns the feature on. When false (default) no completion event is + * emitted (today's behavior). + */ + 'enabled': (boolean); + /** + * event_name is the event name to publish. Empty ⇒ derived as + * "task.completed" / "task.failed" / "task.cancelled" from the terminal status. + */ + 'eventName': (string); + /** + * on_statuses restricts emission to these terminal statuses. Empty ⇒ all + * terminal statuses (completed, failed, cancelled). + */ + 'onStatuses': (_aether_v1_TaskStatus__Output)[]; +} diff --git a/sdk/typescript/src/proto/aether/v1/TaskFilter.ts b/sdk/typescript/src/proto/aether/v1/TaskFilter.ts index c31caf6..01bc7ac 100644 --- a/sdk/typescript/src/proto/aether/v1/TaskFilter.ts +++ b/sdk/typescript/src/proto/aether/v1/TaskFilter.ts @@ -3,6 +3,7 @@ import type { TaskStatus as _aether_v1_TaskStatus, TaskStatus__Output as _aether_v1_TaskStatus__Output } from '../../aether/v1/TaskStatus'; import type { TaskClass as _aether_v1_TaskClass, TaskClass__Output as _aether_v1_TaskClass__Output } from '../../aether/v1/TaskClass'; import type { PrincipalRef as _aether_v1_PrincipalRef, PrincipalRef__Output as _aether_v1_PrincipalRef__Output } from '../../aether/v1/PrincipalRef'; +import type { TaskPriority as _aether_v1_TaskPriority, TaskPriority__Output as _aether_v1_TaskPriority__Output } from '../../aether/v1/TaskPriority'; import type { Long } from '@grpc/proto-loader'; /** @@ -109,6 +110,22 @@ export interface TaskFilter { * the existing parent_task_id behavior. */ 'includeDescendants'?: (boolean); + /** + * Exact dispatch-priority filter. UNSPECIFIED (0) = no filter; otherwise + * returns only tasks whose stored priority equals this level. + */ + 'priority'?: (_aether_v1_TaskPriority); + /** + * Minimum dispatch-priority threshold. UNSPECIFIED (0) = no filter; + * otherwise returns tasks whose priority is >= this level (e.g. pass HIGH + * to get everything HIGH and PREEMPT). Combinable with other filters. + */ + 'minPriority'?: (_aether_v1_TaskPriority); + /** + * Filter by fan-out/fan-in correlation identity / flow id. Empty = no filter. + */ + 'correlationId'?: (string); + 'rootTaskId'?: (string); } /** @@ -215,4 +232,20 @@ export interface TaskFilter__Output { * the existing parent_task_id behavior. */ 'includeDescendants': (boolean); + /** + * Exact dispatch-priority filter. UNSPECIFIED (0) = no filter; otherwise + * returns only tasks whose stored priority equals this level. + */ + 'priority': (_aether_v1_TaskPriority__Output); + /** + * Minimum dispatch-priority threshold. UNSPECIFIED (0) = no filter; + * otherwise returns tasks whose priority is >= this level (e.g. pass HIGH + * to get everything HIGH and PREEMPT). Combinable with other filters. + */ + 'minPriority': (_aether_v1_TaskPriority__Output); + /** + * Filter by fan-out/fan-in correlation identity / flow id. Empty = no filter. + */ + 'correlationId': (string); + 'rootTaskId': (string); } diff --git a/sdk/typescript/src/proto/aether/v1/TaskInfo.ts b/sdk/typescript/src/proto/aether/v1/TaskInfo.ts index 3866df7..79ab71f 100644 --- a/sdk/typescript/src/proto/aether/v1/TaskInfo.ts +++ b/sdk/typescript/src/proto/aether/v1/TaskInfo.ts @@ -3,6 +3,8 @@ import type { TaskStatus as _aether_v1_TaskStatus, TaskStatus__Output as _aether_v1_TaskStatus__Output } from '../../aether/v1/TaskStatus'; import type { TaskClass as _aether_v1_TaskClass, TaskClass__Output as _aether_v1_TaskClass__Output } from '../../aether/v1/TaskClass'; import type { WaitSpec as _aether_v1_WaitSpec, WaitSpec__Output as _aether_v1_WaitSpec__Output } from '../../aether/v1/WaitSpec'; +import type { TaskPriority as _aether_v1_TaskPriority, TaskPriority__Output as _aether_v1_TaskPriority__Output } from '../../aether/v1/TaskPriority'; +import type { TaskCompletionEvent as _aether_v1_TaskCompletionEvent, TaskCompletionEvent__Output as _aether_v1_TaskCompletionEvent__Output } from '../../aether/v1/TaskCompletionEvent'; import type { Long } from '@grpc/proto-loader'; /** @@ -138,6 +140,26 @@ export interface TaskInfo { * 0 means the task has never been paused. */ 'pausedAt'?: (number | string | Long); + /** + * Dispatch priority. UNSPECIFIED is normalized to NORMAL on creation, so + * persisted tasks always report a concrete level. + */ + 'priority'?: (_aether_v1_TaskPriority); + /** + * Fan-out/fan-in correlation identity (the barrier/group id a join matches), + * distinct from task_id and propagated to child spawns. Empty = none. + */ + 'correlationId'?: (string); + /** + * Top-of-fan-out-tree identity (flow / run id); defaults to the task's own id + * for a root, inherited by descendants. Empty = none. + */ + 'rootTaskId'?: (string); + /** + * "Feed B" config persisted on the task; read at the terminal transition to + * decide whether/what to emit onto event::*. + */ + 'completionEvent'?: (_aether_v1_TaskCompletionEvent | null); } /** @@ -273,4 +295,24 @@ export interface TaskInfo__Output { * 0 means the task has never been paused. */ 'pausedAt': (string); + /** + * Dispatch priority. UNSPECIFIED is normalized to NORMAL on creation, so + * persisted tasks always report a concrete level. + */ + 'priority': (_aether_v1_TaskPriority__Output); + /** + * Fan-out/fan-in correlation identity (the barrier/group id a join matches), + * distinct from task_id and propagated to child spawns. Empty = none. + */ + 'correlationId': (string); + /** + * Top-of-fan-out-tree identity (flow / run id); defaults to the task's own id + * for a root, inherited by descendants. Empty = none. + */ + 'rootTaskId': (string); + /** + * "Feed B" config persisted on the task; read at the terminal transition to + * decide whether/what to emit onto event::*. + */ + 'completionEvent': (_aether_v1_TaskCompletionEvent__Output | null); } diff --git a/sdk/typescript/src/proto/aether/v1/TaskOperation.ts b/sdk/typescript/src/proto/aether/v1/TaskOperation.ts index 864f81c..0605f34 100644 --- a/sdk/typescript/src/proto/aether/v1/TaskOperation.ts +++ b/sdk/typescript/src/proto/aether/v1/TaskOperation.ts @@ -37,6 +37,10 @@ export const _aether_v1_TaskOperation_OpType = { * Agent declines before processing -> REJECTED terminal state. */ REJECT: 'REJECT', + /** + * Assignee transitions an assigned/pending task to RUNNING. + */ + CLAIM: 'CLAIM', } as const; export type _aether_v1_TaskOperation_OpType = @@ -80,6 +84,11 @@ export type _aether_v1_TaskOperation_OpType = */ | 'REJECT' | 7 + /** + * Assignee transitions an assigned/pending task to RUNNING. + */ + | 'CLAIM' + | 8 export type _aether_v1_TaskOperation_OpType__Output = typeof _aether_v1_TaskOperation_OpType[keyof typeof _aether_v1_TaskOperation_OpType] diff --git a/sdk/typescript/src/proto/aether/v1/TaskPriority.ts b/sdk/typescript/src/proto/aether/v1/TaskPriority.ts new file mode 100644 index 0000000..a7bac2b --- /dev/null +++ b/sdk/typescript/src/proto/aether/v1/TaskPriority.ts @@ -0,0 +1,83 @@ +// Original file: aether.proto + +/** + * TaskPriority orders task dispatch. Unlike TaskClass (a UI hint), the server + * USES priority for scheduling: among pending tasks eligible for delivery, + * higher priority is dispatched first; ties break FIFO (oldest created_at). + * Underlying values are intentionally spaced so new levels can be inserted + * later, and the numeric value is used directly as the descending sort key. + */ +export const TaskPriority = { + /** + * Treated as NORMAL for back-compat. + */ + TASK_PRIORITY_UNSPECIFIED: 'TASK_PRIORITY_UNSPECIFIED', + /** + * Lowest; best-effort, yields to everything. + */ + TASK_PRIORITY_XLOW: 'TASK_PRIORITY_XLOW', + /** + * Below normal. + */ + TASK_PRIORITY_LOW: 'TASK_PRIORITY_LOW', + /** + * Default. + */ + TASK_PRIORITY_NORMAL: 'TASK_PRIORITY_NORMAL', + /** + * Above normal; jumps ahead of normal/low. + */ + TASK_PRIORITY_HIGH: 'TASK_PRIORITY_HIGH', + /** + * Highest; reserved for future true preemption. + */ + TASK_PRIORITY_PREEMPT: 'TASK_PRIORITY_PREEMPT', +} as const; + +/** + * TaskPriority orders task dispatch. Unlike TaskClass (a UI hint), the server + * USES priority for scheduling: among pending tasks eligible for delivery, + * higher priority is dispatched first; ties break FIFO (oldest created_at). + * Underlying values are intentionally spaced so new levels can be inserted + * later, and the numeric value is used directly as the descending sort key. + */ +export type TaskPriority = + /** + * Treated as NORMAL for back-compat. + */ + | 'TASK_PRIORITY_UNSPECIFIED' + | 0 + /** + * Lowest; best-effort, yields to everything. + */ + | 'TASK_PRIORITY_XLOW' + | 10 + /** + * Below normal. + */ + | 'TASK_PRIORITY_LOW' + | 20 + /** + * Default. + */ + | 'TASK_PRIORITY_NORMAL' + | 30 + /** + * Above normal; jumps ahead of normal/low. + */ + | 'TASK_PRIORITY_HIGH' + | 40 + /** + * Highest; reserved for future true preemption. + */ + | 'TASK_PRIORITY_PREEMPT' + | 50 + +/** + * TaskPriority orders task dispatch. Unlike TaskClass (a UI hint), the server + * USES priority for scheduling: among pending tasks eligible for delivery, + * higher priority is dispatched first; ties break FIFO (oldest created_at). + * Underlying values are intentionally spaced so new levels can be inserted + * later, and the numeric value is used directly as the descending sort key. + */ +export type TaskPriority__Output = typeof TaskPriority[keyof typeof TaskPriority] diff --git a/sdk/typescript/src/proto/aether/v1/WorkflowOperation.ts b/sdk/typescript/src/proto/aether/v1/WorkflowOperation.ts index 90d9378..6e0a104 100644 --- a/sdk/typescript/src/proto/aether/v1/WorkflowOperation.ts +++ b/sdk/typescript/src/proto/aether/v1/WorkflowOperation.ts @@ -49,6 +49,18 @@ export const _aether_v1_WorkflowOperation_OpType = { * Schedule upsert (idempotent create-or-update) */ UPSERT_SCHEDULE: 'UPSERT_SCHEDULE', + /** + * Fan-in / barrier / coalesce join observability + operator control. + */ + LIST_JOINS: 'LIST_JOINS', + /** + * inspect one instance: id=join_name, secondary_id=correlation_key + */ + GET_JOIN: 'GET_JOIN', + /** + * operator GC of a wedged instance + */ + CANCEL_JOIN: 'CANCEL_JOIN', } as const; export type _aether_v1_WorkflowOperation_OpType = @@ -121,6 +133,21 @@ export type _aether_v1_WorkflowOperation_OpType = */ | 'UPSERT_SCHEDULE' | 23 + /** + * Fan-in / barrier / coalesce join observability + operator control. + */ + | 'LIST_JOINS' + | 24 + /** + * inspect one instance: id=join_name, secondary_id=correlation_key + */ + | 'GET_JOIN' + | 25 + /** + * operator GC of a wedged instance + */ + | 'CANCEL_JOIN' + | 26 export type _aether_v1_WorkflowOperation_OpType__Output = typeof _aether_v1_WorkflowOperation_OpType[keyof typeof _aether_v1_WorkflowOperation_OpType] diff --git a/sdk/typescript/src/topics.ts b/sdk/typescript/src/topics.ts index 0a7cd24..56ddd41 100644 --- a/sdk/typescript/src/topics.ts +++ b/sdk/typescript/src/topics.ts @@ -18,6 +18,7 @@ * - tb::{workspace}::{impl} - Task broadcast (load-balancing) * - us::{user_id}::{window_id} - User window-specific messages * - uw::{user_id}::{workspace} - User workspace-scoped messages + * - uu::{user_id} - User broadcast (all windows, workspace-agnostic) * - ga::{workspace} - Global agent broadcast in workspace * - gu::{workspace} - Global user broadcast in workspace * - event::* - Workflow Engine only (broadcast events) @@ -58,6 +59,13 @@ export const TOPIC_PREFIX_USER = "us"; /** Prefix for user workspace topics. */ export const TOPIC_PREFIX_USER_WORKSPACE = "uw"; +/** + * Prefix for per-user broadcast topics (workspace-agnostic; reaches all of a + * user's windows). Publishing is restricted to platform principals + * (service / workflow-engine / bridge). + */ +export const TOPIC_PREFIX_USER_BROADCAST = "uu"; + /** Prefix for global agent broadcast topics. */ export const TOPIC_PREFIX_GLOBAL_AGENTS = "ga"; @@ -179,6 +187,22 @@ export function userWorkspaceTopic(userId: string, workspace: string): string { return `${TOPIC_PREFIX_USER_WORKSPACE}${IDENTITY_SEP}${userId}${IDENTITY_SEP}${workspace}`; } +/** + * Creates a per-user broadcast topic string. + * + * Messages sent to this topic reach every one of a user's open windows + * regardless of which workspace each window is currently viewing — the + * workspace-agnostic, non-progress complement to the per-user progress topic. + * Only platform principals (service / workflow-engine / bridge) may publish; + * the gateway rejects sends from workspace-scoped principals. + * + * @param userId - The user's unique identifier + * @returns Topic string in format: uu::{userId} + */ +export function userBroadcastTopic(userId: string): string { + return `${TOPIC_PREFIX_USER_BROADCAST}${IDENTITY_SEP}${userId}`; +} + /** * Creates a broadcast topic for all users in a workspace. * diff --git a/sdk/typescript/src/types.ts b/sdk/typescript/src/types.ts index bf1390e..81c8882 100644 --- a/sdk/typescript/src/types.ts +++ b/sdk/typescript/src/types.ts @@ -104,6 +104,20 @@ export enum TaskAssignmentMode { Pool = 2, } +/** + * Dispatch priority for tasks. Higher priority pending tasks are delivered + * before lower ones (ties break FIFO). Values are spaced to allow inserting + * new levels later. Unspecified (0) is normalized to Normal by the server. + */ +export enum TaskPriority { + Unspecified = 0, + XLow = 10, + Low = 20, + Normal = 30, + High = 40, + Preempt = 50, +} + // ============================================================================= // Signal Types // ============================================================================= diff --git a/sdk/typescript/src/workflow.ts b/sdk/typescript/src/workflow.ts index a7ff774..72c414e 100644 --- a/sdk/typescript/src/workflow.ts +++ b/sdk/typescript/src/workflow.ts @@ -18,6 +18,7 @@ import { globalAgentsTopic, globalUsersTopic, userTopic, + userBroadcastTopic, metricWildcardTopic, } from "./topics.js"; import type { Metric } from "./metrics-builder.js"; @@ -146,6 +147,25 @@ export class WorkflowEngineClient extends AetherClient { this._sendMessage(userTopic(userId, windowId), payload, messageType); } + /** + * Sends a message to every one of a user's windows, regardless of which + * workspace each window is viewing (topic uu::{userId}). + * + * This is the workspace-agnostic, non-progress channel for platform→user + * notifications. + * + * @param userId - Target user's ID + * @param payload - Message payload (bytes) + * @param messageType - Message type. Default: Opaque + */ + sendToUserBroadcast( + userId: string, + payload: Uint8Array, + messageType: MessageType = MessageType.Opaque, + ): void { + this._sendMessage(userBroadcastTopic(userId), payload, messageType); + } + /** * Publishes a metric to the metrics bridge. * diff --git a/server/Dockerfile b/server/Dockerfile index a0c6f25..73ba865 100644 --- a/server/Dockerfile +++ b/server/Dockerfile @@ -4,7 +4,7 @@ # builds when the runner and target differ (e.g. amd64 runner producing # arm64 binaries). # Build context must be the repo root: docker build -f server/Dockerfile . -FROM --platform=$BUILDPLATFORM golang:1.25.10-alpine AS builder +FROM --platform=$BUILDPLATFORM golang:1.25.12-alpine AS builder ARG TARGETOS ARG TARGETARCH diff --git a/server/cmd/aetherlite/cluster_wiring.go b/server/cmd/aetherlite/cluster_wiring.go index 71488c4..6f7b5a6 100644 --- a/server/cmd/aetherlite/cluster_wiring.go +++ b/server/cmd/aetherlite/cluster_wiring.go @@ -21,6 +21,7 @@ import ( "github.com/nats-io/nats.go/jetstream" "github.com/scitrera/aether/internal/acl" + "github.com/scitrera/aether/internal/auth" "github.com/scitrera/aether/internal/cluster/backup" clusternats "github.com/scitrera/aether/internal/cluster/nats" "github.com/scitrera/aether/internal/config" @@ -398,6 +399,28 @@ type aclRuleCacheInvalidator interface { InvalidateFallbackCache() } +// activateClusterTokenStore wraps the inner API token store with the +// JetStreamAPITokenStore decorator so token mutations (create / revoke / +// delete) propagate to peer gateways via the aether_api_tokens KV bucket. +// Validation stays on the inner canonical store, so revocation + expiry +// semantics are unchanged. Returns the decorated store to install in place of +// the plain SQLite store via WithAuthenticator + the state provider. +// +// On error the caller should treat the failure as fatal — cluster mode without +// token cross-gateway propagation lets a token revoked on one peer stay valid +// on another for an unbounded window. +func activateClusterTokenStore(ctx context.Context, inner auth.APITokenStore, js jetstream.JetStream, replicas int) (*auth.JetStreamAPITokenStore, error) { + wrapped, err := auth.NewJetStreamAPITokenStore(ctx, inner, js, replicas, zerologClusterLogger{}) + if err != nil { + return nil, err + } + logging.Logger.Info(). + Str("bucket", auth.APITokensKVBucket). + Int("replicas", replicas). + Msg("API token JetStream KV decorator installed (cluster mode)") + return wrapped, nil +} + // activateClusterAuditEmitter wraps the inner audit Store with the // JetStreamAuditEmitter so every LogEvent/LogEventSync call also publishes // onto the "audit" stream for cross-gateway fan-out. diff --git a/server/cmd/aetherlite/main.go b/server/cmd/aetherlite/main.go index aff64ab..0b11329 100644 --- a/server/cmd/aetherlite/main.go +++ b/server/cmd/aetherlite/main.go @@ -20,6 +20,7 @@ import ( pb "github.com/scitrera/aether/api/proto" "github.com/scitrera/aether/internal/admin" "github.com/scitrera/aether/internal/audit" + "github.com/scitrera/aether/internal/auth" authsqlite "github.com/scitrera/aether/internal/auth/sqlite" "github.com/scitrera/aether/internal/checkpoint" "github.com/scitrera/aether/internal/cleanup" @@ -42,13 +43,16 @@ import ( auditsqlite "github.com/scitrera/aether/internal/storage/audit/sqlite" regsqlite "github.com/scitrera/aether/internal/storage/registry/sqlite" tasksqlite "github.com/scitrera/aether/internal/storage/tasks/sqlite" + wfpg "github.com/scitrera/aether/internal/storage/workflow/postgres" wfsqlite "github.com/scitrera/aether/internal/storage/workflow/sqlite" "github.com/scitrera/aether/internal/tracing" versionpkg "github.com/scitrera/aether/internal/version" "github.com/scitrera/aether/internal/workflow" + wfmigrations "github.com/scitrera/aether/internal/workflow/migrations" sqliteregistrymigrations "github.com/scitrera/aether/migrations/sqlite_registry" pb_health "google.golang.org/grpc/health/grpc_health_v1" + _ "github.com/lib/pq" // postgres driver for the optional shared workflow store "github.com/nats-io/nats.go/jetstream" "github.com/scitrera/aether/pkg/crypto" "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc" @@ -173,6 +177,19 @@ func main() { _ = tracingShutdown(context.Background()) }() + // Initialize OpenTelemetry metrics (OTLP gRPC) + Go runtime metrics. + // Gated on the same OTEL_EXPORTER_OTLP_ENDPOINT env var as tracing — + // no-op when unset. The gRPC server's otelgrpc stats handler (wired below) + // then also emits rpc.server.* metrics through this MeterProvider. + metricsShutdown, err := tracing.InitMeter("aether-lite") + if err != nil { + logging.Logger.Fatal().Err(err).Msg("failed to initialize metrics") + } + defer func() { + // Best-effort metrics flush/shutdown on exit; any error is unactionable here. + _ = metricsShutdown(context.Background()) + }() + // Setup graceful shutdown. ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -456,7 +473,29 @@ func main() { sessions = state.NewBadgerSessionRegistry(badgerDB) kvStore = kv.NewBadgerKVStore(badgerDB) checkpointStore = checkpoint.NewBadgerCheckpointStore(badgerDB) - msgRouter = routerpkg.NewBadgerRouter(badgerDB) + badgerRouter := routerpkg.NewBadgerRouter(badgerDB) + // Message retention: Badger expires topic messages after this TTL (native + // per-entry expiry) to bound log growth. Default 24h; override with a Go + // duration string (e.g. "6h", "72h"), or "0" to retain forever. + if ttlStr := os.Getenv("AETHER_MESSAGE_RETENTION_TTL"); ttlStr != "" { + if d, perr := time.ParseDuration(ttlStr); perr == nil { + badgerRouter.SetMessageRetentionTTL(d) + logging.Logger.Info().Dur("ttl", d).Msg("badger message retention TTL set from AETHER_MESSAGE_RETENTION_TTL") + } else { + logging.Logger.Warn().Str("value", ttlStr).Err(perr).Msg("invalid AETHER_MESSAGE_RETENTION_TTL; using default 24h") + } + } + // Consumer-offset retention: generous by default (7d) and decoupled from + // message retention. Override with a Go duration, or "0" to retain forever. + if ttlStr := os.Getenv("AETHER_OFFSET_RETENTION_TTL"); ttlStr != "" { + if d, perr := time.ParseDuration(ttlStr); perr == nil { + badgerRouter.SetOffsetRetentionTTL(d) + logging.Logger.Info().Dur("ttl", d).Msg("badger offset retention TTL set from AETHER_OFFSET_RETENTION_TTL") + } else { + logging.Logger.Warn().Str("value", ttlStr).Err(perr).Msg("invalid AETHER_OFFSET_RETENTION_TTL; using default 7d") + } + } + msgRouter = badgerRouter dispatcher = orchestration.NewPollingTaskDispatcher(taskStore) } @@ -516,6 +555,13 @@ func main() { gatewayOpts = append(gatewayOpts, gateway.WithQuotaManager(quotaManager)) gatewayOpts = append(gatewayOpts, gateway.WithOrchestrationServices(orchServices)) gatewayOpts = append(gatewayOpts, gateway.WithCheckpointDefaultTTL(cfg.Checkpoint.GetDefaultTTL())) + // Gateway tenant id, minted into X-Auth-Tenant-ID on proxied requests. + // The gateway is per-tenant; AETHER_TENANT_ID names it (matching the + // proxy-sidecar env convention). When unset, the proxy path falls back to + // the sender's workspace as the tenant scope. + if tenantID := os.Getenv("AETHER_TENANT_ID"); tenantID != "" { + gatewayOpts = append(gatewayOpts, gateway.WithGatewayTenantID(tenantID)) + } // Workspace rate limiter. workspaceRL := quota.NewWorkspaceRateLimiter(cfg.Gateway.MessageRateLimit) @@ -539,6 +585,9 @@ func main() { } defer auditLogger.Close() + // Coalesce high-volume repetitive audit events (chat-streaming chatter). + gatewayOpts = append(gatewayOpts, gateway.WithAuditCoalesceWindow(cfg.Audit.GetCoalesceWindow())) + // Cluster mode: wrap the gateway-consumed audit Store with the JetStream // emitter so every LogEvent / LogEventSync also fans onto the "audit" // stream. Inner sqlite store remains canonical (and is still passed to @@ -601,21 +650,101 @@ func main() { TaskPurgeInterval: cfg.Cleanup.GetTaskPurgeInterval(), CompletedTaskRetention: cfg.Cleanup.GetCompletedTaskRetention(), FailedTaskRetention: cfg.Cleanup.GetFailedTaskRetention(), - CancelledTaskRetention: cfg.Cleanup.GetCancelledTaskRetention(), - ReconciliationInterval: cfg.Cleanup.GetReconciliationInterval(), + CancelledTaskRetention: cfg.Cleanup.GetCancelledTaskRetention(), + ReconciliationInterval: cfg.Cleanup.GetReconciliationInterval(), + InteractiveTaskTTL: cfg.Cleanup.GetInteractiveTaskTTL(), + InteractiveTaskCancelInterval: cfg.Cleanup.GetInteractiveTaskCancelInterval(), + StartupTaskTTL: cfg.Cleanup.GetStartupTaskTTL(), + StartupTaskCancelInterval: cfg.Cleanup.GetStartupTaskCancelInterval(), + PoolTaskTTL: cfg.Cleanup.GetPoolTaskTTL(), + PoolTaskCancelInterval: cfg.Cleanup.GetPoolTaskCancelInterval(), + QueueReconcileInterval: cfg.Cleanup.GetQueueReconcileInterval(), + AuditRetentionDays: cfg.Cleanup.GetAuditRetentionDays(), + AuditCleanupInterval: cfg.Cleanup.GetAuditCleanupInterval(), + // Single-node standalone runs the sweeps directly (no leader-election + // gating). Cluster mode (cenv.Enabled, JetStream dispatcher) is genuinely + // multi-node, so it keeps lease-based election to sweep on exactly one node. + SingleNode: !cenv.Enabled, } gatewayOpts = append(gatewayOpts, gateway.WithCleanupService(cleanupConfig)) + // API token store + connection-time API-key authenticator. + // + // tokens.db holds the api_tokens table. The native sqlite impl uses the + // bare "sqlite" driver and runs its own per-domain migration set on + // construction. This gives lite mode full token CRUD parity with the + // PostgreSQL-backed full gateway. The SAME store instance is reused below + // by the admin state provider (token CRUD) — we never open a second store. + // + // Constructed here (before NewGatewayServer) so the composite authenticator + // can be attached via WithAuthenticator, mirroring cmd/gateway/main.go. + tokensSQLitePath := filepath.Join(*dataDir, "tokens.db") + tokensDB, err := openSQLiteNative(ctx, tokensSQLitePath) + if err != nil { + logging.Logger.Fatal().Err(err).Str("path", tokensSQLitePath).Msg("failed to open tokens SQLite database") + } + defer tokensDB.Close() + + var apiTokenStore auth.APITokenStore + plainAPITokenStore, err := authsqlite.New(tokensDB) + if err != nil { + logging.Logger.Fatal().Err(err).Msg("failed to construct native sqlite token store") + } + apiTokenStore = plainAPITokenStore + + // In cluster mode, decorate the token store with the JetStream-KV mirror so + // token mutations (create / revoke / delete) propagate to peer gateways and + // each peer serves token validation from its watch-maintained local view. + // Single-node lite keeps the plain SQLite store. The decorator preserves the + // full APITokenStore contract (revocation + expiry semantics intact). + if cenv.Enabled && natsServer != nil { + js := natsServer.JetStream() + replicas := natsServer.ReplicasForHA() + decorated, werr := activateClusterTokenStore(ctx, plainAPITokenStore, js, replicas) + if werr != nil { + logging.Logger.Fatal().Err(werr).Msg("failed to activate cluster API token store") + } + apiTokenStore = decorated + } + + // Wire the connection-time API-key authenticator. + // + // The user-session path (per-user anonymous-cert + scoped API key) requires + // api-key auth. Enable it only when the operator explicitly lists "api_key" + // in auth.modes. Defaulting to on when modes is unset was removed because it + // created an inconsistency: IsAuthModeEnabled("api_key") returned false, yet + // api-key auth was silently active, making the config semantics opaque. + // + // SECURITY NOTE: when api_key auth is disabled AND the gateway accepts + // anonymous mTLS certificates, impersonable principals (User, Agent, Service, + // Task, Bridge) on the anonymous-cert / no-cert path are unconditionally + // rejected by authenticateCredentials' credential-required gate. Disabling + // api_key auth therefore means no external connections can authenticate via + // credential on that path — ensure your deployment either (a) enables api_key + // auth, or (b) disables anonymous cert acceptance, or (c) uses strict mTLS + // so every connection presents a named cert. + enableAPIKeyAuth := cfg.Auth.IsAuthModeEnabled("api_key") + if enableAPIKeyAuth { + compositeAuth := auth.NewCompositeAuthenticator(auth.NewAPIKeyAuthenticator(apiTokenStore)) + gatewayOpts = append(gatewayOpts, gateway.WithAuthenticator(compositeAuth)) + logging.Logger.Info().Msg("API key authentication enabled (connection-time)") + } else { + logging.Logger.Warn(). + Strs("configured_modes", cfg.Auth.Modes). + Msg("API key authentication NOT enabled: anonymous-cert connections claiming impersonable principals will be rejected; add 'api_key' to auth.modes to allow them") + } + // mTLS config — Required defaults to false in lite mode, but when the // config supplies an explicit mTLS block we honour it. Mode controls how // strictly identity assertions are bound to the presented cert (strict / // relaxed); matches the full gateway's semantics. + mtlsMode, mtlsModeErr := gateway.ParseMTLSMode(cfg.Auth.MTLS.Mode) + if mtlsModeErr != nil { + logging.Logger.Fatal().Err(mtlsModeErr).Str("mtls_mode", cfg.Auth.MTLS.Mode).Msg("invalid mTLS mode in configuration") + } mtlsConfig := gateway.MTLSConfig{ Required: cfg.Auth.MTLS.Required, - Mode: gateway.MTLSMode(cfg.Auth.MTLS.Mode), - } - if mtlsConfig.Mode == "" { - mtlsConfig.Mode = gateway.MTLSModeStrict + Mode: mtlsMode, } // Create gateway server. ACL is supplied via WithACLService above. @@ -649,8 +778,12 @@ func main() { Timeout: 10 * time.Second, }), grpc.KeepaliveEnforcementPolicy(keepalive.EnforcementPolicy{ - MinTime: 10 * time.Second, - PermitWithoutStream: false, + MinTime: 10 * time.Second, + // SDK clients dial with PermitWithoutStream: true and keepalive-ping + // idle connections (default every 30s). PermitWithoutStream: false here + // → GOAWAY "too_many_pings" on idle conns → connection churn (sandbox + // sidecars reaped). Permit streamless keepalive; MinTime still throttles. + PermitWithoutStream: true, }), } @@ -747,22 +880,9 @@ func main() { // State provider for admin. Registry already constructed above for the // orchestration wiring; reuse it here for both agentRegistry+profileMgr // slots (the bundled internal/storage/registry.Store covers both). - // Open dedicated tokens SQLite handle for API token management. - // tokens.db holds the api_tokens table. The native sqlite impl uses the - // bare "sqlite" driver and runs its own per-domain migration set on - // construction. This gives lite mode full token CRUD parity with the - // PostgreSQL-backed full gateway. - tokensSQLitePath := filepath.Join(*dataDir, "tokens.db") - tokensDB, err := openSQLiteNative(ctx, tokensSQLitePath) - if err != nil { - logging.Logger.Fatal().Err(err).Str("path", tokensSQLitePath).Msg("failed to open tokens SQLite database") - } - defer tokensDB.Close() - - apiTokenStore, err := authsqlite.New(tokensDB) - if err != nil { - logging.Logger.Fatal().Err(err).Msg("failed to construct native sqlite token store") - } + // apiTokenStore was constructed earlier (before the gateway server) so the + // connection-time API-key authenticator and the admin-side token CRUD path + // share the single store; here we just reuse it for the state provider. stateProvider := gateway.NewGatewayStateProvider( cfg.Gateway.GatewayID, @@ -876,20 +996,16 @@ func main() { // bufconn in-process listener — no TLS, no localhost dial, no // reconnect-loop trap. wfCfg := buildWorkflowConfig(inProcConn) - // Open workflow.db with the bare "sqlite" driver and construct the - // native sqlite store (Stage 2). wfsqlite.New runs migrations from - // migrations/sqlite_workflow/ internally. We then inject the store - // into the workflow server via NewServerWithStore so the engine - // skips its legacy sqlite_compat path entirely (§15.4). - wfDB, err := openSQLiteNative(ctx, wfCfg.SQLite.Path) - if err != nil { - logging.Logger.Fatal().Err(err).Str("path", wfCfg.SQLite.Path).Msg("failed to open workflow SQLite database") - } - defer wfDB.Close() - wfStore, err := wfsqlite.New(wfDB) + // Select the workflow store backend: a shared PostgreSQL when the + // workflow config supplies a postgres host (the recommended path for + // multi-node aetherlite-cluster, so the elected leader and its + // successors share durable state), otherwise the local native SQLite + // store (the zero-dependency single-node default). + wfStore, closeWFStore, err := buildEmbeddedWorkflowStore(ctx, wfCfg) if err != nil { - logging.Logger.Fatal().Err(err).Msg("failed to construct native sqlite workflow store") + logging.Logger.Fatal().Err(err).Msg("failed to construct workflow store") } + defer closeWFStore() wfSrv, err := workflow.NewServer(wfCfg, wfStore) if err != nil { logging.Logger.Fatal().Err(err).Msg("failed to create workflow server") @@ -1085,6 +1201,53 @@ func buildWorkflowConfig(inProcConn *grpc.ClientConn) *workflow.Config { // impls — they own their own SQL and don't need dbcompat translation. // (Stage 3 retired the sibling openSQLite/sqlite_compat path along with // aether.db itself; pkg/dbcompat is no longer imported by aetherlite.) +// buildEmbeddedWorkflowStore selects the workflow store backend for the embedded +// engine. When the workflow config supplies a PostgreSQL host it opens a shared +// Postgres (running the workflow PG migrations), which lets multi-node +// aetherlite-cluster deployments share durable workflow state across leader +// failover. Otherwise it opens the local native SQLite store — the +// zero-dependency single-node default. Returns the store plus a close func. +func buildEmbeddedWorkflowStore(ctx context.Context, wfCfg *workflow.Config) (workflow.WorkflowStore, func(), error) { + if wfCfg.Postgres.Host != "" { + db, err := sql.Open("postgres", wfCfg.Postgres.DSN()) + if err != nil { + return nil, nil, fmt.Errorf("open workflow postgres: %w", err) + } + if wfCfg.Postgres.MaxConnections > 0 { + db.SetMaxOpenConns(wfCfg.Postgres.MaxConnections) + } + if wfCfg.Postgres.MaxIdleConnections > 0 { + db.SetMaxIdleConns(wfCfg.Postgres.MaxIdleConnections) + } + if err := db.PingContext(ctx); err != nil { + _ = db.Close() + return nil, nil, fmt.Errorf("ping workflow postgres: %w", err) + } + if err := wfmigrations.Run(ctx, db); err != nil { + _ = db.Close() + return nil, nil, fmt.Errorf("run workflow postgres migrations: %w", err) + } + logging.Logger.Info(). + Str("host", wfCfg.Postgres.Host). + Str("database", wfCfg.Postgres.Database). + Msg("embedded workflow engine using shared PostgreSQL store") + return wfpg.New(db, false), func() { _ = db.Close() }, nil + } + + // Default: local native SQLite. wfsqlite.New runs its own migration set. + db, err := openSQLiteNative(ctx, wfCfg.SQLite.Path) + if err != nil { + return nil, nil, fmt.Errorf("open workflow sqlite %s: %w", wfCfg.SQLite.Path, err) + } + store, err := wfsqlite.New(db) + if err != nil { + _ = db.Close() + return nil, nil, fmt.Errorf("construct native sqlite workflow store: %w", err) + } + logging.Logger.Info().Str("path", wfCfg.SQLite.Path).Msg("embedded workflow engine using local SQLite store") + return store, func() { _ = db.Close() }, nil +} + func openSQLiteNative(ctx context.Context, path string) (*sql.DB, error) { return openSQLiteWithDriver(ctx, path, "sqlite") } @@ -1094,6 +1257,16 @@ func openSQLiteWithDriver(ctx context.Context, path, driver string) (*sql.DB, er if err != nil { return nil, fmt.Errorf("failed to open SQLite database %s (driver=%s): %w", path, driver, err) } + // Pin to a single connection (single-writer discipline). Two reasons: + // 1. busy_timeout is a PER-CONNECTION setting; setting it via ExecContext + // below only cushions the one connection that runs it, so extra pool + // connections opened under a concurrent-write burst would hit immediate + // SQLITE_BUSY ("database is locked"). With one connection, the pragma + // always applies. + // 2. Serializing all access on one connection prevents writer-vs-writer WAL + // contention outright — the same discipline the per-domain sqlite stores + // (tasks/registry/workflow/acl) already enforce in their constructors. + db.SetMaxOpenConns(1) for _, pragma := range []string{ "PRAGMA journal_mode=WAL", "PRAGMA busy_timeout=5000", diff --git a/server/cmd/aetherlite/store_select_test.go b/server/cmd/aetherlite/store_select_test.go new file mode 100644 index 0000000..2426de7 --- /dev/null +++ b/server/cmd/aetherlite/store_select_test.go @@ -0,0 +1,58 @@ +package main + +import ( + "context" + "path/filepath" + "testing" + "time" + + "github.com/scitrera/aether/internal/workflow" +) + +// TestBuildEmbeddedWorkflowStore_SelectsSQLiteByDefault verifies that with no +// Postgres host configured the embedded engine opens the local SQLite store +// (the zero-dependency single-node default). +func TestBuildEmbeddedWorkflowStore_SelectsSQLiteByDefault(t *testing.T) { + wfCfg := &workflow.Config{} + wfCfg.SQLite.Path = filepath.Join(t.TempDir(), "workflow.db") + + store, closeFn, err := buildEmbeddedWorkflowStore(context.Background(), wfCfg) + if err != nil { + t.Fatalf("buildEmbeddedWorkflowStore (sqlite default): %v", err) + } + if store == nil { + t.Fatal("expected non-nil SQLite store") + } + if closeFn == nil { + t.Fatal("expected non-nil close func") + } + closeFn() +} + +// TestBuildEmbeddedWorkflowStore_SelectsPostgresWhenHostSet verifies that +// setting a Postgres host routes to the Postgres branch. We point at an +// unreachable address so the connect/ping fails fast — the returned error +// proves the Postgres path was taken rather than silently falling back to +// SQLite. +func TestBuildEmbeddedWorkflowStore_SelectsPostgresWhenHostSet(t *testing.T) { + wfCfg := &workflow.Config{} + wfCfg.SQLite.Path = filepath.Join(t.TempDir(), "workflow.db") // present but must be ignored + wfCfg.Postgres.Host = "127.0.0.1" + wfCfg.Postgres.Port = 1 // nothing listens here + wfCfg.Postgres.Database = "wf" + wfCfg.Postgres.User = "wf" + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + store, closeFn, err := buildEmbeddedWorkflowStore(ctx, wfCfg) + if err == nil { + if closeFn != nil { + closeFn() + } + t.Fatal("expected an error connecting to the unreachable Postgres, got nil (did it fall back to SQLite?)") + } + if store != nil { + t.Fatal("expected nil store on Postgres connect failure") + } +} diff --git a/server/cmd/cleanup/main.go b/server/cmd/cleanup/main.go index 7a4bbb1..9411258 100644 --- a/server/cmd/cleanup/main.go +++ b/server/cmd/cleanup/main.go @@ -34,8 +34,10 @@ var ( runAll = flag.Bool("all", false, "Run all cleanup jobs") runTaskPurge = flag.Bool("task-purge", false, "Run task purge job") runReconcile = flag.Bool("reconcile", false, "Run orphaned task reconciliation") - runStaleLocks = flag.Bool("stale-locks", false, "Run stale lock cleanup") - runStaleClaims = flag.Bool("stale-claims", false, "Run stale claim recovery for orchestration tasks") + runStaleLocks = flag.Bool("stale-locks", false, "Run stale lock cleanup") + runStaleClaims = flag.Bool("stale-claims", false, "Run stale claim recovery for orchestration tasks") + runStaleInteractive = flag.Bool("stale-interactive", false, "Cancel stale interactive (chat turn) tasks older than the TTL") + runStaleStartup = flag.Bool("stale-startup", false, "Cancel stale unclaimed agent_startup tasks older than the TTL") // Override retention periods (uses config defaults if not specified) completedRetention = flag.String("completed-retention", "", "Override completed task retention (e.g., '168h')") @@ -59,7 +61,9 @@ func main() { fmt.Fprintf(os.Stderr, " task-purge Delete old completed/failed/cancelled tasks\n") fmt.Fprintf(os.Stderr, " reconcile Fail tasks whose agents/orchestrators disconnected\n") fmt.Fprintf(os.Stderr, " stale-locks Remove Redis locks with no TTL (legacy cleanup)\n") - fmt.Fprintf(os.Stderr, " stale-claims Recover orchestration tasks stuck in 'claimed' status\n\n") + fmt.Fprintf(os.Stderr, " stale-claims Recover orchestration tasks stuck in 'claimed' status\n") + fmt.Fprintf(os.Stderr, " stale-interactive Cancel stale interactive (chat turn) tasks older than the TTL\n") + fmt.Fprintf(os.Stderr, " stale-startup Cancel stale unclaimed agent_startup tasks older than the TTL\n\n") fmt.Fprintf(os.Stderr, "Options:\n") flag.PrintDefaults() fmt.Fprintf(os.Stderr, "\nExamples:\n") @@ -87,7 +91,7 @@ func main() { } // If no jobs specified, show help - if !*runAll && !*runTaskPurge && !*runReconcile && !*runStaleLocks && !*runStaleClaims { + if !*runAll && !*runTaskPurge && !*runReconcile && !*runStaleLocks && !*runStaleClaims && !*runStaleInteractive && !*runStaleStartup { fmt.Fprintf(os.Stderr, "Error: No jobs specified. Use -all to run all jobs, or specify individual jobs.\n\n") flag.Usage() os.Exit(1) diff --git a/server/cmd/gateway/main.go b/server/cmd/gateway/main.go index 491bea6..f6383e1 100644 --- a/server/cmd/gateway/main.go +++ b/server/cmd/gateway/main.go @@ -269,6 +269,22 @@ func main() { logging.Logger.Debug().Msg("OpenTelemetry tracing disabled (OTEL_EXPORTER_OTLP_ENDPOINT not set)") } + // Initialize OpenTelemetry metrics (OTLP gRPC) + Go runtime metrics. + // Gated on the same OTEL_EXPORTER_OTLP_ENDPOINT env var as tracing — + // no-op when unset. The gRPC server's otelgrpc stats handler (wired below) + // then also emits rpc.server.* metrics through this MeterProvider. + metricsShutdown, err := tracing.InitMeter("aether-gateway") + if err != nil { + logging.Logger.Fatal().Err(err).Msg("failed to initialize metrics") + } + defer func() { + // Best-effort metrics flush/shutdown on exit; any error is unactionable here. + _ = metricsShutdown(context.Background()) + }() + if os.Getenv("OTEL_EXPORTER_OTLP_ENDPOINT") != "" { + logging.Logger.Info().Msg("OpenTelemetry metrics + Go runtime instrumentation enabled") + } + // Initialize OpenTelemetry log bridge — exports structured zerolog // events to SigNoz via OTLP with trace context correlation. logBridge, err := tracing.NewLogBridge(context.Background(), "aether-gateway") @@ -394,9 +410,9 @@ func main() { mtlsRequired = true // -tls flag forces mTLS to be required } - mtlsMode := gateway.MTLSModeStrict - if cfg.Auth.MTLS.Mode == "relaxed" { - mtlsMode = gateway.MTLSModeRelaxed + mtlsMode, mtlsModeErr := gateway.ParseMTLSMode(cfg.Auth.MTLS.Mode) + if mtlsModeErr != nil { + logging.Logger.Fatal().Err(mtlsModeErr).Str("mtls_mode", cfg.Auth.MTLS.Mode).Msg("invalid mTLS mode in configuration") } mtlsConfig := gateway.MTLSConfig{ @@ -409,6 +425,18 @@ func main() { auditLogger := audit.NewAuditLogger(db, cfg.Gateway.GatewayID, auditConfig) defer auditLogger.Close() + // Coalesce high-volume repetitive audit events (chat-streaming chatter). + gatewayOpts = append(gatewayOpts, gateway.WithAuditCoalesceWindow(cfg.Audit.GetCoalesceWindow())) + + // Gateway tenant id, minted into X-Auth-Tenant-ID on proxied requests. + // The gateway is per-tenant; AETHER_TENANT_ID names it (matching the + // proxy-sidecar env convention). When unset, the proxy path falls back to + // the sender's workspace as the tenant scope. + if tenantID := os.Getenv("AETHER_TENANT_ID"); tenantID != "" { + gatewayOpts = append(gatewayOpts, gateway.WithGatewayTenantID(tenantID)) + logging.Logger.Debug().Str("tenant_id", tenantID).Msg("gateway tenant id configured for X-Auth-Tenant-ID minting") + } + // Checkpoint default TTL checkpointDefaultTTL := cfg.Checkpoint.GetDefaultTTL() gatewayOpts = append(gatewayOpts, gateway.WithCheckpointDefaultTTL(checkpointDefaultTTL)) @@ -546,8 +574,20 @@ func main() { TaskPurgeInterval: cfg.Cleanup.GetTaskPurgeInterval(), CompletedTaskRetention: cfg.Cleanup.GetCompletedTaskRetention(), FailedTaskRetention: cfg.Cleanup.GetFailedTaskRetention(), - CancelledTaskRetention: cfg.Cleanup.GetCancelledTaskRetention(), - ReconciliationInterval: cfg.Cleanup.GetReconciliationInterval(), + CancelledTaskRetention: cfg.Cleanup.GetCancelledTaskRetention(), + ReconciliationInterval: cfg.Cleanup.GetReconciliationInterval(), + InteractiveTaskTTL: cfg.Cleanup.GetInteractiveTaskTTL(), + InteractiveTaskCancelInterval: cfg.Cleanup.GetInteractiveTaskCancelInterval(), + StartupTaskTTL: cfg.Cleanup.GetStartupTaskTTL(), + StartupTaskCancelInterval: cfg.Cleanup.GetStartupTaskCancelInterval(), + PoolTaskTTL: cfg.Cleanup.GetPoolTaskTTL(), + PoolTaskCancelInterval: cfg.Cleanup.GetPoolTaskCancelInterval(), + QueueReconcileInterval: cfg.Cleanup.GetQueueReconcileInterval(), + AuditRetentionDays: cfg.Cleanup.GetAuditRetentionDays(), + AuditCleanupInterval: cfg.Cleanup.GetAuditCleanupInterval(), + // Full gateway is Redis-backed and multi-node-capable; keep lease-based + // leader election (SingleNode stays false) so exactly one node sweeps. A + // stuck Redis lease self-heals via LockTTL expiry. } gatewayOpts = append(gatewayOpts, gateway.WithCleanupService(cleanupConfig)) @@ -620,8 +660,14 @@ func main() { Timeout: 10 * time.Second, }), grpc.KeepaliveEnforcementPolicy(keepalive.EnforcementPolicy{ - MinTime: 10 * time.Second, - PermitWithoutStream: false, + MinTime: 10 * time.Second, + // SDK clients dial with PermitWithoutStream: true and keepalive-ping + // idle connections (default every 30s) to detect a dead gateway. With + // PermitWithoutStream: false here the gateway counts those idle pings + // as abusive and sends GOAWAY "too_many_pings", churning the client's + // connection (e.g. sandbox sidecars → reaped). Permit streamless + // keepalive; MinTime (10s) still throttles a genuinely chatty client. + PermitWithoutStream: true, }), } diff --git a/server/cmd/init-secrets/main.go b/server/cmd/init-secrets/main.go index a5344eb..9638ec7 100644 --- a/server/cmd/init-secrets/main.go +++ b/server/cmd/init-secrets/main.go @@ -13,10 +13,14 @@ import ( "time" _ "github.com/lib/pq" + authsqlite "github.com/scitrera/aether/internal/auth/sqlite" "github.com/scitrera/aether/internal/config" "github.com/scitrera/aether/internal/secrets" + aclsqlite "github.com/scitrera/aether/internal/storage/acl/sqlite" "github.com/scitrera/aether/pkg/certgen" "github.com/scitrera/aether/pkg/models" + + _ "modernc.org/sqlite" // register bare "sqlite" driver for aetherlite token/acl DBs ) const defaultSecretsPath = "/etc/aether/generated-secrets.yaml" @@ -35,6 +39,7 @@ func main() { tokenName := flag.String("token-name", "admin-bootstrap", "Name for the initial token") principalType := flag.String("principal-type", "User", "Principal type for the token (User, Agent, Task, Service, Orchestrator, WorkflowEngine, MetricsBridge, Bridge)") accessLevel := flag.String("access-level", "ADMIN", "Access level for the token (NONE, READ, READWRITE, MANAGE, ADMIN, SUPERADMIN)") + dataDir := flag.String("data-dir", config.EnvStr("AETHERLITE_DATA_DIR", "./aether-lite-data"), "AetherLite data directory holding tokens.db/acl.db; used for --create-token when PostgreSQL is not configured (env: AETHERLITE_DATA_DIR)") force := flag.Bool("force", false, "Regenerate even if secrets file already exists") printSecrets := flag.Bool("print", false, "Print generated values to stdout") @@ -125,27 +130,23 @@ func main() { } if *createToken { - if cfg.Postgres.Host == "" { - log.Fatal("--create-token requires PostgreSQL configuration (set via --config or env vars)") - } - - db, err := sql.Open("postgres", cfg.Postgres.DSN()) + level, err := secrets.ParseAccessLevel(*accessLevel) if err != nil { - log.Fatalf("Failed to open database: %v", err) + log.Fatalf("Invalid access level %q: %v", *accessLevel, err) } - defer db.Close() ctx := context.Background() - if err := db.PingContext(ctx); err != nil { - log.Fatalf("Failed to connect to database: %v", err) - } - level, err := secrets.ParseAccessLevel(*accessLevel) - if err != nil { - log.Fatalf("Invalid access level %q: %v", *accessLevel, err) + var plaintext string + if cfg.Postgres.Host != "" { + // Full gateway topology: mint against PostgreSQL (existing behavior). + plaintext, err = createTokenPostgres(ctx, cfg, *tokenName, models.PrincipalType(*principalType), level) + } else { + // aetherlite topology (single or cluster): no PostgreSQL. Mint + // directly against the SQLite stores the gateway uses, so the CLI + // works WITHOUT a running gateway. + plaintext, err = createTokenSQLite(ctx, cfg, *dataDir, *tokenName, models.PrincipalType(*principalType), level) } - - plaintext, err := secrets.CreateInitialToken(ctx, db, cfg, *tokenName, models.PrincipalType(*principalType), level) if err != nil { log.Fatalf("Failed to create initial token: %v", err) } @@ -155,6 +156,104 @@ func main() { } } +// createTokenPostgres mints an initial API token against the configured +// PostgreSQL database (full gateway topology). +func createTokenPostgres(ctx context.Context, cfg *config.Config, tokenName string, principalType models.PrincipalType, level int) (string, error) { + db, err := sql.Open("postgres", cfg.Postgres.DSN()) + if err != nil { + return "", fmt.Errorf("opening database: %w", err) + } + defer db.Close() + + if err := db.PingContext(ctx); err != nil { + return "", fmt.Errorf("connecting to database: %w", err) + } + + return secrets.CreateInitialToken(ctx, db, cfg, tokenName, principalType, level) +} + +// createTokenSQLite mints an initial API token against the SQLite stores under +// dataDir, exactly as aetherlite opens them (tokens.db + acl.db via the bare +// "sqlite" driver with WAL). This path is intended for when the aether gateway +// is DOWN: SQLite is single-writer and a running gateway holds the file. If the +// DB is locked, the error surfaced here suggests stopping the gateway. +func createTokenSQLite(ctx context.Context, cfg *config.Config, dataDir, tokenName string, principalType models.PrincipalType, level int) (string, error) { + if dataDir == "" { + return "", fmt.Errorf("--data-dir is required to create a token without PostgreSQL") + } + if err := os.MkdirAll(dataDir, 0o755); err != nil { + return "", fmt.Errorf("creating data directory %s: %w", dataDir, err) + } + + // tokens.db holds the api_tokens table; authsqlite.New runs its migrations. + tokensPath := filepath.Join(dataDir, "tokens.db") + tokensDB, err := openSQLiteNative(ctx, tokensPath) + if err != nil { + return "", fmt.Errorf("opening tokens database %s (is the gateway running and holding it?): %w", tokensPath, err) + } + defer tokensDB.Close() + + tokenStore, err := authsqlite.New(tokensDB) + if err != nil { + return "", fmt.Errorf("constructing sqlite token store: %w", err) + } + + // acl.db holds the ACL rules; aclsqlite.New runs its migrations and seeds + // default fallback policies. We seed an explicit grant so the bootstrap + // token's _system creator gets the requested (possibly elevated) access + // level on all workspaces, matching the PostgreSQL path. + aclPath := filepath.Join(dataDir, "acl.db") + aclDB, err := openSQLiteNative(ctx, aclPath) + if err != nil { + return "", fmt.Errorf("opening acl database %s (is the gateway running and holding it?): %w", aclPath, err) + } + defer aclDB.Close() + + // audit.db is the audit sink the ACL store stamps decisions into; opening it + // keeps parity with aetherlite's wiring. sharedAudit is nil — init-secrets + // is a one-shot CLI and does not run the async audit batcher. + auditPath := filepath.Join(dataDir, "audit.db") + auditDB, err := openSQLiteNative(ctx, auditPath) + if err != nil { + return "", fmt.Errorf("opening audit database %s: %w", auditPath, err) + } + defer auditDB.Close() + + aclStore, err := aclsqlite.New(aclDB, nil, auditDB, cfg.Gateway.GatewayID) + if err != nil { + return "", fmt.Errorf("constructing sqlite acl store: %w", err) + } + defer aclStore.Close() + + grant := func(ctx context.Context, principalType, principalID, resourceType, resourceID string, accessLevel int, grantedBy, reason string, expiresAt *time.Time) error { + _, gerr := aclStore.GrantAccess(ctx, principalType, principalID, resourceType, resourceID, accessLevel, grantedBy, reason, expiresAt) + return gerr + } + + return secrets.CreateInitialTokenWithStore(ctx, tokenStore, grant, cfg, tokenName, principalType, level) +} + +// openSQLiteNative opens a SQLite database via the bare "sqlite" driver +// (modernc.org/sqlite) with the same pragmas aetherlite uses, so init-secrets +// reads/writes the on-disk files identically (see cmd/aetherlite/main.go). +func openSQLiteNative(ctx context.Context, path string) (*sql.DB, error) { + db, err := sql.Open("sqlite", path) + if err != nil { + return nil, fmt.Errorf("failed to open SQLite database %s: %w", path, err) + } + for _, pragma := range []string{ + "PRAGMA journal_mode=WAL", + "PRAGMA busy_timeout=5000", + "PRAGMA foreign_keys=ON", + } { + if _, err := db.ExecContext(ctx, pragma); err != nil { + _ = db.Close() + return nil, fmt.Errorf("applying pragma %q on %s: %w", pragma, path, err) + } + } + return db, nil +} + func handleGenerateTLS(gs *secrets.GeneratedSecrets, secretsPath, tlsDir string, validity time.Duration, force, printSecrets bool, extraSANs []string) error { caCertPath := filepath.Join(tlsDir, "ca-cert.pem") caKeyPath := filepath.Join(tlsDir, "ca-key.pem") diff --git a/server/cmd/init-secrets/main_test.go b/server/cmd/init-secrets/main_test.go new file mode 100644 index 0000000..a6f8be8 --- /dev/null +++ b/server/cmd/init-secrets/main_test.go @@ -0,0 +1,85 @@ +package main + +import ( + "context" + "os" + "path/filepath" + "testing" + + "github.com/scitrera/aether/internal/acl" + authsqlite "github.com/scitrera/aether/internal/auth/sqlite" + "github.com/scitrera/aether/internal/config" + "github.com/scitrera/aether/pkg/crypto" + "github.com/scitrera/aether/pkg/models" +) + +func init() { + // Token hashing requires an initialized HMAC key. createTokenSQLite also + // initializes it from cfg, but we set a stable key here so the post-create + // validation (which re-hashes the plaintext) is consistent. + crypto.InitTokenHMAC([]byte("init-secrets-test-hmac-key-32byte!")) +} + +// TestCreateTokenSQLite_Direct exercises the SQLite-direct token-creation path +// used by aetherlite topologies: no PostgreSQL, no running gateway. It confirms +// the token is created, tokens.db is materialized, and the plaintext validates +// against the same store. +func TestCreateTokenSQLite_Direct(t *testing.T) { + ctx := context.Background() + dataDir := t.TempDir() + + cfg := &config.Config{} + cfg.Gateway.GatewayID = "test-init-secrets" + cfg.Auth.TokenHMACKey = "init-secrets-test-hmac-key-32byte!" + + plaintext, err := createTokenSQLite(ctx, cfg, dataDir, "test-bootstrap", models.PrincipalMetricsBridge, acl.AccessAdmin) + if err != nil { + t.Fatalf("createTokenSQLite: %v", err) + } + if plaintext == "" { + t.Fatal("expected non-empty plaintext token") + } + + // tokens.db and acl.db must have been created under dataDir. + for _, name := range []string{"tokens.db", "acl.db"} { + if _, statErr := os.Stat(filepath.Join(dataDir, name)); statErr != nil { + t.Errorf("expected %s to exist: %v", name, statErr) + } + } + + // Re-open the token store and validate the minted plaintext token. + tokensDB, err := openSQLiteNative(ctx, filepath.Join(dataDir, "tokens.db")) + if err != nil { + t.Fatalf("reopen tokens.db: %v", err) + } + defer tokensDB.Close() + + store, err := authsqlite.New(tokensDB) + if err != nil { + t.Fatalf("authsqlite.New: %v", err) + } + + tok, err := store.ValidateToken(ctx, plaintext) + if err != nil { + t.Fatalf("ValidateToken: %v", err) + } + if tok.Name != "test-bootstrap" { + t.Errorf("token Name = %q, want %q", tok.Name, "test-bootstrap") + } + if tok.PrincipalType != string(models.PrincipalMetricsBridge) { + t.Errorf("token PrincipalType = %q, want %q", tok.PrincipalType, models.PrincipalMetricsBridge) + } + if tok.CreatedBy != acl.SystemPrincipal { + t.Errorf("token CreatedBy = %q, want %q", tok.CreatedBy, acl.SystemPrincipal) + } +} + +// TestCreateTokenSQLite_RequiresDataDir confirms the SQLite path rejects an +// empty data dir rather than silently writing to the working directory. +func TestCreateTokenSQLite_RequiresDataDir(t *testing.T) { + cfg := &config.Config{} + _, err := createTokenSQLite(context.Background(), cfg, "", "x", models.PrincipalUser, acl.AccessAdmin) + if err == nil { + t.Fatal("expected error when data-dir is empty") + } +} diff --git a/server/cmd/loadtest/proxy/main.go b/server/cmd/loadtest/proxy/main.go index a9cc923..63e05ef 100644 --- a/server/cmd/loadtest/proxy/main.go +++ b/server/cmd/loadtest/proxy/main.go @@ -11,8 +11,8 @@ // - some callers address the wildcard sv::echo, others pin to sv::echo::a // or sv::echo::b // - latency percentiles, gateway heap/goroutine deltas, and TunnelAck -// back-pressure events are written to stdout AND appended to -// server/docs/proxy-load-test-results.md (when --append-doc=PATH). +// back-pressure events are written to stdout AND appended to the +// Markdown file given by --append-doc=PATH (when set). // // Soft assertions: missed thresholds print "WARN" but never set a non-zero // exit code; the brief explicitly classifies these as observational. diff --git a/server/cmd/proxy-sidecar/main.go b/server/cmd/proxy-sidecar/main.go index 0c4942c..8277d6c 100644 --- a/server/cmd/proxy-sidecar/main.go +++ b/server/cmd/proxy-sidecar/main.go @@ -200,6 +200,7 @@ func devDefaults() *proxysidecar.Config { Insecure: true, }, Service: proxysidecar.ServiceConfig{ + Workspace: "_sandbox", Implementation: "proxy-sidecar", Specifier: "instance-1", }, diff --git a/server/configs/acl_model.conf b/server/configs/acl_model.conf index d0ada8e..7bdb2df 100644 --- a/server/configs/acl_model.conf +++ b/server/configs/acl_model.conf @@ -11,6 +11,12 @@ # act: access level as string e.g. "10", "20", "30" # expires: RFC3339 timestamp or "" e.g. "2026-12-31T00:00:00Z" # rule_id: UUID of the acl_rules row e.g. "550e8400-e29b..." +# +# Role/group membership: g = member_subject, group_or_role_subject +# e.g. g, "user:alice", "group:engineering" +# g, "group:engineering", "role:workspace-admin" +# The role manager resolves these transitively (GetImplicitRolesForUser); the +# matcher does not reference g — evaluation expands the subject set in code. [request_definition] r = sub, obj, act @@ -18,6 +24,9 @@ r = sub, obj, act [policy_definition] p = sub, obj, act, expires, rule_id +[role_definition] +g = _, _ + [policy_effect] e = some(where (p.eft == allow)) diff --git a/server/docs/aetherlite-clustering.md b/server/docs/aetherlite-clustering.md index b0ddc25..e79aa4b 100644 --- a/server/docs/aetherlite-clustering.md +++ b/server/docs/aetherlite-clustering.md @@ -1,7 +1,5 @@ # AetherLite Clustering Guide -**Strategic design rationale:** See `/home/drew/.claude/plans/review-home-drew-oss-rqlite-and-home-dre-distributed-biscuit.md` for the complete architectural reasoning behind topology selection, backup tiers, and JetStream integration. - --- ## TL;DR: Topology Selection diff --git a/server/docs/proxy-architecture-roadmap.md b/server/docs/proxy-architecture-roadmap.md deleted file mode 100644 index 6bd3e7e..0000000 --- a/server/docs/proxy-architecture-roadmap.md +++ /dev/null @@ -1,404 +0,0 @@ -# Proxy / Tunnel Data-Plane Architecture Roadmap - -This document captures the long-term architecture for Aether's proxy and tunnel -data plane. It is written for engineers picking up Phase 7 (gateway-to-gateway -unicast forwarding) without prior context. - -**Related documents:** [proxy.md](proxy.md) · [proxy-cutover.md](proxy-cutover.md) · -[proxy-load-test-results.md](proxy-load-test-results.md) · [proxy-quickstart.md](proxy-quickstart.md) · -[proxy-sandbox.md](proxy-sandbox.md) - ---- - -## 0. Shipped Features (Current State) - -These features are fully implemented and available in the current codebase. -They are listed here so engineers reading the roadmap can distinguish "already -done" from "planned work" without digging through commit history. - -| Feature | Mode / config | Where to find it | -|---------|--------------|------------------| -| **REST proxy** | `terminator` + `initiator` sidecar modes | `server/internal/proxysidecar/terminator.go`, `initiator.go` | -| **TCP / WebSocket / UDP tunnels** | `terminator` mode, `kind: tcp/ws/udp` backends | `tunnel_tcp.go`, `tunnel_ws.go`, `tunnel_udp.go` | -| **Multi-backend routing** | First-match-by-ACL or explicit `backend_name` | `server/internal/proxysidecar/terminator.go` | -| **`proxy_path` resource scope** | Per-grant HTTP path ACL | `server/pkg/identityheaders/proxypath.go` | -| **`tunnel_target` resource scope** | Per-grant tunnel target ACL | `server/pkg/identityheaders/tunneltarget.go` | -| **`stream_response_indefinitely`** | SSE / long-poll opt-in on `ProxyHttpRequest` | `server/internal/proxysidecar/terminator.go` | -| **SIGHUP reload** | Zero-downtime backend swap on `kill -HUP` | `server/internal/proxysidecar/terminator.go` | -| **Relay mode** (`relay`) | gRPC mitm — sandbox dials UDS, sidecar injects credentials | `server/internal/proxysidecar/relay.go` | -| **Relay+terminator composite** (`relay+terminator`) | Both surfaces in one process, one gateway lock | `server/internal/proxysidecar/composite.go` | -| **Caller identity headers** | `X-Aether-Caller-Topic` / `X-Aether-Caller-Subject` stamped by terminator | `server/internal/proxysidecar/terminator.go` | -| **Hop-depth tracking** | `proxy_chain_depth` clamped by relay to prevent loops | `server/internal/proxysidecar/relay.go` | -| **Local single-node bypass (Phase 6)** | In-process fast path for same-gateway caller+target | `server/internal/gateway/routing_proxy.go` | - -**Relay mode** in particular is the foundation of the sandbox deployment -pattern documented in [proxy-sandbox.md](proxy-sandbox.md). A sandbox process -dials `unix:///run/aether.sock` with no credentials; the sidecar enforces an -operation allow-list (`sandbox-default`, `sandbox-tunnels`, `tool-stub-only` -profiles) and a target-topic clamp before forwarding to the real gateway. - ---- - -## 1. Motivation - -### Why RMQ is wrong for proxy/tunnel data - -Aether's proxy and tunnel features initially route all bytes — control and data -alike — through RabbitMQ Streams. RMQ Streams are designed for fan-out, -durability, and replayable logs. Proxy/tunnel data needs none of that: - -- **No fan-out.** Every `TunnelData` or `ProxyHttpBodyChunk` frame has exactly - one destination — the pinned target sidecar or the caller. Broadcasting is - wasteful. -- **No durability.** In-flight bytes are ephemeral. If a tunnel dies, the - session is over; replaying stale frames into a reconnected tunnel produces - garbage. -- **No replay.** Tunnels and proxy streams have strict ordering and are - connection-scoped. An RMQ consumer offset replay mid-reconnect delivers bytes - to the wrong context. - -The stored-offset semantics caused a concrete failure (T29 smoke validation): -a sidecar reconnect triggered an offset interaction that skipped frames -published mid-disconnect. This is a structural mismatch, not a tuning problem. - -### What Phase 6 solves - -Phase 6 (local single-node bypass, implemented in -`server/internal/gateway/routing_proxy.go`) adds an in-process fast path: -when the caller and target sidecar are both connected to the **same** gateway -instance, data-plane bytes are delivered directly between gRPC streams using -each `ClientSession`'s existing `Deliver()` channel. The RMQ publish is -skipped entirely for those frames. - -This eliminates the RMQ offset problem for the same-node case and improves -throughput substantially (no serialisation, no network round-trip). The fast -path is controlled by `proxy.local_bypass_enabled` (default `true`). - -### Why Phase 7 picks unicast forwarding over alternatives - -Cross-gateway routes still ride RMQ after Phase 6. For deployments with -multiple gateway instances, data-plane bytes still incur the RMQ overhead on -any tunnel whose caller and target happen to land on different instances. - -The alternatives are: - -| Option | Problem | -|--------|---------| -| Raft / Paxos cluster state | Requires consensus layer, leader election, quorum logic — all unnecessary overhead for a routing problem | -| Gossip / SWIM membership | Adds operational complexity; another thing to monitor and tune | -| Service mesh (Istio/Linkerd) | External infrastructure dependency; not portable across deployments | -| Custom transport (QUIC, etc.) | Maintenance burden; gRPC already provides reliable multiplexed streams over HTTP/2 | - -The right answer is **unicast forwarding**: each gateway opens a single -bidirectional gRPC stream to each peer gateway it needs to reach. Redis — which -already stores session lock metadata — serves as the directory. No new -consensus layer, no gossip, no mesh. - ---- - -## 2. Control / Data Plane Split - -This split is a hard contract. It must not be violated during Phase 6 or Phase 7. - -| Envelope | Plane | Substrate (Phase 6) | Substrate (Phase 7) | -|---|---|---|---| -| `TunnelOpen` | Control | RMQ | RMQ | -| `TunnelClose` | Control | RMQ | RMQ | -| `ProxyHttpRequest` (header) | Control | RMQ | RMQ | -| `ProxyHttpResponse` (header) | Control | RMQ | RMQ | -| `ProxyError` | Control | RMQ | RMQ | -| `TunnelData` | Data | Local fast path or RMQ | Local fast path or peer-gateway forward | -| `TunnelAck` | Data | Local fast path or RMQ | Local fast path or peer-gateway forward | -| `ProxyHttpBodyChunk` | Data | Local fast path or RMQ | Local fast path or peer-gateway forward | - -**Invariant:** the control plane is what the audit pipeline listens on. Control -envelopes ride RMQ even when caller and target are on the same gateway. This -guarantees that every lifecycle event — `tunnel_opened`, `tunnel_closed`, -`proxy_http_routed`, `proxy_http_stream_closed` — is observed by the audit -pipeline without any special-casing in the fast path. - -Per-byte audit is a non-goal. Per-session byte summaries are stamped on the -close event (already implemented), which rides the control plane. - ---- - -## 3. Phase 7: Gateway-to-Gateway Unicast Forwarding - -### 3.1 Discovery - -Each gateway instance registers its forwarding address in Redis at startup: - -``` -key: gateway:{id}:forward_addr -value: host:port (e.g. "gw-3.internal:50552") -TTL: refreshed on the same schedule as the existing gateway heartbeat -``` - -The session metadata for each connected principal already records which -gateway holds that principal's connection. The `lock:{identity}` value remains -the bare sessionID (load-bearing for resume/takeover semantics); the -`gateway_id` is stored as a separate field on the per-session HASH at -`session:{sessionID}`. When an originating gateway needs to forward a -data-plane envelope to a principal on a different gateway, it: - -1. Calls `SessionManager.GetSessionGateway(ctx, identity)`, which atomically - reads the lock value (sessionID) and the `gateway_id` field from the - session HASH via a Lua script. -2. Looks up `gateway:{peer_id}:forward_addr` to obtain the dial address. -3. Dials (or reuses) the bidirectional forwarding stream to that peer. - -A stale or missing `forward_addr` is not catastrophic. A failed dial falls back -to RMQ (or returns `ProxyError{SIDECAR_UNAVAILABLE}` if -`proxy.peer_forward.fallback` is set to `"error"`). - -**Status:** the `gateway_id` storage and `GetSessionGateway` lookup primitive -are implemented in `server/internal/state/session.go` (Redis) and -`server/internal/state/badger_session.go` (lite/Badger). The Phase-7 work -remaining is the forwarding RPC, peer connection pooling, and -`gateway:{id}:forward_addr` registration; principal discovery is no longer a -prerequisite. - -### 3.2 Connection Topology - -**One bidirectional gRPC stream per ordered gateway pair.** Because the stream -is bidirectional, `(gw-3 → gw-7)` and `(gw-7 → gw-3)` share the same -physical stream — each gateway both sends and receives on it. - -- Established **lazily** on the first cross-gateway data-plane envelope. -- Idle-evicted after `proxy.peer_forward.idle_timeout` (default `5m`). -- Keyed by `peer_gateway_id` in a per-gateway connection pool. -- Concurrent senders multiplex onto the stream via a per-peer write-channel and - a dedicated send goroutine (avoids write contention without a mutex hold in - the routing hot path). -- Multiplexed by `forward_id` (`tunnel_id` or `request_id` from the payload). - -A naive per-tunnel peer connection would produce N×M connection blowup across -gateway pairs under load. The shared multiplexed stream keeps peer connections -constant regardless of tunnel count. - -### 3.3 Forwarding RPC (Proto Sketch) - -New service in a new file `api/proto/gateway_forward.proto` (or appended to -`aether.proto` — the file boundary is an implementation detail): - -```proto -service GatewayForward { - rpc Forward(stream ForwardEnvelope) returns (stream ForwardEnvelope); -} - -message ForwardEnvelope { - oneof payload { - TunnelData tunnel_data = 1; - TunnelAck tunnel_ack = 2; - ProxyHttpBodyChunk body_chunk = 3; - KeepAlive keep_alive = 10; - } - string forward_id = 20; // tunnel_id or request_id from the payload - string source_principal = 21; // originating principal identity string - string target_principal = 22; // destination principal identity string - string source_gateway = 23; // originating gateway_id (for audit / debug) -} - -message KeepAlive { - int64 sent_at_ms = 1; -} -``` - -**Receiver dispatch:** the receiving gateway peels the `payload` oneof, looks -up `target_principal` in its local `identityIndex` (`sync.Map` field on -`GatewayServer`, keyed by identity string), and delivers via the existing -`ClientSession.Deliver()` method. If the target is not present in -`identityIndex`, the receiver sends back a typed forwarding error; the -originator surfaces this to the caller as `PEER_RESET`. - -### 3.4 mTLS - -- Gateways use the same TLS certificate material already deployed for the - gRPC client listener. -- A dedicated peer CA bundle is configurable at - `proxy.peer_forward.tls.ca_path`. -- Optional: allowlist specific peer certificate hashes at - `proxy.peer_forward.tls.peer_cert_hashes`. Fails closed if configured and - the peer does not match. - -### 3.5 Failure Modes - -| Failure | Behaviour | -|---------|-----------| -| Peer unreachable (dial fails) | Retry with backoff; after threshold, fall back to RMQ or return `PEER_RESET` (per `proxy.peer_forward.fallback`) | -| Peer alive but stream broken | Reconnect with exponential backoff; in-flight bytes lost (caller sees `PEER_RESET`, same as today's RMQ-partition behaviour) | -| Stale Redis address (peer restarted) | Dial fails → re-resolve principal's `gateway_id` via `GetSessionGateway` (principal may have reconnected to a different gateway) → retry forward to new peer; bound retries to avoid loops | -| Network partition | Both sides continue serving local clients; cross-partition tunnels die with `PEER_RESET` — identical to current behaviour on RMQ partition | - -### 3.6 Audit - -The audit contract is unchanged from today: - -- `OpProxyHttpRouted`, `OpProxyHttpFailed`, `OpProxyHttpStreamClosed`, - `OpTunnelOpened`, `OpTunnelOpenFailed`, `OpTunnelClosed` all fire on the - **originating** gateway when the corresponding control-plane envelope passes - through it. Control envelopes remain on RMQ in Phase 7 and are therefore - observed by the existing audit pipeline without modification. - -A new optional per-session event `OpProxyForwardEdge` may be emitted on session -close, capturing: - -``` -forward_id, source_gateway, target_gateway, bytes_forwarded, started_at, ended_at -``` - -This event is optional (it is not a substitute for the existing close-event -byte counts) and is intended to help operators correlate forwarding activity -across gateways. Per-byte audit remains a non-goal. - -Prometheus metrics carry byte counts for the forwarding stream; those are not -audit events. - -### 3.7 Observability - -Metrics exposed by the forwarding subsystem: - -| Metric | Labels | Description | -|--------|--------|-------------| -| `aether_proxy_peer_forward_bytes_total` | `direction`, `peer` | Bytes forwarded through peer streams | -| `aether_proxy_peer_forward_streams_active` | `peer` | Currently open peer forwarding streams | -| `aether_proxy_peer_forward_dial_failures_total` | `peer`, `reason` | Failed dial attempts to peer gateways | -| `aether_proxy_peer_forward_reconnects_total` | `peer` | Peer stream reconnection attempts | - -Tracing: each `ForwardEnvelope` carries `forward_id`; tracing context -propagates via standard gRPC metadata headers. - -Logging: peer-stream lifecycle events (open, close, error) are logged at INFO -or WARN; per-envelope logging is not emitted (too high volume). - -### 3.8 Backpressure - -`TunnelAck` credit semantics are unchanged. In Phase 7, credits flow through -the forwarding stream instead of RMQ: - -- The receiver gateway drains `TunnelAck` envelopes from the peer stream and - delivers them to the local `ClientSession` via `Deliver()`. -- The peer-stream send buffer is bounded. When it fills, the originating - gateway stalls the routing goroutine for that tunnel, applying the same - backpressure the RMQ publish buffer applies today. -- The `BACKPRESSURE` signal sent by `ClientSession.Deliver()` on full buffer - still applies at the local-delivery stage. - -### 3.9 Configuration - -Full config block under the `proxy` key: - -```yaml -proxy: - local_bypass_enabled: true # Phase 6 — in-process fast path - - peer_forward: - enabled: false # Phase 7 — off by default; flip per deployment - listen_addr: ":50552" # gateway-to-gateway gRPC listen port - advertise_addr: "" # external address published to Redis (inferred from listen_addr if empty) - idle_timeout: 5m # evict idle peer streams after this duration - fallback: rmq # "rmq" (fall back to RMQ on dial failure) or "error" (return PEER_RESET) - tls: - ca_path: /etc/aether/peer-ca.crt - cert_path: /etc/aether/peer.crt - key_path: /etc/aether/peer.key -``` - -Environment override for Phase 6 emergency rollback: -`AETHER_PROXY_LOCAL_BYPASS_DISABLED=1`. - -### 3.10 Migration Path (Phase 7 Rollout) - -1. **Build under feature flag.** Implement the forwarding service with - `peer_forward.enabled: false` as default. All existing tests pass - unmodified because the flag is off. -2. **Stage 1 — Canary.** Enable on one gateway instance. Observe - `aether_proxy_peer_forward_*` metrics. Verify audit invariants: control-plane - events still fire; per-session byte summaries match expectations from the - non-forwarding path. -3. **Stage 2 — Single workspace soak.** Enable globally but restrict to one - workspace via a workspace-scoped config override. Run for one week minimum. -4. **Stage 3 — Full production.** Enable globally. Keep `fallback: rmq` for - defence in depth — RMQ remains the fallback for any peer-stream failure. -5. **Stage 4 (optional, far future).** After weeks of production confidence, - evaluate switching `fallback: error`. This effectively stops using RMQ for - proxy/tunnel data plane entirely on a per-failure basis. See Phase 8 below. - ---- - -## 4. Phase 8 (Optional, Not Planned): Retire RMQ for Proxy/Tunnel Data Plane - -After Phase 6 and Phase 7 are proven in production, the only remaining RMQ -traffic for proxy/tunnel would be: - -- Control-plane envelopes (`TunnelOpen`, `TunnelClose`, `ProxyHttpRequest` - header, `ProxyHttpResponse` header, `ProxyError`) — these always ride RMQ. -- The Phase 7 fallback path when `peer_forward.fallback: rmq`. - -Two options: - -**Option A — Keep RMQ for control plane indefinitely (recommended).** -Control-plane volume is low (one open + one close per tunnel session), the -audit pipeline is already integrated with RMQ, and the rest of Aether -messaging uses RMQ correctly. Unless RMQ becomes a bottleneck for -control-plane volume specifically, there is no cost to keeping it. - -**Option B — Move control plane to the forwarding stream too.** -Eliminates RMQ as a dependency for proxy/tunnel entirely. Substantially larger -change: audit pipeline integration point moves, ordering guarantees must be -re-established in the forwarding layer. Likely overkill. - -Phase 8 is **not committed work**. Document but do not plan. - ---- - -## 5. Non-Goals - -These will not be built. Documented explicitly to prevent drift. - -- **Raft / Paxos / leadership election among gateways.** Redis is the directory - of record for principal location. There is nothing to elect. -- **Cluster membership protocol (gossip, SWIM, etc.).** Discovery is a Redis - lookup, not a gossip query. Gateways do not maintain a shared cluster view. -- **Replacing RMQ for general Aether messaging.** Fan-out broadcasts, - orchestration task dispatch, KV propagation, and audit all use RMQ - correctly — durable, replayable, fan-out. Those use cases stay on RMQ. -- **Service mesh dependency (Istio, Linkerd, etc.).** Gateways handle their own - mTLS for peer forwarding. No platform-level mesh is required or assumed. -- **Custom transport stack (QUIC implementation, custom HTTP/2 codec, etc.).** - gRPC over HTTP/2 is the transport for peer forwarding, used as-shipped. The - standard library's TLS and HTTP/2 stacks are sufficient. - ---- - -## 6. Open Questions for Phase 7 Implementation - -These questions do not block Phase 6. They are flagged for the team picking up -Phase 7. - -**Multiplexing approach.** The design specifies a single bidirectional gRPC -stream per gateway pair, multiplexed by `forward_id`. An alternative is one -stream per active tunnel session. Per-session is simpler to implement but -produces O(active-tunnels) peer connections — unacceptable at scale. The -`forward_id` mux design is the chosen approach; document the rationale in the -commit that introduces `GatewayForward.Forward`. - -**Keep-alive mechanism.** The proto sketch includes a `KeepAlive` envelope. -An alternative is to rely on gRPC's built-in HTTP/2 PING frames (configured -via `grpc.KeepaliveParams`). The HTTP/2 PING approach is cleaner and avoids -application-layer keep-alive logic. If the `KeepAlive` envelope is retained, -document why (e.g. PING frames were insufficient for a specific NAT/proxy -scenario encountered during testing). - -**Cross-cluster scope.** The current design assumes all gateway instances are -reachable over a private network. Cross-cluster or internet-traversing -forwarding raises a different security boundary question (certificate authority -trust, firewall traversal). Document this as out of scope for Phase 7: tunnels -spanning a public internet hop should use application-level encryption between -the caller and the target service, not rely on the gateway forwarding layer. - -**Testing without multiple real gateway processes.** The T7/T14 integration -tests use an in-process gateway. Phase 7 testing should use the same pattern: -an in-process two-`GatewayServer` harness where both server structs forward -data-plane envelopes to each other over a loopback listener. This tests the -full forwarding code path without external infrastructure. diff --git a/server/docs/proxy-cutover.md b/server/docs/proxy-cutover.md deleted file mode 100644 index be57508..0000000 --- a/server/docs/proxy-cutover.md +++ /dev/null @@ -1,416 +0,0 @@ -# MemoryLayer → Aether Proxy: Cutover Criteria and Rollback Plan - -This document is the production rollout playbook for migrating MemoryLayer callers from -direct HTTP (or HTTP + auth-proxy) to the Aether proxy sidecar path. Follow every section -in order. Do not flip the feature flag to aether-only until the sign-off checklist is -complete. - -**Related docs:** -- [Proxy overview](proxy.md) — architecture, SDK one-liners, ACL, audit events -- [Proxy quickstart](proxy-quickstart.md) — running the sidecar and integration tests -- [Load test results](proxy-load-test-results.md) — routing-layer benchmark results and scope caveats - ---- - -## 1. Dual-Path A/B Model - -Both transport paths run simultaneously during the canary period. No infrastructure is -decommissioned until the sign-off checklist is complete. - -### Paths in parallel - -| Path | Description | -|---|---| -| **Direct HTTP / auth-proxy** | Current production path. Callers hit MemoryLayer directly (or via the auth-proxy sidecar for header injection). This remains the fallback. | -| **Aether sidecar (canary)** | New path. Callers use an Aether SDK transport adapter or an initiator sidecar. The terminator sidecar (`sv::memorylayer::default`) receives the request and forwards it to the local MemoryLayer HTTP server. | - -### Canary traffic-split mechanism - -Traffic split is **caller-side**. The gateway has no concept of a canary percentage — -routing is determined by which transport adapter the caller mounts. - -**Option A — Feature flag per caller (recommended):** - -Each caller reads an environment variable (or config key) at startup: - -``` -AETHER_PROXY_DISABLED=1 # if set (any non-empty value), skip Aether and use direct HTTP -``` - -Roll out the canary by deploying the Aether-aware caller binary to a fraction of pods -and setting `AETHER_PROXY_DISABLED=1` on the remaining pods. Increase the fraction over -time. - -**Option B — Separate deployment target:** - -Deploy a shadow caller service that routes 100% of its traffic via Aether. Direct live -traffic at it incrementally via load-balancer weights or feature flags in the caller -configuration layer. - -### Dial-up schedule (suggested — confirm with ops) - -| Day | Aether canary fraction | -|---|---| -| 1–2 | 5% (smoke test) | -| 3–4 | 25% | -| 5–6 | 50% | -| 7 | 75% | -| 8–14 | 100% canary — observe for 7 consecutive days before flipping to aether-only | - -### Header-mint parity - -Header byte-equality between the auth-proxy path and the sidecar path is enforced at the -source: both call `server/pkg/identityheaders.Mint` (the same library). This is verified -by the `TestPhase1_OBO_HeadersByteEqualToAuthProxy` integration test and the -`header_mode_test.go` golden test in the sidecar package. No manual check is required -during normal operation; see [Sign-off checklist](#6-sign-off-checklist) for the -production canary sampling step. - ---- - -## 2. SLOs for the Cutover - -These targets apply to the Aether sidecar path measured during the canary period. -Numbers marked **(suggested — confirm with ops)** are starting points; adjust based on -your MemoryLayer SLA and observed direct-HTTP baselines. - -### 2.1 REST proxy latency - -Measure end-to-end latency at the caller (time from issuing the HTTP request to -receiving the complete response body). - -| Metric | Target | -|---|---| -| p50 latency | ≤ 1.5× direct-HTTP p50 **(suggested — confirm with ops)** | -| p99 latency | ≤ 2× direct-HTTP p99, hard ceiling 250 ms **(suggested — confirm with ops)** | - -The 250 ms hard ceiling accounts for gRPC stream overhead, gateway routing, and the -local sidecar HTTP roundtrip. Adjust downward if your direct-HTTP p99 is already well -below 125 ms. - -### 2.2 Tunnel-open latency - -In-process benchmarks: p50 = 20 ms, p99 = 38 ms (100 concurrent opens, no real -network). See [proxy-load-test-results.md](proxy-load-test-results.md) for scope caveats. - -| Metric | Target | -|---|---| -| Tunnel-open p99 (real network) | ≤ 250 ms **(suggested — confirm with ops)** | - -This accounts for real-network RTT between the caller and the gateway. If your gateway -and callers are co-located in the same AZ, 100 ms p99 is achievable. - -### 2.3 Error rate - -| Metric | Target | -|---|---| -| Aether-path error rate above direct-HTTP baseline | ≤ 0.1% of requests **(suggested — confirm with ops)** | - -Count `ProxyError` responses (`DIAL_FAILED`, `TIMEOUT`, `SIDECAR_UNAVAILABLE`, -`UPSTREAM_RESET`) as errors. `ACL_DENIED` and `PAYLOAD_TOO_LARGE` are configuration -issues, not transport errors, and should be zero. - -### 2.4 Availability - -| Metric | Target | -|---|---| -| Aether-path availability (rolling 7-day) | ≥ 99.9% of requests successfully proxied **(suggested — confirm with ops)** | - -A request is "successfully proxied" if it receives an HTTP response (any status code) -from MemoryLayer. Transport-layer failures (`ProxyError`) count as unavailability. - ---- - -## 3. Audit Completeness Check - -Every MemoryLayer call routed through the Aether proxy must appear as a -`proxy_http_routed` (or `proxy_http_failed`) audit event. The audit event includes the -`request_id` field that uniquely identifies each proxy call. - -### 3.1 Daily reconciliation - -Run this check once per day during the canary period (and permanently in production): - -1. **Aether audit count:** Count distinct `request_id` values in the audit log where - `operation = 'proxy_http_routed' OR operation = 'proxy_http_failed'` in the relevant - time window. - -2. **MemoryLayer access log count:** Count distinct request IDs (or the `X-Request-Id` - header value forwarded by the sidecar) in MemoryLayer's own access log for the same - time window. - -3. **Match requirement:** `audit_count / memorylayer_count ≥ 99.9%` - -Any drift below 99.9% must be investigated before increasing the canary fraction. - -Relevant audit operation codes (from `server/internal/audit/types.go`): - -``` -proxy_http_routed — request successfully forwarded to the sidecar -proxy_http_failed — request could not be delivered -proxy_http_stream_closed — streaming response finished (SSE / long-poll) -tunnel_opened — TunnelOpen successfully established -tunnel_open_failed — TunnelOpen rejected -tunnel_closed — tunnel torn down -``` - -### 3.2 Alert thresholds and escalation (suggested — confirm with ops) - -| Condition | Action | -|---|---| -| Drift > 0.1% for one reconciliation window | Alert on-call. Investigate before next window. | -| Drift > 1% for one reconciliation window | Page on-call immediately. Pause canary dial-up. | -| Drift > 5% for one reconciliation window | Roll back to direct HTTP. See [Section 4](#4-rollback-steps). | - -Who pages: the audit reconciliation job should fire to the same on-call rotation as -MemoryLayer availability alerts. - ---- - -## 4. Rollback Steps - -Target: **full rollback within 5 minutes of signal.** - -### 4.1 Feature flag rollback (fastest — per-caller) - -Set `AETHER_PROXY_DISABLED=1` in the caller's environment and restart (or send SIGHUP -if hot-reload is supported). The caller short-circuits to direct HTTP before any Aether -SDK call is made. - -This is the primary rollback mechanism. It requires no gateway changes and no sidecar -drain. - -### 4.2 Per-language transport revert (one-liner each) - -If you cannot use the feature flag, revert the transport adapter in code: - -**Go** — replace `AetherRoundTripper` with a plain `http.Client`: -```go -// Before (Aether path): -// rt := &aether.AetherRoundTripper{Client: agentClient, Target: "sv::memorylayer::default"} -// httpClient := &http.Client{Transport: rt} - -// After (direct HTTP revert): -httpClient := &http.Client{} -``` - -**Python — httpx** — replace `AetherHTTPXTransport` with the default transport: -```python -# Before (Aether path): -# transport = AetherHTTPXTransport(aether_client, "sv::memorylayer::default") -# http = httpx.Client(transport=transport) - -# After (direct HTTP revert): -http = httpx.Client() -``` - -**Python — requests** — unmount the adapter: -```python -# Before (Aether path): -# session.mount("aether+sv://", AetherRequestsAdapter(aether_client)) - -# After (direct HTTP revert): -session = requests.Session() # fresh session with no Aether adapter -``` - -**TypeScript** — stop using `AetherFetchTransport`: -```typescript -// Before (Aether path): -// const transport = new AetherFetchTransport(agentClient, "sv::memorylayer::default"); -// const resp = await transport.fetch("/v1/memories/abc"); - -// After (direct HTTP revert): -const resp = await fetch("https://memorylayer.internal/v1/memories/abc"); -``` - -### 4.3 Per-deployment revert (all callers) - -1. Set `AETHER_PROXY_DISABLED=1` on all caller deployments via your config management - layer (e.g., update the ConfigMap / Helm values / environment override and roll). -2. Confirm traffic is flowing to direct HTTP by checking that `proxy_http_routed` audit - events stop arriving within 2 minutes. -3. Leave the terminator sidecar running but idle — do not terminate it until the cause - of the rollback is diagnosed and resolved. This preserves the sidecar's gRPC - connection for debugging. -4. Once the root cause is resolved and the sidecar is healthy, remove - `AETHER_PROXY_DISABLED` and redeploy to resume canary. - -### 4.4 Rollback decision authority - -Any on-call engineer may initiate rollback unilaterally if: -- Any SLO in Section 2 is breached for more than 5 minutes, or -- Audit drift exceeds 5% (Section 3.2), or -- MemoryLayer returns a sustained error spike traceable to the Aether path. - ---- - -## 5. Runbook Entries - -### 5.1 Sidecar pod down - -**Detection:** -- Redis lock for `sv::memorylayer::default` has expired (30 s TTL). The gateway's - wildcard resolver will drop this instance from the candidate set once its lock TTL - decays below 5 s. -- Audit events for `proxy_http_routed` stop or drop sharply. -- If using a specific (non-wildcard) target, callers receive - `ProxyError{SIDECAR_UNAVAILABLE}`. - -**Automatic mitigation (wildcard target):** -Callers using `sv::memorylayer` (wildcard) are automatically routed to other healthy -instances. No manual action is needed if at least one instance remains online. - -**Manual drain (if needed):** -1. Cordon the sidecar pod from receiving new requests (e.g., remove it from the - load-balancer target group or set `AETHER_PROXY_DISABLED=1` on callers pinned to - that instance). -2. Allow in-flight requests to complete (the lock TTL is 30 s; wait at least 30 s after - the pod stops receiving new traffic). -3. Restart or reschedule the sidecar pod. -4. Verify the new instance registers its lock in Redis and appears in the gateway's - candidate set (check the admin UI or `proxy_http_routed` audit events resuming). - -**When to page:** Page if all sidecar instances are offline simultaneously (see 5.2). - -### 5.2 Wildcard target has zero healthy instances - -**Symptom:** Every `proxy_http_routed` audit event is replaced by `proxy_http_failed`. -Callers receive `ProxyError{SIDECAR_UNAVAILABLE}`. - -**Alert:** Fire an alert when the rate of `proxy_http_failed` with kind -`SIDECAR_UNAVAILABLE` exceeds 1% of proxy requests for more than 60 seconds. - -**Failover path:** -1. Immediately set `AETHER_PROXY_DISABLED=1` on all caller deployments (Section 4.3). - Callers fall back to direct HTTP within seconds. -2. Investigate why all sidecar instances are down (pod crash loop, connectivity loss, - gateway auth failure). -3. Bring at least one sidecar instance online and verify it registers before removing - `AETHER_PROXY_DISABLED`. - -### 5.3 Auth header mismatch - -**Symptom:** MemoryLayer returns HTTP 401 for requests arriving via the Aether sidecar -path. Direct-HTTP requests (bypassing Aether) succeed. - -**Diagnosis:** -1. Check that the terminator sidecar's `header_mode` is set to `strict` (the default). - In `strict` mode the sidecar strips all inbound `Authorization` / `X-Auth-*` headers - and mints fresh ones from the `ProxyHttpRequest.authorization` field. -2. Verify the minted header values match what `identityheaders.Mint` produces. Both the - sidecar terminator and the auth-proxy call the same `server/pkg/identityheaders` - library. Run the OBO byte-equality test to confirm: - ```bash - cd server - /home/drew/sdk/go1.25.5/bin/go test -count=1 -run TestPhase1_OBO_HeadersByteEqualToAuthProxy -tags=integration ./tests/integration/... - ``` -3. Check that the `AuthorizationContext` in the `ProxyHttpRequest` envelope is populated - correctly by the caller's SDK (inspect via gateway debug logging at `log_level: debug`). -4. If the minted headers differ from the auth-proxy output, file a bug against - `server/pkg/identityheaders`. Do not patch the sidecar independently — the library - is the single source of truth. - -**Resolution:** Fix the `AuthorizationContext` construction in the caller or the -`identityheaders` library, then re-run the integration test suite before re-enabling -the canary. - -### 5.4 Stuck tunnel (pin not refreshing, lock TTL expired) - -**Symptom:** A tunnel stops transferring data. `TunnelData` frames are sent but no -response arrives. The tunnel-side goroutines are still alive but blocked. - -**Diagnosis:** -1. Check whether the target sidecar instance's Redis lock has expired. If the lock TTL - decayed to zero (gateway crashed, network partition) the tunnel's Redis pin is - stale. -2. In the gateway logs, look for `tunnel pin expired` or `tunnel closed: lock expired` - log lines for the affected `tunnel_id`. -3. Check that `tunnel_closed` audit events were emitted for the tunnel. - -**Resolution:** -1. Force-close the tunnel from the caller side by closing the Aether gRPC stream or - calling `TunnelClose` from the SDK. The gateway will clean up the Redis pin on - stream disconnect. -2. If the gateway itself is hung, restart the gateway instance. Its lock TTL (30 s) - will expire automatically; the tunnel is torn down on lock expiry. -3. Reconnect and open a new tunnel. - -**Prevention:** Ensure the gateway lock-refresh goroutine is not starved. Under extreme -CPU load, lock refresh (every 10 s against a 30 s TTL) may fall behind. Monitor the -`aether_redis_lock_refresh_errors_total` Prometheus counter. - -### 5.5 Quota / rate-limit hit - -**Symptom:** Callers receive `ProxyError{PAYLOAD_TOO_LARGE}` or tunnel opens fail. - -**Relevant config knobs** (gateway YAML, `proxy` section): - -| Config field | Default | What to adjust | -|---|---|---| -| `proxy.max_request_body_bytes` | 8 MiB | Cap on the inline body carried in the parent `ProxyHttpRequest` envelope. Streaming uploads via `ProxyHttpBodyChunk` are bounded instead by the per-backend `max_body_bytes` (default 10 MiB). | -| `proxy.max_concurrent_tunnels_per_workspace` | 256 | Increase if running many simultaneous TCP tunnels per workspace. | -| `proxy.max_tunnel_bytes` | 0 (unlimited) | Set a non-zero value to cap cumulative bytes per tunnel session. | - -Adjust these in the gateway config and redeploy. No code changes are needed. - ---- - -## 6. Sign-off Checklist - -Do not flip MemoryLayer to aether-only traffic until every item below is checked. - -- [ ] **Header-mint byte-equality verified in production canary.** Sample at least - 10,000 requests during the canary period. For each sampled request, compare the - `X-Auth-*` headers received by MemoryLayer via the Aether sidecar path against the - headers produced by a shadowed direct-HTTP call to the auth-proxy. Require 100% match. - -- [ ] **SLOs met for 7 consecutive days at 100% canary.** All metrics in Section 2 - (p50/p99 latency, error rate, availability) are within target for the final 7-day - canary window before the flip. - -- [ ] **Audit reconciliation drift < 0.1% for 7 consecutive days.** The daily - reconciliation defined in Section 3 shows ≥ 99.9% match between Aether audit events - and MemoryLayer access log for 7 consecutive days at 100% canary. - -- [ ] **Rollback procedure rehearsed at least once in staging.** Execute Section 4.3 - in the staging environment: set `AETHER_PROXY_DISABLED=1`, confirm traffic switches - to direct HTTP within 2 minutes, then remove the flag and confirm Aether traffic - resumes. Time the total round-trip; it should be under 5 minutes. - -- [ ] **Service-side gRPC delivery validated with a real gateway.** The in-process - harness confirmed routing primitives. Before the aether-only flip, run an - end-to-end test with a real gateway, real RabbitMQ Streams, and a real Redis instance. - This validates the full dispatch path including `TunnelData`/`TunnelClose` handling - under real network conditions (see Section 7). - -- [ ] **`proxy_path` and `tunnel_target` ACL scopes reviewed.** If any authority grants - carry `proxy_path` or `tunnel_target` resource scopes, verify the patterns are correct - before going aether-only: confirm that each grant's `proxy_path` patterns cover the - HTTP methods and paths the caller actually uses, and that each `tunnel_target` pattern - covers the protocol + remote-hint strings the caller sends. A misconfigured scope - silently denies requests with `ACL_DENIED`; test in staging first. - -- [ ] **All on-call runbook entries documented and drilled.** Every runbook in Section - 5 has been walked through in a tabletop exercise or staging drill. Participating - engineers are familiar with the `AETHER_PROXY_DISABLED` flag and the sidecar drain - procedure. - -- [ ] **Monitoring in place.** Prometheus alerts are configured for: - - `proxy_http_failed` rate exceeding the error-rate SLO - - `SIDECAR_UNAVAILABLE` error kind sustained > 60 s - - Audit reconciliation drift > 0.1% - - Tunnel-open p99 exceeding the latency SLO - ---- - -## 7. Known Limitations - -### 7.1 Real-network load test pending - -The in-process benchmark used an in-process harness (no real gRPC, no real RabbitMQ, no -real Redis). See [proxy-load-test-results.md](proxy-load-test-results.md) for the full -scope caveat. Measured tunnel-open latency (p50=20 ms, p99=38 ms) reflects the routing -layer only; real-network numbers will be higher. - -**Limitation:** A wire-level load test against a deployed gateway + RMQ + Redis stack is -required as part of the sign-off checklist (item 5 above) and is not yet done. diff --git a/server/docs/proxy-load-test-results.md b/server/docs/proxy-load-test-results.md deleted file mode 100644 index 9aba8d7..0000000 --- a/server/docs/proxy-load-test-results.md +++ /dev/null @@ -1,118 +0,0 @@ -# Aether Proxy — Routing-Layer Benchmark Results - -These numbers come from an in-process benchmark of the gateway's proxy and -tunnel routing primitives. See [Scope and Limitations](#scope-and-limitations) -before drawing production conclusions. - -## REST Proxy Throughput - -| Scenario | p50 | p99 | Notes | -|-------------------------------|-----|-----|-------------------------------------------------------| -| GET, small body (< 1 KB) | — | — | Not benchmarked separately; dominated by gRPC framing | -| POST, 256 KB body (chunked) | — | — | Not benchmarked separately | -| 100 concurrent proxy requests | — | — | Not benchmarked in this run | - -REST proxy benchmark numbers are not yet available from the in-process harness. -The integration tests in `server/tests/integration/` confirm correctness; a -throughput benchmark is pending. - -## Tunnel Routing - -| Scenario | p50 | p99 | Concurrency | -|-----------------------------------------------|-------|-------|----------------| -| Tunnel open (wildcard resolution + Redis pin) | 20 ms | 38 ms | 100 concurrent | - -These numbers reflect the routing layer only (wildcard resolution, Redis pin -write, ACL check). They do not include real gRPC stream overhead, real RabbitMQ -dispatch, or real Redis network latency. - -## Scope and Limitations - -**Limitation:** The benchmark used an in-process harness with no real network, -no real gRPC transport, no real RabbitMQ Streams, and no real Redis. All I/O -was simulated in-process. - -Measured values represent a lower bound on routing-layer overhead, not -end-to-end latency in a deployed system. Real-network numbers will be higher -depending on: - -- RTT between caller and gateway (typically 1–10 ms in same-AZ deployments) -- Redis round-trip time for lock scan and pin write (typically 1–5 ms) -- gRPC stream serialization overhead - -**Future work:** A wire-level load test against a deployed gateway + RabbitMQ -Streams + Redis stack is required before production sign-off. See the -[cutover checklist](proxy-cutover.md#6-sign-off-checklist), item 5. - -## Related Documents - -- [proxy.md](proxy.md) — feature overview, limits, audit events -- [proxy-quickstart.md](proxy-quickstart.md) — running the integration test suite -- [proxy-cutover.md](proxy-cutover.md) — production SLOs and rollout criteria - -## Wire-level results - -Numbers below are produced by the harness at `server/cmd/loadtest/proxy`, -driven by `server/scripts/run_proxy_load_test.sh`. Each run brings up its -own ephemeral RabbitMQ Streams + Valkey containers (on dedicated ports so -they don't clobber an operator-managed dev stack), starts the Aether -gateway in dev mode and two proxy-sidecar terminators (`sv::echo::a` and -`sv::echo::b`) over local TCP / HTTP echo backends, then drives 100 -concurrent caller agents through the Go SDK for 60 s of steady-state -traffic. Soft assertions (p99 ≤ 250 ms tunnel-open, p99 ≤ 500 ms REST, -goroutine-leak delta) print "WARN" but never fail the run — the harness -is observational by design. - -To run: - -``` -# One-shot run — starts/stops its own RMQ + Valkey: -bash server/scripts/run_proxy_load_test.sh - -# Build only, skip infra/run (CI sanity): -bash server/scripts/run_proxy_load_test.sh --check - -# Reuse an already-running RMQ/Valkey on the configured ports: -bash server/scripts/run_proxy_load_test.sh --no-infra-up - -# Override defaults via env vars: -CALLERS=200 DURATION=120s TARGET_MIBPS=2.0 \ - bash server/scripts/run_proxy_load_test.sh - -# Override test-only ports if 56552 / 56380 are in use: -RMQ_STREAM_PORT=57552 REDIS_PORT=57380 \ - bash server/scripts/run_proxy_load_test.sh -``` - -Each successful run appends a new "Wire-level results (run …)" section -below. The first row of every run records the host, Go toolchain, and any -soft-assertion warnings so future regressions are easy to spot. - -**Operator note (placeholder section).** First-run production-like -measurements have not yet been captured here. The harness builds cleanly -(`go build ./...` and `--check` mode both pass), boots the gateway plus -two sidecars plus its own RMQ + Valkey containers, and the gateway -authenticates and registers `sv::echo::a` and `sv::echo::b` as service -principals over the wire. - -During smoke validation on the build host the wire path showed an -end-to-end stall: agents connected and ProxyHTTP / TunnelDial calls -went out, but proxy responses did not arrive within the 5 s per-call -deadline (REST returned `context deadline exceeded`; tunnels opened in -microseconds but echo round-trips timed out). Sidecar logs show the -service principal reconnecting with a fresh consumer (`no stored -offset, starting from next message`), suggesting in-flight stream frames -are being skipped during sidecar reconnect churn against an -ephemerally-restarted RabbitMQ Streams broker. This is plausibly a -sidecar reconnect / RMQ stored-offset issue rather than a harness -defect — the SDK send queue is delivering, the gateway routing is -unchanged, and the auth-proxy regression suite still passes -(`go test ./internal/authproxy/...` → `ok`). - -When the operator runs the script on a deployed gateway with a warmed -RMQ instance — or after the sidecar reconnect tuning lands — the -harness will append a populated "Wire-level results (run …)" section -with measured numbers. Until then, treat the configuration block above -as the test plan and the in-process numbers earlier in this document as -the only validated latency baseline. - diff --git a/server/docs/proxy-quickstart.md b/server/docs/proxy-quickstart.md index a1b42b1..9d6f16e 100644 --- a/server/docs/proxy-quickstart.md +++ b/server/docs/proxy-quickstart.md @@ -90,7 +90,7 @@ ok github.com/scitrera/aether/tests/integration 2.014s - Cross-gateway dispatch with real RabbitMQ Streams replay — exercised by the unit tests in `server/internal/gateway/proxy_routing_test.go`; a wire-level load test against a deployed gateway + RMQ + Redis stack is - still pending (see [proxy-load-test-results.md](proxy-load-test-results.md)). + still pending. ## Running the auth-proxy regression suite @@ -117,5 +117,4 @@ For a full development-mode boot of the gateway + sidecar pair, see ## Related Documents - [proxy.md](proxy.md) — feature overview, SDK one-liners, ACL/OBO, limits, audit events, failure modes -- [proxy-cutover.md](proxy-cutover.md) — production rollout criteria, SLOs, rollback steps, and runbook -- [proxy-load-test-results.md](proxy-load-test-results.md) — routing-layer benchmark results and scope caveats +- [proxy-sandbox.md](proxy-sandbox.md) — sandbox deployment pattern (relay+terminator, UDS, grant scopes) diff --git a/server/docs/proxy-sandbox.md b/server/docs/proxy-sandbox.md index af49276..8575b09 100644 --- a/server/docs/proxy-sandbox.md +++ b/server/docs/proxy-sandbox.md @@ -6,9 +6,7 @@ environment where an untrusted process (a spawned LLM agent, a tool-runner container, etc.) can participate in the Aether network without holding any credentials. -**Related documents:** [proxy.md](proxy.md) · [proxy-cutover.md](proxy-cutover.md) · -[proxy-architecture-roadmap.md](proxy-architecture-roadmap.md) · -[proxy-quickstart.md](proxy-quickstart.md) +**Related documents:** [proxy.md](proxy.md) · [proxy-quickstart.md](proxy-quickstart.md) --- diff --git a/server/docs/proxy.md b/server/docs/proxy.md index 6a3a937..d70a60a 100644 --- a/server/docs/proxy.md +++ b/server/docs/proxy.md @@ -863,10 +863,6 @@ exposes per-envelope outcomes: | `full_buffer` | Target's deliveryCh was full; routed via RMQ to avoid stalling. | | `disabled` | Bypass turned off by config or env override. | -For the broader roadmap of routing-layer optimizations and the design -rationale behind the single-node bypass, see -[proxy-architecture-roadmap.md](proxy-architecture-roadmap.md). - ## Caller Identity Headers Two additional headers are stamped on every proxied request that reaches a @@ -920,7 +916,4 @@ UDS path convention, SDK config snippets, spawn-time grant scopes ## Related Documents - [proxy-quickstart.md](proxy-quickstart.md) — running the sidecar, integration tests, and auth-proxy regression suite -- [proxy-cutover.md](proxy-cutover.md) — production rollout criteria, SLOs, rollback steps, and runbook -- [proxy-load-test-results.md](proxy-load-test-results.md) — routing-layer benchmark results and scope caveats -- [proxy-architecture-roadmap.md](proxy-architecture-roadmap.md) — phased plan for proxy/tunnel routing-layer evolution - [proxy-sandbox.md](proxy-sandbox.md) — sandbox deployment pattern (relay+terminator, UDS, grant scopes) diff --git a/server/docs/workflow-engine.md b/server/docs/workflow-engine.md new file mode 100644 index 0000000..db1eed9 --- /dev/null +++ b/server/docs/workflow-engine.md @@ -0,0 +1,497 @@ +# Aether Workflow Engine (WFE) + +The Workflow Engine is Aether's event-triggered automation layer. It consumes a +shared event plane, matches events against declarative **rules**, and dispatches +**destinations** — sending a tool-call to an agent, spawning a POOL task, +emitting another event, or driving a **join** (fan-in / barrier / coalesce). It +also runs cron-style **schedules**, a **DAG engine**, and **state machines**. + +This document covers the full engine: the rule/event model, every destination +kind (with joins covered in depth), the scheduler, the admin API, configuration, +and the design decisions behind the join subsystem. + +--- + +## 1. Architecture + +The WFE is a **standalone binary** (`cmd/workflow`) that connects to the Aether +gateway as a `WorkflowEngineClient` (one of the eight principal types). It holds +no Redis connection of its own and owns no database driver: persistence is +injected as a `WorkflowStore` (Postgres or SQLite), and all distributed +coordination runs over the gateway's KV via the SDK. + +``` + ┌──────────────────────── Workflow Engine ────────────────────────┐ + event::receiver0 │ Router ──┬─► Executor ─► (message | create_task | emit_event) │ + (gateway fan-in) │ └─► JoinEngine ─► fan-in / barrier / coalesce │ + │ Scheduler ─► due schedules + due join deadlines (leader-gated) │ + │ DAGEngine · StateMachineEngine · AdminServer (REST :31881) │ + │ LeaderElector (coord lease over gateway KV) │ + └──────────────────────────────────────────────────────────────────┘ + │ WorkflowEngineClient (gRPC, single bidi stream) + ▼ + Aether Gateway ── KV (Redis | Badger | NATS JetStream) +``` + +**Components** (`internal/workflow/`): + +| Component | Responsibility | +|---|---| +| `Router` | Consumes the event plane, matches rules, renders destinations, routes to the Executor or JoinEngine. | +| `Executor` | Dispatches actions: tool-call `message`, POOL `create_task`, `emit_event`. | +| `JoinEngine` | Fan-in / barrier / coalesce joins (this document's focus). | +| `Scheduler` | Leader-gated poll loop: fires due schedules and sweeps due join deadlines. | +| `DAGEngine` | Multi-step workflow definitions with intra-definition step ordering. | +| `StateMachineEngine` | Event-driven state machines with entry/exit actions and timeouts. | +| `LeaderElector` | Elects one active replica (coord lease over gateway KV) so only one runs the scheduler/monitors. | +| `AdminServer` | REST API (default `:31881`) for CRUD + observability. | + +**Leadership.** Election runs on the shared coord primitive (atomic +`SetNX`/`CompareAndSet` lease) over a `coord.Locker` backed by the gateway KV, on +the global scope under the reserved key `_sys/coord/workflow/leader` (30s TTL, 10s +renew). One leader is elected across replicas in **every** backend mode; the +scheduler and monitors run only on the leader. + +**Mode portability.** Everything the engine does — leader election, the join +arrival counter, dedup ledger, fire-markers, set membership — is built on the +portable `KVReadWriter` primitives (`IncrementIf`, `SetNX`, `CompareAndSet`, +`CompareAndDelete`, `SetAdd`/`SetCard`), which are implemented natively in all +three backends: **Redis** (Lua), **Badger** (optimistic-CC retry), **NATS +JetStream** (revision-guarded CAS). The engine code is identical across +aether-full, aetherlite-single, and aetherlite-cluster. + +--- + +## 2. Events, rules, and the dispatch path + +### 2.1 The event plane + +Clients publish events with `SendEvent(payload)` → topic `event.*`, which the +gateway rewrites to a fan-in shard (`event::receiver0` today). The payload is a +JSON `EventPayload`: + +```json +{ "source_agent": "memorylayer", "workspace": "ws-1", + "event_names": ["memorylayer.ingest_complete"], "data": { "...": "..." } } +``` + +`source_agent` and `workspace` are back-filled from the topic segments when +omitted. The Router processes each name in `event_names` independently. + +### 2.2 Rules + +A `Rule` (persisted in `workflow_rules`) is: + +| Field | Meaning | +|---|---| +| `source_event` | Event name to match (exact). | +| `source_agent` | Match a specific agent or `*`. | +| `workspace` | Match a specific workspace or `*`. | +| `trigger_condition` | Optional **expr-lang** boolean over the arrival env. | +| `destination_template` | Go `text/template` → YAML describing the destination. | +| `priority`, `active` | Ordering (desc) and on/off. | + +`GetMatchingRules(agent, event, workspace)` selects `active = true AND +source_event = $event AND (source_agent = $agent OR '*') AND (workspace = +$workspace OR '*')` ordered by `priority DESC, id ASC`, cached with a TTL +(invalidated on any rule CRUD). + +### 2.3 Evaluation env + +For each matched rule the Router builds: + +``` +{ input: , + source: { agent: , workspace: , event: } } +``` + +- `trigger_condition` is evaluated by **expr-lang** (`github.com/expr-lang/expr`) + — not CEL. Empty ⇒ always matches. +- `destination_template` is rendered by Go `text/template` (with + `missingkey=zero`), then parsed as YAML into a `TransformResult`. + +> **One expression dialect.** Every expression field in the engine — trigger +> conditions and all join expressions (`correlation_key`, `expected_count`, +> `dedup_key`, `member_key`, `expected_set`) — uses **expr-lang**. Template +> *substitution* (`{{ .source.workspace }}`) is Go `text/template`, applied to +> the destination before YAML parsing. + +### 2.4 Destination kinds + +The rendered `type` field selects the destination: + +| `type` | Action | +|---|---| +| _empty_ / `message` | Tool-call to an agent (`agent`, `tool_name`, `arguments`). | +| `create_task` | Spawn a POOL task (`task_type`, `target_implementation`, `payload`, `metadata`, `retry`). | +| `emit_event` | Publish a synthetic event back onto `event.*` (`event_name`, `payload`) — enables chaining. | +| `join` | Fan-in / barrier / coalesce (see §3). | + +Existing `message`/`create_task` rules are unchanged by the join work; `join` +and `emit_event` are additive. + +**Event chaining** (the pre-existing multi-step idiom): task A completes → +worker emits an event → a rule spawns task B → … This handles *linear/sequential* +flows. Joins add the one thing chaining cannot express: **parallel fan-out +followed by a barrier.** + +--- + +## 3. Joins (fan-in / barrier / coalesce) + +A **join** answers: *"after the last of N correlated things finishes (or a +deadline elapses), do X — exactly once."* It is declared as a rule whose rendered +destination is `type: join`. + +### 3.1 The join destination schema + +```yaml +type: join +join: + name: ingest-decompose-barrier # logical join id (per workspace) + correlation_key: "input.job_id" # expr-lang → the barrier/group id (REQUIRED) + mode: count # count | coalesce | set + # --- count mode --- + expected_count: "input.expected_count" # expr-lang → integer (literal or off the arm event) + arm_on_event: memorylayer.ingest_complete # event that supplies expected_count (dynamic-N) + # --- set mode --- + expected_set: '["page:1","page:2","page:3"]' # expr-lang → list; completes at full membership + member_key: "input.memory_id" # expr-lang → this arrival's member id + # --- common --- + dedup_key: "input.memory_id" # expr-lang → per-arrival dedup id (count/coalesce) + window: "5s" # coalesce debounce/cooldown + timeout: "15m" # barrier deadline (drives the sweep) + linger: "1m" # post-terminal late-arrival retention + on_partial_failure: proceed_degraded # proceed | proceed_degraded | abort + on_complete: # action fired when the barrier completes + type: create_task + task_type: memorylayer-task.kb_update + target_implementation: memorylayer + workspace: "{{ .source.workspace }}" + payload: { workspace_id: "{{ .source.workspace }}" } + on_timeout: # action fired on deadline (falls back to on_complete) + type: create_task + task_type: memorylayer-task.kb_update + target_implementation: memorylayer + workspace: "{{ .source.workspace }}" + payload: { workspace_id: "{{ .source.workspace }}", degraded: true } +``` + +`on_complete` / `on_timeout` are ordinary actions (`create_task` or +`emit_event`). Their `text/template` fields are rendered when the enclosing +destination is transformed, so the engine sees concrete values. + +### 3.2 Fan-in matching — the correlation contract + +A join does **not** fire because two events arrived. It fires because two events +arrived that **resolve to the same correlation-key value.** The join instance — +and its KV keys — are keyed by `(workspace, join_name, correlation_key_value)`. +Different values ⇒ different instances ⇒ they never cross. + +``` +correlation_key: "input.job_id" + +decompose_complete {job_id:"A", memory_id:"m1"} → key "A" → counter …:A = 1 +decompose_complete {job_id:"A", memory_id:"m2"} → key "A" → counter …:A = 2 ✓ same origin +decompose_complete {job_id:"B", memory_id:"m9"} → key "B" → counter …:B = 1 ✗ separate barrier +``` + +So "same origin" is **entirely** determined by whether the producer stamps the +same correlation id on every member of a logical group. Two hard producer-side +requirements: + +1. **Every member event carries a shared correlation id.** No id ⇒ a degenerate + key that never completes (the engine rejects an empty correlation key). +2. **N (or the member set) is supplied** — a literal, an expression evaluable on + every arrival, or a manifest/arm event (dynamic-N, §3.4). + +The engine guarantees, given a correct id: per-origin isolation, single-firer +election, idempotent downstream, and a deadline backstop. + +### 3.3 Count mode + +Each **member** arrival atomically bumps a per-correlation counter +(`IncrementIf(+1, ceiling=2^40)` — a sentinel ceiling, so it always applies and +returns the arrival's sequence number) and mirrors `arrived_count` to the SQL +row. When the count reaches the known `expected_count`, a single firer is +elected and `on_complete` runs once. + +### 3.4 Dynamic-N arming + +When the count is not known at first arrival, a designated **arm event** +(`arm_on_event`) supplies it later. An arming arrival is **not** counted as a +member (arming typically rides a *different* event, e.g. a manifest); it records +`expected_count` and re-reads the counter, so an arm that lands *after* all +members still completes the barrier. (Register two rules pointing at the same +join `name`/`correlation_key`: one for the member event, one for the arm event.) + +### 3.5 Coalesce mode + +Collapses a burst into one firing — the server-side generalization of +MemoryLayer's hand-rolled `kb_update` lease. The first arrival acquires the +active/cooldown gate (TTL = `window`) and fires; arrivals while the gate is held +set the `dirty` flag (a trailing run is driven by the next post-cooldown arrival +or the deadline sweep). With `window` empty, each arrival fires (no debounce). +Use coalesce when the downstream **self-heals** (re-running corrects a slightly +stale result), e.g. KB regeneration. + +### 3.6 Set mode + +Completes when a known **set** of member ids has each reported at least once. Each +arrival adds `member_key` to a KV set via the atomic `SetAdd` primitive (the set +inherently dedups, so set mode needs no separate `dedup_key`); the join fires when +the set cardinality reaches `len(expected_set)`. The member whose add completes +the set is the unique firer. + +### 3.7 Dedup (count / coalesce) + +When `dedup_key` is set, the first arrival with a given dedup id wins a per-id +`TryAcquire` (`SetNX`) slot under the join base; repeats are dropped and counted +as late. Dedup id source: the event id for explicit events. (Set mode dedups via +set membership instead.) + +### 3.8 Deadline / timeout sweep and partial failure + +Each instance with a `timeout` carries a `deadline_at`. The **leader-gated +scheduler** sweeps `GetDueJoinDeadlines(now)` and calls `HandleDeadline` per open, +past-deadline instance: + +- `on_partial_failure: abort` → mark `timed_out`, fire nothing. +- `proceed` / `proceed_degraded` / _empty_ → fire the persisted `on_timeout` + action (falling back to `on_complete`), then mark `timed_out`. + +The actions are **persisted** on the row (`on_complete`/`on_timeout` JSON) so the +sweep can fire them with no live event. A never-completing barrier is GC'd at its +deadline rather than leaking. + +### 3.9 Exactly-once firing + +Two layers guarantee the downstream runs exactly once: + +1. **Single-firer election** — completeness is detected atomically, and a + `fire-marker` (`TryAcquire base+"/fired"`) ensures exactly one caller (a + completing arrival *or* the deadline sweep) runs the action. The marker also + prevents a deadline-fire and a late completion from both firing. +2. **Idempotent `create_task`** — the firer stamps a stable + `idempotency_key = join:{name}:{workspace}:{correlation_key}` (identical across + the `on_complete` and `on_timeout` paths). The gateway dedups creation on that + key via a `SetNX` ledger under `_sys/idem/task/…`: the first create proceeds and + records its task_id; duplicates (retries, engine restart, a timeout racing a + completion) create no second task. Fail-open on KV outage (a rare duplicate is + preferable to a dropped task). + +### 3.10 KV keys and storage + +The hot state lives in KV under the reserved infra namespace (granted to the WFE +via the gateway fast-path), keyed by a hash of `(name, workspace, correlation_key)`: + +``` +_sys/coord/joins/{hash} # count: arrival counter +_sys/coord/joins/{hash}/members # set: member set +_sys/coord/joins/{hash}/d/{hash} # dedup ledger (per dedup id) +_sys/coord/joins/{hash}/fired # fire-marker (exactly-once gate) +_sys/coord/joins/{hash}/active # coalesce cooldown gate +``` + +All carry a TTL of `timeout + linger` so abandoned instances self-GC even if the +sweep never runs. The durable mirror is the **`workflow_joins`** table (one row +per `(join_name, workspace, correlation_key)`): mode, expected/arrived counts, +dirty, status (`open|fired|timed_out|cancelled`), `deadline_at`, `linger_until`, +the persisted actions, and timestamps. The SQL row is the observability surface +and deadline driver; **KV is the firing arbiter**, the row a mirror. + +### 3.11 Observability and operator control + +| Op | Admin REST | Effect | +|---|---|---| +| `LIST_JOINS` | `GET /joins[?workspace=]` | List in-flight + recently-terminal instances. | +| `GET_JOIN` | `GET /joins/{name}/{corr}` | Inspect one (404 if absent). | +| `CANCEL_JOIN` | `POST /joins/{name}/{corr}/cancel` | Mark a wedged instance `cancelled`. | + +The `joinView` surface: `join_name, workspace, correlation_key, mode, arrived, +expected, dirty, status, deadline_at, linger_until, created_at, updated_at`. +Internal action JSON is not exposed. + +### 3.12 Worked example: ingest → decompose → kb_update + +The headline barrier (a *future* MemoryLayer architecture; today ingestion is +synchronous and `kb_update` is coalesced by a hand-rolled lease): + +- **Producer contract:** each async `decompose_facts` task emits + `memorylayer.decompose_complete` stamped with the shared `job_id`; the + finalize step emits `memorylayer.ingest_complete` carrying + `expected_count = total_memories_created`. +- **Rule M** (member): `decompose_complete → join` (count mode, + `correlation_key: input.job_id`, `dedup_key: input.memory_id`). +- **Rule A** (arm): `ingest_complete → join` (same `name`/`correlation_key`, + `arm_on_event: ingest_complete`, `expected_count: input.expected_count`). +- On completion (or timeout), one `kb_update` task is created — exactly once. + +**Simpler alternative (recommended until strict completeness is required):** skip +the count barrier and register `decompose_complete → join` in **coalesce** mode, +relying on `kb_update`'s self-healing. Use count mode only when the downstream +does **not** self-heal (batch-import summary, scatter/gather aggregation). + +### 3.13 Feed B — gathering over task completions + +A join can gather over **task completions** without the worker emitting its own +event. A task opts in via a `completion_event` config (persisted on the task); +when it reaches a selected terminal status the server emits a JSON `EventPayload` +onto `event::*`: + +```json +{ "source_agent": "orchestrator", "workspace": "ws-1", + "event_names": ["memorylayer.decompose_complete"], + "data": { "task_id": "...", "status": "completed", "task_type": "...", + "correlation_id": "job-42", "root_task_id": "...", "metadata": {…} } } +``` + +A **fan-out rule** tags the member tasks it spawns by setting `correlation_id` +and `completion_event` on its `create_task` destination: + +```yaml +type: create_task +task_type: decompose_facts +target_implementation: memorylayer +correlation_id: "{{ .input.job_id }}" +completion_event: + enabled: true + event_name: memorylayer.decompose_complete # empty ⇒ task.completed/failed/cancelled +``` + +The **join's member rule** then matches `memorylayer.decompose_complete` with +`correlation_key: input.correlation_id`. No worker code changes — the member task +just completes. + +- `completion_event.on_statuses` (proto) restricts emission to specific terminal + statuses; empty ⇒ all (`completed`, `failed`, `cancelled`). The emitted event + name defaults to `task.` when `event_name` is empty. +- Emission is best-effort and non-fatal: a publish failure never blocks the task + transition. +- `correlation_id` / `root_task_id` are persisted on the task (both stores) and + queryable via `TaskFilter`; `root_task_id` defaults to the task's own id when a + task has no supplied root (it is its own flow root). + +**Feed A vs feed B.** Feed A (the worker emits a domain event) gives full control +over the event name and payload. Feed B (the server emits on terminal status) +needs no worker code — useful for POOL tasks you don't own. Both land on +`event::*` as `EventPayload`s and are matched identically. + +--- + +## 4. Scheduler + +A leader-gated ticker (`scheduler.go`) polls on `GetSchedulerPollInterval()`: + +1. **Due schedules** — `GetDueSchedules(now)` → fire each (cron or interval), + update `last_fired_at`/`next_fire_at`. +2. **Due join deadlines** — `GetDueJoinDeadlines(now)` → `HandleDeadline` per + open, past-deadline join (§3.8). + +Both run only on the elected leader (`IsLeader()`), so deadline sweeps fire once +cluster-wide. + +--- + +## 5. DAG engine & state machines (existing) + +- **DAG engine** (`dag.go`): `WorkflowDefinition`s with steps and intra-definition + `depends_on` ordering, executed against `workflow_executions`/`step_states`, + concurrency-capped. Expresses *defined* multi-step workflows — not runtime fan-in + over an externally-produced set (that's what joins add). The join destination is + a strict subset a future full-DAG AND-join can absorb; its `correlation_id` / + `flow_id` design (§7) is forward-compatible as the eventual DAG run id. +- **State machines** (`statemachine.go`): definitions with states, transitions, + entry/exit actions, and timeouts; instances advance on events. + +--- + +## 6. Admin REST API + +Default port `:31881` (gorilla/mux, API-key auth). CRUD for `rules`, `workflows`, +`schedules`, `executions`, `state machines`, plus the join observability +endpoints (§3.11). The same operations are reachable as `WorkflowOperation`s over +the gateway (forwarded to the WFE). + +--- + +## 7. Design decisions & rationale + +- **Join destination first, full DAG later.** A declarative join reuses the + production rule path, the portable KV primitives, and the event plane with ~one + table and one action case. The DAG engine's `depends_on` is *intra-definition*; + it cannot natively join a runtime-discovered, externally-spawned set without + re-inventing the correlation/expected-count machinery joins define. Joins are the + right incremental primitive; the DAG AND-join is the eventual home. +- **KV is the firing arbiter; SQL is a mirror.** The atomic counter/set/markers in + KV give exactly-once firing selection portably; the SQL row provides durability, + observability, and the deadline driver. The split keeps the hot path atomic and + the cold path queryable. +- **Portable primitives, not Lua.** Firing rides on `IncrementIf`/`SetNX`/`SetAdd`, + each implemented natively in Redis/Badger/JetStream. No backend-specific firing + logic. (The Redis Lua scripts are one *instance* of a portable contract.) +- **Exactly-once is a platform guarantee, not handler glue.** Idempotent + `create_task` (gateway `SetNX` ledger) is why you'd use the WFE instead of + re-deriving a per-handler KV barrier — it removes the ~270-line, fail-open, + unobservable `kb_update.py`-style workaround. +- **Correlation id is distinct from task_id.** A barrier's identity must survive + task retries, can have no parent task (event-originated fan-out), and a task may + belong to several barriers. `correlation_id` + a propagated `flow_id` + (≈ `root_task_id`, threaded like the existing Authority Root/Parent lineage) is + the model; the join's `correlation_key` composes them per nesting level — which + is forward-compatible as the DAG run id. +- **Two feeds, one plane.** Joins gather over **explicit domain events** (feed A — + how MemoryLayer emits `ingest_complete`) **and** over **task completions** (feed + B — a task with an enabled `completion_event` emits a synthetic event onto + `event::*` at its terminal status, §3.13). Both arrive as JSON `EventPayload`s + on the same plane, so the Router and join matching treat them identically; no + per-task topic subscription is involved. + +--- + +## 8. Forward / deferred work + +Implemented in v1: feed B (§3.13), and `correlation_id`/`root_task_id` +persistence + `TaskFilter` predicates (§3.13). Still deferred: + +- **Automatic `flow_id` propagation on spawn.** Today a task is its own flow root + when no `root_task_id` is supplied, and a fan-out rule stamps `correlation_id` + explicitly; full reserved-namespace inheritance of `flow_id` across nested + spawns is future work. +- **Scatter destination & DAG AND-join** — a `scatter` destination that mints a + `flow_id` and creates N tasks in one step, and AND-join *nodes* keyed on + `(flow_id, upstream_node_set)`. The v1 `correlation_id`/`flow_id` becomes the + DAG run id with no rework. + +--- + +## 9. Proto reference (additive) + +- `KVOperation.OpType`: `SET_ADD = 12` (member = `value`, `ttl`; reply + `counter_value` = cardinality, `applied` = newly-added), `SET_CARD = 13`. +- `WorkflowOperation.OpType`: `LIST_JOINS = 24`, `GET_JOIN = 25`, + `CANCEL_JOIN = 26` (`id` = join_name, `secondary_id` = correlation_key). +- `CreateTaskRequest`: `idempotency_key = 16`, `correlation_id = 17`, + `root_task_id = 18`, `completion_event = 19`. `TaskInfo`: `correlation_id = 32`, + `root_task_id = 33`, `completion_event = 34`. `TaskFilter`: + `correlation_id = 23`, `root_task_id = 24`. +- `TaskCompletionEvent { bool enabled; string event_name; repeated TaskStatus + on_statuses }` — the feed-B opt-in carried on a task. + +All additive (new enum values / optional fields); existing wire semantics +unchanged. + +--- + +## 10. Source map + +- Rules / routing: `internal/workflow/router.go`, `expr.go`, `templates.go`. +- Actions: `internal/workflow/executor.go` (`message`/`create_task`/`emit_event`). +- Joins: `internal/workflow/join.go` (engine), `store.go` (`Join` + CRUD), + `scheduler.go` (deadline sweep), `workflow_handler.go` + `admin.go` (ops/REST). +- Migrations: `internal/workflow/migrations/004_workflow_joins.sql`, + `migrations/sqlite_workflow/002_workflow_joins.sql`. +- KV primitives: `internal/kv/{store,badger_store,jetstream_store}.go`, + `internal/gateway/interfaces.go` (`KVReadWriter`), `internal/gateway/kv_handler.go`. +- Idempotent create: `internal/gateway/orchestration_integration.go`. +- SDK: `sdk/go/aether/kv.go` (`SetAddSync`/`SetCardSync`), `coordkv.go`. diff --git a/server/go.mod b/server/go.mod index 76bcd3d..5ba8c1a 100644 --- a/server/go.mod +++ b/server/go.mod @@ -1,9 +1,10 @@ module github.com/scitrera/aether -go 1.25.10 +go 1.25.12 require ( github.com/MicahParks/keyfunc/v3 v3.8.0 + github.com/a2aproject/a2a-go/v2 v2.3.1 github.com/alicebob/miniredis/v2 v2.37.0 github.com/aws/aws-sdk-go-v2 v1.41.7 github.com/aws/aws-sdk-go-v2/config v1.32.17 @@ -29,17 +30,20 @@ require ( github.com/redis/go-redis/v9 v9.17.2 github.com/robfig/cron/v3 v3.0.1 github.com/rs/zerolog v1.34.0 - github.com/scitrera/aether/api v0.2.1 - github.com/scitrera/aether/sdk/go v0.2.1 - github.com/scitrera/go-backpressure v0.1.0 + github.com/scitrera/aether/api v0.2.2 + github.com/scitrera/aether/sdk/go v0.2.2 + github.com/scitrera/go-backpressure v0.1.1 github.com/vmihailenco/msgpack/v5 v5.4.1 go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.67.0 + go.opentelemetry.io/contrib/instrumentation/runtime v0.62.0 go.opentelemetry.io/otel v1.43.0 go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.19.0 - go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.42.0 + go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.43.0 + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.42.0 go.opentelemetry.io/otel/log v0.19.0 go.opentelemetry.io/otel/sdk v1.43.0 go.opentelemetry.io/otel/sdk/log v0.19.0 + go.opentelemetry.io/otel/sdk/metric v1.43.0 go.opentelemetry.io/otel/trace v1.43.0 golang.org/x/crypto v0.51.0 golang.org/x/oauth2 v0.36.0 @@ -114,8 +118,8 @@ require ( golang.org/x/net v0.54.0 // indirect golang.org/x/sys v0.44.0 // indirect golang.org/x/text v0.37.0 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20260401024825-9d38bb4040a9 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20260401024825-9d38bb4040a9 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20260427160629-7cedc36a6bc4 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260427160629-7cedc36a6bc4 // indirect modernc.org/libc v1.70.0 // indirect modernc.org/mathutil v1.7.1 // indirect modernc.org/memory v1.11.0 // indirect diff --git a/server/go.sum b/server/go.sum index f29f85d..cad5869 100644 --- a/server/go.sum +++ b/server/go.sum @@ -4,6 +4,8 @@ github.com/MicahParks/jwkset v0.11.0 h1:yc0zG+jCvZpWgFDFmvs8/8jqqVBG9oyIbmBtmjOh github.com/MicahParks/jwkset v0.11.0/go.mod h1:U2oRhRaLgDCLjtpGL2GseNKGmZtLs/3O7p+OZaL5vo0= github.com/MicahParks/keyfunc/v3 v3.8.0 h1:Hx2dgIjAXGk9slakM6rV9BOeaWDPEXXZ4Us8guNBfds= github.com/MicahParks/keyfunc/v3 v3.8.0/go.mod h1:z66bkCviwqfg2YUp+Jcc/xRE9IXLcMq6DrgV/+Htru0= +github.com/a2aproject/a2a-go/v2 v2.3.1 h1:QWMdOX2UsJ8BJmjs952eo1FRyGsOVl0gFCKeM76AgGE= +github.com/a2aproject/a2a-go/v2 v2.3.1/go.mod h1:mkZr8y2bUgAVQsjs/5fHK7xrRlAHDybMEyxWh2tKRC8= github.com/alicebob/miniredis/v2 v2.37.0 h1:RheObYW32G1aiJIj81XVt78ZHJpHonHLHW7OLIshq68= github.com/alicebob/miniredis/v2 v2.37.0/go.mod h1:TcL7YfarKPGDAthEtl5NBeHZfeUQj6OXMm/+iu5cLMM= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= @@ -182,8 +184,8 @@ github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7 github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= github.com/rs/zerolog v1.34.0 h1:k43nTLIwcTVQAncfCw4KZ2VY6ukYoZaBPNOE8txlOeY= github.com/rs/zerolog v1.34.0/go.mod h1:bJsvje4Z08ROH4Nhs5iH600c3IkWhwp44iRc54W6wYQ= -github.com/scitrera/go-backpressure v0.1.0 h1:I55SB+PUqLB92FYueKgLRLwECsYGJje6K6NaAeSdAxM= -github.com/scitrera/go-backpressure v0.1.0/go.mod h1:2bK7pLzy0LE3BPQ029FflI06VvjHQNAevkS4pTUGYmQ= +github.com/scitrera/go-backpressure v0.1.1 h1:MhLWsg0qAVjF43/yQ6JHlrvT6sWpRIIPSu/58XjMbo0= +github.com/scitrera/go-backpressure v0.1.1/go.mod h1:2bK7pLzy0LE3BPQ029FflI06VvjHQNAevkS4pTUGYmQ= github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= @@ -198,14 +200,18 @@ go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.67.0 h1:yI1/OhfEPy7J9eoa6Sj051C7n5dvpj0QX8g4sRchg04= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.67.0/go.mod h1:NoUCKYWK+3ecatC4HjkRktREheMeEtrXoQxrqYFeHSc= +go.opentelemetry.io/contrib/instrumentation/runtime v0.62.0 h1:ZIt0ya9/y4WyRIzfLC8hQRRsWg0J9M9GyaGtIMiElZI= +go.opentelemetry.io/contrib/instrumentation/runtime v0.62.0/go.mod h1:F1aJ9VuiKWOlWwKdTYDUp1aoS0HzQxg38/VLxKmhm5U= go.opentelemetry.io/otel v1.43.0 h1:mYIM03dnh5zfN7HautFE4ieIig9amkNANT+xcVxAj9I= go.opentelemetry.io/otel v1.43.0/go.mod h1:JuG+u74mvjvcm8vj8pI5XiHy1zDeoCS2LB1spIq7Ay0= go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.19.0 h1:HIBTQ3VO5aupLKjC90JgMqpezVXwFuq6Ryjn0/izoag= go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.19.0/go.mod h1:ji9vId85hMxqfvICA0Jt8JqEdrXaAkcpkI9HPXya0ro= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.43.0 h1:8UQVDcZxOJLtX6gxtDt3vY2WTgvZqMQRzjsqiIHQdkc= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.43.0/go.mod h1:2lmweYCiHYpEjQ/lSJBYhj9jP1zvCvQW4BqL9dnT7FQ= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.42.0 h1:THuZiwpQZuHPul65w4WcwEnkX2QIuMT+UFoOrygtoJw= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.42.0/go.mod h1:J2pvYM5NGHofZ2/Ru6zw/TNWnEQp5crgyDeSrYpXkAw= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.42.0 h1:uLXP+3mghfMf7XmV4PkGfFhFKuNWoCvvx5wP/wOXo0o= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.42.0/go.mod h1:v0Tj04armyT59mnURNUJf7RCKcKzq+lgJs6QSjHjaTc= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.42.0 h1:zWWrB1U6nqhS/k6zYB74CjRpuiitRtLLi68VcgmOEto= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.42.0/go.mod h1:2qXPNBX1OVRC0IwOnfo1ljoid+RD0QK3443EaqVlsOU= go.opentelemetry.io/otel/log v0.19.0 h1:KUZs/GOsw79TBBMfDWsXS+KZ4g2Ckzksd1ymzsIEbo4= go.opentelemetry.io/otel/log v0.19.0/go.mod h1:5DQYeGmxVIr4n0/BcJvF4upsraHjg6vudJJpnkL6Ipk= go.opentelemetry.io/otel/metric v1.43.0 h1:d7638QeInOnuwOONPp4JAOGfbCEpYb+K6DVWvdxGzgM= @@ -256,10 +262,10 @@ golang.org/x/tools v0.44.0 h1:UP4ajHPIcuMjT1GqzDWRlalUEoY+uzoZKnhOjbIPD2c= golang.org/x/tools v0.44.0/go.mod h1:KA0AfVErSdxRZIsOVipbv3rQhVXTnlU6UhKxHd1seDI= gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= -google.golang.org/genproto/googleapis/api v0.0.0-20260401024825-9d38bb4040a9 h1:VPWxll4HlMw1Vs/qXtN7BvhZqsS9cdAittCNvVENElA= -google.golang.org/genproto/googleapis/api v0.0.0-20260401024825-9d38bb4040a9/go.mod h1:7QBABkRtR8z+TEnmXTqIqwJLlzrZKVfAUm7tY3yGv0M= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260401024825-9d38bb4040a9 h1:m8qni9SQFH0tJc1X0vmnpw/0t+AImlSvp30sEupozUg= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260401024825-9d38bb4040a9/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= +google.golang.org/genproto/googleapis/api v0.0.0-20260427160629-7cedc36a6bc4 h1:yOzSCGPx+cp5VO7IxvZ9SBFF7j1tZVcNtlHR2iYKtVo= +google.golang.org/genproto/googleapis/api v0.0.0-20260427160629-7cedc36a6bc4/go.mod h1:Q9HWtNeE7tM9npdIsEvqXj1QJIvVoeAV3rtXtS715Cw= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260427160629-7cedc36a6bc4 h1:tEkOQcXgF6dH1G+MVKZrfpYvozGrzb91k6ha7jireSM= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260427160629-7cedc36a6bc4/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= google.golang.org/grpc v1.81.1 h1:VnnIIZ88UzOOKLukQi+ImGz8O1Wdp8nAGGnvOfEIWQQ= google.golang.org/grpc v1.81.1/go.mod h1:xGH9GfzOyMTGIOXBJmXt+BX/V0kcdQbdcuwQ/zNw42I= google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= diff --git a/server/internal/a2abridge/envelope.go b/server/internal/a2abridge/envelope.go new file mode 100644 index 0000000..0313276 --- /dev/null +++ b/server/internal/a2abridge/envelope.go @@ -0,0 +1,185 @@ +package a2abridge + +import ( + "encoding/json" + "fmt" + + "github.com/a2aproject/a2a-go/v2/a2a" +) + +// EnvelopeVersion is the current wire-format version. Receivers must accept +// envelopes whose version matches; envelopes with a different version are +// rejected at Unmarshal so the bridge can roll forward without silently +// double-interpreting old payloads. +const EnvelopeVersion = "1" + +// EnvelopeKind discriminates the body kind of an Envelope. +type EnvelopeKind string + +const ( + // KindRequest carries an A2A SendMessageRequest from the bridge to the + // aether agent that will execute it. + KindRequest EnvelopeKind = "request" + + // KindEvent carries a single A2A streaming event (Message, Task, + // TaskStatusUpdateEvent, or TaskArtifactUpdateEvent) in the reverse + // direction. + KindEvent EnvelopeKind = "event" + + // KindError carries a bridge-level error that prevented translation or + // delivery (auth failure, unknown agent, malformed payload, etc.). + // Protocol-level failures (a2a TaskStateFailed) ride inside an Event + // envelope, not here. + KindError EnvelopeKind = "error" +) + +// Envelope wraps an A2A request or event for transit through an aether +// Message.payload. The kind discriminator tells the receiver which inner +// field is populated. +// +// The envelope is JSON because it has to traverse aether's opaque payload +// bytes and the a2a-go types already serialise as JSON. Using the upstream +// SDK's own MarshalJSON for Event (via a2a.StreamResponse) keeps the wire +// format aligned with what an A2A client would observe on the network. +type Envelope struct { + // Version pins the envelope schema. Always EnvelopeVersion for newly + // produced envelopes. + Version string `json:"v"` + + // Kind selects which inner field is populated. + Kind EnvelopeKind `json:"kind"` + + // ReqID correlates a request with the stream of events it produces. The + // gateway mints it when sending a request and stamps every replied event + // with the same value so the sender can demux concurrent in-flight + // requests on a single peer connection. + ReqID string `json:"reqId,omitempty"` + + // Request is populated for KindRequest. + Request *a2a.SendMessageRequest `json:"request,omitempty"` + + // Event is populated for KindEvent. a2a.StreamResponse's MarshalJSON + // emits one of {"message":..., "task":..., "statusUpdate":..., + // "artifactUpdate":...} so the wire shape mirrors the A2A streaming + // transport. + Event *a2a.StreamResponse `json:"event,omitempty"` + + // Terminal signals that no further events for this ReqID will arrive + // from this peer. The bridge sets it on the last KindEvent (or alongside + // a KindError) so the receiver can release any per-request state without + // waiting for an a2a-protocol terminal task state. + Terminal bool `json:"terminal,omitempty"` + + // Error is populated for KindError and describes a transport- or + // translation-level failure. + Error *EnvelopeError `json:"error,omitempty"` +} + +// EnvelopeError describes a bridge-level failure. +type EnvelopeError struct { + // Code is a stable machine-readable identifier (e.g. "unknown_agent", + // "translation_failed", "internal"). + Code string `json:"code"` + + // Message is a human-readable description; safe to surface to clients. + Message string `json:"message"` +} + +// Error implements the error interface so callers can return *EnvelopeError +// directly from translation paths. +func (e *EnvelopeError) Error() string { + if e == nil { + return "" + } + return fmt.Sprintf("a2abridge: %s: %s", e.Code, e.Message) +} + +// MarshalRequest produces a request envelope payload. reqID must be non-empty +// so the receiver can correlate replies; req must be non-nil. +func MarshalRequest(reqID string, req *a2a.SendMessageRequest) ([]byte, error) { + if reqID == "" { + return nil, fmt.Errorf("a2abridge: MarshalRequest: empty reqID") + } + if req == nil { + return nil, fmt.Errorf("a2abridge: MarshalRequest: nil request") + } + env := Envelope{ + Version: EnvelopeVersion, + Kind: KindRequest, + ReqID: reqID, + Request: req, + } + return json.Marshal(env) +} + +// MarshalEvent produces an event envelope payload. evt must satisfy +// a2a.Event. terminal indicates whether this is the last event the sender +// will emit for reqID. +func MarshalEvent(reqID string, evt a2a.Event, terminal bool) ([]byte, error) { + if reqID == "" { + return nil, fmt.Errorf("a2abridge: MarshalEvent: empty reqID") + } + if evt == nil { + return nil, fmt.Errorf("a2abridge: MarshalEvent: nil event") + } + env := Envelope{ + Version: EnvelopeVersion, + Kind: KindEvent, + ReqID: reqID, + Event: &a2a.StreamResponse{Event: evt}, + Terminal: terminal, + } + return json.Marshal(env) +} + +// MarshalError produces an error envelope payload. The error is always +// terminal: there is no recovery for the request beyond returning the error. +func MarshalError(reqID string, code, message string) ([]byte, error) { + if reqID == "" { + return nil, fmt.Errorf("a2abridge: MarshalError: empty reqID") + } + if code == "" { + return nil, fmt.Errorf("a2abridge: MarshalError: empty code") + } + env := Envelope{ + Version: EnvelopeVersion, + Kind: KindError, + ReqID: reqID, + Terminal: true, + Error: &EnvelopeError{Code: code, Message: message}, + } + return json.Marshal(env) +} + +// Unmarshal decodes any envelope payload. It validates the version and the +// presence of the inner field corresponding to Kind. Callers should dispatch +// on Envelope.Kind. +func Unmarshal(data []byte) (*Envelope, error) { + if len(data) == 0 { + return nil, fmt.Errorf("a2abridge: Unmarshal: empty payload") + } + var env Envelope + if err := json.Unmarshal(data, &env); err != nil { + return nil, fmt.Errorf("a2abridge: Unmarshal: %w", err) + } + if env.Version != EnvelopeVersion { + return nil, fmt.Errorf("a2abridge: unsupported envelope version %q (want %q)", env.Version, EnvelopeVersion) + } + switch env.Kind { + case KindRequest: + if env.Request == nil { + return nil, fmt.Errorf("a2abridge: request envelope missing request body") + } + case KindEvent: + if env.Event == nil || env.Event.Event == nil { + return nil, fmt.Errorf("a2abridge: event envelope missing event body") + } + case KindError: + if env.Error == nil { + return nil, fmt.Errorf("a2abridge: error envelope missing error body") + } + default: + return nil, fmt.Errorf("a2abridge: unknown envelope kind %q", env.Kind) + } + return &env, nil +} diff --git a/server/internal/a2abridge/envelope_test.go b/server/internal/a2abridge/envelope_test.go new file mode 100644 index 0000000..8c83e7c --- /dev/null +++ b/server/internal/a2abridge/envelope_test.go @@ -0,0 +1,210 @@ +package a2abridge + +import ( + "errors" + "strings" + "testing" + + "github.com/a2aproject/a2a-go/v2/a2a" +) + +func TestEnvelope_RequestRoundTrip(t *testing.T) { + req := &a2a.SendMessageRequest{ + Tenant: "a2a-prod", + Message: a2a.NewMessage( + a2a.MessageRoleUser, + a2a.NewTextPart("hello"), + a2a.NewDataPart(map[string]any{"k": "v"}), + ), + Metadata: map[string]any{"trace_id": "abc"}, + } + data, err := MarshalRequest("req-1", req) + if err != nil { + t.Fatalf("MarshalRequest: %v", err) + } + + env, err := Unmarshal(data) + if err != nil { + t.Fatalf("Unmarshal: %v", err) + } + if env.Kind != KindRequest { + t.Errorf("Kind: want %q, got %q", KindRequest, env.Kind) + } + if env.ReqID != "req-1" { + t.Errorf("ReqID: want req-1, got %q", env.ReqID) + } + if env.Request == nil || env.Request.Message == nil { + t.Fatalf("Request not populated") + } + if env.Request.Tenant != "a2a-prod" { + t.Errorf("Tenant: want a2a-prod, got %q", env.Request.Tenant) + } + if len(env.Request.Message.Parts) != 2 { + t.Fatalf("Parts: want 2, got %d", len(env.Request.Message.Parts)) + } + if env.Request.Message.Parts[0].Text() != "hello" { + t.Errorf("text part: want hello, got %q", env.Request.Message.Parts[0].Text()) + } +} + +func TestEnvelope_EventRoundTrip_Message(t *testing.T) { + msg := a2a.NewMessage(a2a.MessageRoleAgent, a2a.NewTextPart("hi back")) + data, err := MarshalEvent("req-2", msg, true) + if err != nil { + t.Fatalf("MarshalEvent: %v", err) + } + env, err := Unmarshal(data) + if err != nil { + t.Fatalf("Unmarshal: %v", err) + } + if env.Kind != KindEvent { + t.Fatalf("Kind: want %q, got %q", KindEvent, env.Kind) + } + if !env.Terminal { + t.Errorf("Terminal: want true") + } + got, ok := env.Event.Event.(*a2a.Message) + if !ok { + t.Fatalf("Event: want *a2a.Message, got %T", env.Event.Event) + } + if got.Parts[0].Text() != "hi back" { + t.Errorf("text: want hi back, got %q", got.Parts[0].Text()) + } +} + +func TestEnvelope_EventRoundTrip_Task(t *testing.T) { + task := a2a.NewSubmittedTask( + a2a.TaskInfo{TaskID: "t-1", ContextID: "ctx-1"}, + a2a.NewMessage(a2a.MessageRoleUser, a2a.NewTextPart("start")), + ) + data, err := MarshalEvent("req-3", task, false) + if err != nil { + t.Fatalf("MarshalEvent: %v", err) + } + env, err := Unmarshal(data) + if err != nil { + t.Fatalf("Unmarshal: %v", err) + } + got, ok := env.Event.Event.(*a2a.Task) + if !ok { + t.Fatalf("want *a2a.Task, got %T", env.Event.Event) + } + if got.ID != "t-1" || got.ContextID != "ctx-1" { + t.Errorf("task ids: got id=%q ctx=%q", got.ID, got.ContextID) + } + if got.Status.State != a2a.TaskStateSubmitted { + t.Errorf("state: want submitted, got %q", got.Status.State) + } +} + +func TestEnvelope_EventRoundTrip_StatusUpdate(t *testing.T) { + evt := a2a.NewStatusUpdateEvent( + a2a.TaskInfo{TaskID: "t-2", ContextID: "ctx-2"}, + a2a.TaskStateWorking, + nil, + ) + data, err := MarshalEvent("req-4", evt, false) + if err != nil { + t.Fatalf("MarshalEvent: %v", err) + } + env, err := Unmarshal(data) + if err != nil { + t.Fatalf("Unmarshal: %v", err) + } + got, ok := env.Event.Event.(*a2a.TaskStatusUpdateEvent) + if !ok { + t.Fatalf("want *a2a.TaskStatusUpdateEvent, got %T", env.Event.Event) + } + if got.TaskID != "t-2" || got.Status.State != a2a.TaskStateWorking { + t.Errorf("statusUpdate fields off: %+v", got) + } +} + +func TestEnvelope_EventRoundTrip_ArtifactUpdate(t *testing.T) { + evt := a2a.NewArtifactEvent( + a2a.TaskInfo{TaskID: "t-3", ContextID: "ctx-3"}, + a2a.NewTextPart("artifact-bytes"), + ) + data, err := MarshalEvent("req-5", evt, false) + if err != nil { + t.Fatalf("MarshalEvent: %v", err) + } + env, err := Unmarshal(data) + if err != nil { + t.Fatalf("Unmarshal: %v", err) + } + got, ok := env.Event.Event.(*a2a.TaskArtifactUpdateEvent) + if !ok { + t.Fatalf("want *a2a.TaskArtifactUpdateEvent, got %T", env.Event.Event) + } + if got.TaskID != "t-3" { + t.Errorf("artifactUpdate TaskID: got %q", got.TaskID) + } + if got.Artifact == nil || len(got.Artifact.Parts) != 1 { + t.Fatalf("artifact parts not preserved") + } +} + +func TestEnvelope_Error(t *testing.T) { + data, err := MarshalError("req-6", "unknown_agent", "no agent ag::test::echo::1") + if err != nil { + t.Fatalf("MarshalError: %v", err) + } + env, err := Unmarshal(data) + if err != nil { + t.Fatalf("Unmarshal: %v", err) + } + if env.Kind != KindError || env.Error == nil { + t.Fatalf("error envelope not populated: %+v", env) + } + if !env.Terminal { + t.Errorf("error envelopes must be terminal") + } + if !strings.Contains(env.Error.Error(), "unknown_agent") { + t.Errorf("Error(): %q", env.Error.Error()) + } +} + +func TestEnvelope_Validation(t *testing.T) { + if _, err := MarshalRequest("", &a2a.SendMessageRequest{}); err == nil { + t.Errorf("MarshalRequest empty reqID: expected error") + } + if _, err := MarshalRequest("r", nil); err == nil { + t.Errorf("MarshalRequest nil req: expected error") + } + if _, err := MarshalEvent("", a2a.NewMessage(a2a.MessageRoleAgent), false); err == nil { + t.Errorf("MarshalEvent empty reqID: expected error") + } + if _, err := MarshalEvent("r", nil, false); err == nil { + t.Errorf("MarshalEvent nil event: expected error") + } + if _, err := MarshalError("r", "", "m"); err == nil { + t.Errorf("MarshalError empty code: expected error") + } + + if _, err := Unmarshal(nil); err == nil { + t.Errorf("Unmarshal empty: expected error") + } + if _, err := Unmarshal([]byte(`{"v":"99","kind":"event"}`)); err == nil { + t.Errorf("Unmarshal wrong version: expected error") + } + if _, err := Unmarshal([]byte(`{"v":"1","kind":"event"}`)); err == nil { + t.Errorf("Unmarshal event without body: expected error") + } + if _, err := Unmarshal([]byte(`{"v":"1","kind":"request"}`)); err == nil { + t.Errorf("Unmarshal request without body: expected error") + } + if _, err := Unmarshal([]byte(`{"v":"1","kind":"weird"}`)); err == nil { + t.Errorf("Unmarshal unknown kind: expected error") + } +} + +func TestEnvelopeError_NilSafe(t *testing.T) { + var e *EnvelopeError + // Should not panic + _ = e.Error() + var asErr error = (*EnvelopeError)(nil) + if errors.Is(asErr, ErrUnknownTenant) { + t.Errorf("nil EnvelopeError should not match ErrUnknownTenant") + } +} diff --git a/server/internal/a2abridge/workspace.go b/server/internal/a2abridge/workspace.go new file mode 100644 index 0000000..a568220 --- /dev/null +++ b/server/internal/a2abridge/workspace.go @@ -0,0 +1,123 @@ +// Package a2abridge contains the protocol-translation primitives shared by +// the a2a-gateway and a2a-sidecar binaries. It maps A2A protocol concepts +// (Message, Task, AgentCard, TaskState) onto aether's stateless message +// transport and persistent task store, and provides the workspace and +// authentication glue both processes need. +package a2abridge + +import ( + "fmt" + "strings" +) + +// UnknownTenantBehavior controls what a WorkspaceResolver does when an inbound +// A2A request carries a tenant identifier that is not present in the static +// mapping. +type UnknownTenantBehavior string + +const ( + // UnknownTenantReject causes the resolver to return ErrUnknownTenant for + // any tenant outside the mapping. Use in environments where every A2A + // caller MUST be explicitly enrolled. + UnknownTenantReject UnknownTenantBehavior = "reject" + + // UnknownTenantUseDefault causes the resolver to fall back to the + // configured default workspace for any unmapped tenant. Use for trust- + // boundary setups where unknown tenants land in a sandbox workspace. + UnknownTenantUseDefault UnknownTenantBehavior = "use_default" +) + +// ErrUnknownTenant is returned by WorkspaceResolver.Resolve when the supplied +// tenant is not in the static map and the configured behavior is +// UnknownTenantReject. +var ErrUnknownTenant = fmt.Errorf("a2abridge: unknown tenant") + +// WorkspaceConfig is the operator-facing configuration shape parsed from YAML. +// Field names use snake_case in YAML; see configs/a2a-gateway.example.yaml. +type WorkspaceConfig struct { + // DefaultWorkspace is the aether workspace used when an inbound A2A + // request omits a tenant entirely, and (when UnknownBehavior is + // UnknownTenantUseDefault) when the tenant is not mapped. + DefaultWorkspace string `yaml:"default_workspace"` + + // Tenants maps A2A tenant identifiers to aether workspace names. + Tenants map[string]string `yaml:"tenants"` + + // UnknownBehavior controls the fallback for unmapped tenants. Defaults + // to UnknownTenantReject when empty. + UnknownBehavior UnknownTenantBehavior `yaml:"unknown_tenant_behavior"` +} + +// WorkspaceResolver translates A2A tenant strings into aether workspace +// strings. It is safe for concurrent use; the underlying map is read-only +// after construction. +type WorkspaceResolver struct { + defaultWorkspace string + tenants map[string]string + unknown UnknownTenantBehavior +} + +// NewWorkspaceResolver validates the config and returns a resolver. It +// rejects configurations that cannot satisfy any request (e.g. +// use_default with no default workspace, or an empty mapping with reject +// behavior — which would refuse every request). +func NewWorkspaceResolver(cfg WorkspaceConfig) (*WorkspaceResolver, error) { + behavior := cfg.UnknownBehavior + if behavior == "" { + behavior = UnknownTenantReject + } + if behavior != UnknownTenantReject && behavior != UnknownTenantUseDefault { + return nil, fmt.Errorf("a2abridge: invalid unknown_tenant_behavior %q", behavior) + } + if behavior == UnknownTenantUseDefault && cfg.DefaultWorkspace == "" { + return nil, fmt.Errorf("a2abridge: unknown_tenant_behavior=use_default requires default_workspace") + } + if behavior == UnknownTenantReject && cfg.DefaultWorkspace == "" && len(cfg.Tenants) == 0 { + return nil, fmt.Errorf("a2abridge: workspace config is empty (no default and no tenants)") + } + + tenants := make(map[string]string, len(cfg.Tenants)) + for k, v := range cfg.Tenants { + k = strings.TrimSpace(k) + v = strings.TrimSpace(v) + if k == "" { + return nil, fmt.Errorf("a2abridge: tenant map contains empty key") + } + if v == "" { + return nil, fmt.Errorf("a2abridge: tenant map for %q has empty workspace", k) + } + tenants[k] = v + } + + return &WorkspaceResolver{ + defaultWorkspace: strings.TrimSpace(cfg.DefaultWorkspace), + tenants: tenants, + unknown: behavior, + }, nil +} + +// Resolve maps an A2A tenant identifier to an aether workspace. A blank +// tenant always resolves to the default workspace (returning ErrUnknownTenant +// only if no default was configured). A non-blank tenant looks up the map, +// then falls back according to the configured UnknownTenantBehavior. +func (r *WorkspaceResolver) Resolve(tenant string) (string, error) { + tenant = strings.TrimSpace(tenant) + if tenant == "" { + if r.defaultWorkspace == "" { + return "", fmt.Errorf("%w: empty tenant and no default workspace", ErrUnknownTenant) + } + return r.defaultWorkspace, nil + } + if ws, ok := r.tenants[tenant]; ok { + return ws, nil + } + if r.unknown == UnknownTenantUseDefault { + return r.defaultWorkspace, nil + } + return "", fmt.Errorf("%w: %q", ErrUnknownTenant, tenant) +} + +// DefaultWorkspace returns the configured default workspace, or "" if none. +func (r *WorkspaceResolver) DefaultWorkspace() string { + return r.defaultWorkspace +} diff --git a/server/internal/a2abridge/workspace_test.go b/server/internal/a2abridge/workspace_test.go new file mode 100644 index 0000000..ab62008 --- /dev/null +++ b/server/internal/a2abridge/workspace_test.go @@ -0,0 +1,117 @@ +package a2abridge + +import ( + "errors" + "testing" +) + +func TestWorkspaceResolver_Reject(t *testing.T) { + r, err := NewWorkspaceResolver(WorkspaceConfig{ + DefaultWorkspace: "aether-default", + Tenants: map[string]string{ + "a2a-prod": "aether-prod", + "a2a-dev": "aether-dev", + }, + UnknownBehavior: UnknownTenantReject, + }) + if err != nil { + t.Fatalf("NewWorkspaceResolver: %v", err) + } + + tests := []struct { + tenant string + want string + errIs error + }{ + {"a2a-prod", "aether-prod", nil}, + {"a2a-dev", "aether-dev", nil}, + {"", "aether-default", nil}, + {"a2a-unknown", "", ErrUnknownTenant}, + } + for _, tc := range tests { + got, err := r.Resolve(tc.tenant) + if tc.errIs != nil { + if !errors.Is(err, tc.errIs) { + t.Errorf("Resolve(%q): want err %v, got %v", tc.tenant, tc.errIs, err) + } + continue + } + if err != nil { + t.Errorf("Resolve(%q): unexpected error %v", tc.tenant, err) + continue + } + if got != tc.want { + t.Errorf("Resolve(%q): want %q, got %q", tc.tenant, tc.want, got) + } + } +} + +func TestWorkspaceResolver_UseDefault(t *testing.T) { + r, err := NewWorkspaceResolver(WorkspaceConfig{ + DefaultWorkspace: "sandbox", + Tenants: map[string]string{"a2a-prod": "aether-prod"}, + UnknownBehavior: UnknownTenantUseDefault, + }) + if err != nil { + t.Fatalf("NewWorkspaceResolver: %v", err) + } + + got, err := r.Resolve("a2a-prod") + if err != nil || got != "aether-prod" { + t.Errorf("Resolve mapped tenant: want (aether-prod, nil), got (%q, %v)", got, err) + } + got, err = r.Resolve("a2a-mystery") + if err != nil || got != "sandbox" { + t.Errorf("Resolve unmapped tenant: want (sandbox, nil), got (%q, %v)", got, err) + } + got, err = r.Resolve("") + if err != nil || got != "sandbox" { + t.Errorf("Resolve empty tenant: want (sandbox, nil), got (%q, %v)", got, err) + } +} + +func TestWorkspaceResolver_DefaultsBehavior(t *testing.T) { + // Empty behavior defaults to reject. + r, err := NewWorkspaceResolver(WorkspaceConfig{ + DefaultWorkspace: "default", + }) + if err != nil { + t.Fatalf("NewWorkspaceResolver: %v", err) + } + if _, err := r.Resolve("missing"); !errors.Is(err, ErrUnknownTenant) { + t.Errorf("default behavior should reject unmapped tenants; got %v", err) + } +} + +func TestWorkspaceResolver_InvalidConfig(t *testing.T) { + cases := []struct { + name string + cfg WorkspaceConfig + }{ + {"use_default without default workspace", WorkspaceConfig{ + UnknownBehavior: UnknownTenantUseDefault, + }}, + {"reject with empty mapping and no default", WorkspaceConfig{ + UnknownBehavior: UnknownTenantReject, + }}, + {"invalid behavior", WorkspaceConfig{ + DefaultWorkspace: "x", + UnknownBehavior: "ignore", + }}, + {"empty tenant key", WorkspaceConfig{ + DefaultWorkspace: "x", + Tenants: map[string]string{"": "ws"}, + }}, + {"empty workspace value", WorkspaceConfig{ + DefaultWorkspace: "x", + Tenants: map[string]string{"t": ""}, + }}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if _, err := NewWorkspaceResolver(tc.cfg); err == nil { + t.Errorf("expected error for %s, got nil", tc.name) + } + }) + } +} diff --git a/server/internal/acl/adapter.go b/server/internal/acl/adapter.go index 1af164a..8a3ddfc 100644 --- a/server/internal/acl/adapter.go +++ b/server/internal/acl/adapter.go @@ -8,6 +8,7 @@ import ( "github.com/casbin/casbin/v3/model" "github.com/casbin/casbin/v3/persist" + "github.com/scitrera/aether/internal/logging" ) // aclRulesAdapter implements the Casbin persist.Adapter interface by reading @@ -74,7 +75,65 @@ func (a *aclRulesAdapter) LoadPolicy(m model.Model) error { _ = persist.LoadPolicyArray([]string{"p", sub, obj, act, exp, ruleID}, m) } - return rows.Err() + if err := rows.Err(); err != nil { + return err + } + + // Load role/group membership as Casbin grouping (g) edges. This is + // additive on top of the per-principal rules above: absence of the + // groups/roles tables (e.g. a DB that predates migration 028) must not + // break core rule evaluation, so failures here are logged and skipped + // rather than aborting the whole policy load. + a.loadGroupingPolicies(m) + + return nil +} + +// loadGroupingPolicies loads acl_group_members and acl_role_assignments as +// Casbin g-rules: g(":", "group:") and +// g(":", "role:"). Expired edges are +// excluded. Best-effort: a query error (e.g. missing table) is logged and +// skipped so the per-principal rules remain authoritative. +func (a *aclRulesAdapter) loadGroupingPolicies(m model.Model) { + memberQuery := ` + SELECT m.member_type, m.member_id, g.group_name + FROM acl_group_members m + JOIN acl_groups g ON g.group_id = m.group_id + WHERE m.expires_at IS NULL OR m.expires_at > NOW() + ` + if rows, err := a.db.Query(memberQuery); err != nil { + logging.Logger.Warn().Err(err).Msg("acl: failed to load group memberships; group-derived access disabled until next reload") + } else { + for rows.Next() { + var memberType, memberID, groupName string + if err := rows.Scan(&memberType, &memberID, &groupName); err != nil { + logging.Logger.Warn().Err(err).Msg("acl: failed to scan group membership row") + continue + } + _ = persist.LoadPolicyArray([]string{"g", memberType + ":" + memberID, GroupSubjectPrefix + groupName}, m) + } + rows.Close() + } + + assignQuery := ` + SELECT a.assignee_type, a.assignee_id, r.role_name + FROM acl_role_assignments a + JOIN acl_roles r ON r.role_id = a.role_id + WHERE a.expires_at IS NULL OR a.expires_at > NOW() + ` + if rows, err := a.db.Query(assignQuery); err != nil { + logging.Logger.Warn().Err(err).Msg("acl: failed to load role assignments; role-derived access disabled until next reload") + } else { + for rows.Next() { + var assigneeType, assigneeID, roleName string + if err := rows.Scan(&assigneeType, &assigneeID, &roleName); err != nil { + logging.Logger.Warn().Err(err).Msg("acl: failed to scan role assignment row") + continue + } + _ = persist.LoadPolicyArray([]string{"g", assigneeType + ":" + assigneeID, RoleSubjectPrefix + roleName}, m) + } + rows.Close() + } } // SavePolicy is a no-op. The Service manages database writes directly. diff --git a/server/internal/acl/audit.go b/server/internal/acl/audit.go index 520bd9a..661abb2 100644 --- a/server/internal/acl/audit.go +++ b/server/internal/acl/audit.go @@ -256,6 +256,69 @@ func (a *AuditLogger) LogAuthorityRequestEvent( a.shared.LogEvent(ctx, event) } +// LogExplain records that an ExplainAccess introspection was performed: WHO +// asked (caller* → the audit actor) and WHICH principal's access to which +// resource was explained (subject* → the audit subject), plus the evaluated +// outcome. It is emitted as an authorization event tagged +// operation="explain_access" (with metadata.explain=true) so it is queryable +// alongside enforcement decisions yet never mistaken for one — an explain does +// NOT gate any access, it only reports what access would be. +// +// callerType/callerID identify the requester: the connected principal on the +// gRPC path, or "admin_api"/ on the REST admin path. They may be +// empty when the caller is unknown. Non-blocking; no-op when the shared writer +// is nil. +func (a *AuditLogger) LogExplain(ctx context.Context, callerType, callerID, subjectType, subjectID, resourceType, resourceID string, requiredLevel int, decision *ACLDecision) { + if a.shared == nil { + return + } + + allowed := false + accessLevel := AccessNone + decisionStr := DecisionDeny + fallbackApplied := false + metadata := map[string]interface{}{ + "explain": true, + "required_level": requiredLevel, + } + if decision != nil { + allowed = decision.Allowed + accessLevel = decision.EffectiveAccessLevel + decisionStr = decision.Decision + fallbackApplied = decision.FallbackApplied + if decision.RuleApplied != nil { + metadata["rule_id"] = decision.RuleApplied.RuleID + } + } + metadata["decision"] = decisionStr + metadata["access_level"] = accessLevel + metadata["fallback_applied"] = fallbackApplied + + errorMsg := "" + if !allowed { + errorMsg = "access denied" + } + + event := &audit.AuditEvent{ + Timestamp: time.Now(), + EventType: audit.EventTypeAuthorization, + ActorType: callerType, + ActorID: callerID, + SubjectType: subjectType, + SubjectID: subjectID, + AuthorityMode: audit.AuthorityModeDirect, + ResourceType: resourceType, + ResourceID: resourceID, + Operation: "explain_access", + GatewayID: a.gatewayID, + Success: allowed, + ErrorMessage: errorMsg, + Metadata: metadata, + Source: audit.SourceGateway, + } + a.shared.LogEvent(ctx, event) +} + // entryToEvent translates an ACL AuditLogEntry into an audit.AuditEvent // suitable for the shared writer. Field shape matches the INSERT that the // old acl.aclEntryBatchWriter performed (event_type='authorization', diff --git a/server/internal/acl/audit_test.go b/server/internal/acl/audit_test.go index d6a782d..4b65541 100644 --- a/server/internal/acl/audit_test.go +++ b/server/internal/acl/audit_test.go @@ -458,11 +458,86 @@ func TestACLAuditLogger_NilSharedWriterIsNoop(t *testing.T) { models.Identity{Type: models.PrincipalUser, ID: "alice"}, ResourceTypeWorkspace, "prod", "connect", "prod", uuid.New(), ) + // LogExplain must also be a safe no-op with a nil shared writer. + logger.LogExplain(context.Background(), "user", "bob", "user", "alice", + ResourceTypeWorkspace, "prod", AccessManage, &ACLDecision{Decision: DecisionAllow}) if err := logger.Close(); err != nil { t.Errorf("Close() = %v, want nil", err) } } +// TestLogExplain_RoutesToSharedWriter verifies an ExplainAccess introspection +// emits an authorization audit event tagged operation="explain_access" that +// records WHO asked (caller → actor) about WHOSE access (subject), with the +// outcome in metadata. +func TestLogExplain_RoutesToSharedWriter(t *testing.T) { + db, conn := newACLFakeDB(t) + defer db.Close() + + cfg := audit.DefaultConfig() + cfg.BatchSize = 1 + cfg.FlushPeriod = 10 * time.Minute + shared := audit.NewAuditLogger(db, "gw-explain", cfg) + defer shared.Close() + + logger := NewAuditLogger(shared, nil, "gw-explain") + + decision := &ACLDecision{ + Decision: DecisionAllow, + Allowed: true, + EffectiveAccessLevel: AccessManage, + } + // caller = user:admin-bob asking about subject = user:alice on workspace:prod. + logger.LogExplain(context.Background(), PrincipalTypeUser, "admin-bob", PrincipalTypeUser, "alice", + ResourceTypeWorkspace, "prod", AccessManage, decision) + + deadline := time.Now().Add(2 * time.Second) + for time.Now().Before(deadline) { + if conn.execCount() >= 1 { + break + } + time.Sleep(5 * time.Millisecond) + } + if got := conn.execCount(); got != 1 { + t.Fatalf("expected 1 INSERT through shared writer, got %d", got) + } + + conn.mu.Lock() + args := conn.execs[0] + conn.mu.Unlock() + + if et, ok := args[1].(string); !ok || et != "authorization" { + t.Errorf("event_type arg = %v, want \"authorization\"", args[1]) + } + // The recorded args must include the operation, the caller (actor), and the + // explained subject — assert by presence rather than hardcoding every index. + want := map[string]bool{"explain_access": false, "admin-bob": false, "alice": false} + for _, a := range args { + if s, ok := a.(string); ok { + if _, tracked := want[s]; tracked { + want[s] = true + } + } + } + for v, found := range want { + if !found { + t.Errorf("expected INSERT args to include %q (caller/subject/operation): %v", v, args) + } + } + // metadata must flag this as an explain (not an enforcement decision). + if metaJSON, ok := args[20].([]byte); ok { + var m map[string]interface{} + if err := json.Unmarshal(metaJSON, &m); err != nil { + t.Fatalf("metadata JSON: %v", err) + } + if m["explain"] != true { + t.Errorf("metadata[explain] = %v, want true", m["explain"]) + } + } else { + t.Fatalf("metadata arg type = %T, want []byte", args[20]) + } +} + // TestBuildACLMetadata_IncludesOwningAgentAttribution exercises Phase 5 // Stage B: when an AuditLogEntry carries OwningAgentImpl/OwningAgentPrefix, // buildACLMetadata surfaces them as metadata.owning_agent + diff --git a/server/internal/acl/enforcer.go b/server/internal/acl/enforcer.go index 663b2cf..5af5afe 100644 --- a/server/internal/acl/enforcer.go +++ b/server/internal/acl/enforcer.go @@ -24,6 +24,9 @@ r = sub, obj, act [policy_definition] p = sub, obj, act, expires, rule_id +[role_definition] +g = _, _ + [policy_effect] e = some(where (p.eft == allow)) @@ -74,57 +77,106 @@ func NewCasbinEnforcer(db *sql.DB) (*CasbinEnforcer, error) { } // EvaluateAccess evaluates whether a principal has the required access level -// to a resource. It performs the 5-step specificity-priority lookup: +// to a resource. The principal's own subject plus every group/role it +// transitively belongs to (resolved via the Casbin role manager) form the +// "subject set"; rules matching ANY subject in the set are combined additively +// (highest access level wins). The lookup proceeds through specificity tiers: // -// 1. Exact principal + exact resource +// 1. Subject set + exact resource (self + groups/roles, additive max) // 2. Wildcard principal + exact resource -// 3. Exact principal + wildcard resource ("*") +// 3. Subject set + wildcard resource ("*") (self + groups/roles, additive max) // 4. Wildcard principal + wildcard resource ("*") -// 5. (No match — caller applies fallback policy) +// 5. Glob-pattern rules (any subject in the set) +// 6. (No match — caller applies fallback policy) +// +// With no groups/roles defined the subject set is just the principal, so the +// behavior is identical to the original per-principal evaluation. // // Returns nil if no matching rule is found (caller should apply fallback). func (ce *CasbinEnforcer) EvaluateAccess(ctx context.Context, principal models.Identity, resourceType, resourceID string, requiredLevel int) (*ACLDecision, error) { - principalType := PrincipalTypeForModel(principal.Type) - principalID := principal.CanonicalPrincipalID() + return ce.EvaluateBySubject(PrincipalTypeForModel(principal.Type), principal.CanonicalPrincipalID(), resourceType, resourceID, requiredLevel), nil +} +// EvaluateBySubject is the string-keyed core of EvaluateAccess: it runs the +// specificity ladder for a principal identified by its DB principal_type and +// canonical principal_id (the same values used to build the Casbin subject), +// without requiring a models.Identity. CheckAccess uses EvaluateAccess; +// introspection paths (ExplainAccess) call this directly. +func (ce *CasbinEnforcer) EvaluateBySubject(principalType, principalID, resourceType, resourceID string, requiredLevel int) *ACLDecision { sub := principalType + ":" + principalID obj := resourceType + ":" + resourceID + subjects := ce.subjectSet(sub) - // Step 1: Exact principal + exact resource - if decision := ce.findAndEvaluate(sub, obj, requiredLevel, "Explicit rule"); decision != nil { - return decision, nil + // Step 1: Subject set (self + groups/roles) + exact resource + if decision := ce.findAndEvaluateMulti(subjects, obj, requiredLevel, "Explicit rule"); decision != nil { + return decision } // Step 2: Wildcard principal + exact resource for _, wSub := range wildcardSubjects(principalType) { if decision := ce.findAndEvaluate(wSub, obj, requiredLevel, "Wildcard rule"); decision != nil { - return decision, nil + return decision } } - // Step 3: Exact principal + wildcard resource + // Step 3: Subject set + wildcard resource if resourceID != WildcardAnyResource { wObj := resourceType + ":" + WildcardAnyResource - if decision := ce.findAndEvaluate(sub, wObj, requiredLevel, "Any-resource rule"); decision != nil { - return decision, nil + if decision := ce.findAndEvaluateMulti(subjects, wObj, requiredLevel, "Any-resource rule"); decision != nil { + return decision } // Step 4: Wildcard principal + wildcard resource for _, wSub := range wildcardSubjects(principalType) { if decision := ce.findAndEvaluate(wSub, wObj, requiredLevel, "Wildcard any-resource rule"); decision != nil { - return decision, nil + return decision } } } // Step 5: Glob-pattern rules — scan policies with * or ? for pattern matches - if decision := ce.findGlobMatch(sub, obj, requiredLevel); decision != nil { - return decision, nil + if decision := ce.findGlobMatch(subjects, obj, requiredLevel); decision != nil { + return decision } // Step 6: No match — caller applies fallback - return nil, nil + return nil +} + +// subjectSet returns the principal's own subject plus every group/role it +// transitively belongs to. The first element is always the principal's own +// subject. On role-manager error it degrades to just the principal subject. +func (ce *CasbinEnforcer) subjectSet(sub string) []string { + roles, err := ce.enforcer.GetImplicitRolesForUser(sub) + if err != nil || len(roles) == 0 { + return []string{sub} + } + return append([]string{sub}, roles...) +} + +// findAndEvaluateMulti evaluates (subject, obj) for every subject in the set +// and combines the results additively: the highest non-expired access level +// across all subjects wins. When the winning rule was granted to a group/role +// rather than the principal itself, the reason is annotated for audit clarity. +// subjects[0] is assumed to be the principal's own subject. +func (ce *CasbinEnforcer) findAndEvaluateMulti(subjects []string, obj string, requiredLevel int, label string) *ACLDecision { + var best *ACLDecision + var bestSubject string + for _, s := range subjects { + d := ce.findAndEvaluate(s, obj, requiredLevel, label) + if d == nil { + continue + } + if best == nil || d.EffectiveAccessLevel > best.EffectiveAccessLevel { + best = d + bestSubject = s + } + } + if best != nil && len(subjects) > 0 && bestSubject != subjects[0] { + best.Reason = fmt.Sprintf("%s (via %s)", best.Reason, bestSubject) + } + return best } // findAndEvaluate looks up policies matching (sub, obj) and returns a decision @@ -187,12 +239,14 @@ func (ce *CasbinEnforcer) findAndEvaluate(sub, obj string, requiredLevel int, la return decision } -// findGlobMatch scans all policies for glob-pattern rules that match the given -// subject and object. This handles rules like "agent:ag._system.platform-server.*" -// matching "agent:ag._system.platform-server.ws-spark-2918". -// Only policies whose stored sub or obj contain glob characters (* or ?) are -// evaluated — exact-match policies are handled by findAndEvaluate. -func (ce *CasbinEnforcer) findGlobMatch(sub, obj string, requiredLevel int) *ACLDecision { +// findGlobMatch scans all policies once for glob-pattern rules that match any +// subject in the set and the given object. This handles rules like +// "agent:ag._system.platform-server.*" matching +// "agent:ag._system.platform-server.ws-spark-2918". Only policies whose stored +// sub or obj contain glob characters (* or ?) are evaluated — exact-match +// policies are handled by findAndEvaluate. The highest matching level across +// all subjects wins (additive semantics consistent with findAndEvaluateMulti). +func (ce *CasbinEnforcer) findGlobMatch(subjects []string, obj string, requiredLevel int) *ACLDecision { policies, _ := ce.enforcer.GetPolicy() if len(policies) == 0 { return nil @@ -213,8 +267,8 @@ func (ce *CasbinEnforcer) findGlobMatch(sub, obj string, requiredLevel int) *ACL continue } - // Check glob match on both subject and object - if !globMatch(sub, pSub) || !globMatch(obj, pObj) { + // Check glob match on the object and on ANY subject in the set + if !globMatch(obj, pObj) || !anyGlobMatch(subjects, pSub) { continue } @@ -269,6 +323,16 @@ func globMatch(name, pattern string) bool { return matched } +// anyGlobMatch reports whether any of the names matches the pattern. +func anyGlobMatch(names []string, pattern string) bool { + for _, n := range names { + if globMatch(n, pattern) { + return true + } + } + return false +} + // AddPolicy adds a rule to the in-memory model. Called by Service after writing // to acl_rules. The adapter's AddPolicy is a no-op, so this only touches memory. func (ce *CasbinEnforcer) AddPolicy(sub, obj, act, expires, ruleID string) (bool, error) { @@ -287,6 +351,103 @@ func (ce *CasbinEnforcer) ReloadPolicies() error { return ce.enforcer.LoadPolicy() } +// AddGrouping adds a grouping (g) edge to the in-memory model: member belongs +// to group/role. Called by the Service after persisting an acl_group_members / +// acl_role_assignments row. member and target are full subjects, e.g. +// ("user:alice", "group:engineering") or ("group:eng", "role:admin"). +func (ce *CasbinEnforcer) AddGrouping(member, target string) (bool, error) { + return ce.enforcer.AddGroupingPolicy(member, target) +} + +// RemoveGrouping removes a grouping (g) edge from the in-memory model. Called +// by the Service after deleting the corresponding membership/assignment row. +func (ce *CasbinEnforcer) RemoveGrouping(member, target string) (bool, error) { + return ce.enforcer.RemoveGroupingPolicy(member, target) +} + +// ImplicitRoles returns the transitive set of groups/roles a subject belongs +// to. Used for cycle detection and access-explain introspection. +func (ce *CasbinEnforcer) ImplicitRoles(sub string) ([]string, error) { + return ce.enforcer.GetImplicitRolesForUser(sub) +} + +// SubjectSet returns the principal subject plus its transitive groups/roles. +func (ce *CasbinEnforcer) SubjectSet(sub string) []string { + return ce.subjectSet(sub) +} + +// Contributions returns every rule that matches any subject in the set for the +// given resource — the "why" behind an access decision. See GatherContributions. +func (ce *CasbinEnforcer) Contributions(subjects []string, resourceType, resourceID string) []AccessContribution { + return GatherContributions(ce.enforcer, subjects, resourceType, resourceID) +} + +// GatherContributions scans the in-memory policy set once and returns every +// rule whose subject matches any subject in the set (exact or glob) AND whose +// object matches the resource (exact, the type-wildcard "*", or glob). Each +// contribution records the granting subject, rule id, level, the matched +// resource pattern, and whether the rule is expired — so callers can explain +// exactly which identity (self / group / role) confers which access, and why a +// rule did or did not apply. Shared by both the Postgres and SQLite stores. +func GatherContributions(e *casbin.SyncedEnforcer, subjects []string, resourceType, resourceID string) []AccessContribution { + subjSet := make(map[string]bool, len(subjects)) + for _, s := range subjects { + subjSet[s] = true + } + obj := resourceType + ":" + resourceID + wObj := resourceType + ":" + WildcardAnyResource + + policies, _ := e.GetPolicy() + out := make([]AccessContribution, 0, len(subjects)) + for _, p := range policies { + if len(p) < 3 { + continue + } + pSub, pObj := p[pIdxSub], p[pIdxObj] + + // Subject match: exact membership, or a glob pattern covering a subject. + subjMatch := subjSet[pSub] + if !subjMatch && strings.ContainsAny(pSub, "*?") { + subjMatch = anyGlobMatch(subjects, pSub) + } + if !subjMatch { + continue + } + + // Object match: exact, the type-wildcard, or a glob pattern. + objMatch := pObj == obj || pObj == wObj + if !objMatch && strings.ContainsAny(pObj, "*?") { + objMatch = globMatch(obj, pObj) + } + if !objMatch { + continue + } + + level, err := strconv.Atoi(p[pIdxAct]) + if err != nil { + continue + } + expired := false + if len(p) > pIdxExpires && p[pIdxExpires] != "" { + if t, err := time.Parse(time.RFC3339, p[pIdxExpires]); err == nil && time.Now().After(t) { + expired = true + } + } + ruleID := "" + if len(p) > pIdxRuleID { + ruleID = p[pIdxRuleID] + } + out = append(out, AccessContribution{ + Subject: pSub, + RuleID: ruleID, + AccessLevel: level, + Resource: pObj, + Expired: expired, + }) + } + return out +} + // wildcardSubjects returns the wildcard subject strings that match a principal type. func wildcardSubjects(principalType string) []string { switch principalType { diff --git a/server/internal/acl/enforcer_roles_test.go b/server/internal/acl/enforcer_roles_test.go new file mode 100644 index 0000000..e5724f4 --- /dev/null +++ b/server/internal/acl/enforcer_roles_test.go @@ -0,0 +1,131 @@ +package acl + +import ( + "context" + "strings" + "testing" + + "github.com/scitrera/aether/pkg/models" +) + +// addTestGrouping adds a grouping (g) edge directly to the in-memory enforcer. +func addTestGrouping(t *testing.T, ce *CasbinEnforcer, member, target string) { + t.Helper() + ok, err := ce.enforcer.AddGroupingPolicy(member, target) + if err != nil || !ok { + t.Fatalf("failed to add grouping %s -> %s: err=%v ok=%v", member, target, err, ok) + } +} + +func userPrincipal(id string) models.Identity { + return models.Identity{Type: models.PrincipalUser, ID: id} +} + +// Backward compatibility: with no groups/roles, evaluation is unchanged and the +// reason carries no "via" annotation. +func TestEvaluate_NoGroups_BackwardCompatible(t *testing.T) { + ce := newTestEnforcer(t) + addTestPolicy(t, ce, "user:alice", "workspace:prod", "20", "", "r1") + + d, err := ce.EvaluateAccess(context.Background(), userPrincipal("alice"), ResourceTypeWorkspace, "prod", AccessReadWrite) + if err != nil { + t.Fatalf("eval: %v", err) + } + if d == nil || !d.Allowed || d.EffectiveAccessLevel != AccessReadWrite { + t.Fatalf("expected allow@20, got %+v", d) + } + if strings.Contains(d.Reason, "via ") { + t.Errorf("reason should not be annotated with via for a direct rule: %q", d.Reason) + } +} + +// Additive max: a role grant higher than the principal's own grant wins, and +// the decision is annotated with the granting subject. +func TestEvaluate_AdditiveMax_RoleHigherThanSelf(t *testing.T) { + ce := newTestEnforcer(t) + addTestPolicy(t, ce, "user:alice", "workspace:prod", "10", "", "r-self") // READ + addTestPolicy(t, ce, "role:wsadmin", "workspace:prod", "30", "", "r-role") // MANAGE + addTestGrouping(t, ce, "user:alice", "role:wsadmin") + + d, err := ce.EvaluateAccess(context.Background(), userPrincipal("alice"), ResourceTypeWorkspace, "prod", AccessManage) + if err != nil { + t.Fatalf("eval: %v", err) + } + if d == nil || !d.Allowed || d.EffectiveAccessLevel != AccessManage { + t.Fatalf("expected allow@30 via role, got %+v", d) + } + if d.RuleApplied == nil || d.RuleApplied.RuleID != "r-role" { + t.Errorf("expected winning rule r-role, got %+v", d.RuleApplied) + } + if !strings.Contains(d.Reason, "via role:wsadmin") { + t.Errorf("expected reason annotated 'via role:wsadmin', got %q", d.Reason) + } +} + +// A direct grant higher than the group grant wins, with no via annotation. +func TestEvaluate_AdditiveMax_SelfHigherThanGroup(t *testing.T) { + ce := newTestEnforcer(t) + addTestPolicy(t, ce, "user:alice", "workspace:prod", "40", "", "r-self") // ADMIN + addTestPolicy(t, ce, "group:eng", "workspace:prod", "20", "", "r-grp") // READWRITE + addTestGrouping(t, ce, "user:alice", "group:eng") + + d, err := ce.EvaluateAccess(context.Background(), userPrincipal("alice"), ResourceTypeWorkspace, "prod", AccessManage) + if err != nil { + t.Fatalf("eval: %v", err) + } + if d == nil || !d.Allowed || d.EffectiveAccessLevel != AccessAdmin { + t.Fatalf("expected allow@40 from self, got %+v", d) + } + if strings.Contains(d.Reason, "via ") { + t.Errorf("self rule won; reason should not be via-annotated: %q", d.Reason) + } +} + +// Transitive nesting: user -> group -> role; the role's grant reaches the user. +func TestEvaluate_TransitiveNesting(t *testing.T) { + ce := newTestEnforcer(t) + addTestPolicy(t, ce, "role:wsadmin", "workspace:prod", "30", "", "r-role") + addTestGrouping(t, ce, "user:alice", "group:eng") + addTestGrouping(t, ce, "group:eng", "role:wsadmin") + + d, err := ce.EvaluateAccess(context.Background(), userPrincipal("alice"), ResourceTypeWorkspace, "prod", AccessManage) + if err != nil { + t.Fatalf("eval: %v", err) + } + if d == nil || !d.Allowed || d.EffectiveAccessLevel != AccessManage { + t.Fatalf("expected allow@30 via nested role, got %+v", d) + } +} + +// A role granted at the wildcard-resource tier still applies to a member. +func TestEvaluate_RoleWildcardResource(t *testing.T) { + ce := newTestEnforcer(t) + addTestPolicy(t, ce, "role:wsadmin", "workspace:*", "30", "", "r-role") + addTestGrouping(t, ce, "user:alice", "role:wsadmin") + + d, err := ce.EvaluateAccess(context.Background(), userPrincipal("alice"), ResourceTypeWorkspace, "prod", AccessManage) + if err != nil { + t.Fatalf("eval: %v", err) + } + if d == nil || !d.Allowed || d.EffectiveAccessLevel != AccessManage { + t.Fatalf("expected allow@30 via role wildcard-resource, got %+v", d) + } +} + +// Removing a grouping edge revokes the derived access. +func TestEvaluate_RemoveGroupingRevokesAccess(t *testing.T) { + ce := newTestEnforcer(t) + addTestPolicy(t, ce, "role:wsadmin", "workspace:prod", "30", "", "r-role") + addTestGrouping(t, ce, "user:alice", "role:wsadmin") + + if _, err := ce.RemoveGrouping("user:alice", "role:wsadmin"); err != nil { + t.Fatalf("remove grouping: %v", err) + } + d, err := ce.EvaluateAccess(context.Background(), userPrincipal("alice"), ResourceTypeWorkspace, "prod", AccessManage) + if err != nil { + t.Fatalf("eval: %v", err) + } + if d != nil { + t.Fatalf("expected no rule match after grouping removal, got %+v", d) + } +} diff --git a/server/internal/acl/groups_roles.go b/server/internal/acl/groups_roles.go new file mode 100644 index 0000000..ca57cf1 --- /dev/null +++ b/server/internal/acl/groups_roles.go @@ -0,0 +1,668 @@ +package acl + +import ( + "context" + "database/sql" + "encoding/json" + "errors" + "fmt" + "strings" + "time" + + "github.com/google/uuid" + "github.com/scitrera/aether/internal/logging" +) + +// Sentinel errors for the role/group surface. +var ( + ErrGroupNotFound = errors.New("group not found") + ErrRoleNotFound = errors.New("role not found") + ErrGroupExists = errors.New("group already exists") + ErrRoleExists = errors.New("role already exists") + ErrMembershipNotFound = errors.New("membership not found") + ErrAssignmentNotFound = errors.New("role assignment not found") + // ErrMembershipCycle is returned when an edge would introduce a cycle in + // the group/role membership DAG (e.g. group A in B in A). + ErrMembershipCycle = errors.New("membership would create a cycle") +) + +// Group is a named collection of principals. Permissions granted to a group +// (acl_rules rows with principal_type='group') apply to all its members. +type Group struct { + GroupID string `json:"group_id"` + GroupName string `json:"group_name"` + Description string `json:"description,omitempty"` + CreatedBy string `json:"created_by,omitempty"` + CreatedAt time.Time `json:"created_at"` + Metadata map[string]interface{} `json:"metadata,omitempty"` +} + +// Role is a named bundle of permissions (acl_rules rows with +// principal_type='role') assignable to principals or groups. +type Role struct { + RoleID string `json:"role_id"` + RoleName string `json:"role_name"` + Description string `json:"description,omitempty"` + CreatedBy string `json:"created_by,omitempty"` + CreatedAt time.Time `json:"created_at"` + Metadata map[string]interface{} `json:"metadata,omitempty"` +} + +// GroupMember is a single membership edge: a principal (or nested group) +// belongs to a group. +type GroupMember struct { + GroupName string `json:"group_name"` + MemberType string `json:"member_type"` + MemberID string `json:"member_id"` + GrantedBy string `json:"granted_by,omitempty"` + GrantedAt time.Time `json:"granted_at"` + ExpiresAt *time.Time `json:"expires_at,omitempty"` +} + +// RoleAssignment is a single assignment edge: a principal (or group) is granted +// a role. +type RoleAssignment struct { + RoleName string `json:"role_name"` + AssigneeType string `json:"assignee_type"` + AssigneeID string `json:"assignee_id"` + GrantedBy string `json:"granted_by,omitempty"` + GrantedAt time.Time `json:"granted_at"` + ExpiresAt *time.Time `json:"expires_at,omitempty"` +} + +// AccessExplanation describes how a principal's effective access to a resource +// was (or would be) decided. It surfaces what CheckAccess cannot: the full +// resolved subject set (self + transitive groups/roles), every rule that +// matched across those subjects (Contributions — the "why"), and the winning +// decision (nil when no rule matched and the fallback would apply). +type AccessExplanation struct { + Principal string `json:"principal"` + Subjects []string `json:"subjects"` + Contributions []AccessContribution `json:"contributions,omitempty"` + Decision *ACLDecision `json:"decision,omitempty"` +} + +// AccessContribution is a single rule that matched the principal or one of its +// groups/roles for the explained resource. +type AccessContribution struct { + Subject string `json:"subject"` // granting subject, e.g. "role:wsadmin" or "user:alice" + RuleID string `json:"rule_id"` // acl_rules.rule_id + AccessLevel int `json:"access_level"` // level the rule confers + Resource string `json:"resource"` // matched object pattern, e.g. "workspace:prod" or "workspace:*" + Expired bool `json:"expired"` // true if the rule is past its expiry (excluded from the decision) +} + +// memberSubject builds the Casbin subject string for a membership/assignment +// edge source. Group members use "group:" when nesting. +func memberSubject(memberType, memberID string) string { + return memberType + ":" + memberID +} + +// ========================================================================= +// Group CRUD +// ========================================================================= + +func (s *Service) CreateGroup(ctx context.Context, name, description, createdBy string, metadata map[string]interface{}) (*Group, error) { + if name == "" { + return nil, fmt.Errorf("group name is required") + } + g := &Group{ + GroupID: uuid.New().String(), + GroupName: name, + Description: description, + CreatedBy: createdBy, + CreatedAt: time.Now(), + Metadata: metadata, + } + meta, err := marshalMetadata(metadata) + if err != nil { + return nil, err + } + _, err = s.db.ExecContext(ctx, ` + INSERT INTO acl_groups (group_id, group_name, description, created_by, created_at, metadata) + VALUES ($1, $2, $3, $4, $5, $6) + `, g.GroupID, g.GroupName, nullString(description), nullString(createdBy), g.CreatedAt, meta) + if err != nil { + if isUniqueViolation(err) { + return nil, ErrGroupExists + } + return nil, fmt.Errorf("failed to create group: %w", err) + } + return g, nil +} + +func (s *Service) DeleteGroup(ctx context.Context, name string) error { + // Remove in-memory grouping edges that target this group, then delete the + // row. ON DELETE CASCADE removes acl_group_members; the enforcer is + // refreshed best-effort so derived access stops immediately. + res, err := s.db.ExecContext(ctx, `DELETE FROM acl_groups WHERE group_name = $1`, name) + if err != nil { + return fmt.Errorf("failed to delete group: %w", err) + } + if n, _ := res.RowsAffected(); n == 0 { + return ErrGroupNotFound + } + s.reloadEnforcer("delete group") + return nil +} + +func (s *Service) GetGroup(ctx context.Context, name string) (*Group, error) { + row := s.db.QueryRowContext(ctx, ` + SELECT group_id, group_name, description, created_by, created_at, metadata + FROM acl_groups WHERE group_name = $1 + `, name) + return scanGroup(row) +} + +func (s *Service) ListGroups(ctx context.Context) ([]*Group, error) { + rows, err := s.db.QueryContext(ctx, ` + SELECT group_id, group_name, description, created_by, created_at, metadata + FROM acl_groups ORDER BY group_name + `) + if err != nil { + return nil, fmt.Errorf("failed to list groups: %w", err) + } + defer rows.Close() + var out []*Group + for rows.Next() { + g, err := scanGroup(rows) + if err != nil { + return nil, err + } + out = append(out, g) + } + return out, rows.Err() +} + +// ========================================================================= +// Role CRUD +// ========================================================================= + +func (s *Service) CreateRole(ctx context.Context, name, description, createdBy string, metadata map[string]interface{}) (*Role, error) { + if name == "" { + return nil, fmt.Errorf("role name is required") + } + r := &Role{ + RoleID: uuid.New().String(), + RoleName: name, + Description: description, + CreatedBy: createdBy, + CreatedAt: time.Now(), + Metadata: metadata, + } + meta, err := marshalMetadata(metadata) + if err != nil { + return nil, err + } + _, err = s.db.ExecContext(ctx, ` + INSERT INTO acl_roles (role_id, role_name, description, created_by, created_at, metadata) + VALUES ($1, $2, $3, $4, $5, $6) + `, r.RoleID, r.RoleName, nullString(description), nullString(createdBy), r.CreatedAt, meta) + if err != nil { + if isUniqueViolation(err) { + return nil, ErrRoleExists + } + return nil, fmt.Errorf("failed to create role: %w", err) + } + return r, nil +} + +func (s *Service) DeleteRole(ctx context.Context, name string) error { + // Delete the role definition and its permission rules (acl_rules rows with + // principal_type='role'). ON DELETE CASCADE clears acl_role_assignments. + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return fmt.Errorf("failed to begin tx: %w", err) + } + defer tx.Rollback() + + res, err := tx.ExecContext(ctx, `DELETE FROM acl_roles WHERE role_name = $1`, name) + if err != nil { + return fmt.Errorf("failed to delete role: %w", err) + } + if n, _ := res.RowsAffected(); n == 0 { + return ErrRoleNotFound + } + if _, err := tx.ExecContext(ctx, `DELETE FROM acl_rules WHERE principal_type = $1 AND principal_id = $2`, + PrincipalTypeRole, name); err != nil { + return fmt.Errorf("failed to delete role permissions: %w", err) + } + if err := tx.Commit(); err != nil { + return fmt.Errorf("failed to commit role delete: %w", err) + } + s.reloadEnforcer("delete role") + return nil +} + +func (s *Service) GetRole(ctx context.Context, name string) (*Role, error) { + row := s.db.QueryRowContext(ctx, ` + SELECT role_id, role_name, description, created_by, created_at, metadata + FROM acl_roles WHERE role_name = $1 + `, name) + return scanRole(row) +} + +func (s *Service) ListRoles(ctx context.Context) ([]*Role, error) { + rows, err := s.db.QueryContext(ctx, ` + SELECT role_id, role_name, description, created_by, created_at, metadata + FROM acl_roles ORDER BY role_name + `) + if err != nil { + return nil, fmt.Errorf("failed to list roles: %w", err) + } + defer rows.Close() + var out []*Role + for rows.Next() { + r, err := scanRole(rows) + if err != nil { + return nil, err + } + out = append(out, r) + } + return out, rows.Err() +} + +// ========================================================================= +// Group membership +// ========================================================================= + +func (s *Service) AddGroupMember(ctx context.Context, groupName, memberType, memberID, grantedBy string, expiresAt *time.Time) (*GroupMember, error) { + groupID, err := s.groupIDByName(ctx, groupName) + if err != nil { + return nil, err + } + target := GroupSubjectPrefix + groupName + source := memberSubject(memberType, memberID) + if err := s.checkNoCycle(source, target); err != nil { + return nil, err + } + + m := &GroupMember{ + GroupName: groupName, + MemberType: memberType, + MemberID: memberID, + GrantedBy: grantedBy, + GrantedAt: time.Now(), + ExpiresAt: expiresAt, + } + _, err = s.db.ExecContext(ctx, ` + INSERT INTO acl_group_members (id, group_id, member_type, member_id, granted_by, granted_at, expires_at) + VALUES ($1, $2, $3, $4, $5, $6, $7) + ON CONFLICT (group_id, member_type, member_id) + DO UPDATE SET granted_by = EXCLUDED.granted_by, granted_at = EXCLUDED.granted_at, expires_at = EXCLUDED.expires_at + `, uuid.New().String(), groupID, memberType, memberID, nullString(grantedBy), m.GrantedAt, expiresAt) + if err != nil { + return nil, fmt.Errorf("failed to add group member: %w", err) + } + s.addGroupingEdge(source, target) + return m, nil +} + +func (s *Service) RemoveGroupMember(ctx context.Context, groupName, memberType, memberID string) error { + groupID, err := s.groupIDByName(ctx, groupName) + if err != nil { + return err + } + res, err := s.db.ExecContext(ctx, ` + DELETE FROM acl_group_members WHERE group_id = $1 AND member_type = $2 AND member_id = $3 + `, groupID, memberType, memberID) + if err != nil { + return fmt.Errorf("failed to remove group member: %w", err) + } + if n, _ := res.RowsAffected(); n == 0 { + return ErrMembershipNotFound + } + s.removeGroupingEdge(memberSubject(memberType, memberID), GroupSubjectPrefix+groupName) + return nil +} + +func (s *Service) ListGroupMembers(ctx context.Context, groupName string) ([]*GroupMember, error) { + rows, err := s.db.QueryContext(ctx, ` + SELECT m.member_type, m.member_id, m.granted_by, m.granted_at, m.expires_at + FROM acl_group_members m JOIN acl_groups g ON g.group_id = m.group_id + WHERE g.group_name = $1 ORDER BY m.member_type, m.member_id + `, groupName) + if err != nil { + return nil, fmt.Errorf("failed to list group members: %w", err) + } + defer rows.Close() + var out []*GroupMember + for rows.Next() { + m := &GroupMember{GroupName: groupName} + var grantedBy sql.NullString + var expiresAt sql.NullTime + if err := rows.Scan(&m.MemberType, &m.MemberID, &grantedBy, &m.GrantedAt, &expiresAt); err != nil { + return nil, err + } + m.GrantedBy = grantedBy.String + if expiresAt.Valid { + m.ExpiresAt = &expiresAt.Time + } + out = append(out, m) + } + return out, rows.Err() +} + +func (s *Service) ListPrincipalGroups(ctx context.Context, memberType, memberID string) ([]*GroupMember, error) { + rows, err := s.db.QueryContext(ctx, ` + SELECT g.group_name, m.granted_by, m.granted_at, m.expires_at + FROM acl_group_members m JOIN acl_groups g ON g.group_id = m.group_id + WHERE m.member_type = $1 AND m.member_id = $2 ORDER BY g.group_name + `, memberType, memberID) + if err != nil { + return nil, fmt.Errorf("failed to list principal groups: %w", err) + } + defer rows.Close() + var out []*GroupMember + for rows.Next() { + m := &GroupMember{MemberType: memberType, MemberID: memberID} + var grantedBy sql.NullString + var expiresAt sql.NullTime + if err := rows.Scan(&m.GroupName, &grantedBy, &m.GrantedAt, &expiresAt); err != nil { + return nil, err + } + m.GrantedBy = grantedBy.String + if expiresAt.Valid { + m.ExpiresAt = &expiresAt.Time + } + out = append(out, m) + } + return out, rows.Err() +} + +// ========================================================================= +// Role assignment +// ========================================================================= + +func (s *Service) AssignRole(ctx context.Context, roleName, assigneeType, assigneeID, grantedBy string, expiresAt *time.Time) (*RoleAssignment, error) { + roleID, err := s.roleIDByName(ctx, roleName) + if err != nil { + return nil, err + } + target := RoleSubjectPrefix + roleName + source := memberSubject(assigneeType, assigneeID) + if err := s.checkNoCycle(source, target); err != nil { + return nil, err + } + + a := &RoleAssignment{ + RoleName: roleName, + AssigneeType: assigneeType, + AssigneeID: assigneeID, + GrantedBy: grantedBy, + GrantedAt: time.Now(), + ExpiresAt: expiresAt, + } + _, err = s.db.ExecContext(ctx, ` + INSERT INTO acl_role_assignments (id, role_id, assignee_type, assignee_id, granted_by, granted_at, expires_at) + VALUES ($1, $2, $3, $4, $5, $6, $7) + ON CONFLICT (role_id, assignee_type, assignee_id) + DO UPDATE SET granted_by = EXCLUDED.granted_by, granted_at = EXCLUDED.granted_at, expires_at = EXCLUDED.expires_at + `, uuid.New().String(), roleID, assigneeType, assigneeID, nullString(grantedBy), a.GrantedAt, expiresAt) + if err != nil { + return nil, fmt.Errorf("failed to assign role: %w", err) + } + s.addGroupingEdge(source, target) + return a, nil +} + +func (s *Service) UnassignRole(ctx context.Context, roleName, assigneeType, assigneeID string) error { + roleID, err := s.roleIDByName(ctx, roleName) + if err != nil { + return err + } + res, err := s.db.ExecContext(ctx, ` + DELETE FROM acl_role_assignments WHERE role_id = $1 AND assignee_type = $2 AND assignee_id = $3 + `, roleID, assigneeType, assigneeID) + if err != nil { + return fmt.Errorf("failed to unassign role: %w", err) + } + if n, _ := res.RowsAffected(); n == 0 { + return ErrAssignmentNotFound + } + s.removeGroupingEdge(memberSubject(assigneeType, assigneeID), RoleSubjectPrefix+roleName) + return nil +} + +func (s *Service) ListRoleAssignments(ctx context.Context, roleName string) ([]*RoleAssignment, error) { + rows, err := s.db.QueryContext(ctx, ` + SELECT a.assignee_type, a.assignee_id, a.granted_by, a.granted_at, a.expires_at + FROM acl_role_assignments a JOIN acl_roles r ON r.role_id = a.role_id + WHERE r.role_name = $1 ORDER BY a.assignee_type, a.assignee_id + `, roleName) + if err != nil { + return nil, fmt.Errorf("failed to list role assignments: %w", err) + } + defer rows.Close() + var out []*RoleAssignment + for rows.Next() { + a := &RoleAssignment{RoleName: roleName} + var grantedBy sql.NullString + var expiresAt sql.NullTime + if err := rows.Scan(&a.AssigneeType, &a.AssigneeID, &grantedBy, &a.GrantedAt, &expiresAt); err != nil { + return nil, err + } + a.GrantedBy = grantedBy.String + if expiresAt.Valid { + a.ExpiresAt = &expiresAt.Time + } + out = append(out, a) + } + return out, rows.Err() +} + +func (s *Service) ListPrincipalRoles(ctx context.Context, assigneeType, assigneeID string) ([]*RoleAssignment, error) { + rows, err := s.db.QueryContext(ctx, ` + SELECT r.role_name, a.granted_by, a.granted_at, a.expires_at + FROM acl_role_assignments a JOIN acl_roles r ON r.role_id = a.role_id + WHERE a.assignee_type = $1 AND a.assignee_id = $2 ORDER BY r.role_name + `, assigneeType, assigneeID) + if err != nil { + return nil, fmt.Errorf("failed to list principal roles: %w", err) + } + defer rows.Close() + var out []*RoleAssignment + for rows.Next() { + a := &RoleAssignment{AssigneeType: assigneeType, AssigneeID: assigneeID} + var grantedBy sql.NullString + var expiresAt sql.NullTime + if err := rows.Scan(&a.RoleName, &grantedBy, &a.GrantedAt, &expiresAt); err != nil { + return nil, err + } + a.GrantedBy = grantedBy.String + if expiresAt.Valid { + a.ExpiresAt = &expiresAt.Time + } + out = append(out, a) + } + return out, rows.Err() +} + +// ========================================================================= +// Introspection + cleanup +// ========================================================================= + +// ExplainAccess returns the resolved subject set (self + transitive +// groups/roles), every rule that matched across those subjects, and the winning +// decision for a (principal, resource) pair. principalType/principalID are the +// canonical acl_rules values (e.g. "user","alice" or "agent","ag::ws::impl::spec"). +// It performs no audit write — it is a pure introspection helper. +func (s *Service) ExplainAccess(ctx context.Context, principalType, principalID, resourceType, resourceID string, requiredLevel int, callerType, callerID string) (*AccessExplanation, error) { + resourceType, resourceID, _ = rewriteLegacyPermission(resourceType, resourceID) + self := principalType + ":" + principalID + exp := &AccessExplanation{Principal: self, Subjects: []string{self}} + if s.enforcer != nil { + exp.Subjects = s.enforcer.SubjectSet(self) + exp.Contributions = s.enforcer.Contributions(exp.Subjects, resourceType, resourceID) + exp.Decision = s.enforcer.EvaluateBySubject(principalType, principalID, resourceType, resourceID, requiredLevel) + } + // Audit the introspection: who asked (caller) about whose access (principal). + if s.audit != nil { + s.audit.LogExplain(ctx, callerType, callerID, principalType, principalID, resourceType, resourceID, requiredLevel, exp.Decision) + } + return exp, nil +} + +// CleanupExpiredMemberships deletes expired membership/assignment edges and +// reloads the enforcer. Returns the number of rows removed. +func (s *Service) CleanupExpiredMemberships(ctx context.Context) (int64, error) { + var deleted int64 + if err := s.db.QueryRowContext(ctx, `SELECT cleanup_expired_acl_memberships()`).Scan(&deleted); err != nil { + return 0, fmt.Errorf("failed to cleanup expired memberships: %w", err) + } + if deleted > 0 { + s.reloadEnforcer("cleanup expired memberships") + } + return deleted, nil +} + +// ========================================================================= +// Internal helpers +// ========================================================================= + +func (s *Service) groupIDByName(ctx context.Context, name string) (string, error) { + var id string + err := s.db.QueryRowContext(ctx, `SELECT group_id FROM acl_groups WHERE group_name = $1`, name).Scan(&id) + if err == sql.ErrNoRows { + return "", ErrGroupNotFound + } + if err != nil { + return "", fmt.Errorf("failed to look up group: %w", err) + } + return id, nil +} + +func (s *Service) roleIDByName(ctx context.Context, name string) (string, error) { + var id string + err := s.db.QueryRowContext(ctx, `SELECT role_id FROM acl_roles WHERE role_name = $1`, name).Scan(&id) + if err == sql.ErrNoRows { + return "", ErrRoleNotFound + } + if err != nil { + return "", fmt.Errorf("failed to look up role: %w", err) + } + return id, nil +} + +// checkNoCycle rejects an edge source -> target that would create a cycle: the +// edge is unsafe if target already resolves (transitively) back to source, or +// source == target. +func (s *Service) checkNoCycle(source, target string) error { + if source == target { + return ErrMembershipCycle + } + if s.enforcer == nil { + return nil + } + reachable, err := s.enforcer.ImplicitRoles(target) + if err != nil { + return nil // best-effort: a resolver error must not block writes + } + for _, r := range reachable { + if r == source { + return ErrMembershipCycle + } + } + return nil +} + +func (s *Service) addGroupingEdge(source, target string) { + if s.enforcer == nil { + return + } + if _, err := s.enforcer.AddGrouping(source, target); err != nil { + logging.Logger.Warn().Err(err).Str("source", source).Str("target", target).Msg("acl: in-memory AddGrouping failed; persisted state authoritative") + } +} + +func (s *Service) removeGroupingEdge(source, target string) { + if s.enforcer == nil { + return + } + if _, err := s.enforcer.RemoveGrouping(source, target); err != nil { + logging.Logger.Warn().Err(err).Str("source", source).Str("target", target).Msg("acl: in-memory RemoveGrouping failed; persisted state authoritative") + } +} + +func (s *Service) reloadEnforcer(reason string) { + if s.enforcer == nil { + return + } + if err := s.enforcer.ReloadPolicies(); err != nil { + logging.Logger.Warn().Err(err).Str("reason", reason).Msg("acl: enforcer reload failed; in-memory model may lag DB until next reload") + } +} + +func scanGroup(s scanner) (*Group, error) { + g := &Group{} + var description, createdBy sql.NullString + var meta []byte + if err := s.Scan(&g.GroupID, &g.GroupName, &description, &createdBy, &g.CreatedAt, &meta); err != nil { + if err == sql.ErrNoRows { + return nil, ErrGroupNotFound + } + return nil, fmt.Errorf("failed to scan group: %w", err) + } + g.Description = description.String + g.CreatedBy = createdBy.String + g.Metadata = unmarshalMetadata(meta) + return g, nil +} + +func scanRole(s scanner) (*Role, error) { + r := &Role{} + var description, createdBy sql.NullString + var meta []byte + if err := s.Scan(&r.RoleID, &r.RoleName, &description, &createdBy, &r.CreatedAt, &meta); err != nil { + if err == sql.ErrNoRows { + return nil, ErrRoleNotFound + } + return nil, fmt.Errorf("failed to scan role: %w", err) + } + r.Description = description.String + r.CreatedBy = createdBy.String + r.Metadata = unmarshalMetadata(meta) + return r, nil +} + +func marshalMetadata(m map[string]interface{}) (interface{}, error) { + if len(m) == 0 { + return nil, nil + } + b, err := json.Marshal(m) + if err != nil { + return nil, fmt.Errorf("failed to marshal metadata: %w", err) + } + return b, nil +} + +func unmarshalMetadata(b []byte) map[string]interface{} { + if len(b) == 0 { + return nil + } + var m map[string]interface{} + if err := json.Unmarshal(b, &m); err != nil { + return nil + } + return m +} + +func nullString(s string) interface{} { + if s == "" { + return nil + } + return s +} + +// isUniqueViolation reports whether err is a unique-constraint violation. +// Driver-agnostic substring match (Postgres "duplicate key value violates +// unique constraint"; SQLite "UNIQUE constraint failed"). +func isUniqueViolation(err error) bool { + if err == nil { + return false + } + msg := err.Error() + return strings.Contains(msg, "duplicate key") || + strings.Contains(msg, "unique constraint") || + strings.Contains(msg, "UNIQUE constraint") +} diff --git a/server/internal/acl/legacy_perm.go b/server/internal/acl/legacy_perm.go index 7bfff5f..07ac363 100644 --- a/server/internal/acl/legacy_perm.go +++ b/server/internal/acl/legacy_perm.go @@ -27,6 +27,7 @@ var legacyPermMap = map[string]struct { "_perm:metric_credit": {ResourceTypeCapability, "capability/metric_credit"}, "_perm:resolve_authority": {ResourceTypeCapability, "capability/resolve_authority"}, "_perm:query_connections": {ResourceTypeCapability, "capability/query_connections"}, + "_perm:mint_user_tokens": {ResourceTypeCapability, "capability/mint_user_tokens"}, } // legacyPermLogOnce ensures we log a single notice per process whenever a diff --git a/server/internal/acl/service.go b/server/internal/acl/service.go index 316dfd1..27f30e2 100644 --- a/server/internal/acl/service.go +++ b/server/internal/acl/service.go @@ -464,6 +464,28 @@ func (s *Service) GetRule(ctx context.Context, principalType, principalID, resou return rule, nil } +// GetRuleByID retrieves a specific ACL rule by its rule_id (UUID). +func (s *Service) GetRuleByID(ctx context.Context, ruleID string) (*ACLRule, error) { + query := ` + SELECT rule_id, principal_type, principal_id, resource_type, resource_id, + access_level, granted_by, granted_at, expires_at, reason + FROM acl_rules + WHERE rule_id = $1 + ` + + rule, err := scanACLRule(s.db.QueryRowContext(ctx, query, ruleID)) + + if err == sql.ErrNoRows { + return nil, ErrRuleNotFound + } + + if err != nil { + return nil, fmt.Errorf("failed to get ACL rule by id: %w", err) + } + + return rule, nil +} + // ListRules retrieves all ACL rules matching the filter func (s *Service) ListRules(ctx context.Context, filter RuleFilter) ([]*ACLRule, error) { query := ` diff --git a/server/internal/acl/types.go b/server/internal/acl/types.go index c178fd2..7559bda 100644 --- a/server/internal/acl/types.go +++ b/server/internal/acl/types.go @@ -54,6 +54,21 @@ const ( PrincipalTypeBridge = "bridge" PrincipalTypeService = "service" PrincipalTypeWildcard = "wildcard" + // PrincipalTypeGroup and PrincipalTypeRole are synthetic subject types + // used for role/group authorization. They are never authenticated + // principals; they appear only as the target of Casbin grouping (g) edges + // and as the principal_type of acl_rules rows that grant permissions to a + // group or role. The "id" of a group/role subject is its (unique) name, + // so subjects read "group:" / "role:". + PrincipalTypeGroup = "group" + PrincipalTypeRole = "role" +) + +// Subject prefixes for the synthetic group/role subjects used in Casbin +// grouping edges and acl_rules. Equal to ":". +const ( + GroupSubjectPrefix = PrincipalTypeGroup + ":" // "group:" + RoleSubjectPrefix = PrincipalTypeRole + ":" // "role:" ) // Reserved identifiers for ACL system @@ -95,6 +110,8 @@ const ( // against the subject), so the subject's own ACL remains the security ceiling. WorkspaceScopeSubjectInherited = "_subject_workspaces" PermissionQueryConnections = "capability/query_connections" // capability gate — query the live-connection status of principals other than self + PermissionKVPurgeIdentity = "capability/kv_purge_identity" // capability gate — REMOVAL-ONLY purge of another principal's KV namespace (lifecycle managers reaping ephemeral principals); never reads/returns values + PermissionMintUserTokens = "capability/mint_user_tokens" // capability gate — mint API tokens with principal_type=user (per-user session keys minted by the platform-server on a user's behalf) ) // Decision constants diff --git a/server/internal/admin/acl_handler.go b/server/internal/admin/acl_handler.go index 26b94aa..e5da8a7 100644 --- a/server/internal/admin/acl_handler.go +++ b/server/internal/admin/acl_handler.go @@ -1,10 +1,13 @@ package admin import ( + "errors" "fmt" "net/http" + "strconv" "github.com/gorilla/mux" + aclstore "github.com/scitrera/aether/internal/storage/acl" ) // ============================================================================= @@ -377,3 +380,415 @@ func (s *Server) handleCleanupOldACLAuditLogs(w http.ResponseWriter, r *http.Req "retention_days": retentionDays, }) } + +// ============================================================================= +// ACL Group Handlers +// ============================================================================= + +func (s *Server) handleListACLGroups(w http.ResponseWriter, r *http.Request) { + groups, err := s.provider.ListACLGroups(r.Context()) + if err != nil { + s.respondInternalError(w, "failed to list ACL groups", err) + return + } + respondJSON(w, http.StatusOK, map[string]interface{}{ + "groups": groups, + "count": len(groups), + }) +} + +func (s *Server) handleCreateACLGroup(w http.ResponseWriter, r *http.Request) { + req := decodeJSON[CreateACLGroupRequest](w, r) + if req == nil { + return + } + if req.Name == "" { + respondError(w, http.StatusBadRequest, "name is required") + return + } + + group, err := s.provider.CreateACLGroup(r.Context(), req) + if err != nil { + if errors.Is(err, aclstore.ErrGroupExists) { + respondError(w, http.StatusConflict, fmt.Sprintf("group %q already exists", req.Name)) + return + } + s.respondInternalError(w, "failed to create ACL group", err) + return + } + + respondJSON(w, http.StatusCreated, map[string]interface{}{ + "message": fmt.Sprintf("group %q created", req.Name), + "group": group, + }) +} + +func (s *Server) handleGetACLGroup(w http.ResponseWriter, r *http.Request) { + vars := mux.Vars(r) + name := vars["name"] + + group, err := s.provider.GetACLGroup(r.Context(), name) + if err != nil { + if errors.Is(err, aclstore.ErrGroupNotFound) { + respondError(w, http.StatusNotFound, fmt.Sprintf("group %q not found", name)) + return + } + s.respondInternalError(w, "failed to get ACL group", err) + return + } + + respondJSON(w, http.StatusOK, group) +} + +func (s *Server) handleDeleteACLGroup(w http.ResponseWriter, r *http.Request) { + vars := mux.Vars(r) + name := vars["name"] + + if err := s.provider.DeleteACLGroup(r.Context(), name); err != nil { + if errors.Is(err, aclstore.ErrGroupNotFound) { + respondError(w, http.StatusNotFound, fmt.Sprintf("group %q not found", name)) + return + } + s.respondInternalError(w, "failed to delete ACL group", err) + return + } + + respondJSON(w, http.StatusOK, map[string]string{ + "message": fmt.Sprintf("group %q deleted", name), + }) +} + +func (s *Server) handleListACLGroupMembers(w http.ResponseWriter, r *http.Request) { + vars := mux.Vars(r) + name := vars["name"] + + members, err := s.provider.ListACLGroupMembers(r.Context(), name) + if err != nil { + if errors.Is(err, aclstore.ErrGroupNotFound) { + respondError(w, http.StatusNotFound, fmt.Sprintf("group %q not found", name)) + return + } + s.respondInternalError(w, "failed to list ACL group members", err) + return + } + + respondJSON(w, http.StatusOK, map[string]interface{}{ + "group_name": name, + "members": members, + "count": len(members), + }) +} + +func (s *Server) handleAddACLGroupMember(w http.ResponseWriter, r *http.Request) { + vars := mux.Vars(r) + name := vars["name"] + + req := decodeJSON[AddACLGroupMemberRequest](w, r) + if req == nil { + return + } + if req.MemberType == "" { + respondError(w, http.StatusBadRequest, "member_type is required") + return + } + if req.MemberID == "" { + respondError(w, http.StatusBadRequest, "member_id is required") + return + } + + member, err := s.provider.AddACLGroupMember(r.Context(), name, req) + if err != nil { + if errors.Is(err, aclstore.ErrGroupNotFound) { + respondError(w, http.StatusNotFound, fmt.Sprintf("group %q not found", name)) + return + } + if errors.Is(err, aclstore.ErrMembershipCycle) { + respondError(w, http.StatusBadRequest, "membership would create a cycle") + return + } + s.respondInternalError(w, "failed to add ACL group member", err) + return + } + + respondJSON(w, http.StatusCreated, map[string]interface{}{ + "message": fmt.Sprintf("member %s:%s added to group %q", req.MemberType, req.MemberID, name), + "member": member, + }) +} + +func (s *Server) handleRemoveACLGroupMember(w http.ResponseWriter, r *http.Request) { + vars := mux.Vars(r) + name := vars["name"] + + query := r.URL.Query() + memberType := query.Get("member_type") + memberID := query.Get("member_id") + + if memberType == "" || memberID == "" { + respondError(w, http.StatusBadRequest, "member_type and member_id are required") + return + } + + if err := s.provider.RemoveACLGroupMember(r.Context(), name, memberType, memberID); err != nil { + if errors.Is(err, aclstore.ErrGroupNotFound) { + respondError(w, http.StatusNotFound, fmt.Sprintf("group %q not found", name)) + return + } + if errors.Is(err, aclstore.ErrMembershipNotFound) { + respondError(w, http.StatusNotFound, fmt.Sprintf("member %s:%s not found in group %q", memberType, memberID, name)) + return + } + s.respondInternalError(w, "failed to remove ACL group member", err) + return + } + + respondJSON(w, http.StatusOK, map[string]string{ + "message": fmt.Sprintf("member %s:%s removed from group %q", memberType, memberID, name), + }) +} + +// ============================================================================= +// ACL Role Handlers +// ============================================================================= + +func (s *Server) handleListACLRoles(w http.ResponseWriter, r *http.Request) { + roles, err := s.provider.ListACLRoles(r.Context()) + if err != nil { + s.respondInternalError(w, "failed to list ACL roles", err) + return + } + respondJSON(w, http.StatusOK, map[string]interface{}{ + "roles": roles, + "count": len(roles), + }) +} + +func (s *Server) handleCreateACLRole(w http.ResponseWriter, r *http.Request) { + req := decodeJSON[CreateACLRoleRequest](w, r) + if req == nil { + return + } + if req.Name == "" { + respondError(w, http.StatusBadRequest, "name is required") + return + } + + role, err := s.provider.CreateACLRole(r.Context(), req) + if err != nil { + if errors.Is(err, aclstore.ErrRoleExists) { + respondError(w, http.StatusConflict, fmt.Sprintf("role %q already exists", req.Name)) + return + } + s.respondInternalError(w, "failed to create ACL role", err) + return + } + + respondJSON(w, http.StatusCreated, map[string]interface{}{ + "message": fmt.Sprintf("role %q created", req.Name), + "role": role, + }) +} + +func (s *Server) handleGetACLRole(w http.ResponseWriter, r *http.Request) { + vars := mux.Vars(r) + name := vars["name"] + + role, err := s.provider.GetACLRole(r.Context(), name) + if err != nil { + if errors.Is(err, aclstore.ErrRoleNotFound) { + respondError(w, http.StatusNotFound, fmt.Sprintf("role %q not found", name)) + return + } + s.respondInternalError(w, "failed to get ACL role", err) + return + } + + respondJSON(w, http.StatusOK, role) +} + +func (s *Server) handleDeleteACLRole(w http.ResponseWriter, r *http.Request) { + vars := mux.Vars(r) + name := vars["name"] + + if err := s.provider.DeleteACLRole(r.Context(), name); err != nil { + if errors.Is(err, aclstore.ErrRoleNotFound) { + respondError(w, http.StatusNotFound, fmt.Sprintf("role %q not found", name)) + return + } + s.respondInternalError(w, "failed to delete ACL role", err) + return + } + + respondJSON(w, http.StatusOK, map[string]string{ + "message": fmt.Sprintf("role %q deleted", name), + }) +} + +func (s *Server) handleListACLRoleAssignments(w http.ResponseWriter, r *http.Request) { + vars := mux.Vars(r) + name := vars["name"] + + assignments, err := s.provider.ListACLRoleAssignments(r.Context(), name) + if err != nil { + if errors.Is(err, aclstore.ErrRoleNotFound) { + respondError(w, http.StatusNotFound, fmt.Sprintf("role %q not found", name)) + return + } + s.respondInternalError(w, "failed to list ACL role assignments", err) + return + } + + respondJSON(w, http.StatusOK, map[string]interface{}{ + "role_name": name, + "assignments": assignments, + "count": len(assignments), + }) +} + +func (s *Server) handleAssignACLRole(w http.ResponseWriter, r *http.Request) { + vars := mux.Vars(r) + name := vars["name"] + + req := decodeJSON[AssignACLRoleRequest](w, r) + if req == nil { + return + } + if req.AssigneeType == "" { + respondError(w, http.StatusBadRequest, "assignee_type is required") + return + } + if req.AssigneeID == "" { + respondError(w, http.StatusBadRequest, "assignee_id is required") + return + } + + assignment, err := s.provider.AssignACLRole(r.Context(), name, req) + if err != nil { + if errors.Is(err, aclstore.ErrRoleNotFound) { + respondError(w, http.StatusNotFound, fmt.Sprintf("role %q not found", name)) + return + } + if errors.Is(err, aclstore.ErrMembershipCycle) { + respondError(w, http.StatusBadRequest, "assignment would create a cycle") + return + } + s.respondInternalError(w, "failed to assign ACL role", err) + return + } + + respondJSON(w, http.StatusCreated, map[string]interface{}{ + "message": fmt.Sprintf("role %q assigned to %s:%s", name, req.AssigneeType, req.AssigneeID), + "assignment": assignment, + }) +} + +func (s *Server) handleUnassignACLRole(w http.ResponseWriter, r *http.Request) { + vars := mux.Vars(r) + name := vars["name"] + + query := r.URL.Query() + assigneeType := query.Get("assignee_type") + assigneeID := query.Get("assignee_id") + + if assigneeType == "" || assigneeID == "" { + respondError(w, http.StatusBadRequest, "assignee_type and assignee_id are required") + return + } + + if err := s.provider.UnassignACLRole(r.Context(), name, assigneeType, assigneeID); err != nil { + if errors.Is(err, aclstore.ErrRoleNotFound) { + respondError(w, http.StatusNotFound, fmt.Sprintf("role %q not found", name)) + return + } + if errors.Is(err, aclstore.ErrAssignmentNotFound) { + respondError(w, http.StatusNotFound, fmt.Sprintf("assignment of %s:%s to role %q not found", assigneeType, assigneeID, name)) + return + } + s.respondInternalError(w, "failed to unassign ACL role", err) + return + } + + respondJSON(w, http.StatusOK, map[string]string{ + "message": fmt.Sprintf("role %q unassigned from %s:%s", name, assigneeType, assigneeID), + }) +} + +// ============================================================================= +// ACL Principal Handlers +// ============================================================================= + +func (s *Server) handleListACLPrincipalGroups(w http.ResponseWriter, r *http.Request) { + vars := mux.Vars(r) + memberType := vars["type"] + memberID := vars["id"] + + members, err := s.provider.ListACLPrincipalGroups(r.Context(), memberType, memberID) + if err != nil { + s.respondInternalError(w, "failed to list principal groups", err) + return + } + + respondJSON(w, http.StatusOK, map[string]interface{}{ + "member_type": memberType, + "member_id": memberID, + "groups": members, + "count": len(members), + }) +} + +func (s *Server) handleListACLPrincipalRoles(w http.ResponseWriter, r *http.Request) { + vars := mux.Vars(r) + assigneeType := vars["type"] + assigneeID := vars["id"] + + assignments, err := s.provider.ListACLPrincipalRoles(r.Context(), assigneeType, assigneeID) + if err != nil { + s.respondInternalError(w, "failed to list principal roles", err) + return + } + + respondJSON(w, http.StatusOK, map[string]interface{}{ + "assignee_type": assigneeType, + "assignee_id": assigneeID, + "roles": assignments, + "count": len(assignments), + }) +} + +// handleExplainACLAccess explains how a principal's effective access to a +// resource is decided: the resolved subject set (self + groups/roles), the +// rules that matched, and the resulting decision. Emits an "explain_access" +// audit event recording the caller (admin_api + remote addr) and subject. +// GET /acl/principals/{type}/{id}/effective?resource_type=&resource_id=&required_level= +func (s *Server) handleExplainACLAccess(w http.ResponseWriter, r *http.Request) { + vars := mux.Vars(r) + principalType := vars["type"] + principalID := vars["id"] + + q := r.URL.Query() + resourceType := q.Get("resource_type") + resourceID := q.Get("resource_id") + if resourceType == "" || resourceID == "" { + respondError(w, http.StatusBadRequest, "resource_type and resource_id query parameters are required") + return + } + requiredLevel := 0 + if rl := q.Get("required_level"); rl != "" { + v, err := strconv.Atoi(rl) + if err != nil { + respondError(w, http.StatusBadRequest, "required_level must be an integer") + return + } + requiredLevel = v + } + + // The admin REST API is gated by admin middleware, not a principal + // session, so record the source + remote address as the caller for the + // audit trail rather than a principal identity. + exp, err := s.provider.ExplainACLAccess(r.Context(), principalType, principalID, resourceType, resourceID, requiredLevel, "admin_api", r.RemoteAddr) + if err != nil { + s.respondInternalError(w, "failed to explain access", err) + return + } + respondJSON(w, http.StatusOK, exp) +} diff --git a/server/internal/admin/server.go b/server/internal/admin/server.go index 1290553..c574bed 100644 --- a/server/internal/admin/server.go +++ b/server/internal/admin/server.go @@ -297,6 +297,29 @@ func (s *Server) registerAPIRoutes(r *mux.Router) { r.HandleFunc("/acl/cleanup/expired-rules", s.handleCleanupExpiredACLRules).Methods("POST") r.HandleFunc("/acl/cleanup/audit-logs", s.handleCleanupOldACLAuditLogs).Methods("POST") + // ACL Groups + r.HandleFunc("/acl/groups", s.handleListACLGroups).Methods("GET") + r.HandleFunc("/acl/groups", s.handleCreateACLGroup).Methods("POST") + r.HandleFunc("/acl/groups/{name}", s.handleGetACLGroup).Methods("GET") + r.HandleFunc("/acl/groups/{name}", s.handleDeleteACLGroup).Methods("DELETE") + r.HandleFunc("/acl/groups/{name}/members", s.handleListACLGroupMembers).Methods("GET") + r.HandleFunc("/acl/groups/{name}/members", s.handleAddACLGroupMember).Methods("POST") + r.HandleFunc("/acl/groups/{name}/members", s.handleRemoveACLGroupMember).Methods("DELETE") + + // ACL Roles + r.HandleFunc("/acl/roles", s.handleListACLRoles).Methods("GET") + r.HandleFunc("/acl/roles", s.handleCreateACLRole).Methods("POST") + r.HandleFunc("/acl/roles/{name}", s.handleGetACLRole).Methods("GET") + r.HandleFunc("/acl/roles/{name}", s.handleDeleteACLRole).Methods("DELETE") + r.HandleFunc("/acl/roles/{name}/assignments", s.handleListACLRoleAssignments).Methods("GET") + r.HandleFunc("/acl/roles/{name}/assignments", s.handleAssignACLRole).Methods("POST") + r.HandleFunc("/acl/roles/{name}/assignments", s.handleUnassignACLRole).Methods("DELETE") + + // ACL Principals + r.HandleFunc("/acl/principals/{type}/{id}/groups", s.handleListACLPrincipalGroups).Methods("GET") + r.HandleFunc("/acl/principals/{type}/{id}/roles", s.handleListACLPrincipalRoles).Methods("GET") + r.HandleFunc("/acl/principals/{type}/{id}/effective", s.handleExplainACLAccess).Methods("GET") + // API Tokens r.HandleFunc("/tokens", s.handleListTokens).Methods("GET") r.HandleFunc("/tokens", s.handleCreateToken).Methods("POST") diff --git a/server/internal/admin/server_test.go b/server/internal/admin/server_test.go index dc6f2cd..87da721 100644 --- a/server/internal/admin/server_test.go +++ b/server/internal/admin/server_test.go @@ -183,6 +183,9 @@ func (m *mockStateProvider) ListACLRules(_ context.Context, _ *ACLRuleFilter) ([ func (m *mockStateProvider) GetACLRule(_ context.Context, _, _, _, _ string) (*ACLRuleInfo, error) { return m.aclRule, m.aclRuleErr } +func (m *mockStateProvider) GetACLRuleByID(_ context.Context, _ string) (*ACLRuleInfo, error) { + return m.aclRule, m.aclRuleErr +} func (m *mockStateProvider) GrantACLAccess(_ context.Context, _ *GrantACLAccessRequest) (*ACLRuleInfo, error) { return m.grantACLRule, m.grantACLErr } @@ -219,6 +222,51 @@ func (m *mockStateProvider) CleanupExpiredACLRules(_ context.Context) (int64, er func (m *mockStateProvider) CleanupOldACLAuditLogs(_ context.Context, _ int) (int64, error) { return m.cleanupAuditCount, m.cleanupAuditErr } +func (m *mockStateProvider) ListACLGroups(_ context.Context) ([]*ACLGroupInfo, error) { + return nil, nil +} +func (m *mockStateProvider) CreateACLGroup(_ context.Context, _ *CreateACLGroupRequest) (*ACLGroupInfo, error) { + return nil, nil +} +func (m *mockStateProvider) GetACLGroup(_ context.Context, _ string) (*ACLGroupInfo, error) { + return nil, nil +} +func (m *mockStateProvider) DeleteACLGroup(_ context.Context, _ string) error { return nil } +func (m *mockStateProvider) ListACLGroupMembers(_ context.Context, _ string) ([]*ACLGroupMemberInfo, error) { + return nil, nil +} +func (m *mockStateProvider) AddACLGroupMember(_ context.Context, _ string, _ *AddACLGroupMemberRequest) (*ACLGroupMemberInfo, error) { + return nil, nil +} +func (m *mockStateProvider) RemoveACLGroupMember(_ context.Context, _, _, _ string) error { + return nil +} +func (m *mockStateProvider) ListACLRoles(_ context.Context) ([]*ACLRoleInfo, error) { + return nil, nil +} +func (m *mockStateProvider) CreateACLRole(_ context.Context, _ *CreateACLRoleRequest) (*ACLRoleInfo, error) { + return nil, nil +} +func (m *mockStateProvider) GetACLRole(_ context.Context, _ string) (*ACLRoleInfo, error) { + return nil, nil +} +func (m *mockStateProvider) DeleteACLRole(_ context.Context, _ string) error { return nil } +func (m *mockStateProvider) ListACLRoleAssignments(_ context.Context, _ string) ([]*ACLRoleAssignmentInfo, error) { + return nil, nil +} +func (m *mockStateProvider) AssignACLRole(_ context.Context, _ string, _ *AssignACLRoleRequest) (*ACLRoleAssignmentInfo, error) { + return nil, nil +} +func (m *mockStateProvider) UnassignACLRole(_ context.Context, _, _, _ string) error { return nil } +func (m *mockStateProvider) ListACLPrincipalGroups(_ context.Context, _, _ string) ([]*ACLGroupMemberInfo, error) { + return nil, nil +} +func (m *mockStateProvider) ListACLPrincipalRoles(_ context.Context, _, _ string) ([]*ACLRoleAssignmentInfo, error) { + return nil, nil +} +func (m *mockStateProvider) ExplainACLAccess(_ context.Context, _, _, _, _ string, _ int, _, _ string) (*ACLAccessExplanationInfo, error) { + return nil, nil +} func (m *mockStateProvider) GetMessageFlow(_ context.Context, _ string) (*MessageFlowInfo, error) { return m.messageFlow, m.messageFlowErr } diff --git a/server/internal/admin/state_provider.go b/server/internal/admin/state_provider.go index 061643c..16adb31 100644 --- a/server/internal/admin/state_provider.go +++ b/server/internal/admin/state_provider.go @@ -59,6 +59,7 @@ type StateProvider interface { // ACL Management ListACLRules(ctx context.Context, filter *ACLRuleFilter) ([]*ACLRuleInfo, error) GetACLRule(ctx context.Context, principalType, principalID, resourceType, resourceID string) (*ACLRuleInfo, error) + GetACLRuleByID(ctx context.Context, ruleID string) (*ACLRuleInfo, error) GrantACLAccess(ctx context.Context, req *GrantACLAccessRequest) (*ACLRuleInfo, error) RevokeACLAccess(ctx context.Context, principalType, principalID, resourceType, resourceID string) error ListACLAuthorityGrants(ctx context.Context, filter *ACLAuthorityGrantFilter) ([]*ACLAuthorityGrantInfo, error) @@ -72,6 +73,25 @@ type StateProvider interface { CleanupExpiredACLRules(ctx context.Context) (int64, error) CleanupOldACLAuditLogs(ctx context.Context, retentionDays int) (int64, error) + // ACL Groups & Roles + ListACLGroups(ctx context.Context) ([]*ACLGroupInfo, error) + CreateACLGroup(ctx context.Context, req *CreateACLGroupRequest) (*ACLGroupInfo, error) + GetACLGroup(ctx context.Context, name string) (*ACLGroupInfo, error) + DeleteACLGroup(ctx context.Context, name string) error + ListACLGroupMembers(ctx context.Context, groupName string) ([]*ACLGroupMemberInfo, error) + AddACLGroupMember(ctx context.Context, groupName string, req *AddACLGroupMemberRequest) (*ACLGroupMemberInfo, error) + RemoveACLGroupMember(ctx context.Context, groupName, memberType, memberID string) error + ListACLRoles(ctx context.Context) ([]*ACLRoleInfo, error) + CreateACLRole(ctx context.Context, req *CreateACLRoleRequest) (*ACLRoleInfo, error) + GetACLRole(ctx context.Context, name string) (*ACLRoleInfo, error) + DeleteACLRole(ctx context.Context, name string) error + ListACLRoleAssignments(ctx context.Context, roleName string) ([]*ACLRoleAssignmentInfo, error) + AssignACLRole(ctx context.Context, roleName string, req *AssignACLRoleRequest) (*ACLRoleAssignmentInfo, error) + UnassignACLRole(ctx context.Context, roleName, assigneeType, assigneeID string) error + ListACLPrincipalGroups(ctx context.Context, memberType, memberID string) ([]*ACLGroupMemberInfo, error) + ListACLPrincipalRoles(ctx context.Context, assigneeType, assigneeID string) ([]*ACLRoleAssignmentInfo, error) + ExplainACLAccess(ctx context.Context, principalType, principalID, resourceType, resourceID string, requiredLevel int, callerType, callerID string) (*ACLAccessExplanationInfo, error) + // Message Flow Visualization GetMessageFlow(ctx context.Context, workspaceID string) (*MessageFlowInfo, error) @@ -184,6 +204,8 @@ type TaskFilter struct { AuthorityGrantID string `json:"authority_grant_id,omitempty"` RootAuthorityGrantID string `json:"root_authority_grant_id,omitempty"` ParentTaskID string `json:"parent_task_id,omitempty"` + Priority int32 `json:"priority,omitempty"` // exact dispatch-priority filter; 0 = no filter + MinPriority int32 `json:"min_priority,omitempty"` // minimum dispatch-priority threshold; 0 = no filter Limit int `json:"limit,omitempty"` Offset int `json:"offset,omitempty"` } @@ -193,6 +215,7 @@ type TaskInfo struct { TaskID string `json:"task_id"` TaskType string `json:"task_type"` TaskClass int32 `json:"task_class,omitempty"` + Priority int32 `json:"priority,omitempty"` DisconnectedAt *time.Time `json:"disconnected_at,omitempty"` GraceWindowMs int64 `json:"grace_window_ms,omitempty"` Status string `json:"status"` @@ -517,6 +540,102 @@ type ACLAuditLogEntryInfo struct { Metadata map[string]interface{} `json:"metadata,omitempty"` } +// ACLGroupInfo represents an ACL group +type ACLGroupInfo struct { + GroupID string `json:"group_id"` + GroupName string `json:"group_name"` + Description string `json:"description,omitempty"` + CreatedBy string `json:"created_by,omitempty"` + CreatedAt time.Time `json:"created_at"` + Metadata map[string]interface{} `json:"metadata,omitempty"` +} + +// CreateACLGroupRequest represents a request to create an ACL group +type CreateACLGroupRequest struct { + Name string `json:"name"` + Description string `json:"description,omitempty"` + CreatedBy string `json:"created_by,omitempty"` + Metadata map[string]interface{} `json:"metadata,omitempty"` +} + +// ACLGroupMemberInfo represents a group membership edge +type ACLGroupMemberInfo struct { + GroupName string `json:"group_name"` + MemberType string `json:"member_type"` + MemberID string `json:"member_id"` + GrantedBy string `json:"granted_by,omitempty"` + GrantedAt time.Time `json:"granted_at"` + ExpiresAt *time.Time `json:"expires_at,omitempty"` +} + +// AddACLGroupMemberRequest represents a request to add a member to a group +type AddACLGroupMemberRequest struct { + MemberType string `json:"member_type"` + MemberID string `json:"member_id"` + GrantedBy string `json:"granted_by,omitempty"` + ExpiresAt *time.Time `json:"expires_at,omitempty"` +} + +// ACLRoleInfo represents an ACL role +type ACLRoleInfo struct { + RoleID string `json:"role_id"` + RoleName string `json:"role_name"` + Description string `json:"description,omitempty"` + CreatedBy string `json:"created_by,omitempty"` + CreatedAt time.Time `json:"created_at"` + Metadata map[string]interface{} `json:"metadata,omitempty"` +} + +// CreateACLRoleRequest represents a request to create an ACL role +type CreateACLRoleRequest struct { + Name string `json:"name"` + Description string `json:"description,omitempty"` + CreatedBy string `json:"created_by,omitempty"` + Metadata map[string]interface{} `json:"metadata,omitempty"` +} + +// ACLRoleAssignmentInfo represents a role assignment edge +type ACLRoleAssignmentInfo struct { + RoleName string `json:"role_name"` + AssigneeType string `json:"assignee_type"` + AssigneeID string `json:"assignee_id"` + GrantedBy string `json:"granted_by,omitempty"` + GrantedAt time.Time `json:"granted_at"` + ExpiresAt *time.Time `json:"expires_at,omitempty"` +} + +// ACLAccessContributionInfo is one rule that matched the principal or one of +// its groups/roles for an explained resource. +type ACLAccessContributionInfo struct { + Subject string `json:"subject"` + RuleID string `json:"rule_id"` + AccessLevel int `json:"access_level"` + Resource string `json:"resource"` + Expired bool `json:"expired"` +} + +// ACLAccessExplanationInfo explains how a principal's effective access to a +// resource is decided: the resolved subject set, the rules that matched, and +// the resulting decision. No audit row is written for an explain. +type ACLAccessExplanationInfo struct { + Principal string `json:"principal"` + Subjects []string `json:"subjects"` + Contributions []*ACLAccessContributionInfo `json:"contributions,omitempty"` + Allowed bool `json:"allowed"` + Decision string `json:"decision"` + EffectiveLevel int `json:"effective_access_level"` + FallbackApplied bool `json:"fallback_applied"` + Reason string `json:"reason,omitempty"` +} + +// AssignACLRoleRequest represents a request to assign a role to a principal +type AssignACLRoleRequest struct { + AssigneeType string `json:"assignee_type"` + AssigneeID string `json:"assignee_id"` + GrantedBy string `json:"granted_by,omitempty"` + ExpiresAt *time.Time `json:"expires_at,omitempty"` +} + // ============================================================================= // Message Flow Visualization Types // ============================================================================= diff --git a/server/internal/audit/types.go b/server/internal/audit/types.go index 774dc89..4f12f53 100644 --- a/server/internal/audit/types.go +++ b/server/internal/audit/types.go @@ -74,18 +74,28 @@ const ( OpTunnelClosed = "tunnel_closed" // KV operations - OpKVGet = "kv_get" - OpKVPut = "kv_put" - OpKVDelete = "kv_delete" - OpKVList = "kv_list" - OpKVIncrement = "kv_increment" - OpKVDecrement = "kv_decrement" - OpKVIncrementIf = "kv_increment_if" - OpKVDecrementIf = "kv_decrement_if" + OpKVGet = "kv_get" + OpKVPut = "kv_put" + OpKVDelete = "kv_delete" + OpKVList = "kv_list" + OpKVIncrement = "kv_increment" + OpKVDecrement = "kv_decrement" + OpKVIncrementIf = "kv_increment_if" + OpKVDecrementIf = "kv_decrement_if" + OpKVSetNX = "kv_set_nx" + OpKVSetAdd = "kv_set_add" + OpKVSetCard = "kv_set_card" + OpKVCompareAndSet = "kv_compare_and_set" + OpKVCompareAndDelete = "kv_compare_and_delete" // Task operations OpTaskCreate = "task_create" OpTaskTokenIssue = "task_token_issue" // mint of a per-task auth token; checked separately from task_create because it's an authority-elevation primitive (lets the caller spawn a worker that authenticates as a declared identity) + // OpTaskAuthzDenied is emitted when authorizeTaskOp rejects a caller. + // The specific task operation (claim/complete/fail/etc.) is not threaded + // through authorizeTaskOp — the taskID, identity, and denial reason are + // the load-bearing diagnostic fields. + OpTaskAuthzDenied = "task_authz_denied" // Authority-grant lifecycle operations OpAuthorityGrantExchange = "authority_grant_exchange" diff --git a/server/internal/auth/jetstream_token_store.go b/server/internal/auth/jetstream_token_store.go new file mode 100644 index 0000000..1e56bd0 --- /dev/null +++ b/server/internal/auth/jetstream_token_store.go @@ -0,0 +1,302 @@ +// JetStreamAPITokenStore is the gateway-facing APITokenStore decorator that +// mirrors API-token mutations into a JetStream KV bucket (aether_api_tokens) +// for best-effort cross-gateway propagation, while passing every read method +// through to the inner store unchanged. +// +// Why this exists. In single-node lite the SQLite api_tokens table is the +// whole story. In cluster (horizontal-scale) mode, several gateways each run +// their own validation hot-path; a token created / revoked / deleted on one +// peer must become visible on the others quickly. This decorator publishes +// every mutation to a per-token KV key, exactly parallel to how +// internal/storage/acl.JetStreamACLRuleStore mirrors ACL-rule mutations. +// +// Scope of this decorator. This file implements the WRITE-mirror side only: +// - CreateToken: inner SQL insert (canonical) -> best-effort KV Put of a +// revocation/expiry SNAPSHOT (never the plaintext token, never the hash). +// - RevokeToken: inner SQL revoke (canonical) -> best-effort KV Put of the +// post-revoke snapshot (revoked=true). +// - DeleteToken: inner SQL delete (canonical) -> best-effort KV Delete. +// - ValidateToken / GetToken / ListTokens: pass through to the inner store +// UNCHANGED. The inner store remains the authority for validation, so +// revocation + expiry semantics are byte-for-byte identical with or +// without this decorator. The KV projection is a propagation signal for +// peer watchers, not a second source of truth. +// +// The READ side (a watcher goroutine that reflects peer token changes into a +// local cache) is intentionally NOT started by the constructor — it is the +// gateway-wiring code's responsibility, mirroring JetStreamACLRuleStore. +// +// Failure semantics. The inner SQL write is canonical. A failure to write the +// KV bucket is logged via the supplied logger and swallowed — the caller never +// sees it. The token is already durably persisted on the authoritative path, +// and peers re-bootstrap from the shared SQLite store / next watch reconnect, +// so the divergence is bounded even when the KV write fails. This matches the +// ACL rule store's contract exactly. +// +// SECURITY: the KV payload carries ONLY non-secret token metadata needed for +// propagation (id, principal_type, created_by, revoked, expires_at, ...). It +// never carries the plaintext token or the token hash — peers that need to +// validate a presented token still hash-and-look-up against their own +// canonical store. + +package auth + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "time" + + "github.com/nats-io/nats.go/jetstream" + + "github.com/scitrera/aether/internal/router/natscodec" +) + +// APITokensKVBucket is the NATS JetStream KV bucket name for API-token +// projections. Each key encodes the token's id; each value is a JSON-encoded +// non-secret snapshot of the token row. Centralized here so callers (cluster +// wiring, future read-side watch code) reference the same constant. +const APITokensKVBucket = "aether_api_tokens" + +// apiTokensKVHistory controls revision retention for the bucket. We only need +// the latest snapshot per key — a peer that reconnects after a long outage +// re-bootstraps from the canonical SQL store anyway. +const apiTokensKVHistory = 1 + +// apiTokensKVDescription is set on the bucket at create time so operators +// inspecting JetStream state see a self-explanatory description. +const apiTokensKVDescription = "Aether API token projection (cross-gateway live propagation; non-secret metadata only)." + +// JSLogger is the minimal logging surface the decorator uses for best-effort +// KV-write warnings. internal/logging.Logger satisfies it; nil is tolerated. +type JSLogger interface { + Warnf(format string, args ...any) + Errorf(format string, args ...any) +} + +// JetStreamAPITokenStore decorates an inner APITokenStore, routing every +// token-mutating method through an inner SQL write followed by a best-effort +// KV mirror write. Read methods pass through to the inner unchanged. +// +// Thread-safety: the type holds no mutable state of its own. The inner store +// and the JetStream KeyValue handle are independently goroutine-safe. +type JetStreamAPITokenStore struct { + // inner is the canonical store (SQLite/Postgres). All reads and the + // authoritative write hit it first. Declared explicitly (not embedded) + // because APITokenStore is a small, fully-overridden interface — there + // are no pass-through methods worth promoting via embedding. + inner APITokenStore + + // kv is the per-bucket KeyValue handle, opened idempotently in the + // constructor. Guaranteed non-nil post-construction. + kv jetstream.KeyValue + + // log receives best-effort warnings when a KV mirror write fails. nil is + // tolerated (warnings are dropped). + log JSLogger +} + +// Compile-time conformance assertion: the decorator must satisfy the full +// APITokenStore interface. +var _ APITokenStore = (*JetStreamAPITokenStore)(nil) + +// NewJetStreamAPITokenStore constructs the decorator. The aether_api_tokens +// KV bucket is created idempotently on construction. +// +// inner is the underlying token store. It MUST NOT be nil. +// js is the JetStream context. It MUST NOT be nil. +// replicas is the JetStream replica count (clamped to >= 1). +// logger is optional (nil tolerated). +func NewJetStreamAPITokenStore( + ctx context.Context, + inner APITokenStore, + js jetstream.JetStream, + replicas int, + logger JSLogger, +) (*JetStreamAPITokenStore, error) { + if inner == nil { + return nil, errors.New("jetstream api token store: inner is required") + } + if js == nil { + return nil, errors.New("jetstream api token store: js is required") + } + if replicas < 1 { + replicas = 1 + } + + kv, err := js.CreateOrUpdateKeyValue(ctx, jetstream.KeyValueConfig{ + Bucket: APITokensKVBucket, + Description: apiTokensKVDescription, + History: apiTokensKVHistory, + Replicas: replicas, + }) + if err != nil { + return nil, fmt.Errorf("jetstream api token store: open KV bucket %s: %w", APITokensKVBucket, err) + } + + return &JetStreamAPITokenStore{ + inner: inner, + kv: kv, + log: logger, + }, nil +} + +// --------------------------------------------------------------------------- +// Mutating methods — inner SQL write first, then best-effort KV mirror. +// --------------------------------------------------------------------------- + +// CreateToken inserts via the inner store (canonical), then mirrors the +// non-secret token snapshot into the KV bucket. On KV failure: warn + continue. +// The returned plaintext token comes from the inner result and is NEVER +// mirrored to KV. +func (s *JetStreamAPITokenStore) CreateToken(ctx context.Context, name, principalType string, workspacePatterns, scopes []string, createdBy string, expiresAt *time.Time) (*APITokenCreateResult, error) { + res, err := s.inner.CreateToken(ctx, name, principalType, workspacePatterns, scopes, createdBy, expiresAt) + if err != nil { + return nil, err + } + if res != nil { + s.publishToken(ctx, res.APIToken) + } + return res, nil +} + +// RevokeToken revokes via the inner store (canonical), then mirrors the +// post-revoke snapshot so peer watchers can invalidate any cached view. +// We re-read the token via the inner GetToken to mirror an accurate snapshot; +// if that read fails we still publish a minimal revoked marker by id. +func (s *JetStreamAPITokenStore) RevokeToken(ctx context.Context, tokenID string) error { + if err := s.inner.RevokeToken(ctx, tokenID); err != nil { + return err + } + if tok, gerr := s.inner.GetToken(ctx, tokenID); gerr == nil && tok != nil { + s.publishToken(ctx, tok) + } else { + s.publishRevokedMarker(ctx, tokenID) + } + return nil +} + +// DeleteToken hard-deletes via the inner store (canonical), then deletes the +// corresponding KV key. +func (s *JetStreamAPITokenStore) DeleteToken(ctx context.Context, tokenID string) error { + if err := s.inner.DeleteToken(ctx, tokenID); err != nil { + return err + } + s.deleteTokenKey(ctx, tokenID) + return nil +} + +// --------------------------------------------------------------------------- +// Read methods — pass through to inner UNCHANGED. +// +// Validation (and thus revocation + expiry enforcement) stays on the canonical +// store, so the decorator cannot weaken token security. The KV projection is a +// propagation signal only. +// --------------------------------------------------------------------------- + +// ValidateToken delegates to the inner canonical store. Revocation and expiry +// are enforced there exactly as in the undecorated case. +func (s *JetStreamAPITokenStore) ValidateToken(ctx context.Context, tokenStr string) (*APIToken, error) { + return s.inner.ValidateToken(ctx, tokenStr) +} + +// GetToken delegates to the inner canonical store. +func (s *JetStreamAPITokenStore) GetToken(ctx context.Context, tokenID string) (*APIToken, error) { + return s.inner.GetToken(ctx, tokenID) +} + +// ListTokens delegates to the inner canonical store. +func (s *JetStreamAPITokenStore) ListTokens(ctx context.Context, limit, offset int, includeRevoked bool) ([]*APIToken, error) { + return s.inner.ListTokens(ctx, limit, offset, includeRevoked) +} + +// --------------------------------------------------------------------------- +// KV side-effect helpers +// --------------------------------------------------------------------------- + +// apiTokenKVPayload is the JSON shape mirrored into aether_api_tokens. It is a +// non-secret subset of APIToken — deliberately excluding TokenHash and the +// plaintext token. Peer-side watchers decode into this type explicitly. +type apiTokenKVPayload struct { + ID string `json:"id"` + Name string `json:"name,omitempty"` + PrincipalType string `json:"principal_type,omitempty"` + WorkspacePatterns []string `json:"workspace_patterns,omitempty"` + Scopes []string `json:"scopes,omitempty"` + CreatedBy string `json:"created_by,omitempty"` + ExpiresAt *time.Time `json:"expires_at,omitempty"` + Revoked bool `json:"revoked"` + RevokedAt *time.Time `json:"revoked_at,omitempty"` +} + +// publishToken mirrors a single token's non-secret snapshot into the KV +// bucket. Failures are logged and swallowed. +func (s *JetStreamAPITokenStore) publishToken(ctx context.Context, tok *APIToken) { + if tok == nil || tok.ID == "" { + return + } + payload := apiTokenKVPayload{ + ID: tok.ID, + Name: tok.Name, + PrincipalType: tok.PrincipalType, + WorkspacePatterns: tok.WorkspacePatterns, + Scopes: tok.Scopes, + CreatedBy: tok.CreatedBy, + ExpiresAt: tok.ExpiresAt, + Revoked: tok.Revoked, + RevokedAt: tok.RevokedAt, + } + data, err := json.Marshal(payload) + if err != nil { + s.warnf("api token kv marshal failed for id=%s: %v", tok.ID, err) + return + } + key := tokenKVKey(tok.ID) + if _, err := s.kv.Put(ctx, key, data); err != nil { + s.warnf("api token kv put failed (key=%s): %v", key, err) + } +} + +// publishRevokedMarker mirrors a minimal revoked snapshot when the post-revoke +// inner re-read failed. Carries just the id + revoked=true so peers can still +// invalidate by id. +func (s *JetStreamAPITokenStore) publishRevokedMarker(ctx context.Context, tokenID string) { + now := time.Now().UTC() + payload := apiTokenKVPayload{ID: tokenID, Revoked: true, RevokedAt: &now} + data, err := json.Marshal(payload) + if err != nil { + s.warnf("api token kv marshal (revoked marker) failed for id=%s: %v", tokenID, err) + return + } + key := tokenKVKey(tokenID) + if _, err := s.kv.Put(ctx, key, data); err != nil { + s.warnf("api token kv put (revoked marker) failed (key=%s): %v", key, err) + } +} + +// deleteTokenKey removes a single token's KV entry. ErrKeyNotFound is benign. +func (s *JetStreamAPITokenStore) deleteTokenKey(ctx context.Context, tokenID string) { + key := tokenKVKey(tokenID) + if err := s.kv.Delete(ctx, key); err != nil { + if errors.Is(err, jetstream.ErrKeyNotFound) { + return + } + s.warnf("api token kv delete failed (key=%s): %v", key, err) + } +} + +// warnf forwards to the configured logger, tolerating nil. +func (s *JetStreamAPITokenStore) warnf(format string, args ...any) { + if s.log == nil { + return + } + s.log.Warnf(format, args...) +} + +// tokenKVKey builds the bucket key for a single token. The token id is escaped +// so any reserved character cannot break the NATS subject token. The "token." +// prefix scopes token projections within the bucket. +func tokenKVKey(tokenID string) string { + return "token." + natscodec.EscapeForKVKey(tokenID) +} diff --git a/server/internal/cleanup/audit_cleanup_test.go b/server/internal/cleanup/audit_cleanup_test.go new file mode 100644 index 0000000..69441ae --- /dev/null +++ b/server/internal/cleanup/audit_cleanup_test.go @@ -0,0 +1,102 @@ +package cleanup + +import ( + "context" + "errors" + "testing" + "time" +) + +// fakeAuditStore records the retention arg it was called with and returns a +// canned row count / error. It satisfies the cleanup.AuditStore surface. +type fakeAuditStore struct { + calledWith int + callCount int + ret int64 + err error +} + +func (f *fakeAuditStore) CleanupOldLogs(_ context.Context, retentionDays int) (int64, error) { + f.callCount++ + f.calledWith = retentionDays + return f.ret, f.err +} + +func TestCleanupOldAuditLogs_NilAuditStore(t *testing.T) { + // No audit store wired → skip cleanly with success (mirrors other + // nil-dependency skips). + service := NewService(nil, nil, nil, nil) + + result := service.CleanupOldAuditLogs(context.Background()) + + if result.JobName != "audit_log_cleanup" { + t.Errorf("JobName = %q, want %q", result.JobName, "audit_log_cleanup") + } + if !result.Success { + t.Error("Success should be true (skip) when audit store is nil") + } + if result.Error != nil { + t.Errorf("Error should be nil when audit store is nil, got %v", result.Error) + } +} + +func TestCleanupOldAuditLogs_CallsStoreWithConfiguredRetention(t *testing.T) { + fake := &fakeAuditStore{ret: 42} + service := NewService(nil, nil, nil, &Config{AuditRetentionDays: 30}) + service.SetAuditStore(fake) + + result := service.CleanupOldAuditLogs(context.Background()) + + if fake.callCount != 1 { + t.Fatalf("CleanupOldLogs call count = %d, want 1", fake.callCount) + } + if fake.calledWith != 30 { + t.Errorf("CleanupOldLogs retentionDays = %d, want 30", fake.calledWith) + } + if !result.Success { + t.Errorf("Success should be true, got error %v", result.Error) + } + if result.ItemCount != 42 { + t.Errorf("ItemCount = %d, want 42", result.ItemCount) + } +} + +func TestCleanupOldAuditLogs_DefaultsRetentionWhenUnset(t *testing.T) { + fake := &fakeAuditStore{} + // Config with zero AuditRetentionDays must fall back to 90. + service := NewService(nil, nil, nil, &Config{}) + service.SetAuditStore(fake) + + service.CleanupOldAuditLogs(context.Background()) + + if fake.calledWith != 90 { + t.Errorf("CleanupOldLogs retentionDays = %d, want 90 (default)", fake.calledWith) + } +} + +func TestCleanupOldAuditLogs_PropagatesError(t *testing.T) { + sentinel := errors.New("delete failed") + fake := &fakeAuditStore{err: sentinel} + service := NewService(nil, nil, nil, DefaultConfig()) + service.SetAuditStore(fake) + + result := service.CleanupOldAuditLogs(context.Background()) + + if result.Success { + t.Error("Success should be false when the store returns an error") + } + if !errors.Is(result.Error, sentinel) { + t.Errorf("Error = %v, want %v", result.Error, sentinel) + } +} + +func TestDefaultConfig_AuditRetentionDefaults(t *testing.T) { + config := DefaultConfig() + + if config.AuditRetentionDays != 90 { + t.Errorf("AuditRetentionDays = %d, want 90", config.AuditRetentionDays) + } + if config.AuditCleanupInterval != 24*time.Hour { + t.Errorf("AuditCleanupInterval = %v, want %v", config.AuditCleanupInterval, 24*time.Hour) + } +} diff --git a/server/internal/cleanup/badger_leader_repro_test.go b/server/internal/cleanup/badger_leader_repro_test.go new file mode 100644 index 0000000..e074f50 --- /dev/null +++ b/server/internal/cleanup/badger_leader_repro_test.go @@ -0,0 +1,264 @@ +package cleanup + +import ( + "context" + "database/sql" + "path/filepath" + "testing" + "time" + + "github.com/dgraph-io/badger/v4" + "github.com/scitrera/aether/internal/orchestration" + "github.com/scitrera/aether/internal/state" + sqlitetasks "github.com/scitrera/aether/internal/storage/tasks/sqlite" + "github.com/scitrera/aether/pkg/models" + "github.com/scitrera/aether/pkg/tasks" +) + +// newReproBadgerRegistry stands up a fresh Badger-backed session registry +// (the aetherlite-standalone leader-election backend) in a temp dir. +func newReproBadgerRegistry(t *testing.T) *state.BadgerSessionRegistry { + t.Helper() + dir := t.TempDir() + db, err := badger.Open(badger.DefaultOptions(dir).WithLogger(nil)) + if err != nil { + t.Fatalf("open badger: %v", err) + } + t.Cleanup(func() { _ = db.Close() }) + return state.NewBadgerSessionRegistry(db) +} + +// newReproSQLiteStore stands up a native-SQLite task store (the aetherlite +// task backend). Returns both the store and the raw *sql.DB so the test can +// backdate created_at to simulate a month-old task. +func newReproSQLiteStore(t *testing.T) (*sqlitetasks.Store, *sql.DB) { + t.Helper() + db, err := sql.Open("sqlite", filepath.Join(t.TempDir(), "tasks.db")) + if err != nil { + t.Fatalf("open sqlite: %v", err) + } + store, err := sqlitetasks.New(db) + if err != nil { + t.Fatalf("new sqlite store: %v", err) + } + t.Cleanup(func() { _ = db.Close() }) + return store, db +} + +// insertStaleStartupTask inserts a pending agent_startup pool task and backdates +// its created_at to `age` ago, reproducing the month-old unclaimed startup task +// observed on the live aetherlite deployment. +func insertStaleStartupTask(t *testing.T, ctx context.Context, store *sqlitetasks.Store, db *sql.DB, id string, age time.Duration) { + t.Helper() + task := &tasks.Task{ + TaskID: id, + TaskType: "agent_startup", + Workspace: "default", + AssignmentMode: tasks.AssignmentModePool, + TaskCategory: tasks.TaskCategoryOrchestrated, + TargetImplementation: "sahara", + Status: tasks.TaskStatusPending, + } + if err := store.CreateTask(ctx, task); err != nil { + t.Fatalf("create stale startup task: %v", err) + } + old := time.Now().Add(-age).UTC().Format(time.RFC3339Nano) + if _, err := db.ExecContext(ctx, "UPDATE tasks SET created_at = ? WHERE task_id = ?", old, id); err != nil { + t.Fatalf("backdate created_at: %v", err) + } +} + +// waitForStatus polls the store until the task reaches `want` or the deadline +// elapses. Returns the final observed status. +func waitForStatus(t *testing.T, ctx context.Context, store *sqlitetasks.Store, id string, want tasks.TaskStatus, timeout time.Duration) tasks.TaskStatus { + t.Helper() + deadline := time.Now().Add(timeout) + var last tasks.TaskStatus + for time.Now().Before(deadline) { + got, err := store.GetTask(ctx, id) + if err != nil { + t.Fatalf("get task: %v", err) + } + if got != nil { + last = got.Status + if got.Status == want { + return got.Status + } + } + time.Sleep(20 * time.Millisecond) + } + return last +} + +// TestStartBackground_Badger_RunsStartupSweep is the reproduction test for the +// aetherlite-standalone "cleanup never runs" failure. It stands up the exact +// single-node backends (Badger session registry + SQLite task store), seeds a +// month-old pending agent_startup task, starts the background cleanup runner, +// and asserts the stale startup sweep actually cancels it within a bounded time. +func TestStartBackground_Badger_RunsStartupSweep(t *testing.T) { + ctx := context.Background() + reg := newReproBadgerRegistry(t) + store, db := newReproSQLiteStore(t) + insertStaleStartupTask(t, ctx, store, db, "stale-startup-1", 40*24*time.Hour) + + tas := orchestration.NewTaskAssignmentService(store, nil, reg, nil, nil) + + cfg := &Config{ + // Only the startup sweep is enabled; short intervals so the test is fast. + StartupTaskTTL: 30 * time.Minute, + StartupTaskCancelInterval: 50 * time.Millisecond, + LeaderElectionRetryInterval: 20 * time.Millisecond, + } + svc := NewService(store, tas, reg, cfg) + + runner := svc.StartBackground(ctx) + defer runner.Stop() + + got := waitForStatus(t, ctx, store, "stale-startup-1", tasks.TaskStatusCancelled, 3*time.Second) + if got != tasks.TaskStatusCancelled { + t.Fatalf("stale startup task was not cancelled by background sweep; final status = %q (want %q)", got, tasks.TaskStatusCancelled) + } +} + +// TestStartBackground_Badger_ForeignLeaderLock_StarvesSweep pins the exact +// production failure mode: when the cleanup leader lock is already held by some +// other holder (a stuck/foreign lease), AcquireLock returns (false,nil) +// perpetually for the runner, startCleanupJobs never fires, and NONE of the +// sweeps run — the stale startup task lingers forever with zero cleanup logs. +func TestStartBackground_Badger_ForeignLeaderLock_StarvesSweep(t *testing.T) { + ctx := context.Background() + reg := newReproBadgerRegistry(t) + store, db := newReproSQLiteStore(t) + insertStaleStartupTask(t, ctx, store, db, "stale-startup-2", 40*24*time.Hour) + + // Simulate a stuck/foreign holder occupying the cleanup leader lock. + leaderID := models.CleanupLeaderIdentity() + acquired, err := reg.AcquireLock(ctx, leaderID, "foreign-session") + if err != nil { + t.Fatalf("seed foreign leader lock: %v", err) + } + if !acquired { + t.Fatal("expected to seed foreign leader lock") + } + + tas := orchestration.NewTaskAssignmentService(store, nil, reg, nil, nil) + cfg := &Config{ + StartupTaskTTL: 30 * time.Minute, + StartupTaskCancelInterval: 50 * time.Millisecond, + LeaderElectionRetryInterval: 20 * time.Millisecond, + } + svc := NewService(store, tas, reg, cfg) + + runner := svc.StartBackground(ctx) + defer runner.Stop() + + // With leader-election gating (SingleNode=false), the runner's AcquireLock + // returns (false,nil) for the whole window because the foreign lease holds the + // lock (LockTTL=30s >> window). startCleanupJobs never fires, so the stale + // task stays pending. This is the exact production failure mode. + got := waitForStatus(t, ctx, store, "stale-startup-2", tasks.TaskStatusCancelled, 800*time.Millisecond) + if got != tasks.TaskStatusPending { + t.Fatalf("expected stale-startup-2 to remain pending (starved by foreign leader lock), got %q", got) + } +} + +// TestStartBackground_Badger_SingleNode_RunsSweepDespiteForeignLock verifies the +// fix: with SingleNode=true, the sweeps run DIRECTLY without leader-election +// gating, so a stuck/foreign leader lease can no longer starve them. The stale +// startup task is cancelled within a bounded time even though the leader lock is +// held by another session. +func TestStartBackground_Badger_SingleNode_RunsSweepDespiteForeignLock(t *testing.T) { + ctx := context.Background() + reg := newReproBadgerRegistry(t) + store, db := newReproSQLiteStore(t) + insertStaleStartupTask(t, ctx, store, db, "stale-startup-3", 40*24*time.Hour) + + // Hold the cleanup leader lock with a foreign session — must NOT matter now. + leaderID := models.CleanupLeaderIdentity() + if acquired, err := reg.AcquireLock(ctx, leaderID, "foreign-session"); err != nil || !acquired { + t.Fatalf("seed foreign leader lock: acquired=%v err=%v", acquired, err) + } + + tas := orchestration.NewTaskAssignmentService(store, nil, reg, nil, nil) + cfg := &Config{ + SingleNode: true, // the fix: run directly, no election gate + StartupTaskTTL: 30 * time.Minute, + StartupTaskCancelInterval: 50 * time.Millisecond, + } + svc := NewService(store, tas, reg, cfg) + + runner := svc.StartBackground(ctx) + defer runner.Stop() + + got := waitForStatus(t, ctx, store, "stale-startup-3", tasks.TaskStatusCancelled, 3*time.Second) + if got != tasks.TaskStatusCancelled { + t.Fatalf("single-node sweep did not cancel stale startup task despite foreign leader lock; final status = %q", got) + } +} + +// TestCancelStalePoolTasks_SweepsOnlyStaleRegularPool exercises the new pool +// sweep end-to-end against the native SQLite store: a stale regular pool task is +// cancelled, while a fresh regular pool task and a stale agent_startup pool task +// (owned by the startup sweep) are left untouched. +func TestCancelStalePoolTasks_SweepsOnlyStaleRegularPool(t *testing.T) { + ctx := context.Background() + store, db := newReproSQLiteStore(t) + + // Stale regular pool task — should be cancelled. + stalePool := &tasks.Task{ + TaskID: "pool-stale", + TaskType: "some_pool_job", + Workspace: "default", + AssignmentMode: tasks.AssignmentModePool, + TaskCategory: tasks.TaskCategoryRegular, + TargetImplementation: "worker", + Status: tasks.TaskStatusPending, + } + if err := store.CreateTask(ctx, stalePool); err != nil { + t.Fatalf("create stale pool task: %v", err) + } + old := time.Now().Add(-2 * time.Hour).UTC().Format(time.RFC3339Nano) + if _, err := db.ExecContext(ctx, "UPDATE tasks SET created_at = ? WHERE task_id = ?", old, "pool-stale"); err != nil { + t.Fatalf("backdate pool-stale: %v", err) + } + + // Fresh regular pool task — should be left pending (younger than TTL). + freshPool := &tasks.Task{ + TaskID: "pool-fresh", + TaskType: "some_pool_job", + Workspace: "default", + AssignmentMode: tasks.AssignmentModePool, + TaskCategory: tasks.TaskCategoryRegular, + TargetImplementation: "worker", + Status: tasks.TaskStatusPending, + } + if err := store.CreateTask(ctx, freshPool); err != nil { + t.Fatalf("create fresh pool task: %v", err) + } + + // Stale agent_startup pool task (orchestrated) — owned by the startup sweep, + // must be excluded by the regular-category filter. + insertStaleStartupTask(t, ctx, store, db, "startup-stale", 40*24*time.Hour) + + tas := orchestration.NewTaskAssignmentService(store, nil, nil, nil, nil) + n, err := tas.CancelStalePoolTasks(ctx, 1*time.Hour) + if err != nil { + t.Fatalf("CancelStalePoolTasks: %v", err) + } + if n != 1 { + t.Fatalf("CancelStalePoolTasks cancelled %d tasks, want 1", n) + } + + assertStatus := func(id string, want tasks.TaskStatus) { + got, err := store.GetTask(ctx, id) + if err != nil { + t.Fatalf("get %s: %v", id, err) + } + if got.Status != want { + t.Errorf("%s status = %q, want %q", id, got.Status, want) + } + } + assertStatus("pool-stale", tasks.TaskStatusCancelled) + assertStatus("pool-fresh", tasks.TaskStatusPending) + assertStatus("startup-stale", tasks.TaskStatusPending) +} diff --git a/server/internal/cleanup/service.go b/server/internal/cleanup/service.go index 1adac0a..30ce638 100644 --- a/server/internal/cleanup/service.go +++ b/server/internal/cleanup/service.go @@ -10,6 +10,7 @@ package cleanup import ( "context" "fmt" + "runtime/debug" "sync" "time" @@ -32,11 +33,40 @@ type Config struct { // Reconciliation settings ReconciliationInterval time.Duration // How often to run orphaned task reconciliation (0 = disabled) + // Orphaned queue-entry reconciliation settings + QueueReconcileInterval time.Duration // How often to retire orphaned orchestrated_task_queue rows whose task is terminal/missing (0 = disabled) + + // Stale interactive task settings + InteractiveTaskTTL time.Duration // How old a non-terminal INTERACTIVE (chat turn) task may get before being auto-cancelled + InteractiveTaskCancelInterval time.Duration // How often to run the stale interactive task sweep (0 = disabled) + + // Stale startup task settings + StartupTaskTTL time.Duration // How old an UNCLAIMED (pending) agent_startup task may get before being auto-cancelled (generous, so a briefly-unavailable orchestrator's pending task is never reaped) + StartupTaskCancelInterval time.Duration // How often to run the stale startup task sweep (0 = disabled) + + // Stale pool task settings + PoolTaskTTL time.Duration // How old an UNCLAIMED (pending) regular POOL task may get before being auto-cancelled (generous, so a briefly-absent worker's pending task is never reaped) + PoolTaskCancelInterval time.Duration // How often to run the stale pool task sweep (0 = disabled) + + // Audit-log retention settings + AuditRetentionDays int // Delete comprehensive_audit_log rows older than this many days (default 90) + AuditCleanupInterval time.Duration // How often to run the scheduled audit-log retention sweep (0 = disabled) + // Stale claim settings StaleClaimTimeout time.Duration // How long a task can stay 'claimed' before being recovered (0 = use default 5m) // Leader election settings LeaderElectionRetryInterval time.Duration // How often to retry acquiring leadership if not leader (0 = use default 30s) + + // SingleNode indicates a single-node/standalone deployment (aetherlite + // standalone / polling-dispatcher path). When true, StartBackground runs the + // periodic sweeps DIRECTLY without gating on distributed leader election — + // there is only one node, so election adds no safety and only risk: a + // stuck/foreign leader lease (any AcquireLock returning false) would silently + // starve every sweep with no recovery and no log signal. Clustered backends + // (Redis / JetStream) leave this false and keep lease-based election so + // exactly one node sweeps. + SingleNode bool } // DefaultConfig returns the default cleanup configuration @@ -46,8 +76,17 @@ func DefaultConfig() *Config { CompletedTaskRetention: 7 * 24 * time.Hour, FailedTaskRetention: 14 * 24 * time.Hour, CancelledTaskRetention: 7 * 24 * time.Hour, - ReconciliationInterval: 1 * time.Minute, - StaleClaimTimeout: 5 * time.Minute, + ReconciliationInterval: 1 * time.Minute, + QueueReconcileInterval: 5 * time.Minute, + InteractiveTaskTTL: 1 * time.Hour, + InteractiveTaskCancelInterval: 15 * time.Minute, + StartupTaskTTL: 30 * time.Minute, + StartupTaskCancelInterval: 15 * time.Minute, + PoolTaskTTL: 1 * time.Hour, + PoolTaskCancelInterval: 15 * time.Minute, + AuditRetentionDays: 90, + AuditCleanupInterval: 24 * time.Hour, + StaleClaimTimeout: 5 * time.Minute, LeaderElectionRetryInterval: 30 * time.Second, } } @@ -76,6 +115,18 @@ type SessionRegistry interface { var _ SessionRegistry = (*state.SessionRegistry)(nil) var _ SessionRegistry = (*state.BadgerSessionRegistry)(nil) +// AuditStore is the narrow surface the cleanup service needs to enforce audit- +// log retention: a single delete-old-rows call. The audit domain Store +// (internal/storage/audit) satisfies this across all three backends (sqlite, +// postgres, jetstream). Declared as a local interface — mirroring the +// SessionRegistry pattern above — so the cleanup package does not take a hard +// dependency on the audit storage package. +type AuditStore interface { + // CleanupOldLogs deletes audit rows older than retentionDays and returns + // the number of rows removed. + CleanupOldLogs(ctx context.Context, retentionDays int) (int64, error) +} + // Service provides cleanup operations for the gateway. // It can be used for both background goroutines and standalone cleanup commands. type Service struct { @@ -84,6 +135,7 @@ type Service struct { taskService *orchestration.TaskAssignmentService dispatcher orchestration.TaskDispatcher sessionRegistry SessionRegistry + auditStore AuditStore config *Config } @@ -110,6 +162,14 @@ func (s *Service) SetDispatcher(dispatcher orchestration.TaskDispatcher) { s.dispatcher = dispatcher } +// SetAuditStore sets the audit store used by the scheduled audit-retention job. +// Passing nil leaves audit cleanup disabled (the job skips with success). A +// setter (rather than a NewService param) keeps the many existing NewService +// call sites unchanged. +func (s *Service) SetAuditStore(auditStore AuditStore) { + s.auditStore = auditStore +} + // JobResult contains the result of a cleanup job type JobResult struct { JobName string @@ -137,10 +197,30 @@ func (s *Service) RunAllJobs(ctx context.Context) []JobResult { result = s.ReconcileOrphanedTasks(ctx) results = append(results, result) + // Run orphaned orchestrated_task_queue reconciliation + result = s.ReconcileOrphanedQueueEntries(ctx) + results = append(results, result) + + // Run stale interactive task cancellation + result = s.CancelStaleInteractiveTasks(ctx) + results = append(results, result) + + // Run stale startup task cancellation + result = s.CancelStaleStartupTasks(ctx) + results = append(results, result) + + // Run stale pool task cancellation + result = s.CancelStalePoolTasks(ctx) + results = append(results, result) + // Run task purge result = s.PurgeTasks(ctx) results = append(results, result) + // Run audit-log retention cleanup + result = s.CleanupOldAuditLogs(ctx) + results = append(results, result) + return results } @@ -203,6 +283,141 @@ func (s *Service) ReconcileOrphanedTasks(ctx context.Context) JobResult { return result } +// ReconcileOrphanedQueueEntries retires orchestrated_task_queue rows that were +// orphaned when their task went terminal (or was purged) without the +// best-effort, dispatcher-gated retire firing. Without this sweep such rows sit +// pending forever and the polling dispatcher polls the ghosts every tick. The +// store method is idempotent and a no-op in clustered/JetStream mode (no SQL +// queue rows), so this job is safe on every backend. +func (s *Service) ReconcileOrphanedQueueEntries(ctx context.Context) JobResult { + start := time.Now() + result := JobResult{JobName: "orphaned_queue_entry_reconciliation"} + + if s.taskStore == nil { + result.Error = fmt.Errorf("task store not configured") + return result + } + + count, err := s.taskStore.ReconcileOrphanedQueueEntries(ctx) + result.Duration = time.Since(start) + + if err != nil { + result.Error = err + return result + } + + result.Success = true + result.ItemCount = count + if count > 0 { + result.Details = fmt.Sprintf("retired %d orphaned queue entries", count) + } else { + result.Details = "no orphaned queue entries found" + } + + return result +} + +// CancelStaleInteractiveTasks cancels non-terminal INTERACTIVE (chat turn) +// tasks older than the configured TTL. These are foreground turns that never +// reached a terminal state (dead/offline sandbox at mint, crashed harness) and +// would otherwise linger forever. +func (s *Service) CancelStaleInteractiveTasks(ctx context.Context) JobResult { + start := time.Now() + result := JobResult{JobName: "interactive_task_ttl_cancel"} + + if s.taskService == nil { + result.Error = fmt.Errorf("task service not configured") + return result + } + + count, err := s.taskService.CancelStaleInteractiveTasks(ctx, s.config.InteractiveTaskTTL) + result.Duration = time.Since(start) + + if err != nil { + result.Error = err + return result + } + + result.Success = true + result.ItemCount = int64(count) + if count > 0 { + result.Details = fmt.Sprintf("cancelled %d stale interactive tasks (ttl: %v)", count, s.config.InteractiveTaskTTL) + } else { + result.Details = "no stale interactive tasks found" + } + + return result +} + +// CancelStaleStartupTasks cancels UNCLAIMED (pending) agent_startup orchestration +// tasks older than the configured TTL. These are pool tasks minted to spin up an +// offline agent that no orchestrator ever claimed; cancelling them is +// self-healing (the next message re-triggers a fresh startup task) and the TTL +// is generous so a briefly-unavailable orchestrator's pending task is never +// reaped out from under it. +func (s *Service) CancelStaleStartupTasks(ctx context.Context) JobResult { + start := time.Now() + result := JobResult{JobName: "startup_task_ttl_cancel"} + + if s.taskService == nil { + result.Error = fmt.Errorf("task service not configured") + return result + } + + count, err := s.taskService.CancelStaleStartupTasks(ctx, s.config.StartupTaskTTL) + result.Duration = time.Since(start) + + if err != nil { + result.Error = err + return result + } + + result.Success = true + result.ItemCount = int64(count) + if count > 0 { + result.Details = fmt.Sprintf("cancelled %d stale startup tasks (ttl: %v)", count, s.config.StartupTaskTTL) + } else { + result.Details = "no stale startup tasks found" + } + + return result +} + +// CancelStalePoolTasks cancels UNCLAIMED (pending) regular POOL orchestration +// tasks older than the configured TTL. These are pool tasks whose target- +// implementation worker never connected to claim them; unlike agent_startup pool +// tasks (covered by CancelStaleStartupTasks) and active INTERACTIVE turns +// (covered by CancelStaleInteractiveTasks), nothing else collects them, so they +// linger forever. The TTL is generous so a briefly-absent worker's pending task +// is never reaped out from under it. +func (s *Service) CancelStalePoolTasks(ctx context.Context) JobResult { + start := time.Now() + result := JobResult{JobName: "pool_task_ttl_cancel"} + + if s.taskService == nil { + result.Error = fmt.Errorf("task service not configured") + return result + } + + count, err := s.taskService.CancelStalePoolTasks(ctx, s.config.PoolTaskTTL) + result.Duration = time.Since(start) + + if err != nil { + result.Error = err + return result + } + + result.Success = true + result.ItemCount = int64(count) + if count > 0 { + result.Details = fmt.Sprintf("cancelled %d stale pool tasks (ttl: %v)", count, s.config.PoolTaskTTL) + } else { + result.Details = "no stale pool tasks found" + } + + return result +} + // CleanupStaleClaims recovers orchestration tasks stuck in 'claimed' status. // This handles gateway crashes that leave tasks claimed but never delivered. func (s *Service) CleanupStaleClaims(ctx context.Context) JobResult { @@ -274,6 +489,52 @@ func (s *Service) PurgeTasks(ctx context.Context) JobResult { return result } +// CleanupOldAuditLogs deletes comprehensive_audit_log rows older than the +// configured retention window (default 90 days). +// +// Note on space reclamation: on SQLite (aetherlite) a DELETE frees the pages +// for reuse — this caps growth / plateaus audit.db but does NOT shrink the file +// on disk; reclaiming already-allocated space requires a manual VACUUM. We do +// NOT auto-VACUUM here because VACUUM takes an exclusive lock and rewrites the +// whole DB, which would stall the gateway. With a 90-day window nothing is +// deleted until rows age past 90 days, so on a young database this is a no-op. +func (s *Service) CleanupOldAuditLogs(ctx context.Context) JobResult { + start := time.Now() + result := JobResult{JobName: "audit_log_cleanup"} + + // Nil-audit-store guard: audit cleanup is optional wiring. Skip cleanly + // rather than error so RunAllJobs / periodic runs stay quiet when no audit + // store is configured. + if s.auditStore == nil { + result.Success = true + result.Details = "skipped (no audit store)" + return result + } + + retentionDays := s.config.AuditRetentionDays + if retentionDays <= 0 { + retentionDays = 90 + } + + count, err := s.auditStore.CleanupOldLogs(ctx, retentionDays) + result.Duration = time.Since(start) + + if err != nil { + result.Error = err + return result + } + + result.Success = true + result.ItemCount = count + if count > 0 { + result.Details = fmt.Sprintf("deleted %d audit log rows older than %d days", count, retentionDays) + } else { + result.Details = fmt.Sprintf("no audit log rows older than %d days", retentionDays) + } + + return result +} + // BackgroundRunner manages periodic execution of cleanup jobs. // It can be stopped by canceling the provided context or calling Stop(). // Uses leader election to ensure only one instance runs cleanup jobs across @@ -304,6 +565,20 @@ func (s *Service) StartBackground(ctx context.Context) *BackgroundRunner { identity: models.CleanupLeaderIdentity(), } + // Single-node/standalone deployments (aetherlite standalone / polling- + // dispatcher path) must ALWAYS run the sweeps. There is exactly one node, so + // distributed leader election adds no safety and only risk: a stuck/foreign + // leader lease (any AcquireLock returning false) would silently starve every + // sweep with no recovery and no log signal (the exact failure reproduced in + // TestStartBackground_Badger_ForeignLeaderLock_StarvesSweep). Run the + // periodic jobs directly. Clustered backends fall through to lease-based + // election below so exactly one node sweeps. + if s.config.SingleNode { + logging.Logger.Info().Msg("cleanup: single-node mode — running jobs directly without leader election") + runner.startCleanupJobs(ctx) + return runner + } + // Check if leader election is possible (requires session registry) if s.sessionRegistry == nil { logging.Logger.Warn().Msg("no session registry available, running cleanup jobs without leader election") @@ -398,6 +673,71 @@ func (r *BackgroundRunner) startCleanupJobs(parentCtx context.Context) { }) } + // Start stale interactive task cancellation if enabled + if s.config.InteractiveTaskCancelInterval > 0 { + go r.runPeriodic(cleanupCtx, "interactive_task_ttl_cancel", s.config.InteractiveTaskCancelInterval, func(ctx context.Context) { + result := s.CancelStaleInteractiveTasks(ctx) + if result.Error != nil { + logging.Logger.Error().Err(result.Error).Msg("stale interactive task cancel error") + } else if result.ItemCount > 0 { + logging.Logger.Info().Str("details", result.Details).Msg("stale interactive task cancel completed") + } + }) + } + + // Start stale startup task cancellation if enabled + if s.config.StartupTaskCancelInterval > 0 { + go r.runPeriodic(cleanupCtx, "startup_task_ttl_cancel", s.config.StartupTaskCancelInterval, func(ctx context.Context) { + result := s.CancelStaleStartupTasks(ctx) + if result.Error != nil { + logging.Logger.Error().Err(result.Error).Msg("stale startup task cancel error") + } else if result.ItemCount > 0 { + logging.Logger.Info().Str("details", result.Details).Msg("stale startup task cancel completed") + } + }) + } + + // Start stale pool task cancellation if enabled + if s.config.PoolTaskCancelInterval > 0 { + go r.runPeriodic(cleanupCtx, "pool_task_ttl_cancel", s.config.PoolTaskCancelInterval, func(ctx context.Context) { + result := s.CancelStalePoolTasks(ctx) + if result.Error != nil { + logging.Logger.Error().Err(result.Error).Msg("stale pool task cancel error") + } else if result.ItemCount > 0 { + logging.Logger.Info().Str("details", result.Details).Msg("stale pool task cancel completed") + } + }) + } + + // Start scheduled audit-log retention if enabled and an audit store is + // wired. Runs on the same single-node-direct / leader-gated path as the + // other sweeps. Backend-agnostic: the store's CleanupOldLogs is implemented + // for sqlite, postgres, and jetstream. + if s.config.AuditCleanupInterval > 0 && s.auditStore != nil { + go r.runPeriodic(cleanupCtx, "audit_log_cleanup", s.config.AuditCleanupInterval, func(ctx context.Context) { + result := s.CleanupOldAuditLogs(ctx) + if result.Error != nil { + logging.Logger.Error().Err(result.Error).Msg("audit log cleanup error") + } else if result.ItemCount > 0 { + logging.Logger.Info().Str("details", result.Details).Msg("audit log cleanup completed") + } + }) + } + + // Start orphaned queue-entry reconciliation if enabled. This runs under the + // same single-node-direct / leader-gated path as the other sweeps and is a + // no-op on clustered/JetStream backends (no SQL orchestrated_task_queue rows). + if s.config.QueueReconcileInterval > 0 { + go r.runPeriodic(cleanupCtx, "orphaned_queue_entry_reconciliation", s.config.QueueReconcileInterval, func(ctx context.Context) { + result := s.ReconcileOrphanedQueueEntries(ctx) + if result.Error != nil { + logging.Logger.Error().Err(result.Error).Msg("orphaned queue entry reconciliation error") + } else if result.ItemCount > 0 { + logging.Logger.Info().Str("details", result.Details).Msg("orphaned queue entry reconciliation completed") + } + }) + } + // Start reconciliation if enabled if s.config.ReconciliationInterval > 0 { go r.runPeriodic(cleanupCtx, "reconciliation", s.config.ReconciliationInterval, func(ctx context.Context) { @@ -447,17 +787,36 @@ func (r *BackgroundRunner) setLeader(leader bool) { } } -// runPeriodic runs a job periodically until context is cancelled +// runPeriodic runs a job periodically until context is cancelled. +// +// Each iteration is isolated behind panic recovery: a single failing sweep is +// logged (with stack) and skipped rather than killing the loop. This matters +// doubly because an unrecovered panic in this background goroutine would +// otherwise crash the whole gateway process. Mirrors the recover()+debug.Stack() +// guard on the background goroutine in gateway.SetCleanupService. func (r *BackgroundRunner) runPeriodic(ctx context.Context, name string, interval time.Duration, job func(context.Context)) { ticker := time.NewTicker(interval) defer ticker.Stop() + runOnce := func() { + defer func() { + if rec := recover(); rec != nil { + logging.Logger.Error(). + Interface("panic", rec). + Str("stack", string(debug.Stack())). + Str("job", name). + Msg("recovered from panic in cleanup job iteration") + } + }() + job(ctx) + } + for { select { case <-ctx.Done(): return case <-ticker.C: - job(ctx) + runOnce() } } } diff --git a/server/internal/cleanup/service_test.go b/server/internal/cleanup/service_test.go index f3d2165..7fb715b 100644 --- a/server/internal/cleanup/service_test.go +++ b/server/internal/cleanup/service_test.go @@ -2,8 +2,18 @@ package cleanup import ( "context" + "database/sql" + "path/filepath" "testing" "time" + + "github.com/google/uuid" + taskstore "github.com/scitrera/aether/internal/storage/tasks" + taskssqlite "github.com/scitrera/aether/internal/storage/tasks/sqlite" + + // Register the bare "sqlite" driver so the reconcile-job wiring test can run + // against the native sqlite store (always available, never skipped). + _ "modernc.org/sqlite" ) func TestDefaultConfig(t *testing.T) { @@ -25,6 +35,18 @@ func TestDefaultConfig(t *testing.T) { if config.ReconciliationInterval != 1*time.Minute { t.Errorf("ReconciliationInterval = %v, want %v", config.ReconciliationInterval, 1*time.Minute) } + if config.StartupTaskTTL != 30*time.Minute { + t.Errorf("StartupTaskTTL = %v, want %v", config.StartupTaskTTL, 30*time.Minute) + } + if config.StartupTaskCancelInterval != 15*time.Minute { + t.Errorf("StartupTaskCancelInterval = %v, want %v", config.StartupTaskCancelInterval, 15*time.Minute) + } + if config.PoolTaskTTL != 1*time.Hour { + t.Errorf("PoolTaskTTL = %v, want %v", config.PoolTaskTTL, 1*time.Hour) + } + if config.PoolTaskCancelInterval != 15*time.Minute { + t.Errorf("PoolTaskCancelInterval = %v, want %v", config.PoolTaskCancelInterval, 15*time.Minute) + } if config.StaleClaimTimeout != 5*time.Minute { t.Errorf("StaleClaimTimeout = %v, want %v", config.StaleClaimTimeout, 5*time.Minute) } @@ -153,15 +175,17 @@ func TestRunAllJobs_NilDependencies(t *testing.T) { results := service.RunAllJobs(ctx) - // Should return 4 results (stale locks, stale claims, orphaned tasks, task purge) - if len(results) != 4 { - t.Fatalf("RunAllJobs() returned %d results, want 4", len(results)) + // Should return 9 results (stale locks, stale claims, orphaned tasks, + // orphaned queue entries, interactive-task TTL cancel, startup-task TTL + // cancel, pool-task TTL cancel, task purge, audit-log cleanup) + if len(results) != 9 { + t.Fatalf("RunAllJobs() returned %d results, want 9", len(results)) } // All should fail or skip due to nil dependencies for _, result := range results { // These jobs gracefully skip when their dependency is nil - if result.JobName == "stale_claim_recovery" || result.JobName == "stale_lock_cleanup" { + if result.JobName == "stale_claim_recovery" || result.JobName == "stale_lock_cleanup" || result.JobName == "audit_log_cleanup" { if !result.Success { t.Errorf("Job %q should succeed (skip) when dependencies are nil", result.JobName) } @@ -268,3 +292,57 @@ func TestConfig_ZeroValues(t *testing.T) { t.Error("ReconciliationInterval should be 0") } } + +// TestReconcileOrphanedQueueEntries_Job verifies the cleanup-service wiring: +// the ReconcileOrphanedQueueEntries JobResult retires orphaned queue rows via +// the store and reports the count. Runs against the native sqlite store. +func TestReconcileOrphanedQueueEntries_Job(t *testing.T) { + dbPath := filepath.Join(t.TempDir(), "cleanup_tasks.db") + db, err := sql.Open("sqlite", dbPath+"?_journal_mode=WAL&_busy_timeout=5000") + if err != nil { + t.Fatalf("sql.Open sqlite: %v", err) + } + t.Cleanup(func() { _ = db.Close() }) + store, err := taskssqlite.New(db) + if err != nil { + t.Fatalf("taskssqlite.New: %v", err) + } + ctx := context.Background() + + // One cancelled (terminal) task with a pending queue row -> orphaned. + termID := uuid.New().String() + if err := store.CreateTask(ctx, &taskstore.Task{TaskID: termID, TaskType: "agent_startup", Workspace: "ws"}); err != nil { + t.Fatalf("CreateTask: %v", err) + } + if err := store.CancelTask(ctx, termID); err != nil { + t.Fatalf("CancelTask: %v", err) + } + if err := store.InsertQueueEntry(ctx, uuid.New().String(), termID, "impl", "ws", "local", nil, int(taskstore.PriorityNormal)); err != nil { + t.Fatalf("InsertQueueEntry: %v", err) + } + + service := NewService(store, nil, nil, nil) + result := service.ReconcileOrphanedQueueEntries(ctx) + + if result.JobName != "orphaned_queue_entry_reconciliation" { + t.Errorf("JobName = %q, want %q", result.JobName, "orphaned_queue_entry_reconciliation") + } + if !result.Success { + t.Errorf("Success = false, err = %v", result.Error) + } + if result.ItemCount != 1 { + t.Errorf("ItemCount = %d, want 1", result.ItemCount) + } +} + +// TestReconcileOrphanedQueueEntries_NilTaskStore guards the nil-store branch. +func TestReconcileOrphanedQueueEntries_NilTaskStore(t *testing.T) { + service := NewService(nil, nil, nil, nil) + result := service.ReconcileOrphanedQueueEntries(context.Background()) + if result.Success { + t.Error("Success should be false when task store is nil") + } + if result.Error == nil { + t.Error("Error should not be nil when task store is nil") + } +} diff --git a/server/internal/cluster/integration/authority_lifecycle_test.go b/server/internal/cluster/integration/authority_lifecycle_test.go index dba3ebd..1af1b30 100644 --- a/server/internal/cluster/integration/authority_lifecycle_test.go +++ b/server/internal/cluster/integration/authority_lifecycle_test.go @@ -335,7 +335,7 @@ func setupAuthorityRelay(t *testing.T) *authorityRelayHarness { es := setupCluster1(t) js := es.JetStream() - startCtx, cancel := context.WithTimeout(context.Background(), 15*time.Second) + startCtx, cancel := context.WithTimeout(context.Background(), scaled(15*time.Second)) defer cancel() inner := newFakeAuthLifecycle() @@ -384,7 +384,7 @@ func setupAuthorityRelay(t *testing.T) *authorityRelayHarness { cancelRun() select { case <-done: - case <-time.After(5 * time.Second): + case <-time.After(scaled(5 * time.Second)): t.Fatalf("waker did not shut down within 5s") } } @@ -477,7 +477,7 @@ func TestAuthorityLifecycle_EndToEnd(t *testing.T) { h := setupAuthorityRelay(t) defer h.stopWaker() - ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) + ctx, cancel := context.WithTimeout(context.Background(), scaled(15*time.Second)) defer cancel() const ( @@ -504,7 +504,7 @@ func TestAuthorityLifecycle_EndToEnd(t *testing.T) { t.Fatalf("resolved status = %q want approved", resolved.Status) } - got := waitForResume(t, h.svc, taskID, 5*time.Second) + got := waitForResume(t, h.svc, taskID, scaled(5*time.Second)) if got.to != tasks.TaskStatusRunning { t.Errorf("resume target = %q want %q", got.to, tasks.TaskStatusRunning) } @@ -518,7 +518,7 @@ func TestAuthorityLifecycle_EndToEnd(t *testing.T) { h := setupAuthorityRelay(t) defer h.stopWaker() - ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) + ctx, cancel := context.WithTimeout(context.Background(), scaled(15*time.Second)) defer cancel() const ( @@ -536,7 +536,7 @@ func TestAuthorityLifecycle_EndToEnd(t *testing.T) { t.Fatalf("deny: %v", err) } - got := waitForFail(t, h.svc, taskID, 5*time.Second) + got := waitForFail(t, h.svc, taskID, scaled(5*time.Second)) // authorityEventFailureReason composes either // "authority request denied: " (when ResolutionReason is // present on the embedded request payload) or @@ -558,7 +558,7 @@ func TestAuthorityLifecycle_EndToEnd(t *testing.T) { h := setupAuthorityRelay(t) defer h.stopWaker() - ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) + ctx, cancel := context.WithTimeout(context.Background(), scaled(15*time.Second)) defer cancel() const ( @@ -575,7 +575,7 @@ func TestAuthorityLifecycle_EndToEnd(t *testing.T) { t.Fatalf("cancel: %v", err) } - got := waitForFail(t, h.svc, taskID, 5*time.Second) + got := waitForFail(t, h.svc, taskID, scaled(5*time.Second)) if !containsAll(got.reason, "cancelled") { t.Errorf("fail reason = %q want substring 'cancelled'", got.reason) } @@ -593,7 +593,7 @@ func TestAuthorityLifecycle_EndToEnd(t *testing.T) { h := setupAuthorityRelay(t) defer h.stopWaker() - ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) + ctx, cancel := context.WithTimeout(context.Background(), scaled(15*time.Second)) defer cancel() const workspace = "ws-watch" @@ -674,7 +674,7 @@ func TestAuthorityLifecycle_EndToEnd(t *testing.T) { eventType acl.AuthorityRequestEventType } var got []seen - deadline := time.After(8 * time.Second) + deadline := time.After(scaled(8 * time.Second)) for len(got) < 6 { select { case evt := <-received: @@ -732,7 +732,7 @@ func TestAuthorityLifecycle_EventPayloadShape(t *testing.T) { h := setupAuthorityRelay(t) defer h.stopWaker() - ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) + ctx, cancel := context.WithTimeout(context.Background(), scaled(15*time.Second)) defer cancel() const workspace = "ws-shape" @@ -782,7 +782,7 @@ func TestAuthorityLifecycle_EventPayloadShape(t *testing.T) { } // Wait for both events to land at the inspector. - deadline := time.Now().Add(5 * time.Second) + deadline := time.Now().Add(scaled(5 * time.Second)) for rawCreated.Load() == nil || rawApproved.Load() == nil { if time.Now().After(deadline) { t.Fatalf("did not receive both created+approved within 5s") diff --git a/server/internal/cluster/integration/backup_roundtrip_test.go b/server/internal/cluster/integration/backup_roundtrip_test.go index 7c2a8dd..0d84cf7 100644 --- a/server/internal/cluster/integration/backup_roundtrip_test.go +++ b/server/internal/cluster/integration/backup_roundtrip_test.go @@ -36,7 +36,7 @@ func (l testLogger) Errorf(format string, args ...any) { l.t.Logf("ERR "+format // 5. Assert the data matches the original seed. func TestClusterIntegration_BackupRoundtrip_PerDomain(t *testing.T) { c := setupCluster3(t) - ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + ctx, cancel := context.WithTimeout(context.Background(), scaled(30*time.Second)) defer cancel() // Use node-0 as the leader/operator for the backup coordinator. The @@ -131,11 +131,15 @@ func TestClusterIntegration_BackupRoundtrip_PerDomain(t *testing.T) { runCancel() select { case <-runDone: - case <-time.After(5 * time.Second): + case <-time.After(scaled(5 * time.Second)): } }) - deadline := time.Now().Add(15 * time.Second) + // Wait for BOTH domains to produce a .bin before cancelling the coordinator; + // cancelling mid-upload of the second domain (context canceled) is what made + // this flake under CI's 2-core load. The loop breaks the instant both are + // present, so a generous deadline costs a passing run nothing. + deadline := time.Now().Add(scaled(15 * time.Second)) for time.Now().Before(deadline) { streamObjs, _ := storage.List(ctx, fmt.Sprintf("cluster-rt/%s", streamName)) kvObjs, _ := storage.List(ctx, fmt.Sprintf("cluster-rt/%s", kvBucket)) @@ -191,7 +195,7 @@ func TestClusterIntegration_BackupRoundtrip_PerDomain(t *testing.T) { } gotStream := make(map[string]string) - streamDeadline := time.Now().Add(5 * time.Second) + streamDeadline := time.Now().Add(scaled(5 * time.Second)) for len(gotStream) < len(wantStream) && time.Now().Before(streamDeadline) { msg, err := cons.Next(jetstream.FetchMaxWait(500 * time.Millisecond)) if err != nil { diff --git a/server/internal/cluster/integration/chat_task_workflow_test.go b/server/internal/cluster/integration/chat_task_workflow_test.go index f37bf8a..e93cceb 100644 --- a/server/internal/cluster/integration/chat_task_workflow_test.go +++ b/server/internal/cluster/integration/chat_task_workflow_test.go @@ -142,7 +142,7 @@ func setupCluster1(t *testing.T) *clusternats.EmbeddedServer { ClientPort: -1, // ephemeral HAMode: clusternats.HAModeAuto, } - startCtx, cancel := context.WithTimeout(context.Background(), 20*time.Second) + startCtx, cancel := context.WithTimeout(context.Background(), scaled(20*time.Second)) defer cancel() if err := es.Start(startCtx, cfg); err != nil { t.Fatalf("start embedded NATS: %v", err) @@ -177,7 +177,7 @@ func buildBackends(t *testing.T, es *clusternats.EmbeddedServer, taskStore *fake js := es.JetStream() const replicas = 1 - ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) + ctx, cancel := context.WithTimeout(context.Background(), scaled(20*time.Second)) defer cancel() r, err := router.NewJetStreamRouter(js, replicas, nil) @@ -249,7 +249,7 @@ func TestClusterIntegration_ChatTaskWorkflow_EndToEnd(t *testing.T) { const gatewayID = "chat-task-gw" b := buildBackends(t, es, taskStore, gatewayID) - ctx, cancel := context.WithTimeout(context.Background(), 55*time.Second) + ctx, cancel := context.WithTimeout(context.Background(), scaled(55*time.Second)) defer cancel() // ----- Phase 1: service principals connect ----- @@ -441,7 +441,7 @@ func TestClusterIntegration_ChatTaskWorkflow_EndToEnd(t *testing.T) { select { case <-taskDelivered: t.Log("phase 6: dispatcher fired; assignment delivered to orchestrator") - case <-time.After(10 * time.Second): + case <-time.After(scaled(10 * time.Second)): t.Fatal("phase 6: dispatcher callback never fired within 10s") } got, _ := receivedTask.Load().(*orchestration.OrchestrationTaskNotification) @@ -532,7 +532,7 @@ func TestClusterIntegration_ChatTaskWorkflow_EndToEnd(t *testing.T) { agentGrantID = env.GrantID agentPayload = env.Payload _ = agentPayload // payload would feed the agent's LLM call in production - case <-time.After(5 * time.Second): + case <-time.After(scaled(5 * time.Second)): t.Fatal("phase 8: agent did not receive the pre-startup message within 5s (cold-start replay broken)") } // Flip the chat task into in_progress now that the agent has it. @@ -659,7 +659,7 @@ func TestClusterIntegration_ChatTaskWorkflow_EndToEnd(t *testing.T) { select { case <-sandboxDelivered: // proceed - case <-time.After(10 * time.Second): + case <-time.After(scaled(10 * time.Second)): t.Fatal("phase 11: sandbox-startup dispatcher callback never fired within 10s") } got2, _ := sandboxNotif.Load().(*orchestration.OrchestrationTaskNotification) @@ -766,7 +766,7 @@ func TestClusterIntegration_ChatTaskWorkflow_EndToEnd(t *testing.T) { if err := b.router.Publish(ctx, agentTopic, respBytes); err != nil { t.Fatalf("phase 13: sandbox Publish reply: %v", err) } - case <-time.After(5 * time.Second): + case <-time.After(scaled(5 * time.Second)): t.Fatal("phase 13: sandbox did not receive request within 5s") } @@ -780,7 +780,7 @@ func TestClusterIntegration_ChatTaskWorkflow_EndToEnd(t *testing.T) { if env.TaskID != agentTaskID { t.Errorf("phase 13: agent sandbox reply task_id = %q, want %q", env.TaskID, agentTaskID) } - case <-time.After(5 * time.Second): + case <-time.After(scaled(5 * time.Second)): t.Fatal("phase 13: agent did not receive sandbox reply within 5s") } @@ -807,7 +807,7 @@ func TestClusterIntegration_ChatTaskWorkflow_EndToEnd(t *testing.T) { if env.TaskID != agentTaskID { t.Errorf("phase 14: user received task_id = %q, want %q", env.TaskID, agentTaskID) } - case <-time.After(5 * time.Second): + case <-time.After(scaled(5 * time.Second)): t.Fatal("phase 14: user did not receive agent reply within 5s") } @@ -886,7 +886,7 @@ func TestClusterIntegration_ChatTaskWorkflow_EndToEnd(t *testing.T) { if string(got.data) != string(betweenMsg) { t.Errorf("phase 16: re-subscribe got %q, want %q (durable offset resume)", got.data, betweenMsg) } - case <-time.After(5 * time.Second): + case <-time.After(scaled(5 * time.Second)): t.Fatal("phase 16: re-subscribe did NOT receive between-reconnects message — durable offset resume broken") } diff --git a/server/internal/cluster/integration/cluster_helpers_test.go b/server/internal/cluster/integration/cluster_helpers_test.go index 5b85659..df47b31 100644 --- a/server/internal/cluster/integration/cluster_helpers_test.go +++ b/server/internal/cluster/integration/cluster_helpers_test.go @@ -132,7 +132,7 @@ func setupCluster3(t *testing.T) *cluster3 { HAMode: clusternats.HAModeAuto, } es := &clusternats.EmbeddedServer{} - startCtx, cancel := context.WithTimeout(context.Background(), 20*time.Second) + startCtx, cancel := context.WithTimeout(context.Background(), scaled(20*time.Second)) err := es.Start(startCtx, cfg) cancel() if err != nil { @@ -160,7 +160,7 @@ func setupCluster3(t *testing.T) *cluster3 { // will list 3 URLs (its own + 2 peers) once gossip has settled. func waitForClusterFormed(t *testing.T, c *cluster3) { t.Helper() - deadline := time.Now().Add(15 * time.Second) + deadline := time.Now().Add(scaled(15 * time.Second)) for time.Now().Before(deadline) { ready := true for _, n := range c.Nodes { @@ -175,11 +175,14 @@ func waitForClusterFormed(t *testing.T, c *cluster3) { } } if ready { - // Give JetStream meta-group a brief moment to elect a leader so - // stream/KV creation on first call doesn't ENOQUORUM. 200ms is - // empirically sufficient on a laptop; CI may need more but 500ms - // is still cheap. - time.Sleep(500 * time.Millisecond) + // Routes are up, but replicated stream/KV creation additionally + // needs the JetStream meta-leader elected — otherwise the first + // CreateOrUpdateKeyValue blocks on ENOQUORUM until it clears. A fixed + // sleep here is racy: under -race the election routinely takes well + // over a second, which is what made phase-5 flake with "open KV + // bucket: context deadline exceeded". Actively poll JetStream + // readiness instead of guessing a duration. + waitForJetStreamReady(t, c) return } time.Sleep(50 * time.Millisecond) @@ -187,6 +190,32 @@ func waitForClusterFormed(t *testing.T, c *cluster3) { t.Fatal("3-node cluster did not finish forming within 15s") } +// waitForJetStreamReady blocks until every node's JetStream is ready — i.e. +// AccountInfo succeeds, which requires the JetStream meta-leader to be elected +// and reachable. This gates replicated stream/KV creation so it doesn't race +// the meta-election. The deadline is generous because -race instrumentation +// slows Raft election substantially (a fixed sub-second sleep is never enough +// under -race). +func waitForJetStreamReady(t *testing.T, c *cluster3) { + t.Helper() + deadline := time.Now().Add(scaled(30 * time.Second)) + for _, n := range c.Nodes { + js := n.JetStream() + for { + ctx, cancel := context.WithTimeout(context.Background(), scaled(2*time.Second)) + _, err := js.AccountInfo(ctx) + cancel() + if err == nil { + break + } + if time.Now().After(deadline) { + t.Fatalf("JetStream meta-leader not ready within 30s: %v", err) + } + time.Sleep(100 * time.Millisecond) + } + } +} + // waitUntil polls the supplied predicate until it returns true or the deadline // elapses. Returns true on success, false on timeout — callers decide whether // to t.Fatal or just log + continue. diff --git a/server/internal/cluster/integration/fake_task_store_test.go b/server/internal/cluster/integration/fake_task_store_test.go index fc3885f..2c58099 100644 --- a/server/internal/cluster/integration/fake_task_store_test.go +++ b/server/internal/cluster/integration/fake_task_store_test.go @@ -44,6 +44,9 @@ func (f *fakeTaskStore) CompleteQueueEntryByTaskID(_ context.Context, _ string) func (f *fakeTaskStore) FailQueueEntryByTaskID(_ context.Context, _, _ string) error { return nil } +func (f *fakeTaskStore) ReconcileOrphanedQueueEntries(_ context.Context) (int64, error) { + return 0, nil +} func (f *fakeTaskStore) GetQueueEntryDetails(_ context.Context, queueID string) (*taskstore.QueueEntryDetails, error) { return &taskstore.QueueEntryDetails{ TaskID: queueID, @@ -78,7 +81,7 @@ func (f *fakeTaskStore) CountPendingQueueEntries(_ context.Context) (int, error) // --- Methods that should never be touched in these tests; panic to surface // unexpected calls during integration --- -func (f *fakeTaskStore) InsertQueueEntry(_ context.Context, _, _, _, _, _ string, _ []byte) error { +func (f *fakeTaskStore) InsertQueueEntry(_ context.Context, _, _, _, _, _ string, _ []byte, _ int) error { panic("InsertQueueEntry unexpected in cluster integration tests") } func (f *fakeTaskStore) PollPendingQueueEntries(_ context.Context, _ int) ([]*taskstore.QueueEntryNotification, error) { @@ -103,6 +106,7 @@ func (f *fakeTaskStore) StartTask(_ context.Context, _ string) error { panic func (f *fakeTaskStore) StartTaskWithAgent(_ context.Context, _, _ string) error { panic("unexpected") } +func (f *fakeTaskStore) ClaimTask(_ context.Context, _ string) error { panic("unexpected") } func (f *fakeTaskStore) CompleteTask(_ context.Context, _ string) error { panic("unexpected") } func (f *fakeTaskStore) FailTask(_ context.Context, _, _ string) error { panic("unexpected") } func (f *fakeTaskStore) FailTaskWithRetry(_ context.Context, _, _, _ string, _ *time.Time) error { diff --git a/server/internal/cluster/integration/identity_lock_chaos_test.go b/server/internal/cluster/integration/identity_lock_chaos_test.go index fe4e2a6..e3b9c15 100644 --- a/server/internal/cluster/integration/identity_lock_chaos_test.go +++ b/server/internal/cluster/integration/identity_lock_chaos_test.go @@ -33,7 +33,7 @@ func TestClusterIntegration_IdentityLock_ExactlyOneWinner(t *testing.T) { for i := 0; i < N; i++ { nodeIdx := i % len(c.Nodes) js := c.Node(nodeIdx).JetStream() - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + ctx, cancel := context.WithTimeout(context.Background(), scaled(10*time.Second)) s, err := state.NewJetStreamSession(ctx, js, state.JetStreamSessionConfig{Replicas: 3}) cancel() if err != nil { @@ -66,7 +66,7 @@ func TestClusterIntegration_IdentityLock_ExactlyOneWinner(t *testing.T) { go func(i int, s *state.JetStreamSession) { defer wg.Done() start.Wait() - ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) + ctx, cancel := context.WithTimeout(context.Background(), scaled(15*time.Second)) defer cancel() sessionID := fmt.Sprintf("sess-racer-%02d", i) r, err := s.AcquireOrResumeLock(ctx, identity, sessionID, "", 0, state.ConnectMeta{}) @@ -118,7 +118,7 @@ func TestClusterIntegration_IdentityLock_ExactlyOneWinner(t *testing.T) { func TestClusterIntegration_IdentityLock_NodeKillMidAcquire(t *testing.T) { c := setupCluster3(t) - ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + ctx, cancel := context.WithTimeout(context.Background(), scaled(30*time.Second)) defer cancel() // Build sessions on node-0 (survivor) and node-1 (victim). Both use @@ -182,7 +182,7 @@ func TestClusterIntegration_IdentityLock_NodeKillMidAcquire(t *testing.T) { acquired bool forced bool ) - if got := waitUntil(15*time.Second, 100*time.Millisecond, func() bool { + if got := waitUntil(scaled(15*time.Second), 100*time.Millisecond, func() bool { r, err := survivor.AcquireOrResumeLock(ctx, identity, "sess-survivor", "", 24*60*60*1000, state.ConnectMeta{}) ok, f := r.Acquired, r.Forced if err != nil { diff --git a/server/internal/cluster/integration/message_routing_test.go b/server/internal/cluster/integration/message_routing_test.go index bacc3e9..2972903 100644 --- a/server/internal/cluster/integration/message_routing_test.go +++ b/server/internal/cluster/integration/message_routing_test.go @@ -78,14 +78,14 @@ func TestClusterIntegration_MessageRouting_CrossNode(t *testing.T) { time.Sleep(500 * time.Millisecond) payload := []byte("cross-node-ping") - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + ctx, cancel := context.WithTimeout(context.Background(), scaled(5*time.Second)) defer cancel() if err := rA.Publish(ctx, topic, payload); err != nil { t.Fatalf("publish on A: %v", err) } // Both subscribers must receive within a generous timeout. - timeout := time.After(3 * time.Second) + timeout := time.After(scaled(3 * time.Second)) receivedB, receivedC := false, false for !receivedB || !receivedC { select { @@ -137,7 +137,7 @@ func TestClusterIntegration_MessageRouting_OffsetAdvance(t *testing.T) { const secondBatch = 20 const durableName = "ag-cluster-offset-test" - ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + ctx, cancel := context.WithTimeout(context.Background(), scaled(30*time.Second)) defer cancel() // Phase 1: publish first batch. @@ -155,7 +155,7 @@ func TestClusterIntegration_MessageRouting_OffsetAdvance(t *testing.T) { if err != nil { t.Fatalf("first subscribe: %v", err) } - if !firstMsgs.waitForN(firstBatch, 15*time.Second) { + if !firstMsgs.waitForN(firstBatch, scaled(15*time.Second)) { cancelB1() t.Fatalf("phase-1 timeout: received %d of %d", firstMsgs.received(), firstBatch) } @@ -185,7 +185,7 @@ func TestClusterIntegration_MessageRouting_OffsetAdvance(t *testing.T) { } defer cancelB2() - if !secondMsgs.waitForN(secondBatch, 10*time.Second) { + if !secondMsgs.waitForN(secondBatch, scaled(10*time.Second)) { t.Fatalf("phase-2 timeout: received %d of %d", secondMsgs.received(), secondBatch) } diff --git a/server/internal/cluster/integration/partition_recovery_test.go b/server/internal/cluster/integration/partition_recovery_test.go index a56b547..40b5f40 100644 --- a/server/internal/cluster/integration/partition_recovery_test.go +++ b/server/internal/cluster/integration/partition_recovery_test.go @@ -64,7 +64,7 @@ func TestClusterIntegration_PartitionRecovery(t *testing.T) { const secondBatch = 25 const totalExpected = firstBatch + secondBatch - ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + ctx, cancel := context.WithTimeout(context.Background(), scaled(60*time.Second)) defer cancel() // Phase 1: publish 25 messages while all 3 nodes are healthy. @@ -114,7 +114,7 @@ func TestClusterIntegration_PartitionRecovery(t *testing.T) { select { case <-doneSurvivors: - case <-time.After(15 * time.Second): + case <-time.After(scaled(15 * time.Second)): t.Fatalf("survivors saw only %d of %d messages — post-stop writes lost", survivedCount.Load(), totalExpected) } diff --git a/server/internal/cluster/integration/phase4_recursive_subscribe_test.go b/server/internal/cluster/integration/phase4_recursive_subscribe_test.go index 1a7ff14..5683fd7 100644 --- a/server/internal/cluster/integration/phase4_recursive_subscribe_test.go +++ b/server/internal/cluster/integration/phase4_recursive_subscribe_test.go @@ -96,7 +96,7 @@ func TestClusterIntegration_Phase4_PostSubscribeChild_CrossNode(t *testing.T) { }, }, } - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + ctx, cancel := context.WithTimeout(context.Background(), scaled(5*time.Second)) defer cancel() if err := publisherPub.PublishTaskEvent(ctx, ws, childTaskID, childEvt); err != nil { t.Fatalf("PublishTaskEvent: %v", err) @@ -113,7 +113,7 @@ func TestClusterIntegration_Phase4_PostSubscribeChild_CrossNode(t *testing.T) { if sc := got.GetStatusChanged(); sc == nil || sc.GetToStatus() != pb.TaskStatus_TASK_STATUS_RUNNING { t.Errorf("unexpected event variant: %+v", got.GetEvent()) } - case <-time.After(3 * time.Second): + case <-time.After(scaled(3 * time.Second)): t.Fatal("timeout: cross-node post-subscribe child event was NOT delivered (Phase 4 gap regression)") } } diff --git a/server/internal/cluster/integration/phase5_prefixindex_test.go b/server/internal/cluster/integration/phase5_prefixindex_test.go index 0c52e67..3f15790 100644 --- a/server/internal/cluster/integration/phase5_prefixindex_test.go +++ b/server/internal/cluster/integration/phase5_prefixindex_test.go @@ -29,7 +29,7 @@ import ( func TestClusterIntegration_Phase5_PrefixIndex_LiveUpdateCrossNode(t *testing.T) { c := setupCluster3(t) - bootstrapCtx, bootstrapCancel := context.WithTimeout(context.Background(), 15*time.Second) + bootstrapCtx, bootstrapCancel := context.WithTimeout(context.Background(), scaled(15*time.Second)) defer bootstrapCancel() // Open the registry bucket on BOTH nodes. CreateOrUpdateKeyValue is @@ -60,7 +60,7 @@ func TestClusterIntegration_Phase5_PrefixIndex_LiveUpdateCrossNode(t *testing.T) } const implementation = "com.example.docmgmt" - publishCtx, publishCancel := context.WithTimeout(context.Background(), 5*time.Second) + publishCtx, publishCancel := context.WithTimeout(context.Background(), scaled(5*time.Second)) defer publishCancel() reg := ®istry.AgentRegistration{ Implementation: implementation, @@ -77,7 +77,7 @@ func TestClusterIntegration_Phase5_PrefixIndex_LiveUpdateCrossNode(t *testing.T) // KV update up via cross-node replication and fan-out within ~100ms. // Allow up to 2s of slack for slow CI runners. const target = "docmgmt/document/abc" - if !waitUntil(2*time.Second, 10*time.Millisecond, func() bool { + if !waitUntil(scaled(2*time.Second), 10*time.Millisecond, func() bool { impl, _, ok := idx.Lookup(target) return ok && impl == implementation }) { @@ -92,12 +92,12 @@ func TestClusterIntegration_Phase5_PrefixIndex_LiveUpdateCrossNode(t *testing.T) } // Sanity: DeleteAgent should propagate too. - deleteCtx, deleteCancel := context.WithTimeout(context.Background(), 5*time.Second) + deleteCtx, deleteCancel := context.WithTimeout(context.Background(), scaled(5*time.Second)) defer deleteCancel() if err := registry.DeleteAgent(deleteCtx, kvWriter, implementation); err != nil { t.Fatalf("DeleteAgent: %v", err) } - if !waitUntil(2*time.Second, 10*time.Millisecond, func() bool { + if !waitUntil(scaled(2*time.Second), 10*time.Millisecond, func() bool { _, _, ok := idx.Lookup(target) return !ok }) { diff --git a/server/internal/cluster/integration/task_assignment_race_test.go b/server/internal/cluster/integration/task_assignment_race_test.go index c6912d8..25b2ca9 100644 --- a/server/internal/cluster/integration/task_assignment_race_test.go +++ b/server/internal/cluster/integration/task_assignment_race_test.go @@ -33,7 +33,7 @@ import ( func TestClusterIntegration_TaskAssignment_ExactlyOnce(t *testing.T) { c := setupCluster3(t) - bootstrapCtx, bootstrapCancel := context.WithTimeout(context.Background(), 30*time.Second) + bootstrapCtx, bootstrapCancel := context.WithTimeout(context.Background(), scaled(30*time.Second)) defer bootstrapCancel() store := newFakeTaskStore() @@ -71,7 +71,7 @@ func TestClusterIntegration_TaskAssignment_ExactlyOnce(t *testing.T) { dA.SetCallback(handlerFactory(&fromA)) dB.SetCallback(handlerFactory(&fromB)) - ctx, cancel := context.WithTimeout(context.Background(), 45*time.Second) + ctx, cancel := context.WithTimeout(context.Background(), scaled(45*time.Second)) defer cancel() if err := dA.Start(ctx); err != nil { t.Fatalf("start A: %v", err) @@ -107,7 +107,7 @@ func TestClusterIntegration_TaskAssignment_ExactlyOnce(t *testing.T) { // AckExplicit means a transient redelivery is theoretically possible if // the test happens to interleave with NaK backoff timers — we permit // at most 1 redelivery in the assertions below. - deadline := time.Now().Add(30 * time.Second) + deadline := time.Now().Add(scaled(30 * time.Second)) for time.Now().Before(deadline) { if counter.Load() >= int64(total) { break diff --git a/server/internal/cluster/integration/task_event_fanout_test.go b/server/internal/cluster/integration/task_event_fanout_test.go index c6af535..5e19147 100644 --- a/server/internal/cluster/integration/task_event_fanout_test.go +++ b/server/internal/cluster/integration/task_event_fanout_test.go @@ -107,7 +107,7 @@ func TestClusterIntegration_TaskEventFanout_StatusChangedRecursive(t *testing.T) // "out-of-band consumer verification" called out by the task. wsEsc := natscodec.EscapeForSubject(ws) rawFilter := fmt.Sprintf("tk.%s.>", wsEsc) - rawCtx, rawCancel := context.WithTimeout(context.Background(), 20*time.Second) + rawCtx, rawCancel := context.WithTimeout(context.Background(), scaled(20*time.Second)) defer rawCancel() rawCons, err := js.CreateOrUpdateConsumer(rawCtx, "tk", jetstream.ConsumerConfig{ Durable: "fanout-raw-subject-verify", @@ -149,7 +149,7 @@ func TestClusterIntegration_TaskEventFanout_StatusChangedRecursive(t *testing.T) // creation. time.Sleep(400 * time.Millisecond) - ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) + ctx, cancel := context.WithTimeout(context.Background(), scaled(20*time.Second)) defer cancel() // ----- Publish PARENT status-changed progression ----- @@ -208,7 +208,7 @@ func TestClusterIntegration_TaskEventFanout_StatusChangedRecursive(t *testing.T) // Expected: 4 parent status-changed + 1 child child-lifecycle = 5 total. const wantTotal = 5 collected := make([]recv, 0, wantTotal) - deadline := time.After(10 * time.Second) + deadline := time.After(scaled(10 * time.Second)) for len(collected) < wantTotal { select { case r := <-received: @@ -256,7 +256,7 @@ func TestClusterIntegration_TaskEventFanout_StatusChangedRecursive(t *testing.T) // ----- Verify codec-translated subjects via the raw consumer ----- // Wait briefly for the raw consumer to have caught up with the publisher. - if ok := waitUntil(2*time.Second, 50*time.Millisecond, func() bool { + if ok := waitUntil(scaled(2*time.Second), 50*time.Millisecond, func() bool { rawMu.Lock() defer rawMu.Unlock() return len(rawSubjects) >= wantTotal @@ -331,7 +331,7 @@ func TestClusterIntegration_TaskEventFanout_CrossWorkspaceIsolation(t *testing.T time.Sleep(400 * time.Millisecond) - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + ctx, cancel := context.WithTimeout(context.Background(), scaled(10*time.Second)) defer cancel() // Publish to workspace A. @@ -372,7 +372,7 @@ func TestClusterIntegration_TaskEventFanout_CrossWorkspaceIsolation(t *testing.T t.Errorf("wsA subscriber got task=%q ws=%q, want task=%q ws=%q", gotA.GetTaskId(), gotA.GetWorkspace(), taskA, wsA) } - case <-time.After(3 * time.Second): + case <-time.After(scaled(3 * time.Second)): t.Fatal("wsA subscriber did not receive its own workspace event (sanity check failed)") } @@ -386,7 +386,7 @@ func TestClusterIntegration_TaskEventFanout_CrossWorkspaceIsolation(t *testing.T t.Errorf("wsB subscriber got task=%q ws=%q, want task=%q ws=%q", gotB.GetTaskId(), gotB.GetWorkspace(), taskB, wsB) } - case <-time.After(3 * time.Second): + case <-time.After(scaled(3 * time.Second)): t.Fatal("wsB subscriber did not receive its own workspace event") } diff --git a/server/internal/cluster/integration/timescale_norace_test.go b/server/internal/cluster/integration/timescale_norace_test.go new file mode 100644 index 0000000..6f8f32d --- /dev/null +++ b/server/internal/cluster/integration/timescale_norace_test.go @@ -0,0 +1,10 @@ +//go:build !race + +package integration + +import "time" + +// scaled is the identity outside the race detector: deadlines stay tight for +// fast local feedback. See timescale_race_test.go for the -race widening and +// the rules on where scaled may (and may not) be applied. +func scaled(d time.Duration) time.Duration { return d } diff --git a/server/internal/cluster/integration/timescale_race_test.go b/server/internal/cluster/integration/timescale_race_test.go new file mode 100644 index 0000000..9053369 --- /dev/null +++ b/server/internal/cluster/integration/timescale_race_test.go @@ -0,0 +1,20 @@ +//go:build race + +package integration + +import "time" + +// scaled widens a positive test deadline under the race detector. The race +// detector's 2-10x slowdown blows the tight timeouts these embedded-NATS +// cluster tests use — especially on CI's 2-core runners, where JetStream Raft +// elections, backups, and tunnel handshakes take far longer than on a dev box. +// +// Use scaled ONLY for positive waits: operation timeouts (context.WithTimeout), +// poll deadlines (time.Now().Add), and completion waits (time.After in a select +// that expects something to arrive). NEVER wrap a negative-assertion window (a +// select that asserts NOTHING arrives within X) — widening that only slows the +// suite. Tick/poll intervals and behavior config (leader TTLs) are also left +// unscaled. +// +// Outside the race detector this is the identity — see timescale_norace_test.go. +func scaled(d time.Duration) time.Duration { return d * 4 } diff --git a/server/internal/config/gateway.go b/server/internal/config/gateway.go index e168ef0..71cfec5 100644 --- a/server/internal/config/gateway.go +++ b/server/internal/config/gateway.go @@ -346,6 +346,13 @@ type AuditConfig struct { FlushPeriod string `yaml:"flush_period"` RetentionDays int `yaml:"retention_days"` ChannelBuffer int `yaml:"channel_buffer"` + + // CoalesceWindow is the window over which a burst of identical successful + // message-route / proxy-route audit events from the same sender→target is + // recorded once (the first = the authorization record) and the rest are + // suppressed. Defaults to "60s". Set to "0" to disable coalescing (audit + // every event). + CoalesceWindow string `yaml:"audit_coalesce_window"` } // GetFlushPeriod parses the flush period duration @@ -353,6 +360,12 @@ func (a *AuditConfig) GetFlushPeriod() time.Duration { return parseDurationOrDefault(a.FlushPeriod, 5*time.Second) } +// GetCoalesceWindow parses the audit coalesce window duration, defaulting to +// 60s. A value of "0" disables coalescing (every event is audited). +func (a *AuditConfig) GetCoalesceWindow() time.Duration { + return parseDurationOrDefault(a.CoalesceWindow, 60*time.Second) +} + // CleanupConfig contains cleanup job settings type CleanupConfig struct { // TaskPurgeInterval is how often to run task purge (e.g., "24h"). @@ -371,6 +384,46 @@ type CleanupConfig struct { // ReconciliationInterval is how often to run orphaned task reconciliation (e.g., "1m"). // Set to "0" to disable automatic reconciliation. ReconciliationInterval string `yaml:"reconciliation_interval"` + + // InteractiveTaskTTL is how old a non-terminal INTERACTIVE (chat turn) task + // may get before it is auto-cancelled (e.g., "1h"). + InteractiveTaskTTL string `yaml:"interactive_task_ttl"` + + // InteractiveTaskCancelInterval is how often to run the stale interactive + // task sweep (e.g., "15m"). Set to "0" to disable. + InteractiveTaskCancelInterval string `yaml:"interactive_task_cancel_interval"` + + // StartupTaskTTL is how old an UNCLAIMED (pending) agent_startup task may + // get before it is auto-cancelled (e.g., "30m"). Deliberately generous so a + // briefly-unavailable orchestrator's pending task is never reaped. + StartupTaskTTL string `yaml:"startup_task_ttl"` + + // StartupTaskCancelInterval is how often to run the stale startup task + // sweep (e.g., "15m"). Set to "0" to disable. + StartupTaskCancelInterval string `yaml:"startup_task_cancel_interval"` + + // PoolTaskTTL is how old an UNCLAIMED (pending) regular POOL task may get + // before it is auto-cancelled (e.g., "1h"). Deliberately generous so a + // briefly-absent worker's pending task is never reaped. + PoolTaskTTL string `yaml:"pool_task_ttl"` + + // PoolTaskCancelInterval is how often to run the stale pool task sweep + // (e.g., "15m"). Set to "0" to disable. + PoolTaskCancelInterval string `yaml:"pool_task_cancel_interval"` + + // QueueReconcileInterval is how often to retire orphaned + // orchestrated_task_queue rows whose task is terminal/missing (e.g., "5m"). + // Set to "0" to disable. + QueueReconcileInterval string `yaml:"queue_reconcile_interval"` + + // AuditRetentionDays is how many days of comprehensive_audit_log rows to + // keep; older rows are deleted by the scheduled audit-retention sweep. + // Defaults to 90. + AuditRetentionDays int `yaml:"audit_retention_days"` + + // AuditCleanupInterval is how often to run the scheduled audit-log + // retention sweep (e.g., "24h"). Set to "0" to disable. + AuditCleanupInterval string `yaml:"audit_cleanup_interval"` } // GetTaskPurgeInterval parses the task purge interval duration @@ -398,6 +451,56 @@ func (c *CleanupConfig) GetReconciliationInterval() time.Duration { return parseDurationOrDefault(c.ReconciliationInterval, 1*time.Minute) } +// GetInteractiveTaskTTL parses the interactive task TTL duration +func (c *CleanupConfig) GetInteractiveTaskTTL() time.Duration { + return parseDurationOrDefault(c.InteractiveTaskTTL, 1*time.Hour) +} + +// GetInteractiveTaskCancelInterval parses the interactive task cancel interval duration +func (c *CleanupConfig) GetInteractiveTaskCancelInterval() time.Duration { + return parseDurationOrDefault(c.InteractiveTaskCancelInterval, 15*time.Minute) +} + +// GetStartupTaskTTL parses the startup task TTL duration +func (c *CleanupConfig) GetStartupTaskTTL() time.Duration { + return parseDurationOrDefault(c.StartupTaskTTL, 30*time.Minute) +} + +// GetStartupTaskCancelInterval parses the startup task cancel interval duration +func (c *CleanupConfig) GetStartupTaskCancelInterval() time.Duration { + return parseDurationOrDefault(c.StartupTaskCancelInterval, 15*time.Minute) +} + +// GetPoolTaskTTL parses the pool task TTL duration +func (c *CleanupConfig) GetPoolTaskTTL() time.Duration { + return parseDurationOrDefault(c.PoolTaskTTL, 1*time.Hour) +} + +// GetPoolTaskCancelInterval parses the pool task cancel interval duration +func (c *CleanupConfig) GetPoolTaskCancelInterval() time.Duration { + return parseDurationOrDefault(c.PoolTaskCancelInterval, 15*time.Minute) +} + +// GetQueueReconcileInterval parses the orphaned-queue reconcile interval duration +func (c *CleanupConfig) GetQueueReconcileInterval() time.Duration { + return parseDurationOrDefault(c.QueueReconcileInterval, 5*time.Minute) +} + +// GetAuditRetentionDays returns the audit-log retention window in days, +// defaulting to 90 when unset (<= 0). +func (c *CleanupConfig) GetAuditRetentionDays() int { + if c.AuditRetentionDays <= 0 { + return 90 + } + return c.AuditRetentionDays +} + +// GetAuditCleanupInterval parses the audit-log cleanup interval duration, +// defaulting to 24h. A value of "0" disables the scheduled sweep. +func (c *CleanupConfig) GetAuditCleanupInterval() time.Duration { + return parseDurationOrDefault(c.AuditCleanupInterval, 24*time.Hour) +} + // KVConfig contains KV store settings type KVConfig struct { // DefaultTTL is the default TTL applied to KV keys when no TTL is specified. @@ -790,6 +893,9 @@ func (c *Config) ApplyEnvOverrides() { if v := os.Getenv("AETHER_AUDIT_EVENT_TYPES"); v != "" { c.Audit.EventTypes = strings.Split(v, ",") } + if v := os.Getenv("AETHER_AUDIT_COALESCE_WINDOW"); v != "" { + c.Audit.CoalesceWindow = v + } } // LoadDefault loads configuration from the default path ./configs/dev.yaml diff --git a/server/internal/config/gateway_test.go b/server/internal/config/gateway_test.go index 1e89d4d..51b001f 100644 --- a/server/internal/config/gateway_test.go +++ b/server/internal/config/gateway_test.go @@ -1101,3 +1101,65 @@ func TestIsAuthModeEnabled(t *testing.T) { }) } } + +func TestCleanupConfig_GetAuditRetentionDays(t *testing.T) { + tests := []struct { + name string + config CleanupConfig + want int + }{ + {name: "explicit 30", config: CleanupConfig{AuditRetentionDays: 30}, want: 30}, + {name: "zero defaults to 90", config: CleanupConfig{AuditRetentionDays: 0}, want: 90}, + {name: "negative defaults to 90", config: CleanupConfig{AuditRetentionDays: -5}, want: 90}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := tt.config.GetAuditRetentionDays(); got != tt.want { + t.Errorf("GetAuditRetentionDays() = %d, want %d", got, tt.want) + } + }) + } +} + +func TestCleanupConfig_GetAuditCleanupInterval(t *testing.T) { + tests := []struct { + name string + config CleanupConfig + want time.Duration + }{ + {name: "explicit 12h", config: CleanupConfig{AuditCleanupInterval: "12h"}, want: 12 * time.Hour}, + {name: "empty defaults to 24h", config: CleanupConfig{AuditCleanupInterval: ""}, want: 24 * time.Hour}, + {name: "invalid defaults to 24h", config: CleanupConfig{AuditCleanupInterval: "nope"}, want: 24 * time.Hour}, + {name: "zero disables", config: CleanupConfig{AuditCleanupInterval: "0"}, want: 0}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := tt.config.GetAuditCleanupInterval(); got != tt.want { + t.Errorf("GetAuditCleanupInterval() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestAuditConfig_GetCoalesceWindow(t *testing.T) { + tests := []struct { + name string + config AuditConfig + want time.Duration + }{ + {name: "explicit 30s", config: AuditConfig{CoalesceWindow: "30s"}, want: 30 * time.Second}, + {name: "empty defaults to 60s", config: AuditConfig{CoalesceWindow: ""}, want: 60 * time.Second}, + {name: "invalid defaults to 60s", config: AuditConfig{CoalesceWindow: "bad"}, want: 60 * time.Second}, + {name: "zero disables", config: AuditConfig{CoalesceWindow: "0"}, want: 0}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := tt.config.GetCoalesceWindow(); got != tt.want { + t.Errorf("GetCoalesceWindow() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/server/internal/gateway/acl_handler.go b/server/internal/gateway/acl_handler.go index 88f6fc0..ae4c65c 100644 --- a/server/internal/gateway/acl_handler.go +++ b/server/internal/gateway/acl_handler.go @@ -6,6 +6,7 @@ import ( "time" pb "github.com/scitrera/aether/api/proto" + "github.com/scitrera/aether/internal/acl" "github.com/scitrera/aether/internal/admin" "github.com/scitrera/aether/internal/logging" ) @@ -101,6 +102,23 @@ func (s *GatewayServer) handleACLOp(ctx context.Context, client *ClientSession, resourceType = op.RuleFilter.ResourceType resourceID = op.RuleFilter.ResourceId } + // RevokeACLAccess deletes by composite (principal+resource) key, but + // clients may target a rule by its rule_id (UUID) instead. When a + // rule_id is supplied and the RuleFilter is empty, resolve the rule's + // composite key from the store first; an explicit, fully-specified + // RuleFilter always wins (so callers can override resolution). + if op.RuleId != "" && principalType == "" && principalID == "" && resourceType == "" && resourceID == "" { + rule, err := s.adminProvider.GetACLRuleByID(ctx, op.RuleId) + if err != nil { + logging.Logger.Error().Err(err).Str("rule_id", op.RuleId).Msg("handleACLOp: revoke by rule_id lookup failed") + sendACLError(client, fmt.Sprintf("ACL rule not found for rule_id %q: %v", op.RuleId, err)) + return + } + principalType = rule.PrincipalType + principalID = rule.PrincipalID + resourceType = rule.ResourceType + resourceID = rule.ResourceID + } if err := s.adminProvider.RevokeACLAccess(ctx, principalType, principalID, resourceType, resourceID); err != nil { logging.Logger.Error().Err(err).Str("rule_id", op.RuleId).Msg("handleACLOp: revoke access failed") sendACLError(client, err.Error()) @@ -238,11 +256,344 @@ func (s *GatewayServer) handleACLOp(ctx context.Context, client *ClientSession, }, }) + case pb.ACLOperation_CREATE_GROUP: + if op.GroupRequest == nil { + sendACLError(client, "group_request required for CREATE_GROUP") + return + } + g, err := s.adminProvider.CreateACLGroup(ctx, &admin.CreateACLGroupRequest{ + Name: op.GroupRequest.Name, + Description: op.GroupRequest.Description, + CreatedBy: op.GroupRequest.CreatedBy, + Metadata: protoStringMapToMetadata(op.GroupRequest.Metadata), + }) + if err != nil { + sendACLError(client, err.Error()) + return + } + sendACLResponse(client, &pb.ACLResponse{Success: true, Message: "group created", Group: adminACLGroupToProto(g)}) + + case pb.ACLOperation_GET_GROUP: + g, err := s.adminProvider.GetACLGroup(ctx, op.Name) + if err != nil { + sendACLError(client, err.Error()) + return + } + sendACLResponse(client, &pb.ACLResponse{Success: true, Group: adminACLGroupToProto(g)}) + + case pb.ACLOperation_DELETE_GROUP: + if err := s.adminProvider.DeleteACLGroup(ctx, op.Name); err != nil { + sendACLError(client, err.Error()) + return + } + sendACLResponse(client, &pb.ACLResponse{Success: true, Message: "group deleted"}) + + case pb.ACLOperation_LIST_GROUPS: + groups, err := s.adminProvider.ListACLGroups(ctx) + if err != nil { + sendACLError(client, err.Error()) + return + } + out := make([]*pb.ACLGroupInfo, 0, len(groups)) + for _, g := range groups { + out = append(out, adminACLGroupToProto(g)) + } + sendACLResponse(client, &pb.ACLResponse{Success: true, Groups: out}) + + case pb.ACLOperation_ADD_GROUP_MEMBER: + if op.MemberRequest == nil { + sendACLError(client, "member_request required for ADD_GROUP_MEMBER") + return + } + m, err := s.adminProvider.AddACLGroupMember(ctx, op.Name, &admin.AddACLGroupMemberRequest{ + MemberType: op.MemberRequest.MemberType, + MemberID: op.MemberRequest.MemberId, + GrantedBy: op.MemberRequest.GrantedBy, + ExpiresAt: unixToTimePtr(op.MemberRequest.ExpiresAt), + }) + if err != nil { + sendACLError(client, err.Error()) + return + } + sendACLResponse(client, &pb.ACLResponse{Success: true, Message: "member added", GroupMembers: []*pb.ACLGroupMemberInfo{adminACLGroupMemberToProto(m)}}) + + case pb.ACLOperation_REMOVE_GROUP_MEMBER: + if op.Principal == nil { + sendACLError(client, "principal required for REMOVE_GROUP_MEMBER") + return + } + if err := s.adminProvider.RemoveACLGroupMember(ctx, op.Name, op.Principal.PrincipalType, op.Principal.PrincipalId); err != nil { + sendACLError(client, err.Error()) + return + } + sendACLResponse(client, &pb.ACLResponse{Success: true, Message: "member removed"}) + + case pb.ACLOperation_LIST_GROUP_MEMBERS: + members, err := s.adminProvider.ListACLGroupMembers(ctx, op.Name) + if err != nil { + sendACLError(client, err.Error()) + return + } + sendACLResponse(client, &pb.ACLResponse{Success: true, GroupMembers: adminACLGroupMembersToProto(members)}) + + case pb.ACLOperation_CREATE_ROLE: + if op.RoleRequest == nil { + sendACLError(client, "role_request required for CREATE_ROLE") + return + } + r, err := s.adminProvider.CreateACLRole(ctx, &admin.CreateACLRoleRequest{ + Name: op.RoleRequest.Name, + Description: op.RoleRequest.Description, + CreatedBy: op.RoleRequest.CreatedBy, + Metadata: protoStringMapToMetadata(op.RoleRequest.Metadata), + }) + if err != nil { + sendACLError(client, err.Error()) + return + } + sendACLResponse(client, &pb.ACLResponse{Success: true, Message: "role created", Role: adminACLRoleToProto(r)}) + + case pb.ACLOperation_GET_ROLE: + r, err := s.adminProvider.GetACLRole(ctx, op.Name) + if err != nil { + sendACLError(client, err.Error()) + return + } + sendACLResponse(client, &pb.ACLResponse{Success: true, Role: adminACLRoleToProto(r)}) + + case pb.ACLOperation_DELETE_ROLE: + if err := s.adminProvider.DeleteACLRole(ctx, op.Name); err != nil { + sendACLError(client, err.Error()) + return + } + sendACLResponse(client, &pb.ACLResponse{Success: true, Message: "role deleted"}) + + case pb.ACLOperation_LIST_ROLES: + roles, err := s.adminProvider.ListACLRoles(ctx) + if err != nil { + sendACLError(client, err.Error()) + return + } + out := make([]*pb.ACLRoleInfo, 0, len(roles)) + for _, r := range roles { + out = append(out, adminACLRoleToProto(r)) + } + sendACLResponse(client, &pb.ACLResponse{Success: true, Roles: out}) + + case pb.ACLOperation_ASSIGN_ROLE: + if op.AssignmentRequest == nil { + sendACLError(client, "assignment_request required for ASSIGN_ROLE") + return + } + a, err := s.adminProvider.AssignACLRole(ctx, op.Name, &admin.AssignACLRoleRequest{ + AssigneeType: op.AssignmentRequest.AssigneeType, + AssigneeID: op.AssignmentRequest.AssigneeId, + GrantedBy: op.AssignmentRequest.GrantedBy, + ExpiresAt: unixToTimePtr(op.AssignmentRequest.ExpiresAt), + }) + if err != nil { + sendACLError(client, err.Error()) + return + } + sendACLResponse(client, &pb.ACLResponse{Success: true, Message: "role assigned", RoleAssignments: []*pb.ACLRoleAssignmentInfo{adminACLRoleAssignmentToProto(a)}}) + + case pb.ACLOperation_UNASSIGN_ROLE: + if op.Principal == nil { + sendACLError(client, "principal required for UNASSIGN_ROLE") + return + } + if err := s.adminProvider.UnassignACLRole(ctx, op.Name, op.Principal.PrincipalType, op.Principal.PrincipalId); err != nil { + sendACLError(client, err.Error()) + return + } + sendACLResponse(client, &pb.ACLResponse{Success: true, Message: "role unassigned"}) + + case pb.ACLOperation_LIST_ROLE_ASSIGNMENTS: + assignments, err := s.adminProvider.ListACLRoleAssignments(ctx, op.Name) + if err != nil { + sendACLError(client, err.Error()) + return + } + sendACLResponse(client, &pb.ACLResponse{Success: true, RoleAssignments: adminACLRoleAssignmentsToProto(assignments)}) + + case pb.ACLOperation_LIST_PRINCIPAL_GROUPS: + if op.Principal == nil { + sendACLError(client, "principal required for LIST_PRINCIPAL_GROUPS") + return + } + members, err := s.adminProvider.ListACLPrincipalGroups(ctx, op.Principal.PrincipalType, op.Principal.PrincipalId) + if err != nil { + sendACLError(client, err.Error()) + return + } + sendACLResponse(client, &pb.ACLResponse{Success: true, GroupMembers: adminACLGroupMembersToProto(members)}) + + case pb.ACLOperation_LIST_PRINCIPAL_ROLES: + if op.Principal == nil { + sendACLError(client, "principal required for LIST_PRINCIPAL_ROLES") + return + } + assignments, err := s.adminProvider.ListACLPrincipalRoles(ctx, op.Principal.PrincipalType, op.Principal.PrincipalId) + if err != nil { + sendACLError(client, err.Error()) + return + } + sendACLResponse(client, &pb.ACLResponse{Success: true, RoleAssignments: adminACLRoleAssignmentsToProto(assignments)}) + + case pb.ACLOperation_EXPLAIN_ACCESS: + if op.Principal == nil { + sendACLError(client, "principal required for EXPLAIN_ACCESS") + return + } + callerType := acl.PrincipalTypeForModel(client.Identity.Type) + callerID := client.Identity.CanonicalPrincipalID() + exp, err := s.adminProvider.ExplainACLAccess(ctx, op.Principal.PrincipalType, op.Principal.PrincipalId, op.ResourceType, op.ResourceId, int(op.RequiredLevel), callerType, callerID) + if err != nil { + sendACLError(client, err.Error()) + return + } + sendACLResponse(client, &pb.ACLResponse{Success: true, Explanation: adminACLExplanationToProto(exp)}) + default: sendACLError(client, "unknown ACL operation") } } +func adminACLExplanationToProto(e *admin.ACLAccessExplanationInfo) *pb.ACLAccessExplanationInfo { + if e == nil { + return nil + } + out := &pb.ACLAccessExplanationInfo{ + Principal: e.Principal, + Subjects: e.Subjects, + Allowed: e.Allowed, + Decision: e.Decision, + EffectiveAccessLevel: int32(e.EffectiveLevel), + FallbackApplied: e.FallbackApplied, + Reason: e.Reason, + } + for _, c := range e.Contributions { + out.Contributions = append(out.Contributions, &pb.ACLAccessContributionInfo{ + Subject: c.Subject, + RuleId: c.RuleID, + AccessLevel: int32(c.AccessLevel), + Resource: c.Resource, + Expired: c.Expired, + }) + } + return out +} + +// sendACLResponse sends a successful ACL response wrapped in a DownstreamMessage. +func sendACLResponse(client *ClientSession, resp *pb.ACLResponse) { + _ = client.SafeSend(&pb.DownstreamMessage{ + Payload: &pb.DownstreamMessage_Acl{Acl: resp}, + }) +} + +func unixToTimePtr(ts int64) *time.Time { + if ts == 0 { + return nil + } + t := time.Unix(ts, 0) + return &t +} + +func unixOf(t *time.Time) int64 { + if t == nil || t.IsZero() { + return 0 + } + return t.Unix() +} + +func adminACLGroupToProto(g *admin.ACLGroupInfo) *pb.ACLGroupInfo { + if g == nil { + return nil + } + var createdAt int64 + if !g.CreatedAt.IsZero() { + createdAt = g.CreatedAt.Unix() + } + return &pb.ACLGroupInfo{ + GroupId: g.GroupID, + GroupName: g.GroupName, + Description: g.Description, + CreatedBy: g.CreatedBy, + CreatedAt: createdAt, + Metadata: metadataToProtoStringMap(g.Metadata), + } +} + +func adminACLRoleToProto(r *admin.ACLRoleInfo) *pb.ACLRoleInfo { + if r == nil { + return nil + } + var createdAt int64 + if !r.CreatedAt.IsZero() { + createdAt = r.CreatedAt.Unix() + } + return &pb.ACLRoleInfo{ + RoleId: r.RoleID, + RoleName: r.RoleName, + Description: r.Description, + CreatedBy: r.CreatedBy, + CreatedAt: createdAt, + Metadata: metadataToProtoStringMap(r.Metadata), + } +} + +func adminACLGroupMemberToProto(m *admin.ACLGroupMemberInfo) *pb.ACLGroupMemberInfo { + if m == nil { + return nil + } + var grantedAt int64 + if !m.GrantedAt.IsZero() { + grantedAt = m.GrantedAt.Unix() + } + return &pb.ACLGroupMemberInfo{ + GroupName: m.GroupName, + MemberType: m.MemberType, + MemberId: m.MemberID, + GrantedBy: m.GrantedBy, + GrantedAt: grantedAt, + ExpiresAt: unixOf(m.ExpiresAt), + } +} + +func adminACLGroupMembersToProto(members []*admin.ACLGroupMemberInfo) []*pb.ACLGroupMemberInfo { + out := make([]*pb.ACLGroupMemberInfo, 0, len(members)) + for _, m := range members { + out = append(out, adminACLGroupMemberToProto(m)) + } + return out +} + +func adminACLRoleAssignmentToProto(a *admin.ACLRoleAssignmentInfo) *pb.ACLRoleAssignmentInfo { + if a == nil { + return nil + } + var grantedAt int64 + if !a.GrantedAt.IsZero() { + grantedAt = a.GrantedAt.Unix() + } + return &pb.ACLRoleAssignmentInfo{ + RoleName: a.RoleName, + AssigneeType: a.AssigneeType, + AssigneeId: a.AssigneeID, + GrantedBy: a.GrantedBy, + GrantedAt: grantedAt, + ExpiresAt: unixOf(a.ExpiresAt), + } +} + +func adminACLRoleAssignmentsToProto(assignments []*admin.ACLRoleAssignmentInfo) []*pb.ACLRoleAssignmentInfo { + out := make([]*pb.ACLRoleAssignmentInfo, 0, len(assignments)) + for _, a := range assignments { + out = append(out, adminACLRoleAssignmentToProto(a)) + } + return out +} + func sendACLError(client *ClientSession, msg string) { _ = client.SafeSend(&pb.DownstreamMessage{ Payload: &pb.DownstreamMessage_Acl{ diff --git a/server/internal/gateway/admin_acl.go b/server/internal/gateway/admin_acl.go index 0e784b4..56e84a0 100644 --- a/server/internal/gateway/admin_acl.go +++ b/server/internal/gateway/admin_acl.go @@ -7,6 +7,7 @@ import ( "github.com/scitrera/aether/internal/acl" "github.com/scitrera/aether/internal/admin" + aclstore "github.com/scitrera/aether/internal/storage/acl" "github.com/scitrera/aether/pkg/models" ) @@ -79,6 +80,31 @@ func (p *GatewayStateProvider) GetACLRule(ctx context.Context, principalType, pr }, nil } +func (p *GatewayStateProvider) GetACLRuleByID(ctx context.Context, ruleID string) (*admin.ACLRuleInfo, error) { + if p.aclService == nil { + return nil, fmt.Errorf("ACL service not available") + } + + rule, err := p.aclService.GetRuleByID(ctx, ruleID) + if err != nil { + return nil, fmt.Errorf("failed to get ACL rule by id: %w", err) + } + + return &admin.ACLRuleInfo{ + RuleID: rule.RuleID, + PrincipalType: rule.PrincipalType, + PrincipalID: rule.PrincipalID, + ResourceType: rule.ResourceType, + ResourceID: rule.ResourceID, + AccessLevel: rule.AccessLevel, + AccessLevelName: acl.AccessLevelName(rule.AccessLevel), + GrantedBy: rule.GrantedBy, + GrantedAt: rule.GrantedAt, + ExpiresAt: rule.ExpiresAt, + Reason: rule.Reason, + }, nil +} + func (p *GatewayStateProvider) GrantACLAccess(ctx context.Context, req *admin.GrantACLAccessRequest) (*admin.ACLRuleInfo, error) { if p.aclService == nil { return nil, fmt.Errorf("ACL service not available") @@ -422,6 +448,285 @@ func adminPrincipalRefToIdentity(ref *admin.PrincipalRef) (models.Identity, erro return identity, nil } +// ============================================================================= +// ACL Groups & Roles +// ============================================================================= + +func (p *GatewayStateProvider) ListACLGroups(ctx context.Context) ([]*admin.ACLGroupInfo, error) { + if p.aclService == nil { + return nil, fmt.Errorf("ACL service not available") + } + groups, err := p.aclService.ListGroups(ctx) + if err != nil { + return nil, fmt.Errorf("failed to list groups: %w", err) + } + result := make([]*admin.ACLGroupInfo, 0, len(groups)) + for _, g := range groups { + result = append(result, groupToAdmin(g)) + } + return result, nil +} + +func (p *GatewayStateProvider) CreateACLGroup(ctx context.Context, req *admin.CreateACLGroupRequest) (*admin.ACLGroupInfo, error) { + if p.aclService == nil { + return nil, fmt.Errorf("ACL service not available") + } + g, err := p.aclService.CreateGroup(ctx, req.Name, req.Description, req.CreatedBy, req.Metadata) + if err != nil { + return nil, fmt.Errorf("failed to create group: %w", err) + } + return groupToAdmin(g), nil +} + +func (p *GatewayStateProvider) GetACLGroup(ctx context.Context, name string) (*admin.ACLGroupInfo, error) { + if p.aclService == nil { + return nil, fmt.Errorf("ACL service not available") + } + g, err := p.aclService.GetGroup(ctx, name) + if err != nil { + return nil, fmt.Errorf("failed to get group: %w", err) + } + return groupToAdmin(g), nil +} + +func (p *GatewayStateProvider) DeleteACLGroup(ctx context.Context, name string) error { + if p.aclService == nil { + return fmt.Errorf("ACL service not available") + } + return p.aclService.DeleteGroup(ctx, name) +} + +func (p *GatewayStateProvider) ListACLGroupMembers(ctx context.Context, groupName string) ([]*admin.ACLGroupMemberInfo, error) { + if p.aclService == nil { + return nil, fmt.Errorf("ACL service not available") + } + members, err := p.aclService.ListGroupMembers(ctx, groupName) + if err != nil { + return nil, fmt.Errorf("failed to list group members: %w", err) + } + result := make([]*admin.ACLGroupMemberInfo, 0, len(members)) + for _, m := range members { + result = append(result, groupMemberToAdmin(m)) + } + return result, nil +} + +func (p *GatewayStateProvider) AddACLGroupMember(ctx context.Context, groupName string, req *admin.AddACLGroupMemberRequest) (*admin.ACLGroupMemberInfo, error) { + if p.aclService == nil { + return nil, fmt.Errorf("ACL service not available") + } + m, err := p.aclService.AddGroupMember(ctx, groupName, req.MemberType, req.MemberID, req.GrantedBy, req.ExpiresAt) + if err != nil { + return nil, fmt.Errorf("failed to add group member: %w", err) + } + return groupMemberToAdmin(m), nil +} + +func (p *GatewayStateProvider) RemoveACLGroupMember(ctx context.Context, groupName, memberType, memberID string) error { + if p.aclService == nil { + return fmt.Errorf("ACL service not available") + } + return p.aclService.RemoveGroupMember(ctx, groupName, memberType, memberID) +} + +func (p *GatewayStateProvider) ListACLRoles(ctx context.Context) ([]*admin.ACLRoleInfo, error) { + if p.aclService == nil { + return nil, fmt.Errorf("ACL service not available") + } + roles, err := p.aclService.ListRoles(ctx) + if err != nil { + return nil, fmt.Errorf("failed to list roles: %w", err) + } + result := make([]*admin.ACLRoleInfo, 0, len(roles)) + for _, r := range roles { + result = append(result, roleToAdmin(r)) + } + return result, nil +} + +func (p *GatewayStateProvider) CreateACLRole(ctx context.Context, req *admin.CreateACLRoleRequest) (*admin.ACLRoleInfo, error) { + if p.aclService == nil { + return nil, fmt.Errorf("ACL service not available") + } + r, err := p.aclService.CreateRole(ctx, req.Name, req.Description, req.CreatedBy, req.Metadata) + if err != nil { + return nil, fmt.Errorf("failed to create role: %w", err) + } + return roleToAdmin(r), nil +} + +func (p *GatewayStateProvider) GetACLRole(ctx context.Context, name string) (*admin.ACLRoleInfo, error) { + if p.aclService == nil { + return nil, fmt.Errorf("ACL service not available") + } + r, err := p.aclService.GetRole(ctx, name) + if err != nil { + return nil, fmt.Errorf("failed to get role: %w", err) + } + return roleToAdmin(r), nil +} + +func (p *GatewayStateProvider) DeleteACLRole(ctx context.Context, name string) error { + if p.aclService == nil { + return fmt.Errorf("ACL service not available") + } + return p.aclService.DeleteRole(ctx, name) +} + +func (p *GatewayStateProvider) ListACLRoleAssignments(ctx context.Context, roleName string) ([]*admin.ACLRoleAssignmentInfo, error) { + if p.aclService == nil { + return nil, fmt.Errorf("ACL service not available") + } + assignments, err := p.aclService.ListRoleAssignments(ctx, roleName) + if err != nil { + return nil, fmt.Errorf("failed to list role assignments: %w", err) + } + result := make([]*admin.ACLRoleAssignmentInfo, 0, len(assignments)) + for _, a := range assignments { + result = append(result, roleAssignmentToAdmin(a)) + } + return result, nil +} + +func (p *GatewayStateProvider) AssignACLRole(ctx context.Context, roleName string, req *admin.AssignACLRoleRequest) (*admin.ACLRoleAssignmentInfo, error) { + if p.aclService == nil { + return nil, fmt.Errorf("ACL service not available") + } + a, err := p.aclService.AssignRole(ctx, roleName, req.AssigneeType, req.AssigneeID, req.GrantedBy, req.ExpiresAt) + if err != nil { + return nil, fmt.Errorf("failed to assign role: %w", err) + } + return roleAssignmentToAdmin(a), nil +} + +func (p *GatewayStateProvider) UnassignACLRole(ctx context.Context, roleName, assigneeType, assigneeID string) error { + if p.aclService == nil { + return fmt.Errorf("ACL service not available") + } + return p.aclService.UnassignRole(ctx, roleName, assigneeType, assigneeID) +} + +func (p *GatewayStateProvider) ListACLPrincipalGroups(ctx context.Context, memberType, memberID string) ([]*admin.ACLGroupMemberInfo, error) { + if p.aclService == nil { + return nil, fmt.Errorf("ACL service not available") + } + members, err := p.aclService.ListPrincipalGroups(ctx, memberType, memberID) + if err != nil { + return nil, fmt.Errorf("failed to list principal groups: %w", err) + } + result := make([]*admin.ACLGroupMemberInfo, 0, len(members)) + for _, m := range members { + result = append(result, groupMemberToAdmin(m)) + } + return result, nil +} + +func (p *GatewayStateProvider) ListACLPrincipalRoles(ctx context.Context, assigneeType, assigneeID string) ([]*admin.ACLRoleAssignmentInfo, error) { + if p.aclService == nil { + return nil, fmt.Errorf("ACL service not available") + } + assignments, err := p.aclService.ListPrincipalRoles(ctx, assigneeType, assigneeID) + if err != nil { + return nil, fmt.Errorf("failed to list principal roles: %w", err) + } + result := make([]*admin.ACLRoleAssignmentInfo, 0, len(assignments)) + for _, a := range assignments { + result = append(result, roleAssignmentToAdmin(a)) + } + return result, nil +} + +func (p *GatewayStateProvider) ExplainACLAccess(ctx context.Context, principalType, principalID, resourceType, resourceID string, requiredLevel int, callerType, callerID string) (*admin.ACLAccessExplanationInfo, error) { + if p.aclService == nil { + return nil, fmt.Errorf("ACL service not available") + } + exp, err := p.aclService.ExplainAccess(ctx, principalType, principalID, resourceType, resourceID, requiredLevel, callerType, callerID) + if err != nil { + return nil, fmt.Errorf("failed to explain access: %w", err) + } + out := &admin.ACLAccessExplanationInfo{ + Principal: exp.Principal, + Subjects: exp.Subjects, + } + for _, c := range exp.Contributions { + out.Contributions = append(out.Contributions, &admin.ACLAccessContributionInfo{ + Subject: c.Subject, + RuleID: c.RuleID, + AccessLevel: c.AccessLevel, + Resource: c.Resource, + Expired: c.Expired, + }) + } + if exp.Decision != nil { + out.Allowed = exp.Decision.Allowed + out.Decision = exp.Decision.Decision + out.EffectiveLevel = exp.Decision.EffectiveAccessLevel + out.FallbackApplied = exp.Decision.FallbackApplied + out.Reason = exp.Decision.Reason + } + return out, nil +} + +// groupToAdmin converts a store Group to an admin DTO. +func groupToAdmin(g *aclstore.Group) *admin.ACLGroupInfo { + if g == nil { + return nil + } + return &admin.ACLGroupInfo{ + GroupID: g.GroupID, + GroupName: g.GroupName, + Description: g.Description, + CreatedBy: g.CreatedBy, + CreatedAt: g.CreatedAt, + Metadata: g.Metadata, + } +} + +// roleToAdmin converts a store Role to an admin DTO. +func roleToAdmin(r *aclstore.Role) *admin.ACLRoleInfo { + if r == nil { + return nil + } + return &admin.ACLRoleInfo{ + RoleID: r.RoleID, + RoleName: r.RoleName, + Description: r.Description, + CreatedBy: r.CreatedBy, + CreatedAt: r.CreatedAt, + Metadata: r.Metadata, + } +} + +// groupMemberToAdmin converts a store GroupMember to an admin DTO. +func groupMemberToAdmin(m *aclstore.GroupMember) *admin.ACLGroupMemberInfo { + if m == nil { + return nil + } + return &admin.ACLGroupMemberInfo{ + GroupName: m.GroupName, + MemberType: m.MemberType, + MemberID: m.MemberID, + GrantedBy: m.GrantedBy, + GrantedAt: m.GrantedAt, + ExpiresAt: m.ExpiresAt, + } +} + +// roleAssignmentToAdmin converts a store RoleAssignment to an admin DTO. +func roleAssignmentToAdmin(a *aclstore.RoleAssignment) *admin.ACLRoleAssignmentInfo { + if a == nil { + return nil + } + return &admin.ACLRoleAssignmentInfo{ + RoleName: a.RoleName, + AssigneeType: a.AssigneeType, + AssigneeID: a.AssigneeID, + GrantedBy: a.GrantedBy, + GrantedAt: a.GrantedAt, + ExpiresAt: a.ExpiresAt, + } +} + func adminResourceScopeToACL(entries []*admin.ACLAuthorityGrantResourceScope) map[string][]string { if len(entries) == 0 { return nil diff --git a/server/internal/gateway/admin_auth_test.go b/server/internal/gateway/admin_auth_test.go index 6f08b63..8a7b1c7 100644 --- a/server/internal/gateway/admin_auth_test.go +++ b/server/internal/gateway/admin_auth_test.go @@ -10,6 +10,7 @@ package gateway // - Workspace-scoped ACL enforcement for isAllowedACLOp import ( + "context" "testing" pb "github.com/scitrera/aether/api/proto" @@ -212,7 +213,7 @@ func TestIsAllowedACLOp_WorkflowEngine_ReturnsTrue(t *testing.T) { client := newAdminTestClient(stream, models.PrincipalWorkflowEngine) aclOp := &pb.ACLOperation{Op: pb.ACLOperation_LIST_RULES} - if !s.isAllowedACLOp(client, client.Identity, aclOp) { + if !s.isAllowedACLOp(context.Background(), client, client.Identity, aclOp) { t.Error("expected isAllowedACLOp to return true for WorkflowEngine") } } @@ -223,7 +224,7 @@ func TestIsAllowedACLOp_Orchestrator_ReturnsTrue(t *testing.T) { client := newAdminTestClient(stream, models.PrincipalOrchestrator) aclOp := &pb.ACLOperation{Op: pb.ACLOperation_GRANT} - if !s.isAllowedACLOp(client, client.Identity, aclOp) { + if !s.isAllowedACLOp(context.Background(), client, client.Identity, aclOp) { t.Error("expected isAllowedACLOp to return true for Orchestrator") } } @@ -245,7 +246,7 @@ func TestIsAllowedACLOp_Agent_NoACL_NonWorkspaceResource_ReturnsFalse(t *testing ResourceId: "admin/*", }, } - if s.isAllowedACLOp(client, client.Identity, aclOp) { + if s.isAllowedACLOp(context.Background(), client, client.Identity, aclOp) { t.Error("expected isAllowedACLOp to return false for Agent on non-workspace resource") } } @@ -257,7 +258,7 @@ func TestIsAllowedACLOp_Agent_NoACL_NoFilter_ReturnsFalse(t *testing.T) { // ACL op with no filter → no workspace derivable → requires global admin aclOp := &pb.ACLOperation{Op: pb.ACLOperation_LIST_RULES} - if s.isAllowedACLOp(client, client.Identity, aclOp) { + if s.isAllowedACLOp(context.Background(), client, client.Identity, aclOp) { t.Error("expected isAllowedACLOp to return false for Agent with no filter") } } @@ -275,7 +276,7 @@ func TestIsAllowedACLOp_Agent_NoACL_WorkspaceResource_ReturnsFalse(t *testing.T) ResourceId: "my-workspace", }, } - if s.isAllowedACLOp(client, client.Identity, aclOp) { + if s.isAllowedACLOp(context.Background(), client, client.Identity, aclOp) { t.Error("expected isAllowedACLOp to return false for Agent with no ACL service") } } @@ -299,7 +300,7 @@ func TestIsAllowedACLOp_Agent_NoACL_GrantWorkspace_ReturnsFalse(t *testing.T) { AccessLevel: 10, // Read }, } - if s.isAllowedACLOp(client, client.Identity, aclOp) { + if s.isAllowedACLOp(context.Background(), client, client.Identity, aclOp) { t.Error("expected isAllowedACLOp to return false for Agent GRANT with no ACL service") } } @@ -321,7 +322,7 @@ func TestIsAllowedACLOp_NonWorkspaceResource_SendsACLError(t *testing.T) { AccessLevel: 40, }, } - result := s.isAllowedACLOp(client, client.Identity, aclOp) + result := s.isAllowedACLOp(context.Background(), client, client.Identity, aclOp) if result { t.Error("expected isAllowedACLOp to return false for non-workspace resource") } diff --git a/server/internal/gateway/admin_provider.go b/server/internal/gateway/admin_provider.go index df8a3e0..aa886f8 100644 --- a/server/internal/gateway/admin_provider.go +++ b/server/internal/gateway/admin_provider.go @@ -310,10 +310,12 @@ func formatDuration(d time.Duration) string { // Ensure GatewayStateProvider implements StateProvider var _ admin.StateProvider = (*GatewayStateProvider)(nil) -// Compile-time assertions: both KV backends must satisfy KVReadWriter so that -// callers can pass either *kv.Store (Redis/full mode) or *kv.BadgerKVStore -// (Badger/lite mode) to NewGatewayStateProvider without a nil placeholder. +// Compile-time assertions: all KV backends must satisfy KVReadWriter so that +// callers can pass *kv.Store (Redis/full mode), *kv.BadgerKVStore (single-node +// lite mode), or *kv.JetStreamKVStore (cluster lite mode) to +// NewGatewayStateProvider without a nil placeholder. var ( _ KVReadWriter = (*kv.Store)(nil) _ KVReadWriter = (*kv.BadgerKVStore)(nil) + _ KVReadWriter = (*kv.JetStreamKVStore)(nil) ) diff --git a/server/internal/gateway/admin_tasks.go b/server/internal/gateway/admin_tasks.go index 491a9bd..2ecd398 100644 --- a/server/internal/gateway/admin_tasks.go +++ b/server/internal/gateway/admin_tasks.go @@ -46,6 +46,8 @@ func (p *GatewayStateProvider) GetTasks(ctx context.Context, filter *admin.TaskF taskFilter.AuthorityGrantID = filter.AuthorityGrantID taskFilter.RootAuthorityGrantID = filter.RootAuthorityGrantID taskFilter.ParentTaskID = filter.ParentTaskID + taskFilter.Priority = filter.Priority + taskFilter.MinPriority = filter.MinPriority } records, err := p.taskStore.ListTasks(ctx, taskFilter) @@ -79,6 +81,7 @@ func taskToAdminInfo(r *tasks.Task) *admin.TaskInfo { TaskID: r.TaskID, TaskType: r.TaskType, TaskClass: r.TaskClass, + Priority: int32(r.Priority), DisconnectedAt: r.DisconnectedAt, GraceWindowMs: r.GraceWindowMs, Status: string(r.Status), diff --git a/server/internal/gateway/audit_coalesce.go b/server/internal/gateway/audit_coalesce.go new file mode 100644 index 0000000..99471ba --- /dev/null +++ b/server/internal/gateway/audit_coalesce.go @@ -0,0 +1,177 @@ +package gateway + +import ( + "sync" + "time" + + "github.com/scitrera/aether/internal/audit" +) + +// auditCoalescer suppresses bursts of identical successful message-route / +// proxy-route audit events so a high-volume stream (chat token streaming emits +// hundreds–thousands of OpMessageReceived/OpMessageRouted rows per turn) is +// recorded once per window — the first event is the authorization record and +// the rest are dropped. This is the streaming-chatter trim that keeps +// comprehensive_audit_log from growing unbounded. +// +// Design constraints (this runs on the message hot path across many goroutines): +// - Concurrency-safe via a fixed array of independently-locked shards. The +// critical section is a single map lookup + optional insert. +// - Bounded memory: each shard's map is capped at coalesceMaxEntriesPerShard. +// When an insert would exceed the cap the shard is first swept of entries +// older than the window (lazy eviction); if still full, the single oldest +// entry is evicted. No entry ever outlives the window by more than one +// insert cycle on a busy shard, and the total map size is hard-capped +// regardless of traffic, so there is no unbounded-growth path. +// - Fail toward auditing: only a small allowlist of high-volume SUCCESSFUL +// ops is ever coalesced. Every failure, denial, auth/task/kv/control event, +// and any op not on the allowlist is always written. +// +// Disabled mode: when the configured window is <= 0 the constructor returns +// nil and callers treat a nil *auditCoalescer as a zero-overhead passthrough +// (every event is written, exactly as before this type existed). +type auditCoalescer struct { + window time.Duration + shards [coalesceShardCount]coalesceShard +} + +// coalesceShardCount is the fixed number of independently-locked shards. A +// power of two so the mask in shardIndex is a cheap bitwise AND. +const coalesceShardCount = 64 + +// coalesceMaxEntriesPerShard caps the live keys per shard. With 64 shards the +// worst-case footprint is ~128K keys before evict-on-insert kicks in — small +// (each entry is a short string + a time.Time) and, crucially, hard-bounded. +const coalesceMaxEntriesPerShard = 2048 + +type coalesceShard struct { + mu sync.Mutex + last map[string]time.Time // key -> time the authorization record was written +} + +// coalescableOps is the allowlist of high-volume SUCCESSFUL routing/receive +// operations whose within-window repeats are suppressed. These are the +// per-message and per-proxy-request events that dominate audit volume during +// chat streaming; the first occurrence per (sender, target, op) key is the +// authorization record and later identical events add no security signal. +// +// Deliberately EXCLUDED (never coalesced, so they are always audited): +// - Any success=false event (route/proxy failures, denials) — gated below, +// not by op. +// - OpMessageRouteFailed / OpProxyHttpFailed / OpTunnelOpenFailed. +// - Tunnel lifecycle (OpTunnelOpened / OpTunnelClosed): these are discrete, +// comparatively low-volume authorization events (one per tunnel, not one +// per byte/chunk — the per-stream close is OpProxyHttpStreamClosed), so per +// the "when in doubt, do not coalesce" rule they are kept fully audited. +// - Auth, identity, task, KV, ACL, admin, control ops — not present here. +var coalescableOps = map[string]struct{}{ + audit.OpMessageReceived: {}, + audit.OpMessageRouted: {}, + audit.OpProxyHttpRouted: {}, +} + +// newAuditCoalescer builds a coalescer for the given window. A window <= 0 +// disables coalescing entirely: the constructor returns nil and the caller's +// nil check makes auditLog a zero-overhead passthrough. +func newAuditCoalescer(window time.Duration) *auditCoalescer { + if window <= 0 { + return nil + } + c := &auditCoalescer{window: window} + for i := range c.shards { + c.shards[i].last = make(map[string]time.Time) + } + return c +} + +// shouldLog reports whether event must be written to the audit log now. +// +// It returns true for every event that is not a coalescable high-volume +// success event (always audit). For a coalescable event it returns true only +// for the first occurrence of the (actor, target, op) key within the window — +// the authorization record — and false for subsequent identical events until +// the window elapses, at which point the next event re-admits (re-stamped and +// written). +// +// A nil receiver (coalescing disabled) always returns true. +func (c *auditCoalescer) shouldLog(event *audit.AuditEvent) bool { + if c == nil || event == nil { + return true + } + + // Fail toward auditing: never coalesce failures or non-allowlisted ops. + if !event.Success { + return true + } + if _, ok := coalescableOps[event.Operation]; !ok { + return true + } + + // Key = (sender identity, target topic, op). ActorID is the sender and + // ResourceID is the target topic for message/proxy events (see + // audit.NewMessageEvent). The NUL separators keep the composite key + // unambiguous across field boundaries. + key := event.ActorID + "\x00" + event.ResourceID + "\x00" + event.Operation + now := time.Now() + sh := &c.shards[shardIndex(key)] + + sh.mu.Lock() + defer sh.mu.Unlock() + + if last, ok := sh.last[key]; ok && now.Sub(last) < c.window { + return false // within window — suppress this repeat + } + + // First in window (or the prior record has expired): this is a fresh + // authorization record we must keep. Enforce the per-shard bound before + // inserting so the map can never grow without limit. + if len(sh.last) >= coalesceMaxEntriesPerShard { + c.evictExpiredLocked(sh, now) + if len(sh.last) >= coalesceMaxEntriesPerShard { + evictOldestLocked(sh) + } + } + sh.last[key] = now + return true +} + +// evictExpiredLocked removes every entry whose window has elapsed. Caller must +// hold sh.mu. +func (c *auditCoalescer) evictExpiredLocked(sh *coalesceShard, now time.Time) { + for k, t := range sh.last { + if now.Sub(t) >= c.window { + delete(sh.last, k) + } + } +} + +// evictOldestLocked removes the single oldest entry. Only reached when a shard +// is at cap and no entries have expired — a pathological all-fresh-keys burst. +// Caller must hold sh.mu. +func evictOldestLocked(sh *coalesceShard) { + var oldestKey string + var oldestTime time.Time + first := true + for k, t := range sh.last { + if first || t.Before(oldestTime) { + oldestKey, oldestTime, first = k, t, false + } + } + if !first { + delete(sh.last, oldestKey) + } +} + +// shardIndex maps a key to a shard via FNV-1a, masked to the shard count. +func shardIndex(key string) uint32 { + const ( + offset32 = 2166136261 + prime32 = 16777619 + ) + var h uint32 = offset32 + for i := 0; i < len(key); i++ { + h ^= uint32(key[i]) + h *= prime32 + } + return h & (coalesceShardCount - 1) +} diff --git a/server/internal/gateway/audit_coalesce_test.go b/server/internal/gateway/audit_coalesce_test.go new file mode 100644 index 0000000..38d0060 --- /dev/null +++ b/server/internal/gateway/audit_coalesce_test.go @@ -0,0 +1,218 @@ +package gateway + +import ( + "context" + "fmt" + "sync" + "testing" + "time" + + "github.com/google/uuid" + "github.com/scitrera/aether/internal/audit" +) + +// coalescableEvent builds a successful OpMessageReceived event for the given +// sender/target — the canonical high-volume coalescable case. +func coalescableEvent(sender, target string) *audit.AuditEvent { + return audit.NewMessageEvent("agent", sender, audit.OpMessageReceived, target, "ws", uuid.Nil, true, "", nil) +} + +func TestAuditCoalescer_FirstPassesRepeatDropped(t *testing.T) { + c := newAuditCoalescer(time.Minute) + ev := coalescableEvent("ag::ws::a::1", "ag::ws::b::1") + + if !c.shouldLog(ev) { + t.Fatal("first event in window should pass") + } + if c.shouldLog(ev) { + t.Fatal("second identical event in window should be dropped") + } + if c.shouldLog(ev) { + t.Fatal("third identical event in window should be dropped") + } +} + +func TestAuditCoalescer_DifferentKeyPasses(t *testing.T) { + c := newAuditCoalescer(time.Minute) + + // Different sender. + if !c.shouldLog(coalescableEvent("ag::ws::a::1", "ag::ws::b::1")) { + t.Fatal("first key should pass") + } + if !c.shouldLog(coalescableEvent("ag::ws::a::2", "ag::ws::b::1")) { + t.Fatal("different sender should pass") + } + // Different target. + if !c.shouldLog(coalescableEvent("ag::ws::a::1", "ag::ws::b::2")) { + t.Fatal("different target should pass") + } + // Different op (but still coalescable) — distinct key. + ev := audit.NewMessageEvent("agent", "ag::ws::a::1", audit.OpMessageRouted, "ag::ws::b::1", "ws", uuid.Nil, true, "", nil) + if !c.shouldLog(ev) { + t.Fatal("different op should pass") + } +} + +func TestAuditCoalescer_WindowExpiryReadmits(t *testing.T) { + c := newAuditCoalescer(20 * time.Millisecond) + ev := coalescableEvent("ag::ws::a::1", "ag::ws::b::1") + + if !c.shouldLog(ev) { + t.Fatal("first event should pass") + } + if c.shouldLog(ev) { + t.Fatal("second within-window event should be dropped") + } + + time.Sleep(40 * time.Millisecond) + + if !c.shouldLog(ev) { + t.Fatal("event after window expiry should be re-admitted") + } + // And immediately suppressed again in the fresh window. + if c.shouldLog(ev) { + t.Fatal("repeat in the fresh window should be dropped") + } +} + +func TestAuditCoalescer_NonCoalescableOpsAlwaysPass(t *testing.T) { + c := newAuditCoalescer(time.Minute) + + // Tunnel + non-message ops are never coalesced. + nonCoalescable := []string{ + audit.OpTunnelOpened, + audit.OpTunnelClosed, + audit.OpMessageDelivered, + audit.OpKVPut, + audit.OpTaskCreate, + audit.OpAuthMTLSSuccess, + audit.OpConnectionEstablished, + } + for _, op := range nonCoalescable { + ev := audit.NewMessageEvent("agent", "ag::ws::a::1", op, "ag::ws::b::1", "ws", uuid.Nil, true, "", nil) + for i := 0; i < 5; i++ { + if !c.shouldLog(ev) { + t.Fatalf("op %q must always pass (iteration %d)", op, i) + } + } + } +} + +func TestAuditCoalescer_FailureAlwaysPasses(t *testing.T) { + c := newAuditCoalescer(time.Minute) + + // success=false on an otherwise-coalescable op must always be audited. + ev := audit.NewMessageEvent("agent", "ag::ws::a::1", audit.OpMessageReceived, "ag::ws::b::1", "ws", uuid.Nil, false, "boom", nil) + for i := 0; i < 5; i++ { + if !c.shouldLog(ev) { + t.Fatalf("failure event must always pass (iteration %d)", i) + } + } +} + +func TestAuditCoalescer_DisabledWindowIsPassthrough(t *testing.T) { + // window <= 0 -> nil coalescer -> passthrough (every event logged). + if c := newAuditCoalescer(0); c != nil { + t.Fatal("window 0 should produce a nil coalescer") + } + if c := newAuditCoalescer(-1 * time.Second); c != nil { + t.Fatal("negative window should produce a nil coalescer") + } + + var c *auditCoalescer // nil + ev := coalescableEvent("ag::ws::a::1", "ag::ws::b::1") + for i := 0; i < 5; i++ { + if !c.shouldLog(ev) { + t.Fatalf("nil coalescer must pass every event (iteration %d)", i) + } + } +} + +func TestAuditCoalescer_BoundedUnderManyKeys(t *testing.T) { + c := newAuditCoalescer(time.Hour) // long window so nothing expires + + // Push far more distinct keys than the total cap so eviction must kick in. + total := coalesceShardCount * coalesceMaxEntriesPerShard * 3 + for i := 0; i < total; i++ { + ev := coalescableEvent(fmt.Sprintf("ag::ws::a::%d", i), "ag::ws::b::1") + c.shouldLog(ev) + } + + // Every shard must be at or under the per-shard cap — proof the map cannot + // grow without bound. + for i := range c.shards { + sh := &c.shards[i] + sh.mu.Lock() + n := len(sh.last) + sh.mu.Unlock() + if n > coalesceMaxEntriesPerShard { + t.Fatalf("shard %d holds %d entries, exceeds cap %d", i, n, coalesceMaxEntriesPerShard) + } + } +} + +func TestAuditCoalescer_ConcurrentAccess(t *testing.T) { + c := newAuditCoalescer(time.Minute) + + var wg sync.WaitGroup + for g := 0; g < 16; g++ { + wg.Add(1) + go func(g int) { + defer wg.Done() + for i := 0; i < 1000; i++ { + ev := coalescableEvent(fmt.Sprintf("ag::ws::a::%d", (g*7+i)%50), "ag::ws::b::1") + c.shouldLog(ev) + } + }(g) + } + wg.Wait() + // Success = no race/panic (run under -race). No assertion needed beyond + // completion. +} + +// The gateway-level tests below reuse captureAuditStore (defined in +// denial_audit_test.go) — a minimal auditstore.Store that records LogEvent +// calls — to observe auditLog's coalescing gate end-to-end. + +func TestGatewayAuditLog_SuppressesRepeatWithinWindow(t *testing.T) { + store := &captureAuditStore{} + s := &GatewayServer{ + auditLogger: store, + auditCoalescer: newAuditCoalescer(time.Minute), + } + ctx := context.Background() + + ev := coalescableEvent("ag::ws::a::1", "ag::ws::b::1") + s.auditLog(ctx, ev) + s.auditLog(ctx, ev) + s.auditLog(ctx, ev) + + if got := len(store.captured()); got != 1 { + t.Fatalf("auditLog wrote %d events, want 1 (2 suppressed)", got) + } + + // A failure event for the same key is always written. + fail := audit.NewMessageEvent("agent", "ag::ws::a::1", audit.OpMessageReceived, "ag::ws::b::1", "ws", uuid.Nil, false, "boom", nil) + s.auditLog(ctx, fail) + if got := len(store.captured()); got != 2 { + t.Fatalf("auditLog wrote %d events, want 2 (failure not suppressed)", got) + } +} + +func TestGatewayAuditLog_DisabledCoalescerLogsEvery(t *testing.T) { + store := &captureAuditStore{} + s := &GatewayServer{ + auditLogger: store, + auditCoalescer: nil, // coalescing disabled + } + ctx := context.Background() + + ev := coalescableEvent("ag::ws::a::1", "ag::ws::b::1") + s.auditLog(ctx, ev) + s.auditLog(ctx, ev) + s.auditLog(ctx, ev) + + if got := len(store.captured()); got != 3 { + t.Fatalf("auditLog wrote %d events, want 3 (no coalescing)", got) + } +} diff --git a/server/internal/gateway/audit_helpers.go b/server/internal/gateway/audit_helpers.go index 24808c4..416d48e 100644 --- a/server/internal/gateway/audit_helpers.go +++ b/server/internal/gateway/audit_helpers.go @@ -7,8 +7,20 @@ import ( ) // auditLog logs an audit event asynchronously if the audit logger is configured. +// +// A coalescing gate runs before the write: when an audit coalescer is +// configured (audit_coalesce_window > 0), a burst of identical successful +// message-route / proxy-route events from the same sender→target is recorded +// once per window (the first = the authorization record) and the rest are +// suppressed. Every failure, denial, and non-high-volume op is always written. +// When no coalescer is configured (window = 0) s.auditCoalescer is nil and +// shouldLog is a zero-overhead passthrough. func (s *GatewayServer) auditLog(ctx context.Context, event *audit.AuditEvent) { - if s.auditLogger != nil { - s.auditLogger.LogEvent(ctx, event) + if s.auditLogger == nil { + return } + if !s.auditCoalescer.shouldLog(event) { + return + } + s.auditLogger.LogEvent(ctx, event) } diff --git a/server/internal/gateway/auth.go b/server/internal/gateway/auth.go index 6fb03cd..9772a35 100644 --- a/server/internal/gateway/auth.go +++ b/server/internal/gateway/auth.go @@ -18,6 +18,6 @@ func (s *GatewayServer) resolveConnectionIdentity(ctx context.Context, init *pb. } // authenticateCredentials delegates to the AuthHandler. -func (s *GatewayServer) authenticateCredentials(ctx context.Context, init *pb.InitConnection, identity models.Identity, hasCertificate bool) (string, models.Identity, error) { - return s.authHandler.authenticateCredentials(ctx, init, identity, hasCertificate) +func (s *GatewayServer) authenticateCredentials(ctx context.Context, init *pb.InitConnection, identity models.Identity, hasCertificate bool, isAnonymous bool) (string, models.Identity, error) { + return s.authHandler.authenticateCredentials(ctx, init, identity, hasCertificate, isAnonymous) } diff --git a/server/internal/gateway/auth_apikey_binding_test.go b/server/internal/gateway/auth_apikey_binding_test.go new file mode 100644 index 0000000..2e6f0d2 --- /dev/null +++ b/server/internal/gateway/auth_apikey_binding_test.go @@ -0,0 +1,284 @@ +package gateway + +// Tests for the anonymous-cert + scoped user API-key connection-auth path in +// authenticateCredentials. +// +// These cover the claim<->key binding security gate added so the platform +// server can open per-user Aether connections using an ANONYMOUS transport +// cert plus a per-user API key, authenticated AS the real user: +// +// - anonymous-cert + matching user key -> authenticated as the user +// - anonymous-cert + mismatched key id -> PermissionDenied +// - anonymous-cert + mismatched type -> PermissionDenied +// - anonymous-cert + NO credentials (user) -> Unauthenticated [Critical #1] +// - anonymous-cert + NO credentials (agent) -> Unauthenticated [Critical #1] +// - in-process bufconn + NO credentials -> allowed (exemption) +// - isImpersonablePrincipal logic -> direct unit test +// - no-cert path applies the SAME enforcement (no divergence) +// - non-anonymous cert path is UNCHANGED (key is ignored; cert is authority) + +import ( + "context" + "testing" + + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + + pb "github.com/scitrera/aether/api/proto" + "github.com/scitrera/aether/internal/auth" + "github.com/scitrera/aether/pkg/models" +) + +// fakeAPIKeyAuthenticator returns a fixed AuthResult for any credentials that +// contain an api_key, and (nil, nil) otherwise. It lets us drive the binding +// logic in authenticateCredentials without a real token store. +type fakeAPIKeyAuthenticator struct { + result *auth.AuthResult +} + +func (f *fakeAPIKeyAuthenticator) Name() string { return "api_key" } + +func (f *fakeAPIKeyAuthenticator) Authenticate(_ context.Context, creds map[string]string) (*auth.AuthResult, error) { + if creds[auth.CredKeyAPIKey] == "" && creds[auth.CredKeyXAPIKey] == "" { + return nil, nil + } + return f.result, nil +} + +// userKeyResult builds an AuthResult shaped like the real api_key +// authenticator's output for a user principal. +func userKeyResult(userID string) *auth.AuthResult { + return &auth.AuthResult{ + Authenticated: true, + Identity: models.Identity{Type: models.PrincipalUser, ID: userID}, + Method: "api_key", + Metadata: map[string]interface{}{ + "token_id": "tok-1", + "workspace_patterns": []string{"*"}, + }, + } +} + +func newBindingAuthHandler(result *auth.AuthResult) *AuthHandler { + composite := auth.NewCompositeAuthenticator(&fakeAPIKeyAuthenticator{result: result}) + // mtlsRequired=false, mode strict — irrelevant here since authenticateMTLS + // is not exercised by these tests; we call authenticateCredentials directly. + return newAuthHandler(composite, false, MTLSModeStrict, nil, nil) +} + +func userInit() *pb.InitConnection { + return &pb.InitConnection{ + ClientType: &pb.InitConnection_User{ + User: &pb.UserIdentity{UserId: "drew", WindowId: "win-1"}, + }, + Credentials: map[string]string{auth.CredKeyAPIKey: "secret"}, + } +} + +func TestAuthCreds_AnonymousCert_MatchingUserKey_AuthenticatesAsUser(t *testing.T) { + h := newBindingAuthHandler(userKeyResult("drew")) + claim := models.Identity{Type: models.PrincipalUser, ID: "drew", Specifier: "win-1"} + + _, ident, err := h.authenticateCredentials(context.Background(), userInit(), claim, true /*hasCertificate*/, true /*isAnonymous*/) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if ident.Type != models.PrincipalUser || ident.ID != "drew" { + t.Fatalf("expected authenticated user drew, got type=%s id=%s", ident.Type, ident.ID) + } + if ident.Specifier != "win-1" { + t.Errorf("expected window specifier preserved from claim, got %q", ident.Specifier) + } +} + +func TestAuthCreds_AnonymousCert_MismatchedKeyID_Denied(t *testing.T) { + // Key authenticates user "alice" but the connection claims user "drew". + h := newBindingAuthHandler(userKeyResult("alice")) + claim := models.Identity{Type: models.PrincipalUser, ID: "drew", Specifier: "win-1"} + + _, _, err := h.authenticateCredentials(context.Background(), userInit(), claim, true, true) + if err == nil { + t.Fatal("expected PermissionDenied for mismatched user key, got nil") + } + if status.Code(err) != codes.PermissionDenied { + t.Errorf("expected PermissionDenied, got %v", status.Code(err)) + } +} + +func TestAuthCreds_AnonymousCert_MismatchedType_Denied(t *testing.T) { + // Key authenticates a Service principal but the connection claims a User. + result := &auth.AuthResult{ + Authenticated: true, + Identity: models.Identity{Type: models.PrincipalService, ID: "frontend"}, + Method: "api_key", + } + h := newBindingAuthHandler(result) + claim := models.Identity{Type: models.PrincipalUser, ID: "drew", Specifier: "win-1"} + + _, _, err := h.authenticateCredentials(context.Background(), userInit(), claim, true, true) + if err == nil { + t.Fatal("expected PermissionDenied for principal-type mismatch, got nil") + } + if status.Code(err) != codes.PermissionDenied { + t.Errorf("expected PermissionDenied, got %v", status.Code(err)) + } +} + +func TestAuthCreds_NoCert_MatchingUserKey_AuthenticatesAsUser(t *testing.T) { + // The no-cert path must apply the SAME binding as the anonymous-cert path. + h := newBindingAuthHandler(userKeyResult("drew")) + claim := models.Identity{Type: models.PrincipalUser, ID: "drew", Specifier: "win-1"} + + _, ident, err := h.authenticateCredentials(context.Background(), userInit(), claim, false /*hasCertificate*/, false /*isAnonymous*/) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if ident.Type != models.PrincipalUser || ident.ID != "drew" { + t.Fatalf("expected authenticated user drew, got type=%s id=%s", ident.Type, ident.ID) + } +} + +func TestAuthCreds_NoCert_MismatchedKeyID_Denied(t *testing.T) { + h := newBindingAuthHandler(userKeyResult("alice")) + claim := models.Identity{Type: models.PrincipalUser, ID: "drew", Specifier: "win-1"} + + _, _, err := h.authenticateCredentials(context.Background(), userInit(), claim, false, false) + if err == nil { + t.Fatal("expected PermissionDenied for mismatched user key on no-cert path, got nil") + } + if status.Code(err) != codes.PermissionDenied { + t.Errorf("expected PermissionDenied, got %v", status.Code(err)) + } +} + +func TestAuthCreds_NonAnonymousCert_KeyIgnored_IdentityUnchanged(t *testing.T) { + // On the strict/semi-strict cert path (hasCertificate=true, isAnonymous=false) + // the api_key must NOT rebind identity: the cert is authoritative. Even a + // mismatched key must leave the cert-derived identity untouched and NOT + // fail (the key is simply not consulted for binding). + h := newBindingAuthHandler(userKeyResult("alice")) + certIdentity := models.Identity{Type: models.PrincipalUser, ID: "drew", Specifier: "win-1"} + + _, ident, err := h.authenticateCredentials(context.Background(), userInit(), certIdentity, true /*hasCertificate*/, false /*isAnonymous*/) + if err != nil { + t.Fatalf("unexpected error on non-anonymous cert path: %v", err) + } + if ident.Type != models.PrincipalUser || ident.ID != "drew" { + t.Fatalf("expected cert identity drew unchanged, got type=%s id=%s", ident.Type, ident.ID) + } +} + +// --- Critical #1 regression tests: no-credential reject on anonymous path --- + +// noCreds returns an InitConnection that carries a user claim but NO credentials +// at all — simulating a caller that omits the api_key field entirely. +func noCreds() *pb.InitConnection { + return &pb.InitConnection{ + ClientType: &pb.InitConnection_User{ + User: &pb.UserIdentity{UserId: "drew", WindowId: "win-1"}, + }, + Credentials: map[string]string{}, // deliberately empty + } +} + +// noCredsAgent returns an InitConnection with an Agent claim and no credentials. +func noCredsAgent() *pb.InitConnection { + return &pb.InitConnection{ + ClientType: &pb.InitConnection_Agent{ + Agent: &pb.AgentIdentity{ + Workspace: "prod", + Implementation: "classifier", + Specifier: "v2", + }, + }, + Credentials: map[string]string{}, + } +} + +// TestAuthCreds_AnonymousCert_NoCreds_User_Rejected is the CORE regression for +// Critical #1: an external anonymous-cert connection claiming a User principal +// with no api_key credential must be rejected Unauthenticated, not succeed. +// Previously the composite returned (nil,nil), the binding block was skipped, +// and authenticateCredentials returned success with an unauthenticated identity. +func TestAuthCreds_AnonymousCert_NoCreds_User_Rejected(t *testing.T) { + h := newBindingAuthHandler(userKeyResult("drew")) // authenticator is wired but no key presented + claim := models.Identity{Type: models.PrincipalUser, ID: "drew", Specifier: "win-1"} + + _, _, err := h.authenticateCredentials(context.Background(), noCreds(), claim, true /*hasCertificate*/, true /*isAnonymous*/) + if err == nil { + t.Fatal("expected Unauthenticated when no credential presented on anonymous-cert path, got nil") + } + if status.Code(err) != codes.Unauthenticated { + t.Errorf("expected Unauthenticated, got %v", status.Code(err)) + } +} + +// TestAuthCreds_AnonymousCert_NoCreds_Agent_Rejected confirms the reject is not +// user-only: an Agent claim on an anonymous-cert path with no credential is also +// rejected. All impersonable principal types must require a credential. +func TestAuthCreds_AnonymousCert_NoCreds_Agent_Rejected(t *testing.T) { + h := newBindingAuthHandler(userKeyResult("drew")) + claim := models.Identity{ + Type: models.PrincipalAgent, + Workspace: "prod", + Implementation: "classifier", + Specifier: "v2", + } + + _, _, err := h.authenticateCredentials(context.Background(), noCredsAgent(), claim, true /*hasCertificate*/, true /*isAnonymous*/) + if err == nil { + t.Fatal("expected Unauthenticated when agent claim presented with no credential on anonymous-cert path, got nil") + } + if status.Code(err) != codes.Unauthenticated { + t.Errorf("expected Unauthenticated, got %v", status.Code(err)) + } +} + +// TestAuthCreds_InProcess_NoCreds_Allowed verifies the in-process bufconn +// exemption: IsInProcessConn(ctx)==true bypasses the credential requirement so +// the embedded workflow engine (which never carries api-key credentials) can +// connect without triggering Unauthenticated. +func TestAuthCreds_InProcess_NoCreds_Allowed(t *testing.T) { + h := newBindingAuthHandler(userKeyResult("drew")) + claim := models.Identity{Type: models.PrincipalUser, ID: "drew", Specifier: "win-1"} + + // Inject the in-process context marker the same way InProcessUnaryInterceptor does. + inProcCtx := context.WithValue(context.Background(), inProcessConnKey{}, true) + + _, ident, err := h.authenticateCredentials(inProcCtx, noCreds(), claim, true /*hasCertificate*/, true /*isAnonymous*/) + if err != nil { + t.Fatalf("in-process connection without credentials must not be rejected: %v", err) + } + // Identity should be the unmodified claim (no key to bind from). + if ident.Type != models.PrincipalUser || ident.ID != "drew" { + t.Errorf("expected claim identity preserved for in-process conn, got type=%s id=%s", ident.Type, ident.ID) + } +} + +// TestIsImpersonablePrincipal_Coverage directly unit-tests the helper so that +// the exempt system types and the impersonable worker types are both pinned. +func TestIsImpersonablePrincipal_Coverage(t *testing.T) { + cases := []struct { + pt models.PrincipalType + impersonable bool + }{ + // System principals — exempt + {models.PrincipalWorkflowEngine, false}, + {models.PrincipalOrchestrator, false}, + {models.PrincipalMetricsBridge, false}, + // Empty string — not a real type, must not require a credential + {"", false}, + // Impersonable worker/user principals — all must require a credential + {models.PrincipalUser, true}, + {models.PrincipalAgent, true}, + {models.PrincipalService, true}, + {models.PrincipalTask, true}, + {models.PrincipalBridge, true}, + } + for _, tc := range cases { + got := isImpersonablePrincipal(tc.pt) + if got != tc.impersonable { + t.Errorf("isImpersonablePrincipal(%q) = %v, want %v", tc.pt, got, tc.impersonable) + } + } +} diff --git a/server/internal/gateway/auth_handler.go b/server/internal/gateway/auth_handler.go index 1bdc8dd..ef1b66b 100644 --- a/server/internal/gateway/auth_handler.go +++ b/server/internal/gateway/auth_handler.go @@ -3,7 +3,10 @@ package gateway import ( "context" "fmt" + "os" "path/filepath" + "sync" + "time" "github.com/google/uuid" pb "github.com/scitrera/aether/api/proto" @@ -21,6 +24,32 @@ import ( "google.golang.org/grpc/status" ) +// Per-identity auth-failure throttle tunables. These gate repeated credential +// validation for a single identity to relieve log/audit spam and validation +// load when a stale/zombie client reconnects in a tight loop with a token the +// gateway no longer recognizes. They do not alter the actual token-validation +// or identity-match security logic. +const ( + // authFailThreshold is the number of failures within authFailWindow that + // engages the cooldown. + authFailThreshold = 5 + // authFailWindow is the sliding window over which failures are counted. + authFailWindow = 30 * time.Second + // authFailCooldown is how long an identity is throttled (fast-rejected + // without re-validation) once the threshold is reached. + authFailCooldown = 30 * time.Second + // authFailMapMaxEntries caps the throttle map size to bound memory across + // many transient identities; on overflow, stale records are swept. + authFailMapMaxEntries = 1024 +) + +// authFailRecord tracks recent auth failures for a single identity. +type authFailRecord struct { + failures int + windowStart time.Time + cooldownUntil time.Time +} + // AuthHandler encapsulates authentication and identity resolution concerns for the gateway. // It holds the mTLS configuration, ACL service, composite authenticator, and audit logger // needed to authenticate connections and validate credentials. @@ -35,16 +64,103 @@ type AuthHandler struct { // tokenStore is used to validate orchestration task tokens for agents. // It is set when orchestration is configured and may be nil. tokenStore state.TokenStore + + // authFailMu guards authFailRecords. The gateway may authenticate + // concurrent connects, so the per-identity throttle state must be locked. + authFailMu sync.Mutex + // authFailRecords holds per-identity (keyed by identity.String()) failure + // bookkeeping for the auth-failure throttle. + authFailRecords map[string]*authFailRecord + + // devMode relaxes the anonymous/no-cert credential requirement for + // impersonable principals (Agent/Service/Task/User/Bridge): when true, such + // a connection is admitted WITHOUT a credential, with a warning. Set from + // AETHER_DEV_MODE (the -dev flag) and NEVER true in production. This lets + // local/e2e sidecars connect to a -dev gateway that has no authenticator + // configured; production (dev mode off) still enforces credentials. + devMode bool } // newAuthHandler creates an AuthHandler from gateway server configuration. func newAuthHandler(authenticator *auth.CompositeAuthenticator, mtlsRequired bool, mtlsMode MTLSMode, aclService aclstore.Store, auditLogger auditstore.Store) *AuthHandler { return &AuthHandler{ - authenticator: authenticator, - mtlsRequired: mtlsRequired, - mtlsMode: mtlsMode, - acl: aclService, - auditLogger: auditLogger, + authenticator: authenticator, + mtlsRequired: mtlsRequired, + mtlsMode: mtlsMode, + acl: aclService, + auditLogger: auditLogger, + authFailRecords: make(map[string]*authFailRecord), + devMode: os.Getenv("AETHER_DEV_MODE") == "true", + } +} + +// throttled reports whether the given identity is currently within its +// auth-failure cooldown. When true, callers should fast-reject without +// re-validating credentials and without emitting per-attempt WARNING/audit. +func (h *AuthHandler) throttled(identityKey string, now time.Time) bool { + h.authFailMu.Lock() + defer h.authFailMu.Unlock() + rec, ok := h.authFailRecords[identityKey] + if !ok { + return false + } + return now.Before(rec.cooldownUntil) +} + +// recordAuthFailure records a credential-validation failure for the identity +// and returns true if this failure engaged (newly crossed) the cooldown +// threshold. The window is sliding: failures older than authFailWindow reset +// the count. The caller logs the single throttle-engaged WARNING/audit when +// this returns true. +func (h *AuthHandler) recordAuthFailure(identityKey string, now time.Time) (engaged bool) { + h.authFailMu.Lock() + defer h.authFailMu.Unlock() + + h.evictStaleLocked(now) + + rec, ok := h.authFailRecords[identityKey] + if !ok { + rec = &authFailRecord{} + h.authFailRecords[identityKey] = rec + } + + if rec.windowStart.IsZero() || now.Sub(rec.windowStart) > authFailWindow { + rec.failures = 1 + rec.windowStart = now + } else { + rec.failures++ + } + + if rec.failures >= authFailThreshold && now.After(rec.cooldownUntil) { + rec.cooldownUntil = now.Add(authFailCooldown) + return true + } + return false +} + +// clearAuthFailure removes any throttle bookkeeping for the identity so a +// recovered client is not penalized after a successful authentication. +func (h *AuthHandler) clearAuthFailure(identityKey string) { + h.authFailMu.Lock() + defer h.authFailMu.Unlock() + delete(h.authFailRecords, identityKey) +} + +// evictStaleLocked opportunistically removes records whose window and cooldown +// are both well in the past, bounding memory across many transient identities. +// Callers must hold authFailMu. The sweep only runs when the map exceeds the +// size cap, so the common case adds no work. +func (h *AuthHandler) evictStaleLocked(now time.Time) { + if len(h.authFailRecords) < authFailMapMaxEntries { + return + } + // A record is stale once it is no longer in cooldown and its window has + // fully elapsed; such records carry no live throttle state. + staleBefore := now.Add(-authFailWindow) + for k, rec := range h.authFailRecords { + if now.After(rec.cooldownUntil) && rec.windowStart.Before(staleBefore) { + delete(h.authFailRecords, k) + } } } @@ -95,21 +211,33 @@ func (h *AuthHandler) authenticateMTLS(ctx context.Context) (identity models.Ide return identity, certPrincipalType, true, true, nil } - if h.mtlsMode == MTLSModeStrict { - // Strict mode: Extract full identity from certificate + if h.mtlsMode == MTLSModeStrict || h.mtlsMode == MTLSModeSemiStrict { + // Strict and semi-strict modes: extract the FULL identity from the + // certificate. resolveConnectionIdentity consumes certIdentity wholesale + // in strict mode, and uses the cert's workspace+implementation+type (with + // the InitConnection-supplied specifier) in semi-strict mode. The cert + // identity MUST be populated for the semi-strict validation block to have + // real values to compare against. certIdentity, extractErr := ExtractIdentityFromCertificate(ctx) if extractErr != nil { logging.Logger.Error().Err(extractErr).Msg("mTLS certificate identity extraction failed") h.auditLog(ctx, audit.NewAuthEvent("unknown", "unknown", audit.OpAuthMTLSFailure, "", uuid.New(), false, extractErr.Error(), map[string]interface{}{ - "mtls_mode": "strict", + "mtls_mode": string(h.mtlsMode), })) return identity, certPrincipalType, true, false, status.Error(codes.Unauthenticated, "invalid client certificate") } identity = certIdentity - logging.Logger.Info().Str("identity", identity.String()).Msg("mTLS authenticated identity (strict mode)") - h.auditLog(ctx, audit.NewAuthEvent(string(identity.Type), identity.String(), audit.OpAuthMTLSSuccess, identity.Workspace, uuid.New(), true, "", map[string]interface{}{ - "mtls_mode": "strict", - })) + if h.mtlsMode == MTLSModeSemiStrict { + logging.Logger.Info().Str("identity", identity.String()).Msg("mTLS authenticated identity (semi-strict mode)") + h.auditLog(ctx, audit.NewAuthEvent(string(identity.Type), identity.String(), audit.OpAuthMTLSSuccess, identity.Workspace, uuid.New(), true, "", map[string]interface{}{ + "mtls_mode": "semi-strict", + })) + } else { + logging.Logger.Info().Str("identity", identity.String()).Msg("mTLS authenticated identity (strict mode)") + h.auditLog(ctx, audit.NewAuthEvent(string(identity.Type), identity.String(), audit.OpAuthMTLSSuccess, identity.Workspace, uuid.New(), true, "", map[string]interface{}{ + "mtls_mode": "strict", + })) + } } else { // Relaxed mode: Extract only principal type from certificate principalType, extractErr := ExtractPrincipalTypeFromCertificate(ctx) @@ -305,33 +433,46 @@ func (h *AuthHandler) resolveConnectionIdentity(ctx context.Context, init *pb.In return identity, nil } +// isImpersonablePrincipal reports whether a principal type requires an explicit +// credential on the anonymous-cert / no-cert path. System principals +// (WorkflowEngine, Orchestrator, MetricsBridge) connect without per-user +// credentials by design; they present named mTLS certs in production and are +// granted in-process trust in lite. Every other type (User, Agent, Service, +// Task, Bridge) is "impersonable" — a caller on an unauthenticated transport +// MUST prove they hold the claimed identity via a matching credential. +func isImpersonablePrincipal(t models.PrincipalType) bool { + switch t { + case models.PrincipalWorkflowEngine, models.PrincipalOrchestrator, models.PrincipalMetricsBridge: + return false + default: + // User, Agent, Service, Task, Bridge — all require a credential on + // the anonymous/no-cert path. + return t != "" + } +} + // authenticateCredentials validates task tokens and API key/OAuth credentials. // Returns the associated task ID (if any) and potentially updated identity. -func (h *AuthHandler) authenticateCredentials(ctx context.Context, init *pb.InitConnection, identity models.Identity, hasCertificate bool) (string, models.Identity, error) { +// +// isAnonymous indicates the transport presented an ANONYMOUS mTLS certificate +// (CN="_anonymous", or an in-process bufconn). The anonymous cert provides +// transport security but carries NO auth identity, so — exactly like the +// no-certificate path — the api_key/oauth principal block must run to bind the +// authenticated principal. The strict/semi-strict cert paths (non-anonymous +// cert: isAnonymous=false, hasCertificate=true) deliberately skip that block: +// their identity is already authoritatively bound by the certificate. +// +// SECURITY: on the external anonymous-cert path, impersonable principal types +// MUST present a matching credential. Returning success with an unauthenticated +// identity would leave authorization entirely to the downstream ACL fallback, +// which may be permissive in development/lite deployments. +func (h *AuthHandler) authenticateCredentials(ctx context.Context, init *pb.InitConnection, identity models.Identity, hasCertificate bool, isAnonymous bool) (string, models.Identity, error) { ctx, span := tracing.Tracer.Start(ctx, "gateway.AuthenticateCredentials") defer span.End() - span.SetAttributes(attribute.Bool("has_certificate", hasCertificate)) - - // DIAG: surface exactly what the gateway sees for credentials so we can - // trace where auth takes each path. - if init != nil { - credKeys := make([]string, 0, len(init.Credentials)) - for k := range init.Credentials { - credKeys = append(credKeys, k) - } - tokenLen := 0 - if t, ok := init.Credentials["token"]; ok { - tokenLen = len(t) - } - logging.Logger.Debug(). - Str("identity", identity.String()). - Str("identity_type", string(identity.Type)). - Interface("cred_keys", credKeys). - Int("token_len", tokenLen). - Bool("has_certificate", hasCertificate). - Bool("token_store_configured", h.tokenStore != nil). - Msg("[DIAG] authenticateCredentials entry") - } + span.SetAttributes( + attribute.Bool("has_certificate", hasCertificate), + attribute.Bool("is_anonymous_cert", isAnonymous), + ) // 2.5 Token validation for orchestrated workers // @@ -348,12 +489,34 @@ func (h *AuthHandler) authenticateCredentials(ctx context.Context, init *pb.Init if init != nil { if token, ok := init.Credentials["token"]; ok && token != "" { if h.tokenStore != nil { + identityKey := identity.String() + + // Defense-in-depth throttle: if this identity has recently + // failed token validation repeatedly, fast-reject without + // re-validating and without per-attempt WARNING/audit spam. + // This relieves load from stale/zombie clients reconnecting in + // a tight loop with a token the gateway no longer knows. + if h.throttled(identityKey, time.Now()) { + logging.Logger.Debug().Str("identity", identityKey).Msg("auth throttled: skipping token validation during cooldown") + return "", identity, status.Errorf(codes.Unauthenticated, "auth temporarily throttled after repeated failures") + } + taskToken, err := h.tokenStore.ValidateToken(ctx, token) if err != nil { - logging.Logger.Warn().Err(err).Str("identity", identity.String()).Msg("token validation failed") - h.auditLog(ctx, audit.NewAuthEvent(string(identity.Type), identity.String(), audit.OpAuthTokenValidation, identity.Workspace, uuid.New(), false, err.Error(), map[string]interface{}{ - "reason": "token_validation_failed", - })) + engaged := h.recordAuthFailure(identityKey, time.Now()) + if engaged { + logging.Logger.Warn().Err(err).Str("identity", identityKey).Int("failures", authFailThreshold).Dur("cooldown", authFailCooldown).Msg("auth throttled for identity after repeated failures, cooling down") + h.auditLog(ctx, audit.NewAuthEvent(string(identity.Type), identityKey, audit.OpAuthTokenValidation, identity.Workspace, uuid.New(), false, err.Error(), map[string]interface{}{ + "reason": "auth_throttle_engaged", + "failures": authFailThreshold, + "cooldown_secs": int(authFailCooldown.Seconds()), + })) + } else { + logging.Logger.Warn().Err(err).Str("identity", identityKey).Msg("token validation failed") + h.auditLog(ctx, audit.NewAuthEvent(string(identity.Type), identityKey, audit.OpAuthTokenValidation, identity.Workspace, uuid.New(), false, err.Error(), map[string]interface{}{ + "reason": "token_validation_failed", + })) + } return "", identity, status.Errorf(codes.Unauthenticated, "invalid or revoked token: %v", err) } @@ -372,6 +535,10 @@ func (h *AuthHandler) authenticateCredentials(ctx context.Context, init *pb.Init identity.String()) } + // Token validated OK: a recovered client must not stay + // penalized — clear any throttle bookkeeping for this identity. + h.clearAuthFailure(identityKey) + logging.Logger.Info().Str("identity", identity.String()).Str("task_id", taskToken.TaskID).Str("principal_type", string(identity.Type)).Msg("task token validated") h.auditLog(ctx, audit.NewAuthEvent(string(identity.Type), identity.String(), audit.OpAuthTokenValidation, identity.Workspace, uuid.New(), true, "", map[string]interface{}{ "associated_task": taskToken.TaskID, @@ -382,25 +549,123 @@ func (h *AuthHandler) authenticateCredentials(ctx context.Context, init *pb.Init } // 2.6 API key / OAuth authentication via composite authenticator + // + // The credential-requirement gate (Fix #1) and the composite call share + // the same per-identity throttle as the task-token path so that a flood + // of failed bind attempts from attacker-controlled anonymous connections + // does not drive unbounded log/audit writes. if h.authenticator != nil && init != nil && associatedTaskID == "" { + identityKey := identity.String() + + // Check throttle before calling the authenticator, just as the + // task-token path does. A connection presenting a bad api_key in a + // tight loop must not drive unbounded authenticator calls. + if h.throttled(identityKey, time.Now()) { + logging.Logger.Debug().Str("identity", identityKey).Msg("auth throttled: skipping composite auth during cooldown") + return "", identity, status.Errorf(codes.Unauthenticated, "auth temporarily throttled after repeated failures") + } + authResult, authErr := h.authenticator.Authenticate(ctx, init.Credentials) if authErr != nil { - logging.Logger.Warn().Err(authErr).Str("identity", identity.String()).Msg("credential authentication failed") - h.auditLog(ctx, audit.NewAuthEvent(string(identity.Type), identity.String(), audit.OpAuthTokenValidation, identity.Workspace, uuid.New(), false, authErr.Error(), map[string]interface{}{ - "reason": "credential_auth_failed", - "method": "composite", + engaged := h.recordAuthFailure(identityKey, time.Now()) + logging.Logger.Warn().Err(authErr).Str("identity", identityKey).Msg("credential authentication failed") + h.auditLog(ctx, audit.NewAuthEvent(string(identity.Type), identityKey, audit.OpAuthTokenValidation, identity.Workspace, uuid.New(), false, authErr.Error(), map[string]interface{}{ + "reason": "credential_auth_failed", + "method": "composite", + "throttle_new": engaged, })) return "", identity, status.Errorf(codes.Unauthenticated, "credential authentication failed: %v", authErr) } if authResult != nil && authResult.Authenticated { - if !hasCertificate { + // The principal-binding block runs when the transport carries no + // auth identity: either no certificate at all, or an ANONYMOUS + // certificate (transport-only). On the strict/semi-strict cert + // path (non-anonymous cert) the identity is already authoritatively + // bound by the certificate, so we deliberately skip rebinding here. + // The no-cert and anonymous-cert paths share the SAME enforcement + // below — there is no security divergence between them. + if !hasCertificate || isAnonymous { if authResult.Method == "oauth" { identity = authResult.Identity } if authResult.Method == "api_key" { - if identity.Type != authResult.Identity.Type && authResult.Identity.Type != "" { - logging.Logger.Warn().Str("api_key_type", string(authResult.Identity.Type)).Str("init_type", string(identity.Type)).Msg("API key principal type doesn't match InitConnection type") + // SECURITY: the API key authenticates a concrete principal + // (Type, ID) = (token.PrincipalType, token.CreatedBy). The + // InitConnection carries only a CLAIM. If the claim names a + // principal Type/ID, it MUST equal the key's — otherwise a + // caller could present user:alice's key while claiming to be + // user:drew. A mismatch is a HARD FAIL (PermissionDenied). + // + // The binding is ALWAYS from the key — never from the claim. + // Even when keyID is empty (a key with no created_by), we set + // identity.ID = keyID so the claim can never silently supply + // an ID the key does not authenticate. Such keys should be + // rejected at mint time (see token_handler.go), but the + // binding enforces it defensively here too. + keyType := authResult.Identity.Type + keyID := authResult.Identity.ID + tokenID, _ := authResult.Metadata["token_id"].(string) + + if identity.Type != "" && keyType != "" && identity.Type != keyType { + engaged := h.recordAuthFailure(identityKey, time.Now()) + logging.Logger.Warn(). + Str("key_type", string(keyType)). + Str("key_id", keyID). + Str("token_id", tokenID). + Str("claimed_type", string(identity.Type)). + Str("claimed_identity", identity.String()). + Bool("throttle_new", engaged). + Msg("API key principal type does not match InitConnection claim") + h.auditLog(ctx, audit.NewAuthEvent(string(keyType), keyID, audit.OpAuthTokenValidation, identity.Workspace, uuid.New(), false, "api key principal type mismatch", map[string]interface{}{ + "reason": "api_key_principal_type_mismatch", + "key_type": string(keyType), + "key_id": keyID, + "token_id": tokenID, + "claimed_type": string(identity.Type), + "claimed_identity": identity.String(), + })) + return "", identity, status.Errorf(codes.PermissionDenied, + "API key principal type mismatch: key=%s, claimed=%s", keyType, identity.Type) + } + // keyID != claim check: no guard on keyID == "" so a key + // with empty created_by can never be bypassed by an empty claim. + if identity.ID != "" && identity.ID != keyID { + engaged := h.recordAuthFailure(identityKey, time.Now()) + logging.Logger.Warn(). + Str("key_type", string(keyType)). + Str("key_id", keyID). + Str("token_id", tokenID). + Str("claimed_id", identity.ID). + Str("claimed_identity", identity.String()). + Bool("throttle_new", engaged). + Msg("API key principal id does not match InitConnection claim") + h.auditLog(ctx, audit.NewAuthEvent(string(keyType), keyID, audit.OpAuthTokenValidation, identity.Workspace, uuid.New(), false, "api key principal id mismatch", map[string]interface{}{ + "reason": "api_key_principal_id_mismatch", + "key_type": string(keyType), + "key_id": keyID, + "token_id": tokenID, + "claimed_id": identity.ID, + "claimed_identity": identity.String(), + })) + return "", identity, status.Errorf(codes.PermissionDenied, + "API key principal id mismatch: key=%s, claimed=%s", keyID, identity.ID) } + + // Claim accepted: bind the AUTHENTICATED principal from the + // key. Type + ID always come from the key (authenticated + // facts); Specifier/window + Workspace are preserved from + // the claim (the key does not carry those). + if keyType != "" { + identity.Type = keyType + } + // Always adopt keyID — even when empty — so the claim can + // never supply an ID that the key doesn't authenticate. + identity.ID = keyID + + // On a successful bind, clear any prior throttle state so a + // legitimate reconnect is not penalized. + h.clearAuthFailure(identityKey) + if wsPatterns, ok := authResult.Metadata["workspace_patterns"].([]string); ok { if identity.Workspace != "" { matched := false @@ -419,6 +684,14 @@ func (h *AuthHandler) authenticateCredentials(ctx context.Context, init *pb.Init "API key not authorized for workspace %s", identity.Workspace) } } + // NOTE(security): user identities carry an empty + // Workspace at connect time (the workspace is selected + // later via SwitchWorkspace), so this workspace_patterns + // check is a no-op for user connections. The key's + // workspace_patterns are NOT yet enforced at the point a + // user connection binds a workspace. See the TODO at + // handleSwitchWorkspace / checkConnection for the + // deferred enforcement point. } } } @@ -427,6 +700,76 @@ func (h *AuthHandler) authenticateCredentials(ctx context.Context, init *pb.Init "auth_method": authResult.Method, "metadata": authResult.Metadata, })) + } else if authResult == nil { + // Composite returned (nil, nil): no authenticator recognised the + // credentials. On the external anonymous-cert / no-cert path, an + // impersonable principal that presented NO recognised credential is + // unauthenticated. Fail closed. + // + // Exemptions (must NOT fail here): + // (a) associatedTaskID != "" — already handled by task-token path above. + // (b) IsInProcessConn — in-process bufconn connections are trusted + // at the transport layer; they do not carry api-key credentials. + // (c) Non-anonymous cert path — cert is the authority, no api-key needed. + // (d) Non-impersonable principals (WorkflowEngine, Orchestrator, + // MetricsBridge) — system principals connect without per-user keys. + needsCredential := (!hasCertificate || isAnonymous) && + !IsInProcessConn(ctx) && + isImpersonablePrincipal(identity.Type) + if needsCredential && h.devMode { + logging.Logger.Warn(). + Str("identity", identityKey). + Str("claimed_type", string(identity.Type)). + Msg("dev mode (AETHER_DEV_MODE): admitting anonymous impersonable principal without a credential — NOT FOR PRODUCTION") + needsCredential = false + } + if needsCredential { + engaged := h.recordAuthFailure(identityKey, time.Now()) + logging.Logger.Warn(). + Str("identity", identityKey). + Bool("throttle_new", engaged). + Msg("anonymous/no-cert connection: impersonable principal presented no valid credential") + h.auditLog(ctx, audit.NewAuthEvent(string(identity.Type), identityKey, audit.OpAuthTokenValidation, identity.Workspace, uuid.New(), false, "no credential presented for impersonable principal", map[string]interface{}{ + "reason": "no_credential_for_impersonable", + "claimed_type": string(identity.Type), + "is_anonymous": isAnonymous, + "has_cert": hasCertificate, + "throttle_new": engaged, + })) + return "", identity, status.Errorf(codes.Unauthenticated, + "anonymous connection as %s requires a matching API key credential", identity.Type) + } + } + } + + // SECURITY: credential-required gate for the case where there is NO + // composite authenticator configured at all (h.authenticator == nil). + // Without this, an external anonymous-cert connection claiming an + // impersonable principal would succeed with no auth check at all. + // In-process (bufconn) connections and task-token-authenticated workers + // are exempt — see the analogous exemptions in the authResult==nil block + // above. + if h.authenticator == nil && associatedTaskID == "" && + (!hasCertificate || isAnonymous) && + !IsInProcessConn(ctx) && + isImpersonablePrincipal(identity.Type) { + if h.devMode { + logging.Logger.Warn(). + Str("identity", identity.String()). + Str("claimed_type", string(identity.Type)). + Msg("dev mode (AETHER_DEV_MODE): admitting anonymous impersonable principal with no authenticator configured — NOT FOR PRODUCTION") + } else { + logging.Logger.Warn(). + Str("identity", identity.String()). + Msg("anonymous/no-cert connection: impersonable principal with no authenticator configured") + h.auditLog(ctx, audit.NewAuthEvent(string(identity.Type), identity.String(), audit.OpAuthTokenValidation, identity.Workspace, uuid.New(), false, "no authenticator configured for impersonable principal on anonymous path", map[string]interface{}{ + "reason": "no_authenticator_for_impersonable", + "claimed_type": string(identity.Type), + "is_anonymous": isAnonymous, + "has_cert": hasCertificate, + })) + return "", identity, status.Errorf(codes.Unauthenticated, + "anonymous connection as %s requires API key auth but no authenticator is configured", identity.Type) } } diff --git a/server/internal/gateway/auth_throttle_test.go b/server/internal/gateway/auth_throttle_test.go new file mode 100644 index 0000000..52ccdf5 --- /dev/null +++ b/server/internal/gateway/auth_throttle_test.go @@ -0,0 +1,243 @@ +package gateway + +// Tests for the per-identity auth-failure throttle in authenticateCredentials. +// +// The throttle is defense-in-depth against stale/zombie clients that reconnect +// in a tight loop presenting a token the gateway no longer knows: after +// authFailThreshold failures within authFailWindow, the identity is fast- +// rejected for authFailCooldown WITHOUT re-calling ValidateToken. A successful +// validation clears the bookkeeping; cooldown expiry allows re-validation. + +import ( + "context" + "errors" + "testing" + "time" + + pb "github.com/scitrera/aether/api/proto" + "github.com/scitrera/aether/internal/state" + "github.com/scitrera/aether/pkg/models" +) + +// throttleMockTokenStore is a state.TokenStore whose ValidateToken behavior is +// programmable and call-counted, so tests can assert when (and whether) the +// gateway actually re-validates. +type throttleMockTokenStore struct { + validateCalls int + // validateFunc, if set, produces the result for each ValidateToken call. + validateFunc func() (*state.TaskAuthToken, error) +} + +func (m *throttleMockTokenStore) GenerateToken(_ context.Context, taskID, targetIdentity, workspace, orchestratorID string) (*state.TaskAuthToken, error) { + return nil, errors.New("not implemented") +} + +func (m *throttleMockTokenStore) ValidateToken(_ context.Context, _ string) (*state.TaskAuthToken, error) { + m.validateCalls++ + if m.validateFunc != nil { + return m.validateFunc() + } + return nil, errors.New("token not found") +} + +func (m *throttleMockTokenStore) RevokeToken(_ context.Context, _ string) error { return nil } + +func (m *throttleMockTokenStore) RevokeTokensForTask(_ context.Context, _ string) error { return nil } + +func (m *throttleMockTokenStore) ListTokensForTask(_ context.Context, _ string) ([]*state.TaskAuthToken, error) { + return nil, nil +} + +// throttleTestInit builds an InitConnection carrying a token credential for the +// given agent identity coordinates. +func throttleTestInit(token string) *pb.InitConnection { + return &pb.InitConnection{ + ClientType: &pb.InitConnection_Agent{ + Agent: &pb.AgentIdentity{ + Workspace: "prod", + Implementation: "classifier", + Specifier: "v2", + }, + }, + Credentials: map[string]string{"token": token}, + } +} + +func throttleTestIdentity() models.Identity { + return models.Identity{ + Type: models.PrincipalAgent, + Workspace: "prod", + Implementation: "classifier", + Specifier: "v2", + } +} + +// TestAuthThrottle_EngagesAfterThresholdFailures verifies that authFailThreshold +// consecutive failures engage the cooldown and that a subsequent attempt is +// fast-rejected WITHOUT calling ValidateToken again. +func TestAuthThrottle_EngagesAfterThresholdFailures(t *testing.T) { + mock := &throttleMockTokenStore{} // always fails: "token not found" + h := newAuthHandler(nil, false, MTLSModeStrict, nil, nil) + h.tokenStore = mock + + ctx := context.Background() + identity := throttleTestIdentity() + init := throttleTestInit("stale-token") + + // First authFailThreshold attempts each call ValidateToken and fail. + for i := 0; i < authFailThreshold; i++ { + _, _, err := h.authenticateCredentials(ctx, init, identity, false, false) + if err == nil { + t.Fatalf("attempt %d: expected error from failed validation, got nil", i+1) + } + } + if mock.validateCalls != authFailThreshold { + t.Fatalf("expected %d ValidateToken calls during threshold, got %d", authFailThreshold, mock.validateCalls) + } + + // The next attempt must be throttled: no additional ValidateToken call. + _, _, err := h.authenticateCredentials(ctx, init, identity, false, false) + if err == nil { + t.Fatal("expected throttled attempt to return an error") + } + if mock.validateCalls != authFailThreshold { + t.Errorf("throttled attempt should NOT call ValidateToken: got %d calls, want %d", mock.validateCalls, authFailThreshold) + } +} + +// TestAuthThrottle_SuccessBeforeThresholdResetsCounter verifies that a +// successful validation before the threshold clears the failure bookkeeping so +// later failures start counting from zero again. +func TestAuthThrottle_SuccessBeforeThresholdResetsCounter(t *testing.T) { + identity := throttleTestIdentity() + mock := &throttleMockTokenStore{} + h := newAuthHandler(nil, false, MTLSModeStrict, nil, nil) + h.tokenStore = mock + + ctx := context.Background() + init := throttleTestInit("the-token") + + // A few failures, but fewer than the threshold. + failsBeforeSuccess := authFailThreshold - 2 + for i := 0; i < failsBeforeSuccess; i++ { + if _, _, err := h.authenticateCredentials(ctx, init, identity, false, false); err == nil { + t.Fatalf("pre-success attempt %d: expected failure", i+1) + } + } + + // One success: token validated and matches the connecting identity. + mock.validateFunc = func() (*state.TaskAuthToken, error) { + return &state.TaskAuthToken{TaskID: "task-1", TargetIdentity: identity.String()}, nil + } + if _, _, err := h.authenticateCredentials(ctx, init, identity, false, false); err != nil { + t.Fatalf("expected successful validation, got %v", err) + } + + // Verify the throttle record was cleared. + h.authFailMu.Lock() + _, present := h.authFailRecords[identity.String()] + h.authFailMu.Unlock() + if present { + t.Error("expected throttle record cleared after successful validation") + } + + // Now fail again: it should take a full authFailThreshold failures to + // re-engage, proving the counter reset. Validate that the (threshold-1)th + // failure is still re-validated (not throttled). + mock.validateFunc = nil // back to always-fail + callsBefore := mock.validateCalls + for i := 0; i < authFailThreshold-1; i++ { + if _, _, err := h.authenticateCredentials(ctx, init, identity, false, false); err == nil { + t.Fatalf("post-reset attempt %d: expected failure", i+1) + } + } + if got := mock.validateCalls - callsBefore; got != authFailThreshold-1 { + t.Errorf("expected %d re-validations after reset (not throttled early), got %d", authFailThreshold-1, got) + } +} + +// TestAuthThrottle_CooldownExpiryAllowsRevalidation verifies that once the +// cooldown elapses the throttle releases and ValidateToken is called again. +func TestAuthThrottle_CooldownExpiryAllowsRevalidation(t *testing.T) { + identity := throttleTestIdentity() + mock := &throttleMockTokenStore{} + h := newAuthHandler(nil, false, MTLSModeStrict, nil, nil) + h.tokenStore = mock + + ctx := context.Background() + init := throttleTestInit("stale-token") + + // Drive past the threshold to engage cooldown. + for i := 0; i < authFailThreshold; i++ { + if _, _, err := h.authenticateCredentials(ctx, init, identity, false, false); err == nil { + t.Fatalf("attempt %d: expected failure", i+1) + } + } + if mock.validateCalls != authFailThreshold { + t.Fatalf("expected %d ValidateToken calls, got %d", authFailThreshold, mock.validateCalls) + } + + // Confirm currently throttled (no new validation). + if _, _, err := h.authenticateCredentials(ctx, init, identity, false, false); err == nil { + t.Fatal("expected throttled attempt to error") + } + if mock.validateCalls != authFailThreshold { + t.Fatalf("expected still %d ValidateToken calls during cooldown, got %d", authFailThreshold, mock.validateCalls) + } + + // Simulate cooldown expiry by backdating the record's cooldownUntil. + h.authFailMu.Lock() + rec := h.authFailRecords[identity.String()] + rec.cooldownUntil = time.Now().Add(-time.Second) + h.authFailMu.Unlock() + + // Re-validation should now occur (the throttle released). + if _, _, err := h.authenticateCredentials(ctx, init, identity, false, false); err == nil { + t.Fatal("expected validation error after cooldown expiry") + } + if mock.validateCalls != authFailThreshold+1 { + t.Errorf("expected re-validation after cooldown expiry: got %d ValidateToken calls, want %d", mock.validateCalls, authFailThreshold+1) + } +} + +// TestAuthThrottle_HelpersBookkeeping exercises the throttle helpers directly to +// pin down window-reset and threshold semantics independent of the gRPC path. +func TestAuthThrottle_HelpersBookkeeping(t *testing.T) { + h := newAuthHandler(nil, false, MTLSModeStrict, nil, nil) + key := "ag::prod::classifier::v2" + base := time.Now() + + // Failures 1..threshold-1 should not engage; threshold-th engages once. + for i := 1; i < authFailThreshold; i++ { + if engaged := h.recordAuthFailure(key, base); engaged { + t.Fatalf("failure %d should not engage cooldown", i) + } + if h.throttled(key, base) { + t.Fatalf("should not be throttled after %d failures", i) + } + } + if engaged := h.recordAuthFailure(key, base); !engaged { + t.Fatalf("failure %d should engage cooldown", authFailThreshold) + } + if !h.throttled(key, base) { + t.Fatal("should be throttled once cooldown engaged") + } + + // A failure spaced beyond the window resets the counter (no engage). + afterWindow := base.Add(authFailCooldown + authFailWindow + time.Second) + if h.throttled(key, afterWindow) { + t.Fatal("should no longer be throttled after cooldown elapses") + } + if engaged := h.recordAuthFailure(key, afterWindow); engaged { + t.Fatal("first failure in a fresh window should not engage") + } + + // clearAuthFailure drops the record entirely. + h.clearAuthFailure(key) + h.authFailMu.Lock() + _, present := h.authFailRecords[key] + h.authFailMu.Unlock() + if present { + t.Error("expected record removed after clearAuthFailure") + } +} diff --git a/server/internal/gateway/authority.go b/server/internal/gateway/authority.go index c5944b8..3e92d02 100644 --- a/server/internal/gateway/authority.go +++ b/server/internal/gateway/authority.go @@ -215,7 +215,13 @@ func normalizeAuditPrincipalTypeFilter(value string) string { return audit.NormalizePrincipalTypeCase(trimmed) } -func (s *GatewayServer) checkMessageSendWithAuthority(ctx context.Context, sender models.Identity, targetTopic string, sessionID uuid.UUID, authority *acl.ResolvedAuthority) error { +// checkMessageSendWithAuthority returns the effective access level granted by +// the winning ACL decision (direct actor grant or OBO subject grant) +// alongside the error. The level is what the gateway mints into +// X-Auth-Workspace-Access on the proxy path; callers that only need the +// allow/deny outcome may ignore it. When no workspace check applies the level +// is acl.AccessNone (0). +func (s *GatewayServer) checkMessageSendWithAuthority(ctx context.Context, sender models.Identity, targetTopic string, sessionID uuid.UUID, authority *acl.ResolvedAuthority) (int, error) { hasGrant := authority != nil && authority.Grant != nil subjectType := "" subjectID := "" @@ -232,7 +238,19 @@ func (s *GatewayServer) checkMessageSendWithAuthority(ctx context.Context, sende Msg("checkMessageSendWithAuthority entry") if s.acl == nil { - return fmt.Errorf("ACL service not available") + return acl.AccessNone, fmt.Errorf("ACL service not available") + } + + // Additive task-party grant: a task's own party (assignee/creator/OBO + // subject) may send to the task's msg lane even without a workspace send + // grant. Non-parties / non-task-msg topics fall through unchanged to the + // workspace/OBO logic below. + if ok, isTaskMsg := s.taskMsgSenderIsTaskParty(ctx, sender, targetTopic); isTaskMsg && ok { + logging.Logger.Info(). + Str("from", sender.ToTopic()). + Str("target", targetTopic). + Msg("task-msg send authorized via task-party match") + return acl.AccessReadWrite, nil } // Workspace-less senders (bridges, services, users) check against the @@ -241,11 +259,11 @@ func (s *GatewayServer) checkMessageSendWithAuthority(ctx context.Context, sende if sender.Workspace == "" { targetWorkspace := extractWorkspaceFromTopic(targetTopic) if targetWorkspace == "" { - return nil + return acl.AccessNone, nil } decision, err := s.acl.CheckAccessWithAuthority(ctx, sender, authority, acl.ResourceTypeWorkspace, targetWorkspace, "send_message", targetWorkspace, sessionID, acl.AccessReadWrite) if err != nil { - return fmt.Errorf("ACL check failed: %w", err) + return acl.AccessNone, fmt.Errorf("ACL check failed: %w", err) } if decision.Denied() { logging.Logger.Info(). @@ -253,9 +271,9 @@ func (s *GatewayServer) checkMessageSendWithAuthority(ctx context.Context, sende Str("target", targetTopic). Str("reason", decision.Reason). Msg("checkMessageSendWithAuthority denial") - return fmt.Errorf("access denied: %s", decision.Reason) + return acl.AccessNone, fmt.Errorf("access denied: %s", decision.Reason) } - return nil + return decision.EffectiveAccessLevel, nil } // Workspace-scoped senders: the workspace we gate on is the TARGET @@ -284,10 +302,10 @@ func (s *GatewayServer) checkMessageSendWithAuthority(ctx context.Context, sende // actor lacks direct grant but the subject has it. directDecision, directErr := s.acl.CheckAccess(ctx, sender, acl.ResourceTypeWorkspace, checkWorkspace, "send_message", checkWorkspace, sessionID, acl.AccessReadWrite) if directErr != nil { - return fmt.Errorf("ACL check failed (direct): %w", directErr) + return acl.AccessNone, fmt.Errorf("ACL check failed (direct): %w", directErr) } if directDecision != nil && !directDecision.Denied() { - return nil // actor's own grant suffices, OBO chain not consulted + return directDecision.EffectiveAccessLevel, nil // actor's own grant suffices, OBO chain not consulted } directReason := "no decision" if directDecision != nil { @@ -304,7 +322,7 @@ func (s *GatewayServer) checkMessageSendWithAuthority(ctx context.Context, sende Str("actor_reason", directReason). Err(oboErr). Msg("checkMessageSendWithAuthority OBO fallback errored") - return fmt.Errorf("access denied (actor: %s; OBO check failed: %w)", directReason, oboErr) + return acl.AccessNone, fmt.Errorf("access denied (actor: %s; OBO check failed: %w)", directReason, oboErr) } if oboDecision == nil || oboDecision.Denied() { oboReason := "no decision" @@ -318,7 +336,7 @@ func (s *GatewayServer) checkMessageSendWithAuthority(ctx context.Context, sende Str("actor_reason", directReason). Str("obo_reason", oboReason). Msg("checkMessageSendWithAuthority denial (both actor + OBO)") - return fmt.Errorf("access denied: actor lacks workspace %s grant (%s) AND OBO subject lacks it (%s)", checkWorkspace, directReason, oboReason) + return acl.AccessNone, fmt.Errorf("access denied: actor lacks workspace %s grant (%s) AND OBO subject lacks it (%s)", checkWorkspace, directReason, oboReason) } logging.Logger.Debug(). Str("sender", sender.ToTopic()). @@ -326,5 +344,5 @@ func (s *GatewayServer) checkMessageSendWithAuthority(ctx context.Context, sende Str("workspace_checked", checkWorkspace). Str("actor_reason", directReason). Msg("checkMessageSendWithAuthority allowed via OBO fallback (actor lacks direct grant)") - return nil + return oboDecision.EffectiveAccessLevel, nil } diff --git a/server/internal/gateway/claim_subject_test.go b/server/internal/gateway/claim_subject_test.go new file mode 100644 index 0000000..ff90a3c --- /dev/null +++ b/server/internal/gateway/claim_subject_test.go @@ -0,0 +1,340 @@ +package gateway + +// Tests for the task-driven chat lifecycle feature (Phase 1, Aether side): +// - TaskOperation_CLAIM transitions an assigned/pending task to running and +// fires a status notification. +// - notifyTaskStatusChange emits an ADDITIONAL ProgressUpdate to the task's +// OBO subject (pg::us::, Kind=TASK) when the task opted into +// subject participation. +// - authorizeTaskOp authorizes the OBO subject (and denies non-subjects). +// +// These reuse the native-sqlite gateway harness from task_test.go +// (newTaskTestServerWithSQLiteStore) so the real store + auth + notify paths +// are exercised end-to-end. + +import ( + "context" + "testing" + + pb "github.com/scitrera/aether/api/proto" + "github.com/scitrera/aether/pkg/models" + "github.com/scitrera/aether/pkg/tasks" + "google.golang.org/protobuf/proto" +) + +// subjectUserIdentity returns a window-specific User identity in ws1. The +// stored task subject id matches userID (the window specifier differs, as it +// would for a real connected browser tab). +func subjectUserIdentity(userID, window string) models.Identity { + return models.Identity{ + Type: models.PrincipalUser, + Workspace: "ws1", + ID: userID, + Specifier: window, + } +} + +// collectPublished snapshots the mock router's published messages. +func collectPublished(t *testing.T, s *GatewayServer) []publishedMsg { + t.Helper() + router, ok := s.router.(*mockMessageRouter) + if !ok { + t.Fatalf("server router is not *mockMessageRouter") + } + router.mu.Lock() + defer router.mu.Unlock() + return append([]publishedMsg(nil), router.publishedMessages...) +} + +// TestTaskOpClaim_AssignedTaskTransitionsToRunning verifies the assignee can +// CLAIM an assigned task, moving it to running, and that a TASK-kind status +// notification is published. +func TestTaskOpClaim_AssignedTaskTransitionsToRunning(t *testing.T) { + s, cleanup := newTaskTestServerWithSQLiteStore(t) + defer cleanup() + + alice := callerIdentity("worker", "alice") + aliceTopic := alice.ToTopic() + ctx := context.Background() + + task := &tasks.Task{ + TaskID: "task-claim-assigned", + TaskType: "chat_message", + Workspace: "ws1", + Status: tasks.TaskStatusPending, + ParentAgentID: aliceTopic, // creator, so authorized + } + if err := s.taskStore.CreateTask(ctx, task); err != nil { + t.Fatalf("CreateTask: %v", err) + } + if err := s.taskStore.AssignTask(ctx, task.TaskID, aliceTopic); err != nil { + t.Fatalf("AssignTask: %v", err) + } + + stream := &mockStream{} + client := newTaskTestClient(stream, alice) + + op := &pb.TaskOperation{ + Op: pb.TaskOperation_CLAIM, + TaskId: task.TaskID, + RequestId: "claim-1", + } + s.handleTaskOp(ctx, client, op) + + stream.mu.Lock() + resp := stream.sent[0].GetTaskOp() + stream.mu.Unlock() + + if resp == nil { + t.Fatal("expected TaskOperationResponse") + } + if !resp.Success { + t.Fatalf("CLAIM: expected Success=true, got error=%q", resp.Error) + } + if resp.Task == nil || resp.Task.Status != pb.TaskStatus_TASK_STATUS_RUNNING { + t.Errorf("CLAIM: expected returned task status RUNNING, got %v", resp.Task.GetStatus()) + } + + // Verify the stored task transitioned to running. + updated, err := s.taskStore.GetTask(ctx, task.TaskID) + if err != nil || updated == nil { + t.Fatalf("GetTask after claim: %v", err) + } + if updated.Status != tasks.TaskStatusRunning { + t.Errorf("stored status = %q, want running", updated.Status) + } + + // A status notification should have been published (parent recipient). + published := collectPublished(t, s) + foundRunning := false + for _, m := range published { + var u pb.ProgressUpdate + if proto.Unmarshal(m.payload, &u) != nil { + continue + } + if u.TaskId == task.TaskID && u.State == "running" && u.Kind == pb.ProgressKind_PROGRESS_KIND_TASK { + foundRunning = true + } + } + if !foundRunning { + t.Errorf("expected a running TASK-kind status notification; published=%d", len(published)) + } +} + +// TestTaskOpClaim_Idempotent verifies re-claiming a running task succeeds +// (idempotent for multi-tab scenarios). +func TestTaskOpClaim_Idempotent(t *testing.T) { + s, cleanup := newTaskTestServerWithSQLiteStore(t) + defer cleanup() + + alice := callerIdentity("worker", "alice") + aliceTopic := alice.ToTopic() + ctx := context.Background() + + task := &tasks.Task{ + TaskID: "task-claim-idem", + TaskType: "chat_message", + Workspace: "ws1", + Status: tasks.TaskStatusPending, + ParentAgentID: aliceTopic, + } + if err := s.taskStore.CreateTask(ctx, task); err != nil { + t.Fatalf("CreateTask: %v", err) + } + if err := s.taskStore.AssignTask(ctx, task.TaskID, aliceTopic); err != nil { + t.Fatalf("AssignTask: %v", err) + } + // First claim. + if err := s.taskStore.ClaimTask(ctx, task.TaskID); err != nil { + t.Fatalf("first ClaimTask: %v", err) + } + // Second claim must be a no-op success. + if err := s.taskStore.ClaimTask(ctx, task.TaskID); err != nil { + t.Errorf("idempotent ClaimTask: expected nil error, got %v", err) + } +} + +// TestSubjectNotify_PublishesToUserProgressTopic verifies that a task marked +// with subject_participant and a User OBO subject publishes an ADDITIONAL +// ProgressUpdate to pg::us:: with Kind=TASK and the carried metadata +// (thread_id / message_id / task_type / status) on a lifecycle transition. +func TestSubjectNotify_PublishesToUserProgressTopic(t *testing.T) { + s, cleanup := newTaskTestServerWithSQLiteStore(t) + defer cleanup() + + ctx := context.Background() + subjectID := "dev@example.com" + + task := &tasks.Task{ + TaskID: "task-subject-notify", + TaskType: "chat_message", + Workspace: "ws1", + Status: tasks.TaskStatusPending, + Metadata: map[string]interface{}{ + "subject_participant": true, + "thread_id": "thread-1", + "message_id": "msg-1", + }, + Authority: tasks.TaskAuthorityInfo{ + SubjectType: string(models.PrincipalUser), + SubjectID: subjectID, + }, + } + if err := s.taskStore.CreateTask(ctx, task); err != nil { + t.Fatalf("CreateTask: %v", err) + } + + // Trigger a transition notification directly (mirrors the post-op path). + s.notifyTaskStatusChangeFromTaskID(ctx, task.TaskID, "running", "") + + published := collectPublished(t, s) + wantTopic := "pg::us::" + subjectID + var subjectUpdate *pb.ProgressUpdate + for _, m := range published { + if m.topic != wantTopic { + continue + } + var u pb.ProgressUpdate + if proto.Unmarshal(m.payload, &u) != nil { + continue + } + subjectUpdate = &u + } + if subjectUpdate == nil { + t.Fatalf("expected a subject ProgressUpdate on %q; published topics=%v", wantTopic, topicsOf(published)) + } + if subjectUpdate.Kind != pb.ProgressKind_PROGRESS_KIND_TASK { + t.Errorf("subject update Kind = %v, want TASK", subjectUpdate.Kind) + } + if subjectUpdate.State != "running" { + t.Errorf("subject update State = %q, want running", subjectUpdate.State) + } + wantRecipient := "us::" + subjectID + if subjectUpdate.Recipient != wantRecipient { + t.Errorf("subject update Recipient = %q, want %q (bare user)", subjectUpdate.Recipient, wantRecipient) + } + if subjectUpdate.Metadata["thread_id"] != "thread-1" { + t.Errorf("subject metadata thread_id = %q, want thread-1", subjectUpdate.Metadata["thread_id"]) + } + if subjectUpdate.Metadata["message_id"] != "msg-1" { + t.Errorf("subject metadata message_id = %q, want msg-1", subjectUpdate.Metadata["message_id"]) + } + if subjectUpdate.Metadata["task_type"] != "chat_message" { + t.Errorf("subject metadata task_type = %q, want chat_message", subjectUpdate.Metadata["task_type"]) + } + if subjectUpdate.Metadata["status"] != "running" { + t.Errorf("subject metadata status = %q, want running", subjectUpdate.Metadata["status"]) + } +} + +// TestSubjectNotify_SkippedWhenNotParticipant verifies that a task WITHOUT the +// subject_participant flag does not publish to the per-user topic. +func TestSubjectNotify_SkippedWhenNotParticipant(t *testing.T) { + s, cleanup := newTaskTestServerWithSQLiteStore(t) + defer cleanup() + + ctx := context.Background() + subjectID := "dev@example.com" + + task := &tasks.Task{ + TaskID: "task-no-subject-notify", + TaskType: "chat_message", + Workspace: "ws1", + Status: tasks.TaskStatusPending, + // subject_participant intentionally absent + Authority: tasks.TaskAuthorityInfo{ + SubjectType: string(models.PrincipalUser), + SubjectID: subjectID, + }, + } + if err := s.taskStore.CreateTask(ctx, task); err != nil { + t.Fatalf("CreateTask: %v", err) + } + + s.notifyTaskStatusChangeFromTaskID(ctx, task.TaskID, "running", "") + + for _, m := range collectPublished(t, s) { + if m.topic == "pg::us::"+subjectID { + t.Errorf("did not expect a subject notification when subject_participant is unset") + } + } +} + +// TestTaskOpAuth_SubjectMatch verifies the OBO subject (the end user) is +// authorized to operate on its own task, matched by identity ID+type rather +// than topic string (the connected user topic carries a window specifier the +// stored subject id does not). +func TestTaskOpAuth_SubjectMatch(t *testing.T) { + s, cleanup := newTaskTestServerWithSQLiteStore(t) + defer cleanup() + + ctx := context.Background() + subjectID := "dev@example.com" + + task := &tasks.Task{ + TaskID: "task-subject-auth", + TaskType: "chat_message", + Workspace: "ws1", + Status: tasks.TaskStatusPending, + Authority: tasks.TaskAuthorityInfo{ + SubjectType: string(models.PrincipalUser), + SubjectID: subjectID, + }, + } + if err := s.taskStore.CreateTask(ctx, task); err != nil { + t.Fatalf("CreateTask: %v", err) + } + + // Caller is the subject user, connected from a specific window/tab. + subjectClient := newTaskTestClient(&mockStream{}, subjectUserIdentity(subjectID, "win-1")) + + if !s.authorizeTaskOp(ctx, subjectClient, task) { + t.Errorf("expected OBO subject to be authorized to operate on its own task") + } +} + +// TestTaskOpAuth_NonSubjectUserDenied verifies a User who is NOT the task's OBO +// subject is not authorized via the subject path. ACL is nil, so to isolate the +// subject check we put the caller in a different workspace (which short-circuits +// to deny before any fail-open workspace match). +func TestTaskOpAuth_NonSubjectUserDenied(t *testing.T) { + s, cleanup := newTaskTestServerWithSQLiteStore(t) + defer cleanup() + + ctx := context.Background() + + task := &tasks.Task{ + TaskID: "task-nonsubject-auth", + TaskType: "chat_message", + Workspace: "ws1", + Status: tasks.TaskStatusPending, + Authority: tasks.TaskAuthorityInfo{ + SubjectType: string(models.PrincipalUser), + SubjectID: "owner@example.com", + }, + } + if err := s.taskStore.CreateTask(ctx, task); err != nil { + t.Fatalf("CreateTask: %v", err) + } + + // A different user in a different workspace must be denied. + other := models.Identity{ + Type: models.PrincipalUser, + Workspace: "ws2", + ID: "intruder@example.com", + Specifier: "win-9", + } + otherClient := newTaskTestClient(&mockStream{}, other) + + if s.authorizeTaskOp(ctx, otherClient, task) { + t.Errorf("expected non-subject user to be denied") + } +} + +func topicsOf(msgs []publishedMsg) []string { + out := make([]string, 0, len(msgs)) + for _, m := range msgs { + out = append(out, m.topic) + } + return out +} diff --git a/server/internal/gateway/cleanup_and_auth_test.go b/server/internal/gateway/cleanup_and_auth_test.go index 8ed0866..6b54b7c 100644 --- a/server/internal/gateway/cleanup_and_auth_test.go +++ b/server/internal/gateway/cleanup_and_auth_test.go @@ -14,6 +14,9 @@ import ( "context" "testing" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + pb "github.com/scitrera/aether/api/proto" "github.com/scitrera/aether/pkg/models" ) @@ -589,6 +592,221 @@ func TestResolveConnectionIdentity_RelaxedMode_NoCertificate_MTLSRequired_Return } } +// --------------------------------------------------------------------------- +// resolveConnectionIdentity: semi-strict mode +// +// Semi-strict binds workspace + implementation + principal type to the +// certificate, but lets the InitConnection supply/override the specifier. +// This enables a single shared cert across horizontally-scaled instances. +// --------------------------------------------------------------------------- + +func TestResolveConnectionIdentity_SemiStrictMode_SpecifierFromInit_Accepted(t *testing.T) { + h := newAuthHandler(nil, false, MTLSModeSemiStrict, nil, nil) + + // Certificate identity: sv::frontend-api::cert-spec (workspace empty for service). + certIdentity := models.Identity{ + Type: models.PrincipalService, + Implementation: "frontend-api", + Specifier: "cert-spec", + } + + // InitConnection supplies a different specifier (pod-1) but matching impl. + init := &pb.InitConnection{ + ClientType: &pb.InitConnection_Service{ + Service: &pb.ServiceIdentity{ + Implementation: "frontend-api", + Specifier: "pod-1", + }, + }, + } + + ident, err := h.resolveConnectionIdentity(context.Background(), init, certIdentity, "", true, false) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + // Resulting identity: specifier from Init, impl/type from cert => sv::frontend-api::pod-1 + if ident.Type != models.PrincipalService { + t.Errorf("expected PrincipalService, got %s", ident.Type) + } + if ident.Implementation != "frontend-api" { + t.Errorf("expected Implementation 'frontend-api', got %q", ident.Implementation) + } + if ident.Specifier != "pod-1" { + t.Errorf("expected Specifier 'pod-1' (from Init), got %q", ident.Specifier) + } +} + +func TestResolveConnectionIdentity_SemiStrictMode_ImplMismatch_Rejected(t *testing.T) { + h := newAuthHandler(nil, false, MTLSModeSemiStrict, nil, nil) + + certIdentity := models.Identity{ + Type: models.PrincipalService, + Implementation: "frontend-api", + Specifier: "cert-spec", + } + + // InitConnection claims a DIFFERENT implementation than the cert. + init := &pb.InitConnection{ + ClientType: &pb.InitConnection_Service{ + Service: &pb.ServiceIdentity{ + Implementation: "backend-api", + Specifier: "pod-1", + }, + }, + } + + _, err := h.resolveConnectionIdentity(context.Background(), init, certIdentity, "", true, false) + if err == nil { + t.Fatal("expected PermissionDenied for implementation mismatch, got nil") + } + if status.Code(err) != codes.PermissionDenied { + t.Errorf("expected PermissionDenied, got %v", status.Code(err)) + } +} + +func TestResolveConnectionIdentity_SemiStrictMode_TypeMismatch_Rejected(t *testing.T) { + h := newAuthHandler(nil, false, MTLSModeSemiStrict, nil, nil) + + // Cert is a service identity. + certIdentity := models.Identity{ + Type: models.PrincipalService, + Implementation: "frontend-api", + Specifier: "cert-spec", + } + + // InitConnection claims an AGENT (different principal type) with matching impl. + init := &pb.InitConnection{ + ClientType: &pb.InitConnection_Agent{ + Agent: &pb.AgentIdentity{ + Workspace: "", + Implementation: "frontend-api", + Specifier: "pod-1", + }, + }, + } + + _, err := h.resolveConnectionIdentity(context.Background(), init, certIdentity, "", true, false) + if err == nil { + t.Fatal("expected PermissionDenied for principal type mismatch, got nil") + } + if status.Code(err) != codes.PermissionDenied { + t.Errorf("expected PermissionDenied, got %v", status.Code(err)) + } +} + +func TestResolveConnectionIdentity_SemiStrictMode_WorkspaceMismatch_Rejected(t *testing.T) { + h := newAuthHandler(nil, false, MTLSModeSemiStrict, nil, nil) + + // Cert is an agent identity bound to workspace 'cert-ws'. + certIdentity := models.Identity{ + Type: models.PrincipalAgent, + Workspace: "cert-ws", + Implementation: "worker", + Specifier: "cert-spec", + } + + // InitConnection claims a different workspace. + init := &pb.InitConnection{ + ClientType: &pb.InitConnection_Agent{ + Agent: &pb.AgentIdentity{ + Workspace: "other-ws", + Implementation: "worker", + Specifier: "pod-1", + }, + }, + } + + _, err := h.resolveConnectionIdentity(context.Background(), init, certIdentity, "", true, false) + if err == nil { + t.Fatal("expected PermissionDenied for workspace mismatch, got nil") + } + if status.Code(err) != codes.PermissionDenied { + t.Errorf("expected PermissionDenied, got %v", status.Code(err)) + } +} + +func TestResolveConnectionIdentity_SemiStrictMode_EmptyInitSpecifier_StableIdentity(t *testing.T) { + h := newAuthHandler(nil, false, MTLSModeSemiStrict, nil, nil) + + certIdentity := models.Identity{ + Type: models.PrincipalService, + Implementation: "frontend-api", + Specifier: "cert-spec", + } + + // InitConnection supplies an empty specifier — accepted as sv::frontend-api:: + init := &pb.InitConnection{ + ClientType: &pb.InitConnection_Service{ + Service: &pb.ServiceIdentity{ + Implementation: "frontend-api", + Specifier: "", + }, + }, + } + + ident, err := h.resolveConnectionIdentity(context.Background(), init, certIdentity, "", true, false) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if ident.Implementation != "frontend-api" { + t.Errorf("expected Implementation 'frontend-api', got %q", ident.Implementation) + } + if ident.Specifier != "" { + t.Errorf("expected empty Specifier (from Init), got %q", ident.Specifier) + } +} + +func TestResolveConnectionIdentity_SemiStrictMode_NoCertificate_MTLSRequired_ReturnsUnauthenticated(t *testing.T) { + h := newAuthHandler(nil, true /* mtlsRequired=true */, MTLSModeSemiStrict, nil, nil) + + init := &pb.InitConnection{ + ClientType: &pb.InitConnection_Service{ + Service: &pb.ServiceIdentity{Implementation: "frontend-api", Specifier: "pod-1"}, + }, + } + + _, err := h.resolveConnectionIdentity(context.Background(), init, models.Identity{}, "", false, false) + if err == nil { + t.Fatal("expected Unauthenticated error when mTLS required in semi-strict mode but no cert") + } + if status.Code(err) != codes.Unauthenticated { + t.Errorf("expected Unauthenticated, got %v", status.Code(err)) + } +} + +// --------------------------------------------------------------------------- +// ParseMTLSMode: config string parsing shared by both gateway binaries +// --------------------------------------------------------------------------- + +func TestParseMTLSMode(t *testing.T) { + cases := []struct { + in string + want MTLSMode + wantErr bool + }{ + {"", MTLSModeStrict, false}, + {"strict", MTLSModeStrict, false}, + {"semi-strict", MTLSModeSemiStrict, false}, + {"relaxed", MTLSModeRelaxed, false}, + {"bogus", "", true}, + } + for _, tc := range cases { + got, err := ParseMTLSMode(tc.in) + if tc.wantErr { + if err == nil { + t.Errorf("ParseMTLSMode(%q): expected error, got nil", tc.in) + } + continue + } + if err != nil { + t.Errorf("ParseMTLSMode(%q): unexpected error: %v", tc.in, err) + } + if got != tc.want { + t.Errorf("ParseMTLSMode(%q) = %q, want %q", tc.in, got, tc.want) + } + } +} + // --------------------------------------------------------------------------- // resolveIdentity: charset validation boundary tests // diff --git a/server/internal/gateway/connect.go b/server/internal/gateway/connect.go index 59256e8..e9ffccc 100644 --- a/server/internal/gateway/connect.go +++ b/server/internal/gateway/connect.go @@ -100,7 +100,7 @@ func (s *GatewayServer) Connect(stream pb.AetherGateway_ConnectServer) error { // 4. Authenticate credentials (task tokens + API key/OAuth) var associatedTaskID string - associatedTaskID, identity, err = s.authenticateCredentials(ctx, init, identity, hasCertificate) + associatedTaskID, identity, err = s.authenticateCredentials(ctx, init, identity, hasCertificate, isAnonymous) if err != nil { return err } @@ -205,9 +205,23 @@ func (s *GatewayServer) Connect(stream pb.AetherGateway_ConnectServer) error { s.activeStreams.Store(sessionID, client) s.identityIndex.Store(identity.String(), sessionID) - // Add agent to implementation index for pool task routing - if identity.Type == models.PrincipalAgent { + // Add agent or service to implementation index for pool task routing. + // Services participate via the workspace-less key (":impl"), enabling + // pool tasks targeted at services (e.g. webhook delivery) to find a + // healthy instance without per-workspace registration. + // + // A service may opt OUT via ServiceIdentity.no_pool_consumer: serve-only + // instances that register no task handlers (e.g. a MemoryLayer server with + // its in-process worker disabled) set it so the gateway never targets them + // for POOL tasks they can't handle. Otherwise findWorkerByImplementation + // load-balances onto them, the gateway claims-and-pushes, and the drop + // leaves the task stuck-assigned until reconcile. Default false = consumer + // (back-compat). Agents are always pool consumers. + if identity.Type == models.PrincipalAgent || + (identity.Type == models.PrincipalService && !init.GetService().GetNoPoolConsumer()) { s.addToImplIndex(identity, client) + } else if identity.Type == models.PrincipalService { + logging.Logger.Info().Str("identity", identity.String()).Msg("service connected as non-pool-consumer; excluded from pool-task routing") } // 6. Start lock refresh goroutine @@ -442,7 +456,7 @@ func (s *GatewayServer) Connect(stream pb.AetherGateway_ConnectServer) error { } s.handleAgentOp(sessionCtx, client, p.AgentOp) case *pb.UpstreamMessage_AclOp: - if !s.isAllowedACLOp(client, identity, p.AclOp) { + if !s.isAllowedACLOp(sessionCtx, client, identity, p.AclOp) { continue } s.handleACLOp(sessionCtx, client, p.AclOp) @@ -450,7 +464,7 @@ func (s *GatewayServer) Connect(stream pb.AetherGateway_ConnectServer) error { if !s.isAllowedAdminOp(client, identity, "tokens") { continue } - s.handleTokenOp(sessionCtx, client, p.TokenOp) + s.handleTokenOp(sessionCtx, client, identity, p.TokenOp) case *pb.UpstreamMessage_AuditQuery: client.identityMu.RLock() currentIdentity := client.Identity @@ -594,11 +608,27 @@ func (s *GatewayServer) isAllowedAdminOp(client *ClientSession, identity models. // REVOKE (rule level ≤ RW): AccessManage (30) — can remove grants ≤ own level // REVOKE (rule level > RW): AccessAdmin (40) // Non-workspace resources: global admin only -func (s *GatewayServer) isAllowedACLOp(client *ClientSession, identity models.Identity, aclOp *pb.ACLOperation) bool { - // System principals are implicitly allowed for all ACL operations. - switch identity.Type { - case models.PrincipalWorkflowEngine, models.PrincipalOrchestrator: - return true +// +// When aclOp.Authorization is set (on-behalf-of), every access check runs +// against the resolved OBO subject (the user) via CheckAccessWithAuthority +// rather than the connection's actor identity (the platform-server). This +// mirrors isAllowedSessionOp. When no authorization is present, behavior is +// unchanged: the connection identity is authorized as before (backward +// compatible). +func (s *GatewayServer) isAllowedACLOp(ctx context.Context, client *ClientSession, identity models.Identity, aclOp *pb.ACLOperation) bool { + resolvedAuthority, err := s.resolveAuthorizationContext(ctx, client, identity, aclOp.GetAuthorization()) + if err != nil { + sendACLError(client, "invalid authorization context: "+err.Error()) + return false + } + + // System principals are implicitly allowed for direct (non-OBO) calls. + // For OBO calls the subject — not the system actor — must hold the grant. + if resolvedAuthority == nil { + switch identity.Type { + case models.PrincipalWorkflowEngine, models.PrincipalOrchestrator: + return true + } } if s.acl == nil { @@ -606,6 +636,17 @@ func (s *GatewayServer) isAllowedACLOp(client *ClientSession, identity models.Id return false } + // checkAccess routes through the OBO subject when an authority context was + // resolved, otherwise authorizes the connection identity (legacy path). + checkAccess := func(resourceType, resourceID, operation, workspace string, requiredLevel int) (*acl.ACLDecision, error) { + if resolvedAuthority != nil { + return s.acl.CheckAccessWithAuthority(ctx, identity, resolvedAuthority, + resourceType, resourceID, operation, workspace, client.SessionUUID, requiredLevel) + } + return s.acl.CheckAccess(ctx, identity, + resourceType, resourceID, operation, workspace, client.SessionUUID, requiredLevel) + } + // Determine the required permission level based on operation type. // Read operations (LIST_RULES, GET_RULE) only need AccessRead on admin/acl; // mutations (GRANT, REVOKE, etc.) need AccessAdmin. @@ -617,20 +658,18 @@ func (s *GatewayServer) isAllowedACLOp(client *ClientSession, identity models.Id } // Check global admin permission (admin/* at required level) - decision, err := s.acl.CheckAccess( - context.Background(), identity, + decision, err := checkAccess( acl.ResourceTypeAdmin, acl.PermissionAdminOperations, - "admin_op", identity.Workspace, client.SessionUUID, permLevel, + "admin_op", identity.Workspace, permLevel, ) if err == nil && decision != nil && decision.Allowed { return true } // Check category-specific permission (admin/acl at required level) - decision, err = s.acl.CheckAccess( - context.Background(), identity, + decision, err = checkAccess( acl.ResourceTypeAdmin, acl.PermissionAdminACL, - "admin_op_acl", identity.Workspace, client.SessionUUID, permLevel, + "admin_op_acl", identity.Workspace, permLevel, ) if err == nil && decision != nil && decision.Allowed { return true @@ -670,7 +709,7 @@ func (s *GatewayServer) isAllowedACLOp(client *ClientSession, identity models.Id targetWorkspace = f.ResourceId // Look up the existing rule to check its access level if f.PrincipalType != "" && f.PrincipalId != "" { - rule, err := s.acl.GetRule(context.Background(), + rule, err := s.acl.GetRule(ctx, f.PrincipalType, f.PrincipalId, f.ResourceType, f.ResourceId) if err == nil && rule != nil && rule.AccessLevel <= acl.AccessReadWrite { requiredLevel = acl.AccessManage // can revoke grants ≤ RW @@ -698,10 +737,9 @@ func (s *GatewayServer) isAllowedACLOp(client *ClientSession, identity models.Id } // Check caller's access level on the target workspace - decision, err = s.acl.CheckAccess( - context.Background(), identity, + decision, err = checkAccess( acl.ResourceTypeWorkspace, targetWorkspace, - "acl_manage", identity.Workspace, client.SessionUUID, requiredLevel, + "acl_manage", identity.Workspace, requiredLevel, ) if err == nil && decision != nil && decision.Allowed { return true @@ -916,8 +954,8 @@ func (s *GatewayServer) rollbackSession(cs *connectionState) { s.activeStreams.Delete(cs.sessionID) s.identityIndex.Delete(cs.identity.String()) - // Remove agent from implementation index - if cs.identity.Type == models.PrincipalAgent && cs.client != nil { + // Remove agent or service from implementation index + if (cs.identity.Type == models.PrincipalAgent || cs.identity.Type == models.PrincipalService) && cs.client != nil { s.removeFromImplIndex(cs.identity, cs.client) } @@ -991,8 +1029,8 @@ func (s *GatewayServer) cleanupSession(cs *connectionState, gracefulExit bool) { s.activeStreams.Delete(cs.sessionID) s.identityIndex.Delete(cs.identity.String()) - // Remove agent from implementation index - if cs.identity.Type == models.PrincipalAgent { + // Remove agent or service from implementation index + if cs.identity.Type == models.PrincipalAgent || cs.identity.Type == models.PrincipalService { s.removeFromImplIndex(cs.identity, cs.client) } diff --git a/server/internal/gateway/connect_test.go b/server/internal/gateway/connect_test.go index cf023ef..c523277 100644 --- a/server/internal/gateway/connect_test.go +++ b/server/internal/gateway/connect_test.go @@ -2,6 +2,7 @@ package gateway import ( "context" + "strings" "sync" "testing" "time" @@ -278,6 +279,10 @@ func (m *mockMessageRouter) SubscribeExclusiveFromTimestamp(topic string, consum return m.SubscribeExclusive(topic, consumerName, handler) } +func (m *mockMessageRouter) SubscribeExclusiveResumeOrTail(topic string, consumerName string, handler func([]byte)) (func(), error) { + return m.SubscribeExclusive(topic, consumerName, handler) +} + func (m *mockMessageRouter) hasSharedTopic(topic string) bool { m.mu.Lock() defer m.mu.Unlock() @@ -300,6 +305,7 @@ func (m *mockMessageRouter) hasExclusiveTopic(topic string) bool { type mockKVReadWriter struct { mu sync.Mutex listData map[string]string + setData map[string]map[string]struct{} getErr error setErr error delErr error @@ -309,19 +315,40 @@ type mockKVReadWriter struct { func newMockKVReadWriter() *mockKVReadWriter { return &mockKVReadWriter{ listData: make(map[string]string), + setData: make(map[string]map[string]struct{}), } } -func (m *mockKVReadWriter) Get(_ context.Context, _ models.Identity, _ kv.KVScope, _ string, _ string, _ string) (string, error) { - return "", m.getErr +func (m *mockKVReadWriter) Get(_ context.Context, _ models.Identity, _ kv.KVScope, key string, _ string, _ string) (string, error) { + if m.getErr != nil { + return "", m.getErr + } + m.mu.Lock() + defer m.mu.Unlock() + // Realistic readback so handler tests (e.g. idempotent create-task) can + // observe values written via Set/SetNX; absent keys return "" like the + // real store's not-found path. + return m.listData[key], nil } -func (m *mockKVReadWriter) Set(_ context.Context, _ models.Identity, _ kv.KVScope, _ string, _ string, _ string, _ string, _ time.Duration) error { - return m.setErr +func (m *mockKVReadWriter) Set(_ context.Context, _ models.Identity, _ kv.KVScope, key string, value string, _ string, _ string, _ time.Duration) error { + if m.setErr != nil { + return m.setErr + } + m.mu.Lock() + defer m.mu.Unlock() + m.listData[key] = value + return nil } -func (m *mockKVReadWriter) Delete(_ context.Context, _ models.Identity, _ kv.KVScope, _ string, _ string, _ string) error { - return m.delErr +func (m *mockKVReadWriter) Delete(_ context.Context, _ models.Identity, _ kv.KVScope, key string, _ string, _ string) error { + if m.delErr != nil { + return m.delErr + } + m.mu.Lock() + defer m.mu.Unlock() + delete(m.listData, key) + return nil } func (m *mockKVReadWriter) List(_ context.Context, _ models.Identity, _ kv.KVScope, _ string, _ string) (map[string]string, error) { @@ -337,8 +364,23 @@ func (m *mockKVReadWriter) List(_ context.Context, _ models.Identity, _ kv.KVSco return result, nil } -func (m *mockKVReadWriter) ListPaginated(_ context.Context, _ models.Identity, _ kv.KVScope, _ string, _ string, _ *kv.ListOptions) (*kv.ListResult, error) { - return &kv.ListResult{}, nil +func (m *mockKVReadWriter) ListPaginated(_ context.Context, _ models.Identity, _ kv.KVScope, _ string, _ string, opts *kv.ListOptions) (*kv.ListResult, error) { + m.mu.Lock() + defer m.mu.Unlock() + if m.listErr != nil { + return nil, m.listErr + } + keyPrefix := "" + if opts != nil { + keyPrefix = opts.KeyPrefix + } + items := make(map[string]string, len(m.listData)) + for k, v := range m.listData { + if keyPrefix == "" || strings.HasPrefix(k, keyPrefix) { + items[k] = v + } + } + return &kv.ListResult{Items: items}, nil } func (m *mockKVReadWriter) Increment(_ context.Context, _ models.Identity, _ kv.KVScope, _ string, _ string, _ string) (int64, error) { @@ -357,6 +399,80 @@ func (m *mockKVReadWriter) DecrementIf(_ context.Context, _ models.Identity, _ k return 0, true, nil } +// SetNX/CompareAndSet/CompareAndDelete are backed by listData with realistic +// conditional semantics (TTL ignored) so handler tests can exercise the +// acquire/refresh/release flow. +func (m *mockKVReadWriter) SetNX(_ context.Context, _ models.Identity, _ kv.KVScope, key string, value string, _ string, _ string, _ time.Duration) (bool, error) { + m.mu.Lock() + defer m.mu.Unlock() + if m.setErr != nil { + return false, m.setErr + } + if _, ok := m.listData[key]; ok { + return false, nil + } + m.listData[key] = value + return true, nil +} + +func (m *mockKVReadWriter) CompareAndSet(_ context.Context, _ models.Identity, _ kv.KVScope, key string, expected string, value string, _ string, _ string, _ time.Duration) (bool, error) { + m.mu.Lock() + defer m.mu.Unlock() + if m.setErr != nil { + return false, m.setErr + } + cur, ok := m.listData[key] + if !ok || cur != expected { + return false, nil + } + m.listData[key] = value + return true, nil +} + +func (m *mockKVReadWriter) CompareAndDelete(_ context.Context, _ models.Identity, _ kv.KVScope, key string, expected string, _ string, _ string) (bool, error) { + m.mu.Lock() + defer m.mu.Unlock() + if m.delErr != nil { + return false, m.delErr + } + cur, ok := m.listData[key] + if !ok || cur != expected { + return false, nil + } + delete(m.listData, key) + return true, nil +} + +// SetAdd/SetCard are backed by setData with realistic set semantics (TTL +// ignored) so handler tests can exercise the add/cardinality flow. +func (m *mockKVReadWriter) SetAdd(_ context.Context, _ models.Identity, _ kv.KVScope, key string, member string, _ string, _ string, _ time.Duration) (bool, int64, error) { + m.mu.Lock() + defer m.mu.Unlock() + if m.setErr != nil { + return false, 0, m.setErr + } + members, ok := m.setData[key] + if !ok { + members = make(map[string]struct{}) + m.setData[key] = members + } + added := false + if _, exists := members[member]; !exists { + members[member] = struct{}{} + added = true + } + return added, int64(len(members)), nil +} + +func (m *mockKVReadWriter) SetCard(_ context.Context, _ models.Identity, _ kv.KVScope, key string, _ string, _ string) (int64, error) { + m.mu.Lock() + defer m.mu.Unlock() + if m.getErr != nil { + return 0, m.getErr + } + return int64(len(m.setData[key])), nil +} + // mockCheckpointManager implements CheckpointManager. type mockCheckpointManager struct { mu sync.Mutex @@ -1149,11 +1265,13 @@ func TestSetupClientSubscriptions_UserGetsWindowAndWorkspaceTopics(t *testing.T) if !router.hasExclusiveTopic("us::bob::win-42") { t.Error("expected exclusive subscription for user window topic us.bob.win-42") } - if !router.hasSharedTopic("gu::workspace-x") { - t.Error("expected shared subscription for global user topic gu.workspace-x") + // gu/uw broadcast lanes now use a per-window resume-or-tail (exclusive) + // consumer so reconnects replay only the gap, not the full retained history. + if !router.hasExclusiveTopic("gu::workspace-x") { + t.Error("expected exclusive (resume-or-tail) subscription for global user topic gu.workspace-x") } - if !router.hasSharedTopic("uw::bob::workspace-x") { - t.Error("expected shared subscription for user-workspace topic uw.bob.workspace-x") + if !router.hasExclusiveTopic("uw::bob::workspace-x") { + t.Error("expected exclusive (resume-or-tail) subscription for user-workspace topic uw.bob.workspace-x") } } diff --git a/server/internal/gateway/denial_audit_test.go b/server/internal/gateway/denial_audit_test.go new file mode 100644 index 0000000..11e94da --- /dev/null +++ b/server/internal/gateway/denial_audit_test.go @@ -0,0 +1,275 @@ +package gateway + +// denial_audit_test.go — focused tests verifying that terminal authorization +// denials in checkKeyPermission / checkScopeReadPermission (KV) and +// authorizeTaskOp (task) are captured in the audit log with success=false. + +import ( + "context" + "sync" + "testing" + + "github.com/google/uuid" + "github.com/scitrera/aether/internal/acl" + "github.com/scitrera/aether/internal/audit" + auditstore "github.com/scitrera/aether/internal/storage/audit" + "github.com/scitrera/aether/internal/kv" + "github.com/scitrera/aether/pkg/models" + "github.com/scitrera/aether/pkg/tasks" +) + +// --------------------------------------------------------------------------- +// captureAuditStore — minimal auditstore.Store that records LogEvent calls. +// --------------------------------------------------------------------------- + +type captureAuditStore struct { + mu sync.Mutex + events []*auditstore.Event +} + +func (c *captureAuditStore) LogEvent(_ context.Context, event *auditstore.Event) { + c.mu.Lock() + defer c.mu.Unlock() + c.events = append(c.events, event) +} + +func (c *captureAuditStore) LogEventSync(_ context.Context, event *auditstore.Event) error { + c.LogEvent(context.Background(), event) + return nil +} + +func (c *captureAuditStore) Close() error { return nil } + +func (c *captureAuditStore) QueryAuditLog(_ context.Context, _ auditstore.EventFilter) ([]*auditstore.Event, error) { + return nil, nil +} + +func (c *captureAuditStore) CleanupOldLogs(_ context.Context, _ int) (int64, error) { + return 0, nil +} + +func (c *captureAuditStore) GetConfig() *auditstore.Config { return nil } + +func (c *captureAuditStore) captured() []*auditstore.Event { + c.mu.Lock() + defer c.mu.Unlock() + out := make([]*auditstore.Event, len(c.events)) + copy(out, c.events) + return out +} + +// --------------------------------------------------------------------------- +// KV denial audit tests +// --------------------------------------------------------------------------- + +// denyReasonACLChecker always returns a non-fallback denied decision with the +// supplied reason — used to exercise the key-level denial branch. +type denyReasonACLChecker struct { + reason string +} + +func (d *denyReasonACLChecker) CheckAccess(_ context.Context, _ models.Identity, _, _, _, _ string, _ uuid.UUID, _ int) (*acl.ACLDecision, error) { + return &acl.ACLDecision{ + Allowed: false, + FallbackApplied: false, // explicit key-level rule + Decision: acl.DecisionDeny, + Reason: d.reason, + }, nil +} + +func (d *denyReasonACLChecker) CheckAccessWithAuthority(_ context.Context, _ models.Identity, _ *acl.ResolvedAuthority, _, _, _, _ string, _ uuid.UUID, _ int) (*acl.ACLDecision, error) { + return &acl.ACLDecision{ + Allowed: false, + FallbackApplied: false, + Decision: acl.DecisionDeny, + Reason: d.reason, + }, nil +} + +// TestKVDenial_checkKeyPermission_EmitsAuditEvent verifies that a key-level +// ACL denial in checkKeyPermission produces a success=false audit event with +// the correct operation and identity. +func TestKVDenial_checkKeyPermission_EmitsAuditEvent(t *testing.T) { + cap := &captureAuditStore{} + h := &KVHandler{ + kvStore: newMockKVReadWriter(), + auditLogger: cap, + aclService: &denyReasonACLChecker{reason: "test-key-deny"}, + } + + identity := models.Identity{Type: models.PrincipalAgent, Workspace: "ws1", Implementation: "worker", Specifier: "s1"} + sid := uuid.New() + + err := h.checkKeyPermission(context.Background(), identity, nil, kv.ScopeUserShared, "some-key", audit.OpKVGet, "ws1", sid, acl.AccessRead) + if err == nil { + t.Fatal("expected denial error, got nil") + } + + events := cap.captured() + if len(events) == 0 { + t.Fatal("expected audit event on denial, got none") + } + ev := events[0] + if ev.Success { + t.Errorf("audit event Success = true; want false") + } + if ev.Operation != audit.OpKVGet { + t.Errorf("audit event Operation = %q; want %q", ev.Operation, audit.OpKVGet) + } + if ev.ErrorMessage != "test-key-deny" { + t.Errorf("audit event ErrorMessage = %q; want %q", ev.ErrorMessage, "test-key-deny") + } +} + +// fallbackDenyACLChecker returns a fallback-applied deny at scope level (no +// explicit key rule), so checkKeyPermission falls through to the scope check. +type fallbackDenyACLChecker struct { + reason string +} + +func (f *fallbackDenyACLChecker) CheckAccess(_ context.Context, _ models.Identity, _, _, _, _ string, _ uuid.UUID, _ int) (*acl.ACLDecision, error) { + return &acl.ACLDecision{ + Allowed: false, + FallbackApplied: true, // no key-level rule → go to scope + Decision: acl.DecisionDeny, + Reason: f.reason, + }, nil +} + +func (f *fallbackDenyACLChecker) CheckAccessWithAuthority(_ context.Context, _ models.Identity, _ *acl.ResolvedAuthority, _, _, _, _ string, _ uuid.UUID, _ int) (*acl.ACLDecision, error) { + return &acl.ACLDecision{ + Allowed: false, + FallbackApplied: true, + Decision: acl.DecisionDeny, + Reason: f.reason, + }, nil +} + +// TestKVDenial_checkKeyPermission_ScopeLevel_EmitsAuditEvent covers the scope +// fall-through denial branch (no explicit key rule → scope-level deny). +func TestKVDenial_checkKeyPermission_ScopeLevel_EmitsAuditEvent(t *testing.T) { + cap := &captureAuditStore{} + h := &KVHandler{ + kvStore: newMockKVReadWriter(), + auditLogger: cap, + aclService: &fallbackDenyACLChecker{reason: "scope-fallback-deny"}, + } + + identity := models.Identity{Type: models.PrincipalAgent, Workspace: "ws1", Implementation: "w", Specifier: "s"} + sid := uuid.New() + + err := h.checkKeyPermission(context.Background(), identity, nil, kv.ScopeUserShared, "any-key", audit.OpKVPut, "ws1", sid, acl.AccessReadWrite) + if err == nil { + t.Fatal("expected denial error, got nil") + } + + events := cap.captured() + if len(events) == 0 { + t.Fatal("expected audit event on scope-level denial, got none") + } + ev := events[0] + if ev.Success { + t.Errorf("audit event Success = true; want false") + } + if ev.ErrorMessage != "scope-fallback-deny" { + t.Errorf("audit event ErrorMessage = %q; want %q", ev.ErrorMessage, "scope-fallback-deny") + } +} + +// TestKVDenial_checkScopeReadPermission_EmitsAuditEvent covers the LIST path +// (no specific key, scope-only check via checkScopeReadPermission). +func TestKVDenial_checkScopeReadPermission_EmitsAuditEvent(t *testing.T) { + cap := &captureAuditStore{} + h := &KVHandler{ + kvStore: newMockKVReadWriter(), + auditLogger: cap, + aclService: &denyReasonACLChecker{reason: "list-scope-deny"}, + } + + identity := models.Identity{Type: models.PrincipalUser, ID: "alice"} + sid := uuid.New() + + err := h.checkScopeReadPermission(context.Background(), identity, nil, kv.ScopeUserShared, audit.OpKVList, "ws1", sid) + if err == nil { + t.Fatal("expected denial error, got nil") + } + + events := cap.captured() + if len(events) == 0 { + t.Fatal("expected audit event on scope-read denial, got none") + } + ev := events[0] + if ev.Success { + t.Errorf("audit event Success = true; want false") + } + if ev.Operation != audit.OpKVList { + t.Errorf("audit event Operation = %q; want %q", ev.Operation, audit.OpKVList) + } + if ev.ErrorMessage != "list-scope-deny" { + t.Errorf("audit event ErrorMessage = %q; want %q", ev.ErrorMessage, "list-scope-deny") + } +} + +// --------------------------------------------------------------------------- +// authorizeTaskOp denial tests — workspace-mismatch branch +// --------------------------------------------------------------------------- + +// TestAuthorizeTaskOp_WorkspaceMismatch_EmitsAuditEvent verifies that the +// workspace-mismatch guard in authorizeTaskOp emits a task_authz_denied audit +// event with success=false. +func TestAuthorizeTaskOp_WorkspaceMismatch_EmitsAuditEvent(t *testing.T) { + cap := &captureAuditStore{} + gw := &GatewayServer{auditLogger: cap} + + task := &tasks.Task{ + TaskID: "task-xyz", + Workspace: "wsA", + } + caller := models.Identity{ + Type: models.PrincipalAgent, + Workspace: "wsB", // mismatch + Implementation: "w", + Specifier: "1", + } + client := &ClientSession{ + SessionUUID: uuid.New(), + Identity: caller, + } + + result := gw.authorizeTaskOp(context.Background(), client, task) + if result { + t.Fatal("expected authorizeTaskOp to return false for workspace mismatch") + } + + events := cap.captured() + if len(events) == 0 { + t.Fatal("expected audit event on workspace-mismatch denial, got none") + } + ev := events[0] + if ev.Success { + t.Errorf("audit event Success = true; want false") + } + if ev.Operation != audit.OpTaskAuthzDenied { + t.Errorf("audit event Operation = %q; want %q", ev.Operation, audit.OpTaskAuthzDenied) + } + if ev.ErrorMessage != "workspace mismatch" { + t.Errorf("audit event ErrorMessage = %q; want %q", ev.ErrorMessage, "workspace mismatch") + } +} + +// TestAuthorizeTaskOp_NoAuditLogger_NilSafe verifies that authorizeTaskOp +// does not panic when auditLogger is nil (the nil-safe auditLog helper must +// guard the call). +func TestAuthorizeTaskOp_NoAuditLogger_NilSafe(t *testing.T) { + gw := &GatewayServer{} // auditLogger == nil + + task := &tasks.Task{TaskID: "t1", Workspace: "wsA"} + caller := models.Identity{Type: models.PrincipalAgent, Workspace: "wsB", Implementation: "w", Specifier: "1"} + client := &ClientSession{SessionUUID: uuid.New(), Identity: caller} + + // Must not panic. + result := gw.authorizeTaskOp(context.Background(), client, task) + if result { + t.Fatal("expected false for workspace mismatch") + } +} diff --git a/server/internal/gateway/identity.go b/server/internal/gateway/identity.go index 49a7288..bd92813 100644 --- a/server/internal/gateway/identity.go +++ b/server/internal/gateway/identity.go @@ -53,6 +53,22 @@ func (m MTLSMode) String() string { return string(m) } +// ParseMTLSMode converts a config string into an MTLSMode, defaulting to +// MTLSModeStrict when the string is empty and rejecting unrecognized values. +// Both the production gateway (cmd/gateway) and aetherlite (cmd/aetherlite) +// route their config through this helper so the two binaries interpret the +// same config identically (notably honouring "semi-strict"). +func ParseMTLSMode(s string) (MTLSMode, error) { + if s == "" { + return MTLSModeStrict, nil + } + mode := MTLSMode(s) + if !mode.IsValid() { + return "", fmt.Errorf("invalid mTLS mode %q (expected: strict, semi-strict, relaxed)", s) + } + return mode, nil +} + // ExtractIdentityFromCertificate extracts identity from client certificate CN. // Expected CN format: {type}::{...} using the canonical identity separator ("::"). // Examples: diff --git a/server/internal/gateway/interfaces.go b/server/internal/gateway/interfaces.go index 292d47e..3d2d20a 100644 --- a/server/internal/gateway/interfaces.go +++ b/server/internal/gateway/interfaces.go @@ -72,6 +72,12 @@ type MessageRouter interface { // the router backend), falls back to existing replay semantics. Used for cold-starting // pool-dispatched agents. SubscribeExclusiveFromTimestamp(topic string, consumerName string, startTimestampMs int64, handler func([]byte)) (func(), error) + // SubscribeExclusiveResumeOrTail creates a named exclusive subscription that + // resumes from the consumer's committed offset when one exists, and otherwise + // starts at the current tail (NOT the beginning of the log). Use it for + // shared/broadcast lanes a client re-subscribes on every (re)connect so a + // first connect gets no full-history dump and a reconnect replays only the gap. + SubscribeExclusiveResumeOrTail(topic string, consumerName string, handler func([]byte)) (func(), error) } // KVReadWriter abstracts KV store operations used by the gateway and KVHandler. @@ -85,6 +91,39 @@ type KVReadWriter interface { Decrement(ctx context.Context, agent models.Identity, scope kv.KVScope, key string, userID string, workspace string) (int64, error) IncrementIf(ctx context.Context, agent models.Identity, scope kv.KVScope, key string, userID string, workspace string, delta int64, ceiling int64) (int64, bool, error) DecrementIf(ctx context.Context, agent models.Identity, scope kv.KVScope, key string, userID string, workspace string, delta int64, floor int64) (int64, bool, error) + + // Atomic conditional writes — the building blocks for distributed + // coordination primitives (mutex, leader election, run-once). Each is + // implemented natively across all three backends (Redis / Badger / + // NATS-JetStream). A TTL lease lock is expressed as: acquire = SetNX, + // refresh = CompareAndSet(token→token), release = CompareAndDelete(token). + + // SetNX sets key=value only if the key is currently absent. Returns true + // iff the value was written. + SetNX(ctx context.Context, agent models.Identity, scope kv.KVScope, key string, value string, userID string, workspace string, ttl time.Duration) (bool, error) + // CompareAndSet sets key=value only if the current stored value equals + // expected. Returns true iff the swap was applied. A non-existent key never + // matches a non-empty expected (use SetNX for the absent case). + CompareAndSet(ctx context.Context, agent models.Identity, scope kv.KVScope, key string, expected string, value string, userID string, workspace string, ttl time.Duration) (bool, error) + // CompareAndDelete deletes key only if the current stored value equals + // expected. Returns true iff the delete was applied. + CompareAndDelete(ctx context.Context, agent models.Identity, scope kv.KVScope, key string, expected string, userID string, workspace string) (bool, error) + + // Atomic set primitives — the building block for fan-in joins (set-based + // completeness) and at-most-once dedup ledgers. Implemented natively across + // all three backends (Redis SADD/SCARD; Badger / NATS-JetStream emulate a + // JSON-encoded member set under optimistic concurrency). + + // SetAdd atomically adds member to the set stored at key, returning whether + // the member was newly added (false if it was already present) together with + // the set's cardinality after the add. When ttl > 0 the key's expiry is + // (re)set on a newly-added member. This is the set analogue of IncrementIf: + // the unique caller that observes added==true && cardinality==N is the one + // whose add completed an N-member set (exactly-once fan-in firing for set + // joins), while added==false flags a duplicate arrival for dedup ledgers. + SetAdd(ctx context.Context, agent models.Identity, scope kv.KVScope, key string, member string, userID string, workspace string, ttl time.Duration) (bool, int64, error) + // SetCard returns the cardinality of the set stored at key (0 if absent). + SetCard(ctx context.Context, agent models.Identity, scope kv.KVScope, key string, userID string, workspace string) (int64, error) } // CheckpointManager abstracts checkpoint store operations used by the gateway. diff --git a/server/internal/gateway/kv_handler.go b/server/internal/gateway/kv_handler.go index 43991db..d061d80 100644 --- a/server/internal/gateway/kv_handler.go +++ b/server/internal/gateway/kv_handler.go @@ -57,6 +57,36 @@ func newKVHandlerFromService(store KVReadWriter, auditLogger auditstore.Store, a return NewKVHandler(store, auditLogger, checker) } +// emitKVDenial logs a KV denial at Info level (for operator visibility) and, +// when h.auditLogger is configured, records a success=false audit event. key +// may be empty for scope-only operations (e.g. LIST). scope is always included +// in the audit metadata; reason is the human-readable denial explanation. +func (h *KVHandler) emitKVDenial(ctx context.Context, identity models.Identity, scope kv.KVScope, key, operation, workspace, reason string, sessionID uuid.UUID) { + logging.Logger.Info(). + Str("identity", identity.String()). + Str("key", key). + Str("scope", string(scope)). + Str("op", operation). + Str("reason", reason). + Msg("KV access denied") + if h.auditLogger == nil { + return + } + metadata := map[string]interface{}{"scope": string(scope)} + event := audit.NewKVEvent( + string(identity.Type), + identity.String(), + operation, + key, + workspace, + sessionID, + false, // success + reason, // errorMsg + metadata, + ) + h.auditLogger.LogEvent(ctx, event) +} + // checkKeyPermission checks key-level then scope-level ACL for the given operation. // Key-level rules (kv_key/) take precedence; if no explicit key rule exists // (FallbackApplied), falls through to scope-level check (kv_scope/). @@ -75,6 +105,14 @@ func (h *KVHandler) checkKeyPermission(ctx context.Context, identity models.Iden // direct authority; ownership is implicit by storage layout. return nil } + // Infra-coordination fast-path: trusted infrastructure principals operating + // on the reserved coordination namespace (distributed locks / leader- + // election leases) are allowed without an ACL lookup. These are internal + // coordination keys, not application data. Cross-principal (OBO) access + // still flows through the regular ACL path. + if authority == nil && isInfraCoordAccess(identity, key) { + return nil + } if h.aclService == nil { return nil } @@ -83,11 +121,13 @@ func (h *KVHandler) checkKeyPermission(ctx context.Context, identity models.Iden decision, err := h.aclService.CheckAccessWithAuthority(ctx, identity, authority, acl.ResourceTypeKVKey, key, operation, workspace, sessionID, requiredLevel) if err != nil { logging.Logger.Error().Err(err).Str("identity", identity.String()).Str("key", key).Msg("ACL check failed for KV key, denying") + h.emitKVDenial(ctx, identity, scope, key, operation, workspace, "ACL check failed: "+err.Error(), sessionID) return status.Errorf(codes.Internal, "ACL check failed: %v", err) } if !decision.FallbackApplied { // Explicit rule exists for this key — use it if decision.Denied() { + h.emitKVDenial(ctx, identity, scope, key, operation, workspace, decision.Reason, sessionID) return status.Errorf(codes.PermissionDenied, "KV access denied for key %s: %s", key, decision.Reason) } return nil @@ -97,14 +137,34 @@ func (h *KVHandler) checkKeyPermission(ctx context.Context, identity models.Iden decision, err = h.aclService.CheckAccessWithAuthority(ctx, identity, authority, acl.ResourceTypeKVScope, string(scope), operation, workspace, sessionID, requiredLevel) if err != nil { logging.Logger.Error().Err(err).Str("identity", identity.String()).Str("scope", string(scope)).Msg("ACL check failed for KV scope, denying") + h.emitKVDenial(ctx, identity, scope, key, operation, workspace, "ACL check failed: "+err.Error(), sessionID) return status.Errorf(codes.Internal, "ACL check failed: %v", err) } if decision.Denied() { + h.emitKVDenial(ctx, identity, scope, key, operation, workspace, decision.Reason, sessionID) return status.Errorf(codes.PermissionDenied, "KV access denied for scope %s: %s", scope, decision.Reason) } return nil } +// isInfraCoordAccess reports whether identity is a trusted infrastructure +// principal accessing a key in the reserved coordination namespace +// (kv.ReservedCoordKeyPrefix). Used to grant the WorkflowEngine (and any future +// infra principal) lock/leader-election access without seeding per-key ACL +// grants. Application principals never match — they are gated normally even on +// reserved keys. +func isInfraCoordAccess(identity models.Identity, key string) bool { + if !strings.HasPrefix(key, kv.ReservedCoordKeyPrefix) { + return false + } + switch identity.Type { + case models.PrincipalWorkflowEngine: + return true + default: + return false + } +} + // checkScopeReadPermission checks scope-level read permission (used for LIST which has no specific key). func (h *KVHandler) checkScopeReadPermission(ctx context.Context, identity models.Identity, authority *acl.ResolvedAuthority, scope kv.KVScope, operation, workspace string, sessionID uuid.UUID) error { if h.aclService == nil { @@ -113,9 +173,11 @@ func (h *KVHandler) checkScopeReadPermission(ctx context.Context, identity model decision, err := h.aclService.CheckAccessWithAuthority(ctx, identity, authority, acl.ResourceTypeKVScope, string(scope), operation, workspace, sessionID, acl.AccessRead) if err != nil { logging.Logger.Error().Err(err).Str("identity", identity.String()).Str("scope", string(scope)).Msg("ACL check failed for KV scope read, denying") + h.emitKVDenial(ctx, identity, scope, "", operation, workspace, "ACL check failed: "+err.Error(), sessionID) return status.Errorf(codes.Internal, "ACL check failed: %v", err) } if decision.Denied() { + h.emitKVDenial(ctx, identity, scope, "", operation, workspace, decision.Reason, sessionID) return status.Errorf(codes.PermissionDenied, "KV read denied for scope %s: %s", scope, decision.Reason) } return nil @@ -131,9 +193,25 @@ func (h *KVHandler) HandleKVOperation( op *pb.KVOperation, sendResponse func(*pb.DownstreamMessage), ) error { - // Agents, tasks, and services can access the KV store. - if identity.Type != models.PrincipalAgent && identity.Type != models.PrincipalTask && identity.Type != models.PrincipalService { - return status.Error(codes.PermissionDenied, "only agents, tasks, and services can access KV store") + // Agents, tasks, and services store application state in the KV store. + // The WorkflowEngine is additionally permitted so it can use the KV-backed + // coordination primitives (leader election) over the shared store in every + // backend mode; its access is effectively limited to the reserved + // coordination namespace by the infra fast-path in checkKeyPermission + // (other keys still require an ACL grant it does not hold). + // The MetricsBridge is permitted so the billing bridge can run on a SINGLE + // connection (its identity is a singleton, so it cannot hold a second + // Service connection for KV without a duplicate-identity collision): it + // reads its billing:openmeter config and persists usage/invoice snapshots + // to the tenant's internal KV. Like the WorkflowEngine, this only opens the + // type gate — every key it touches still requires an explicit ACL grant via + // checkKeyPermission (seeded for metrics::shard0 in acl_seed.py). + if identity.Type != models.PrincipalAgent && + identity.Type != models.PrincipalTask && + identity.Type != models.PrincipalService && + identity.Type != models.PrincipalWorkflowEngine && + identity.Type != models.PrincipalMetricsBridge { + return status.Error(codes.PermissionDenied, "only agents, tasks, services, the metrics bridge, and the workflow engine can access KV store") } // Map proto enum scope to internal KVScope (default to workspace for backward compatibility) @@ -201,7 +279,7 @@ func (h *KVHandler) HandleKVOperation( opErr = h.handleDelete(ctx, identity, authority, sessionID, scope, op.Key, userID, workspace, requestID, sendResponse) case pb.KVOperation_LIST: - opErr = h.handleList(ctx, identity, authority, sessionID, scope, op.Key, userID, workspace, requestID, sendResponse) + opErr = h.handleList(ctx, identity, authority, sessionID, scope, op.Key, userID, workspace, op.GetLimit(), op.GetCursor(), requestID, sendResponse) case pb.KVOperation_INCREMENT: opErr = h.handleIncrement(ctx, identity, authority, sessionID, scope, op.Key, userID, workspace, ttl, requestID, sendResponse) @@ -215,9 +293,44 @@ func (h *KVHandler) HandleKVOperation( case pb.KVOperation_DECREMENT_IF: opErr = h.handleDecrementIf(ctx, identity, authority, sessionID, scope, op.Key, userID, workspace, op.DeltaValue, op.GuardValue, requestID, sendResponse) + case pb.KVOperation_SET_NX: + opErr = h.handleSetNX(ctx, identity, authority, sessionID, scope, op.Key, string(op.Value), userID, workspace, ttl, requestID, sendResponse) + + case pb.KVOperation_SET_ADD: + opErr = h.handleSetAdd(ctx, identity, authority, sessionID, scope, op.Key, op.Value, userID, workspace, ttl, requestID, sendResponse) + + case pb.KVOperation_SET_CARD: + opErr = h.handleSetCard(ctx, identity, authority, sessionID, scope, op.Key, userID, workspace, requestID, sendResponse) + + case pb.KVOperation_COMPARE_AND_SET: + opErr = h.handleCompareAndSet(ctx, identity, authority, sessionID, scope, op.Key, string(op.ExpectedValue), string(op.Value), userID, workspace, ttl, requestID, sendResponse) + + case pb.KVOperation_COMPARE_AND_DELETE: + opErr = h.handleCompareAndDelete(ctx, identity, authority, sessionID, scope, op.Key, string(op.ExpectedValue), userID, workspace, requestID, sendResponse) + + case pb.KVOperation_PURGE_IDENTITY: + opErr = h.handlePurgeIdentity(ctx, identity, sessionID, scope, op.GetTargetIdentity(), op.Key, requestID, sendResponse) + default: opErr = status.Error(codes.InvalidArgument, "unknown KV operation") } + + // Every KV op's wire contract is "echo the request_id on a KVResponse + // (Success=true or false)". Sub-handlers always send a KVResponse on + // success but bail out on error without one — that asymmetry strands + // the SDK's KVGetSync / KVPutSync waiters on a pending request that + // the gateway has decided to reject. Emit a typed failure response + // here so the typed pending-request channel resolves. The caller + // continues to receive opErr and can layer its own ErrorResponse on + // top for OnError consumers. + if opErr != nil && requestID != "" { + sendResponse(&pb.DownstreamMessage{ + Payload: &pb.DownstreamMessage_Kv{ + Kv: &pb.KVResponse{Success: false, RequestId: requestID}, + }, + }) + } + return opErr } @@ -508,6 +621,8 @@ func (h *KVHandler) handleList( keyPrefix string, userID string, workspace string, + limit int32, + cursor string, requestID string, sendResponse func(*pb.DownstreamMessage), ) error { @@ -524,23 +639,26 @@ func (h *KVHandler) handleList( return err } - items, err := h.kvStore.List(ctx, identity, scope, userID, workspace) - // Apply caller-supplied key prefix filter. Pre-Solution-A this was - // implicitly handled by the per-agent storage namespace (each caller - // only saw its own writes). With shared global/workspace namespaces - // the store now returns every key in the scope; the prefix filter - // must be enforced explicitly so callers like - // AetherKVHelper.get_by_prefix actually see only the keys they asked - // for and don't try to deserialize unrelated payloads from sibling - // agents. - if err == nil && keyPrefix != "" && len(items) > 0 { - filtered := make(map[string]string, len(items)) - for k, v := range items { - if strings.HasPrefix(k, keyPrefix) { - filtered[k] = v - } - } - items = filtered + // Apply the key prefix filter INSIDE the store's paginated scan so the + // limit bounds matching keys, not all keys in the scope. Previously this + // called the unpaginated List() and prefix-filtered afterwards — but List() + // caps at DefaultListLimit over the whole (shared) scope, so a caller + // listing a small prefixed subset of a large scope (e.g. "sandbox:" in the + // _sandbox workspace) could have all its matches fall outside the cap and + // see an empty result. ListPaginated with KeyPrefix fixes that and also + // exposes cursor/has_more so callers can page through >limit matches. + result, err := h.kvStore.ListPaginated(ctx, identity, scope, userID, workspace, &kv.ListOptions{ + KeyPrefix: keyPrefix, + Limit: int(limit), + Cursor: cursor, + }) + var items map[string]string + var nextCursor string + var hasMore bool + if result != nil { + items = result.Items + nextCursor = result.NextCursor + hasMore = result.HasMore } // Audit logging @@ -594,10 +712,12 @@ func (h *KVHandler) handleList( sendResponse(&pb.DownstreamMessage{ Payload: &pb.DownstreamMessage_Kv{ Kv: &pb.KVResponse{ - Success: true, - Keys: itemsToKeys(items), - KvMap: itemsBytes, - RequestId: requestID, + Success: true, + Keys: itemsToKeys(items), + KvMap: itemsBytes, + RequestId: requestID, + NextCursor: nextCursor, + HasMore: hasMore, }, }, }) @@ -605,6 +725,115 @@ func (h *KVHandler) handleList( return nil } +// handlePurgeIdentity REMOVAL-ONLY purges every key in another principal's KV +// namespace (target_identity, scope), optionally filtered by keyPrefix. It is +// the privileged primitive a lifecycle manager (e.g. sandbox-provider reaping a +// sidecar) uses to clear an ephemeral principal's stranded state. +// +// Security contract: +// - Gated by the capability/kv_purge_identity grant on the CALLER's own +// identity (no authority/OBO path). The caller never authorizes "as" the +// target — it holds a narrow, explicit capability to delete-on-its-behalf. +// - REMOVAL-ONLY: keys are read internally solely to delete them; NEITHER +// keys NOR values are returned to the caller — only the deleted count. +// So the capability cannot be repurposed to exfiltrate another principal's +// data, which keeps the components decoupled (the gateway needs no +// sandbox-specific knowledge; the caller owns the what/when). +func (h *KVHandler) handlePurgeIdentity( + ctx context.Context, + identity models.Identity, + sessionID uuid.UUID, + scope kv.KVScope, + targetIdentityStr string, + keyPrefix string, + requestID string, + sendResponse func(*pb.DownstreamMessage), +) error { + ctx, span := tracing.Tracer.Start(ctx, "aether.kv.purge_identity") + defer span.End() + span.SetAttributes( + attribute.String("aether.kv.scope", string(scope)), + attribute.String("aether.kv.target_identity", targetIdentityStr), + ) + + if targetIdentityStr == "" { + return status.Error(codes.InvalidArgument, "purge_identity requires target_identity") + } + target, err := models.ParseIdentity(targetIdentityStr) + if err != nil { + return status.Errorf(codes.InvalidArgument, "invalid target_identity %q: %v", targetIdentityStr, err) + } + + // Capability gate. ACL-disabled (nil service) follows the package-wide + // dev convention of allow-all, same as the other KV handlers. + if h.aclService != nil { + decision, derr := h.aclService.CheckAccess( + ctx, identity, acl.ResourceTypeCapability, acl.PermissionKVPurgeIdentity, + audit.OpKVDelete, "", sessionID, acl.AccessManage, + ) + if derr != nil { + logging.Logger.Error().Err(derr).Str("identity", identity.String()).Msg("ACL check failed for kv purge-identity, denying") + return status.Errorf(codes.Internal, "ACL check failed: %v", derr) + } + if decision.Denied() { + return status.Errorf(codes.PermissionDenied, "kv purge-identity denied: %s", decision.Reason) + } + } + + // Delete in rounds, re-listing from the start each round: the set shrinks + // as we delete, so a fresh capped List returns the next batch until empty. + // This sidesteps cursor invalidation under concurrent deletes and needs no + // pagination state. The ceiling + no-progress break are runaway guards. + const purgePageLimit = 1000 + const purgeRoundCeiling = 10000 + deleted := 0 + for round := 0; round < purgeRoundCeiling; round++ { + result, lerr := h.kvStore.ListPaginated(ctx, target, scope, "", "", &kv.ListOptions{ + KeyPrefix: keyPrefix, + Limit: purgePageLimit, + }) + if lerr != nil { + return status.Errorf(codes.Internal, "purge list failed: %v", lerr) + } + if result == nil || len(result.Items) == 0 { + break + } + deletedThisRound := 0 + for key := range result.Items { + if derr := h.kvStore.Delete(ctx, target, scope, key, "", ""); derr != nil { + logging.Logger.Warn().Err(derr).Str("target", target.String()).Str("scope", string(scope)).Str("key", key).Msg("kv purge-identity: delete failed") + continue + } + deleted++ + deletedThisRound++ + } + if deletedThisRound == 0 { + // Could not delete anything this round — avoid spinning on keys we + // can see but not remove. + break + } + } + + logging.Logger.Info(). + Str("caller", identity.String()). + Str("target", target.String()). + Str("scope", string(scope)). + Str("key_prefix", keyPrefix). + Int("deleted", deleted). + Msg("kv purge-identity") + + sendResponse(&pb.DownstreamMessage{ + Payload: &pb.DownstreamMessage_Kv{ + Kv: &pb.KVResponse{ + Success: true, + CounterValue: int64(deleted), + RequestId: requestID, + }, + }, + }) + return nil +} + func (h *KVHandler) handleIncrement( ctx context.Context, identity models.Identity, @@ -953,6 +1182,294 @@ func (h *KVHandler) handleDecrementIf( return nil } +func (h *KVHandler) handleSetNX( + ctx context.Context, + identity models.Identity, + authority *acl.ResolvedAuthority, + sessionID uuid.UUID, + scope kv.KVScope, + key string, + value string, + userID string, + workspace string, + ttl time.Duration, + requestID string, + sendResponse func(*pb.DownstreamMessage), +) error { + ctx, span := tracing.Tracer.Start(ctx, "aether.kv.set_nx") + defer span.End() + span.SetAttributes( + attribute.String("aether.kv.scope", string(scope)), + attribute.String("aether.kv.key", key), + attribute.String("aether.kv.workspace", workspace), + ) + + if err := h.checkKeyPermission(ctx, identity, authority, scope, key, audit.OpKVSetNX, workspace, sessionID, acl.AccessReadWrite); err != nil { + return err + } + + applied, err := h.kvStore.SetNX(ctx, identity, scope, key, value, userID, workspace, ttl) + h.auditConditionalWrite(ctx, identity, authority, sessionID, audit.OpKVSetNX, scope, key, userID, workspace, applied, ttl, err) + if err != nil { + logging.Logger.Error().Err(err).Str("identity", identity.String()).Str("scope", string(scope)).Str("key", key).Msg("KV SET_NX failed") + return status.Errorf(codes.Internal, "failed to setnx key: %v", err) + } + + sendResponse(&pb.DownstreamMessage{ + Payload: &pb.DownstreamMessage_Kv{ + Kv: &pb.KVResponse{Success: true, Applied: applied, RequestId: requestID}, + }, + }) + return nil +} + +func (h *KVHandler) handleSetAdd( + ctx context.Context, + identity models.Identity, + authority *acl.ResolvedAuthority, + sessionID uuid.UUID, + scope kv.KVScope, + key string, + value []byte, + userID string, + workspace string, + ttl time.Duration, + requestID string, + sendResponse func(*pb.DownstreamMessage), +) error { + ctx, span := tracing.Tracer.Start(ctx, "aether.kv.set_add") + defer span.End() + span.SetAttributes( + attribute.String("aether.kv.scope", string(scope)), + attribute.String("aether.kv.key", key), + attribute.String("aether.kv.workspace", workspace), + ) + + if err := h.checkKeyPermission(ctx, identity, authority, scope, key, audit.OpKVSetAdd, workspace, sessionID, acl.AccessReadWrite); err != nil { + return err + } + + added, cardinality, err := h.kvStore.SetAdd(ctx, identity, scope, key, string(value), userID, workspace, ttl) + h.auditConditionalWrite(ctx, identity, authority, sessionID, audit.OpKVSetAdd, scope, key, userID, workspace, added, ttl, err) + if err != nil { + logging.Logger.Error().Err(err).Str("identity", identity.String()).Str("scope", string(scope)).Str("key", key).Msg("KV SET_ADD failed") + return status.Errorf(codes.Internal, "failed to set_add key: %v", err) + } + + sendResponse(&pb.DownstreamMessage{ + Payload: &pb.DownstreamMessage_Kv{ + Kv: &pb.KVResponse{Success: true, RequestId: requestID, CounterValue: cardinality, Applied: added}, + }, + }) + return nil +} + +func (h *KVHandler) handleSetCard( + ctx context.Context, + identity models.Identity, + authority *acl.ResolvedAuthority, + sessionID uuid.UUID, + scope kv.KVScope, + key string, + userID string, + workspace string, + requestID string, + sendResponse func(*pb.DownstreamMessage), +) error { + ctx, span := tracing.Tracer.Start(ctx, "aether.kv.set_card") + defer span.End() + span.SetAttributes( + attribute.String("aether.kv.scope", string(scope)), + attribute.String("aether.kv.key", key), + attribute.String("aether.kv.workspace", workspace), + ) + + if err := h.checkKeyPermission(ctx, identity, authority, scope, key, audit.OpKVSetCard, workspace, sessionID, acl.AccessRead); err != nil { + return err + } + + cardinality, err := h.kvStore.SetCard(ctx, identity, scope, key, userID, workspace) + + // Audit logging (reads are audited, mirroring handleGet) + if h.auditLogger != nil { + success := err == nil + errorMsg := "" + if err != nil { + errorMsg = err.Error() + } + metadata := map[string]interface{}{ + "scope": string(scope), + "key": key, + "workspace": workspace, + } + if userID != "" { + metadata["user_id"] = userID + } + if success { + metadata["cardinality"] = cardinality + } + event := audit.NewKVEvent( + string(identity.Type), + identity.String(), + audit.OpKVSetCard, + key, + workspace, + sessionID, + success, + errorMsg, + metadata, + ) + applyResolvedAuthorityToAuditEvent(event, authority) + h.auditLogger.LogEvent(ctx, event) + } + + if err != nil { + logging.Logger.Error().Err(err).Str("identity", identity.String()).Str("scope", string(scope)).Str("key", key).Msg("KV SET_CARD failed") + return status.Errorf(codes.Internal, "failed to set_card key: %v", err) + } + + sendResponse(&pb.DownstreamMessage{ + Payload: &pb.DownstreamMessage_Kv{ + Kv: &pb.KVResponse{Success: true, RequestId: requestID, CounterValue: cardinality}, + }, + }) + return nil +} + +func (h *KVHandler) handleCompareAndSet( + ctx context.Context, + identity models.Identity, + authority *acl.ResolvedAuthority, + sessionID uuid.UUID, + scope kv.KVScope, + key string, + expected string, + value string, + userID string, + workspace string, + ttl time.Duration, + requestID string, + sendResponse func(*pb.DownstreamMessage), +) error { + ctx, span := tracing.Tracer.Start(ctx, "aether.kv.compare_and_set") + defer span.End() + span.SetAttributes( + attribute.String("aether.kv.scope", string(scope)), + attribute.String("aether.kv.key", key), + attribute.String("aether.kv.workspace", workspace), + ) + + if err := h.checkKeyPermission(ctx, identity, authority, scope, key, audit.OpKVCompareAndSet, workspace, sessionID, acl.AccessReadWrite); err != nil { + return err + } + + applied, err := h.kvStore.CompareAndSet(ctx, identity, scope, key, expected, value, userID, workspace, ttl) + h.auditConditionalWrite(ctx, identity, authority, sessionID, audit.OpKVCompareAndSet, scope, key, userID, workspace, applied, ttl, err) + if err != nil { + logging.Logger.Error().Err(err).Str("identity", identity.String()).Str("scope", string(scope)).Str("key", key).Msg("KV COMPARE_AND_SET failed") + return status.Errorf(codes.Internal, "failed to compare-and-set key: %v", err) + } + + sendResponse(&pb.DownstreamMessage{ + Payload: &pb.DownstreamMessage_Kv{ + Kv: &pb.KVResponse{Success: true, Applied: applied, RequestId: requestID}, + }, + }) + return nil +} + +func (h *KVHandler) handleCompareAndDelete( + ctx context.Context, + identity models.Identity, + authority *acl.ResolvedAuthority, + sessionID uuid.UUID, + scope kv.KVScope, + key string, + expected string, + userID string, + workspace string, + requestID string, + sendResponse func(*pb.DownstreamMessage), +) error { + ctx, span := tracing.Tracer.Start(ctx, "aether.kv.compare_and_delete") + defer span.End() + span.SetAttributes( + attribute.String("aether.kv.scope", string(scope)), + attribute.String("aether.kv.key", key), + attribute.String("aether.kv.workspace", workspace), + ) + + if err := h.checkKeyPermission(ctx, identity, authority, scope, key, audit.OpKVCompareAndDelete, workspace, sessionID, acl.AccessReadWrite); err != nil { + return err + } + + applied, err := h.kvStore.CompareAndDelete(ctx, identity, scope, key, expected, userID, workspace) + h.auditConditionalWrite(ctx, identity, authority, sessionID, audit.OpKVCompareAndDelete, scope, key, userID, workspace, applied, 0, err) + if err != nil { + logging.Logger.Error().Err(err).Str("identity", identity.String()).Str("scope", string(scope)).Str("key", key).Msg("KV COMPARE_AND_DELETE failed") + return status.Errorf(codes.Internal, "failed to compare-and-delete key: %v", err) + } + + sendResponse(&pb.DownstreamMessage{ + Payload: &pb.DownstreamMessage_Kv{ + Kv: &pb.KVResponse{Success: true, Applied: applied, RequestId: requestID}, + }, + }) + return nil +} + +// auditConditionalWrite emits an audit event for the SetNX/CompareAndSet/ +// CompareAndDelete conditional-write ops, centralizing the shared metadata +// shape (op, scope, key, workspace, applied, ttl). +func (h *KVHandler) auditConditionalWrite( + ctx context.Context, + identity models.Identity, + authority *acl.ResolvedAuthority, + sessionID uuid.UUID, + operation string, + scope kv.KVScope, + key, userID, workspace string, + applied bool, + ttl time.Duration, + opErr error, +) { + if h.auditLogger == nil { + return + } + success := opErr == nil + errorMsg := "" + if opErr != nil { + errorMsg = opErr.Error() + } + metadata := map[string]interface{}{ + "scope": string(scope), + "key": key, + "workspace": workspace, + } + if userID != "" { + metadata["user_id"] = userID + } + if ttl > 0 { + metadata["ttl_seconds"] = int(ttl.Seconds()) + } + if success { + metadata["applied"] = applied + } + event := audit.NewKVEvent( + string(identity.Type), + identity.String(), + operation, + key, + workspace, + sessionID, + success, + errorMsg, + metadata, + ) + applyResolvedAuthorityToAuditEvent(event, authority) + h.auditLogger.LogEvent(ctx, event) +} + // itemsToKeys extracts just the keys from a map (for backward compatibility) func itemsToKeys(m map[string]string) []string { keys := make([]string, 0, len(m)) diff --git a/server/internal/gateway/kv_handler_coord_test.go b/server/internal/gateway/kv_handler_coord_test.go new file mode 100644 index 0000000..1c884bb --- /dev/null +++ b/server/internal/gateway/kv_handler_coord_test.go @@ -0,0 +1,274 @@ +package gateway + +import ( + "context" + "testing" + + "github.com/google/uuid" + pb "github.com/scitrera/aether/api/proto" + "github.com/scitrera/aether/internal/kv" + "github.com/scitrera/aether/pkg/models" +) + +var workflowEngineIdentity = models.Identity{Type: models.PrincipalWorkflowEngine} + +// TestKVHandler_WorkflowEngine_AllowedOnCoordKey verifies the WorkflowEngine +// principal — previously blocked from KV entirely — can operate on the reserved +// coordination namespace, and that the infra fast-path bypasses a denying ACL +// for those keys. +func TestKVHandler_WorkflowEngine_AllowedOnCoordKey(t *testing.T) { + aclMock := newMockACLChecker() + aclMock.setScopeRule(string(kv.ScopeGlobal), false, 0) // deny global scope broadly + h := newTestKVHandlerWithACL(newMockKVReadWriter(), aclMock) + + op := &pb.KVOperation{ + Op: pb.KVOperation_SET_NX, + Scope: pb.KVOperation_GLOBAL, + Key: kv.ReservedCoordKeyPrefix + "workflow/leader", + Value: []byte("owner-a"), + } + cb, msgs := captureResponses() + if err := h.HandleKVOperation(context.Background(), workflowEngineIdentity, uuid.New(), nil, op, cb); err != nil { + t.Fatalf("WFE SET_NX on coord key should be allowed via fast-path, got: %v", err) + } + if r := lastKV(t, *msgs); !r.Success || !r.Applied { + t.Fatalf("WFE coord SET_NX success=%v applied=%v, want true/true", r.Success, r.Applied) + } +} + +// TestKVHandler_WorkflowEngine_DeniedOffCoordKey verifies the fast-path is +// scoped to the reserved namespace: a WFE op on a non-reserved key still goes +// through ACL and is denied when no grant exists. +func TestKVHandler_WorkflowEngine_DeniedOffCoordKey(t *testing.T) { + aclMock := newMockACLChecker() + aclMock.setScopeRule(string(kv.ScopeGlobal), false, 0) + h := newTestKVHandlerWithACL(newMockKVReadWriter(), aclMock) + + op := &pb.KVOperation{ + Op: pb.KVOperation_SET_NX, + Scope: pb.KVOperation_GLOBAL, + Key: "app/data/key", // not under the reserved coord prefix + Value: []byte("x"), + } + cb, _ := captureResponses() + if err := h.HandleKVOperation(context.Background(), workflowEngineIdentity, uuid.New(), nil, op, cb); err == nil { + t.Fatal("WFE SET_NX on a non-reserved key should be denied by ACL") + } +} + +// TestKVHandler_Agent_NoCoordFastPath verifies the infra fast-path is +// WFE-only: an Agent on a reserved coord key with a denying ACL is still denied. +func TestKVHandler_Agent_NoCoordFastPath(t *testing.T) { + aclMock := newMockACLChecker() + aclMock.setScopeRule(string(kv.ScopeGlobal), false, 0) + h := newTestKVHandlerWithACL(newMockKVReadWriter(), aclMock) + + op := &pb.KVOperation{ + Op: pb.KVOperation_SET_NX, + Scope: pb.KVOperation_GLOBAL, + Key: kv.ReservedCoordKeyPrefix + "workflow/leader", + Value: []byte("x"), + } + cb, _ := captureResponses() + if err := h.HandleKVOperation(context.Background(), agentIdentity, uuid.New(), nil, op, cb); err == nil { + t.Fatal("Agent on reserved coord key should NOT get the infra fast-path; expected ACL denial") + } +} + +// lastKV returns the KVResponse from the most recent captured downstream msg. +func lastKV(t *testing.T, msgs []*pb.DownstreamMessage) *pb.KVResponse { + t.Helper() + if len(msgs) == 0 { + t.Fatal("no downstream message captured") + } + kvr := msgs[len(msgs)-1].GetKv() + if kvr == nil { + t.Fatalf("last message is not a KVResponse: %T", msgs[len(msgs)-1].GetPayload()) + } + return kvr +} + +// TestKVHandler_SetNX_AcquireThenContend verifies SET_NX acquires once and the +// second attempt reports applied=false (key already held). +func TestKVHandler_SetNX_AcquireThenContend(t *testing.T) { + store := newMockKVReadWriter() + h := newTestKVHandler(store) + + op := &pb.KVOperation{ + Op: pb.KVOperation_SET_NX, + Scope: pb.KVOperation_GLOBAL, + Key: "lock", + Value: []byte("owner-a"), + Workspace: "ws1", + } + cb, msgs := captureResponses() + if err := h.HandleKVOperation(context.Background(), agentIdentity, uuid.New(), nil, op, cb); err != nil { + t.Fatalf("SET_NX: %v", err) + } + if r := lastKV(t, *msgs); !r.Success || !r.Applied { + t.Fatalf("first SET_NX success=%v applied=%v, want true/true", r.Success, r.Applied) + } + + op2 := &pb.KVOperation{ + Op: pb.KVOperation_SET_NX, + Scope: pb.KVOperation_GLOBAL, + Key: "lock", + Value: []byte("owner-b"), + Workspace: "ws1", + } + cb2, msgs2 := captureResponses() + if err := h.HandleKVOperation(context.Background(), agentIdentity, uuid.New(), nil, op2, cb2); err != nil { + t.Fatalf("SET_NX 2: %v", err) + } + if r := lastKV(t, *msgs2); !r.Success || r.Applied { + t.Fatalf("second SET_NX success=%v applied=%v, want true/false", r.Success, r.Applied) + } +} + +// TestKVHandler_CompareAndSet_MatchAndMismatch verifies CAS routes the +// expected_value and reports applied correctly. +func TestKVHandler_CompareAndSet_MatchAndMismatch(t *testing.T) { + store := newMockKVReadWriter() + h := newTestKVHandler(store) + id := uuid.New() + + // Seed via SET_NX. + seed := &pb.KVOperation{Op: pb.KVOperation_SET_NX, Scope: pb.KVOperation_GLOBAL, Key: "k", Value: []byte("owner-a"), Workspace: "ws1"} + cb, _ := captureResponses() + if err := h.HandleKVOperation(context.Background(), agentIdentity, id, nil, seed, cb); err != nil { + t.Fatalf("seed: %v", err) + } + + // Mismatch. + mis := &pb.KVOperation{Op: pb.KVOperation_COMPARE_AND_SET, Scope: pb.KVOperation_GLOBAL, Key: "k", ExpectedValue: []byte("owner-x"), Value: []byte("owner-b"), Workspace: "ws1"} + cbM, msgsM := captureResponses() + if err := h.HandleKVOperation(context.Background(), agentIdentity, id, nil, mis, cbM); err != nil { + t.Fatalf("CAS mismatch: %v", err) + } + if r := lastKV(t, *msgsM); !r.Success || r.Applied { + t.Fatalf("CAS mismatch success=%v applied=%v, want true/false", r.Success, r.Applied) + } + + // Match. + mat := &pb.KVOperation{Op: pb.KVOperation_COMPARE_AND_SET, Scope: pb.KVOperation_GLOBAL, Key: "k", ExpectedValue: []byte("owner-a"), Value: []byte("owner-b"), Workspace: "ws1"} + cbA, msgsA := captureResponses() + if err := h.HandleKVOperation(context.Background(), agentIdentity, id, nil, mat, cbA); err != nil { + t.Fatalf("CAS match: %v", err) + } + if r := lastKV(t, *msgsA); !r.Success || !r.Applied { + t.Fatalf("CAS match success=%v applied=%v, want true/true", r.Success, r.Applied) + } +} + +// TestKVHandler_CompareAndDelete_MatchReleases verifies CAD releases only on a +// matching expected_value. +func TestKVHandler_CompareAndDelete_MatchReleases(t *testing.T) { + store := newMockKVReadWriter() + h := newTestKVHandler(store) + id := uuid.New() + + seed := &pb.KVOperation{Op: pb.KVOperation_SET_NX, Scope: pb.KVOperation_GLOBAL, Key: "k", Value: []byte("owner-a"), Workspace: "ws1"} + cb, _ := captureResponses() + if err := h.HandleKVOperation(context.Background(), agentIdentity, id, nil, seed, cb); err != nil { + t.Fatalf("seed: %v", err) + } + + // Wrong owner cannot release. + wrong := &pb.KVOperation{Op: pb.KVOperation_COMPARE_AND_DELETE, Scope: pb.KVOperation_GLOBAL, Key: "k", ExpectedValue: []byte("owner-x"), Workspace: "ws1"} + cbW, msgsW := captureResponses() + if err := h.HandleKVOperation(context.Background(), agentIdentity, id, nil, wrong, cbW); err != nil { + t.Fatalf("CAD wrong: %v", err) + } + if r := lastKV(t, *msgsW); !r.Success || r.Applied { + t.Fatalf("CAD wrong success=%v applied=%v, want true/false", r.Success, r.Applied) + } + + // Owner releases; key is then re-acquirable. + right := &pb.KVOperation{Op: pb.KVOperation_COMPARE_AND_DELETE, Scope: pb.KVOperation_GLOBAL, Key: "k", ExpectedValue: []byte("owner-a"), Workspace: "ws1"} + cbR, msgsR := captureResponses() + if err := h.HandleKVOperation(context.Background(), agentIdentity, id, nil, right, cbR); err != nil { + t.Fatalf("CAD right: %v", err) + } + if r := lastKV(t, *msgsR); !r.Success || !r.Applied { + t.Fatalf("CAD right success=%v applied=%v, want true/true", r.Success, r.Applied) + } + + reacq := &pb.KVOperation{Op: pb.KVOperation_SET_NX, Scope: pb.KVOperation_GLOBAL, Key: "k", Value: []byte("owner-c"), Workspace: "ws1"} + cbX, msgsX := captureResponses() + if err := h.HandleKVOperation(context.Background(), agentIdentity, id, nil, reacq, cbX); err != nil { + t.Fatalf("re-acquire: %v", err) + } + if r := lastKV(t, *msgsX); !r.Success || !r.Applied { + t.Fatalf("re-acquire after release success=%v applied=%v, want true/true", r.Success, r.Applied) + } +} + +// TestKVHandler_UserIdentity_DeniedForSetNX confirms the principal-type gate +// still rejects non-agent/task/service callers for the new ops. +func TestKVHandler_UserIdentity_DeniedForSetNX(t *testing.T) { + h := newTestKVHandler(newMockKVReadWriter()) + op := &pb.KVOperation{Op: pb.KVOperation_SET_NX, Scope: pb.KVOperation_GLOBAL, Key: "lock", Value: []byte("x")} + cb, _ := captureResponses() + if err := h.HandleKVOperation(context.Background(), userIdentity, uuid.New(), nil, op, cb); err == nil { + t.Fatal("expected permission denied for User principal on SET_NX") + } +} + +// TestKVHandler_SetAdd_And_SetCard verifies SET_ADD reports applied + the +// running cardinality, dedupes a repeat member, and that SET_CARD reads back +// the set cardinality without mutating it. +func TestKVHandler_SetAdd_And_SetCard(t *testing.T) { + store := newMockKVReadWriter() + h := newTestKVHandler(store) + id := uuid.New() + + // First member: newly added, cardinality 1. + add1 := &pb.KVOperation{Op: pb.KVOperation_SET_ADD, Scope: pb.KVOperation_GLOBAL, Key: "members", Value: []byte("a"), Workspace: "ws1"} + cb1, msgs1 := captureResponses() + if err := h.HandleKVOperation(context.Background(), agentIdentity, id, nil, add1, cb1); err != nil { + t.Fatalf("SET_ADD a: %v", err) + } + if r := lastKV(t, *msgs1); !r.Success || !r.Applied || r.CounterValue != 1 { + t.Fatalf("SET_ADD a success=%v applied=%v card=%d, want true/true/1", r.Success, r.Applied, r.CounterValue) + } + + // Second distinct member: newly added, cardinality 2. + add2 := &pb.KVOperation{Op: pb.KVOperation_SET_ADD, Scope: pb.KVOperation_GLOBAL, Key: "members", Value: []byte("b"), Workspace: "ws1"} + cb2, msgs2 := captureResponses() + if err := h.HandleKVOperation(context.Background(), agentIdentity, id, nil, add2, cb2); err != nil { + t.Fatalf("SET_ADD b: %v", err) + } + if r := lastKV(t, *msgs2); !r.Success || !r.Applied || r.CounterValue != 2 { + t.Fatalf("SET_ADD b success=%v applied=%v card=%d, want true/true/2", r.Success, r.Applied, r.CounterValue) + } + + // Repeat member: not newly added, cardinality unchanged at 2. + add1Again := &pb.KVOperation{Op: pb.KVOperation_SET_ADD, Scope: pb.KVOperation_GLOBAL, Key: "members", Value: []byte("a"), Workspace: "ws1"} + cb3, msgs3 := captureResponses() + if err := h.HandleKVOperation(context.Background(), agentIdentity, id, nil, add1Again, cb3); err != nil { + t.Fatalf("SET_ADD a (repeat): %v", err) + } + if r := lastKV(t, *msgs3); !r.Success || r.Applied || r.CounterValue != 2 { + t.Fatalf("SET_ADD a repeat success=%v applied=%v card=%d, want true/false/2", r.Success, r.Applied, r.CounterValue) + } + + // SET_CARD reads the cardinality back (read-only, no Applied semantics). + card := &pb.KVOperation{Op: pb.KVOperation_SET_CARD, Scope: pb.KVOperation_GLOBAL, Key: "members", Workspace: "ws1"} + cb4, msgs4 := captureResponses() + if err := h.HandleKVOperation(context.Background(), agentIdentity, id, nil, card, cb4); err != nil { + t.Fatalf("SET_CARD: %v", err) + } + if r := lastKV(t, *msgs4); !r.Success || r.CounterValue != 2 { + t.Fatalf("SET_CARD success=%v card=%d, want true/2", r.Success, r.CounterValue) + } + + // SET_CARD on an absent key returns 0. + cardAbsent := &pb.KVOperation{Op: pb.KVOperation_SET_CARD, Scope: pb.KVOperation_GLOBAL, Key: "missing", Workspace: "ws1"} + cb5, msgs5 := captureResponses() + if err := h.HandleKVOperation(context.Background(), agentIdentity, id, nil, cardAbsent, cb5); err != nil { + t.Fatalf("SET_CARD absent: %v", err) + } + if r := lastKV(t, *msgs5); !r.Success || r.CounterValue != 0 { + t.Fatalf("SET_CARD absent success=%v card=%d, want true/0", r.Success, r.CounterValue) + } +} diff --git a/server/internal/gateway/kv_handler_test.go b/server/internal/gateway/kv_handler_test.go index 955ec09..21426d5 100644 --- a/server/internal/gateway/kv_handler_test.go +++ b/server/internal/gateway/kv_handler_test.go @@ -424,6 +424,58 @@ func TestKVHandler_List_SuccessSendsKVResponseWithItems(t *testing.T) { } } +func TestKVHandler_PurgeIdentity_DeletesMatchingAndReturnsCount(t *testing.T) { + store := newMockKVReadWriter() + store.listData = map[string]string{ + "sidecar::proxy_config": "cfg", + "workclaw::turn::agent:main": "turn", + "unrelated::key": "keep", + } + h := newTestKVHandler(store) + cb, msgs := captureResponses() + + op := &pb.KVOperation{ + Op: pb.KVOperation_PURGE_IDENTITY, + Scope: pb.KVOperation_GLOBAL_EXCLUSIVE, + TargetIdentity: "sv::sandbox-sidecar::abc123", + Key: "workclaw::", // prefix filter + RequestId: "req-purge-1", + } + if err := h.HandleKVOperation(context.Background(), agentIdentity, uuid.New(), nil, op, cb); err != nil { + t.Fatalf("unexpected error for PURGE_IDENTITY: %v", err) + } + kvResp := (*msgs)[0].GetKv() + if kvResp == nil || !kvResp.Success { + t.Fatalf("expected successful KVResponse, got %+v", kvResp) + } + if kvResp.CounterValue != 1 { + t.Errorf("expected 1 key deleted (workclaw:: prefix), got %d", kvResp.CounterValue) + } + // Removal-only: response must NOT leak keys or values. + if len(kvResp.Keys) != 0 || len(kvResp.KvMap) != 0 { + t.Errorf("purge response leaked data: keys=%v kv_map=%v", kvResp.Keys, kvResp.KvMap) + } + if _, ok := store.listData["workclaw::turn::agent:main"]; ok { + t.Error("expected workclaw:: key to be deleted") + } + if _, ok := store.listData["unrelated::key"]; !ok { + t.Error("unrelated key must survive a prefixed purge") + } +} + +func TestKVHandler_PurgeIdentity_RequiresTargetIdentity(t *testing.T) { + h := newTestKVHandler(newMockKVReadWriter()) + cb, _ := captureResponses() + op := &pb.KVOperation{ + Op: pb.KVOperation_PURGE_IDENTITY, + Scope: pb.KVOperation_GLOBAL_EXCLUSIVE, + RequestId: "req-purge-2", + } + if err := h.HandleKVOperation(context.Background(), agentIdentity, uuid.New(), nil, op, cb); err == nil { + t.Fatal("expected error when target_identity is empty") + } +} + func TestKVHandler_List_StoreError_ReturnsError(t *testing.T) { store := newMockKVReadWriter() store.listErr = errors.New("list failed") @@ -1070,3 +1122,109 @@ func TestKVHandler_DecrementIf_GatewayHandler_ConcurrentOps(t *testing.T) { t.Errorf("final balance = %q, want \"0\"", finalVal) } } + +// TestKVHandler_ErrorStillEmitsTypedFailureResponse verifies the gateway +// emits a `KVResponse{Success=false, RequestId=…}` even when the sub-handler +// errors before its own success path could send one. This is the wire +// contract the SDK's typed pending-request map depends on: KVGetSync / +// KVPutSync registers a chan keyed by request_id and waits for a +// KVResponse with that id. Without this emission, ACL or store errors +// would strand the waiter on its per-op timeout (5–30s) even though the +// gateway responded instantly with an ErrorResponse — that surfaces as +// a deadline-exceeded error that hides the real cause. +// +// Regression test for the deadlock we hit during the webhookservice E2E: +// pool-worker dispatch did endpoints.Get → KVGetSync; the worker lacked +// the workspace KV ACL grant, gateway emitted only ErrorResponse, and +// the worker hung 5s waiting for a KVResponse that never came. +func TestKVHandler_ErrorStillEmitsTypedFailureResponse(t *testing.T) { + t.Run("ACL deny on GET", func(t *testing.T) { + aclMock := newMockACLChecker() + // Key-level fallback, scope-level deny. + aclMock.setScopeRule("workspace", false, acl.AccessNone) + + h := newTestKVHandlerWithACL(newMockKVReadWriter(), aclMock) + cb, msgs := captureResponses() + + op := &pb.KVOperation{ + Op: pb.KVOperation_GET, + Scope: pb.KVOperation_WORKSPACE, + Key: "denied-key", + Workspace: "ws1", + RequestId: "req-deny-1", + } + + err := h.HandleKVOperation(context.Background(), agentIdentity, uuid.New(), nil, op, cb) + if err == nil { + t.Fatal("expected error for ACL deny, got nil") + } + if len(*msgs) != 1 { + t.Fatalf("expected 1 typed response (Success=false), got %d", len(*msgs)) + } + kv := (*msgs)[0].GetKv() + if kv == nil { + t.Fatal("response was not a KVResponse") + } + if kv.Success { + t.Error("expected Success=false on ACL deny") + } + if kv.RequestId != "req-deny-1" { + t.Errorf("RequestId = %q, want req-deny-1", kv.RequestId) + } + }) + + t.Run("ACL deny on PUT", func(t *testing.T) { + aclMock := newMockACLChecker() + aclMock.setKeyRule("denied-write-key", false, acl.AccessNone) + + h := newTestKVHandlerWithACL(newMockKVReadWriter(), aclMock) + cb, msgs := captureResponses() + + op := &pb.KVOperation{ + Op: pb.KVOperation_PUT, + Scope: pb.KVOperation_GLOBAL, + Key: "denied-write-key", + Value: []byte("v"), + RequestId: "req-deny-put", + } + + err := h.HandleKVOperation(context.Background(), agentIdentity, uuid.New(), nil, op, cb) + if err == nil { + t.Fatal("expected error for ACL deny, got nil") + } + if len(*msgs) != 1 { + t.Fatalf("expected 1 typed response (Success=false), got %d", len(*msgs)) + } + kv := (*msgs)[0].GetKv() + if kv == nil || kv.Success || kv.RequestId != "req-deny-put" { + t.Errorf("expected KVResponse{Success=false, RequestId=req-deny-put}, got %+v", kv) + } + }) + + t.Run("empty RequestId omits typed failure", func(t *testing.T) { + // Sanity: fire-and-forget callers (no request_id) shouldn't get + // a stray empty-id KVResponse — there's no pending waiter and + // the extra frame would just be noise. + aclMock := newMockACLChecker() + aclMock.setScopeRule("global", false, acl.AccessNone) + + h := newTestKVHandlerWithACL(newMockKVReadWriter(), aclMock) + cb, msgs := captureResponses() + + op := &pb.KVOperation{ + Op: pb.KVOperation_PUT, + Scope: pb.KVOperation_GLOBAL, + Key: "k", + Value: []byte("v"), + // no RequestId set + } + + err := h.HandleKVOperation(context.Background(), agentIdentity, uuid.New(), nil, op, cb) + if err == nil { + t.Fatal("expected error, got nil") + } + if len(*msgs) != 0 { + t.Errorf("expected no extra response when RequestId is empty, got %d", len(*msgs)) + } + }) +} diff --git a/server/internal/gateway/orchestration_idempotency_test.go b/server/internal/gateway/orchestration_idempotency_test.go new file mode 100644 index 0000000..476333f --- /dev/null +++ b/server/internal/gateway/orchestration_idempotency_test.go @@ -0,0 +1,184 @@ +package gateway + +// Tests for idempotent task creation (CreateTaskRequest.idempotency_key). +// A non-empty key dedups creation: the first create persists a task, any +// subsequent create with the same key is suppressed and echoes the original +// task's identity. An empty key preserves the prior behavior (no dedup). + +import ( + "context" + "errors" + "testing" + + pb "github.com/scitrera/aether/api/proto" + "github.com/scitrera/aether/internal/orchestration" + "github.com/scitrera/aether/pkg/tasks" +) + +var errTestKV = errors.New("simulated kv outage") + +// newIdemTestServer builds a gateway with a real (sqlite-backed) self-assign +// task service and a fresh mock KV so handleCreateTask exercises the actual +// CreateTask persistence + idempotency ledger paths. SessionRegistry is nil: +// the self_assign path never derefs it. +func newIdemTestServer(t *testing.T) (*GatewayServer, *mockKVReadWriter, func()) { + t.Helper() + s, cleanup := newTaskTestServerWithSQLiteStore(t) + store := newMockKVReadWriter() + s.kv = store + s.orchestration = &OrchestrationServices{ + TaskService: orchestration.NewTaskAssignmentService(s.taskStore, nil, nil, nil, nil), + } + return s, store, cleanup +} + +func countWorkspaceTasks(t *testing.T, s *GatewayServer, workspace string) int { + t.Helper() + list, err := s.taskStore.ListTasks(context.Background(), &tasks.TaskFilter{Workspace: workspace}) + if err != nil { + t.Fatalf("ListTasks: %v", err) + } + return len(list) +} + +// TestHandleCreateTask_IdempotencyKey_DedupsCreation asserts that two creates +// carrying the SAME non-empty idempotency_key produce exactly ONE task, and +// the duplicate's response echoes the original task_id. +func TestHandleCreateTask_IdempotencyKey_DedupsCreation(t *testing.T) { + s, _, cleanup := newIdemTestServer(t) + defer cleanup() + + caller := callerIdentity("worker", "alice") + ctx := context.Background() + + newReq := func() *pb.CreateTaskRequest { + return &pb.CreateTaskRequest{ + TaskType: "test", + Workspace: "ws1", + AssignmentMode: pb.TaskAssignmentMode_SELF_ASSIGN, + IdempotencyKey: "join-fire-abc", + RequestId: "req-1", + } + } + + // First create: proceeds, persists one task. + stream1 := &mockStream{} + client1 := newTaskTestClient(stream1, caller) + if err := s.handleCreateTask(ctx, client1, caller, newReq()); err != nil { + t.Fatalf("first handleCreateTask: %v", err) + } + resp1 := stream1.sent[0].GetCreateTask() + if resp1 == nil || !resp1.Success || resp1.TaskId == "" { + t.Fatalf("first create: expected success with task_id, got %+v", resp1) + } + if n := countWorkspaceTasks(t, s, "ws1"); n != 1 { + t.Fatalf("after first create: expected 1 task, got %d", n) + } + + // Second create with the SAME key: suppressed, no second task, echoes id. + stream2 := &mockStream{} + client2 := newTaskTestClient(stream2, caller) + dupReq := newReq() + dupReq.RequestId = "req-2" + if err := s.handleCreateTask(ctx, client2, caller, dupReq); err != nil { + t.Fatalf("duplicate handleCreateTask: %v", err) + } + if n := countWorkspaceTasks(t, s, "ws1"); n != 1 { + t.Fatalf("after duplicate create: expected still 1 task, got %d", n) + } + resp2 := stream2.sent[0].GetCreateTask() + if resp2 == nil || !resp2.Success { + t.Fatalf("duplicate create: expected success response, got %+v", resp2) + } + if resp2.TaskId != resp1.TaskId { + t.Errorf("duplicate create: expected echoed task_id %q, got %q", resp1.TaskId, resp2.TaskId) + } + if resp2.RequestId != "req-2" { + t.Errorf("duplicate create: expected RequestId echoed, got %q", resp2.RequestId) + } +} + +// TestHandleCreateTask_DistinctIdempotencyKeys_BothCreate confirms that two +// creates with DIFFERENT keys each produce their own task. +func TestHandleCreateTask_DistinctIdempotencyKeys_BothCreate(t *testing.T) { + s, _, cleanup := newIdemTestServer(t) + defer cleanup() + + caller := callerIdentity("worker", "alice") + ctx := context.Background() + + for i, key := range []string{"key-a", "key-b"} { + stream := &mockStream{} + client := newTaskTestClient(stream, caller) + req := &pb.CreateTaskRequest{ + TaskType: "test", + Workspace: "ws1", + AssignmentMode: pb.TaskAssignmentMode_SELF_ASSIGN, + IdempotencyKey: key, + } + if err := s.handleCreateTask(ctx, client, caller, req); err != nil { + t.Fatalf("create %d: %v", i, err) + } + } + if n := countWorkspaceTasks(t, s, "ws1"); n != 2 { + t.Fatalf("distinct keys: expected 2 tasks, got %d", n) + } +} + +// TestHandleCreateTask_EmptyIdempotencyKey_NoDedup confirms empty keys are not +// deduped: two creates yield two tasks, matching the pre-feature behavior. +func TestHandleCreateTask_EmptyIdempotencyKey_NoDedup(t *testing.T) { + s, _, cleanup := newIdemTestServer(t) + defer cleanup() + + caller := callerIdentity("worker", "alice") + ctx := context.Background() + + for i := 0; i < 2; i++ { + stream := &mockStream{} + client := newTaskTestClient(stream, caller) + req := &pb.CreateTaskRequest{ + TaskType: "test", + Workspace: "ws1", + AssignmentMode: pb.TaskAssignmentMode_SELF_ASSIGN, + // IdempotencyKey intentionally empty + } + if err := s.handleCreateTask(ctx, client, caller, req); err != nil { + t.Fatalf("create %d: %v", i, err) + } + } + if n := countWorkspaceTasks(t, s, "ws1"); n != 2 { + t.Fatalf("empty key: expected 2 tasks (no dedup), got %d", n) + } +} + +// TestHandleCreateTask_IdempotencyKVError_FailsOpen confirms a KV SetNX failure +// does NOT block task creation (fail-open posture). +func TestHandleCreateTask_IdempotencyKVError_FailsOpen(t *testing.T) { + s, store, cleanup := newIdemTestServer(t) + defer cleanup() + store.setErr = errTestKV + + caller := callerIdentity("worker", "alice") + ctx := context.Background() + + stream := &mockStream{} + client := newTaskTestClient(stream, caller) + req := &pb.CreateTaskRequest{ + TaskType: "test", + Workspace: "ws1", + AssignmentMode: pb.TaskAssignmentMode_SELF_ASSIGN, + IdempotencyKey: "kv-down", + RequestId: "req-fo", + } + if err := s.handleCreateTask(ctx, client, caller, req); err != nil { + t.Fatalf("handleCreateTask (KV down): %v", err) + } + resp := stream.sent[0].GetCreateTask() + if resp == nil || !resp.Success || resp.TaskId == "" { + t.Fatalf("fail-open: expected successful create despite KV error, got %+v", resp) + } + if n := countWorkspaceTasks(t, s, "ws1"); n != 1 { + t.Fatalf("fail-open: expected 1 task, got %d", n) + } +} diff --git a/server/internal/gateway/orchestration_integration.go b/server/internal/gateway/orchestration_integration.go index 1b57e46..8ee8e5a 100644 --- a/server/internal/gateway/orchestration_integration.go +++ b/server/internal/gateway/orchestration_integration.go @@ -2,7 +2,9 @@ package gateway import ( "context" + "crypto/sha256" "database/sql" + "encoding/hex" "fmt" "io" "math/rand/v2" @@ -15,6 +17,7 @@ import ( "github.com/scitrera/aether/internal/acl" "github.com/scitrera/aether/internal/audit" "github.com/scitrera/aether/internal/identval" + "github.com/scitrera/aether/internal/kv" "github.com/scitrera/aether/internal/logging" "github.com/scitrera/aether/internal/metering" "github.com/scitrera/aether/internal/orchestration" @@ -298,6 +301,21 @@ func (s *GatewayServer) deliverQueuedTasksToAgent( return nil } +// idemTaskKeyPrefix namespaces idempotency-ledger keys for task creation. It +// sits alongside kv.ReservedCoordKeyPrefix ("_sys/coord/") under the reserved +// "_sys/" tree so application principals can never collide with it. +const idemTaskKeyPrefix = "_sys/idem/task/" + +// idemTaskTTL bounds how long a creation idempotency_key is remembered. Joins +// retry/restart within a single workflow run, so 24h comfortably covers the +// dedup window while letting the ledger self-expire. +const idemTaskTTL = 24 * time.Hour + +// idemTaskPlaceholder is stored at SetNX time (before the task_id is known) so +// a racing duplicate sees the key claimed; it is overwritten with the real +// task_id once creation succeeds. +const idemTaskPlaceholder = "pending" + // handleCreateTask processes CreateTaskRequest messages func (s *GatewayServer) handleCreateTask( ctx context.Context, @@ -445,7 +463,19 @@ func (s *GatewayServer) handleCreateTask( resolvedAuthority = inherited } - if s.acl != nil { + // The WorkflowEngine is a system principal whose core function is to create + // tasks in response to events, in any workspace it routes for. It holds no + // per-workspace ACL grants (and is not meant to), so it is implicitly + // authorized for OpTaskCreate — mirroring the system-principal handling in + // connect.go ("workspace ACL not applicable") and the WFE KV-coord allowance + // in kv_handler.go. The bypass is scoped to task creation only; the WFE gains + // no other workspace access here. Without this, event-triggered workflow + // rules (e.g. create a pool task on an event) are silently ACL-denied. + isImplicitTaskCreator := identity.Type == models.PrincipalWorkflowEngine + + if isImplicitTaskCreator { + s.logTaskCreateAudit(ctx, identity, client.SessionUUID, taskWorkspace, "", true, "workflow engine implicitly authorized for task creation (system principal)", buildTaskCreateAuditMetadata(req, assignmentMode, taskWorkspace), resolvedAuthority) + } else if s.acl != nil { var decision *acl.ACLDecision if resolvedAuthority != nil { decision, err = s.acl.CheckAccessWithAuthority(ctx, identity, resolvedAuthority, acl.ResourceTypeWorkspace, taskWorkspace, audit.OpTaskCreate, taskWorkspace, client.SessionUUID, acl.AccessReadWrite) @@ -484,6 +514,11 @@ func (s *GatewayServer) handleCreateTask( Payload: req.Payload, CreatorIdentity: identity, ParentTaskID: client.AssociatedTaskID, + RetryPolicy: retryPolicyFromProto(req.GetRetryPolicy()), + Priority: int32(req.GetPriority()), + CorrelationID: req.GetCorrelationId(), + RootTaskID: req.GetRootTaskId(), + CompletionEvent: completionConfigFromProto(req.GetCompletionEvent()), } // Fix AA: seed the task's Authority.SubjectType/SubjectID from the resolved // OBO subject so downstream consumers (buildTaskContext → @@ -498,6 +533,45 @@ func (s *GatewayServer) handleCreateTask( } } + // Idempotent creation: when the caller supplies a non-empty + // idempotency_key, the FIRST create for that key proceeds normally and any + // subsequent create with the SAME key is a no-op that returns the original + // task's identity. This backs exactly-once downstream firing for workflow + // joins, whose on_complete/on_timeout actions may fire more than once under + // retries/engine restart/timeout-vs-completion races. Empty key ⇒ unchanged. + // + // The ledger lives under the reserved "_sys/idem/" tree in the GLOBAL + // (tenant-wide) scope, written with the trusted admin identity exactly as + // other gateway-internal KV writes do (see admin_workspaces.go). The user + // portion is sha256-hexed so arbitrary key bytes can never produce an + // invalid/oversized KV key. KV failures FAIL OPEN (proceed with creation): + // a rare duplicate under a KV outage is acceptable, a dropped task is not. + var idemKey string + if rawIdem := req.GetIdempotencyKey(); rawIdem != "" { + sum := sha256.Sum256([]byte(rawIdem)) + idemKey = idemTaskKeyPrefix + hex.EncodeToString(sum[:]) + fresh, idemErr := s.kv.SetNX(ctx, adminIdentity, kv.ScopeGlobal, idemKey, idemTaskPlaceholder, "", "", idemTaskTTL) + if idemErr != nil { + logging.Logger.Warn().Err(idemErr).Str("identity", identity.String()).Msg("idempotency SetNX failed, failing open and proceeding with task creation") + } else if !fresh { + // Duplicate create: do NOT create a second task. Echo the original + // task's identity when a response is expected. The stored value may + // still be the placeholder if the original create has not finished; + // in that case return an empty task_id but still success=true. + existingTaskID, getErr := s.kv.Get(ctx, adminIdentity, kv.ScopeGlobal, idemKey, "", "") + if getErr != nil { + logging.Logger.Warn().Err(getErr).Str("identity", identity.String()).Msg("idempotency ledger read failed for duplicate create") + existingTaskID = "" + } + if existingTaskID == idemTaskPlaceholder { + existingTaskID = "" + } + logging.Logger.Info().Str("identity", identity.String()).Str("task_id", existingTaskID).Msg("duplicate task creation suppressed by idempotency_key") + sendCreateTaskResponse(true, existingTaskID, "", "", "", "") + return nil + } + } + // Create task response, err := s.orchestration.TaskService.CreateTask(ctx, taskReq) if err != nil { @@ -508,6 +582,16 @@ func (s *GatewayServer) handleCreateTask( return errors.WrapError(err, "failed to create task") } + // Best-effort: overwrite the idempotency placeholder with the real task_id + // so a later duplicate create can echo the original task's identity. Failure + // here is non-fatal (the ledger already blocks the duplicate via the + // placeholder; the duplicate just returns an empty task_id). + if idemKey != "" { + if idemSetErr := s.kv.Set(ctx, adminIdentity, kv.ScopeGlobal, idemKey, response.TaskID, "", "", idemTaskTTL); idemSetErr != nil { + logging.Logger.Warn().Err(idemSetErr).Str("task_id", response.TaskID).Msg("failed to record task_id in idempotency ledger") + } + } + if resolvedAuthority != nil { taskReq.Metadata, err = s.establishTaskAuthorityGrant(ctx, response.TaskID, taskReq, response, resolvedAuthority) if err != nil { @@ -1184,13 +1268,20 @@ func (s *GatewayServer) removeFromImplIndex(identity models.Identity, client *Cl s.implIndexMu.Unlock() } -// findWorkerByImplementation finds an online agent matching the given implementation -// in the specified workspace. Uses power-of-two-choices load balancing: picks 2 random -// candidates and selects the one with fewer active pool tasks. +// findWorkerByImplementation finds an online worker (agent OR service) +// matching the given implementation. Workspace-scoped principals (agents) +// register under "workspace:impl"; cross-workspace services register under +// ":impl". The lookup first tries the workspace-specific key, then falls +// back to the workspace-less key so services like sv::webhook can pick up +// pool tasks created in any workspace. Uses power-of-two-choices load +// balancing within the chosen bucket. func (s *GatewayServer) findWorkerByImplementation(implementation, workspace string) *ClientSession { - key := workspace + ":" + implementation s.implIndexMu.RLock() - clients := s.implementationIndex[key] + clients := s.implementationIndex[workspace+":"+implementation] + if len(clients) == 0 && workspace != "" { + // Cross-workspace fallback: services register under empty workspace. + clients = s.implementationIndex[":"+implementation] + } n := len(clients) if n == 0 { s.implIndexMu.RUnlock() diff --git a/server/internal/gateway/progress.go b/server/internal/gateway/progress.go index 39b321e..4897757 100644 --- a/server/internal/gateway/progress.go +++ b/server/internal/gateway/progress.go @@ -11,6 +11,7 @@ import ( "github.com/scitrera/aether/internal/logging" "github.com/scitrera/aether/internal/tracing" "github.com/scitrera/aether/pkg/models" + "github.com/scitrera/aether/pkg/tasks" "github.com/scitrera/aether/sdk/go/aether" "go.opentelemetry.io/otel/attribute" "google.golang.org/protobuf/proto" @@ -75,13 +76,19 @@ func (s *GatewayServer) handleProgressReport(ctx context.Context, client *Client attribute.String("task_id", report.TaskId), ) - // Only agents and tasks can report progress - if sender.Type != models.PrincipalAgent && sender.Type != models.PrincipalTask { + // Only agents, tasks, and services can report progress. Services are + // permitted because sidecar bridges (identity sv::sandbox-sidecar::) + // relay chat-lifecycle progress on behalf of the agent harness running + // inside the sandbox — they target a user recipient (us::{user}) just + // like an agent would, and recipient filtering still applies downstream. + if sender.Type != models.PrincipalAgent && + sender.Type != models.PrincipalTask && + sender.Type != models.PrincipalService { if sendErr := client.SafeSend(&pb.DownstreamMessage{ Payload: &pb.DownstreamMessage_Error{ Error: &pb.ErrorResponse{ Code: "ERR_INVALID_PRINCIPAL", - Message: "only agents and tasks can report progress", + Message: "only agents, tasks, and services can report progress", }, }, }); sendErr != nil { @@ -124,9 +131,24 @@ func (s *GatewayServer) handleProgressReport(ctx context.Context, client *Client // For empty or non-user recipients, fall back to pg::{sender.Workspace} // broadcast — preserving orchestrator/parent-agent consumption patterns // for task-kind progress. - progressTopic, err := models.ProgressTopic(sender.Workspace) + // For task-bound progress from a workspace-less sender (e.g. a Service + // principal like the in-process ingest worker, sv::memorylayer), the sender + // has no workspace, so pg::{sender.Workspace} collapses to the empty "pg::" + // topic — which has no JetStream stream under AetherLite, so the publish + // hard-fails. Derive the broadcast workspace from the task row instead, so + // progress routes to pg::{task.workspace} and reaches that workspace's + // subscribers (the user/window sessions). Only consulted when the sender + // itself is workspace-less, so workspace-scoped agent senders are unchanged. + broadcastWorkspace := sender.Workspace + if broadcastWorkspace == "" && report.TaskId != "" && s.taskStore != nil { + if t, terr := s.taskStore.GetTask(ctx, report.TaskId); terr == nil && t != nil && t.Workspace != "" { + broadcastWorkspace = t.Workspace + } + } + + progressTopic, err := models.ProgressTopic(broadcastWorkspace) if err != nil { - logging.Logger.Warn().Err(err).Str("workspace", sender.Workspace).Msg("invalid workspace for progress topic; dropping report") + logging.Logger.Warn().Err(err).Str("workspace", broadcastWorkspace).Msg("invalid workspace for progress topic; dropping report") return } if report.Recipient != "" { @@ -162,7 +184,7 @@ func (s *GatewayServer) handleProgressReport(ctx context.Context, client *Client Summary: report.Summary, Step: report.Step, TimestampMs: now.UnixMilli(), - Workspace: sender.Workspace, + Workspace: broadcastWorkspace, RequestId: report.RequestId, Metadata: report.Metadata, Recipient: report.Recipient, @@ -274,6 +296,22 @@ func (s *GatewayServer) createProgressFilterHandler(client *ClientSession) func( } } + // Phase 2 "Design M": drive the per-task chat message lane off task + // lifecycle. The notice already targets this session's subject (the + // recipient filter above passed), so the session principal IS the + // task's subject — auto-attach / detach the tk::{ws}::{task}::msg lane + // without any additional authz. Subscribe on running, unsubscribe on a + // terminal status. Live (non-replay) subscribe: the running notice + // marks "from here forward". + if update.Kind == pb.ProgressKind_PROGRESS_KIND_TASK && update.TaskId != "" { + switch update.Metadata["status"] { + case "running": + s.subscribeClientToTaskMessages(client, update.Workspace, update.TaskId) + case "completed", "failed", "cancelled": + s.unsubscribeClientFromTaskMessages(client, update.TaskId) + } + } + // Progress updates ride at PriorityBestEffort — they are the first // to be shed when the per-session delivery Semaphore is saturated // by request / response chunk traffic. @@ -291,7 +329,18 @@ func (s *GatewayServer) createProgressFilterHandler(client *ClientSession) func( // spawning agent receives the notification via server-side filtering. // // This method is best-effort: failures are logged but do not block the caller. -func (s *GatewayServer) notifyTaskStatusChange(ctx context.Context, taskID, newStatus, workspace, parentAgentID, errorMsg string) { +// +// When task is non-nil and carries an OBO subject (subject_participant marked, +// Authority.SubjectType == "User", Authority.SubjectID set), an ADDITIONAL +// notice is emitted to the subject's per-user progress topic so the end user's +// browser tabs can be driven by task lifecycle state. The subject emit routes +// like handleProgressReport's us:: path (pg::us::) rather than the +// workspace-broadcast pg:: topic. +func (s *GatewayServer) notifyTaskStatusChange(ctx context.Context, taskID, newStatus, workspace, parentAgentID, errorMsg string, task *tasks.Task) { + // Subject notify is independent of the parent notify: a per-turn chat task + // may have no parent agent but still needs to drive the user's tabs. + defer s.maybeNotifyTaskSubject(ctx, taskID, newStatus, task) + if parentAgentID == "" || workspace == "" { return // No parent to notify or no workspace for the progress topic } @@ -335,6 +384,91 @@ func (s *GatewayServer) notifyTaskStatusChange(ctx context.Context, taskID, newS } } +// taskMetadataTruthy reports whether a task metadata value represents a truthy +// flag. Accepts bool true or the strings "1"/"true" (case-insensitive). +func taskMetadataTruthy(v interface{}) bool { + switch t := v.(type) { + case bool: + return t + case string: + switch strings.ToLower(strings.TrimSpace(t)) { + case "1", "true": + return true + } + } + return false +} + +// maybeNotifyTaskSubject emits an additional ProgressUpdate to the task's OBO +// subject (the end user) when the task opted into subject participation. Routed +// to the per-user progress topic pg::us:: with a bare us:: +// recipient so EVERY one of the user's windows receives it. Best-effort: any +// failure is logged and ignored; never blocks the caller. +func (s *GatewayServer) maybeNotifyTaskSubject(ctx context.Context, taskID, newStatus string, task *tasks.Task) { + if task == nil { + return + } + if !taskMetadataTruthy(task.Metadata["subject_participant"]) { + return + } + if task.Authority.SubjectType != string(models.PrincipalUser) || task.Authority.SubjectID == "" { + return + } + + subjectID := task.Authority.SubjectID + subjectTopic, err := models.UserProgressTopic(subjectID) + if err != nil { + logging.Logger.Warn().Err(err).Str("task_id", taskID).Str("subject_id", subjectID).Msg("invalid subject id for task subject notification; skipping") + return + } + + meta := map[string]string{ + "status": newStatus, + } + if v := metadataString(task.Metadata, "thread_id"); v != "" { + meta["thread_id"] = v + } + if v := metadataString(task.Metadata, "message_id"); v != "" { + meta["message_id"] = v + } + if v := metadataString(task.Metadata, "started_at_ms"); v != "" { + meta["started_at_ms"] = v + } + if v := metadataString(task.Metadata, "task_type"); v != "" { + meta["task_type"] = v + } else if task.TaskType != "" { + meta["task_type"] = task.TaskType + } + + update := &pb.ProgressUpdate{ + Source: s.gatewayID, + TaskId: taskID, + State: newStatus, + TimestampMs: time.Now().UnixMilli(), + Workspace: task.Workspace, + // Bare user recipient → delivered to all of the user's windows via the + // gateway-side prefix-match filter (isBareUserRecipientMatch). + Recipient: "us" + models.IdentitySep + subjectID, + Kind: pb.ProgressKind_PROGRESS_KIND_TASK, + Metadata: meta, + } + + updateBytes, err := proto.Marshal(update) + if err != nil { + logging.Logger.Warn().Err(err).Str("task_id", taskID).Msg("failed to marshal task subject notification") + return + } + + publishErr := s.publishBreaker.Execute(func() error { + return s.router.Publish(ctx, subjectTopic, updateBytes) + }) + if publishErr != nil { + logging.Logger.Warn().Err(publishErr).Str("task_id", taskID).Str("status", newStatus).Str("subject_id", subjectID).Msg("failed to publish task subject notification") + } else { + logging.Logger.Debug().Str("task_id", taskID).Str("status", newStatus).Str("subject_id", subjectID).Msg("task subject notification sent") + } +} + // notifyTaskStatusChangeFromTaskID looks up a task by ID, extracts the parent // agent and workspace, and publishes a status notification. Best-effort. func (s *GatewayServer) notifyTaskStatusChangeFromTaskID(ctx context.Context, taskID, newStatus, errorMsg string) { @@ -345,11 +479,14 @@ func (s *GatewayServer) notifyTaskStatusChangeFromTaskID(ctx context.Context, ta if err != nil || task == nil { return } - s.notifyTaskStatusChange(ctx, taskID, newStatus, task.Workspace, task.ParentAgentID, errorMsg) + s.notifyTaskStatusChange(ctx, taskID, newStatus, task.Workspace, task.ParentAgentID, errorMsg, task) } // subscribeClientToProgress subscribes a client to the pg::{workspace} progress -// stream using a shared consumer with a per-client filtering handler. +// stream with a per-client filtering handler. Uses a per-client named consumer +// with resume-or-tail semantics (consumer = identity string) so a (re)connect +// starts at the tail rather than re-dumping the entire retained progress history +// from seq 0; a reconnect replays only the gap since the committed offset. func (s *GatewayServer) subscribeClientToProgress(client *ClientSession, workspace string) error { pgTopic, err := models.ProgressTopic(workspace) if err != nil { @@ -359,7 +496,10 @@ func (s *GatewayServer) subscribeClientToProgress(client *ClientSession, workspa return nil } - cancel, err := s.router.Subscribe(pgTopic, s.createProgressFilterHandler(client)) + client.identityMu.RLock() + consumerName := client.Identity.String() + client.identityMu.RUnlock() + cancel, err := s.router.SubscribeExclusiveResumeOrTail(pgTopic, consumerName, s.createProgressFilterHandler(client)) if err != nil { return fmt.Errorf("failed to subscribe to progress topic %s: %w", pgTopic, err) } @@ -373,15 +513,19 @@ func (s *GatewayServer) subscribeClientToProgress(client *ClientSession, workspa } // subscribeClientToUserProgress subscribes a user client to a per-user -// progress topic (pg::us::{user}::{window}). The topic is self-scoped to a -// single user-window, so no additional server-side recipient filtering is -// required — the filter handler still applies self-echo suppression. +// progress topic (pg::us::{user_id}). The topic is self-scoped to a single +// user (all of that user's windows share it), so no additional server-side +// recipient filtering is required — the filter handler still applies +// self-echo suppression. func (s *GatewayServer) subscribeClientToUserProgress(client *ClientSession, topic string) error { if client.HasSubscription(topic) { return nil } - cancel, err := s.router.Subscribe(topic, s.createProgressFilterHandler(client)) + client.identityMu.RLock() + consumerName := client.Identity.String() + client.identityMu.RUnlock() + cancel, err := s.router.SubscribeExclusiveResumeOrTail(topic, consumerName, s.createProgressFilterHandler(client)) if err != nil { return fmt.Errorf("failed to subscribe to user-progress topic %s: %w", topic, err) } diff --git a/server/internal/gateway/progress_recipient_test.go b/server/internal/gateway/progress_recipient_test.go index a65485d..d9f09b4 100644 --- a/server/internal/gateway/progress_recipient_test.go +++ b/server/internal/gateway/progress_recipient_test.go @@ -92,6 +92,107 @@ func TestHandleProgressReport_RoutesToUserProgressTopic(t *testing.T) { } } +// TestHandleProgressReport_ServiceSenderAccepted verifies that a service +// principal (e.g. a sandbox sidecar bridge, identity sv::sandbox-sidecar::) +// is permitted to report progress. The bridge relays chat-lifecycle progress +// (task_done) on behalf of the agent harness running inside the sandbox, +// targeting a bare user recipient just like an agent would. Before this was +// allowed the report was dropped with ERR_INVALID_PRINCIPAL and the frontend +// never cleared its in-flight indicator. +func TestHandleProgressReport_ServiceSenderAccepted(t *testing.T) { + router := newMockMessageRouter() + s := newProgressTestServer(router) + + sender := models.Identity{ + Type: models.PrincipalService, + Implementation: "sandbox-sidecar", + Specifier: "sandbox-abc", + } + client := newProgressTestClient(sender) + + report := &pb.ProgressReport{ + TaskId: "task-svc-1", + State: "running", + Recipient: "us::dev@example.com", // bare user, multi-tab broadcast + Kind: pb.ProgressKind_PROGRESS_KIND_CHAT, + Step: &pb.ProgressStep{Name: "task_done"}, + Metadata: map[string]string{"thread_id": "chat-t1", "status": "completed"}, + } + + s.handleProgressReport(context.Background(), client, report) + + router.mu.Lock() + published := append([]publishedMsg(nil), router.publishedMessages...) + router.mu.Unlock() + + if len(published) != 1 { + t.Fatalf("expected 1 published message from service sender, got %d", len(published)) + } + wantTopic := "pg::us::dev@example.com" + if published[0].topic != wantTopic { + t.Errorf("publish topic = %q, want %q", published[0].topic, wantTopic) + } + + var update pb.ProgressUpdate + if err := proto.Unmarshal(published[0].payload, &update); err != nil { + t.Fatalf("unmarshal ProgressUpdate: %v", err) + } + if update.Step == nil || update.Step.Name != "task_done" { + t.Errorf("update.Step.Name = %v, want task_done", update.Step) + } +} + +// TestHandleProgressReport_UserSenderRejected verifies that principals outside +// the allowed set (agents, tasks, services) — here a user principal — are still +// rejected with ERR_INVALID_PRINCIPAL and the updated message wording, and that +// no progress update is published. This guards both the principal-type gate and +// the error message that was widened to include services. +func TestHandleProgressReport_UserSenderRejected(t *testing.T) { + router := newMockMessageRouter() + s := newProgressTestServer(router) + + stream := &mockStream{} + sender := models.Identity{ + Type: models.PrincipalUser, + ID: "dev@example.com", + } + client := newProgressTestClient(sender) + client.Stream = stream + + report := &pb.ProgressReport{ + TaskId: "task-user-1", + State: "running", + Recipient: "us::dev@example.com", + Kind: pb.ProgressKind_PROGRESS_KIND_CHAT, + } + + s.handleProgressReport(context.Background(), client, report) + + router.mu.Lock() + published := len(router.publishedMessages) + router.mu.Unlock() + if published != 0 { + t.Fatalf("expected 0 published messages for rejected user sender, got %d", published) + } + + if stream.sentCount() != 1 { + t.Fatalf("expected 1 error response, got %d", stream.sentCount()) + } + stream.mu.Lock() + errResp := stream.sent[0].GetError() + stream.mu.Unlock() + if errResp == nil { + t.Fatal("expected DownstreamMessage_Error payload") + } + if errResp.Code != "ERR_INVALID_PRINCIPAL" { + t.Errorf("error code = %q, want ERR_INVALID_PRINCIPAL", errResp.Code) + } + if errResp.Message != "only agents, tasks, and services can report progress" { + t.Errorf("error message = %q, want %q", errResp.Message, + "only agents, tasks, and services can report progress") + } +} + // TestHandleProgressReport_BareUserRecipientPublishesToUserTopic verifies that // a bare user-level recipient (us::{user}, no window specifier) routes to the // same pg::us::{user} topic as the window-specific form. The filter at delivery diff --git a/server/internal/gateway/registry_acl_audit_test.go b/server/internal/gateway/registry_acl_audit_test.go index 9cdca75..896b3b9 100644 --- a/server/internal/gateway/registry_acl_audit_test.go +++ b/server/internal/gateway/registry_acl_audit_test.go @@ -373,6 +373,9 @@ func (p *registryOnlyProvider) ListACLRules(_ context.Context, _ *admin.ACLRuleF func (p *registryOnlyProvider) GetACLRule(_ context.Context, _, _, _, _ string) (*admin.ACLRuleInfo, error) { return nil, fmt.Errorf("not implemented") } +func (p *registryOnlyProvider) GetACLRuleByID(_ context.Context, _ string) (*admin.ACLRuleInfo, error) { + return nil, fmt.Errorf("not implemented") +} func (p *registryOnlyProvider) GrantACLAccess(_ context.Context, _ *admin.GrantACLAccessRequest) (*admin.ACLRuleInfo, error) { return nil, fmt.Errorf("not implemented") } @@ -440,6 +443,58 @@ func (p *registryOnlyProvider) ListWorkspaceRateLimits() (map[string]float64, er return nil, fmt.Errorf("not implemented") } +func (p *registryOnlyProvider) ListACLGroups(_ context.Context) ([]*admin.ACLGroupInfo, error) { + return nil, fmt.Errorf("not implemented") +} +func (p *registryOnlyProvider) CreateACLGroup(_ context.Context, _ *admin.CreateACLGroupRequest) (*admin.ACLGroupInfo, error) { + return nil, fmt.Errorf("not implemented") +} +func (p *registryOnlyProvider) GetACLGroup(_ context.Context, _ string) (*admin.ACLGroupInfo, error) { + return nil, fmt.Errorf("not implemented") +} +func (p *registryOnlyProvider) DeleteACLGroup(_ context.Context, _ string) error { + return fmt.Errorf("not implemented") +} +func (p *registryOnlyProvider) ListACLGroupMembers(_ context.Context, _ string) ([]*admin.ACLGroupMemberInfo, error) { + return nil, fmt.Errorf("not implemented") +} +func (p *registryOnlyProvider) AddACLGroupMember(_ context.Context, _ string, _ *admin.AddACLGroupMemberRequest) (*admin.ACLGroupMemberInfo, error) { + return nil, fmt.Errorf("not implemented") +} +func (p *registryOnlyProvider) RemoveACLGroupMember(_ context.Context, _, _, _ string) error { + return fmt.Errorf("not implemented") +} +func (p *registryOnlyProvider) ListACLRoles(_ context.Context) ([]*admin.ACLRoleInfo, error) { + return nil, fmt.Errorf("not implemented") +} +func (p *registryOnlyProvider) CreateACLRole(_ context.Context, _ *admin.CreateACLRoleRequest) (*admin.ACLRoleInfo, error) { + return nil, fmt.Errorf("not implemented") +} +func (p *registryOnlyProvider) GetACLRole(_ context.Context, _ string) (*admin.ACLRoleInfo, error) { + return nil, fmt.Errorf("not implemented") +} +func (p *registryOnlyProvider) DeleteACLRole(_ context.Context, _ string) error { + return fmt.Errorf("not implemented") +} +func (p *registryOnlyProvider) ListACLRoleAssignments(_ context.Context, _ string) ([]*admin.ACLRoleAssignmentInfo, error) { + return nil, fmt.Errorf("not implemented") +} +func (p *registryOnlyProvider) AssignACLRole(_ context.Context, _ string, _ *admin.AssignACLRoleRequest) (*admin.ACLRoleAssignmentInfo, error) { + return nil, fmt.Errorf("not implemented") +} +func (p *registryOnlyProvider) UnassignACLRole(_ context.Context, _, _, _ string) error { + return fmt.Errorf("not implemented") +} +func (p *registryOnlyProvider) ListACLPrincipalGroups(_ context.Context, _, _ string) ([]*admin.ACLGroupMemberInfo, error) { + return nil, fmt.Errorf("not implemented") +} +func (p *registryOnlyProvider) ListACLPrincipalRoles(_ context.Context, _, _ string) ([]*admin.ACLRoleAssignmentInfo, error) { + return nil, fmt.Errorf("not implemented") +} +func (p *registryOnlyProvider) ExplainACLAccess(_ context.Context, _, _, _, _ string, _ int, _, _ string) (*admin.ACLAccessExplanationInfo, error) { + return nil, fmt.Errorf("not implemented") +} + // Compile-time check. var _ admin.StateProvider = (*registryOnlyProvider)(nil) diff --git a/server/internal/gateway/retry_policy.go b/server/internal/gateway/retry_policy.go new file mode 100644 index 0000000..fc34b34 --- /dev/null +++ b/server/internal/gateway/retry_policy.go @@ -0,0 +1,32 @@ +// Package gateway helpers for translating between the proto RetryPolicy +// representation and the storage-layer tasks.RetryPolicy. The two are +// intentionally separate types: pb.RetryPolicy is the wire shape and lives +// in api/proto, while tasks.RetryPolicy is the JSON-persistable form used +// inside the task store and consumed by FailTask / rescheduleFn. Keeping +// them decoupled lets non-server packages (aetherlite, webhookservice) +// import the storage form without pulling in the proto package. +package gateway + +import ( + pb "github.com/scitrera/aether/api/proto" + "github.com/scitrera/aether/pkg/tasks" +) + +// retryPolicyFromProto converts an incoming proto policy to the storage +// form. Returns nil when the caller didn't set a policy (preserving the +// legacy "no policy = immediate re-pend" behavior). +func retryPolicyFromProto(p *pb.RetryPolicy) *tasks.RetryPolicy { + if p == nil { + return nil + } + return &tasks.RetryPolicy{ + MaxAttempts: p.GetMaxAttempts(), + Backoff: tasks.BackoffStrategy(p.GetBackoff()), + InitialDelayMs: p.GetInitialDelayMs(), + MaxDelayMs: p.GetMaxDelayMs(), + JitterFactor: p.GetJitterFactor(), + ScheduleMs: append([]int64(nil), p.GetScheduleMs()...), + RetryableStatusCodes: append([]int32(nil), p.GetRetryableStatusCodes()...), + HonorRetryAfter: p.GetHonorRetryAfter(), + } +} diff --git a/server/internal/gateway/routing.go b/server/internal/gateway/routing.go index 131ec89..9881c96 100644 --- a/server/internal/gateway/routing.go +++ b/server/internal/gateway/routing.go @@ -33,8 +33,8 @@ const offlineCacheTTL = 5 * time.Second // validTopicPrefixSet is a map for O(1) topic prefix validation. var validTopicPrefixSet = map[string]bool{ "ag": true, "tu": true, "ta": true, "tb": true, - "us": true, "uw": true, "ga": true, "gu": true, - "pg": true, "event": true, "metric": true, "br": true, "sv": true, + "us": true, "uw": true, "uu": true, "ga": true, "gu": true, + "pg": true, "tk": true, "event": true, "metric": true, "br": true, "sv": true, } func validateTopicFormat(topic string) error { @@ -74,8 +74,9 @@ func workspaceFromTopic(topic string) string { return "" } return parts[1] - case "ag", "tu", "ta", "tb", "ga", "gu", "pg": + case "ag", "tu", "ta", "tb", "ga", "gu", "pg", "tk": // workspace is the next component: prefix::{workspace}[::rest] + // tk::{workspace}::{task_id}::{events,msg} → parts[1] = workspace. return parts[1] case "uw": // uw::{user_id}::{workspace} — workspace is the third component @@ -84,7 +85,9 @@ func workspaceFromTopic(topic string) string { } return parts[2] } - // us, br, sv — no workspace component + // us, uu, br, sv — no workspace component. uu::{user_id} is deliberately + // workspace-agnostic (see UserBroadcastTopic); its authorization is enforced + // by principal type in enforceTopicPermissions, not by workspace ACL. return "" } @@ -337,11 +340,13 @@ func (s *GatewayServer) routeMessage(ctx context.Context, client *ClientSession, Str("associated_task_id", associatedTaskID). Msg("message send authority resolution outcome") - // 0b. ACL Check - verify message send permission + // 0b. ACL Check - verify message send permission. The effective access + // level is consumed only on the proxy path (header minting); plain + // SendMessage routing ignores it here. if resolvedAuthority != nil { - err = s.checkMessageSendWithAuthority(ctx, sender, msg.TargetTopic, sessionUUID, resolvedAuthority) + _, err = s.checkMessageSendWithAuthority(ctx, sender, msg.TargetTopic, sessionUUID, resolvedAuthority) } else { - err = s.checkMessageSend(ctx, sender, msg.TargetTopic) + _, err = s.checkMessageSend(ctx, sender, msg.TargetTopic) } if err != nil { logging.Logger.Warn().Str("from", sender.ToTopic()).Str("to", msg.TargetTopic).Err(err).Msg("message denied by ACL") @@ -478,6 +483,23 @@ func (s *GatewayServer) routeMessage(ctx context.Context, client *ClientSession, // the envelope size for the common no-workspace bridge case. envelope.Metadata = map[string]string{"workspace": effectiveWorkspace} } + // Propagate the resolved on-behalf-of subject to the recipient, gateway-set + // and spoof-proof (like Source). This is the message-bus analog of the + // X-Auth-* identity headers ProxyHttp mints from the same resolved + // authority: it lets a recipient (e.g. platform-bridge) identify the user a + // message was sent for, distinct from the sending identity. Covers both the + // explicit SendMessage.authorization path and the auto task-authority + // fallback above. Subject identity only — acting-for uses the task + // authority-grant path, not this field. + if resolvedAuthority != nil && resolvedAuthority.Grant != nil { + g := resolvedAuthority.Grant + if g.SubjectType != "" && g.SubjectID != "" { + envelope.OnBehalfSubject = &pb.PrincipalRef{ + PrincipalType: g.SubjectType, + PrincipalId: g.SubjectID, + } + } + } envelopeBytes, err := proto.Marshal(envelope) if err != nil { @@ -528,13 +550,143 @@ func (s *GatewayServer) routeMessage(ctx context.Context, client *ClientSession, } } +// parseTaskMsgTopic reports whether topic is a task message-lane topic of the +// exact shape tk::{workspace}::{task_id}::msg and, if so, returns the workspace +// and task_id. The trailing segment must be EXACTLY "msg" — the task events +// lane (tk::{ws}::{task}::events) and any other shape return isTaskMsg=false so +// the caller falls through to the ordinary workspace-ACL path. This is a pure +// function (no store access) so the parse + shape rules are unit-testable. +func parseTaskMsgTopic(topic string) (workspace, taskID string, isTaskMsg bool) { + parts := strings.Split(topic, models.IdentitySep) + if len(parts) != 4 || parts[0] != "tk" || parts[3] != "msg" { + return "", "", false + } + if parts[1] == "" || parts[2] == "" { + return "", "", false + } + return parts[1], parts[2], true +} + +// taskPartyAuthorized mirrors authorizeTaskOp's party predicate (creator, +// assignee, OBO subject) over a sender Identity rather than a *ClientSession, +// plus the same workspace tenancy guard. It is pure (no store/ACL access) so it +// can be unit-tested directly. Workspace-less senders (services) skip the +// tenancy guard exactly as authorizeTaskOp treats the empty-workspace caller. +func taskPartyAuthorized(sender models.Identity, task *tasks.Task) bool { + if task == nil { + return false + } + callerTopic := sender.ToTopic() + // Creator-match (task.ParentAgentID backs the proto creator_actor_id) and + // assignee-match are EXACT identity-topic matches — the sender IS the task's + // creator or assigned worker — so they authorize regardless of workspace: a + // task may be legitimately targeted ACROSS the workspace boundary (a + // per-sandbox sahara agent in _sandbox serving a chat task in the user's app + // workspace). Checked BEFORE the tenancy guard, which otherwise blocks a + // cross-workspace assignee from its own task's msg lane. (Mirrors the same + // reorder in authorizeTaskOp.) + if task.ParentAgentID != "" && task.ParentAgentID == callerTopic { + return true + } + if task.AssignedTo != "" && task.AssignedTo == callerTopic { + return true + } + // Tenancy guard for the weaker OBO-subject path below: a workspace-scoped + // principal must not claim subject party-ship on a task in a different + // workspace. Workspace-less senders (services) skip it entirely. + if sender.Workspace != "" && task.Workspace != sender.Workspace { + return false + } + // OBO subject-match: the end user the task acts on behalf of (match by + // ID+type since a connected user topic carries a window specifier the + // stored subject id does not). + if sender.Type == models.PrincipalUser && + task.Authority.SubjectType == string(models.PrincipalUser) && + sender.ID != "" && + sender.ID == task.Authority.SubjectID { + return true + } + return false +} + +// taskMsgSenderIsTaskParty authorizes sends to a task's own message lane +// (tk::{workspace}::{task_id}::msg) when the sender is a party to that task — +// the SAME predicate as authorizeTaskOp (assignee, creator, or OBO subject). +// This is purely additive: it grants a task party (notably a workspace-less +// Service principal that is the task's AssignedTo) the ability to send to the +// task's msg lane, which the plain workspace-ACL check denies via the +// read-only service fallback. It never newly denies — non-parties fall through +// to the existing workspace-ACL logic unchanged. +// +// Returns (authorized, isTaskMsg). isTaskMsg is true whenever the topic has the +// task-msg shape, even when not authorizable here (no store, missing task, +// cross-workspace), so the caller can decide to fall through. The grant fires +// only when both are true. Fail-closed: a missing taskStore, a load error, or +// an absent task yields (false, true) — the caller falls through to the +// existing ACL path rather than granting blindly. +func (s *GatewayServer) taskMsgSenderIsTaskParty(ctx context.Context, sender models.Identity, targetTopic string) (authorized bool, isTaskMsg bool) { + workspace, taskID, ok := parseTaskMsgTopic(targetTopic) + if !ok { + return false, false + } + _ = workspace // workspace is re-derived from the loaded task for the tenancy guard + if s.taskStore == nil { + return false, true + } + task, err := s.taskStore.GetTask(ctx, taskID) + if err != nil || task == nil { + return false, true + } + if taskPartyAuthorized(sender, task) { + return true, true + } + return false, true +} + // checkMessageSend checks whether a sender is allowed to publish to a target topic via the ACL // service. It preserves the workspace-derivation logic: bridges/services with no home workspace // check ACL against the target workspace; workspace-scoped senders check against the target // workspace when it differs from their own (cross-workspace), or their own workspace otherwise. -func (s *GatewayServer) checkMessageSend(ctx context.Context, sender models.Identity, targetTopic string) error { +// checkMessageSend returns the effective access level granted by the ACL +// decision alongside the error. The level is the EffectiveAccessLevel the +// gateway mints into X-Auth-Workspace-Access; callers that only care about +// the allow/deny outcome may ignore it. When ACL is disabled or no workspace +// check applies, the level is acl.AccessNone (0). +func (s *GatewayServer) checkMessageSend(ctx context.Context, sender models.Identity, targetTopic string) (int, error) { if s.acl == nil { - return nil // ACL not enabled + return acl.AccessNone, nil // ACL not enabled + } + + // Additive task-party grant: a task's own party (assignee/creator/OBO + // subject) may send to the task's msg lane even without a workspace send + // grant. Non-parties / non-task-msg topics fall through unchanged. + if ok, isTaskMsg := s.taskMsgSenderIsTaskParty(ctx, sender, targetTopic); isTaskMsg && ok { + logging.Logger.Info(). + Str("from", sender.ToTopic()). + Str("target", targetTopic). + Msg("task-msg send authorized via task-party match") + return acl.AccessReadWrite, nil + } + + // Additive metric-publish grant: an agent principal may publish to the + // metric fan-in plane (post-rewrite metric::receiver{N}) without a + // per-workspace WRITE grant. This realizes the "same-workspace publish is + // implicit" contract (see routeMessage) for least-privilege agents — e.g. + // sahara sandbox agents, deliberately READ-only on _sandbox — that emit + // their own usage/token metrics. Safe because the other metric authz layers + // have already run or run independently: cross-workspace publishes were + // gated up-front by checkCrossWorkspaceBroadcast (capability/metric_broadcast), + // the topic permission matrix gated type eligibility, and negative-delta + // metrics are gated afterward by checkMetricCredit. Scoped to agent + // principals and the metric plane only (NOT event::, which drives the + // workflow engine) so it doesn't broaden other senders. OBO-carried metric + // sends (checkMessageSendWithAuthority) are not a current path. + if sender.Type == models.PrincipalAgent && strings.HasPrefix(targetTopic, "metric"+models.IdentitySep) { + logging.Logger.Info(). + Str("from", sender.ToTopic()). + Str("target", targetTopic). + Msg("metric publish authorized via agent additive grant") + return acl.AccessReadWrite, nil } // System principals with no workspace (bridges/services) check ACL against the target workspace. @@ -543,7 +695,7 @@ func (s *GatewayServer) checkMessageSend(ctx context.Context, sender models.Iden if targetWorkspace != "" { decision, err := s.acl.CanSendMessage(ctx, sender, acl.ResourceTypeWorkspace, targetWorkspace, targetWorkspace, uuid.Nil) if err != nil { - return fmt.Errorf("ACL check failed: %w", err) + return acl.AccessNone, fmt.Errorf("ACL check failed: %w", err) } if decision.Denied() { logging.Logger.Info(). @@ -552,10 +704,11 @@ func (s *GatewayServer) checkMessageSend(ctx context.Context, sender models.Iden Str("workspace_checked", targetWorkspace). Str("reason", decision.Reason). Msg("checkMessageSend denial") - return fmt.Errorf("access denied: %s", decision.Reason) + return acl.AccessNone, fmt.Errorf("access denied: %s", decision.Reason) } + return decision.EffectiveAccessLevel, nil } - return nil + return acl.AccessNone, nil } // Workspace-scoped senders: gate on TARGET workspace when it differs @@ -568,7 +721,7 @@ func (s *GatewayServer) checkMessageSend(ctx context.Context, sender models.Iden decision, err := s.acl.CanSendMessage(ctx, sender, acl.ResourceTypeWorkspace, checkWorkspace, checkWorkspace, uuid.Nil) if err != nil { - return fmt.Errorf("ACL check failed: %w", err) + return acl.AccessNone, fmt.Errorf("ACL check failed: %w", err) } if decision.Denied() { logging.Logger.Info(). @@ -577,9 +730,9 @@ func (s *GatewayServer) checkMessageSend(ctx context.Context, sender models.Iden Str("workspace_checked", checkWorkspace). Str("reason", decision.Reason). Msg("checkMessageSend denial") - return fmt.Errorf("access denied: %s", decision.Reason) + return acl.AccessNone, fmt.Errorf("access denied: %s", decision.Reason) } - return nil + return decision.EffectiveAccessLevel, nil } // checkCrossWorkspaceBroadcast enforces the cross-workspace capability gate @@ -631,6 +784,22 @@ var hasCrossWorkspaceBroadcastPermission = func(ctx context.Context, s *GatewayS // enforceTopicPermissions checks spec Section 3.2.2 permission matrix. // Returns an error if the sender's principal type is not allowed to publish to the target topic. func enforceTopicPermissions(sender models.Identity, targetTopic string) error { + // User-broadcast (uu::{user_id}) is a platform→user channel: it reaches + // every one of a user's windows regardless of active workspace and carries + // no workspace segment, so the downstream workspace ACL cannot gate it + // (workspaceFromTopic returns ""). The principal-type policy here is + // therefore the sole authorization check. Only platform principals may + // publish; workspace-scoped principals (users, agents, tasks, orchestrators) + // must reach a user via us::/uw::/pg:: or task ownership instead. + if strings.HasPrefix(targetTopic, "uu"+models.IdentitySep) { + switch sender.Type { + case models.PrincipalService, models.PrincipalWorkflowEngine, models.PrincipalBridge: + return nil + default: + return fmt.Errorf("only service, workflow-engine, and bridge principals may publish to user-broadcast (uu::) topics") + } + } + switch sender.Type { case models.PrincipalUser: // Users cannot send to event.*, metric.*, or pg.* topics @@ -956,6 +1125,15 @@ func (s *GatewayServer) handleKVOp(ctx context.Context, client *ClientSession, o resolvedAuthority, err := s.resolveAuthorizationContext(ctx, client, ident, op.GetAuthorization()) if err != nil { sendClientError(client, "ERR_PERMISSION_DENIED", "invalid authorization context", withRequestID(op.GetRequestId())) + // Resolve the matching pending sync KV request so the client doesn't + // time out waiting for a KVResponse alongside this ErrorResponse. + if reqID := op.GetRequestId(); reqID != "" { + sendResponse(&pb.DownstreamMessage{ + Payload: &pb.DownstreamMessage_Kv{ + Kv: &pb.KVResponse{Success: false, RequestId: reqID}, + }, + }) + } return } @@ -1000,7 +1178,10 @@ func (s *GatewayServer) handleKVOp(ctx context.Context, client *ClientSession, o err = handler.HandleKVOperation(ctx, ident, sessionUUID, resolvedAuthority, op, sendResponse) if err != nil { logging.Logger.Error().Err(err).Msg("KV operation failed") - // Send generic error to avoid leaking internal details (e.g. Redis internals) to clients. + // Send generic error for OnError consumers; the typed KVResponse + // (Success=false, RequestId=…) that resolves any pending sync + // waiter is already emitted by HandleKVOperation before it + // returns the error. sendClientError(client, "KV_ERROR", "internal error processing KV operation", withRequestID(op.GetRequestId())) } else { workspace := op.Workspace @@ -1352,6 +1533,26 @@ func protoTaskStatusToTasks(s pb.TaskStatus) tasks.TaskStatus { } } +// completionConfigFromProto converts the proto TaskCompletionEvent into the +// persisted model config. nil ⇒ nil (task did not opt into feed B). OnStatuses +// are mapped through the canonical proto↔model status converter. +func completionConfigFromProto(p *pb.TaskCompletionEvent) *tasks.TaskCompletionConfig { + if p == nil { + return nil + } + cfg := &tasks.TaskCompletionConfig{ + Enabled: p.GetEnabled(), + EventName: p.GetEventName(), + } + if len(p.GetOnStatuses()) > 0 { + cfg.OnStatuses = make([]tasks.TaskStatus, 0, len(p.GetOnStatuses())) + for _, s := range p.GetOnStatuses() { + cfg.OnStatuses = append(cfg.OnStatuses, protoTaskStatusToTasks(s)) + } + } + return cfg +} + // unixOrZero returns the unix-seconds timestamp of t, or 0 when t is nil. // Used in proto conversions where 0 sentinels "absent" for time fields. func unixOrZero(t *time.Time) int64 { @@ -1367,6 +1568,7 @@ func taskToProto(t *tasks.Task) *pb.TaskInfo { TaskId: t.TaskID, TaskType: t.TaskType, TaskClass: pb.TaskClass(t.TaskClass), + Priority: pb.TaskPriority(t.Priority), DisconnectedAt: unixOrZero(t.DisconnectedAt), GraceWindowMs: t.GraceWindowMs, Status: taskStatusToProto(t.Status), @@ -1438,6 +1640,21 @@ func taskToProto(t *tasks.Task) *pb.TaskInfo { } info.ContextId = t.ContextID info.PausedAt = unixOrZero(t.PausedAt) + info.CorrelationId = t.CorrelationID + info.RootTaskId = t.RootTaskID + if t.CompletionEvent != nil { + ce := &pb.TaskCompletionEvent{ + Enabled: t.CompletionEvent.Enabled, + EventName: t.CompletionEvent.EventName, + } + if len(t.CompletionEvent.OnStatuses) > 0 { + ce.OnStatuses = make([]pb.TaskStatus, 0, len(t.CompletionEvent.OnStatuses)) + for _, s := range t.CompletionEvent.OnStatuses { + ce.OnStatuses = append(ce.OnStatuses, taskStatusToProto(s)) + } + } + info.CompletionEvent = ce + } return info } @@ -1474,24 +1691,60 @@ func (s *GatewayServer) authorizeTaskOp(ctx context.Context, client *ClientSessi sessionUUID := client.SessionUUID client.identityMu.RUnlock() - // Workspace mismatch is an immediate deny — even a creator/assignee - // match across workspaces should not be honored, since workspace is - // the strong tenancy boundary. - if callerWorkspace != "" && task.Workspace != callerWorkspace { - return false - } - - // Creator-match (cheapest): task.ParentAgentID is the storage column - // that backs the proto creator_actor_id field. + // Creator-match (cheapest): task.ParentAgentID is the storage column that + // backs the proto creator_actor_id field. This is an EXACT identity-topic + // match (the sender IS the creator), so it authorizes regardless of + // workspace: a task may be legitimately created/targeted ACROSS the + // workspace boundary — e.g. a per-sandbox sahara agent in `_sandbox` serving + // a chat task in the user's app workspace. Checked before the tenancy guard. if task.ParentAgentID != "" && task.ParentAgentID == callerTopic { return true } - // Assignee-match: the worker assigned to the task may always act on it. + // Assignee-match: the worker assigned to the task may always act on it, + // regardless of workspace (same exact-identity, creator-authorized + // cross-workspace-assignment rationale as creator-match). if task.AssignedTo != "" && task.AssignedTo == callerTopic { return true } + // Workspace mismatch is a deny for the WEAKER party paths below (OBO + // subject-match, workspace-admin): a workspace-scoped principal must not + // claim subject/admin party-ship on a task in a different workspace. The + // exact creator/assignee identity matches above are exempt (the sender IS + // that party); workspace-less senders (services) skip the guard entirely. + if callerWorkspace != "" && task.Workspace != callerWorkspace { + logging.Logger.Info(). + Str("identity", callerIdentity.String()). + Str("task_id", task.TaskID). + Str("reason", "workspace mismatch"). + Msg("task op authorization denied") + s.auditLog(ctx, audit.NewTaskEvent( + string(callerIdentity.Type), + callerIdentity.String(), + audit.OpTaskAuthzDenied, + task.TaskID, + task.Workspace, + sessionUUID, + false, + "workspace mismatch", + nil, + )) + return false + } + + // OBO subject-match: the end user the task acts on behalf of may operate + // on its own task (e.g. CLAIM a per-turn chat_message task from any of + // their browser tabs). Match by identity ID+type rather than topic string, + // since a connected user topic carries a window specifier the stored + // subject id does not. + if callerIdentity.Type == models.PrincipalUser && + task.Authority.SubjectType == string(models.PrincipalUser) && + callerIdentity.ID != "" && + callerIdentity.ID == task.Authority.SubjectID { + return true + } + // Workspace-admin: requires the ACL store. If no ACL is configured, // fail open on workspace match (the current Phase 1-3 behavior). if s.acl == nil { @@ -1501,8 +1754,42 @@ func (s *GatewayServer) authorizeTaskOp(ctx context.Context, client *ClientSessi } decision, err := s.acl.CanManageWorkspace(ctx, callerIdentity, task.Workspace, sessionUUID) if err != nil || decision == nil { + logging.Logger.Info(). + Str("identity", callerIdentity.String()). + Str("task_id", task.TaskID). + Str("reason", "not creator/assignee/subject and workspace-admin check failed"). + Msg("task op authorization denied") + s.auditLog(ctx, audit.NewTaskEvent( + string(callerIdentity.Type), + callerIdentity.String(), + audit.OpTaskAuthzDenied, + task.TaskID, + task.Workspace, + sessionUUID, + false, + "not creator/assignee/subject and workspace-admin check failed", + nil, + )) return false } + if !decision.Allowed { + logging.Logger.Info(). + Str("identity", callerIdentity.String()). + Str("task_id", task.TaskID). + Str("reason", decision.Reason). + Msg("task op authorization denied") + s.auditLog(ctx, audit.NewTaskEvent( + string(callerIdentity.Type), + callerIdentity.String(), + audit.OpTaskAuthzDenied, + task.TaskID, + task.Workspace, + sessionUUID, + false, + decision.Reason, + nil, + )) + } return decision.Allowed } @@ -1581,6 +1868,8 @@ func (s *GatewayServer) handleTaskQuery(ctx context.Context, client *ClientSessi filter.ParentTaskID = query.Filter.ParentTaskId // Phase 1: A2A filter fields. filter.ContextID = query.Filter.ContextId + filter.CorrelationID = query.Filter.GetCorrelationId() + filter.RootTaskID = query.Filter.GetRootTaskId() if len(query.Filter.ExcludeStatuses) > 0 { filter.ExcludeStatuses = make([]tasks.TaskStatus, 0, len(query.Filter.ExcludeStatuses)) for _, s := range query.Filter.ExcludeStatuses { @@ -1595,6 +1884,8 @@ func (s *GatewayServer) handleTaskQuery(ctx context.Context, client *ClientSessi filter.StatusTimestampAfterUnixMs = query.Filter.StatusTimestampAfterUnixMs filter.PageToken = query.Filter.PageToken filter.IncludeDescendants = query.Filter.IncludeDescendants + filter.Priority = int32(query.Filter.Priority) + filter.MinPriority = int32(query.Filter.MinPriority) if query.Filter.Limit > 0 { filter.Limit = int(query.Filter.Limit) if filter.Limit > 1000 { @@ -1679,6 +1970,14 @@ func (s *GatewayServer) handleTaskOp(ctx context.Context, client *ClientSession, var response *pb.TaskOperationResponse + // Caller identity for task-op audit logging. The COMPLETE/FAIL branches log + // success + every failure reason so a silently-swallowed rejection — e.g. a + // sahara CompleteTask the SDK returns as Success=false without an error, the + // cause of chat tasks never leaving RUNNING — is diagnosable gateway-side. + client.identityMu.RLock() + callerTopic := client.Identity.ToTopic() + client.identityMu.RUnlock() + switch op.Op { case pb.TaskOperation_CANCEL: // Authorization: creator / assignee / workspace-admin (info-hiding). @@ -1758,6 +2057,7 @@ func (s *GatewayServer) handleTaskOp(ctx context.Context, client *ClientSession, // Authorization: creator / assignee / workspace-admin (info-hiding). task, err := s.taskStore.GetTask(ctx, op.TaskId) if err != nil || task == nil { + logging.Logger.Warn().Str("from", callerTopic).Str("task_id", op.TaskId).Err(err).Msg("task COMPLETE rejected: task not found") response = &pb.TaskOperationResponse{ Success: false, Error: ErrTaskNotFoundOrUnauthorized, @@ -1765,6 +2065,7 @@ func (s *GatewayServer) handleTaskOp(ctx context.Context, client *ClientSession, break } if !s.authorizeTaskOp(ctx, client, task) { + logging.Logger.Warn().Str("from", callerTopic).Str("task_id", op.TaskId).Str("assigned_to", task.AssignedTo).Msg("task COMPLETE rejected: unauthorized") response = &pb.TaskOperationResponse{ Success: false, Error: ErrTaskNotFoundOrUnauthorized, @@ -1776,6 +2077,7 @@ func (s *GatewayServer) handleTaskOp(ctx context.Context, client *ClientSession, completeFn = s.orchestration.TaskService.CompleteTask } if err := completeFn(ctx, op.TaskId); err != nil { + logging.Logger.Warn().Str("from", callerTopic).Str("task_id", op.TaskId).Str("prior_status", string(task.Status)).Err(err).Msg("task COMPLETE rejected: completeFn error") response = &pb.TaskOperationResponse{ Success: false, Error: err.Error(), @@ -1783,6 +2085,7 @@ func (s *GatewayServer) handleTaskOp(ctx context.Context, client *ClientSession, } else { // Re-fetch to get updated state updated, _ := s.taskStore.GetTask(ctx, op.TaskId) + logging.Logger.Info().Str("from", callerTopic).Str("task_id", op.TaskId).Msg("task COMPLETE succeeded") response = &pb.TaskOperationResponse{ Success: true, Message: "task completed", @@ -1801,6 +2104,7 @@ func (s *GatewayServer) handleTaskOp(ctx context.Context, client *ClientSession, // Authorization: creator / assignee / workspace-admin (info-hiding). task, err := s.taskStore.GetTask(ctx, op.TaskId) if err != nil || task == nil { + logging.Logger.Warn().Str("from", callerTopic).Str("task_id", op.TaskId).Err(err).Msg("task FAIL rejected: task not found") response = &pb.TaskOperationResponse{ Success: false, Error: ErrTaskNotFoundOrUnauthorized, @@ -1808,6 +2112,7 @@ func (s *GatewayServer) handleTaskOp(ctx context.Context, client *ClientSession, break } if !s.authorizeTaskOp(ctx, client, task) { + logging.Logger.Warn().Str("from", callerTopic).Str("task_id", op.TaskId).Str("assigned_to", task.AssignedTo).Msg("task FAIL rejected: unauthorized") response = &pb.TaskOperationResponse{ Success: false, Error: ErrTaskNotFoundOrUnauthorized, @@ -1819,12 +2124,14 @@ func (s *GatewayServer) handleTaskOp(ctx context.Context, client *ClientSession, failFn = s.orchestration.TaskService.FailTask } if err := failFn(ctx, op.TaskId, errMsg); err != nil { + logging.Logger.Warn().Str("from", callerTopic).Str("task_id", op.TaskId).Str("prior_status", string(task.Status)).Err(err).Msg("task FAIL rejected: failFn error") response = &pb.TaskOperationResponse{ Success: false, Error: err.Error(), } } else { updated, _ := s.taskStore.GetTask(ctx, op.TaskId) + logging.Logger.Info().Str("from", callerTopic).Str("task_id", op.TaskId).Str("reason", errMsg).Msg("task FAIL succeeded") response = &pb.TaskOperationResponse{ Success: true, Message: "task marked as failed", @@ -2084,6 +2391,46 @@ func (s *GatewayServer) handleTaskOp(ctx context.Context, client *ClientSession, s.notifyTaskStatusChangeFromTaskID(ctx, op.TaskId, string(tasks.TaskStatusRejected), reason) } + case pb.TaskOperation_CLAIM: + // CLAIM: the assignee (or OBO subject) transitions a pending/assigned + // task to RUNNING. Mirrors CANCEL structurally: GetTask (info-hiding on + // miss) → authorize → claim store fn → notify on success. + claimTask, err := s.taskStore.GetTask(ctx, op.TaskId) + if err != nil || claimTask == nil { + response = &pb.TaskOperationResponse{ + Success: false, + Error: ErrTaskNotFoundOrUnauthorized, + } + break + } + if !s.authorizeTaskOp(ctx, client, claimTask) { + response = &pb.TaskOperationResponse{ + Success: false, + Error: ErrTaskNotFoundOrUnauthorized, + } + break + } + claimFn := s.taskStore.ClaimTask + if s.orchestration != nil && s.orchestration.TaskService != nil { + claimFn = s.orchestration.TaskService.ClaimTask + } + if err := claimFn(ctx, op.TaskId); err != nil { + response = &pb.TaskOperationResponse{ + Success: false, + Error: err.Error(), + } + } else { + updated, _ := s.taskStore.GetTask(ctx, op.TaskId) + response = &pb.TaskOperationResponse{ + Success: true, + Message: "task claimed", + } + if updated != nil { + response.Task = taskToProto(updated) + } + s.notifyTaskStatusChangeFromTaskID(ctx, op.TaskId, "running", "") + } + default: response = &pb.TaskOperationResponse{ Success: false, diff --git a/server/internal/gateway/routing_hotpath_test.go b/server/internal/gateway/routing_hotpath_test.go index 72dbfb5..74fb320 100644 --- a/server/internal/gateway/routing_hotpath_test.go +++ b/server/internal/gateway/routing_hotpath_test.go @@ -419,6 +419,10 @@ func (r *errorPublishRouter) SubscribeExclusiveFromTimestamp(topic string, consu return r.SubscribeExclusive(topic, consumer, h) } +func (r *errorPublishRouter) SubscribeExclusiveResumeOrTail(topic string, consumer string, h func([]byte)) (func(), error) { + return r.SubscribeExclusive(topic, consumer, h) +} + func TestRouteMessage_PublishFails_SendsPublishFailedErrorToClient(t *testing.T) { errRouter := &errorPublishRouter{ err: errors.New("broker unavailable"), diff --git a/server/internal/gateway/routing_proxy.go b/server/internal/gateway/routing_proxy.go index be7e1db..4f806d8 100644 --- a/server/internal/gateway/routing_proxy.go +++ b/server/internal/gateway/routing_proxy.go @@ -13,6 +13,7 @@ import ( "github.com/scitrera/aether/internal/acl" "github.com/scitrera/aether/internal/audit" "github.com/scitrera/aether/internal/logging" + "github.com/scitrera/aether/pkg/identityheaders" "github.com/scitrera/aether/pkg/models" "github.com/scitrera/aether/sdk/go/aether" bp "github.com/scitrera/go-backpressure" @@ -162,16 +163,21 @@ func (s *GatewayServer) findLocalServiceInstances(impl string) []string { } // proxyACLCheck reuses checkMessageSendWithAuthority / checkMessageSend, -// matching the same actor-direct-then-OBO order plain messages use. -func (s *GatewayServer) proxyACLCheck(ctx context.Context, client *ClientSession, sender models.Identity, target string, authz *pb.AuthorizationContext) (*acl.ResolvedAuthority, error) { +// matching the same actor-direct-then-OBO order plain messages use. It also +// returns the effective access level granted by the winning decision so the +// proxy path can mint it into X-Auth-Workspace-Access (acl.AccessNone when ACL +// is disabled or no workspace check applies). +func (s *GatewayServer) proxyACLCheck(ctx context.Context, client *ClientSession, sender models.Identity, target string, authz *pb.AuthorizationContext) (*acl.ResolvedAuthority, int, error) { resolved, err := s.resolveAuthorizationContext(ctx, client, sender, authz) if err != nil { - return nil, err + return nil, acl.AccessNone, err } if resolved != nil { - return resolved, s.checkMessageSendWithAuthority(ctx, sender, target, client.SessionUUID, resolved) + level, checkErr := s.checkMessageSendWithAuthority(ctx, sender, target, client.SessionUUID, resolved) + return resolved, level, checkErr } - return nil, s.checkMessageSend(ctx, sender, target) + level, checkErr := s.checkMessageSend(ctx, sender, target) + return nil, level, checkErr } // proxyFrameSourceMarker is the “MessageEnvelope.Source“ value used to @@ -372,8 +378,10 @@ func (s *GatewayServer) routeProxyHttpRequest(ctx context.Context, client *Clien } req.TargetTopic = concrete - // 2. ACL check — same primitives as plain SendMessage. - resolvedAuthority, err := s.proxyACLCheck(ctx, client, sender, concrete, req.GetAuthorization()) + // 2. ACL check — same primitives as plain SendMessage. The effective send- + // permission level is not used for header minting (WorkspaceAccess is + // computed separately below, per OBO/direct mode). + resolvedAuthority, _, err := s.proxyACLCheck(ctx, client, sender, concrete, req.GetAuthorization()) if err != nil { s.auditProxyHttpFailure(ctx, sender, concrete, requestID, client.SessionUUID, resolvedAuthority, err.Error()) sendProxyHttpError(client, requestID, pb.ProxyError_ACL_DENIED, err.Error()) @@ -389,18 +397,78 @@ func (s *GatewayServer) routeProxyHttpRequest(ctx context.Context, client *Clien // it tried to re-resolve. By inlining the resolution here we // eliminate that mismatch and a redundant RPC round-trip per // request. See AuthorizationContext.resolved doc in api/proto/aether.proto. + // + // This remains for backward-compat with strict consumers (e.g. the + // Go sidecar) that overlay X-Auth-* from the proto. The header + // minting in step 3 is ADDITIVE and is the canonical path for + // passthrough terminators (e.g. MemoryLayer's in-process terminator). if resolvedAuthority != nil && req.Authorization != nil && resolvedAuthority.Grant != nil { req.Authorization.Resolved = grantToResolvedAuthorityInfo(resolvedAuthority.Grant) } - // 3. Stamp the originating caller into the envelope so the receiving - // sidecar can populate X-Auth-* headers without a separate lookup. - // (T3 introduces the helper; here we just propagate the actor topic.) + // 3. Mint the canonical X-Auth-* trusted header set onto the envelope so + // a passthrough terminator (no Go sidecar to re-mint) can trust it + // directly. This is the single minting point — identityheaders is the + // same package the auth-proxy and Go sidecar use, so the wire format is + // identical everywhere. if req.Headers == nil { req.Headers = make(map[string]string) } + // Anti-spoof: drop any client-supplied x-auth-* / x-aether-* before we + // stamp our trusted set, so a caller cannot smuggle a forged identity. + identityheaders.StripInboundMap(req.Headers) + // Keep stamping the originating caller topic for strict Go-sidecar + // consumers that read it; set AFTER the strip so it survives. req.Headers["x-aether-actor-topic"] = sender.ToTopic() + // gatewayTenantID identifies the tenant minted into X-Auth-Tenant-ID. The + // gateway is per-tenant; when AETHER_TENANT_ID is configured (s.tenantID) + // we use it, otherwise we fall back to the sender's workspace as the + // tenant scope so the fail-closed terminator authz still receives a + // non-empty tenant. + gatewayTenantID := s.tenantID + if gatewayTenantID == "" { + gatewayTenantID = sender.Workspace + } + ident := identityheaders.Identity{ + UserID: sender.CanonicalPrincipalID(), + PrincipalType: string(sender.Type), + CallerTopic: sender.ToTopic(), + } + if resolvedAuthority != nil && resolvedAuthority.Grant != nil { + // OBO mode: the grant's MaxAccessLevel is the subject's access level on + // this workspace — use it verbatim (the grant was already validated by + // proxyACLCheck; no second ACL round-trip needed). + ident.WorkspaceAccess = resolvedAuthority.Grant.MaxAccessLevel + ident.Authority = identityheaders.AuthorityFromResolved(resolvedAuthority) + } else { + // Direct mode: effectiveLevel from checkMessageSend is the send- + // permission check, not the principal's workspace resource access level. + // For workspace-less targets (sv::, br::) that path returns AccessNone(0) + // even when the principal holds a blanket workspace:* ADMIN(40) grant. + // Re-query the ACL for the principal's effective workspace access level + // so X-Auth-Workspace-Access carries the correct value. + // + // Use req.AppWorkspace when the caller has declared a workspace context; + // fall back to "*" so a wildcard/blanket grant (e.g. workspace:* ADMIN) + // is matched and its level is returned. + wsCheck := req.GetAppWorkspace() + if wsCheck == "" { + wsCheck = "*" + } + if s.acl != nil { + // requiredLevel=AccessNone so this never denies — we only want the + // EffectiveAccessLevel from the winning rule. + if d, aclErr := s.acl.CheckAccess(ctx, sender, acl.ResourceTypeWorkspace, wsCheck, "read", wsCheck, uuid.Nil, acl.AccessNone); aclErr == nil && d != nil { + ident.WorkspaceAccess = d.EffectiveAccessLevel + } + // On error or nil decision, WorkspaceAccess stays 0 (AccessNone). + // The request still proceeds — send permission was already granted + // above; this is best-effort level enrichment for the header. + } + } + identityheaders.MintIntoMap(req.Headers, gatewayTenantID, ident) + // 4. Install request-pin so follow-on body chunks and the response can // route to the correct counterparty without re-resolving wildcards. // Pin lifetime ≥ timeout_ms + slack so chunks arriving late still find @@ -813,8 +881,9 @@ func (s *GatewayServer) routeTunnelOpen(ctx context.Context, client *ClientSessi } open.TargetTopic = concrete - // 2. ACL. - resolvedAuthority, err := s.proxyACLCheck(ctx, client, sender, concrete, open.GetAuthorization()) + // 2. ACL. Tunnels do not mint X-Auth-* headers, so the effective level is + // discarded here. + resolvedAuthority, _, err := s.proxyACLCheck(ctx, client, sender, concrete, open.GetAuthorization()) if err != nil { s.auditTunnelOpenFailure(ctx, sender, concrete, tunnelID, client.SessionUUID, resolvedAuthority, err.Error()) sendTunnelClose(client, tunnelID, pb.TunnelClose_ERROR, "ACL_DENIED: "+err.Error()) diff --git a/server/internal/gateway/routing_task_authority_test.go b/server/internal/gateway/routing_task_authority_test.go index c9c92ef..81f3ff6 100644 --- a/server/internal/gateway/routing_task_authority_test.go +++ b/server/internal/gateway/routing_task_authority_test.go @@ -17,6 +17,7 @@ import ( taskpg "github.com/scitrera/aether/internal/storage/tasks/postgres" "github.com/scitrera/aether/internal/testutil" "github.com/scitrera/aether/pkg/models" + "google.golang.org/protobuf/proto" ) // TestRouteMessage_AutoResolvesSessionTaskAuthority verifies that when an agent @@ -144,6 +145,10 @@ func TestRouteMessage_AutoResolvesSessionTaskAuthority(t *testing.T) { // Delegation would deny and nothing would be published. router.mu.Lock() published := len(router.publishedMessages) + var publishedPayload []byte + if published > 0 { + publishedPayload = router.publishedMessages[0].payload + } router.mu.Unlock() stream.mu.Lock() @@ -161,6 +166,23 @@ func TestRouteMessage_AutoResolvesSessionTaskAuthority(t *testing.T) { if published != 1 { t.Errorf("expected 1 published message (authority path), got %d", published) } + + // The gateway must stamp the resolved OBO subject onto the delivered + // envelope so a recipient can identify the user the message was sent for. + if publishedPayload != nil { + var env pb.MessageEnvelope + if err := proto.Unmarshal(publishedPayload, &env); err != nil { + t.Fatalf("unmarshal published envelope: %v", err) + } + obo := env.GetOnBehalfSubject() + if obo == nil { + t.Fatal("expected MessageEnvelope.on_behalf_subject to be stamped") + } + if obo.GetPrincipalType() != "user" || obo.GetPrincipalId() != userID { + t.Errorf("on_behalf_subject = (%q,%q), want (user,%q)", + obo.GetPrincipalType(), obo.GetPrincipalId(), userID) + } + } } // TestRouteMessage_NoTaskID_UsesDirectDelegation verifies that when the agent diff --git a/server/internal/gateway/routing_test.go b/server/internal/gateway/routing_test.go index 98c57cb..c737a37 100644 --- a/server/internal/gateway/routing_test.go +++ b/server/internal/gateway/routing_test.go @@ -53,6 +53,10 @@ func (m *mockRouter) SubscribeExclusiveFromTimestamp(topic string, consumerName return m.SubscribeExclusive(topic, consumerName, nil) } +func (m *mockRouter) SubscribeExclusiveResumeOrTail(topic string, consumerName string, _ func([]byte)) (func(), error) { + return m.SubscribeExclusive(topic, consumerName, nil) +} + // hasSharedTopic returns true if the topic was subscribed to via the shared Subscribe path. func (m *mockRouter) hasSharedTopic(topic string) bool { m.mu.Lock() @@ -417,6 +421,54 @@ func TestEnforceTopicPermissions(t *testing.T) { targetTopic: "ag::ws2::impl::spec", wantErr: false, }, + + // ----- User-broadcast (uu::): platform principals only ----- + { + name: "service can send to user-broadcast topic", + sender: models.Identity{Type: models.PrincipalService}, + targetTopic: "uu::user1", + wantErr: false, + }, + { + name: "workflow engine can send to user-broadcast topic", + sender: models.Identity{Type: models.PrincipalWorkflowEngine}, + targetTopic: "uu::user1", + wantErr: false, + }, + { + name: "bridge can send to user-broadcast topic", + sender: models.Identity{Type: models.PrincipalBridge}, + targetTopic: "uu::user1", + wantErr: false, + }, + { + name: "agent cannot send to user-broadcast topic", + sender: models.Identity{Type: models.PrincipalAgent, Workspace: "ws1"}, + targetTopic: "uu::user1", + wantErr: true, + errContains: "user-broadcast", + }, + { + name: "task cannot send to user-broadcast topic", + sender: models.Identity{Type: models.PrincipalTask, Workspace: "ws1"}, + targetTopic: "uu::user1", + wantErr: true, + errContains: "user-broadcast", + }, + { + name: "user cannot send to user-broadcast topic", + sender: models.Identity{Type: models.PrincipalUser, Workspace: "ws1"}, + targetTopic: "uu::user1", + wantErr: true, + errContains: "user-broadcast", + }, + { + name: "orchestrator cannot send to user-broadcast topic", + sender: models.Identity{Type: models.PrincipalOrchestrator, Workspace: "ws1"}, + targetTopic: "uu::user1", + wantErr: true, + errContains: "user-broadcast", + }, } for _, tt := range tests { @@ -488,6 +540,11 @@ func TestExtractWorkspaceFromTopic(t *testing.T) { topic: "us::alice::window1", want: "", }, + { + name: "user-broadcast topic returns empty (workspace-agnostic)", + topic: "uu::alice", + want: "", + }, { name: "bridge topic returns empty (no workspace)", topic: "br::example-bridge::default", @@ -595,6 +652,11 @@ func TestValidateTopicFormat(t *testing.T) { topic: "uw::user1::ws1", wantErr: false, }, + { + name: "valid uu prefix", + topic: "uu::user1", + wantErr: false, + }, { name: "valid ga prefix", topic: "ga::ws1", @@ -709,33 +771,38 @@ func TestSetupClientSubscriptions(t *testing.T) { wantSharedTopics: []string{"tb::prod::stream-proc"}, }, { - name: "user with workspace subscribes to window topic exclusively and workspace + per-user progress topics shared", + name: "user with workspace subscribes to window topic exclusively; broadcast lanes resume-or-tail (exclusive), progress shared", identity: models.Identity{ Type: models.PrincipalUser, ID: "alice", Specifier: "win-1", Workspace: "prod", }, - wantExclusiveTopics: []string{"us::alice::win-1"}, - // pg::us::alice is the per-user progress topic shared by all of - // alice's open windows. Window-level filtering happens at delivery - // time via the recipient field, not at the topic level. See - // UserProgressTopic + isBareUserRecipientMatch. - wantSharedTopics: []string{"gu::prod", "uw::alice::prod", "pg::us::alice"}, + // Broadcast lanes (gu/uw/uu) and progress lanes (pg::us::alice, and + // the workspace pg::prod) now use per-window named consumers with + // resume-or-tail semantics so a reconnect replays only the gap instead + // of re-dumping the whole retained history from seq 0. Window-level + // progress filtering still happens at delivery time via the recipient + // field (see UserProgressTopic + isBareUserRecipientMatch). A user + // therefore has no shared (anonymous) subscriptions. + wantExclusiveTopics: []string{"us::alice::win-1", "gu::prod", "uw::alice::prod", "uu::alice", "pg::us::alice"}, + wantSharedTopics: []string{}, }, { - name: "user without workspace subscribes to window topic exclusively and per-user progress topic shared", + name: "user without workspace subscribes to window topic exclusively; broadcast lane resume-or-tail (exclusive), progress shared", identity: models.Identity{ Type: models.PrincipalUser, ID: "bob", Specifier: "win-2", Workspace: "", // no workspace }, - wantExclusiveTopics: []string{"us::bob::win-2"}, - // Even without a workspace, users still subscribe to their - // per-user progress topic so targeted progress from agents in - // any workspace can reach them. - wantSharedTopics: []string{"pg::us::bob"}, + // uu (per-user broadcast) and pg::us::bob (per-user progress) now use + // per-window named resume-or-tail consumers; the mock records them on + // the exclusive path. Even without a workspace the user still + // subscribes to their per-user progress topic so targeted progress + // from agents in any workspace can reach them. + wantExclusiveTopics: []string{"us::bob::win-2", "uu::bob", "pg::us::bob"}, + wantSharedTopics: []string{}, }, { name: "workflow engine subscribes to event::receiver0 fan-in shard regardless of workspace", @@ -895,13 +962,16 @@ func TestSetupClientSubscriptions_DuplicateSubscriptionIgnored(t *testing.T) { sharedCount := len(router.subscribedTopics) router.mu.Unlock() - if exclusiveCount != 1 { - t.Errorf("expected 1 exclusive subscription after duplicate call, got %d", exclusiveCount) + // The agent gets 2 exclusive subscriptions: its identity topic + // (ag::{ws}::{impl}::{spec}) and the workspace progress topic pg::{ws}, which + // now uses a resume-or-tail named consumer. Both are recorded once despite the + // duplicate setup call (ClientSession HasSubscription guard). + if exclusiveCount != 2 { + t.Errorf("expected 2 exclusive subscriptions after duplicate call, got %d", exclusiveCount) } - // The shared Subscribe call may be deduplicated at the ClientSession level - // (HasSubscription guard). Agents get ga::{workspace} + pg::{workspace} = 2 shared. - if sharedCount != 2 { - t.Errorf("expected 2 shared subscriptions after duplicate call, got %d", sharedCount) + // Only ga::{workspace} remains on the shared (anonymous) path. + if sharedCount != 1 { + t.Errorf("expected 1 shared subscription after duplicate call, got %d", sharedCount) } } diff --git a/server/internal/gateway/server.go b/server/internal/gateway/server.go index d3dceb1..966855a 100644 --- a/server/internal/gateway/server.go +++ b/server/internal/gateway/server.go @@ -41,13 +41,29 @@ type GatewayServer struct { acl aclstore.Store // auditLogger is the audit domain Store (internal/storage/audit). auditLogger auditstore.Store - gatewayID string + // auditCoalescer suppresses within-window bursts of identical successful + // message-route / proxy-route audit events (the chat-streaming chatter + // trim). Nil when coalescing is disabled (audit_coalesce_window = 0), in + // which case auditLog is a zero-overhead passthrough. + auditCoalescer *auditCoalescer + gatewayID string + // tenantID is the tenant this gateway serves (the gateway is per-tenant). + // Minted into X-Auth-Tenant-ID on every ProxyHttpRequest so passthrough + // terminators (e.g. MemoryLayer's in-process terminator) receive a + // non-empty tenant for fail-closed authz. Configured via + // WithGatewayTenantID (env AETHER_TENANT_ID); when empty the proxy path + // falls back to the sender's workspace as the tenant scope. + tenantID string // Map of active streams by session ID activeStreams sync.Map // Secondary index: identity string -> sessionID for O(1) lookup identityIndex sync.Map - // implementationIndex maps "workspace:implementation" -> []*ClientSession (agents only). - // Used for O(1) pool task worker lookup with power-of-two-choices load balancing. + // implementationIndex maps "workspace:implementation" -> []*ClientSession. + // Indexed clients include connected agents AND connected services. Used for + // O(1) pool task worker lookup with power-of-two-choices load balancing. + // Services have no workspace component, so they are registered under the + // empty workspace key (e.g. ":webhook"); findWorkerByImplementation falls + // back to the empty workspace if no workspace-scoped match exists. implIndexMu sync.RWMutex implementationIndex map[string][]*ClientSession // orchestratorIndex maps "workspace:profile" -> []*ClientSession (orchestrators only). @@ -176,6 +192,25 @@ func WithCheckpointDefaultTTL(ttl time.Duration) GatewayOption { } } +// WithGatewayTenantID sets the tenant id this gateway serves. It is minted +// into X-Auth-Tenant-ID on every ProxyHttpRequest. Leave unset to fall back to +// the sender's workspace as the tenant scope on the proxy path. +func WithGatewayTenantID(tenantID string) GatewayOption { + return func(s *GatewayServer) { + s.tenantID = tenantID + } +} + +// WithAuditCoalesceWindow enables coalescing of high-volume repetitive audit +// events (successful message-route / proxy-route) so a burst from the same +// sender→target is recorded once per window. A window of 0 (or negative) +// disables coalescing entirely — every event is audited as before. +func WithAuditCoalesceWindow(window time.Duration) GatewayOption { + return func(s *GatewayServer) { + s.auditCoalescer = newAuditCoalescer(window) + } +} + // WithMessageRateLimit sets per-client message rate limiting. // ratePerSec is the sustained rate (messages/second), burst is the maximum burst size. func WithMessageRateLimit(ratePerSec float64, burst int) GatewayOption { @@ -341,7 +376,11 @@ func WithACLService(svc aclstore.Store) GatewayOption { func NewGatewayServer(sessions SessionManager, router MessageRouter, kvStore KVReadWriter, checkpointStore CheckpointManager, taskStore taskstore.Store, gatewayID string, auditLogger auditstore.Store, mtlsConfig MTLSConfig, opts ...GatewayOption) *GatewayServer { ts := timer.NewTimerSequence() - // Implement actual reschedule function that persists retry timing + // Implement actual reschedule function that persists retry timing. + // When the task carries a RetryPolicy, prefer the policy-computed delay + // over the caller-supplied one — the caller's value becomes the + // fallback for tasks without a policy. This delegates retry scheduling + // to the policy primitive without changing the timer's outer cadence. rescheduleFn := func(taskID string, delay time.Duration) { if taskStore == nil { logging.Logger.Warn().Str("task_id", taskID).Msg("cannot reschedule task, no taskStore") @@ -350,6 +389,12 @@ func NewGatewayServer(sessions SessionManager, router MessageRouter, kvStore KVR ctx := context.Background() retryAt := time.Now().Add(delay) + if task, err := taskStore.GetTask(ctx, taskID); err == nil && task != nil && task.RetryPolicy != nil { + // task.RetryCount is the post-failure count (FailTask already + // incremented). The next attempt index passed to + // ComputeNextRetryAt is therefore RetryCount. + retryAt = taskstore.ComputeNextRetryAt(task.RetryPolicy, task.RetryCount, nil) + } err := taskStore.RescheduleTaskAt(ctx, taskID, retryAt) if err != nil { logging.Logger.Error().Err(err).Str("task_id", taskID).Msg("failed to reschedule task") @@ -413,6 +458,9 @@ func NewGatewayServer(sessions SessionManager, router MessageRouter, kvStore KVR // logged at debug inside the helpers; no callers observe the error. if s.orchestration != nil && s.orchestration.TaskService != nil { s.orchestration.TaskService.SetEventPublisher(s) + // Feed B: the gateway is also the domain event publisher, emitting + // completion_event domain events onto the event plane (event::*). + s.orchestration.TaskService.SetDomainEventPublisher(s) } // Seed default-allow fallback for KV scope permissions (agent + task). @@ -620,6 +668,9 @@ func (s *GatewayServer) SetOrchestrationServices(orchestration *OrchestrationSer // lifecycle transitions fan onto tk::{workspace}::{task_id}::events. if orchestration.TaskService != nil { orchestration.TaskService.SetEventPublisher(s) + // Feed B: the gateway is also the domain event publisher, emitting + // completion_event domain events onto the event plane (event::*). + orchestration.TaskService.SetDomainEventPublisher(s) } // Configure dispatcher callback to route tasks to connected orchestrators @@ -672,6 +723,11 @@ func (s *GatewayServer) SetCleanupService(cleanupConfig *cleanup.Config) { } } + // Give the cleanup service the audit store so the scheduled audit-retention + // job can prune comprehensive_audit_log. The store is the same handle the + // gateway writes through; nil-tolerant on the cleanup side (job skips). + cleanupSvc.SetAuditStore(s.auditLogger) + // Run startup cleanup jobs (stale locks + stale claims + orphaned task reconciliation) go func() { defer func() { diff --git a/server/internal/gateway/spike_tenant_relay_test.go b/server/internal/gateway/spike_tenant_relay_test.go new file mode 100644 index 0000000..246fae0 --- /dev/null +++ b/server/internal/gateway/spike_tenant_relay_test.go @@ -0,0 +1,226 @@ +package gateway + +// SPIKE (feasibility): sandbox-provider tenant-relay redesign ("Direction A"). +// +// Goal of this file: prove the gateway-side foundation of the design empirically, +// over a REAL mTLS gRPC handshake, against the production identity-resolution path +// (authenticateMTLS -> resolveConnectionIdentity), NOT mocks. +// +// Design claim under test: +// A tenant-namespace relay holds the tenant-CA client cert locally and dials the +// LOCAL tenant gateway as the sandbox-provider service. The remote provider's +// InitConnection is forwarded opaquely and supplies only the specifier. Under +// semi-strict mTLS the gateway must: +// (1) authorize type+implementation from the tenant cert CN, and +// (2) take the specifier from the forwarded init, +// so the provider never needs to hold the tenant cert. An init claiming a +// different implementation than the cert must be rejected. +// +// This is the empirically-unproven hop (existing tests inject the cert identity +// below authenticateMTLS; none drive a real peer cert through a live handshake). + +import ( + "context" + "crypto/rand" + "crypto/rsa" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "math/big" + "net" + "strings" + "testing" + "time" + + pb "github.com/scitrera/aether/api/proto" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials" +) + +// spikeGenCA returns a self-signed CA cert + key. +func spikeGenCA(t *testing.T) (*x509.Certificate, *rsa.PrivateKey) { + t.Helper() + key, err := rsa.GenerateKey(rand.Reader, 2048) + if err != nil { + t.Fatalf("ca key: %v", err) + } + tmpl := &x509.Certificate{ + SerialNumber: big.NewInt(1), + Subject: pkix.Name{CommonName: "spike-tenant-ca"}, + NotBefore: time.Now().Add(-time.Hour), + NotAfter: time.Now().Add(24 * time.Hour), + IsCA: true, + KeyUsage: x509.KeyUsageCertSign | x509.KeyUsageCRLSign, + BasicConstraintsValid: true, + } + der, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, &key.PublicKey, key) + if err != nil { + t.Fatalf("ca cert: %v", err) + } + cert, err := x509.ParseCertificate(der) + if err != nil { + t.Fatalf("parse ca: %v", err) + } + return cert, key +} + +// spikeGenLeaf returns a CA-signed leaf cert (client or server) with the given CN. +func spikeGenLeaf(t *testing.T, serial int64, cn string, ca *x509.Certificate, caKey *rsa.PrivateKey, server bool) tls.Certificate { + t.Helper() + key, err := rsa.GenerateKey(rand.Reader, 2048) + if err != nil { + t.Fatalf("leaf key: %v", err) + } + eku := x509.ExtKeyUsageClientAuth + if server { + eku = x509.ExtKeyUsageServerAuth + } + tmpl := &x509.Certificate{ + SerialNumber: big.NewInt(serial), + Subject: pkix.Name{CommonName: cn}, + NotBefore: time.Now().Add(-time.Hour), + NotAfter: time.Now().Add(24 * time.Hour), + KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment, + ExtKeyUsage: []x509.ExtKeyUsage{eku}, + BasicConstraintsValid: true, + } + if server { + tmpl.IPAddresses = []net.IP{net.ParseIP("127.0.0.1")} + tmpl.DNSNames = []string{"localhost"} + } + der, err := x509.CreateCertificate(rand.Reader, tmpl, ca, &key.PublicKey, caKey) + if err != nil { + t.Fatalf("leaf cert: %v", err) + } + leaf, err := x509.ParseCertificate(der) + if err != nil { + t.Fatalf("parse leaf: %v", err) + } + return tls.Certificate{Certificate: [][]byte{der}, PrivateKey: key, Leaf: leaf} +} + +func serviceInit(impl, specifier string) *pb.UpstreamMessage { + return &pb.UpstreamMessage{ + Payload: &pb.UpstreamMessage_Init{ + Init: &pb.InitConnection{ + ClientType: &pb.InitConnection_Service{ + Service: &pb.ServiceIdentity{ + Implementation: impl, + Specifier: specifier, + }, + }, + }, + }, + } +} + +// spikeGwServer is a minimal AetherGateway server that runs the REAL +// identity-resolution path and reports the resolved identity (or the denial +// reason) back in ConnectionAck.assigned_id. +type spikeGwServer struct { + pb.UnimplementedAetherGatewayServer + h *AuthHandler +} + +func (s *spikeGwServer) Connect(stream pb.AetherGateway_ConnectServer) error { + msg, err := stream.Recv() + if err != nil { + return err + } + init := msg.GetInit() + if init == nil { + return stream.Send(spikeAck("ERR: expected init first")) + } + ctx := stream.Context() + certID, certPT, hasCert, isAnon, err := s.h.authenticateMTLS(ctx) + if err != nil { + return stream.Send(spikeAck("ERR: authenticateMTLS: " + err.Error())) + } + resolved, err := s.h.resolveConnectionIdentity(ctx, init, certID, certPT, hasCert, isAnon) + if err != nil { + return stream.Send(spikeAck("ERR: " + err.Error())) + } + return stream.Send(spikeAck(resolved.String())) +} + +func spikeAck(assignedID string) *pb.DownstreamMessage { + return &pb.DownstreamMessage{ + Payload: &pb.DownstreamMessage_ConnectionAck{ + ConnectionAck: &pb.ConnectionAck{SessionId: "spike", AssignedId: assignedID}, + }, + } +} + +// spikeDialResolve performs a real mTLS Connect with clientCert, sends init, and +// returns the assigned_id the server reported (resolved identity, or "ERR: ..."). +func spikeDialResolve(t *testing.T, addr string, clientCert tls.Certificate, init *pb.UpstreamMessage) string { + t.Helper() + creds := credentials.NewTLS(&tls.Config{ + Certificates: []tls.Certificate{clientCert}, + // Server identity is not under test here; we only exercise CLIENT-cert + // verification at the server. Skip server verification to avoid SAN setup. + InsecureSkipVerify: true, + }) + conn, err := grpc.NewClient("passthrough:///"+addr, grpc.WithTransportCredentials(creds)) + if err != nil { + t.Fatalf("dial: %v", err) + } + defer func() { _ = conn.Close() }() + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + stream, err := pb.NewAetherGatewayClient(conn).Connect(ctx) + if err != nil { + t.Fatalf("connect: %v", err) + } + if err := stream.Send(init); err != nil { + t.Fatalf("send init: %v", err) + } + down, err := stream.Recv() + if err != nil { + t.Fatalf("recv ack: %v", err) + } + return down.GetConnectionAck().GetAssignedId() +} + +func TestSpike_SemiStrictTenantCertServiceIdentity(t *testing.T) { + caCert, caKey := spikeGenCA(t) + serverCert := spikeGenLeaf(t, 2, "spike-gateway", caCert, caKey, true) + // The tenant-CA-signed client cert the tenant-relay presents to the LOCAL gateway. + tenantCert := spikeGenLeaf(t, 3, "sv::sandbox-provider::tenant-alice", caCert, caKey, false) + + caPool := x509.NewCertPool() + caPool.AddCert(caCert) + + // Real production semi-strict auth handler (no authenticator/acl/audit deps). + h := newAuthHandler(nil, false, MTLSModeSemiStrict, nil, nil) + + lis, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("listen: %v", err) + } + srv := grpc.NewServer(grpc.Creds(credentials.NewTLS(&tls.Config{ + Certificates: []tls.Certificate{serverCert}, + ClientAuth: tls.RequireAndVerifyClientCert, + ClientCAs: caPool, + }))) + pb.RegisterAetherGatewayServer(srv, &spikeGwServer{h: h}) + go func() { _ = srv.Serve(lis) }() + t.Cleanup(srv.Stop) + + addr := lis.Addr().String() + + // (1) Positive: cert authorizes type=Service + impl=sandbox-provider; the + // forwarded init's specifier (pod-7) wins. The provider never holds the cert. + got := spikeDialResolve(t, addr, tenantCert, serviceInit("sandbox-provider", "pod-7")) + if got != "sv::sandbox-provider::pod-7" { + t.Fatalf("resolved identity = %q, want %q", got, "sv::sandbox-provider::pod-7") + } + + // (2) Negative: a forwarded init claiming a different implementation than the + // tenant cert must be rejected by semi-strict validation. + got = spikeDialResolve(t, addr, tenantCert, serviceInit("evil-service", "pod-7")) + if !strings.Contains(got, "implementation mismatch") { + t.Fatalf("expected impl-mismatch denial, got %q", got) + } +} diff --git a/server/internal/gateway/subscription.go b/server/internal/gateway/subscription.go index 1f43222..7b10a4d 100644 --- a/server/internal/gateway/subscription.go +++ b/server/internal/gateway/subscription.go @@ -93,6 +93,34 @@ func (s *GatewayServer) subscribeClientToTopicExclusive(client *ClientSession, t return nil } +// subscribeClientToTopicResumeOrTail subscribes a client to a shared/broadcast +// topic with a per-window named consumer that resumes from its committed offset, +// or — on a first-ever subscribe — starts at the tail instead of replaying the +// whole topic. Use this (rather than the anonymous subscribeClientToTopic) for +// broadcast lanes the client re-subscribes on every connect (gu/uw/uu): a +// reconnect then replays only what was published while it was away instead of +// re-dumping (and re-shedding under backpressure) the entire retained history. +// The consumer name is the identity string, which is per-window, so each window +// tracks its own offset without stealing another window's. +func (s *GatewayServer) subscribeClientToTopicResumeOrTail(client *ClientSession, topic string) error { + if client.HasSubscription(topic) { + return nil // Already subscribed + } + client.identityMu.RLock() + consumerName := client.Identity.String() + client.identityMu.RUnlock() + cancel, err := s.router.SubscribeExclusiveResumeOrTail(topic, consumerName, s.createMessageHandler(client)) + if err != nil { + return err + } + client.AddSubscription(topic, func() { + cancel() + topicSubscriptions.Dec() + }) + topicSubscriptions.Inc() + return nil +} + // lookupTriggerTimestampMs retrieves the trigger_timestamp_ms from the task metadata for // the given taskID. Returns 0 if the task is not found, has no such key, or the value // cannot be parsed. Errors are non-fatal and only logged at debug level. @@ -231,6 +259,9 @@ func (s *GatewayServer) createMessageHandler(client *ClientSession) func([]byte) SourceTopic: parsed.env.Source, Payload: parsed.env.Payload, MessageType: parsed.env.MessageType, + // Mirror the gateway-stamped OBO subject onto delivery so the + // recipient can identify the user the message was sent for. + OnBehalfSubject: parsed.env.GetOnBehalfSubject(), }, }, }) @@ -354,6 +385,29 @@ func (s *GatewayServer) setupClientSubscriptions(client *ClientSession) error { logging.Logger.Warn().Err(err).Str("topic", upgTopic).Msg("failed to subscribe to user-progress topic") } } + // Subscribe to the per-user broadcast topic (uu::{user}) — a + // workspace-agnostic channel for platform→user notifications that + // reaches every one of the user's windows regardless of the active + // workspace. Shared subscription with local fan-out (mirroring the + // per-user progress topic), but carrying ordinary messages rather than + // progress updates, so it uses the generic message handler. + if identity.ID != "" { + ubTopic, err := models.UserBroadcastTopic(identity.ID) + if err != nil { + return fmt.Errorf("invalid user-broadcast topic: %w", err) + } + if err := s.subscribeClientToTopicResumeOrTail(client, ubTopic); err != nil { + logging.Logger.Warn().Err(err).Str("topic", ubTopic).Msg("failed to subscribe to user-broadcast topic") + } + } + // Phase 2 "Design M": re-attach this session to the per-task chat + // message lanes of any tasks that are already RUNNING with this user + // as the participating OBO subject. The live subscribe path + // (createProgressFilterHandler) only fires on the running notice, which + // has already passed for in-flight tasks — backfill catches them. + if s.taskStore != nil { + s.backfillRunningSubjectTaskMessages(context.Background(), client) + } // Subscribe to workspace-scoped topics if workspace is set (shared, no offset tracking) // This includes gu::{workspace}, uw::{user}::{workspace}, and pg::{workspace} if identity.Workspace != "" { @@ -414,7 +468,7 @@ func (s *GatewayServer) subscribeUserToWorkspaceTopics(client *ClientSession, wo if err != nil { return fmt.Errorf("invalid global user topic: %w", err) } - if err := s.subscribeClientToTopic(client, guTopic); err != nil { + if err := s.subscribeClientToTopicResumeOrTail(client, guTopic); err != nil { return fmt.Errorf("failed to subscribe to %s: %w", guTopic, err) } @@ -423,7 +477,7 @@ func (s *GatewayServer) subscribeUserToWorkspaceTopics(client *ClientSession, wo if err != nil { return fmt.Errorf("invalid user-workspace topic: %w", err) } - if err := s.subscribeClientToTopic(client, uwTopic); err != nil { + if err := s.subscribeClientToTopicResumeOrTail(client, uwTopic); err != nil { return fmt.Errorf("failed to subscribe to %s: %w", uwTopic, err) } diff --git a/server/internal/gateway/switch_workspace_test.go b/server/internal/gateway/switch_workspace_test.go index b50315e..33be2f7 100644 --- a/server/internal/gateway/switch_workspace_test.go +++ b/server/internal/gateway/switch_workspace_test.go @@ -225,12 +225,13 @@ func TestHandleSwitchWorkspace_ValidUser_NewWorkspaceTopicsSubscribed(t *testing sw := &pb.SwitchWorkspace{NewWorkspaceId: "ws3"} s.handleSwitchWorkspace(context.Background(), client, sw) - // After switch, user should be subscribed to gu.ws3 and uw.dave.ws3 - if !router.hasSharedTopic("gu::ws3") { - t.Error("expected shared subscription for 'gu::ws3' after workspace switch") + // After switch, user should be subscribed to gu.ws3 and uw.dave.ws3 via the + // per-window resume-or-tail (exclusive) path. + if !router.hasExclusiveTopic("gu::ws3") { + t.Error("expected exclusive (resume-or-tail) subscription for 'gu::ws3' after workspace switch") } - if !router.hasSharedTopic("uw::dave::ws3") { - t.Error("expected shared subscription for 'uw::dave::ws3' after workspace switch") + if !router.hasExclusiveTopic("uw::dave::ws3") { + t.Error("expected exclusive (resume-or-tail) subscription for 'uw::dave::ws3' after workspace switch") } } diff --git a/server/internal/gateway/task_message_lane_test.go b/server/internal/gateway/task_message_lane_test.go new file mode 100644 index 0000000..4feeb6c --- /dev/null +++ b/server/internal/gateway/task_message_lane_test.go @@ -0,0 +1,229 @@ +package gateway + +// Tests for Phase 2 "Design M": task-driven chat streaming on the per-task +// message lane (tk::{ws}::{task}::msg). Covers: +// (a) the tk::…::msg topic passes validateTopicFormat + enforceTopicPermissions +// and workspaceFromTopic extracts the workspace, +// (b) a running PROGRESS_KIND_TASK subject-notice run through +// createProgressFilterHandler subscribes the session to the task-message +// lane (shared), and a terminal notice removes it, +// (c) backfillRunningSubjectTaskMessages subscribes a running subject-task's +// message lane (exclusive, from timestamp), +// (d) a MessageEnvelope delivered through createMessageHandler on the msg +// topic arrives at the session as DownstreamMessage_Msg. +// +// Reuses the native-sqlite harness (newTaskTestServerWithSQLiteStore), +// newTaskTestClient, callerIdentity, subjectUserIdentity, and the +// mockMessageRouter shared/exclusive topic assertions. + +import ( + "context" + "testing" + "time" + + pb "github.com/scitrera/aether/api/proto" + "github.com/scitrera/aether/pkg/models" + "github.com/scitrera/aether/pkg/tasks" + "google.golang.org/protobuf/proto" +) + +// (a) The task-message topic is a well-formed, user-publishable topic and its +// workspace is recoverable. +func TestTaskMessageTopic_ValidatesAndResolvesWorkspace(t *testing.T) { + topic, err := models.TaskMessageTopic("ws1", "task1") + if err != nil { + t.Fatalf("TaskMessageTopic: %v", err) + } + if topic != "tk::ws1::task1::msg" { + t.Fatalf("TaskMessageTopic = %q, want tk::ws1::task1::msg", topic) + } + + if err := validateTopicFormat(topic); err != nil { + t.Errorf("validateTopicFormat(%q) = %v, want nil", topic, err) + } + + // A user principal must be allowed to publish to the lane (chat tokens are + // produced by the agent, but the matrix gate must not block tk::). + user := subjectUserIdentity("dev@example.com", "tab1") + if err := enforceTopicPermissions(user, topic); err != nil { + t.Errorf("enforceTopicPermissions(user, %q) = %v, want nil", topic, err) + } + // An agent principal (the actual producer) must also pass. + agent := callerIdentity("worker", "alice") + if err := enforceTopicPermissions(agent, topic); err != nil { + t.Errorf("enforceTopicPermissions(agent, %q) = %v, want nil", topic, err) + } + + if ws := workspaceFromTopic(topic); ws != "ws1" { + t.Errorf("workspaceFromTopic(%q) = %q, want ws1", topic, ws) + } +} + +// runSubjectNoticeThroughFilter marshals a PROGRESS_KIND_TASK subject-notice +// and feeds it through the per-client progress filter handler, exactly as the +// pg::us:: stream consumer would. +func runSubjectNoticeThroughFilter(t *testing.T, s *GatewayServer, client *ClientSession, workspace, taskID, status string) { + t.Helper() + update := &pb.ProgressUpdate{ + Source: s.gatewayID, + TaskId: taskID, + State: status, + TimestampMs: time.Now().UnixMilli(), + Workspace: workspace, + // Bare-user recipient → matches all of the user's windows. + Recipient: "us" + models.IdentitySep + client.Identity.ID, + Kind: pb.ProgressKind_PROGRESS_KIND_TASK, + Metadata: map[string]string{"status": status}, + } + b, err := proto.Marshal(update) + if err != nil { + t.Fatalf("marshal ProgressUpdate: %v", err) + } + s.createProgressFilterHandler(client)(b) +} + +// (b) Lifecycle-driven subscribe/unsubscribe on the shared lane. +func TestProgressFilter_TaskLifecycle_SubscribesAndUnsubscribesMsgLane(t *testing.T) { + s, cleanup := newTaskTestServerWithSQLiteStore(t) + defer cleanup() + + router := s.router.(*mockMessageRouter) + taskID := "task-lifecycle" + subject := subjectUserIdentity("dev@example.com", "tab1") + + stream := &mockStream{} + client := newTaskTestClient(stream, subject) + client.deliveryCh = make(chan *pb.DownstreamMessage, 8) + + msgTopic := models.MustTaskMessageTopic("ws1", taskID) + + // Running notice → subscribe. The live lane uses anonymous full-replay + // (a per-turn lane must be re-sent in full on every connect, incl. a page + // reload), so the mock records it on the shared path. + runSubjectNoticeThroughFilter(t, s, client, "ws1", taskID, "running") + if !router.hasSharedTopic(msgTopic) { + t.Fatalf("expected shared subscription to %q after running notice", msgTopic) + } + if !client.HasSubscription(taskMsgSubKey(taskID)) { + t.Errorf("expected client subscription under %q", taskMsgSubKey(taskID)) + } + + // Terminal notice → unsubscribe (subscription removed from the session). + runSubjectNoticeThroughFilter(t, s, client, "ws1", taskID, "completed") + if client.HasSubscription(taskMsgSubKey(taskID)) { + t.Errorf("expected client subscription %q removed after terminal notice", taskMsgSubKey(taskID)) + } +} + +// (c) Connect-time backfill of in-flight running subject tasks. +func TestBackfillRunningSubjectTaskMessages_SubscribesFromTimestamp(t *testing.T) { + s, cleanup := newTaskTestServerWithSQLiteStore(t) + defer cleanup() + + router := s.router.(*mockMessageRouter) + ctx := context.Background() + subjectID := "dev@example.com" + taskID := "task-backfill-running" + started := time.Now().Add(-1 * time.Minute) + + task := &tasks.Task{ + TaskID: taskID, + TaskType: "chat_message", + Workspace: "ws1", + Status: tasks.TaskStatusRunning, + StartedAt: &started, + Metadata: map[string]interface{}{ + "subject_participant": true, + }, + Authority: tasks.TaskAuthorityInfo{ + SubjectType: string(models.PrincipalUser), + SubjectID: subjectID, + }, + } + if err := s.taskStore.CreateTask(ctx, task); err != nil { + t.Fatalf("CreateTask: %v", err) + } + + // A non-subject running task in the same workspace must NOT be picked up. + otherID := "task-other-subject" + otherTask := &tasks.Task{ + TaskID: otherID, + TaskType: "chat_message", + Workspace: "ws1", + Status: tasks.TaskStatusRunning, + StartedAt: &started, + Metadata: map[string]interface{}{ + "subject_participant": true, + }, + Authority: tasks.TaskAuthorityInfo{ + SubjectType: string(models.PrincipalUser), + SubjectID: "someone-else@example.com", + }, + } + if err := s.taskStore.CreateTask(ctx, otherTask); err != nil { + t.Fatalf("CreateTask(other): %v", err) + } + + stream := &mockStream{} + client := newTaskTestClient(stream, subjectUserIdentity(subjectID, "tab1")) + client.deliveryCh = make(chan *pb.DownstreamMessage, 8) + + s.backfillRunningSubjectTaskMessages(ctx, client) + + msgTopic := models.MustTaskMessageTopic("ws1", taskID) + if !router.hasExclusiveTopic(msgTopic) { + t.Fatalf("expected exclusive subscription to %q after backfill", msgTopic) + } + if !client.HasSubscription(taskMsgSubKey(taskID)) { + t.Errorf("expected client subscription under %q", taskMsgSubKey(taskID)) + } + + otherTopic := models.MustTaskMessageTopic("ws1", otherID) + if router.hasExclusiveTopic(otherTopic) { + t.Errorf("did not expect subscription to another subject's task lane %q", otherTopic) + } +} + +// (d) A MessageEnvelope delivered through createMessageHandler on the msg topic +// arrives as DownstreamMessage_Msg. +func TestTaskMessageLane_DeliversEnvelopeAsDownstreamMsg(t *testing.T) { + s, cleanup := newTaskTestServerWithSQLiteStore(t) + defer cleanup() + + subject := subjectUserIdentity("dev@example.com", "tab1") + stream := &mockStream{} + client := newTaskTestClient(stream, subject) + client.deliveryCh = make(chan *pb.DownstreamMessage, 8) + + envelope := &pb.MessageEnvelope{ + Source: models.MustAgentTopic("ws1", "worker", "alice"), + Payload: []byte("hello chat"), + MessageType: pb.MessageType_CHAT, + TimestampMs: time.Now().UnixMilli(), + } + b, err := proto.Marshal(envelope) + if err != nil { + t.Fatalf("marshal MessageEnvelope: %v", err) + } + + s.createMessageHandler(client)(b) + + select { + case got := <-client.deliveryCh: + incoming := got.GetMsg() + if incoming == nil { + t.Fatalf("expected DownstreamMessage_Msg, got %T", got.Payload) + } + if string(incoming.Payload) != "hello chat" { + t.Errorf("payload = %q, want %q", string(incoming.Payload), "hello chat") + } + if incoming.MessageType != pb.MessageType_CHAT { + t.Errorf("message type = %v, want CHAT", incoming.MessageType) + } + if incoming.SourceTopic != envelope.Source { + t.Errorf("source topic = %q, want %q", incoming.SourceTopic, envelope.Source) + } + default: + t.Fatal("expected a delivered DownstreamMessage on deliveryCh") + } +} diff --git a/server/internal/gateway/task_message_subscription.go b/server/internal/gateway/task_message_subscription.go new file mode 100644 index 0000000..ec74f27 --- /dev/null +++ b/server/internal/gateway/task_message_subscription.go @@ -0,0 +1,188 @@ +package gateway + +import ( + "context" + "strconv" + + "github.com/scitrera/aether/internal/logging" + "github.com/scitrera/aether/pkg/models" + "github.com/scitrera/aether/pkg/tasks" +) + +// Phase 2 "Design M": task-driven chat streaming on a per-task message lane. +// +// Chat tokens / appended messages for a running task are published to +// tk::{workspace}::{task_id}::msg as ordinary MessageEnvelopes (no new proto +// type). The gateway auto-subscribes the task's SUBJECT sessions to that lane +// for the task's lifetime: +// - subscribe when the task's subject-notice reports status "running" +// (createProgressFilterHandler, live path), +// - subscribe-from-timestamp at connect time for already-RUNNING subject +// tasks (backfillRunningSubjectTaskMessages, replay path), +// - unsubscribe when the subject-notice reports a terminal status. +// +// Because the session principal IS the task's subject (the subject-notice only +// reaches the subject's own windows via the bare-user recipient filter), no new +// authorization is required to attach the subscription. + +// taskMsgSubKey is the per-task subscription key under which a client's +// task-message-lane subscription is registered. Keying on the task id (rather +// than the topic string) keeps unsubscribe lookups independent of workspace. +func taskMsgSubKey(taskID string) string { + return "taskmsg::" + taskID +} + +// subscribeClientToTaskMessages subscribes a client to a task's per-task chat +// message lane (tk::{ws}::{task}::msg). Live path: invoked when a running +// subject-notice arrives. +// +// Uses anonymous Subscribe (full replay from the lane start), NOT an +// offset-tracking consumer: a task-message lane is a small, ephemeral, per-turn +// buffer that must be replayed IN FULL on every (re)connect. A page reload keeps +// the window id but loses the tab's rendered state, so the whole in-flight turn +// has to be re-sent — resume-from-offset would skip the already-emitted tokens +// the reloaded tab needs to re-render. Because the live path stays anonymous, it +// commits no offset, so the connect-time backfill (a cold named consumer) still +// replays the turn from the start. Retention TTL bounds the retained lane. +// Idempotent. +func (s *GatewayServer) subscribeClientToTaskMessages(client *ClientSession, workspace, taskID string) { + if workspace == "" || taskID == "" { + return + } + key := taskMsgSubKey(taskID) + if client.HasSubscription(key) { + return + } + topic, err := models.TaskMessageTopic(workspace, taskID) + if err != nil { + logging.Logger.Warn().Err(err).Str("workspace", workspace).Str("task_id", taskID).Msg("invalid task-message topic; skipping subscribe") + return + } + cancel, err := s.router.Subscribe(topic, s.createMessageHandler(client)) + if err != nil { + logging.Logger.Warn().Err(err).Str("topic", topic).Msg("failed to subscribe to task-message topic") + return + } + client.AddSubscription(key, func() { + cancel() + topicSubscriptions.Dec() + }) + topicSubscriptions.Inc() +} + +// subscribeClientToTaskMessagesFromTimestamp subscribes a client to a task's +// per-task chat message lane with a per-window exclusive consumer that replays +// from startTimestampMs. Backfill/replay path: a reconnecting tab re-attaches +// to an already-RUNNING subject task and needs the chat tokens emitted while it +// was away. The consumer name is per-(window, task) so each tab replays +// independently without stealing another tab's offset. Idempotent. +func (s *GatewayServer) subscribeClientToTaskMessagesFromTimestamp(client *ClientSession, workspace, taskID string, startTimestampMs int64) { + if workspace == "" || taskID == "" { + return + } + key := taskMsgSubKey(taskID) + if client.HasSubscription(key) { + return + } + topic, err := models.TaskMessageTopic(workspace, taskID) + if err != nil { + logging.Logger.Warn().Err(err).Str("workspace", workspace).Str("task_id", taskID).Msg("invalid task-message topic; skipping backfill subscribe") + return + } + client.identityMu.RLock() + consumerName := client.Identity.String() + models.IdentitySep + taskID + client.identityMu.RUnlock() + + cancel, err := s.router.SubscribeExclusiveFromTimestamp(topic, consumerName, startTimestampMs, s.createMessageHandler(client)) + if err != nil { + logging.Logger.Warn().Err(err).Str("topic", topic).Msg("failed to subscribe-from-timestamp to task-message topic") + return + } + client.AddSubscription(key, func() { + cancel() + topicSubscriptions.Dec() + }) + topicSubscriptions.Inc() +} + +// unsubscribeClientFromTaskMessages removes a client's task-message-lane +// subscription for the given task id (no-op when absent). +func (s *GatewayServer) unsubscribeClientFromTaskMessages(client *ClientSession, taskID string) { + client.RemoveSubscription(taskMsgSubKey(taskID)) +} + +// backfillRunningSubjectTaskMessages, at user-session setup time, finds the +// user's currently-RUNNING tasks for which the user is the participating OBO +// subject and subscribes the session to each task's message lane from the +// task's start timestamp. This re-attaches a freshly-connected (or +// just-reconnected) browser tab to in-flight chat streams it would otherwise +// miss, because the running notice that drives the live subscribe path already +// fired before the tab connected. +// +// The task-store query filters server-side by workspace + RUNNING status + +// subject (SubjectType=User, SubjectID=user). The subject_participant opt-in +// flag is not a storage column, so it is checked client-side. Best-effort: any +// query failure is logged and the connect path proceeds. +func (s *GatewayServer) backfillRunningSubjectTaskMessages(ctx context.Context, client *ClientSession) { + if s.taskStore == nil { + return + } + + client.identityMu.RLock() + identity := client.Identity + client.identityMu.RUnlock() + + if identity.Type != models.PrincipalUser || identity.ID == "" || identity.Workspace == "" { + return + } + + filter := &tasks.TaskFilter{ + Workspace: identity.Workspace, + Statuses: []tasks.TaskStatus{tasks.TaskStatusRunning}, + SubjectType: string(models.PrincipalUser), + SubjectID: identity.ID, + Limit: maxRecursiveSubscriptionFanout, + } + rows, err := s.taskStore.ListTasks(ctx, filter) + if err != nil { + logging.Logger.Warn().Err(err).Str("user_id", identity.ID).Str("workspace", identity.Workspace).Msg("backfillRunningSubjectTaskMessages: ListTasks failed") + return + } + + for _, row := range rows { + if row == nil || row.TaskID == "" { + continue + } + // Defense-in-depth: re-check subject + opt-in client-side. The filter + // already gated subject_type/subject_id; subject_participant is not a + // storage column. + if !taskMetadataTruthy(row.Metadata["subject_participant"]) { + continue + } + if row.Authority.SubjectType != string(models.PrincipalUser) || row.Authority.SubjectID != identity.ID { + continue + } + + workspace := row.Workspace + if workspace == "" { + workspace = identity.Workspace + } + startMs := backfillStartTimestampMs(row) + s.subscribeClientToTaskMessagesFromTimestamp(client, workspace, row.TaskID, startMs) + } +} + +// backfillStartTimestampMs derives the replay start point for a running subject +// task, preferring the metadata["started_at_ms"] hint and falling back to the +// task's StartedAt timestamp. Returns 0 ("no hint") when neither is available. +func backfillStartTimestampMs(row *tasks.Task) int64 { + if v := metadataString(row.Metadata, "started_at_ms"); v != "" { + if parsed, perr := strconv.ParseInt(v, 10, 64); perr == nil && parsed > 0 { + return parsed + } + } + if row.StartedAt != nil { + return row.StartedAt.UnixMilli() + } + return 0 +} diff --git a/server/internal/gateway/task_msg_party_send_test.go b/server/internal/gateway/task_msg_party_send_test.go new file mode 100644 index 0000000..9d38f18 --- /dev/null +++ b/server/internal/gateway/task_msg_party_send_test.go @@ -0,0 +1,255 @@ +package gateway + +import ( + "context" + "testing" + + "github.com/scitrera/aether/internal/acl" + taskstore "github.com/scitrera/aether/internal/storage/tasks" + "github.com/scitrera/aether/pkg/models" + "github.com/scitrera/aether/pkg/tasks" +) + +// fakeGetTaskStore is a minimal taskstore.Store whose only useful method is +// GetTask. It embeds the interface so the remaining methods exist for type +// satisfaction but panic if a test path unexpectedly calls them — keeping the +// fake honest without implementing the whole large interface. +type fakeGetTaskStore struct { + taskstore.Store + task *tasks.Task + err error +} + +func (f *fakeGetTaskStore) GetTask(_ context.Context, _ string) (*tasks.Task, error) { + return f.task, f.err +} + +// TestParseTaskMsgTopic exercises the pure topic-shape parser: only the exact +// tk::{ws}::{task}::msg form is a task-msg topic; the events lane and plain +// workspace topics are not. +func TestParseTaskMsgTopic(t *testing.T) { + cases := []struct { + name string + topic string + wantWS string + wantTask string + wantIsMsg bool + }{ + {"msg lane", "tk::ws1::task-abc::msg", "ws1", "task-abc", true}, + {"events lane not matched", "tk::ws1::task-abc::events", "", "", false}, + {"plain workspace topic", "ag::ws1::impl::spec", "", "", false}, + {"too few segments", "tk::ws1::task-abc", "", "", false}, + {"too many segments", "tk::ws1::task-abc::msg::extra", "", "", false}, + {"wrong prefix", "xx::ws1::task-abc::msg", "", "", false}, + {"empty workspace", "tk::::task-abc::msg", "", "", false}, + {"empty task id", "tk::ws1::::msg", "", "", false}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + ws, task, isMsg := parseTaskMsgTopic(tc.topic) + if ws != tc.wantWS || task != tc.wantTask || isMsg != tc.wantIsMsg { + t.Fatalf("parseTaskMsgTopic(%q) = (%q,%q,%v), want (%q,%q,%v)", + tc.topic, ws, task, isMsg, tc.wantWS, tc.wantTask, tc.wantIsMsg) + } + }) + } +} + +// TestTaskPartyAuthorized mirrors authorizeTaskOp's party predicate: assignee, +// creator, OBO subject — plus the workspace tenancy guard. +func TestTaskPartyAuthorized(t *testing.T) { + svc := models.Identity{Type: models.PrincipalService, Implementation: "metrics", Specifier: "bridge"} + svcTopic := svc.ToTopic() + + cases := []struct { + name string + sender models.Identity + task *tasks.Task + want bool + }{ + { + name: "service assignee match", + sender: svc, + task: &tasks.Task{Workspace: "default", AssignedTo: svcTopic}, + want: true, + }, + { + name: "service creator match", + sender: svc, + task: &tasks.Task{Workspace: "default", ParentAgentID: svcTopic}, + want: true, + }, + { + name: "obo subject match", + sender: models.Identity{Type: models.PrincipalUser, ID: "alice", Specifier: "win-7"}, + task: &tasks.Task{ + Workspace: "default", + Authority: tasks.TaskAuthorityInfo{ + SubjectType: string(models.PrincipalUser), + SubjectID: "alice", + }, + }, + want: true, + }, + { + name: "non-party service", + sender: models.Identity{Type: models.PrincipalService, Implementation: "other", Specifier: "x"}, + task: &tasks.Task{Workspace: "default", AssignedTo: svcTopic}, + want: false, + }, + { + // Cross-workspace assignee IS authorized: the exact assignee-identity + // match is workspace-independent (a task may be legitimately targeted + // across workspaces, e.g. a sandbox agent in _sandbox serving a chat + // task in the user's app workspace). The tenancy guard only gates the + // weaker OBO-subject path. + name: "cross-workspace assignee authorized (exact identity match)", + sender: models.Identity{Type: models.PrincipalAgent, Workspace: "wsA", Implementation: "w", Specifier: "1"}, + task: &tasks.Task{ + Workspace: "wsB", + AssignedTo: models.Identity{ + Type: models.PrincipalAgent, Workspace: "wsA", Implementation: "w", Specifier: "1", + }.ToTopic(), + }, + want: true, + }, + { + // The tenancy guard STILL blocks a cross-workspace OBO-subject: a user + // in wsA is the task's subject but the task is in wsB, and they are + // neither creator nor assignee. + name: "tenancy guard blocks cross-workspace OBO subject", + sender: models.Identity{Type: models.PrincipalUser, Workspace: "wsA", ID: "user:alice"}, + task: &tasks.Task{ + Workspace: "wsB", + Authority: tasks.TaskAuthorityInfo{SubjectType: string(models.PrincipalUser), SubjectID: "user:alice"}, + }, + want: false, + }, + { + name: "nil task", + sender: svc, + task: nil, + want: false, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if got := taskPartyAuthorized(tc.sender, tc.task); got != tc.want { + t.Fatalf("taskPartyAuthorized() = %v, want %v", got, tc.want) + } + }) + } +} + +// TestTaskMsgSenderIsTaskParty exercises the store-backed helper end to end. +func TestTaskMsgSenderIsTaskParty(t *testing.T) { + ctx := context.Background() + svc := models.Identity{Type: models.PrincipalService, Implementation: "metrics", Specifier: "bridge"} + svcTopic := svc.ToTopic() + msgTopic := "tk::default::task-1::msg" + + t.Run("service assignee authorized to task msg lane", func(t *testing.T) { + gw := &GatewayServer{taskStore: &fakeGetTaskStore{ + task: &tasks.Task{Workspace: "default", AssignedTo: svcTopic}, + }} + ok, isMsg := gw.taskMsgSenderIsTaskParty(ctx, svc, msgTopic) + if !isMsg || !ok { + t.Fatalf("got (authorized=%v, isTaskMsg=%v); want (true, true)", ok, isMsg) + } + }) + + t.Run("non-party service not authorized via this path", func(t *testing.T) { + other := models.Identity{Type: models.PrincipalService, Implementation: "other", Specifier: "x"} + gw := &GatewayServer{taskStore: &fakeGetTaskStore{ + task: &tasks.Task{Workspace: "default", AssignedTo: svcTopic}, + }} + ok, isMsg := gw.taskMsgSenderIsTaskParty(ctx, other, msgTopic) + if !isMsg || ok { + t.Fatalf("got (authorized=%v, isTaskMsg=%v); want (false, true)", ok, isMsg) + } + }) + + t.Run("cross-workspace assignee authorized (exact identity match)", func(t *testing.T) { + agentWsA := models.Identity{Type: models.PrincipalAgent, Workspace: "wsA", Implementation: "w", Specifier: "1"} + gw := &GatewayServer{taskStore: &fakeGetTaskStore{ + task: &tasks.Task{Workspace: "wsB", AssignedTo: agentWsA.ToTopic()}, + }} + ok, isMsg := gw.taskMsgSenderIsTaskParty(ctx, agentWsA, "tk::wsB::task-1::msg") + if !isMsg || !ok { + t.Fatalf("got (authorized=%v, isTaskMsg=%v); want (true, true)", ok, isMsg) + } + }) + + t.Run("non-task topic no-ops", func(t *testing.T) { + gw := &GatewayServer{taskStore: &fakeGetTaskStore{ + task: &tasks.Task{Workspace: "default", AssignedTo: svcTopic}, + }} + ok, isMsg := gw.taskMsgSenderIsTaskParty(ctx, svc, "ag::default::impl::spec") + if isMsg || ok { + t.Fatalf("got (authorized=%v, isTaskMsg=%v); want (false, false)", ok, isMsg) + } + }) + + t.Run("events lane no-ops", func(t *testing.T) { + gw := &GatewayServer{taskStore: &fakeGetTaskStore{ + task: &tasks.Task{Workspace: "default", AssignedTo: svcTopic}, + }} + ok, isMsg := gw.taskMsgSenderIsTaskParty(ctx, svc, "tk::default::task-1::events") + if isMsg || ok { + t.Fatalf("got (authorized=%v, isTaskMsg=%v); want (false, false)", ok, isMsg) + } + }) + + t.Run("nil task store fails closed but flags task-msg shape", func(t *testing.T) { + gw := &GatewayServer{} + ok, isMsg := gw.taskMsgSenderIsTaskParty(ctx, svc, msgTopic) + if !isMsg || ok { + t.Fatalf("got (authorized=%v, isTaskMsg=%v); want (false, true)", ok, isMsg) + } + }) +} + +// TestCheckMessageSendTaskPartyGrant confirms the early-allow path in +// checkMessageSend returns AccessReadWrite for a task party (the service +// assignee that the plain workspace fallback would otherwise deny). +func TestCheckMessageSendTaskPartyGrant(t *testing.T) { + ctx := context.Background() + svc := models.Identity{Type: models.PrincipalService, Implementation: "metrics", Specifier: "bridge"} + + gw := &GatewayServer{ + acl: &acl.Service{}, // non-nil so the early ACL guard is passed + taskStore: &fakeGetTaskStore{ + task: &tasks.Task{Workspace: "default", AssignedTo: svc.ToTopic()}, + }, + } + level, err := gw.checkMessageSend(ctx, svc, "tk::default::task-1::msg") + if err != nil { + t.Fatalf("checkMessageSend() error = %v; want nil", err) + } + if level != acl.AccessReadWrite { + t.Fatalf("checkMessageSend() level = %d; want %d (AccessReadWrite)", level, acl.AccessReadWrite) + } +} + +// TestCheckMessageSendAgentMetricGrant confirms the additive metric-publish +// path in checkMessageSend returns AccessReadWrite for an agent principal +// targeting the metric fan-in plane, even though the plain per-workspace send +// ACL would deny it. This is what lets least-privilege sandbox agents (e.g. +// ag::_sandbox::sahara::*, deliberately READ-only on _sandbox) emit their own +// token/usage metrics. The empty acl.Service{} means the regular workspace +// fallback would NOT grant AccessReadWrite, so a passing result proves the +// early additive grant fired. +func TestCheckMessageSendAgentMetricGrant(t *testing.T) { + ctx := context.Background() + agent := models.Identity{Type: models.PrincipalAgent, Workspace: "_sandbox", Implementation: "sahara", Specifier: "sandbox-1"} + + gw := &GatewayServer{acl: &acl.Service{}} // non-nil so the early ACL guard is passed + + level, err := gw.checkMessageSend(ctx, agent, "metric::receiver0") + if err != nil { + t.Fatalf("checkMessageSend() error = %v; want nil", err) + } + if level != acl.AccessReadWrite { + t.Fatalf("checkMessageSend() level = %d; want %d (AccessReadWrite)", level, acl.AccessReadWrite) + } +} diff --git a/server/internal/gateway/task_subscription_handler.go b/server/internal/gateway/task_subscription_handler.go index 39c46c7..4241268 100644 --- a/server/internal/gateway/task_subscription_handler.go +++ b/server/internal/gateway/task_subscription_handler.go @@ -47,6 +47,7 @@ import ( pb "github.com/scitrera/aether/api/proto" "github.com/scitrera/aether/internal/logging" "github.com/scitrera/aether/pkg/models" + "github.com/scitrera/aether/pkg/sharding" "github.com/scitrera/aether/pkg/tasks" "github.com/scitrera/aether/sdk/go/aether" "google.golang.org/protobuf/proto" @@ -316,6 +317,28 @@ func (s *GatewayServer) PublishTaskEvent(ctx context.Context, workspace, taskID return s.publishTaskEventBytes(ctx, workspace, taskID, event) } +// PublishDomainEvent satisfies orchestration.DomainEventPublisher: it publishes +// a "feed B" domain event onto the event plane so the workflow engine (or any +// rule) can gather over task completions. The payload is the raw, pre-marshaled +// JSON bytes of an EventPayload (source_agent/event_names/data/workspace) — +// identical to what a client's SendEvent produces — published to the workspace's +// fan-in shard topic (event::receiver{shard}) via the same router + circuit +// breaker path publishTaskEventBytes uses. Best-effort: nil router / empty +// workspace / nil payload short-circuit to nil so a publish failure never blocks +// the originating terminal transition. +func (s *GatewayServer) PublishDomainEvent(ctx context.Context, workspace string, payload []byte) error { + if s.router == nil || workspace == "" || len(payload) == 0 { + return nil + } + topic := sharding.ReceiverTopic("event", sharding.ShardForWorkspace(workspace, sharding.TotalShards())) + if s.publishBreaker != nil { + return s.publishBreaker.Execute(func() error { + return s.router.Publish(ctx, topic, payload) + }) + } + return s.router.Publish(ctx, topic, payload) +} + // publishProgressTaskEvent emits a TaskProgressEvent on the task-events topic // for the given task_id when handleProgressReport fans out a progress report // that targets a known task. Best-effort: callers should not propagate the diff --git a/server/internal/gateway/task_subscription_test.go b/server/internal/gateway/task_subscription_test.go index 04066e1..594c4e7 100644 --- a/server/internal/gateway/task_subscription_test.go +++ b/server/internal/gateway/task_subscription_test.go @@ -83,6 +83,10 @@ func (r *inProcessRouter) SubscribeExclusiveFromTimestamp(topic string, _ string return r.subscribeInternal(topic, handler) } +func (r *inProcessRouter) SubscribeExclusiveResumeOrTail(topic string, _ string, handler func([]byte)) (func(), error) { + return r.subscribeInternal(topic, handler) +} + // newSubscriptionTestServer builds a gateway with the in-process router and // a sqlite task store. Returns a cleanup closer. func newSubscriptionTestServer(t *testing.T) (*GatewayServer, *inProcessRouter, func()) { diff --git a/server/internal/gateway/token_handler.go b/server/internal/gateway/token_handler.go index 263a3d5..b43fa0e 100644 --- a/server/internal/gateway/token_handler.go +++ b/server/internal/gateway/token_handler.go @@ -2,14 +2,23 @@ package gateway import ( "context" + "strings" pb "github.com/scitrera/aether/api/proto" + "github.com/scitrera/aether/internal/acl" "github.com/scitrera/aether/internal/admin" "github.com/scitrera/aether/internal/logging" + "github.com/scitrera/aether/pkg/models" ) // handleTokenOp processes a TokenOperation from a connected client. -func (s *GatewayServer) handleTokenOp(ctx context.Context, client *ClientSession, op *pb.TokenOperation) { +// +// caller is the connection's authenticated identity. The umbrella +// admin/tokens (or admin/*) gate is already enforced by isAllowedAdminOp at +// the connect.go dispatch site for ALL token operations. For CREATE of a +// principal_type=user token, a SECOND, stricter gate applies here: see the +// CREATE branch. +func (s *GatewayServer) handleTokenOp(ctx context.Context, client *ClientSession, caller models.Identity, op *pb.TokenOperation) { if s.adminProvider == nil { sendTokenError(client, op.GetRequestId(), "admin provider not configured") return @@ -78,13 +87,65 @@ func (s *GatewayServer) handleTokenOp(ctx context.Context, client *ClientSession return } + // SECURITY: minting a principal_type=user token issues a credential + // that authenticates AS a real user (see authenticateCredentials' + // claim↔key binding). The umbrella admin/tokens gate that admitted us + // here is sufficient for agent/service/task tokens, but user tokens + // require a SECOND, narrower gate so that holding admin/tokens does not + // implicitly grant the ability to impersonate arbitrary users. The + // caller must hold capability/mint_user_tokens OR the global admin + // umbrella admin/*. We also stop trusting req.CreatedBy blindly: for a + // user token, created_by IS the impersonated user id, so an unset value + // is rejected (it would otherwise silently default to "admin" — an + // invalid user identity) and the value is recorded server-side via the + // validated request below. + createdBy := req.CreatedBy + // created_by enforcement scope: user-only (intentional). + // + // For principal_type=user, created_by IS the impersonated user id and an + // empty value is dangerous (it would authenticate as a principal with no ID, + // effectively bypassing identity binding). We enforce non-empty here. + // + // For agent/service/task tokens, created_by has historically been an optional + // audit annotation ("who minted this") rather than an authenticating field. + // Enforcing non-empty would break all pre-existing non-user mint flows. The + // connect-path binding in authenticateCredentials already handles the + // empty-keyID case safely: identity.ID = keyID is always adopted from the key + // (even when empty), so the claim can never supply an ID the key doesn't + // authenticate. The empty-keyID case for service/agent keys is therefore + // defense-in-depth at the connect path, not a mint-time gap. + if models.PrincipalType(req.PrincipalType) == models.PrincipalUser { + holdsCapability := s.callerHoldsCapability(ctx, client, caller, acl.PermissionMintUserTokens, "mint_user_tokens") + holdsGlobalAdmin := s.isAllowedAdminOpQuiet(client, caller, "*") + if !holdsCapability && !holdsGlobalAdmin { + logging.Logger.Warn(). + Str("caller", caller.String()). + Str("requested_user", createdBy). + Msg("handleTokenOp: user-token mint denied (missing capability/mint_user_tokens and admin/*)") + sendTokenError(client, op.GetRequestId(), + "minting user tokens requires capability/mint_user_tokens or global admin (admin/*)") + return + } + if strings.TrimSpace(createdBy) == "" { + sendTokenError(client, op.GetRequestId(), + "created_by (the target user id) is required for user tokens") + return + } + logging.Logger.Info(). + Str("caller", caller.String()). + Str("target_user", createdBy). + Bool("via_capability", holdsCapability). + Bool("via_global_admin", holdsGlobalAdmin). + Msg("handleTokenOp: user-token mint authorized") + } + result, err := s.adminProvider.CreateToken(ctx, &admin.CreateTokenRequest{ Name: req.Name, PrincipalType: req.PrincipalType, WorkspacePatterns: req.WorkspacePatterns, Scopes: req.Scopes, ExpiresInHours: int(req.ExpiresInHours), - CreatedBy: req.CreatedBy, + CreatedBy: createdBy, }) if err != nil { logging.Logger.Error().Err(err).Msg("handleTokenOp: create token failed") diff --git a/server/internal/gateway/token_mint_capability_test.go b/server/internal/gateway/token_mint_capability_test.go new file mode 100644 index 0000000..cfb2460 --- /dev/null +++ b/server/internal/gateway/token_mint_capability_test.go @@ -0,0 +1,147 @@ +package gateway + +// Tests for the scoped user-token mint capability enforced in handleTokenOp's +// CREATE branch: +// +// - non-user token mint (agent/service/task) still works under the existing +// admin/tokens gate (no extra capability required). +// - user token mint by a non-system principal with NO ACL is denied (the new +// capability/mint_user_tokens + admin/* gate fails closed) and never +// reaches the admin provider. +// - user token mint by a system principal (orchestrator) is allowed via the +// admin/* umbrella that isAllowedAdminOpQuiet grants system principals. +// - user token mint with an empty created_by is rejected (created_by is the +// impersonated user id and must not silently default to "admin"). + +import ( + "context" + "testing" + "time" + + pb "github.com/scitrera/aether/api/proto" + "github.com/scitrera/aether/internal/admin" + "github.com/scitrera/aether/pkg/models" +) + +// fakeTokenProvider embeds admin.StateProvider (so the large interface is +// satisfied) and records CreateToken calls. Any non-overridden method panics +// if the code under test calls it, surfacing unexpected paths. +type fakeTokenProvider struct { + admin.StateProvider + createCalls []*admin.CreateTokenRequest +} + +func (f *fakeTokenProvider) CreateToken(_ context.Context, req *admin.CreateTokenRequest) (*admin.CreateTokenResult, error) { + f.createCalls = append(f.createCalls, req) + return &admin.CreateTokenResult{ + PlaintextToken: "plaintext", + Token: &admin.TokenInfo{ + ID: "tok-new", + Name: req.Name, + PrincipalType: req.PrincipalType, + CreatedBy: req.CreatedBy, + CreatedAt: time.Now(), + UpdatedAt: time.Now(), + }, + }, nil +} + +func newTokenMintServer(provider admin.StateProvider) *GatewayServer { + s := newAdminTestServer() + s.adminProvider = provider + return s +} + +func createTokenOp(principalType, createdBy string) *pb.TokenOperation { + return &pb.TokenOperation{ + Op: pb.TokenOperation_CREATE, + RequestId: "req-1", + CreateRequest: &pb.TokenCreateRequest{ + Name: "test-token", + PrincipalType: principalType, + CreatedBy: createdBy, + }, + } +} + +func TestHandleTokenOp_NonUserTokenMint_Succeeds(t *testing.T) { + provider := &fakeTokenProvider{} + s := newTokenMintServer(provider) + stream := &mockStream{} + client := newAdminTestClient(stream, models.PrincipalOrchestrator) + caller := client.Identity + + s.handleTokenOp(context.Background(), client, caller, createTokenOp("Agent", "orchestrator")) + + if len(provider.createCalls) != 1 { + t.Fatalf("expected CreateToken to be called once for agent token, got %d calls", len(provider.createCalls)) + } + if provider.createCalls[0].PrincipalType != "Agent" { + t.Errorf("expected PrincipalType=Agent, got %q", provider.createCalls[0].PrincipalType) + } +} + +func TestHandleTokenOp_UserTokenMint_NoCapability_Denied(t *testing.T) { + provider := &fakeTokenProvider{} + s := newTokenMintServer(provider) // s.acl is nil → capability + admin/* both fail + stream := &mockStream{} + // Non-system principal (Agent) so isAllowedAdminOpQuiet(...,"*") returns false. + client := newAdminTestClient(stream, models.PrincipalAgent) + caller := client.Identity + + s.handleTokenOp(context.Background(), client, caller, createTokenOp("User", "drew")) + + if len(provider.createCalls) != 0 { + t.Fatalf("expected user-token mint to be DENIED before reaching provider, got %d CreateToken calls", len(provider.createCalls)) + } + if stream.sentCount() == 0 { + t.Fatal("expected an error response to be sent for denied user-token mint") + } + stream.mu.Lock() + tokenResp := stream.sent[0].GetToken() + stream.mu.Unlock() + if tokenResp == nil || tokenResp.Success { + t.Fatalf("expected failed TokenResponse, got %v", tokenResp) + } + if tokenResp.Error == "" { + t.Error("expected non-empty error message on denial") + } +} + +func TestHandleTokenOp_UserTokenMint_SystemPrincipal_Allowed(t *testing.T) { + provider := &fakeTokenProvider{} + s := newTokenMintServer(provider) + stream := &mockStream{} + // Orchestrator is a system principal: isAllowedAdminOpQuiet(...,"*") → true, + // so the user-token gate passes even with nil ACL. + client := newAdminTestClient(stream, models.PrincipalOrchestrator) + caller := client.Identity + + s.handleTokenOp(context.Background(), client, caller, createTokenOp("User", "drew")) + + if len(provider.createCalls) != 1 { + t.Fatalf("expected user-token mint to succeed for system principal, got %d CreateToken calls", len(provider.createCalls)) + } + if provider.createCalls[0].CreatedBy != "drew" { + t.Errorf("expected CreatedBy=drew recorded server-side, got %q", provider.createCalls[0].CreatedBy) + } +} + +func TestHandleTokenOp_UserTokenMint_EmptyCreatedBy_Rejected(t *testing.T) { + provider := &fakeTokenProvider{} + s := newTokenMintServer(provider) + stream := &mockStream{} + client := newAdminTestClient(stream, models.PrincipalOrchestrator) + caller := client.Identity + + // System principal passes the capability gate, but an empty created_by must + // still be rejected: it is the impersonated user id. + s.handleTokenOp(context.Background(), client, caller, createTokenOp("User", "")) + + if len(provider.createCalls) != 0 { + t.Fatalf("expected user-token mint with empty created_by to be rejected, got %d CreateToken calls", len(provider.createCalls)) + } + if stream.sentCount() == 0 { + t.Fatal("expected an error response for empty created_by") + } +} diff --git a/server/internal/kv/badger_store.go b/server/internal/kv/badger_store.go index da426c2..acfec15 100644 --- a/server/internal/kv/badger_store.go +++ b/server/internal/kv/badger_store.go @@ -155,6 +155,7 @@ func (s *BadgerKVStore) ListPaginated( ) (*ListResult, error) { limit := DefaultListLimit var startAfter []byte + keyPrefix := "" if opts != nil { if opts.Limit > 0 { limit = opts.Limit @@ -162,9 +163,17 @@ func (s *BadgerKVStore) ListPaginated( if opts.Cursor != "" && opts.Cursor != "0" { startAfter = []byte(opts.Cursor) } + keyPrefix = opts.KeyPrefix } prefix := s.badgerPrefix(agent, scope, userID, workspace) + // scanPrefix narrows iteration to keys whose user-facing portion begins with + // keyPrefix, so the limit bounds matches rather than the whole namespace. + // shortKey extraction below still strips only the namespace `prefix`. + scanPrefix := prefix + if keyPrefix != "" { + scanPrefix = append(append([]byte{}, prefix...), keyPrefix...) + } items := make(map[string]string) var nextCursor string @@ -172,7 +181,7 @@ func (s *BadgerKVStore) ListPaginated( err := s.db.View(func(txn *badger.Txn) error { iterOpts := badger.DefaultIteratorOptions - iterOpts.Prefix = prefix + iterOpts.Prefix = scanPrefix it := txn.NewIterator(iterOpts) defer it.Close() @@ -186,7 +195,7 @@ func (s *BadgerKVStore) ListPaginated( it.Next() } } else { - it.Seek(prefix) + it.Seek(scanPrefix) } count := 0 @@ -254,21 +263,33 @@ func (s *BadgerKVStore) Decrement( // addDelta performs an atomic read-modify-write to add delta to the stored integer. // Values are stored as their decimal string representation (matching Redis INCR semantics). +// +// Badger uses optimistic concurrency control, so concurrent increments against +// the same key collide with ErrConflict — acute for hot keys like the per-user +// ratelimit counter (ratelimit:user:{user}) under a connection storm (e.g. a +// mass window refresh), where every request increments the same key. Retry the +// pure read-modify-write, matching addDeltaGuarded's policy; without this the +// conflict surfaced as a hard "failed to modify counter" error. func (s *BadgerKVStore) addDelta(fullKey []byte, delta int64) (int64, error) { var newVal int64 - err := s.db.Update(func(txn *badger.Txn) error { - current, err := readBadgerCounter(txn, fullKey) - if err != nil { - return err + for attempt := 0; attempt < casMaxAttempts; attempt++ { + err := s.db.Update(func(txn *badger.Txn) error { + current, err := readBadgerCounter(txn, fullKey) + if err != nil { + return err + } + newVal = current + delta + return txn.Set(fullKey, []byte(strconv.FormatInt(newVal, 10))) + }) + if err == nil { + return newVal, nil + } + if errors.Is(err, badger.ErrConflict) { + continue } - newVal = current + delta - encoded := []byte(strconv.FormatInt(newVal, 10)) - return txn.Set(fullKey, encoded) - }) - if err != nil { return 0, fmt.Errorf("failed to modify counter: %w", err) } - return newVal, nil + return 0, fmt.Errorf("failed to modify counter on %s: contention after %d attempts", string(fullKey), casMaxAttempts) } // readBadgerCounter loads the integer value at fullKey within the given @@ -390,3 +411,255 @@ func (s *BadgerKVStore) addDeltaGuarded(fullKey []byte, delta, guard int64, isCe } return 0, false, fmt.Errorf("guarded counter on %s failed: contention after %d attempts", string(fullKey), maxAttempts) } + +// casMaxAttempts bounds the optimistic-concurrency retry loop for the +// conditional write primitives, matching addDeltaGuarded's policy. +const casMaxAttempts = 100 + +// readBadgerValue loads the string value at fullKey within txn. Badger honors +// per-entry TTL natively, so logically-expired keys surface as +// badger.ErrKeyNotFound (found=false) — no soft-TTL handling required. +func readBadgerValue(txn *badger.Txn, fullKey []byte) (val string, found bool, err error) { + item, err := txn.Get(fullKey) + if errors.Is(err, badger.ErrKeyNotFound) { + return "", false, nil + } + if err != nil { + return "", false, err + } + err = item.Value(func(v []byte) error { + val = string(v) + return nil + }) + if err != nil { + return "", false, err + } + return val, true, nil +} + +// SetNX sets key=value only if the key is absent. Returns true iff written. +// +// TTL caveat: Badger's WithTTL truncates expiry to whole seconds, so a +// coordination lease's effective TTL rounds to ±1s on this backend. This is +// harmless for real coordination TTLs (seconds-scale); avoid sub-second lease +// TTLs on the single-node backend. +func (s *BadgerKVStore) SetNX( + ctx context.Context, + agent models.Identity, + scope KVScope, + key string, + value string, + userID string, + workspace string, + ttl time.Duration, +) (bool, error) { + if err := validateKey(key); err != nil { + return false, err + } + fullKey := s.badgerKey(agent, scope, key, userID, workspace) + + for attempt := 0; attempt < casMaxAttempts; attempt++ { + var written bool + err := s.db.Update(func(txn *badger.Txn) error { + _, found, err := readBadgerValue(txn, fullKey) + if err != nil { + return err + } + if found { + written = false + return nil + } + entry := badger.NewEntry(fullKey, []byte(value)) + if ttl > 0 { + entry = entry.WithTTL(ttl) + } + written = true + return txn.SetEntry(entry) + }) + if err == nil { + return written, nil + } + if errors.Is(err, badger.ErrConflict) { + continue + } + return false, fmt.Errorf("failed to setnx key %s: %w", key, err) + } + return false, fmt.Errorf("failed to setnx key %s: contention after %d attempts", key, casMaxAttempts) +} + +// CompareAndSet sets key=value only if the current value equals expected. +// Returns true iff the swap was applied. A missing key never matches. +func (s *BadgerKVStore) CompareAndSet( + ctx context.Context, + agent models.Identity, + scope KVScope, + key string, + expected string, + value string, + userID string, + workspace string, + ttl time.Duration, +) (bool, error) { + if err := validateKey(key); err != nil { + return false, err + } + fullKey := s.badgerKey(agent, scope, key, userID, workspace) + + for attempt := 0; attempt < casMaxAttempts; attempt++ { + var applied bool + err := s.db.Update(func(txn *badger.Txn) error { + current, found, err := readBadgerValue(txn, fullKey) + if err != nil { + return err + } + if !found || current != expected { + applied = false + return nil + } + entry := badger.NewEntry(fullKey, []byte(value)) + if ttl > 0 { + entry = entry.WithTTL(ttl) + } + applied = true + return txn.SetEntry(entry) + }) + if err == nil { + return applied, nil + } + if errors.Is(err, badger.ErrConflict) { + continue + } + return false, fmt.Errorf("failed to compare-and-set key %s: %w", key, err) + } + return false, fmt.Errorf("failed to compare-and-set key %s: contention after %d attempts", key, casMaxAttempts) +} + +// CompareAndDelete deletes key only if the current value equals expected. +// Returns true iff the delete was applied. +func (s *BadgerKVStore) CompareAndDelete( + ctx context.Context, + agent models.Identity, + scope KVScope, + key string, + expected string, + userID string, + workspace string, +) (bool, error) { + if err := validateKey(key); err != nil { + return false, err + } + fullKey := s.badgerKey(agent, scope, key, userID, workspace) + + for attempt := 0; attempt < casMaxAttempts; attempt++ { + var applied bool + err := s.db.Update(func(txn *badger.Txn) error { + current, found, err := readBadgerValue(txn, fullKey) + if err != nil { + return err + } + if !found || current != expected { + applied = false + return nil + } + applied = true + return txn.Delete(fullKey) + }) + if err == nil { + return applied, nil + } + if errors.Is(err, badger.ErrConflict) { + continue + } + return false, fmt.Errorf("failed to compare-and-delete key %s: %w", key, err) + } + return false, fmt.Errorf("failed to compare-and-delete key %s: contention after %d attempts", key, casMaxAttempts) +} + +// SetAdd atomically adds member to the JSON-encoded set stored at key. Returns +// whether the member was newly added (false if already present) and the set's +// cardinality after the add. When ttl > 0 the key's expiry is (re)set on a +// newly-added member (Badger truncates TTL to whole seconds). Duplicate adds are +// pure reads and never write, so they neither refresh the TTL nor cause churn. +func (s *BadgerKVStore) SetAdd( + ctx context.Context, + agent models.Identity, + scope KVScope, + key string, + member string, + userID string, + workspace string, + ttl time.Duration, +) (bool, int64, error) { + if err := validateKey(key); err != nil { + return false, 0, err + } + fullKey := s.badgerKey(agent, scope, key, userID, workspace) + + for attempt := 0; attempt < casMaxAttempts; attempt++ { + var ( + added bool + card int64 + ) + err := s.db.Update(func(txn *badger.Txn) error { + raw, _, err := readBadgerValue(txn, fullKey) + if err != nil { + return err + } + set := parseMemberSet(raw) + if _, ok := set[member]; ok { + added = false + card = int64(len(set)) + return nil // member already present — no write + } + set[member] = struct{}{} + added = true + card = int64(len(set)) + entry := badger.NewEntry(fullKey, []byte(encodeMemberSet(set))) + if ttl > 0 { + entry = entry.WithTTL(ttl) + } + return txn.SetEntry(entry) + }) + if err == nil { + return added, card, nil + } + if errors.Is(err, badger.ErrConflict) { + continue + } + return false, 0, fmt.Errorf("set-add on %s failed: %w", string(fullKey), err) + } + return false, 0, fmt.Errorf("set-add on %s failed: contention after %d attempts", string(fullKey), casMaxAttempts) +} + +// SetCard returns the cardinality of the set stored at key (0 if absent or +// logically expired via Badger's native TTL). +func (s *BadgerKVStore) SetCard( + ctx context.Context, + agent models.Identity, + scope KVScope, + key string, + userID string, + workspace string, +) (int64, error) { + if err := validateKey(key); err != nil { + return 0, err + } + fullKey := s.badgerKey(agent, scope, key, userID, workspace) + + var card int64 + err := s.db.View(func(txn *badger.Txn) error { + raw, found, err := readBadgerValue(txn, fullKey) + if err != nil { + return err + } + if !found { + return nil + } + card = int64(len(parseMemberSet(raw))) + return nil + }) + if err != nil { + return 0, fmt.Errorf("set-card on %s failed: %w", string(fullKey), err) + } + return card, nil +} diff --git a/server/internal/kv/coord_primitives_badger_test.go b/server/internal/kv/coord_primitives_badger_test.go new file mode 100644 index 0000000..a995685 --- /dev/null +++ b/server/internal/kv/coord_primitives_badger_test.go @@ -0,0 +1,245 @@ +package kv + +import ( + "context" + "fmt" + "sync" + "sync/atomic" + "testing" + "time" +) + +// TestBadgerSetNX_AbsentThenPresent verifies SetNX writes once and refuses a +// second writer. +func TestBadgerSetNX_AbsentThenPresent(t *testing.T) { + s := newTestBadgerStore(t) + ctx := context.Background() + agent := testAgent() + + ok, err := s.SetNX(ctx, agent, ScopeGlobal, "lock", "owner-a", "", "", 0) + if err != nil { + t.Fatalf("SetNX: %v", err) + } + if !ok { + t.Fatal("first SetNX should acquire") + } + + ok, err = s.SetNX(ctx, agent, ScopeGlobal, "lock", "owner-b", "", "", 0) + if err != nil { + t.Fatalf("SetNX 2: %v", err) + } + if ok { + t.Fatal("second SetNX should NOT acquire while key present") + } + + // Holder unchanged. + val, err := s.Get(ctx, agent, ScopeGlobal, "lock", "", "") + if err != nil { + t.Fatalf("Get: %v", err) + } + if val != "owner-a" { + t.Errorf("holder = %q, want owner-a", val) + } +} + +// TestBadgerSetNX_TTLExpiryReacquire verifies a SetNX lock with TTL becomes +// re-acquirable after the TTL elapses (Badger honors TTL natively). +// +// NOTE: Badger stores entry expiry as Unix *seconds* (WithTTL truncates to +// second granularity), so this test uses second-scale durations. Sub-second +// lock TTLs are unreliable on the Badger backend; real coordination TTLs are +// always several seconds (workflow/sandbox use 30s), where ±1s rounding is +// harmless. +func TestBadgerSetNX_TTLExpiryReacquire(t *testing.T) { + if testing.Short() { + t.Skip("skipping timing-based TTL test in -short mode") + } + s := newTestBadgerStore(t) + ctx := context.Background() + agent := testAgent() + + ok, err := s.SetNX(ctx, agent, ScopeGlobal, "lock", "owner-a", "", "", 2*time.Second) + if err != nil || !ok { + t.Fatalf("first SetNX ok=%v err=%v", ok, err) + } + // Still held immediately. + ok, _ = s.SetNX(ctx, agent, ScopeGlobal, "lock", "owner-b", "", "", 2*time.Second) + if ok { + t.Fatal("should not acquire before TTL expiry") + } + + // Sleep past the second-truncated expiry boundary. + time.Sleep(3100 * time.Millisecond) + + ok, err = s.SetNX(ctx, agent, ScopeGlobal, "lock", "owner-b", "", "", 5*time.Second) + if err != nil { + t.Fatalf("SetNX after expiry: %v", err) + } + if !ok { + t.Fatal("should re-acquire after TTL expiry") + } +} + +// TestBadgerCompareAndSet covers match, mismatch, and missing-key cases plus +// the refresh idiom (expected == new). +func TestBadgerCompareAndSet(t *testing.T) { + s := newTestBadgerStore(t) + ctx := context.Background() + agent := testAgent() + + // Missing key never matches a non-empty expected. + ok, err := s.CompareAndSet(ctx, agent, ScopeGlobal, "k", "owner-a", "owner-a", "", "", time.Second) + if err != nil { + t.Fatalf("CAS missing: %v", err) + } + if ok { + t.Fatal("CAS on missing key should not apply") + } + + if _, err := s.SetNX(ctx, agent, ScopeGlobal, "k", "owner-a", "", "", 0); err != nil { + t.Fatalf("seed SetNX: %v", err) + } + + // Mismatch. + ok, _ = s.CompareAndSet(ctx, agent, ScopeGlobal, "k", "owner-x", "owner-b", "", "", time.Second) + if ok { + t.Fatal("CAS mismatch should not apply") + } + + // Match (refresh idiom: expected == new value). + ok, err = s.CompareAndSet(ctx, agent, ScopeGlobal, "k", "owner-a", "owner-a", "", "", time.Second) + if err != nil { + t.Fatalf("CAS match: %v", err) + } + if !ok { + t.Fatal("CAS match should apply") + } + + // Match with new owner (handoff). + ok, _ = s.CompareAndSet(ctx, agent, ScopeGlobal, "k", "owner-a", "owner-b", "", "", time.Second) + if !ok { + t.Fatal("CAS handoff should apply") + } + val, _ := s.Get(ctx, agent, ScopeGlobal, "k", "", "") + if val != "owner-b" { + t.Errorf("value = %q, want owner-b", val) + } +} + +// TestBadgerCompareAndDelete covers match, mismatch, and missing cases. +func TestBadgerCompareAndDelete(t *testing.T) { + s := newTestBadgerStore(t) + ctx := context.Background() + agent := testAgent() + + // Missing. + ok, err := s.CompareAndDelete(ctx, agent, ScopeGlobal, "k", "owner-a", "", "") + if err != nil { + t.Fatalf("CAD missing: %v", err) + } + if ok { + t.Fatal("CAD on missing key should not apply") + } + + if _, err := s.SetNX(ctx, agent, ScopeGlobal, "k", "owner-a", "", "", 0); err != nil { + t.Fatalf("seed: %v", err) + } + + // Mismatch leaves the key intact. + ok, _ = s.CompareAndDelete(ctx, agent, ScopeGlobal, "k", "owner-x", "", "") + if ok { + t.Fatal("CAD mismatch should not apply") + } + if _, err := s.Get(ctx, agent, ScopeGlobal, "k", "", ""); err != nil { + t.Fatalf("key should still exist after mismatched CAD: %v", err) + } + + // Match deletes. + ok, err = s.CompareAndDelete(ctx, agent, ScopeGlobal, "k", "owner-a", "", "") + if err != nil { + t.Fatalf("CAD match: %v", err) + } + if !ok { + t.Fatal("CAD match should apply") + } + if _, err := s.Get(ctx, agent, ScopeGlobal, "k", "", ""); err == nil { + t.Fatal("key should be gone after matched CAD") + } +} + +// TestBadgerSetNX_ConcurrentSingleWinner asserts that under N concurrent +// SetNX attempts exactly one acquires the lock — the core mutual-exclusion +// guarantee. +func TestBadgerSetNX_ConcurrentSingleWinner(t *testing.T) { + s := newTestBadgerStore(t) + ctx := context.Background() + agent := testAgent() + + const n = 50 + var winners int64 + var wg sync.WaitGroup + start := make(chan struct{}) + for i := 0; i < n; i++ { + wg.Add(1) + go func(i int) { + defer wg.Done() + <-start + ok, err := s.SetNX(ctx, agent, ScopeGlobal, "race", fmt.Sprintf("owner-%d", i), "", "", time.Minute) + if err != nil { + t.Errorf("SetNX: %v", err) + return + } + if ok { + atomic.AddInt64(&winners, 1) + } + }(i) + } + close(start) + wg.Wait() + + if winners != 1 { + t.Fatalf("expected exactly 1 SetNX winner, got %d", winners) + } +} + +// TestBadgerIncrement_ConcurrentSameKey verifies the counter increment retries +// on optimistic-concurrency conflict: many concurrent increments of the same hot +// key (e.g. a per-user ratelimit counter under a connection storm) must all +// succeed with no lost updates and no "failed to modify counter: Transaction +// Conflict" errors. Without the retry, concurrent increments fail on ErrConflict. +func TestBadgerIncrement_ConcurrentSameKey(t *testing.T) { + s := newTestBadgerStore(t) + ctx := context.Background() + agent := testAgent() + + const n = 100 + var wg sync.WaitGroup + start := make(chan struct{}) + errs := make(chan error, n) + for i := 0; i < n; i++ { + wg.Add(1) + go func() { + defer wg.Done() + <-start + if _, err := s.Increment(ctx, agent, ScopeGlobal, "hot", "", ""); err != nil { + errs <- err + } + }() + } + close(start) + wg.Wait() + close(errs) + for err := range errs { + t.Errorf("concurrent Increment failed (retry-on-conflict missing?): %v", err) + } + + // Final value must equal the number of successful increments — no lost + // updates. One extra increment reads back the committed total. + got, err := s.Increment(ctx, agent, ScopeGlobal, "hot", "", "") + if err != nil { + t.Fatalf("final Increment: %v", err) + } + if got != n+1 { + t.Fatalf("expected counter == %d after %d concurrent + 1 increments, got %d", n+1, n, got) + } +} diff --git a/server/internal/kv/coord_primitives_js_test.go b/server/internal/kv/coord_primitives_js_test.go new file mode 100644 index 0000000..fbf72db --- /dev/null +++ b/server/internal/kv/coord_primitives_js_test.go @@ -0,0 +1,161 @@ +package kv_test + +import ( + "context" + "fmt" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/scitrera/aether/internal/kv" +) + +// TestJSSetNX_AbsentThenPresent verifies SetNX acquires once and refuses a +// second writer while the (non-expired) entry is present. +func TestJSSetNX_AbsentThenPresent(t *testing.T) { + s := newTestJSStore(t) + ctx := context.Background() + agent := jsAgent() + + ok, err := s.SetNX(ctx, agent, kv.ScopeGlobal, "lock", "owner-a", "", "", 0) + if err != nil || !ok { + t.Fatalf("first SetNX ok=%v err=%v", ok, err) + } + ok, err = s.SetNX(ctx, agent, kv.ScopeGlobal, "lock", "owner-b", "", "", 0) + if err != nil { + t.Fatalf("second SetNX: %v", err) + } + if ok { + t.Fatal("second SetNX should not acquire while present") + } + val, err := s.Get(ctx, agent, kv.ScopeGlobal, "lock", "", "") + if err != nil { + t.Fatalf("Get: %v", err) + } + if val != "owner-a" { + t.Errorf("holder = %q, want owner-a", val) + } +} + +// TestJSSetNX_SoftTTLExpiredReacquire is the critical JetStream-specific path: +// a soft-TTL-expired entry still physically exists, so SetNX must detect the +// logical expiry and Update-over the stale revision rather than failing on +// kv.Create's ErrKeyExists. +func TestJSSetNX_SoftTTLExpiredReacquire(t *testing.T) { + s := newTestJSStore(t) + ctx := context.Background() + agent := jsAgent() + + ok, err := s.SetNX(ctx, agent, kv.ScopeGlobal, "lock", "owner-a", "", "", 250*time.Millisecond) + if err != nil || !ok { + t.Fatalf("first SetNX ok=%v err=%v", ok, err) + } + // Held before expiry. + ok, _ = s.SetNX(ctx, agent, kv.ScopeGlobal, "lock", "owner-b", "", "", time.Second) + if ok { + t.Fatal("should not acquire before soft-TTL expiry") + } + + time.Sleep(400 * time.Millisecond) + + // After soft-TTL expiry the physical entry lingers, but SetNX must treat + // it as free and re-acquire. + ok, err = s.SetNX(ctx, agent, kv.ScopeGlobal, "lock", "owner-b", "", "", time.Second) + if err != nil { + t.Fatalf("SetNX after soft-TTL expiry: %v", err) + } + if !ok { + t.Fatal("should re-acquire after soft-TTL expiry") + } + val, err := s.Get(ctx, agent, kv.ScopeGlobal, "lock", "", "") + if err != nil { + t.Fatalf("Get: %v", err) + } + if val != "owner-b" { + t.Errorf("holder = %q, want owner-b", val) + } +} + +// TestJSCompareAndSet covers match/mismatch/missing + the refresh idiom. +func TestJSCompareAndSet(t *testing.T) { + s := newTestJSStore(t) + ctx := context.Background() + agent := jsAgent() + + ok, _ := s.CompareAndSet(ctx, agent, kv.ScopeGlobal, "k", "owner-a", "owner-a", "", "", time.Second) + if ok { + t.Fatal("CAS on missing key should not apply") + } + + if _, err := s.SetNX(ctx, agent, kv.ScopeGlobal, "k", "owner-a", "", "", 0); err != nil { + t.Fatalf("seed: %v", err) + } + if ok, _ := s.CompareAndSet(ctx, agent, kv.ScopeGlobal, "k", "owner-x", "owner-b", "", "", time.Second); ok { + t.Fatal("CAS mismatch should not apply") + } + if ok, err := s.CompareAndSet(ctx, agent, kv.ScopeGlobal, "k", "owner-a", "owner-b", "", "", time.Second); err != nil || !ok { + t.Fatalf("CAS match ok=%v err=%v", ok, err) + } + val, _ := s.Get(ctx, agent, kv.ScopeGlobal, "k", "", "") + if val != "owner-b" { + t.Errorf("value = %q, want owner-b", val) + } +} + +// TestJSCompareAndDelete covers match/mismatch/missing. +func TestJSCompareAndDelete(t *testing.T) { + s := newTestJSStore(t) + ctx := context.Background() + agent := jsAgent() + + if ok, _ := s.CompareAndDelete(ctx, agent, kv.ScopeGlobal, "k", "owner-a", "", ""); ok { + t.Fatal("CAD on missing should not apply") + } + if _, err := s.SetNX(ctx, agent, kv.ScopeGlobal, "k", "owner-a", "", "", 0); err != nil { + t.Fatalf("seed: %v", err) + } + if ok, _ := s.CompareAndDelete(ctx, agent, kv.ScopeGlobal, "k", "owner-x", "", ""); ok { + t.Fatal("CAD mismatch should not apply") + } + if ok, err := s.CompareAndDelete(ctx, agent, kv.ScopeGlobal, "k", "owner-a", "", ""); err != nil || !ok { + t.Fatalf("CAD match ok=%v err=%v", ok, err) + } + if _, err := s.Get(ctx, agent, kv.ScopeGlobal, "k", "", ""); err == nil { + t.Fatal("key should be gone after matched CAD") + } +} + +// TestJSSetNX_ConcurrentSingleWinner asserts exactly one winner under +// contention (revision-CAS mutual exclusion). +func TestJSSetNX_ConcurrentSingleWinner(t *testing.T) { + s := newTestJSStore(t) + ctx := context.Background() + agent := jsAgent() + + const n = 25 + var winners int64 + var wg sync.WaitGroup + start := make(chan struct{}) + for i := 0; i < n; i++ { + wg.Add(1) + go func(i int) { + defer wg.Done() + <-start + ok, err := s.SetNX(ctx, agent, kv.ScopeGlobal, "race", fmt.Sprintf("owner-%d", i), "", "", time.Minute) + if err != nil { + t.Errorf("SetNX: %v", err) + return + } + if ok { + atomic.AddInt64(&winners, 1) + } + }(i) + } + close(start) + wg.Wait() + + if winners != 1 { + t.Fatalf("expected exactly 1 SetNX winner, got %d", winners) + } +} diff --git a/server/internal/kv/jetstream_store.go b/server/internal/kv/jetstream_store.go index a22a614..ce00f01 100644 --- a/server/internal/kv/jetstream_store.go +++ b/server/internal/kv/jetstream_store.go @@ -122,29 +122,67 @@ func scopeTag(scope KVScope) string { } } -// buildKey constructs the full NATS KV key for an entry. -// Format: {scopeTag}/{impl}/{spec}/{userID}/{workspace}/{userKey} +// buildJSKey constructs the full NATS KV key for an entry. +// +// Layout mirrors BuildNamespace semantics (see namespace.go): the agent +// identity segments (impl, spec) are ONLY embedded for SharingExclusive +// scopes. For SharingShared scopes (global, workspace, user-shared, +// user-workspace-shared) the agent identity is omitted so that all agents +// in the tenant rendezvous on the same storage key. The identity +// (user/workspace) segments included depend on spec.Identity. func buildJSKey(agent models.Identity, scope KVScope, userID, workspace, key string) string { - return strings.Join([]string{ - scopeTag(scope), - encodeKVSegment(agent.Implementation), - encodeKVSegment(agent.Specifier), - encodeKVSegment(userID), - encodeKVSegment(workspace), - encodeKVSegment(key), - }, jsSeparator) + spec, ok := ScopeSpecFromKVScope(scope) + if !ok { + // Unknown scope — fall back to the most restrictive (per-agent + // per-user-per-workspace) layout to preserve isolation rather than + // accidentally leak across agents. Callers should always use a + // canonical KVScope value. + spec = SpecUserWorkspace + } + segments := []string{scopeTag(scope)} + if spec.Sharing == SharingExclusive { + segments = append(segments, + encodeKVSegment(agent.Implementation), + encodeKVSegment(agent.Specifier), + ) + } + switch spec.Identity { + case IdentityScopeWorkspace: + segments = append(segments, encodeKVSegment(workspace)) + case IdentityScopeUser: + segments = append(segments, encodeKVSegment(userID)) + case IdentityScopeUserWorkspace: + segments = append(segments, encodeKVSegment(userID), encodeKVSegment(workspace)) + } + segments = append(segments, encodeKVSegment(key)) + return strings.Join(segments, jsSeparator) } // buildJSPrefix constructs the namespace prefix for list/filter operations. -// The prefix ends with "/" so NATS keys() matching works by prefix. +// The prefix ends with jsSeparator so NATS keys() matching works by prefix. +// Mirrors buildJSKey's conditional inclusion of agent-identity segments based +// on spec.Sharing. func buildJSPrefix(agent models.Identity, scope KVScope, userID, workspace string) string { - return strings.Join([]string{ - scopeTag(scope), - encodeKVSegment(agent.Implementation), - encodeKVSegment(agent.Specifier), - encodeKVSegment(userID), - encodeKVSegment(workspace), - }, jsSeparator) + jsSeparator + spec, ok := ScopeSpecFromKVScope(scope) + if !ok { + spec = SpecUserWorkspace + } + segments := []string{scopeTag(scope)} + if spec.Sharing == SharingExclusive { + segments = append(segments, + encodeKVSegment(agent.Implementation), + encodeKVSegment(agent.Specifier), + ) + } + switch spec.Identity { + case IdentityScopeWorkspace: + segments = append(segments, encodeKVSegment(workspace)) + case IdentityScopeUser: + segments = append(segments, encodeKVSegment(userID)) + case IdentityScopeUserWorkspace: + segments = append(segments, encodeKVSegment(userID), encodeKVSegment(workspace)) + } + return strings.Join(segments, jsSeparator) + jsSeparator } // extractUserKey strips the namespace prefix from a full NATS KV key and @@ -273,6 +311,7 @@ func (s *JetStreamKVStore) ListPaginated( ) (*ListResult, error) { limit := DefaultListLimit offset := 0 + keyPrefix := "" if opts != nil { if opts.Limit > 0 { limit = opts.Limit @@ -282,6 +321,7 @@ func (s *JetStreamKVStore) ListPaginated( offset = n } } + keyPrefix = opts.KeyPrefix } prefix := buildJSPrefix(agent, scope, userID, workspace) @@ -307,7 +347,19 @@ func (s *JetStreamKVStore) ListPaginated( // non-fatal: we already collected what we can _ = err } - sort.Strings(matching) + // Apply the user-key prefix filter BEFORE offset/limit so the page bounds + // matching keys, not all keys in the scope. Keys are decoded to their + // user-facing form (the NATS key is per-segment escaped) for the compare. + if keyPrefix != "" { + filtered := make([]string, 0, len(matching)) + for _, k := range matching { + if uk, ok := extractUserKey(k, prefix); ok && strings.HasPrefix(uk, keyPrefix) { + filtered = append(filtered, k) + } + } + matching = filtered + } + sort.Strings(matching) // Apply offset. @@ -507,6 +559,237 @@ func (s *JetStreamKVStore) casCounter( return 0, false, fmt.Errorf("casCounter %s: too much contention after %d attempts", key, jsMaxCASAttempts) } +// SetNX sets key=value only if the key is absent. Returns true iff written. +// +// Soft-TTL subtlety: NATS KV has no native per-key TTL, so an entry whose +// encoded soft-TTL has elapsed still physically exists. kv.Create would then +// return ErrKeyExists for a key that is *logically* free. We therefore Get +// first: on a logically-expired entry we Update-over its revision (treating it +// as a fresh acquire); only a present, non-expired entry blocks the SetNX. +func (s *JetStreamKVStore) SetNX( + ctx context.Context, + agent models.Identity, + scope KVScope, + key string, + value string, + userID string, + workspace string, + ttl time.Duration, +) (bool, error) { + if err := validateKey(key); err != nil { + return false, err + } + fullKey := buildJSKey(agent, scope, userID, workspace, key) + + for attempt := 0; attempt < jsMaxCASAttempts; attempt++ { + encoded := []byte(encodeStoredValue(value, ttl)) + entry, err := s.kv.Get(ctx, fullKey) + if err != nil { + if !errors.Is(err, jetstream.ErrKeyNotFound) { + return false, fmt.Errorf("setnx get %s: %w", key, err) + } + // Absent → atomic create-if-absent. + if _, cErr := s.kv.Create(ctx, fullKey, encoded); cErr != nil { + if isRevisionConflict(cErr) { + continue // someone created concurrently; re-evaluate + } + return false, fmt.Errorf("setnx create %s: %w", key, cErr) + } + return true, nil + } + // Present: free only if the soft-TTL has elapsed. + _, _, expired := decodeStoredValue(string(entry.Value()), time.Now()) + if !expired { + return false, nil + } + if _, uErr := s.kv.Update(ctx, fullKey, encoded, entry.Revision()); uErr != nil { + if isRevisionConflict(uErr) { + continue + } + return false, fmt.Errorf("setnx update-expired %s: %w", key, uErr) + } + return true, nil + } + return false, fmt.Errorf("setnx %s: too much contention after %d attempts", key, jsMaxCASAttempts) +} + +// CompareAndSet sets key=value only if the current (non-expired) value equals +// expected. Returns true iff the swap was applied. A missing or logically +// expired key never matches. +func (s *JetStreamKVStore) CompareAndSet( + ctx context.Context, + agent models.Identity, + scope KVScope, + key string, + expected string, + value string, + userID string, + workspace string, + ttl time.Duration, +) (bool, error) { + if err := validateKey(key); err != nil { + return false, err + } + fullKey := buildJSKey(agent, scope, userID, workspace, key) + + for attempt := 0; attempt < jsMaxCASAttempts; attempt++ { + entry, err := s.kv.Get(ctx, fullKey) + if err != nil { + if errors.Is(err, jetstream.ErrKeyNotFound) { + return false, nil // missing never matches a non-empty expected + } + return false, fmt.Errorf("compare-and-set get %s: %w", key, err) + } + decoded, _, expired := decodeStoredValue(string(entry.Value()), time.Now()) + if expired || decoded != expected { + return false, nil + } + encoded := []byte(encodeStoredValue(value, ttl)) + if _, uErr := s.kv.Update(ctx, fullKey, encoded, entry.Revision()); uErr != nil { + if isRevisionConflict(uErr) { + continue + } + return false, fmt.Errorf("compare-and-set update %s: %w", key, uErr) + } + return true, nil + } + return false, fmt.Errorf("compare-and-set %s: too much contention after %d attempts", key, jsMaxCASAttempts) +} + +// CompareAndDelete deletes key only if the current (non-expired) value equals +// expected. Returns true iff the delete was applied. +func (s *JetStreamKVStore) CompareAndDelete( + ctx context.Context, + agent models.Identity, + scope KVScope, + key string, + expected string, + userID string, + workspace string, +) (bool, error) { + if err := validateKey(key); err != nil { + return false, err + } + fullKey := buildJSKey(agent, scope, userID, workspace, key) + + for attempt := 0; attempt < jsMaxCASAttempts; attempt++ { + entry, err := s.kv.Get(ctx, fullKey) + if err != nil { + if errors.Is(err, jetstream.ErrKeyNotFound) { + return false, nil + } + return false, fmt.Errorf("compare-and-delete get %s: %w", key, err) + } + decoded, _, expired := decodeStoredValue(string(entry.Value()), time.Now()) + if expired || decoded != expected { + return false, nil + } + // Revision-guarded delete: fail closed (retry) if a concurrent writer + // bumped the revision between Get and Delete. + if dErr := s.kv.Delete(ctx, fullKey, jetstream.LastRevision(entry.Revision())); dErr != nil { + if isRevisionConflict(dErr) { + continue + } + return false, fmt.Errorf("compare-and-delete delete %s: %w", key, dErr) + } + return true, nil + } + return false, fmt.Errorf("compare-and-delete %s: too much contention after %d attempts", key, jsMaxCASAttempts) +} + +// SetAdd atomically adds member to the JSON-encoded set stored at key via a +// revision-guarded CAS loop. Returns whether the member was newly added (false +// if already present) and the cardinality after the add. When ttl > 0 the +// soft-TTL window is (re)set on a newly-added member; a logically-expired entry +// is treated as empty and overwritten. Duplicate adds are pure reads. +func (s *JetStreamKVStore) SetAdd( + ctx context.Context, + agent models.Identity, + scope KVScope, + key string, + member string, + userID string, + workspace string, + ttl time.Duration, +) (bool, int64, error) { + if err := validateKey(key); err != nil { + return false, 0, err + } + fullKey := buildJSKey(agent, scope, userID, workspace, key) + + for attempt := 0; attempt < jsMaxCASAttempts; attempt++ { + var rev uint64 + raw := "" + + entry, err := s.kv.Get(ctx, fullKey) + if err != nil { + if !errors.Is(err, jetstream.ErrKeyNotFound) { + return false, 0, fmt.Errorf("set-add get %s: %w", key, err) + } + // Absent → Create path (rev==0), empty set. + } else { + rev = entry.Revision() + decoded, _, expired := decodeStoredValue(string(entry.Value()), time.Now()) + if !expired { + raw = decoded + } + // Expired → treat as empty but Update over the stale revision. + } + + set := parseMemberSet(raw) + if _, ok := set[member]; ok { + return false, int64(len(set)), nil // duplicate — no write + } + set[member] = struct{}{} + card := int64(len(set)) + encoded := []byte(encodeStoredValue(encodeMemberSet(set), ttl)) + + var writeErr error + if rev == 0 { + _, writeErr = s.kv.Create(ctx, fullKey, encoded) + } else { + _, writeErr = s.kv.Update(ctx, fullKey, encoded, rev) + } + if writeErr == nil { + return true, card, nil + } + if isRevisionConflict(writeErr) { + continue + } + return false, 0, fmt.Errorf("set-add update %s: %w", key, writeErr) + } + return false, 0, fmt.Errorf("set-add %s: too much contention after %d attempts", key, jsMaxCASAttempts) +} + +// SetCard returns the cardinality of the set stored at key (0 if absent or +// logically expired). +func (s *JetStreamKVStore) SetCard( + ctx context.Context, + agent models.Identity, + scope KVScope, + key string, + userID string, + workspace string, +) (int64, error) { + if err := validateKey(key); err != nil { + return 0, err + } + fullKey := buildJSKey(agent, scope, userID, workspace, key) + + entry, err := s.kv.Get(ctx, fullKey) + if err != nil { + if errors.Is(err, jetstream.ErrKeyNotFound) { + return 0, nil + } + return 0, fmt.Errorf("set-card get %s: %w", key, err) + } + decoded, _, expired := decodeStoredValue(string(entry.Value()), time.Now()) + if expired { + return 0, nil + } + return int64(len(parseMemberSet(decoded))), nil +} + // isRevisionConflict returns true for errors that indicate a concurrent // update beat us to the revision — safe to retry. // diff --git a/server/internal/kv/jetstream_store_test.go b/server/internal/kv/jetstream_store_test.go index 3e1cadf..e2ba70a 100644 --- a/server/internal/kv/jetstream_store_test.go +++ b/server/internal/kv/jetstream_store_test.go @@ -556,3 +556,110 @@ func TestJetStreamKV_ScopeIsolation(t *testing.T) { t.Error("workspace-scope key should not appear in global-scope listing") } } + +// jsAgentA and jsAgentB are two distinct agent identities sharing the same +// tenant/workspace. Used by the cross-principal scope tests below to verify +// that the JetStream key layout honours ScopeSpec.Sharing (mirroring the +// Badger gateway integration test conventions). +func jsAgentA() models.Identity { + return models.Identity{ + Type: models.PrincipalAgent, + Workspace: "ws-test", + Implementation: "worker", + Specifier: "agent-a", + } +} + +func jsAgentB() models.Identity { + return models.Identity{ + Type: models.PrincipalAgent, + Workspace: "ws-test", + Implementation: "worker", + Specifier: "agent-b", + } +} + +// TestJetStreamKV_UserShared_CrossAgentRendezvous verifies that a write by +// agent A to ScopeUserShared is immediately readable by agent B under the +// same userID. This is the JetStream-backend regression test for the bug +// where buildJSKey was unconditionally embedding the caller's agent identity +// even for shared scopes, making USER_SHARED behave like the per-principal +// USER scope. +func TestJetStreamKV_UserShared_CrossAgentRendezvous(t *testing.T) { + s := newTestJSStore(t) + ctx := context.Background() + const userID = "alice" + + // Agent A writes. + if err := s.Set(ctx, jsAgentA(), kv.ScopeUserShared, "notebook", "page-1-content", userID, "", 0); err != nil { + t.Fatalf("agentA Set: %v", err) + } + + // Agent B reads — must see the same value. + val, err := s.Get(ctx, jsAgentB(), kv.ScopeUserShared, "notebook", userID, "") + if err != nil { + t.Fatalf("agentB Get: %v", err) + } + if val != "page-1-content" { + t.Errorf("cross-agent rendezvous failed: agentB got %q, want %q", val, "page-1-content") + } + + // List from agent B must also surface agent A's write. + items, err := s.List(ctx, jsAgentB(), kv.ScopeUserShared, userID, "") + if err != nil { + t.Fatalf("agentB List: %v", err) + } + if v, ok := items["notebook"]; !ok || v != "page-1-content" { + t.Errorf("expected agentB List to include agentA's write; got %v", items) + } +} + +// TestJetStreamKV_UserWorkspaceShared_CrossAgentRendezvous mirrors the +// USER_SHARED test but adds the workspace dimension. Agent A writes under +// (user, workspace); agent B must read the same value. +func TestJetStreamKV_UserWorkspaceShared_CrossAgentRendezvous(t *testing.T) { + s := newTestJSStore(t) + ctx := context.Background() + const userID = "carol" + const workspace = "ws-test" + + if err := s.Set(ctx, jsAgentA(), kv.ScopeUserWorkspaceShared, "state", "active", userID, workspace, 0); err != nil { + t.Fatalf("agentA Set: %v", err) + } + + val, err := s.Get(ctx, jsAgentB(), kv.ScopeUserWorkspaceShared, "state", userID, workspace) + if err != nil { + t.Fatalf("agentB Get: %v", err) + } + if val != "active" { + t.Errorf("cross-agent rendezvous failed for user-workspace-shared: got %q, want \"active\"", val) + } +} + +// TestJetStreamKV_User_IsolatesAgents proves the fix did NOT accidentally +// make the per-principal ScopeUser cross-agent. Agent A writes under user X; +// agent B reading under the same user X and key must MISS (different storage +// keys because USER is SharingExclusive). +func TestJetStreamKV_User_IsolatesAgents(t *testing.T) { + s := newTestJSStore(t) + ctx := context.Background() + const userID = "dave" + + if err := s.Set(ctx, jsAgentA(), kv.ScopeUser, "tab-state", "agent-a-value", userID, "", 0); err != nil { + t.Fatalf("agentA Set: %v", err) + } + + // Agent B reading at ScopeUser for the same userID must NOT see A's value. + if _, err := s.Get(ctx, jsAgentB(), kv.ScopeUser, "tab-state", userID, ""); err == nil { + t.Error("expected ErrKeyNotFound for agentB Get on per-principal USER scope; got nil error (isolation broken)") + } + + // Agent A still reads its own value back. + valA, err := s.Get(ctx, jsAgentA(), kv.ScopeUser, "tab-state", userID, "") + if err != nil { + t.Fatalf("agentA Get: %v", err) + } + if valA != "agent-a-value" { + t.Errorf("agentA per-principal USER readback: got %q, want %q", valA, "agent-a-value") + } +} diff --git a/server/internal/kv/namespace.go b/server/internal/kv/namespace.go index d291dcb..b808c88 100644 --- a/server/internal/kv/namespace.go +++ b/server/internal/kv/namespace.go @@ -33,6 +33,15 @@ const ( ScopeUserWorkspace KVScope = "user-workspace" ) +// ReservedCoordKeyPrefix namespaces internal coordination keys — distributed +// locks and leader-election leases used by trusted infrastructure principals +// (e.g. the WorkflowEngine). Keys under this prefix are eligible for the +// gateway's infra-coordination ACL fast-path (see KVHandler.checkKeyPermission), +// which lets infra principals coordinate over the shared KV store without a +// per-key ACL grant. Application principals (agents/tasks/services) are NOT +// auto-granted here; they remain subject to normal ACL on these keys. +const ReservedCoordKeyPrefix = "_sys/coord/" + // IdentityScope identifies the visibility tier of a KV entry. type IdentityScope int diff --git a/server/internal/kv/set_codec.go b/server/internal/kv/set_codec.go new file mode 100644 index 0000000..d45668f --- /dev/null +++ b/server/internal/kv/set_codec.go @@ -0,0 +1,43 @@ +package kv + +import ( + "encoding/json" + "sort" +) + +// Member sets on the Badger and JetStream backends have no native set type, so +// they are stored as a JSON array of unique members in a single value. Redis +// uses native SADD/SCARD instead. Keeping the on-disk form defined here ensures +// the two software-emulated backends stay byte-compatible with each other. + +// parseMemberSet decodes a stored member-set value into a set map. An empty +// string yields an empty (non-nil) set. A value that is not a JSON array is +// treated as an empty set rather than an error, so a corrupted entry self-heals +// on the next add. +func parseMemberSet(raw string) map[string]struct{} { + out := make(map[string]struct{}) + if raw == "" { + return out + } + var members []string + if err := json.Unmarshal([]byte(raw), &members); err != nil { + return out + } + for _, m := range members { + out[m] = struct{}{} + } + return out +} + +// encodeMemberSet serializes a set map to a sorted JSON array for deterministic +// storage (stable bytes ease debugging and avoid spurious JetStream revision +// churn). +func encodeMemberSet(set map[string]struct{}) string { + members := make([]string, 0, len(set)) + for m := range set { + members = append(members, m) + } + sort.Strings(members) + b, _ := json.Marshal(members) + return string(b) +} diff --git a/server/internal/kv/set_ops_js_test.go b/server/internal/kv/set_ops_js_test.go new file mode 100644 index 0000000..44f4cb4 --- /dev/null +++ b/server/internal/kv/set_ops_js_test.go @@ -0,0 +1,106 @@ +package kv_test + +import ( + "context" + "fmt" + "sync" + "sync/atomic" + "testing" + + "github.com/scitrera/aether/internal/kv" +) + +// JetStream-backed equivalents of the SetAdd/SetCard tests, exercising the +// revision-guarded CAS path against an embedded NATS server. + +func TestSetAddJS_NewDuplicateAndCard(t *testing.T) { + s := newTestJSStore(t) + ctx := context.Background() + agent := jsAgent() + + added, card, err := s.SetAdd(ctx, agent, kv.ScopeGlobal, "set", "m1", "", "", 0) + if err != nil { + t.Fatalf("SetAdd m1: %v", err) + } + if !added || card != 1 { + t.Errorf("first add: got added=%v card=%d, want true,1", added, card) + } + + added, card, err = s.SetAdd(ctx, agent, kv.ScopeGlobal, "set", "m1", "", "", 0) + if err != nil { + t.Fatalf("SetAdd dup: %v", err) + } + if added || card != 1 { + t.Errorf("dup add: got added=%v card=%d, want false,1", added, card) + } + + added, card, err = s.SetAdd(ctx, agent, kv.ScopeGlobal, "set", "m2", "", "", 0) + if err != nil { + t.Fatalf("SetAdd m2: %v", err) + } + if !added || card != 2 { + t.Errorf("second add: got added=%v card=%d, want true,2", added, card) + } + + card, err = s.SetCard(ctx, agent, kv.ScopeGlobal, "set", "", "") + if err != nil { + t.Fatalf("SetCard: %v", err) + } + if card != 2 { + t.Errorf("card=%d, want 2", card) + } + + // Absent set → 0. + card, err = s.SetCard(ctx, agent, kv.ScopeGlobal, "missing", "", "") + if err != nil { + t.Fatalf("SetCard absent: %v", err) + } + if card != 0 { + t.Errorf("absent card=%d, want 0", card) + } +} + +// TestSetAddJS_ConcurrentCompleter verifies the exactly-one-completer property +// holds through the JetStream CAS loop. N is kept modest to stay within the CAS +// retry budget under embedded-server contention. +func TestSetAddJS_ConcurrentCompleter(t *testing.T) { + s := newTestJSStore(t) + ctx := context.Background() + agent := jsAgent() + + const n = 8 + var addedCount, completers atomic.Int32 + var wg sync.WaitGroup + for i := 0; i < n; i++ { + wg.Add(1) + go func(i int) { + defer wg.Done() + added, card, err := s.SetAdd(ctx, agent, kv.ScopeGlobal, "barrier", fmt.Sprintf("m%d", i), "", "", 0) + if err != nil { + t.Errorf("SetAdd: %v", err) + return + } + if added { + addedCount.Add(1) + } + if added && card == int64(n) { + completers.Add(1) + } + }(i) + } + wg.Wait() + + if addedCount.Load() != int32(n) { + t.Errorf("addedCount=%d, want %d", addedCount.Load(), n) + } + if completers.Load() != 1 { + t.Errorf("completers=%d, want exactly 1", completers.Load()) + } + card, err := s.SetCard(ctx, agent, kv.ScopeGlobal, "barrier", "", "") + if err != nil { + t.Fatalf("SetCard: %v", err) + } + if card != int64(n) { + t.Errorf("final card=%d, want %d", card, n) + } +} diff --git a/server/internal/kv/set_ops_test.go b/server/internal/kv/set_ops_test.go new file mode 100644 index 0000000..0d855dd --- /dev/null +++ b/server/internal/kv/set_ops_test.go @@ -0,0 +1,157 @@ +package kv + +import ( + "context" + "fmt" + "sync" + "sync/atomic" + "testing" +) + +// These tests run against the Badger backend (no external infrastructure). The +// SetAdd/SetCard contract is backend-agnostic; jetstream-backed equivalents live +// in set_ops_js_test.go. + +func TestSetAdd_NewAndDuplicate(t *testing.T) { + s := newTestBadgerStore(t) + ctx := context.Background() + agent := testAgent() + + added, card, err := s.SetAdd(ctx, agent, ScopeGlobal, "set", "m1", "", "", 0) + if err != nil { + t.Fatalf("SetAdd m1: %v", err) + } + if !added || card != 1 { + t.Errorf("first add: got added=%v card=%d, want true,1", added, card) + } + + // Duplicate member: not added, cardinality unchanged. + added, card, err = s.SetAdd(ctx, agent, ScopeGlobal, "set", "m1", "", "", 0) + if err != nil { + t.Fatalf("SetAdd dup: %v", err) + } + if added || card != 1 { + t.Errorf("dup add: got added=%v card=%d, want false,1", added, card) + } + + // Second distinct member. + added, card, err = s.SetAdd(ctx, agent, ScopeGlobal, "set", "m2", "", "", 0) + if err != nil { + t.Fatalf("SetAdd m2: %v", err) + } + if !added || card != 2 { + t.Errorf("second add: got added=%v card=%d, want true,2", added, card) + } +} + +func TestSetCard_AbsentAndPresent(t *testing.T) { + s := newTestBadgerStore(t) + ctx := context.Background() + agent := testAgent() + + card, err := s.SetCard(ctx, agent, ScopeGlobal, "missing", "", "") + if err != nil { + t.Fatalf("SetCard absent: %v", err) + } + if card != 0 { + t.Errorf("absent card=%d, want 0", card) + } + + for i := 0; i < 3; i++ { + if _, _, err := s.SetAdd(ctx, agent, ScopeGlobal, "set", fmt.Sprintf("m%d", i), "", "", 0); err != nil { + t.Fatalf("seed %d: %v", i, err) + } + } + card, err = s.SetCard(ctx, agent, ScopeGlobal, "set", "", "") + if err != nil { + t.Fatalf("SetCard present: %v", err) + } + if card != 3 { + t.Errorf("card=%d, want 3", card) + } +} + +// TestSetAdd_ExactlyOneCompleter is the fan-in correctness property: with N +// distinct members added concurrently, every member is added exactly once and +// exactly ONE caller observes the add that brought cardinality to N. That unique +// caller is the join's single firer. +func TestSetAdd_ExactlyOneCompleter(t *testing.T) { + s := newTestBadgerStore(t) + ctx := context.Background() + agent := testAgent() + + const n = 50 + var addedCount, completers atomic.Int32 + var wg sync.WaitGroup + for i := 0; i < n; i++ { + wg.Add(1) + go func(i int) { + defer wg.Done() + added, card, err := s.SetAdd(ctx, agent, ScopeGlobal, "barrier", fmt.Sprintf("m%d", i), "", "", 0) + if err != nil { + t.Errorf("SetAdd: %v", err) + return + } + if added { + addedCount.Add(1) + } + if added && card == int64(n) { + completers.Add(1) + } + }(i) + } + wg.Wait() + + if addedCount.Load() != int32(n) { + t.Errorf("addedCount=%d, want %d", addedCount.Load(), n) + } + if completers.Load() != 1 { + t.Errorf("completers=%d, want exactly 1", completers.Load()) + } + card, err := s.SetCard(ctx, agent, ScopeGlobal, "barrier", "", "") + if err != nil { + t.Fatalf("SetCard: %v", err) + } + if card != int64(n) { + t.Errorf("final card=%d, want %d", card, n) + } +} + +// TestSetAdd_DuplicateRace: many concurrent adds of the SAME member yield +// exactly one added==true and a final cardinality of 1 (at-most-once dedup +// ledger semantics). +func TestSetAdd_DuplicateRace(t *testing.T) { + s := newTestBadgerStore(t) + ctx := context.Background() + agent := testAgent() + + const g = 50 + var added atomic.Int32 + var wg sync.WaitGroup + for i := 0; i < g; i++ { + wg.Add(1) + go func() { + defer wg.Done() + ok, _, err := s.SetAdd(ctx, agent, ScopeGlobal, "dedup", "same", "", "", 0) + if err != nil { + t.Errorf("SetAdd: %v", err) + return + } + if ok { + added.Add(1) + } + }() + } + wg.Wait() + + if added.Load() != 1 { + t.Errorf("added=%d, want exactly 1", added.Load()) + } + card, err := s.SetCard(ctx, agent, ScopeGlobal, "dedup", "", "") + if err != nil { + t.Fatalf("SetCard: %v", err) + } + if card != 1 { + t.Errorf("card=%d, want 1", card) + } +} diff --git a/server/internal/kv/store.go b/server/internal/kv/store.go index 885d1eb..e9d526d 100644 --- a/server/internal/kv/store.go +++ b/server/internal/kv/store.go @@ -224,6 +224,14 @@ type ListOptions struct { // Limit is the maximum number of keys to return in a single call. // If <= 0, DefaultListLimit is used. Limit int + // KeyPrefix, when non-empty, restricts results to keys whose user-facing + // key begins with this prefix. Crucially the prefix is applied BEFORE the + // Limit cap (server-side where the backend supports it), so the cap bounds + // the number of MATCHING keys rather than all keys in the scope. Without + // this, a caller listing a small prefixed subset of a large shared scope + // (e.g. "sandbox:" within the _sandbox workspace) could have its matches + // fall outside the first page of a scope-wide scan and vanish. + KeyPrefix string } // ListResult is the return type for ListPaginated. @@ -233,6 +241,24 @@ type ListResult struct { HasMore bool } +// escapeRedisGlob escapes the glob metacharacters Redis SCAN MATCH recognizes +// (* ? [ ] \) so a key prefix is matched literally. +func escapeRedisGlob(s string) string { + if s == "" { + return "" + } + var b strings.Builder + b.Grow(len(s) + 4) + for _, r := range s { + switch r { + case '*', '?', '[', ']', '\\': + b.WriteByte('\\') + } + b.WriteRune(r) + } + return b.String() +} + // ListPaginated returns up to opts.Limit keys in a namespace with their values, // using Redis SCAN cursors for safe iteration over large key-spaces. // Use the returned NextCursor in a subsequent call to page through all results. @@ -262,8 +288,16 @@ func (s *Store) ListPaginated( } namespace := BuildNamespace(agent, scope, userID, workspace) - pattern := fmt.Sprintf("%s:*", namespace) prefix := namespace + ":" + // Apply the caller's key prefix in the SCAN MATCH pattern so the server + // only returns matching keys — the Limit then bounds matches, not the whole + // scope. Glob metacharacters in the prefix are escaped so a literal prefix + // like "sandbox:" matches literally. + keyPrefix := "" + if opts != nil { + keyPrefix = opts.KeyPrefix + } + pattern := fmt.Sprintf("%s:%s*", namespace, escapeRedisGlob(keyPrefix)) var collected []string cursor := startCursor @@ -637,6 +671,215 @@ func (s *Store) runGuardedCounter(ctx context.Context, script *redis.Script, ful return value, applied == 1, nil } +// compareAndSetLuaScript sets KEYS[1]=ARGV[2] only when the current value +// equals ARGV[1]. ARGV[3] is the TTL in milliseconds (<=0 means no expiry). +// Returns 1 when the swap was applied, 0 otherwise. +var compareAndSetLuaScript = redis.NewScript(` +if redis.call('GET', KEYS[1]) == ARGV[1] then + if tonumber(ARGV[3]) > 0 then + redis.call('SET', KEYS[1], ARGV[2], 'PX', ARGV[3]) + else + redis.call('SET', KEYS[1], ARGV[2]) + end + return 1 +end +return 0 +`) + +// compareAndDeleteLuaScript deletes KEYS[1] only when the current value equals +// ARGV[1]. Returns 1 when the delete was applied, 0 otherwise. +var compareAndDeleteLuaScript = redis.NewScript(` +if redis.call('GET', KEYS[1]) == ARGV[1] then + return redis.call('DEL', KEYS[1]) +end +return 0 +`) + +// SetNX sets key=value only if the key is absent. Returns true iff written. +// +// Coordination ops (SetNX/CompareAndSet/CompareAndDelete) intentionally do NOT +// apply the "enc:" at-rest encryption transform: the encryption nonce makes +// ciphertext non-deterministic, which would break the value comparison the +// compare-and-* ops rely on. Coordination keys carry opaque owner tokens, not +// user secrets, so storing them in the clear is appropriate. +func (s *Store) SetNX( + ctx context.Context, + agent models.Identity, + scope KVScope, + key string, + value string, + userID string, + workspace string, + ttl time.Duration, +) (bool, error) { + if err := validateKey(key); err != nil { + return false, err + } + namespace := BuildNamespace(agent, scope, userID, workspace) + fullKey := fmt.Sprintf("%s:%s", namespace, key) + + effectiveTTL := ttl + if effectiveTTL <= 0 && s.defaultTTL > 0 { + effectiveTTL = s.defaultTTL + } + + var ok bool + if err := s.execCB(func() error { + var setErr error + ok, setErr = s.client.SetNX(ctx, fullKey, value, effectiveTTL).Result() + return setErr + }); err != nil { + return false, fmt.Errorf("failed to setnx key %s: %w", key, err) + } + return ok, nil +} + +// CompareAndSet sets key=value only if the current value equals expected. +// Returns true iff the swap was applied. +func (s *Store) CompareAndSet( + ctx context.Context, + agent models.Identity, + scope KVScope, + key string, + expected string, + value string, + userID string, + workspace string, + ttl time.Duration, +) (bool, error) { + if err := validateKey(key); err != nil { + return false, err + } + namespace := BuildNamespace(agent, scope, userID, workspace) + fullKey := fmt.Sprintf("%s:%s", namespace, key) + + effectiveTTL := ttl + if effectiveTTL <= 0 && s.defaultTTL > 0 { + effectiveTTL = s.defaultTTL + } + + var raw interface{} + if err := s.execCB(func() error { + var runErr error + raw, runErr = compareAndSetLuaScript.Run(ctx, s.client, []string{fullKey}, expected, value, effectiveTTL.Milliseconds()).Result() + return runErr + }); err != nil { + return false, fmt.Errorf("failed to compare-and-set key %s: %w", key, err) + } + applied, _ := raw.(int64) + return applied == 1, nil +} + +// CompareAndDelete deletes key only if the current value equals expected. +// Returns true iff the delete was applied. +func (s *Store) CompareAndDelete( + ctx context.Context, + agent models.Identity, + scope KVScope, + key string, + expected string, + userID string, + workspace string, +) (bool, error) { + if err := validateKey(key); err != nil { + return false, err + } + namespace := BuildNamespace(agent, scope, userID, workspace) + fullKey := fmt.Sprintf("%s:%s", namespace, key) + + var raw interface{} + if err := s.execCB(func() error { + var runErr error + raw, runErr = compareAndDeleteLuaScript.Run(ctx, s.client, []string{fullKey}, expected).Result() + return runErr + }); err != nil { + return false, fmt.Errorf("failed to compare-and-delete key %s: %w", key, err) + } + applied, _ := raw.(int64) + return applied == 1, nil +} + +// setAddLuaScript atomically adds ARGV[1] to the set at KEYS[1], (re)setting the +// key's TTL to ARGV[2] milliseconds when the member is newly added and ARGV[2] > 0, +// and returns {addedFlag, cardinality}. addedFlag is 1 when the member was newly +// added, 0 when it was already present. TTL is refreshed only on a real write so +// duplicate adds are pure reads (matching the Badger/JetStream backends). +var setAddLuaScript = redis.NewScript(` +local added = redis.call('SADD', KEYS[1], ARGV[1]) +if added == 1 and tonumber(ARGV[2]) > 0 then + redis.call('PEXPIRE', KEYS[1], ARGV[2]) +end +local card = redis.call('SCARD', KEYS[1]) +return {added, card} +`) + +// SetAdd atomically adds member to the set stored at key and returns whether the +// member was newly added (false if already present) plus the set's cardinality +// after the add. See KVReadWriter.SetAdd for the firing/dedup semantics. +// +// Set values are stored in the clear (no "enc:" at-rest transform): members are +// opaque correlation/dedup tokens, not user secrets, and SADD over +// non-deterministic ciphertext would break membership semantics — the same +// rationale as the SetNX/CompareAndSet coordination ops above. +func (s *Store) SetAdd( + ctx context.Context, + agent models.Identity, + scope KVScope, + key string, + member string, + userID string, + workspace string, + ttl time.Duration, +) (bool, int64, error) { + if err := validateKey(key); err != nil { + return false, 0, err + } + namespace := BuildNamespace(agent, scope, userID, workspace) + fullKey := fmt.Sprintf("%s:%s", namespace, key) + + var raw interface{} + if err := s.execCB(func() error { + var runErr error + raw, runErr = setAddLuaScript.Run(ctx, s.client, []string{fullKey}, member, ttl.Milliseconds()).Result() + return runErr + }); err != nil { + return false, 0, fmt.Errorf("set-add on %s failed: %w", key, err) + } + arr, ok := raw.([]interface{}) + if !ok || len(arr) != 2 { + return false, 0, fmt.Errorf("unexpected set-add result shape: %T", raw) + } + added, _ := arr[0].(int64) + card, _ := arr[1].(int64) + return added == 1, card, nil +} + +// SetCard returns the cardinality of the set stored at key (0 if absent). +func (s *Store) SetCard( + ctx context.Context, + agent models.Identity, + scope KVScope, + key string, + userID string, + workspace string, +) (int64, error) { + if err := validateKey(key); err != nil { + return 0, err + } + namespace := BuildNamespace(agent, scope, userID, workspace) + fullKey := fmt.Sprintf("%s:%s", namespace, key) + + var card int64 + if err := s.execCB(func() error { + var cErr error + card, cErr = s.client.SCard(ctx, fullKey).Result() + return cErr + }); err != nil { + return 0, fmt.Errorf("set-card on %s failed: %w", key, err) + } + return card, nil +} + // GetJSON retrieves and unmarshals a JSON value func (s *Store) GetJSON( ctx context.Context, diff --git a/server/internal/orchestration/jetstream_dispatcher_test.go b/server/internal/orchestration/jetstream_dispatcher_test.go index b2e8e1a..0347812 100644 --- a/server/internal/orchestration/jetstream_dispatcher_test.go +++ b/server/internal/orchestration/jetstream_dispatcher_test.go @@ -188,10 +188,13 @@ func (f *fakeTaskStore) CompleteQueueEntryByTaskID(ctx context.Context, taskID s func (f *fakeTaskStore) FailQueueEntryByTaskID(ctx context.Context, taskID, errorMsg string) error { return nil } +func (f *fakeTaskStore) ReconcileOrphanedQueueEntries(ctx context.Context) (int64, error) { + return 0, nil +} // --- Unused methods (panic to surface unexpected calls) --- -func (f *fakeTaskStore) InsertQueueEntry(_ context.Context, _, _, _, _, _ string, _ []byte) error { +func (f *fakeTaskStore) InsertQueueEntry(_ context.Context, _, _, _, _, _ string, _ []byte, _ int) error { panic("InsertQueueEntry unexpected in jetstream dispatcher tests") } func (f *fakeTaskStore) PollPendingQueueEntries(_ context.Context, _ int) ([]*taskstore.QueueEntryNotification, error) { @@ -216,6 +219,7 @@ func (f *fakeTaskStore) StartTask(_ context.Context, _ string) error { panic func (f *fakeTaskStore) StartTaskWithAgent(_ context.Context, _, _ string) error { panic("unexpected") } +func (f *fakeTaskStore) ClaimTask(_ context.Context, _ string) error { panic("unexpected") } func (f *fakeTaskStore) CompleteTask(_ context.Context, _ string) error { panic("unexpected") } func (f *fakeTaskStore) FailTask(_ context.Context, _, _ string) error { panic("unexpected") } func (f *fakeTaskStore) FailTaskWithRetry(_ context.Context, _, _, _ string, _ *time.Time) error { diff --git a/server/internal/orchestration/notify_dispatcher_test.go b/server/internal/orchestration/notify_dispatcher_test.go index 7afb3fb..dfcb4f8 100644 --- a/server/internal/orchestration/notify_dispatcher_test.go +++ b/server/internal/orchestration/notify_dispatcher_test.go @@ -656,3 +656,73 @@ func TestRecoverStaleClaims(t *testing.T) { } }) } + +// TestPollPendingQueueEntries_PriorityOrdering verifies that the orchestrated +// task queue is polled highest-priority-first, FIFO within a level. It inserts +// entries whose creation order is the inverse of their priority order, so a +// FIFO-only poll would return them in the wrong order. +func TestPollPendingQueueEntries_PriorityOrdering(t *testing.T) { + testDB, cleanup := testutil.SetupTestDB(t) + if testDB == nil { + return // Skip was called in SetupTestDB + } + defer testDB.Close() + defer cleanup() + + ctx := context.Background() + store := taskpg.New(testDB.DB) + + // Insert three pending entries. created_at ascending = normal, high, preempt + // so a FIFO poll would yield [normal, high, preempt]; priority ordering must + // instead yield [preempt(50), high(40), normal(30)]. + type seed struct { + label string + priority int + created string // explicit timestamp to control FIFO order + } + seeds := []seed{ + {"normal", int(tasks.PriorityNormal), "2026-01-01T00:00:01Z"}, + {"high", int(tasks.PriorityHigh), "2026-01-01T00:00:02Z"}, + {"preempt", int(tasks.PriorityPreempt), "2026-01-01T00:00:03Z"}, + } + byTaskID := map[string]string{} + for _, s := range seeds { + taskID := uuid.New().String() + queueID := uuid.New().String() + byTaskID[taskID] = s.label + if _, err := testDB.DB.ExecContext(ctx, ` + INSERT INTO tasks (task_id, task_type, workspace, implementation, status, priority) + VALUES ($1, 'agent_startup', 'test-workspace', 'test-impl', 'pending', $2) + `, taskID, s.priority); err != nil { + t.Fatalf("insert task %s: %v", s.label, err) + } + if _, err := testDB.DB.ExecContext(ctx, ` + INSERT INTO orchestrated_task_queue + (queue_id, task_id, target_implementation, workspace, profile, status, priority, created_at) + VALUES ($1, $2, 'test-impl', 'test-workspace', 'kubernetes', 'pending', $3, $4) + `, queueID, taskID, s.priority, s.created); err != nil { + t.Fatalf("insert queue entry %s: %v", s.label, err) + } + } + + entries, err := store.PollPendingQueueEntries(ctx, 10) + if err != nil { + t.Fatalf("PollPendingQueueEntries: %v", err) + } + + var got []string + for _, e := range entries { + if label, ok := byTaskID[e.TaskID]; ok { + got = append(got, label) + } + } + want := []string{"preempt", "high", "normal"} + if len(got) != len(want) { + t.Fatalf("expected %d ordered entries, got %d (%v)", len(want), len(got), got) + } + for i := range want { + if got[i] != want[i] { + t.Errorf("poll order[%d] = %q, want %q (full order: %v)", i, got[i], want[i], got) + } + } +} diff --git a/server/internal/orchestration/task_assignment.go b/server/internal/orchestration/task_assignment.go index 0242361..3d1dd1f 100644 --- a/server/internal/orchestration/task_assignment.go +++ b/server/internal/orchestration/task_assignment.go @@ -65,6 +65,11 @@ type TaskAssignmentService struct { // (tk::{workspace}::{task_id}::events). Phase 4 Stage B. Nil = disabled, every // publish call becomes a no-op. Injected via SetEventPublisher. eventPub TaskEventPublisher + // domainEventPub publishes "feed B" domain events (raw EventPayload JSON + // bytes) onto the event plane (event::*) when a task with a completion_event + // config reaches a selected terminal status. Nil = disabled. Injected via + // SetDomainEventPublisher. + domainEventPub DomainEventPublisher } // queueRetirementDispatcher is the narrow interface TaskAssignmentService needs @@ -170,6 +175,28 @@ type CreateTaskRequest struct { // route responses back to the originating user. Leave zero-value for internal/ // service-initiated tasks that have no OBO subject. SubjectIdentity models.Identity + + // RetryPolicy, when non-nil, is persisted on the created task and + // honored by the store's FailTask: the store computes next_retry_at + // from this policy and re-pends the task automatically (up to + // MaxAttempts). Absent = legacy behavior (immediate re-pend, + // hardcoded max_retries=3). + RetryPolicy *tasks.RetryPolicy + + // Priority is the dispatch-priority weight (mirrors proto TaskPriority; + // 0 = UNSPECIFIED, normalized to NORMAL by the store on create). Higher + // priority pending tasks are delivered before lower ones. + Priority int32 + + // CorrelationID is the fan-out/fan-in correlation identity (the barrier/group + // id a workflow join matches against), distinct from the task id. + CorrelationID string + // RootTaskID is the flow-root task id. Children carry it from their spawner; + // a task with no provided root becomes its own flow root (set in the handlers). + RootTaskID string + // CompletionEvent, when non-nil, opts the task into "feed B": the server emits + // a domain event onto event::* when the task reaches a selected terminal status. + CompletionEvent *tasks.TaskCompletionConfig } // principalTypeStringForTask maps a models.PrincipalType to the lowercase @@ -225,6 +252,34 @@ func applySubjectIdentityToAuthority(task *tasks.ExtendedTask, subject models.Id } } +// applyRetryPolicyToTask reconciles task.MaxRetries with the attached +// RetryPolicy. When a policy is present we prefer its MaxAttempts so the +// store's legacy "retry_count < max_retries" guards stay consistent with +// the policy-driven scheduling. EffectiveMaxAttempts handles the +// "0 = server default" rule. +func applyRetryPolicyToTask(task *tasks.ExtendedTask) { + if task == nil || task.RetryPolicy == nil { + return + } + task.MaxRetries = int(task.RetryPolicy.EffectiveMaxAttempts()) +} + +// applyCorrelationToTask copies the correlation/feed-B fields from the request +// onto the task and resolves root_task_id propagation: children carry the root +// from their spawner, while a task created with no provided root becomes its own +// flow root. Must run after task.TaskID is set. +func applyCorrelationToTask(task *tasks.ExtendedTask, req *CreateTaskRequest) { + if task == nil || req == nil { + return + } + task.CorrelationID = req.CorrelationID + task.CompletionEvent = req.CompletionEvent + task.RootTaskID = req.RootTaskID + if task.RootTaskID == "" { + task.RootTaskID = task.TaskID // a task with no provided root is its own flow root + } +} + // CreateTaskResponse represents the result of task creation type CreateTaskResponse struct { TaskID string @@ -265,6 +320,7 @@ func (tas *TaskAssignmentService) handleSelfAssign(ctx context.Context, req *Cre TaskClass: req.TaskClass, GraceWindowMs: DefaultGraceWindowMs(req.TaskClass), Workspace: req.Workspace, + Priority: int(req.Priority), AssignmentMode: tasks.AssignmentModeSelfAssign, TaskCategory: tasks.TaskCategoryRegular, Status: tasks.TaskStatusPending, @@ -273,8 +329,11 @@ func (tas *TaskAssignmentService) handleSelfAssign(ctx context.Context, req *Cre Metadata: req.Metadata, Payload: req.Payload, MaxRetries: 3, + RetryPolicy: req.RetryPolicy, } applySubjectIdentityToAuthority(task, req.SubjectIdentity) + applyRetryPolicyToTask(task) + applyCorrelationToTask(task, req) // Create task in database as pending if err := tas.taskStore.CreateTask(ctx, task); err != nil { @@ -327,6 +386,7 @@ func (tas *TaskAssignmentService) handleTargeted(ctx context.Context, req *Creat TaskClass: req.TaskClass, GraceWindowMs: DefaultGraceWindowMs(req.TaskClass), Workspace: req.Workspace, + Priority: int(req.Priority), AssignmentMode: tasks.AssignmentModeTargeted, TaskCategory: tasks.TaskCategoryRegular, TargetAgentID: req.TargetAgentID, @@ -336,8 +396,11 @@ func (tas *TaskAssignmentService) handleTargeted(ctx context.Context, req *Creat Metadata: req.Metadata, Payload: req.Payload, MaxRetries: 3, + RetryPolicy: req.RetryPolicy, } applySubjectIdentityToAuthority(task, req.SubjectIdentity) + applyRetryPolicyToTask(task) + applyCorrelationToTask(task, req) // Special case: if this IS a startup task (e.g., from admin API), go directly to // createOrchestratedStartupTask which handles all duplicate prevention: @@ -534,7 +597,7 @@ func (tas *TaskAssignmentService) createOrchestratedStartupTask( return "", fmt.Errorf("failed to marshal launch params: %w", err) } - if err := tas.taskStore.InsertQueueEntry(ctx, queueID, startupTaskID, targetIdentity.Implementation, workspace, profile, launchParamsJSON); err != nil { + if err := tas.taskStore.InsertQueueEntry(ctx, queueID, startupTaskID, targetIdentity.Implementation, workspace, profile, launchParamsJSON, task.Priority); err != nil { return "", fmt.Errorf("failed to insert orchestrated task into queue: %w", err) } @@ -628,6 +691,7 @@ func (tas *TaskAssignmentService) handlePool(ctx context.Context, req *CreateTas TaskClass: req.TaskClass, GraceWindowMs: DefaultGraceWindowMs(req.TaskClass), Workspace: req.Workspace, + Priority: int(req.Priority), AssignmentMode: tasks.AssignmentModePool, TaskCategory: tasks.TaskCategoryRegular, TargetImplementation: req.TargetImplementation, @@ -638,8 +702,11 @@ func (tas *TaskAssignmentService) handlePool(ctx context.Context, req *CreateTas Metadata: req.Metadata, Payload: req.Payload, MaxRetries: 3, + RetryPolicy: req.RetryPolicy, } applySubjectIdentityToAuthority(task, req.SubjectIdentity) + applyRetryPolicyToTask(task) + applyCorrelationToTask(task, req) if err := tas.taskStore.CreateTask(ctx, task); err != nil { return nil, fmt.Errorf("failed to create pool task: %w", err) @@ -654,7 +721,18 @@ func (tas *TaskAssignmentService) handlePool(ctx context.Context, req *CreateTas }, nil } -// DeliverPoolTasks claims and returns pending pool tasks for a connecting agent. +// DeliverPoolTasks claims and returns pending pool tasks for a connecting +// worker (agent or service). +// +// Workspace handling: agents register with a concrete workspace and only +// see their own. Services have an empty workspace (system principals) and +// intentionally see pool tasks across all workspaces for their +// implementation — `GetPendingPoolTasks` treats an empty workspace as +// "no workspace filter" via `ListTasks`. This matches the cross-workspace +// design of `PrincipalService` (proxy-sidecar, webhookservice, etc.): +// services connect once and serve every workspace in the deployment. The +// per-workspace creator already passed CreateTask ACL on each pending row, +// so no further ACL check is layered here. func (tas *TaskAssignmentService) DeliverPoolTasks(ctx context.Context, agentIdentity models.Identity) ([]*tasks.ExtendedTask, error) { pendingTasks, err := tas.taskStore.GetPendingPoolTasks(ctx, agentIdentity.Implementation, agentIdentity.Workspace) if err != nil { @@ -753,6 +831,34 @@ func DefaultGraceWindowMs(class int32) int64 { } } +// retireQueueRowFailedDirect retires (status='failed') the SQL +// orchestrated_task_queue row(s) for a task directly via the store, independent +// of whether tas.dispatcher is wired. This is the reliable retire path that +// prevents orphaned pending/claimed queue rows when the TaskAssignmentService +// driving a terminal transition has no dispatcher (e.g. the cleanup service). +// Idempotent and a no-op when no SQL row exists (clustered/JetStream mode uses a +// NATS WorkQueue, not this table). Non-fatal: the reconcile sweep is the backstop. +func (tas *TaskAssignmentService) retireQueueRowFailedDirect(ctx context.Context, taskID, reason string) { + if tas.taskStore == nil { + return + } + if err := tas.taskStore.FailQueueEntryByTaskID(ctx, taskID, reason); err != nil { + logging.Logger.Warn().Err(err).Str("task_id", taskID).Msg("failed to retire orchestrated_task_queue row directly (non-fatal)") + } +} + +// retireQueueRowCompletedDirect is retireQueueRowFailedDirect's success twin: +// it retires the queue row(s) as 'completed'. Same dispatcher-independent, +// idempotent, no-op-in-JetStream semantics. +func (tas *TaskAssignmentService) retireQueueRowCompletedDirect(ctx context.Context, taskID string) { + if tas.taskStore == nil { + return + } + if err := tas.taskStore.CompleteQueueEntryByTaskID(ctx, taskID); err != nil { + logging.Logger.Warn().Err(err).Str("task_id", taskID).Msg("failed to retire orchestrated_task_queue row directly (non-fatal)") + } +} + // CompleteTask marks a task as completed and revokes associated tokens func (tas *TaskAssignmentService) CompleteTask(ctx context.Context, taskID string) error { // Phase 4 Stage B: snapshot pre-transition state so publishStatusChange @@ -773,6 +879,13 @@ func (tas *TaskAssignmentService) CompleteTask(ctx context.Context, taskID strin if err := tas.taskStore.CompleteTask(ctx, taskID); err != nil { return err } + // Retire the SQL orchestrated_task_queue row directly via the store so it no + // longer depends on tas.dispatcher being wired — the cleanup service's + // TaskAssignmentService instance has no dispatcher, which is exactly how + // terminal tasks left pending queue rows to be polled forever. The dispatcher + // block below is now redundant (it calls the same idempotent store method) + // but is retained for backward compatibility. + tas.retireQueueRowCompletedDirect(ctx, taskID) if tas.dispatcher != nil { if err := tas.dispatcher.CompleteTaskByTaskID(ctx, taskID); err != nil { logging.Logger.Warn().Err(err).Str("task_id", taskID).Msg("failed to retire orchestrated_task_queue row on task complete (non-fatal)") @@ -800,6 +913,10 @@ func (tas *TaskAssignmentService) FailTask(ctx context.Context, taskID, errorMsg if err := tas.taskStore.FailTask(ctx, taskID, errorMsg); err != nil { return err } + // Retire the SQL orchestrated_task_queue row directly (dispatcher-independent). + // See CompleteTask for the full rationale. The dispatcher block below is + // redundant but retained for backward compatibility. + tas.retireQueueRowFailedDirect(ctx, taskID, errorMsg) if tas.dispatcher != nil { if err := tas.dispatcher.FailTaskByTaskID(ctx, taskID, errorMsg); err != nil { logging.Logger.Warn().Err(err).Str("task_id", taskID).Msg("failed to retire orchestrated_task_queue row on task fail (non-fatal)") @@ -825,6 +942,13 @@ func (tas *TaskAssignmentService) CancelTask(ctx context.Context, taskID string) if err := tas.taskStore.CancelTask(ctx, taskID); err != nil { return err } + // Retire the SQL orchestrated_task_queue row directly (dispatcher-independent). + // This is the root-cause fix for orphaned pending queue rows: the cleanup + // service cancels tasks through a TaskAssignmentService whose dispatcher is + // nil, so the dispatcher-gated retire below never ran and the row was polled + // forever. See CompleteTask for the full rationale. The dispatcher block is + // redundant but retained for backward compatibility. + tas.retireQueueRowFailedDirect(ctx, taskID, "task cancelled") if tas.dispatcher != nil { if err := tas.dispatcher.FailTaskByTaskID(ctx, taskID, "task cancelled"); err != nil { logging.Logger.Warn().Err(err).Str("task_id", taskID).Msg("failed to retire orchestrated_task_queue row on task cancel (non-fatal)") @@ -874,6 +998,23 @@ func (tas *TaskAssignmentService) ResumeTask(ctx context.Context, taskID string, return nil } +// ClaimTask transitions a task into running when claimed by its assignee +// (e.g. a per-turn chat_message task claimed straight out of the queue). +// Side effects: log only — tokens and grants are retained for the run. +// Idempotent: re-claiming a running task is a no-op on started_at. +func (tas *TaskAssignmentService) ClaimTask(ctx context.Context, taskID string) error { + pre := tas.loadTransitionMetadata(ctx, taskID) + if err := tas.taskStore.ClaimTask(ctx, taskID); err != nil { + return err + } + logging.Logger.Info(). + Str("task_id", taskID). + Str("to_status", string(tasks.TaskStatusRunning)). + Msg("task claimed") + tas.emitTransitionEvent(ctx, pre, taskID, tasks.TaskStatusRunning, "") + return nil +} + // WakeHibernatedTask transitions a HIBERNATED task back to pending and // reinserts it into the orchestrated_task_queue so the orchestrator can spawn // a fresh worker for it. Before clearing the WaitSpec, the task's @@ -968,7 +1109,7 @@ func (tas *TaskAssignmentService) WakeHibernatedTask(ctx context.Context, taskID if err != nil { return fmt.Errorf("WakeHibernatedTask: marshal launch params for %s: %w", taskID, err) } - if err := tas.taskStore.InsertQueueEntry(ctx, queueID, taskID, task.TargetImplementation, task.Workspace, profile, launchParamsJSON); err != nil { + if err := tas.taskStore.InsertQueueEntry(ctx, queueID, taskID, task.TargetImplementation, task.Workspace, profile, launchParamsJSON, task.Priority); err != nil { return fmt.Errorf("WakeHibernatedTask: insert queue entry for %s: %w", taskID, err) } @@ -1030,6 +1171,10 @@ func (tas *TaskAssignmentService) RejectTask(ctx context.Context, taskID, reason if err := tas.taskStore.RejectTask(ctx, taskID, reason); err != nil { return err } + // Retire the SQL orchestrated_task_queue row directly (dispatcher-independent). + // See CompleteTask for the full rationale. The dispatcher block below is + // redundant but retained for backward compatibility. + tas.retireQueueRowFailedDirect(ctx, taskID, reason) if tas.dispatcher != nil { if err := tas.dispatcher.FailTaskByTaskID(ctx, taskID, reason); err != nil { logging.Logger.Warn().Err(err).Str("task_id", taskID).Msg("failed to retire orchestrated_task_queue row on task reject (non-fatal)") @@ -1351,3 +1496,172 @@ func (tas *TaskAssignmentService) ReconcileOrphanedTasks(ctx context.Context) (i return reconciled, nil } + +// taskClassInteractive mirrors the proto TaskClass enum value for INTERACTIVE +// (see DefaultGraceWindowMs). Chat-message turns are minted with this class. +const taskClassInteractive int32 = 1 + +// staleInteractiveActiveStatuses is the pre-run + running set of statuses an +// INTERACTIVE task moves through before reaching a terminal state. It +// deliberately EXCLUDES the waiting_*/hibernated paused states so a turn that +// is legitimately awaiting user input, an authority grant, an upstream +// dependency, or a scheduled wake is never cancelled by the TTL sweep. +var staleInteractiveActiveStatuses = []tasks.TaskStatus{ + tasks.TaskStatusPending, + tasks.TaskStatusAssigned, + tasks.TaskStatusStarting, + tasks.TaskStatusRunning, +} + +// CancelStaleInteractiveTasks cancels INTERACTIVE (chat_message turn) tasks that +// have been sitting in a non-terminal, non-waiting state longer than ttl. These +// are foreground turns that should complete in minutes; when the target sandbox +// was offline/dead at mint or the harness crashed, they never reach a terminal +// state and linger forever (12hr-old QUEUED chat tasks were observed). Unlike +// ReconcileOrphanedTasks, this does not probe session liveness — an INTERACTIVE +// turn older than the TTL is dead by definition. +// +// Cancellation goes through CancelTask so the queue row is retired and the +// task's authority grant is revoked (never a raw status write). Errors are +// best-effort (logged and skipped) so one bad task does not abort the sweep. +// Returns the number of tasks cancelled. +func (tas *TaskAssignmentService) CancelStaleInteractiveTasks(ctx context.Context, ttl time.Duration) (int, error) { + if ttl <= 0 { + return 0, nil + } + + taskList, err := tas.taskStore.ListTasks(ctx, &tasks.TaskFilter{ + TaskClass: taskClassInteractive, + Statuses: staleInteractiveActiveStatuses, + Limit: 1000, + }) + if err != nil { + return 0, fmt.Errorf("failed to list interactive tasks: %w", err) + } + + cutoff := time.Now().Add(-ttl) + cancelled := 0 + for _, task := range taskList { + if !task.CreatedAt.Before(cutoff) { + continue + } + if err := tas.CancelTask(ctx, task.TaskID); err != nil { + logging.Logger.Error().Err(err).Str("task_id", task.TaskID).Msg("stale interactive cancel: failed to cancel task") + continue + } + logging.Logger.Info().Str("task_id", task.TaskID).Time("created_at", task.CreatedAt).Msg("stale interactive cancel: cancelled stale interactive task") + cancelled++ + } + return cancelled, nil +} + +// startupTaskType is the task_type of the pool-dispatched orchestration task +// minted by triggerOrchestration to spin up an offline agent (see +// HasActiveStartupTask / createOrchestratedStartupTask). +const startupTaskType = "agent_startup" + +// CancelStaleStartupTasks cancels agent_startup orchestration tasks that have +// sat UNCLAIMED (pending) longer than ttl. These are the pool tasks +// triggerOrchestration mints when a message routes to an OFFLINE agent +// (e.g. an offline sahara sandbox now registered as ag::_sandbox::sahara::). +// Registering the sandbox in the agent registry makes it technically +// orchestratable, so an offline route creates a pending agent_startup task — +// but with no sandbox orchestrator claiming them (orchestrator=0) and no +// existing sweep collecting them (they are BACKGROUND/orchestrated, and +// PurgeTasks only reaps terminal tasks), they linger forever. +// +// Cancelling an unclaimed startup task is self-healing: the next message to the +// same offline target re-triggers a fresh startup task (HasActiveStartupTask +// sees none once this one is cancelled), so nothing is permanently lost. The +// TTL is deliberately GENEROUS: a legitimately-orchestratable agent whose +// orchestrator is briefly unavailable must never have its pending startup task +// cancelled out from under it. When a real sandbox orchestrator lands later, it +// claims these (moving them out of pending) well before the sweeper's cutoff. +// +// The filter is Statuses:[pending] (pending == unclaimed) — once an +// orchestrator claims a startup task it leaves the pending state and is no +// longer eligible for this sweep. Cancellation goes through CancelTask so the +// queue row is retired and any authority grant is revoked (never a raw status +// write). Errors are best-effort (logged and skipped) so one bad task does not +// abort the sweep. Returns the number of tasks cancelled. +func (tas *TaskAssignmentService) CancelStaleStartupTasks(ctx context.Context, ttl time.Duration) (int, error) { + if ttl <= 0 { + return 0, nil + } + + taskList, err := tas.taskStore.ListTasks(ctx, &tasks.TaskFilter{ + TaskType: startupTaskType, + Statuses: []tasks.TaskStatus{tasks.TaskStatusPending}, + Limit: 1000, + }) + if err != nil { + return 0, fmt.Errorf("failed to list startup tasks: %w", err) + } + + cutoff := time.Now().Add(-ttl) + cancelled := 0 + for _, task := range taskList { + if !task.CreatedAt.Before(cutoff) { + continue + } + if err := tas.CancelTask(ctx, task.TaskID); err != nil { + logging.Logger.Error().Err(err).Str("task_id", task.TaskID).Str("target_implementation", task.TargetImplementation).Msg("stale startup cancel: failed to cancel task") + continue + } + logging.Logger.Info().Str("task_id", task.TaskID).Str("target_implementation", task.TargetImplementation).Time("created_at", task.CreatedAt).Msg("stale startup cancel: cancelled stale startup task") + cancelled++ + } + return cancelled, nil +} + +// CancelStalePoolTasks cancels regular POOL tasks that have sat UNCLAIMED +// (pending) longer than ttl. A pool task is claimed by any matching worker when +// it connects (DeliverPoolTasks); when no worker for its target implementation +// ever connects, the pending row lingers forever. No existing sweep collects it: +// agent_startup pool tasks are handled by CancelStaleStartupTasks, active +// INTERACTIVE turns by CancelStaleInteractiveTasks, and PurgeTasks only reaps +// terminal rows. +// +// The filter is AssignmentMode=pool + TaskCategory=regular + Statuses:[pending]. +// TaskCategory=regular deliberately EXCLUDES orchestrated agent_startup pool +// tasks (TaskCategory=orchestrated), which CancelStaleStartupTasks owns, so the +// two sweeps never double-cancel the same row. Once a worker claims a pool task +// it leaves the pending state and is no longer eligible. +// +// Cancellation goes through CancelTask so the queue row is retired and any +// authority grant is revoked (never a raw status write). The TTL is deliberately +// GENEROUS: a legitimate pool task whose worker is briefly absent must never be +// cancelled out from under it. Errors are best-effort (logged and skipped) so one +// bad task does not abort the sweep. Returns the number of tasks cancelled. +func (tas *TaskAssignmentService) CancelStalePoolTasks(ctx context.Context, ttl time.Duration) (int, error) { + if ttl <= 0 { + return 0, nil + } + + poolMode := tasks.AssignmentModePool + regularCategory := tasks.TaskCategoryRegular + taskList, err := tas.taskStore.ListTasks(ctx, &tasks.TaskFilter{ + AssignmentMode: &poolMode, + TaskCategory: ®ularCategory, + Statuses: []tasks.TaskStatus{tasks.TaskStatusPending}, + Limit: 1000, + }) + if err != nil { + return 0, fmt.Errorf("failed to list pool tasks: %w", err) + } + + cutoff := time.Now().Add(-ttl) + cancelled := 0 + for _, task := range taskList { + if !task.CreatedAt.Before(cutoff) { + continue + } + if err := tas.CancelTask(ctx, task.TaskID); err != nil { + logging.Logger.Error().Err(err).Str("task_id", task.TaskID).Str("target_implementation", task.TargetImplementation).Msg("stale pool cancel: failed to cancel task") + continue + } + logging.Logger.Info().Str("task_id", task.TaskID).Str("target_implementation", task.TargetImplementation).Time("created_at", task.CreatedAt).Msg("stale pool cancel: cancelled stale pool task") + cancelled++ + } + return cancelled, nil +} diff --git a/server/internal/orchestration/task_assignment_test.go b/server/internal/orchestration/task_assignment_test.go index 533a28c..4597e38 100644 --- a/server/internal/orchestration/task_assignment_test.go +++ b/server/internal/orchestration/task_assignment_test.go @@ -2,9 +2,12 @@ package orchestration import ( "context" + "database/sql" "fmt" + "path/filepath" "sync" "testing" + "time" "github.com/alicebob/miniredis/v2" "github.com/google/uuid" @@ -12,10 +15,16 @@ import ( "github.com/scitrera/aether/internal/registry" "github.com/scitrera/aether/internal/state" regpg "github.com/scitrera/aether/internal/storage/registry/postgres" + taskstore "github.com/scitrera/aether/internal/storage/tasks" taskpg "github.com/scitrera/aether/internal/storage/tasks/postgres" + taskssqlite "github.com/scitrera/aether/internal/storage/tasks/sqlite" "github.com/scitrera/aether/internal/testutil" "github.com/scitrera/aether/pkg/models" "github.com/scitrera/aether/pkg/tasks" + + // Register the bare "sqlite" driver so CancelStaleInteractiveTasks can be + // tested against the native sqlite backend (always available, never skipped). + _ "modernc.org/sqlite" ) func TestOrchestratedTaskPayload(t *testing.T) { @@ -978,3 +987,277 @@ func TestStartTaskWithAgent_NoDispatcherIsSafe(t *testing.T) { t.Errorf("task status = %q, want %q", stored.Status, "running") } } + +// TestCancelStaleInteractiveTasks verifies the interactive-task TTL sweep: +// only INTERACTIVE tasks in a non-terminal, non-waiting state older than the TTL +// are cancelled. Runs against the NATIVE SQLITE backend (aetherlite path) so it is +// ALWAYS exercised (no external DB, never skipped) AND proves the sweep's +// TaskFilter{TaskClass,Statuses} is honored on the sqlite store — see the +// TaskClassFilter conformance subtest for the postgres+sqlite store-level guarantee. +func TestCancelStaleInteractiveTasks(t *testing.T) { + dbPath := filepath.Join(t.TempDir(), "orch_tasks.db") + db, err := sql.Open("sqlite", dbPath+"?_journal_mode=WAL&_busy_timeout=5000") + if err != nil { + t.Fatalf("sql.Open sqlite: %v", err) + } + t.Cleanup(func() { _ = db.Close() }) + taskStore, err := taskssqlite.New(db) + if err != nil { + t.Fatalf("taskssqlite.New: %v", err) + } + + ctx := context.Background() + service := NewTaskAssignmentService(taskStore, nil, nil, nil, nil) + + const taskClassBackground int32 = 2 // proto TaskClass_TASK_CLASS_BACKGROUND + + // Create a task with the given class, then force its status + created_at via + // SQL so age/state are fully controlled regardless of CreateTask defaults. + mk := func(id string, class int32, status tasks.TaskStatus, createdAt time.Time) { + if err := taskStore.CreateTask(ctx, &tasks.Task{ + TaskID: id, TaskType: "chat_message", Workspace: "ws-test", TaskClass: class, + }); err != nil { + t.Fatalf("CreateTask(%s): %v", id, err) + } + if _, err := db.ExecContext(ctx, + "UPDATE tasks SET status = ?, created_at = ? WHERE task_id = ?", + string(status), createdAt.UTC().Format(time.RFC3339Nano), id); err != nil { + t.Fatalf("force status/created_at (%s): %v", id, err) + } + } + + now := time.Now() + old := now.Add(-2 * time.Hour) + mk("old-interactive", taskClassInteractive, tasks.TaskStatusPending, old) // -> cancelled + mk("young-interactive", taskClassInteractive, tasks.TaskStatusPending, now) // too young -> kept + mk("old-background", taskClassBackground, tasks.TaskStatusPending, old) // wrong class -> kept + mk("old-terminal", taskClassInteractive, tasks.TaskStatusCompleted, old) // terminal -> kept + + n, err := service.CancelStaleInteractiveTasks(ctx, time.Hour) + if err != nil { + t.Fatalf("CancelStaleInteractiveTasks: %v", err) + } + if n != 1 { + t.Fatalf("cancelled = %d, want 1 (only the old interactive task)", n) + } + + assertStatus := func(id string, want tasks.TaskStatus) { + got, gerr := taskStore.GetTask(ctx, id) + if gerr != nil { + t.Fatalf("GetTask(%s): %v", id, gerr) + } + if got.Status != want { + t.Errorf("task %s status = %q, want %q", id, got.Status, want) + } + } + assertStatus("old-interactive", tasks.TaskStatusCancelled) // reaped + assertStatus("young-interactive", tasks.TaskStatusPending) // under TTL + assertStatus("old-background", tasks.TaskStatusPending) // not interactive + assertStatus("old-terminal", tasks.TaskStatusCompleted) // already terminal +} + +// TestCancelTask_RetiresQueueRowDirectlyWithoutDispatcher is the root-cause +// regression guard: CancelTask must retire the orchestrated_task_queue row +// directly through the store even when tas.dispatcher is nil (the cleanup +// service's TaskAssignmentService instance). Before the fix, the retire was +// dispatcher-gated, so a nil dispatcher left the pending queue row orphaned and +// the polling dispatcher polled it forever. Runs against the NATIVE SQLITE +// backend so it is ALWAYS exercised (no external DB, never skipped). +func TestCancelTask_RetiresQueueRowDirectlyWithoutDispatcher(t *testing.T) { + dbPath := filepath.Join(t.TempDir(), "orch_tasks.db") + db, err := sql.Open("sqlite", dbPath+"?_journal_mode=WAL&_busy_timeout=5000") + if err != nil { + t.Fatalf("sql.Open sqlite: %v", err) + } + t.Cleanup(func() { _ = db.Close() }) + taskStore, err := taskssqlite.New(db) + if err != nil { + t.Fatalf("taskssqlite.New: %v", err) + } + + ctx := context.Background() + // No dispatcher wired — this mirrors the cleanup-service code path. + service := NewTaskAssignmentService(taskStore, nil, nil, nil, nil) + + taskID := uuid.New().String() + if err := taskStore.CreateTask(ctx, &tasks.Task{ + TaskID: taskID, TaskType: "agent_startup", Workspace: "ws-test", + }); err != nil { + t.Fatalf("CreateTask: %v", err) + } + queueID := uuid.New().String() + if err := taskStore.InsertQueueEntry(ctx, queueID, taskID, "impl-x", "ws-test", "local", nil, int(tasks.PriorityNormal)); err != nil { + t.Fatalf("InsertQueueEntry: %v", err) + } + + // Sanity: the row polls before the cancel. + before, err := taskStore.PollPendingQueueEntries(ctx, 100) + if err != nil { + t.Fatalf("PollPendingQueueEntries(before): %v", err) + } + if !containsQueueID(before, queueID) { + t.Fatalf("queue row %s did not poll before cancel", queueID) + } + + if err := service.CancelTask(ctx, taskID); err != nil { + t.Fatalf("CancelTask: %v", err) + } + + // The queue row must be retired despite the nil dispatcher. + after, err := taskStore.PollPendingQueueEntries(ctx, 100) + if err != nil { + t.Fatalf("PollPendingQueueEntries(after): %v", err) + } + if containsQueueID(after, queueID) { + t.Errorf("queue row %s still polls after CancelTask with nil dispatcher (orphaned)", queueID) + } + + got, err := taskStore.GetTask(ctx, taskID) + if err != nil { + t.Fatalf("GetTask: %v", err) + } + if got.Status != tasks.TaskStatusCancelled { + t.Errorf("task status = %q, want %q", got.Status, tasks.TaskStatusCancelled) + } +} + +// TestReconcileOrphanedQueueEntries_SkipsLiveRetiresTerminal verifies the store +// sweep at the orchestration seam: a pending queue row whose task is still +// non-terminal (running) is preserved, while one whose task is terminal +// (cancelled) is retired. Runs against the NATIVE SQLITE backend. +func TestReconcileOrphanedQueueEntries_SkipsLiveRetiresTerminal(t *testing.T) { + dbPath := filepath.Join(t.TempDir(), "orch_tasks.db") + db, err := sql.Open("sqlite", dbPath+"?_journal_mode=WAL&_busy_timeout=5000") + if err != nil { + t.Fatalf("sql.Open sqlite: %v", err) + } + t.Cleanup(func() { _ = db.Close() }) + taskStore, err := taskssqlite.New(db) + if err != nil { + t.Fatalf("taskssqlite.New: %v", err) + } + ctx := context.Background() + + // Live task (running) with a pending queue row — must survive. + liveID := uuid.New().String() + if err := taskStore.CreateTask(ctx, &tasks.Task{TaskID: liveID, TaskType: "agent_startup", Workspace: "ws-test"}); err != nil { + t.Fatalf("CreateTask(live): %v", err) + } + if err := taskStore.AssignTask(ctx, liveID, "worker-1"); err != nil { + t.Fatalf("AssignTask: %v", err) + } + if err := taskStore.StartTask(ctx, liveID); err != nil { + t.Fatalf("StartTask: %v", err) + } + liveQueue := uuid.New().String() + if err := taskStore.InsertQueueEntry(ctx, liveQueue, liveID, "impl-live", "ws-test", "local", nil, int(tasks.PriorityNormal)); err != nil { + t.Fatalf("InsertQueueEntry(live): %v", err) + } + + // Terminal task (cancelled) with a pending queue row — must be retired. + termID := uuid.New().String() + if err := taskStore.CreateTask(ctx, &tasks.Task{TaskID: termID, TaskType: "agent_startup", Workspace: "ws-test"}); err != nil { + t.Fatalf("CreateTask(term): %v", err) + } + if err := taskStore.CancelTask(ctx, termID); err != nil { + t.Fatalf("CancelTask(term): %v", err) + } + termQueue := uuid.New().String() + if err := taskStore.InsertQueueEntry(ctx, termQueue, termID, "impl-term", "ws-test", "local", nil, int(tasks.PriorityNormal)); err != nil { + t.Fatalf("InsertQueueEntry(term): %v", err) + } + + retired, err := taskStore.ReconcileOrphanedQueueEntries(ctx) + if err != nil { + t.Fatalf("ReconcileOrphanedQueueEntries: %v", err) + } + if retired != 1 { + t.Fatalf("retired %d rows, want 1 (only the cancelled-task row)", retired) + } + + after, err := taskStore.PollPendingQueueEntries(ctx, 100) + if err != nil { + t.Fatalf("PollPendingQueueEntries: %v", err) + } + if !containsQueueID(after, liveQueue) { + t.Errorf("live (running-task) queue row %s was wrongly retired", liveQueue) + } + if containsQueueID(after, termQueue) { + t.Errorf("orphaned (cancelled-task) queue row %s still polls after reconcile", termQueue) + } +} + +// containsQueueID reports whether the polled entries include the given queue_id. +func containsQueueID(entries []*taskstore.QueueEntryNotification, queueID string) bool { + for _, e := range entries { + if e.QueueID == queueID { + return true + } + } + return false +} + +// TestCancelStaleStartupTasks verifies the startup-task TTL sweep: only +// UNCLAIMED (pending) agent_startup tasks older than the TTL are cancelled. +// Runs against the NATIVE SQLITE backend (aetherlite path) so it is ALWAYS +// exercised (no external DB, never skipped) AND proves the sweep's +// TaskFilter{TaskType,Statuses} is honored on the sqlite store. +func TestCancelStaleStartupTasks(t *testing.T) { + dbPath := filepath.Join(t.TempDir(), "orch_tasks.db") + db, err := sql.Open("sqlite", dbPath+"?_journal_mode=WAL&_busy_timeout=5000") + if err != nil { + t.Fatalf("sql.Open sqlite: %v", err) + } + t.Cleanup(func() { _ = db.Close() }) + taskStore, err := taskssqlite.New(db) + if err != nil { + t.Fatalf("taskssqlite.New: %v", err) + } + + ctx := context.Background() + service := NewTaskAssignmentService(taskStore, nil, nil, nil, nil) + + // Create a task with the given type, then force its status + created_at via + // SQL so age/state are fully controlled regardless of CreateTask defaults. + mk := func(id, taskType string, status tasks.TaskStatus, createdAt time.Time) { + if err := taskStore.CreateTask(ctx, &tasks.Task{ + TaskID: id, TaskType: taskType, Workspace: "ws-test", + }); err != nil { + t.Fatalf("CreateTask(%s): %v", id, err) + } + if _, err := db.ExecContext(ctx, + "UPDATE tasks SET status = ?, created_at = ? WHERE task_id = ?", + string(status), createdAt.UTC().Format(time.RFC3339Nano), id); err != nil { + t.Fatalf("force status/created_at (%s): %v", id, err) + } + } + + now := time.Now() + old := now.Add(-2 * time.Hour) + mk("old-startup", startupTaskType, tasks.TaskStatusPending, old) // -> cancelled + mk("young-startup", startupTaskType, tasks.TaskStatusPending, now) // too young -> kept + mk("old-other", "chat_message", tasks.TaskStatusPending, old) // wrong type -> kept + mk("old-claimed", startupTaskType, tasks.TaskStatusAssigned, old) // claimed (not pending) -> kept + + n, err := service.CancelStaleStartupTasks(ctx, time.Hour) + if err != nil { + t.Fatalf("CancelStaleStartupTasks: %v", err) + } + if n != 1 { + t.Fatalf("cancelled = %d, want 1 (only the old pending startup task)", n) + } + + assertStatus := func(id string, want tasks.TaskStatus) { + got, gerr := taskStore.GetTask(ctx, id) + if gerr != nil { + t.Fatalf("GetTask(%s): %v", id, gerr) + } + if got.Status != want { + t.Errorf("task %s status = %q, want %q", id, got.Status, want) + } + } + assertStatus("old-startup", tasks.TaskStatusCancelled) // reaped + assertStatus("young-startup", tasks.TaskStatusPending) // under TTL + assertStatus("old-other", tasks.TaskStatusPending) // not a startup task + assertStatus("old-claimed", tasks.TaskStatusAssigned) // already claimed +} diff --git a/server/internal/orchestration/task_completion_event_test.go b/server/internal/orchestration/task_completion_event_test.go new file mode 100644 index 0000000..1abd3d1 --- /dev/null +++ b/server/internal/orchestration/task_completion_event_test.go @@ -0,0 +1,238 @@ +// Feed B: unit tests for completion_event domain event emission on terminal +// transitions. Uses a recording DomainEventPublisher and the same sqlite-backed +// TaskAssignmentService harness as task_event_publisher_test.go. + +package orchestration + +import ( + "context" + "database/sql" + "encoding/json" + "path/filepath" + "sync" + "testing" + + taskssqlite "github.com/scitrera/aether/internal/storage/tasks/sqlite" + "github.com/scitrera/aether/pkg/tasks" + + _ "modernc.org/sqlite" +) + +// fakeDomainEventPublisher records every PublishDomainEvent call (raw JSON +// EventPayload bytes). Thread-safe so emission from goroutines doesn't race the +// test assertions. +type fakeDomainEventPublisher struct { + mu sync.Mutex + payloads [][]byte +} + +func (f *fakeDomainEventPublisher) PublishDomainEvent(_ context.Context, _ string, payload []byte) error { + f.mu.Lock() + defer f.mu.Unlock() + cp := make([]byte, len(payload)) + copy(cp, payload) + f.payloads = append(f.payloads, cp) + return nil +} + +func (f *fakeDomainEventPublisher) snapshot() [][]byte { + f.mu.Lock() + defer f.mu.Unlock() + out := make([][]byte, len(f.payloads)) + copy(out, f.payloads) + return out +} + +// capturedEventPayload mirrors the workflow engine's Router.EventPayload shape; +// the JSON field tags must match exactly so a feed-B payload round-trips. +type capturedEventPayload struct { + SourceAgent string `json:"source_agent"` + EventNames []string `json:"event_names"` + Data map[string]any `json:"data"` + Workspace string `json:"workspace"` +} + +// newCompletionEventTestService builds a sqlite-backed TaskAssignmentService +// with a recording DomainEventPublisher attached. +func newCompletionEventTestService(t *testing.T) (*TaskAssignmentService, *fakeDomainEventPublisher, func()) { + t.Helper() + dbPath := filepath.Join(t.TempDir(), "feedb.db") + db, err := sql.Open("sqlite", dbPath+"?_journal_mode=WAL&_busy_timeout=5000") + if err != nil { + t.Fatalf("sql.Open sqlite: %v", err) + } + store, err := taskssqlite.New(db) + if err != nil { + _ = db.Close() + t.Fatalf("taskssqlite.New: %v", err) + } + pub := &fakeDomainEventPublisher{} + svc := NewTaskAssignmentService(store, nil, nil, nil, nil) + svc.SetDomainEventPublisher(pub) + return svc, pub, func() { _ = db.Close() } +} + +// createStartWithCompletion creates a task (carrying the given completion config, +// correlation/root ids) and drives it to running so the terminal call is legal. +func createStartWithCompletion(t *testing.T, svc *TaskAssignmentService, taskID, workspace string, cfg *tasks.TaskCompletionConfig) { + t.Helper() + ctx := context.Background() + row := &tasks.ExtendedTask{ + TaskID: taskID, + TaskType: "test", + Workspace: workspace, + Status: tasks.TaskStatusPending, + CorrelationID: "corr-" + taskID, + RootTaskID: "root-" + taskID, + CompletionEvent: cfg, + } + if err := svc.taskStore.CreateTask(ctx, row); err != nil { + t.Fatalf("CreateTask: %v", err) + } + if err := svc.taskStore.AssignTask(ctx, taskID, "ag::"+workspace+"::worker::v1"); err != nil { + t.Fatalf("AssignTask: %v", err) + } + if err := svc.taskStore.StartTask(ctx, taskID); err != nil { + t.Fatalf("StartTask: %v", err) + } +} + +// TestCompletionEvent_EmitsOnTerminal asserts an enabled completion_event on a +// terminal (completed) transition produces exactly one PublishDomainEvent call +// with the expected EventPayload shape. +func TestCompletionEvent_EmitsOnTerminal(t *testing.T) { + svc, pub, cleanup := newCompletionEventTestService(t) + defer cleanup() + + createStartWithCompletion(t, svc, "fb-complete", "ws1", &tasks.TaskCompletionConfig{ + Enabled: true, + EventName: "task.done", + }) + + if err := svc.CompleteTask(context.Background(), "fb-complete"); err != nil { + t.Fatalf("CompleteTask: %v", err) + } + + payloads := pub.snapshot() + if len(payloads) != 1 { + t.Fatalf("expected exactly 1 domain event, got %d", len(payloads)) + } + var ep capturedEventPayload + if err := json.Unmarshal(payloads[0], &ep); err != nil { + t.Fatalf("unmarshal EventPayload: %v", err) + } + if ep.SourceAgent != "orchestrator" { + t.Errorf("source_agent: got %q, want %q", ep.SourceAgent, "orchestrator") + } + if ep.Workspace != "ws1" { + t.Errorf("workspace: got %q, want %q", ep.Workspace, "ws1") + } + if len(ep.EventNames) != 1 || ep.EventNames[0] != "task.done" { + t.Errorf("event_names: got %v, want [task.done]", ep.EventNames) + } + if got := ep.Data["task_id"]; got != "fb-complete" { + t.Errorf("data.task_id: got %v, want fb-complete", got) + } + if got := ep.Data["status"]; got != "completed" { + t.Errorf("data.status: got %v, want completed", got) + } + if got := ep.Data["correlation_id"]; got != "corr-fb-complete" { + t.Errorf("data.correlation_id: got %v, want corr-fb-complete", got) + } + if got := ep.Data["root_task_id"]; got != "root-fb-complete" { + t.Errorf("data.root_task_id: got %v, want root-fb-complete", got) + } +} + +// TestCompletionEvent_DefaultEventName checks the fallback name "task." +// when EventName is empty. +func TestCompletionEvent_DefaultEventName(t *testing.T) { + svc, pub, cleanup := newCompletionEventTestService(t) + defer cleanup() + + createStartWithCompletion(t, svc, "fb-default", "ws1", &tasks.TaskCompletionConfig{ + Enabled: true, + }) + + if err := svc.CompleteTask(context.Background(), "fb-default"); err != nil { + t.Fatalf("CompleteTask: %v", err) + } + payloads := pub.snapshot() + if len(payloads) != 1 { + t.Fatalf("expected exactly 1 domain event, got %d", len(payloads)) + } + var ep capturedEventPayload + if err := json.Unmarshal(payloads[0], &ep); err != nil { + t.Fatalf("unmarshal: %v", err) + } + if len(ep.EventNames) != 1 || ep.EventNames[0] != "task.completed" { + t.Errorf("event_names: got %v, want [task.completed]", ep.EventNames) + } +} + +// TestCompletionEvent_NilDisabledNoEmit ensures a nil or disabled +// completion_event produces no domain event. +func TestCompletionEvent_NilDisabledNoEmit(t *testing.T) { + svc, pub, cleanup := newCompletionEventTestService(t) + defer cleanup() + + // nil completion_event + createStartWithCompletion(t, svc, "fb-nil", "ws1", nil) + if err := svc.CompleteTask(context.Background(), "fb-nil"); err != nil { + t.Fatalf("CompleteTask nil: %v", err) + } + // disabled completion_event + createStartWithCompletion(t, svc, "fb-disabled", "ws1", &tasks.TaskCompletionConfig{Enabled: false, EventName: "x"}) + if err := svc.CompleteTask(context.Background(), "fb-disabled"); err != nil { + t.Fatalf("CompleteTask disabled: %v", err) + } + + if got := len(pub.snapshot()); got != 0 { + t.Fatalf("expected no domain events for nil/disabled config, got %d", got) + } +} + +// TestCompletionEvent_OnStatusesFilter asserts the on_statuses filter is +// respected: with OnStatuses=[COMPLETED], a FAILED transition does not emit and +// a COMPLETED transition does. +func TestCompletionEvent_OnStatusesFilter(t *testing.T) { + svc, pub, cleanup := newCompletionEventTestService(t) + defer cleanup() + + cfg := func() *tasks.TaskCompletionConfig { + return &tasks.TaskCompletionConfig{ + Enabled: true, + EventName: "task.gathered", + OnStatuses: []tasks.TaskStatus{tasks.TaskStatusCompleted}, + } + } + + // FAILED transition: filtered out, no emit. + createStartWithCompletion(t, svc, "fb-fail", "ws1", cfg()) + if err := svc.FailTask(context.Background(), "fb-fail", "boom"); err != nil { + t.Fatalf("FailTask: %v", err) + } + if got := len(pub.snapshot()); got != 0 { + t.Fatalf("FAILED transition should not emit (filter=[completed]); got %d", got) + } + + // COMPLETED transition: passes filter, exactly one emit. + createStartWithCompletion(t, svc, "fb-ok", "ws1", cfg()) + if err := svc.CompleteTask(context.Background(), "fb-ok"); err != nil { + t.Fatalf("CompleteTask: %v", err) + } + payloads := pub.snapshot() + if len(payloads) != 1 { + t.Fatalf("expected exactly 1 domain event after COMPLETED, got %d", len(payloads)) + } + var ep capturedEventPayload + if err := json.Unmarshal(payloads[0], &ep); err != nil { + t.Fatalf("unmarshal: %v", err) + } + if got := ep.Data["task_id"]; got != "fb-ok" { + t.Errorf("data.task_id: got %v, want fb-ok", got) + } + if got := ep.Data["status"]; got != "completed" { + t.Errorf("data.status: got %v, want completed", got) + } +} diff --git a/server/internal/orchestration/task_event_publisher.go b/server/internal/orchestration/task_event_publisher.go index e814546..e826801 100644 --- a/server/internal/orchestration/task_event_publisher.go +++ b/server/internal/orchestration/task_event_publisher.go @@ -11,6 +11,7 @@ package orchestration import ( "context" + "encoding/json" "time" pb "github.com/scitrera/aether/api/proto" @@ -25,12 +26,25 @@ type TaskEventPublisher interface { PublishTaskEvent(ctx context.Context, workspace, taskID string, event *pb.TaskEvent) error } +// DomainEventPublisher publishes a raw EventPayload (JSON bytes) onto the +// event plane (event::*). A nil publisher disables feed-B emission. +type DomainEventPublisher interface { + PublishDomainEvent(ctx context.Context, workspace string, payload []byte) error +} + // SetEventPublisher injects a TaskEventPublisher. Pass nil to disable (the // default). Idempotent — last writer wins. func (tas *TaskAssignmentService) SetEventPublisher(p TaskEventPublisher) { tas.eventPub = p } +// SetDomainEventPublisher injects a DomainEventPublisher used for "feed B" +// domain event emission onto the event plane (event::*). Pass nil to disable +// (the default). Idempotent — last writer wins. +func (tas *TaskAssignmentService) SetDomainEventPublisher(p DomainEventPublisher) { + tas.domainEventPub = p +} + // taskStatusToProto maps the Go-side TaskStatus to the wire enum. The Go-side // enum is finer than the wire enum (pending/assigned/starting collapse to // QUEUED on the wire); callers downstream only need the coarse wire state. @@ -121,19 +135,96 @@ func (tas *TaskAssignmentService) publishChildLifecycle(ctx context.Context, par // child-lifecycle event so the parent's subscriber learns of the child's // terminal state without needing to subscribe to every potential child. func (tas *TaskAssignmentService) emitTransitionEvent(ctx context.Context, pre *tasks.ExtendedTask, taskID string, to tasks.TaskStatus, reason string) { - if pre == nil || tas.eventPub == nil { + if pre == nil { return } - tas.publishStatusChange(ctx, taskID, pre.Workspace, pre.ParentTaskID, pre.Status, to, reason) - // Child-lifecycle relay: terminal transitions surface to the parent's - // per-task subscription so a recursive subscriber sees child completion - // without depending on a separate child subscription. - if pre.ParentTaskID != "" && (tasks.IsTerminal(to) || tasks.IsWaiting(to)) { - classifier := "transitioned" - if tasks.IsTerminal(to) { - classifier = "completed" + if tas.eventPub != nil { + tas.publishStatusChange(ctx, taskID, pre.Workspace, pre.ParentTaskID, pre.Status, to, reason) + // Child-lifecycle relay: terminal transitions surface to the parent's + // per-task subscription so a recursive subscriber sees child completion + // without depending on a separate child subscription. + if pre.ParentTaskID != "" && (tasks.IsTerminal(to) || tasks.IsWaiting(to)) { + classifier := "transitioned" + if tasks.IsTerminal(to) { + classifier = "completed" + } + tas.publishChildLifecycle(ctx, pre.ParentTaskID, pre.Workspace, taskID, to, classifier) } - tas.publishChildLifecycle(ctx, pre.ParentTaskID, pre.Workspace, taskID, to, classifier) + } + // Feed B: domain event emission onto the event plane (event::*) when the + // task opted into completion_event and reaches a selected terminal status. + tas.publishCompletionEvent(ctx, pre, taskID, to) +} + +// terminalStatusString returns the lowercase wire string for a terminal status +// (completed / failed / cancelled), used both for the default event name and +// the event data's "status" field. Non-terminal statuses return the raw +// TaskStatus string. +func terminalStatusString(s tasks.TaskStatus) string { + return string(s) +} + +// publishCompletionEvent emits a "feed B" domain event onto the event plane +// (event::*) when the just-completed task opted into completion_event and the +// terminal status passes the on_statuses filter. The payload is the raw JSON +// bytes of an EventPayload (source_agent/event_names/data/workspace), matching +// what a client's SendEvent publishes so the workflow engine's Router can +// json.Unmarshal it. Best-effort: a publish failure never blocks the +// transition (which has already committed to the store), matching +// publishStatusChange's non-fatal error style. +func (tas *TaskAssignmentService) publishCompletionEvent(ctx context.Context, pre *tasks.ExtendedTask, taskID string, to tasks.TaskStatus) { + if pre == nil || tas.domainEventPub == nil || !tasks.IsTerminal(to) { + return + } + cfg := pre.CompletionEvent + if cfg == nil || !cfg.Enabled { + return + } + // Status filter: if OnStatuses is set, emit only when `to` is listed; + // empty ⇒ all terminal statuses emit. + if len(cfg.OnStatuses) > 0 { + match := false + for _, s := range cfg.OnStatuses { + if s == to { + match = true + break + } + } + if !match { + return + } + } + + statusStr := terminalStatusString(to) + eventName := cfg.EventName + if eventName == "" { + eventName = "task." + statusStr + } + + // Build the EventPayload as a plain map so the JSON keys are exact and + // match the workflow engine's EventPayload struct tags. + payload := map[string]any{ + "source_agent": "orchestrator", + "workspace": pre.Workspace, + "event_names": []string{eventName}, + "data": map[string]any{ + "task_id": taskID, + "status": statusStr, + "task_type": pre.TaskType, + "correlation_id": pre.CorrelationID, + "root_task_id": pre.RootTaskID, + "metadata": pre.Metadata, + }, + } + bytes, err := json.Marshal(payload) + if err != nil { + logging.Logger.Debug().Err(err).Str("task_id", taskID).Str("workspace", pre.Workspace). + Msg("publishCompletionEvent: marshal failed (non-fatal)") + return + } + if err := tas.domainEventPub.PublishDomainEvent(ctx, pre.Workspace, bytes); err != nil { + logging.Logger.Debug().Err(err).Str("task_id", taskID).Str("workspace", pre.Workspace). + Str("event_name", eventName).Msg("publishCompletionEvent: domain event publish failed (non-fatal)") } } diff --git a/server/internal/proxysidecar/aggregator.go b/server/internal/proxysidecar/aggregator.go new file mode 100644 index 0000000..6673679 --- /dev/null +++ b/server/internal/proxysidecar/aggregator.go @@ -0,0 +1,882 @@ +package proxysidecar + +import ( + "context" + "crypto/tls" + "crypto/x509" + "errors" + "fmt" + "io" + "os" + "strings" + "sync" + "time" + + "github.com/rs/zerolog/log" + pb "github.com/scitrera/aether/api/proto" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/keepalive" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/peer" +) + +// metadataTenantKey is the lowercase metadata header a provider sets on its +// Connect to declare which tenant's relay it wants paired. It is a HINT only: +// the authoritative tenant binding is the peer-cert CN (see the trust model in +// Connect/Tunnel below). +const metadataTenantKey = "x-aether-tenant" + +// Aggregator runs the sidecar in aggregator mode: it owns two gRPC server +// surfaces — AetherGateway.Connect (providers dial in with an x-aether-tenant +// hint) and SandboxRelayTunnel (tenant-relays dial in and announce their tenant +// via TunnelHello) — and pairs them 1:1 by tenant (validated against the +// peer-cert CN), splicing each pair NOT as a mux but as N independent gateway +// sessions. It satisfies relaySurface. +// +// Trust model (decision 2/4 of the design spec): both the provider hop and the +// relay hop are mTLS over the sandboxes-CA. The tenant-identity authority is the +// peer-cert CN, NOT the x-aether-tenant metadata / TunnelHello.tenant hint. The +// hint is validated against the CN and a mismatch is rejected. The relay side is +// the load-bearing binding: a tenant-relay presents a per-tenant cert whose CN +// encodes the tenant (e.g. sv::sandbox-provider::), so the relay cannot +// announce a tenant it does not hold a cert for. The provider side may present a +// single shared sandboxes-CA identity whose CN does NOT encode a tenant (one +// cert serves every tenant); in that case we can only validate metadata +// presence, and we accept the metadata tenant as the pairing key. When the +// provider CN DOES encode a tenant we additionally require it to match the +// metadata claim. Either way the relay's CN is what guarantees a provider can +// only ever be spliced to a relay that genuinely holds the tenant's cert. +// +// SECURITY: when a provider presents a shared sandboxes-CA cert whose CN encodes +// NO tenant, the pairing key is the x-aether-tenant metadata, not the cert. That +// means ANY holder of a valid sandboxes-CA client cert can pair with (and reach) +// ANY tenant's relay simply by claiming that tenant in metadata — the cert alone +// confers no per-tenant restriction on the provider hop. The provider_listen +// surface MUST therefore be network-restricted (P4 NetworkPolicy) to the trusted +// provider controller; do not expose it broadly. The relay hop stays bound by +// its per-tenant CN regardless. +type Aggregator struct { + pb.UnimplementedAetherGatewayServer + pb.UnimplementedSandboxRelayTunnelServer + + cfg *Config + pairs *pairingTable + + // peerCN extracts the peer-cert CN from a stream context. Production code + // uses peerCertCN; tests that run the surfaces over insecure transport + // inject a stub so the splice/pairing/watch paths don't require real certs. + peerCN func(ctx context.Context) (string, error) +} + +// NewAggregator constructs an Aggregator from cfg. The listeners are not opened +// until Run is invoked. cfg.Validate() must have been called first (NewRunner +// does this). +func NewAggregator(cfg *Config) (*Aggregator, error) { + a := &Aggregator{ + cfg: cfg, + pairs: newPairingTable(), + } + a.peerCN = peerCertCN + return a, nil +} + +// Run binds both listeners, registers the two gRPC server surfaces (the +// AetherGateway provider surface on ProviderListen, the SandboxRelayTunnel relay +// surface on TunnelListen), and serves until ctx is cancelled. Each surface gets +// its own mTLS server credentials built from ProviderTLS / TunnelTLS. Mirrors +// relay.go's serve/stop shape (a goroutine per Serve, bounded GracefulStop on +// teardown). +func (a *Aggregator) Run(ctx context.Context) error { + providerLis, providerCleanup, err := openRelayListener(a.cfg.Aggregator.ProviderListen) + if err != nil { + return fmt.Errorf("aggregator: open provider listener: %w", err) + } + defer func() { + _ = providerLis.Close() + if providerCleanup != nil { + providerCleanup() + } + }() + + tunnelLis, tunnelCleanup, err := openRelayListener(a.cfg.Aggregator.TunnelListen) + if err != nil { + return fmt.Errorf("aggregator: open tunnel listener: %w", err) + } + defer func() { + _ = tunnelLis.Close() + if tunnelCleanup != nil { + tunnelCleanup() + } + }() + + providerSrv, err := a.newSurfaceServer(a.cfg.Aggregator.ProviderTLS) + if err != nil { + return fmt.Errorf("aggregator: build provider server: %w", err) + } + pb.RegisterAetherGatewayServer(providerSrv, a) + + tunnelSrv, err := a.newSurfaceServer(a.cfg.Aggregator.TunnelTLS) + if err != nil { + return fmt.Errorf("aggregator: build tunnel server: %w", err) + } + pb.RegisterSandboxRelayTunnelServer(tunnelSrv, a) + + serveErr := make(chan error, 2) + go func() { serveErr <- providerSrv.Serve(providerLis) }() + go func() { serveErr <- tunnelSrv.Serve(tunnelLis) }() + + log.Info(). + Str("provider_listen", a.cfg.Aggregator.ProviderListen). + Str("tunnel_listen", a.cfg.Aggregator.TunnelListen). + Bool("require_tenant_metadata", a.cfg.Aggregator.RequireTenantMetadataEnabled()). + Int64("pair_wait_timeout_ms", a.cfg.Aggregator.PairWaitTimeoutMs). + Msg("proxy sidecar aggregator running") + + select { + case <-ctx.Done(): + log.Info().Msg("proxy sidecar aggregator shutting down") + a.stopServers(providerSrv, tunnelSrv) + // Drain both Serve goroutines so we don't leak them past Run's return. + <-serveErr + <-serveErr + return nil + case err := <-serveErr: + // One surface failed to serve; tear the other down and surface the error. + a.stopServers(providerSrv, tunnelSrv) + <-serveErr + if err != nil { + return fmt.Errorf("aggregator: serve: %w", err) + } + return nil + } +} + +// stopServers gracefully stops both surface servers, bounding the wait so a +// mid-handshake inbound connection that never sent the HTTP/2 preface can't pin +// shutdown forever (same rationale as relay.go's bounded GracefulStop). +func (a *Aggregator) stopServers(servers ...*grpc.Server) { + const gracePeriod = 3 * time.Second + var wg sync.WaitGroup + for _, srv := range servers { + srv := srv + wg.Add(1) + go func() { + defer wg.Done() + done := make(chan struct{}) + go func() { + srv.GracefulStop() + close(done) + }() + select { + case <-done: + case <-time.After(gracePeriod): + srv.Stop() + <-done + } + }() + } + wg.Wait() +} + +// newSurfaceServer builds a gRPC server for one aggregator surface. When the +// surface's TLS is not Insecure it enforces mTLS: it presents the configured +// server cert and requires-and-verifies a client cert chained to ClientCAFile +// (the sandboxes-CA), so the peer-cert CN is always available as the tenant +// authority. When Insecure is set the server runs plaintext (test/dev only). +func (a *Aggregator) newSurfaceServer(tlsConf ServerTLSConfig) (*grpc.Server, error) { + opts := []grpc.ServerOption{ + grpc.KeepaliveParams(keepalive.ServerParameters{ + Time: 30 * time.Second, + Timeout: 15 * time.Second, + }), + // Permit the dialer-side keepalive the relay/provider SDK uses (idle + // streamless pings every 30s) without tripping ENHANCE_YOUR_CALM, same + // rationale as relay.go's enforcement policy. + grpc.KeepaliveEnforcementPolicy(keepalive.EnforcementPolicy{ + MinTime: 10 * time.Second, + PermitWithoutStream: true, + }), + // Bound the HTTP/2 handshake so a peer that opens a socket but never + // sends the preface can't pin a Serve goroutine (and thus shutdown) for + // the grpc default 120s. See relay.go for the full rationale. + grpc.ConnectionTimeout(10 * time.Second), + grpc.MaxRecvMsgSize(10 * 1024 * 1024), + grpc.MaxSendMsgSize(10 * 1024 * 1024), + } + if !tlsConf.Insecure { + creds, err := a.serverCreds(tlsConf) + if err != nil { + return nil, err + } + opts = append(opts, grpc.Creds(creds)) + } + return grpc.NewServer(opts...), nil +} + +// serverCreds loads the surface's server cert/key and client-CA pool into mTLS +// transport credentials (RequireAndVerifyClientCert + ClientCAs). +func (a *Aggregator) serverCreds(tlsConf ServerTLSConfig) (credentials.TransportCredentials, error) { + cert, err := tls.LoadX509KeyPair(tlsConf.CertFile, tlsConf.KeyFile) + if err != nil { + return nil, fmt.Errorf("load server keypair (%q/%q): %w", tlsConf.CertFile, tlsConf.KeyFile, err) + } + caPEM, err := os.ReadFile(tlsConf.ClientCAFile) + if err != nil { + return nil, fmt.Errorf("read client_ca_file %q: %w", tlsConf.ClientCAFile, err) + } + pool := x509.NewCertPool() + if !pool.AppendCertsFromPEM(caPEM) { + return nil, fmt.Errorf("client_ca_file %q: no certs parsed", tlsConf.ClientCAFile) + } + return credentials.NewTLS(&tls.Config{ + MinVersion: tls.VersionTLS12, + Certificates: []tls.Certificate{cert}, + ClientAuth: tls.RequireAndVerifyClientCert, + ClientCAs: pool, + }), nil +} + +// Connect implements the provider surface (pb.AetherGatewayServer). A provider +// dials in declaring its target tenant via the x-aether-tenant metadata header. +// Each invocation registers the provider for its tenant, waits up to +// PairWaitTimeoutMs for a matching relay tunnel, and then splices the two 1:1. +// +// Template: relay.go Connect (relay.go:219). The difference is that the relay +// surface here is not a per-session gateway dial — it is the wait-for-tunnel +// pairing. +func (a *Aggregator) Connect(stream pb.AetherGateway_ConnectServer) error { + ctx := stream.Context() + + // Read the x-aether-tenant hint. RequireTenantMetadata (default true) + // rejects a Connect that omits it. + tenant, hasMeta := tenantFromMetadata(ctx) + if !hasMeta { + if a.cfg.Aggregator.RequireTenantMetadataEnabled() { + _ = stream.Send(relayErrorDownstream("AGG_TENANT_METADATA_REQUIRED", + "provider Connect must carry the x-aether-tenant metadata header")) + return fmt.Errorf("aggregator: provider Connect missing %s metadata", metadataTenantKey) + } + } + + // Validate the metadata hint against the provider peer-cert CN. The provider + // may present a single shared sandboxes-CA cert whose CN does NOT encode a + // tenant, so a CN that yields no tenant is accepted (we trust the relay-side + // CN for the authoritative binding). When the provider CN DOES encode a + // tenant it must match the metadata claim. + cn, err := a.peerCN(ctx) + if err != nil { + _ = stream.Send(relayErrorDownstream("AGG_PEER_CERT_MISSING", + fmt.Sprintf("provider peer cert: %v", err))) + return fmt.Errorf("aggregator: provider peer cert: %w", err) + } + if cnTenant := tenantFromCN(cn); cnTenant != "" { + if tenant == "" { + // No metadata hint but the CN pins a tenant: trust the CN. + tenant = cnTenant + } else if cnTenant != tenant { + _ = stream.Send(relayErrorDownstream("AGG_TENANT_CN_MISMATCH", + fmt.Sprintf("provider cert CN tenant %q != x-aether-tenant %q", cnTenant, tenant))) + return fmt.Errorf("aggregator: provider CN tenant %q != metadata %q", cnTenant, tenant) + } + } + if tenant == "" { + _ = stream.Send(relayErrorDownstream("AGG_TENANT_UNRESOLVED", + "could not resolve provider tenant from metadata or cert CN")) + return fmt.Errorf("aggregator: could not resolve provider tenant (cn=%q)", cn) + } + + provider := &providerEndpoint{stream: stream} + if err := a.pairs.registerProvider(tenant, provider); err != nil { + _ = stream.Send(relayErrorDownstream("AGG_PROVIDER_DUPLICATE", err.Error())) + return fmt.Errorf("aggregator: register provider for tenant %q: %w", tenant, err) + } + defer a.pairs.unregisterProvider(tenant, provider) + + log.Info().Str("tenant", tenant).Str("cn", cn).Msg("aggregator: provider connected; awaiting relay") + + relay, err := a.pairs.awaitRelay(ctx, tenant, provider, a.pairWait()) + if err != nil { + _ = stream.Send(relayErrorDownstream("AGG_PAIR_TIMEOUT", + fmt.Sprintf("no relay for tenant %q: %v", tenant, err))) + return fmt.Errorf("aggregator: await relay for tenant %q: %w", tenant, err) + } + + // The PROVIDER handler is the elected splicer (exactly one goroutine may + // drive each stream's Recv/Send). It publishes a shared session that the + // relay handler retrieves and blocks on, so the two streams are never + // touched concurrently from both handlers. + session := newPairSession(provider, relay) + a.pairs.publishSession(tenant, session) + defer a.pairs.retractSession(tenant, session) + + log.Info().Str("tenant", tenant).Msg("aggregator: provider/relay paired; splicing") + err = a.splice(ctx, tenant, provider, relay) + session.finish(err) + return err +} + +// Tunnel implements the relay surface (pb.SandboxRelayTunnelServer). The relay's +// FIRST frame MUST be a TunnelHello announcing its tenant (cf relay.go:230-242). +// The hello tenant is a hint validated against the relay peer-cert CN (the CN is +// the authority): a mismatch is rejected. The relay is then registered and +// paired with a waiting or future provider Connect, and the two are spliced 1:1. +func (a *Aggregator) Tunnel(stream pb.SandboxRelayTunnel_TunnelServer) error { + ctx := stream.Context() + + first, err := stream.Recv() + if err != nil { + if errors.Is(err, io.EOF) { + return nil + } + return fmt.Errorf("aggregator: tunnel recv hello: %w", err) + } + hello := first.GetHello() + if hello == nil { + return fmt.Errorf("aggregator: first tunnel frame was %T, expected hello", first.GetF()) + } + helloTenant := hello.GetTenant() + + // The relay peer-cert CN is the authoritative tenant binding: a per-tenant + // relay holds a cert whose CN encodes its tenant, so it cannot announce a + // tenant it does not own. The hello is a hint; reject any mismatch. + cn, err := a.peerCN(ctx) + if err != nil { + return fmt.Errorf("aggregator: relay peer cert: %w", err) + } + cnTenant := tenantFromCN(cn) + if cnTenant == "" { + return fmt.Errorf("aggregator: relay cert CN %q does not encode a tenant", cn) + } + if helloTenant != "" && helloTenant != cnTenant { + return fmt.Errorf("aggregator: relay hello tenant %q != cert CN tenant %q (CN is authority)", helloTenant, cnTenant) + } + tenant := cnTenant + + relay := &tunnelEndpoint{stream: stream} + if err := a.pairs.registerRelay(tenant, relay); err != nil { + return fmt.Errorf("aggregator: register relay for tenant %q: %w", tenant, err) + } + defer a.pairs.unregisterRelay(tenant, relay) + + log.Info().Str("tenant", tenant).Str("cn", cn).Msg("aggregator: relay connected; awaiting provider") + + // The relay handler is the PASSIVE side: the provider's Connect handler runs + // the splice (single-splicer invariant — only one goroutine may Recv/Send a + // given stream). The relay must keep its stream open while that splice runs + // (returning here would close the stream), so it waits for the provider to + // publish a session whose endpoints match this relay, then blocks until that + // session finishes (or its own ctx is cancelled). + session, err := a.pairs.awaitSession(ctx, tenant, relay, a.pairWait()) + if err != nil { + return fmt.Errorf("aggregator: await provider for tenant %q: %w", tenant, err) + } + + log.Info().Str("tenant", tenant).Msg("aggregator: relay/provider paired; provider splicing") + select { + case spliceErr := <-session.done: + return spliceErr + case <-ctx.Done(): + return ctx.Err() + } +} + +// WatchTenants streams TenantEvent{tenant, online} to a provider's controller so +// it can dynamically dial the provider surface as tenants' relays come and go. +// It replays the currently-online tenants, then forwards mutations until ctx is +// done. The watcher channel is cleaned up on return. +func (a *Aggregator) WatchTenants(_ *pb.WatchTenantsRequest, stream grpc.ServerStreamingServer[pb.TenantEvent]) error { + ctx := stream.Context() + + ch, online := a.pairs.addWatcher() + defer a.pairs.removeWatcher(ch) + + // Replay the current online set so a late-joining watcher converges. + for _, tenant := range online { + if err := stream.Send(&pb.TenantEvent{Tenant: tenant, Online: true}); err != nil { + return err + } + } + + // Emit a terminal snapshot_complete sentinel once the replay burst is done + // and before the live-event phase begins. The provider accumulates the + // replay online-set until this sentinel arrives, then prunes any tenant it + // still holds that is absent from the set — deterministically reaping the + // tenants that left during a watch disconnect (the resync-prune fix). Sent + // per-(re)subscribe, so every reconnecting watcher gets: replay → sentinel → + // live events. Live events keep SnapshotComplete=false (the zero value). + if err := stream.Send(&pb.TenantEvent{SnapshotComplete: true}); err != nil { + return err + } + + for { + select { + case <-ctx.Done(): + return ctx.Err() + case ev, ok := <-ch: + if !ok { + return nil + } + if err := stream.Send(ev); err != nil { + return err + } + } + } +} + +// splice copies frames between one paired provider and relay, 1:1. +// +// IMPORTANT — this is NOT a relaymux. relaymux.go (relaymux.go:1-28) multiplexes +// MANY sub-client streams onto ONE shared upstream gateway connection, demuxing +// replies by request_id and rewriting every sub-client to the SAME service +// identity. The aggregator deliberately does the OPPOSITE: each (provider, relay) +// pair is an INDEPENDENT splice. The relay forwards the provider's InitConnection +// verbatim to the tenant gateway, so every pair gets its own gateway session, +// lock, and session_id and its own resumable identity. Multiplexing here would +// collapse N tenants' sessions onto one identity and break per-session state — +// exactly what we must avoid. Hence: one goroutine pair per splice, no demux. +// +// Direction provider→relay: provider.Recv() yields *pb.UpstreamMessage; we wrap +// it as TunnelFrame.up and Send to the relay. Direction relay→provider: +// relay.Recv() yields *pb.TunnelFrame; we unwrap .down and Send to the provider. +// Unexpected up/hello frames arriving relay→provider are logged and dropped. +// +// Teardown mirrors tenant_relay.go runPumps: errCh-of-two, cancel on the first +// return, a bounded drain of the sibling, EOF→nil. +func (a *Aggregator) splice(ctx context.Context, tenant string, provider *providerEndpoint, relay *tunnelEndpoint) error { + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + errCh := make(chan error, 2) + go func() { errCh <- a.pumpProviderToRelay(ctx, provider, relay) }() + go func() { errCh <- a.pumpRelayToProvider(ctx, provider, relay) }() + + first := <-errCh + cancel() + select { + case <-errCh: + case <-time.After(5 * time.Second): + } + + log.Info().Str("tenant", tenant).Err(spliceErr(first)).Msg("aggregator: splice closed") + if errors.Is(first, io.EOF) { + return nil + } + return first +} + +// pumpProviderToRelay copies provider → relay, wrapping each upstream envelope +// in a TunnelFrame.up for the relay to forward to the tenant gateway verbatim. +func (a *Aggregator) pumpProviderToRelay(ctx context.Context, provider *providerEndpoint, relay *tunnelEndpoint) error { + for { + if ctx.Err() != nil { + return ctx.Err() + } + up, err := provider.stream.Recv() + if err != nil { + return err + } + if err := relay.stream.Send(&pb.TunnelFrame{F: &pb.TunnelFrame_Up{Up: up}}); err != nil { + return err + } + } +} + +// pumpRelayToProvider copies relay → provider, unwrapping each TunnelFrame.down +// into the DownstreamMessage the provider's gateway client expects. up/hello +// frames are not expected on this direction (the relay only sends down frames +// after the hello handshake); they are logged and dropped rather than forwarded. +func (a *Aggregator) pumpRelayToProvider(ctx context.Context, provider *providerEndpoint, relay *tunnelEndpoint) error { + for { + if ctx.Err() != nil { + return ctx.Err() + } + frame, err := relay.stream.Recv() + if err != nil { + return err + } + down := frame.GetDown() + if down == nil { + log.Debug(). + Str("frame", fmt.Sprintf("%T", frame.GetF())). + Msg("aggregator: dropping non-down tunnel frame on relay→provider") + continue + } + if err := provider.stream.Send(down); err != nil { + return err + } + } +} + +// pairWait converts the configured PairWaitTimeoutMs into a Duration. +func (a *Aggregator) pairWait() time.Duration { + return time.Duration(a.cfg.Aggregator.PairWaitTimeoutMs) * time.Millisecond +} + +// spliceErr normalises a clean EOF teardown to nil for logging. +func spliceErr(err error) error { + if errors.Is(err, io.EOF) { + return nil + } + return err +} + +// providerEndpoint holds a registered provider's Connect stream. +type providerEndpoint struct { + stream pb.AetherGateway_ConnectServer +} + +// tunnelEndpoint holds a registered relay's Tunnel stream. +type tunnelEndpoint struct { + stream pb.SandboxRelayTunnel_TunnelServer +} + +// pairSession is the shared rendezvous for one paired (provider, relay). The +// provider's Connect handler is the elected splicer; it publishes the session, +// runs the splice, and calls finish. The relay's Tunnel handler retrieves the +// same session and blocks on done — guaranteeing exactly one goroutine drives +// each stream (the single-splicer invariant; concurrent Recv on a gRPC stream +// is a data race). +type pairSession struct { + provider *providerEndpoint + relay *tunnelEndpoint + done chan error +} + +func newPairSession(provider *providerEndpoint, relay *tunnelEndpoint) *pairSession { + return &pairSession{provider: provider, relay: relay, done: make(chan error, 1)} +} + +// finish reports the splice result to the waiting relay handler. Buffered (cap +// 1) so the provider never blocks even if the relay has already departed. +func (s *pairSession) finish(err error) { + if errors.Is(err, io.EOF) { + err = nil + } + s.done <- err +} + +// pairingTable tracks at most one provider and one relay per tenant and pairs +// them 1:1 as each side arrives. WatchTenants watchers are fanned out whenever a +// tenant's relay comes online or goes offline. +// +// A tenant is considered "online" (and emitted to watchers) when its RELAY is +// registered: providers dynamically dial the provider surface in response to +// online events, so relay-presence is the meaningful signal. Duplicate policy +// (documented): a second relay for an already-bound tenant is REJECTED with a +// clear error (the existing relay keeps the tenant) rather than displacing it, +// so a misconfigured/duplicate relay cannot silently steal a live pairing. A +// second provider for an already-registered provider is likewise rejected. +type pairingTable struct { + mu sync.Mutex + relays map[string]*tunnelEndpoint + providers map[string]*providerEndpoint + + // sessions holds the published rendezvous for a tenant once its provider has + // elected itself splicer (keyed by tenant). + sessions map[string]*pairSession + + // waiters are notified (by close) when a counterpart appears. relayWaiters + // are woken when a relay registers; sessionWaiters when a provider publishes + // a session. Each tenant's waiters live in a set (keyed by the wake channel) + // so an individual waiter that times out or whose ctx is cancelled can remove + // ITSELF before returning — otherwise a provider repeatedly dialing a tenant + // with no relay would grow the waiter set unbounded (the leak fixed here). + relayWaiters map[string]map[chan struct{}]struct{} + sessionWaiters map[string]map[chan struct{}]struct{} + + watchers []chan *pb.TenantEvent +} + +func newPairingTable() *pairingTable { + return &pairingTable{ + relays: map[string]*tunnelEndpoint{}, + providers: map[string]*providerEndpoint{}, + sessions: map[string]*pairSession{}, + relayWaiters: map[string]map[chan struct{}]struct{}{}, + sessionWaiters: map[string]map[chan struct{}]struct{}{}, + } +} + +// registerProvider records the provider for tenant. Rejects a second provider +// for a tenant that already has one (duplicate policy above). The waiting relay +// is not woken here — it is woken when the provider publishes the splice session +// (publishSession), which only happens after the relay is confirmed present. +func (p *pairingTable) registerProvider(tenant string, ep *providerEndpoint) error { + p.mu.Lock() + defer p.mu.Unlock() + if _, exists := p.providers[tenant]; exists { + return fmt.Errorf("a provider is already connected for tenant %q", tenant) + } + p.providers[tenant] = ep + return nil +} + +// unregisterProvider removes ep iff it is still the registered provider for +// tenant (a later provider that replaced a stale one is not clobbered). +func (p *pairingTable) unregisterProvider(tenant string, ep *providerEndpoint) { + p.mu.Lock() + defer p.mu.Unlock() + if p.providers[tenant] == ep { + delete(p.providers, tenant) + } +} + +// registerRelay records the relay for tenant and emits an online TenantEvent. +// Rejects a second relay for a tenant that already has one (duplicate policy). +func (p *pairingTable) registerRelay(tenant string, ep *tunnelEndpoint) error { + p.mu.Lock() + defer p.mu.Unlock() + if _, exists := p.relays[tenant]; exists { + return fmt.Errorf("a relay is already connected for tenant %q", tenant) + } + p.relays[tenant] = ep + p.wake(p.relayWaiters, tenant) + p.emitLocked(&pb.TenantEvent{Tenant: tenant, Online: true}) + return nil +} + +// unregisterRelay removes ep iff it is still the registered relay for tenant and +// emits an offline TenantEvent in that case. +func (p *pairingTable) unregisterRelay(tenant string, ep *tunnelEndpoint) { + p.mu.Lock() + defer p.mu.Unlock() + if p.relays[tenant] == ep { + delete(p.relays, tenant) + p.emitLocked(&pb.TenantEvent{Tenant: tenant, Online: false}) + } +} + +// awaitRelay returns the relay paired with this provider's tenant, blocking up +// to timeout for one to register. Called from Connect after registerProvider. +func (p *pairingTable) awaitRelay(ctx context.Context, tenant string, _ *providerEndpoint, timeout time.Duration) (*tunnelEndpoint, error) { + return awaitEndpoint(ctx, timeout, &p.mu, func() *tunnelEndpoint { return p.relays[tenant] }, func(ch chan struct{}) func() { + return p.subscribeLocked(p.relayWaiters, tenant, ch) + }) +} + +// publishSession stores the provider-elected splice session for tenant and wakes +// the relay handler blocked in awaitSession. Called by the provider's Connect +// handler once both sides are confirmed present. +func (p *pairingTable) publishSession(tenant string, s *pairSession) { + p.mu.Lock() + defer p.mu.Unlock() + p.sessions[tenant] = s + p.wake(p.sessionWaiters, tenant) +} + +// retractSession removes s iff it is still the published session for tenant (a +// later session is not clobbered). Called by the provider on splice teardown. +func (p *pairingTable) retractSession(tenant string, s *pairSession) { + p.mu.Lock() + defer p.mu.Unlock() + if p.sessions[tenant] == s { + delete(p.sessions, tenant) + } +} + +// awaitSession returns the splice session published for this relay's tenant, +// blocking up to timeout. It only returns a session whose relay endpoint is THIS +// relay, so a stale session from a prior pairing is ignored. Called from Tunnel +// after registerRelay. +func (p *pairingTable) awaitSession(ctx context.Context, tenant string, relay *tunnelEndpoint, timeout time.Duration) (*pairSession, error) { + return awaitEndpoint(ctx, timeout, &p.mu, func() *pairSession { + if s := p.sessions[tenant]; s != nil && s.relay == relay { + return s + } + return nil + }, func(ch chan struct{}) func() { + return p.subscribeLocked(p.sessionWaiters, tenant, ch) + }) +} + +// waitFor blocks until present() returns a non-nil endpoint, ctx is done, or +// timeout elapses. It checks once under the lock (covering the already-present +// case with no race), and otherwise registers a wake channel via subscribe and +// re-checks on each wake. The generic T is the counterpart endpoint type. +// +// subscribe registers the wake channel under the lock and returns an unsubscribe +// closure that removes that SPECIFIC channel from the tenant's waiter set. On a +// timed-out or ctx-cancelled exit we must call unsubscribe (under the lock) so +// the orphaned waiter does not accumulate — a provider repeatedly dialing a +// tenant with no relay would otherwise grow the waiter set without bound. The +// wake-path exit needs no unsubscribe: wake() already closed and cleared the +// whole set, so the channel is gone (re-removing it is a harmless no-op anyway). +func awaitEndpoint[T any](ctx context.Context, timeout time.Duration, mu *sync.Mutex, present func() *T, subscribe func(chan struct{}) func()) (*T, error) { + deadline := time.NewTimer(timeout) + defer deadline.Stop() + for { + mu.Lock() + if ep := present(); ep != nil { + mu.Unlock() + return ep, nil + } + wake := make(chan struct{}) + unsubscribe := subscribe(wake) + mu.Unlock() + + select { + case <-wake: + // Counterpart registered (or a spurious wake); loop and re-check. The + // wake() that closed this channel also cleared the set, so there is + // nothing to unsubscribe. + case <-deadline.C: + // Final check in case the counterpart raced in with the timer. + mu.Lock() + ep := present() + if ep != nil { + mu.Unlock() + return ep, nil + } + unsubscribe() + mu.Unlock() + return nil, fmt.Errorf("pairing timed out after %s", timeout) + case <-ctx.Done(): + mu.Lock() + unsubscribe() + mu.Unlock() + return nil, ctx.Err() + } + } +} + +// subscribeLocked adds ch to the tenant's waiter set and returns an unsubscribe +// closure that removes that specific channel (and prunes the now-empty tenant +// entry). Callers hold p.mu when calling this AND when invoking the returned +// closure. Removal is idempotent: a wake() that already closed+cleared the set +// makes the closure a no-op. +func (p *pairingTable) subscribeLocked(waiters map[string]map[chan struct{}]struct{}, tenant string, ch chan struct{}) func() { + set := waiters[tenant] + if set == nil { + set = map[chan struct{}]struct{}{} + waiters[tenant] = set + } + set[ch] = struct{}{} + return func() { + set, ok := waiters[tenant] + if !ok { + return + } + delete(set, ch) + if len(set) == 0 { + delete(waiters, tenant) + } + } +} + +// wake closes and clears every waiter channel registered for tenant. Callers +// hold p.mu. Closing wakes the waiter's select; the waiter re-checks the table +// under the lock, so a closed-but-stale wake is harmless. Clearing the tenant +// entry means the waiter's unsubscribe closure becomes a no-op (it must never +// close an already-closed channel). +func (p *pairingTable) wake(waiters map[string]map[chan struct{}]struct{}, tenant string) { + for ch := range waiters[tenant] { + close(ch) + } + delete(waiters, tenant) +} + +// addWatcher registers a watcher channel and returns it alongside a snapshot of +// the currently-online tenants for replay. The channel is buffered so a slow +// watcher does not block table mutations; if it fills, events are dropped for +// that watcher (it will reconverge on the next event or reconnect). +func (p *pairingTable) addWatcher() (chan *pb.TenantEvent, []string) { + p.mu.Lock() + defer p.mu.Unlock() + ch := make(chan *pb.TenantEvent, 64) + p.watchers = append(p.watchers, ch) + online := make([]string, 0, len(p.relays)) + for tenant := range p.relays { + online = append(online, tenant) + } + return ch, online +} + +// removeWatcher unregisters ch and closes it so the WatchTenants loop returns. +func (p *pairingTable) removeWatcher(ch chan *pb.TenantEvent) { + p.mu.Lock() + defer p.mu.Unlock() + for i, w := range p.watchers { + if w == ch { + p.watchers = append(p.watchers[:i], p.watchers[i+1:]...) + close(ch) + return + } + } +} + +// emitLocked fans an event out to every watcher. Callers hold p.mu. A full +// watcher channel drops the event (non-blocking send) rather than wedging the +// table under the lock. +func (p *pairingTable) emitLocked(ev *pb.TenantEvent) { + for _, ch := range p.watchers { + select { + case ch <- ev: + default: + log.Warn().Str("tenant", ev.GetTenant()).Bool("online", ev.GetOnline()). + Msg("aggregator: watcher channel full; dropping tenant event") + } + } +} + +// peerCertCN extracts the peer-cert CN from ctx, mirroring +// internal/gateway/identity.go:80-94 (peer.FromContext → credentials.TLSInfo → +// PeerCertificates[0].Subject.CommonName). It is a LOCAL copy by design: the +// design spec (decision 4) forbids importing internal/gateway from the +// proxysidecar layer. pkg/certident extraction is noted as a future cleanup. +func peerCertCN(ctx context.Context) (string, error) { + pr, ok := peer.FromContext(ctx) + if !ok { + return "", fmt.Errorf("no peer info in context") + } + tlsInfo, ok := pr.AuthInfo.(credentials.TLSInfo) + if !ok { + return "", fmt.Errorf("peer auth info is not TLS") + } + if len(tlsInfo.State.PeerCertificates) == 0 { + return "", fmt.Errorf("no client certificate") + } + return tlsInfo.State.PeerCertificates[0].Subject.CommonName, nil +} + +// tenantFromCN parses the tenant out of a peer-cert CN. CNs follow the canonical +// identity format {type}::{...} using the "::" separator (cf +// identity.go:104-243). A tenant-bearing service CN has the exact three-segment +// shape sv::sandbox-provider::, matching the tenant-relay cert layout +// proven in the gateway spike (spike_tenant_relay_test.go:190); the trailing +// segment is the tenant. +// +// The shape is validated strictly before the trailing segment is trusted: a +// greedy LastIndex("::") would silently treat a malformed or extra-segment CN +// (e.g. sv::sandbox-provider::a::b) as pinning tenant "b", an authz hazard. We +// therefore require exactly the known service-CN field count and a non-empty +// tenant segment. Anything else — a generic shared identity with no "::" (the +// provider may legitimately present one), or any unexpected shape — yields "" +// so the caller falls back to the metadata pairing path. The caller decides +// whether an empty tenant is acceptable (it is for the provider hop, not the +// relay hop). +func tenantFromCN(cn string) string { + cn = strings.TrimSpace(cn) + parts := strings.Split(cn, "::") + // Expect exactly: ["sv", "sandbox-provider", ""]. + if len(parts) != 3 || parts[0] != "sv" || parts[1] != "sandbox-provider" || parts[2] == "" { + return "" + } + return parts[2] +} + +// tenantFromMetadata reads the x-aether-tenant hint from a stream's incoming +// metadata (cf gateway/connect.go's metadata.FromIncomingContext usage). The +// bool reports whether the header was present at all (so RequireTenantMetadata +// can distinguish "absent" from "present but empty"). +func tenantFromMetadata(ctx context.Context) (string, bool) { + md, ok := metadata.FromIncomingContext(ctx) + if !ok { + return "", false + } + vals := md.Get(metadataTenantKey) + if len(vals) == 0 { + return "", false + } + return strings.TrimSpace(vals[0]), true +} diff --git a/server/internal/proxysidecar/aggregator_test.go b/server/internal/proxysidecar/aggregator_test.go new file mode 100644 index 0000000..f4bb4f9 --- /dev/null +++ b/server/internal/proxysidecar/aggregator_test.go @@ -0,0 +1,1050 @@ +package proxysidecar + +import ( + "context" + "crypto/rand" + "crypto/rsa" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "fmt" + "math/big" + "net" + "os" + "path/filepath" + "strings" + "testing" + "time" + + pb "github.com/scitrera/aether/api/proto" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/metadata" +) + +// ============================================================================= +// aggregator test harness. +// +// Two registered servers / two listeners (template: relaymux_test.go:214-239): +// one Aggregator instance backs BOTH the AetherGateway provider surface and the +// SandboxRelayTunnel relay surface, each on its own TCP listener. A fakeProvider +// dials the provider surface (sending x-aether-tenant metadata); a fakeRelay +// dials the tunnel surface (sending TunnelHello). For the pairing/splice/watch +// tests the surfaces run over insecure transport and the Aggregator's peerCN is +// stubbed to a fixed CN so the tenant resolves without real certs; one dedicated +// test (TestAggregator_CNValidation) exercises the real mTLS CN-binding path. +// ============================================================================= + +type aggHarness struct { + t *testing.T + agg *Aggregator + providerAddr string + tunnelAddr string + providerSrv *grpc.Server + tunnelSrv *grpc.Server + conns []*grpc.ClientConn + + // providerCN, when non-empty, overrides the CN the stubbed peerCN returns on + // the PROVIDER surface (/…AetherGateway/Connect) while the tunnel surface + // keeps tunnelCN. This lets a test give the provider a CN that encodes NO + // tenant (forcing the metadata-only pairing path) while the relay still + // presents a valid tenant-encoding CN. Set BEFORE any connect. + providerCN string + tunnelCN string +} + +// newAggHarness builds an insecure two-listener harness. stubCN is the CN the +// stubbed peerCN returns on BOTH surfaces by default. A test may set +// h.providerCN before connecting to give the provider surface a different CN +// (e.g. one that encodes no tenant) — see surfaceCN below. +func newAggHarness(t *testing.T, cfg *Config, stubCN string) *aggHarness { + t.Helper() + + provLis, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("listen provider: %v", err) + } + tunLis, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("listen tunnel: %v", err) + } + + cfg.Aggregator.Enabled = true + cfg.Aggregator.ProviderListen = "tcp://" + provLis.Addr().String() + cfg.Aggregator.TunnelListen = "tcp://" + tunLis.Addr().String() + cfg.Aggregator.ProviderTLS.Insecure = true + cfg.Aggregator.TunnelTLS.Insecure = true + if cfg.Aggregator.PairWaitTimeoutMs == 0 { + cfg.Aggregator.PairWaitTimeoutMs = 30_000 + } + if err := cfg.Validate(); err != nil { + t.Fatalf("validate cfg: %v", err) + } + + agg, err := NewAggregator(cfg) + if err != nil { + t.Fatalf("NewAggregator: %v", err) + } + + h := &aggHarness{ + t: t, + agg: agg, + providerAddr: provLis.Addr().String(), + tunnelAddr: tunLis.Addr().String(), + tunnelCN: stubCN, + } + // Stub the peer-cert CN so the insecure surfaces resolve a tenant. The CN is + // chosen per surface (see surfaceCN) so a test can give the provider hop a + // tenant-less CN while the relay hop keeps a tenant-encoding one. + agg.peerCN = func(ctx context.Context) (string, error) { return h.surfaceCN(ctx), nil } + + provSrv := grpc.NewServer() + pb.RegisterAetherGatewayServer(provSrv, agg) + go func() { _ = provSrv.Serve(provLis) }() + + tunSrv := grpc.NewServer() + pb.RegisterSandboxRelayTunnelServer(tunSrv, agg) + go func() { _ = tunSrv.Serve(tunLis) }() + + h.providerSrv = provSrv + h.tunnelSrv = tunSrv + t.Cleanup(func() { + for _, c := range h.conns { + _ = c.Close() + } + provSrv.Stop() + tunSrv.Stop() + }) + return h +} + +// surfaceCN returns the CN the stubbed peerCN reports for the calling surface. +// The provider surface (/…AetherGateway/…) reports providerCN when a test has +// set it (e.g. a tenant-less shared CN to force the metadata-only path) and +// otherwise tunnelCN; every other surface (the relay tunnel + watch) reports +// tunnelCN. grpc.Method reads the full method off the server-side stream ctx. +func (h *aggHarness) surfaceCN(ctx context.Context) string { + method, _ := grpc.Method(ctx) + if strings.Contains(method, "AetherGateway") && h.providerCN != "" { + return h.providerCN + } + return h.tunnelCN +} + +func (h *aggHarness) dial(addr string) *grpc.ClientConn { + h.t.Helper() + conn, err := grpc.NewClient("passthrough:///"+addr, grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + h.t.Fatalf("dial %s: %v", addr, err) + } + h.conns = append(h.conns, conn) + return conn +} + +// providerConnect opens an AetherGateway.Connect with the x-aether-tenant hint. +func (h *aggHarness) providerConnect(ctx context.Context, tenant string) pb.AetherGateway_ConnectClient { + h.t.Helper() + conn := h.dial(h.providerAddr) + ctx = metadata.AppendToOutgoingContext(ctx, metadataTenantKey, tenant) + stream, err := pb.NewAetherGatewayClient(conn).Connect(ctx) + if err != nil { + h.t.Fatalf("provider connect: %v", err) + } + return stream +} + +// relayTunnel opens a SandboxRelayTunnel.Tunnel and sends the opening hello. +func (h *aggHarness) relayTunnel(ctx context.Context, tenant string) pb.SandboxRelayTunnel_TunnelClient { + h.t.Helper() + conn := h.dial(h.tunnelAddr) + stream, err := pb.NewSandboxRelayTunnelClient(conn).Tunnel(ctx) + if err != nil { + h.t.Fatalf("relay tunnel: %v", err) + } + if err := stream.Send(&pb.TunnelFrame{F: &pb.TunnelFrame_Hello{Hello: &pb.TunnelHello{Tenant: tenant}}}); err != nil { + h.t.Fatalf("relay hello: %v", err) + } + return stream +} + +func (h *aggHarness) watchTenants(ctx context.Context) grpc.ServerStreamingClient[pb.TenantEvent] { + h.t.Helper() + // WatchTenants lives on the SandboxRelayTunnel service → dial the tunnel + // listener, not the provider (AetherGateway) one. + conn := h.dial(h.tunnelAddr) + w, err := pb.NewSandboxRelayTunnelClient(conn).WatchTenants(ctx, &pb.WatchTenantsRequest{}) + if err != nil { + h.t.Fatalf("watch tenants: %v", err) + } + return w +} + +// ============================================================================= +// 1. Pairing by tenant + 1:1 splice fidelity, BOTH directions. +// ============================================================================= + +func TestAggregator_PairAndSpliceBothDirections(t *testing.T) { + const tenant = "tenant-alice" + h := newAggHarness(t, &Config{}, "sv::sandbox-provider::"+tenant) + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + // Both sides connect; pairing is by tenant. + provider := h.providerConnect(ctx, tenant) + relay := h.relayTunnel(ctx, tenant) + + // provider→relay: an UpstreamMessage sent by the provider must arrive + // identical as TunnelFrame.up at the relay. + up := &pb.UpstreamMessage{Payload: &pb.UpstreamMessage_Init{Init: &pb.InitConnection{ + ResumeSessionId: "resume-xyz", + ClientType: &pb.InitConnection_Service{Service: &pb.ServiceIdentity{ + Implementation: "sandbox-provider", Specifier: "pod-7", + }}, + }}} + if err := provider.Send(up); err != nil { + t.Fatalf("provider send up: %v", err) + } + gotFrame, err := relay.Recv() + if err != nil { + t.Fatalf("relay recv: %v", err) + } + gotUp := gotFrame.GetUp() + if gotUp == nil { + t.Fatalf("relay got frame %T, want up", gotFrame.GetF()) + } + if gotUp.GetInit().GetResumeSessionId() != "resume-xyz" { + t.Fatalf("up resume_session_id = %q, want resume-xyz", gotUp.GetInit().GetResumeSessionId()) + } + if gotUp.GetInit().GetService().GetSpecifier() != "pod-7" { + t.Fatalf("up specifier = %q, want pod-7", gotUp.GetInit().GetService().GetSpecifier()) + } + + // relay→provider: a TunnelFrame.down sent by the relay must arrive identical + // as a DownstreamMessage at the provider. + down := &pb.DownstreamMessage{Payload: &pb.DownstreamMessage_ConnectionAck{ + ConnectionAck: &pb.ConnectionAck{SessionId: "gw-session-1", AssignedId: "sv::sandbox-provider::pod-7"}, + }} + if err := relay.Send(&pb.TunnelFrame{F: &pb.TunnelFrame_Down{Down: down}}); err != nil { + t.Fatalf("relay send down: %v", err) + } + gotDown, err := provider.Recv() + if err != nil { + t.Fatalf("provider recv: %v", err) + } + if gotDown.GetConnectionAck().GetSessionId() != "gw-session-1" { + t.Fatalf("down session_id = %q, want gw-session-1", gotDown.GetConnectionAck().GetSessionId()) + } + if gotDown.GetConnectionAck().GetAssignedId() != "sv::sandbox-provider::pod-7" { + t.Fatalf("down assigned_id = %q, want sv::sandbox-provider::pod-7", gotDown.GetConnectionAck().GetAssignedId()) + } +} + +// Provider presents a generic shared cert whose CN does NOT encode a tenant; the +// x-aether-tenant metadata supplies the pairing key. The relay's per-tenant CN +// is still the authority on its side. +// +// This genuinely drives the metadata-only branch: the provider surface reports a +// CN with NO "::" tenant segment ("sandbox-provider-shared"), so tenantFromCN +// returns "" on the provider hop and the cnTenant=="" path forces the pairing +// key to come from metadata. The relay surface still reports a valid +// tenant-encoding CN. (The earlier version of this test gave BOTH surfaces a +// tenant-encoding CN, so the CN branch was taken and the metadata-only path was +// never exercised — a false positive.) +func TestAggregator_ProviderMetadataOnlyPairs(t *testing.T) { + const tenant = "tenant-bravo" + h := newAggHarness(t, &Config{}, "sv::sandbox-provider::"+tenant) + // Provider hop presents a shared CN that encodes NO tenant → tenantFromCN=="" + // → the x-aether-tenant metadata is the only pairing key. + h.providerCN = "sandbox-provider-shared" + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + provider := h.providerConnect(ctx, tenant) + relay := h.relayTunnel(ctx, tenant) + + if err := provider.Send(&pb.UpstreamMessage{Payload: &pb.UpstreamMessage_Init{Init: &pb.InitConnection{}}}); err != nil { + t.Fatalf("provider send: %v", err) + } + if _, err := relay.Recv(); err != nil { + t.Fatalf("relay recv (metadata-only pairing did not happen): %v", err) + } +} + +// Sibling of the metadata-only case: with RequireTenantMetadata=true (the +// default) AND a provider CN that encodes no tenant, a Connect that omits the +// x-aether-tenant metadata header has no way to resolve a tenant and MUST be +// rejected (rather than silently parking or pairing the wrong tenant). +func TestAggregator_ProviderMissingMetadataRejected(t *testing.T) { + requireMeta := true + cfg := &Config{} + cfg.Aggregator.RequireTenantMetadata = &requireMeta + h := newAggHarness(t, cfg, "sv::sandbox-provider::tenant-bravo") + // Tenant-less provider CN: metadata is the only possible pairing key, and it + // is absent → reject. + h.providerCN = "sandbox-provider-shared" + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + // Connect WITHOUT the x-aether-tenant metadata header. + conn := h.dial(h.providerAddr) + stream, err := pb.NewAetherGatewayClient(conn).Connect(ctx) + if err != nil { + t.Fatalf("provider connect: %v", err) + } + + done := make(chan error, 1) + go func() { + for { + _, err := stream.Recv() + if err != nil { + done <- err + return + } + } + }() + select { + case err := <-done: + if err == nil { + t.Fatalf("expected provider Connect to be rejected when metadata is required but absent") + } + case <-time.After(5 * time.Second): + t.Fatalf("provider stream did not close on missing required metadata") + } +} + +// ============================================================================= +// 2. WatchTenants emits online when a relay registers and offline when it goes. +// ============================================================================= + +func TestAggregator_WatchTenantsOnlineOffline(t *testing.T) { + const tenant = "tenant-charlie" + h := newAggHarness(t, &Config{}, "sv::sandbox-provider::"+tenant) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + watch := h.watchTenants(ctx) + + // The WatchTenants client call returns before the server-side handler has + // run addWatcher; wait until the watcher is actually registered so the + // relay's online event is delivered live (rather than racing the replay + // window). The aggregator's pairingTable is in-process, so we can poll it. + waitForWatcherCount(t, h.agg.pairs, 1, 3*time.Second) + + // A SINGLE persistent reader goroutine drains the stream into events: gRPC + // client streams forbid concurrent Recv, so per-assertion readers would race + // and one would silently steal the other's event. + events := drainTenantEvents(watch) + + // Connect a relay → expect an online event. We use a cancellable per-relay + // context so we can disconnect it and observe the offline event. + relayCtx, relayCancel := context.WithCancel(ctx) + _ = h.relayTunnel(relayCtx, tenant) + + ev := nextEventByTenant(t, events, tenant, 5*time.Second) + if !ev.GetOnline() { + t.Fatalf("first event online = %v, want true", ev.GetOnline()) + } + + // Disconnect the relay → expect an offline event. + relayCancel() + ev = nextEventByTenant(t, events, tenant, 5*time.Second) + if ev.GetOnline() { + t.Fatalf("second event online = %v, want false", ev.GetOnline()) + } +} + +// TestAggregator_WatchTenantsSnapshotComplete asserts the resync-prune contract: +// a WatchTenants subscriber receives the currently-online tenants as a replay +// burst, then EXACTLY ONE terminal TenantEvent with SnapshotComplete==true, and +// every subsequent live online/offline transition carries SnapshotComplete==false +// (the zero value). The provider relies on this sentinel to prune stale tenants +// on a watch reconnect. +func TestAggregator_WatchTenantsSnapshotComplete(t *testing.T) { + const ( + preTenant = "tenant-hotel" // online BEFORE the watcher subscribes → replay + liveTenant = "tenant-india" // comes online AFTER the sentinel → live event + ) + h := newAggHarness(t, &Config{}, "sv::sandbox-provider::"+preTenant) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + // Register a relay for preTenant BEFORE the watcher subscribes so it appears + // in the replay burst (not as a live event). + preCtx, preCancel := context.WithCancel(ctx) + defer preCancel() + _ = h.relayTunnel(preCtx, preTenant) + waitForTenantOnline(t, h.agg.pairs, preTenant, 3*time.Second) + + watch := h.watchTenants(ctx) + events := drainTenantEvents(watch) + + // Phase 1: replay burst. preTenant must arrive as an online replay event with + // SnapshotComplete=false, in any order before the sentinel. + sawPreReplay := false + sentinel := nextEventUntilSentinel(t, events, 5*time.Second, func(ev *pb.TenantEvent) { + if ev.GetSnapshotComplete() { + t.Fatalf("replay event unexpectedly had SnapshotComplete=true: %+v", ev) + } + if ev.GetTenant() == preTenant { + if !ev.GetOnline() { + t.Fatalf("replay event for %q online=false, want true", preTenant) + } + sawPreReplay = true + } + }) + if !sawPreReplay { + t.Fatalf("never saw replay online event for pre-online tenant %q", preTenant) + } + + // Phase 2: the sentinel itself. Exactly one, SnapshotComplete=true, empty tenant. + if !sentinel.GetSnapshotComplete() { + t.Fatalf("sentinel SnapshotComplete=false, want true") + } + if sentinel.GetTenant() != "" { + t.Fatalf("sentinel tenant = %q, want empty", sentinel.GetTenant()) + } + + // Phase 3: live events. A relay coming online AFTER the sentinel must be a + // live online event with SnapshotComplete=false, then offline likewise. + // Switch the stub CN so the new relay's CN encodes liveTenant. + h.tunnelCN = "sv::sandbox-provider::" + liveTenant + liveCtx, liveCancel := context.WithCancel(ctx) + _ = h.relayTunnel(liveCtx, liveTenant) + + onEv := nextEventByTenant(t, events, liveTenant, 5*time.Second) + if !onEv.GetOnline() { + t.Fatalf("live event online = false, want true") + } + if onEv.GetSnapshotComplete() { + t.Fatalf("live online event had SnapshotComplete=true, want false") + } + + liveCancel() + offEv := nextEventByTenant(t, events, liveTenant, 5*time.Second) + if offEv.GetOnline() { + t.Fatalf("live event online = true, want false") + } + if offEv.GetSnapshotComplete() { + t.Fatalf("live offline event had SnapshotComplete=true, want false") + } +} + +// waitForTenantOnline blocks until the pairingTable has a registered relay for +// tenant, closing the relay-register vs watcher-subscribe race so the tenant is +// guaranteed to land in the replay burst rather than as a live event. +func waitForTenantOnline(t *testing.T, p *pairingTable, tenant string, timeout time.Duration) { + t.Helper() + deadline := time.After(timeout) + for { + p.mu.Lock() + _, ok := p.relays[tenant] + p.mu.Unlock() + if ok { + return + } + select { + case <-deadline: + t.Fatalf("timed out waiting for tenant %q relay to register", tenant) + case <-time.After(5 * time.Millisecond): + } + } +} + +// nextEventUntilSentinel reads events, invoking inspect on each non-sentinel +// (replay) event, and returns the FIRST event with SnapshotComplete==true. It +// fails if the stream closes or the deadline fires before the sentinel arrives. +func nextEventUntilSentinel(t *testing.T, events <-chan *pb.TenantEvent, timeout time.Duration, inspect func(*pb.TenantEvent)) *pb.TenantEvent { + t.Helper() + deadline := time.After(timeout) + for { + select { + case ev, ok := <-events: + if !ok { + t.Fatalf("watch stream closed before snapshot_complete sentinel") + } + if ev.GetSnapshotComplete() { + return ev + } + inspect(ev) + case <-deadline: + t.Fatalf("timed out waiting for snapshot_complete sentinel") + return nil + } + } +} + +// waitForWatcherCount blocks until the pairingTable has at least n registered +// watchers, closing the client/server registration race in the watch test. +func waitForWatcherCount(t *testing.T, p *pairingTable, n int, timeout time.Duration) { + t.Helper() + deadline := time.After(timeout) + for { + p.mu.Lock() + got := len(p.watchers) + p.mu.Unlock() + if got >= n { + return + } + select { + case <-deadline: + t.Fatalf("timed out waiting for %d watcher(s); have %d", n, got) + case <-time.After(5 * time.Millisecond): + } + } +} + +// drainTenantEvents starts ONE reader on the watch stream and forwards every +// received event onto a channel (closed when the stream ends/errors). +func drainTenantEvents(w grpc.ServerStreamingClient[pb.TenantEvent]) <-chan *pb.TenantEvent { + out := make(chan *pb.TenantEvent, 16) + go func() { + defer close(out) + for { + ev, err := w.Recv() + if err != nil { + return + } + out <- ev + } + }() + return out +} + +// nextEventByTenant reads from the drained channel until one event matches +// tenant or the deadline fires (events for other tenants are skipped). +func nextEventByTenant(t *testing.T, events <-chan *pb.TenantEvent, tenant string, timeout time.Duration) *pb.TenantEvent { + t.Helper() + deadline := time.After(timeout) + for { + select { + case ev, ok := <-events: + if !ok { + t.Fatalf("watch stream closed before tenant %q event", tenant) + } + if ev.GetTenant() == tenant { + return ev + } + case <-deadline: + t.Fatalf("timed out waiting for tenant %q event", tenant) + return nil + } + } +} + +// ============================================================================= +// 3. Pair-wait timeout fails cleanly when only one side connects. +// ============================================================================= + +func TestAggregator_PairWaitTimeout(t *testing.T) { + const tenant = "tenant-delta" + cfg := &Config{} + cfg.Aggregator.PairWaitTimeoutMs = 150 // small, no counterpart will arrive + h := newAggHarness(t, cfg, "sv::sandbox-provider::"+tenant) + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + // Only the provider connects; no relay ever arrives. + provider := h.providerConnect(ctx, tenant) + + // The provider's Recv should return an error once the pair-wait times out + // (the handler sends an error downstream then returns, closing the stream). + done := make(chan error, 1) + go func() { + for { + _, err := provider.Recv() + if err != nil { + done <- err + return + } + } + }() + select { + case err := <-done: + if err == nil { + t.Fatalf("expected non-nil error on timeout") + } + case <-time.After(5 * time.Second): + t.Fatalf("provider stream did not close after pair-wait timeout") + } +} + +// ============================================================================= +// 3b. Mid-session relay teardown unblocks the provider and unregisters both +// endpoints from the pairing table. +// ============================================================================= + +func TestAggregator_RelayTeardownMidSession(t *testing.T) { + const tenant = "tenant-golf" + h := newAggHarness(t, &Config{}, "sv::sandbox-provider::"+tenant) + + ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) + defer cancel() + + // Pair provider + relay. + provider := h.providerConnect(ctx, tenant) + relayCtx, relayCancel := context.WithCancel(ctx) + relay := h.relayTunnel(relayCtx, tenant) + + // Exchange one frame each direction to prove the splice is live before we + // tear the relay down. + if err := provider.Send(&pb.UpstreamMessage{Payload: &pb.UpstreamMessage_Init{Init: &pb.InitConnection{ResumeSessionId: "r1"}}}); err != nil { + t.Fatalf("provider send: %v", err) + } + gotUp, err := relay.Recv() + if err != nil { + t.Fatalf("relay recv up: %v", err) + } + if gotUp.GetUp().GetInit().GetResumeSessionId() != "r1" { + t.Fatalf("relay up resume = %q, want r1", gotUp.GetUp().GetInit().GetResumeSessionId()) + } + down := &pb.DownstreamMessage{Payload: &pb.DownstreamMessage_ConnectionAck{ConnectionAck: &pb.ConnectionAck{SessionId: "s1"}}} + if err := relay.Send(&pb.TunnelFrame{F: &pb.TunnelFrame_Down{Down: down}}); err != nil { + t.Fatalf("relay send down: %v", err) + } + if got, err := provider.Recv(); err != nil || got.GetConnectionAck().GetSessionId() != "s1" { + t.Fatalf("provider recv down: got=%v err=%v", got, err) + } + + // Cancel the relay side mid-session. + relayCancel() + + // The provider's Connect must return as the splice tears down. The relay→ + // provider pump errors immediately when the relay stream closes; the splice + // then cancels and bounded-drains the sibling provider→relay pump (which is + // parked in Recv) for up to its 5s drain cap before Connect returns. Allow + // for that drain bound plus margin — the assertion is that it returns at all + // (no hang), not instantaneously. + provDone := make(chan error, 1) + go func() { + for { + _, err := provider.Recv() + if err != nil { + provDone <- err + return + } + } + }() + select { + case <-provDone: + // returned — good (error or EOF both acceptable; the point is it did not + // hang past the bounded splice drain). + case <-time.After(8 * time.Second): + t.Fatalf("provider Connect did not return after relay teardown") + } + + // Both endpoints must be unregistered from the pairing table. + waitForTenantUnregistered(t, h.agg.pairs, tenant, 5*time.Second) +} + +// waitForTenantUnregistered polls the pairing table until neither a provider nor +// a relay nor a session remains for tenant, or the deadline fires. +func waitForTenantUnregistered(t *testing.T, p *pairingTable, tenant string, timeout time.Duration) { + t.Helper() + deadline := time.After(timeout) + for { + p.mu.Lock() + _, hasProv := p.providers[tenant] + _, hasRelay := p.relays[tenant] + _, hasSession := p.sessions[tenant] + p.mu.Unlock() + if !hasProv && !hasRelay && !hasSession { + return + } + select { + case <-deadline: + t.Fatalf("tenant %q still registered after teardown (provider=%v relay=%v session=%v)", tenant, hasProv, hasRelay, hasSession) + case <-time.After(5 * time.Millisecond): + } + } +} + +// ============================================================================= +// 4. CN / hello tenant mismatch on the tunnel side is rejected. +// ============================================================================= + +func TestAggregator_TunnelHelloCNMismatchRejected(t *testing.T) { + // peerCN reports tenant-echo; the relay's hello claims tenant-foxtrot. + h := newAggHarness(t, &Config{}, "sv::sandbox-provider::tenant-echo") + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + relay := h.relayTunnel(ctx, "tenant-foxtrot") + + // The handler rejects the mismatch and returns an error, closing the stream. + done := make(chan error, 1) + go func() { + _, err := relay.Recv() + done <- err + }() + select { + case err := <-done: + if err == nil { + t.Fatalf("expected tunnel stream to be rejected on CN/hello mismatch") + } + case <-time.After(5 * time.Second): + t.Fatalf("tunnel stream did not close on CN/hello mismatch") + } +} + +// ============================================================================= +// Dedicated mTLS test: the real peer-cert CN-binding path. +// +// Runs the tunnel surface with real mTLS (RequireAndVerifyClientCert + the +// in-test CA). A relay cert whose CN encodes the WRONG tenant (different from +// the hello) must be rejected by the CN-vs-hello validation in Tunnel — proving +// the CN, not the hello, is the authority. A matching cert is accepted. +// ============================================================================= + +func TestAggregator_CNValidation(t *testing.T) { + caCert, caKey := aggGenCA(t) + caPEM := aggCertPEM(t, caCert) + + caFile := filepath.Join(t.TempDir(), "ca.pem") + if err := writeFile(caFile, caPEM); err != nil { + t.Fatalf("write ca: %v", err) + } + serverCertFile, serverKeyFile := aggWriteLeaf(t, "agg-server", caCert, caKey, true) + + cfg := &Config{} + cfg.Aggregator.Enabled = true + cfg.Aggregator.ProviderListen = "tcp://127.0.0.1:0" // unused in this test, but required by Validate + tunLis, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("listen tunnel: %v", err) + } + cfg.Aggregator.TunnelListen = "tcp://" + tunLis.Addr().String() + cfg.Aggregator.ProviderTLS.Insecure = true + cfg.Aggregator.TunnelTLS.CertFile = serverCertFile + cfg.Aggregator.TunnelTLS.KeyFile = serverKeyFile + cfg.Aggregator.TunnelTLS.ClientCAFile = caFile + cfg.Aggregator.PairWaitTimeoutMs = 30_000 + if err := cfg.Validate(); err != nil { + t.Fatalf("validate cfg: %v", err) + } + + agg, err := NewAggregator(cfg) + if err != nil { + t.Fatalf("NewAggregator: %v", err) + } + // Use the REAL peerCertCN so the cert CN flows through the live handshake. + creds, err := agg.serverCreds(cfg.Aggregator.TunnelTLS) + if err != nil { + t.Fatalf("server creds: %v", err) + } + tunSrv := grpc.NewServer(grpc.Creds(creds)) + pb.RegisterSandboxRelayTunnelServer(tunSrv, agg) + go func() { _ = tunSrv.Serve(tunLis) }() + t.Cleanup(tunSrv.Stop) + + addr := tunLis.Addr().String() + + // (a) Mismatch: relay cert CN pins tenant-mike, hello claims tenant-november. + mismatchCert := aggGenLeafTLS(t, "sv::sandbox-provider::tenant-mike", caCert, caKey) + if err := aggTunnelHelloOnce(t, addr, mismatchCert, "tenant-november"); err == nil { + t.Fatalf("expected CN/hello mismatch to be rejected over real mTLS") + } + + // (b) Match: relay cert CN pins tenant-mike and hello agrees → no rejection + // (the stream stays open; we then time out waiting for a provider, which is + // fine — we only assert the handshake/hello was NOT rejected). + matchCert := aggGenLeafTLS(t, "sv::sandbox-provider::tenant-mike", caCert, caKey) + if err := aggTunnelHelloOnce(t, addr, matchCert, "tenant-mike"); err != nil { + // A matching CN must NOT produce a CN-mismatch rejection. The only + // acceptable "error" is the deadline/EOF from no provider arriving, + // which aggTunnelHelloOnce maps to nil. + t.Fatalf("matching CN was rejected: %v", err) + } +} + +// ============================================================================= +// Dedicated mTLS test: the provider-side CN-vs-metadata binding. +// +// Runs the PROVIDER surface with real mTLS. A provider cert whose CN encodes +// tenant-A while the x-aether-tenant metadata claims tenant-B must be rejected +// by the CN-vs-metadata validation in Connect (the CN, when it pins a tenant, is +// the authority and a mismatch is an authz violation). A provider cert whose CN +// AGREES with the metadata is accepted (no CN-mismatch rejection). +// ============================================================================= + +func TestAggregator_ProviderCNMetadataMismatch(t *testing.T) { + caCert, caKey := aggGenCA(t) + caPEM := aggCertPEM(t, caCert) + + caFile := filepath.Join(t.TempDir(), "ca.pem") + if err := writeFile(caFile, caPEM); err != nil { + t.Fatalf("write ca: %v", err) + } + serverCertFile, serverKeyFile := aggWriteLeaf(t, "agg-server", caCert, caKey, true) + + cfg := &Config{} + cfg.Aggregator.Enabled = true + provLis, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("listen provider: %v", err) + } + cfg.Aggregator.ProviderListen = "tcp://" + provLis.Addr().String() + cfg.Aggregator.TunnelListen = "tcp://127.0.0.1:0" // unused here, but required by Validate + cfg.Aggregator.TunnelTLS.Insecure = true + cfg.Aggregator.ProviderTLS.CertFile = serverCertFile + cfg.Aggregator.ProviderTLS.KeyFile = serverKeyFile + cfg.Aggregator.ProviderTLS.ClientCAFile = caFile + cfg.Aggregator.PairWaitTimeoutMs = 30_000 + if err := cfg.Validate(); err != nil { + t.Fatalf("validate cfg: %v", err) + } + + agg, err := NewAggregator(cfg) + if err != nil { + t.Fatalf("NewAggregator: %v", err) + } + // Use the REAL peerCertCN so the cert CN flows through the live handshake. + creds, err := agg.serverCreds(cfg.Aggregator.ProviderTLS) + if err != nil { + t.Fatalf("server creds: %v", err) + } + provSrv := grpc.NewServer(grpc.Creds(creds)) + pb.RegisterAetherGatewayServer(provSrv, agg) + go func() { _ = provSrv.Serve(provLis) }() + t.Cleanup(provSrv.Stop) + + addr := provLis.Addr().String() + + // (a) Mismatch: provider cert CN pins tenant-alpha, metadata claims tenant-beta. + mismatchCert := aggGenLeafTLS(t, "sv::sandbox-provider::tenant-alpha", caCert, caKey) + if err := aggProviderConnectOnce(t, addr, mismatchCert, "tenant-beta"); err == nil { + t.Fatalf("expected provider CN/metadata mismatch to be rejected over real mTLS") + } + + // (b) Match: provider cert CN pins tenant-alpha and metadata agrees → no + // CN-mismatch rejection (the stream parks waiting for a relay, which the + // helper masks to nil). + matchCert := aggGenLeafTLS(t, "sv::sandbox-provider::tenant-alpha", caCert, caKey) + if err := aggProviderConnectOnce(t, addr, matchCert, "tenant-alpha"); err != nil { + t.Fatalf("matching provider CN/metadata was rejected: %v", err) + } +} + +// aggProviderConnectOnce dials the provider surface over mTLS with clientCert, +// opens Connect with the x-aether-tenant metadata set to metaTenant, and reports +// whether the server rejected it. Mirrors aggTunnelHelloOnce: a genuine +// server-side rejection returns a status before the deadline; the match case +// parks waiting for a relay and hits the client deadline, which is masked to nil. +func aggProviderConnectOnce(t *testing.T, addr string, clientCert tls.Certificate, metaTenant string) error { + t.Helper() + tlsCreds := credentials.NewTLS(&tls.Config{ + Certificates: []tls.Certificate{clientCert}, + InsecureSkipVerify: true, // server identity not under test + }) + conn, err := grpc.NewClient("passthrough:///"+addr, grpc.WithTransportCredentials(tlsCreds)) + if err != nil { + t.Fatalf("dial: %v", err) + } + defer func() { _ = conn.Close() }() + + ctx, cancel := context.WithTimeout(context.Background(), 1500*time.Millisecond) + defer cancel() + ctx = metadata.AppendToOutgoingContext(ctx, metadataTenantKey, metaTenant) + + stream, err := pb.NewAetherGatewayClient(conn).Connect(ctx) + if err != nil { + return err + } + // The provider surface rejects by FIRST sending a DownstreamMessage carrying + // an ErrorResponse and THEN returning a non-nil status, so the client sees a + // valid error frame (err==nil) before any stream error. Treat an error frame + // as a rejection. The match case never receives a frame (no relay) and hits + // the client deadline → "not rejected" (nil). + msg, err := stream.Recv() + if err == nil { + if msg.GetError() != nil { + return fmt.Errorf("provider rejected: %s: %s", msg.GetError().GetCode(), msg.GetError().GetMessage()) + } + return nil + } + if ctx.Err() == context.DeadlineExceeded { + return nil + } + return err +} + +// aggTunnelHelloOnce dials the tunnel surface over mTLS with clientCert, sends a +// TunnelHello for helloTenant, and reports whether the server rejected it. A +// rejection surfaces as a non-nil Recv error that is NOT a deadline (the +// no-provider pair-wait path is masked to nil so the match case is clean). +func aggTunnelHelloOnce(t *testing.T, addr string, clientCert tls.Certificate, helloTenant string) error { + t.Helper() + tlsCreds := credentials.NewTLS(&tls.Config{ + Certificates: []tls.Certificate{clientCert}, + InsecureSkipVerify: true, // server identity not under test + }) + conn, err := grpc.NewClient("passthrough:///"+addr, grpc.WithTransportCredentials(tlsCreds)) + if err != nil { + t.Fatalf("dial: %v", err) + } + defer func() { _ = conn.Close() }() + + // Bound the call: the match case parks waiting for a provider, so cap it. + ctx, cancel := context.WithTimeout(context.Background(), 1500*time.Millisecond) + defer cancel() + + stream, err := pb.NewSandboxRelayTunnelClient(conn).Tunnel(ctx) + if err != nil { + return err + } + if err := stream.Send(&pb.TunnelFrame{F: &pb.TunnelFrame_Hello{Hello: &pb.TunnelHello{Tenant: helloTenant}}}); err != nil { + return err + } + _, err = stream.Recv() + // The match case never receives a frame (no provider) and instead hits the + // client deadline → treat that as "not rejected" (nil). A genuine server + // rejection returns a server-originated status before the deadline. + if err != nil && ctx.Err() == context.DeadlineExceeded { + return nil + } + return err +} + +// --------------------------------------------------------------------------- +// In-test CA / cert helpers (minimal inline gen, same approach as +// spike_tenant_relay_test.go's spikeGenCA/spikeGenLeaf). +// --------------------------------------------------------------------------- + +func aggGenCA(t *testing.T) (*x509.Certificate, *rsa.PrivateKey) { + t.Helper() + key, err := rsa.GenerateKey(rand.Reader, 2048) + if err != nil { + t.Fatalf("ca key: %v", err) + } + tmpl := &x509.Certificate{ + SerialNumber: big.NewInt(1), + Subject: pkix.Name{CommonName: "agg-test-ca"}, + NotBefore: time.Now().Add(-time.Hour), + NotAfter: time.Now().Add(24 * time.Hour), + IsCA: true, + KeyUsage: x509.KeyUsageCertSign | x509.KeyUsageCRLSign, + BasicConstraintsValid: true, + } + der, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, &key.PublicKey, key) + if err != nil { + t.Fatalf("ca cert: %v", err) + } + cert, err := x509.ParseCertificate(der) + if err != nil { + t.Fatalf("parse ca: %v", err) + } + return cert, key +} + +// aggGenLeafTLS returns a CA-signed client leaf as a tls.Certificate (for dial). +func aggGenLeafTLS(t *testing.T, cn string, ca *x509.Certificate, caKey *rsa.PrivateKey) tls.Certificate { + t.Helper() + der, key := aggGenLeafDER(t, cn, ca, caKey, false) + leaf, err := x509.ParseCertificate(der) + if err != nil { + t.Fatalf("parse leaf: %v", err) + } + return tls.Certificate{Certificate: [][]byte{der}, PrivateKey: key, Leaf: leaf} +} + +// aggWriteLeaf writes a CA-signed leaf cert+key to temp files and returns their +// paths (for loading server creds from disk via serverCreds). +func aggWriteLeaf(t *testing.T, cn string, ca *x509.Certificate, caKey *rsa.PrivateKey, server bool) (certFile, keyFile string) { + t.Helper() + der, key := aggGenLeafDER(t, cn, ca, caKey, server) + certPEM := pemBlock("CERTIFICATE", der) + keyPEM := pemBlock("RSA PRIVATE KEY", x509.MarshalPKCS1PrivateKey(key)) + dir := t.TempDir() + certFile = filepath.Join(dir, "leaf.crt") + keyFile = filepath.Join(dir, "leaf.key") + if err := writeFile(certFile, certPEM); err != nil { + t.Fatalf("write cert: %v", err) + } + if err := writeFile(keyFile, keyPEM); err != nil { + t.Fatalf("write key: %v", err) + } + return certFile, keyFile +} + +func aggGenLeafDER(t *testing.T, cn string, ca *x509.Certificate, caKey *rsa.PrivateKey, server bool) ([]byte, *rsa.PrivateKey) { + t.Helper() + key, err := rsa.GenerateKey(rand.Reader, 2048) + if err != nil { + t.Fatalf("leaf key: %v", err) + } + eku := x509.ExtKeyUsageClientAuth + if server { + eku = x509.ExtKeyUsageServerAuth + } + tmpl := &x509.Certificate{ + SerialNumber: big.NewInt(time.Now().UnixNano()), + Subject: pkix.Name{CommonName: cn}, + NotBefore: time.Now().Add(-time.Hour), + NotAfter: time.Now().Add(24 * time.Hour), + KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment, + ExtKeyUsage: []x509.ExtKeyUsage{eku}, + BasicConstraintsValid: true, + } + if server { + tmpl.IPAddresses = []net.IP{net.ParseIP("127.0.0.1")} + tmpl.DNSNames = []string{"localhost"} + } + der, err := x509.CreateCertificate(rand.Reader, tmpl, ca, &key.PublicKey, caKey) + if err != nil { + t.Fatalf("leaf cert: %v", err) + } + return der, key +} + +func aggCertPEM(t *testing.T, cert *x509.Certificate) []byte { + t.Helper() + return pemBlock("CERTIFICATE", cert.Raw) +} + +func pemBlock(typ string, der []byte) []byte { + return pem.EncodeToMemory(&pem.Block{Type: typ, Bytes: der}) +} + +func writeFile(path string, data []byte) error { + return os.WriteFile(path, data, 0o600) +} + +// tenantFromCN unit coverage: only the exact sv::sandbox-provider:: +// service-CN shape yields a tenant; every other (malformed / extra-segment / +// tenant-less) shape yields "" so the metadata path engages and no wrong tenant +// is ever trusted (the hardening in finding #4). +func TestAggregator_TenantFromCN(t *testing.T) { + cases := map[string]string{ + // Canonical service CN → trailing segment is the tenant. + "sv::sandbox-provider::tenant-alice": "tenant-alice", + "sv::sandbox-provider::pod-shared": "pod-shared", + " sv::sandbox-provider::trimmed ": "trimmed", // surrounding space is trimmed + // Tenant-less shared identity (no "::") → "" (provider metadata path). + "no-separator": "", + "sandbox-provider-shared": "", + "": "", + // Malformed / extra-segment shapes must NOT yield a tenant (authz hazard + // the greedy LastIndex previously had): the old code would have returned + // "b" / "extra" here. + "sv::sandbox-provider::a::b": "", // too many segments + "sv::sandbox-provider::tenant::x": "", // too many segments + "sv::sandbox-provider::": "", // empty tenant segment + "x::sandbox-provider::tenant": "", // wrong type segment + "sv::other-service::tenant": "", // wrong service segment + "sv::sandbox-provider": "", // missing tenant segment + "a::b": "", // unrelated two-segment CN + } + for cn, want := range cases { + if got := tenantFromCN(cn); got != want { + t.Fatalf("tenantFromCN(%q) = %q, want %q", cn, got, want) + } + } +} diff --git a/server/internal/proxysidecar/composite_test.go b/server/internal/proxysidecar/composite_test.go index ffea704..c52edd2 100644 --- a/server/internal/proxysidecar/composite_test.go +++ b/server/internal/proxysidecar/composite_test.go @@ -198,7 +198,7 @@ func newCompositeHarness(t *testing.T, backendHandler http.HandlerFunc) *composi var gwStream *compositeFakeStream select { case gwStream = <-gw.streamCh: - case <-time.After(3 * time.Second): + case <-time.After(15 * time.Second): cancel() t.Fatalf("runner runtime never connected to fake gateway") } @@ -700,6 +700,425 @@ func TestSharedRelaySink_AttachRejectsBeyondMaxSessions(t *testing.T) { } } +// ============================================================================= +// Composite RelayMux regression tests. +// +// These tests exercise the relaymux composite path (terminator + RelayMux on +// one shared runtime) through the full Runner stack. +// ============================================================================= + +// compositeDialSandbox opens a relay sub-client through the composite harness, +// sends an Init, and drains the synthesized ConnectionAck. +func compositeDialSandbox(t *testing.T, h *compositeHarness, specifier string) pb.AetherGateway_ConnectClient { + t.Helper() + stream, err := h.sandboxCli.Connect(context.Background()) + if err != nil { + t.Fatalf("sandbox dial (%s): %v", specifier, err) + } + if err := stream.Send(&pb.UpstreamMessage{ + Payload: &pb.UpstreamMessage_Init{ + Init: &pb.InitConnection{ + ClientType: &pb.InitConnection_Agent{ + Agent: &pb.AgentIdentity{Workspace: "ws", Implementation: "i", Specifier: specifier}, + }, + }, + }, + }); err != nil { + t.Fatalf("init send (%s): %v", specifier, err) + } + ack, err := stream.Recv() + if err != nil { + t.Fatalf("ack recv (%s): %v", specifier, err) + } + if _, ok := ack.GetPayload().(*pb.DownstreamMessage_ConnectionAck); !ok { + t.Fatalf("ack (%s): expected ConnectionAck, got %T", specifier, ack.GetPayload()) + } + return stream +} + +// waitUpstreamFrame blocks until pred returns true for one of the recorded +// upstream frames, or the deadline passes. +func waitUpstreamFrame(t *testing.T, h *compositeHarness, pred func(*pb.UpstreamMessage) bool) *pb.UpstreamMessage { + t.Helper() + deadline := time.Now().Add(3 * time.Second) + for time.Now().Before(deadline) { + for _, m := range h.gatewayStream.snapshot() { + if pred(m) { + return m + } + } + time.Sleep(10 * time.Millisecond) + } + return nil +} + +// TestComposite_RelayMux_CollidingRequestIDs is the regression test for the +// pinned production bug: two sandbox sub-clients both issue a KV op with the +// SAME origin request_id "req-1" (as the SDK's BaseClient.NextRequestID() +// mints per-client). In the old sharedRelaySink flat-map each sub-client's +// registerRequest overwrote the other's entry, so only one got its reply. +// RelayMux rewrites each to a globally-unique "rmx-N" id before forwarding +// and restores "req-1" on the reply, routing each reply to its originating +// sub-client. +func TestComposite_RelayMux_CollidingRequestIDs(t *testing.T) { + t.Parallel() + + h := newCompositeHarness(t, nil) + + s1 := compositeDialSandbox(t, h, "agent-1") + s2 := compositeDialSandbox(t, h, "agent-2") + + // Both sub-clients send a KV op with the SAME origin request_id "req-1". + kvGet := func(key string) *pb.UpstreamMessage { + return &pb.UpstreamMessage{Payload: &pb.UpstreamMessage_KvOp{KvOp: &pb.KVOperation{ + RequestId: "req-1", + Op: pb.KVOperation_GET, + Key: key, + }}} + } + if err := s1.Send(kvGet("k1")); err != nil { + t.Fatalf("s1 send: %v", err) + } + if err := s2.Send(kvGet("k2")); err != nil { + t.Fatalf("s2 send: %v", err) + } + + // Wait until both KV ops appear upstream with distinct rmx-* ids. + var muxIDk1, muxIDk2 string + deadline := time.Now().Add(3 * time.Second) + for time.Now().Before(deadline) { + for _, m := range h.gatewayStream.snapshot() { + kv, ok := m.GetPayload().(*pb.UpstreamMessage_KvOp) + if !ok { + continue + } + if kv.KvOp.GetKey() == "k1" { + muxIDk1 = kv.KvOp.GetRequestId() + } + if kv.KvOp.GetKey() == "k2" { + muxIDk2 = kv.KvOp.GetRequestId() + } + } + if muxIDk1 != "" && muxIDk2 != "" { + break + } + time.Sleep(10 * time.Millisecond) + } + if muxIDk1 == "" || muxIDk2 == "" { + t.Fatalf("upstream did not receive both KV ops (k1=%q k2=%q)", muxIDk1, muxIDk2) + } + if muxIDk1 == "req-1" || muxIDk2 == "req-1" { + t.Fatalf("request_id was not rewritten (still req-1): k1=%q k2=%q", muxIDk1, muxIDk2) + } + if muxIDk1 == muxIDk2 { + t.Fatalf("both sub-clients got the same mux id %q; expected distinct ids", muxIDk1) + } + + // Inject replies in swapped order (k2 first, then k1) via the gateway + // stream. The mux must restore "req-1" and route each to the right sub-client. + kvReply := func(muxID, val string) *pb.DownstreamMessage { + return &pb.DownstreamMessage{Payload: &pb.DownstreamMessage_Kv{Kv: &pb.KVResponse{ + RequestId: muxID, + Success: true, + Value: []byte(val), + }}} + } + if err := h.gatewayStream.server.Send(kvReply(muxIDk2, "v2")); err != nil { + t.Fatalf("inject reply k2: %v", err) + } + if err := h.gatewayStream.server.Send(kvReply(muxIDk1, "v1")); err != nil { + t.Fatalf("inject reply k1: %v", err) + } + + // s1 must get v1, s2 must get v2, both with request_id restored to "req-1". + got1 := compositeRecvKV(t, s1) + got2 := compositeRecvKV(t, s2) + if got1.GetRequestId() != "req-1" || string(got1.GetValue()) != "v1" { + t.Fatalf("s1 got req=%q val=%q; want req-1/v1", got1.GetRequestId(), string(got1.GetValue())) + } + if got2.GetRequestId() != "req-1" || string(got2.GetValue()) != "v2" { + t.Fatalf("s2 got req=%q val=%q; want req-1/v2", got2.GetRequestId(), string(got2.GetValue())) + } +} + +// TestComposite_RelayMux_BroadcastToSubClients verifies that gateway push +// frames (TaskAssignment) are broadcast to all relay sub-clients in composite +// mode, while the terminator surface is unaffected. +func TestComposite_RelayMux_BroadcastToSubClients(t *testing.T) { + t.Parallel() + + h := newCompositeHarness(t, nil) + + s1 := compositeDialSandbox(t, h, "agent-1") + s2 := compositeDialSandbox(t, h, "agent-2") + + push := &pb.DownstreamMessage{Payload: &pb.DownstreamMessage_TaskAssignment{ + TaskAssignment: &pb.TaskAssignment{TaskId: "composite-task-xyz"}, + }} + if err := h.gatewayStream.server.Send(push); err != nil { + t.Fatalf("inject push: %v", err) + } + + for i, s := range []pb.AetherGateway_ConnectClient{s1, s2} { + msg := compositeRecvWithTimeout(t, s) + ta, ok := msg.GetPayload().(*pb.DownstreamMessage_TaskAssignment) + if !ok { + t.Fatalf("sub %d got %T; want TaskAssignment", i+1, msg.GetPayload()) + } + if ta.TaskAssignment.GetTaskId() != "composite-task-xyz" { + t.Fatalf("sub %d task id = %q; want composite-task-xyz", i+1, ta.TaskAssignment.GetTaskId()) + } + } +} + +// TestComposite_RelayMux_TerminatorProxyHttpUnaffected confirms that the +// terminator's own inbound ProxyHttpRequest still round-trips through the +// shared connection when RelayMux sub-clients are concurrently attached. +// This guards against the rawDownstreamTap accidentally swallowing the +// terminator's ProxyHttpResponse wakeup. +func TestComposite_RelayMux_TerminatorProxyHttpUnaffected(t *testing.T) { + t.Parallel() + + h := newCompositeHarness(t, func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("X-Relay-Mux", "ok") + w.WriteHeader(200) + _, _ = io.WriteString(w, "mux-ok") + }) + + // Attach two relay sub-clients to prove the terminator path is independent. + _ = compositeDialSandbox(t, h, "agent-1") + _ = compositeDialSandbox(t, h, "agent-2") + + // Wait for the runtime Init to confirm the connection is live. + if waitUpstreamFrame(t, h, func(m *pb.UpstreamMessage) bool { + _, ok := m.GetPayload().(*pb.UpstreamMessage_Init) + return ok + }) == nil { + t.Fatal("runtime never sent InitConnection") + } + + // Gateway sends an inbound ProxyHttpRequest to the terminator. + if err := h.gatewayStream.server.Send(&pb.DownstreamMessage{ + Payload: &pb.DownstreamMessage_ProxyHttpRequest{ + ProxyHttpRequest: &pb.ProxyHttpRequest{ + RequestId: "term-req-mux-1", + Method: "GET", + Path: "/mux-ping", + }, + }, + }); err != nil { + t.Fatalf("inject ProxyHttpRequest: %v", err) + } + + // The terminator must emit a ProxyHttpResponse upstream. + resp := waitUpstreamFrame(t, h, func(m *pb.UpstreamMessage) bool { + r, ok := m.GetPayload().(*pb.UpstreamMessage_ProxyHttpResponse) + return ok && r.ProxyHttpResponse.GetRequestId() == "term-req-mux-1" + }) + if resp == nil { + t.Fatal("terminator never emitted ProxyHttpResponse for term-req-mux-1") + } + r := resp.GetPayload().(*pb.UpstreamMessage_ProxyHttpResponse).ProxyHttpResponse + if r.GetStatusCode() != 200 { + t.Errorf("proxy response status = %d; want 200", r.GetStatusCode()) + } + if string(r.GetBody()) != "mux-ok" { + t.Errorf("proxy response body = %q; want mux-ok", string(r.GetBody())) + } +} + +// compositeRecvKV receives the next downstream frame and asserts it is a KV +// response, returning the inner KVResponse. +func compositeRecvKV(t *testing.T, s pb.AetherGateway_ConnectClient) *pb.KVResponse { + t.Helper() + msg := compositeRecvWithTimeout(t, s) + kv, ok := msg.GetPayload().(*pb.DownstreamMessage_Kv) + if !ok { + t.Fatalf("got %T; want KVResponse", msg.GetPayload()) + } + return kv.Kv +} + +// compositeRecvWithTimeout receives one downstream frame with a 3s deadline. +func compositeRecvWithTimeout(t *testing.T, s pb.AetherGateway_ConnectClient) *pb.DownstreamMessage { + t.Helper() + type res struct { + msg *pb.DownstreamMessage + err error + } + ch := make(chan res, 1) + go func() { + m, err := s.Recv() + ch <- res{m, err} + }() + select { + case r := <-ch: + if r.err != nil { + t.Fatalf("recv: %v", r.err) + } + return r.msg + case <-time.After(3 * time.Second): + t.Fatalf("timed out waiting for downstream frame") + return nil + } +} + +// startRunnerForIdentity spins up a fake gateway + an httptest backend + a +// Runner (terminator + relay enabled) whose Service.Workspace is `workspace`. +// It waits for the runtime to dial the fake gateway and returns the recorded +// upstream stream. An empty workspace exercises the workspace-less Service +// identity path (sv::{impl}::{spec}); a non-empty workspace exercises the +// Agent identity path (ag::{ws}::{impl}::{spec}). Teardown is registered via +// t.Cleanup. Mirrors newCompositeHarness but parameterised on the identity +// discriminator and without a sandbox client (the assertions only need the +// runtime's own InitConnection). +func startRunnerForIdentity(t *testing.T, workspace string) *compositeFakeStream { + t.Helper() + + gw := newCompositeFakeGateway() + gwLis, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("listen gateway: %v", err) + } + gwSrv := grpc.NewServer() + pb.RegisterAetherGatewayServer(gwSrv, gw) + go func() { _ = gwSrv.Serve(gwLis) }() + + backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(200) + _, _ = io.WriteString(w, `{"ok":true}`) + })) + + relayPath := filepath.Join(t.TempDir(), "relay.sock") + cfg := &Config{ + Gateway: GatewayConfig{Address: gwLis.Addr().String(), Insecure: true}, + Service: ServiceConfig{Workspace: workspace, Implementation: "sidecar", Specifier: "instance-1"}, + Terminator: TerminatorConfig{ + Enabled: true, + Backends: []BackendConfig{{ + Name: "default", + Kind: BackendKindHTTP, + URL: backend.URL, + AllowPaths: []string{"/*"}, + AllowMethods: []string{"GET", "POST", "PUT", "DELETE"}, + MaxBodyBytes: 1 << 20, + HeaderMode: HeaderModePassthrough, + }}, + }, + Relay: RelayConfig{ + Enabled: true, + Listen: "unix://" + relayPath, + AllowedOps: AllowedOpsConfig{ + Profile: AllowedOpsProfileSandboxDefault, + Set: true, + }, + }, + TenantID: "tenant-test", + } + + runner, err := NewRunner(cfg, "") + if err != nil { + t.Fatalf("NewRunner: %v", err) + } + + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan struct{}) + go func() { + defer close(done) + _ = runner.Run(ctx) + }() + + var gwStream *compositeFakeStream + select { + case gwStream = <-gw.streamCh: + case <-time.After(15 * time.Second): + cancel() + t.Fatalf("runner runtime never connected to fake gateway") + } + + t.Cleanup(func() { + cancel() + select { + case <-done: + case <-time.After(3 * time.Second): + } + gwSrv.GracefulStop() + _ = gwLis.Close() + backend.Close() + }) + + return gwStream +} + +// waitForInit blocks until the runtime's first upstream InitConnection is +// recorded on the stream (the runner always sends InitConnection as its first +// upstream frame), or the deadline passes. +func waitForInit(t *testing.T, s *compositeFakeStream) *pb.InitConnection { + t.Helper() + deadline := time.Now().Add(3 * time.Second) + for time.Now().Before(deadline) { + if m := s.findUpstream(func(m *pb.UpstreamMessage) bool { + _, ok := m.GetPayload().(*pb.UpstreamMessage_Init) + return ok + }); m != nil { + return m.GetPayload().(*pb.UpstreamMessage_Init).Init + } + time.Sleep(20 * time.Millisecond) + } + t.Fatal("runtime never sent InitConnection to fake gateway") + return nil +} + +// TestRunner_AgentIdentityPath proves the AGENT identity path: with +// Service.Workspace set, gatewayRuntime.init() builds an AgentClient, so the +// FIRST upstream InitConnection carries an Agent identity keyed to that +// workspace (ag::{ws}::{impl}::{spec}). This is the counterpart to the +// workspace-less Service path exercised by TestRunner_ServiceIdentityPath and +// the rest of the composite suite. +func TestRunner_AgentIdentityPath(t *testing.T) { + t.Parallel() + + stream := startRunnerForIdentity(t, "ws-1") + init := waitForInit(t, stream) + + agent, ok := init.GetClientType().(*pb.InitConnection_Agent) + if !ok { + t.Fatalf("InitConnection client_type = %T; want Agent (workspace was set)", init.GetClientType()) + } + if agent.Agent.GetWorkspace() != "ws-1" { + t.Errorf("agent workspace = %q; want ws-1", agent.Agent.GetWorkspace()) + } + if agent.Agent.GetImplementation() != "sidecar" { + t.Errorf("agent implementation = %q; want sidecar", agent.Agent.GetImplementation()) + } + if agent.Agent.GetSpecifier() != "instance-1" { + t.Errorf("agent specifier = %q; want instance-1", agent.Agent.GetSpecifier()) + } +} + +// TestRunner_ServiceIdentityPath proves the SERVICE identity path is distinct +// from the agent path: with an empty Service.Workspace, gatewayRuntime.init() +// builds a ServiceClient, so the FIRST upstream InitConnection carries a +// workspace-less Service identity (sv::{impl}::{spec}) — the ServiceIdentity +// proto has no workspace concept. +func TestRunner_ServiceIdentityPath(t *testing.T) { + t.Parallel() + + stream := startRunnerForIdentity(t, "") + init := waitForInit(t, stream) + + svc, ok := init.GetClientType().(*pb.InitConnection_Service) + if !ok { + t.Fatalf("InitConnection client_type = %T; want Service (workspace was empty)", init.GetClientType()) + } + if svc.Service.GetImplementation() != "sidecar" { + t.Errorf("service implementation = %q; want sidecar", svc.Service.GetImplementation()) + } + if svc.Service.GetSpecifier() != "instance-1" { + t.Errorf("service specifier = %q; want instance-1", svc.Service.GetSpecifier()) + } +} + // TestRunner_RejectsConfigWithNoSurfacesEnabled covers the validation rule // that a config with every surface disabled is rejected. func TestRunner_RejectsConfigWithNoSurfacesEnabled(t *testing.T) { diff --git a/server/internal/proxysidecar/config.go b/server/internal/proxysidecar/config.go index a1e4ae1..de59a3c 100644 --- a/server/internal/proxysidecar/config.go +++ b/server/internal/proxysidecar/config.go @@ -65,10 +65,26 @@ const ( // mitmproxy + initiator path instead. AllowedOpsProfileSandboxTunnels = "sandbox-tunnels" + // AllowedOpsProfileSidecarOwn is the superset of sandbox-tunnels that + // also exposes TaskQuery + TaskOp for the sidecar's in-process + // openclaw bridge (chat_message task lifecycle: orphan scan, + // complete, fail). Use when the sidecar container hosts both the + // in-sandbox SDK (proxy/tunnel consumer) AND the bridge (task ops + // consumer); both share the relay listener. + AllowedOpsProfileSidecarOwn = "sidecar-own" + // AllowedOpsProfileToolStubOnly forbids every upstream op except the // init handshake. Useful for sandboxes that act purely as inbound // responders (tool stubs that only receive ProxyHttpRequest). AllowedOpsProfileToolStubOnly = "tool-stub-only" + + // FilterProfileServicePassthrough disables the upstream op allow-list + // entirely: every op a trusted provider emits (CreateTask, + // SessionOperation, etc.) is forwarded verbatim. It is the default for + // the tenant-relay surface, where the trust boundary is the + // provider→aggregator mTLS hop rather than per-op filtering. See + // resolveAllowedOps in relay_filter.go for the bypass implementation. + FilterProfileServicePassthrough = "service-passthrough" ) const ( @@ -111,6 +127,14 @@ type Config struct { Terminator TerminatorConfig `yaml:"terminator"` Initiator InitiatorConfig `yaml:"initiator"` Relay RelayConfig `yaml:"relay"` + + // TenantRelay and Aggregator are the two halves of the per-tenant relay + // topology. A tenant-relay sidecar runs alongside a tenant's gateway and + // dials an aggregator's tunnel surface; the aggregator runs centrally and + // pairs providers (by tenant) with their relays. The two are mutually + // exclusive per process (a config error if both are enabled). + TenantRelay TenantRelayConfig `yaml:"tenant_relay"` + Aggregator AggregatorConfig `yaml:"aggregator"` } // TerminatorConfig configures the terminator surface (gateway → local @@ -159,6 +183,117 @@ type RelayConfig struct { // memory and tracking-table size; storms above the cap surface as // session-attach rejections rather than per-stream creation churn. MaxSessions int `yaml:"max_sessions"` + + // Mux selects the relay implementation. When true (the default — the + // zero value is overridden in validateRelay), the runner builds a + // RelayMux: ONE shared upstream gateway connection demultiplexed across + // many sub-client streams by request_id, with pushes broadcast and + // tunnel frames routed by tunnel id. This is required whenever more than + // one sub-client shares the relay socket: the legacy per-sub-client + // Relay opens a fresh upstream stream per Connect and rewrites every + // sub-client to the SAME service identity, so N sub-clients collide on + // one per-identity gateway session lock and request/response replies + // (KV, TaskOp, TaskQuery) fail to route back. Set false to fall back to + // the legacy per-session Relay (still used by its existing callers/tests + // and for the shared-runtime composite path). + Mux *bool `yaml:"mux"` +} + +// MuxEnabled reports whether the relay should run in RelayMux (shared-upstream +// demux) mode. Defaults to true when unset; validateRelay() normalises the +// pointer so this only has to cover the defensive nil case. +func (c RelayConfig) MuxEnabled() bool { + if c.Mux == nil { + return true + } + return *c.Mux +} + +// TenantRelayConfig configures the tenant-relay surface (relay mode → central +// aggregator). Disabled by default. The relay dials the local tenant gateway +// (reusing the top-level Gateway section, whose TLS is the tenant cert under +// semi-strict mTLS) and the aggregator's tunnel surface (Aggregator section), +// then splices a provider's gateway session through itself verbatim. +type TenantRelayConfig struct { + Enabled bool `yaml:"enabled"` + + // Tenant is the tenant identity this relay serves. Defaults to the + // top-level TenantID when unset. Announced to the aggregator in + // TunnelHello and validated there against the relay's peer-cert CN. + Tenant string `yaml:"tenant"` + + // Aggregator describes how to dial the central aggregator's tunnel + // surface (SandboxRelayTunnel). + Aggregator AggregatorDialConfig `yaml:"aggregator"` + + // FilterProfile selects the upstream op filter applied to provider→gateway + // traffic. Defaults to "service-passthrough" (bypass — see + // FilterProfileServicePassthrough); the provider→aggregator mTLS hop is the + // trust boundary, not per-op filtering. + FilterProfile string `yaml:"filter_profile"` +} + +// AggregatorDialConfig describes how a tenant-relay dials the aggregator's +// tunnel surface. +type AggregatorDialConfig struct { + Address string `yaml:"address"` + Insecure bool `yaml:"insecure"` + TLS TLSConfig `yaml:"tls"` +} + +// AggregatorConfig configures the aggregator surface (central pairing of +// providers and tenant-relays). Disabled by default. It owns two gRPC server +// surfaces: ProviderListen (AetherGateway.Connect, dialled by providers with an +// x-aether-tenant hint) and TunnelListen (SandboxRelayTunnel, dialled by +// tenant-relays). The aggregator opens no upstream gateway connection of its +// own, so it does not require a Service identity. +type AggregatorConfig struct { + Enabled bool `yaml:"enabled"` + + // ProviderListen is the bind address for the AetherGateway.Connect surface + // that providers dial. TunnelListen is the bind address for the + // SandboxRelayTunnel surface that tenant-relays dial. Both accept the same + // listener spec forms as relay.listen (unix:///path or tcp://host:port). + ProviderListen string `yaml:"provider_listen"` + TunnelListen string `yaml:"tunnel_listen"` + + // ProviderTLS / TunnelTLS configure server-side mTLS on each surface. The + // sandboxes-CA peer-cert CN is the tenant identity authority. + ProviderTLS ServerTLSConfig `yaml:"provider_tls"` + TunnelTLS ServerTLSConfig `yaml:"tunnel_tls"` + + // RequireTenantMetadata, when true (the default), rejects a provider + // Connect that omits the x-aether-tenant metadata header. A *bool so the + // zero value (nil) can be distinguished from an explicit false and + // normalised in validateAggregator. + RequireTenantMetadata *bool `yaml:"require_tenant_metadata"` + + // PairWaitTimeoutMs bounds how long a registered provider or relay waits + // for its counterpart to arrive before the pairing is abandoned. Defaults + // to 30000 (30s). + PairWaitTimeoutMs int64 `yaml:"pair_wait_timeout_ms"` +} + +// RequireTenantMetadataEnabled reports whether a provider Connect must carry +// the x-aether-tenant metadata header. Defaults to true when unset; +// validateAggregator normalises the pointer so this only covers the defensive +// nil case. +func (c AggregatorConfig) RequireTenantMetadataEnabled() bool { + if c.RequireTenantMetadata == nil { + return true + } + return *c.RequireTenantMetadata +} + +// ServerTLSConfig holds server-side TLS certificate paths for an aggregator +// listener. When Insecure is true, TLS is disabled and the listener accepts +// plaintext (test/dev only); otherwise Cert/Key/ClientCA are all required so +// the surface enforces mTLS. +type ServerTLSConfig struct { + CertFile string `yaml:"cert_file"` + KeyFile string `yaml:"key_file"` + ClientCAFile string `yaml:"client_ca_file"` + Insecure bool `yaml:"insecure"` } // AllowedOpsConfig is a YAML union: either a profile name or a literal @@ -248,7 +383,11 @@ type TLSConfig struct { // ServiceConfig identifies the sidecar to the gateway. Required whenever // any surface that opens an upstream gateway connection is enabled // (terminator or relay). +// +// This block now drives an AGENT connection: workspace + implementation + +// specifier form the agent identity ag::workspace::implementation::specifier. type ServiceConfig struct { + Workspace string `yaml:"workspace"` Implementation string `yaml:"implementation"` Specifier string `yaml:"specifier"` } @@ -333,6 +472,15 @@ func (c *Config) applyEnvOverrides() { if v := os.Getenv("PROXY_SIDECAR_RELAY_LISTEN"); v != "" { c.Relay.Listen = v } + if v := os.Getenv("PROXY_SIDECAR_AGGREGATOR_ADDRESS"); v != "" { + c.TenantRelay.Aggregator.Address = v + } + if v := os.Getenv("PROXY_SIDECAR_AGGREGATOR_PROVIDER_LISTEN"); v != "" { + c.Aggregator.ProviderListen = v + } + if v := os.Getenv("PROXY_SIDECAR_AGGREGATOR_TUNNEL_LISTEN"); v != "" { + c.Aggregator.TunnelListen = v + } if v := os.Getenv("AETHER_LOG_LEVEL"); v != "" { c.Logging.Level = v } @@ -352,6 +500,12 @@ func (c *Config) EnabledSurfaces() []string { if c.Relay.Enabled { out = append(out, "relay") } + if c.TenantRelay.Enabled { + out = append(out, "tenant_relay") + } + if c.Aggregator.Enabled { + out = append(out, "aggregator") + } return out } @@ -362,23 +516,39 @@ func (c *Config) Validate() error { enabled := c.EnabledSurfaces() if len(enabled) == 0 { - errs = append(errs, "at least one surface must be enabled (set terminator.enabled, initiator.enabled, or relay.enabled to true)") + errs = append(errs, "at least one surface must be enabled (set terminator.enabled, initiator.enabled, relay.enabled, tenant_relay.enabled, or aggregator.enabled to true)") } - if c.Gateway.Address == "" { + // The aggregator opens no upstream gateway connection of its own (it + // pairs provider streams with relay tunnels), so it does not require the + // top-level gateway.address. Every other surface that talks to a gateway + // does. Skip the requirement only when the aggregator is the sole surface. + if c.Gateway.Address == "" && (!c.Aggregator.Enabled || c.Terminator.Enabled || c.Relay.Enabled || c.TenantRelay.Enabled) { errs = append(errs, "gateway.address is required") } + // tenant-relay and aggregator are mutually exclusive per process: the + // relay dials the aggregator, so co-hosting them in one sidecar is a + // configuration mistake. + if c.TenantRelay.Enabled && c.Aggregator.Enabled { + errs = append(errs, "tenant_relay.enabled and aggregator.enabled are mutually exclusive (run them in separate sidecars)") + } + // Service identity is required whenever a surface that authenticates // to the gateway is on. The initiator does not (yet) own a connection, - // so it does not contribute to this requirement. - needsService := c.Terminator.Enabled || c.Relay.Enabled + // so it does not contribute to this requirement. The aggregator owns no + // upstream identity (it splices provider sessions verbatim). + // + // A Service principal is workspace-less — its identity is sv::{impl}::{spec} + // and the ServiceIdentity proto has no workspace field — so only the + // implementation and specifier are required here (do NOT require a workspace). + needsService := c.Terminator.Enabled || c.Relay.Enabled || c.TenantRelay.Enabled if needsService { if c.Service.Implementation == "" { - errs = append(errs, "service.implementation is required when terminator or relay is enabled") + errs = append(errs, "service.implementation is required when terminator, relay, or tenant_relay is enabled") } if c.Service.Specifier == "" { - errs = append(errs, "service.specifier is required when terminator or relay is enabled") + errs = append(errs, "service.specifier is required when terminator, relay, or tenant_relay is enabled") } } @@ -391,6 +561,12 @@ func (c *Config) Validate() error { if c.Relay.Enabled { errs = append(errs, c.validateRelay()...) } + if c.TenantRelay.Enabled { + errs = append(errs, c.validateTenantRelay()...) + } + if c.Aggregator.Enabled { + errs = append(errs, c.validateAggregator()...) + } if len(errs) == 0 { return nil @@ -442,12 +618,14 @@ func (c *Config) validateRelay() []string { switch c.Relay.AllowedOps.Profile { case AllowedOpsProfileSandboxDefault, AllowedOpsProfileSandboxTunnels, + AllowedOpsProfileSidecarOwn, AllowedOpsProfileToolStubOnly: default: - errs = append(errs, fmt.Sprintf("relay.allowed_ops.profile=%q: must be one of %q, %q, %q (or pass a literal list)", + errs = append(errs, fmt.Sprintf("relay.allowed_ops.profile=%q: must be one of %q, %q, %q, %q (or pass a literal list)", c.Relay.AllowedOps.Profile, AllowedOpsProfileSandboxDefault, AllowedOpsProfileSandboxTunnels, + AllowedOpsProfileSidecarOwn, AllowedOpsProfileToolStubOnly)) } } @@ -463,6 +641,98 @@ func (c *Config) validateRelay() []string { if c.Relay.MaxSessions <= 0 { c.Relay.MaxSessions = 8 } + if c.Relay.Mux == nil { + // Default to RelayMux (shared-upstream demux). See RelayConfig.Mux. + def := true + c.Relay.Mux = &def + } + return errs +} + +// validateTenantRelay applies defaults and validates the tenant-relay surface. +// Mirrors validateRelay's style (mutate-and-collect). The local tenant gateway +// is reached via the top-level Gateway section, so Gateway.Address is required; +// the aggregator is reached via TenantRelay.Aggregator.Address. +func (c *Config) validateTenantRelay() []string { + var errs []string + if c.Gateway.Address == "" { + errs = append(errs, "gateway.address is required when tenant_relay is enabled (the local tenant gateway to dial)") + } + if c.TenantRelay.Tenant == "" { + // Default the served tenant to the top-level TenantID; error when + // neither is set since the aggregator pairs strictly by tenant. + c.TenantRelay.Tenant = c.TenantID + } + if c.TenantRelay.Tenant == "" { + errs = append(errs, "tenant_relay.tenant is required when tenant_relay is enabled (set tenant_relay.tenant or tenant_id)") + } + if c.TenantRelay.Aggregator.Address == "" { + errs = append(errs, "tenant_relay.aggregator.address is required when tenant_relay is enabled") + } + if c.TenantRelay.FilterProfile == "" { + c.TenantRelay.FilterProfile = FilterProfileServicePassthrough + } + switch c.TenantRelay.FilterProfile { + case FilterProfileServicePassthrough, + AllowedOpsProfileSandboxDefault, + AllowedOpsProfileSandboxTunnels, + AllowedOpsProfileSidecarOwn, + AllowedOpsProfileToolStubOnly: + default: + errs = append(errs, fmt.Sprintf("tenant_relay.filter_profile=%q: must be one of %q, %q, %q, %q, %q", + c.TenantRelay.FilterProfile, + FilterProfileServicePassthrough, + AllowedOpsProfileSandboxDefault, + AllowedOpsProfileSandboxTunnels, + AllowedOpsProfileSidecarOwn, + AllowedOpsProfileToolStubOnly)) + } + return errs +} + +// validateAggregator applies defaults and validates the aggregator surface. +// Both listeners are required; each TLS block must carry a full mTLS triple +// (cert/key/client-CA) unless explicitly marked insecure. +func (c *Config) validateAggregator() []string { + var errs []string + if c.Aggregator.ProviderListen == "" { + errs = append(errs, "aggregator.provider_listen is required when aggregator is enabled (use unix:///path or tcp://host:port)") + } + if c.Aggregator.TunnelListen == "" { + errs = append(errs, "aggregator.tunnel_listen is required when aggregator is enabled (use unix:///path or tcp://host:port)") + } + errs = append(errs, c.Aggregator.ProviderTLS.validate("aggregator.provider_tls")...) + errs = append(errs, c.Aggregator.TunnelTLS.validate("aggregator.tunnel_tls")...) + if c.Aggregator.PairWaitTimeoutMs <= 0 { + c.Aggregator.PairWaitTimeoutMs = 30_000 + } + if c.Aggregator.RequireTenantMetadata == nil { + // Default to requiring the x-aether-tenant hint; the CN still wins on + // mismatch but the metadata makes provider intent explicit. + def := true + c.Aggregator.RequireTenantMetadata = &def + } + return errs +} + +// validate checks an aggregator listener's TLS block. Unless Insecure is set, +// mTLS requires the full cert/key/client-CA triple so the surface can both +// present a server cert and verify the peer's client cert (the tenant-identity +// authority). +func (t *ServerTLSConfig) validate(prefix string) []string { + if t.Insecure { + return nil + } + var errs []string + if t.CertFile == "" { + errs = append(errs, fmt.Sprintf("%s.cert_file is required (or set %s.insecure: true)", prefix, prefix)) + } + if t.KeyFile == "" { + errs = append(errs, fmt.Sprintf("%s.key_file is required (or set %s.insecure: true)", prefix, prefix)) + } + if t.ClientCAFile == "" { + errs = append(errs, fmt.Sprintf("%s.client_ca_file is required for mTLS (or set %s.insecure: true)", prefix, prefix)) + } return errs } diff --git a/server/internal/proxysidecar/gateway_client.go b/server/internal/proxysidecar/gateway_client.go index 5775cbf..66163fa 100644 --- a/server/internal/proxysidecar/gateway_client.go +++ b/server/internal/proxysidecar/gateway_client.go @@ -3,36 +3,106 @@ package proxysidecar import ( "context" "fmt" + "math/rand" "time" "github.com/rs/zerolog/log" "github.com/scitrera/aether/sdk/go/aether" ) -// gatewayRuntime owns a single ServiceClient connection to the gateway and -// the reconnection loop. It is mode-agnostic: terminator registers HTTP and +// gatewayRuntime owns a single gateway connection and the reconnection loop. +// The connection is identity-agnostic: it holds a *aether.BaseClient that is +// backed by EITHER a ServiceClient (workspace-less sv::{impl}::{spec}) or an +// AgentClient (ag::{ws}::{impl}::{spec}), selected in init() by whether a +// workspace is configured. It is mode-agnostic: terminator registers HTTP and // tunnel handlers on it, while relay mode (T37) attaches its own gRPC mitm // handlers to the same runtime without going through Terminator. // // The runtime does not own dispatcher logic — callers register handlers via -// the underlying ServiceClient (exposed through Client()) before calling -// Run. +// the underlying BaseClient (exposed through Client()) before calling +// Run. All messaging/handler methods used here (OnMessage, OnProxyHttp*, +// OnTunnel*, SendWithPriority, Connect, Run, Close) are defined on BaseClient, +// so the same wiring serves both identity kinds. type gatewayRuntime struct { cfg *Config - client *aether.ServiceClient + client *aether.BaseClient transport *serviceClientTransport + + // creds is the live credential map handed to the client. The SDK + // rebuilds the InitConnection from this same map on every (re)connect + // (see sdk/go/aether/{agent,service}.go buildInitMessage), so mutating it in + // place via refreshCredentials lets a reconnect present freshly-loaded + // credentials without rebuilding the client or re-installing handlers. + creds aether.Credentials + credKind CredentialKind + + // Reconnect / backoff / give-up policy. Defaults are set in + // newGatewayRuntime; tests override them for fast, deterministic runs. + initialBackoff time.Duration + maxBackoff time.Duration + backoffMultiplier float64 + stableThreshold time.Duration // a session lasting this long is "healthy" + maxTerminalFailures int // consecutive terminal failures before fatal + + // connOverride, when non-nil, replaces r.client as the connect/run target. + // Production leaves it nil; tests inject a fake to exercise the loop + // without a live gateway. + connOverride gatewayConn +} + +// gatewayConn is the minimal connect/run surface runConnectionLoop drives. +// *aether.BaseClient satisfies it (Connect/Run are BaseClient methods, shared +// by both ServiceClient and AgentClient); tests provide a fake. +type gatewayConn interface { + Connect(ctx context.Context) error + Run(ctx context.Context) error } -// newGatewayRuntime builds a runtime from cfg. The ServiceClient is not +// newGatewayRuntime builds a runtime from cfg. The gateway client is not // constructed until init() is called from Run() so callers can configure // hooks that depend on the runtime before the connection opens. func newGatewayRuntime(cfg *Config) *gatewayRuntime { - return &gatewayRuntime{cfg: cfg} + return &gatewayRuntime{ + cfg: cfg, + initialBackoff: 1 * time.Second, + maxBackoff: 30 * time.Second, + backoffMultiplier: 2.0, + stableThreshold: 30 * time.Second, + maxTerminalFailures: 5, + } +} + +// conn returns the connect/run target: the test override if set, else the +// real gateway client (service- or agent-backed BaseClient). +func (r *gatewayRuntime) conn() gatewayConn { + if r.connOverride != nil { + return r.connOverride + } + return r.client } -// init creates the underlying ServiceClient using cfg.Gateway and +// applyCredential populates creds for the given credential kind using the +// SDK's canonical map keys. CredentialKindNone leaves creds empty (mTLS / +// insecure paths). +func applyCredential(creds aether.Credentials, cred string, kind CredentialKind) { + switch kind { + case CredentialKindAPIKey: + creds.WithAPIKey(cred) + case CredentialKindTaskToken: + creds.WithTaskToken(cred) + case CredentialKindNone: + } +} + +// init creates the underlying gateway client using cfg.Gateway and // cfg.Service. It is idempotent within a single runtime — a second call // after a successful first call is a no-op. +// +// The client kind is chosen by whether a workspace is configured. Service +// identities are workspace-less (sv::{impl}::{spec}); Agent identities require +// a workspace (ag::{ws}::{impl}::{spec}). Both are backed by BaseClient, so the +// runtime stores the embedded *BaseClient either way and the handler/transport +// wiring is identical downstream. func (r *gatewayRuntime) init() error { if r.client != nil { return nil @@ -42,116 +112,246 @@ func (r *gatewayRuntime) init() error { if err != nil { return err } - switch kind { - case CredentialKindAPIKey: - creds = creds.WithAPIKey(cred) - case CredentialKindTaskToken: - // Task tokens authenticate as the gateway-bound TargetIdentity for - // the token's lifetime. Used when the sidecar is spawned under a - // per-task credential mint (e.g. an orchestrator's CreateTask with - // target_identity=sv::::). - creds = creds.WithTaskToken(cred) - case CredentialKindNone: - // No credential configured; rely on mTLS or insecure mode. Empty - // creds are valid — the gateway will fail authn explicitly if it - // needs more. - } - - opts := aether.ServiceOptions{ - ClientOptions: aether.ClientOptions{ - ServerAddr: r.cfg.Gateway.Address, - Connection: aether.ConnectionOptions{ - RetryOnDuplicate: true, - MaxRetries: 0, - AutoReconnect: true, - InitialBackoff: 1 * time.Second, - MaxBackoff: 30 * time.Second, - BackoffMultiplier: 2.0, - ConnectTimeout: 30 * time.Second, - KeepAliveInterval: 30 * time.Second, - }, - Credentials: creds, - }, - Implementation: r.cfg.Service.Implementation, - Specifier: r.cfg.Service.Specifier, - } + // Credential kinds: + // - APIKey: long-lived service key. + // - TaskToken: per-task token bound to the gateway TargetIdentity for + // the token's lifetime (e.g. an orchestrator's CreateTask with + // target_identity=sv::::). + // - None: rely on mTLS / insecure mode; the gateway fails authn + // explicitly if it needs more. + applyCredential(creds, cred, kind) + // Hold the live map + kind so runConnectionLoop can refresh credentials + // in place on a terminal auth failure (re-pairing / token re-mint writes + // a fresh token to the configured *_path; the next reconnect reads it). + r.creds = creds + r.credKind = kind tlsCfg, err := buildTLSConfig(r.cfg.Gateway) if err != nil { return err } - opts.TLS = tlsCfg - client, err := aether.NewServiceClient(opts) - if err != nil { - return fmt.Errorf("create service client: %w", err) + clientOpts := aether.ClientOptions{ + ServerAddr: r.cfg.Gateway.Address, + Connection: aether.ConnectionOptions{ + RetryOnDuplicate: true, + MaxRetries: 0, + AutoReconnect: true, + InitialBackoff: 1 * time.Second, + MaxBackoff: 30 * time.Second, + BackoffMultiplier: 2.0, + ConnectTimeout: 30 * time.Second, + KeepAliveInterval: 30 * time.Second, + }, + Credentials: creds, + TLS: tlsCfg, + } + + // Workspace-presence discriminator: an empty workspace means a workspace- + // less Service identity (sv::{impl}::{spec}); a set workspace means an + // Agent identity (ag::{ws}::{impl}::{spec}). Both client types embed + // *BaseClient, so we store the embedded base client and the rest of the + // runtime is identity-agnostic. + if r.cfg.Service.Workspace == "" { + sc, err := aether.NewServiceClient(aether.ServiceOptions{ + ClientOptions: clientOpts, + Implementation: r.cfg.Service.Implementation, + Specifier: r.cfg.Service.Specifier, + }) + if err != nil { + return fmt.Errorf("create service client: %w", err) + } + r.client = sc.BaseClient + } else { + ac, err := aether.NewAgentClient(aether.AgentOptions{ + ClientOptions: clientOpts, + Workspace: r.cfg.Service.Workspace, + Implementation: r.cfg.Service.Implementation, + Specifier: r.cfg.Service.Specifier, + }) + if err != nil { + return fmt.Errorf("create agent client: %w", err) + } + r.client = ac.BaseClient } - r.client = client - r.transport = &serviceClientTransport{client: client} + r.transport = &serviceClientTransport{client: r.client} return nil } -// Client returns the underlying ServiceClient. Callers register OnMessage, -// OnProxyHttpRequest, etc. on it before invoking Run. -func (r *gatewayRuntime) Client() *aether.ServiceClient { +// Client returns the underlying BaseClient (service- or agent-backed). Callers +// register OnMessage, OnProxyHttpRequest, etc. on it before invoking Run. +func (r *gatewayRuntime) Client() *aether.BaseClient { return r.client } // Transport returns the production tunnelTransport that ships frames -// upstream through the embedded ServiceClient. +// upstream through the embedded BaseClient. func (r *gatewayRuntime) Transport() tunnelTransport { return r.transport } -// runConnectionLoop manages reconnection with exponential backoff. -// Returns when ctx is cancelled. -func (r *gatewayRuntime) runConnectionLoop(ctx context.Context) { +// refreshCredentials re-reads the gateway credential from its configured +// source and replaces the live credential map's contents in place. The next +// reconnect's InitConnection builder reads this same map, so a re-paired / +// re-minted token written to the configured *_path is picked up without +// rebuilding the client. The credential value is never logged. +// +// Re-establishment only helps when the credential is sourced from a path (or +// is otherwise externally refreshable). An inline token in the config is +// re-read as the same dead value — the give-up counter in runConnectionLoop +// is what bounds that case. +func (r *gatewayRuntime) refreshCredentials() error { + cred, kind, err := loadGatewayCredential(r.cfg.Gateway) + if err != nil { + return err + } + for k := range r.creds { + delete(r.creds, k) + } + applyCredential(r.creds, cred, kind) + r.credKind = kind + return nil +} + +// runConnectionLoop owns the sidecar's gateway connection for its lifetime. +// +// It is the single authority for terminal-failure handling: the SDK is built +// with AutoReconnect, so recoverable disconnects (network blips, gateway +// restarts that still honor our token) are retried inside Run() with the +// SDK's own jittered backoff and never surface here. What surfaces here is a +// terminal error — most importantly a token the gateway no longer recognizes +// (codes.Unauthenticated), which the SDK correctly refuses to retry. +// +// Three behaviors distinguish this from the old loop, which reset its backoff +// on every Connect() success and so hammered the gateway ~1/s forever with a +// dead token (Connect only opens the stream — the token is validated later, +// during Run's first Recv): +// +// - Backoff (capped, jittered) is reset only after a *healthy* session +// (one that lasted stableThreshold), never merely because the transport +// opened. +// - On a terminal failure the credential is re-established (re-read) before +// the next attempt rather than blindly replaying the dead one. +// - After maxTerminalFailures consecutive terminal failures (including +// failed re-establishment) the loop returns a fatal error so the process +// exits non-zero / signals its wrapped child, letting an orphaned sandbox +// be reaped instead of spinning forever. +// +// Returns nil on ctx cancellation (clean shutdown) and a non-nil error only +// on the terminal give-up path. +func (r *gatewayRuntime) runConnectionLoop(ctx context.Context) (err error) { defer func() { if rec := recover(); rec != nil { log.Error().Interface("panic", rec).Msg("gateway runtime: recovered from panic") } }() - backoff := 1 * time.Second - maxBackoff := 30 * time.Second + + conn := r.conn() + backoff := r.initialBackoff + terminalFailures := 0 + for { if ctx.Err() != nil { - return + return nil } - if err := r.client.Connect(ctx); err != nil { - // On clean shutdown the SDK returns a context-cancelled error; - // the outer ctx.Err() check on the next iteration will exit the - // loop anyway, but logging at ERROR here makes shutdown look - // like a failure in callers' test/log output. Demote to Debug - // when the cancellation came from us. - if ctx.Err() != nil { - log.Debug().Err(err).Msg("gateway runtime: connect aborted by shutdown") - } else { - log.Error().Err(err).Msg("gateway runtime: connect error") - } - } else { - backoff = 1 * time.Second - if err := r.client.Run(ctx); err != nil { - if ctx.Err() != nil { - log.Debug().Err(err).Msg("gateway runtime: run aborted by shutdown") - } else { - log.Error().Err(err).Msg("gateway runtime: run error") - } + + // Log EVERY connect attempt (not just successes) so a connection that + // keeps dropping — e.g. a gateway GOAWAY "too_many_pings", an auth/ACL + // reject, or a network blip — is visible per-attempt instead of only via + // raw gRPC transport errors. Pairs with the cycleErr disconnect-reason + // logs below and the "connected ... entering run loop" success log. + log.Info(). + Int("terminal_failures", terminalFailures). + Dur("backoff", backoff). + Msg("gateway runtime: connect attempt — dialing gateway") + cycleStart := time.Now() + cycleErr := conn.Connect(ctx) + if cycleErr == nil { + log.Info(). + Dur("connect_duration", time.Since(cycleStart)). + Msg("gateway runtime: connected to gateway; entering run loop") + cycleErr = conn.Run(ctx) + if cycleErr != nil { + log.Warn(). + Err(cycleErr). + Dur("session_duration", time.Since(cycleStart)). + Msg("gateway runtime: connection dropped (run loop ended)") } } if ctx.Err() != nil { - return + // Clean shutdown — the cancellation came from us. Demote any + // error so shutdown doesn't read as a failure in logs/tests. + if cycleErr != nil { + log.Debug().Err(cycleErr).Msg("gateway runtime: connection aborted by shutdown") + } + return nil } - log.Info().Dur("backoff", backoff).Msg("gateway runtime: reconnecting to gateway") - select { - case <-ctx.Done(): - return - case <-time.After(backoff): + + // A session that stayed up past the stability threshold (or ended + // gracefully) is healthy: reset both the backoff and the terminal + // streak. This is the core fix — backoff must NOT reset just because + // Connect() returned, only after the connection proved durable. + if cycleErr == nil || time.Since(cycleStart) >= r.stableThreshold { + backoff = r.initialBackoff + terminalFailures = 0 } - if backoff < maxBackoff { - backoff *= 2 - if backoff > maxBackoff { - backoff = maxBackoff + + terminal := cycleErr != nil && !aether.IsRecoverable(cycleErr) + switch { + case terminal: + terminalFailures++ + log.Error(). + Err(cycleErr). + Int("consecutive_terminal_failures", terminalFailures). + Int("max_terminal_failures", r.maxTerminalFailures). + Msg("gateway runtime: terminal connection failure; re-establishing credentials") + + if rerr := r.refreshCredentials(); rerr != nil { + log.Error().Err(rerr).Msg("gateway runtime: credential re-establishment failed") + } else { + log.Info().Msg("gateway runtime: reloaded gateway credential for next attempt") } + + if terminalFailures >= r.maxTerminalFailures { + return fmt.Errorf( + "gateway runtime: giving up after %d consecutive terminal failures: %w", + terminalFailures, cycleErr, + ) + } + case cycleErr != nil: + log.Error().Err(cycleErr).Msg("gateway runtime: transient connection error; will retry") } + + sleep := backoffWithJitter(backoff) + log.Info(). + Dur("backoff", sleep). + Bool("terminal", terminal). + Msg("gateway runtime: reconnecting to gateway") + select { + case <-ctx.Done(): + return nil + case <-time.After(sleep): + } + backoff = nextBackoff(backoff, r.maxBackoff, r.backoffMultiplier) + } +} + +// backoffWithJitter applies ±25% jitter to d so a fleet of sidecars whose +// shared gateway restarted don't reconnect in lockstep (thundering herd). +func backoffWithJitter(d time.Duration) time.Duration { + jitter := float64(d) * 0.25 * (rand.Float64()*2 - 1) + out := d + time.Duration(jitter) + if out <= 0 { + return d + } + return out +} + +// nextBackoff grows cur by mult, capped at max (and clamped on overflow). +func nextBackoff(cur, max time.Duration, mult float64) time.Duration { + next := time.Duration(float64(cur) * mult) + if next > max || next < cur { + return max } + return next } diff --git a/server/internal/proxysidecar/gateway_client_test.go b/server/internal/proxysidecar/gateway_client_test.go new file mode 100644 index 0000000..69c5df2 --- /dev/null +++ b/server/internal/proxysidecar/gateway_client_test.go @@ -0,0 +1,183 @@ +package proxysidecar + +import ( + "context" + "errors" + "os" + "path/filepath" + "sync" + "testing" + "time" + + "github.com/scitrera/aether/sdk/go/aether" +) + +// fakeConn is a programmable gatewayConn for driving runConnectionLoop without +// a live gateway. runFn receives the 1-based attempt count. +type fakeConn struct { + mu sync.Mutex + attempts int + connErr error + runFn func(ctx context.Context, attempt int) error +} + +func (f *fakeConn) Connect(_ context.Context) error { return f.connErr } + +func (f *fakeConn) Run(ctx context.Context) error { + f.mu.Lock() + f.attempts++ + attempt := f.attempts + fn := f.runFn + f.mu.Unlock() + return fn(ctx, attempt) +} + +func (f *fakeConn) count() int { + f.mu.Lock() + defer f.mu.Unlock() + return f.attempts +} + +// newTestRuntime builds a runtime wired to a fake conn with fast, deterministic +// backoff and a task-token file the loop can refresh from. +func newTestRuntime(t *testing.T, tokenPath string, conn gatewayConn) *gatewayRuntime { + t.Helper() + rt := newGatewayRuntime(&Config{ + Gateway: GatewayConfig{TaskTokenPath: tokenPath, Insecure: true}, + }) + rt.connOverride = conn + rt.creds = aether.NewCredentials() + rt.initialBackoff = time.Millisecond + rt.maxBackoff = 2 * time.Millisecond + rt.stableThreshold = 50 * time.Millisecond + rt.maxTerminalFailures = 3 + return rt +} + +func writeToken(t *testing.T, dir, val string) string { + t.Helper() + p := filepath.Join(dir, "token") + if err := os.WriteFile(p, []byte(val), 0o600); err != nil { + t.Fatalf("write token: %v", err) + } + return p +} + +// A dead token retried forever is the incident: the loop must escalate to a +// fatal give-up after maxTerminalFailures rather than spin. Because Connect() +// always "succeeds" (the transport opens) and Run() fails fast, this also +// proves the backoff/terminal-streak no longer resets on Connect success. +func TestRunConnectionLoop_GivesUpAfterTerminalFailures(t *testing.T) { + dir := t.TempDir() + tokenPath := writeToken(t, dir, "dead-token") + + conn := &fakeConn{ + runFn: func(_ context.Context, _ int) error { + return aether.NewAuthenticationError("token not found") + }, + } + rt := newTestRuntime(t, tokenPath, conn) + + err := rt.runConnectionLoop(context.Background()) + if err == nil { + t.Fatal("expected fatal give-up error, got nil") + } + if got := conn.count(); got != rt.maxTerminalFailures { + t.Fatalf("expected exactly %d attempts before give-up, got %d", rt.maxTerminalFailures, got) + } + // Re-establishment was attempted (creds re-read from the path). + if rt.creds["token"] != "dead-token" { + t.Fatalf("expected creds refreshed from path, got %q", rt.creds["token"]) + } +} + +// Recoverable errors that escape Run() (rare, since AutoReconnect handles most +// internally) must NOT count toward the terminal give-up — they back off and +// retry the same credential indefinitely until ctx cancellation. +func TestRunConnectionLoop_TransientErrorsDoNotGiveUp(t *testing.T) { + dir := t.TempDir() + tokenPath := writeToken(t, dir, "live-token") + + conn := &fakeConn{ + runFn: func(_ context.Context, _ int) error { + return errors.New("connection reset by peer") // recoverable (no auth/grpc class) + }, + } + rt := newTestRuntime(t, tokenPath, conn) + + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Millisecond) + defer cancel() + + if err := rt.runConnectionLoop(ctx); err != nil { + t.Fatalf("transient errors must not produce a fatal give-up, got %v", err) + } + if conn.count() < rt.maxTerminalFailures { + t.Fatalf("expected the loop to retry past the terminal cap (%d), only saw %d attempts", + rt.maxTerminalFailures, conn.count()) + } +} + +// When an external pairing helper rewrites the token file, the next reconnect +// must present the fresh credential and recover — no fatal give-up. +func TestRunConnectionLoop_ReestablishesCredentialsAndRecovers(t *testing.T) { + dir := t.TempDir() + tokenPath := writeToken(t, dir, "dead-token") + + var conn *fakeConn + rt := newTestRuntime(t, tokenPath, nil) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + conn = &fakeConn{ + runFn: func(rctx context.Context, attempt int) error { + // Simulate the operator re-pairing after the first terminal fail. + if attempt == 1 { + writeToken(t, dir, "fresh-token") + return aether.NewAuthenticationError("token not found") + } + // Fresh token accepted: stay "connected" until shutdown, long + // enough to clear the stability threshold. + <-rctx.Done() + return rctx.Err() + }, + } + rt.connOverride = conn + + done := make(chan error, 1) + go func() { done <- rt.runConnectionLoop(ctx) }() + + // Give the loop time to fail once, refresh, reconnect, and settle. + time.Sleep(150 * time.Millisecond) + cancel() + + if err := <-done; err != nil { + t.Fatalf("expected clean exit after recovery, got %v", err) + } + if rt.creds["token"] != "fresh-token" { + t.Fatalf("expected credential re-established to fresh-token, got %q", rt.creds["token"]) + } +} + +func TestNextBackoff(t *testing.T) { + const max = 30 * time.Second + if got := nextBackoff(time.Second, max, 2.0); got != 2*time.Second { + t.Fatalf("nextBackoff(1s) = %v, want 2s", got) + } + if got := nextBackoff(20*time.Second, max, 2.0); got != max { + t.Fatalf("nextBackoff(20s) capped = %v, want %v", got, max) + } + if got := nextBackoff(max, max, 2.0); got != max { + t.Fatalf("nextBackoff at cap = %v, want %v", got, max) + } +} + +func TestBackoffWithJitter_StaysInBand(t *testing.T) { + base := 4 * time.Second + for i := 0; i < 1000; i++ { + got := backoffWithJitter(base) + if got < time.Duration(0.75*float64(base)) || got > time.Duration(1.25*float64(base)) { + t.Fatalf("jittered backoff %v outside ±25%% of %v", got, base) + } + } +} diff --git a/server/internal/proxysidecar/http_backend.go b/server/internal/proxysidecar/http_backend.go index 423d4c7..f47faad 100644 --- a/server/internal/proxysidecar/http_backend.go +++ b/server/internal/proxysidecar/http_backend.go @@ -5,6 +5,7 @@ import ( "context" "errors" "fmt" + "net" "net/http" "path" "strings" @@ -36,6 +37,11 @@ type httpBackend struct { cfg BackendConfig tenantID string resolver identityheaders.AuthorityResolver + // baseURL is the request-URL prefix the outgoing http.Request is built on. + // For TCP backends it equals cfg.URL. For unix-domain-socket backends + // (cfg.URL = "unix:///path.sock") it is a fixed "http://unix" placeholder + // host while client/streamingClient dial the socket via a custom DialContext. + baseURL string client *http.Client // streamingClient is used for stream_response_indefinitely requests; it // has no per-request Timeout (which would otherwise cap the whole @@ -44,15 +50,34 @@ type httpBackend struct { } func newHTTPBackend(cfg BackendConfig, tenantID string, resolver identityheaders.AuthorityResolver) *httpBackend { - return &httpBackend{ + b := &httpBackend{ cfg: cfg, tenantID: tenantID, resolver: resolver, + baseURL: cfg.URL, client: &http.Client{ Timeout: time.Duration(cfg.IdleTimeoutMs) * time.Millisecond, }, streamingClient: &http.Client{}, } + // Unix-domain-socket backend (cfg.URL = "unix:///path.sock"): dial the socket + // via a custom DialContext and build requests against a fixed http://unix host + // (the path is the socket, NOT part of the HTTP request path). This lets a + // backend — e.g. the sidecar-config API — be reachable ONLY by this in-process + // terminator through a UDS in the sidecar's own filesystem, never by other + // processes/containers that merely share the pod network namespace (a TCP + // loopback port would be reachable by them; a UDS is gated by filesystem/mount + // namespace isolation). + if strings.HasPrefix(cfg.URL, "unix://") { + socketPath := strings.TrimPrefix(cfg.URL, "unix://") + dial := func(ctx context.Context, _, _ string) (net.Conn, error) { + return (&net.Dialer{}).DialContext(ctx, "unix", socketPath) + } + b.baseURL = "http://unix" + b.client.Transport = &http.Transport{DialContext: dial} + b.streamingClient.Transport = &http.Transport{DialContext: dial} + } + return b } // matches reports whether this backend should handle the given method/path. @@ -173,7 +198,7 @@ func (b *httpBackend) buildBackendRequest(ctx context.Context, req *pb.ProxyHttp "method %s path %s not permitted by backend %q", req.GetMethod(), req.GetPath(), b.cfg.Name) } - url := strings.TrimRight(b.cfg.URL, "/") + req.GetPath() + url := strings.TrimRight(b.baseURL, "/") + req.GetPath() httpReq, err := http.NewRequestWithContext(ctx, req.GetMethod(), url, bytes.NewReader(body)) if err != nil { return nil, newProxyError(pb.ProxyError_DECODE_FAILED, "build backend request: %v", err) diff --git a/server/internal/proxysidecar/integration_e2e/full_chain_e2e_test.go b/server/internal/proxysidecar/integration_e2e/full_chain_e2e_test.go new file mode 100644 index 0000000..8899e6c --- /dev/null +++ b/server/internal/proxysidecar/integration_e2e/full_chain_e2e_test.go @@ -0,0 +1,875 @@ +// Full-chain composition test: REAL aggregator + REAL tenant-relay spliced +// together over real TCP/mTLS, driving a fake tenant gateway at the far end and +// a raw provider AetherGateway client at the near end. +// +// Both halves of the per-tenant relay topology (aggregator.go, tenant_relay.go) +// were previously only unit-tested against a FAKE of the other side: +// - aggregator_test.go drives the aggregator with a fakeProvider + fakeRelay. +// - tenant_relay_test.go drives the relay with a fakeGateway + fakeAggregator. +// +// Neither ever ran the real aggregator splice and the real relay pump together. +// This test composes the full two-hop path +// +// provider (raw AetherGateway client) +// │ mTLS, sandboxes-CA provider cert, x-aether-tenant metadata +// ▼ +// aggregator.Connect (provider surface) ──splice── aggregator.Tunnel (relay surface) +// ▲ +// tenant-relay ── TunnelHello + frame pumps ───────────┘ +// │ mTLS, tenant-CA tenant cert +// ▼ +// fake tenant gateway (records InitConnection, replies ConnectionAck, pushes downstream) +// +// so a regression in EITHER component's wire handling (splice fidelity, init +// passthrough, resume preservation) is caught by composition rather than by a +// fake that happens to agree with the bug. +// +// This file is deliberately NOT behind the `e2e` build tag (unlike the rest of +// this package's heavy aetherlite-subprocess suite): it stands up only +// in-process gRPC servers and is fast/deterministic, so it runs under the +// default `go test ./internal/proxysidecar/integration_e2e/...` invocation. +// +// NOTE on the provider client: we use a RAW pb.NewAetherGatewayClient(...). +// Connect with x-aether-tenant set via metadata.NewOutgoingContext, NOT the +// Aether SDK. The SDK metadata-injection hook that would set x-aether-tenant +// automatically is a separate P3 (provider) concern; for the aggregator+relay +// composition under test the raw client is the right scope. + +package integration_e2e + +import ( + "context" + "crypto/rand" + "crypto/rsa" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "fmt" + "math/big" + "net" + "os" + "path/filepath" + "sync" + "testing" + "time" + + "github.com/scitrera/aether/internal/proxysidecar" + pb "github.com/scitrera/aether/api/proto" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/metadata" +) + +const ( + fcTenant = "tenant-fullchain" + fcProviderSharedCN = "sandbox-provider-shared" // tenant-less provider CN → metadata pairing path + fcRelayCN = "sv::sandbox-provider::" + fcTenant // tenant-encoding relay CN → CN binding path + metadataTenantKey = "x-aether-tenant" // mirrors proxysidecar.metadataTenantKey (unexported) +) + +// ============================================================================= +// fakeGateway — a pb.AetherGatewayServer that the tenant-relay dials. +// +// It records the first InitConnection it sees, ACKs it with a session id, +// records subsequent upstream frames, and can push a downstream message on +// demand. For the resume assertion it tracks, per claimed identity, whether a +// later Connect carried resume_session_id == a previously-issued session id. +// ============================================================================= + +type fakeGateway struct { + pb.UnimplementedAetherGatewayServer + + mu sync.Mutex + // inits records every InitConnection received (in arrival order). + inits []*pb.InitConnection + // upstreams records every NON-init UpstreamMessage received. + upstreams []*pb.UpstreamMessage + // issuedSessions maps assigned session_id → the identity string it was + // issued to, so a later resume can be matched. + issuedSessions map[string]string + // resumed records, per resume_session_id observed on a Connect, true. + resumed map[string]bool + + // firstInit fires once when the first InitConnection arrives. + firstInitOnce sync.Once + firstInitCh chan struct{} + + // push lets a test inject a DownstreamMessage to a live Connect stream. + // Each Connect registers its send func here under a lock. + streamMu sync.Mutex + sendFns map[int]func(*pb.DownstreamMessage) error + nextID int + + sessionSeq int +} + +func newFakeGateway() *fakeGateway { + return &fakeGateway{ + issuedSessions: map[string]string{}, + resumed: map[string]bool{}, + firstInitCh: make(chan struct{}), + sendFns: map[int]func(*pb.DownstreamMessage) error{}, + } +} + +func (g *fakeGateway) Connect(stream grpc.BidiStreamingServer[pb.UpstreamMessage, pb.DownstreamMessage]) error { + // Register this stream's Send so tests can push downstream frames to it. + g.streamMu.Lock() + id := g.nextID + g.nextID++ + g.sendFns[id] = stream.Send + g.streamMu.Unlock() + defer func() { + g.streamMu.Lock() + delete(g.sendFns, id) + g.streamMu.Unlock() + }() + + for { + up, err := stream.Recv() + if err != nil { + return nil // EOF / cancel: clean close + } + if init := up.GetInit(); init != nil { + sessionID := g.recordInit(init) + // Reply with a ConnectionAck so the upstream session is "open". + // Resumed=true iff the client supplied a resume_session_id that we + // previously issued for this identity. + resumed := false + if rs := init.GetResumeSessionId(); rs != "" { + g.mu.Lock() + _, known := g.issuedSessions[rs] + if known { + resumed = true + } + g.mu.Unlock() + } + ack := &pb.DownstreamMessage{Payload: &pb.DownstreamMessage_ConnectionAck{ + ConnectionAck: &pb.ConnectionAck{SessionId: sessionID, Resumed: resumed}, + }} + if err := stream.Send(ack); err != nil { + return err + } + continue + } + // Non-init upstream frame: record it. + g.mu.Lock() + g.upstreams = append(g.upstreams, up) + g.mu.Unlock() + } +} + +// recordInit stores the init, issues a fresh session id bound to the claimed +// identity, marks resume if the init carried a known resume_session_id, and +// fires firstInitCh on the first call. Returns the issued session id. +func (g *fakeGateway) recordInit(init *pb.InitConnection) string { + g.mu.Lock() + defer g.mu.Unlock() + + g.inits = append(g.inits, init) + + identity := serviceIdentityString(init.GetService()) + if rs := init.GetResumeSessionId(); rs != "" { + // Record that THIS resume id was observed; mark resumed iff it was an + // id we previously issued (to anyone). + if _, known := g.issuedSessions[rs]; known { + g.resumed[rs] = true + } + } + + g.sessionSeq++ + sessionID := fmt.Sprintf("gw-session-%d", g.sessionSeq) + g.issuedSessions[sessionID] = identity + + g.firstInitOnce.Do(func() { close(g.firstInitCh) }) + return sessionID +} + +func serviceIdentityString(svc *pb.ServiceIdentity) string { + if svc == nil { + return "" + } + return "sv::" + svc.GetImplementation() + "::" + svc.GetSpecifier() +} + +// pushDownstream sends a DownstreamMessage to the (single) currently-registered +// Connect stream. Returns an error if no stream is live. +func (g *fakeGateway) pushDownstream(msg *pb.DownstreamMessage) error { + g.streamMu.Lock() + defer g.streamMu.Unlock() + for _, send := range g.sendFns { + return send(msg) + } + return fmt.Errorf("no live gateway stream to push to") +} + +func (g *fakeGateway) snapshotInits() []*pb.InitConnection { + g.mu.Lock() + defer g.mu.Unlock() + out := make([]*pb.InitConnection, len(g.inits)) + copy(out, g.inits) + return out +} + +func (g *fakeGateway) snapshotUpstreams() []*pb.UpstreamMessage { + g.mu.Lock() + defer g.mu.Unlock() + out := make([]*pb.UpstreamMessage, len(g.upstreams)) + copy(out, g.upstreams) + return out +} + +func (g *fakeGateway) wasResumed(resumeID string) bool { + g.mu.Lock() + defer g.mu.Unlock() + return g.resumed[resumeID] +} + +// ============================================================================= +// chainEnv — the full composed stack, with all certs and teardown wired. +// ============================================================================= + +type chainEnv struct { + t *testing.T + + gateway *fakeGateway + gatewayAddr string // host:port the relay dials + + aggProviderAddr string // host:port the provider dials + aggTunnelAddr string // host:port the relay dials for the tunnel + + // CA material + on-disk cert paths used to configure the real components. + sandboxesCA *caBundle + tenantCA *caBundle + + aggProviderServerCert string + aggProviderServerKey string + aggTunnelServerCert string + aggTunnelServerKey string + sandboxesCAFile string + + providerClientCert tls.Certificate // sandboxes-CA, CN=sandbox-provider-shared + + // relay dial config (sandboxes-CA relay cert + tenant-CA tenant cert). + relayAggTLS proxysidecar.TLSConfig // relay → aggregator tunnel (sandboxes CA) + relayGatewayTLS proxysidecar.TLSConfig // relay → gateway (tenant CA) +} + +// newChainEnv builds and starts the fake gateway + real aggregator. The relay +// and provider are started per-phase (so the resume test can bring up a fresh +// pair) via startRelay / dialProvider. +func newChainEnv(t *testing.T) *chainEnv { + t.Helper() + dir := t.TempDir() + + sandboxesCA := newCA(t, "sandboxes-ca") + tenantCA := newCA(t, "tenant-ca") + + env := &chainEnv{ + t: t, + sandboxesCA: sandboxesCA, + tenantCA: tenantCA, + } + + // --- CA files on disk (server creds + relay dial configs read from disk) --- + env.sandboxesCAFile = filepath.Join(dir, "sandboxes-ca.pem") + mustWrite(t, env.sandboxesCAFile, sandboxesCA.certPEM) + tenantCAFile := filepath.Join(dir, "tenant-ca.pem") + mustWrite(t, tenantCAFile, tenantCA.certPEM) + + // --- Aggregator server certs (sandboxes CA), SAN 127.0.0.1 --- + env.aggProviderServerCert, env.aggProviderServerKey = sandboxesCA.writeLeaf(t, dir, "agg-provider-server", "agg-provider-server", true) + env.aggTunnelServerCert, env.aggTunnelServerKey = sandboxesCA.writeLeaf(t, dir, "agg-tunnel-server", "agg-tunnel-server", true) + + // --- Provider client cert (sandboxes CA), tenant-less CN --- + env.providerClientCert = sandboxesCA.leafTLS(t, fcProviderSharedCN, false) + + // --- Relay client cert for the aggregator tunnel hop (sandboxes CA), + // tenant-encoding CN so the aggregator's CN binding is exercised --- + relayAggCert, relayAggKey := sandboxesCA.writeLeaf(t, dir, "relay-agg-client", fcRelayCN, false) + env.relayAggTLS = proxysidecar.TLSConfig{ + CertFile: relayAggCert, + KeyFile: relayAggKey, + CAFile: env.sandboxesCAFile, + } + + // --- Relay client cert for the tenant gateway hop (tenant CA) --- + relayGwCert, relayGwKey := tenantCA.writeLeaf(t, dir, "relay-gw-client", "sv::sandbox-provider::"+fcTenant, false) + env.relayGatewayTLS = proxysidecar.TLSConfig{ + CertFile: relayGwCert, + KeyFile: relayGwKey, + CAFile: tenantCAFile, + } + + // --- Fake tenant gateway: TLS server cert (tenant CA), requires + verifies + // a client cert chained to the tenant CA --- + gwServerCert, gwServerKey := tenantCA.writeLeaf(t, dir, "gateway-server", "tenant-gateway", true) + env.gateway = newFakeGateway() + env.gatewayAddr = startGatewayServer(t, env.gateway, gwServerCert, gwServerKey, tenantCA.certPEM) + + // --- Real aggregator over real mTLS on two listeners --- + env.aggProviderAddr, env.aggTunnelAddr = env.startAggregator(t) + + return env +} + +// startAggregator pre-binds two ephemeral ports, configures a real Aggregator +// to listen on them with mTLS server creds, and drives Run(ctx) in a goroutine. +func (env *chainEnv) startAggregator(t *testing.T) (providerAddr, tunnelAddr string) { + t.Helper() + providerAddr = grabPort(t) + tunnelAddr = grabPort(t) + + cfg := &proxysidecar.Config{} + cfg.Aggregator.Enabled = true + cfg.Aggregator.ProviderListen = "tcp://" + providerAddr + cfg.Aggregator.TunnelListen = "tcp://" + tunnelAddr + cfg.Aggregator.ProviderTLS = proxysidecar.ServerTLSConfig{ + CertFile: env.aggProviderServerCert, + KeyFile: env.aggProviderServerKey, + ClientCAFile: env.sandboxesCAFile, + } + cfg.Aggregator.TunnelTLS = proxysidecar.ServerTLSConfig{ + CertFile: env.aggTunnelServerCert, + KeyFile: env.aggTunnelServerKey, + ClientCAFile: env.sandboxesCAFile, + } + cfg.Aggregator.PairWaitTimeoutMs = 10_000 + if err := cfg.Validate(); err != nil { + t.Fatalf("aggregator cfg validate: %v", err) + } + + agg, err := proxysidecar.NewAggregator(cfg) + if err != nil { + t.Fatalf("NewAggregator: %v", err) + } + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan struct{}) + go func() { + defer close(done) + _ = agg.Run(ctx) + }() + t.Cleanup(func() { + cancel() + select { + case <-done: + case <-time.After(10 * time.Second): + t.Logf("warning: aggregator Run did not exit within 10s") + } + }) + + // Wait until both listeners accept TCP connections. + mustDialable(t, providerAddr, 5*time.Second) + mustDialable(t, tunnelAddr, 5*time.Second) + return providerAddr, tunnelAddr +} + +// startRelay constructs a real TenantRelay pointed at the fake gateway (tenant +// cert) and the aggregator tunnel (sandboxes cert), and drives Run(ctx) in a +// goroutine. Returns a cancel that tears the relay down. +func (env *chainEnv) startRelay(t *testing.T) (cancel context.CancelFunc, done <-chan struct{}) { + t.Helper() + cfg := &proxysidecar.Config{ + Gateway: proxysidecar.GatewayConfig{ + Address: env.gatewayAddr, + TLS: env.relayGatewayTLS, + }, + Service: proxysidecar.ServiceConfig{ + Implementation: "sandbox-provider", + Specifier: "tenant-relay", + }, + TenantID: fcTenant, + } + cfg.TenantRelay = proxysidecar.TenantRelayConfig{ + Enabled: true, + Tenant: fcTenant, + Aggregator: proxysidecar.AggregatorDialConfig{ + Address: env.aggTunnelAddr, + TLS: env.relayAggTLS, + }, + FilterProfile: proxysidecar.FilterProfileServicePassthrough, + } + if err := cfg.Validate(); err != nil { + t.Fatalf("tenant-relay cfg validate: %v", err) + } + + relay, err := proxysidecar.NewTenantRelay(cfg) + if err != nil { + t.Fatalf("NewTenantRelay: %v", err) + } + ctx, cancelFn := context.WithCancel(context.Background()) + d := make(chan struct{}) + go func() { + defer close(d) + _ = relay.Run(ctx) + }() + return cancelFn, d +} + +// dialProvider opens a RAW provider AetherGateway.Connect against the +// aggregator provider surface, with the sandboxes-CA provider cert and the +// x-aether-tenant metadata. Returns the stream and a cleanup. +func (env *chainEnv) dialProvider(t *testing.T, ctx context.Context, tenant string) (pb.AetherGateway_ConnectClient, func()) { + t.Helper() + tlsCreds := credentials.NewTLS(&tls.Config{ + Certificates: []tls.Certificate{env.providerClientCert}, + InsecureSkipVerify: true, // aggregator server identity not under test here + }) + conn, err := grpc.NewClient("passthrough:///"+env.aggProviderAddr, grpc.WithTransportCredentials(tlsCreds)) + if err != nil { + t.Fatalf("provider dial: %v", err) + } + ctx = metadata.NewOutgoingContext(ctx, metadata.Pairs(metadataTenantKey, tenant)) + stream, err := pb.NewAetherGatewayClient(conn).Connect(ctx) + if err != nil { + _ = conn.Close() + t.Fatalf("provider Connect: %v", err) + } + return stream, func() { _ = conn.Close() } +} + +// ============================================================================= +// TestFullChain_InitFlowDownstreamAndResume — the core composition test +// (assertions 1, 2, 3, 4). +// ============================================================================= + +func TestFullChain_InitFlowDownstreamAndResume(t *testing.T) { + env := newChainEnv(t) + + // ---- Phase 1: bring up relay + provider, drive init/up/down ---- + relayCancel, relayDone := env.startRelay(t) + + ctx, cancel := context.WithTimeout(context.Background(), 25*time.Second) + defer cancel() + + providerStream, providerClose := env.dialProvider(t, ctx, fcTenant) + + // The provider's FIRST upstream frame is its InitConnection. It carries a + // Service identity (impl=sandbox-provider, spec=pod-7) and a resume id we + // will later assert survived BOTH hops verbatim. + const resumeID1 = "rs-1" + initUp := &pb.UpstreamMessage{Payload: &pb.UpstreamMessage_Init{Init: &pb.InitConnection{ + ResumeSessionId: resumeID1, + ClientType: &pb.InitConnection_Service{Service: &pb.ServiceIdentity{ + Implementation: "sandbox-provider", + Specifier: "pod-7", + }}, + }}} + if err := providerStream.Send(initUp); err != nil { + t.Fatalf("provider send init: %v", err) + } + + // ---- Assertion 1: init reaches the gateway verbatim through BOTH hops ---- + waitFor(t, env.gateway.firstInitCh, 15*time.Second, "gateway never received provider InitConnection") + inits := env.gateway.snapshotInits() + if len(inits) != 1 { + t.Fatalf("gateway recorded %d inits, want 1", len(inits)) + } + got := inits[0] + if got.GetService().GetImplementation() != "sandbox-provider" { + t.Fatalf("init impl = %q, want sandbox-provider", got.GetService().GetImplementation()) + } + if got.GetService().GetSpecifier() != "pod-7" { + t.Fatalf("init spec = %q, want pod-7 (an intermediate hop mutated the identity)", got.GetService().GetSpecifier()) + } + if got.GetResumeSessionId() != resumeID1 { + t.Fatalf("init resume_session_id = %q, want %q (a hop dropped/rewrote resume)", got.GetResumeSessionId(), resumeID1) + } + t.Logf("assertion 1 OK: init reached gateway verbatim (impl=sandbox-provider spec=pod-7 resume=%s)", resumeID1) + + // The gateway ACKs the init; the ack flows gateway→relay→aggregator→provider. + // Read it off the provider stream so the session id is the one the gateway + // issued (used to drive the resume phase). + ack, err := providerStream.Recv() + if err != nil { + t.Fatalf("provider recv ack: %v", err) + } + sessionID := ack.GetConnectionAck().GetSessionId() + if sessionID == "" { + t.Fatalf("provider got no ConnectionAck.session_id (got %v)", ack) + } + t.Logf("downstream OK: provider received ConnectionAck session_id=%s", sessionID) + + // ---- Assertion 2: a non-init upstream frame reaches the gateway ---- + upMsg := makeNonInitUpstream("up-probe-1") + if err := providerStream.Send(upMsg); err != nil { + t.Fatalf("provider send upstream probe: %v", err) + } + waitUntil(t, 10*time.Second, "non-init upstream frame never reached gateway", func() bool { + for _, u := range env.gateway.snapshotUpstreams() { + if nonInitUpstreamMarker(u) == "up-probe-1" { + return true + } + } + return false + }) + t.Logf("assertion 2 OK: non-init upstream frame reached gateway") + + // ---- Assertion 3: a downstream the gateway pushes reaches the provider ---- + pushed := &pb.DownstreamMessage{Payload: &pb.DownstreamMessage_ConnectionAck{ + ConnectionAck: &pb.ConnectionAck{SessionId: sessionID, AssignedId: "down-probe-1"}, + }} + if err := env.gateway.pushDownstream(pushed); err != nil { + t.Fatalf("gateway push downstream: %v", err) + } + gotDown, err := providerStream.Recv() + if err != nil { + t.Fatalf("provider recv pushed downstream: %v", err) + } + if gotDown.GetConnectionAck().GetAssignedId() != "down-probe-1" { + t.Fatalf("provider downstream assigned_id = %q, want down-probe-1", gotDown.GetConnectionAck().GetAssignedId()) + } + t.Logf("assertion 3 OK: gateway-pushed downstream reached provider") + + // ---- Teardown phase-1 relay + provider for the resume phase ---- + providerClose() + relayCancel() + select { + case <-relayDone: + case <-time.After(10 * time.Second): + t.Fatalf("phase-1 relay did not exit after cancel") + } + // Wait until the aggregator has fully unregistered the tenant so the fresh + // relay can register without a duplicate-relay rejection. + waitTenantClear(t, env, 10*time.Second) + + // ---- Assertion 4: resume survives a fresh two-hop path ---- + relay2Cancel, relay2Done := env.startRelay(t) + defer func() { + relay2Cancel() + select { + case <-relay2Done: + case <-time.After(10 * time.Second): + } + }() + + provider2, provider2Close := env.dialProvider(t, ctx, fcTenant) + defer provider2Close() + + // The provider's new InitConnection carries resume_session_id = the + // session id the gateway issued in phase 1. After traversing a FRESH + // aggregator splice + FRESH relay, the gateway must observe resumed=true. + initUp2 := &pb.UpstreamMessage{Payload: &pb.UpstreamMessage_Init{Init: &pb.InitConnection{ + ResumeSessionId: sessionID, + ClientType: &pb.InitConnection_Service{Service: &pb.ServiceIdentity{ + Implementation: "sandbox-provider", + Specifier: "pod-7", + }}, + }}} + if err := provider2.Send(initUp2); err != nil { + t.Fatalf("provider2 send resume init: %v", err) + } + + ack2, err := provider2.Recv() + if err != nil { + t.Fatalf("provider2 recv resume ack: %v", err) + } + // Primary assertion: gateway flagged the session as resumed. + if !ack2.GetConnectionAck().GetResumed() { + t.Fatalf("resume ConnectionAck.resumed = false, want true (resume did not survive the fresh two-hop path)") + } + // Secondary (belt-and-suspenders): the gateway recorded the resume id on + // the second init verbatim. + if !env.gateway.wasResumed(sessionID) { + t.Fatalf("gateway did not record resume of session %q across reconnect", sessionID) + } + inits = env.gateway.snapshotInits() + if len(inits) < 2 { + t.Fatalf("gateway recorded %d inits across resume, want >=2", len(inits)) + } + if inits[len(inits)-1].GetResumeSessionId() != sessionID { + t.Fatalf("second init resume_session_id = %q, want %q", inits[len(inits)-1].GetResumeSessionId(), sessionID) + } + t.Logf("assertion 4 OK: resume survived a fresh two-hop relay+aggregator path (resumed=true, session=%s)", sessionID) +} + +// ============================================================================= +// TestFullChain_NoRelayPairWaitTimesOut — assertion 5 (cheap pair-wait path). +// +// A provider whose x-aether-tenant names a tenant with NO online relay must +// time out / error cleanly (the aggregator's bounded pair-wait), rather than +// hanging forever. +// ============================================================================= + +func TestFullChain_NoRelayPairWaitTimesOut(t *testing.T) { + env := newChainEnv(t) + // NB: no relay started for this tenant — only the aggregator + provider. + + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) + defer cancel() + + provider, providerClose := env.dialProvider(t, ctx, "tenant-with-no-relay") + defer providerClose() + + // Send the init; with no relay to pair with, the aggregator's pair-wait + // (configured to 10s in startAggregator) will fire and the provider stream + // must close with an error/ack-of-error rather than hang. + initUp := &pb.UpstreamMessage{Payload: &pb.UpstreamMessage_Init{Init: &pb.InitConnection{ + ClientType: &pb.InitConnection_Service{Service: &pb.ServiceIdentity{ + Implementation: "sandbox-provider", Specifier: "pod-lonely", + }}, + }}} + if err := provider.Send(initUp); err != nil { + t.Fatalf("provider send init: %v", err) + } + + done := make(chan struct{}) + go func() { + defer close(done) + for { + // The aggregator rejects by sending an error DownstreamMessage then + // returning a status; either an error frame or a stream error ends + // the wait. We loop until Recv returns a non-nil error OR an error + // frame. + msg, err := provider.Recv() + if err != nil { + return + } + if msg.GetError() != nil { + return + } + } + }() + select { + case <-done: + t.Logf("assertion 5 OK: provider with no online relay closed cleanly on pair-wait") + case <-time.After(15 * time.Second): + t.Fatalf("provider with no online relay did not close within pair-wait + margin (hang)") + } +} + +// ============================================================================= +// Helpers: non-init upstream frame construction + sync utilities. +// ============================================================================= + +// makeNonInitUpstream builds a NON-init UpstreamMessage carrying a recognisable +// marker. We use a ProgressReport-style payload only to get a non-init frame +// across the wire; the relay's service-passthrough bypass forwards it verbatim. +// The marker is read back via nonInitUpstreamMarker. +func makeNonInitUpstream(marker string) *pb.UpstreamMessage { + return &pb.UpstreamMessage{Payload: &pb.UpstreamMessage_Send{ + Send: &pb.SendMessage{ + TargetTopic: marker, + }, + }} +} + +// nonInitUpstreamMarker extracts the marker set by makeNonInitUpstream, or "". +func nonInitUpstreamMarker(up *pb.UpstreamMessage) string { + if sm := up.GetSend(); sm != nil { + return sm.GetTargetTopic() + } + return "" +} + +func waitFor(t *testing.T, ch <-chan struct{}, timeout time.Duration, msg string) { + t.Helper() + select { + case <-ch: + case <-time.After(timeout): + t.Fatalf("%s (timed out after %s)", msg, timeout) + } +} + +func waitUntil(t *testing.T, timeout time.Duration, msg string, cond func() bool) { + t.Helper() + deadline := time.After(timeout) + for { + if cond() { + return + } + select { + case <-deadline: + t.Fatalf("%s (timed out after %s)", msg, timeout) + case <-time.After(10 * time.Millisecond): + } + } +} + +// waitTenantClear polls the gateway until a fresh relay registration would not +// collide. We cannot inspect the aggregator's private pairing table from this +// external package, so we instead retry the relay registration implicitly by +// giving the aggregator time to observe the prior relay's stream close. A short +// settle poll is sufficient because the aggregator unregisters the relay on +// stream return (cancelled ctx) synchronously in its defer. +func waitTenantClear(t *testing.T, env *chainEnv, timeout time.Duration) { + t.Helper() + // Best-effort settle: the relay's Run returns only after its tunnel stream + // closes, which triggers the aggregator's unregisterRelay defer. We already + // waited for relayDone in the caller, so a brief poll covers the residual + // async unregister on the aggregator side. + deadline := time.After(timeout) + for { + select { + case <-deadline: + return // give up waiting; the duplicate guard would surface as a test failure downstream + case <-time.After(50 * time.Millisecond): + return // single short settle is enough; aggregator unregister is in a defer on stream return + } + } +} + +func mustDialable(t *testing.T, addr string, timeout time.Duration) { + t.Helper() + deadline := time.After(timeout) + for { + conn, err := net.DialTimeout("tcp", addr, 200*time.Millisecond) + if err == nil { + _ = conn.Close() + return + } + select { + case <-deadline: + t.Fatalf("addr %s never became dialable: %v", addr, err) + case <-time.After(20 * time.Millisecond): + } + } +} + +// grabPort binds an ephemeral port, captures its address, and closes the +// listener so the component can re-bind it. The TOCTOU window is negligible for +// loopback test usage. +func grabPort(t *testing.T) string { + t.Helper() + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("grab port: %v", err) + } + addr := ln.Addr().String() + _ = ln.Close() + return addr +} + +// startGatewayServer binds an ephemeral port and serves the fake gateway over +// mTLS (RequireAndVerifyClientCert against clientCAPEM). Returns its host:port. +func startGatewayServer(t *testing.T, gw *fakeGateway, serverCertFile, serverKeyFile string, clientCAPEM []byte) string { + t.Helper() + cert, err := tls.LoadX509KeyPair(serverCertFile, serverKeyFile) + if err != nil { + t.Fatalf("gateway load keypair: %v", err) + } + pool := x509.NewCertPool() + if !pool.AppendCertsFromPEM(clientCAPEM) { + t.Fatalf("gateway client CA pool: no certs parsed") + } + creds := credentials.NewTLS(&tls.Config{ + MinVersion: tls.VersionTLS12, + Certificates: []tls.Certificate{cert}, + ClientAuth: tls.RequireAndVerifyClientCert, + ClientCAs: pool, + }) + + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("gateway listen: %v", err) + } + srv := grpc.NewServer(grpc.Creds(creds)) + pb.RegisterAetherGatewayServer(srv, gw) + go func() { _ = srv.Serve(ln) }() + t.Cleanup(srv.Stop) + return ln.Addr().String() +} + +// ============================================================================= +// In-test CA / cert generation (same inline approach as aggregator_test.go's +// aggGenCA/aggGenLeaf and spike_tenant_relay_test.go). +// ============================================================================= + +type caBundle struct { + cert *x509.Certificate + key *rsa.PrivateKey + certPEM []byte +} + +func newCA(t *testing.T, cn string) *caBundle { + t.Helper() + key, err := rsa.GenerateKey(rand.Reader, 2048) + if err != nil { + t.Fatalf("ca key: %v", err) + } + tmpl := &x509.Certificate{ + SerialNumber: big.NewInt(time.Now().UnixNano()), + Subject: pkix.Name{CommonName: cn}, + NotBefore: time.Now().Add(-time.Hour), + NotAfter: time.Now().Add(24 * time.Hour), + IsCA: true, + KeyUsage: x509.KeyUsageCertSign | x509.KeyUsageCRLSign, + BasicConstraintsValid: true, + } + der, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, &key.PublicKey, key) + if err != nil { + t.Fatalf("ca cert: %v", err) + } + cert, err := x509.ParseCertificate(der) + if err != nil { + t.Fatalf("parse ca: %v", err) + } + return &caBundle{cert: cert, key: key, certPEM: pemEncode("CERTIFICATE", der)} +} + +// leafDER signs a leaf cert with this CA. server controls EKU and SAN (server +// leaves get 127.0.0.1 + localhost so the gRPC server-name verification on the +// relay→gateway hop passes). +func (ca *caBundle) leafDER(t *testing.T, cn string, server bool) ([]byte, *rsa.PrivateKey) { + t.Helper() + key, err := rsa.GenerateKey(rand.Reader, 2048) + if err != nil { + t.Fatalf("leaf key: %v", err) + } + eku := x509.ExtKeyUsageClientAuth + if server { + eku = x509.ExtKeyUsageServerAuth + } + tmpl := &x509.Certificate{ + SerialNumber: big.NewInt(time.Now().UnixNano()), + Subject: pkix.Name{CommonName: cn}, + NotBefore: time.Now().Add(-time.Hour), + NotAfter: time.Now().Add(24 * time.Hour), + KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment, + ExtKeyUsage: []x509.ExtKeyUsage{eku}, + BasicConstraintsValid: true, + } + if server { + tmpl.IPAddresses = []net.IP{net.ParseIP("127.0.0.1")} + tmpl.DNSNames = []string{"localhost"} + } + der, err := x509.CreateCertificate(rand.Reader, tmpl, ca.cert, &key.PublicKey, ca.key) + if err != nil { + t.Fatalf("leaf cert: %v", err) + } + return der, key +} + +// leafTLS returns a CA-signed leaf as a tls.Certificate (for in-memory dial). +func (ca *caBundle) leafTLS(t *testing.T, cn string, server bool) tls.Certificate { + t.Helper() + der, key := ca.leafDER(t, cn, server) + leaf, err := x509.ParseCertificate(der) + if err != nil { + t.Fatalf("parse leaf: %v", err) + } + return tls.Certificate{Certificate: [][]byte{der}, PrivateKey: key, Leaf: leaf} +} + +// writeLeaf writes a CA-signed leaf cert+key to dir and returns the file paths. +func (ca *caBundle) writeLeaf(t *testing.T, dir, name, cn string, server bool) (certFile, keyFile string) { + t.Helper() + der, key := ca.leafDER(t, cn, server) + certFile = filepath.Join(dir, name+".crt") + keyFile = filepath.Join(dir, name+".key") + mustWrite(t, certFile, pemEncode("CERTIFICATE", der)) + mustWrite(t, keyFile, pemEncode("RSA PRIVATE KEY", x509.MarshalPKCS1PrivateKey(key))) + return certFile, keyFile +} + +func pemEncode(typ string, der []byte) []byte { + return pem.EncodeToMemory(&pem.Block{Type: typ, Bytes: der}) +} + +func mustWrite(t *testing.T, path string, data []byte) { + t.Helper() + if err := os.WriteFile(path, data, 0o600); err != nil { + t.Fatalf("write %s: %v", path, err) + } +} diff --git a/server/internal/proxysidecar/relay.go b/server/internal/proxysidecar/relay.go index 569201b..3b0d3f3 100644 --- a/server/internal/proxysidecar/relay.go +++ b/server/internal/proxysidecar/relay.go @@ -138,6 +138,21 @@ func (r *Relay) Run(ctx context.Context) error { Time: 30 * time.Second, Timeout: 15 * time.Second, }), + // The sandbox-side aether SDK dials this relay with keepalive + // ClientParameters{Time: KeepAliveInterval (default 30s), + // PermitWithoutStream: true} — it pings to detect a dead relay even on an + // idle (no-active-stream) connection. WITHOUT an explicit + // EnforcementPolicy the relay server falls back to gRPC's default + // {MinTime: 5min, PermitWithoutStream: false}, which counts those idle 30s + // pings as abusive and sends GOAWAY ENHANCE_YOUR_CALM "too_many_pings", + // tearing the sandbox<->sidecar connection down (relay sessions churn, + // the agent loses its tunnel, and the provider's liveness eventually reaps + // the sandbox). Permit the SDK's keepalive: allow streamless pings and set + // MinTime <= the SDK's 30s interval. + grpc.KeepaliveEnforcementPolicy(keepalive.EnforcementPolicy{ + MinTime: 10 * time.Second, + PermitWithoutStream: true, + }), grpc.ConnectionTimeout(10*time.Second), grpc.MaxRecvMsgSize(10*1024*1024), grpc.MaxSendMsgSize(10*1024*1024), @@ -476,7 +491,16 @@ func (s *relaySession) observedDepth() uint32 { // using the sidecar's TLS / API-key configuration. The closer must be // invoked when the caller no longer needs the connection. func (r *Relay) dialUpstreamGateway(_ context.Context) (pb.AetherGatewayClient, func() error, error) { - tlsCfg, err := buildTLSConfig(r.cfg.Gateway) + return dialGatewayWithTLS(r.cfg.Gateway) +} + +// dialGatewayWithTLS opens a gRPC AetherGateway connection to g.Address using +// g's TLS / API-key configuration. Shared by the relay's per-session dial and +// the tenant-relay surface (which dials its local tenant gateway with the +// tenant cert under semi-strict mTLS). The closer must be invoked when the +// caller no longer needs the connection. +func dialGatewayWithTLS(g GatewayConfig) (pb.AetherGatewayClient, func() error, error) { + tlsCfg, err := buildTLSConfig(g) if err != nil { return nil, nil, fmt.Errorf("build tls: %w", err) } @@ -497,7 +521,7 @@ func (r *Relay) dialUpstreamGateway(_ context.Context) (pb.AetherGatewayClient, PermitWithoutStream: true, })) - conn, err := grpc.NewClient(r.cfg.Gateway.Address, dialOpts...) + conn, err := grpc.NewClient(g.Address, dialOpts...) if err != nil { return nil, nil, err } diff --git a/server/internal/proxysidecar/relay_filter.go b/server/internal/proxysidecar/relay_filter.go index 0c539e5..5c6e81b 100644 --- a/server/internal/proxysidecar/relay_filter.go +++ b/server/internal/proxysidecar/relay_filter.go @@ -26,13 +26,27 @@ const ( OpTunnelClose = "TunnelClose" OpTunnelAck = "TunnelAck" OpSwitchWorkspace = "SwitchWorkspace" + // OpTaskQuery covers TaskQuery (list/filter tasks). The sidecar's + // bridge uses this on startup to scan for orphaned chat_message tasks + // it owned in a prior incarnation and fail them. + OpTaskQuery = "TaskQuery" + // OpTaskOp covers TaskOperation (complete / fail / pause / etc.). The + // bridge uses this to drive the chat_message task lifecycle. + OpTaskOp = "TaskOp" ) // allowedOpsSet is an O(1)-membership view of the relay's permitted upstream // op set. InitConnection is always allowed because the sandbox must complete // the handshake before any other op is dispatched. +// +// When bypass is set (the service-passthrough profile) the op allow-list is +// skipped entirely and every op is permitted. The trust boundary in that mode +// is the provider→aggregator mTLS hop (peer-cert CN), not per-op filtering — +// so ops absent from upstreamOpName (CreateTask, SessionOperation, ACLOperation, +// ...) are forwarded verbatim instead of being silently dropped. type allowedOpsSet struct { - ops map[string]struct{} + ops map[string]struct{} + bypass bool } // resolveAllowedOps returns a set built from cfg. Profile names take @@ -43,6 +57,16 @@ func resolveAllowedOps(cfg AllowedOpsConfig) (*allowedOpsSet, error) { set := &allowedOpsSet{ops: map[string]struct{}{}} set.ops[OpInitConnection] = struct{}{} + // service-passthrough is a bypass profile: the resolved set allows every + // op (see allowedOpsSet.bypass). Used by the tenant-relay surface, whose + // trust boundary is the provider→aggregator mTLS hop rather than per-op + // filtering. Handled before profileOps so the literal op table doesn't + // have to enumerate provider-only ops. + if cfg.Profile == FilterProfileServicePassthrough { + set.bypass = true + return set, nil + } + if cfg.Profile != "" { ops, err := profileOps(cfg.Profile) if err != nil { @@ -89,6 +113,33 @@ func profileOps(profile string) ([]string, error) { OpTunnelClose, OpTunnelAck, }, nil + case AllowedOpsProfileSidecarOwn: + // Superset of sandbox-tunnels that also exposes the task-lifecycle + // ops (TaskQuery + TaskOp) the sidecar's in-process openclaw + // bridge needs to manage chat_message tasks (orphan scan at + // startup, complete/fail per-turn). Use this profile when the + // sidecar container hosts BOTH: + // - the in-sandbox SDK that talks data-connectors / memorylayer + // via ProxyHttp (the sandbox-tunnels piece), AND + // - the in-sidecar openclaw bridge that drives chat_message + // task lifecycle via TaskQuery + TaskOp. + // The relay listener is shared between both consumers (one + // listener per sidecar container), so the profile must cover + // the union of their needs. + return []string{ + OpSendMessage, + OpProgressReport, + OpKVOperation, + OpProxyHttpRequest, + OpProxyHttpBodyChunk, + OpProxyHttpResponse, + OpTunnelOpen, + OpTunnelData, + OpTunnelClose, + OpTunnelAck, + OpTaskQuery, + OpTaskOp, + }, nil case AllowedOpsProfileToolStubOnly: // Only the InitConnection handshake; any other op is denied. The // sandbox is expected to *receive* envelopes (e.g. ProxyHttpRequest) @@ -113,24 +164,34 @@ func knownRelayOp(name string) bool { OpTunnelData, OpTunnelClose, OpTunnelAck, - OpSwitchWorkspace: + OpSwitchWorkspace, + OpTaskQuery, + OpTaskOp: return true } return false } -// allows reports whether op is permitted. +// allows reports whether op is permitted. A bypass set (service-passthrough) +// allows every op unconditionally. func (s *allowedOpsSet) allows(op string) bool { if s == nil { return false } + if s.bypass { + return true + } _, ok := s.ops[op] return ok } // list returns the sorted list of allowed ops. Used in logs and ErrorResponse -// detail so operators can see the resolved set without consulting config. +// detail so operators can see the resolved set without consulting config. A +// bypass set reports a single sentinel so logs make the passthrough explicit. func (s *allowedOpsSet) list() []string { + if s != nil && s.bypass { + return []string{""} + } out := make([]string, 0, len(s.ops)) for op := range s.ops { out = append(out, op) @@ -216,6 +277,10 @@ func upstreamOpName(msg *pb.UpstreamMessage) string { return OpTunnelAck case *pb.UpstreamMessage_SwitchWorkspace: return OpSwitchWorkspace + case *pb.UpstreamMessage_TaskQuery: + return OpTaskQuery + case *pb.UpstreamMessage_TaskOp: + return OpTaskOp } return "" } diff --git a/server/internal/proxysidecar/relay_test.go b/server/internal/proxysidecar/relay_test.go index 41b2160..a86e0ed 100644 --- a/server/internal/proxysidecar/relay_test.go +++ b/server/internal/proxysidecar/relay_test.go @@ -795,6 +795,11 @@ func TestUpstreamOpName(t *testing.T) { {&pb.UpstreamMessage{Payload: &pb.UpstreamMessage_KvOp{KvOp: &pb.KVOperation{}}}, OpKVOperation}, {&pb.UpstreamMessage{Payload: &pb.UpstreamMessage_ProxyHttpRequest{ProxyHttpRequest: &pb.ProxyHttpRequest{}}}, OpProxyHttpRequest}, {&pb.UpstreamMessage{Payload: &pb.UpstreamMessage_TunnelOpen{TunnelOpen: &pb.TunnelOpen{}}}, OpTunnelOpen}, + // Bridge-driven task-lifecycle ops: TaskQuery (orphan scan) and + // TaskOp (complete / fail / pause). Without these cases the + // relay returns "" and labels them in RELAY_OP_DENIED. + {&pb.UpstreamMessage{Payload: &pb.UpstreamMessage_TaskQuery{TaskQuery: &pb.TaskQuery{}}}, OpTaskQuery}, + {&pb.UpstreamMessage{Payload: &pb.UpstreamMessage_TaskOp{TaskOp: &pb.TaskOperation{}}}, OpTaskOp}, } for _, tc := range cases { got := upstreamOpName(tc.msg) @@ -804,6 +809,39 @@ func TestUpstreamOpName(t *testing.T) { } } +func TestProfileOps_SidecarOwn(t *testing.T) { + // sidecar-own must be a superset of sandbox-tunnels (so the in-sandbox + // SDK keeps its proxy/tunnel access) and additionally allow the task + // lifecycle ops the in-sidecar bridge needs. + got, err := profileOps(AllowedOpsProfileSidecarOwn) + if err != nil { + t.Fatalf("profileOps(sidecar-own) error: %v", err) + } + gotSet := make(map[string]struct{}, len(got)) + for _, op := range got { + gotSet[op] = struct{}{} + } + required := []string{ + OpSendMessage, + OpProgressReport, + OpKVOperation, + OpProxyHttpRequest, + OpProxyHttpBodyChunk, + OpProxyHttpResponse, + OpTunnelOpen, + OpTunnelData, + OpTunnelClose, + OpTunnelAck, + OpTaskQuery, + OpTaskOp, + } + for _, op := range required { + if _, ok := gotSet[op]; !ok { + t.Errorf("sidecar-own profile missing required op %q", op) + } + } +} + func TestSplitListenSpec(t *testing.T) { cases := []struct { in string diff --git a/server/internal/proxysidecar/relaymux.go b/server/internal/proxysidecar/relaymux.go new file mode 100644 index 0000000..32e5309 --- /dev/null +++ b/server/internal/proxysidecar/relaymux.go @@ -0,0 +1,1275 @@ +// Package proxysidecar relay-mux mode. +// +// RelayMux is an alternative to Relay (relay.go) that fixes the per-identity +// session-collision problem the legacy relay has when MORE THAN ONE sub-client +// shares the relay socket. +// +// Why the legacy relay collides: +// +// Relay.Connect opens a SEPARATE upstream gateway stream per accepted +// sub-client (dialUpstreamGateway → a fresh grpc stream per Connect) and +// rewrites EVERY sub-client's InitConnection to the SAME service identity +// (service:/). The gateway's session lock is per-identity, so N +// concurrent sub-client streams collide on one session: request/response +// replies (KV, TaskOp, TaskQuery, ...) get delivered to whichever stream +// currently owns the lock, NOT the stream that asked → DEADLINE_EXCEEDED on +// the loser. One-way SendMessage survives because it expects no reply. +// +// How RelayMux fixes it: +// +// RelayMux owns ONE shared upstream gateway connection (one InitConnection, +// rewritten once to the sidecar's service identity + api key). It accepts +// many sub-client streams on the same AetherGateway server surface, fans all +// their upstream frames onto the single upstream, and demultiplexes the +// downstream: +// - CORRELATED ops (carry a request_id): the request_id is rewritten to a +// globally-unique mux id on the way up and restored on the matching reply +// on the way down, which is delivered ONLY to the originating sub-client. +// In composite mode the mux ids are prefixed "rmx-" to guarantee they +// never collide with the terminator's own "req-N" ids on the same +// connection (the pending table lookup is the source of truth for routing; +// the prefix eliminates any possibility of cross-surface id collision). +// - TUNNEL frames: routed by tunnel id to the sub-client that opened the +// tunnel. +// - PUSH frames (IncomingMessage, ConfigSnapshot, Signal, TaskAssignment, +// ProgressUpdate, inbound ProxyHttpRequest, ...): broadcast to every +// sub-client; each uses or ignores at will (e.g. a broadcast +// TaskAssignment is picked up by the harness it belongs to). +// +// Two operating modes: +// +// STANDALONE (no terminator): RelayMux dials its own upstream gateway +// connection (one InitConnection, rewritten to the sidecar identity). The +// downstream pump reads from up.stream.Recv and dispatches frames via +// dispatchDownstream, which enqueues into each sub-client's bounded inbox. +// +// COMPOSITE (terminator + relay on one shared connection): the terminator +// owns the single gateway connection via gatewayRuntime. RelayMux does NOT +// dial its own upstream; instead the runner wires it via SetSharedUpstream: +// - Outbound (sub-client → gateway): RelayMux calls sharedSend (which +// routes through runtime.Client().SendWithPriority) instead of +// up.stream.Send. +// - Inbound (gateway → sub-client): the SDK's rawDownstreamTap calls +// RouteDownstream inline (no separate inbox channel needed). For +// correlated/tunnel frames RouteDownstream enqueues directly to the +// owning sub-client's inbox. Broadcast push types (TaskAssignment, +// ProgressUpdate) are also claimed inline and enqueued to all sub-client +// inboxes. Everything else falls through to SDK typed dispatch so that +// OnProxyHttpResponse (terminator wakeup), Signal (SDK state mutations), +// Config, and OnMessage callbacks still fire normally. +// The mux id namespace is prefixed "rmx-" so relay correlated ids never +// collide with the terminator's "req-N" ids on the same stream. +// +// Backpressure (both modes): +// +// Each muxSubClient has a bounded inbox channel and a priority-aware +// deliverSem (backpressure.Semaphore with CoDel). Neither dispatchDownstream +// (standalone pump) nor RouteDownstream (composite tap) blocks the caller on +// a slow in-sandbox reader — they enqueue via deliver() which times out and +// synthesises a BACKPRESSURE error frame on shed, exactly as sharedRuntimeSession +// does. A per-sub-client writer goroutine drains the inbox and does the +// blocking stream.Send, so a wedged reader on one sub-client never stalls the +// shared runtime receive loop or other sub-clients. +// +// Reconnect policy (v1, deliberately simple): +// +// Standalone: on shared-upstream error/EOF, ALL sub-client streams are +// cancelled and the pending/tunnels tables are torn down; sub-clients +// reconnect and re-handshake and the next connect re-establishes the +// upstream. We do NOT buffer frames across a reconnect. +// Composite: reconnect is owned by gatewayRuntime; RelayMux cancels all +// sub-clients on Run ctx cancellation. +// +// Relay (relay.go) is left intact for its existing callers/tests. +// RelayMux is now the default for BOTH standalone and composite modes when +// cfg.Relay.Mux is true (default). See runner.go. +package proxysidecar + +import ( + "context" + "errors" + "fmt" + "io" + "net" + "strconv" + "sync" + "sync/atomic" + "time" + + "github.com/rs/zerolog/log" + pb "github.com/scitrera/aether/api/proto" + "github.com/scitrera/go-backpressure" + "google.golang.org/grpc" + "google.golang.org/grpc/keepalive" +) + +// muxSharedSendFn is the function signature for sending an upstream frame +// through the shared runtime in composite mode. It mirrors +// sharedRuntimeSession.Send's SendWithPriority call so the mux can route +// outbound envelopes through the terminator's shared gateway queue. +type muxSharedSendFn func(ctx context.Context, prio backpressure.Priority, msg *pb.UpstreamMessage) error + +// RelayMux runs the sidecar in shared-upstream relay mode. It owns a local +// gRPC server (on UDS or TCP) and exactly ONE upstream gateway connection +// shared across all accepted sub-client streams. +// +// In STANDALONE mode (no shared runtime) it dials its own upstream. In +// COMPOSITE mode (terminator + relay) it is wired via SetSharedUpstream to +// send through the terminator's shared runtime queue and receive frames from +// sharedInbox fed by the downstreamRouter. +type RelayMux struct { + pb.UnimplementedAetherGatewayServer + + cfg *Config + allowed *allowedOpsSet + clamp *targetClamp + + // upstreamDialer builds the single outbound gRPC connection to the real + // gateway. Production uses dialUpstreamGateway (shared with Relay); tests + // inject a fake. Nil in composite mode (no own-dialed upstream). + upstreamDialer func(ctx context.Context) (pb.AetherGatewayClient, func() error, error) + + // composite-mode upstream: when sharedSend is non-nil the mux is in + // composite mode and routes outbound frames through the shared runtime. + // Downstream frames reach sub-clients via RouteDownstream (called from + // the rawDownstreamTap for correlated/tunnel frames) and via SDK + // callbacks (routeToRelay) for broadcast frames — no separate inbox + // channel is needed. + sharedSend muxSharedSendFn + + srv *grpc.Server + listener net.Listener + + // subSeq assigns each accepted sub-client a unique id. + subSeq atomic.Uint64 + // muxSeq assigns each correlated upstream op a globally-unique request id. + muxSeq atomic.Uint64 + + // mu guards everything below: the shared upstream lifecycle and the + // routing tables. + mu sync.Mutex + // up is the live shared-upstream binding, or nil when not established + // (always nil in composite mode). + up *muxUpstream + // subs is the set of registered sub-clients keyed by sub-client id. + subs map[uint64]*muxSubClient + // pending maps a mux request id to the sub-client + original request id + // that originated it, so the demux pump can restore and route the reply. + pending map[string]muxPending + // tunnels maps a tunnel id to the sub-client id that opened it, so tunnel + // downstream frames route back to the right sub-client. + tunnels map[string]uint64 +} + +// muxUpstream is one live shared-upstream binding. A new one is created each +// time the upstream is (re-)established; teardown compares by pointer identity +// so a stale pump's teardown can't tear down a newer binding. +type muxUpstream struct { + stream pb.AetherGateway_ConnectClient + closer func() error + cancel context.CancelFunc +} + +// muxPending records the owner of a correlated in-flight upstream op. +type muxPending struct { + subID uint64 + origReqID string + payloadTag string // for logging only +} + +// muxSubClient is one accepted sub-client stream. Downstream delivery is +// decoupled from the upstream receive loop: deliver() enqueues into a bounded +// inbox via a priority-aware CoDel semaphore, and a per-sub-client writer +// goroutine drains the inbox and does the blocking stream.Send. A wedged +// in-sandbox reader therefore cannot stall the shared runtime receive loop or +// any other sub-client. +type muxSubClient struct { + id uint64 + stream pb.AetherGateway_ConnectServer + cancel context.CancelFunc + closed atomic.Bool + + // inbox + deliverSem implement bounded priority-aware downstream admission. + // Sized and tuned identically to sharedRuntimeSession so both relay paths + // behave consistently under sustained inbox pressure. + inbox chan *pb.DownstreamMessage + deliverSem *backpressure.Semaphore + + // inboundDepth is the largest proxy/tunnel chain depth this sub-client has + // observed on broadcast inbound frames; outbound clamps floor against it. + depthMu sync.Mutex + inboundDepth uint32 +} + +// NewRelayMux constructs a RelayMux from cfg. The local listener is not opened +// until Run is invoked. cfg.Validate() must have been called first. +func NewRelayMux(cfg *Config) (*RelayMux, error) { + allowed, err := resolveAllowedOps(cfg.Relay.AllowedOps) + if err != nil { + return nil, err + } + m := &RelayMux{ + cfg: cfg, + allowed: allowed, + clamp: newTargetClamp(cfg.Relay.TargetTopicClamp), + subs: map[uint64]*muxSubClient{}, + pending: map[string]muxPending{}, + tunnels: map[string]uint64{}, + } + m.upstreamDialer = m.dialUpstreamGateway + return m, nil +} + +// dialUpstreamGateway reuses Relay's dialer so the TLS/keepalive/api-key dial +// stays in one place. A throwaway Relay value is the cheapest way to share the +// method without restructuring relay.go. +func (m *RelayMux) dialUpstreamGateway(ctx context.Context) (pb.AetherGatewayClient, func() error, error) { + return (&Relay{cfg: m.cfg}).dialUpstreamGateway(ctx) +} + +// SetUpstreamDialer replaces the mux's upstream dialer (used by tests). +func (m *RelayMux) SetUpstreamDialer(dialer func(ctx context.Context) (pb.AetherGatewayClient, func() error, error)) { + m.upstreamDialer = dialer +} + +// SetSharedUpstream configures the mux for COMPOSITE mode. Outbound frames +// are sent through sendFn (the shared runtime's SendWithPriority). Downstream +// frames reach sub-clients via two paths: +// - Correlated/tunnel: RouteDownstream is called from the rawDownstreamTap +// and delivers directly to the owning sub-client. +// - Broadcast/push: the SDK's typed callbacks (OnProxyHttpResponse, +// OnTunnelDataIn, OnMessage, …) call routeToRelay which broadcasts to all +// relay sub-clients after the terminator surface has had its chance. +// +// Must be called before Run. Calling this disables own-upstream dialing. +func (m *RelayMux) SetSharedUpstream(sendFn muxSharedSendFn) { + m.sharedSend = sendFn +} + +// RouteDownstream is called by the rawDownstreamTap for correlated-response +// downstream frames in composite mode. It claims a frame only when the mux +// recognises it as relay-owned: a correlated reply (rmx-* mux id in the pending +// table), a relay-owned tunnel frame, or a PROXY response/chunk whose +// request_id decodes (via the stateless per-sub proxy-id prefix) to a live +// sub-client — and delivers it directly to the right sub-client. +// +// Broadcast/push frames (TaskAssignment, Signal, Config, IncomingMessage, ...) +// are NOT claimed here. They travel through the normal SDK typed-dispatch path +// so that OnTunnelDataIn etc. still fire for the terminator surface, and those +// SDK callbacks call routeToRelay to fan the frame to relay sub-clients. +// Claiming broadcasts in the tap would swallow terminator wakeups. +// +// A PROXY response whose request_id is NOT relay-encoded (the terminator's own +// "req-N" proxy response) decodes to ok=false and falls through to broadcast → +// returns false so the SDK's OnProxyHttpResponse terminator dispatch still +// fires (e.g. the "your caller went away" ProxyHttpResponse error that cancels +// an active terminator dispatch). +// +// Returns true only when the frame was consumed (correlated demux delivered, +// or tunnel routed). Returns false for broadcasts, unknown ids, and frames +// the terminator or SDK must handle — letting the SDK continue normal dispatch. +// +// In standalone mode RouteDownstream is never called; frames arrive via the +// mux's own pumpDownstream goroutine. +func (m *RelayMux) RouteDownstream(msg *pb.DownstreamMessage) bool { + if m.sharedSend == nil { + return false + } + m.mu.Lock() + dec := m.classifyDownstream(msg) + + switch dec.route { + case routeCorrelated, routeTunnel: + // Correlated/tunnel: claim and deliver to the owning sub-client. + // Apply table mutations before unlocking so a concurrent send for + // the same id can't double-deliver. + delete(m.pending, dec.muxReqID) + if dec.tunnelDone { + delete(m.tunnels, dec.tunnelID) + } + target := m.subs[dec.subID] + m.mu.Unlock() + + if target == nil { + if dec.route == routeCorrelated { + log.Info().Str("mux_request_id", dec.muxReqID). + Msg("relaymux: correlated reply for departed sub-client; dropping") + } else { + log.Info().Str("tunnel_id", dec.tunnelID). + Msg("relaymux: tunnel frame for departed sub-client; dropping") + } + return true // consumed; don't let SDK misroute it + } + if dec.route == routeCorrelated { + restoreDownstreamRequestID(msg, dec.origReqID) + } + target.deliver(msg) + return true + + case routeProxyToSub: + // Stateless proxy response/chunk routed to its owning sub-client only. + // No pending entry to delete; just restore the proxy request_id and + // deliver. Claiming it (return true) prevents the SDK's terminator + // OnProxyHttpResponse dispatch from also seeing a relay-owned response. + target := m.subs[dec.subID] + m.mu.Unlock() + if target == nil { + // Sub departed between classify and here: fall through so the SDK + // can handle/drop it (it won't match any terminator dispatch). + return false + } + restoreDownstreamProxyReqID(msg, dec.origReqID) + target.deliver(msg) + return true + + case routeBroadcast: + // Broadcast: only claim push frame types that have no SDK-internal + // side effects AND that the terminator does not need from the typed + // dispatch path. These types have no registered SDK callbacks in + // installOn, so claiming them in the tap is the only way relay + // sub-clients receive them. + // + // NOT claimed (must fall through to SDK dispatch): + // ProxyHttpResponse — terminator's activeDispatches wakeup path + // ProxyHttpBodyChunk — terminator's chunked-request accumulator + // Signal — SDK mutates forceDisconnect/connected atomics + // Config — bootstraps SDK KV state + // Error — handled by SDK, re-checked by relay tap for correlated + // + // Safe to claim (relay-bound pushes the sub-clients need, no terminator/ + // SDK state effects): TaskAssignment, ProgressUpdate, and Msg. + // Msg (IncomingMessage) is the workclaw/sahara chat envelope the + // in-sandbox harness consumes via its OnMessage→TaskSource path. It MUST + // be claimed here: the installOn OnMessage callback re-forwards Msg via + // routeToRelay, which re-enters this RouteDownstream — so leaving Msg in + // the default branch dropped it (the harness never saw the chat task and + // stayed idle). Claiming it broadcasts to the sub-clients; the SDK then + // skips its OnMessage dispatch (tap consumed it), so no double delivery. + switch msg.GetPayload().(type) { + case *pb.DownstreamMessage_TaskAssignment, + *pb.DownstreamMessage_ProgressUpdate, + *pb.DownstreamMessage_Msg: + default: + // All other broadcast types: let SDK typed dispatch handle them. + // OnProxyHttpResponse, OnMessage etc. call routeToRelay afterward. + m.mu.Unlock() + return false + } + + broadcast := make([]*muxSubClient, 0, len(m.subs)) + for _, s := range m.subs { + broadcast = append(broadcast, s) + } + m.mu.Unlock() + + for _, s := range broadcast { + s.deliver(msg) + } + return true // claimed; SDK need not process further + + default: + m.mu.Unlock() + return false + } +} + +// Run serves the relay-mux until ctx is cancelled. +// +// In STANDALONE mode it binds the configured listener, registers as the +// AetherGateway server, and pumps the shared upstream. In COMPOSITE mode the +// gateway listener is shared with the terminator (via Runner.Run), so Run +// only drains the sharedInbox channel fed by the downstreamRouter and +// dispatches frames to sub-clients. +func (m *RelayMux) Run(ctx context.Context) error { + if m.sharedSend != nil { + return m.runComposite(ctx) + } + return m.runStandalone(ctx) +} + +// runStandalone is the original Run implementation for standalone relay mode. +func (m *RelayMux) runStandalone(ctx context.Context) error { + listener, cleanup, err := openRelayListener(m.cfg.Relay.Listen) + if err != nil { + return fmt.Errorf("relaymux: open listener: %w", err) + } + m.listener = listener + defer func() { + _ = listener.Close() + if cleanup != nil { + cleanup() + } + }() + + server := grpc.NewServer( + grpc.KeepaliveParams(keepalive.ServerParameters{ + Time: 30 * time.Second, + Timeout: 15 * time.Second, + }), + grpc.KeepaliveEnforcementPolicy(keepalive.EnforcementPolicy{ + MinTime: 10 * time.Second, + PermitWithoutStream: true, + }), + grpc.ConnectionTimeout(10*time.Second), + grpc.MaxRecvMsgSize(10*1024*1024), + grpc.MaxSendMsgSize(10*1024*1024), + grpc.MaxHeaderListSize(16*1024), + ) + pb.RegisterAetherGatewayServer(server, m) + m.srv = server + + serveErr := make(chan error, 1) + go func() { + log.Info(). + Str("listen", m.cfg.Relay.Listen). + Str("identity", m.cfg.Service.Implementation+"/"+m.cfg.Service.Specifier). + Strs("allowed_ops", m.allowed.list()). + Str("clamp_mode", m.cfg.Relay.TargetTopicClamp.Mode). + Int("allowed_targets", len(m.cfg.Relay.TargetTopicClamp.AllowedTargets)). + Msg("proxy sidecar relay-mux running") + serveErr <- server.Serve(listener) + }() + + select { + case <-ctx.Done(): + log.Info().Msg("proxy sidecar relay-mux shutting down") + const gracePeriod = 3 * time.Second + gracefulDone := make(chan struct{}) + go func() { + server.GracefulStop() + close(gracefulDone) + }() + select { + case <-gracefulDone: + case <-time.After(gracePeriod): + log.Warn(). + Dur("grace", gracePeriod). + Msg("relaymux: GracefulStop exceeded grace window; forcing Stop()") + server.Stop() + <-gracefulDone + } + m.teardownUpstream(nil) + <-serveErr + return nil + case err := <-serveErr: + if err != nil { + return fmt.Errorf("relaymux: serve: %w", err) + } + return nil + } +} + +// runComposite is the Run implementation for composite mode. +// +// The sandbox-facing local gRPC listener is the same as standalone mode — +// sandbox processes still connect over UDS/TCP to the relay address. What +// differs is the upstream path: instead of dialing its own gateway stream, +// the mux routes outbound frames through sharedSend (the shared runtime's +// SendWithPriority). Downstream frames reach sub-clients via two paths that +// need no extra goroutine here: +// - Correlated/tunnel: RouteDownstream is called inline from the SDK's +// rawDownstreamTap on the runtime's receive goroutine and delivers +// directly to the owning sub-client. +// - Broadcast/push: SDK callbacks (OnProxyHttpResponse, OnTunnelDataIn, +// OnMessage, …) call routeToRelay which fans to all relay sub-clients. +func (m *RelayMux) runComposite(ctx context.Context) error { + listener, cleanup, err := openRelayListener(m.cfg.Relay.Listen) + if err != nil { + return fmt.Errorf("relaymux: open listener: %w", err) + } + m.listener = listener + defer func() { + _ = listener.Close() + if cleanup != nil { + cleanup() + } + }() + + server := grpc.NewServer( + grpc.KeepaliveParams(keepalive.ServerParameters{ + Time: 30 * time.Second, + Timeout: 15 * time.Second, + }), + grpc.KeepaliveEnforcementPolicy(keepalive.EnforcementPolicy{ + MinTime: 10 * time.Second, + PermitWithoutStream: true, + }), + grpc.ConnectionTimeout(10*time.Second), + grpc.MaxRecvMsgSize(10*1024*1024), + grpc.MaxSendMsgSize(10*1024*1024), + grpc.MaxHeaderListSize(16*1024), + ) + pb.RegisterAetherGatewayServer(server, m) + m.srv = server + + serveErr := make(chan error, 1) + go func() { + log.Info(). + Str("listen", m.cfg.Relay.Listen). + Str("identity", m.cfg.Service.Implementation+"/"+m.cfg.Service.Specifier). + Strs("allowed_ops", m.allowed.list()). + Str("clamp_mode", m.cfg.Relay.TargetTopicClamp.Mode). + Int("allowed_targets", len(m.cfg.Relay.TargetTopicClamp.AllowedTargets)). + Msg("proxy sidecar relay-mux composite running") + serveErr <- server.Serve(listener) + }() + + select { + case <-ctx.Done(): + log.Info().Msg("proxy sidecar relay-mux composite shutting down") + // Cancel all sub-clients so their Connect handlers return promptly. + m.mu.Lock() + subs := make([]*muxSubClient, 0, len(m.subs)) + for _, s := range m.subs { + subs = append(subs, s) + } + m.subs = map[uint64]*muxSubClient{} + m.pending = map[string]muxPending{} + m.tunnels = map[string]uint64{} + m.mu.Unlock() + for _, s := range subs { + s.closeSubClient() + s.cancel() + } + + const gracePeriod = 3 * time.Second + gracefulDone := make(chan struct{}) + go func() { + server.GracefulStop() + close(gracefulDone) + }() + select { + case <-gracefulDone: + case <-time.After(gracePeriod): + log.Warn(). + Dur("grace", gracePeriod). + Msg("relaymux: composite GracefulStop exceeded grace window; forcing Stop()") + server.Stop() + <-gracefulDone + } + <-serveErr + return nil + case err := <-serveErr: + if err != nil { + return fmt.Errorf("relaymux: composite serve: %w", err) + } + return nil + } +} + +// Connect implements pb.AetherGatewayServer. Each invocation is one sub-client. +func (m *RelayMux) Connect(stream pb.AetherGateway_ConnectServer) error { + // First message MUST be InitConnection — mirror Relay.Connect. We do NOT + // forward it upstream; the shared upstream did its single Init already. + first, err := stream.Recv() + if err != nil { + if errors.Is(err, io.EOF) { + return nil + } + return err + } + init, ok := first.GetPayload().(*pb.UpstreamMessage_Init) + if !ok || init == nil { + _ = stream.Send(relayErrorDownstream("RELAY_INVALID_INIT", + "first message on sandbox stream must be InitConnection")) + return fmt.Errorf("relaymux: first message was %T, expected InitConnection", first.GetPayload()) + } + + // Ensure the shared upstream is live before we admit the sub-client. + if err := m.ensureUpstream(stream.Context()); err != nil { + _ = stream.Send(relayErrorDownstream("RELAY_UPSTREAM_UNAVAILABLE", + fmt.Sprintf("shared upstream not available: %v", err))) + return err + } + + subID := m.subSeq.Add(1) + ctx, cancel := context.WithCancel(stream.Context()) + sub := &muxSubClient{ + id: subID, + stream: stream, + cancel: cancel, + inbox: make(chan *pb.DownstreamMessage, sharedRuntimeSessionDeliverCapacity*2), + deliverSem: backpressure.NewSemaphore( + 5, // 5 priorities (PriorityControl..PriorityBestEffort) + sharedRuntimeSessionDeliverCapacity, + backpressure.SemaphoreShortTimeout(sharedRuntimeSessionDeliverShortTimeout), + backpressure.SemaphoreLongTimeout(sharedRuntimeSessionDeliverLongTimeout), + ), + } + + m.mu.Lock() + m.subs[subID] = sub + m.mu.Unlock() + + logger := log.With().Uint64("mux_sub", subID).Logger() + logger.Info(). + Str("sandbox_claim", describeSandboxIdentity(init.Init)). + Msg("relaymux: sub-client connected") + + // Start the per-sub-client writer goroutine. It drains inbox and calls + // the blocking stream.Send, decoupling downstream delivery from the + // upstream receive loop (rawDownstreamTap / pumpDownstream). The goroutine + // exits when ctx is cancelled or the inbox is closed by closeSubClient. + writerDone := make(chan struct{}) + go func() { + defer close(writerDone) + sub.runWriter(ctx) + }() + + // Synthesize a ConnectionAck so the sub-client's SDK considers itself + // connected. The Go SDK confirms the connection on the FIRST downstream + // frame and handleConnectionAck reads only session_id (+ resumed), so a + // per-sub-client synthetic session id is sufficient. We mint one rather + // than echo the shared upstream's so distinct sub-clients never observe a + // colliding session id. + sub.deliver(&pb.DownstreamMessage{ + Payload: &pb.DownstreamMessage_ConnectionAck{ + ConnectionAck: &pb.ConnectionAck{ + SessionId: fmt.Sprintf("mux-sub-%d", subID), + }, + }, + }) + + // Pump this sub-client's upstream frames onto the shared upstream until + // the sub-client closes or the upstream is torn down (cancel fires). + err = m.pumpSubUpstream(ctx, sub) + sub.closeSubClient() + <-writerDone + m.removeSub(subID) + logger.Info().Err(err).Msg("relaymux: sub-client closed") + if errors.Is(err, io.EOF) || errors.Is(err, context.Canceled) { + return nil + } + return err +} + +// ensureUpstream lazily establishes the shared upstream connection. It is safe +// to call concurrently; only the first caller dials. +// In composite mode this is a no-op — the upstream is the shared runtime. +func (m *RelayMux) ensureUpstream(ctx context.Context) error { + if m.sharedSend != nil { + // Composite mode: no own upstream to establish. + return nil + } + + m.mu.Lock() + if m.up != nil { + m.mu.Unlock() + return nil + } + m.mu.Unlock() + + // Dial outside the lock (it can block); then re-check under the lock so a + // racing caller doesn't double-establish. + dialCtx, cancelDial := context.WithTimeout(context.Background(), 30*time.Second) + client, closer, err := m.upstreamDialer(dialCtx) + cancelDial() + if err != nil { + return err + } + + upCtx, upCancel := context.WithCancel(context.Background()) + stream, err := client.Connect(upCtx) + if err != nil { + upCancel() + if closer != nil { + _ = closer() + } + return err + } + + apiKey, _ := loadAPIKey(m.cfg.Gateway) + rewritten := rewriteInitConnection(&pb.InitConnection{}, m.cfg, apiKey) + if err := stream.Send(&pb.UpstreamMessage{ + Payload: &pb.UpstreamMessage_Init{Init: rewritten}, + }); err != nil { + upCancel() + if closer != nil { + _ = closer() + } + return err + } + + m.mu.Lock() + if m.up != nil { + // Lost the race; tear down the connection we just built. + m.mu.Unlock() + upCancel() + if closer != nil { + _ = closer() + } + return nil + } + up := &muxUpstream{stream: stream, closer: closer, cancel: upCancel} + m.up = up + m.mu.Unlock() + + log.Info().Msg("relaymux: shared upstream established") + go m.pumpDownstream(up) + return nil +} + +// pumpDownstream is the single reader of the shared upstream. It demultiplexes +// each downstream frame to the right sub-client(s) per routeDownstream, and on +// upstream error/EOF tears the whole mux down (all sub-clients + tables). +func (m *RelayMux) pumpDownstream(up *muxUpstream) { + for { + msg, err := up.stream.Recv() + if err != nil { + if errors.Is(err, io.EOF) { + log.Info().Msg("relaymux: shared upstream closed (EOF)") + } else { + log.Warn().Err(err).Msg("relaymux: shared upstream error; tearing down sub-clients") + } + m.teardownUpstream(up) + return + } + m.dispatchDownstream(msg) + } +} + +// muxRoute is the routing decision for a downstream frame. +type muxRoute int + +const ( + // routeCorrelated: deliver to a single sub-client identified by a pending + // mux request id (which must be restored to origReqID first). + routeCorrelated muxRoute = iota + // routeTunnel: deliver to the sub-client that owns tunnelID. + routeTunnel + // routeBroadcast: deliver a copy to every registered sub-client. + routeBroadcast + // routeDropUnknownCorrelated: a correlated/tunnel frame whose id is not in + // the routing tables — stale/unknown; log + drop (do NOT broadcast). + routeDropUnknownCorrelated + // routeProxyToSub: a proxy RESPONSE/CHUNK frame whose request_id decoded to a + // LIVE sub-client (the stateless per-sub proxy-id prefix scheme; see + // relaymux_reqid.go). Deliver to that sub-client only, after restoring the + // original proxy request_id. Unlike routeCorrelated there is NO pending entry + // (proxy is stateless), so no table delete. + routeProxyToSub +) + +// downstreamDecision is the pure routing classification of a downstream frame +// against the current pending/tunnel tables. It is computed under m.mu by +// classifyDownstream and consumed by dispatchDownstream. Splitting it out keeps +// the routing logic table-testable without a live stream. +type downstreamDecision struct { + route muxRoute + subID uint64 // target for routeCorrelated / routeTunnel + origReqID string // restore target for routeCorrelated + muxReqID string // pending key to delete for routeCorrelated + tunnelID string // for routeTunnel close-cleanup + tunnelDone bool // routeTunnel + TunnelClose → delete the tunnel entry +} + +// classifyDownstream computes the routing decision for msg. It must be called +// with m.mu held. It does NOT mutate the tables; the caller applies any +// deletes after sending so a failed send still cleans up. +func (m *RelayMux) classifyDownstream(msg *pb.DownstreamMessage) downstreamDecision { + // 1. ErrorResponse: demux when it carries a known pending request id, else + // broadcast (connection-level errors have no request_id). + if reqID, isErr := downstreamErrorRequestID(msg); isErr { + if reqID != "" { + if p, ok := m.pending[reqID]; ok { + return downstreamDecision{route: routeCorrelated, subID: p.subID, origReqID: p.origReqID, muxReqID: reqID} + } + } + return downstreamDecision{route: routeBroadcast} + } + + // 2. Correlated reply: restore + deliver to the originating sub-client. + if reqID, correlated := downstreamGetRequestID(msg); correlated { + if p, ok := m.pending[reqID]; ok { + return downstreamDecision{route: routeCorrelated, subID: p.subID, origReqID: p.origReqID, muxReqID: reqID} + } + // Unknown/stale correlated id: drop, never broadcast. + return downstreamDecision{route: routeDropUnknownCorrelated, muxReqID: reqID} + } + + // 3. Tunnel frame: route by tunnel id to the owning sub-client. + if tunID, isTunnel := downstreamTunnelID(msg); isTunnel { + if subID, ok := m.tunnels[tunID]; ok { + _, isClose := msg.Payload.(*pb.DownstreamMessage_TunnelClose) + return downstreamDecision{route: routeTunnel, subID: subID, tunnelID: tunID, tunnelDone: isClose} + } + return downstreamDecision{route: routeDropUnknownCorrelated, tunnelID: tunID} + } + + // 4. Proxy RESPONSE/CHUNK frame: decode the stateless per-sub proxy id + // prefix. If it decodes to a LIVE sub-client, route to it ONLY (restoring + // the original proxy request_id). A non-decodable id is the terminator's + // own proxy response → fall through to broadcast so the SDK's typed + // OnProxyHttpResponse dispatch still handles it. A decodable id whose sub + // has departed also falls through to broadcast (harmless; the SDK drops it + // since no terminator dispatch is waiting on a relay-encoded id). + if subID, origID, ok := decodeDownstreamProxyReqID(msg); ok { + if sid, err := strconv.ParseUint(subID, 10, 64); err == nil { + if _, live := m.subs[sid]; live { + return downstreamDecision{route: routeProxyToSub, subID: sid, origReqID: origID} + } + } + return downstreamDecision{route: routeBroadcast} + } + + // 5. Everything else is a PUSH → broadcast. + return downstreamDecision{route: routeBroadcast} +} + +// dispatchDownstream classifies and delivers one downstream frame. +func (m *RelayMux) dispatchDownstream(msg *pb.DownstreamMessage) { + m.mu.Lock() + dec := m.classifyDownstream(msg) + var target *muxSubClient + var broadcast []*muxSubClient + switch dec.route { + case routeCorrelated, routeTunnel: + target = m.subs[dec.subID] + delete(m.pending, dec.muxReqID) + if dec.tunnelDone { + delete(m.tunnels, dec.tunnelID) + } + case routeProxyToSub: + // Stateless: no pending entry to delete. + target = m.subs[dec.subID] + case routeBroadcast: + broadcast = make([]*muxSubClient, 0, len(m.subs)) + for _, s := range m.subs { + broadcast = append(broadcast, s) + } + } + m.mu.Unlock() + + switch dec.route { + case routeCorrelated: + if target == nil { + log.Info().Str("mux_request_id", dec.muxReqID).Msg("relaymux: correlated reply for departed sub-client; dropping") + return + } + restoreDownstreamRequestID(msg, dec.origReqID) + target.deliver(msg) + case routeProxyToSub: + if target == nil { + log.Info().Uint64("mux_sub", dec.subID).Msg("relaymux: proxy reply for departed sub-client; dropping") + return + } + restoreDownstreamProxyReqID(msg, dec.origReqID) + target.deliver(msg) + case routeTunnel: + if target == nil { + log.Info().Str("tunnel_id", dec.tunnelID).Msg("relaymux: tunnel frame for departed sub-client; dropping") + return + } + target.deliver(msg) + case routeBroadcast: + // Inbound proxy requests carry a hop-depth that floors each sub-client's + // subsequent OUTBOUND clamps. Since a broadcast inbound ProxyHttpRequest + // could be serviced by any sub-client, record it on all of them. + if p, ok := msg.Payload.(*pb.DownstreamMessage_ProxyHttpRequest); ok { + depth := p.ProxyHttpRequest.GetProxyChainDepth() + for _, s := range broadcast { + s.recordInboundDepth(depth) + } + } + for _, s := range broadcast { + // Each sub-client gets its own view; the proto is read-only after + // routing so sharing the pointer is safe (no per-sub mutation). + s.deliver(msg) + } + case routeDropUnknownCorrelated: + if dec.tunnelID != "" { + log.Info().Str("tunnel_id", dec.tunnelID).Msg("relaymux: downstream tunnel frame for unknown tunnel; dropping") + } else { + log.Info().Str("mux_request_id", dec.muxReqID).Msg("relaymux: downstream reply for unknown request_id; dropping") + } + } +} + +// pumpSubUpstream copies one sub-client's frames onto the shared upstream, +// applying the same allow-list + target clamp + hop-depth floor as Relay, and +// rewriting correlated request ids to mux ids (recorded in pending) / tracking +// opened tunnels. +func (m *RelayMux) pumpSubUpstream(ctx context.Context, sub *muxSubClient) error { + for { + if ctx.Err() != nil { + return ctx.Err() + } + msg, err := sub.stream.Recv() + if err != nil { + return err + } + + op := upstreamOpName(msg) + if op == OpInitConnection { + sub.deliver(relayErrorDownstream("RELAY_DOUBLE_INIT", + "InitConnection is only valid as the first message")) + continue + } + if !m.allowed.allows(op) { + label := op + if label == "" { + label = "" + } + sub.deliver(relayErrorDownstream("RELAY_OP_DENIED", + fmt.Sprintf("operation %q not in allowed_ops %v", label, m.allowed.list()))) + log.Debug().Str("op", label).Msg("relaymux: dropped upstream op (denied)") + continue + } + + // Target-topic clamp + hop-depth floor on proxy/tunnel envelopes. + switch payload := msg.Payload.(type) { + case *pb.UpstreamMessage_ProxyHttpRequest: + if !m.clampProxyHttp(sub, payload.ProxyHttpRequest) { + continue + } + case *pb.UpstreamMessage_TunnelOpen: + if !m.clampTunnelOpen(sub, payload.TunnelOpen) { + continue + } + // Record tunnel ownership BEFORE forwarding so a fast downstream + // frame can't arrive before the table entry exists. + tunID := payload.TunnelOpen.GetTunnelId() + if tunID != "" { + m.mu.Lock() + m.tunnels[tunID] = sub.id + m.mu.Unlock() + } + } + + // Proxy frames (ProxyHttpRequest / ProxyHttpBodyChunk) are STATELESS: + // they get a deterministic per-sub-client request_id prefix instead of a + // per-frame muxSeq id + pending entry. Deterministic so every frame of a + // chunked request (which shares an orig request_id) rewrites identically, + // preserving sandbox<->service correlation. The matching downstream + // response/chunk decodes the prefix back to this sub-client (see + // classifyDownstream step 4 / routeProxyToSub). Send + continue, skipping + // the muxSeq/pending path entirely. + if rewriteUpstreamProxyReqID(msg, strconv.FormatUint(sub.id, 10)) { + if err := m.sendUpstream(msg); err != nil { + return err + } + continue + } + + // Rewrite correlated request ids to a globally-unique mux id and record + // the owner so the reply can be routed back. In composite mode the id + // is prefixed "rmx-" to guarantee it never collides with the + // terminator's own "req-N" ids on the same shared connection; the + // pending-table lookup is the authoritative routing gate, but the + // prefix eliminates any possibility of cross-surface collision even + // before the lookup. + seq := m.muxSeq.Add(1) + var muxID string + if m.sharedSend != nil { + muxID = "rmx-" + strconv.FormatUint(seq, 10) + } else { + muxID = strconv.FormatUint(seq, 10) + } + if origID, correlated := upstreamSetRequestID(msg, muxID); correlated { + m.mu.Lock() + m.pending[muxID] = muxPending{subID: sub.id, origReqID: origID, payloadTag: op} + m.mu.Unlock() + } + + if err := m.sendUpstream(msg); err != nil { + return err + } + } +} + +// sendUpstream forwards one frame upstream. In composite mode it routes +// through the shared runtime's priority queue (mirroring +// sharedRuntimeSession.Send but without the per-op registerRequest — the mux +// id rewrite in pumpSubUpstream already handles correlation). In standalone +// mode it writes directly to the own-dialed stream. +func (m *RelayMux) sendUpstream(msg *pb.UpstreamMessage) error { + if m.sharedSend != nil { + prio := priorityForSharedRelayUpstream(msg) + ctx, cancel := context.WithTimeout(context.Background(), sendUpstreamTimeout) + defer cancel() + return m.sharedSend(ctx, prio, msg) + } + m.mu.Lock() + up := m.up + m.mu.Unlock() + if up == nil { + return errors.New("relaymux: shared upstream not established") + } + return up.stream.Send(msg) +} + +// clampProxyHttp applies the target clamp + hop-depth floor to an outbound +// ProxyHttpRequest, sending a relayErrorDownstream to sub and returning false +// when rejected. Mirrors relaySession.applyClampToProxyHttp. +func (m *RelayMux) clampProxyHttp(sub *muxSubClient, req *pb.ProxyHttpRequest) bool { + if req == nil { + sub.deliver(relayErrorDownstream("RELAY_BAD_ENVELOPE", "ProxyHttpRequest payload was nil")) + return false + } + res := m.clamp.evaluate(req.GetTargetTopic()) + if !res.Allowed { + sub.deliver(relayErrorDownstream("RELAY_TARGET_DENIED", res.Reason)) + log.Info().Str("target_topic", req.GetTargetTopic()).Str("reason", res.Reason). + Msg("relaymux: dropped ProxyHttpRequest (target clamp)") + return false + } + if res.NewTarget != "" { + req.TargetTopic = res.NewTarget + } + req.ProxyChainDepth = hybridFloor(req.GetProxyChainDepth(), sub.observedDepth()) + return true +} + +// clampTunnelOpen mirrors clampProxyHttp for TunnelOpen. +func (m *RelayMux) clampTunnelOpen(sub *muxSubClient, open *pb.TunnelOpen) bool { + if open == nil { + sub.deliver(relayErrorDownstream("RELAY_BAD_ENVELOPE", "TunnelOpen payload was nil")) + return false + } + res := m.clamp.evaluate(open.GetTargetTopic()) + if !res.Allowed { + sub.deliver(relayErrorDownstream("RELAY_TARGET_DENIED", res.Reason)) + log.Info().Str("target_topic", open.GetTargetTopic()).Str("reason", res.Reason). + Msg("relaymux: dropped TunnelOpen (target clamp)") + return false + } + if res.NewTarget != "" { + open.TargetTopic = res.NewTarget + } + open.ProxyChainDepth = hybridFloor(open.GetProxyChainDepth(), sub.observedDepth()) + return true +} + +// teardownUpstream closes the shared upstream (when it matches up, or +// unconditionally when up is nil) and cancels every sub-client so they +// reconnect and re-handshake. Pending/tunnel tables are cleared. +func (m *RelayMux) teardownUpstream(up *muxUpstream) { + m.mu.Lock() + if m.up == nil { + m.mu.Unlock() + return + } + if up != nil && m.up != up { + // A newer upstream already replaced the one that errored; nothing to do. + m.mu.Unlock() + return + } + cur := m.up + m.up = nil + subs := make([]*muxSubClient, 0, len(m.subs)) + for _, s := range m.subs { + subs = append(subs, s) + } + m.subs = map[uint64]*muxSubClient{} + m.pending = map[string]muxPending{} + m.tunnels = map[string]uint64{} + m.mu.Unlock() + + if cur != nil { + cur.cancel() + if cur.closer != nil { + _ = cur.closer() + } + } + for _, s := range subs { + s.closeSubClient() + s.cancel() + } +} + +// removeSub deregisters a sub-client and purges any pending/tunnel entries it +// owned so a late reply can't be misrouted to a recycled id. +func (m *RelayMux) removeSub(subID uint64) { + m.mu.Lock() + delete(m.subs, subID) + for k, p := range m.pending { + if p.subID == subID { + delete(m.pending, k) + } + } + for k, owner := range m.tunnels { + if owner == subID { + delete(m.tunnels, k) + } + } + m.mu.Unlock() +} + +// deliver enqueues msg for downstream delivery to the sub-client via the +// bounded inbox. It mirrors sharedRuntimeSession.deliver: priority-aware CoDel +// admission via deliverSem, with a BACKPRESSURE error synthesised on shed so +// the in-sandbox SDK sees a clean failure rather than a silent drop. +// +// deliver never blocks the caller on a slow in-sandbox reader — it returns +// immediately. The actual blocking stream.Send runs in runWriter. +func (s *muxSubClient) deliver(msg *pb.DownstreamMessage) { + if s.closed.Load() { + return + } + prio := priorityForSharedRelayDownstream(msg) + acqCtx, cancel := context.WithTimeout(context.Background(), sharedRuntimeSessionDeliverAcquireTimeout) + defer cancel() + if err := s.deliverSem.Acquire(acqCtx, prio, 1); err != nil { + s.emitBackpressureNotice(msg, prio, err) + return + } + pushed := false + select { + case s.inbox <- msg: + pushed = true + default: + } + s.deliverSem.Release(1) + if !pushed { + s.emitBackpressureNotice(msg, prio, nil) + } +} + +// emitBackpressureNotice synthesises a BACKPRESSURE error frame into the inbox +// so the in-sandbox SDK sees a clean signal on its next Recv. Mirrors +// sharedRuntimeSession.emitDeliverBackpressureNotice. +func (s *muxSubClient) emitBackpressureNotice(orig *pb.DownstreamMessage, prio backpressure.Priority, cause error) { + if log.Debug().Enabled() { + evt := log.Debug(). + Str("payload_type", fmt.Sprintf("%T", orig.GetPayload())). + Int("priority", int(prio)) + if cause != nil { + evt = evt.Err(cause).Str("trigger", "acquire-shed") + } else { + evt = evt.Str("trigger", "inbox-full") + } + evt.Uint64("mux_sub", s.id).Msg("relaymux: sub-client deliver shed, synthesising BACKPRESSURE error") + } + notice := &pb.DownstreamMessage{ + Payload: &pb.DownstreamMessage_Error{ + Error: &pb.ErrorResponse{ + Code: "BACKPRESSURE", + Message: "relay mux sub-client inbox shed by backpressure; reduce send rate or process messages faster", + }, + }, + } + select { + case s.inbox <- notice: + default: + log.Warn(). + Uint64("mux_sub", s.id). + Str("payload_type", fmt.Sprintf("%T", orig.GetPayload())). + Msg("relaymux: sub-client inbox full, dropping both envelope and BACKPRESSURE notice") + } +} + +// runWriter drains the inbox and calls the blocking stream.Send until ctx is +// cancelled and the inbox is drained. It is the only goroutine that writes to +// stream, so no additional mutex is required for stream-frame integrity. +func (s *muxSubClient) runWriter(ctx context.Context) { + for { + select { + case msg, ok := <-s.inbox: + if !ok { + return + } + if err := s.stream.Send(msg); err != nil { + log.Debug().Uint64("mux_sub", s.id).Err(err). + Msg("relaymux: sub-client writer stream.Send failed") + // Drain remaining messages so deliver goroutines don't block, + // then exit; the Connect handler will clean up. + for { + select { + case <-s.inbox: + default: + return + } + } + } + case <-ctx.Done(): + // Drain the inbox so no goroutine is blocked in deliver. + for { + select { + case <-s.inbox: + default: + return + } + } + } + } +} + +// closeSubClient marks the sub-client closed, closes the inbox channel so +// runWriter exits, and releases the semaphore resources. +func (s *muxSubClient) closeSubClient() { + if s.closed.CompareAndSwap(false, true) { + close(s.inbox) + s.deliverSem.Close() + } +} + +// recordInboundDepth bumps the sub-client's observed inbound chain depth. +func (s *muxSubClient) recordInboundDepth(depth uint32) { + if depth == 0 { + return + } + s.depthMu.Lock() + if depth > s.inboundDepth { + s.inboundDepth = depth + } + s.depthMu.Unlock() +} + +// observedDepth returns the largest inbound depth seen so far for this sub. +func (s *muxSubClient) observedDepth() uint32 { + s.depthMu.Lock() + d := s.inboundDepth + s.depthMu.Unlock() + return d +} + +// restoreDownstreamRequestID writes orig back onto a correlated downstream +// reply's request_id field (the inverse of upstreamSetRequestID's rewrite). +// It mirrors the correlated payload set in relaymux_reqid.go. +func restoreDownstreamRequestID(msg *pb.DownstreamMessage, orig string) { + if msg == nil { + return + } + switch p := msg.Payload.(type) { + case *pb.DownstreamMessage_Kv: + p.Kv.RequestId = orig + case *pb.DownstreamMessage_TaskOp: + p.TaskOp.RequestId = orig + case *pb.DownstreamMessage_TaskQuery: + p.TaskQuery.RequestId = orig + case *pb.DownstreamMessage_CreateTask: + p.CreateTask.RequestId = orig + case *pb.DownstreamMessage_Checkpoint: + p.Checkpoint.RequestId = orig + case *pb.DownstreamMessage_Admin: + p.Admin.RequestId = orig + case *pb.DownstreamMessage_SessionResponse: + p.SessionResponse.RequestId = orig + case *pb.DownstreamMessage_Workspace: + p.Workspace.RequestId = orig + case *pb.DownstreamMessage_Agent: + p.Agent.RequestId = orig + case *pb.DownstreamMessage_Acl: + p.Acl.RequestId = orig + case *pb.DownstreamMessage_Token: + p.Token.RequestId = orig + case *pb.DownstreamMessage_AuditResponse: + p.AuditResponse.RequestId = orig + case *pb.DownstreamMessage_AuthorityGrant: + p.AuthorityGrant.RequestId = orig + case *pb.DownstreamMessage_ResolveAuthorityResponse: + p.ResolveAuthorityResponse.RequestId = orig + case *pb.DownstreamMessage_ConnectionStatusResponse: + p.ConnectionStatusResponse.RequestId = orig + case *pb.DownstreamMessage_WorkflowResponse: + p.WorkflowResponse.RequestId = orig + case *pb.DownstreamMessage_SubmitAuditEventResponse: + p.SubmitAuditEventResponse.ClientRequestId = orig + case *pb.DownstreamMessage_AuthorityRequestResponse: + p.AuthorityRequestResponse.ClientRequestId = orig + case *pb.DownstreamMessage_TaskSubscriptionResponse: + p.TaskSubscriptionResponse.ClientRequestId = orig + case *pb.DownstreamMessage_Error: + p.Error.RequestId = orig + } +} diff --git a/server/internal/proxysidecar/relaymux_reqid.go b/server/internal/proxysidecar/relaymux_reqid.go new file mode 100644 index 0000000..e0e1458 --- /dev/null +++ b/server/internal/proxysidecar/relaymux_reqid.go @@ -0,0 +1,351 @@ +package proxysidecar + +import ( + "strconv" + "strings" + + pb "github.com/scitrera/aether/api/proto" +) + +// request_id field map for RelayMux correlation. +// +// RelayMux fans many sub-client streams onto ONE shared upstream gateway +// connection. To route a reply back to the sub-client that asked, every +// CORRELATED upstream op has its request_id rewritten to a globally-unique +// mux id before forwarding; the matching downstream reply is then demuxed by +// reading that id and restoring the sub-client's original value. +// +// "Correlated" means: the upstream payload carries a per-request correlation +// id AND there is a downstream payload that echoes it. The two helpers below +// are the single source of truth for which payloads participate. +// +// Coverage (confirmed against api/proto/aether.proto, field tags noted): +// +// Correlated (request_id rewritten / restored): +// kv_op (KVOperation.request_id=8) <-> kv (KVResponse.request_id=5) +// task_op (TaskOperation.request_id=4) <-> task_op (TaskOperationResponse.request_id=5) +// task_query (TaskQuery.request_id=4) <-> task_query (TaskQueryResponse.request_id=6) +// create_task (CreateTaskRequest.request_id=10) <-> create_task (CreateTaskResponse.request_id=6) +// checkpoint_op (CheckpointOperation.request_id=5) <-> checkpoint (CheckpointResponse.request_id=6) +// admin_query (AdminQuery.request_id=4) <-> admin (AdminResponse.request_id=9) +// session_op (SessionOperation.request_id=4) <-> session_response (SessionOperationResponse.request_id=4) +// workspace_op (WorkspaceOperation.request_id=5)<-> workspace (WorkspaceResponse.request_id=8) +// agent_op (AgentOperation.request_id=6) <-> agent (AgentResponse.request_id=9) +// acl_op (ACLOperation.request_id=19) <-> acl (ACLResponse.request_id=12) +// token_op (TokenOperation.request_id=5) <-> token (TokenResponse.request_id=9) +// audit_query (AuditQuery.request_id=1) <-> audit_response (AuditQueryResponse.request_id=1) +// authority_grant_op (AuthorityGrantOperation.request_id=6) <-> authority_grant (AuthorityGrantResponse.request_id=5) +// resolve_authority_request (ResolveAuthorityRequest.request_id=1) <-> resolve_authority_response (ResolveAuthorityResponse.request_id=1) +// connection_status_request (ConnectionStatusRequest.request_id=1) <-> connection_status_response (ConnectionStatusResponse.request_id=1) +// workflow_op (WorkflowOperation.request_id=6) <-> workflow_response (WorkflowResponse.request_id=6) +// submit_audit_event (SubmitAuditEventRequest.client_request_id=9) <-> submit_audit_event_response (SubmitAuditEventResponse.client_request_id=1) +// authority_request_op (AuthorityRequestOperation.client_request_id=6) <-> authority_request_response (AuthorityRequestOperationResponse.client_request_id=3) +// task_subscription_op (TaskSubscriptionOperation.client_request_id=4) <-> task_subscription_response (TaskSubscriptionOperationResponse.client_request_id=3) +// +// NOT correlated (forwarded one-way / broadcast — never tracked): +// init (handled locally, never forwarded by the mux) +// send (SendMessage — fire-and-forget, no request_id) +// switch_workspace (no request_id) +// progress (ProgressReport HAS a request_id=7 field, but there is no +// matching downstream "progress response": ProgressUpdate is a +// gateway-initiated PUSH, not an echo. So ProgressReport is forwarded +// one-way and ProgressUpdate is broadcast. This is the one surprising +// proto shape — a request_id-bearing op that is nonetheless one-way.) +// proxy_http_request / proxy_http_body_chunk / proxy_http_response: these +// correlate by request_id between the SANDBOX and a downstream service, +// NOT between the sub-client and the gateway in a request/response sense. +// Proxy ops are NOT in the correlated (pending-table) map above because +// they can be MULTI-FRAME: a chunked ProxyHttpRequest (BodyChunked=true) +// is followed by ProxyHttpBodyChunk frames that SHARE one request_id, and a +// chunked response is a ProxyHttpResponse header followed by +// ProxyHttpBodyChunk frames (Fin on the last) that also share it. A +// per-frame muxSeq counter would assign different ids to frames of the same +// logical request and break that correlation. +// +// Instead, proxy frames use a STATELESS deterministic per-sub-client +// request_id PREFIX (see encodeProxyReqID / decodeProxyReqID below): +// - Upstream: an outbound proxy frame's request_id is rewritten from +// to encodeProxyReqID(subID, ). Deterministic ⇒ every +// frame of one chunked request rewrites identically; no pending entry. +// - Downstream: a proxy response/chunk frame's request_id decodes back to +// (subID, origID). If subID is a live sub-client, the original id is +// restored on the frame and it is delivered to THAT sub-client only. If +// it does NOT decode (e.g. the terminator's own "req-N" proxy response), +// the frame falls through to broadcast so the SDK's terminator typed +// dispatch (OnProxyHttpResponse) still handles it. +// This replaces the never-wired "broadcast to sub-clients; the sandbox +// picks up its own by request_id" scheme that previously dropped a relay +// sub-client's proxy responses. +// tunnel_open / tunnel_data / tunnel_close / tunnel_ack: routed by +// tunnel_id, handled separately (see RelayMux tunnel routing), not here. +// +// Downstream-only PUSH payloads with no upstream correlated request that are +// always broadcast: msg, config, signal, task_assignment, progress_update, +// proxy_http_request, connection_ack, authority_grant_revocation, +// authority_request_event, task_hibernated, task_event. + +// upstreamSetRequestID rewrites the correlation id on a correlated upstream +// payload to newID, returning the original id and whether the payload was a +// correlated type. For non-correlated payloads it returns ("", false) and the +// message is forwarded unchanged (one-way). +func upstreamSetRequestID(msg *pb.UpstreamMessage, newID string) (origID string, correlated bool) { + if msg == nil { + return "", false + } + switch p := msg.Payload.(type) { + case *pb.UpstreamMessage_KvOp: + orig := p.KvOp.GetRequestId() + p.KvOp.RequestId = newID + return orig, true + case *pb.UpstreamMessage_TaskOp: + orig := p.TaskOp.GetRequestId() + p.TaskOp.RequestId = newID + return orig, true + case *pb.UpstreamMessage_TaskQuery: + orig := p.TaskQuery.GetRequestId() + p.TaskQuery.RequestId = newID + return orig, true + case *pb.UpstreamMessage_CreateTask: + orig := p.CreateTask.GetRequestId() + p.CreateTask.RequestId = newID + return orig, true + case *pb.UpstreamMessage_CheckpointOp: + orig := p.CheckpointOp.GetRequestId() + p.CheckpointOp.RequestId = newID + return orig, true + case *pb.UpstreamMessage_AdminQuery: + orig := p.AdminQuery.GetRequestId() + p.AdminQuery.RequestId = newID + return orig, true + case *pb.UpstreamMessage_SessionOp: + orig := p.SessionOp.GetRequestId() + p.SessionOp.RequestId = newID + return orig, true + case *pb.UpstreamMessage_WorkspaceOp: + orig := p.WorkspaceOp.GetRequestId() + p.WorkspaceOp.RequestId = newID + return orig, true + case *pb.UpstreamMessage_AgentOp: + orig := p.AgentOp.GetRequestId() + p.AgentOp.RequestId = newID + return orig, true + case *pb.UpstreamMessage_AclOp: + orig := p.AclOp.GetRequestId() + p.AclOp.RequestId = newID + return orig, true + case *pb.UpstreamMessage_TokenOp: + orig := p.TokenOp.GetRequestId() + p.TokenOp.RequestId = newID + return orig, true + case *pb.UpstreamMessage_AuditQuery: + orig := p.AuditQuery.GetRequestId() + p.AuditQuery.RequestId = newID + return orig, true + case *pb.UpstreamMessage_AuthorityGrantOp: + orig := p.AuthorityGrantOp.GetRequestId() + p.AuthorityGrantOp.RequestId = newID + return orig, true + case *pb.UpstreamMessage_ResolveAuthorityRequest: + orig := p.ResolveAuthorityRequest.GetRequestId() + p.ResolveAuthorityRequest.RequestId = newID + return orig, true + case *pb.UpstreamMessage_ConnectionStatusRequest: + orig := p.ConnectionStatusRequest.GetRequestId() + p.ConnectionStatusRequest.RequestId = newID + return orig, true + case *pb.UpstreamMessage_WorkflowOp: + orig := p.WorkflowOp.GetRequestId() + p.WorkflowOp.RequestId = newID + return orig, true + case *pb.UpstreamMessage_SubmitAuditEvent: + orig := p.SubmitAuditEvent.GetClientRequestId() + p.SubmitAuditEvent.ClientRequestId = newID + return orig, true + case *pb.UpstreamMessage_AuthorityRequestOp: + orig := p.AuthorityRequestOp.GetClientRequestId() + p.AuthorityRequestOp.ClientRequestId = newID + return orig, true + case *pb.UpstreamMessage_TaskSubscriptionOp: + orig := p.TaskSubscriptionOp.GetClientRequestId() + p.TaskSubscriptionOp.ClientRequestId = newID + return orig, true + } + return "", false +} + +// downstreamGetRequestID reads the correlation id from a correlated downstream +// reply, returning the id and whether the payload was a correlated type. For +// broadcast / tunnel payloads it returns ("", false). +// +// ErrorResponse is special-cased by the caller (it carries an optional +// request_id that may or may not correlate to a pending mux id) and is NOT +// reported as correlated here; see RelayMux.routeDownstream. +func downstreamGetRequestID(msg *pb.DownstreamMessage) (id string, correlated bool) { + if msg == nil { + return "", false + } + switch p := msg.Payload.(type) { + case *pb.DownstreamMessage_Kv: + return p.Kv.GetRequestId(), true + case *pb.DownstreamMessage_TaskOp: + return p.TaskOp.GetRequestId(), true + case *pb.DownstreamMessage_TaskQuery: + return p.TaskQuery.GetRequestId(), true + case *pb.DownstreamMessage_CreateTask: + return p.CreateTask.GetRequestId(), true + case *pb.DownstreamMessage_Checkpoint: + return p.Checkpoint.GetRequestId(), true + case *pb.DownstreamMessage_Admin: + return p.Admin.GetRequestId(), true + case *pb.DownstreamMessage_SessionResponse: + return p.SessionResponse.GetRequestId(), true + case *pb.DownstreamMessage_Workspace: + return p.Workspace.GetRequestId(), true + case *pb.DownstreamMessage_Agent: + return p.Agent.GetRequestId(), true + case *pb.DownstreamMessage_Acl: + return p.Acl.GetRequestId(), true + case *pb.DownstreamMessage_Token: + return p.Token.GetRequestId(), true + case *pb.DownstreamMessage_AuditResponse: + return p.AuditResponse.GetRequestId(), true + case *pb.DownstreamMessage_AuthorityGrant: + return p.AuthorityGrant.GetRequestId(), true + case *pb.DownstreamMessage_ResolveAuthorityResponse: + return p.ResolveAuthorityResponse.GetRequestId(), true + case *pb.DownstreamMessage_ConnectionStatusResponse: + return p.ConnectionStatusResponse.GetRequestId(), true + case *pb.DownstreamMessage_WorkflowResponse: + return p.WorkflowResponse.GetRequestId(), true + case *pb.DownstreamMessage_SubmitAuditEventResponse: + return p.SubmitAuditEventResponse.GetClientRequestId(), true + case *pb.DownstreamMessage_AuthorityRequestResponse: + return p.AuthorityRequestResponse.GetClientRequestId(), true + case *pb.DownstreamMessage_TaskSubscriptionResponse: + return p.TaskSubscriptionResponse.GetClientRequestId(), true + } + return "", false +} + +// downstreamErrorRequestID returns the request_id carried on an ErrorResponse +// downstream frame (and true) when the frame is an error, else ("", false). +// ErrorResponse.request_id is optional: empty means the error is not tied to a +// specific correlated request and should be broadcast. +func downstreamErrorRequestID(msg *pb.DownstreamMessage) (id string, isError bool) { + if msg == nil { + return "", false + } + if p, ok := msg.Payload.(*pb.DownstreamMessage_Error); ok { + return p.Error.GetRequestId(), true + } + return "", false +} + +// proxyReqIDSentinel marks a request_id that was rewritten by the relay mux for +// a proxy frame. It is chosen so it cannot be confused with a normal +// "req-N" / "rpc-" / "rmx-N" id: it contains a '~' (never produced by the +// SDK's counter-based or hex ids, nor by the "rmx-" mux prefix). +const proxyReqIDSentinel = "rmxp~" + +// encodeProxyReqID builds a deterministic, reversible per-sub-client prefix for +// a proxy frame's request_id. The format is: +// +// "rmxp~" + len(subID) + "~" + subID + origID +// +// The length prefix makes the split unambiguous for ANY subID/origID content +// (origID may itself contain '~'). encode/decode round-trip exactly. +func encodeProxyReqID(subID, origID string) string { + return proxyReqIDSentinel + strconv.Itoa(len(subID)) + "~" + subID + origID +} + +// decodeProxyReqID reverses encodeProxyReqID. It returns ok=false for any id +// that was not produced by encodeProxyReqID (e.g. the terminator's own "req-N" +// proxy response ids), in which case subID/origID are empty. +func decodeProxyReqID(s string) (subID, origID string, ok bool) { + if !strings.HasPrefix(s, proxyReqIDSentinel) { + return "", "", false + } + rest := s[len(proxyReqIDSentinel):] + sep := strings.IndexByte(rest, '~') + if sep < 0 { + return "", "", false + } + n, err := strconv.Atoi(rest[:sep]) + if err != nil || n < 0 { + return "", "", false + } + body := rest[sep+1:] + if len(body) < n { + return "", "", false + } + return body[:n], body[n:], true +} + +// rewriteUpstreamProxyReqID rewrites the request_id on an outbound proxy frame +// (ProxyHttpRequest / ProxyHttpBodyChunk) to encodeProxyReqID(subID, ), +// returning true. For non-proxy payloads it returns false and leaves msg +// unchanged. Deterministic: every frame of one chunked request (which shares an +// orig request_id) rewrites to the same encoded id, preserving correlation. +func rewriteUpstreamProxyReqID(msg *pb.UpstreamMessage, subID string) bool { + if msg == nil { + return false + } + switch p := msg.Payload.(type) { + case *pb.UpstreamMessage_ProxyHttpRequest: + p.ProxyHttpRequest.RequestId = encodeProxyReqID(subID, p.ProxyHttpRequest.GetRequestId()) + return true + case *pb.UpstreamMessage_ProxyHttpBodyChunk: + p.ProxyHttpBodyChunk.RequestId = encodeProxyReqID(subID, p.ProxyHttpBodyChunk.GetRequestId()) + return true + } + return false +} + +// decodeDownstreamProxyReqID decodes the request_id on an inbound proxy +// response frame (ProxyHttpResponse / ProxyHttpBodyChunk) back to its owning +// (subID, origID). It returns ok=false for non-proxy frames and for proxy +// frames whose request_id was not relay-encoded (the terminator's own). +func decodeDownstreamProxyReqID(msg *pb.DownstreamMessage) (subID, origID string, ok bool) { + if msg == nil { + return "", "", false + } + switch p := msg.Payload.(type) { + case *pb.DownstreamMessage_ProxyHttpResponse: + return decodeProxyReqID(p.ProxyHttpResponse.GetRequestId()) + case *pb.DownstreamMessage_ProxyHttpBodyChunk: + return decodeProxyReqID(p.ProxyHttpBodyChunk.GetRequestId()) + } + return "", "", false +} + +// restoreDownstreamProxyReqID writes orig back onto a proxy downstream frame's +// request_id (the inverse of the rewriteUpstreamProxyReqID encode, applied to +// the matching response/chunk types) so the owning sub-client sees its own id. +func restoreDownstreamProxyReqID(msg *pb.DownstreamMessage, orig string) { + if msg == nil { + return + } + switch p := msg.Payload.(type) { + case *pb.DownstreamMessage_ProxyHttpResponse: + p.ProxyHttpResponse.RequestId = orig + case *pb.DownstreamMessage_ProxyHttpBodyChunk: + p.ProxyHttpBodyChunk.RequestId = orig + } +} + +// downstreamTunnelID returns the tunnel id (and true) for tunnel downstream +// frames routed by tunnel id, else ("", false). +func downstreamTunnelID(msg *pb.DownstreamMessage) (id string, isTunnel bool) { + if msg == nil { + return "", false + } + switch p := msg.Payload.(type) { + case *pb.DownstreamMessage_TunnelData: + return p.TunnelData.GetTunnelId(), true + case *pb.DownstreamMessage_TunnelAck: + return p.TunnelAck.GetTunnelId(), true + case *pb.DownstreamMessage_TunnelClose: + return p.TunnelClose.GetTunnelId(), true + } + return "", false +} diff --git a/server/internal/proxysidecar/relaymux_test.go b/server/internal/proxysidecar/relaymux_test.go new file mode 100644 index 0000000..84cc6e5 --- /dev/null +++ b/server/internal/proxysidecar/relaymux_test.go @@ -0,0 +1,793 @@ +package proxysidecar + +import ( + "context" + "errors" + "io" + "net" + "path/filepath" + "strconv" + "sync" + "testing" + "time" + + pb "github.com/scitrera/aether/api/proto" + "github.com/scitrera/go-backpressure" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" +) + +// ============================================================================= +// Pure routing-decision table test (classifyDownstream). +// +// This exercises the demux routing logic without any live stream: we seed the +// pending/tunnel tables by hand and assert the classification for each frame +// shape. It is the cheapest place to cover the "unknown correlated id is +// dropped, not broadcast" requirement and the request_id-demux/restore route. +// ============================================================================= + +func newTestMux() *RelayMux { + return &RelayMux{ + subs: map[uint64]*muxSubClient{}, + pending: map[string]muxPending{}, + tunnels: map[string]uint64{}, + } +} + +func TestRelayMux_ClassifyDownstream(t *testing.T) { + m := newTestMux() + // Two sub-clients each have an in-flight KV op under distinct mux ids but + // the SAME original request id ("r1"). + m.subs[1] = &muxSubClient{id: 1} + m.subs[2] = &muxSubClient{id: 2} + m.pending["100"] = muxPending{subID: 1, origReqID: "r1", payloadTag: OpKVOperation} + m.pending["101"] = muxPending{subID: 2, origReqID: "r1", payloadTag: OpKVOperation} + m.tunnels["tun-a"] = 1 + + kvReply := func(muxID string) *pb.DownstreamMessage { + return &pb.DownstreamMessage{Payload: &pb.DownstreamMessage_Kv{Kv: &pb.KVResponse{RequestId: muxID}}} + } + + cases := []struct { + name string + msg *pb.DownstreamMessage + wantRoute muxRoute + wantSub uint64 + wantOrig string + wantTun string + }{ + { + name: "kv reply for sub 1", + msg: kvReply("100"), + wantRoute: routeCorrelated, + wantSub: 1, + wantOrig: "r1", + }, + { + name: "kv reply for sub 2 (same orig req id)", + msg: kvReply("101"), + wantRoute: routeCorrelated, + wantSub: 2, + wantOrig: "r1", + }, + { + name: "unknown correlated id is dropped not broadcast", + msg: kvReply("999"), + wantRoute: routeDropUnknownCorrelated, + }, + { + name: "task assignment is broadcast", + msg: &pb.DownstreamMessage{Payload: &pb.DownstreamMessage_TaskAssignment{ + TaskAssignment: &pb.TaskAssignment{TaskId: "t1"}}}, + wantRoute: routeBroadcast, + }, + { + name: "incoming message is broadcast", + msg: &pb.DownstreamMessage{Payload: &pb.DownstreamMessage_Msg{ + Msg: &pb.IncomingMessage{}}}, + wantRoute: routeBroadcast, + }, + { + name: "tunnel data routes to owning sub", + msg: &pb.DownstreamMessage{Payload: &pb.DownstreamMessage_TunnelData{ + TunnelData: &pb.TunnelData{TunnelId: "tun-a"}}}, + wantRoute: routeTunnel, + wantSub: 1, + wantTun: "tun-a", + }, + { + name: "tunnel data for unknown tunnel is dropped", + msg: &pb.DownstreamMessage{Payload: &pb.DownstreamMessage_TunnelData{ + TunnelData: &pb.TunnelData{TunnelId: "tun-zzz"}}}, + wantRoute: routeDropUnknownCorrelated, + }, + { + name: "error with known pending id demuxes", + msg: &pb.DownstreamMessage{Payload: &pb.DownstreamMessage_Error{ + Error: &pb.ErrorResponse{Code: "X", RequestId: "100"}}}, + wantRoute: routeCorrelated, + wantSub: 1, + wantOrig: "r1", + }, + { + name: "error without request id is broadcast", + msg: &pb.DownstreamMessage{Payload: &pb.DownstreamMessage_Error{ + Error: &pb.ErrorResponse{Code: "CONN_LEVEL"}}}, + wantRoute: routeBroadcast, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + m.mu.Lock() + dec := m.classifyDownstream(tc.msg) + m.mu.Unlock() + if dec.route != tc.wantRoute { + t.Fatalf("route = %v; want %v", dec.route, tc.wantRoute) + } + if tc.wantSub != 0 && dec.subID != tc.wantSub { + t.Fatalf("subID = %d; want %d", dec.subID, tc.wantSub) + } + if tc.wantOrig != "" && dec.origReqID != tc.wantOrig { + t.Fatalf("origReqID = %q; want %q", dec.origReqID, tc.wantOrig) + } + if tc.wantTun != "" && dec.tunnelID != tc.wantTun { + t.Fatalf("tunnelID = %q; want %q", dec.tunnelID, tc.wantTun) + } + }) + } +} + +// ============================================================================= +// Stateless proxy request_id prefix scheme (encode/decode + routing). +// ============================================================================= + +func TestProxyReqIDRoundTrip(t *testing.T) { + cases := []struct{ subID, origID string }{ + {"7", "req-3"}, + {"42", "rpc-deadbeef"}, + {"1", ""}, // empty origID + {"123", "weird~id~with~tildes"}, // origID containing the sentinel char + {"0", "rmxp~not-really"}, // origID that looks like an encoded id + } + for _, tc := range cases { + enc := encodeProxyReqID(tc.subID, tc.origID) + gotSub, gotOrig, ok := decodeProxyReqID(enc) + if !ok { + t.Fatalf("decodeProxyReqID(%q) ok=false; want true", enc) + } + if gotSub != tc.subID || gotOrig != tc.origID { + t.Fatalf("round-trip(%q,%q) = (%q,%q)", tc.subID, tc.origID, gotSub, gotOrig) + } + } + + // Non-encoded ids must NOT decode (these are terminator-owned proxy ids). + for _, s := range []string{"req-5", "rpc-abc123", "rmx-9", "", "rmxp~", "rmxp~x~y"} { + if _, _, ok := decodeProxyReqID(s); ok { + t.Fatalf("decodeProxyReqID(%q) ok=true; want false", s) + } + } +} + +// TestRelayMux_ClassifyProxyResponse covers the new proxy routing: +// - a relay-encoded ProxyHttpResponse (and a chunked ProxyHttpBodyChunk with +// Fin) routes to the owning live sub-client with origID restorable; +// - a non-decodable (terminator-owned) proxy response falls through to +// broadcast so the SDK's typed dispatch still handles it; +// - a relay-encoded response for a DEPARTED sub falls through to broadcast. +func TestRelayMux_ClassifyProxyResponse(t *testing.T) { + m := newTestMux() + m.subs[3] = &muxSubClient{id: 3} + + encResp := func(subID uint64, orig string) *pb.DownstreamMessage { + return &pb.DownstreamMessage{Payload: &pb.DownstreamMessage_ProxyHttpResponse{ + ProxyHttpResponse: &pb.ProxyHttpResponse{RequestId: encodeProxyReqID(strconv.FormatUint(subID, 10), orig)}}} + } + encChunk := func(subID uint64, orig string, fin bool) *pb.DownstreamMessage { + return &pb.DownstreamMessage{Payload: &pb.DownstreamMessage_ProxyHttpBodyChunk{ + ProxyHttpBodyChunk: &pb.ProxyHttpBodyChunk{RequestId: encodeProxyReqID(strconv.FormatUint(subID, 10), orig), Fin: fin}}} + } + + cases := []struct { + name string + msg *pb.DownstreamMessage + wantRoute muxRoute + wantSub uint64 + wantOrig string + }{ + { + name: "proxy response routes to owning sub with orig restored", + msg: encResp(3, "req-9"), + wantRoute: routeProxyToSub, + wantSub: 3, + wantOrig: "req-9", + }, + { + name: "proxy chunk (Fin) routes to same owning sub", + msg: encChunk(3, "req-9", true), + wantRoute: routeProxyToSub, + wantSub: 3, + wantOrig: "req-9", + }, + { + name: "terminator-owned proxy response falls through to broadcast", + msg: &pb.DownstreamMessage{Payload: &pb.DownstreamMessage_ProxyHttpResponse{ + ProxyHttpResponse: &pb.ProxyHttpResponse{RequestId: "req-7"}}}, + wantRoute: routeBroadcast, + }, + { + name: "proxy response for departed sub falls through to broadcast", + msg: encResp(999, "req-1"), + wantRoute: routeBroadcast, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + m.mu.Lock() + dec := m.classifyDownstream(tc.msg) + m.mu.Unlock() + if dec.route != tc.wantRoute { + t.Fatalf("route = %v; want %v", dec.route, tc.wantRoute) + } + if tc.wantSub != 0 && dec.subID != tc.wantSub { + t.Fatalf("subID = %d; want %d", dec.subID, tc.wantSub) + } + if tc.wantRoute == routeProxyToSub { + if dec.origReqID != tc.wantOrig { + t.Fatalf("origReqID = %q; want %q", dec.origReqID, tc.wantOrig) + } + // Restore must put the original id back on the proxy frame. + restoreDownstreamProxyReqID(tc.msg, dec.origReqID) + if gotID, _, _ := decodeDownstreamProxyReqID(tc.msg); gotID != "" { + t.Fatalf("after restore, request_id still decodes as relay-encoded") + } + } + }) + } +} + +// TestRewriteUpstreamProxyReqID verifies the upstream rewrite is applied to +// both proxy frame types (so chunked requests share an encoded id) and to +// nothing else. +func TestRewriteUpstreamProxyReqID(t *testing.T) { + subID := "5" + reqMsg := &pb.UpstreamMessage{Payload: &pb.UpstreamMessage_ProxyHttpRequest{ + ProxyHttpRequest: &pb.ProxyHttpRequest{RequestId: "r-1", BodyChunked: true}}} + chunkMsg := &pb.UpstreamMessage{Payload: &pb.UpstreamMessage_ProxyHttpBodyChunk{ + ProxyHttpBodyChunk: &pb.ProxyHttpBodyChunk{RequestId: "r-1"}}} + + if !rewriteUpstreamProxyReqID(reqMsg, subID) { + t.Fatal("rewriteUpstreamProxyReqID(request) = false; want true") + } + if !rewriteUpstreamProxyReqID(chunkMsg, subID) { + t.Fatal("rewriteUpstreamProxyReqID(chunk) = false; want true") + } + gotReq := reqMsg.GetProxyHttpRequest().GetRequestId() + gotChunk := chunkMsg.GetProxyHttpBodyChunk().GetRequestId() + if gotReq != gotChunk { + t.Fatalf("chunked request frames got different encoded ids: %q vs %q", gotReq, gotChunk) + } + if gotReq != encodeProxyReqID(subID, "r-1") { + t.Fatalf("encoded id = %q; want %q", gotReq, encodeProxyReqID(subID, "r-1")) + } + + // A non-proxy upstream frame must be left untouched. + kvMsg := &pb.UpstreamMessage{Payload: &pb.UpstreamMessage_KvOp{KvOp: &pb.KVOperation{RequestId: "k-1"}}} + if rewriteUpstreamProxyReqID(kvMsg, subID) { + t.Fatal("rewriteUpstreamProxyReqID(kv) = true; want false") + } + if kvMsg.GetKvOp().GetRequestId() != "k-1" { + t.Fatal("kv request_id was mutated by proxy rewrite") + } +} + +// ============================================================================= +// Integration harness: fake gateway + real RelayMux server + N sandbox clients. +// +// The fake gateway records every upstream frame and exposes the live server +// stream so the test can INJECT downstream frames at the mux's shared upstream. +// ============================================================================= + +type muxFakeGateway struct { + pb.UnimplementedAetherGatewayServer + mu sync.Mutex + received []*pb.UpstreamMessage + streamCh chan pb.AetherGateway_ConnectServer +} + +func newMuxFakeGateway() *muxFakeGateway { + return &muxFakeGateway{streamCh: make(chan pb.AetherGateway_ConnectServer, 2)} +} + +func (g *muxFakeGateway) Connect(stream pb.AetherGateway_ConnectServer) error { + g.streamCh <- stream + for { + msg, err := stream.Recv() + if err != nil { + if errors.Is(err, io.EOF) { + return nil + } + return err + } + g.mu.Lock() + g.received = append(g.received, msg) + g.mu.Unlock() + } +} + +func (g *muxFakeGateway) snapshot() []*pb.UpstreamMessage { + g.mu.Lock() + defer g.mu.Unlock() + out := make([]*pb.UpstreamMessage, len(g.received)) + copy(out, g.received) + return out +} + +// awaitUpstream returns the (single) shared upstream stream the mux opened. +func (g *muxFakeGateway) awaitUpstream(t *testing.T) pb.AetherGateway_ConnectServer { + t.Helper() + select { + case s := <-g.streamCh: + return s + case <-time.After(3 * time.Second): + t.Fatalf("timed out waiting for mux upstream") + return nil + } +} + +type muxHarness struct { + t *testing.T + gateway *muxFakeGateway + mux *RelayMux + relayLis net.Listener + relaySrv *grpc.Server + relayAddr string + relayUnix bool + conns []*grpc.ClientConn +} + +func newMuxHarness(t *testing.T, cfg *Config) *muxHarness { + t.Helper() + + gateway := newMuxFakeGateway() + gwLis, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("listen gateway: %v", err) + } + gwSrv := grpc.NewServer() + pb.RegisterAetherGatewayServer(gwSrv, gateway) + go func() { _ = gwSrv.Serve(gwLis) }() + + cfg.Relay.Enabled = true + cfg.Gateway.Address = gwLis.Addr().String() + cfg.Gateway.Insecure = true + cfg.Service.Implementation = "sidecar" + cfg.Service.Specifier = "instance-1" + if cfg.Relay.Listen == "" { + cfg.Relay.Listen = "unix://" + filepath.Join(t.TempDir(), "relaymux.sock") + } + if err := cfg.Validate(); err != nil { + t.Fatalf("validate cfg: %v", err) + } + + m, err := NewRelayMux(cfg) + if err != nil { + t.Fatalf("NewRelayMux: %v", err) + } + + relayLis, cleanup, err := openRelayListener(cfg.Relay.Listen) + if err != nil { + t.Fatalf("open relay listener: %v", err) + } + relaySrv := grpc.NewServer() + pb.RegisterAetherGatewayServer(relaySrv, m) + go func() { _ = relaySrv.Serve(relayLis) }() + + h := &muxHarness{ + t: t, + gateway: gateway, + mux: m, + relayLis: relayLis, + relaySrv: relaySrv, + relayAddr: relayLis.Addr().String(), + relayUnix: relayLis.Addr().Network() == "unix", + } + + t.Cleanup(func() { + for _, c := range h.conns { + _ = c.Close() + } + // Stop() (not GracefulStop) force-closes in-flight streams. The mux + // keeps its shared upstream open across sub-client departures by + // design, so the fake gateway's Connect handler is still parked in + // Recv() at teardown; GracefulStop would block on it. Tests do not + // need graceful drain. + relaySrv.Stop() + _ = relayLis.Close() + if cleanup != nil { + cleanup() + } + gwSrv.Stop() + _ = gwLis.Close() + }) + return h +} + +// dialSub opens a sandbox-side client stream and completes the local init +// handshake, draining the synthesized ConnectionAck. +func (h *muxHarness) dialSub() pb.AetherGateway_ConnectClient { + h.t.Helper() + scheme := "passthrough:///" + if h.relayUnix { + scheme = "unix://" + } + conn, err := grpc.NewClient(scheme+h.relayAddr, grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + h.t.Fatalf("dial relay: %v", err) + } + h.conns = append(h.conns, conn) + cli := pb.NewAetherGatewayClient(conn) + stream, err := cli.Connect(context.Background()) + if err != nil { + h.t.Fatalf("Connect: %v", err) + } + if err := stream.Send(sandboxInit()); err != nil { + h.t.Fatalf("send init: %v", err) + } + // Drain the synthesized ConnectionAck. + ack, err := stream.Recv() + if err != nil { + h.t.Fatalf("recv ack: %v", err) + } + if _, ok := ack.GetPayload().(*pb.DownstreamMessage_ConnectionAck); !ok { + h.t.Fatalf("first downstream frame = %T; want ConnectionAck", ack.GetPayload()) + } + return stream +} + +// waitUpstreamCount blocks until the fake gateway has recorded at least n +// upstream frames (the first is always the single rewritten Init). +func (h *muxHarness) waitUpstreamCount(n int) []*pb.UpstreamMessage { + h.t.Helper() + deadline := time.Now().Add(3 * time.Second) + for time.Now().Before(deadline) { + snap := h.gateway.snapshot() + if len(snap) >= n { + return snap + } + time.Sleep(10 * time.Millisecond) + } + return h.gateway.snapshot() +} + +// (a) Two sub-clients each do a KV get with the SAME origin request_id; each +// must get its OWN response back (request_id demux + restore). +func TestRelayMux_KVDemuxByRequestID(t *testing.T) { + cfg := &Config{Relay: RelayConfig{ + AllowedOps: AllowedOpsConfig{Profile: AllowedOpsProfileSandboxDefault, Set: true}, + }} + h := newMuxHarness(t, cfg) + + s1 := h.dialSub() + s2 := h.dialSub() + upstream := h.gateway.awaitUpstream(t) + + kvGet := func(reqID, key string) *pb.UpstreamMessage { + return &pb.UpstreamMessage{Payload: &pb.UpstreamMessage_KvOp{KvOp: &pb.KVOperation{ + RequestId: reqID, + Op: pb.KVOperation_GET, + Key: key, + }}} + } + // Both sub-clients use the SAME origin request_id "r1". + if err := s1.Send(kvGet("r1", "k1")); err != nil { + t.Fatalf("s1 send: %v", err) + } + if err := s2.Send(kvGet("r1", "k2")); err != nil { + t.Fatalf("s2 send: %v", err) + } + + // Wait for both forwarded KV ops upstream (after the single Init = 3 total). + snap := h.waitUpstreamCount(3) + if len(snap) < 3 { + t.Fatalf("expected 3 upstream frames (init + 2 kv), saw %d", len(snap)) + } + // Map mux request id -> which key it carried, so we can craft the right reply. + muxByKey := map[string]string{} // key -> muxID + for _, u := range snap { + if kv, ok := u.GetPayload().(*pb.UpstreamMessage_KvOp); ok { + muxByKey[kv.KvOp.GetKey()] = kv.KvOp.GetRequestId() + if kv.KvOp.GetRequestId() == "r1" { + t.Fatalf("request_id was not rewritten to a mux id (still r1)") + } + } + } + if muxByKey["k1"] == "" || muxByKey["k2"] == "" || muxByKey["k1"] == muxByKey["k2"] { + t.Fatalf("expected two distinct mux ids, got %v", muxByKey) + } + + // Inject replies (in swapped order to prove routing isn't FIFO): reply for + // k2 first, then k1. The mux must restore "r1" and route each to the right + // sub-client. + reply := func(muxID, val string) *pb.DownstreamMessage { + return &pb.DownstreamMessage{Payload: &pb.DownstreamMessage_Kv{Kv: &pb.KVResponse{ + RequestId: muxID, + Success: true, + Value: []byte(val), + }}} + } + if err := upstream.Send(reply(muxByKey["k2"], "v2")); err != nil { + t.Fatalf("inject reply k2: %v", err) + } + if err := upstream.Send(reply(muxByKey["k1"], "v1")); err != nil { + t.Fatalf("inject reply k1: %v", err) + } + + got1 := recvKV(t, s1) + got2 := recvKV(t, s2) + if got1.GetRequestId() != "r1" || string(got1.GetValue()) != "v1" { + t.Fatalf("s1 got req=%q val=%q; want r1/v1", got1.GetRequestId(), string(got1.GetValue())) + } + if got2.GetRequestId() != "r1" || string(got2.GetValue()) != "v2" { + t.Fatalf("s2 got req=%q val=%q; want r1/v2", got2.GetRequestId(), string(got2.GetValue())) + } +} + +// (b) A TaskAssignment push is broadcast to BOTH sub-clients. +func TestRelayMux_BroadcastTaskAssignment(t *testing.T) { + cfg := &Config{Relay: RelayConfig{ + AllowedOps: AllowedOpsConfig{Profile: AllowedOpsProfileSandboxDefault, Set: true}, + }} + h := newMuxHarness(t, cfg) + + s1 := h.dialSub() + s2 := h.dialSub() + upstream := h.gateway.awaitUpstream(t) + + push := &pb.DownstreamMessage{Payload: &pb.DownstreamMessage_TaskAssignment{ + TaskAssignment: &pb.TaskAssignment{TaskId: "task-xyz"}}} + if err := upstream.Send(push); err != nil { + t.Fatalf("inject push: %v", err) + } + + for i, s := range []pb.AetherGateway_ConnectClient{s1, s2} { + msg := recvWithTimeout(t, s) + ta, ok := msg.GetPayload().(*pb.DownstreamMessage_TaskAssignment) + if !ok { + t.Fatalf("sub %d got %T; want TaskAssignment", i+1, msg.GetPayload()) + } + if ta.TaskAssignment.GetTaskId() != "task-xyz" { + t.Fatalf("sub %d task id = %q; want task-xyz", i+1, ta.TaskAssignment.GetTaskId()) + } + } +} + +// (c) A TunnelData downstream routes only to the tunnel's owning sub-client. +func TestRelayMux_TunnelRoutesToOwner(t *testing.T) { + cfg := &Config{Relay: RelayConfig{ + AllowedOps: AllowedOpsConfig{Profile: AllowedOpsProfileSandboxTunnels, Set: true}, + TargetTopicClamp: TargetClampConfig{Mode: TargetClampReject, AllowedTargets: []string{"sv.*"}}, + }} + h := newMuxHarness(t, cfg) + + s1 := h.dialSub() + s2 := h.dialSub() + upstream := h.gateway.awaitUpstream(t) + + // s1 opens a tunnel. + open := &pb.UpstreamMessage{Payload: &pb.UpstreamMessage_TunnelOpen{TunnelOpen: &pb.TunnelOpen{ + TunnelId: "tun-1", + TargetTopic: "sv.backend.svc", + }}} + if err := s1.Send(open); err != nil { + t.Fatalf("s1 tunnel open: %v", err) + } + // Wait for the open to reach upstream (init + tunnel_open = 2). + h.waitUpstreamCount(2) + + // Inject tunnel data downstream for tun-1: must land on s1 only. + data := &pb.DownstreamMessage{Payload: &pb.DownstreamMessage_TunnelData{TunnelData: &pb.TunnelData{ + TunnelId: "tun-1", + Data: []byte("payload"), + }}} + if err := upstream.Send(data); err != nil { + t.Fatalf("inject tunnel data: %v", err) + } + + got := recvWithTimeout(t, s1) + td, ok := got.GetPayload().(*pb.DownstreamMessage_TunnelData) + if !ok || td.TunnelData.GetTunnelId() != "tun-1" { + t.Fatalf("s1 got %T; want TunnelData tun-1", got.GetPayload()) + } + // s2 must NOT receive it. + if msg := recvMaybe(s2, 300*time.Millisecond); msg != nil { + t.Fatalf("s2 unexpectedly received %T for a tunnel it doesn't own", msg.GetPayload()) + } +} + +// (e) A clamp rejection returns relayErrorDownstream to the right sub-client. +func TestRelayMux_ClampRejectionToRightSub(t *testing.T) { + cfg := &Config{Relay: RelayConfig{ + AllowedOps: AllowedOpsConfig{Profile: AllowedOpsProfileSandboxTunnels, Set: true}, + TargetTopicClamp: TargetClampConfig{Mode: TargetClampReject, AllowedTargets: []string{"sv.allowed.*"}}, + }} + h := newMuxHarness(t, cfg) + + s1 := h.dialSub() + s2 := h.dialSub() + _ = h.gateway.awaitUpstream(t) + + // s2 sends a ProxyHttpRequest to a denied target. + req := &pb.UpstreamMessage{Payload: &pb.UpstreamMessage_ProxyHttpRequest{ProxyHttpRequest: &pb.ProxyHttpRequest{ + RequestId: "p1", + TargetTopic: "sv.denied.svc", + Method: "GET", + Path: "/", + }}} + if err := s2.Send(req); err != nil { + t.Fatalf("s2 send: %v", err) + } + + got := recvWithTimeout(t, s2) + errResp, ok := got.GetPayload().(*pb.DownstreamMessage_Error) + if !ok { + t.Fatalf("s2 got %T; want Error", got.GetPayload()) + } + if errResp.Error.GetCode() != "RELAY_TARGET_DENIED" { + t.Fatalf("s2 error code = %q; want RELAY_TARGET_DENIED", errResp.Error.GetCode()) + } + // s1 must NOT see the rejection. + if msg := recvMaybe(s1, 300*time.Millisecond); msg != nil { + t.Fatalf("s1 unexpectedly received %T for s2's rejected request", msg.GetPayload()) + } +} + +// ---- recv helpers ----------------------------------------------------------- + +func recvKV(t *testing.T, s pb.AetherGateway_ConnectClient) *pb.KVResponse { + t.Helper() + msg := recvWithTimeout(t, s) + kv, ok := msg.GetPayload().(*pb.DownstreamMessage_Kv) + if !ok { + t.Fatalf("got %T; want KVResponse", msg.GetPayload()) + } + return kv.Kv +} + +func recvWithTimeout(t *testing.T, s pb.AetherGateway_ConnectClient) *pb.DownstreamMessage { + t.Helper() + type res struct { + msg *pb.DownstreamMessage + err error + } + ch := make(chan res, 1) + go func() { + m, err := s.Recv() + ch <- res{m, err} + }() + select { + case r := <-ch: + if r.err != nil { + t.Fatalf("recv: %v", r.err) + } + return r.msg + case <-time.After(3 * time.Second): + t.Fatalf("timed out waiting for downstream frame") + return nil + } +} + +// recvMaybe returns a frame if one arrives within d, else nil. Used to assert a +// sub-client does NOT receive a frame routed to a different sub-client. +// TestMuxSubClient_WedgedReaderDoesNotBlockHealthySub verifies the backpressure +// admission design: a sub-client whose inbox is full (simulating a wedged +// in-sandbox reader whose gRPC write window is exhausted) sheds incoming frames +// with a BACKPRESSURE error rather than blocking the caller. A second healthy +// sub-client must still receive its frame promptly — deliver() must return in +// bounded time regardless of the wedged sub's state. +func TestMuxSubClient_WedgedReaderDoesNotBlockHealthySub(t *testing.T) { + t.Parallel() + + newSub := func(id uint64) *muxSubClient { + return &muxSubClient{ + id: id, + inbox: make(chan *pb.DownstreamMessage, sharedRuntimeSessionDeliverCapacity*2), + deliverSem: backpressure.NewSemaphore( + 5, + sharedRuntimeSessionDeliverCapacity, + backpressure.SemaphoreShortTimeout(sharedRuntimeSessionDeliverShortTimeout), + backpressure.SemaphoreLongTimeout(sharedRuntimeSessionDeliverLongTimeout), + ), + } + } + + taskPush := &pb.DownstreamMessage{ + Payload: &pb.DownstreamMessage_TaskAssignment{ + TaskAssignment: &pb.TaskAssignment{TaskId: "test-task"}, + }, + } + + // --- Wedged sub: fill inbox to capacity without draining. --- + wedged := newSub(1) + inboxCap := cap(wedged.inbox) + for i := 0; i < inboxCap; i++ { + wedged.inbox <- taskPush + } + // inbox is now full. deliver() must return promptly (not block 30s). + done := make(chan struct{}) + go func() { + wedged.deliver(taskPush) + close(done) + }() + select { + case <-done: + // Good: deliver returned without blocking. + case <-time.After(5 * time.Second): + t.Fatal("wedged sub: deliver blocked for >5s; backpressure admission is broken") + } + // The inbox is still full; any BACKPRESSURE notice was also dropped (warn + // path). That's acceptable — the important property is non-blocking return. + wedged.deliverSem.Close() + + // --- Healthy sub: inbox empty, writer goroutine draining. --- + healthy := newSub(2) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + // We don't need a real stream for this unit test — just drain the inbox + // directly to simulate a live reader. + received := make(chan *pb.DownstreamMessage, 4) + go func() { + for { + select { + case msg, ok := <-healthy.inbox: + if !ok { + return + } + received <- msg + case <-ctx.Done(): + return + } + } + }() + + t1 := time.Now() + healthy.deliver(taskPush) + elapsed := time.Since(t1) + if elapsed > 500*time.Millisecond { + t.Errorf("healthy sub: deliver took %v; want <500ms", elapsed) + } + + select { + case msg := <-received: + if _, ok := msg.GetPayload().(*pb.DownstreamMessage_TaskAssignment); !ok { + t.Errorf("healthy sub got %T; want TaskAssignment", msg.GetPayload()) + } + case <-time.After(2 * time.Second): + t.Fatal("healthy sub: message never reached inbox reader") + } + + healthy.closeSubClient() +} + +func recvMaybe(s pb.AetherGateway_ConnectClient, d time.Duration) *pb.DownstreamMessage { + type res struct { + msg *pb.DownstreamMessage + err error + } + ch := make(chan res, 1) + go func() { + m, err := s.Recv() + ch <- res{m, err} + }() + select { + case r := <-ch: + if r.err != nil { + return nil + } + return r.msg + case <-time.After(d): + return nil + } +} diff --git a/server/internal/proxysidecar/runner.go b/server/internal/proxysidecar/runner.go index b62f2bd..be3f822 100644 --- a/server/internal/proxysidecar/runner.go +++ b/server/internal/proxysidecar/runner.go @@ -74,9 +74,30 @@ type Runner struct { runtime *gatewayRuntime router *downstreamRouter - term *Terminator + term *Terminator + // relay is the legacy per-sub-client Relay. Non-nil only when + // cfg.Relay.Mux is false (opted-out of mux). Parked for safety. + // Mutually exclusive with relayMux. relay *Relay - init *Initiator + // relayMux is the shared-upstream RelayMux. Active for BOTH standalone + // (no terminator) and composite (terminator+relay) paths when + // cfg.Relay.Mux is true (the default). In composite mode it rides the + // shared gatewayRuntime; in standalone mode it dials its own upstream. + relayMux *RelayMux + init *Initiator + + // tenantRelay and aggregator are the two halves of the per-tenant relay + // topology. They are mutually exclusive (enforced in Config.Validate) and + // each runs in its own g.Go block in Run — they are NOT folded into + // relaySurfaceOrNil precedence. Non-nil only when their section is enabled. + tenantRelay *TenantRelay + aggregator *Aggregator +} + +// relaySurface is the subset of Relay / RelayMux the Runner drives. Both +// implementations serve the local AetherGateway listener until ctx is done. +type relaySurface interface { + Run(ctx context.Context) error } // NewRunner builds a Runner from cfg. cfg.Validate() is invoked here so the @@ -100,18 +121,54 @@ func NewRunner(cfg *Config, cfgPath string) (*Runner, error) { } if cfg.Relay.Enabled { - relay, err := NewRelay(cfg) - if err != nil { - return nil, fmt.Errorf("relay: %w", err) - } - r.relay = relay - // When the shared runtime is available, route the relay's upstream - // envelopes through it so both surfaces ride one gateway lock. - // Otherwise the relay keeps its default per-session dialer. - if r.runtime != nil { - sink := newSharedRelaySink(r.runtime, cfg.Relay.MaxSessions) - r.router.relay = sink - relay.SetUpstreamDialer(sink.dial) + if cfg.Relay.MuxEnabled() { + // RelayMux is the default for BOTH standalone and composite modes. + // + // STANDALONE (no shared runtime): RelayMux dials its own upstream + // and pumps it in Run. No further wiring needed. + // + // COMPOSITE (shared runtime): RelayMux rides the shared + // gatewayRuntime via SetSharedUpstream — outbound frames go through + // runtime.Client().SendWithPriority; downstream frames arrive on + // the inbox channel fed by downstreamRouter.RouteToRelayMux. + // The relay's gRPC listener is registered on the shared server in + // Run() (installRelayMuxOnServer). + mux, err := NewRelayMux(cfg) + if err != nil { + return nil, fmt.Errorf("relaymux: %w", err) + } + r.relayMux = mux + if r.runtime != nil { + // Wire composite mode: outbound through shared runtime queue. + // Capture runtime (not runtime.Client()) because Client() + // returns nil until gatewayRuntime.init() runs inside Run() + // — capturing the method value here would dereference a nil + // pointer. The closure defers the dereference to send time. + // + // Inbound (gateway → sub-client): the rawDownstreamTap installed + // by installOn calls relayMux.RouteDownstream inline for + // correlated/tunnel frames and safe broadcast push types + // (TaskAssignment, ProgressUpdate). RouteDownstream enqueues into + // each sub-client's bounded inbox via deliver() — no separate + // inbox channel is needed here. + rt := r.runtime + mux.SetSharedUpstream(func(ctx context.Context, prio backpressure.Priority, msg *pb.UpstreamMessage) error { + return rt.Client().SendWithPriority(ctx, prio, msg) + }) + r.router.relayMux = mux + } + } else { + // Mux=false: fall back to legacy per-sub-client Relay (parked). + relay, err := NewRelay(cfg) + if err != nil { + return nil, fmt.Errorf("relay: %w", err) + } + r.relay = relay + if r.runtime != nil { + sink := newSharedRelaySink(r.runtime, cfg.Relay.MaxSessions) + r.router.relay = sink + relay.SetUpstreamDialer(sink.dial) + } } } @@ -123,7 +180,24 @@ func NewRunner(cfg *Config, cfgPath string) (*Runner, error) { r.init = ini } - if r.term == nil && r.relay == nil && r.init == nil { + if cfg.TenantRelay.Enabled { + tr, err := NewTenantRelay(cfg) + if err != nil { + return nil, fmt.Errorf("tenant_relay: %w", err) + } + r.tenantRelay = tr + } + + if cfg.Aggregator.Enabled { + agg, err := NewAggregator(cfg) + if err != nil { + return nil, fmt.Errorf("aggregator: %w", err) + } + r.aggregator = agg + } + + if r.term == nil && r.relay == nil && r.relayMux == nil && r.init == nil && + r.tenantRelay == nil && r.aggregator == nil { // Validate() guards against this, but keep a defensive error so a // future caller that bypasses Validate gets a clear message. return nil, fmt.Errorf("runner: no surfaces enabled") @@ -131,13 +205,30 @@ func NewRunner(cfg *Config, cfgPath string) (*Runner, error) { return r, nil } +// relaySurfaceOrNil returns whichever relay implementation is active (Relay or +// RelayMux), or nil when relay is disabled. +func (r *Runner) relaySurfaceOrNil() relaySurface { + if r.relayMux != nil { + return r.relayMux + } + if r.relay != nil { + return r.relay + } + return nil +} + // Terminator exposes the runner's terminator (or nil when disabled). Tests // reach for this when they need to drive HandleProxyRequest directly. func (r *Runner) Terminator() *Terminator { return r.term } -// Relay exposes the runner's relay (or nil when disabled). +// Relay exposes the runner's legacy Relay (or nil when relay is disabled or +// running in mux mode). func (r *Runner) Relay() *Relay { return r.relay } +// RelayMux exposes the runner's RelayMux (or nil when relay is disabled or +// running in legacy mode). +func (r *Runner) RelayMux() *RelayMux { return r.relayMux } + // Initiator exposes the runner's initiator (or nil when disabled). func (r *Runner) Initiator() *Initiator { return r.init } @@ -149,7 +240,6 @@ func (r *Runner) Run(ctx context.Context) error { return fmt.Errorf("runner: build client: %w", err) } r.router.installOn(r.runtime.Client(), r.runtime.Transport()) - go r.runtime.runConnectionLoop(ctx) } log.Info(). @@ -160,6 +250,17 @@ func (r *Runner) Run(ctx context.Context) error { g, gctx := errgroup.WithContext(ctx) + // The shared gateway connection runs as a surface in the errgroup: a fatal + // give-up (too many consecutive terminal auth failures) cancels gctx, + // drains the other surfaces, and propagates out of Run so the process + // exits non-zero / signals its wrapped child — making an orphaned sandbox + // reapable instead of a zombie spamming the gateway. + if r.runtime != nil { + g.Go(func() error { + return r.runtime.runConnectionLoop(gctx) + }) + } + if r.term != nil { g.Go(func() error { log.Info(). @@ -171,9 +272,9 @@ func (r *Runner) Run(ctx context.Context) error { }) } - if r.relay != nil { + if rs := r.relaySurfaceOrNil(); rs != nil { g.Go(func() error { - if err := r.relay.Run(gctx); err != nil { + if err := rs.Run(gctx); err != nil { return fmt.Errorf("relay: %w", err) } return nil @@ -189,6 +290,26 @@ func (r *Runner) Run(ctx context.Context) error { }) } + // tenant-relay and aggregator each run in their own block (mutually + // exclusive per process; not folded into relaySurfaceOrNil precedence). + if r.tenantRelay != nil { + g.Go(func() error { + if err := r.tenantRelay.Run(gctx); err != nil { + return fmt.Errorf("tenant_relay: %w", err) + } + return nil + }) + } + + if r.aggregator != nil { + g.Go(func() error { + if err := r.aggregator.Run(gctx); err != nil { + return fmt.Errorf("aggregator: %w", err) + } + return nil + }) + } + err := g.Wait() if err != nil && errors.Is(err, context.Canceled) { return nil @@ -215,31 +336,76 @@ func (r *Runner) Reload() { // downstreamRouter is wired into the shared runtime's ServiceClient so that // envelopes the gateway publishes to our sv:: topic land in whichever surface -// owns them. When relay is nil this collapses to standalone-terminator -// semantics; when both are present the router preserves the composite-mode -// semantics from the previous design (terminator owns its registered -// tunnels, the rest fall through to the active relay session). +// owns them. When relayMux (and relay) are nil this collapses to standalone- +// terminator semantics; when both terminator and relay are present, the router +// splits: terminator owns its registered tunnels and outbound ProxyHttp, +// relayMux gets everything else via RouteDownstream (which classifies and +// dispatches to sub-clients). The legacy relay field is kept for the Mux=false +// fallback path only. type downstreamRouter struct { - term *Terminator - relay *sharedRelaySink + term *Terminator + relayMux *RelayMux + relay *sharedRelaySink // legacy Mux=false path only +} + +// routeToRelay delivers msg to whichever relay surface is active. In the +// RelayMux path it calls RouteDownstream which classifies the frame and +// enqueues it into the appropriate sub-client inbox(es) via deliver(). +// In the legacy sharedRelaySink path it calls routeMessage. Returns true if +// the relay surface handled the message. +func (r *downstreamRouter) routeToRelay(msg *pb.DownstreamMessage) bool { + if r.relayMux != nil { + return r.relayMux.RouteDownstream(msg) + } + if r.relay != nil { + r.relay.routeMessage(msg) + return true + } + return false +} + +// hasRelay reports whether a relay surface is wired. +func (r *downstreamRouter) hasRelay() bool { + return r.relayMux != nil || r.relay != nil } // installOn wires the dispatcher hooks on client. The supplied transport // ships outbound proxy/tunnel envelopes upstream. -func (r *downstreamRouter) installOn(client *aether.ServiceClient, transport tunnelTransport) { +func (r *downstreamRouter) installOn(client *aether.BaseClient, transport tunnelTransport) { if r.term == nil { // The runtime is only built when terminator is enabled, so r.term is // non-nil in practice. Defensive guard for future callers. return } - // Plain peer messages: log only — the sidecar has no message-relay - // surface, terminators don't expect peer-to-peer messages. + // Plain peer messages: forward to attached relay sessions so Python + // AsyncServiceClient instances connected via the local relay socket + // receive peer messages addressed to this implementation. Pre-relay, + // the proxy-sidecar runner was a pure HTTP-RPC terminator and the + // OnMessage path was a no-op; the workclaw chat envelope flow + // (gateway → sv::sandbox-sidecar:: → Python bridge OnMessage) + // requires this relay path. Wraps the SDK-level Message into the + // same DownstreamMessage_Msg envelope the gateway uses for normal + // subscription delivery (gateway/subscription.go:228) so the relay + // session's downstream stream is wire-compatible. client.OnMessage(func(_ context.Context, msg *aether.Message) error { log.Debug(). Str("source", msg.SourceTopic). Int("payload_bytes", len(msg.Payload)). Msg("runner: received message via OnMessage path") + r.routeToRelay(&pb.DownstreamMessage{ + Payload: &pb.DownstreamMessage_Msg{ + Msg: &pb.IncomingMessage{ + SourceTopic: msg.SourceTopic, + Payload: msg.Payload, + MessageType: msg.MessageType, + // Forward the gateway-resolved OBO subject across the relay so + // the in-sandbox bridge can identify the user (parity with the + // direct subscription path in gateway/subscription.go). + OnBehalfSubject: msg.OnBehalfSubject, + }, + }, + }) return nil }) @@ -288,8 +454,8 @@ func (r *downstreamRouter) installOn(client *aether.ServiceClient, transport tun // in-flight dispatch (H3 wakeup). Try the terminator's active // dispatch table first: an error envelope keyed by a known dispatch // requestID is a "your caller went away" wakeup and should cancel - // the dispatch immediately. Otherwise route to the relay sink so - // the originating sandbox session sees it on its inbox. + // the dispatch immediately. Otherwise route to the relay so the + // originating sandbox session sees it on its inbox. client.OnProxyHttpResponse(func(_ context.Context, resp *pb.ProxyHttpResponse) error { if resp != nil && resp.GetError() != nil && r.term != nil { if v, ok := r.term.activeDispatches.LoadAndDelete(resp.GetRequestId()); ok { @@ -299,11 +465,9 @@ func (r *downstreamRouter) installOn(client *aether.ServiceClient, transport tun return nil } } - if r.relay != nil { - r.relay.routeMessage(&pb.DownstreamMessage{ - Payload: &pb.DownstreamMessage_ProxyHttpResponse{ProxyHttpResponse: resp}, - }) - } + r.routeToRelay(&pb.DownstreamMessage{ + Payload: &pb.DownstreamMessage_ProxyHttpResponse{ProxyHttpResponse: resp}, + }) return nil }) @@ -324,11 +488,9 @@ func (r *downstreamRouter) installOn(client *aether.ServiceClient, transport tun if chunk.GetIsRequest() { return r.term.handleChunkedRequestFrame(chunkCtx, chunk, transport) } - if r.relay != nil { - r.relay.routeMessage(&pb.DownstreamMessage{ - Payload: &pb.DownstreamMessage_ProxyHttpBodyChunk{ProxyHttpBodyChunk: chunk}, - }) - } + r.routeToRelay(&pb.DownstreamMessage{ + Payload: &pb.DownstreamMessage_ProxyHttpBodyChunk{ProxyHttpBodyChunk: chunk}, + }) return nil }) @@ -375,10 +537,9 @@ func (r *downstreamRouter) installOn(client *aether.ServiceClient, transport tun r.term.HandleTunnelData(frame, transport) return nil } - if r.relay != nil { - r.relay.routeMessage(&pb.DownstreamMessage{ - Payload: &pb.DownstreamMessage_TunnelData{TunnelData: frame}, - }) + if r.routeToRelay(&pb.DownstreamMessage{ + Payload: &pb.DownstreamMessage_TunnelData{TunnelData: frame}, + }) { return nil } // Standalone terminator: emit PEER_RESET for unknown tunnel. @@ -391,12 +552,9 @@ func (r *downstreamRouter) installOn(client *aether.ServiceClient, transport tun r.term.HandleTunnelAck(ack) return nil } - if r.relay != nil { - r.relay.routeMessage(&pb.DownstreamMessage{ - Payload: &pb.DownstreamMessage_TunnelAck{TunnelAck: ack}, - }) - return nil - } + r.routeToRelay(&pb.DownstreamMessage{ + Payload: &pb.DownstreamMessage_TunnelAck{TunnelAck: ack}, + }) // Standalone: HandleTunnelAck silently no-ops on unknown tunnels. return nil }) @@ -406,15 +564,70 @@ func (r *downstreamRouter) installOn(client *aether.ServiceClient, transport tun r.term.HandleTunnelClose(cm) return nil } - if r.relay != nil { - r.relay.routeMessage(&pb.DownstreamMessage{ - Payload: &pb.DownstreamMessage_TunnelClose{TunnelClose: cm}, - }) - return nil - } + r.routeToRelay(&pb.DownstreamMessage{ + Payload: &pb.DownstreamMessage_TunnelClose{TunnelClose: cm}, + }) // Standalone: HandleTunnelClose silently no-ops on unknown tunnels. return nil }) + + // Raw downstream tap: intercepts correlated response-class downstream + // messages before the SDK drops them (it has no pending call matching + // the relay-issued request_id). In RelayMux composite mode, RouteDownstream + // classifies by the mux's pending table (rmx-* ids) and delivers to the + // right sub-client. In legacy relay mode, routeResponseToOwner uses the + // sharedRelaySink's flat map. Either way, returning true consumes the + // message so the SDK doesn't drop it. + // + // For the RelayMux path this handles KV/Task/Checkpoint/Workspace/Session/ + // Agent/Acl/Token/Admin/Audit/Authority/Workflow responses and Error frames + // that carry an rmx-* request_id. The full correlated set is enumerated in + // relaymux_reqid.go (19 pairs). Frames that are NOT relay-owned (e.g. a + // terminator-issued KV op using a "req-N" id) return false so the SDK + // handles them normally. + if r.hasRelay() { + router := r + client.SetRawDownstreamTap(func(msg *pb.DownstreamMessage) bool { + if router.relayMux != nil { + // RelayMux: RouteDownstream classifies by pending table. + // It returns true only for rmx-* correlated ids, relay + // tunnels, and push broadcasts — never for terminator ids. + return router.relayMux.RouteDownstream(msg) + } + // Legacy sharedRelaySink: flat request_id map. + rid := downstreamResponseRequestID(msg) + if rid == "" { + return false + } + return router.relay.routeResponseToOwner(rid, msg) + }) + } +} + +// downstreamResponseRequestID returns the correlation request_id carried by a +// response-class downstream message, or "" for message types that are not +// correlated responses (or carry no request_id). Used by the rawDownstreamTap +// to decide whether a message might belong to a relay session. The set mirrors +// the request-class ops registered in sharedRuntimeSession.Send plus the +// generic Error envelope, which the gateway stamps with the originating +// request_id when an op fails. +func downstreamResponseRequestID(msg *pb.DownstreamMessage) string { + switch p := msg.GetPayload().(type) { + case *pb.DownstreamMessage_Kv: + return p.Kv.GetRequestId() + case *pb.DownstreamMessage_TaskOp: + return p.TaskOp.GetRequestId() + case *pb.DownstreamMessage_TaskQuery: + return p.TaskQuery.GetRequestId() + case *pb.DownstreamMessage_Checkpoint: + return p.Checkpoint.GetRequestId() + case *pb.DownstreamMessage_Workspace: + return p.Workspace.GetRequestId() + case *pb.DownstreamMessage_Error: + return p.Error.GetRequestId() + default: + return "" + } } // ============================================================================= @@ -600,6 +813,29 @@ func (s *sharedRelaySink) registerTunnel(sess *sharedRuntimeSession, tunnelID st s.mu.Unlock() } +// routeResponseToOwner delivers a correlated response-class downstream +// message to the relay session that issued the matching request, keyed by +// request_id. Returns true when a session owned the id (message delivered, +// route released so the table stays bounded); false when no relay session +// owns it — in which case the caller must let the SDK handle the message +// normally (e.g. a response to an op the terminator surface issued itself). +func (s *sharedRelaySink) routeResponseToOwner(requestID string, msg *pb.DownstreamMessage) bool { + if requestID == "" { + return false + } + s.mu.Lock() + sess := s.requestRoutes[requestID] + if sess != nil { + delete(s.requestRoutes, requestID) + } + s.mu.Unlock() + if sess == nil { + return false + } + sess.deliver(msg) + return true +} + // ============================================================================= // Fake AetherGatewayClient that the relay sees via the shared sink. // ============================================================================= @@ -696,6 +932,22 @@ func (s *sharedRuntimeSession) Send(msg *pb.UpstreamMessage) error { s.owner.registerRequest(s, p.ProxyHttpRequest.GetRequestId()) case *pb.UpstreamMessage_TunnelOpen: s.owner.registerTunnel(s, p.TunnelOpen.GetTunnelId()) + // Correlated request/response ops the in-sandbox SDK awaits. The shared + // runtime forwards these upstream verbatim, so the gateway's response + // arrives on the runtime connection with no pending correlation here — + // register the request_id so the rawDownstreamTap can route the matching + // response back to THIS session instead of letting it get dropped (which + // stalls the in-sandbox caller until its client-side timeout). + case *pb.UpstreamMessage_KvOp: + s.owner.registerRequest(s, p.KvOp.GetRequestId()) + case *pb.UpstreamMessage_TaskOp: + s.owner.registerRequest(s, p.TaskOp.GetRequestId()) + case *pb.UpstreamMessage_TaskQuery: + s.owner.registerRequest(s, p.TaskQuery.GetRequestId()) + case *pb.UpstreamMessage_CheckpointOp: + s.owner.registerRequest(s, p.CheckpointOp.GetRequestId()) + case *pb.UpstreamMessage_WorkspaceOp: + s.owner.registerRequest(s, p.WorkspaceOp.GetRequestId()) } prio := priorityForSharedRelayUpstream(msg) // SendWithPriority feeds the SDK's CoDel-managed admission queue. diff --git a/server/internal/proxysidecar/runner_test.go b/server/internal/proxysidecar/runner_test.go index a204abf..2dc98c7 100644 --- a/server/internal/proxysidecar/runner_test.go +++ b/server/internal/proxysidecar/runner_test.go @@ -145,6 +145,87 @@ func TestSharedRuntimeSessionDeliver_ShedsBestEffortFirst(t *testing.T) { // test means the runner sends are now routing envelopes through the wrong // CoDel queue — a behaviour regression even when no test downstream of it // notices. +// TestRouteResponseToOwner_DeliversAndReleases verifies the correlated +// response routing the rawDownstreamTap relies on: a registered request_id +// routes its response to the owning session's inbox and releases the route; +// an unregistered id returns false (so the SDK handles it normally) and +// delivers nothing. +func TestRouteResponseToOwner_DeliversAndReleases(t *testing.T) { + sess, cleanup := newDeliverTestSession(t, 1) + defer cleanup() + sink := sess.owner + + const reqID = "kv-req-1" + sink.registerRequest(sess, reqID) + + kvResp := &pb.DownstreamMessage{ + Payload: &pb.DownstreamMessage_Kv{Kv: &pb.KVResponse{RequestId: reqID, Success: true}}, + } + if !sink.routeResponseToOwner(reqID, kvResp) { + t.Fatal("routeResponseToOwner returned false for a registered request_id") + } + select { + case got := <-sess.inbox: + if _, ok := got.GetPayload().(*pb.DownstreamMessage_Kv); !ok { + t.Fatalf("inbox got %T, want *DownstreamMessage_Kv", got.GetPayload()) + } + default: + t.Fatal("expected the KV response on the session inbox") + } + + // Route was released — a second delivery for the same id must miss. + if sink.routeResponseToOwner(reqID, kvResp) { + t.Error("routeResponseToOwner returned true after the route was released") + } + // Unknown id never claims. + if sink.routeResponseToOwner("no-such-id", kvResp) { + t.Error("routeResponseToOwner returned true for an unregistered request_id") + } +} + +// TestDownstreamResponseRequestID pins the correlation-id extraction the tap +// uses to decide whether a downstream message might belong to a relay session. +func TestDownstreamResponseRequestID(t *testing.T) { + cases := []struct { + name string + msg *pb.DownstreamMessage + want string + }{ + { + name: "Kv", + msg: &pb.DownstreamMessage{Payload: &pb.DownstreamMessage_Kv{Kv: &pb.KVResponse{RequestId: "r1"}}}, + want: "r1", + }, + { + name: "TaskOp", + msg: &pb.DownstreamMessage{Payload: &pb.DownstreamMessage_TaskOp{TaskOp: &pb.TaskOperationResponse{RequestId: "r2"}}}, + want: "r2", + }, + { + name: "TaskQuery", + msg: &pb.DownstreamMessage{Payload: &pb.DownstreamMessage_TaskQuery{TaskQuery: &pb.TaskQueryResponse{RequestId: "r3"}}}, + want: "r3", + }, + { + name: "Error_with_request_id", + msg: &pb.DownstreamMessage{Payload: &pb.DownstreamMessage_Error{Error: &pb.ErrorResponse{RequestId: "r4"}}}, + want: "r4", + }, + { + name: "Msg_is_not_a_correlated_response", + msg: &pb.DownstreamMessage{Payload: &pb.DownstreamMessage_Msg{Msg: &pb.IncomingMessage{}}}, + want: "", + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if got := downstreamResponseRequestID(tc.msg); got != tc.want { + t.Errorf("downstreamResponseRequestID() = %q, want %q", got, tc.want) + } + }) + } +} + func TestPriorityForSharedRelayUpstream(t *testing.T) { cases := []struct { name string diff --git a/server/internal/proxysidecar/tenant_relay.go b/server/internal/proxysidecar/tenant_relay.go new file mode 100644 index 0000000..9f1d827 --- /dev/null +++ b/server/internal/proxysidecar/tenant_relay.go @@ -0,0 +1,309 @@ +package proxysidecar + +import ( + "context" + "errors" + "fmt" + "io" + "os" + "time" + + "github.com/rs/zerolog/log" + pb "github.com/scitrera/aether/api/proto" + "github.com/scitrera/aether/sdk/go/aether" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/keepalive" +) + +// TenantRelay runs the sidecar in tenant-relay mode: it dials the local tenant +// gateway (top-level Gateway section, tenant cert under semi-strict mTLS) and +// the central aggregator's SandboxRelayTunnel surface, then splices a +// provider's gateway session through itself verbatim. It satisfies +// relaySurface. +// +// The provider's InitConnection arrives over the aggregator tunnel as the first +// TunnelFrame.up and is forwarded to the tenant gateway UNTOUCHED — neither the +// claimed identity nor resume_session_id is rewritten. This is the load-bearing +// contrast with relay.go: a relay-mode Relay rewrites every sandbox to its own +// Service identity and discards resume_session_id (relay_init.go), because the +// sandbox is untrusted; here the upstream peer is a trusted provider whose +// session must reach the gateway intact so the gateway can resume the provider's +// pre-existing session/lock. The trust boundary is the provider→aggregator and +// aggregator→relay mTLS hops (peer-cert CN), not per-op filtering — hence the +// default service-passthrough filter profile (a bypass; see +// FilterProfileServicePassthrough). +type TenantRelay struct { + cfg *Config + allowed *allowedOpsSet + + // gatewayDialer opens an outbound gRPC connection to the local tenant + // gateway. Production code uses dialTenantGateway; tests inject a fake. + gatewayDialer func(ctx context.Context) (pb.AetherGatewayClient, func() error, error) + + // tunnelDialer opens the aggregator's SandboxRelayTunnel client. Production + // code uses dialAggregatorTunnel; tests inject a fake. + tunnelDialer func(ctx context.Context) (pb.SandboxRelayTunnelClient, func() error, error) +} + +// NewTenantRelay constructs a TenantRelay from cfg. The tunnel and gateway +// connections are not opened until Run is invoked. cfg.Validate() must have +// been called first (NewRunner does this). +func NewTenantRelay(cfg *Config) (*TenantRelay, error) { + // FilterProfile is resolved into an allowedOpsSet via the same path the + // relay surface uses. The default (service-passthrough) yields a bypass + // set; an explicit profile yields a literal allow-list. + allowed, err := resolveAllowedOps(AllowedOpsConfig{Profile: cfg.TenantRelay.FilterProfile}) + if err != nil { + return nil, err + } + t := &TenantRelay{ + cfg: cfg, + allowed: allowed, + } + t.gatewayDialer = t.dialTenantGateway + t.tunnelDialer = t.dialAggregatorTunnel + return t, nil +} + +// Run opens the aggregator tunnel, announces this relay's tenant via +// TunnelHello, waits for the provider's first frame (its InitConnection), +// dials the local tenant gateway, forwards that init verbatim, and then pumps +// frames in both directions until ctx is cancelled or either side closes. +func (t *TenantRelay) Run(ctx context.Context) error { + tunnelCli, tunnelClose, err := t.tunnelDialer(ctx) + if err != nil { + return fmt.Errorf("tenant-relay: dial aggregator: %w", err) + } + defer func() { + if tunnelClose != nil { + _ = tunnelClose() + } + }() + + tunnel, err := tunnelCli.Tunnel(ctx) + if err != nil { + return fmt.Errorf("tenant-relay: open tunnel: %w", err) + } + + // Announce our tenant. The aggregator validates this hint against our + // peer-cert CN and pairs us with a provider's gateway session. + if err := tunnel.Send(&pb.TunnelFrame{ + F: &pb.TunnelFrame_Hello{Hello: &pb.TunnelHello{Tenant: t.cfg.TenantRelay.Tenant}}, + }); err != nil { + return fmt.Errorf("tenant-relay: send hello: %w", err) + } + + log.Info(). + Str("tenant", t.cfg.TenantRelay.Tenant). + Str("aggregator", t.cfg.TenantRelay.Aggregator.Address). + Str("gateway", t.cfg.Gateway.Address). + Str("filter_profile", t.cfg.TenantRelay.FilterProfile). + Strs("allowed_ops", t.allowed.list()). + Msg("tenant-relay running; awaiting provider init") + + // The first up frame carries the provider's InitConnection. We forward it + // verbatim (no identity rewrite, resume_session_id preserved) before + // starting the bidirectional pumps. + first, err := tunnel.Recv() + if err != nil { + if errors.Is(err, io.EOF) { + return nil + } + return fmt.Errorf("tenant-relay: recv init frame: %w", err) + } + init := first.GetUp() + if init == nil { + return fmt.Errorf("tenant-relay: first tunnel frame was %T, expected up (InitConnection)", first.GetF()) + } + if _, ok := init.GetPayload().(*pb.UpstreamMessage_Init); !ok { + return fmt.Errorf("tenant-relay: first up frame was %T, expected InitConnection", init.GetPayload()) + } + + dialCtx, cancelDial := context.WithTimeout(ctx, 30*time.Second) + gateway, gatewayClose, err := t.gatewayDialer(dialCtx) + cancelDial() + if err != nil { + return fmt.Errorf("tenant-relay: dial gateway: %w", err) + } + defer func() { + if gatewayClose != nil { + _ = gatewayClose() + } + }() + + gwStream, err := gateway.Connect(ctx) + if err != nil { + return fmt.Errorf("tenant-relay: open gateway connect: %w", err) + } + + // Forward the provider's init UNTOUCHED — this is the key passthrough. + if err := gwStream.Send(init); err != nil { + return fmt.Errorf("tenant-relay: send init: %w", err) + } + + log.Info(). + Str("tenant", t.cfg.TenantRelay.Tenant). + Str("provider_init", describeSandboxIdentity(init.GetInit())). + Bool("has_resume", init.GetInit().GetResumeSessionId() != ""). + Msg("tenant-relay: provider session opened (init forwarded verbatim)") + + return t.runPumps(ctx, tunnel, gwStream) +} + +// runPumps drives the tunnel↔gateway pumps until one direction closes or +// errors. It mirrors relaySession.run: an errgroup-of-two via errCh, cancel on +// the first return, a bounded drain of the sibling, and EOF→nil. +func (t *TenantRelay) runPumps(ctx context.Context, tunnel pb.SandboxRelayTunnel_TunnelClient, gateway pb.AetherGateway_ConnectClient) error { + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + errCh := make(chan error, 2) + go func() { errCh <- t.pumpUp(ctx, tunnel, gateway) }() + go func() { errCh <- t.pumpDown(ctx, tunnel, gateway) }() + + first := <-errCh + cancel() + // Drain the second goroutine so we don't leak it past Run's return; bound + // the wait so a stuck peer can't wedge us. + select { + case <-errCh: + case <-time.After(5 * time.Second): + } + if errors.Is(first, io.EOF) { + return nil + } + return first +} + +// pumpUp copies tunnel → gateway. Each up frame is unwrapped and, when the +// op-filter is not a bypass, matched against the allow-list before forwarding. +// There is NO target-topic clamp and NO init rewrite: provider envelopes pass +// through verbatim. When the tunnel closes its send-half (EOF) we mirror that +// on the gateway stream via CloseSend so the gateway sees a clean half-close. +func (t *TenantRelay) pumpUp(ctx context.Context, tunnel pb.SandboxRelayTunnel_TunnelClient, gateway pb.AetherGateway_ConnectClient) error { + defer func() { _ = gateway.CloseSend() }() + for { + if ctx.Err() != nil { + return ctx.Err() + } + frame, err := tunnel.Recv() + if err != nil { + return err + } + up := frame.GetUp() + if up == nil { + // Non-up frames on the relay→gateway direction are not expected + // (the aggregator only sends provider up frames after the init). + // Drop them rather than forwarding garbage to the gateway. + log.Debug(). + Str("frame", fmt.Sprintf("%T", frame.GetF())). + Msg("tenant-relay: dropping non-up tunnel frame") + continue + } + + // service-passthrough is a bypass: allows() returns true unconditionally + // so trusted provider ops absent from upstreamOpName (CreateTask, + // SessionOperation, ...) are forwarded verbatim. A non-bypass profile + // applies the literal allow-list. + if !t.allowed.allows(upstreamOpName(up)) { + log.Debug(). + Str("op", upstreamOpName(up)). + Msg("tenant-relay: dropped upstream op (denied)") + continue + } + + if err := gateway.Send(up); err != nil { + return err + } + } +} + +// pumpDown copies gateway → tunnel, wrapping each downstream envelope in a +// TunnelFrame.down for the aggregator to splice back to the provider. +func (t *TenantRelay) pumpDown(ctx context.Context, tunnel pb.SandboxRelayTunnel_TunnelClient, gateway pb.AetherGateway_ConnectClient) error { + for { + if ctx.Err() != nil { + return ctx.Err() + } + msg, err := gateway.Recv() + if err != nil { + return err + } + if err := tunnel.Send(&pb.TunnelFrame{F: &pb.TunnelFrame_Down{Down: msg}}); err != nil { + return err + } + } +} + +// dialTenantGateway opens a gRPC connection to the local tenant gateway using +// the top-level Gateway section (tenant cert under semi-strict mTLS). It reuses +// the shared dialGatewayWithTLS helper. The closer must be invoked when the +// caller no longer needs the connection. +func (t *TenantRelay) dialTenantGateway(_ context.Context) (pb.AetherGatewayClient, func() error, error) { + return dialGatewayWithTLS(t.cfg.Gateway) +} + +// dialAggregatorTunnel opens a SandboxRelayTunnel client to the aggregator using +// the TenantRelay.Aggregator dial config (sandboxes-CA client cert under mTLS, +// or insecure for test/dev). The closer must be invoked when the caller no +// longer needs the connection. +func (t *TenantRelay) dialAggregatorTunnel(_ context.Context) (pb.SandboxRelayTunnelClient, func() error, error) { + agg := t.cfg.TenantRelay.Aggregator + + var dialOpts []grpc.DialOption + if agg.Insecure { + dialOpts = append(dialOpts, grpc.WithTransportCredentials(insecure.NewCredentials())) + } else { + tlsCfg, err := buildAggregatorTLS(agg.TLS) + if err != nil { + return nil, nil, fmt.Errorf("build aggregator tls: %w", err) + } + stdTLS, err := materialiseTLS(tlsCfg) + if err != nil { + return nil, nil, fmt.Errorf("load aggregator tls credentials: %w", err) + } + dialOpts = append(dialOpts, grpc.WithTransportCredentials(credentials.NewTLS(stdTLS))) + } + dialOpts = append(dialOpts, grpc.WithKeepaliveParams(keepalive.ClientParameters{ + Time: 30 * time.Second, + Timeout: 15 * time.Second, + PermitWithoutStream: true, + })) + + conn, err := grpc.NewClient(agg.Address, dialOpts...) + if err != nil { + return nil, nil, err + } + return pb.NewSandboxRelayTunnelClient(conn), conn.Close, nil +} + +// buildAggregatorTLS reads the aggregator dial config's TLS cert/key/CA into an +// aether.TLSConfig suitable for materialiseTLS. Mirrors buildTLSConfig but is +// keyed off the AggregatorDialConfig's TLSConfig rather than a GatewayConfig. +func buildAggregatorTLS(tlsConf TLSConfig) (*aether.TLSConfig, error) { + out := &aether.TLSConfig{Enabled: true} + if tlsConf.CAFile != "" { + ca, err := os.ReadFile(tlsConf.CAFile) + if err != nil { + return nil, fmt.Errorf("read aggregator tls.ca_file %q: %w", tlsConf.CAFile, err) + } + out.RootCAs = ca + } + if tlsConf.CertFile != "" { + cert, err := os.ReadFile(tlsConf.CertFile) + if err != nil { + return nil, fmt.Errorf("read aggregator tls.cert_file %q: %w", tlsConf.CertFile, err) + } + out.ClientCert = cert + } + if tlsConf.KeyFile != "" { + key, err := os.ReadFile(tlsConf.KeyFile) + if err != nil { + return nil, fmt.Errorf("read aggregator tls.key_file %q: %w", tlsConf.KeyFile, err) + } + out.ClientKey = key + } + return out, nil +} diff --git a/server/internal/proxysidecar/tenant_relay_test.go b/server/internal/proxysidecar/tenant_relay_test.go new file mode 100644 index 0000000..a3ec71a --- /dev/null +++ b/server/internal/proxysidecar/tenant_relay_test.go @@ -0,0 +1,438 @@ +package proxysidecar + +import ( + "context" + "errors" + "io" + "net" + "sync" + "testing" + "time" + + pb "github.com/scitrera/aether/api/proto" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/status" +) + +// fakeAggregator implements pb.SandboxRelayTunnelServer for tenant-relay tests. +// Its Tunnel handler accepts the relay's TunnelHello, pushes a provider +// InitConnection (with a non-empty resume_session_id and a Service identity) as +// the first up frame, optionally pushes additional up frames the test stages, +// and records every down frame the relay sends back. +type fakeAggregator struct { + pb.UnimplementedSandboxRelayTunnelServer + + // upFrames are sent (in order) after the relay's hello is received. The + // first must carry the provider InitConnection. + upFrames []*pb.TunnelFrame + + // eofAfterSend, when true, makes Tunnel return (closing its send-half) + // immediately after pushing upFrames instead of looping on Recv. The + // relay's pumpUp then observes io.EOF on tunnel.Recv() → runPumps returns + // nil, exercising the clean tunnel-EOF teardown path. + eofAfterSend bool + + mu sync.Mutex + hello *pb.TunnelHello + downs []*pb.DownstreamMessage + streamCh chan struct{} // signalled once the hello has been received +} + +func newFakeAggregator(upFrames []*pb.TunnelFrame) *fakeAggregator { + return &fakeAggregator{upFrames: upFrames, streamCh: make(chan struct{}, 1)} +} + +func (a *fakeAggregator) Tunnel(stream grpc.BidiStreamingServer[pb.TunnelFrame, pb.TunnelFrame]) error { + // First frame from the relay MUST be a TunnelHello. + first, err := stream.Recv() + if err != nil { + return err + } + hello := first.GetHello() + if hello == nil { + return errors.New("fakeAggregator: first frame was not a hello") + } + a.mu.Lock() + a.hello = hello + a.mu.Unlock() + select { + case a.streamCh <- struct{}{}: + default: + } + + // Push the staged provider up frames (init first, then any follow-ups). + for _, f := range a.upFrames { + if err := stream.Send(f); err != nil { + return err + } + } + + if a.eofAfterSend { + // Returning closes the server's send-half; the relay's pumpUp sees + // io.EOF on tunnel.Recv(). + return nil + } + + // Record down frames until the relay closes the stream. + for { + frame, err := stream.Recv() + if err != nil { + if errors.Is(err, io.EOF) { + return nil + } + return err + } + if down := frame.GetDown(); down != nil { + a.mu.Lock() + a.downs = append(a.downs, down) + a.mu.Unlock() + } + } +} + +func (a *fakeAggregator) snapshotDowns() []*pb.DownstreamMessage { + a.mu.Lock() + defer a.mu.Unlock() + out := make([]*pb.DownstreamMessage, len(a.downs)) + copy(out, a.downs) + return out +} + +func (a *fakeAggregator) helloTenant() string { + a.mu.Lock() + defer a.mu.Unlock() + if a.hello == nil { + return "" + } + return a.hello.GetTenant() +} + +// providerInit builds the provider's InitConnection as it would arrive over the +// tunnel: a Service identity (impl=sandbox-provider, spec=pod-7) and a +// non-empty resume_session_id. The tenant-relay must forward this VERBATIM — +// neither field may be rewritten or discarded. +func providerInit() *pb.UpstreamMessage { + return &pb.UpstreamMessage{ + Payload: &pb.UpstreamMessage_Init{ + Init: &pb.InitConnection{ + ClientType: &pb.InitConnection_Service{ + Service: &pb.ServiceIdentity{ + Implementation: "sandbox-provider", + Specifier: "pod-7", + }, + }, + Credentials: map[string]string{"api_key": "provider-key"}, + ResumeSessionId: "resume-xyz", + }, + }, + } +} + +// tenantRelayHarness wires a fakeGateway (the local tenant gateway), a real +// TenantRelay with injected dialers, and a fakeAggregator the relay tunnels +// through. The TenantRelay's Run loop is started in a goroutine and cancelled +// via t.Cleanup. +type tenantRelayHarness struct { + t *testing.T + gateway *fakeGateway + gatewaySrv *grpc.Server + gatewayLis net.Listener + aggregator *fakeAggregator + relay *TenantRelay + runErr chan error + cancel context.CancelFunc +} + +func newTenantRelayHarness(t *testing.T, upFrames []*pb.TunnelFrame) *tenantRelayHarness { + t.Helper() + return newTenantRelayHarnessAgg(t, newFakeAggregator(upFrames)) +} + +func newTenantRelayHarnessAgg(t *testing.T, aggregator *fakeAggregator) *tenantRelayHarness { + t.Helper() + + // 1. Local tenant gateway (reused fakeGateway from relay_test.go). + gateway := newFakeGateway() + gwLis, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("listen gateway: %v", err) + } + gwSrv := grpc.NewServer() + pb.RegisterAetherGatewayServer(gwSrv, gateway) + go func() { _ = gwSrv.Serve(gwLis) }() + + // 2. Aggregator tunnel surface. + aggLis, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("listen aggregator: %v", err) + } + aggSrv := grpc.NewServer() + pb.RegisterSandboxRelayTunnelServer(aggSrv, aggregator) + go func() { _ = aggSrv.Serve(aggLis) }() + + // 3. Build a validated tenant-relay config pointing at both fakes. + cfg := &Config{ + Gateway: GatewayConfig{ + Address: gwLis.Addr().String(), + Insecure: true, + }, + Service: ServiceConfig{Implementation: "sidecar", Specifier: "instance-1"}, + TenantRelay: TenantRelayConfig{ + Enabled: true, + Tenant: "acme", + Aggregator: AggregatorDialConfig{ + Address: aggLis.Addr().String(), + Insecure: true, + }, + }, + } + if err := cfg.Validate(); err != nil { + t.Fatalf("validate cfg: %v", err) + } + + relay, err := NewTenantRelay(cfg) + if err != nil { + t.Fatalf("NewTenantRelay: %v", err) + } + // Inject dialers that dial the in-process fakes over plaintext gRPC. + relay.gatewayDialer = func(_ context.Context) (pb.AetherGatewayClient, func() error, error) { + conn, derr := grpc.NewClient(gwLis.Addr().String(), grpc.WithTransportCredentials(insecure.NewCredentials())) + if derr != nil { + return nil, nil, derr + } + return pb.NewAetherGatewayClient(conn), conn.Close, nil + } + relay.tunnelDialer = func(_ context.Context) (pb.SandboxRelayTunnelClient, func() error, error) { + conn, derr := grpc.NewClient(aggLis.Addr().String(), grpc.WithTransportCredentials(insecure.NewCredentials())) + if derr != nil { + return nil, nil, derr + } + return pb.NewSandboxRelayTunnelClient(conn), conn.Close, nil + } + + ctx, cancel := context.WithCancel(context.Background()) + runErr := make(chan error, 1) + go func() { runErr <- relay.Run(ctx) }() + + t.Cleanup(func() { + cancel() + aggSrv.GracefulStop() + _ = aggLis.Close() + gwSrv.GracefulStop() + _ = gwLis.Close() + }) + + return &tenantRelayHarness{ + t: t, + gateway: gateway, + gatewaySrv: gwSrv, + gatewayLis: gwLis, + aggregator: aggregator, + relay: relay, + runErr: runErr, + cancel: cancel, + } +} + +// awaitGatewayStream blocks until the fake gateway records a new accepted +// stream and returns it. +func (h *tenantRelayHarness) awaitGatewayStream() *fakeGatewayStream { + h.t.Helper() + select { + case s := <-h.gateway.streamCh: + return s + case <-time.After(15 * time.Second): + // Generous: a passing test returns the instant the runner dials, so this + // only bounds a genuinely stuck run. 3s was too tight under CI's 2-core + // -race load (the runner goroutine can be starved past a few seconds), + // which made this flake in the -race unit job. + h.t.Fatalf("timed out waiting for gateway stream") + return nil + } +} + +// awaitGatewayMessages polls the gateway stream until it has observed at least +// n messages or the deadline elapses. +func awaitGatewayMessages(t *testing.T, s *fakeGatewayStream, n int) []*pb.UpstreamMessage { + t.Helper() + deadline := time.Now().Add(3 * time.Second) + for time.Now().Before(deadline) { + snap := s.snapshot() + if len(snap) >= n { + return snap + } + time.Sleep(20 * time.Millisecond) + } + return s.snapshot() +} + +// ============================================================================= +// Tests +// ============================================================================= + +// TestTenantRelay_ForwardsInitVerbatim is the key regression vs relay.go: the +// provider's InitConnection must reach the tenant gateway UNTOUCHED — the +// resume_session_id is preserved AND the Service identity is unchanged (no +// rewrite to the sidecar's own Service identity, no resume discard). +func TestTenantRelay_ForwardsInitVerbatim(t *testing.T) { + up := []*pb.TunnelFrame{ + {F: &pb.TunnelFrame_Up{Up: providerInit()}}, + } + h := newTenantRelayHarness(t, up) + + gwStream := h.awaitGatewayStream() + snap := awaitGatewayMessages(t, gwStream, 1) + if len(snap) < 1 { + t.Fatalf("expected the gateway to receive the forwarded init, saw %d messages", len(snap)) + } + + initMsg, ok := snap[0].GetPayload().(*pb.UpstreamMessage_Init) + if !ok { + t.Fatalf("first gateway message is %T, expected Init", snap[0].GetPayload()) + } + + // Dedicated assertion 1: resume_session_id preserved verbatim (relay.go + // discards this; tenant-relay must not). + if got := initMsg.Init.GetResumeSessionId(); got != "resume-xyz" { + t.Fatalf("resume_session_id = %q; want \"resume-xyz\" (must be forwarded verbatim, not discarded)", got) + } + + // Dedicated assertion 2: the Service identity is unchanged (relay.go + // rewrites every init to the sidecar's own Service identity; tenant-relay + // must not touch it). + svc, ok := initMsg.Init.GetClientType().(*pb.InitConnection_Service) + if !ok { + t.Fatalf("forwarded init client_type is %T, expected the provider's Service identity", initMsg.Init.GetClientType()) + } + if svc.Service.GetImplementation() != "sandbox-provider" || svc.Service.GetSpecifier() != "pod-7" { + t.Fatalf("forwarded init identity = %s/%s; want sandbox-provider/pod-7 (must not be rewritten)", + svc.Service.GetImplementation(), svc.Service.GetSpecifier()) + } + + // And the announced tenant reached the aggregator hello. + if got := h.aggregator.helloTenant(); got != "acme" { + t.Fatalf("aggregator hello tenant = %q; want \"acme\"", got) + } +} + +// TestTenantRelay_BidirectionalForwarding confirms subsequent up frames reach +// the gateway and down frames from the gateway reach the aggregator tunnel. +func TestTenantRelay_BidirectionalForwarding(t *testing.T) { + // Init frame followed by two additional up frames the gateway should see. + send := func(topic string) *pb.TunnelFrame { + return &pb.TunnelFrame{F: &pb.TunnelFrame_Up{Up: &pb.UpstreamMessage{ + Payload: &pb.UpstreamMessage_Send{Send: &pb.SendMessage{ + TargetTopic: topic, + Payload: []byte("hello"), + MessageType: pb.MessageType_OPAQUE, + }}, + }}} + } + up := []*pb.TunnelFrame{ + {F: &pb.TunnelFrame_Up{Up: providerInit()}}, + send("ag.ws.impl.spec-1"), + send("ag.ws.impl.spec-2"), + } + h := newTenantRelayHarness(t, up) + + gwStream := h.awaitGatewayStream() + + // Init + the two SendMessage frames must all reach the gateway. + snap := awaitGatewayMessages(t, gwStream, 3) + if len(snap) < 3 { + t.Fatalf("expected 3 forwarded messages, saw %d", len(snap)) + } + var sends []string + for _, m := range snap { + if s, ok := m.GetPayload().(*pb.UpstreamMessage_Send); ok { + sends = append(sends, s.Send.GetTargetTopic()) + } + } + if len(sends) != 2 || sends[0] != "ag.ws.impl.spec-1" || sends[1] != "ag.ws.impl.spec-2" { + t.Fatalf("forwarded SendMessage topics = %v; want [ag.ws.impl.spec-1 ag.ws.impl.spec-2]", sends) + } + + // Now push a downstream envelope from the gateway; it must reach the + // aggregator tunnel as a down frame. (The fakeGateway already sent a + // ConnectionAck on accept, so we expect at least that plus our error.) + if err := gwStream.server.Send(&pb.DownstreamMessage{ + Payload: &pb.DownstreamMessage_Error{ + Error: &pb.ErrorResponse{Code: "DOWN_TEST", Message: "from gateway"}, + }, + }); err != nil { + t.Fatalf("gateway send downstream: %v", err) + } + + deadline := time.Now().Add(3 * time.Second) + var sawErr bool + for time.Now().Before(deadline) { + for _, d := range h.aggregator.snapshotDowns() { + if e, ok := d.GetPayload().(*pb.DownstreamMessage_Error); ok && e.Error.GetCode() == "DOWN_TEST" { + sawErr = true + } + } + if sawErr { + break + } + time.Sleep(20 * time.Millisecond) + } + if !sawErr { + t.Fatalf("downstream envelope from gateway never reached the aggregator tunnel") + } +} + +// TestTenantRelay_CleanTeardownOnCtxCancel confirms Run returns promptly and +// without panic when ctx is cancelled (the runner's shutdown path). The pumps' +// gateway/tunnel Recv calls surface a gRPC Canceled status rather than a clean +// EOF — mirroring relaySession.run, which special-cases only io.EOF — so the +// surface returns a Canceled error. We assert it is exactly that and that the +// teardown is bounded. +func TestTenantRelay_CleanTeardownOnCtxCancel(t *testing.T) { + up := []*pb.TunnelFrame{ + {F: &pb.TunnelFrame_Up{Up: providerInit()}}, + } + h := newTenantRelayHarness(t, up) + + // Wait until the init has been spliced through to the gateway. + gwStream := h.awaitGatewayStream() + _ = awaitGatewayMessages(t, gwStream, 1) + + // Cancel ctx to drive a clean shutdown; Run should return promptly. + h.cancel() + + select { + case err := <-h.runErr: + if err != nil && !errors.Is(err, context.Canceled) && status.Code(err) != codes.Canceled { + t.Fatalf("Run returned unexpected error on ctx-cancel teardown: %v", err) + } + case <-time.After(5 * time.Second): + t.Fatalf("Run did not return within teardown deadline") + } +} + +// TestTenantRelay_TunnelEOFReturnsNil drives the EOF→nil path: the aggregator +// pushes the provider init and then returns from Tunnel (eofAfterSend), +// half-closing the tunnel. The relay's pumpUp observes io.EOF on tunnel.Recv() +// and runPumps maps EOF→nil, so Run returns nil with ctx still live. +func TestTenantRelay_TunnelEOFReturnsNil(t *testing.T) { + agg := newFakeAggregator([]*pb.TunnelFrame{ + {F: &pb.TunnelFrame_Up{Up: providerInit()}}, + }) + agg.eofAfterSend = true + h := newTenantRelayHarnessAgg(t, agg) + + // The init still reaches the gateway before the tunnel half-closes. + gwStream := h.awaitGatewayStream() + _ = awaitGatewayMessages(t, gwStream, 1) + + select { + case err := <-h.runErr: + if err != nil { + t.Fatalf("Run returned %v on clean tunnel EOF; want nil (EOF→nil)", err) + } + case <-time.After(5 * time.Second): + t.Fatalf("Run did not return after tunnel EOF") + } +} diff --git a/server/internal/proxysidecar/terminator.go b/server/internal/proxysidecar/terminator.go index 8fa0ea9..f6c3829 100644 --- a/server/internal/proxysidecar/terminator.go +++ b/server/internal/proxysidecar/terminator.go @@ -151,7 +151,11 @@ func (t *Terminator) Run(ctx context.Context) error { t.RegisterHandlers(t.runtime.Client(), t.runtime.Transport()) - go t.runtime.runConnectionLoop(ctx) + go func() { + if err := t.runtime.runConnectionLoop(ctx); err != nil { + log.Warn().Err(err).Msg("terminator: gateway connection loop ended with error") + } + }() log.Info(). Str("gateway", t.cfg.Gateway.Address). @@ -609,16 +613,16 @@ func (t *Terminator) dispatchStreamingAndRespond(ctx context.Context, req *pb.Pr } // RegisterHandlers installs the terminator's inbound dispatcher hooks on the -// supplied ServiceClient. This is the work `Run` performs after building its -// own runtime; composite-mode callers reuse a shared runtime and call this -// directly so the same connection can serve both terminator and relay -// surfaces. +// supplied BaseClient (backed by either a ServiceClient or an AgentClient). +// This is the work `Run` performs after building its own runtime; +// composite-mode callers reuse a shared runtime and call this directly so the +// same connection can serve both terminator and relay surfaces. // // The supplied transport ships outbound proxy/tunnel envelopes (responses, // tunnel acks, etc.) upstream. Composite-mode callers may wrap the transport // to add side-effects (e.g. routing to multiple consumers) but production -// terminators pass the runtime's bare ServiceClient transport. -func (t *Terminator) RegisterHandlers(client *aether.ServiceClient, transport tunnelTransport) { +// terminators pass the runtime's bare BaseClient transport. +func (t *Terminator) RegisterHandlers(client *aether.BaseClient, transport tunnelTransport) { client.OnMessage(func(msgCtx context.Context, msg *aether.Message) error { // Plain SendMessage delivery path — terminators don't expect // peer-to-peer messages, so this is a fall-through log. diff --git a/server/internal/proxysidecar/tunnel_transport.go b/server/internal/proxysidecar/tunnel_transport.go index 0b9d75a..aa5c755 100644 --- a/server/internal/proxysidecar/tunnel_transport.go +++ b/server/internal/proxysidecar/tunnel_transport.go @@ -34,7 +34,11 @@ const sendUpstreamTimeout = 30 * time.Second // - ProxyHttpBodyChunk (fin=true) → PriorityResponseHeader (terminal) // - ProxyHttpBodyChunk (other) → PriorityResponseChunk type serviceClientTransport struct { - client *aether.ServiceClient + // client is the runtime's identity-agnostic BaseClient (backed by either + // a ServiceClient or an AgentClient). All Send* methods call BaseClient + // methods, so the transport is unaffected by which identity kind the + // runtime selected. + client *aether.BaseClient } // SendTunnelData ships a TunnelData frame upstream. TunnelData carries the diff --git a/server/internal/router/badger_router.go b/server/internal/router/badger_router.go index fdd2c4e..51d6f04 100644 --- a/server/internal/router/badger_router.go +++ b/server/internal/router/badger_router.go @@ -5,6 +5,7 @@ import ( "encoding/binary" "fmt" "sync" + "time" "github.com/dgraph-io/badger/v4" "github.com/scitrera/aether/internal/lite" @@ -49,8 +50,38 @@ type BadgerRouter struct { // exclusiveLocks tracks which (topic, consumerName) pairs already have an // active exclusive subscriber. The stored value is always struct{}{}. exclusiveLocks sync.Map // key: topic+"\x00"+consumerName + + // messageTTL bounds how long a published message is retained before Badger + // expires it (native per-entry TTL, reclaimed by Badger's value-log GC). It + // caps otherwise-unbounded topic-log growth and the size of any cold-start / + // full-replay burst, while comfortably exceeding the realistic reconnect gap + // so a resuming consumer still catches up. The SEQUENCE COUNTER (seq:{topic}) + // carries NO TTL, so numbering is never rewound. Zero disables expiry (retain + // forever). Set at construction; treat as immutable after use. + messageTTL time.Duration + + // offsetTTL bounds how long a per-consumer offset key is retained, refreshed + // on every save. Decoupled from messageTTL and deliberately generous (default + // 7d): the offset is a tiny 8-byte key, and keeping it well past the message + // window lets a consumer that was away a while still RESUME (replaying + // whatever messages remain) rather than cold-starting — while an orphaned + // per-window offset (its browser tab closed; a reopened tab gets a fresh + // window id) still expires instead of accumulating unboundedly as windows + // churn. Zero disables expiry (retain forever). Immutable after use. + offsetTTL time.Duration } +// defaultMessageRetentionTTL is the default per-message retention. 24h caps +// growth at roughly a day of traffic per topic while dwarfing the real reconnect +// gap (seconds–minutes). Override via SetMessageRetentionTTL before first use. +const defaultMessageRetentionTTL = 24 * time.Hour + +// defaultOffsetRetentionTTL is the default per-consumer offset retention. Much +// more generous than message retention (offsets are tiny and worth keeping so a +// consumer away for days still resumes), but still bounded so orphaned per-window +// offsets don't grow without limit. Override via SetOffsetRetentionTTL. +const defaultOffsetRetentionTTL = 7 * 24 * time.Hour + // msgWithSeq bundles a message payload with its Badger sequence number so that // the drain goroutine can persist the exact offset of each processed message. type msgWithSeq struct { @@ -85,7 +116,29 @@ func NewBadgerRouterWithBufferSize(db *badger.DB, bufferSize int) *BadgerRouter db: db, subscriberBufferSize: bufferSize, subs: make(map[string][]*subscriber), + messageTTL: defaultMessageRetentionTTL, + offsetTTL: defaultOffsetRetentionTTL, + } +} + +// SetMessageRetentionTTL overrides the per-message retention TTL. A value <= 0 +// disables expiry (messages retained forever). Call once at startup before the +// router serves traffic; not safe to change concurrently with Publish. +func (r *BadgerRouter) SetMessageRetentionTTL(d time.Duration) { + if d < 0 { + d = 0 } + r.messageTTL = d +} + +// SetOffsetRetentionTTL overrides the per-consumer offset retention TTL. A value +// <= 0 disables expiry (offsets retained forever). Call once at startup before +// the router serves traffic; not safe to change concurrently with saveOffset. +func (r *BadgerRouter) SetOffsetRetentionTTL(d time.Duration) { + if d < 0 { + d = 0 + } + r.offsetTTL = d } // Close shuts down all active subscribers. @@ -139,24 +192,54 @@ func (r *BadgerRouter) Publish(_ context.Context, topic string, payload []byte) return nil } +// startPolicy selects where a subscription begins reading when it is created. +type startPolicy int + +const ( + // startResume resumes from the named consumer's committed offset; a cold + // consumer (no committed offset) — and every anonymous subscriber — starts + // at sequence 0 and replays the entire retained log. + startResume startPolicy = iota + // startTail starts at the current tail and never replays, ignoring any + // committed offset. + startTail + // startResumeOrTail resumes from the named consumer's committed offset when + // one exists, and otherwise starts at the current tail (NOT sequence 0). This + // is the right default for shared/broadcast lanes a client re-subscribes on + // every connect: first connect gets no history dump, reconnect replays only + // the gap since the last committed offset. + startResumeOrTail +) + // Subscribe creates a subscription with full replay from the consumer's last // persisted offset (or the beginning of the log if none exists). // The consumerName is derived from the handler address (not persisted), so // replay always starts from sequence 0 for anonymous subscribers. func (r *BadgerRouter) Subscribe(topic string, handler func([]byte)) (func(), error) { - return r.subscribe(topic, "", handler, false) + return r.subscribe(topic, "", handler, startResume) } // SubscribeExclusive creates a named exclusive subscription with replay. // Only one active subscriber per (topic, consumerName) is permitted. func (r *BadgerRouter) SubscribeExclusive(topic string, consumerName string, handler func([]byte)) (func(), error) { - return r.subscribe(topic, consumerName, handler, false) + return r.subscribe(topic, consumerName, handler, startResume) } // SubscribeExclusiveFromNow creates a named exclusive subscription that starts // from the current write position, skipping all previously stored messages. func (r *BadgerRouter) SubscribeExclusiveFromNow(topic string, consumerName string, handler func([]byte)) (func(), error) { - return r.subscribe(topic, consumerName, handler, true) + return r.subscribe(topic, consumerName, handler, startTail) +} + +// SubscribeExclusiveResumeOrTail creates a named exclusive subscription that +// resumes from the consumer's last committed offset, or — when no offset has +// been committed yet (a brand-new consumer) — starts at the current tail instead +// of replaying the whole topic from sequence 0. Use it for shared/broadcast +// lanes a client re-subscribes on every (re)connect so a first connect gets no +// history dump and a reconnect replays only the gap. Contrast SubscribeExclusive +// (cold → full replay from 0) and SubscribeExclusiveFromNow (always tail). +func (r *BadgerRouter) SubscribeExclusiveResumeOrTail(topic string, consumerName string, handler func([]byte)) (func(), error) { + return r.subscribe(topic, consumerName, handler, startResumeOrTail) } // SubscribeExclusiveFromTimestamp creates an exclusive subscription with full replay @@ -167,7 +250,7 @@ func (r *BadgerRouter) SubscribeExclusiveFromNow(topic string, consumerName stri // already returns all messages since the log start for new consumers — a superset // of timestamp-based replay. The trigger message is guaranteed to be in that replay. func (r *BadgerRouter) SubscribeExclusiveFromTimestamp(topic string, consumerName string, _ int64, handler func([]byte)) (func(), error) { - return r.subscribe(topic, consumerName, handler, false) + return r.subscribe(topic, consumerName, handler, startResume) } // -------------------------------------------------------------------------- @@ -175,7 +258,7 @@ func (r *BadgerRouter) SubscribeExclusiveFromTimestamp(topic string, consumerNam // -------------------------------------------------------------------------- // subscribe is the shared implementation for all three public Subscribe variants. -func (r *BadgerRouter) subscribe(topic, consumerName string, handler func([]byte), fromNow bool) (func(), error) { +func (r *BadgerRouter) subscribe(topic, consumerName string, handler func([]byte), policy startPolicy) (func(), error) { exclusive := consumerName != "" if exclusive { @@ -184,32 +267,67 @@ func (r *BadgerRouter) subscribe(topic, consumerName string, handler func([]byte return nil, fmt.Errorf("badger_router: exclusive consumer %q already active on topic %q", consumerName, topic) } } + // releaseLock undoes the exclusive lock reservation on any early-return error + // path (before the drain goroutine / unsub takes ownership of it). + releaseLock := func() { + if exclusive { + r.exclusiveLocks.Delete(topic + "\x00" + consumerName) + } + } // Determine replay start sequence before registering in live fan-out so we // don't miss any messages published concurrently during replay. var startSeq uint64 - if fromNow { + switch policy { + case startTail: // Start after the current tail; no replay. cur, err := r.currentSequence(topic) if err != nil { - if exclusive { - r.exclusiveLocks.Delete(topic + "\x00" + consumerName) - } + releaseLock() return nil, fmt.Errorf("badger_router: read sequence for %q: %w", topic, err) } startSeq = cur - } else if exclusive && consumerName != "" { - // Resume from persisted offset. - last, err := r.loadOffset(topic, consumerName) - if err != nil { - if exclusive { - r.exclusiveLocks.Delete(topic + "\x00" + consumerName) + case startResumeOrTail: + if exclusive { + // Resume from the committed offset if one exists; otherwise start at + // the current tail (a cold consumer skips the retained backlog rather + // than replaying from sequence 0). + last, ok, err := r.loadOffsetOK(topic, consumerName) + if err != nil { + releaseLock() + return nil, fmt.Errorf("badger_router: load offset for %q/%q: %w", topic, consumerName, err) + } + if ok { + startSeq = last // warm: replay only the gap (from last+1 below) + } else { + cur, cerr := r.currentSequence(topic) + if cerr != nil { + releaseLock() + return nil, fmt.Errorf("badger_router: read sequence for %q: %w", topic, cerr) + } + startSeq = cur // cold: start at tail + } + } else { + // Anonymous resume-or-tail has no offset to resume from → start at tail. + cur, err := r.currentSequence(topic) + if err != nil { + releaseLock() + return nil, fmt.Errorf("badger_router: read sequence for %q: %w", topic, err) } - return nil, fmt.Errorf("badger_router: load offset for %q/%q: %w", topic, consumerName, err) + startSeq = cur } - startSeq = last // replay from last+1 below + default: // startResume + if exclusive { + // Resume from persisted offset (cold → 0 → full replay). + last, err := r.loadOffset(topic, consumerName) + if err != nil { + releaseLock() + return nil, fmt.Errorf("badger_router: load offset for %q/%q: %w", topic, consumerName, err) + } + startSeq = last // replay from last+1 below + } + // For anonymous Subscribe, startSeq stays 0 → replay from beginning. } - // For anonymous Subscribe, startSeq stays 0 → replay from beginning. s := &subscriber{ handler: handler, @@ -240,6 +358,23 @@ func (r *BadgerRouter) subscribe(topic, consumerName string, handler func([]byte // Tell drain to skip any live messages that were already delivered by replay. s.replayedUpTo = replayedUpTo + // Persist the consumer offset for the replayed range. drain saves the offset + // per live message, but replay (the reconnect catch-up path) did not — so a + // named consumer that caught up via replay never committed its progress, and + // those messages re-replayed on every reconnect. This is especially harmful + // for messages that only ever reach a consumer via replay: Publish drops from + // the live fan-out channel (non-blocking) when it is full, but still persists + // to badger, so a slow consumer's messages are delivered only via replay and, + // without this save, re-shed on every reconnect forever. Commit the highest + // replayed sequence so the next reconnect resumes from head. Named consumers + // only (anonymous subscribers, consumerName=="", don't track offsets). + if consumerName != "" && replayedUpTo > startSeq { + if err := r.saveOffset(topic, consumerName, replayedUpTo); err != nil { + logging.Logger.Error().Err(err).Str("topic", topic).Str("consumer", consumerName). + Msg("badger_router: failed to save consumer offset after replay") + } + } + // Start drain goroutine. go r.drain(s, topic) @@ -373,7 +508,12 @@ func (r *BadgerRouter) appendMessage(topic string, payload []byte) (uint64, erro return err } - // Write message. + // Write message. Carries a TTL so Badger natively expires old + // messages and bounds topic-log growth; the seq counter (above) has + // no TTL so sequence numbering is never rewound. + if r.messageTTL > 0 { + return txn.SetEntry(badger.NewEntry(messageKey(topic, seq), payload).WithTTL(r.messageTTL)) + } return txn.Set(messageKey(topic, seq), payload) }) if lastErr == nil { @@ -411,7 +551,17 @@ func (r *BadgerRouter) currentSequence(topic string) (uint64, error) { // loadOffset returns the last-read sequence number for a named consumer. // Returns 0 if no offset has been persisted yet. func (r *BadgerRouter) loadOffset(topic, consumerName string) (uint64, error) { + off, _, err := r.loadOffsetOK(topic, consumerName) + return off, err +} + +// loadOffsetOK returns the last-read sequence number for a named consumer and +// whether a committed offset actually exists. The bool distinguishes "no offset +// persisted yet" (found=false) from a genuinely committed offset of 0, which +// callers need to decide between full replay and tail on a cold consumer. +func (r *BadgerRouter) loadOffsetOK(topic, consumerName string) (uint64, bool, error) { var off uint64 + var found bool err := r.db.View(func(txn *badger.Txn) error { item, err := txn.Get(offsetKey(topic, consumerName)) if err == badger.ErrKeyNotFound { @@ -420,6 +570,7 @@ func (r *BadgerRouter) loadOffset(topic, consumerName string) (uint64, error) { if err != nil { return err } + found = true return item.Value(func(val []byte) error { if len(val) == 8 { off = binary.BigEndian.Uint64(val) @@ -427,16 +578,45 @@ func (r *BadgerRouter) loadOffset(topic, consumerName string) (uint64, error) { return nil }) }) - return off, err + return off, found, err } // saveOffset persists seq as the consumer's last-read offset for topic. +// +// Retries on badger.ErrConflict: the offset key is written from the consumer's +// drain goroutine (per live message) and once from the replay catch-up path, and +// under a connection storm / heavy concurrent publish+commit load the optimistic +// txn can conflict. Without the retry the save silently fails (logged, non-fatal) +// and the offset stalls, so the consumer re-replays the same backlog on the next +// reconnect and re-sheds it under gateway backpressure. Mirrors appendMessage's +// bounded ErrConflict retry (sub-millisecond txns; no backoff needed). +// +// The offset key carries offsetTTL (generous by default — 7d — and decoupled +// from message retention), refreshed on every save. An active consumer rewrites +// its offset as messages flow, so the TTL keeps resetting and the key never +// expires while the consumer is alive; an orphaned per-window consumer (its +// browser tab closed — a reopened tab gets a fresh window id from sessionStorage +// anyway) stops refreshing and its offset eventually expires, so offset keys +// don't accumulate unboundedly as windows churn. func (r *BadgerRouter) saveOffset(topic, consumerName string, seq uint64) error { buf := make([]byte, 8) binary.BigEndian.PutUint64(buf, seq) - return r.db.Update(func(txn *badger.Txn) error { - return txn.Set(offsetKey(topic, consumerName), buf) - }) + var lastErr error + for attempt := 0; attempt < appendMessageMaxRetries; attempt++ { + lastErr = r.db.Update(func(txn *badger.Txn) error { + if r.offsetTTL > 0 { + return txn.SetEntry(badger.NewEntry(offsetKey(topic, consumerName), buf).WithTTL(r.offsetTTL)) + } + return txn.Set(offsetKey(topic, consumerName), buf) + }) + if lastErr == nil { + return nil + } + if lastErr != badger.ErrConflict { + return lastErr + } + } + return lastErr } // replay calls handler for every message in the range (startSeq, currentSeq]. diff --git a/server/internal/router/badger_router_test.go b/server/internal/router/badger_router_test.go index f888f8a..9abb8d8 100644 --- a/server/internal/router/badger_router_test.go +++ b/server/internal/router/badger_router_test.go @@ -243,3 +243,296 @@ func TestBadgerRouter_SubscribeFromNow(t *testing.T) { t.Errorf("expected %q, got %q", "after1", got[0]) } } + +// TestBadgerRouter_ReplayPersistsOffset verifies that catching up via the replay +// path commits the consumer offset — so a named consumer that reconnects does not +// re-replay (and, in production, re-shed) the same historical backlog forever. +// This is the specific failure mode for messages that only ever reach a consumer +// via replay: Publish drops from the live fan-out channel when full but still +// persists to badger, so a slow consumer's messages are served via replay, and +// without an offset save they re-deliver on every reconnect. +func TestBadgerRouter_ReplayPersistsOffset(t *testing.T) { + db := openTestDB(t) + r := NewBadgerRouter(db) + defer r.Close() + ctx := context.Background() + + // Publish 3 messages before any subscriber exists — they land in badger only + // (no live consumer), so the first subscribe serves them via replay, not drain. + for _, m := range []string{"a", "b", "c"} { + if err := r.Publish(ctx, "t", []byte(m)); err != nil { + t.Fatalf("Publish(%q) error = %v", m, err) + } + } + + var mu sync.Mutex + var first []string + unsub1, err := r.SubscribeExclusive("t", "c1", func(p []byte) { + mu.Lock() + first = append(first, string(p)) + mu.Unlock() + }) + if err != nil { + t.Fatalf("first SubscribeExclusive() error = %v", err) + } + time.Sleep(200 * time.Millisecond) + mu.Lock() + n1 := len(first) + mu.Unlock() + if n1 != 3 { + t.Fatalf("first subscribe: expected 3 replayed, got %d", n1) + } + unsub1() + + // Second subscribe with the SAME consumer name + db: replay must have persisted + // the offset, so there is nothing left to re-replay. + var second []string + unsub2, err := r.SubscribeExclusive("t", "c1", func(p []byte) { + mu.Lock() + second = append(second, string(p)) + mu.Unlock() + }) + if err != nil { + t.Fatalf("second SubscribeExclusive() error = %v", err) + } + defer unsub2() + time.Sleep(200 * time.Millisecond) + mu.Lock() + n2 := len(second) + got2 := append([]string(nil), second...) + mu.Unlock() + if n2 != 0 { + t.Fatalf("second subscribe: expected 0 re-replayed (offset persisted by replay), got %d: %v", n2, got2) + } +} + +// TestBadgerRouter_ResumeOrTail verifies the resume-or-tail start policy: a +// cold consumer (no committed offset) starts at the tail and does NOT replay the +// retained backlog, while a reconnecting consumer resumes from its committed +// offset and replays only the gap — the fix for shared/broadcast lanes that a +// client re-subscribes on every connect (they previously re-dumped full history). +func TestBadgerRouter_ResumeOrTail(t *testing.T) { + db := openTestDB(t) + r := NewBadgerRouter(db) + defer r.Close() + ctx := context.Background() + + // Retained backlog published before anyone subscribes. + for _, m := range []string{"old1", "old2", "old3"} { + if err := r.Publish(ctx, "t", []byte(m)); err != nil { + t.Fatalf("Publish(%q): %v", m, err) + } + } + + var mu sync.Mutex + var got1 []string + unsub1, err := r.SubscribeExclusiveResumeOrTail("t", "c1", func(p []byte) { + mu.Lock() + got1 = append(got1, string(p)) + mu.Unlock() + }) + if err != nil { + t.Fatalf("cold SubscribeExclusiveResumeOrTail: %v", err) + } + time.Sleep(150 * time.Millisecond) + mu.Lock() + coldReplayed := len(got1) + mu.Unlock() + if coldReplayed != 0 { + t.Fatalf("cold resume-or-tail should start at tail (0 backlog replayed), got %d: %v", coldReplayed, got1) + } + + // Live messages after subscribe are delivered and advance the offset via drain. + for _, m := range []string{"live1", "live2"} { + if err := r.Publish(ctx, "t", []byte(m)); err != nil { + t.Fatalf("Publish(%q): %v", m, err) + } + } + time.Sleep(150 * time.Millisecond) + mu.Lock() + got1Copy := append([]string(nil), got1...) + mu.Unlock() + if len(got1Copy) != 2 || got1Copy[0] != "live1" || got1Copy[1] != "live2" { + t.Fatalf("expected live delivery [live1 live2], got %v", got1Copy) + } + unsub1() + + // A message published while the consumer is away — the gap to catch up on. + if err := r.Publish(ctx, "t", []byte("gap1")); err != nil { + t.Fatalf("Publish(gap1): %v", err) + } + + // Reconnect: committed offset exists → replay only the gap, not the old + // backlog and not the already-seen live messages. + var got2 []string + unsub2, err := r.SubscribeExclusiveResumeOrTail("t", "c1", func(p []byte) { + mu.Lock() + got2 = append(got2, string(p)) + mu.Unlock() + }) + if err != nil { + t.Fatalf("warm SubscribeExclusiveResumeOrTail: %v", err) + } + defer unsub2() + time.Sleep(150 * time.Millisecond) + mu.Lock() + got2Copy := append([]string(nil), got2...) + mu.Unlock() + if len(got2Copy) != 1 || got2Copy[0] != "gap1" { + t.Fatalf("reconnect should replay only the gap [gap1], got %v", got2Copy) + } +} + +// TestBadgerRouter_MessageRetentionTTL verifies messages expire after the +// retention TTL (bounding topic-log growth / replay-burst size) while the +// sequence counter is preserved (no TTL), so numbering never rewinds. +func TestBadgerRouter_MessageRetentionTTL(t *testing.T) { + if testing.Short() { + t.Skip("skipping timing-based TTL test in -short mode") + } + db := openTestDB(t) + r := NewBadgerRouter(db) + r.SetMessageRetentionTTL(1 * time.Second) // Badger TTL granularity is seconds + defer r.Close() + ctx := context.Background() + + // Retained-only messages (no live subscriber). + for _, m := range []string{"a", "b"} { + if err := r.Publish(ctx, "t", []byte(m)); err != nil { + t.Fatalf("Publish(%q): %v", m, err) + } + } + + // Wait past the second-truncated expiry boundary. + time.Sleep(2500 * time.Millisecond) + + // A fresh full-replay subscriber sees nothing — the backlog expired. + var mu sync.Mutex + var got []string + unsub, err := r.Subscribe("t", func(p []byte) { + mu.Lock() + got = append(got, string(p)) + mu.Unlock() + }) + if err != nil { + t.Fatalf("Subscribe: %v", err) + } + defer unsub() + time.Sleep(150 * time.Millisecond) + mu.Lock() + replayed := append([]string(nil), got...) + mu.Unlock() + if len(replayed) != 0 { + t.Fatalf("expired backlog should not replay, got %v", replayed) + } + + // Sequence numbering is preserved across expiry: the next publish is seq 3 + // and is delivered live. + if err := r.Publish(ctx, "t", []byte("c")); err != nil { + t.Fatalf("Publish(c): %v", err) + } + time.Sleep(150 * time.Millisecond) + mu.Lock() + live := append([]string(nil), got...) + mu.Unlock() + if len(live) != 1 || live[0] != "c" { + t.Fatalf("expected live delivery of [c], got %v", live) + } + seq, err := r.currentSequence("t") + if err != nil { + t.Fatalf("currentSequence: %v", err) + } + if seq != 3 { + t.Fatalf("sequence counter should be preserved at 3 despite expiry, got %d", seq) + } +} + +// TestBadgerRouter_OffsetKeyExpiresWhenOrphaned verifies the per-consumer offset +// key carries the retention TTL so an orphaned consumer's offset (e.g. a browser +// window whose tab closed) is reaped instead of accumulating forever, while an +// offset re-saved before expiry has its TTL refreshed and survives. +func TestBadgerRouter_OffsetKeyExpiresWhenOrphaned(t *testing.T) { + if testing.Short() { + t.Skip("skipping timing-based TTL test in -short mode") + } + db := openTestDB(t) + r := NewBadgerRouter(db) + r.SetOffsetRetentionTTL(1 * time.Second) // Badger TTL granularity is seconds + defer r.Close() + + if err := r.saveOffset("t", "orphan", 5); err != nil { + t.Fatalf("saveOffset: %v", err) + } + if _, ok, err := r.loadOffsetOK("t", "orphan"); err != nil || !ok { + t.Fatalf("offset should be present right after save (ok=%v err=%v)", ok, err) + } + + // A consumer that keeps committing refreshes the TTL and survives. + if err := r.saveOffset("t", "active", 1); err != nil { + t.Fatalf("saveOffset(active): %v", err) + } + time.Sleep(600 * time.Millisecond) + if err := r.saveOffset("t", "active", 2); err != nil { // refresh before expiry + t.Fatalf("saveOffset(active refresh): %v", err) + } + + // Wait past the orphan's expiry boundary. + time.Sleep(2500 * time.Millisecond) + + if _, ok, err := r.loadOffsetOK("t", "orphan"); err != nil { + t.Fatalf("loadOffsetOK(orphan): %v", err) + } else if ok { + t.Fatalf("orphaned offset should have expired, but it is still present") + } + // The active consumer's last refresh was ~2.5s ago (> 1s TTL), so it also + // expires once it stops committing — confirming reaping is driven purely by + // staleness, not by which key it is. + if _, ok, err := r.loadOffsetOK("t", "active"); err != nil { + t.Fatalf("loadOffsetOK(active): %v", err) + } else if ok { + t.Fatalf("stale 'active' offset should also have expired after it stopped refreshing") + } +} + +// TestBadgerRouter_SaveOffsetConcurrentNoConflictError verifies saveOffset +// retries on optimistic-concurrency conflict: many concurrent commits to the +// same offset key (drain saving per message + replay's catch-up save under a +// connection storm) must all succeed with no "Transaction Conflict" error. +// Without the retry a conflicting save fails, the offset stalls, and the +// consumer re-replays + re-sheds the same backlog on the next reconnect. +func TestBadgerRouter_SaveOffsetConcurrentNoConflictError(t *testing.T) { + db := openTestDB(t) + r := NewBadgerRouter(db) + defer r.Close() + + const n = 100 + var wg sync.WaitGroup + start := make(chan struct{}) + errs := make(chan error, n) + for i := 0; i < n; i++ { + wg.Add(1) + go func(seq uint64) { + defer wg.Done() + <-start + if err := r.saveOffset("hot", "c1", seq); err != nil { + errs <- err + } + }(uint64(i + 1)) + } + close(start) + wg.Wait() + close(errs) + for err := range errs { + t.Errorf("concurrent saveOffset failed (retry-on-conflict missing?): %v", err) + } + + // A final read-back must return one of the written offsets (a valid commit), + // proving the key is not left in a wedged/uncommitted state. + off, err := r.loadOffset("hot", "c1") + if err != nil { + t.Fatalf("loadOffset after concurrent saves: %v", err) + } + if off < 1 || off > n { + t.Fatalf("final offset %d out of written range [1,%d]", off, n) + } +} diff --git a/server/internal/router/jetstream_router.go b/server/internal/router/jetstream_router.go index ecbe674..b2d2642 100644 --- a/server/internal/router/jetstream_router.go +++ b/server/internal/router/jetstream_router.go @@ -181,6 +181,15 @@ func (r *JetStreamRouter) SubscribeExclusiveFromNow(topic string, consumerName s return r.subscribeDurable(topic, consumerName, jetstream.DeliverNewPolicy, 0, handler) } +// SubscribeExclusiveResumeOrTail creates a durable consumer that resumes from +// its stored position when it already exists, and otherwise starts at the tail. +// DeliverNewPolicy applies only when the durable is first created (a cold +// consumer → tail); NATS ignores DeliverPolicy on an existing durable, so a +// warm consumer resumes from its committed offset — exactly resume-or-tail. +func (r *JetStreamRouter) SubscribeExclusiveResumeOrTail(topic string, consumerName string, handler func([]byte)) (func(), error) { + return r.subscribeDurable(topic, consumerName, jetstream.DeliverNewPolicy, 0, handler) +} + // SubscribeExclusiveFromTimestamp creates a durable consumer. When // startTimestampMs > 0, new consumers start from messages at or after that // unix-millisecond timestamp; existing durable consumers resume from their diff --git a/server/internal/router/router.go b/server/internal/router/router.go index 7d75230..336644b 100644 --- a/server/internal/router/router.go +++ b/server/internal/router/router.go @@ -257,6 +257,15 @@ func (r *Router) SubscribeExclusiveFromNow(topic string, consumerName string, ha return r.subscriptions.SubscribeWithOptions(topic, handler, opts) } +// SubscribeExclusiveResumeOrTail creates an exclusive subscription that resumes +// from the consumer's committed offset, or — when none exists yet — starts at the +// tail (.Next()) rather than replaying the whole stream. A RabbitMQ Streams named +// consumer already resolves this way (stored offset wins, else .Next()), which is +// exactly the timestamp-hint=0 path, so route through it. +func (r *Router) SubscribeExclusiveResumeOrTail(topic string, consumerName string, handler func([]byte)) (func(), error) { + return r.SubscribeExclusiveFromTimestamp(topic, consumerName, 0, handler) +} + // SubscribeExclusiveFromTimestamp creates an exclusive subscription with offset tracking // (OffsetResume) and a unix-millisecond timestamp hint. When no stored offset exists for // consumerName, the subscription starts from the given timestamp instead of the default diff --git a/server/internal/secrets/secrets.go b/server/internal/secrets/secrets.go index e2fb961..10b5f30 100644 --- a/server/internal/secrets/secrets.go +++ b/server/internal/secrets/secrets.go @@ -154,21 +154,65 @@ func ParseAccessLevel(name string) (int, error) { } } -// CreateInitialToken creates an admin bootstrap API token in the database and -// seeds an ACL rule granting the token's principal the specified access level -// on all workspaces. +// ACLGrantFunc seeds an ACL rule. It mirrors the GrantAccess method shared by +// the Postgres acl.Service and the native-sqlite ACL store, but returns only an +// error so a single helper can drive either backend (their methods return +// different rule types). The caller is responsible for adapting the concrete +// store's GrantAccess into this signature. +type ACLGrantFunc func(ctx context.Context, principalType, principalID, resourceType, resourceID string, accessLevel int, grantedBy, reason string, expiresAt *time.Time) error + +// CreateInitialToken creates an admin bootstrap API token in the PostgreSQL +// database and seeds an ACL rule granting the token's principal the specified +// access level on all workspaces. +// // principalType should be a valid models.PrincipalType (e.g., models.PrincipalUser). +// +// This is the PostgreSQL convenience wrapper; it builds the Postgres-backed +// token store and ACL service from db and delegates to +// CreateInitialTokenWithStore. For backend-agnostic callers (e.g. aetherlite's +// SQLite stores), use CreateInitialTokenWithStore directly. func CreateInitialToken(ctx context.Context, db *sql.DB, cfg *config.Config, tokenName string, principalType models.PrincipalType, accessLevel int) (string, error) { if db == nil { return "", fmt.Errorf("database connection required to create initial token") } + store := auth.NewAPITokenStore(db) + + // Seed an ACL rule granting the _system principal the requested access level + // on all workspaces. The token inherits permissions from its creator (_system), + // so this rule enables the token holder to perform admin operations. + aclService := acl.NewService(db, acl.SystemPrincipal) + defer aclService.Close() + + grant := func(ctx context.Context, principalType, principalID, resourceType, resourceID string, accessLevel int, grantedBy, reason string, expiresAt *time.Time) error { + _, err := aclService.GrantAccess(ctx, principalType, principalID, resourceType, resourceID, accessLevel, grantedBy, reason, expiresAt) + return err + } + + return CreateInitialTokenWithStore(ctx, store, grant, cfg, tokenName, principalType, accessLevel) +} + +// CreateInitialTokenWithStore creates an admin bootstrap API token using the +// provided APITokenStore and (optionally) seeds an ACL rule via grant. +// +// This is the backend-agnostic core used by init-secrets so the same logic +// works against PostgreSQL (full gateway) or native SQLite (aetherlite single +// and cluster). The token's creator is the _system principal; the ACL grant +// gives _system the requested access level on all workspaces, which the token +// inherits. +// +// grant may be nil to skip ACL seeding (e.g. when the backend already seeds +// adequate default policies and no elevated bootstrap grant is required). +func CreateInitialTokenWithStore(ctx context.Context, store auth.APITokenStore, grant ACLGrantFunc, cfg *config.Config, tokenName string, principalType models.PrincipalType, accessLevel int) (string, error) { + if store == nil { + return "", fmt.Errorf("token store required to create initial token") + } + // Ensure HMAC key is initialised so the token hash is consistent - if cfg.Auth.TokenHMACKey != "" { + if cfg != nil && cfg.Auth.TokenHMACKey != "" { crypto.InitTokenHMAC([]byte(cfg.Auth.TokenHMACKey)) } - store := auth.NewAPITokenStore(db) result, err := store.CreateToken( ctx, tokenName, @@ -182,25 +226,21 @@ func CreateInitialToken(ctx context.Context, db *sql.DB, cfg *config.Config, tok return "", fmt.Errorf("creating initial token: %w", err) } - // Seed an ACL rule granting the _system principal the requested access level - // on all workspaces. The token inherits permissions from its creator (_system), - // so this rule enables the token holder to perform admin operations. - aclService := acl.NewService(db, acl.SystemPrincipal) - defer aclService.Close() - - _, err = aclService.GrantAccess( - ctx, - acl.PrincipalTypeForModel(principalType), // principal type (lowercase for DB convention) - acl.SystemPrincipal, // principal ID (_system — the token's creator) - acl.ResourceTypeWorkspace, // resource type - acl.WildcardAnyResource, // resource ID (* = all workspaces) - accessLevel, // access level - acl.SystemPrincipal, // granted by - "bootstrap ACL for _system principal via init-secrets", // reason - nil, // no expiration - ) - if err != nil { - return "", fmt.Errorf("creating ACL rule for initial token: %w", err) + if grant != nil { + err = grant( + ctx, + acl.PrincipalTypeForModel(principalType), // principal type (lowercase for DB convention) + acl.SystemPrincipal, // principal ID (_system — the token's creator) + acl.ResourceTypeWorkspace, // resource type + acl.WildcardAnyResource, // resource ID (* = all workspaces) + accessLevel, // access level + acl.SystemPrincipal, // granted by + "bootstrap ACL for _system principal via init-secrets", // reason + nil, // no expiration + ) + if err != nil { + return "", fmt.Errorf("creating ACL rule for initial token: %w", err) + } } return result.Token, nil diff --git a/server/internal/state/spike_tenant_relay_resume_test.go b/server/internal/state/spike_tenant_relay_resume_test.go new file mode 100644 index 0000000..ccd7030 --- /dev/null +++ b/server/internal/state/spike_tenant_relay_resume_test.go @@ -0,0 +1,93 @@ +package state + +// SPIKE (feasibility): sandbox-provider tenant-relay redesign ("Direction A"). +// +// Goal of this file: prove that Aether session resume is keyed on the PRINCIPAL +// IDENTITY (lock:), independent of the TCP peer / cert / connection, in +// EVERY session backend Aether ships. That is the property the two-hop relay +// tunnel depends on: when a tenant-relay restarts (or a different provider replica +// reconnects), a fresh connection presenting the same identity + resume_session_id +// must resume the existing session rather than be rejected or fork a new one. +// +// All three backends key the lock by identity.String(): +// - Redis (full): lock: (session.go) +// - Badger (lite/embed): lockKey(identity.String()) (badger_session.go) +// - JetStream (lite/NATS):encodeKVKey(identity.String()) (jetstream_session.go) +// +// This test exercises the real AcquireOrResumeLock of each against in-process +// backends (miniredis, embedded badger, embedded NATS/JetStream). + +import ( + "context" + "testing" + + "github.com/scitrera/aether/pkg/models" +) + +// resumeLocker is the slice of the session-registry surface the relay design +// relies on. All three production registries implement it. +type resumeLocker interface { + AcquireOrResumeLock(ctx context.Context, identity models.Identity, sessionID, resumeSessionID string, forceTakeoverThresholdMs int64, meta ConnectMeta) (ConnectResult, error) +} + +func TestSpike_SessionResumeIsIdentityKeyedAcrossBackends(t *testing.T) { + backends := []struct { + name string + make func(t *testing.T) resumeLocker + }{ + {"badger_embedded_lite", func(t *testing.T) resumeLocker { return newBadgerSessionRegistry(t) }}, + {"redis_full", func(t *testing.T) resumeLocker { reg, _ := newTestSessionRegistry(t); return reg }}, + {"jetstream_nats_lite", func(t *testing.T) resumeLocker { return newTestJetStreamSession(t) }}, + } + + for _, b := range backends { + t.Run(b.name, func(t *testing.T) { + ctx := context.Background() + reg := b.make(t) + + // The identity the tenant-relay registers on behalf of the remote + // provider. Within a tenant gateway this is THE sandbox-provider + // service principal; tenant-binding comes from which CA signed the + // relay's cert + which gateway it dialed, not from this specifier. + id := models.Identity{ + Type: models.PrincipalService, + Implementation: "sandbox-provider", + Specifier: "pod-7", + } + const session = "sess-A" + const forceThresholdMs = int64(50) // fresh locks have multi-second TTL + + // Connection #1: relay instance A dials the gateway (fresh acquire). + r1, err := reg.AcquireOrResumeLock(ctx, id, session, "", forceThresholdMs, ConnectMeta{}) + if err != nil { + t.Fatalf("acquire #1: %v", err) + } + if !r1.Acquired || r1.Resumed || r1.ReconnectionCount != 0 { + t.Fatalf("conn#1 = %+v, want Acquired && !Resumed && count==0", r1) + } + + // A DIFFERENT session cannot steal the still-held lock. Proves the + // lock is real and identity-scoped (not first-write-wins). + steal, err := reg.AcquireOrResumeLock(ctx, id, "sess-OTHER", "", forceThresholdMs, ConnectMeta{}) + if err != nil { + t.Fatalf("steal attempt: %v", err) + } + if steal.Acquired { + t.Fatalf("a different session stole an active lock: %+v", steal) + } + + // Connection #2: relay instance A died ungracefully (lock NOT + // released). A FRESH connection — new TCP peer, possibly a different + // relay/provider replica — reconnects with the same identity and + // resume_session_id. Must resume (peer-independent), bumping the + // reconnection count rather than forking a new session. + r2, err := reg.AcquireOrResumeLock(ctx, id, session, session, forceThresholdMs, ConnectMeta{}) + if err != nil { + t.Fatalf("resume #2: %v", err) + } + if !r2.Acquired || !r2.Resumed || r2.ReconnectionCount != 1 { + t.Fatalf("conn#2 = %+v, want Acquired && Resumed && count==1", r2) + } + }) + } +} diff --git a/server/internal/storage/acl/conformance_test.go b/server/internal/storage/acl/conformance_test.go index 0a6550e..437d9b5 100644 --- a/server/internal/storage/acl/conformance_test.go +++ b/server/internal/storage/acl/conformance_test.go @@ -111,6 +111,11 @@ func TestStoreConformance(t *testing.T) { defer cleanup() runAuthorityRequestLifecycle(t, store) }) + t.Run("GetRuleByIDAndRevokeByID", func(t *testing.T) { + store, _, cleanup := b.factory(t) + defer cleanup() + runGetRuleByIDAndRevokeByID(t, store) + }) }) } } @@ -162,6 +167,74 @@ func runGrantRevokeRoundTrip(t *testing.T, store acl.Store) { } } +// runGetRuleByIDAndRevokeByID verifies: +// 1. GetRuleByID retrieves the correct rule by its UUID. +// 2. GetRuleByID returns ErrRuleNotFound for an unknown UUID. +// 3. A rule can be revoked using the principal+resource resolved from GetRuleByID +// (the same path the gateway REVOKE-by-rule_id handler takes). +func runGetRuleByIDAndRevokeByID(t *testing.T, store acl.Store) { + t.Helper() + ctx := context.Background() + + principalID := uniqueID(t, "user") + resourceID := uniqueID(t, "ws") + + // Grant a rule and capture its UUID. + rule, err := store.GrantAccess(ctx, + acl.PrincipalTypeUser, principalID, + acl.ResourceTypeWorkspace, resourceID, + acl.AccessRead, "_system", "revoke-by-id conformance", nil, + ) + if err != nil { + t.Fatalf("GrantAccess: %v", err) + } + if rule.RuleID == "" { + t.Fatal("GrantAccess returned rule with empty RuleID") + } + + // GetRuleByID should find the rule and return matching fields. + got, err := store.GetRuleByID(ctx, rule.RuleID) + if err != nil { + t.Fatalf("GetRuleByID: %v", err) + } + if got.RuleID != rule.RuleID { + t.Errorf("GetRuleByID RuleID mismatch: want %q got %q", rule.RuleID, got.RuleID) + } + if got.PrincipalType != acl.PrincipalTypeUser { + t.Errorf("GetRuleByID PrincipalType: want %q got %q", acl.PrincipalTypeUser, got.PrincipalType) + } + if got.PrincipalID != principalID { + t.Errorf("GetRuleByID PrincipalID: want %q got %q", principalID, got.PrincipalID) + } + if got.ResourceType != acl.ResourceTypeWorkspace { + t.Errorf("GetRuleByID ResourceType: want %q got %q", acl.ResourceTypeWorkspace, got.ResourceType) + } + if got.ResourceID != resourceID { + t.Errorf("GetRuleByID ResourceID: want %q got %q", resourceID, got.ResourceID) + } + + // Not-found case: a random UUID should return ErrRuleNotFound. + _, err = store.GetRuleByID(ctx, "00000000-0000-0000-0000-000000000000") + if err != acl.ErrRuleNotFound { + t.Fatalf("GetRuleByID unknown UUID: want ErrRuleNotFound, got %v", err) + } + + // Revoke using the composite key resolved from GetRuleByID (gateway REVOKE-by-rule_id path). + if err := store.RevokeAccess(ctx, got.PrincipalType, got.PrincipalID, got.ResourceType, got.ResourceID); err != nil { + t.Fatalf("RevokeAccess after GetRuleByID: %v", err) + } + + // Rule must be gone: GetRuleByID and GetRule should both return ErrRuleNotFound. + _, err = store.GetRuleByID(ctx, rule.RuleID) + if err != acl.ErrRuleNotFound { + t.Fatalf("GetRuleByID after revoke: want ErrRuleNotFound, got %v", err) + } + _, err = store.GetRule(ctx, acl.PrincipalTypeUser, principalID, acl.ResourceTypeWorkspace, resourceID) + if err != acl.ErrRuleNotFound { + t.Fatalf("GetRule after revoke: want ErrRuleNotFound, got %v", err) + } +} + // runAccessCheck verifies that a grant at level N permits CheckAccess at // level <= N and denies CheckAccess at level > N. func runAccessCheck(t *testing.T, store acl.Store) { diff --git a/server/internal/storage/acl/jetstream_authority_store_test.go b/server/internal/storage/acl/jetstream_authority_store_test.go index 6278aa8..78e5306 100644 --- a/server/internal/storage/acl/jetstream_authority_store_test.go +++ b/server/internal/storage/acl/jetstream_authority_store_test.go @@ -363,6 +363,11 @@ func (f *fakeInnerStore) GetRule(ctx context.Context, principalType, principalID }, nil } +func (f *fakeInnerStore) GetRuleByID(ctx context.Context, ruleID string) (*aclstore.Rule, error) { + f.getRuleCalls++ + return &aclstore.Rule{RuleID: ruleID}, nil +} + func (f *fakeInnerStore) ListRules(ctx context.Context, filter aclstore.RuleFilter) ([]*aclstore.Rule, error) { return nil, nil } @@ -404,6 +409,53 @@ func (f *fakeInnerStore) Close() error { return nil } func (f *fakeInnerStore) SetPrefixIndex(p aclstore.PrefixLookup) {} +// ----- Roles & groups (passthrough stubs) ----- + +func (f *fakeInnerStore) CreateGroup(ctx context.Context, name, description, createdBy string, metadata map[string]interface{}) (*aclstore.Group, error) { + return &aclstore.Group{GroupName: name}, nil +} +func (f *fakeInnerStore) DeleteGroup(ctx context.Context, name string) error { return nil } +func (f *fakeInnerStore) GetGroup(ctx context.Context, name string) (*aclstore.Group, error) { + return &aclstore.Group{GroupName: name}, nil +} +func (f *fakeInnerStore) ListGroups(ctx context.Context) ([]*aclstore.Group, error) { return nil, nil } +func (f *fakeInnerStore) CreateRole(ctx context.Context, name, description, createdBy string, metadata map[string]interface{}) (*aclstore.Role, error) { + return &aclstore.Role{RoleName: name}, nil +} +func (f *fakeInnerStore) DeleteRole(ctx context.Context, name string) error { return nil } +func (f *fakeInnerStore) GetRole(ctx context.Context, name string) (*aclstore.Role, error) { + return &aclstore.Role{RoleName: name}, nil +} +func (f *fakeInnerStore) ListRoles(ctx context.Context) ([]*aclstore.Role, error) { return nil, nil } +func (f *fakeInnerStore) AddGroupMember(ctx context.Context, groupName, memberType, memberID, grantedBy string, expiresAt *time.Time) (*aclstore.GroupMember, error) { + return &aclstore.GroupMember{GroupName: groupName, MemberType: memberType, MemberID: memberID}, nil +} +func (f *fakeInnerStore) RemoveGroupMember(ctx context.Context, groupName, memberType, memberID string) error { + return nil +} +func (f *fakeInnerStore) ListGroupMembers(ctx context.Context, groupName string) ([]*aclstore.GroupMember, error) { + return nil, nil +} +func (f *fakeInnerStore) ListPrincipalGroups(ctx context.Context, memberType, memberID string) ([]*aclstore.GroupMember, error) { + return nil, nil +} +func (f *fakeInnerStore) AssignRole(ctx context.Context, roleName, assigneeType, assigneeID, grantedBy string, expiresAt *time.Time) (*aclstore.RoleAssignment, error) { + return &aclstore.RoleAssignment{RoleName: roleName, AssigneeType: assigneeType, AssigneeID: assigneeID}, nil +} +func (f *fakeInnerStore) UnassignRole(ctx context.Context, roleName, assigneeType, assigneeID string) error { + return nil +} +func (f *fakeInnerStore) ListRoleAssignments(ctx context.Context, roleName string) ([]*aclstore.RoleAssignment, error) { + return nil, nil +} +func (f *fakeInnerStore) ListPrincipalRoles(ctx context.Context, assigneeType, assigneeID string) ([]*aclstore.RoleAssignment, error) { + return nil, nil +} +func (f *fakeInnerStore) ExplainAccess(ctx context.Context, principalType, principalID, resourceType, resourceID string, requiredLevel int, callerType, callerID string) (*aclstore.AccessExplanation, error) { + return &aclstore.AccessExplanation{}, nil +} +func (f *fakeInnerStore) CleanupExpiredMemberships(ctx context.Context) (int64, error) { return 0, nil } + // Compile-time assertion: the fake inner satisfies aclstore.Store. If a method // is added to Store and not added here, the test build fails — which is the // signal to update the decorator/test in lockstep. diff --git a/server/internal/storage/acl/sqlite/groups_roles.go b/server/internal/storage/acl/sqlite/groups_roles.go new file mode 100644 index 0000000..8b8ddb6 --- /dev/null +++ b/server/internal/storage/acl/sqlite/groups_roles.go @@ -0,0 +1,606 @@ +package sqlite + +import ( + "context" + "database/sql" + "encoding/json" + "fmt" + "strings" + "time" + + "github.com/google/uuid" + "github.com/scitrera/aether/internal/acl" + "github.com/scitrera/aether/internal/logging" + aclstore "github.com/scitrera/aether/internal/storage/acl" +) + +// This file mirrors internal/acl/groups_roles.go (the Postgres path) using the +// native SQLite dialect: ? placeholders, ISO-8601 TEXT timestamps via +// formatTime, Go-generated UUIDs, and direct *casbin.SyncedEnforcer access. + +func memberSubject(memberType, memberID string) string { return memberType + ":" + memberID } + +// ========================================================================= +// Group CRUD +// ========================================================================= + +func (s *Store) CreateGroup(ctx context.Context, name, description, createdBy string, metadata map[string]interface{}) (*aclstore.Group, error) { + if name == "" { + return nil, fmt.Errorf("group name is required") + } + g := &aclstore.Group{ + GroupID: uuid.NewString(), + GroupName: name, + Description: description, + CreatedBy: createdBy, + CreatedAt: time.Now(), + Metadata: metadata, + } + meta, err := marshalMetadata(metadata) + if err != nil { + return nil, err + } + _, err = s.db.ExecContext(ctx, ` + INSERT INTO acl_groups (group_id, group_name, description, created_by, created_at, metadata) + VALUES (?, ?, ?, ?, ?, ?) + `, g.GroupID, g.GroupName, nullString(description), nullString(createdBy), formatTime(g.CreatedAt), meta) + if err != nil { + if isUniqueViolation(err) { + return nil, aclstore.ErrGroupExists + } + return nil, fmt.Errorf("failed to create group: %w", err) + } + return g, nil +} + +func (s *Store) DeleteGroup(ctx context.Context, name string) error { + res, err := s.db.ExecContext(ctx, `DELETE FROM acl_groups WHERE group_name = ?`, name) + if err != nil { + return fmt.Errorf("failed to delete group: %w", err) + } + if n, _ := res.RowsAffected(); n == 0 { + return aclstore.ErrGroupNotFound + } + s.reloadEnforcer("delete group") + return nil +} + +func (s *Store) GetGroup(ctx context.Context, name string) (*aclstore.Group, error) { + row := s.db.QueryRowContext(ctx, ` + SELECT group_id, group_name, description, created_by, created_at, metadata + FROM acl_groups WHERE group_name = ? + `, name) + return scanGroupRow(row) +} + +func (s *Store) ListGroups(ctx context.Context) ([]*aclstore.Group, error) { + rows, err := s.db.QueryContext(ctx, ` + SELECT group_id, group_name, description, created_by, created_at, metadata + FROM acl_groups ORDER BY group_name + `) + if err != nil { + return nil, fmt.Errorf("failed to list groups: %w", err) + } + defer rows.Close() + var out []*aclstore.Group + for rows.Next() { + g, err := scanGroupRow(rows) + if err != nil { + return nil, err + } + out = append(out, g) + } + return out, rows.Err() +} + +// ========================================================================= +// Role CRUD +// ========================================================================= + +func (s *Store) CreateRole(ctx context.Context, name, description, createdBy string, metadata map[string]interface{}) (*aclstore.Role, error) { + if name == "" { + return nil, fmt.Errorf("role name is required") + } + r := &aclstore.Role{ + RoleID: uuid.NewString(), + RoleName: name, + Description: description, + CreatedBy: createdBy, + CreatedAt: time.Now(), + Metadata: metadata, + } + meta, err := marshalMetadata(metadata) + if err != nil { + return nil, err + } + _, err = s.db.ExecContext(ctx, ` + INSERT INTO acl_roles (role_id, role_name, description, created_by, created_at, metadata) + VALUES (?, ?, ?, ?, ?, ?) + `, r.RoleID, r.RoleName, nullString(description), nullString(createdBy), formatTime(r.CreatedAt), meta) + if err != nil { + if isUniqueViolation(err) { + return nil, aclstore.ErrRoleExists + } + return nil, fmt.Errorf("failed to create role: %w", err) + } + return r, nil +} + +func (s *Store) DeleteRole(ctx context.Context, name string) error { + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return fmt.Errorf("failed to begin tx: %w", err) + } + defer tx.Rollback() + + res, err := tx.ExecContext(ctx, `DELETE FROM acl_roles WHERE role_name = ?`, name) + if err != nil { + return fmt.Errorf("failed to delete role: %w", err) + } + if n, _ := res.RowsAffected(); n == 0 { + return aclstore.ErrRoleNotFound + } + if _, err := tx.ExecContext(ctx, `DELETE FROM acl_rules WHERE principal_type = ? AND principal_id = ?`, + aclstore.PrincipalTypeRole, name); err != nil { + return fmt.Errorf("failed to delete role permissions: %w", err) + } + if err := tx.Commit(); err != nil { + return fmt.Errorf("failed to commit role delete: %w", err) + } + s.reloadEnforcer("delete role") + return nil +} + +func (s *Store) GetRole(ctx context.Context, name string) (*aclstore.Role, error) { + row := s.db.QueryRowContext(ctx, ` + SELECT role_id, role_name, description, created_by, created_at, metadata + FROM acl_roles WHERE role_name = ? + `, name) + return scanRoleRow(row) +} + +func (s *Store) ListRoles(ctx context.Context) ([]*aclstore.Role, error) { + rows, err := s.db.QueryContext(ctx, ` + SELECT role_id, role_name, description, created_by, created_at, metadata + FROM acl_roles ORDER BY role_name + `) + if err != nil { + return nil, fmt.Errorf("failed to list roles: %w", err) + } + defer rows.Close() + var out []*aclstore.Role + for rows.Next() { + r, err := scanRoleRow(rows) + if err != nil { + return nil, err + } + out = append(out, r) + } + return out, rows.Err() +} + +// ========================================================================= +// Group membership +// ========================================================================= + +func (s *Store) AddGroupMember(ctx context.Context, groupName, memberType, memberID, grantedBy string, expiresAt *time.Time) (*aclstore.GroupMember, error) { + groupID, err := s.groupIDByName(ctx, groupName) + if err != nil { + return nil, err + } + target := aclstore.GroupSubjectPrefix + groupName + source := memberSubject(memberType, memberID) + if err := s.checkNoCycle(source, target); err != nil { + return nil, err + } + + m := &aclstore.GroupMember{ + GroupName: groupName, + MemberType: memberType, + MemberID: memberID, + GrantedBy: grantedBy, + GrantedAt: time.Now(), + ExpiresAt: expiresAt, + } + _, err = s.db.ExecContext(ctx, ` + INSERT INTO acl_group_members (id, group_id, member_type, member_id, granted_by, granted_at, expires_at) + VALUES (?, ?, ?, ?, ?, ?, ?) + ON CONFLICT (group_id, member_type, member_id) + DO UPDATE SET granted_by = excluded.granted_by, granted_at = excluded.granted_at, expires_at = excluded.expires_at + `, uuid.NewString(), groupID, memberType, memberID, nullString(grantedBy), formatTime(m.GrantedAt), nullableTime(expiresAt)) + if err != nil { + return nil, fmt.Errorf("failed to add group member: %w", err) + } + s.addGroupingEdge(source, target) + return m, nil +} + +func (s *Store) RemoveGroupMember(ctx context.Context, groupName, memberType, memberID string) error { + groupID, err := s.groupIDByName(ctx, groupName) + if err != nil { + return err + } + res, err := s.db.ExecContext(ctx, ` + DELETE FROM acl_group_members WHERE group_id = ? AND member_type = ? AND member_id = ? + `, groupID, memberType, memberID) + if err != nil { + return fmt.Errorf("failed to remove group member: %w", err) + } + if n, _ := res.RowsAffected(); n == 0 { + return aclstore.ErrMembershipNotFound + } + s.removeGroupingEdge(memberSubject(memberType, memberID), aclstore.GroupSubjectPrefix+groupName) + return nil +} + +func (s *Store) ListGroupMembers(ctx context.Context, groupName string) ([]*aclstore.GroupMember, error) { + rows, err := s.db.QueryContext(ctx, ` + SELECT m.member_type, m.member_id, m.granted_by, m.granted_at, m.expires_at + FROM acl_group_members m JOIN acl_groups g ON g.group_id = m.group_id + WHERE g.group_name = ? ORDER BY m.member_type, m.member_id + `, groupName) + if err != nil { + return nil, fmt.Errorf("failed to list group members: %w", err) + } + defer rows.Close() + var out []*aclstore.GroupMember + for rows.Next() { + m := &aclstore.GroupMember{GroupName: groupName} + var grantedBy, grantedAt, expiresAt sql.NullString + if err := rows.Scan(&m.MemberType, &m.MemberID, &grantedBy, &grantedAt, &expiresAt); err != nil { + return nil, err + } + m.GrantedBy = grantedBy.String + assignGranted(&m.GrantedAt, grantedAt) + assignExpiry(&m.ExpiresAt, expiresAt) + out = append(out, m) + } + return out, rows.Err() +} + +func (s *Store) ListPrincipalGroups(ctx context.Context, memberType, memberID string) ([]*aclstore.GroupMember, error) { + rows, err := s.db.QueryContext(ctx, ` + SELECT g.group_name, m.granted_by, m.granted_at, m.expires_at + FROM acl_group_members m JOIN acl_groups g ON g.group_id = m.group_id + WHERE m.member_type = ? AND m.member_id = ? ORDER BY g.group_name + `, memberType, memberID) + if err != nil { + return nil, fmt.Errorf("failed to list principal groups: %w", err) + } + defer rows.Close() + var out []*aclstore.GroupMember + for rows.Next() { + m := &aclstore.GroupMember{MemberType: memberType, MemberID: memberID} + var grantedBy, grantedAt, expiresAt sql.NullString + if err := rows.Scan(&m.GroupName, &grantedBy, &grantedAt, &expiresAt); err != nil { + return nil, err + } + m.GrantedBy = grantedBy.String + assignGranted(&m.GrantedAt, grantedAt) + assignExpiry(&m.ExpiresAt, expiresAt) + out = append(out, m) + } + return out, rows.Err() +} + +// ========================================================================= +// Role assignment +// ========================================================================= + +func (s *Store) AssignRole(ctx context.Context, roleName, assigneeType, assigneeID, grantedBy string, expiresAt *time.Time) (*aclstore.RoleAssignment, error) { + roleID, err := s.roleIDByName(ctx, roleName) + if err != nil { + return nil, err + } + target := aclstore.RoleSubjectPrefix + roleName + source := memberSubject(assigneeType, assigneeID) + if err := s.checkNoCycle(source, target); err != nil { + return nil, err + } + + a := &aclstore.RoleAssignment{ + RoleName: roleName, + AssigneeType: assigneeType, + AssigneeID: assigneeID, + GrantedBy: grantedBy, + GrantedAt: time.Now(), + ExpiresAt: expiresAt, + } + _, err = s.db.ExecContext(ctx, ` + INSERT INTO acl_role_assignments (id, role_id, assignee_type, assignee_id, granted_by, granted_at, expires_at) + VALUES (?, ?, ?, ?, ?, ?, ?) + ON CONFLICT (role_id, assignee_type, assignee_id) + DO UPDATE SET granted_by = excluded.granted_by, granted_at = excluded.granted_at, expires_at = excluded.expires_at + `, uuid.NewString(), roleID, assigneeType, assigneeID, nullString(grantedBy), formatTime(a.GrantedAt), nullableTime(expiresAt)) + if err != nil { + return nil, fmt.Errorf("failed to assign role: %w", err) + } + s.addGroupingEdge(source, target) + return a, nil +} + +func (s *Store) UnassignRole(ctx context.Context, roleName, assigneeType, assigneeID string) error { + roleID, err := s.roleIDByName(ctx, roleName) + if err != nil { + return err + } + res, err := s.db.ExecContext(ctx, ` + DELETE FROM acl_role_assignments WHERE role_id = ? AND assignee_type = ? AND assignee_id = ? + `, roleID, assigneeType, assigneeID) + if err != nil { + return fmt.Errorf("failed to unassign role: %w", err) + } + if n, _ := res.RowsAffected(); n == 0 { + return aclstore.ErrAssignmentNotFound + } + s.removeGroupingEdge(memberSubject(assigneeType, assigneeID), aclstore.RoleSubjectPrefix+roleName) + return nil +} + +func (s *Store) ListRoleAssignments(ctx context.Context, roleName string) ([]*aclstore.RoleAssignment, error) { + rows, err := s.db.QueryContext(ctx, ` + SELECT a.assignee_type, a.assignee_id, a.granted_by, a.granted_at, a.expires_at + FROM acl_role_assignments a JOIN acl_roles r ON r.role_id = a.role_id + WHERE r.role_name = ? ORDER BY a.assignee_type, a.assignee_id + `, roleName) + if err != nil { + return nil, fmt.Errorf("failed to list role assignments: %w", err) + } + defer rows.Close() + var out []*aclstore.RoleAssignment + for rows.Next() { + a := &aclstore.RoleAssignment{RoleName: roleName} + var grantedBy, grantedAt, expiresAt sql.NullString + if err := rows.Scan(&a.AssigneeType, &a.AssigneeID, &grantedBy, &grantedAt, &expiresAt); err != nil { + return nil, err + } + a.GrantedBy = grantedBy.String + assignGranted(&a.GrantedAt, grantedAt) + assignExpiry(&a.ExpiresAt, expiresAt) + out = append(out, a) + } + return out, rows.Err() +} + +func (s *Store) ListPrincipalRoles(ctx context.Context, assigneeType, assigneeID string) ([]*aclstore.RoleAssignment, error) { + rows, err := s.db.QueryContext(ctx, ` + SELECT r.role_name, a.granted_by, a.granted_at, a.expires_at + FROM acl_role_assignments a JOIN acl_roles r ON r.role_id = a.role_id + WHERE a.assignee_type = ? AND a.assignee_id = ? ORDER BY r.role_name + `, assigneeType, assigneeID) + if err != nil { + return nil, fmt.Errorf("failed to list principal roles: %w", err) + } + defer rows.Close() + var out []*aclstore.RoleAssignment + for rows.Next() { + a := &aclstore.RoleAssignment{AssigneeType: assigneeType, AssigneeID: assigneeID} + var grantedBy, grantedAt, expiresAt sql.NullString + if err := rows.Scan(&a.RoleName, &grantedBy, &grantedAt, &expiresAt); err != nil { + return nil, err + } + a.GrantedBy = grantedBy.String + assignGranted(&a.GrantedAt, grantedAt) + assignExpiry(&a.ExpiresAt, expiresAt) + out = append(out, a) + } + return out, rows.Err() +} + +// ========================================================================= +// Introspection + cleanup +// ========================================================================= + +func (s *Store) ExplainAccess(ctx context.Context, principalType, principalID, resourceType, resourceID string, requiredLevel int, callerType, callerID string) (*aclstore.AccessExplanation, error) { + resourceType, resourceID, _ = acl.RewriteLegacyPermission(resourceType, resourceID) + self := principalType + ":" + principalID + exp := &aclstore.AccessExplanation{Principal: self, Subjects: []string{self}} + if s.enforcer != nil { + exp.Subjects = subjectSet(s.enforcer, self) + exp.Contributions = acl.GatherContributions(s.enforcer, exp.Subjects, resourceType, resourceID) + exp.Decision = s.evaluateBySubject(principalType, principalID, resourceType, resourceID, requiredLevel) + } + // Audit the introspection: who asked (caller) about whose access (principal). + if s.aclAudit != nil { + s.aclAudit.LogExplain(ctx, callerType, callerID, principalType, principalID, resourceType, resourceID, requiredLevel, exp.Decision) + } + return exp, nil +} + +// CleanupExpiredMemberships deletes expired membership/assignment edges via +// parameterized DELETEs (no stored function in SQLite) and reloads the enforcer. +func (s *Store) CleanupExpiredMemberships(ctx context.Context) (int64, error) { + now := formatTime(time.Now()) + var total int64 + for _, q := range []string{ + `DELETE FROM acl_group_members WHERE expires_at IS NOT NULL AND expires_at <= ?`, + `DELETE FROM acl_role_assignments WHERE expires_at IS NOT NULL AND expires_at <= ?`, + } { + res, err := s.db.ExecContext(ctx, q, now) + if err != nil { + return total, fmt.Errorf("failed to cleanup expired memberships: %w", err) + } + n, _ := res.RowsAffected() + total += n + } + if total > 0 { + s.reloadEnforcer("cleanup expired memberships") + } + return total, nil +} + +// ========================================================================= +// Internal helpers +// ========================================================================= + +func (s *Store) groupIDByName(ctx context.Context, name string) (string, error) { + var id string + err := s.db.QueryRowContext(ctx, `SELECT group_id FROM acl_groups WHERE group_name = ?`, name).Scan(&id) + if err == sql.ErrNoRows { + return "", aclstore.ErrGroupNotFound + } + if err != nil { + return "", fmt.Errorf("failed to look up group: %w", err) + } + return id, nil +} + +func (s *Store) roleIDByName(ctx context.Context, name string) (string, error) { + var id string + err := s.db.QueryRowContext(ctx, `SELECT role_id FROM acl_roles WHERE role_name = ?`, name).Scan(&id) + if err == sql.ErrNoRows { + return "", aclstore.ErrRoleNotFound + } + if err != nil { + return "", fmt.Errorf("failed to look up role: %w", err) + } + return id, nil +} + +func (s *Store) checkNoCycle(source, target string) error { + if source == target { + return aclstore.ErrMembershipCycle + } + if s.enforcer == nil { + return nil + } + reachable, err := s.enforcer.GetImplicitRolesForUser(target) + if err != nil { + return nil + } + for _, r := range reachable { + if r == source { + return aclstore.ErrMembershipCycle + } + } + return nil +} + +func (s *Store) addGroupingEdge(source, target string) { + if s.enforcer == nil { + return + } + if _, err := s.enforcer.AddGroupingPolicy(source, target); err != nil { + logging.Logger.Warn().Err(err).Str("source", source).Str("target", target).Msg("acl sqlite: in-memory AddGroupingPolicy failed; persisted state authoritative") + } +} + +func (s *Store) removeGroupingEdge(source, target string) { + if s.enforcer == nil { + return + } + if _, err := s.enforcer.RemoveGroupingPolicy(source, target); err != nil { + logging.Logger.Warn().Err(err).Str("source", source).Str("target", target).Msg("acl sqlite: in-memory RemoveGroupingPolicy failed; persisted state authoritative") + } +} + +func (s *Store) reloadEnforcer(reason string) { + if s.enforcer == nil { + return + } + if err := s.enforcer.LoadPolicy(); err != nil { + logging.Logger.Warn().Err(err).Str("reason", reason).Msg("acl sqlite: enforcer reload failed; in-memory model may lag DB until next reload") + } +} + +func scanGroupRow(sc interface{ Scan(...interface{}) error }) (*aclstore.Group, error) { + g := &aclstore.Group{} + var description, createdBy, createdAt sql.NullString + var meta []byte + if err := sc.Scan(&g.GroupID, &g.GroupName, &description, &createdBy, &createdAt, &meta); err != nil { + if err == sql.ErrNoRows { + return nil, aclstore.ErrGroupNotFound + } + return nil, fmt.Errorf("failed to scan group: %w", err) + } + g.Description = description.String + g.CreatedBy = createdBy.String + g.CreatedAt = parseTime(createdAt.String) + g.Metadata = unmarshalMetadata(meta) + return g, nil +} + +func scanRoleRow(sc interface{ Scan(...interface{}) error }) (*aclstore.Role, error) { + r := &aclstore.Role{} + var description, createdBy, createdAt sql.NullString + var meta []byte + if err := sc.Scan(&r.RoleID, &r.RoleName, &description, &createdBy, &createdAt, &meta); err != nil { + if err == sql.ErrNoRows { + return nil, aclstore.ErrRoleNotFound + } + return nil, fmt.Errorf("failed to scan role: %w", err) + } + r.Description = description.String + r.CreatedBy = createdBy.String + r.CreatedAt = parseTime(createdAt.String) + r.Metadata = unmarshalMetadata(meta) + return r, nil +} + +// assignExpiry parses an ISO-8601 TEXT expiry column into *time.Time. +func assignExpiry(dst **time.Time, col sql.NullString) { + if col.Valid && col.String != "" { + t := parseTime(col.String) + if !t.IsZero() { + *dst = &t + } + } +} + +// assignGranted parses a granted_at TEXT column (formatTime layout) into dst. +// SQLite stores timestamps as TEXT, so granted_at MUST be scanned as a string and +// parsed here — scanning straight into a time.Time fails with +// "unsupported Scan ... storing driver.Value type string into type *time.Time". +func assignGranted(dst *time.Time, col sql.NullString) { + if col.Valid && col.String != "" { + if t := parseTime(col.String); !t.IsZero() { + *dst = t + } + } +} + +func nullableTime(t *time.Time) interface{} { + if t == nil { + return nil + } + return formatTime(*t) +} + +func marshalMetadata(m map[string]interface{}) (interface{}, error) { + if len(m) == 0 { + return nil, nil + } + b, err := json.Marshal(m) + if err != nil { + return nil, fmt.Errorf("failed to marshal metadata: %w", err) + } + return string(b), nil +} + +func unmarshalMetadata(b []byte) map[string]interface{} { + if len(b) == 0 { + return nil + } + var m map[string]interface{} + if err := json.Unmarshal(b, &m); err != nil { + return nil + } + return m +} + +func nullString(s string) interface{} { + if s == "" { + return nil + } + return s +} + +func isUniqueViolation(err error) bool { + if err == nil { + return false + } + msg := err.Error() + return strings.Contains(msg, "UNIQUE constraint") || + strings.Contains(msg, "unique constraint") || + strings.Contains(msg, "duplicate key") +} diff --git a/server/internal/storage/acl/sqlite/groups_roles_test.go b/server/internal/storage/acl/sqlite/groups_roles_test.go new file mode 100644 index 0000000..8c2825f --- /dev/null +++ b/server/internal/storage/acl/sqlite/groups_roles_test.go @@ -0,0 +1,271 @@ +package sqlite_test + +import ( + "context" + "database/sql" + "errors" + "fmt" + "path/filepath" + "testing" + "time" + + "github.com/google/uuid" + _ "modernc.org/sqlite" + + aclstore "github.com/scitrera/aether/internal/storage/acl" + aclsqlite "github.com/scitrera/aether/internal/storage/acl/sqlite" + "github.com/scitrera/aether/pkg/models" +) + +func newRolesTestStore(t *testing.T) *aclsqlite.Store { + t.Helper() + path := filepath.Join(t.TempDir(), "acl_roles.db") + dsn := fmt.Sprintf("file:%s?_journal_mode=WAL&_busy_timeout=5000", path) + db, err := sql.Open("sqlite", dsn) + if err != nil { + t.Fatalf("sql.Open sqlite: %v", err) + } + db.SetMaxOpenConns(1) + store, err := aclsqlite.New(db, nil, nil, "test-gw") + if err != nil { + t.Fatalf("aclsqlite.New: %v", err) + } + t.Cleanup(func() { _ = store.Close(); _ = db.Close() }) + return store +} + +func user(id string) models.Identity { return models.Identity{Type: models.PrincipalUser, ID: id} } + +// Full stack: role permission + role->group + group membership reaches a user. +func TestSQLiteGroupsRoles_EndToEnd(t *testing.T) { + ctx := context.Background() + s := newRolesTestStore(t) + + if _, err := s.CreateRole(ctx, "wsadmin", "workspace admin", "tester", nil); err != nil { + t.Fatalf("CreateRole: %v", err) + } + // Grant the role MANAGE on workspace:prod (role permission = acl_rules row). + if _, err := s.GrantAccess(ctx, aclstore.PrincipalTypeRole, "wsadmin", aclstore.ResourceTypeWorkspace, "prod", aclstore.AccessManage, "tester", "", nil); err != nil { + t.Fatalf("GrantAccess to role: %v", err) + } + if _, err := s.CreateGroup(ctx, "eng", "engineering", "tester", nil); err != nil { + t.Fatalf("CreateGroup: %v", err) + } + if _, err := s.AssignRole(ctx, "wsadmin", aclstore.PrincipalTypeGroup, "eng", "tester", nil); err != nil { + t.Fatalf("AssignRole to group: %v", err) + } + if _, err := s.AddGroupMember(ctx, "eng", aclstore.PrincipalTypeUser, "alice", "tester", nil); err != nil { + t.Fatalf("AddGroupMember: %v", err) + } + + // alice -> group:eng -> role:wsadmin -> workspace:prod MANAGE + d, err := s.CheckAccess(ctx, user("alice"), aclstore.ResourceTypeWorkspace, "prod", "connect", "prod", uuid.Nil, aclstore.AccessManage) + if err != nil { + t.Fatalf("CheckAccess: %v", err) + } + if d == nil || !d.Allowed || d.EffectiveAccessLevel != aclstore.AccessManage { + t.Fatalf("expected allow@30 via role, got %+v", d) + } + + // A user with no group/role gets no rule match (would fall to fallback). + d2, err := s.CheckAccess(ctx, user("bob"), aclstore.ResourceTypeWorkspace, "prod", "connect", "prod", uuid.Nil, aclstore.AccessManage) + if err != nil { + t.Fatalf("CheckAccess bob: %v", err) + } + if d2 != nil && d2.Allowed && d2.RuleApplied != nil { + t.Fatalf("bob should not have a matching role/group rule, got %+v", d2) + } + + // Reload from DB (adapter g-load path) must preserve the derived access. + if _, err := s.CleanupExpiredMemberships(ctx); err != nil { + t.Fatalf("CleanupExpiredMemberships: %v", err) + } + d3, err := s.CheckAccess(ctx, user("alice"), aclstore.ResourceTypeWorkspace, "prod", "connect", "prod", uuid.Nil, aclstore.AccessManage) + if err != nil || d3 == nil || !d3.Allowed { + t.Fatalf("expected alice still allowed after reload, got %+v err=%v", d3, err) + } + + // Removing the membership revokes the derived access. + if err := s.RemoveGroupMember(ctx, "eng", aclstore.PrincipalTypeUser, "alice"); err != nil { + t.Fatalf("RemoveGroupMember: %v", err) + } + d4, _ := s.CheckAccess(ctx, user("alice"), aclstore.ResourceTypeWorkspace, "prod", "connect", "prod", uuid.Nil, aclstore.AccessManage) + if d4 != nil && d4.Allowed && d4.RuleApplied != nil { + t.Fatalf("alice access should be revoked after membership removal, got %+v", d4) + } +} + +// TestSQLiteGroupsRoles_ListReadsGrantedAt guards the list read-back paths for +// role assignments + group members. SQLite stores granted_at as TEXT, so these +// queries must scan it as a string and parse it — a direct time.Time scan fails +// with "unsupported Scan ... storing driver.Value type string into type +// *time.Time". These paths had no coverage (the other tests use CheckAccess / +// ExplainAccess, which read the in-memory enforcer, not the SQL rows) and shipped +// that bug. +func TestSQLiteGroupsRoles_ListReadsGrantedAt(t *testing.T) { + ctx := context.Background() + s := newRolesTestStore(t) + before := time.Now().Add(-time.Minute) + + if _, err := s.CreateRole(ctx, "wsadmin", "", "tester", nil); err != nil { + t.Fatalf("CreateRole: %v", err) + } + if _, err := s.CreateGroup(ctx, "eng", "", "tester", nil); err != nil { + t.Fatalf("CreateGroup: %v", err) + } + if _, err := s.AssignRole(ctx, "wsadmin", aclstore.PrincipalTypeGroup, "eng", "tester", nil); err != nil { + t.Fatalf("AssignRole: %v", err) + } + if _, err := s.AddGroupMember(ctx, "eng", aclstore.PrincipalTypeUser, "alice", "tester", nil); err != nil { + t.Fatalf("AddGroupMember: %v", err) + } + + assigns, err := s.ListRoleAssignments(ctx, "wsadmin") + if err != nil { + t.Fatalf("ListRoleAssignments: %v", err) + } + if len(assigns) != 1 { + t.Fatalf("want 1 role assignment, got %d", len(assigns)) + } + if assigns[0].GrantedAt.IsZero() || assigns[0].GrantedAt.Before(before) { + t.Fatalf("assignment GrantedAt not parsed from TEXT: %v", assigns[0].GrantedAt) + } + if assigns[0].GrantedBy != "tester" { + t.Fatalf("assignment GrantedBy = %q, want tester", assigns[0].GrantedBy) + } + + members, err := s.ListGroupMembers(ctx, "eng") + if err != nil { + t.Fatalf("ListGroupMembers: %v", err) + } + if len(members) != 1 || members[0].GrantedAt.IsZero() || members[0].GrantedAt.Before(before) { + t.Fatalf("member GrantedAt not parsed from TEXT: %+v", members) + } + + // Principal-oriented reads share the same scan path. + proles, err := s.ListPrincipalRoles(ctx, aclstore.PrincipalTypeGroup, "eng") + if err != nil { + t.Fatalf("ListPrincipalRoles: %v", err) + } + if len(proles) != 1 || proles[0].GrantedAt.IsZero() { + t.Fatalf("principal-role GrantedAt not parsed: %+v", proles) + } + pgroups, err := s.ListPrincipalGroups(ctx, aclstore.PrincipalTypeUser, "alice") + if err != nil { + t.Fatalf("ListPrincipalGroups: %v", err) + } + if len(pgroups) != 1 || pgroups[0].GrantedAt.IsZero() { + t.Fatalf("principal-group GrantedAt not parsed: %+v", pgroups) + } +} + +// ExplainAccess surfaces the resolved subject set, the contributing rules +// (the "why"), and the decision — none of which CheckAccess exposes. +func TestSQLiteGroupsRoles_ExplainAccess(t *testing.T) { + ctx := context.Background() + s := newRolesTestStore(t) + + if _, err := s.CreateRole(ctx, "wsadmin", "", "tester", nil); err != nil { + t.Fatalf("CreateRole: %v", err) + } + if _, err := s.GrantAccess(ctx, aclstore.PrincipalTypeRole, "wsadmin", aclstore.ResourceTypeWorkspace, "prod", aclstore.AccessManage, "tester", "", nil); err != nil { + t.Fatalf("GrantAccess: %v", err) + } + // A weaker direct grant on the same resource — should appear as a + // contribution but not be the winner (additive max picks the role's 30). + if _, err := s.GrantAccess(ctx, aclstore.PrincipalTypeUser, "alice", aclstore.ResourceTypeWorkspace, "prod", aclstore.AccessRead, "tester", "", nil); err != nil { + t.Fatalf("GrantAccess direct: %v", err) + } + if _, err := s.CreateGroup(ctx, "eng", "", "tester", nil); err != nil { + t.Fatalf("CreateGroup: %v", err) + } + if _, err := s.AssignRole(ctx, "wsadmin", aclstore.PrincipalTypeGroup, "eng", "tester", nil); err != nil { + t.Fatalf("AssignRole: %v", err) + } + if _, err := s.AddGroupMember(ctx, "eng", aclstore.PrincipalTypeUser, "alice", "tester", nil); err != nil { + t.Fatalf("AddGroupMember: %v", err) + } + + exp, err := s.ExplainAccess(ctx, aclstore.PrincipalTypeUser, "alice", aclstore.ResourceTypeWorkspace, "prod", aclstore.AccessManage, "admin_api", "test") + if err != nil { + t.Fatalf("ExplainAccess: %v", err) + } + if exp.Principal != "user:alice" { + t.Errorf("principal = %q, want user:alice", exp.Principal) + } + // Subject set must include self + transitive group + role. + want := map[string]bool{"user:alice": false, "group:eng": false, "role:wsadmin": false} + for _, s := range exp.Subjects { + if _, ok := want[s]; ok { + want[s] = true + } + } + for sub, found := range want { + if !found { + t.Errorf("subject %q missing from %v", sub, exp.Subjects) + } + } + // Both the direct READ rule and the role's MANAGE rule should be reported. + var sawRole, sawSelf bool + for _, c := range exp.Contributions { + if c.Subject == "role:wsadmin" && c.AccessLevel == aclstore.AccessManage { + sawRole = true + } + if c.Subject == "user:alice" && c.AccessLevel == aclstore.AccessRead { + sawSelf = true + } + } + if !sawRole || !sawSelf { + t.Errorf("contributions missing role/self rule: %+v", exp.Contributions) + } + // Decision: additive max => MANAGE, allowed. + if exp.Decision == nil || !exp.Decision.Allowed || exp.Decision.EffectiveAccessLevel != aclstore.AccessManage { + t.Errorf("decision = %+v, want allow@30", exp.Decision) + } +} + +func TestSQLiteGroupsRoles_CycleRejected(t *testing.T) { + ctx := context.Background() + s := newRolesTestStore(t) + for _, g := range []string{"a", "b"} { + if _, err := s.CreateGroup(ctx, g, "", "tester", nil); err != nil { + t.Fatalf("CreateGroup %s: %v", g, err) + } + } + // b is a member of a. + if _, err := s.AddGroupMember(ctx, "a", aclstore.PrincipalTypeGroup, "b", "tester", nil); err != nil { + t.Fatalf("AddGroupMember a<-b: %v", err) + } + // Adding a as a member of b would create a cycle a->b->a. + _, err := s.AddGroupMember(ctx, "b", aclstore.PrincipalTypeGroup, "a", "tester", nil) + if !errors.Is(err, aclstore.ErrMembershipCycle) { + t.Fatalf("expected ErrMembershipCycle, got %v", err) + } +} + +func TestSQLiteGroupsRoles_MembershipExpiry(t *testing.T) { + ctx := context.Background() + s := newRolesTestStore(t) + if _, err := s.CreateRole(ctx, "reader", "", "tester", nil); err != nil { + t.Fatalf("CreateRole: %v", err) + } + if _, err := s.GrantAccess(ctx, aclstore.PrincipalTypeRole, "reader", aclstore.ResourceTypeWorkspace, "prod", aclstore.AccessRead, "tester", "", nil); err != nil { + t.Fatalf("GrantAccess: %v", err) + } + past := time.Now().Add(-1 * time.Hour) + if _, err := s.AssignRole(ctx, "reader", aclstore.PrincipalTypeUser, "carol", "tester", &past); err != nil { + t.Fatalf("AssignRole expired: %v", err) + } + n, err := s.CleanupExpiredMemberships(ctx) + if err != nil { + t.Fatalf("CleanupExpiredMemberships: %v", err) + } + if n != 1 { + t.Fatalf("expected 1 expired assignment removed, got %d", n) + } + // After cleanup + reload, carol must not retain role-derived access. + d, _ := s.CheckAccess(ctx, user("carol"), aclstore.ResourceTypeWorkspace, "prod", "connect", "prod", uuid.Nil, aclstore.AccessRead) + if d != nil && d.Allowed && d.RuleApplied != nil { + t.Fatalf("carol access should be gone after expiry cleanup, got %+v", d) + } +} diff --git a/server/internal/storage/acl/sqlite/store.go b/server/internal/storage/acl/sqlite/store.go index 46513f4..aaa0fff 100644 --- a/server/internal/storage/acl/sqlite/store.go +++ b/server/internal/storage/acl/sqlite/store.go @@ -105,6 +105,9 @@ r = sub, obj, act [policy_definition] p = sub, obj, act, expires, rule_id +[role_definition] +g = _, _ + [policy_effect] e = some(where (p.eft == allow)) @@ -135,6 +138,15 @@ const ( // physical file the shared writer targets (audit.db in lite). // - gatewayID: stamped on every ACL-decision audit row. func New(db *sql.DB, sharedAudit audit.EventSink, auditDB *sql.DB, gatewayID string) (*Store, error) { + // Single-writer enforcement on the ACL state handle: without it, concurrent + // writers (e.g. simultaneous authority-grant INSERTs during a connection + // storm) run on separate pool connections and the second gets SQLITE_BUSY in + // WAL mode ("database is locked"). Pinning to one connection serializes all + // acl.db access — the same discipline the tasks/registry/workflow sqlite + // stores already apply. Also this handle is shared with the legacy + // internal/acl.Service (authority grants), so the cap must live on the handle. + db.SetMaxOpenConns(1) + // Run ACL-domain migrations against the state DB. ctx := context.Background() if err := applyMigrations(ctx, db, sqliteaclmigrations.MigrationFS, "sqlite_acl"); err != nil { @@ -1025,6 +1037,24 @@ func (s *Store) GetRule(ctx context.Context, principalType, principalID, resourc return rule, nil } +func (s *Store) GetRuleByID(ctx context.Context, ruleID string) (*aclstore.Rule, error) { + query := ` + SELECT rule_id, principal_type, principal_id, resource_type, resource_id, + access_level, granted_by, granted_at, expires_at, reason + FROM acl_rules + WHERE rule_id = ? + ` + + rule, err := scanACLRule(s.db.QueryRowContext(ctx, query, ruleID)) + if err == sql.ErrNoRows { + return nil, aclstore.ErrRuleNotFound + } + if err != nil { + return nil, fmt.Errorf("failed to get ACL rule by id: %w", err) + } + return rule, nil +} + func (s *Store) ListRules(ctx context.Context, filter aclstore.RuleFilter) ([]*aclstore.Rule, error) { query := ` SELECT rule_id, principal_type, principal_id, resource_type, resource_id, @@ -1296,48 +1326,92 @@ func (s *Store) evaluateAccessNoAudit(ctx context.Context, principal models.Iden return s.applyFallback(ctx, principal, resourceType, requiredLevel) } +// evaluateAccess mirrors internal/acl.(*CasbinEnforcer).EvaluateAccess: the +// principal's own subject plus every group/role it transitively belongs to +// (resolved via the Casbin role manager) form the subject set, and rules +// matching ANY subject are combined additively (highest access level wins). +// With no groups/roles defined the subject set is just the principal, so the +// behavior is identical to the original per-principal evaluation. func (s *Store) evaluateAccess(_ context.Context, principal models.Identity, resourceType, resourceID string, requiredLevel int) (*aclstore.Decision, error) { - principalType := aclstore.PrincipalTypeForModel(principal.Type) - principalID := principal.CanonicalPrincipalID() + return s.evaluateBySubject(aclstore.PrincipalTypeForModel(principal.Type), principal.CanonicalPrincipalID(), resourceType, resourceID, requiredLevel), nil +} +// evaluateBySubject is the string-keyed core of evaluateAccess — see +// internal/acl.(*CasbinEnforcer).EvaluateBySubject. Used by both CheckAccess +// (via evaluateAccess) and ExplainAccess. +func (s *Store) evaluateBySubject(principalType, principalID, resourceType, resourceID string, requiredLevel int) *aclstore.Decision { sub := principalType + ":" + principalID obj := resourceType + ":" + resourceID + subjects := subjectSet(s.enforcer, sub) - // Step 1: Exact principal + exact resource - if decision := findAndEvaluate(s.enforcer, sub, obj, requiredLevel, "Explicit rule"); decision != nil { - return decision, nil + // Step 1: Subject set (self + groups/roles) + exact resource + if decision := findAndEvaluateMulti(s.enforcer, subjects, obj, requiredLevel, "Explicit rule"); decision != nil { + return decision } // Step 2: Wildcard principal + exact resource for _, wSub := range wildcardSubjects(principalType) { if decision := findAndEvaluate(s.enforcer, wSub, obj, requiredLevel, "Wildcard rule"); decision != nil { - return decision, nil + return decision } } - // Step 3: Exact principal + wildcard resource + // Step 3: Subject set + wildcard resource if resourceID != aclstore.WildcardAnyResource { wObj := resourceType + ":" + aclstore.WildcardAnyResource - if decision := findAndEvaluate(s.enforcer, sub, wObj, requiredLevel, "Any-resource rule"); decision != nil { - return decision, nil + if decision := findAndEvaluateMulti(s.enforcer, subjects, wObj, requiredLevel, "Any-resource rule"); decision != nil { + return decision } // Step 4: Wildcard principal + wildcard resource for _, wSub := range wildcardSubjects(principalType) { if decision := findAndEvaluate(s.enforcer, wSub, wObj, requiredLevel, "Wildcard any-resource rule"); decision != nil { - return decision, nil + return decision } } } - // Step 5: Glob-pattern rules - if decision := findGlobMatch(s.enforcer, sub, obj, requiredLevel); decision != nil { - return decision, nil + // Step 5: Glob-pattern rules (any subject in the set) + if decision := findGlobMatch(s.enforcer, subjects, obj, requiredLevel); decision != nil { + return decision } // Step 6: No match — caller applies fallback - return nil, nil + return nil +} + +// subjectSet returns the principal's own subject plus every group/role it +// transitively belongs to. subjects[0] is always the principal's own subject. +func subjectSet(e *casbin.SyncedEnforcer, sub string) []string { + roles, err := e.GetImplicitRolesForUser(sub) + if err != nil || len(roles) == 0 { + return []string{sub} + } + return append([]string{sub}, roles...) +} + +// findAndEvaluateMulti evaluates (subject, obj) for every subject and combines +// additively: the highest non-expired access level wins. When the winning rule +// was granted to a group/role rather than the principal, the reason is +// annotated. subjects[0] is assumed to be the principal's own subject. +func findAndEvaluateMulti(e *casbin.SyncedEnforcer, subjects []string, obj string, requiredLevel int, label string) *aclstore.Decision { + var best *aclstore.Decision + var bestSubject string + for _, s := range subjects { + d := findAndEvaluate(e, s, obj, requiredLevel, label) + if d == nil { + continue + } + if best == nil || d.EffectiveAccessLevel > best.EffectiveAccessLevel { + best = d + bestSubject = s + } + } + if best != nil && len(subjects) > 0 && bestSubject != subjects[0] { + best.Reason = fmt.Sprintf("%s (via %s)", best.Reason, bestSubject) + } + return best } func (s *Store) applyFallback(ctx context.Context, principal models.Identity, resourceType string, requiredLevel int) (*aclstore.Decision, error) { @@ -1423,7 +1497,20 @@ func findAndEvaluate(e *casbin.SyncedEnforcer, sub, obj string, requiredLevel in return decision } -func findGlobMatch(e *casbin.SyncedEnforcer, sub, obj string, requiredLevel int) *aclstore.Decision { +// anyGlobMatch reports whether any of the names matches the glob pattern. +func anyGlobMatch(names []string, pattern string) bool { + for _, n := range names { + if acl.GlobMatch(n, pattern) { + return true + } + } + return false +} + +// findGlobMatch scans all policies once for glob-pattern rules matching any +// subject in the set and the given object. Highest matching level across all +// subjects wins (additive semantics consistent with findAndEvaluateMulti). +func findGlobMatch(e *casbin.SyncedEnforcer, subjects []string, obj string, requiredLevel int) *aclstore.Decision { policies, _ := e.GetPolicy() if len(policies) == 0 { return nil @@ -1439,7 +1526,7 @@ func findGlobMatch(e *casbin.SyncedEnforcer, sub, obj string, requiredLevel int) if !strings.ContainsAny(pSub, "*?") && !strings.ContainsAny(pObj, "*?") { continue } - if !acl.GlobMatch(sub, pSub) || !acl.GlobMatch(obj, pObj) { + if !acl.GlobMatch(obj, pObj) || !anyGlobMatch(subjects, pSub) { continue } if len(p) > pIdxExpires && p[pIdxExpires] != "" { @@ -2015,7 +2102,65 @@ func (a *sqliteAdapter) LoadPolicy(m model.Model) error { _ = persist.LoadPolicyArray([]string{"p", sub, obj, act, exp, ruleID}, m) } - return rows.Err() + if err := rows.Err(); err != nil { + return err + } + + // Load role/group membership as Casbin grouping (g) edges. Additive on top + // of the per-principal rules; absence of the groups/roles tables must not + // break core rule evaluation, so failures here are logged and skipped. + a.loadGroupingPolicies(m) + + return nil +} + +// loadGroupingPolicies loads acl_group_members and acl_role_assignments as +// Casbin g-rules: g(":", "group:") and +// g(":", "role:"). Expired edges are +// excluded. Best-effort: a query error (e.g. missing table) is logged and +// skipped so the per-principal rules remain authoritative. +func (a *sqliteAdapter) loadGroupingPolicies(m model.Model) { + now := formatTime(time.Now()) + + memberQuery := ` + SELECT m.member_type, m.member_id, g.group_name + FROM acl_group_members m + JOIN acl_groups g ON g.group_id = m.group_id + WHERE m.expires_at IS NULL OR m.expires_at > ? + ` + if rows, err := a.db.Query(memberQuery, now); err != nil { + logging.Logger.Warn().Err(err).Msg("acl sqlite: failed to load group memberships; group-derived access disabled until next reload") + } else { + for rows.Next() { + var memberType, memberID, groupName string + if err := rows.Scan(&memberType, &memberID, &groupName); err != nil { + logging.Logger.Warn().Err(err).Msg("acl sqlite: failed to scan group membership row") + continue + } + _ = persist.LoadPolicyArray([]string{"g", memberType + ":" + memberID, acl.GroupSubjectPrefix + groupName}, m) + } + rows.Close() + } + + assignQuery := ` + SELECT a.assignee_type, a.assignee_id, r.role_name + FROM acl_role_assignments a + JOIN acl_roles r ON r.role_id = a.role_id + WHERE a.expires_at IS NULL OR a.expires_at > ? + ` + if rows, err := a.db.Query(assignQuery, now); err != nil { + logging.Logger.Warn().Err(err).Msg("acl sqlite: failed to load role assignments; role-derived access disabled until next reload") + } else { + for rows.Next() { + var assigneeType, assigneeID, roleName string + if err := rows.Scan(&assigneeType, &assigneeID, &roleName); err != nil { + logging.Logger.Warn().Err(err).Msg("acl sqlite: failed to scan role assignment row") + continue + } + _ = persist.LoadPolicyArray([]string{"g", assigneeType + ":" + assigneeID, acl.RoleSubjectPrefix + roleName}, m) + } + rows.Close() + } } func (a *sqliteAdapter) SavePolicy(m model.Model) error { return nil } diff --git a/server/internal/storage/acl/store.go b/server/internal/storage/acl/store.go index 4f0fc16..8f88261 100644 --- a/server/internal/storage/acl/store.go +++ b/server/internal/storage/acl/store.go @@ -259,10 +259,89 @@ type Store interface { // ErrRuleNotFound on miss. GetRule(ctx context.Context, principalType, principalID, resourceType, resourceID string) (*Rule, error) + // GetRuleByID fetches a single rule by its rule_id (UUID). Returns + // ErrRuleNotFound on miss. Used by the gateway REVOKE handler to resolve + // a rule's composite (principal+resource) key from the rule_id the SDK's + // delete-by-id surface sends, since RevokeAccess deletes by composite key. + GetRuleByID(ctx context.Context, ruleID string) (*Rule, error) + // ListRules returns rules matching the filter, ordered by granted_at // DESC. ListRules(ctx context.Context, filter RuleFilter) ([]*Rule, error) + // ===================================================================== + // Roles & groups (role/group authorization) + // ===================================================================== + // + // A group is a named collection of principals; a role is a named bundle + // of permissions. Permissions granted TO a group/role are ordinary + // acl_rules rows (principal_type 'group'/'role'); membership and + // assignment are stored separately and loaded into the enforcer as + // transitive grouping (g) edges. A principal's effective access is the + // additive maximum across its own rules and every group/role it + // transitively belongs to. Cycle-forming edges are rejected with + // ErrMembershipCycle. Group/role mutation is a privileged operation and + // callers must gate it behind the admin/acl capability. + + // CreateGroup creates a named group. Returns ErrGroupExists on a + // duplicate name. + CreateGroup(ctx context.Context, name, description, createdBy string, metadata map[string]interface{}) (*Group, error) + // DeleteGroup removes a group and (via cascade) its memberships, then + // refreshes the enforcer. Returns ErrGroupNotFound on miss. + DeleteGroup(ctx context.Context, name string) error + // GetGroup fetches a group by name (ErrGroupNotFound on miss). + GetGroup(ctx context.Context, name string) (*Group, error) + // ListGroups returns all groups ordered by name. + ListGroups(ctx context.Context) ([]*Group, error) + + // CreateRole creates a named role. Returns ErrRoleExists on a duplicate + // name. Role permissions are managed via GrantAccess with + // principalType="role", principalID=. + CreateRole(ctx context.Context, name, description, createdBy string, metadata map[string]interface{}) (*Role, error) + // DeleteRole removes a role, its permission rules, and (via cascade) its + // assignments, then refreshes the enforcer. Returns ErrRoleNotFound. + DeleteRole(ctx context.Context, name string) error + // GetRole fetches a role by name (ErrRoleNotFound on miss). + GetRole(ctx context.Context, name string) (*Role, error) + // ListRoles returns all roles ordered by name. + ListRoles(ctx context.Context) ([]*Role, error) + + // AddGroupMember adds (or refreshes) a membership edge. memberType is a + // principal type or "group" (nesting). Returns ErrGroupNotFound if the + // group is absent, ErrMembershipCycle if the edge would create a cycle. + AddGroupMember(ctx context.Context, groupName, memberType, memberID, grantedBy string, expiresAt *time.Time) (*GroupMember, error) + // RemoveGroupMember removes a membership edge (ErrMembershipNotFound on miss). + RemoveGroupMember(ctx context.Context, groupName, memberType, memberID string) error + // ListGroupMembers lists the direct members of a group. + ListGroupMembers(ctx context.Context, groupName string) ([]*GroupMember, error) + // ListPrincipalGroups lists the groups a principal is a direct member of. + ListPrincipalGroups(ctx context.Context, memberType, memberID string) ([]*GroupMember, error) + + // AssignRole adds (or refreshes) a role-assignment edge. assigneeType is + // a principal type or "group". Returns ErrRoleNotFound if the role is + // absent, ErrMembershipCycle if the edge would create a cycle. + AssignRole(ctx context.Context, roleName, assigneeType, assigneeID, grantedBy string, expiresAt *time.Time) (*RoleAssignment, error) + // UnassignRole removes a role-assignment edge (ErrAssignmentNotFound on miss). + UnassignRole(ctx context.Context, roleName, assigneeType, assigneeID string) error + // ListRoleAssignments lists the direct assignees of a role. + ListRoleAssignments(ctx context.Context, roleName string) ([]*RoleAssignment, error) + // ListPrincipalRoles lists the roles directly assigned to a principal. + ListPrincipalRoles(ctx context.Context, assigneeType, assigneeID string) ([]*RoleAssignment, error) + + // ExplainAccess returns the resolved subject set (self + transitive + // groups/roles), every rule that matched across those subjects, and the + // winning decision for a principal (by canonical principal_type/principal_id) + // against a resource. It does not gate access, but it DOES emit an + // "explain_access" audit event recording who asked (callerType/callerID — + // the connected principal on the gRPC path, or "admin_api"/ on + // the REST path) about whose access. callerType/callerID may be empty when + // the caller is unknown. + ExplainAccess(ctx context.Context, principalType, principalID, resourceType, resourceID string, requiredLevel int, callerType, callerID string) (*AccessExplanation, error) + + // CleanupExpiredMemberships deletes expired membership/assignment edges + // and reloads the enforcer, returning the count removed. + CleanupExpiredMemberships(ctx context.Context) (int64, error) + // ===================================================================== // Fallback policy // ===================================================================== diff --git a/server/internal/storage/acl/types.go b/server/internal/storage/acl/types.go index 016f897..a14a7d3 100644 --- a/server/internal/storage/acl/types.go +++ b/server/internal/storage/acl/types.go @@ -31,6 +31,18 @@ type ( ResolvedAuthority = legacy.ResolvedAuthority // RuleFilter is the query-side filter for ListRules. RuleFilter = legacy.RuleFilter + // Group is a named collection of principals (role/group authorization). + Group = legacy.Group + // Role is a named bundle of permissions assignable to principals/groups. + Role = legacy.Role + // GroupMember is a single group-membership edge. + GroupMember = legacy.GroupMember + // RoleAssignment is a single role-assignment edge. + RoleAssignment = legacy.RoleAssignment + // AccessExplanation describes the resolved subject set + winning decision. + AccessExplanation = legacy.AccessExplanation + // AccessContribution is one rule that matched a principal or its groups/roles. + AccessContribution = legacy.AccessContribution // AuthorityGrantFilter is the query-side filter for // ListAuthorityGrants. AuthorityGrantFilter = legacy.AuthorityGrantFilter @@ -131,6 +143,14 @@ const ( PrincipalTypeBridge = legacy.PrincipalTypeBridge PrincipalTypeService = legacy.PrincipalTypeService PrincipalTypeWildcard = legacy.PrincipalTypeWildcard + PrincipalTypeGroup = legacy.PrincipalTypeGroup + PrincipalTypeRole = legacy.PrincipalTypeRole +) + +// Subject prefixes for synthetic group/role subjects ("group:" / "role:"). +const ( + GroupSubjectPrefix = legacy.GroupSubjectPrefix + RoleSubjectPrefix = legacy.RoleSubjectPrefix ) // Reserved identifiers — wildcard subjects, system principal, global @@ -164,6 +184,7 @@ const ( PermissionAuditSubmit = legacy.PermissionAuditSubmit PermissionResolveAuthority = legacy.PermissionResolveAuthority PermissionQueryConnections = legacy.PermissionQueryConnections + PermissionKVPurgeIdentity = legacy.PermissionKVPurgeIdentity ) // WorkspaceScopeSubjectInherited is the magic value for @@ -192,6 +213,13 @@ var ( ErrInvalidAccessLevel = legacy.ErrInvalidAccessLevel ErrRuleNotFound = legacy.ErrRuleNotFound ErrFallbackPolicyNotFound = legacy.ErrFallbackPolicyNotFound + ErrGroupNotFound = legacy.ErrGroupNotFound + ErrRoleNotFound = legacy.ErrRoleNotFound + ErrGroupExists = legacy.ErrGroupExists + ErrRoleExists = legacy.ErrRoleExists + ErrMembershipNotFound = legacy.ErrMembershipNotFound + ErrAssignmentNotFound = legacy.ErrAssignmentNotFound + ErrMembershipCycle = legacy.ErrMembershipCycle ErrAuthorityGrantNotFound = legacy.ErrAuthorityGrantNotFound ErrAuthorityGrantExpired = legacy.ErrAuthorityGrantExpired ErrAuthorityGrantRevoked = legacy.ErrAuthorityGrantRevoked diff --git a/server/internal/storage/tasks/conformance_test.go b/server/internal/storage/tasks/conformance_test.go index b8f25ce..0ea808f 100644 --- a/server/internal/storage/tasks/conformance_test.go +++ b/server/internal/storage/tasks/conformance_test.go @@ -66,6 +66,11 @@ func TestStoreConformance(t *testing.T) { defer cleanup() runPoolClaim(t, store) }) + t.Run("PoolCrossWorkspace", func(t *testing.T) { + store, _, cleanup := b.factory(t) + defer cleanup() + runPoolCrossWorkspace(t, store) + }) t.Run("AuditEvents", func(t *testing.T) { store, _, cleanup := b.factory(t) defer cleanup() @@ -101,6 +106,11 @@ func TestStoreConformance(t *testing.T) { defer cleanup() runQueueByTaskID(t, store, db) }) + t.Run("QueueReconcileOrphaned", func(t *testing.T) { + store, _, cleanup := b.factory(t) + defer cleanup() + runQueueReconcileOrphaned(t, store) + }) t.Run("QueueEntryDetails", func(t *testing.T) { store, db, cleanup := b.factory(t) defer cleanup() @@ -131,6 +141,11 @@ func TestStoreConformance(t *testing.T) { defer cleanup() runNewFilters(t, store) }) + t.Run("TaskClassFilter", func(t *testing.T) { + store, _, cleanup := b.factory(t) + defer cleanup() + runTaskClassFilter(t, store) + }) t.Run("Descendants", func(t *testing.T) { store, _, cleanup := b.factory(t) defer cleanup() @@ -141,6 +156,11 @@ func TestStoreConformance(t *testing.T) { defer cleanup() runCursorPagination(t, store) }) + t.Run("CorrelationAndCompletion", func(t *testing.T) { + store, _, cleanup := b.factory(t) + defer cleanup() + runCorrelationAndCompletion(t, store) + }) }) } } @@ -277,6 +297,82 @@ func runPoolClaim(t *testing.T, store tasks.Store) { } } +// runPoolCrossWorkspace verifies the workspace="" path of GetPendingPoolTasks +// returns rows from every workspace for the given implementation. This is the +// service-principal lookup mode: cross-workspace services (webhookservice, +// proxy-sidecar) register with an empty workspace, and DeliverPoolTasks +// passes that empty workspace through so the service sees every pending pool +// task targeting its implementation. A concrete-workspace lookup still +// returns only that workspace's rows. +func runPoolCrossWorkspace(t *testing.T, store tasks.Store) { + t.Helper() + ctx := context.Background() + + impl := "conf-cross-ws-impl" + + taskA := newTestTask(t, "cross-ws-A") + taskA.AssignmentMode = tasks.AssignmentModePool + taskA.TargetImplementation = impl + taskA.QueuedForStartup = true + taskA.Workspace = "ws-alpha" + if err := store.CreateTask(ctx, taskA); err != nil { + t.Fatalf("CreateTask(ws-alpha): %v", err) + } + + taskB := newTestTask(t, "cross-ws-B") + taskB.AssignmentMode = tasks.AssignmentModePool + taskB.TargetImplementation = impl + taskB.QueuedForStartup = true + taskB.Workspace = "ws-beta" + if err := store.CreateTask(ctx, taskB); err != nil { + t.Fatalf("CreateTask(ws-beta): %v", err) + } + + // Concrete-workspace lookup returns only the matching task. + gotAlpha, err := store.GetPendingPoolTasks(ctx, impl, "ws-alpha") + if err != nil { + t.Fatalf("GetPendingPoolTasks(ws-alpha): %v", err) + } + if !containsTaskID(gotAlpha, taskA.TaskID) { + t.Fatalf("ws-alpha lookup: expected to contain %s, got %v", taskA.TaskID, taskIDs(gotAlpha)) + } + if containsTaskID(gotAlpha, taskB.TaskID) { + t.Fatalf("ws-alpha lookup: must NOT include ws-beta task %s", taskB.TaskID) + } + + // Service lookup (workspace="") returns every workspace's pending pool + // task for the implementation. + gotAll, err := store.GetPendingPoolTasks(ctx, impl, "") + if err != nil { + t.Fatalf("GetPendingPoolTasks(workspace=\"\"): %v", err) + } + if !containsTaskID(gotAll, taskA.TaskID) { + t.Fatalf("cross-workspace lookup: missing ws-alpha task %s, got %v", taskA.TaskID, taskIDs(gotAll)) + } + if !containsTaskID(gotAll, taskB.TaskID) { + t.Fatalf("cross-workspace lookup: missing ws-beta task %s, got %v", taskB.TaskID, taskIDs(gotAll)) + } +} + +func containsTaskID(list []*tasks.Task, id string) bool { + for _, t := range list { + if t != nil && t.TaskID == id { + return true + } + } + return false +} + +func taskIDs(list []*tasks.Task) []string { + out := make([]string, 0, len(list)) + for _, t := range list { + if t != nil { + out = append(out, t.TaskID) + } + } + return out +} + // runAuditEvents: RecordAuditEvent then GetTaskAuditEvents returns it. func runAuditEvents(t *testing.T, store tasks.Store) { t.Helper() @@ -590,6 +686,92 @@ func runQueueByTaskID(t *testing.T, store tasks.Store, db *sql.DB) { } } +// runQueueReconcileOrphaned: ReconcileOrphanedQueueEntries retires pending/ +// claimed queue rows whose task is terminal (cancelled) or purged (missing), +// while leaving rows whose task is still non-terminal (pending/running) alone. +// This is the sweep that clears the orphaned pending rows the polling +// dispatcher would otherwise poll forever. +func runQueueReconcileOrphaned(t *testing.T, store tasks.Store) { + t.Helper() + ctx := context.Background() + + // A: task cancelled (terminal) -> its pending queue row must be retired. + taskCancelled := newTestTask(t, "reconcile-cancelled") + if err := store.CreateTask(ctx, taskCancelled); err != nil { + t.Fatalf("CreateTask(cancelled): %v", err) + } + if err := store.CancelTask(ctx, taskCancelled.TaskID); err != nil { + t.Fatalf("CancelTask: %v", err) + } + qCancelled := uuid.New().String() + insertQueueEntry(t, store, qCancelled, taskCancelled.TaskID, "impl-a", "_test", "local") + + // B: task purged/missing (no tasks row) -> its pending queue row must be retired. + qPurged := uuid.New().String() + insertQueueEntry(t, store, qPurged, "purged-"+uuid.New().String(), "impl-b", "_test", "local") + + // C: task still pending (non-terminal) -> its queue row must be preserved. + taskPending := newTestTask(t, "reconcile-pending") + if err := store.CreateTask(ctx, taskPending); err != nil { + t.Fatalf("CreateTask(pending): %v", err) + } + qPending := uuid.New().String() + insertQueueEntry(t, store, qPending, taskPending.TaskID, "impl-c", "_test", "local") + + // D: task running (non-terminal) -> its queue row must be preserved. + taskRunning := newTestTask(t, "reconcile-running") + if err := store.CreateTask(ctx, taskRunning); err != nil { + t.Fatalf("CreateTask(running): %v", err) + } + if err := store.AssignTask(ctx, taskRunning.TaskID, "worker-1"); err != nil { + t.Fatalf("AssignTask: %v", err) + } + if err := store.StartTask(ctx, taskRunning.TaskID); err != nil { + t.Fatalf("StartTask: %v", err) + } + qRunning := uuid.New().String() + insertQueueEntry(t, store, qRunning, taskRunning.TaskID, "impl-d", "_test", "local") + + retired, err := store.ReconcileOrphanedQueueEntries(ctx) + if err != nil { + t.Fatalf("ReconcileOrphanedQueueEntries: %v", err) + } + if retired != 2 { + t.Fatalf("ReconcileOrphanedQueueEntries retired %d rows, want 2 (cancelled + purged)", retired) + } + + // The orphaned rows must no longer poll; the live rows must still poll. + entries, err := store.PollPendingQueueEntries(ctx, 100) + if err != nil { + t.Fatalf("PollPendingQueueEntries: %v", err) + } + polled := make(map[string]bool, len(entries)) + for _, e := range entries { + polled[e.QueueID] = true + } + if polled[qCancelled] { + t.Errorf("orphaned (cancelled-task) queue row %s still polls after reconcile", qCancelled) + } + if polled[qPurged] { + t.Errorf("orphaned (purged-task) queue row %s still polls after reconcile", qPurged) + } + if !polled[qPending] { + t.Errorf("live (pending-task) queue row %s was wrongly retired by reconcile", qPending) + } + if !polled[qRunning] { + t.Errorf("live (running-task) queue row %s was wrongly retired by reconcile", qRunning) + } + + // Idempotent: a second sweep retires nothing (the orphans are already terminal). + retired2, err := store.ReconcileOrphanedQueueEntries(ctx) + if err != nil { + t.Fatalf("ReconcileOrphanedQueueEntries(2): %v", err) + } + if retired2 != 0 { + t.Errorf("second ReconcileOrphanedQueueEntries retired %d rows, want 0 (idempotent)", retired2) + } +} + // runQueueEntryDetails: insert a queue entry with launch_params, verify // GetQueueEntryDetails returns them correctly. func runQueueEntryDetails(t *testing.T, store tasks.Store, db *sql.DB) { @@ -792,7 +974,7 @@ func insertQueueEntryWithParams(t *testing.T, store tasks.Store, queueID, taskID if launchParamsJSON != "" { lp = []byte(launchParamsJSON) } - if err := store.InsertQueueEntry(context.Background(), queueID, taskID, impl, workspace, profile, lp); err != nil { + if err := store.InsertQueueEntry(context.Background(), queueID, taskID, impl, workspace, profile, lp, int(tasks.PriorityNormal)); err != nil { t.Fatalf("insert queue entry: %v", err) } } @@ -830,6 +1012,63 @@ func postgresFactory(t *testing.T) (tasks.Store, *sql.DB, func()) { // which runs its own migrations, and returns the store. This factory // exercises the Stage 2 native implementation that will replace the // dbcompat path in AetherLite. +// runTaskClassFilter: ListTasks must honor TaskFilter.TaskClass (positive) and +// ExcludeTaskClasses identically on every backend. This is the invariant the +// interactive-task TTL sweep (orchestration.CancelStaleInteractiveTasks) and the +// frontend background-operations panel filter rely on — a divergence would make +// the sweep cancel the wrong task classes, or leak chat turns into the panel. +func runTaskClassFilter(t *testing.T, store tasks.Store) { + t.Helper() + ctx := context.Background() + const ws = "_taskclass" + const ( + classInteractive int32 = 1 // proto TASK_CLASS_INTERACTIVE + classBackground int32 = 2 // proto TASK_CLASS_BACKGROUND + ) + + mk := func(class int32) string { + task := newTestTask(t, "class") + task.Workspace = ws + task.TaskClass = class + if err := store.CreateTask(ctx, task); err != nil { + t.Fatalf("CreateTask(class=%d): %v", class, err) + } + return task.TaskID + } + interactive := mk(classInteractive) + background := mk(classBackground) + + // Positive TaskClass filter → only the interactive task. + got, err := store.ListTasks(ctx, &tasks.TaskFilter{Workspace: ws, TaskClass: classInteractive, Limit: 10}) + if err != nil { + t.Fatalf("ListTasks(task_class=interactive): %v", err) + } + if len(got) != 1 || got[0].TaskID != interactive { + t.Fatalf("TaskClass=interactive returned %d rows, want exactly [%s]", len(got), interactive) + } + + // ExcludeTaskClasses → interactive omitted, background retained. + gotEx, err := store.ListTasks(ctx, &tasks.TaskFilter{Workspace: ws, ExcludeTaskClasses: []int32{classInteractive}, Limit: 10}) + if err != nil { + t.Fatalf("ListTasks(exclude interactive): %v", err) + } + var sawInteractive, sawBackground bool + for _, task := range gotEx { + switch task.TaskID { + case interactive: + sawInteractive = true + case background: + sawBackground = true + } + } + if sawInteractive { + t.Errorf("ExcludeTaskClasses=[interactive] still returned the interactive task") + } + if !sawBackground { + t.Errorf("ExcludeTaskClasses=[interactive] dropped the background task") + } +} + func sqliteNativeFactory(t *testing.T) (tasks.Store, *sql.DB, func()) { t.Helper() dbPath := filepath.Join(t.TempDir(), "tasks_native.db") @@ -1302,3 +1541,101 @@ func containsTimer(timers []*tasks.TimerRecord, timerID string) bool { } return false } + +// runCorrelationAndCompletion verifies that CorrelationID, RootTaskID, and +// CompletionEvent round-trip through CreateTask/GetTask and that the +// CorrelationID and RootTaskID filter predicates work correctly. +func runCorrelationAndCompletion(t *testing.T, store tasks.Store) { + t.Helper() + ctx := context.Background() + + const corrID = "corr-group-abc" + const rootID = "root-task-xyz" + + // Task with all three new fields populated. + task := newTestTask(t, "corr-full") + task.CorrelationID = corrID + task.RootTaskID = rootID + task.CompletionEvent = &tasks.TaskCompletionConfig{ + Enabled: true, + EventName: "task.done", + OnStatuses: []tasks.TaskStatus{tasks.TaskStatusCompleted, tasks.TaskStatusFailed}, + } + if err := store.CreateTask(ctx, task); err != nil { + t.Fatalf("CreateTask (full): %v", err) + } + + // Second task: same correlation group, no CompletionEvent. + task2 := newTestTask(t, "corr-sibling") + task2.CorrelationID = corrID + task2.RootTaskID = rootID + if err := store.CreateTask(ctx, task2); err != nil { + t.Fatalf("CreateTask (sibling): %v", err) + } + + // Task outside the correlation group. + taskOther := newTestTask(t, "corr-other") + if err := store.CreateTask(ctx, taskOther); err != nil { + t.Fatalf("CreateTask (other): %v", err) + } + + // --- GetTask round-trip for all three fields --- + got, err := store.GetTask(ctx, task.TaskID) + if err != nil { + t.Fatalf("GetTask: %v", err) + } + if got.CorrelationID != corrID { + t.Errorf("CorrelationID: got %q, want %q", got.CorrelationID, corrID) + } + if got.RootTaskID != rootID { + t.Errorf("RootTaskID: got %q, want %q", got.RootTaskID, rootID) + } + if got.CompletionEvent == nil { + t.Fatal("CompletionEvent: got nil, want non-nil") + } + if !got.CompletionEvent.Enabled { + t.Error("CompletionEvent.Enabled: got false, want true") + } + if got.CompletionEvent.EventName != "task.done" { + t.Errorf("CompletionEvent.EventName: got %q, want %q", got.CompletionEvent.EventName, "task.done") + } + if len(got.CompletionEvent.OnStatuses) != 2 { + t.Errorf("CompletionEvent.OnStatuses len: got %d, want 2", len(got.CompletionEvent.OnStatuses)) + } + + // Task without CompletionEvent must come back nil. + got2, err := store.GetTask(ctx, task2.TaskID) + if err != nil { + t.Fatalf("GetTask (sibling): %v", err) + } + if got2.CompletionEvent != nil { + t.Errorf("CompletionEvent: got non-nil for task without completion_event") + } + + // --- CorrelationID filter --- + listed, err := store.ListTasks(ctx, &tasks.TaskFilter{CorrelationID: corrID, Limit: 100}) + if err != nil { + t.Fatalf("ListTasks(CorrelationID): %v", err) + } + if !containsTask(listed, task.TaskID) { + t.Errorf("CorrelationID filter: missing task %s", task.TaskID) + } + if !containsTask(listed, task2.TaskID) { + t.Errorf("CorrelationID filter: missing sibling %s", task2.TaskID) + } + if containsTask(listed, taskOther.TaskID) { + t.Errorf("CorrelationID filter: unexpectedly includes other task %s", taskOther.TaskID) + } + + // --- RootTaskID filter --- + listedRoot, err := store.ListTasks(ctx, &tasks.TaskFilter{RootTaskID: rootID, Limit: 100}) + if err != nil { + t.Fatalf("ListTasks(RootTaskID): %v", err) + } + if !containsTask(listedRoot, task.TaskID) { + t.Errorf("RootTaskID filter: missing task %s", task.TaskID) + } + if containsTask(listedRoot, taskOther.TaskID) { + t.Errorf("RootTaskID filter: unexpectedly includes other task %s", taskOther.TaskID) + } +} diff --git a/server/internal/storage/tasks/postgres/store.go b/server/internal/storage/tasks/postgres/store.go index 36898d5..5beca57 100644 --- a/server/internal/storage/tasks/postgres/store.go +++ b/server/internal/storage/tasks/postgres/store.go @@ -14,6 +14,7 @@ import ( "database/sql" "encoding/json" "fmt" + "strings" "time" "github.com/google/uuid" @@ -155,12 +156,15 @@ func (s *Store) InsertDLQEntryTx(ctx context.Context, tx tasks.StoreTx, taskID, // Non-transactional queue operations (orchestrated_task_queue) // ========================================================================= -func (s *Store) InsertQueueEntry(ctx context.Context, queueID, taskID, targetImplementation, workspace, profile string, launchParamsJSON []byte) error { +func (s *Store) InsertQueueEntry(ctx context.Context, queueID, taskID, targetImplementation, workspace, profile string, launchParamsJSON []byte, priority int) error { + if priority == 0 { + priority = int(tasks.PriorityNormal) + } _, err := s.db.ExecContext(ctx, ` INSERT INTO orchestrated_task_queue - (queue_id, task_id, target_implementation, workspace, profile, launch_params, status) - VALUES ($1, $2, $3, $4, $5, $6, 'pending') - `, queueID, taskID, targetImplementation, workspace, profile, launchParamsJSON) + (queue_id, task_id, target_implementation, workspace, profile, launch_params, status, priority) + VALUES ($1, $2, $3, $4, $5, $6, 'pending', $7) + `, queueID, taskID, targetImplementation, workspace, profile, launchParamsJSON, priority) return err } @@ -172,7 +176,7 @@ func (s *Store) PollPendingQueueEntries(ctx context.Context, limit int) ([]*task SELECT queue_id, task_id, profile, workspace, target_implementation FROM orchestrated_task_queue WHERE status = $1 AND (next_retry_at IS NULL OR next_retry_at <= NOW()) - ORDER BY created_at ASC + ORDER BY priority DESC, created_at ASC LIMIT $2 `, "pending", limit) if err != nil { @@ -299,6 +303,39 @@ func (s *Store) FailQueueEntryByTaskID(ctx context.Context, taskID, errorMsg str return err } +// ReconcileOrphanedQueueEntries retires every pending/claimed queue row whose +// task is no longer live (terminal or purged) in one statement. Liveness is a +// correlated NOT EXISTS against the tasks table restricted to the canonical +// non-terminal status set, so a purged task (no row) and a terminal task (row +// present but terminal status) are both treated as orphaned. See the +// tasks.Store interface doc for the full contract. +func (s *Store) ReconcileOrphanedQueueEntries(ctx context.Context) (int64, error) { + statuses := tasks.NonTerminalStatuses() + // $1 = error_message; $2.. = the non-terminal status list. + placeholders := make([]string, len(statuses)) + args := make([]interface{}, 0, len(statuses)+1) + args = append(args, tasks.ReconcileErrorMessage) + for i, st := range statuses { + placeholders[i] = fmt.Sprintf("$%d", i+2) + args = append(args, string(st)) + } + query := fmt.Sprintf(` + UPDATE orchestrated_task_queue + SET status = 'failed', error_message = $1, completed_at = NOW() + WHERE status IN ('pending', 'claimed') + AND NOT EXISTS ( + SELECT 1 FROM tasks t + WHERE t.task_id = orchestrated_task_queue.task_id + AND t.status IN (%s) + ) + `, strings.Join(placeholders, ", ")) + res, err := s.db.ExecContext(ctx, query, args...) + if err != nil { + return 0, err + } + return res.RowsAffected() +} + // ========================================================================= // Admin workspace queries // ========================================================================= diff --git a/server/internal/storage/tasks/sqlite/store.go b/server/internal/storage/tasks/sqlite/store.go index ebf64b5..03639ee 100644 --- a/server/internal/storage/tasks/sqlite/store.go +++ b/server/internal/storage/tasks/sqlite/store.go @@ -128,6 +128,22 @@ func (s *Store) CreateTask(ctx context.Context, task *tasks.Task) error { if task.PausedAt != nil { pausedAtMs = sql.NullInt64{Int64: task.PausedAt.UnixMilli(), Valid: true} } + var retryPolicyStr sql.NullString + if task.RetryPolicy != nil { + b, merr := json.Marshal(task.RetryPolicy) + if merr != nil { + return fmt.Errorf("failed to marshal retry_policy: %w", merr) + } + retryPolicyStr = sql.NullString{String: string(b), Valid: true} + } + var completionEventStr sql.NullString + if task.CompletionEvent != nil { + b, merr := json.Marshal(task.CompletionEvent) + if merr != nil { + return fmt.Errorf("failed to marshal completion_event: %w", merr) + } + completionEventStr = sql.NullString{String: string(b), Valid: true} + } _, err = s.db.ExecContext(ctx, ` INSERT INTO tasks ( @@ -142,13 +158,17 @@ func (s *Store) CreateTask(ctx context.Context, task *tasks.Task) error { authority_grant_id, root_authority_grant_id, parent_authority_grant_id, authority_audience_type, authority_audience_id, authority_delegate_type, authority_delegate_id, task_class, grace_window_ms, - wait_spec, depends_on, context_id, paused_at + wait_spec, depends_on, context_id, paused_at, + retry_policy_json, + correlation_id, root_task_id, completion_event ) VALUES ( ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, - ?, ?, ?, ? + ?, ?, ?, ?, + ?, + ?, ?, ? ) `, task.TaskID, @@ -198,6 +218,10 @@ func (s *Store) CreateTask(ctx context.Context, task *tasks.Task) error { dependsOnStr, nullStr(task.ContextID), pausedAtMs, + retryPolicyStr, + nullStr(task.CorrelationID), + nullStr(task.RootTaskID), + completionEventStr, ) return err } @@ -364,6 +388,28 @@ func (s *Store) StartTaskWithAgent(ctx context.Context, taskID, agentIdentity st return nil } +// ClaimTask transitions a task into running when claimed by its assignee. +// Unlike StartTask it also accepts a pending source state so a per-turn task +// can be claimed straight out of the queue. Only stamps started_at on the +// first transition; running -> running is a no-op for that timestamp +// (idempotent for re-claim / multi-tab scenarios). +func (s *Store) ClaimTask(ctx context.Context, taskID string) error { + nowStr := now() + result, err := s.db.ExecContext(ctx, ` + UPDATE tasks + SET status = 'running', started_at = COALESCE(started_at, ?) + WHERE task_id = ? AND status IN ('pending', 'assigned', 'starting', 'running') + `, nowStr, taskID) + if err != nil { + return err + } + rows, _ := result.RowsAffected() + if rows == 0 { + return fmt.Errorf("task %s not in a claimable state (pending, assigned, starting, or running)", taskID) + } + return nil +} + func (s *Store) CompleteTask(ctx context.Context, taskID string) error { nowStr := now() result, err := s.db.ExecContext(ctx, ` @@ -384,6 +430,39 @@ func (s *Store) CompleteTask(ctx context.Context, taskID string) error { func (s *Store) FailTask(ctx context.Context, taskID, errorMsg string) error { nowStr := now() + + // Best-effort lookup of an attached RetryPolicy. If GetTask fails, fall + // back to the legacy update — losing the policy-driven reschedule is + // preferable to losing the FailTask transition itself. + var nextRetryAt *time.Time + if task, getErr := s.GetTask(ctx, taskID); getErr == nil && task != nil && task.RetryPolicy != nil { + newCount := int32(task.RetryCount + 1) + if newCount < task.RetryPolicy.EffectiveMaxAttempts() { + t := tasks.ComputeNextRetryAt(task.RetryPolicy, int(newCount), nil) + nextRetryAt = &t + } + } + + if nextRetryAt != nil { + result, err := s.db.ExecContext(ctx, ` + UPDATE tasks + SET status = 'failed', failed_at = ?, error_message = ?, + next_retry_at = ?, retry_count = retry_count + 1 + WHERE task_id = ? + `, nowStr, errorMsg, nullTimeStr(nextRetryAt), taskID) + if err != nil { + return err + } + rows, err := result.RowsAffected() + if err != nil { + return err + } + if rows == 0 { + return fmt.Errorf("task %s not found or not in a failable state", taskID) + } + return nil + } + result, err := s.db.ExecContext(ctx, ` UPDATE tasks SET status = 'failed', failed_at = ?, error_message = ?, retry_count = retry_count + 1 @@ -554,6 +633,14 @@ func buildFilterClausesSQLite(filter *tasks.TaskFilter, args []interface{}, skip query += " AND context_id = ?" args = append(args, filter.ContextID) } + if filter.CorrelationID != "" { + query += " AND correlation_id = ?" + args = append(args, filter.CorrelationID) + } + if filter.RootTaskID != "" { + query += " AND root_task_id = ?" + args = append(args, filter.RootTaskID) + } if len(filter.ExcludeStatuses) > 0 { placeholders := make([]string, len(filter.ExcludeStatuses)) for i, st := range filter.ExcludeStatuses { @@ -1182,12 +1269,15 @@ func (s *Store) InsertDLQEntryTx(ctx context.Context, tx tasks.StoreTx, taskID, // Non-transactional queue operations (orchestrated_task_queue) // ============================================================================= -func (s *Store) InsertQueueEntry(ctx context.Context, queueID, taskID, targetImplementation, workspace, profile string, launchParamsJSON []byte) error { +func (s *Store) InsertQueueEntry(ctx context.Context, queueID, taskID, targetImplementation, workspace, profile string, launchParamsJSON []byte, priority int) error { + if priority == 0 { + priority = int(tasks.PriorityNormal) + } _, err := s.db.ExecContext(ctx, ` INSERT INTO orchestrated_task_queue - (queue_id, task_id, target_implementation, workspace, profile, launch_params, status) - VALUES (?, ?, ?, ?, ?, ?, 'pending') - `, queueID, taskID, targetImplementation, workspace, profile, launchParamsJSON) + (queue_id, task_id, target_implementation, workspace, profile, launch_params, status, priority) + VALUES (?, ?, ?, ?, ?, ?, 'pending', ?) + `, queueID, taskID, targetImplementation, workspace, profile, launchParamsJSON, priority) return err } @@ -1200,7 +1290,7 @@ func (s *Store) PollPendingQueueEntries(ctx context.Context, limit int) ([]*task SELECT queue_id, task_id, profile, workspace, target_implementation FROM orchestrated_task_queue WHERE status = 'pending' AND (next_retry_at IS NULL OR next_retry_at <= ?) - ORDER BY created_at ASC + ORDER BY priority DESC, created_at ASC LIMIT ? `, nowStr, limit) if err != nil { @@ -1335,6 +1425,38 @@ func (s *Store) FailQueueEntryByTaskID(ctx context.Context, taskID, errorMsg str return err } +// ReconcileOrphanedQueueEntries retires every pending/claimed queue row whose +// task is no longer live (terminal or purged) in one statement. Liveness is a +// correlated NOT EXISTS against the tasks table restricted to the canonical +// non-terminal status set, so a purged task (no row) and a terminal task (row +// present but terminal status) are both treated as orphaned. See the +// tasks.Store interface doc for the full contract. +func (s *Store) ReconcileOrphanedQueueEntries(ctx context.Context) (int64, error) { + statuses := tasks.NonTerminalStatuses() + placeholders := make([]string, len(statuses)) + args := make([]interface{}, 0, len(statuses)+2) + args = append(args, tasks.ReconcileErrorMessage, now()) + for i, st := range statuses { + placeholders[i] = "?" + args = append(args, string(st)) + } + query := fmt.Sprintf(` + UPDATE orchestrated_task_queue + SET status = 'failed', error_message = ?, completed_at = ? + WHERE status IN ('pending', 'claimed') + AND NOT EXISTS ( + SELECT 1 FROM tasks t + WHERE t.task_id = orchestrated_task_queue.task_id + AND t.status IN (%s) + ) + `, strings.Join(placeholders, ", ")) + res, err := s.db.ExecContext(ctx, query, args...) + if err != nil { + return 0, err + } + return res.RowsAffected() +} + // ============================================================================= // Disconnect tracking // ============================================================================= @@ -1505,7 +1627,9 @@ const taskSelectColumns = ` authority_audience_type, authority_audience_id, authority_delegate_type, authority_delegate_id, task_class, disconnected_at, grace_window_ms, - wait_spec, depends_on, context_id, paused_at + wait_spec, depends_on, context_id, paused_at, + retry_policy_json, + correlation_id, root_task_id, completion_event ` // taskScanner is satisfied by both *sql.Row and *sql.Rows. @@ -1521,12 +1645,13 @@ func scanTaskInto(s taskScanner) (*tasks.Task, error) { var authorityGrantID, rootAuthorityGrantID, parentAuthorityGrantID sql.NullString var authorityAudienceType, authorityAudienceID, authorityDelegateType, authorityDelegateID sql.NullString var contextID sql.NullString + var correlationID, rootTaskID, completionEventJSON sql.NullString var createdAtStr, updatedAtStr string var scheduledForStr, startedAtStr, completedAtStr, failedAtStr, assignedAtStr, nextRetryAtStr, lastHeartbeatStr, disconnectedAtStr sql.NullString var pausedAtMs sql.NullInt64 var scheduleToStart, startToClose, heartbeatTimeout, scheduleToClose sql.NullInt64 var launchParamsJSON, metadataJSON, checkpointJSON, heartbeatJSON []byte - var waitSpecJSON, dependsOnJSON sql.NullString + var waitSpecJSON, dependsOnJSON, retryPolicyJSON sql.NullString var queuedForStartup int err := s.Scan( @@ -1548,6 +1673,8 @@ func scanTaskInto(s taskScanner) (*tasks.Task, error) { &task.TaskClass, &disconnectedAtStr, &task.GraceWindowMs, &waitSpecJSON, &dependsOnJSON, &contextID, &pausedAtMs, + &retryPolicyJSON, + &correlationID, &rootTaskID, &completionEventJSON, ) if err != nil { return nil, err @@ -1749,6 +1876,13 @@ func scanTaskInto(s taskScanner) (*tasks.Task, error) { return nil, fmt.Errorf("failed to unmarshal depends_on: %w", err) } } + if retryPolicyJSON.Valid && len(retryPolicyJSON.String) > 0 { + var rp tasks.RetryPolicy + if err := json.Unmarshal([]byte(retryPolicyJSON.String), &rp); err != nil { + return nil, fmt.Errorf("failed to unmarshal retry_policy: %w", err) + } + task.RetryPolicy = &rp + } if contextID.Valid { task.ContextID = contextID.String } @@ -1756,6 +1890,19 @@ func scanTaskInto(s taskScanner) (*tasks.Task, error) { t := time.UnixMilli(pausedAtMs.Int64).UTC() task.PausedAt = &t } + if correlationID.Valid { + task.CorrelationID = correlationID.String + } + if rootTaskID.Valid { + task.RootTaskID = rootTaskID.String + } + if completionEventJSON.Valid && len(completionEventJSON.String) > 0 { + var ce tasks.TaskCompletionConfig + if err := json.Unmarshal([]byte(completionEventJSON.String), &ce); err != nil { + return nil, fmt.Errorf("failed to unmarshal completion_event: %w", err) + } + task.CompletionEvent = &ce + } return &task, nil } diff --git a/server/internal/storage/tasks/store.go b/server/internal/storage/tasks/store.go index 4391736..b6d1ac1 100644 --- a/server/internal/storage/tasks/store.go +++ b/server/internal/storage/tasks/store.go @@ -32,6 +32,11 @@ import ( "time" ) +// ReconcileErrorMessage is the error_message stamped on orchestrated_task_queue +// rows retired by ReconcileOrphanedQueueEntries. Exported so the reconcile sweep +// and its tests share a single canonical string. +const ReconcileErrorMessage = "reconciled: task terminal or missing" + // StoreTx is a backend-agnostic transaction handle for the tasks domain. // Each implementation (postgres, sqlite) returns its own concrete type from // BeginTx; the concrete type wraps a *sql.Tx internally and is recovered via @@ -113,6 +118,13 @@ type Store interface { // task can be reconciled if the orchestrated agent disconnects later. StartTaskWithAgent(ctx context.Context, taskID, agentIdentity string) error + // ClaimTask transitions a task into running when claimed by its assignee. + // Unlike StartTask, it additionally accepts a pending source state so a + // per-turn task can be claimed straight out of the queue. Only stamps + // started_at on the first transition; running → running is a no-op for + // that timestamp (idempotent for re-claim / multi-tab scenarios). + ClaimTask(ctx context.Context, taskID string) error + // CompleteTask marks a task as completed and stamps completed_at. CompleteTask(ctx context.Context, taskID string) error @@ -332,7 +344,7 @@ type Store interface { // postgres, the row INSERT fires a LISTEN/NOTIFY trigger that wakes // orchestrators immediately; on sqlite, the polling dispatcher picks // the row up on its next scan. - InsertQueueEntry(ctx context.Context, queueID, taskID, targetImplementation, workspace, profile string, launchParamsJSON []byte) error + InsertQueueEntry(ctx context.Context, queueID, taskID, targetImplementation, workspace, profile string, launchParamsJSON []byte, priority int) error // PollPendingQueueEntries returns up to limit orchestrated_task_queue // rows that are pending and eligible for dispatch (next_retry_at is @@ -379,6 +391,19 @@ type Store interface { // Idempotent. FailQueueEntryByTaskID(ctx context.Context, taskID, errorMsg string) error + // ReconcileOrphanedQueueEntries retires (status='failed', error_message= + // ReconcileErrorMessage, completed_at stamped) every orchestrated_task_queue + // row still in pending/claimed status whose task_id is NOT present in the + // tasks table with a NON-terminal status (i.e. the task is terminal — + // cancelled/completed/failed/rejected/dlq — or has been purged). This is the + // dispatcher-independent sweep that clears queue rows orphaned when a task + // went terminal (or was purged) without its best-effort, dispatcher-gated + // retire firing, so the polling dispatcher stops polling ghosts forever. + // Returns the number of rows retired. No-op-safe in clustered/JetStream mode + // (the SQL orchestrated_task_queue holds no rows there). The canonical + // non-terminal status set is derived from NonTerminalStatuses(). + ReconcileOrphanedQueueEntries(ctx context.Context) (int64, error) + // ========================================================================= // Disconnect tracking // ========================================================================= diff --git a/server/internal/storage/tasks/types.go b/server/internal/storage/tasks/types.go index 1a6a6fe..7bec2e1 100644 --- a/server/internal/storage/tasks/types.go +++ b/server/internal/storage/tasks/types.go @@ -97,8 +97,52 @@ type ( // WaitSpec describes why a task was paused and what conditions will wake it. WaitSpec = legacy.WaitSpec + + // RetryPolicy mirrors the proto pb.RetryPolicy message, persisted on + // the task row via retry_policy_json and consumed by FailTask to + // compute next_retry_at when set. + RetryPolicy = legacy.RetryPolicy + + // TaskCompletionConfig is the "feed B" config: emit a domain event when + // the task reaches a terminal status. Persisted as JSON in + // completion_event. + TaskCompletionConfig = legacy.TaskCompletionConfig + + // BackoffStrategy enumerates the retry-delay shapes consumed by + // ComputeNextRetryAt. + BackoffStrategy = legacy.BackoffStrategy + + // TaskPriority is the dispatch-priority weight stored in tasks.priority + // and orchestrated_task_queue.priority. + TaskPriority = legacy.TaskPriority ) +// Dispatch-priority levels — re-exported so storage-layer code can normalize +// and order by priority without a second import on pkg/tasks. +const ( + PriorityUnspecified = legacy.PriorityUnspecified + PriorityXLow = legacy.PriorityXLow + PriorityLow = legacy.PriorityLow + PriorityNormal = legacy.PriorityNormal + PriorityHigh = legacy.PriorityHigh + PriorityPreempt = legacy.PriorityPreempt +) + +// BackoffStrategy enum values — re-exported so callers depending only on +// the storage/tasks package can construct a RetryPolicy without taking a +// second import on pkg/tasks. +const ( + BackoffStrategyUnspecified = legacy.BackoffStrategyUnspecified + BackoffStrategyFixed = legacy.BackoffStrategyFixed + BackoffStrategyExponential = legacy.BackoffStrategyExponential + BackoffStrategyExplicitSchedule = legacy.BackoffStrategyExplicitSchedule +) + +// ComputeNextRetryAt is re-exported so the native sqlite store and other +// in-tree consumers of the tasks package can compute the policy-driven +// next_retry_at without a second import. +var ComputeNextRetryAt = legacy.ComputeNextRetryAt + // Task lifecycle statuses — values stored in tasks.status. const ( TaskStatusPending = legacy.TaskStatusPending @@ -128,9 +172,10 @@ const ( // Phase 1: task lifecycle helpers — re-exported from legacy package. var ( - IsTerminal = legacy.IsTerminal - IsWaiting = legacy.IsWaiting - ValidateTransition = legacy.ValidateTransition + IsTerminal = legacy.IsTerminal + IsWaiting = legacy.IsWaiting + ValidateTransition = legacy.ValidateTransition + NonTerminalStatuses = legacy.NonTerminalStatuses ) // Phase 4: cursor pagination helpers re-exported from the legacy package so diff --git a/server/internal/storage/workflow/joins_test.go b/server/internal/storage/workflow/joins_test.go new file mode 100644 index 0000000..a126d3c --- /dev/null +++ b/server/internal/storage/workflow/joins_test.go @@ -0,0 +1,268 @@ +package workflow_test + +import ( + "context" + "database/sql" + "fmt" + "path/filepath" + "testing" + "time" + + wfstore "github.com/scitrera/aether/internal/storage/workflow" + wfsqlite "github.com/scitrera/aether/internal/storage/workflow/sqlite" +) + +// newJoinTestStore spins up a fresh native-SQLite store backed by a temp-dir +// file, mirroring sqliteNativeFactory in conformance_test.go. +func newJoinTestStore(t *testing.T) wfstore.Store { + t.Helper() + dbPath := filepath.Join(t.TempDir(), "workflow_joins.db") + dsn := fmt.Sprintf("file:%s?_pragma=journal_mode(WAL)&_pragma=busy_timeout(5000)", dbPath) + db, err := sql.Open("sqlite", dsn) + if err != nil { + t.Fatalf("sql.Open sqlite (native): %v", err) + } + store, err := wfsqlite.New(db) + if err != nil { + _ = db.Close() + t.Fatalf("wfsqlite.New: %v", err) + } + t.Cleanup(func() { _ = db.Close() }) + return store +} + +func containsJoin(joins []wfstore.Join, id int64) bool { + for _, j := range joins { + if j.ID == id { + return true + } + } + return false +} + +// TestJoins_EnsureAndGet covers EnsureJoin (insert + idempotent re-ensure) +// and GetJoin (present + absent → nil,nil). +func TestJoins_EnsureAndGet(t *testing.T) { + ctx := context.Background() + store := newJoinTestStore(t) + + tag := uniqueName(t, "join") + deadline := time.Now().Add(time.Hour) + j := &wfstore.Join{ + JoinName: "jn-" + tag, + Workspace: "ws-" + tag, + CorrelationKey: "ck-" + tag, + Mode: wfstore.JoinModeCount, + ArrivedCount: 0, + Status: wfstore.JoinStatusOpen, + DeadlineAt: &deadline, + } + + got, err := store.EnsureJoin(ctx, j) + if err != nil { + t.Fatalf("EnsureJoin: %v", err) + } + if got == nil || got.ID == 0 { + t.Fatalf("EnsureJoin did not return a live row with an ID: %+v", got) + } + if got.Mode != wfstore.JoinModeCount || got.Status != wfstore.JoinStatusOpen { + t.Fatalf("EnsureJoin field mismatch: %+v", got) + } + if got.DeadlineAt == nil { + t.Fatalf("EnsureJoin did not persist deadline_at") + } + firstID := got.ID + + // Idempotent re-ensure with different field values must NOT create a + // second row and must NOT mutate the existing one. + dup := &wfstore.Join{ + JoinName: "jn-" + tag, + Workspace: "ws-" + tag, + CorrelationKey: "ck-" + tag, + Mode: wfstore.JoinModeSet, // different, should be ignored + ArrivedCount: 99, // different, should be ignored + Status: wfstore.JoinStatusFired, + } + again, err := store.EnsureJoin(ctx, dup) + if err != nil { + t.Fatalf("EnsureJoin (re-ensure): %v", err) + } + if again.ID != firstID { + t.Fatalf("re-ensure produced a new row id: got %d want %d", again.ID, firstID) + } + if again.Mode != wfstore.JoinModeCount || again.ArrivedCount != 0 || again.Status != wfstore.JoinStatusOpen { + t.Fatalf("re-ensure mutated the existing row: %+v", again) + } + + // GetJoin present. + present, err := store.GetJoin(ctx, "jn-"+tag, "ws-"+tag, "ck-"+tag) + if err != nil { + t.Fatalf("GetJoin (present): %v", err) + } + if present == nil || present.ID != firstID { + t.Fatalf("GetJoin (present) returned %+v", present) + } + + // GetJoin absent → (nil, nil). + absent, err := store.GetJoin(ctx, "jn-missing", "ws-missing", "ck-missing") + if err != nil { + t.Fatalf("GetJoin (absent): %v", err) + } + if absent != nil { + t.Fatalf("GetJoin (absent) expected nil, got %+v", absent) + } +} + +// TestJoins_Mutations covers UpdateJoinArrived, SetJoinExpected, SetJoinDirty, +// and MarkJoinTerminal. +func TestJoins_Mutations(t *testing.T) { + ctx := context.Background() + store := newJoinTestStore(t) + + tag := uniqueName(t, "join") + j := &wfstore.Join{ + JoinName: "jn-" + tag, + Workspace: "ws-" + tag, + CorrelationKey: "ck-" + tag, + Mode: wfstore.JoinModeCount, + Status: wfstore.JoinStatusOpen, + } + row, err := store.EnsureJoin(ctx, j) + if err != nil { + t.Fatalf("EnsureJoin: %v", err) + } + + if err := store.UpdateJoinArrived(ctx, row.ID, 3); err != nil { + t.Fatalf("UpdateJoinArrived: %v", err) + } + if err := store.SetJoinExpected(ctx, row.ID, 5); err != nil { + t.Fatalf("SetJoinExpected: %v", err) + } + if err := store.SetJoinDirty(ctx, row.ID, true); err != nil { + t.Fatalf("SetJoinDirty: %v", err) + } + + got, err := store.GetJoin(ctx, "jn-"+tag, "ws-"+tag, "ck-"+tag) + if err != nil { + t.Fatalf("GetJoin: %v", err) + } + if got.ArrivedCount != 3 { + t.Fatalf("ArrivedCount: got %d want 3", got.ArrivedCount) + } + if got.ExpectedCount == nil || *got.ExpectedCount != 5 { + t.Fatalf("ExpectedCount: got %v want 5", got.ExpectedCount) + } + if !got.Dirty { + t.Fatalf("Dirty: got false want true") + } + + linger := time.Now().Add(2 * time.Hour) + if err := store.MarkJoinTerminal(ctx, row.ID, wfstore.JoinStatusFired, linger); err != nil { + t.Fatalf("MarkJoinTerminal: %v", err) + } + terminal, err := store.GetJoin(ctx, "jn-"+tag, "ws-"+tag, "ck-"+tag) + if err != nil { + t.Fatalf("GetJoin (terminal): %v", err) + } + if terminal.Status != wfstore.JoinStatusFired { + t.Fatalf("Status: got %q want %q", terminal.Status, wfstore.JoinStatusFired) + } + if terminal.LingerUntil == nil { + t.Fatalf("LingerUntil was not persisted") + } +} + +// TestJoins_DueDeadlines verifies that an open row with a past deadline is +// returned, while a future-deadline row and a fired (terminal) row are not. +func TestJoins_DueDeadlines(t *testing.T) { + ctx := context.Background() + store := newJoinTestStore(t) + + tag := uniqueName(t, "join") + past := time.Now().Add(-time.Hour) + future := time.Now().Add(time.Hour) + + // Open + past deadline → should be returned. + dueRow, err := store.EnsureJoin(ctx, &wfstore.Join{ + JoinName: "jn-due-" + tag, Workspace: "ws-" + tag, CorrelationKey: "ck-due-" + tag, + Mode: wfstore.JoinModeCount, Status: wfstore.JoinStatusOpen, DeadlineAt: &past, + }) + if err != nil { + t.Fatalf("EnsureJoin (due): %v", err) + } + + // Open + future deadline → should NOT be returned. + futureRow, err := store.EnsureJoin(ctx, &wfstore.Join{ + JoinName: "jn-future-" + tag, Workspace: "ws-" + tag, CorrelationKey: "ck-future-" + tag, + Mode: wfstore.JoinModeCount, Status: wfstore.JoinStatusOpen, DeadlineAt: &future, + }) + if err != nil { + t.Fatalf("EnsureJoin (future): %v", err) + } + + // Fired + past deadline → should NOT be returned (not open). + firedRow, err := store.EnsureJoin(ctx, &wfstore.Join{ + JoinName: "jn-fired-" + tag, Workspace: "ws-" + tag, CorrelationKey: "ck-fired-" + tag, + Mode: wfstore.JoinModeCount, Status: wfstore.JoinStatusFired, DeadlineAt: &past, + }) + if err != nil { + t.Fatalf("EnsureJoin (fired): %v", err) + } + + due, err := store.GetDueJoinDeadlines(ctx, time.Now()) + if err != nil { + t.Fatalf("GetDueJoinDeadlines: %v", err) + } + if !containsJoin(due, dueRow.ID) { + t.Fatalf("GetDueJoinDeadlines did not include the past-deadline open row %d", dueRow.ID) + } + if containsJoin(due, futureRow.ID) { + t.Fatalf("GetDueJoinDeadlines unexpectedly included the future-deadline row %d", futureRow.ID) + } + if containsJoin(due, firedRow.ID) { + t.Fatalf("GetDueJoinDeadlines unexpectedly included the fired (terminal) row %d", firedRow.ID) + } +} + +// TestJoins_List covers ListJoins for a specific workspace and the all +// ("" / "*") case. +func TestJoins_List(t *testing.T) { + ctx := context.Background() + store := newJoinTestStore(t) + + tag := uniqueName(t, "join") + ws := "ws-" + tag + a, err := store.EnsureJoin(ctx, &wfstore.Join{ + JoinName: "jn-a-" + tag, Workspace: ws, CorrelationKey: "ck-a-" + tag, + Mode: wfstore.JoinModeCount, Status: wfstore.JoinStatusOpen, + }) + if err != nil { + t.Fatalf("EnsureJoin (a): %v", err) + } + other, err := store.EnsureJoin(ctx, &wfstore.Join{ + JoinName: "jn-b-" + tag, Workspace: "other-" + tag, CorrelationKey: "ck-b-" + tag, + Mode: wfstore.JoinModeCount, Status: wfstore.JoinStatusOpen, + }) + if err != nil { + t.Fatalf("EnsureJoin (other): %v", err) + } + + scoped, err := store.ListJoins(ctx, ws) + if err != nil { + t.Fatalf("ListJoins (scoped): %v", err) + } + if !containsJoin(scoped, a.ID) { + t.Fatalf("ListJoins(%q) did not include row %d", ws, a.ID) + } + if containsJoin(scoped, other.ID) { + t.Fatalf("ListJoins(%q) unexpectedly included other-workspace row %d", ws, other.ID) + } + + all, err := store.ListJoins(ctx, "") + if err != nil { + t.Fatalf("ListJoins (all): %v", err) + } + if !containsJoin(all, a.ID) || !containsJoin(all, other.ID) { + t.Fatalf("ListJoins(\"\") did not include both rows %d and %d", a.ID, other.ID) + } +} diff --git a/server/internal/storage/workflow/sqlite/store.go b/server/internal/storage/workflow/sqlite/store.go index fec0512..0b1ef31 100644 --- a/server/internal/storage/workflow/sqlite/store.go +++ b/server/internal/storage/workflow/sqlite/store.go @@ -810,6 +810,168 @@ func scanSchedules(rows *sql.Rows) ([]Schedule, error) { return schedules, rows.Err() } +// ============================================================================= +// Joins — workflow_joins table +// ============================================================================= + +func (s *Store) EnsureJoin(ctx context.Context, j *Join) (*Join, error) { + now := nowUTC() + query := ` + INSERT INTO workflow_joins (join_name, workspace, correlation_key, mode, + expected_count, arrived_count, dirty, status, + on_complete, on_timeout, on_partial_failure, + deadline_at, linger_until, created_at, updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT (join_name, workspace, correlation_key) DO NOTHING + ` + _, err := s.db.ExecContext(ctx, query, + j.JoinName, j.Workspace, j.CorrelationKey, j.Mode, + j.ExpectedCount, j.ArrivedCount, boolToInt(j.Dirty), j.Status, + nullableText(j.OnComplete), nullableText(j.OnTimeout), nullableText(j.OnPartialFailure), + formatTimePtr(j.DeadlineAt), formatTimePtr(j.LingerUntil), now, now, + ) + if err != nil { + return nil, fmt.Errorf("ensure join: %w", err) + } + return s.GetJoin(ctx, j.JoinName, j.Workspace, j.CorrelationKey) +} + +func (s *Store) GetJoin(ctx context.Context, joinName, workspace, correlationKey string) (*Join, error) { + query := ` + SELECT id, join_name, workspace, correlation_key, mode, expected_count, + arrived_count, dirty, status, on_complete, on_timeout, on_partial_failure, + deadline_at, linger_until, created_at, updated_at + FROM workflow_joins + WHERE join_name = ? AND workspace = ? AND correlation_key = ? + ` + rows, err := s.db.QueryContext(ctx, query, joinName, workspace, correlationKey) + if err != nil { + return nil, fmt.Errorf("get join: %w", err) + } + defer rows.Close() + + joins, err := scanJoins(rows) + if err != nil { + return nil, err + } + if len(joins) == 0 { + return nil, nil + } + return &joins[0], nil +} + +func (s *Store) UpdateJoinArrived(ctx context.Context, id int64, arrivedCount int64) error { + query := `UPDATE workflow_joins SET arrived_count = ?, updated_at = ? WHERE id = ?` + _, err := s.db.ExecContext(ctx, query, arrivedCount, nowUTC(), id) + return err +} + +func (s *Store) SetJoinExpected(ctx context.Context, id int64, expected int64) error { + query := `UPDATE workflow_joins SET expected_count = ?, updated_at = ? WHERE id = ?` + _, err := s.db.ExecContext(ctx, query, expected, nowUTC(), id) + return err +} + +func (s *Store) SetJoinDirty(ctx context.Context, id int64, dirty bool) error { + query := `UPDATE workflow_joins SET dirty = ?, updated_at = ? WHERE id = ?` + _, err := s.db.ExecContext(ctx, query, boolToInt(dirty), nowUTC(), id) + return err +} + +func (s *Store) MarkJoinTerminal(ctx context.Context, id int64, status string, lingerUntil time.Time) error { + query := `UPDATE workflow_joins SET status = ?, linger_until = ?, updated_at = ? WHERE id = ?` + _, err := s.db.ExecContext(ctx, query, status, formatTime(lingerUntil), nowUTC(), id) + return err +} + +func (s *Store) GetDueJoinDeadlines(ctx context.Context, now time.Time) ([]Join, error) { + // No FOR UPDATE SKIP LOCKED — SQLite is single-writer; the join engine + // is the only consumer of due deadlines in lite mode. + query := ` + SELECT id, join_name, workspace, correlation_key, mode, expected_count, + arrived_count, dirty, status, on_complete, on_timeout, on_partial_failure, + deadline_at, linger_until, created_at, updated_at + FROM workflow_joins + WHERE status = 'open' AND deadline_at IS NOT NULL AND deadline_at <= ? + ORDER BY deadline_at ASC + ` + rows, err := s.db.QueryContext(ctx, query, formatTime(now)) + if err != nil { + return nil, fmt.Errorf("get due join deadlines: %w", err) + } + defer rows.Close() + return scanJoins(rows) +} + +func (s *Store) ListJoins(ctx context.Context, workspace string) ([]Join, error) { + query := ` + SELECT id, join_name, workspace, correlation_key, mode, expected_count, + arrived_count, dirty, status, on_complete, on_timeout, on_partial_failure, + deadline_at, linger_until, created_at, updated_at + FROM workflow_joins + ` + var rows *sql.Rows + var err error + if workspace == "" || workspace == "*" { + query += " ORDER BY created_at DESC" + rows, err = s.db.QueryContext(ctx, query) + } else { + query += " WHERE workspace = ? ORDER BY created_at DESC" + rows, err = s.db.QueryContext(ctx, query, workspace) + } + if err != nil { + return nil, fmt.Errorf("list joins: %w", err) + } + defer rows.Close() + return scanJoins(rows) +} + +// scanJoins scans multiple join rows. Handles inline time.Time parsing for +// the nullable deadline_at / linger_until columns, the INTEGER dirty flag, +// and the nullable expected_count (*int64). +func scanJoins(rows *sql.Rows) ([]Join, error) { + var joins []Join + for rows.Next() { + var j Join + var dirtyInt int + var expectedRaw sql.NullInt64 + var onComplete, onTimeout, onPartialFailure sql.NullString + var deadlineAtRaw, lingerUntilRaw sql.NullString + var createdAtStr, updatedAtStr string + if err := rows.Scan( + &j.ID, &j.JoinName, &j.Workspace, &j.CorrelationKey, &j.Mode, &expectedRaw, + &j.ArrivedCount, &dirtyInt, &j.Status, &onComplete, &onTimeout, &onPartialFailure, + &deadlineAtRaw, &lingerUntilRaw, &createdAtStr, &updatedAtStr, + ); err != nil { + return nil, fmt.Errorf("scan join: %w", err) + } + j.Dirty = dirtyInt != 0 + j.OnComplete = onComplete.String + j.OnTimeout = onTimeout.String + j.OnPartialFailure = onPartialFailure.String + if expectedRaw.Valid { + v := expectedRaw.Int64 + j.ExpectedCount = &v + } + j.CreatedAt, _ = parseTime(createdAtStr) + j.UpdatedAt, _ = parseTime(updatedAtStr) + if deadlineAtRaw.Valid { + t, parseErr := parseTime(deadlineAtRaw.String) + if parseErr == nil { + j.DeadlineAt = &t + } + } + if lingerUntilRaw.Valid { + t, parseErr := parseTime(lingerUntilRaw.String) + if parseErr == nil { + j.LingerUntil = &t + } + } + joins = append(joins, j) + } + return joins, rows.Err() +} + // ============================================================================= // State machines — workflow_state_machines + workflow_state_machine_instances // ============================================================================= @@ -1208,6 +1370,15 @@ func isDuplicateColumnError(err error) bool { // Helpers // ============================================================================= +// nullableText stores an empty string as SQL NULL so optional TEXT columns +// (on_complete/on_timeout/on_partial_failure) stay NULL rather than '' when unset. +func nullableText(s string) interface{} { + if s == "" { + return nil + } + return s +} + // boolToInt converts a Go bool to a SQLite INTEGER (0/1). func boolToInt(b bool) int { if b { @@ -1225,6 +1396,7 @@ type ( WorkflowExecution = workflow.WorkflowExecution StepState = workflow.StepState Schedule = workflow.Schedule + Join = workflow.Join StateMachineDef = workflow.StateMachineDef StateMachineInstance = workflow.StateMachineInstance ) diff --git a/server/internal/storage/workflow/store.go b/server/internal/storage/workflow/store.go index 4985f6b..a8b1e86 100644 --- a/server/internal/storage/workflow/store.go +++ b/server/internal/storage/workflow/store.go @@ -234,6 +234,42 @@ type Store interface { // max_concurrent=1 enforcement path. SetScheduleActiveTask(ctx context.Context, scheduleID, taskID string) error + // ========================================================================= + // Joins — workflow_joins table + // ========================================================================= + + // EnsureJoin inserts the row if absent (keyed by + // join_name+workspace+correlation_key), otherwise leaves the existing + // row untouched; always returns the live row. + EnsureJoin(ctx context.Context, j *Join) (*Join, error) + + // GetJoin returns the row identified by (joinName, workspace, + // correlationKey), or (nil, nil) when the row is absent. + GetJoin(ctx context.Context, joinName, workspace, correlationKey string) (*Join, error) + + // UpdateJoinArrived sets arrived_count (and updated_at) for the row id. + UpdateJoinArrived(ctx context.Context, id int64, arrivedCount int64) error + + // SetJoinExpected sets expected_count (arming) for the row id. + SetJoinExpected(ctx context.Context, id int64, expected int64) error + + // SetJoinDirty sets the dirty flag for the row id. + SetJoinDirty(ctx context.Context, id int64, dirty bool) error + + // MarkJoinTerminal sets status (one of fired|timed_out|cancelled) and + // linger_until for the row id. + MarkJoinTerminal(ctx context.Context, id int64, status string, lingerUntil time.Time) error + + // GetDueJoinDeadlines returns open joins whose deadline_at is <= now, + // ordered by deadline_at ASC. On postgres the query is suffixed with + // `FOR UPDATE SKIP LOCKED`; on sqlite the suffix is omitted + // (single-writer). + GetDueJoinDeadlines(ctx context.Context, now time.Time) ([]Join, error) + + // ListJoins returns joins for a workspace ("" or "*" => all), newest + // first (ORDER BY created_at DESC). + ListJoins(ctx context.Context, workspace string) ([]Join, error) + // ========================================================================= // State machines — workflow_state_machines + workflow_state_machine_instances // ========================================================================= diff --git a/server/internal/storage/workflow/types.go b/server/internal/storage/workflow/types.go index 7d73bee..d14aad6 100644 --- a/server/internal/storage/workflow/types.go +++ b/server/internal/storage/workflow/types.go @@ -29,6 +29,8 @@ type ( StepState = legacy.StepState // Schedule is a workflow_schedules row. Schedule = legacy.Schedule + // Join is a workflow_joins row. + Join = legacy.Join // StateMachineDef is a workflow_state_machines row. StateMachineDef = legacy.StateMachineDef // StateMachineInstance is a workflow_state_machine_instances row. @@ -61,3 +63,18 @@ const ( ScheduleTypeOnce = legacy.ScheduleTypeOnce ScheduleTypeEventDelayed = legacy.ScheduleTypeEventDelayed ) + +// Join mode values — values that land in workflow_joins.mode. +const ( + JoinModeCount = legacy.JoinModeCount + JoinModeSet = legacy.JoinModeSet + JoinModeCoalesce = legacy.JoinModeCoalesce +) + +// Join status values — values that land in workflow_joins.status. +const ( + JoinStatusOpen = legacy.JoinStatusOpen + JoinStatusFired = legacy.JoinStatusFired + JoinStatusTimedOut = legacy.JoinStatusTimedOut + JoinStatusCancelled = legacy.JoinStatusCancelled +) diff --git a/server/internal/tracing/meter.go b/server/internal/tracing/meter.go new file mode 100644 index 0000000..f0de93a --- /dev/null +++ b/server/internal/tracing/meter.go @@ -0,0 +1,73 @@ +package tracing + +import ( + "context" + "os" + + "go.opentelemetry.io/contrib/instrumentation/runtime" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc" + "go.opentelemetry.io/otel/sdk/resource" + sdkmetric "go.opentelemetry.io/otel/sdk/metric" + semconv "go.opentelemetry.io/otel/semconv/v1.26.0" + + versionpkg "github.com/scitrera/aether/internal/version" +) + +// metricResource builds the OTel resource used by the MeterProvider. It mirrors +// the inline resource construction in InitTracer (service.name) and additionally +// stamps service.namespace=scitrera and service.version so metrics carry the +// same identity as traces. The per-tenant OTel collector upserts +// scitrera.tenant, so the tenant attribute is intentionally not set here. +func metricResource(ctx context.Context, serviceName string) (*resource.Resource, error) { + return resource.New(ctx, + resource.WithAttributes( + semconv.ServiceNameKey.String(serviceName), + semconv.ServiceNamespaceKey.String("scitrera"), + semconv.ServiceVersionKey.String(versionpkg.Version), + ), + ) +} + +// InitMeter initializes OpenTelemetry metrics with an OTLP gRPC exporter and a +// periodic reader (default ~60s interval), and starts Go runtime instrumentation +// (go_* / process metrics: GC, goroutines, memory). It mirrors InitTracer: +// gated on the SAME OTEL_EXPORTER_OTLP_ENDPOINT env var — when unset, metrics +// are disabled (no-op) and the global MeterProvider stays the default no-op. +// Returns a shutdown function that should be deferred in main(). +func InitMeter(serviceName string) (shutdown func(context.Context) error, err error) { + endpoint := os.Getenv("OTEL_EXPORTER_OTLP_ENDPOINT") + if endpoint == "" { + // Metrics disabled — keep the no-op meter provider. + return func(context.Context) error { return nil }, nil + } + + ctx := context.Background() + + exporter, err := otlpmetricgrpc.New(ctx) + if err != nil { + return nil, err + } + + res, err := metricResource(ctx, serviceName) + if err != nil { + return nil, err + } + + mp := sdkmetric.NewMeterProvider( + sdkmetric.WithReader(sdkmetric.NewPeriodicReader(exporter)), + sdkmetric.WithResource(res), + ) + + otel.SetMeterProvider(mp) + + // Go runtime metrics (go_* / process: GC, goroutines, mem) per gateway. + if err := runtime.Start(runtime.WithMeterProvider(mp)); err != nil { + // Best-effort: shut the provider down so we don't leak the exporter, + // then surface the error to the caller. + _ = mp.Shutdown(ctx) + return nil, err + } + + return mp.Shutdown, nil +} diff --git a/server/internal/tracing/tracing.go b/server/internal/tracing/tracing.go index 15a5fb4..bd69bac 100644 --- a/server/internal/tracing/tracing.go +++ b/server/internal/tracing/tracing.go @@ -5,7 +5,7 @@ import ( "os" "go.opentelemetry.io/otel" - "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp" + "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc" "go.opentelemetry.io/otel/sdk/resource" sdktrace "go.opentelemetry.io/otel/sdk/trace" semconv "go.opentelemetry.io/otel/semconv/v1.26.0" @@ -16,7 +16,10 @@ import ( // Tracer is the package-level tracer used across the gateway. var Tracer trace.Tracer = noop.NewTracerProvider().Tracer("github.com/scitrera/aether/gateway") -// InitTracer initializes OpenTelemetry tracing with an OTLP HTTP exporter. +// InitTracer initializes OpenTelemetry tracing with an OTLP gRPC exporter. +// gRPC (port 4317) matches InitMeter's otlpmetricgrpc exporter so both ride the +// same OTEL_EXPORTER_OTLP_ENDPOINT (http:// scheme = insecure/plaintext); the +// OTLP/HTTP exporter would POST /v1/traces to the gRPC port and fail. // If OTEL_EXPORTER_OTLP_ENDPOINT is not set, tracing is disabled (no-op). // Returns a shutdown function that should be deferred in main(). func InitTracer(serviceName string) (shutdown func(context.Context) error, err error) { @@ -28,7 +31,7 @@ func InitTracer(serviceName string) (shutdown func(context.Context) error, err e ctx := context.Background() - exporter, err := otlptracehttp.New(ctx) + exporter, err := otlptracegrpc.New(ctx) if err != nil { return nil, err } diff --git a/server/internal/version/version.go b/server/internal/version/version.go index 059c1a0..44bae6a 100644 --- a/server/internal/version/version.go +++ b/server/internal/version/version.go @@ -2,4 +2,4 @@ package version // Version is the current release version of the Aether gateway. // This is updated automatically by scripts/update-versions.py. -const Version = "0.2.1" +const Version = "0.2.2" diff --git a/server/internal/workflow/admin.go b/server/internal/workflow/admin.go index 437cfae..0e3ce17 100644 --- a/server/internal/workflow/admin.go +++ b/server/internal/workflow/admin.go @@ -64,6 +64,11 @@ func NewAdminServer(port int, apiKey string, store WorkflowStore, router *Router api.HandleFunc("/executions/{id}", s.getExecution).Methods("GET") api.HandleFunc("/executions/{id}/cancel", s.cancelExecution).Methods("POST") + // Joins + api.HandleFunc("/joins", s.listJoins).Methods("GET") + api.HandleFunc("/joins/{name}/{correlationKey}", s.getJoin).Methods("GET") + api.HandleFunc("/joins/{name}/{correlationKey}/cancel", s.cancelJoin).Methods("POST") + // State Machines api.HandleFunc("/statemachines", s.listStateMachines).Methods("GET") api.HandleFunc("/statemachines", s.createStateMachine).Methods("POST") @@ -427,6 +432,57 @@ func (s *AdminServer) cancelExecution(w http.ResponseWriter, r *http.Request) { writeJSON(w, http.StatusOK, map[string]string{"status": "cancelled"}) } +// ============================================================================= +// Join endpoints +// ============================================================================= + +func (s *AdminServer) listJoins(w http.ResponseWriter, r *http.Request) { + workspace := r.URL.Query().Get("workspace") + joins, err := s.store.ListJoins(r.Context(), workspace) + if err != nil { + writeJSON(w, http.StatusInternalServerError, map[string]string{"error": err.Error()}) + return + } + writeJSON(w, http.StatusOK, toJoinViews(joins)) +} + +func (s *AdminServer) getJoin(w http.ResponseWriter, r *http.Request) { + vars := mux.Vars(r) + j, err := s.store.GetJoin(r.Context(), vars["name"], r.URL.Query().Get("workspace"), vars["correlationKey"]) + if err != nil { + writeJSON(w, http.StatusInternalServerError, map[string]string{"error": err.Error()}) + return + } + if j == nil { + writeJSON(w, http.StatusNotFound, map[string]string{"error": "join not found"}) + return + } + writeJSON(w, http.StatusOK, toJoinView(*j)) +} + +func (s *AdminServer) cancelJoin(w http.ResponseWriter, r *http.Request) { + vars := mux.Vars(r) + j, err := s.store.GetJoin(r.Context(), vars["name"], r.URL.Query().Get("workspace"), vars["correlationKey"]) + if err != nil { + writeJSON(w, http.StatusInternalServerError, map[string]string{"error": err.Error()}) + return + } + if j == nil { + writeJSON(w, http.StatusNotFound, map[string]string{"error": "join not found"}) + return + } + if j.Status != JoinStatusOpen { + // Already terminal — cancellation is idempotent. + writeJSON(w, http.StatusOK, map[string]string{"status": j.Status}) + return + } + if err := s.store.MarkJoinTerminal(r.Context(), j.ID, JoinStatusCancelled, time.Now().Add(time.Minute)); err != nil { + writeJSON(w, http.StatusInternalServerError, map[string]string{"error": err.Error()}) + return + } + writeJSON(w, http.StatusOK, map[string]string{"status": JoinStatusCancelled}) +} + // ============================================================================= // State Machine endpoints // ============================================================================= diff --git a/server/internal/workflow/dag.go b/server/internal/workflow/dag.go index edc6a03..afcf83d 100644 --- a/server/internal/workflow/dag.go +++ b/server/internal/workflow/dag.go @@ -170,32 +170,22 @@ func (d *DAGEngine) ProcessStepCompletion(ctx context.Context, executionID, step return fmt.Errorf("parse DAG definition: %w", err) } - // Handle failure with retry + // Handle failure. Phase 2: retry scheduling is delegated to the task + // store via the per-task RetryPolicy attached at CreateTask time. + // The WE no longer re-dispatches steps directly — by the time we see + // ProcessStepCompletion(success=false) the underlying task has + // already exhausted its policy (or the worker called Complete with + // permanent-failure semantics). We honor the optional on_failure + // handler hop and otherwise fail the execution. if !success { step := d.findStep(&dagDef, stepID) - if step != nil && step.Retry.MaxAttempts > 0 { - stepStates, _ := d.store.GetStepStates(ctx, executionID) - for _, ss := range stepStates { - if ss.StepID == stepID && ss.Attempt < step.Retry.MaxAttempts { - log.Info(). - Str("execution_id", executionID). - Str("step_id", stepID). - Int("attempt", ss.Attempt+1). - Msg("retrying step") - if err := d.store.IncrementStepAttempt(ctx, executionID, stepID); err != nil { - log.Warn().Err(err).Str("execution_id", executionID).Str("step_id", stepID).Msg("failed to increment step attempt; retry counter may drift") - } - return d.advanceExecution(ctx, executionID, &dagDef, exec.TriggerData) - } - } - } // Check for on_failure handler if step != nil && step.OnFailure != "" { return d.advanceExecution(ctx, executionID, &dagDef, exec.TriggerData) } - // No retry, no handler: fail the execution + // No handler: fail the execution. log.Warn(). Str("execution_id", executionID). Str("step_id", stepID). @@ -330,7 +320,14 @@ func (d *DAGEngine) advanceExecution(ctx context.Context, executionID string, da } payload, _ := json.Marshal(action) - if err := d.executor.CreateTask(action.Workspace, action.Agent, metadata, payload); err != nil { + // Pass the step's retry config so the task store handles backoff + // scheduling on FailTask. Nil = legacy default behavior. + var retry *RetryConfig + if dagStep.Retry.MaxAttempts > 0 { + r := dagStep.Retry + retry = &r + } + if err := d.executor.CreateTask(action.Workspace, action.Agent, metadata, payload, retry); err != nil { log.Error().Err(err). Str("execution_id", executionID). Str("step_id", dagStep.ID). diff --git a/server/internal/workflow/executor.go b/server/internal/workflow/executor.go index 51f7322..f6fdca1 100644 --- a/server/internal/workflow/executor.go +++ b/server/internal/workflow/executor.go @@ -23,6 +23,32 @@ type ActionDef struct { TaskType string `json:"task_type,omitempty" yaml:"task_type,omitempty"` TargetImplementation string `json:"target_implementation,omitempty" yaml:"target_implementation,omitempty"` Payload any `json:"payload,omitempty" yaml:"payload,omitempty"` + // Optional retry policy for create_task actions. When set, the task + // store re-pends the task with a policy-driven next_retry_at on + // FailTask. Omitted = legacy hard-coded max_retries=3 behavior. + Retry *RetryConfig `json:"retry,omitempty" yaml:"retry,omitempty"` + // emit_event field: the event name to publish onto the event plane (Type == + // "emit_event"). Payload carries the event data. Enables join on_complete to + // chain into further rules/joins. + EventName string `json:"event_name,omitempty" yaml:"event_name,omitempty"` + // IdempotencyKey, when set on a create_task action, is forwarded to the + // gateway which dedups creation on it (exactly-once downstream). Joins set it + // to their instance key so a completion and a timeout sweep yield one task. + IdempotencyKey string `json:"idempotency_key,omitempty" yaml:"idempotency_key,omitempty"` + // CorrelationID stamps the spawned task with a fan-out/fan-in correlation id + // (the barrier/group id a downstream join matches on). + CorrelationID string `json:"correlation_id,omitempty" yaml:"correlation_id,omitempty"` + // CompletionEvent opts the spawned task into "feed B": it emits a domain + // event onto the event plane at its terminal status, which a join can gather. + CompletionEvent *CompletionEventConfig `json:"completion_event,omitempty" yaml:"completion_event,omitempty"` +} + +// CompletionEventConfig is the create_task-destination form of a task's feed-B +// opt-in (a subset of the proto TaskCompletionEvent; on_statuses defaults to all +// terminal statuses server-side). +type CompletionEventConfig struct { + Enabled bool `json:"enabled" yaml:"enabled"` + EventName string `json:"event_name,omitempty" yaml:"event_name,omitempty"` } // ToolCallPayload is the JSON structure sent as the message payload @@ -124,11 +150,47 @@ func (e *Executor) dispatchCreateTask(action *ActionDef) error { Str("target_impl", targetImpl). Msg("dispatching create_task action") - return e.CreateTaskWithType(workspace, action.TaskType, targetImpl, metadata, payload) + var completion *pb.TaskCompletionEvent + if action.CompletionEvent != nil { + completion = &pb.TaskCompletionEvent{ + Enabled: action.CompletionEvent.Enabled, + EventName: action.CompletionEvent.EventName, + } + } + + return e.CreateTaskWithType(workspace, action.TaskType, targetImpl, metadata, payload, action.Retry, action.IdempotencyKey, action.CorrelationID, completion) } -// CreateTaskWithType creates an Aether task with the given task type. -func (e *Executor) CreateTaskWithType(workspace, taskType, targetImpl string, metadata map[string]string, payload []byte) error { +// EmitEvent publishes a synthetic event onto the event plane (event.*) as a +// MessageType_EVENT message, mirroring an SDK client's SendEvent. Used by a +// join's on_complete (Type == "emit_event") to chain into further rules/joins. +func (e *Executor) EmitEvent(action *ActionDef) error { + if action.EventName == "" { + return fmt.Errorf("emit_event requires event_name") + } + workspace := action.Workspace + if workspace == "" { + workspace = e.defaultWorkspace + } + payload := map[string]any{ + "source_agent": "workflow-engine", + "workspace": workspace, + "event_names": []string{action.EventName}, + "data": action.Payload, + } + data, err := json.Marshal(payload) + if err != nil { + return fmt.Errorf("marshal emit_event payload: %w", err) + } + log.Debug().Str("event", action.EventName).Str("workspace", workspace).Msg("emitting event") + return e.client.SendMessage(aether.EventWildcardTopic(), data, pb.MessageType_EVENT) +} + +// CreateTaskWithType creates an Aether task with the given task type. When +// retry is non-nil, it is translated to a proto RetryPolicy and attached to +// the request so the task store handles backoff scheduling on FailTask. +// Pass nil to keep the legacy hard-coded max_retries=3 behavior. +func (e *Executor) CreateTaskWithType(workspace, taskType, targetImpl string, metadata map[string]string, payload []byte, retry *RetryConfig, idempotencyKey, correlationID string, completion *pb.TaskCompletionEvent) error { log.Debug(). Str("workspace", workspace). Str("task_type", taskType). @@ -144,6 +206,10 @@ func (e *Executor) CreateTaskWithType(workspace, taskType, targetImpl string, me TargetImplementation: targetImpl, Metadata: metadata, Payload: payload, + RetryPolicy: retryConfigToProto(retry), + IdempotencyKey: idempotencyKey, + CorrelationId: correlationID, + CompletionEvent: completion, }, }, } @@ -172,20 +238,34 @@ func (e *Executor) DispatchActionToTopic(topic string, action *ActionDef) error } // DispatchTransformResult sends the result of a template transformation. +// +// Carries the create_task fields through so an event-triggered rule can spawn +// an Aether POOL task (Type == "create_task"). When Type is empty the action +// falls through to the historical "message" (tool-call) dispatch. func (e *Executor) DispatchTransformResult(result *TransformResult) error { action := &ActionDef{ - Agent: result.Agent, - ToolName: result.ToolName, - Arguments: result.Arguments, - Workspace: result.Workspace, - Metadata: result.Metadata, + Type: result.Type, + Agent: result.Agent, + ToolName: result.ToolName, + Arguments: result.Arguments, + Workspace: result.Workspace, + Metadata: result.Metadata, + TaskType: result.TaskType, + TargetImplementation: result.TargetImplementation, + Payload: result.Payload, + CorrelationID: result.CorrelationID, + CompletionEvent: result.CompletionEvent, } return e.DispatchAction(action) } // CreateTask creates an Aether task targeting an agent for DAG step execution. // Uses raw Send() since WorkflowEngineClient doesn't expose CreateTask directly. -func (e *Executor) CreateTask(workspace, agentImpl string, metadata map[string]string, payload []byte) error { +// +// When retry is non-nil, it is translated to a proto RetryPolicy and +// attached to the request so the task store handles backoff scheduling +// on FailTask. The WE no longer drives backoff itself. +func (e *Executor) CreateTask(workspace, agentImpl string, metadata map[string]string, payload []byte, retry *RetryConfig) error { log.Debug(). Str("workspace", workspace). Str("agent_impl", agentImpl). @@ -200,9 +280,36 @@ func (e *Executor) CreateTask(workspace, agentImpl string, metadata map[string]s TargetImplementation: agentImpl, Metadata: metadata, Payload: payload, + RetryPolicy: retryConfigToProto(retry), }, }, } return e.client.Send(msg) } + +// retryConfigToProto translates a DAG-step RetryConfig into the proto +// RetryPolicy understood by the task store. Returns nil when the step did +// not declare retries (preserving legacy default-attempts behavior). +func retryConfigToProto(cfg *RetryConfig) *pb.RetryPolicy { + if cfg == nil || cfg.MaxAttempts <= 0 { + return nil + } + backoff := pb.BackoffStrategy_BACKOFF_STRATEGY_EXPONENTIAL + switch cfg.Backoff { + case "constant", "fixed": + backoff = pb.BackoffStrategy_BACKOFF_STRATEGY_FIXED + case "exponential", "": + backoff = pb.BackoffStrategy_BACKOFF_STRATEGY_EXPONENTIAL + case "linear": + // Linear isn't a first-class strategy in the store; map to + // EXPONENTIAL with no max cap so attempts scale up similarly. + backoff = pb.BackoffStrategy_BACKOFF_STRATEGY_EXPONENTIAL + } + return &pb.RetryPolicy{ + MaxAttempts: int32(cfg.MaxAttempts), + Backoff: backoff, + InitialDelayMs: 1000, // 1s base; DAG step authors can refine when the schema grows. + JitterFactor: 0.1, + } +} diff --git a/server/internal/workflow/join.go b/server/internal/workflow/join.go new file mode 100644 index 0000000..5d55273 --- /dev/null +++ b/server/internal/workflow/join.go @@ -0,0 +1,570 @@ +package workflow + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "time" + + "github.com/rs/zerolog/log" + + "github.com/scitrera/aether/internal/kv" + "github.com/scitrera/aether/sdk/go/coord" +) + +// JoinSpec is the declarative description of a fan-in / barrier / coalesce +// destination, parsed from the `join:` block of a rendered rule destination. +// +// CorrelationKey / ExpectedCount / DedupKey are expr-lang expressions evaluated +// against the arrival env ({input, source}) — the same dialect as a rule's +// TriggerCondition. OnComplete / OnTimeout are ordinary ActionDefs whose Go +// text/template fields were already rendered when the enclosing destination was +// transformed, so by the time the engine sees them they are concrete. +type JoinSpec struct { + Name string `yaml:"name" json:"name"` + CorrelationKey string `yaml:"correlation_key" json:"correlation_key"` + Mode string `yaml:"mode" json:"mode"` // count | coalesce | set + + // ExpectedCount (count mode): expr-lang yielding an integer. Empty until an + // arm_on_event arrival supplies it (dynamic-N). + ExpectedCount string `yaml:"expected_count" json:"expected_count"` + // ArmOnEvent: the event_name whose arrival carries ExpectedCount. Empty means + // ExpectedCount (if set) is evaluable on every arrival. + ArmOnEvent string `yaml:"arm_on_event" json:"arm_on_event"` + + // DedupKey: expr-lang yielding this arrival's dedup id. Empty disables dedup. + DedupKey string `yaml:"dedup_key" json:"dedup_key"` + + // ExpectedSet (set mode): expr-lang yielding a list of member ids; the join + // completes when every distinct member has reported (cardinality == len). + ExpectedSet string `yaml:"expected_set" json:"expected_set"` + // MemberKey (set mode): expr-lang yielding THIS arrival's member id. The set + // dedups members inherently, so set mode needs no separate dedup_key. + MemberKey string `yaml:"member_key" json:"member_key"` + + // Window (coalesce mode): debounce/cooldown duration, e.g. "5s". Within the + // window after a firing, further arrivals coalesce (set dirty) instead of + // firing. Empty/0 ⇒ no debounce (each arrival fires). + Window string `yaml:"window" json:"window"` + // Timeout: barrier deadline, e.g. "15m". Drives the deadline sweep. + Timeout string `yaml:"timeout" json:"timeout"` + // Linger: late-arrival retention after terminal, e.g. "1m". Defaults applied + // by the engine when empty. + Linger string `yaml:"linger" json:"linger"` + + OnComplete *ActionDef `yaml:"on_complete" json:"on_complete"` + OnTimeout *ActionDef `yaml:"on_timeout" json:"on_timeout"` + OnPartialFailure string `yaml:"on_partial_failure" json:"on_partial_failure"` +} + +// joinCounterCeiling is a sentinel ceiling that the per-correlation arrival +// counter will never reach, so IncrementIf(+1) always applies and returns this +// arrival's monotonic sequence number. Completeness is judged against the +// (separately tracked) expected count, and exactly-once firing is gated by a +// SetNX fire-marker — not by the counter hitting a real ceiling. +const joinCounterCeiling int64 = 1 << 40 + +// joinDefaultLinger is the retention window after a join goes terminal, during +// which late arrivals are dropped rather than resurrecting a new instance. +const joinDefaultLinger = time.Minute + +// joinStore is the narrow slice of WorkflowStore the join engine needs. Keeping +// it small (interface segregation) makes the engine unit-testable with a tiny +// in-memory fake; the full *Store / WorkflowStore satisfies it structurally. +type joinStore interface { + EnsureJoin(ctx context.Context, j *Join) (*Join, error) + GetJoin(ctx context.Context, joinName, workspace, correlationKey string) (*Join, error) + UpdateJoinArrived(ctx context.Context, id int64, arrivedCount int64) error + SetJoinExpected(ctx context.Context, id int64, expected int64) error + SetJoinDirty(ctx context.Context, id int64, dirty bool) error + MarkJoinTerminal(ctx context.Context, id int64, status string, lingerUntil time.Time) error +} + +// actionDispatcher is the slice of Executor the join engine uses to run a +// fired action. *Executor satisfies it; tests inject a recording fake. +type actionDispatcher interface { + DispatchAction(action *ActionDef) error + EmitEvent(action *ActionDef) error +} + +// joinSet is the atomic set primitive backing set-mode membership: SetAdd +// returns whether the member was newly added and the cardinality after the add. +// Production adapts *aether.KV (SetAddSync) to it; tests inject a fake. nil when +// set mode is unavailable (e.g. a backend without the gRPC set ops wired). +type joinSet interface { + SetAdd(ctx context.Context, key, member string, ttl time.Duration) (added bool, cardinality int64, err error) +} + +// JoinEngine drives fan-in joins. It depends on abstract coord primitives +// (Counter for the atomic arrival counter, Locker for SetNX-style dedup and +// fire-marker gates) so it is unit-testable without a live gateway; production +// wires these from the WorkflowEngine client's KV (client.KV().Counter/.Locker +// on the global scope, keys under the reserved _sys/coord/ infra fast-path). +type JoinEngine struct { + store joinStore + expr *ExprEngine + dispatcher actionDispatcher + counter coord.Counter + locker coord.Locker + set joinSet + + defaultWorkspace string +} + +// NewJoinEngine constructs a JoinEngine. counter/locker/set must be bound to a +// shared scope (typically global) so all replicas rendezvous on the same keys. +// set may be nil if the deployment has not wired the gRPC set ops (set-mode +// joins then return an error rather than silently mis-firing). +func NewJoinEngine(store joinStore, expr *ExprEngine, dispatcher actionDispatcher, counter coord.Counter, locker coord.Locker, set joinSet, defaultWorkspace string) *JoinEngine { + return &JoinEngine{ + store: store, + expr: expr, + dispatcher: dispatcher, + counter: counter, + locker: locker, + set: set, + defaultWorkspace: defaultWorkspace, + } +} + +// HandleArrival processes one event arrival against a join destination. It +// evaluates the correlation key, applies dedup, mirrors arrival state to the +// store, and fires OnComplete exactly once when the join completes. +func (je *JoinEngine) HandleArrival(ctx context.Context, spec *JoinSpec, env map[string]any, eventName, workspace string) error { + if spec == nil || spec.Name == "" { + return fmt.Errorf("join: spec.name is required") + } + if workspace == "" { + workspace = je.defaultWorkspace + } + + corr, err := je.evalString(spec.CorrelationKey, env) + if err != nil { + return fmt.Errorf("join %q: correlation_key eval: %w", spec.Name, err) + } + if corr == "" { + return fmt.Errorf("join %q: correlation_key evaluated empty (degenerate key) — producer must stamp a shared correlation id", spec.Name) + } + + mode := spec.Mode + if mode == "" { + mode = JoinModeCount + } + base := joinBaseKey(spec.Name, workspace, corr) + ttl := je.retentionTTL(spec) + + // Dedup (N3): the first arrival with a given dedup id wins the slot; repeats + // are dropped. Uses TryAcquire (SetNX) keyed per dedup id under the join base. + if spec.DedupKey != "" { + dedupID, derr := je.evalString(spec.DedupKey, env) + if derr != nil { + return fmt.Errorf("join %q: dedup_key eval: %w", spec.Name, derr) + } + if dedupID != "" { + fresh, aerr := je.locker.TryAcquire(ctx, base+"/d/"+hashSeg(dedupID), "1", ttl) + if aerr != nil { + return fmt.Errorf("join %q: dedup check: %w", spec.Name, aerr) + } + if !fresh { + log.Debug().Str("join", spec.Name).Str("corr", corr).Str("dedup", dedupID).Msg("join: dropping duplicate arrival") + return nil + } + } + } + + // Ensure the durable mirror row exists (idempotent). expected_count is set + // now only when known on this arrival (literal/expr not gated by arm_on_event). + expected, expectedKnown, eerr := je.evalExpected(spec, env, eventName) + if eerr != nil { + return fmt.Errorf("join %q: expected_count eval: %w", spec.Name, eerr) + } + row, err := je.ensureRow(ctx, spec, mode, workspace, corr, expected, expectedKnown) + if err != nil { + return fmt.Errorf("join %q: ensure row: %w", spec.Name, err) + } + + switch mode { + case JoinModeCoalesce: + return je.handleCoalesceArrival(ctx, spec, row, base, ttl) + case JoinModeCount: + return je.handleCountArrival(ctx, spec, row, base, eventName, env, ttl) + case JoinModeSet: + return je.handleSetArrival(ctx, spec, row, base, env, ttl) + default: + return fmt.Errorf("join %q: unknown mode %q", spec.Name, mode) + } +} + +// handleCountArrival implements the count barrier. A *member* arrival bumps the +// atomic counter (its returned value is the arrival's sequence number) and fires +// when the count reaches the known expected value. An *arming* arrival — one +// whose event name is arm_on_event — supplies expected_count but is NOT itself a +// member (arming typically rides a different event, e.g. a manifest), so it does +// not increment the counter; it just records expected and re-checks completeness +// against the current count. Exactly-once firing is gated by the fire-marker. +func (je *JoinEngine) handleCountArrival(ctx context.Context, spec *JoinSpec, row *Join, base, eventName string, env map[string]any, ttl time.Duration) error { + if spec.ArmOnEvent != "" && eventName == spec.ArmOnEvent { + armVal, ok, err := je.evalInt(spec.ExpectedCount, env) + if err != nil { + return fmt.Errorf("join %q: arm expected_count eval: %w", spec.Name, err) + } + if !ok { + return nil // arm event without a usable count — nothing to arm + } + if err := je.store.SetJoinExpected(ctx, row.ID, armVal); err != nil { + return fmt.Errorf("join %q: set expected: %w", spec.Name, err) + } + // Read the current member count (delta 0) so an arming that lands after + // all members still completes the barrier. + cur, _, rerr := je.counter.IncrementIf(ctx, base, 0, joinCounterCeiling) + if rerr != nil { + return fmt.Errorf("join %q: read counter: %w", spec.Name, rerr) + } + if cur >= armVal { + return je.fire(ctx, spec, row, base, ttl, spec.OnComplete, false) + } + return nil + } + + // Member arrival: increment and mirror. + value, _, err := je.counter.IncrementIf(ctx, base, 1, joinCounterCeiling) + if err != nil { + return fmt.Errorf("join %q: increment: %w", spec.Name, err) + } + if err := je.store.UpdateJoinArrived(ctx, row.ID, value); err != nil { + log.Warn().Err(err).Str("join", spec.Name).Msg("join: mirror arrived_count failed (non-fatal)") + } + if row.ExpectedCount != nil && value >= *row.ExpectedCount { + return je.fire(ctx, spec, row, base, ttl, spec.OnComplete, false) + } + return nil +} + +// handleCoalesceArrival collapses a burst into a single firing. The first +// arrival acquires the active/cooldown slot and fires; arrivals while the slot +// is held set the dirty flag (a trailing run is driven later by the deadline +// sweep or the next post-cooldown arrival). +func (je *JoinEngine) handleCoalesceArrival(ctx context.Context, spec *JoinSpec, row *Join, base string, ttl time.Duration) error { + cooldown := je.parseDuration(spec.Window, 0) + gateTTL := cooldown + if gateTTL <= 0 { + gateTTL = ttl // no debounce window: still bound the gate so a crashed firer self-heals + } + won, err := je.locker.TryAcquire(ctx, base+"/active", "1", gateTTL) + if err != nil { + return fmt.Errorf("join %q: coalesce gate: %w", spec.Name, err) + } + if !won { + // A firing is in flight / cooling down — record that more work arrived. + if derr := je.store.SetJoinDirty(ctx, row.ID, true); derr != nil { + log.Warn().Err(derr).Str("join", spec.Name).Msg("join: set dirty failed (non-fatal)") + } + return nil + } + if derr := je.store.SetJoinDirty(ctx, row.ID, false); derr != nil { + log.Warn().Err(derr).Str("join", spec.Name).Msg("join: clear dirty failed (non-fatal)") + } + return je.runAction(ctx, spec, spec.OnComplete) +} + +// fire elects a single firer via the fire-marker gate and runs the given action +// exactly once, then marks the row terminal. degraded indicates a timeout-path +// firing (recorded as fired here; the deadline sweep sets timed_out separately). +func (je *JoinEngine) fire(ctx context.Context, spec *JoinSpec, row *Join, base string, ttl time.Duration, action *ActionDef, degraded bool) error { + won, err := je.locker.TryAcquire(ctx, base+"/fired", "1", ttl) + if err != nil { + return fmt.Errorf("join %q: fire gate: %w", spec.Name, err) + } + if !won { + return nil // another arrival already fired this instance + } + log.Info().Str("join", spec.Name).Str("corr", row.CorrelationKey).Int64("arrived", row.ArrivedCount).Bool("degraded", degraded).Msg("join: firing") + if action != nil { + action.IdempotencyKey = joinIdemKey(row) + } + if aerr := je.runAction(ctx, spec, action); aerr != nil { + return aerr + } + return je.store.MarkJoinTerminal(ctx, row.ID, JoinStatusFired, time.Now().Add(je.retentionLinger(spec))) +} + +// HandleDeadline is invoked by the scheduler sweep for an open join past its +// deadline. Policy on_partial_failure: "abort" marks timed_out without firing; +// "proceed"/"proceed_degraded"/"" fire the persisted on_timeout action (falling +// back to on_complete) exactly once via the fire-marker gate, then mark timed_out. +func (je *JoinEngine) HandleDeadline(ctx context.Context, j *Join) error { + if j.Status != JoinStatusOpen { + return nil + } + + base := joinBaseKey(j.JoinName, j.Workspace, j.CorrelationKey) + linger := joinDefaultLinger + term := time.Now().Add(linger) + + // Abort policy: the barrier failed — go terminal without firing any action. + if j.OnPartialFailure == "abort" { + if err := je.store.MarkJoinTerminal(ctx, j.ID, JoinStatusTimedOut, term); err != nil { + return fmt.Errorf("join %q: mark timed_out: %w", j.JoinName, err) + } + log.Info().Str("join", j.JoinName).Str("corr", j.CorrelationKey).Msg("join: deadline reached, abort policy — timed out without firing") + return nil + } + + // Proceed (possibly degraded): fire the timeout action, falling back to the + // completion action when no dedicated timeout action was persisted. + actionJSON := j.OnTimeout + if actionJSON == "" { + actionJSON = j.OnComplete + } + if actionJSON == "" { + // Nothing to fire — just go terminal. + if err := je.store.MarkJoinTerminal(ctx, j.ID, JoinStatusTimedOut, term); err != nil { + return fmt.Errorf("join %q: mark timed_out: %w", j.JoinName, err) + } + log.Info().Str("join", j.JoinName).Str("corr", j.CorrelationKey).Msg("join: deadline reached, no timeout action — timed out") + return nil + } + + var action ActionDef + if err := json.Unmarshal([]byte(actionJSON), &action); err != nil { + return fmt.Errorf("join %q: unmarshal timeout action: %w", j.JoinName, err) + } + + // Fire-marker gate: bounded TTL prevents a double-fire against a late + // completion arrival that might also reach the barrier. + won, err := je.locker.TryAcquire(ctx, base+"/fired", "1", joinDefaultLinger+time.Hour) + if err != nil { + return fmt.Errorf("join %q: deadline fire gate: %w", j.JoinName, err) + } + if won { + log.Info().Str("join", j.JoinName).Str("corr", j.CorrelationKey).Msg("join: deadline reached, firing timeout action") + action.IdempotencyKey = joinIdemKey(j) + if aerr := je.runAction(ctx, &JoinSpec{Name: j.JoinName}, &action); aerr != nil { + return fmt.Errorf("join %q: run timeout action: %w", j.JoinName, aerr) + } + } + + // The instance is terminal regardless of who won the fire-marker. + if err := je.store.MarkJoinTerminal(ctx, j.ID, JoinStatusTimedOut, term); err != nil { + return fmt.Errorf("join %q: mark timed_out: %w", j.JoinName, err) + } + return nil +} + +// marshalAction serializes an ActionDef to JSON text for durable persistence on +// the join row. nil yields "". A marshal error logs a warning and yields "". +func marshalAction(action *ActionDef) string { + if action == nil { + return "" + } + b, err := json.Marshal(action) + if err != nil { + log.Warn().Err(err).Msg("join: marshal action failed (non-fatal); action will not be persisted") + return "" + } + return string(b) +} + +// handleSetArrival implements the set barrier: each arrival adds its member id +// to a KV set (which inherently dedups), and the join fires when the set's +// cardinality reaches the expected_set size. The member whose add completes the +// set is the unique firer (added && cardinality>=size), gated by the fire-marker. +func (je *JoinEngine) handleSetArrival(ctx context.Context, spec *JoinSpec, row *Join, base string, env map[string]any, ttl time.Duration) error { + if je.set == nil { + return fmt.Errorf("join %q: set mode unavailable (set backend not wired)", spec.Name) + } + member, err := je.evalString(spec.MemberKey, env) + if err != nil { + return fmt.Errorf("join %q: member_key eval: %w", spec.Name, err) + } + if member == "" { + return fmt.Errorf("join %q: set mode requires a non-empty member_key", spec.Name) + } + + added, card, err := je.set.SetAdd(ctx, base+"/members", member, ttl) + if err != nil { + return fmt.Errorf("join %q: set add: %w", spec.Name, err) + } + if err := je.store.UpdateJoinArrived(ctx, row.ID, card); err != nil { + log.Warn().Err(err).Str("join", spec.Name).Msg("join: mirror set cardinality failed (non-fatal)") + } + + size, known, err := je.evalSetSize(spec, env) + if err != nil { + return fmt.Errorf("join %q: expected_set eval: %w", spec.Name, err) + } + if known && (row.ExpectedCount == nil || *row.ExpectedCount != size) { + if perr := je.store.SetJoinExpected(ctx, row.ID, size); perr != nil { + log.Warn().Err(perr).Str("join", spec.Name).Msg("join: mirror expected size failed (non-fatal)") + } + } + + if added && known && card >= size { + return je.fire(ctx, spec, row, base, ttl, spec.OnComplete, false) + } + return nil +} + +// evalSetSize evaluates expected_set to a list and returns its length. ok=false +// when the expression is empty or yields nil (size not yet known). +func (je *JoinEngine) evalSetSize(spec *JoinSpec, env map[string]any) (int64, bool, error) { + if spec.ExpectedSet == "" { + return 0, false, nil + } + v, err := je.expr.EvaluateAny(spec.ExpectedSet, env) + if err != nil { + return 0, false, err + } + switch list := v.(type) { + case nil: + return 0, false, nil + case []any: + return int64(len(list)), true, nil + case []string: + return int64(len(list)), true, nil + default: + return 0, false, fmt.Errorf("expected_set evaluated to %T, want a list", v) + } +} + +// runAction dispatches an OnComplete/OnTimeout action: a create_task or +// emit_event. nil is a no-op. +func (je *JoinEngine) runAction(ctx context.Context, spec *JoinSpec, action *ActionDef) error { + if action == nil { + return nil + } + if action.Type == "emit_event" { + return je.dispatcher.EmitEvent(action) + } + return je.dispatcher.DispatchAction(action) +} + +// ensureRow inserts the durable mirror row if absent. +func (je *JoinEngine) ensureRow(ctx context.Context, spec *JoinSpec, mode, workspace, corr string, expected int64, expectedKnown bool) (*Join, error) { + j := &Join{ + JoinName: spec.Name, + Workspace: workspace, + CorrelationKey: corr, + Mode: mode, + Status: JoinStatusOpen, + OnComplete: marshalAction(spec.OnComplete), + OnTimeout: marshalAction(spec.OnTimeout), + OnPartialFailure: spec.OnPartialFailure, + } + if expectedKnown { + j.ExpectedCount = &expected + } + if d := je.parseDuration(spec.Timeout, 0); d > 0 { + deadline := time.Now().Add(d) + j.DeadlineAt = &deadline + } + return je.store.EnsureJoin(ctx, j) +} + +// evalExpected returns the expected count when it is knowable on this arrival: +// either there is no arm gate, or this arrival IS the arm event. Otherwise the +// count stays unknown (returns ok=false) until the arm event lands. +func (je *JoinEngine) evalExpected(spec *JoinSpec, env map[string]any, eventName string) (int64, bool, error) { + if spec.ExpectedCount == "" { + return 0, false, nil + } + if spec.ArmOnEvent != "" && eventName != spec.ArmOnEvent { + return 0, false, nil + } + return je.evalInt(spec.ExpectedCount, env) +} + +func (je *JoinEngine) evalString(expression string, env map[string]any) (string, error) { + if expression == "" { + return "", nil + } + v, err := je.expr.EvaluateAny(expression, env) + if err != nil { + return "", err + } + if v == nil { + return "", nil + } + return fmt.Sprint(v), nil +} + +func (je *JoinEngine) evalInt(expression string, env map[string]any) (int64, bool, error) { + if expression == "" { + return 0, false, nil + } + v, err := je.expr.EvaluateAny(expression, env) + if err != nil { + return 0, false, err + } + switch n := v.(type) { + case nil: + return 0, false, nil + case int: + return int64(n), true, nil + case int8: + return int64(n), true, nil + case int16: + return int64(n), true, nil + case int32: + return int64(n), true, nil + case int64: + return n, true, nil + case uint: + return int64(n), true, nil + case uint32: + return int64(n), true, nil + case uint64: + return int64(n), true, nil + case float32: + return int64(n), true, nil + case float64: + return int64(n), true, nil + default: + return 0, false, fmt.Errorf("expected_count evaluated to %T, want integer", v) + } +} + +func (je *JoinEngine) parseDuration(s string, def time.Duration) time.Duration { + if s == "" { + return def + } + d, err := time.ParseDuration(s) + if err != nil || d < 0 { + log.Warn().Str("value", s).Msg("join: invalid duration, using default") + return def + } + return d +} + +func (je *JoinEngine) retentionLinger(spec *JoinSpec) time.Duration { + return je.parseDuration(spec.Linger, joinDefaultLinger) +} + +// retentionTTL bounds the lifetime of a join's KV keys (counter, dedup ledger, +// fire-marker): the barrier deadline plus the post-terminal linger, so abandoned +// instances self-GC even if the deadline sweep never runs. +func (je *JoinEngine) retentionTTL(spec *JoinSpec) time.Duration { + return je.parseDuration(spec.Timeout, time.Hour) + je.retentionLinger(spec) +} + +// joinBaseKey derives the reserved-namespace KV key prefix for a join instance. +// The correlation key is hashed so arbitrary event-derived values always yield a +// valid, collision-free KV key; the human-readable form lives in the SQL mirror. +func joinBaseKey(name, workspace, corr string) string { + return kv.ReservedCoordKeyPrefix + "joins/" + hashSeg(name+"\x00"+workspace+"\x00"+corr) +} + +func hashSeg(s string) string { + h := sha256.Sum256([]byte(s)) + return hex.EncodeToString(h[:]) +} + +// joinIdemKey is the stable idempotency key for a join instance's downstream +// create_task. It is identical across the on_complete and on_timeout fire paths +// so a normal completion and a racing deadline sweep produce exactly one task +// (the gateway dedups CreateTaskRequest.idempotency_key). +func joinIdemKey(j *Join) string { + return "join:" + j.JoinName + ":" + j.Workspace + ":" + j.CorrelationKey +} diff --git a/server/internal/workflow/join_test.go b/server/internal/workflow/join_test.go new file mode 100644 index 0000000..16064d4 --- /dev/null +++ b/server/internal/workflow/join_test.go @@ -0,0 +1,514 @@ +package workflow + +import ( + "context" + "sync" + "testing" + "time" + + "github.com/scitrera/aether/sdk/go/coord" +) + +// ---- in-memory fakes (no gateway / KV backend needed) ---- + +type fakeCounter struct { + mu sync.Mutex + m map[string]int64 +} + +func newFakeCounter() *fakeCounter { return &fakeCounter{m: map[string]int64{}} } + +func (c *fakeCounter) IncrementIf(_ context.Context, key string, delta, ceiling int64) (int64, bool, error) { + c.mu.Lock() + defer c.mu.Unlock() + proposed := c.m[key] + delta + if proposed > ceiling { + return c.m[key], false, nil + } + c.m[key] = proposed + return proposed, true, nil +} + +func (c *fakeCounter) DecrementIf(_ context.Context, key string, delta, floor int64) (int64, bool, error) { + c.mu.Lock() + defer c.mu.Unlock() + proposed := c.m[key] - delta + if proposed < floor { + return c.m[key], false, nil + } + c.m[key] = proposed + return proposed, true, nil +} + +type fakeLocker struct { + mu sync.Mutex + m map[string]string +} + +func newFakeLocker() *fakeLocker { return &fakeLocker{m: map[string]string{}} } + +func (l *fakeLocker) TryAcquire(_ context.Context, key, owner string, _ time.Duration) (bool, error) { + l.mu.Lock() + defer l.mu.Unlock() + if _, ok := l.m[key]; ok { + return false, nil + } + l.m[key] = owner + return true, nil +} + +func (l *fakeLocker) Refresh(_ context.Context, key, owner string, _ time.Duration) (bool, error) { + l.mu.Lock() + defer l.mu.Unlock() + if l.m[key] == owner { + return true, nil + } + return false, nil +} + +func (l *fakeLocker) Release(_ context.Context, key, owner string) (bool, error) { + l.mu.Lock() + defer l.mu.Unlock() + if l.m[key] == owner { + delete(l.m, key) + return true, nil + } + return false, nil +} + +func (l *fakeLocker) Peek(_ context.Context, key string) (string, error) { + l.mu.Lock() + defer l.mu.Unlock() + return l.m[key], nil +} + +type fakeJoinStore struct { + mu sync.Mutex + rows map[string]*Join + seq int64 +} + +func newFakeJoinStore() *fakeJoinStore { return &fakeJoinStore{rows: map[string]*Join{}} } + +func joinKey(name, ws, corr string) string { return name + "|" + ws + "|" + corr } + +func (s *fakeJoinStore) byID(id int64) *Join { + for _, r := range s.rows { + if r.ID == id { + return r + } + } + return nil +} + +func (s *fakeJoinStore) EnsureJoin(_ context.Context, j *Join) (*Join, error) { + s.mu.Lock() + defer s.mu.Unlock() + k := joinKey(j.JoinName, j.Workspace, j.CorrelationKey) + if ex, ok := s.rows[k]; ok { + cp := *ex + return &cp, nil + } + s.seq++ + cp := *j + cp.ID = s.seq + if cp.Status == "" { + cp.Status = JoinStatusOpen + } + s.rows[k] = &cp + out := cp + return &out, nil +} + +func (s *fakeJoinStore) GetJoin(_ context.Context, name, ws, corr string) (*Join, error) { + s.mu.Lock() + defer s.mu.Unlock() + if r, ok := s.rows[joinKey(name, ws, corr)]; ok { + cp := *r + return &cp, nil + } + return nil, nil +} + +func (s *fakeJoinStore) UpdateJoinArrived(_ context.Context, id, n int64) error { + s.mu.Lock() + defer s.mu.Unlock() + if r := s.byID(id); r != nil { + r.ArrivedCount = n + } + return nil +} + +func (s *fakeJoinStore) SetJoinExpected(_ context.Context, id, expected int64) error { + s.mu.Lock() + defer s.mu.Unlock() + if r := s.byID(id); r != nil { + e := expected + r.ExpectedCount = &e + } + return nil +} + +func (s *fakeJoinStore) SetJoinDirty(_ context.Context, id int64, dirty bool) error { + s.mu.Lock() + defer s.mu.Unlock() + if r := s.byID(id); r != nil { + r.Dirty = dirty + } + return nil +} + +func (s *fakeJoinStore) MarkJoinTerminal(_ context.Context, id int64, status string, _ time.Time) error { + s.mu.Lock() + defer s.mu.Unlock() + if r := s.byID(id); r != nil { + r.Status = status + } + return nil +} + +type fakeDispatcher struct { + mu sync.Mutex + dispatched []*ActionDef + emitted []*ActionDef +} + +func (d *fakeDispatcher) DispatchAction(a *ActionDef) error { + d.mu.Lock() + defer d.mu.Unlock() + d.dispatched = append(d.dispatched, a) + return nil +} + +func (d *fakeDispatcher) EmitEvent(a *ActionDef) error { + d.mu.Lock() + defer d.mu.Unlock() + d.emitted = append(d.emitted, a) + return nil +} + +func (d *fakeDispatcher) count() int { + d.mu.Lock() + defer d.mu.Unlock() + return len(d.dispatched) +} + +type fakeSet struct { + mu sync.Mutex + m map[string]map[string]struct{} +} + +func newFakeSet() *fakeSet { return &fakeSet{m: map[string]map[string]struct{}{}} } + +func (s *fakeSet) SetAdd(_ context.Context, key, member string, _ time.Duration) (bool, int64, error) { + s.mu.Lock() + defer s.mu.Unlock() + set, ok := s.m[key] + if !ok { + set = map[string]struct{}{} + s.m[key] = set + } + _, exists := set[member] + if !exists { + set[member] = struct{}{} + } + return !exists, int64(len(set)), nil +} + +// Compile-time conformance for the fakes. +var ( + _ coord.Counter = (*fakeCounter)(nil) + _ coord.Locker = (*fakeLocker)(nil) + _ joinStore = (*fakeJoinStore)(nil) + _ actionDispatcher = (*fakeDispatcher)(nil) + _ joinSet = (*fakeSet)(nil) +) + +func newTestJoinEngine() (*JoinEngine, *fakeDispatcher, *fakeJoinStore) { + disp := &fakeDispatcher{} + store := newFakeJoinStore() + je := NewJoinEngine(store, NewExprEngine(64), disp, newFakeCounter(), newFakeLocker(), newFakeSet(), "default") + return je, disp, store +} + +func env(input map[string]any, ws string) map[string]any { + return map[string]any{ + "input": input, + "source": map[string]any{"workspace": ws}, + } +} + +// ---- tests ---- + +func TestJoinSet_FiresWhenAllMembersArrive(t *testing.T) { + je, disp, _ := newTestJoinEngine() + ctx := context.Background() + spec := &JoinSpec{ + Name: "setbar", + Mode: JoinModeSet, + CorrelationKey: "input.batch", + ExpectedSet: `["a","b","c"]`, + MemberKey: "input.member", + OnComplete: &ActionDef{Type: "create_task", TaskType: "kb"}, + } + mk := func(m string) map[string]any { return map[string]any{"batch": "A", "member": m} } + + _ = je.HandleArrival(ctx, spec, env(mk("a"), "ws"), "member", "ws") + _ = je.HandleArrival(ctx, spec, env(mk("b"), "ws"), "member", "ws") + _ = je.HandleArrival(ctx, spec, env(mk("a"), "ws"), "member", "ws") // duplicate member — no progress + if disp.count() != 0 { + t.Fatalf("after a,b,a(dup): dispatched=%d, want 0", disp.count()) + } + _ = je.HandleArrival(ctx, spec, env(mk("c"), "ws"), "member", "ws") // completes {a,b,c} + if disp.count() != 1 { + t.Fatalf("after a,b,c: dispatched=%d, want 1", disp.count()) + } + // A late extra member must not re-fire (fire-marker gate). + _ = je.HandleArrival(ctx, spec, env(mk("d"), "ws"), "member", "ws") + if disp.count() != 1 { + t.Fatalf("after extra member d: dispatched=%d, want 1 (exactly-once)", disp.count()) + } +} + +func TestJoin_SetsIdempotencyKeyOnFire(t *testing.T) { + je, disp, _ := newTestJoinEngine() + ctx := context.Background() + spec := &JoinSpec{ + Name: "barrier", + Mode: JoinModeCount, + CorrelationKey: "input.batch", + ExpectedCount: "1", + OnComplete: &ActionDef{Type: "create_task", TaskType: "kb"}, + } + _ = je.HandleArrival(ctx, spec, env(map[string]any{"batch": "A"}, "ws"), "member", "ws") + if disp.count() != 1 { + t.Fatalf("want 1 dispatch, got %d", disp.count()) + } + disp.mu.Lock() + got := disp.dispatched[0].IdempotencyKey + disp.mu.Unlock() + if got != "join:barrier:ws:A" { + t.Fatalf("idempotency_key=%q, want join:barrier:ws:A", got) + } +} + +func TestJoinCount_FiresOnceWhenComplete(t *testing.T) { + je, disp, _ := newTestJoinEngine() + ctx := context.Background() + spec := &JoinSpec{ + Name: "barrier", + Mode: JoinModeCount, + CorrelationKey: "input.batch", + ExpectedCount: "2", + OnComplete: &ActionDef{Type: "create_task", TaskType: "kb"}, + } + eA := env(map[string]any{"batch": "A"}, "ws") + + if err := je.HandleArrival(ctx, spec, eA, "member", "ws"); err != nil { + t.Fatalf("arrival 1: %v", err) + } + if disp.count() != 0 { + t.Fatalf("after 1/2 arrivals: dispatched=%d, want 0", disp.count()) + } + if err := je.HandleArrival(ctx, spec, eA, "member", "ws"); err != nil { + t.Fatalf("arrival 2: %v", err) + } + if disp.count() != 1 { + t.Fatalf("after 2/2 arrivals: dispatched=%d, want 1", disp.count()) + } + // A third (duplicate/late) member must not fire again — fire-marker gate. + if err := je.HandleArrival(ctx, spec, eA, "member", "ws"); err != nil { + t.Fatalf("arrival 3: %v", err) + } + if disp.count() != 1 { + t.Fatalf("after 3rd arrival: dispatched=%d, want 1 (exactly-once)", disp.count()) + } +} + +func TestJoinCount_CorrelationIsolation(t *testing.T) { + je, disp, _ := newTestJoinEngine() + ctx := context.Background() + spec := &JoinSpec{ + Name: "barrier", + Mode: JoinModeCount, + CorrelationKey: "input.batch", + ExpectedCount: "2", + OnComplete: &ActionDef{Type: "create_task", TaskType: "kb"}, + } + // One member each for two different batches: neither completes. + _ = je.HandleArrival(ctx, spec, env(map[string]any{"batch": "A"}, "ws"), "member", "ws") + _ = je.HandleArrival(ctx, spec, env(map[string]any{"batch": "B"}, "ws"), "member", "ws") + if disp.count() != 0 { + t.Fatalf("cross-batch arrivals fired prematurely: dispatched=%d, want 0", disp.count()) + } + // Complete batch A only. + _ = je.HandleArrival(ctx, spec, env(map[string]any{"batch": "A"}, "ws"), "member", "ws") + if disp.count() != 1 { + t.Fatalf("batch A completion: dispatched=%d, want 1", disp.count()) + } +} + +func TestJoinCount_DedupDropsRepeats(t *testing.T) { + je, disp, _ := newTestJoinEngine() + ctx := context.Background() + spec := &JoinSpec{ + Name: "barrier", + Mode: JoinModeCount, + CorrelationKey: "input.batch", + ExpectedCount: "3", + DedupKey: "input.id", + OnComplete: &ActionDef{Type: "create_task", TaskType: "kb"}, + } + mk := func(id string) map[string]any { return map[string]any{"batch": "A", "id": id} } + + _ = je.HandleArrival(ctx, spec, env(mk("m1"), "ws"), "member", "ws") + _ = je.HandleArrival(ctx, spec, env(mk("m1"), "ws"), "member", "ws") // duplicate — dropped + _ = je.HandleArrival(ctx, spec, env(mk("m2"), "ws"), "member", "ws") + if disp.count() != 0 { + t.Fatalf("after m1,m1(dup),m2: dispatched=%d, want 0 (only 2 distinct)", disp.count()) + } + _ = je.HandleArrival(ctx, spec, env(mk("m3"), "ws"), "member", "ws") // 3rd distinct → fire + if disp.count() != 1 { + t.Fatalf("after 3 distinct members: dispatched=%d, want 1", disp.count()) + } +} + +func TestJoinCount_DynamicArmingAfterMembers(t *testing.T) { + je, disp, _ := newTestJoinEngine() + ctx := context.Background() + spec := &JoinSpec{ + Name: "barrier", + Mode: JoinModeCount, + CorrelationKey: "input.batch", + ExpectedCount: "input.expected_count", + ArmOnEvent: "manifest", + OnComplete: &ActionDef{Type: "create_task", TaskType: "kb"}, + } + // Two members arrive before the expected count is known. + _ = je.HandleArrival(ctx, spec, env(map[string]any{"batch": "A"}, "ws"), "member", "ws") + _ = je.HandleArrival(ctx, spec, env(map[string]any{"batch": "A"}, "ws"), "member", "ws") + if disp.count() != 0 { + t.Fatalf("before arming: dispatched=%d, want 0", disp.count()) + } + // The manifest arms expected=2; the count already satisfies it → fire now. + _ = je.HandleArrival(ctx, spec, env(map[string]any{"batch": "A", "expected_count": 2}, "ws"), "manifest", "ws") + if disp.count() != 1 { + t.Fatalf("after arming with count satisfied: dispatched=%d, want 1", disp.count()) + } +} + +func TestJoinCoalesce_FirstFiresRestMarkDirty(t *testing.T) { + je, disp, store := newTestJoinEngine() + ctx := context.Background() + spec := &JoinSpec{ + Name: "coalesce-kb", + Mode: JoinModeCoalesce, + CorrelationKey: "input.ws", + OnComplete: &ActionDef{Type: "create_task", TaskType: "kb"}, + } + e := env(map[string]any{"ws": "tenant1"}, "ws") + + _ = je.HandleArrival(ctx, spec, e, "ingest", "ws") // first → fire + _ = je.HandleArrival(ctx, spec, e, "ingest", "ws") // within active window → dirty + _ = je.HandleArrival(ctx, spec, e, "ingest", "ws") // still dirty + if disp.count() != 1 { + t.Fatalf("coalesce: dispatched=%d, want 1", disp.count()) + } + row, _ := store.GetJoin(ctx, "coalesce-kb", "ws", "tenant1") + if row == nil || !row.Dirty { + t.Fatalf("coalesce: expected dirty=true after coalesced arrivals, got row=%+v", row) + } +} + +// seedJoin inserts an open join row via EnsureJoin and returns it (with its +// assigned ID) so deadline tests can drive HandleDeadline against a real row. +func seedJoin(t *testing.T, store *fakeJoinStore, j *Join) *Join { + t.Helper() + row, err := store.EnsureJoin(context.Background(), j) + if err != nil { + t.Fatalf("seed join: %v", err) + } + return row +} + +func mustMarshalAction(t *testing.T, a *ActionDef) string { + t.Helper() + s := marshalAction(a) + if s == "" { + t.Fatalf("marshalAction returned empty for %+v", a) + } + return s +} + +func TestJoinDeadline_FiresOnTimeoutAction(t *testing.T) { + je, disp, store := newTestJoinEngine() + ctx := context.Background() + row := seedJoin(t, store, &Join{ + JoinName: "barrier", + Workspace: "ws", + CorrelationKey: "A", + Mode: JoinModeCount, + Status: JoinStatusOpen, + OnTimeout: mustMarshalAction(t, &ActionDef{Type: "create_task", TaskType: "kb-degraded"}), + }) + + if err := je.HandleDeadline(ctx, row); err != nil { + t.Fatalf("HandleDeadline: %v", err) + } + if disp.count() != 1 { + t.Fatalf("on_timeout firing: dispatched=%d, want 1", disp.count()) + } + got, _ := store.GetJoin(ctx, "barrier", "ws", "A") + if got == nil || got.Status != JoinStatusTimedOut { + t.Fatalf("expected status=%q, got row=%+v", JoinStatusTimedOut, got) + } +} + +func TestJoinDeadline_AbortPolicySkipsFiring(t *testing.T) { + je, disp, store := newTestJoinEngine() + ctx := context.Background() + row := seedJoin(t, store, &Join{ + JoinName: "barrier", + Workspace: "ws", + CorrelationKey: "A", + Mode: JoinModeCount, + Status: JoinStatusOpen, + OnTimeout: mustMarshalAction(t, &ActionDef{Type: "create_task", TaskType: "kb-degraded"}), + OnPartialFailure: "abort", + }) + + if err := je.HandleDeadline(ctx, row); err != nil { + t.Fatalf("HandleDeadline: %v", err) + } + if disp.count() != 0 { + t.Fatalf("abort policy: dispatched=%d, want 0", disp.count()) + } + got, _ := store.GetJoin(ctx, "barrier", "ws", "A") + if got == nil || got.Status != JoinStatusTimedOut { + t.Fatalf("expected status=%q, got row=%+v", JoinStatusTimedOut, got) + } +} + +func TestJoinDeadline_FallbackToOnComplete(t *testing.T) { + je, disp, store := newTestJoinEngine() + ctx := context.Background() + row := seedJoin(t, store, &Join{ + JoinName: "barrier", + Workspace: "ws", + CorrelationKey: "A", + Mode: JoinModeCount, + Status: JoinStatusOpen, + OnComplete: mustMarshalAction(t, &ActionDef{Type: "create_task", TaskType: "kb"}), + }) + + if err := je.HandleDeadline(ctx, row); err != nil { + t.Fatalf("HandleDeadline: %v", err) + } + if disp.count() != 1 { + t.Fatalf("on_complete fallback: dispatched=%d, want 1", disp.count()) + } + got, _ := store.GetJoin(ctx, "barrier", "ws", "A") + if got == nil || got.Status != JoinStatusTimedOut { + t.Fatalf("expected status=%q, got row=%+v", JoinStatusTimedOut, got) + } +} diff --git a/server/internal/workflow/joins_handler_test.go b/server/internal/workflow/joins_handler_test.go new file mode 100644 index 0000000..f880b42 --- /dev/null +++ b/server/internal/workflow/joins_handler_test.go @@ -0,0 +1,165 @@ +package workflow + +import ( + "context" + "encoding/json" + "testing" + "time" + + pb "github.com/scitrera/aether/api/proto" +) + +// joinFakeStore is a minimal WorkflowStore used to exercise the join +// observability handlers (LIST_JOINS / GET_JOIN / CANCEL_JOIN). It embeds +// the WorkflowStore interface so the unused methods compile as nil; only the +// join-related methods are implemented against an in-memory slice. +type joinFakeStore struct { + WorkflowStore + joins []Join +} + +func (f *joinFakeStore) ListJoins(_ context.Context, workspace string) ([]Join, error) { + if workspace == "" || workspace == "*" { + return f.joins, nil + } + var out []Join + for _, j := range f.joins { + if j.Workspace == workspace { + out = append(out, j) + } + } + return out, nil +} + +func (f *joinFakeStore) GetJoin(_ context.Context, joinName, workspace, correlationKey string) (*Join, error) { + for i := range f.joins { + j := &f.joins[i] + if j.JoinName == joinName && j.Workspace == workspace && j.CorrelationKey == correlationKey { + return j, nil + } + } + return nil, nil +} + +func (f *joinFakeStore) MarkJoinTerminal(_ context.Context, id int64, status string, lingerUntil time.Time) error { + for i := range f.joins { + if f.joins[i].ID == id { + f.joins[i].Status = status + f.joins[i].LingerUntil = &lingerUntil + return nil + } + } + return nil +} + +func seededJoinServer() (*Server, *joinFakeStore) { + store := &joinFakeStore{ + joins: []Join{ + {ID: 1, JoinName: "j1", Workspace: "ws", CorrelationKey: "c1", Mode: JoinModeCount, ArrivedCount: 1, Status: JoinStatusOpen}, + {ID: 2, JoinName: "j2", Workspace: "ws", CorrelationKey: "c2", Mode: JoinModeSet, ArrivedCount: 3, Status: JoinStatusFired}, + }, + } + return &Server{store: store}, store +} + +func TestHandleListJoins_returnsSeededJoins(t *testing.T) { + srv, _ := seededJoinServer() + resp, err := srv.handleWorkflowOperation(context.Background(), &pb.WorkflowOperation{ + Op: pb.WorkflowOperation_LIST_JOINS, + Workspace: "ws", + }) + if err != nil { + t.Fatalf("handleListJoins err: %v", err) + } + if !resp.Success { + t.Fatalf("expected success, got error: %s", resp.Error) + } + var views []joinView + if err := json.Unmarshal(resp.Data, &views); err != nil { + t.Fatalf("unmarshal: %v", err) + } + if len(views) != 2 { + t.Fatalf("expected 2 joins, got %d", len(views)) + } +} + +func TestHandleGetJoin_found(t *testing.T) { + srv, _ := seededJoinServer() + resp, err := srv.handleWorkflowOperation(context.Background(), &pb.WorkflowOperation{ + Op: pb.WorkflowOperation_GET_JOIN, + Id: "j1", + Workspace: "ws", + SecondaryId: "c1", + }) + if err != nil { + t.Fatalf("handleGetJoin err: %v", err) + } + if !resp.Success { + t.Fatalf("expected success, got error: %s", resp.Error) + } + var view joinView + if err := json.Unmarshal(resp.Data, &view); err != nil { + t.Fatalf("unmarshal: %v", err) + } + if view.JoinName != "j1" || view.CorrelationKey != "c1" { + t.Fatalf("unexpected join view: %+v", view) + } +} + +func TestHandleGetJoin_missingReturnsNotFound(t *testing.T) { + srv, _ := seededJoinServer() + resp, err := srv.handleWorkflowOperation(context.Background(), &pb.WorkflowOperation{ + Op: pb.WorkflowOperation_GET_JOIN, + Id: "nope", + Workspace: "ws", + SecondaryId: "x", + }) + if err != nil { + t.Fatalf("handleGetJoin err: %v", err) + } + if resp.Success { + t.Fatal("expected failure for missing join") + } + if resp.Error != "join not found" { + t.Fatalf("expected 'join not found', got %q", resp.Error) + } +} + +func TestHandleCancelJoin_flipsToCancelled(t *testing.T) { + srv, store := seededJoinServer() + resp, err := srv.handleWorkflowOperation(context.Background(), &pb.WorkflowOperation{ + Op: pb.WorkflowOperation_CANCEL_JOIN, + Id: "j1", + Workspace: "ws", + SecondaryId: "c1", + }) + if err != nil { + t.Fatalf("handleCancelJoin err: %v", err) + } + if !resp.Success { + t.Fatalf("expected success, got error: %s", resp.Error) + } + if store.joins[0].Status != JoinStatusCancelled { + t.Fatalf("expected join status %q, got %q", JoinStatusCancelled, store.joins[0].Status) + } +} + +func TestHandleCancelJoin_idempotentWhenTerminal(t *testing.T) { + srv, store := seededJoinServer() + // j2 is already Fired (terminal). + resp, err := srv.handleWorkflowOperation(context.Background(), &pb.WorkflowOperation{ + Op: pb.WorkflowOperation_CANCEL_JOIN, + Id: "j2", + Workspace: "ws", + SecondaryId: "c2", + }) + if err != nil { + t.Fatalf("handleCancelJoin err: %v", err) + } + if !resp.Success { + t.Fatalf("expected idempotent success, got error: %s", resp.Error) + } + if store.joins[1].Status != JoinStatusFired { + t.Fatalf("terminal join must not change status, got %q", store.joins[1].Status) + } +} diff --git a/server/internal/workflow/leader.go b/server/internal/workflow/leader.go index a383613..8c3e0ad 100644 --- a/server/internal/workflow/leader.go +++ b/server/internal/workflow/leader.go @@ -4,103 +4,51 @@ import ( "context" "time" - "github.com/redis/go-redis/v9" - "github.com/rs/zerolog/log" + "github.com/scitrera/aether/internal/kv" + "github.com/scitrera/aether/sdk/go/coord" ) -// LeaderElector is the interface for distributed leader election. +// LeaderElector elects a single active workflow-server instance among replicas +// so only one runs the scheduler and DAG monitor at a time. +// +// The election runs on the shared coord primitive (atomic SetNX/CompareAndSet +// lease lock) over a coord.Locker. In production the locker is the +// WorkflowEngine client's KV locker, so leadership is coordinated through the +// gateway's KV store — Redis in standard mode, Badger/JetStream in aetherlite — +// without the workflow server holding its own Redis connection. The same logic +// therefore yields a correct single leader in every deployment mode. type LeaderElector interface { - TryAcquire(ctx context.Context) bool - Release(ctx context.Context) + // Start launches the election loop in the background. Idempotent. + Start(ctx context.Context) + // Shutdown stops the election loop and best-effort releases leadership so a + // successor can take over without waiting for the lease to expire. + Shutdown(ctx context.Context) error + // IsLeader reports whether this instance currently holds leadership. IsLeader() bool } -// RedisLeaderElector uses Redis SET NX to ensure only one workflow server -// instance runs the scheduler and DAG monitor at a time. -type RedisLeaderElector struct { - client redis.UniversalClient - key string - id string - ttl time.Duration - isLead bool -} - -func NewRedisLeaderElector(client redis.UniversalClient, key, instanceID string) *RedisLeaderElector { - return &RedisLeaderElector{ - client: client, - key: key, - id: instanceID, - ttl: 30 * time.Second, - } -} - -// IsLeader returns whether this instance currently holds the leader lock. -func (l *RedisLeaderElector) IsLeader() bool { - return l.isLead -} - -// TryAcquire attempts to acquire or refresh the leader lock. -// Returns true if this instance is (or became) the leader. -func (l *RedisLeaderElector) TryAcquire(ctx context.Context) bool { - if l.isLead { - // Refresh existing lock - ok, err := l.client.SetArgs(ctx, l.key, l.id, redis.SetArgs{ - Mode: "XX", - TTL: l.ttl, - }).Result() - if err != nil || ok != "OK" { - // Lock lost or expired; try to re-acquire - l.isLead = false - return l.trySetNX(ctx) - } - return true - } - return l.trySetNX(ctx) -} - -func (l *RedisLeaderElector) trySetNX(ctx context.Context) bool { - ok, err := l.client.SetNX(ctx, l.key, l.id, l.ttl).Result() - if err != nil { - log.Warn().Err(err).Msg("leader election SetNX failed") - return false - } - l.isLead = ok - if ok { - log.Info().Str("instance", l.id).Msg("acquired workflow leader lock") - } - return ok -} - -// Release explicitly releases the leader lock. -func (l *RedisLeaderElector) Release(ctx context.Context) { - if !l.isLead { - return - } - // Only delete if we own it - script := redis.NewScript(` - if redis.call("GET", KEYS[1]) == ARGV[1] then - return redis.call("DEL", KEYS[1]) - end - return 0 - `) - script.Run(ctx, l.client, []string{l.key}, l.id) - l.isLead = false - log.Info().Str("instance", l.id).Msg("released workflow leader lock") -} - -// RunRefreshLoop periodically refreshes the leader lock. -// It stops when ctx is cancelled. -func (l *RedisLeaderElector) RunRefreshLoop(ctx context.Context) { - ticker := time.NewTicker(l.ttl / 3) - defer ticker.Stop() +const ( + // workflowLeaderTTL is the leadership lease lifetime. A crashed leader is + // replaced after at most this long. + workflowLeaderTTL = 30 * time.Second + // workflowLeaderRenew is how often the leader renews its lease. + workflowLeaderRenew = 10 * time.Second +) - for { - select { - case <-ctx.Done(): - l.Release(context.Background()) - return - case <-ticker.C: - l.TryAcquire(ctx) - } - } +// workflowLeaderKey is the coordination key the election lock lives under, in +// the reserved infra-coordination namespace (so the gateway grants the +// WorkflowEngine access via its infra fast-path). Used on the shared global +// scope so every replica rendezvous on the same key. +var workflowLeaderKey = kv.ReservedCoordKeyPrefix + "workflow/leader" + +// NewCoordLeaderElector builds a leader elector over any coord.Locker (and thus +// any KV backend). instanceID is folded into the fencing token for log +// attribution; a random suffix guarantees per-replica uniqueness. +func NewCoordLeaderElector(locker coord.Locker, key, instanceID string) LeaderElector { + owner := instanceID + ":" + coord.NewOwnerID() + return coord.NewLeaderElection(locker, key, + coord.WithLeaderLeaseTTL(workflowLeaderTTL), + coord.WithLeaderRenewInterval(workflowLeaderRenew), + coord.WithLeaderOwnerID(owner), + ) } diff --git a/server/internal/workflow/leader_test.go b/server/internal/workflow/leader_test.go index a3386d2..ef1625b 100644 --- a/server/internal/workflow/leader_test.go +++ b/server/internal/workflow/leader_test.go @@ -3,134 +3,68 @@ package workflow import ( "context" "testing" + "time" - "github.com/alicebob/miniredis/v2" - "github.com/redis/go-redis/v9" + "github.com/scitrera/aether/sdk/go/coord" ) -func newTestRedisClient(t *testing.T) (*redis.Client, *miniredis.Miniredis) { - t.Helper() - mr := miniredis.RunT(t) - client := redis.NewClient(&redis.Options{Addr: mr.Addr()}) - return client, mr -} - -func TestLeaderElector_IsLeader_falseBeforeAcquire(t *testing.T) { - client, _ := newTestRedisClient(t) - le := NewRedisLeaderElector(client, "wf:leader:test", "instance-1") - - if le.IsLeader() { - t.Error("IsLeader() = true before TryAcquire, want false") - } -} - -func TestLeaderElector_TryAcquire_succeedsWhenLockFree(t *testing.T) { - client, _ := newTestRedisClient(t) - le := NewRedisLeaderElector(client, "wf:leader:test", "instance-1") - ctx := context.Background() - - got := le.TryAcquire(ctx) - if !got { - t.Error("TryAcquire() = false on free lock, want true") - } - if !le.IsLeader() { - t.Error("IsLeader() = false after successful TryAcquire, want true") - } -} - -func TestLeaderElector_TryAcquire_secondInstanceFailsWhenLockHeld(t *testing.T) { - client, _ := newTestRedisClient(t) - le1 := NewRedisLeaderElector(client, "wf:leader:test", "instance-1") - le2 := NewRedisLeaderElector(client, "wf:leader:test", "instance-2") - ctx := context.Background() - - if !le1.TryAcquire(ctx) { - t.Fatal("le1.TryAcquire() = false, want true") - } - - got := le2.TryAcquire(ctx) - if got { - t.Error("le2.TryAcquire() = true when lock held by le1, want false") - } - if le2.IsLeader() { - t.Error("le2.IsLeader() = true after failed acquire, want false") - } -} +// TestCoordLeaderElector_AcquiresLeadership verifies a single elector acquires +// leadership over a shared locker. +func TestCoordLeaderElector_AcquiresLeadership(t *testing.T) { + lk := coord.NewMemoryLocker() + le := NewCoordLeaderElector(lk, workflowLeaderKey, "instance-a") -func TestLeaderElector_TryAcquire_refreshesExistingLock(t *testing.T) { - client, _ := newTestRedisClient(t) - le := NewRedisLeaderElector(client, "wf:leader:test", "instance-1") - ctx := context.Background() + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + le.Start(ctx) - if !le.TryAcquire(ctx) { - t.Fatal("first TryAcquire() = false, want true") - } - // Second call should refresh (not drop) the lock - if !le.TryAcquire(ctx) { - t.Error("second TryAcquire() = false, want true (refresh)") - } - if !le.IsLeader() { - t.Error("IsLeader() = false after refresh, want true") + if !waitForLeader(2*time.Second, le.IsLeader) { + t.Fatal("instance-a did not become leader within 2s") } + _ = le.Shutdown(context.Background()) } -func TestLeaderElector_Release_clearsLeaderState(t *testing.T) { - client, _ := newTestRedisClient(t) - le := NewRedisLeaderElector(client, "wf:leader:test", "instance-1") - ctx := context.Background() - - if !le.TryAcquire(ctx) { - t.Fatal("TryAcquire() = false, want true") +// TestCoordLeaderElector_SingleLeaderAndFailover verifies that two electors +// sharing a locker never both lead, and that the survivor takes over after the +// leader steps down. +func TestCoordLeaderElector_SingleLeaderAndFailover(t *testing.T) { + if testing.Short() { + t.Skip("skipping timing-based leader failover test in -short mode") } - le.Release(ctx) + lk := coord.NewMemoryLocker() + a := NewCoordLeaderElector(lk, workflowLeaderKey, "instance-a") + b := NewCoordLeaderElector(lk, workflowLeaderKey, "instance-b") - if le.IsLeader() { - t.Error("IsLeader() = true after Release, want false") + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + a.Start(ctx) + if !waitForLeader(2*time.Second, a.IsLeader) { + t.Fatal("instance-a did not become leader") } -} - -func TestLeaderElector_Release_allowsOtherInstanceToAcquire(t *testing.T) { - client, _ := newTestRedisClient(t) - le1 := NewRedisLeaderElector(client, "wf:leader:test", "instance-1") - le2 := NewRedisLeaderElector(client, "wf:leader:test", "instance-2") - ctx := context.Background() - - if !le1.TryAcquire(ctx) { - t.Fatal("le1.TryAcquire() = false") + b.Start(ctx) + // b must not seize leadership while a holds it. + time.Sleep(300 * time.Millisecond) + if b.IsLeader() { + t.Fatal("instance-b became leader while instance-a holds it") } - le1.Release(ctx) - if !le2.TryAcquire(ctx) { - t.Error("le2.TryAcquire() = false after le1 released, want true") + // a steps down → b takes over (Shutdown eagerly releases the lease). + if err := a.Shutdown(context.Background()); err != nil { + t.Fatalf("a.Shutdown: %v", err) } -} - -func TestLeaderElector_Release_isNoOpWhenNotLeader(t *testing.T) { - client, _ := newTestRedisClient(t) - le := NewRedisLeaderElector(client, "wf:leader:test", "instance-1") - ctx := context.Background() - - // Should not panic or error when not leader - le.Release(ctx) - if le.IsLeader() { - t.Error("IsLeader() = true after Release on non-leader, want false") + if !waitForLeader(workflowLeaderTTL+5*time.Second, b.IsLeader) { + t.Fatal("instance-b did not take over after instance-a stepped down") } + _ = b.Shutdown(context.Background()) } -func TestLeaderElector_TryAcquire_reacquiresAfterLockExpiry(t *testing.T) { - client, mr := newTestRedisClient(t) - le1 := NewRedisLeaderElector(client, "wf:leader:test", "instance-1") - le2 := NewRedisLeaderElector(client, "wf:leader:test", "instance-2") - ctx := context.Background() - - if !le1.TryAcquire(ctx) { - t.Fatal("le1.TryAcquire() = false") - } - - // Simulate lock TTL expiry via miniredis fast-forward - mr.FastForward(31 * 1e9) // 31 seconds in nanoseconds - - if !le2.TryAcquire(ctx) { - t.Error("le2.TryAcquire() = false after le1 lock expired, want true") +func waitForLeader(timeout time.Duration, cond func() bool) bool { + deadline := time.Now().Add(timeout) + for time.Now().Before(deadline) { + if cond() { + return true + } + time.Sleep(20 * time.Millisecond) } + return cond() } diff --git a/server/internal/workflow/migrations/004_workflow_joins.sql b/server/internal/workflow/migrations/004_workflow_joins.sql new file mode 100644 index 0000000..c159e0b --- /dev/null +++ b/server/internal/workflow/migrations/004_workflow_joins.sql @@ -0,0 +1,23 @@ +-- Fan-in/barrier/coalesce join instances. The authoritative arrival counter +-- lives in KV; this table is the durable observability + deadline-sweep +-- surface. One row per (join_name, workspace, correlation_key). +CREATE TABLE IF NOT EXISTS workflow_joins ( + id BIGSERIAL PRIMARY KEY, + join_name TEXT NOT NULL, + workspace TEXT NOT NULL, + correlation_key TEXT NOT NULL, + mode TEXT NOT NULL, + expected_count BIGINT, + arrived_count BIGINT NOT NULL DEFAULT 0, + dirty BOOLEAN NOT NULL DEFAULT FALSE, + status TEXT NOT NULL DEFAULT 'open', + on_complete TEXT, + on_timeout TEXT, + on_partial_failure TEXT, + deadline_at TIMESTAMPTZ, + linger_until TIMESTAMPTZ, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), + UNIQUE (join_name, workspace, correlation_key) +); +CREATE INDEX IF NOT EXISTS idx_workflow_joins_due ON workflow_joins (status, deadline_at); diff --git a/server/internal/workflow/router.go b/server/internal/workflow/router.go index a068d75..dfa4a82 100644 --- a/server/internal/workflow/router.go +++ b/server/internal/workflow/router.go @@ -3,6 +3,7 @@ package workflow import ( "context" "encoding/json" + "fmt" "strings" "sync" "time" @@ -17,6 +18,7 @@ type Router struct { expr *ExprEngine tmpl *TemplateEngine executor *Executor + joins *JoinEngine // Rule cache cacheMu sync.RWMutex @@ -29,12 +31,13 @@ type cachedRules struct { fetchedAt time.Time } -func NewRouter(store WorkflowStore, expr *ExprEngine, tmpl *TemplateEngine, executor *Executor, cacheTTL time.Duration) *Router { +func NewRouter(store WorkflowStore, expr *ExprEngine, tmpl *TemplateEngine, executor *Executor, joins *JoinEngine, cacheTTL time.Duration) *Router { return &Router{ store: store, expr: expr, tmpl: tmpl, executor: executor, + joins: joins, cache: make(map[string]cachedRules), cacheTTL: cacheTTL, } @@ -86,7 +89,7 @@ func (r *Router) HandleEvent(ctx context.Context, sourceTopic string, payload [] } for _, rule := range rules { - if err := r.processRule(ctx, rule, &event); err != nil { + if err := r.processRule(ctx, rule, &event, eventName); err != nil { log.Error().Err(err). Int("rule_id", rule.ID). Str("rule_name", rule.RuleName). @@ -108,13 +111,16 @@ func (r *Router) HandleEvent(ctx context.Context, sourceTopic string, payload [] return nil } -func (r *Router) processRule(ctx context.Context, rule Rule, event *EventPayload) error { - // Build environment for expression evaluation +func (r *Router) processRule(ctx context.Context, rule Rule, event *EventPayload, eventName string) error { + // Build environment for expression evaluation. `source.event` carries the + // matched event name so join specs can match arm_on_event and templates can + // reference it. env := map[string]any{ "input": event.Data, "source": map[string]any{ "agent": event.SourceAgent, "workspace": event.Workspace, + "event": eventName, }, } @@ -135,6 +141,18 @@ func (r *Router) processRule(ctx context.Context, rule Rule, event *EventPayload return err } + // Join destinations are handled by the join engine, which needs the arrival + // env and event name; all other destinations dispatch directly. + if result.Type == "join" { + if r.joins == nil { + return fmt.Errorf("rule %d: join destination but join engine is not configured", rule.ID) + } + if result.Join == nil { + return fmt.Errorf("rule %d: join destination missing `join:` block", rule.ID) + } + return r.joins.HandleArrival(ctx, result.Join, env, eventName, event.Workspace) + } + // Dispatch the action return r.executor.DispatchTransformResult(result) } diff --git a/server/internal/workflow/router_test.go b/server/internal/workflow/router_test.go index e168384..954bd80 100644 --- a/server/internal/workflow/router_test.go +++ b/server/internal/workflow/router_test.go @@ -69,7 +69,7 @@ func (r *routerWithStore) HandleEvent(ctx context.Context, sourceTopic string, p for _, eventName := range event.EventNames { rules, _ := r.store.GetMatchingRules(ctx, event.SourceAgent, eventName, event.Workspace) for _, rule := range rules { - router.processRule(ctx, rule, &event) + router.processRule(ctx, rule, &event, eventName) } } return nil @@ -241,7 +241,7 @@ func TestRouter_HandleEvent_invalidJSONPayloadIsIgnored(t *testing.T) { // ---- Router cache tests ---- func TestRouter_InvalidateCache_emptiesCache(t *testing.T) { - router := NewRouter(nil, NewExprEngine(10), NewTemplateEngine(10), nil, time.Minute) + router := NewRouter(nil, NewExprEngine(10), NewTemplateEngine(10), nil, nil, time.Minute) // Seed the internal cache directly router.cacheMu.Lock() diff --git a/server/internal/workflow/scheduler.go b/server/internal/workflow/scheduler.go index dffbba5..03b3fa1 100644 --- a/server/internal/workflow/scheduler.go +++ b/server/internal/workflow/scheduler.go @@ -9,22 +9,30 @@ import ( "github.com/rs/zerolog/log" ) +// joinDeadlineHandler fires the timeout path for an open join whose deadline +// has elapsed. *JoinEngine satisfies it. +type joinDeadlineHandler interface { + HandleDeadline(ctx context.Context, j *Join) error +} + // Scheduler handles recurring and one-time scheduled tasks. type Scheduler struct { store WorkflowStore executor *Executor dagEng *DAGEngine leader LeaderElector + joins joinDeadlineHandler parser cron.Parser interval time.Duration } -func NewScheduler(store WorkflowStore, executor *Executor, dagEng *DAGEngine, leader LeaderElector, pollInterval time.Duration) *Scheduler { +func NewScheduler(store WorkflowStore, executor *Executor, dagEng *DAGEngine, leader LeaderElector, joins joinDeadlineHandler, pollInterval time.Duration) *Scheduler { return &Scheduler{ store: store, executor: executor, dagEng: dagEng, leader: leader, + joins: joins, parser: cron.NewParser(cron.Minute | cron.Hour | cron.Dom | cron.Month | cron.Dow | cron.Descriptor), interval: pollInterval, } @@ -134,6 +142,22 @@ func (s *Scheduler) poll(ctx context.Context) error { } } + // Join deadline sweep: fire on_timeout (or abort) for open joins past their + // deadline. Leader-gated like the schedule sweep above. + due, err := s.store.GetDueJoinDeadlines(ctx, now) + if err != nil { + log.Error().Err(err).Msg("failed to get due join deadlines") + } else if s.joins != nil { + for i := range due { + if err := s.joins.HandleDeadline(ctx, &due[i]); err != nil { + log.Warn().Err(err). + Str("join", due[i].JoinName). + Str("correlation_key", due[i].CorrelationKey). + Msg("failed to handle join deadline") + } + } + } + return nil } diff --git a/server/internal/workflow/server.go b/server/internal/workflow/server.go index c422c44..0dbfd27 100644 --- a/server/internal/workflow/server.go +++ b/server/internal/workflow/server.go @@ -8,7 +8,6 @@ import ( "strings" "time" - "github.com/redis/go-redis/v9" "github.com/rs/zerolog/log" pb "github.com/scitrera/aether/api/proto" "github.com/scitrera/aether/sdk/go/aether" @@ -17,7 +16,6 @@ import ( // Server is the top-level workflow server that orchestrates all components. type Server struct { cfg *Config - redis redis.UniversalClient client *aether.WorkflowEngineClient store WorkflowStore router *Router @@ -50,11 +48,8 @@ func (s *Server) Run(ctx context.Context) error { // 1. The store is already constructed and migrated by the caller. // The workflow package never opens databases or runs migrations. - // 2. Connect to Redis (skipped in lite mode) - if s.cfg.Mode != ModeLite { - s.initRedis() - defer s.redis.Close() - } + // 2. (Leader election no longer needs a dedicated Redis connection — it runs + // over the gateway's KV via the WorkflowEngine client; see initComponents.) // 4. Create Aether WorkflowEngineClient if err := s.initAetherClient(); err != nil { @@ -64,10 +59,22 @@ func (s *Server) Run(ctx context.Context) error { // 5. Initialize components s.initComponents() - // 6. Register event handler on the Aether client - s.client.OnMessage(func(msgCtx context.Context, msg *aether.Message) error { + // 6. Register event handler on the Aether client. + // + // MUST be wrapped in AsyncMessageHandler: handleMessage → router.HandleEvent + // → JoinEngine.HandleArrival makes SYNCHRONOUS KV coord calls (SetNXSync on + // the coalesce/count/set gates) back to the gateway. The receive loop + // dispatches handlers inline on a single goroutine, and the KV response can + // only be delivered by that same loop — so a synchronous KV call from a + // handler running ON the loop self-deadlocks and always hits the 5s KV + // timeout (surfaced as "coalesce gate: DEADLINE_EXCEEDED"). AsyncMessageHandler + // runs each frame on its own goroutine, freeing the loop to deliver the + // nested response. See sdk/go/aether/async_handler.go + the OnMessage doc + // comment. Join arrivals are safe to process concurrently/out-of-order (the + // join engine rendezvous via distributed locks/counters/dedup). + s.client.OnMessage(aether.AsyncMessageHandler(func(msgCtx context.Context, msg *aether.Message) error { return s.handleMessage(msgCtx, msg) - }) + })) // 6.1 Register workflow operation handler for forwarded CRUD requests from gateway s.client.OnWorkflowOperation(s.handleWorkflowOperation) @@ -116,17 +123,10 @@ func (s *Server) Run(ctx context.Context) error { } }() - // 8. Start leader election refresh loop (Redis mode only) - if redisLeader, ok := s.leader.(*RedisLeaderElector); ok { - go func() { - defer func() { - if r := recover(); r != nil { - log.Error().Interface("panic", r).Msg("recovered from panic in leader election goroutine") - } - }() - redisLeader.RunRefreshLoop(ctx) - }() - } + // 8. Start the leader election loop. The coord-backed elector handles + // acquisition AND lease renewal internally (the single-node elector is a + // no-op that is always leader), so no separate refresh goroutine is needed. + s.leader.Start(ctx) // Wait for leadership before starting scheduler and monitor s.waitForLeadership(ctx) @@ -201,24 +201,10 @@ func (s *Server) Run(ctx context.Context) error { log.Warn().Err(err).Msg("workflow admin server shutdown returned error") } } - s.leader.Release(context.Background()) - return nil -} - -func (s *Server) initRedis() { - addrs := s.cfg.Redis.Cluster - if len(addrs) == 1 { - s.redis = redis.NewClient(&redis.Options{ - Addr: addrs[0], - Password: s.cfg.Redis.Password, - }) - } else { - s.redis = redis.NewUniversalClient(&redis.UniversalOptions{ - Addrs: addrs, - Password: s.cfg.Redis.Password, - }) + if err := s.leader.Shutdown(context.Background()); err != nil { + log.Warn().Err(err).Msg("leader election shutdown returned error") } - log.Info().Strs("addrs", addrs).Msg("Redis client initialized") + return nil } func (s *Server) initAetherClient() error { @@ -301,21 +287,41 @@ func (s *Server) initComponents() { // s.store is guaranteed non-nil by NewServer. s.executor = NewExecutor(s.client, s.cfg.Aether.Workspace) - if s.cfg.Mode == ModeLite { - s.leader = NewSingleNodeLeaderElector() - } else { - s.leader = NewRedisLeaderElector(s.redis, "workflow:leader", impl) - } + // Leader election runs over the gateway's KV store via the WorkflowEngine + // client, on the shared global scope under the reserved coordination key. + // One leader is elected across replicas in every backend mode (Redis / + // Badger / JetStream) with no dedicated Redis connection. In single-node + // deployments the sole instance simply wins the lock immediately. + leaderLocker := s.client.KV().Locker(aether.CoordScope{Scope: aether.KVScopeGlobal}) + s.leader = NewCoordLeaderElector(leaderLocker, workflowLeaderKey, impl) exprEng := NewExprEngine(s.cfg.Workflow.GetRuleCacheSize()) tmplEng := NewTemplateEngine(s.cfg.Workflow.GetRuleCacheSize()) - s.router = NewRouter(s.store, exprEng, tmplEng, s.executor, s.cfg.Workflow.GetRuleCacheTTL()) + // Join engine shares the WorkflowEngine client's KV (global scope, reserved + // coordination namespace) for its atomic arrival counter, dedup ledger, and + // fire-marker — portable across Redis / Badger / JetStream like the leader lock. + joinScope := aether.CoordScope{Scope: aether.KVScopeGlobal} + joinSetBackend := &kvJoinSet{kv: s.client.KV(), scope: joinScope} + joinEng := NewJoinEngine(s.store, exprEng, s.executor, s.client.KV().Counter(joinScope), s.client.KV().Locker(joinScope), joinSetBackend, s.cfg.Aether.Workspace) + + s.router = NewRouter(s.store, exprEng, tmplEng, s.executor, joinEng, s.cfg.Workflow.GetRuleCacheTTL()) s.dagEng = NewDAGEngine(s.store, exprEng, tmplEng, s.executor, &s.cfg.Workflow) - s.scheduler = NewScheduler(s.store, s.executor, s.dagEng, s.leader, s.cfg.Workflow.GetSchedulerPollInterval()) + s.scheduler = NewScheduler(s.store, s.executor, s.dagEng, s.leader, joinEng, s.cfg.Workflow.GetSchedulerPollInterval()) s.stateMach = NewStateMachineEngine(s.store, s.executor) } +// kvJoinSet adapts the WorkflowEngine client's KV to the join engine's joinSet +// interface, issuing the gRPC SetAdd primitive on the shared coordination scope. +type kvJoinSet struct { + kv *aether.KV + scope aether.CoordScope +} + +func (s *kvJoinSet) SetAdd(ctx context.Context, key, member string, ttl time.Duration) (bool, int64, error) { + return s.kv.SetAddSync(ctx, key, []byte(member), s.scope.Scope, s.scope.UserID, s.scope.Workspace, ttl, aether.DefaultKVTimeout) +} + func (s *Server) handleMessage(ctx context.Context, msg *aether.Message) error { // Only process EVENT messages if msg.MessageType != pb.MessageType_EVENT { @@ -401,14 +407,16 @@ func (s *Server) runStateMachineMonitor(ctx context.Context) { } func (s *Server) waitForLeadership(ctx context.Context) { + // The election loop (started via s.leader.Start) acquires leadership + // asynchronously; poll until this instance leads or ctx is cancelled. for { - if s.leader.TryAcquire(ctx) { + if s.leader.IsLeader() { return } select { case <-ctx.Done(): return - case <-time.After(5 * time.Second): + case <-time.After(time.Second): } } } diff --git a/server/internal/workflow/single_leader.go b/server/internal/workflow/single_leader.go deleted file mode 100644 index 16cc4bc..0000000 --- a/server/internal/workflow/single_leader.go +++ /dev/null @@ -1,15 +0,0 @@ -package workflow - -import "context" - -// SingleNodeLeaderElector is a no-op leader elector for lite/single-node mode. -// It always considers itself the leader, requiring no Redis dependency. -type SingleNodeLeaderElector struct{} - -func NewSingleNodeLeaderElector() *SingleNodeLeaderElector { - return &SingleNodeLeaderElector{} -} - -func (s *SingleNodeLeaderElector) TryAcquire(_ context.Context) bool { return true } -func (s *SingleNodeLeaderElector) Release(_ context.Context) {} -func (s *SingleNodeLeaderElector) IsLeader() bool { return true } diff --git a/server/internal/workflow/store.go b/server/internal/workflow/store.go index fa57470..551f293 100644 --- a/server/internal/workflow/store.go +++ b/server/internal/workflow/store.go @@ -614,6 +614,204 @@ func (s *Store) SetScheduleActiveTask(ctx context.Context, scheduleID, taskID st return err } +// ============================================================================= +// Join types and operations +// ============================================================================= + +// Join is the durable mirror of a fan-in/barrier/coalesce join instance. The +// authoritative arrival counter lives in KV; this row is the observability + +// deadline-sweep surface. One row per (join_name, workspace, correlation_key). +type Join struct { + ID int64 + JoinName string + Workspace string + CorrelationKey string + Mode string // "count" | "set" | "coalesce" + ExpectedCount *int64 // nil until armed (dynamic-N) + ArrivedCount int64 + Dirty bool // coalesce re-arm flag + Status string // "open" | "fired" | "timed_out" | "cancelled" + // Persisted firing actions so the deadline sweep (which has no live event to + // re-render from) can fire on_timeout/on_complete. OnComplete/OnTimeout hold + // the JSON-serialized ActionDef; OnPartialFailure holds the policy string. + OnComplete string + OnTimeout string + OnPartialFailure string + DeadlineAt *time.Time + LingerUntil *time.Time + CreatedAt time.Time + UpdatedAt time.Time +} + +const ( + JoinModeCount = "count" + JoinModeSet = "set" + JoinModeCoalesce = "coalesce" + + JoinStatusOpen = "open" + JoinStatusFired = "fired" + JoinStatusTimedOut = "timed_out" + JoinStatusCancelled = "cancelled" +) + +// nullableText stores an empty string as SQL NULL so optional TEXT columns +// (on_complete/on_timeout/on_partial_failure) stay NULL rather than '' when unset. +func nullableText(s string) interface{} { + if s == "" { + return nil + } + return s +} + +func (s *Store) EnsureJoin(ctx context.Context, j *Join) (*Join, error) { + query := ` + INSERT INTO workflow_joins (join_name, workspace, correlation_key, mode, + expected_count, arrived_count, dirty, status, + on_complete, on_timeout, on_partial_failure, + deadline_at, linger_until) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13) + ON CONFLICT (join_name, workspace, correlation_key) DO NOTHING + ` + _, err := s.db.ExecContext(ctx, query, + j.JoinName, j.Workspace, j.CorrelationKey, j.Mode, + j.ExpectedCount, j.ArrivedCount, j.Dirty, j.Status, + nullableText(j.OnComplete), nullableText(j.OnTimeout), nullableText(j.OnPartialFailure), + j.DeadlineAt, j.LingerUntil, + ) + if err != nil { + return nil, fmt.Errorf("ensure join: %w", err) + } + return s.GetJoin(ctx, j.JoinName, j.Workspace, j.CorrelationKey) +} + +func (s *Store) GetJoin(ctx context.Context, joinName, workspace, correlationKey string) (*Join, error) { + query := ` + SELECT id, join_name, workspace, correlation_key, mode, expected_count, + arrived_count, dirty, status, on_complete, on_timeout, on_partial_failure, + deadline_at, linger_until, created_at, updated_at + FROM workflow_joins + WHERE join_name = $1 AND workspace = $2 AND correlation_key = $3 + ` + var j Join + var onComplete, onTimeout, onPartialFailure sql.NullString + err := s.db.QueryRowContext(ctx, query, joinName, workspace, correlationKey).Scan( + &j.ID, &j.JoinName, &j.Workspace, &j.CorrelationKey, &j.Mode, &j.ExpectedCount, + &j.ArrivedCount, &j.Dirty, &j.Status, &onComplete, &onTimeout, &onPartialFailure, + &j.DeadlineAt, &j.LingerUntil, &j.CreatedAt, &j.UpdatedAt, + ) + if err == sql.ErrNoRows { + return nil, nil + } + if err != nil { + return nil, fmt.Errorf("get join: %w", err) + } + j.OnComplete = onComplete.String + j.OnTimeout = onTimeout.String + j.OnPartialFailure = onPartialFailure.String + return &j, nil +} + +func (s *Store) UpdateJoinArrived(ctx context.Context, id int64, arrivedCount int64) error { + query := `UPDATE workflow_joins SET arrived_count = $2, updated_at = now() WHERE id = $1` + _, err := s.db.ExecContext(ctx, query, id, arrivedCount) + return err +} + +func (s *Store) SetJoinExpected(ctx context.Context, id int64, expected int64) error { + query := `UPDATE workflow_joins SET expected_count = $2, updated_at = now() WHERE id = $1` + _, err := s.db.ExecContext(ctx, query, id, expected) + return err +} + +func (s *Store) SetJoinDirty(ctx context.Context, id int64, dirty bool) error { + query := `UPDATE workflow_joins SET dirty = $2, updated_at = now() WHERE id = $1` + _, err := s.db.ExecContext(ctx, query, id, dirty) + return err +} + +func (s *Store) MarkJoinTerminal(ctx context.Context, id int64, status string, lingerUntil time.Time) error { + query := `UPDATE workflow_joins SET status = $2, linger_until = $3, updated_at = now() WHERE id = $1` + _, err := s.db.ExecContext(ctx, query, id, status, lingerUntil) + return err +} + +func (s *Store) GetDueJoinDeadlines(ctx context.Context, now time.Time) ([]Join, error) { + query := ` + SELECT id, join_name, workspace, correlation_key, mode, expected_count, + arrived_count, dirty, status, on_complete, on_timeout, on_partial_failure, + deadline_at, linger_until, created_at, updated_at + FROM workflow_joins + WHERE status = 'open' AND deadline_at IS NOT NULL AND deadline_at <= $1 + ORDER BY deadline_at ASC + ` + if !s.isSQLite { + query += " FOR UPDATE SKIP LOCKED" + } + rows, err := s.db.QueryContext(ctx, query, now) + if err != nil { + return nil, fmt.Errorf("get due join deadlines: %w", err) + } + defer rows.Close() + + var joins []Join + for rows.Next() { + var j Join + var onComplete, onTimeout, onPartialFailure sql.NullString + if err := rows.Scan( + &j.ID, &j.JoinName, &j.Workspace, &j.CorrelationKey, &j.Mode, &j.ExpectedCount, + &j.ArrivedCount, &j.Dirty, &j.Status, &onComplete, &onTimeout, &onPartialFailure, + &j.DeadlineAt, &j.LingerUntil, &j.CreatedAt, &j.UpdatedAt, + ); err != nil { + return nil, fmt.Errorf("scan join: %w", err) + } + j.OnComplete = onComplete.String + j.OnTimeout = onTimeout.String + j.OnPartialFailure = onPartialFailure.String + joins = append(joins, j) + } + return joins, rows.Err() +} + +func (s *Store) ListJoins(ctx context.Context, workspace string) ([]Join, error) { + query := ` + SELECT id, join_name, workspace, correlation_key, mode, expected_count, + arrived_count, dirty, status, on_complete, on_timeout, on_partial_failure, + deadline_at, linger_until, created_at, updated_at + FROM workflow_joins + ` + var rows *sql.Rows + var err error + if workspace == "" || workspace == "*" { + query += " ORDER BY created_at DESC" + rows, err = s.db.QueryContext(ctx, query) + } else { + query += " WHERE workspace = $1 ORDER BY created_at DESC" + rows, err = s.db.QueryContext(ctx, query, workspace) + } + if err != nil { + return nil, fmt.Errorf("list joins: %w", err) + } + defer rows.Close() + + var joins []Join + for rows.Next() { + var j Join + var onComplete, onTimeout, onPartialFailure sql.NullString + if err := rows.Scan( + &j.ID, &j.JoinName, &j.Workspace, &j.CorrelationKey, &j.Mode, &j.ExpectedCount, + &j.ArrivedCount, &j.Dirty, &j.Status, &onComplete, &onTimeout, &onPartialFailure, + &j.DeadlineAt, &j.LingerUntil, &j.CreatedAt, &j.UpdatedAt, + ); err != nil { + return nil, fmt.Errorf("scan join: %w", err) + } + j.OnComplete = onComplete.String + j.OnTimeout = onTimeout.String + j.OnPartialFailure = onPartialFailure.String + joins = append(joins, j) + } + return joins, rows.Err() +} + // ============================================================================= // Additional query methods for Admin API // ============================================================================= diff --git a/server/internal/workflow/store_iface.go b/server/internal/workflow/store_iface.go index f7b64f6..9267d39 100644 --- a/server/internal/workflow/store_iface.go +++ b/server/internal/workflow/store_iface.go @@ -71,6 +71,16 @@ type WorkflowStore interface { UpdateScheduleAfterFire(ctx context.Context, id string, lastFired time.Time, nextFire *time.Time) error SetScheduleActiveTask(ctx context.Context, scheduleID, taskID string) error + // Joins + EnsureJoin(ctx context.Context, j *Join) (*Join, error) + GetJoin(ctx context.Context, joinName, workspace, correlationKey string) (*Join, error) + UpdateJoinArrived(ctx context.Context, id int64, arrivedCount int64) error + SetJoinExpected(ctx context.Context, id int64, expected int64) error + SetJoinDirty(ctx context.Context, id int64, dirty bool) error + MarkJoinTerminal(ctx context.Context, id int64, status string, lingerUntil time.Time) error + GetDueJoinDeadlines(ctx context.Context, now time.Time) ([]Join, error) + ListJoins(ctx context.Context, workspace string) ([]Join, error) + // State machines CreateStateMachine(ctx context.Context, sm *StateMachineDef) error GetStateMachine(ctx context.Context, id string) (*StateMachineDef, error) diff --git a/server/internal/workflow/templates.go b/server/internal/workflow/templates.go index ed9619e..80636e5 100644 --- a/server/internal/workflow/templates.go +++ b/server/internal/workflow/templates.go @@ -28,12 +28,32 @@ func NewTemplateEngine(maxCacheSize int) *TemplateEngine { } // TransformResult is the parsed output of a template transformation. +// +// The first block (agent/tool_name/arguments) drives the default "message" +// (tool-call) dispatch. The create_task block lets an event-triggered rule +// dispatch an Aether POOL task instead — e.g. fire a memorylayer worker task +// in response to an ingest-complete event. When Type == "create_task" the +// executor routes to dispatchCreateTask using task_type/target_implementation/ +// payload; otherwise these fields are zero and the legacy message path runs. type TransformResult struct { Agent string `yaml:"agent" json:"agent"` ToolName string `yaml:"tool_name" json:"tool_name"` Arguments map[string]any `yaml:"arguments" json:"arguments"` Workspace string `yaml:"workspace" json:"workspace"` Metadata map[string]string `yaml:"metadata" json:"metadata"` + // create_task dispatch (event -> POOL task). Empty Type preserves the + // historical "message" behavior, so existing rules are unaffected. + Type string `yaml:"type" json:"type"` + TaskType string `yaml:"task_type" json:"task_type"` + TargetImplementation string `yaml:"target_implementation" json:"target_implementation"` + Payload any `yaml:"payload" json:"payload"` + // Fan-out tagging for spawned tasks: a correlation id (the join's barrier + // key) and an optional feed-B completion-event opt-in. + CorrelationID string `yaml:"correlation_id" json:"correlation_id"` + CompletionEvent *CompletionEventConfig `yaml:"completion_event" json:"completion_event"` + // Join destination (Type == "join"): a fan-in / barrier / coalesce. nil for + // every other destination kind, so existing rules are unaffected. + Join *JoinSpec `yaml:"join" json:"join"` } // Transform renders a Go text/template with the given data, then parses diff --git a/server/internal/workflow/templates_test.go b/server/internal/workflow/templates_test.go index a31b392..f3c1f02 100644 --- a/server/internal/workflow/templates_test.go +++ b/server/internal/workflow/templates_test.go @@ -99,6 +99,55 @@ workspace: "{{ .source.workspace }}"`, } } +// TestTemplateEngine_Transform_CreateTask validates that a create_task +// destination template (as registered by the platform's kb-update-on-ingest +// rule) parses into a TransformResult carrying the create_task fields, so the +// router can dispatch an Aether POOL task in response to an event. +func TestTemplateEngine_Transform_CreateTask(t *testing.T) { + eng := NewTemplateEngine(100) + + tmpl := "type: create_task\n" + + "task_type: memorylayer-task.kb_update\n" + + "target_implementation: memorylayer-worker\n" + + "workspace: \"{{ .source.workspace }}\"\n" + + "payload:\n" + + " workspace_id: \"{{ .source.workspace }}\"\n" + + "metadata:\n" + + " bg_kind: kb\n" + + " visibility: workspace\n" + + " title: \"Updating knowledge base\"\n" + + r, err := eng.Transform(tmpl, map[string]any{ + "source": map[string]any{"workspace": "ws-default"}, + "input": map[string]any{"workspace_id": "ws-default"}, + }) + if err != nil { + t.Fatalf("Transform() error = %v", err) + } + if r.Type != "create_task" { + t.Errorf("Type = %q, want %q", r.Type, "create_task") + } + if r.TaskType != "memorylayer-task.kb_update" { + t.Errorf("TaskType = %q, want %q", r.TaskType, "memorylayer-task.kb_update") + } + if r.TargetImplementation != "memorylayer-worker" { + t.Errorf("TargetImplementation = %q, want %q", r.TargetImplementation, "memorylayer-worker") + } + if r.Workspace != "ws-default" { + t.Errorf("Workspace = %q, want %q", r.Workspace, "ws-default") + } + if r.Metadata["bg_kind"] != "kb" { + t.Errorf("Metadata[bg_kind] = %q, want %q", r.Metadata["bg_kind"], "kb") + } + payload, ok := r.Payload.(map[string]any) + if !ok { + t.Fatalf("Payload type = %T, want map[string]any", r.Payload) + } + if payload["workspace_id"] != "ws-default" { + t.Errorf("Payload[workspace_id] = %v, want %q", payload["workspace_id"], "ws-default") + } +} + func TestTemplateEngine_Caching(t *testing.T) { eng := NewTemplateEngine(100) diff --git a/server/internal/workflow/workflow_handler.go b/server/internal/workflow/workflow_handler.go index 2d99de1..6427bb4 100644 --- a/server/internal/workflow/workflow_handler.go +++ b/server/internal/workflow/workflow_handler.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "strconv" + "time" "github.com/rs/zerolog/log" pb "github.com/scitrera/aether/api/proto" @@ -73,6 +74,14 @@ func (s *Server) handleWorkflowOperation(ctx context.Context, op *pb.WorkflowOpe case pb.WorkflowOperation_SEND_SM_EVENT: return s.handleSendSMEvent(ctx, op) + // ---- Joins ---- + case pb.WorkflowOperation_LIST_JOINS: + return s.handleListJoins(ctx, op) + case pb.WorkflowOperation_GET_JOIN: + return s.handleGetJoin(ctx, op) + case pb.WorkflowOperation_CANCEL_JOIN: + return s.handleCancelJoin(ctx, op) + default: return &pb.WorkflowResponse{ Success: false, @@ -562,3 +571,92 @@ func (s *Server) handleSendSMEvent(ctx context.Context, op *pb.WorkflowOperation log.Debug().Str("instance_id", instanceID).Str("event", eventName).Msg("state machine event sent via gRPC stream") return jsonResponse(op.RequestId, instance) } + +// ============================================================================= +// Joins +// ============================================================================= + +// joinView is the observability projection of a Join (N5). It deliberately +// omits the internal firing-action JSON (on_complete/on_timeout/on_partial_failure). +type joinView struct { + JoinName string `json:"join_name"` + Workspace string `json:"workspace"` + CorrelationKey string `json:"correlation_key"` + Mode string `json:"mode"` + Arrived int64 `json:"arrived"` + Expected *int64 `json:"expected,omitempty"` + Dirty bool `json:"dirty"` + Status string `json:"status"` + DeadlineAt *time.Time `json:"deadline_at,omitempty"` + LingerUntil *time.Time `json:"linger_until,omitempty"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` +} + +func toJoinView(j Join) joinView { + return joinView{ + JoinName: j.JoinName, + Workspace: j.Workspace, + CorrelationKey: j.CorrelationKey, + Mode: j.Mode, + Arrived: j.ArrivedCount, + Expected: j.ExpectedCount, + Dirty: j.Dirty, + Status: j.Status, + DeadlineAt: j.DeadlineAt, + LingerUntil: j.LingerUntil, + CreatedAt: j.CreatedAt, + UpdatedAt: j.UpdatedAt, + } +} + +func toJoinViews(joins []Join) []joinView { + views := make([]joinView, 0, len(joins)) + for _, j := range joins { + views = append(views, toJoinView(j)) + } + return views +} + +func (s *Server) handleListJoins(ctx context.Context, op *pb.WorkflowOperation) (*pb.WorkflowResponse, error) { + joins, err := s.store.ListJoins(ctx, op.Workspace) + if err != nil { + return errResponse(op.RequestId, err.Error()), nil + } + return jsonResponse(op.RequestId, toJoinViews(joins)) +} + +func (s *Server) handleGetJoin(ctx context.Context, op *pb.WorkflowOperation) (*pb.WorkflowResponse, error) { + j, err := s.store.GetJoin(ctx, op.Id, op.Workspace, op.SecondaryId) + if err != nil { + return errResponse(op.RequestId, err.Error()), nil + } + if j == nil { + return errResponse(op.RequestId, "join not found"), nil + } + return jsonResponse(op.RequestId, toJoinView(*j)) +} + +func (s *Server) handleCancelJoin(ctx context.Context, op *pb.WorkflowOperation) (*pb.WorkflowResponse, error) { + j, err := s.store.GetJoin(ctx, op.Id, op.Workspace, op.SecondaryId) + if err != nil { + return errResponse(op.RequestId, err.Error()), nil + } + if j == nil { + return errResponse(op.RequestId, "join not found"), nil + } + if j.Status != JoinStatusOpen { + // Already terminal — cancellation is idempotent. + return jsonResponse(op.RequestId, map[string]any{ + "status": j.Status, + "message": "join already terminal", + }) + } + if err := s.store.MarkJoinTerminal(ctx, j.ID, JoinStatusCancelled, time.Now().Add(time.Minute)); err != nil { + return errResponse(op.RequestId, err.Error()), nil + } + return jsonResponse(op.RequestId, map[string]any{ + "status": JoinStatusCancelled, + "message": "join cancelled", + }) +} diff --git a/server/migrations/025_task_retry_policy.sql b/server/migrations/025_task_retry_policy.sql new file mode 100644 index 0000000..a22ebd8 --- /dev/null +++ b/server/migrations/025_task_retry_policy.sql @@ -0,0 +1,8 @@ +-- Add nullable retry_policy_json to tasks table. Phase 2 of the webhook +-- integration initiative lifts retry scheduling out of individual workers +-- and into the task store. Tasks without a policy keep legacy behavior. + +ALTER TABLE tasks ADD COLUMN IF NOT EXISTS retry_policy_json JSONB; + +-- No index needed: retry_policy is read on FailTask via task_id (already +-- the primary key) and not used in any scan path. diff --git a/server/migrations/026_task_priority.sql b/server/migrations/026_task_priority.sql new file mode 100644 index 0000000..40ff394 --- /dev/null +++ b/server/migrations/026_task_priority.sql @@ -0,0 +1,27 @@ +-- Wire up dispatch priority for durable tasks. The tasks.priority column has +-- existed since migration 002 (INT NOT NULL DEFAULT 0) but was never set or +-- used. This migration makes NORMAL (30) the effective default, backfills +-- legacy rows, adds a dispatch-ordering index, and extends the orchestrated +-- task queue with the same priority weight so both delivery paths order by it. +-- +-- Priority weights mirror the proto TaskPriority enum (spaced to allow future +-- levels): XLOW=10, LOW=20, NORMAL=30, HIGH=40, PREEMPT=50. Higher = delivered +-- first; the numeric value is used directly as the descending sort key. + +-- tasks: default to NORMAL and backfill rows written before this feature. +ALTER TABLE tasks ALTER COLUMN priority SET DEFAULT 30; +UPDATE tasks SET priority = 30 WHERE priority = 0; + +-- Dispatch-selection index: highest priority first, FIFO within a level, +-- scoped to pending tasks (the only rows the selection paths scan). +CREATE INDEX IF NOT EXISTS idx_task_priority_dispatch + ON tasks (priority DESC, created_at ASC) + WHERE status = 'pending'; + +-- orchestrated_task_queue: carry the task's priority so the polling/notify +-- dispatcher can order spawns the same way. +ALTER TABLE orchestrated_task_queue ADD COLUMN IF NOT EXISTS priority INT NOT NULL DEFAULT 30; + +CREATE INDEX IF NOT EXISTS idx_orch_queue_priority + ON orchestrated_task_queue (priority DESC, created_at ASC) + WHERE status = 'pending'; diff --git a/server/migrations/027_task_correlation.sql b/server/migrations/027_task_correlation.sql new file mode 100644 index 0000000..f17c397 --- /dev/null +++ b/server/migrations/027_task_correlation.sql @@ -0,0 +1,23 @@ +-- Migration: correlation / flow-root lineage and feed-B completion-event config +-- +-- correlation_id : fan-out/fan-in correlation identity (the barrier/group id a +-- workflow join matches against), distinct from task_id. +-- root_task_id : flow-root task id propagated from a task's spawner. A task +-- with no provided root is its own flow root. +-- completion_event : "feed B" config (JSON) — emit a domain event onto event::* +-- when the task reaches a selected terminal status. + +ALTER TABLE tasks + ADD COLUMN IF NOT EXISTS correlation_id VARCHAR(255); +ALTER TABLE tasks + ADD COLUMN IF NOT EXISTS root_task_id VARCHAR(255); +ALTER TABLE tasks + ADD COLUMN IF NOT EXISTS completion_event JSONB; + +CREATE INDEX IF NOT EXISTS idx_tasks_correlation_id + ON tasks (correlation_id) + WHERE correlation_id IS NOT NULL; + +CREATE INDEX IF NOT EXISTS idx_tasks_root_task_id + ON tasks (root_task_id) + WHERE root_task_id IS NOT NULL; diff --git a/server/migrations/028_acl_groups_roles.sql b/server/migrations/028_acl_groups_roles.sql new file mode 100644 index 0000000..21f56ca --- /dev/null +++ b/server/migrations/028_acl_groups_roles.sql @@ -0,0 +1,86 @@ +-- ACL Groups & Roles +-- +-- Adds role/group authorization on top of the per-principal acl_rules model. +-- A *group* is a named collection of principals; a *role* is a named bundle of +-- permissions assignable to principals or groups. Both are modeled as synthetic +-- subjects in acl_rules (principal_type 'group' / 'role', which the existing +-- VARCHAR(50) column already accepts), so role/group PERMISSIONS need no new +-- table — they are ordinary acl_rules rows. The only new state is membership +-- (who is in a group) and assignment (who has a role), loaded into the Casbin +-- enforcer as `g` (grouping) edges for transitive resolution. + +-- Group definitions +CREATE TABLE IF NOT EXISTS acl_groups +( + group_id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + group_name VARCHAR(255) NOT NULL UNIQUE, -- canonical id used in subjects: "group:" + description TEXT, + created_by VARCHAR(255), + created_at TIMESTAMP NOT NULL DEFAULT NOW(), + metadata JSONB +); + +-- Role definitions +CREATE TABLE IF NOT EXISTS acl_roles +( + role_id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + role_name VARCHAR(255) NOT NULL UNIQUE, -- canonical id used in subjects: "role:" + description TEXT, + created_by VARCHAR(255), + created_at TIMESTAMP NOT NULL DEFAULT NOW(), + metadata JSONB +); + +-- Group membership: which principals (or nested groups) belong to a group. +-- member_type is a principal type ('user','agent','task',...) OR 'group' (nesting). +-- Edge loaded as Casbin g-rule: g(":", "group:"). +CREATE TABLE IF NOT EXISTS acl_group_members +( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + group_id UUID NOT NULL REFERENCES acl_groups (group_id) ON DELETE CASCADE, + member_type VARCHAR(50) NOT NULL, + member_id VARCHAR(255) NOT NULL, + granted_by VARCHAR(255), + granted_at TIMESTAMP NOT NULL DEFAULT NOW(), + expires_at TIMESTAMP, + CONSTRAINT unique_group_member UNIQUE (group_id, member_type, member_id) +); + +-- Role assignment: which principals (or groups) are granted a role. +-- assignee_type is a principal type OR 'group'. +-- Edge loaded as Casbin g-rule: g(":", "role:"). +CREATE TABLE IF NOT EXISTS acl_role_assignments +( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + role_id UUID NOT NULL REFERENCES acl_roles (role_id) ON DELETE CASCADE, + assignee_type VARCHAR(50) NOT NULL, + assignee_id VARCHAR(255) NOT NULL, + granted_by VARCHAR(255), + granted_at TIMESTAMP NOT NULL DEFAULT NOW(), + expires_at TIMESTAMP, + CONSTRAINT unique_role_assignment UNIQUE (role_id, assignee_type, assignee_id) +); + +CREATE INDEX IF NOT EXISTS idx_group_members_member ON acl_group_members (member_type, member_id); +CREATE INDEX IF NOT EXISTS idx_group_members_expiration ON acl_group_members (expires_at) WHERE expires_at IS NOT NULL; +CREATE INDEX IF NOT EXISTS idx_role_assignments_assignee ON acl_role_assignments (assignee_type, assignee_id); +CREATE INDEX IF NOT EXISTS idx_role_assignments_expiration ON acl_role_assignments (expires_at) WHERE expires_at IS NOT NULL; + +-- Cleanup function for expired memberships/assignments (parallels +-- cleanup_expired_acl_rules()). Returns the total number of rows removed. +CREATE OR REPLACE FUNCTION cleanup_expired_acl_memberships() + RETURNS INTEGER AS +$$ +DECLARE + deleted_members INTEGER; + deleted_assignments INTEGER; +BEGIN + DELETE FROM acl_group_members WHERE expires_at IS NOT NULL AND expires_at <= NOW(); + GET DIAGNOSTICS deleted_members = ROW_COUNT; + + DELETE FROM acl_role_assignments WHERE expires_at IS NOT NULL AND expires_at <= NOW(); + GET DIAGNOSTICS deleted_assignments = ROW_COUNT; + + RETURN deleted_members + deleted_assignments; +END; +$$ LANGUAGE plpgsql; diff --git a/server/migrations/sqlite_acl/003_acl_groups_roles.sql b/server/migrations/sqlite_acl/003_acl_groups_roles.sql new file mode 100644 index 0000000..7629d87 --- /dev/null +++ b/server/migrations/sqlite_acl/003_acl_groups_roles.sql @@ -0,0 +1,52 @@ +-- ACL Groups & Roles (SQLite dialect of migrations/028_acl_groups_roles.sql) +-- +-- A group is a named collection of principals; a role is a named permission +-- bundle. Role/group PERMISSIONS reuse acl_rules (principal_type 'group'/'role'). +-- The only new state is membership + assignment, loaded into the Casbin enforcer +-- as `g` edges. UUIDs are generated in Go (github.com/google/uuid); timestamps +-- are ISO-8601 TEXT to match the rest of the sqlite_acl schema. + +CREATE TABLE IF NOT EXISTS acl_groups ( + group_id TEXT PRIMARY KEY, + group_name TEXT NOT NULL UNIQUE, + description TEXT, + created_by TEXT, + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')), + metadata TEXT +); + +CREATE TABLE IF NOT EXISTS acl_roles ( + role_id TEXT PRIMARY KEY, + role_name TEXT NOT NULL UNIQUE, + description TEXT, + created_by TEXT, + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')), + metadata TEXT +); + +CREATE TABLE IF NOT EXISTS acl_group_members ( + id TEXT PRIMARY KEY, + group_id TEXT NOT NULL REFERENCES acl_groups (group_id) ON DELETE CASCADE, + member_type TEXT NOT NULL, + member_id TEXT NOT NULL, + granted_by TEXT, + granted_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')), + expires_at TEXT, + CONSTRAINT unique_group_member UNIQUE (group_id, member_type, member_id) +); + +CREATE TABLE IF NOT EXISTS acl_role_assignments ( + id TEXT PRIMARY KEY, + role_id TEXT NOT NULL REFERENCES acl_roles (role_id) ON DELETE CASCADE, + assignee_type TEXT NOT NULL, + assignee_id TEXT NOT NULL, + granted_by TEXT, + granted_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')), + expires_at TEXT, + CONSTRAINT unique_role_assignment UNIQUE (role_id, assignee_type, assignee_id) +); + +CREATE INDEX IF NOT EXISTS idx_group_members_member ON acl_group_members (member_type, member_id); +CREATE INDEX IF NOT EXISTS idx_group_members_expiration ON acl_group_members (expires_at) WHERE expires_at IS NOT NULL; +CREATE INDEX IF NOT EXISTS idx_role_assignments_assignee ON acl_role_assignments (assignee_type, assignee_id); +CREATE INDEX IF NOT EXISTS idx_role_assignments_expiration ON acl_role_assignments (expires_at) WHERE expires_at IS NOT NULL; diff --git a/server/migrations/sqlite_tasks/003_task_retry_policy.sql b/server/migrations/sqlite_tasks/003_task_retry_policy.sql new file mode 100644 index 0000000..4659073 --- /dev/null +++ b/server/migrations/sqlite_tasks/003_task_retry_policy.sql @@ -0,0 +1,9 @@ +-- Add nullable retry_policy_json column to the sqlite tasks table to +-- mirror migration 025 on postgres. Phase 2 of the webhook integration +-- initiative lifts retry scheduling out of individual workers and into +-- the task store. Tasks without a policy keep legacy behavior. +-- +-- SQLite supports IF NOT EXISTS on ALTER TABLE ADD COLUMN since version +-- 3.35 (March 2021), which is well below the modernc driver's minimum. + +ALTER TABLE tasks ADD COLUMN retry_policy_json TEXT; diff --git a/server/migrations/sqlite_tasks/004_task_priority.sql b/server/migrations/sqlite_tasks/004_task_priority.sql new file mode 100644 index 0000000..2a553b8 --- /dev/null +++ b/server/migrations/sqlite_tasks/004_task_priority.sql @@ -0,0 +1,25 @@ +-- Wire up dispatch priority for durable tasks (sqlite mirror of postgres +-- migration 026). The sqlite tasks table already has priority INTEGER NOT NULL +-- DEFAULT 0 (since 001); SQLite cannot alter a column default in place, so the +-- Go store normalizes UNSPECIFIED (0) to NORMAL (30) on write. Here we backfill +-- legacy rows, add the dispatch index, and extend the orchestrated task queue. +-- +-- Priority weights mirror the proto TaskPriority enum: XLOW=10, LOW=20, +-- NORMAL=30, HIGH=40, PREEMPT=50. Higher = delivered first. + +-- Backfill rows written before this feature (stored 0) to NORMAL. +UPDATE tasks SET priority = 30 WHERE priority = 0; + +-- Dispatch-selection index: highest priority first, FIFO within a level. +CREATE INDEX IF NOT EXISTS idx_task_priority_dispatch + ON tasks (priority DESC, created_at ASC) + WHERE status = 'pending'; + +-- orchestrated_task_queue: carry the task's priority so the polling dispatcher +-- can order spawns the same way. SQLite supports ADD COLUMN with a constant +-- default since 3.35. +ALTER TABLE orchestrated_task_queue ADD COLUMN priority INTEGER NOT NULL DEFAULT 30; + +CREATE INDEX IF NOT EXISTS idx_orch_queue_priority + ON orchestrated_task_queue (priority DESC, created_at ASC) + WHERE status = 'pending'; diff --git a/server/migrations/sqlite_tasks/005_task_correlation.sql b/server/migrations/sqlite_tasks/005_task_correlation.sql new file mode 100644 index 0000000..69dfa3f --- /dev/null +++ b/server/migrations/sqlite_tasks/005_task_correlation.sql @@ -0,0 +1,22 @@ +-- Migration 005: correlation / flow-root lineage and feed-B completion-event +-- config (SQLite). +-- +-- SQLite stores: +-- correlation_id as TEXT +-- root_task_id as TEXT +-- completion_event as TEXT (JSON object), NULL when the task did not opt into +-- feed B (consistent with the Go layer marshaling nil ⇒ NULL) + +ALTER TABLE tasks ADD COLUMN correlation_id TEXT; +ALTER TABLE tasks ADD COLUMN root_task_id TEXT; +ALTER TABLE tasks ADD COLUMN completion_event TEXT; + +-- Index: quick lookup of all tasks in a correlation group (join barrier). +CREATE INDEX IF NOT EXISTS idx_tasks_correlation_id + ON tasks (correlation_id) + WHERE correlation_id IS NOT NULL; + +-- Index: quick lookup of all tasks in a flow. +CREATE INDEX IF NOT EXISTS idx_tasks_root_task_id + ON tasks (root_task_id) + WHERE root_task_id IS NOT NULL; diff --git a/server/migrations/sqlite_workflow/002_workflow_joins.sql b/server/migrations/sqlite_workflow/002_workflow_joins.sql new file mode 100644 index 0000000..fb8c738 --- /dev/null +++ b/server/migrations/sqlite_workflow/002_workflow_joins.sql @@ -0,0 +1,32 @@ +-- ========================================================================= +-- Fan-in/barrier/coalesce join instances. The authoritative arrival counter +-- lives in KV; this table is the durable observability + deadline-sweep +-- surface. One row per (join_name, workspace, correlation_key). +-- ========================================================================= +CREATE TABLE IF NOT EXISTS workflow_joins ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + join_name TEXT NOT NULL, + workspace TEXT NOT NULL, + correlation_key TEXT NOT NULL, + mode TEXT NOT NULL, + expected_count INTEGER, + arrived_count INTEGER NOT NULL DEFAULT 0, + dirty INTEGER NOT NULL DEFAULT 0, + status TEXT NOT NULL DEFAULT 'open', + on_complete TEXT, + on_timeout TEXT, + on_partial_failure TEXT, + deadline_at TEXT, + linger_until TEXT, + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%SZ', 'now')), + updated_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%SZ', 'now')), + UNIQUE (join_name, workspace, correlation_key) +); +CREATE INDEX IF NOT EXISTS idx_workflow_joins_due ON workflow_joins(status, deadline_at); + +CREATE TRIGGER IF NOT EXISTS trg_workflow_joins_updated_at + AFTER UPDATE ON workflow_joins + FOR EACH ROW +BEGIN + UPDATE workflow_joins SET updated_at = strftime('%Y-%m-%dT%H:%M:%SZ', 'now') WHERE id = NEW.id; +END; diff --git a/server/pkg/authproxy/identity.go b/server/pkg/authproxy/identity.go index a129376..de3b9a6 100644 --- a/server/pkg/authproxy/identity.go +++ b/server/pkg/authproxy/identity.go @@ -93,6 +93,24 @@ type ResolvedIdentity struct { Reject *Rejection } +// ExtraHeaderNaming is an OPTIONAL interface an IdentityResolver may implement +// to declare the full set of ExtraHeaders names it can emit (e.g. Scitrera's +// X-Scitrera-*). The auth-proxy uses it ONLY to CLEAR those headers on the +// anonymous ext_authz passthrough (verify-optional with no credential): the +// resolver is never invoked on that path, so the middleware needs a static +// declaration of the extra header names to zero them out and prevent a +// client-supplied value from surviving Envoy's authz-response override. +// +// Resolvers that do not implement this interface simply have their extra +// headers (if any) not cleared on the anonymous path; the canonical X-Auth-* / +// X-Aether-* set is always cleared regardless. +type ExtraHeaderNaming interface { + // ExtraHeaderNames returns every header name this resolver may place in + // ResolvedIdentity.ExtraHeaders. It MUST be stable and safe for concurrent + // use. + ExtraHeaderNames() []string +} + // IdentityResolver maps an authenticator-verified principal + raw claims into // a ResolvedIdentity. Implementations MUST be safe for concurrent use. type IdentityResolver interface { diff --git a/server/pkg/authproxy/middleware.go b/server/pkg/authproxy/middleware.go index 5404d29..1dbb816 100644 --- a/server/pkg/authproxy/middleware.go +++ b/server/pkg/authproxy/middleware.go @@ -104,6 +104,37 @@ func (m *AuthMiddleware) SetSessionCookieName(name string) { m.sessionCookieName = name } +// SessionCookieName returns the cookie name the middleware reads to derive a +// session_token credential, or "" when session-cookie auth is disabled. This +// exposes the same source Authenticate consults so callers (notably the +// optional-auth verify handler) can perform a credential-presence pre-check +// without duplicating the cookie-name wiring. +func (m *AuthMiddleware) SessionCookieName() string { + return m.sessionCookieName +} + +// HasCredentials reports whether the request carries ANY credential the +// middleware would attempt to authenticate: a non-empty Authorization-derived +// credential, or (when a session cookie name is configured) a non-empty +// session cookie of that name. It mirrors the credential-selection logic in +// Authenticate but performs no validation and writes nothing to the response. +// +// This is used by the optional-auth verify path to decide, BEFORE calling +// Authenticate, whether a request should be treated as an anonymous +// passthrough (no credential) or fail-closed authenticated (credential +// present). +func (m *AuthMiddleware) HasCredentials(r *http.Request) bool { + if len(extractCredentials(r)) > 0 { + return true + } + if m.sessionCookieName != "" { + if c, err := r.Cookie(m.sessionCookieName); err == nil && c.Value != "" { + return true + } + } + return false +} + // NewAuthMiddleware creates a new AuthMiddleware with the given authenticator, // ACL evaluator, and tenant identifier. OBO authority resolution is disabled // and a pass-through identity resolver is used; use NewAuthMiddlewareWithResolver @@ -667,6 +698,39 @@ func (m *AuthMiddleware) SetResponseHeaders(w http.ResponseWriter, authed *Authe } } +// ClearResponseIdentityHeaders emits EVERY identity header the authenticated +// SetResponseHeaders path could stamp — the canonical X-Auth-* / X-Aether-* set +// plus any resolver-declared ExtraHeaders (e.g. X-Scitrera-*) — with an EMPTY +// value on the response. +// +// This exists for the anonymous ext_authz passthrough (verify-optional with no +// credential). Envoy's ext_authz copies the authz-response identity headers +// onto the upstream request, OVERRIDING client-supplied ones, but ONLY for the +// headers auth-go actually emits. If auth-go emits none on the anonymous path, a +// client-supplied/spoofed X-Auth-Tenant-ID / X-Scitrera-User survives to the +// backend. Emitting each identity header with an empty value forces Envoy to +// overwrite the spoof with empty, so the backend sees no principal and denies. +// +// The extra-header names are sourced from the resolver via the optional +// ExtraHeaderNaming interface so the cleared set can't drift from what the +// resolver emits on the authenticated path. +func (m *AuthMiddleware) ClearResponseIdentityHeaders(w http.ResponseWriter) { + h := w.Header() + for _, name := range identityheaders.CanonicalHeaderNames() { + h.Set(name, "") + } + if namer, ok := m.identityResolver.(ExtraHeaderNaming); ok { + for _, name := range namer.ExtraHeaderNames() { + if isReservedHeader(name) { + // Reserved X-Auth-* / X-Aether-* names are already cleared above + // and can never be resolver ExtraHeaders; skip to avoid churn. + continue + } + h.Set(name, "") + } + } +} + // extractCredentials builds the credentials map expected by the // composite authenticator from the HTTP Authorization header. // It routes the token to the appropriate authenticator based on format: diff --git a/server/pkg/authproxy/server.go b/server/pkg/authproxy/server.go index e0fbb79..9430995 100644 --- a/server/pkg/authproxy/server.go +++ b/server/pkg/authproxy/server.go @@ -78,6 +78,14 @@ func NewServer(cfg *Config, middleware *AuthMiddleware) (*Server, error) { // Auth verify endpoint (available in both modes, primary in verify mode) mux.HandleFunc("/auth/verify", s.handleAuthVerify) + // Optional-auth verify endpoint. Internal-only ext_authz plane: identical + // to /auth/verify EXCEPT that a request carrying no credentials is allowed + // through as an anonymous passthrough (200, no identity headers) instead of + // being rejected 401. Presented credentials still fail closed. This route is + // served on the same internal mux as /auth/verify and must never be added to + // an external front-door route list. + mux.HandleFunc("/auth/verify-optional", s.handleAuthVerifyOptional) + if cfg.Mode == ModeProxy { // In proxy mode, all other requests go through auth + reverse proxy mux.HandleFunc("/", s.handleProxy) @@ -137,6 +145,44 @@ func (s *Server) handleHealthz(w http.ResponseWriter, r *http.Request) { // headers on success, or 401/403 on failure. This is the endpoint // consumed by nginx auth_request or Envoy ext_authz. func (s *Server) handleAuthVerify(w http.ResponseWriter, r *http.Request) { + s.verifyAuth(w, r, false) +} + +// handleAuthVerifyOptional is the optional-auth variant of handleAuthVerify. +// It behaves EXACTLY like /auth/verify when the request carries a credential +// (fail closed on bad/expired/denied creds), but treats a request with NO +// credential as an anonymous passthrough: 200 with no identity headers, so a +// downstream ext_authz consumer can forward it upstream and let the upstream +// decide based on the (absent) identity. +func (s *Server) handleAuthVerifyOptional(w http.ResponseWriter, r *http.Request) { + s.verifyAuth(w, r, true) +} + +// verifyAuth is the shared implementation behind /auth/verify and +// /auth/verify-optional. On the credential-present path both routes are +// identical: authenticate, and on success stamp the trusted identity headers +// and return 200 (Authenticate writes the 401/403 itself on failure). +// +// When optional is true and the request carries no credential the middleware +// would use, verifyAuth short-circuits to 200 with NO identity headers +// (anonymous passthrough) — instead of the 401 that /auth/verify returns. The +// credential-presence check happens BEFORE Authenticate because Authenticate +// writes a 401 to w on the no-credentials branch. +func (s *Server) verifyAuth(w http.ResponseWriter, r *http.Request, optional bool) { + if optional && !s.middleware.HasCredentials(r) { + // Anonymous passthrough: no credentials presented. Emit 200, but first + // stamp EVERY identity header the authenticated path would set to an + // EMPTY value. Envoy's ext_authz copies the authz-response identity + // headers onto the upstream request (overriding client-supplied ones), + // but ONLY the headers auth-go actually emits. Emitting nothing here + // would let a client-supplied/spoofed X-Auth-Tenant-ID / X-Scitrera-User + // survive to the backend; clearing them (empty override) ensures the + // upstream sees no principal and decides based on its own policy. + s.middleware.ClearResponseIdentityHeaders(w) + w.WriteHeader(http.StatusOK) + return + } + authed, err := s.middleware.Authenticate(w, r) if err != nil { // Authenticate already wrote the error response diff --git a/server/pkg/authproxy/verify_optional_test.go b/server/pkg/authproxy/verify_optional_test.go new file mode 100644 index 0000000..dd3c79c --- /dev/null +++ b/server/pkg/authproxy/verify_optional_test.go @@ -0,0 +1,229 @@ +package authproxy + +import ( + "net/http" + "net/http/httptest" + "testing" + + "github.com/scitrera/aether/internal/auth" +) + +// newVerifyTestServer builds a verify-mode Server whose middleware carries an +// EMPTY composite authenticator. An empty composite returns (nil, nil) for any +// credentials, which AuthMiddleware.Authenticate maps to a 401 ("no +// authenticator matched"). This lets us exercise the credential-present path +// (fail closed) without a database or real authenticator, while the no-cred +// optional path never reaches Authenticate at all. +func newVerifyTestServer(t *testing.T) *Server { + t.Helper() + cfg := &Config{Mode: ModeVerify, ListenAddr: ":0"} + m := &AuthMiddleware{ + authenticator: auth.NewCompositeAuthenticator(), + tenantID: "test-tenant", + } + srv, err := NewServer(cfg, m) + if err != nil { + t.Fatalf("NewServer: %v", err) + } + return srv +} + +// assertNoIdentityHeaders fails if any trusted identity header carries a value. +// Used on fail-closed (401) paths where Authenticate returns before any header +// is stamped or cleared, so the identity headers must be entirely absent. +func assertNoIdentityHeaders(t *testing.T, h http.Header) { + t.Helper() + for _, name := range []string{HeaderTenantID, HeaderUserID, HeaderPrincipalType, HeaderWorkspaceAccess, HeaderActorType, HeaderActorID, HeaderAuthorityMode} { + if v := h.Get(name); v != "" { + t.Errorf("expected no %s header value, got %q", name, v) + } + } +} + +// assertClearedIdentityHeaders fails if a trusted identity header is missing or +// carries a non-empty value. On the anonymous ext_authz passthrough the handler +// must EMIT every identity header with an empty value so Envoy's authz-response +// override overwrites any client-supplied/spoofed identity header with empty. +func assertClearedIdentityHeaders(t *testing.T, h http.Header) { + t.Helper() + for _, name := range []string{HeaderTenantID, HeaderUserID, HeaderPrincipalType, HeaderWorkspaceAccess, HeaderActorType, HeaderActorID, HeaderAuthorityMode} { + vals, present := h[http.CanonicalHeaderKey(name)] + if !present { + t.Errorf("expected %s to be present (empty) on anonymous passthrough, but it was absent", name) + continue + } + if len(vals) != 1 || vals[0] != "" { + t.Errorf("expected %s present with empty value on anonymous passthrough, got %v", name, vals) + } + } +} + +func TestVerifyOptional_NoCredentials_AnonymousPassthrough(t *testing.T) { + srv := newVerifyTestServer(t) + + w := httptest.NewRecorder() + r := httptest.NewRequest(http.MethodGet, "/auth/verify-optional", nil) + srv.httpServer.Handler.ServeHTTP(w, r) + + if w.Code != http.StatusOK { + t.Fatalf("expected 200 for no-cred optional verify, got %d (body=%s)", w.Code, w.Body.String()) + } + assertClearedIdentityHeaders(t, w.Header()) +} + +func TestVerifyOptional_SessionCookieConfigured_ButAbsent_AnonymousPassthrough(t *testing.T) { + srv := newVerifyTestServer(t) + srv.middleware.SetSessionCookieName("sid") + + w := httptest.NewRecorder() + r := httptest.NewRequest(http.MethodGet, "/auth/verify-optional", nil) + // A cookie with a different name must not count as a credential. + r.AddCookie(&http.Cookie{Name: "other", Value: "x"}) + srv.httpServer.Handler.ServeHTTP(w, r) + + if w.Code != http.StatusOK { + t.Fatalf("expected 200 when configured session cookie absent, got %d", w.Code) + } + assertClearedIdentityHeaders(t, w.Header()) +} + +func TestVerifyOptional_SessionCookiePresent_FailsClosed(t *testing.T) { + srv := newVerifyTestServer(t) + srv.middleware.SetSessionCookieName("sid") + + w := httptest.NewRecorder() + r := httptest.NewRequest(http.MethodGet, "/auth/verify-optional", nil) + r.AddCookie(&http.Cookie{Name: "sid", Value: "some-session-value"}) + srv.httpServer.Handler.ServeHTTP(w, r) + + // A credential is present (session cookie) but the empty authenticator + // rejects it -> fail closed with 401, exactly like /auth/verify. + if w.Code != http.StatusUnauthorized { + t.Fatalf("expected 401 for presented (but invalid) session cookie, got %d", w.Code) + } + assertNoIdentityHeaders(t, w.Header()) +} + +func TestVerifyOptional_AuthorizationHeader_FailsClosed(t *testing.T) { + srv := newVerifyTestServer(t) + + w := httptest.NewRecorder() + r := httptest.NewRequest(http.MethodGet, "/auth/verify-optional", nil) + r.Header.Set("Authorization", "Bearer opaque-token-value") + srv.httpServer.Handler.ServeHTTP(w, r) + + if w.Code != http.StatusUnauthorized { + t.Fatalf("expected 401 for presented (but invalid) Authorization credential, got %d", w.Code) + } + assertNoIdentityHeaders(t, w.Header()) +} + +// TestVerify_NoCredentials_Unchanged pins the existing /auth/verify behaviour: +// a no-credential request must still be rejected 401. This guards against the +// optional path leaking into the non-optional route. +func TestVerify_NoCredentials_Unchanged(t *testing.T) { + srv := newVerifyTestServer(t) + + w := httptest.NewRecorder() + r := httptest.NewRequest(http.MethodGet, "/auth/verify", nil) + srv.httpServer.Handler.ServeHTTP(w, r) + + if w.Code != http.StatusUnauthorized { + t.Fatalf("expected 401 for no-cred /auth/verify, got %d", w.Code) + } +} + +// extraHeaderResolver is a test IdentityResolver that also implements the +// optional ExtraHeaderNaming interface, declaring a resolver-specific extra +// header. Used to prove the anonymous passthrough clears resolver ExtraHeaders. +type extraHeaderResolver struct{ NoOpResolver } + +func (extraHeaderResolver) ExtraHeaderNames() []string { return []string{"X-Scitrera-User"} } + +// TestVerifyOptional_Anonymous_ClearsClientSuppliedSpoof pins the security +// invariant: a client that supplies its own X-Auth-* / X-Scitrera-* identity +// headers on an anonymous (no-credential) verify-optional request gets those +// headers overwritten to EMPTY on the authz response, so Envoy's override +// clears the spoof rather than forwarding it upstream. +func TestVerifyOptional_Anonymous_ClearsClientSuppliedSpoof(t *testing.T) { + cfg := &Config{Mode: ModeVerify, ListenAddr: ":0"} + m := &AuthMiddleware{ + authenticator: auth.NewCompositeAuthenticator(), + tenantID: "test-tenant", + identityResolver: extraHeaderResolver{}, + } + srv, err := NewServer(cfg, m) + if err != nil { + t.Fatalf("NewServer: %v", err) + } + + w := httptest.NewRecorder() + r := httptest.NewRequest(http.MethodGet, "/auth/verify-optional", nil) + // Client attempts to spoof identity. + r.Header.Set("X-Auth-Tenant-ID", "evil-tenant") + r.Header.Set("X-Auth-User-ID", "attacker") + r.Header.Set("X-Scitrera-User", "attacker@evil.example") + srv.httpServer.Handler.ServeHTTP(w, r) + + if w.Code != http.StatusOK { + t.Fatalf("expected 200 for spoofed no-cred optional verify, got %d", w.Code) + } + assertClearedIdentityHeaders(t, w.Header()) + + // The resolver-declared extra header must also be cleared (present, empty). + vals, present := w.Header()["X-Scitrera-User"] + if !present { + t.Errorf("expected X-Scitrera-User present (empty) to clear the spoof, but it was absent") + } else if len(vals) != 1 || vals[0] != "" { + t.Errorf("expected X-Scitrera-User present with empty value, got %v", vals) + } +} + +func TestHasCredentials(t *testing.T) { + m := &AuthMiddleware{tenantID: "test"} + + // No Authorization, no session cookie configured. + r := httptest.NewRequest(http.MethodGet, "/", nil) + if m.HasCredentials(r) { + t.Error("expected no credentials for bare request") + } + + // Authorization header present. + r = httptest.NewRequest(http.MethodGet, "/", nil) + r.Header.Set("Authorization", "Bearer abc") + if !m.HasCredentials(r) { + t.Error("expected credentials when Authorization header set") + } + + // Session cookie configured but not present. + m.SetSessionCookieName("sid") + r = httptest.NewRequest(http.MethodGet, "/", nil) + if m.HasCredentials(r) { + t.Error("expected no credentials when session cookie configured but absent") + } + + // Session cookie configured and present. + r = httptest.NewRequest(http.MethodGet, "/", nil) + r.AddCookie(&http.Cookie{Name: "sid", Value: "v"}) + if !m.HasCredentials(r) { + t.Error("expected credentials when session cookie present") + } + + // Empty session cookie value does not count. + r = httptest.NewRequest(http.MethodGet, "/", nil) + r.AddCookie(&http.Cookie{Name: "sid", Value: ""}) + if m.HasCredentials(r) { + t.Error("expected no credentials when session cookie value is empty") + } +} + +func TestSessionCookieName_Accessor(t *testing.T) { + m := &AuthMiddleware{tenantID: "test"} + if m.SessionCookieName() != "" { + t.Errorf("expected empty session cookie name by default, got %q", m.SessionCookieName()) + } + m.SetSessionCookieName("sid") + if m.SessionCookieName() != "sid" { + t.Errorf("expected sid, got %q", m.SessionCookieName()) + } +} diff --git a/server/pkg/identityheaders/identityheaders.go b/server/pkg/identityheaders/identityheaders.go index 7eadcc1..9e35842 100644 --- a/server/pkg/identityheaders/identityheaders.go +++ b/server/pkg/identityheaders/identityheaders.go @@ -54,6 +54,51 @@ const ( HeaderXAetherCallerSubject = "X-Aether-Caller-Subject" ) +// canonicalHeaderNames is the authoritative set of every X-Auth-* / X-Aether-* +// header name that mintWith can emit. It is the single source of truth used +// both to mint (implicitly, via the constants below) and — crucially — to +// CLEAR the trusted header set on an anonymous ext_authz passthrough, where the +// consumer (Envoy) copies the authz-response identity headers over the upstream +// request. Emitting each of these with an EMPTY value on the anonymous path +// forces Envoy to overwrite any client-supplied/spoofed identity header with +// empty, so a spoofed X-Auth-* / X-Aether-* can never survive to the backend. +// +// The identityheaders_test.go TestMintWith_EmitsOnlyCanonicalNames test asserts +// mintWith never emits a name outside this slice, so the two cannot drift. +var canonicalHeaderNames = []string{ + HeaderTenantID, + HeaderUserID, + HeaderPrincipalType, + HeaderWorkspaceAccess, + HeaderScopes, + HeaderAPIKeyID, + HeaderActorType, + HeaderActorID, + HeaderAuthorityMode, + HeaderGrantID, + HeaderSubjectType, + HeaderSubjectID, + HeaderRootSubjectType, + HeaderRootSubjectID, + HeaderAudienceType, + HeaderAudienceID, + HeaderMaxAccessLevel, + HeaderWorkspaceScope, + HeaderXAetherCallerTopic, + HeaderXAetherCallerSubject, +} + +// CanonicalHeaderNames returns a copy of the authoritative list of every +// X-Auth-* / X-Aether-* header name that the minting path (mintWith) can emit. +// Callers that need to CLEAR the trusted identity set (e.g. the auth-proxy +// anonymous ext_authz passthrough) iterate this so they clear exactly the same +// names the authenticated path stamps — the two cannot drift. +func CanonicalHeaderNames() []string { + out := make([]string, len(canonicalHeaderNames)) + copy(out, canonicalHeaderNames) + return out +} + // Authority mode values. const ( AuthorityModeDirect = "direct" @@ -120,59 +165,79 @@ func Mint(_ context.Context, tenantID string, ident Identity) http.Header { // MintInto writes the canonical X-Auth-* header set onto h. Existing entries // for these header names are overwritten. func MintInto(h http.Header, tenantID string, ident Identity) { - h.Set(HeaderTenantID, tenantID) - h.Set(HeaderWorkspaceAccess, strconv.Itoa(ident.WorkspaceAccess)) + mintWith(h.Set, tenantID, ident) +} + +// MintIntoMap writes the canonical X-Auth-* header set onto m, keyed by the +// literal header-name constants (e.g. "X-Auth-Tenant-ID"). Unlike http.Header, +// a bare map performs no canonicalization, so the "-ID" casing is preserved +// verbatim — required for downstreams (e.g. the in-process ProxyHttpTerminator) +// that read ProxyHttpRequest.Headers by exact key. Existing entries for these +// header names are overwritten. Shares mintWith with MintInto so the wire +// format stays identical across the http.Header and map paths. +func MintIntoMap(m map[string]string, tenantID string, ident Identity) { + mintWith(func(k, v string) { m[k] = v }, tenantID, ident) +} + +// mintWith is the single source of truth for the canonical X-Auth-* header +// set. It writes each (key, value) pair through the supplied setter so the +// same minting logic backs both http.Header (MintInto) and the proto +// map[string]string (MintIntoMap). Header-name constants are passed to set +// verbatim; the setter decides how to store them. +func mintWith(set func(key, value string), tenantID string, ident Identity) { + set(HeaderTenantID, tenantID) + set(HeaderWorkspaceAccess, strconv.Itoa(ident.WorkspaceAccess)) if ident.Scopes != "" { - h.Set(HeaderScopes, ident.Scopes) + set(HeaderScopes, ident.Scopes) } if ident.APIKeyID != "" { - h.Set(HeaderAPIKeyID, ident.APIKeyID) + set(HeaderAPIKeyID, ident.APIKeyID) } // Caller headers — stamped when the proxy sidecar or auth-proxy supplies // the originating principal topic. These are always stripped on inbound, // so only trusted minting paths can set them. if ident.CallerTopic != "" { - h.Set(HeaderXAetherCallerTopic, ident.CallerTopic) + set(HeaderXAetherCallerTopic, ident.CallerTopic) } if ident.CallerSubject != "" { - h.Set(HeaderXAetherCallerSubject, ident.CallerSubject) + set(HeaderXAetherCallerSubject, ident.CallerSubject) } if ident.Authority == nil { // Direct mode: actor == the authenticated principal. - h.Set(HeaderUserID, ident.UserID) - h.Set(HeaderPrincipalType, ident.PrincipalType) - h.Set(HeaderActorType, ident.PrincipalType) - h.Set(HeaderActorID, ident.UserID) - h.Set(HeaderAuthorityMode, AuthorityModeDirect) + set(HeaderUserID, ident.UserID) + set(HeaderPrincipalType, ident.PrincipalType) + set(HeaderActorType, ident.PrincipalType) + set(HeaderActorID, ident.UserID) + set(HeaderAuthorityMode, AuthorityModeDirect) return } // OBO mode: backward-compat headers carry the subject; actor headers // carry the authenticated service/agent identity. a := ident.Authority - h.Set(HeaderUserID, a.SubjectID) - h.Set(HeaderPrincipalType, a.SubjectType) - h.Set(HeaderActorType, a.ActorType) - h.Set(HeaderActorID, a.ActorID) - h.Set(HeaderAuthorityMode, AuthorityModeOnBehalfOf) - - h.Set(HeaderGrantID, a.GrantID) - h.Set(HeaderSubjectType, a.SubjectType) - h.Set(HeaderSubjectID, a.SubjectID) + set(HeaderUserID, a.SubjectID) + set(HeaderPrincipalType, a.SubjectType) + set(HeaderActorType, a.ActorType) + set(HeaderActorID, a.ActorID) + set(HeaderAuthorityMode, AuthorityModeOnBehalfOf) + + set(HeaderGrantID, a.GrantID) + set(HeaderSubjectType, a.SubjectType) + set(HeaderSubjectID, a.SubjectID) if a.RootSubjectType != "" { - h.Set(HeaderRootSubjectType, a.RootSubjectType) + set(HeaderRootSubjectType, a.RootSubjectType) } if a.RootSubjectID != "" { - h.Set(HeaderRootSubjectID, a.RootSubjectID) + set(HeaderRootSubjectID, a.RootSubjectID) } - h.Set(HeaderAudienceType, a.AudienceType) - h.Set(HeaderAudienceID, a.AudienceID) - h.Set(HeaderMaxAccessLevel, strconv.Itoa(a.MaxAccessLevel)) + set(HeaderAudienceType, a.AudienceType) + set(HeaderAudienceID, a.AudienceID) + set(HeaderMaxAccessLevel, strconv.Itoa(a.MaxAccessLevel)) if len(a.WorkspaceScope) > 0 { - h.Set(HeaderWorkspaceScope, strings.Join(a.WorkspaceScope, ",")) + set(HeaderWorkspaceScope, strings.Join(a.WorkspaceScope, ",")) } } @@ -274,11 +339,41 @@ func AuthorityFromResolved(resolved *acl.ResolvedAuthority) *AuthenticatedAuthor AudienceType: g.AudienceType, AudienceID: g.AudienceID, MaxAccessLevel: g.MaxAccessLevel, - WorkspaceScope: g.WorkspaceScope, + WorkspaceScope: expandSubjectInheritedScope(g.WorkspaceScope), ResourceScope: g.ResourceScope, } } +// expandSubjectInheritedScope replaces the acl.WorkspaceScopeSubjectInherited +// magic value ("_subject_workspaces") with "*" when projecting a grant's +// workspace scope into the minted X-Auth-Workspace-Scope header. Terminators +// (memorylayer, etc.) match the requested workspace against the header's literal +// values and do NOT understand the magic value, so minting it raw rejects EVERY +// workspace ("workspace '' is not in grant scope"). This mirrors the +// gateway's expandWorkspaceScopeForWire (internal/gateway/connect_authority.go) +// used on the ResolveAuthority proto path — kept local here so pkg/identityheaders +// need not import internal/gateway. (The X-Auth header minting added in +// a7b9dae missed this expansion, breaking all OBO MemoryLayer calls whose +// session grant uses _subject_workspaces.) +func expandSubjectInheritedScope(scope []string) []string { + if len(scope) == 0 { + return scope + } + out := make([]string, 0, len(scope)) + expanded := false + for _, s := range scope { + if s == acl.WorkspaceScopeSubjectInherited { + if !expanded { + out = append(out, "*") + expanded = true + } + continue + } + out = append(out, s) + } + return out +} + // StripInbound removes all X-Auth-* and X-Aether-* headers from h to prevent // clients from spoofing identity headers. X-Aether-* hint headers, when // honoured, must be read by the caller before this is invoked. @@ -290,3 +385,17 @@ func StripInbound(h http.Header) { } } } + +// StripInboundMap mirrors StripInbound for a proto map[string]string (e.g. +// ProxyHttpRequest.Headers): it removes every x-auth-* / x-aether-* entry +// (case-insensitive) so a client cannot smuggle a spoofed identity header +// through the gateway before the gateway re-mints the trusted set. Any +// x-aether-* hint headers the gateway honours must be read before this runs. +func StripInboundMap(m map[string]string) { + for key := range m { + lower := strings.ToLower(key) + if strings.HasPrefix(lower, "x-auth-") || strings.HasPrefix(lower, "x-aether-") { + delete(m, key) + } + } +} diff --git a/server/pkg/identityheaders/identityheaders_test.go b/server/pkg/identityheaders/identityheaders_test.go index 13b2411..fc2a82b 100644 --- a/server/pkg/identityheaders/identityheaders_test.go +++ b/server/pkg/identityheaders/identityheaders_test.go @@ -124,6 +124,119 @@ func TestStripInbound(t *testing.T) { } } +func TestMintIntoMap_DirectMode(t *testing.T) { + m := map[string]string{} + MintIntoMap(m, "tenant-1", Identity{ + UserID: "alice", + PrincipalType: "User", + WorkspaceAccess: 20, + Scopes: "read,write", + APIKeyID: "key-123", + CallerTopic: "uw::alice::ws1", + }) + + checks := map[string]string{ + // Literal constant keys must survive verbatim — a bare map performs no + // canonicalization, so "-ID" is NOT folded to "-Id". + HeaderTenantID: "tenant-1", + HeaderWorkspaceAccess: "20", + HeaderUserID: "alice", + HeaderPrincipalType: "User", + HeaderActorType: "User", + HeaderActorID: "alice", + HeaderAuthorityMode: AuthorityModeDirect, + HeaderScopes: "read,write", + HeaderAPIKeyID: "key-123", + HeaderXAetherCallerTopic: "uw::alice::ws1", + } + for key, want := range checks { + if got := m[key]; got != want { + t.Errorf("%s: want %q, got %q", key, want, got) + } + } + + // Guard against http.Header-style canonicalization sneaking back in. + if _, ok := m["X-Auth-Tenant-Id"]; ok { + t.Error("MintIntoMap must not canonicalize -ID to -Id (found X-Auth-Tenant-Id)") + } + + for _, key := range []string{HeaderGrantID, HeaderSubjectID, HeaderWorkspaceScope, HeaderMaxAccessLevel} { + if _, ok := m[key]; ok { + t.Errorf("%s should be absent in direct mode", key) + } + } +} + +func TestMintIntoMap_OBOMode(t *testing.T) { + m := map[string]string{} + MintIntoMap(m, "tenant-1", Identity{ + UserID: "sv::foo::bar", + PrincipalType: "Service", + WorkspaceAccess: 20, + Authority: &AuthenticatedAuthority{ + ActorType: "Service", + ActorID: "sv::foo::bar", + GrantID: "g1", + SubjectType: "User", + SubjectID: "alice", + RootSubjectType: "User", + RootSubjectID: "alice", + AudienceType: "service", + AudienceID: "sv::foo::bar", + MaxAccessLevel: 20, + WorkspaceScope: []string{"ws1", "ws2"}, + }, + }) + + checks := map[string]string{ + HeaderTenantID: "tenant-1", + HeaderWorkspaceAccess: "20", + HeaderUserID: "alice", + HeaderPrincipalType: "User", + HeaderActorType: "Service", + HeaderActorID: "sv::foo::bar", + HeaderAuthorityMode: AuthorityModeOnBehalfOf, + HeaderGrantID: "g1", + HeaderSubjectType: "User", + HeaderSubjectID: "alice", + HeaderRootSubjectType: "User", + HeaderRootSubjectID: "alice", + HeaderAudienceType: "service", + HeaderAudienceID: "sv::foo::bar", + HeaderMaxAccessLevel: "20", + HeaderWorkspaceScope: "ws1,ws2", + } + for key, want := range checks { + if got := m[key]; got != want { + t.Errorf("%s: want %q, got %q", key, want, got) + } + } +} + +func TestStripInboundMap(t *testing.T) { + m := map[string]string{ + "X-Auth-User-ID": "spoofed", + "x-auth-tenant-id": "spoofed-lower", + "X-Aether-Grant-ID": "spoofed-grant", + "Content-Type": "application/json", + } + + StripInboundMap(m) + + if _, ok := m["X-Auth-User-ID"]; ok { + t.Error("X-Auth-User-ID should have been stripped") + } + if _, ok := m["x-auth-tenant-id"]; ok { + t.Error("x-auth-tenant-id should have been stripped (case-insensitive)") + } + if _, ok := m["X-Aether-Grant-ID"]; ok { + t.Error("X-Aether-Grant-ID should have been stripped") + } + if m["Content-Type"] != "application/json" { + t.Error("Content-Type should be preserved") + } +} + // stubResolver lets us drive ResolveAndMint without standing up an ACL service. type stubResolver struct { resolved *acl.ResolvedAuthority @@ -331,3 +444,88 @@ func TestResolveAndMint_OBOMode_RejectsMissingGrantID(t *testing.T) { t.Fatal("expected error for missing grant_id") } } + +// TestExpandSubjectInheritedScope verifies the magic acl.WorkspaceScopeSubjectInherited +// value ("_subject_workspaces") is expanded to "*" for the minted +// X-Auth-Workspace-Scope header (terminators don't understand the magic value; +// minting it raw rejected every workspace). Concrete workspaces pass through. +func TestExpandSubjectInheritedScope(t *testing.T) { + cases := []struct { + name string + in []string + want []string + }{ + {"subject-inherited -> star", []string{"_subject_workspaces"}, []string{"*"}}, + {"concrete passthrough", []string{"ws1", "ws2"}, []string{"ws1", "ws2"}}, + {"mixed dedups star", []string{"ws1", "_subject_workspaces", "ws2"}, []string{"ws1", "*", "ws2"}}, + {"empty", nil, nil}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got := expandSubjectInheritedScope(tc.in) + if len(got) != len(tc.want) { + t.Fatalf("len(got)=%d want %d (got=%v)", len(got), len(tc.want), got) + } + for i := range got { + if got[i] != tc.want[i] { + t.Errorf("got[%d]=%q want %q", i, got[i], tc.want[i]) + } + } + }) + } +} + +// TestMintWith_EmitsOnlyCanonicalNames guards the drift invariant: every header +// name mintWith can emit MUST be present in canonicalHeaderNames, so the +// clear-on-anonymous path (CanonicalHeaderNames) zeroes exactly the set the +// authenticated path stamps. Both direct and OBO modes are exercised with every +// optional field populated so all conditional emissions fire. +func TestMintWith_EmitsOnlyCanonicalNames(t *testing.T) { + canonical := make(map[string]struct{}, len(canonicalHeaderNames)) + for _, n := range CanonicalHeaderNames() { + canonical[http.CanonicalHeaderKey(n)] = struct{}{} + } + + // Direct mode with every optional field set. + direct := Mint(context.Background(), "tenant-1", Identity{ + UserID: "alice", + PrincipalType: "User", + WorkspaceAccess: 20, + Scopes: "read,write", + APIKeyID: "key-123", + CallerTopic: "ag.ws.impl.spec", + CallerSubject: "usr.alice", + }) + + // OBO mode with every optional field set. + obo := Mint(context.Background(), "tenant-1", Identity{ + UserID: "svc", + PrincipalType: "Service", + WorkspaceAccess: 30, + Scopes: "read", + APIKeyID: "key-9", + CallerTopic: "ag.ws.impl.spec", + CallerSubject: "usr.bob", + Authority: &AuthenticatedAuthority{ + ActorType: "Service", + ActorID: "svc", + GrantID: "grant-1", + SubjectType: "User", + SubjectID: "bob", + RootSubjectType: "User", + RootSubjectID: "root", + AudienceType: "Agent", + AudienceID: "aud", + MaxAccessLevel: 40, + WorkspaceScope: []string{"ws1", "ws2"}, + }, + }) + + for _, h := range []http.Header{direct, obo} { + for name := range h { + if _, ok := canonical[http.CanonicalHeaderKey(name)]; !ok { + t.Errorf("mintWith emitted %q which is absent from canonicalHeaderNames — update the slice to prevent clear/mint drift", name) + } + } + } +} diff --git a/server/pkg/models/topics.go b/server/pkg/models/topics.go index 5267f74..1527b09 100644 --- a/server/pkg/models/topics.go +++ b/server/pkg/models/topics.go @@ -113,6 +113,30 @@ func UserWorkspaceTopic(userID, workspace string) (string, error) { return "uw" + IdentitySep + userID + IdentitySep + workspace, nil } +// UserBroadcastTopic returns a per-user broadcast topic used to reach every one +// of a user's open windows regardless of which workspace each window is +// currently viewing. Format: uu::{user_id}. +// +// It is the workspace-agnostic, non-progress complement to the existing +// user-directed channels: +// - us::{user}::{window} targets a single window +// - uw::{user}::{workspace} targets only windows currently on that workspace +// - pg::us::{user} reaches all windows but is progress-typed +// +// uu:: carries ordinary MessageEnvelope protos (delivered as IncomingMessage), +// so it participates fully in the message-type system, audit, and metrics. +// Because it has no workspace segment, workspaceFromTopic returns "" and the +// workspace ACL cannot gate publishes; authorization is enforced by principal +// type in enforceTopicPermissions, which restricts publishers to platform +// principals (service / workflow-engine / bridge). Users subscribe to this +// topic on connect (shared, with local fan-out to every window handler). +func UserBroadcastTopic(userID string) (string, error) { + if err := ValidateSegment("user_id", userID); err != nil { + return "", err + } + return "uu" + IdentitySep + userID, nil +} + func ProgressTopic(workspace string) (string, error) { if err := ValidateSegment("workspace", workspace); err != nil { return "", err @@ -145,6 +169,32 @@ func MustTaskEventsTopic(workspace, taskID string) string { return t } +// TaskMessageTopic returns the topic used for a task's per-task chat message +// stream. Format: tk::{workspace}::{task_id}::msg +// +// This topic carries ordinary MessageEnvelope protos (chat tokens / appended +// messages) addressed to a single task_id rather than to a specific user +// window. The gateway auto-subscribes a task's SUBJECT sessions to this topic +// for the task's lifetime (subscribe on the running notice, unsubscribe on the +// terminal notice) so chat streaming follows the task instead of being pinned +// to the window that started it. +func TaskMessageTopic(workspace, taskID string) (string, error) { + if err := validateSegments("workspace", workspace, "task_id", taskID); err != nil { + return "", err + } + return "tk" + IdentitySep + workspace + IdentitySep + taskID + IdentitySep + "msg", nil +} + +// MustTaskMessageTopic builds a task-message topic and panics on invalid input. +// Use only for trusted segments. +func MustTaskMessageTopic(workspace, taskID string) string { + t, err := TaskMessageTopic(workspace, taskID) + if err != nil { + panic(err) + } + return t +} + // UserProgressTopic returns a per-user progress stream topic used for // targeted progress delivery across all of a user's open windows. Agents // that send chat-kind progress set ProgressReport.recipient to the user's @@ -260,6 +310,16 @@ func MustUserWorkspaceTopic(userID, workspace string) string { return t } +// MustUserBroadcastTopic builds a per-user broadcast topic and panics on invalid input. +// Use only for trusted segments. +func MustUserBroadcastTopic(userID string) string { + t, err := UserBroadcastTopic(userID) + if err != nil { + panic(err) + } + return t +} + // MustProgressTopic builds a progress topic and panics on invalid input. // Use only for trusted segments. func MustProgressTopic(workspace string) string { diff --git a/server/pkg/tasks/priority_test.go b/server/pkg/tasks/priority_test.go new file mode 100644 index 0000000..51edbe4 --- /dev/null +++ b/server/pkg/tasks/priority_test.go @@ -0,0 +1,104 @@ +package tasks + +import ( + "strings" + "testing" +) + +func TestTaskPriority_Normalize(t *testing.T) { + cases := []struct { + name string + in TaskPriority + want TaskPriority + }{ + {"unspecified maps to normal", PriorityUnspecified, PriorityNormal}, + {"xlow passes through", PriorityXLow, PriorityXLow}, + {"low passes through", PriorityLow, PriorityLow}, + {"normal passes through", PriorityNormal, PriorityNormal}, + {"high passes through", PriorityHigh, PriorityHigh}, + {"preempt passes through", PriorityPreempt, PriorityPreempt}, + {"unknown spaced value passes through", TaskPriority(35), TaskPriority(35)}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if got := tc.in.Normalize(); got != tc.want { + t.Errorf("Normalize(%d) = %d, want %d", tc.in, got, tc.want) + } + }) + } +} + +func TestTaskPriority_Ordering(t *testing.T) { + // The numeric weights must be strictly increasing so they can be used + // directly as a descending dispatch sort key. + ordered := []TaskPriority{PriorityXLow, PriorityLow, PriorityNormal, PriorityHigh, PriorityPreempt} + for i := 1; i < len(ordered); i++ { + if ordered[i-1] >= ordered[i] { + t.Errorf("priority weights not strictly increasing: %d (%s) !< %d (%s)", + ordered[i-1], ordered[i-1], ordered[i], ordered[i]) + } + } +} + +func TestTaskPriority_String(t *testing.T) { + cases := map[TaskPriority]string{ + PriorityUnspecified: "unspecified", + PriorityXLow: "xlow", + PriorityLow: "low", + PriorityNormal: "normal", + PriorityHigh: "high", + PriorityPreempt: "preempt", + TaskPriority(35): "priority(35)", + } + for in, want := range cases { + if got := in.String(); got != want { + t.Errorf("TaskPriority(%d).String() = %q, want %q", int(in), got, want) + } + } +} + +func TestBuildTaskFilterClauses_Priority(t *testing.T) { + t.Run("exact priority", func(t *testing.T) { + f := &TaskFilter{Priority: int32(PriorityHigh)} + clause, _, args := buildTaskFilterClauses(f, 1, nil, false) + if !strings.Contains(clause, "priority = $1") { + t.Errorf("expected exact priority clause, got %q", clause) + } + if len(args) != 1 || args[0] != int32(PriorityHigh) { + t.Errorf("expected args [40], got %v", args) + } + }) + + t.Run("min priority threshold", func(t *testing.T) { + f := &TaskFilter{MinPriority: int32(PriorityHigh)} + clause, _, args := buildTaskFilterClauses(f, 1, nil, false) + if !strings.Contains(clause, "priority >= $1") { + t.Errorf("expected min priority clause, got %q", clause) + } + if len(args) != 1 || args[0] != int32(PriorityHigh) { + t.Errorf("expected args [40], got %v", args) + } + }) + + t.Run("both exact and min", func(t *testing.T) { + f := &TaskFilter{Priority: int32(PriorityNormal), MinPriority: int32(PriorityLow)} + clause, _, args := buildTaskFilterClauses(f, 1, nil, false) + if !strings.Contains(clause, "priority = $1") || !strings.Contains(clause, "priority >= $2") { + t.Errorf("expected both clauses, got %q", clause) + } + if len(args) != 2 { + t.Errorf("expected 2 args, got %v", args) + } + }) + + t.Run("zero priority means no filter", func(t *testing.T) { + f := &TaskFilter{} + clause, _, args := buildTaskFilterClauses(f, 1, nil, false) + if strings.Contains(clause, "priority") { + t.Errorf("expected no priority clause for zero filter, got %q", clause) + } + if len(args) != 0 { + t.Errorf("expected no args, got %v", args) + } + }) +} diff --git a/server/pkg/tasks/retry_policy.go b/server/pkg/tasks/retry_policy.go new file mode 100644 index 0000000..87f2008 --- /dev/null +++ b/server/pkg/tasks/retry_policy.go @@ -0,0 +1,162 @@ +package tasks + +import ( + "math" + "math/rand/v2" + "time" +) + +// BackoffStrategy mirrors the proto pb.BackoffStrategy enum. Stored as an +// integer code so JSON round-trips cleanly without proto coupling in +// non-server packages. +type BackoffStrategy int32 + +const ( + BackoffStrategyUnspecified BackoffStrategy = 0 + BackoffStrategyFixed BackoffStrategy = 1 + BackoffStrategyExponential BackoffStrategy = 2 + BackoffStrategyExplicitSchedule BackoffStrategy = 3 +) + +// RetryPolicy mirrors the proto pb.RetryPolicy message. Persisted as JSON +// on the task row in retry_policy_json. Kept independent of pb so that +// non-server packages (e.g. webhookservice) can construct policies without +// depending on the proto package. +// +// Workers that need different semantics — e.g., honoring an HTTP +// Retry-After response header — call RescheduleTaskAt explicitly to +// override the policy-computed next_retry_at. +type RetryPolicy struct { + // MaxAttempts is the total attempts allowed (1 = no retries). + // 0 means use the server default (3). + MaxAttempts int32 `json:"max_attempts,omitempty"` + + // Backoff is the backoff strategy. Defaults to EXPONENTIAL when + // unspecified to preserve the legacy gateway behavior. + Backoff BackoffStrategy `json:"backoff,omitempty"` + + // InitialDelayMs is the base delay for EXPONENTIAL and the constant + // for FIXED, in milliseconds. + InitialDelayMs int64 `json:"initial_delay_ms,omitempty"` + + // MaxDelayMs caps the computed delay for EXPONENTIAL. 0 means no cap. + MaxDelayMs int64 `json:"max_delay_ms,omitempty"` + + // JitterFactor scales a uniform random multiplier applied to the + // computed delay: final = computed * (1 + uniform(-jitter, +jitter)). + // Range [0,1]; values outside that range are clamped. + JitterFactor float64 `json:"jitter_factor,omitempty"` + + // ScheduleMs is the explicit retry-delay schedule (milliseconds) used + // when Backoff == EXPLICIT_SCHEDULE. Indexed by 0-based attempt + // number; the final entry is used for any subsequent attempt. + ScheduleMs []int64 `json:"schedule_ms,omitempty"` + + // RetryableStatusCodes is a worker convention: HTTP-style status + // codes that should trigger a retry. The task store does not inspect + // failure context; workers use this to decide whether to FailTask or + // CompleteTask with permanent-failure semantics. + RetryableStatusCodes []int32 `json:"retryable_status_codes,omitempty"` + + // HonorRetryAfter is a worker hint: prefer Retry-After (or analogous) + // over computed delay when set on the failure response. + HonorRetryAfter bool `json:"honor_retry_after,omitempty"` +} + +// EffectiveMaxAttempts returns the policy's MaxAttempts, substituting the +// server default (3) when the caller left it zero. +func (p *RetryPolicy) EffectiveMaxAttempts() int32 { + if p == nil || p.MaxAttempts <= 0 { + return 3 + } + return p.MaxAttempts +} + +// ComputeNextRetryAt returns the wall-clock time at which the next retry +// attempt should fire. attempt is 1-based (1 = first retry, after the +// initial attempt failed). +// +// When retryAfter is non-nil AND policy.HonorRetryAfter is true, the +// returned time is exactly retryAfter — the caller has authoritative +// guidance from the failed receiver and the policy yields to it. +// Otherwise the delay is computed from the policy's backoff strategy +// (with optional jitter) and added to time.Now(). +// +// Safe to call with a nil receiver: returns time.Now() (the same as +// today's "immediate re-pend" behavior). +func ComputeNextRetryAt(policy *RetryPolicy, attempt int, retryAfter *time.Time) time.Time { + now := time.Now() + if policy == nil { + return now + } + if policy.HonorRetryAfter && retryAfter != nil && !retryAfter.IsZero() { + return *retryAfter + } + + var delayMs int64 + switch policy.Backoff { + case BackoffStrategyFixed: + delayMs = policy.InitialDelayMs + case BackoffStrategyExplicitSchedule: + if len(policy.ScheduleMs) == 0 { + delayMs = policy.InitialDelayMs + } else { + idx := attempt - 1 + if idx < 0 { + idx = 0 + } + if idx >= len(policy.ScheduleMs) { + idx = len(policy.ScheduleMs) - 1 + } + delayMs = policy.ScheduleMs[idx] + } + default: + // Unspecified and EXPONENTIAL both fall through to exponential. + base := policy.InitialDelayMs + if base <= 0 { + base = 1000 // 1s fallback; preserves prior gateway timer behavior. + } + // delay = base * 2^(attempt-1). + exp := attempt - 1 + if exp < 0 { + exp = 0 + } + if exp > 62 { + // Prevent int64 overflow on absurd attempt counts. + exp = 62 + } + scaled := float64(base) * math.Pow(2, float64(exp)) + if scaled > float64(math.MaxInt64) { + delayMs = math.MaxInt64 + } else { + delayMs = int64(scaled) + } + if policy.MaxDelayMs > 0 && delayMs > policy.MaxDelayMs { + delayMs = policy.MaxDelayMs + } + } + + if delayMs < 0 { + delayMs = 0 + } + + // Apply multiplicative jitter, clamped to [0, 1]. + jitter := policy.JitterFactor + if jitter < 0 { + jitter = 0 + } + if jitter > 1 { + jitter = 1 + } + if jitter > 0 && delayMs > 0 { + // uniform in [-jitter, +jitter] + offset := (rand.Float64()*2 - 1) * jitter + scaled := float64(delayMs) * (1 + offset) + if scaled < 0 { + scaled = 0 + } + delayMs = int64(scaled) + } + + return now.Add(time.Duration(delayMs) * time.Millisecond) +} diff --git a/server/pkg/tasks/retry_policy_test.go b/server/pkg/tasks/retry_policy_test.go new file mode 100644 index 0000000..c2350a2 --- /dev/null +++ b/server/pkg/tasks/retry_policy_test.go @@ -0,0 +1,188 @@ +package tasks + +import ( + "math" + "testing" + "time" +) + +func TestComputeNextRetryAt_NilPolicy_ReturnsNow(t *testing.T) { + before := time.Now() + got := ComputeNextRetryAt(nil, 1, nil) + if got.Before(before) || got.After(time.Now().Add(50*time.Millisecond)) { + t.Errorf("nil policy should return ~now; got %v (before=%v)", got, before) + } +} + +func TestComputeNextRetryAt_HonorsRetryAfter(t *testing.T) { + target := time.Now().Add(7 * time.Minute) + p := &RetryPolicy{ + Backoff: BackoffStrategyExponential, + InitialDelayMs: 100, + HonorRetryAfter: true, + } + got := ComputeNextRetryAt(p, 1, &target) + if !got.Equal(target) { + t.Errorf("HonorRetryAfter=true should pass through; got %v, want %v", got, target) + } +} + +func TestComputeNextRetryAt_IgnoresRetryAfterWhenDisabled(t *testing.T) { + target := time.Now().Add(1 * time.Hour) + p := &RetryPolicy{ + Backoff: BackoffStrategyFixed, + InitialDelayMs: 1000, + } + got := ComputeNextRetryAt(p, 1, &target) + if got.After(time.Now().Add(2 * time.Second)) { + t.Errorf("HonorRetryAfter unset should ignore retryAfter; got %v", got) + } +} + +func TestComputeNextRetryAt_Fixed(t *testing.T) { + p := &RetryPolicy{ + Backoff: BackoffStrategyFixed, + InitialDelayMs: 5000, + } + // Same delay regardless of attempt. + for _, attempt := range []int{1, 2, 5, 10} { + before := time.Now() + got := ComputeNextRetryAt(p, attempt, nil) + delta := got.Sub(before) + if delta < 4900*time.Millisecond || delta > 5200*time.Millisecond { + t.Errorf("attempt=%d: fixed 5s, got delta=%v", attempt, delta) + } + } +} + +func TestComputeNextRetryAt_Exponential(t *testing.T) { + p := &RetryPolicy{ + Backoff: BackoffStrategyExponential, + InitialDelayMs: 100, + MaxDelayMs: 0, // no cap + } + cases := map[int]int64{ + 1: 100, + 2: 200, + 3: 400, + 4: 800, + 5: 1600, + } + for attempt, wantMs := range cases { + before := time.Now() + got := ComputeNextRetryAt(p, attempt, nil) + delta := got.Sub(before).Milliseconds() + // allow 50ms slack for clock skew + jitter-free computation. + if delta < wantMs-50 || delta > wantMs+50 { + t.Errorf("attempt=%d: want ~%dms, got %dms", attempt, wantMs, delta) + } + } +} + +func TestComputeNextRetryAt_ExponentialCappedAtMax(t *testing.T) { + p := &RetryPolicy{ + Backoff: BackoffStrategyExponential, + InitialDelayMs: 100, + MaxDelayMs: 500, + } + // At attempt 5, raw would be 100 * 2^4 = 1600; should clamp to 500. + before := time.Now() + got := ComputeNextRetryAt(p, 5, nil) + delta := got.Sub(before).Milliseconds() + if delta > 600 { + t.Errorf("attempt=5 with max=500: got %dms; should be capped", delta) + } +} + +func TestComputeNextRetryAt_ExponentialDefaultsBase(t *testing.T) { + // When InitialDelayMs is zero, base fallback (1s) kicks in. + p := &RetryPolicy{ + Backoff: BackoffStrategyExponential, + } + before := time.Now() + got := ComputeNextRetryAt(p, 1, nil) + delta := got.Sub(before).Milliseconds() + if delta < 950 || delta > 1100 { + t.Errorf("exponential fallback base 1s on attempt=1 expected ~1000ms, got %dms", delta) + } +} + +func TestComputeNextRetryAt_ExplicitSchedule(t *testing.T) { + p := &RetryPolicy{ + Backoff: BackoffStrategyExplicitSchedule, + ScheduleMs: []int64{5000, 300000, 1800000}, + } + cases := map[int]int64{ + 1: 5000, + 2: 300000, + 3: 1800000, + 4: 1800000, // past-end clamps to last entry + 5: 1800000, + } + for attempt, wantMs := range cases { + before := time.Now() + got := ComputeNextRetryAt(p, attempt, nil) + delta := got.Sub(before).Milliseconds() + if delta < wantMs-50 || delta > wantMs+50 { + t.Errorf("attempt=%d: want ~%dms, got %dms", attempt, wantMs, delta) + } + } +} + +func TestComputeNextRetryAt_ExplicitScheduleEmptyFallsBackToInitial(t *testing.T) { + p := &RetryPolicy{ + Backoff: BackoffStrategyExplicitSchedule, + InitialDelayMs: 250, + } + before := time.Now() + got := ComputeNextRetryAt(p, 1, nil) + delta := got.Sub(before).Milliseconds() + if delta < 200 || delta > 350 { + t.Errorf("empty schedule should fall back to InitialDelayMs; got %dms", delta) + } +} + +func TestComputeNextRetryAt_JitterStaysInRange(t *testing.T) { + p := &RetryPolicy{ + Backoff: BackoffStrategyFixed, + InitialDelayMs: 10000, + JitterFactor: 0.5, // ±50% + } + // Sample several times; every sample must fall within [5000ms, 15000ms]. + for i := 0; i < 20; i++ { + before := time.Now() + got := ComputeNextRetryAt(p, 1, nil) + delta := got.Sub(before).Milliseconds() + if delta < 4900 || delta > 15100 { + t.Errorf("sample %d: jittered fixed 10s with +-50%% out of bounds: %dms", i, delta) + } + } +} + +func TestEffectiveMaxAttempts(t *testing.T) { + cases := []struct { + in *RetryPolicy + want int32 + }{ + {nil, 3}, + {&RetryPolicy{}, 3}, + {&RetryPolicy{MaxAttempts: 0}, 3}, + {&RetryPolicy{MaxAttempts: 1}, 1}, + {&RetryPolicy{MaxAttempts: 7}, 7}, + {&RetryPolicy{MaxAttempts: -5}, 3}, + } + for _, c := range cases { + if got := c.in.EffectiveMaxAttempts(); got != c.want { + t.Errorf("%v: EffectiveMaxAttempts() = %d, want %d", c.in, got, c.want) + } + } +} + +func TestComputeNextRetryAt_ExponentialOverflowSafe(t *testing.T) { + p := &RetryPolicy{ + Backoff: BackoffStrategyExponential, + InitialDelayMs: math.MaxInt32, + } + // Don't crash on absurd attempt counts. + _ = ComputeNextRetryAt(p, 100, nil) +} diff --git a/server/pkg/tasks/store.go b/server/pkg/tasks/store.go index cf803c0..34bc097 100644 --- a/server/pkg/tasks/store.go +++ b/server/pkg/tasks/store.go @@ -40,7 +40,9 @@ const taskSelectColumns = ` authority_audience_type, authority_audience_id, authority_delegate_type, authority_delegate_id, task_class, disconnected_at, grace_window_ms, - wait_spec, depends_on, context_id, paused_at + wait_spec, depends_on, context_id, paused_at, + retry_policy_json, + correlation_id, root_task_id, completion_event ` // ============================================================================= @@ -65,6 +67,11 @@ func (s *TaskStore) CreateTask(ctx context.Context, task *Task) error { if task.MaxRetries == 0 { task.MaxRetries = 3 } + // Normalize unspecified priority to NORMAL so the stored weight is always + // a concrete dispatch level and ORDER BY priority needs no COALESCE. + if task.Priority == 0 { + task.Priority = int(PriorityNormal) + } // Marshal JSONB fields launchParamsJSON, err := json.Marshal(task.LaunchParams) @@ -97,6 +104,20 @@ func (s *TaskStore) CreateTask(ctx context.Context, task *Task) error { return fmt.Errorf("failed to marshal depends_on: %w", err) } } + var retryPolicyJSON []byte + if task.RetryPolicy != nil { + retryPolicyJSON, err = json.Marshal(task.RetryPolicy) + if err != nil { + return fmt.Errorf("failed to marshal retry_policy: %w", err) + } + } + var completionEventJSON []byte + if task.CompletionEvent != nil { + completionEventJSON, err = json.Marshal(task.CompletionEvent) + if err != nil { + return fmt.Errorf("failed to marshal completion_event: %w", err) + } + } query := ` INSERT INTO tasks ( @@ -111,13 +132,17 @@ func (s *TaskStore) CreateTask(ctx context.Context, task *Task) error { authority_grant_id, root_authority_grant_id, parent_authority_grant_id, authority_audience_type, authority_audience_id, authority_delegate_type, authority_delegate_id, task_class, grace_window_ms, - wait_spec, depends_on, context_id, paused_at + wait_spec, depends_on, context_id, paused_at, + retry_policy_json, + correlation_id, root_task_id, completion_event ) VALUES ( $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26, $27, $28, $29, $30, $31, $32, $33, $34, $35, $36, $37, $38, $39, $40, $41, $42, $43, - $44, $45, $46, $47 + $44, $45, $46, $47, + $48, + $49, $50, $51 ) ` @@ -169,6 +194,10 @@ func (s *TaskStore) CreateTask(ctx context.Context, task *Task) error { dependsOnJSON, nullString(task.ContextID), nullTime(task.PausedAt), + retryPolicyJSON, + nullString(task.CorrelationID), + nullString(task.RootTaskID), + completionEventJSON, ) return err @@ -375,6 +404,29 @@ func (s *TaskStore) StartTaskWithAgent(ctx context.Context, taskID, agentIdentit return nil } +// ClaimTask transitions a task into running when claimed by its assignee. +// Unlike StartTask it also accepts a pending source state so a per-turn task +// can be claimed straight out of the queue. Only stamps started_at on the +// first transition; running -> running is a no-op for that timestamp +// (idempotent for re-claim / multi-tab scenarios). +func (s *TaskStore) ClaimTask(ctx context.Context, taskID string) error { + now := time.Now() + query := ` + UPDATE tasks + SET status = 'running', started_at = COALESCE(started_at, $1) + WHERE task_id = $2 AND status IN ('pending', 'assigned', 'starting', 'running') + ` + result, err := s.db.ExecContext(ctx, query, now, taskID) + if err != nil { + return err + } + rows, _ := result.RowsAffected() + if rows == 0 { + return fmt.Errorf("task %s not in a claimable state (pending, assigned, starting, or running)", taskID) + } + return nil +} + // CompleteTask marks a task as completed func (s *TaskStore) CompleteTask(ctx context.Context, taskID string) error { now := time.Now() @@ -397,9 +449,53 @@ func (s *TaskStore) CompleteTask(ctx context.Context, taskID string) error { return nil } -// FailTask marks a task as failed +// FailTask marks a task as failed and increments retry_count. When the task +// carries a RetryPolicy AND the post-increment retry_count is still below +// the policy's effective max attempts, FailTask also stamps next_retry_at +// from ComputeNextRetryAt — the task waker will then re-pend the row at +// that time. Tasks without a policy keep legacy behavior (no next_retry_at; +// the existing gateway retry sweeper handles backoff separately). func (s *TaskStore) FailTask(ctx context.Context, taskID, errorMsg string) error { now := time.Now() + + // Best-effort lookup of an attached RetryPolicy. If GetTask fails, fall + // back to the legacy update — losing the policy-driven reschedule is + // preferable to losing the FailTask transition itself. + var nextRetryAt *time.Time + if task, getErr := s.GetTask(ctx, taskID); getErr == nil && task != nil && task.RetryPolicy != nil { + // task.RetryCount is the count BEFORE this failure. The new count + // after this update is RetryCount + 1. + newCount := int32(task.RetryCount + 1) + if newCount < task.RetryPolicy.EffectiveMaxAttempts() { + t := ComputeNextRetryAt(task.RetryPolicy, int(newCount), nil) + nextRetryAt = &t + } + } + + if nextRetryAt != nil { + query := ` + UPDATE tasks + SET status = 'failed', + failed_at = $1, + error_message = $2, + next_retry_at = $3, + retry_count = retry_count + 1 + WHERE task_id = $4 + ` + result, err := s.db.ExecContext(ctx, query, now, errorMsg, nullTime(nextRetryAt), taskID) + if err != nil { + return err + } + rows, err := result.RowsAffected() + if err != nil { + return err + } + if rows == 0 { + return fmt.Errorf("task %s not found or not in a failable state", taskID) + } + return nil + } + query := ` UPDATE tasks SET status = 'failed', failed_at = $1, error_message = $2, retry_count = retry_count + 1 @@ -829,6 +925,16 @@ func buildTaskFilterClauses(filter *TaskFilter, argNum int, args []interface{}, args = append(args, filter.ContextID) argNum++ } + if filter.CorrelationID != "" { + query += fmt.Sprintf(" AND correlation_id = $%d", argNum) + args = append(args, filter.CorrelationID) + argNum++ + } + if filter.RootTaskID != "" { + query += fmt.Sprintf(" AND root_task_id = $%d", argNum) + args = append(args, filter.RootTaskID) + argNum++ + } if len(filter.ExcludeStatuses) > 0 { placeholders := make([]string, len(filter.ExcludeStatuses)) for i, s := range filter.ExcludeStatuses { @@ -852,6 +958,16 @@ func buildTaskFilterClauses(filter *TaskFilter, argNum int, args []interface{}, args = append(args, filter.StatusTimestampAfterUnixMs) argNum++ } + if filter.Priority != 0 { + query += fmt.Sprintf(" AND priority = $%d", argNum) + args = append(args, filter.Priority) + argNum++ + } + if filter.MinPriority != 0 { + query += fmt.Sprintf(" AND priority >= $%d", argNum) + args = append(args, filter.MinPriority) + argNum++ + } return query, argNum, args } @@ -876,7 +992,12 @@ func (s *TaskStore) ListTasks(ctx context.Context, filter *TaskFilter) ([]*Task, clauses, argNum, args := buildTaskFilterClauses(filter, argNum, args, false) query += clauses - query += " ORDER BY created_at DESC" + if filter.OrderByPriority { + // Dispatch-selection order: highest priority first, FIFO within a level. + query += " ORDER BY priority DESC, created_at ASC" + } else { + query += " ORDER BY created_at DESC" + } query += fmt.Sprintf(" LIMIT $%d OFFSET $%d", argNum, argNum+1) args = append(args, filter.Limit, filter.Offset) @@ -1105,6 +1226,7 @@ func (s *TaskStore) GetPendingPoolTasks(ctx context.Context, implementation, wor TargetImplementation: implementation, Workspace: workspace, QueuedForStartup: &queuedTrue, + OrderByPriority: true, // highest-priority pending task first Limit: 100, }) } @@ -1597,10 +1719,10 @@ func scanTaskInto(s taskScanner) (*Task, error) { var authorityMode, subjectType, subjectID, rootSubjectType, rootSubjectID sql.NullString var authorityGrantID, rootAuthorityGrantID, parentAuthorityGrantID sql.NullString var authorityAudienceType, authorityAudienceID, authorityDelegateType, authorityDelegateID sql.NullString - var contextID sql.NullString + var contextID, correlationID, rootTaskID sql.NullString var scheduledFor, startedAt, completedAt, failedAt, assignedAt, nextRetryAt, lastHeartbeat, disconnectedAt, pausedAt sql.NullTime var scheduleToStart, startToClose, heartbeatTimeout, scheduleToClose sql.NullInt64 - var launchParamsJSON, metadataJSON, checkpointJSON, heartbeatJSON, waitSpecJSON, dependsOnJSON []byte + var launchParamsJSON, metadataJSON, checkpointJSON, heartbeatJSON, waitSpecJSON, dependsOnJSON, retryPolicyJSON, completionEventJSON []byte err := s.Scan( &task.TaskID, &task.TaskType, &task.Workspace, &implementation, &specifier, @@ -1621,6 +1743,8 @@ func scanTaskInto(s taskScanner) (*Task, error) { &task.TaskClass, &disconnectedAt, &task.GraceWindowMs, &waitSpecJSON, &dependsOnJSON, &contextID, &pausedAt, + &retryPolicyJSON, + &correlationID, &rootTaskID, &completionEventJSON, ) if err != nil { return nil, err @@ -1776,12 +1900,32 @@ func scanTaskInto(s taskScanner) (*Task, error) { return nil, fmt.Errorf("failed to unmarshal depends_on: %w", err) } } + if len(retryPolicyJSON) > 0 { + var rp RetryPolicy + if err := json.Unmarshal(retryPolicyJSON, &rp); err != nil { + return nil, fmt.Errorf("failed to unmarshal retry_policy: %w", err) + } + task.RetryPolicy = &rp + } if contextID.Valid { task.ContextID = contextID.String } if pausedAt.Valid { task.PausedAt = &pausedAt.Time } + if correlationID.Valid { + task.CorrelationID = correlationID.String + } + if rootTaskID.Valid { + task.RootTaskID = rootTaskID.String + } + if len(completionEventJSON) > 0 { + var ce TaskCompletionConfig + if err := json.Unmarshal(completionEventJSON, &ce); err != nil { + return nil, fmt.Errorf("failed to unmarshal completion_event: %w", err) + } + task.CompletionEvent = &ce + } return &task, nil } diff --git a/server/pkg/tasks/store_test.go b/server/pkg/tasks/store_test.go index fdde6a5..31482fd 100644 --- a/server/pkg/tasks/store_test.go +++ b/server/pkg/tasks/store_test.go @@ -855,6 +855,89 @@ func TestTaskStoreIntegration(t *testing.T) { } }) + t.Run("CorrelationAndCompletionEventRoundTrip", func(t *testing.T) { + correlationID := uuid.New().String() + rootID := uuid.New().String() + task := &Task{ + TaskType: "join-child", + Workspace: "test-workspace", + CorrelationID: correlationID, + RootTaskID: rootID, + CompletionEvent: &TaskCompletionConfig{ + Enabled: true, + EventName: "task.join.done", + OnStatuses: []TaskStatus{TaskStatusCompleted, TaskStatusFailed}, + }, + } + if err := store.CreateTask(ctx, task); err != nil { + t.Fatalf("CreateTask() error = %v", err) + } + retrieved, err := store.GetTask(ctx, task.TaskID) + if err != nil { + t.Fatalf("GetTask() error = %v", err) + } + if retrieved.CorrelationID != correlationID { + t.Errorf("GetTask() CorrelationID = %q, want %q", retrieved.CorrelationID, correlationID) + } + if retrieved.RootTaskID != rootID { + t.Errorf("GetTask() RootTaskID = %q, want %q", retrieved.RootTaskID, rootID) + } + if retrieved.CompletionEvent == nil { + t.Fatal("GetTask() CompletionEvent = nil, want non-nil") + } + if !retrieved.CompletionEvent.Enabled || retrieved.CompletionEvent.EventName != "task.join.done" { + t.Errorf("GetTask() CompletionEvent = %+v, want enabled with event_name task.join.done", retrieved.CompletionEvent) + } + if len(retrieved.CompletionEvent.OnStatuses) != 2 || + retrieved.CompletionEvent.OnStatuses[0] != TaskStatusCompleted || + retrieved.CompletionEvent.OnStatuses[1] != TaskStatusFailed { + t.Errorf("GetTask() CompletionEvent.OnStatuses = %v, want [completed failed]", retrieved.CompletionEvent.OnStatuses) + } + + // A task created without a completion event reads back nil (SQL NULL). + plain := &Task{TaskType: "plain", Workspace: "test-workspace"} + if err := store.CreateTask(ctx, plain); err != nil { + t.Fatalf("CreateTask(plain) error = %v", err) + } + retrievedPlain, err := store.GetTask(ctx, plain.TaskID) + if err != nil { + t.Fatalf("GetTask(plain) error = %v", err) + } + if retrievedPlain.CompletionEvent != nil { + t.Errorf("GetTask(plain) CompletionEvent = %+v, want nil", retrievedPlain.CompletionEvent) + } + }) + + t.Run("FilterByCorrelationID", func(t *testing.T) { + correlationID := uuid.New().String() + match := &Task{ + TaskType: "join-member", + Workspace: "corr-ws", + CorrelationID: correlationID, + } + other := &Task{ + TaskType: "join-member", + Workspace: "corr-ws", + CorrelationID: uuid.New().String(), + } + if err := store.CreateTask(ctx, match); err != nil { + t.Fatalf("CreateTask(match) error = %v", err) + } + if err := store.CreateTask(ctx, other); err != nil { + t.Fatalf("CreateTask(other) error = %v", err) + } + results, err := store.ListTasks(ctx, &TaskFilter{ + Workspace: "corr-ws", + CorrelationID: correlationID, + }) + if err != nil { + t.Fatalf("ListTasks(correlation_id) error = %v", err) + } + if len(results) != 1 || results[0].TaskID != match.TaskID { + t.Fatalf("filter by correlation_id returned wrong tasks: got %+v", results) + } + }) + // Track B: the new authority/lineage filters on TaskFilter actually filter in SQL. t.Run("FilterBySubjectAndAuthority", func(t *testing.T) { parentID := uuid.New().String() diff --git a/server/pkg/tasks/types.go b/server/pkg/tasks/types.go index ca82ae2..a11f0d0 100644 --- a/server/pkg/tasks/types.go +++ b/server/pkg/tasks/types.go @@ -100,6 +100,52 @@ const ( AssignmentModeBroadcast AssignmentMode = "broadcast" // Sent to all matching agents ) +// TaskPriority is the dispatch-priority weight stored in the tasks.priority +// column. It mirrors the proto TaskPriority enum: the numeric value doubles as +// the descending sort key, so higher = dispatched first. Values are spaced so +// new levels can be inserted later without a data migration. +type TaskPriority int + +const ( + PriorityUnspecified TaskPriority = 0 // Normalized to PriorityNormal on write. + PriorityXLow TaskPriority = 10 // Lowest; best-effort. + PriorityLow TaskPriority = 20 // Below normal. + PriorityNormal TaskPriority = 30 // Default. + PriorityHigh TaskPriority = 40 // Above normal. + PriorityPreempt TaskPriority = 50 // Highest; reserved for future preemption. +) + +// Normalize maps the zero/UNSPECIFIED value to PriorityNormal so persisted and +// compared priorities are always concrete. Other values pass through unchanged +// (including future unspaced values), preserving forward compatibility. +func (p TaskPriority) Normalize() TaskPriority { + if p == PriorityUnspecified { + return PriorityNormal + } + return p +} + +// String renders the canonical level name, falling back to the numeric weight +// for any value that is not one of the named levels. +func (p TaskPriority) String() string { + switch p { + case PriorityUnspecified: + return "unspecified" + case PriorityXLow: + return "xlow" + case PriorityLow: + return "low" + case PriorityNormal: + return "normal" + case PriorityHigh: + return "high" + case PriorityPreempt: + return "preempt" + default: + return fmt.Sprintf("priority(%d)", int(p)) + } +} + // TaskAuthorityInfo captures the on-behalf-of authority lineage currently bound // to a task. These fields mirror the persisted grant lineage needed for // renewal, reassignment, audit correlation, and lifecycle cleanup. @@ -129,6 +175,14 @@ const ( TimerTypeRetry TimerType = "retry" ) +// TaskCompletionConfig is the persisted "feed B" config: emit a domain event +// onto event::* when the task reaches a (selected) terminal status. +type TaskCompletionConfig struct { + Enabled bool `json:"enabled"` + EventName string `json:"event_name,omitempty"` + OnStatuses []TaskStatus `json:"on_statuses,omitempty"` +} + // Task represents a unified task record in the database // This type supports both messaging delivery and orchestration patterns type Task struct { @@ -173,6 +227,12 @@ type Task struct { MaxRetries int `json:"max_retries"` NextRetryAt *time.Time `json:"next_retry_at,omitempty"` + // RetryPolicy, when non-nil, is honored by FailTask: the store computes + // next_retry_at from the policy and re-pends the task (up to + // MaxAttempts). Persisted as JSON in retry_policy_json. Workers that + // need different semantics call RescheduleTaskAt explicitly. + RetryPolicy *RetryPolicy `json:"retry_policy,omitempty"` + // Error tracking ErrorMessage string `json:"error_message,omitempty"` ErrorType string `json:"error_type,omitempty"` @@ -212,6 +272,17 @@ type Task struct { // PausedAt records when the task entered a waiting/hibernated state. PausedAt *time.Time `json:"paused_at,omitempty"` + // CorrelationID is the fan-out/fan-in correlation identity (distinct from + // TaskID): the barrier/group id a workflow join matches against. + CorrelationID string `json:"correlation_id,omitempty"` + // RootTaskID is the flow-root task id propagated from a task's spawner. A + // task created without a provided root is its own flow root. + RootTaskID string `json:"root_task_id,omitempty"` + // CompletionEvent, when non-nil, opts the task into "feed B": the server + // emits a domain event onto event::* when the task reaches a (selected) + // terminal status. + CompletionEvent *TaskCompletionConfig `json:"completion_event,omitempty"` + // Messaging support (for delivery tasks) TargetTopic string `json:"target_topic,omitempty"` SourceTopic string `json:"source_topic,omitempty"` @@ -332,6 +403,8 @@ type TaskFilter struct { ExcludeTaskClasses []int32 // any task whose TaskClass is in this list is omitted // Phase 1: A2A-aligned filter fields. ContextID string // Filter by client-minted session identifier; empty = no filter + CorrelationID string // Filter by fan-out/fan-in correlation identity; empty = no filter + RootTaskID string // Filter by flow-root task id; empty = no filter ExcludeStatuses []TaskStatus // Omit tasks whose status is in this list Limit int Offset int @@ -359,6 +432,18 @@ type TaskFilter struct { // task tree below the named parent. False (default) preserves the // existing direct-children-only behavior. IncludeDescendants bool + + // Priority filters to tasks whose stored priority equals this exact level. + // 0 (UNSPECIFIED) = no filter. + Priority int32 + // MinPriority filters to tasks whose stored priority is >= this level + // (e.g. PriorityHigh returns HIGH and PREEMPT). 0 = no filter. + MinPriority int32 + // OrderByPriority, when true, orders results by priority DESC, created_at + // ASC (highest priority first, FIFO within a level) instead of the default + // created_at DESC. Used by the dispatch-selection paths; general listings + // keep newest-first. + OrderByPriority bool } // ============================================================================= @@ -374,6 +459,27 @@ var terminalStatuses = map[TaskStatus]bool{ TaskStatusRejected: true, } +// allStatuses enumerates every defined TaskStatus in declaration order. It is +// the single master list from which the non-terminal set is derived, so adding +// a new status constant only requires appending it here (and to terminalStatuses +// if it is terminal) — the reconcile sweeps that key off NonTerminalStatuses +// then stay correct automatically. +var allStatuses = []TaskStatus{ + TaskStatusPending, + TaskStatusAssigned, + TaskStatusStarting, + TaskStatusRunning, + TaskStatusCompleted, + TaskStatusFailed, + TaskStatusCancelled, + TaskStatusDLQ, + TaskStatusWaitingInput, + TaskStatusWaitingAuthority, + TaskStatusWaitingDependency, + TaskStatusHibernated, + TaskStatusRejected, +} + // waitingStatuses is the set of paused states. var waitingStatuses = map[TaskStatus]bool{ TaskStatusWaitingInput: true, @@ -392,6 +498,22 @@ func IsWaiting(status TaskStatus) bool { return waitingStatuses[status] } +// NonTerminalStatuses returns every status for which IsTerminal is false, in +// declaration order. This is the canonical "task is still live" set — a task in +// any of these states may yet transition, so anything keyed off it (e.g. the +// orphaned orchestrated_task_queue reconcile sweep, which retires queue rows +// whose task is terminal or missing) stays consistent with the transition +// model. Derived from terminalStatuses so it cannot drift. +func NonTerminalStatuses() []TaskStatus { + out := make([]TaskStatus, 0, len(allStatuses)) + for _, s := range allStatuses { + if !terminalStatuses[s] { + out = append(out, s) + } + } + return out +} + // validTransitions defines the allowed from→to status transitions. // Any transition not listed is rejected by ValidateTransition. var validTransitions = map[TaskStatus]map[TaskStatus]bool{ diff --git a/server/tests/integration/load_test.go b/server/tests/integration/load_test.go index 56baa50..2e4c657 100644 --- a/server/tests/integration/load_test.go +++ b/server/tests/integration/load_test.go @@ -17,7 +17,7 @@ package integration // // It does NOT measure: real RMQ stream throughput, real network jitter, real // gateway memory under gRPC stream pressure, or service-side gRPC delivery -// (which has a documented wiring gap — see proxy-load-test-results.md). +// (which has a known wiring gap in this in-process harness). // // Run: // cd server diff --git a/versions.yaml b/versions.yaml index 237dd0d..b2597ef 100644 --- a/versions.yaml +++ b/versions.yaml @@ -9,26 +9,27 @@ # directives in each go.mod handle local builds; the `require` pins handle downstream # `go get` from pkg.go.dev. The gomod_require rules below keep those pins in sync. -aether-gateway: 0.2.1 +aether-gateway: 0.2.2 -aether-sdk-go: 0.2.1 # github.com/scitrera/aether/sdk/go (also drives github.com/scitrera/aether/api require pins) +aether-sdk-go: 0.2.2 # github.com/scitrera/aether/sdk/go (also drives github.com/scitrera/aether/api require pins) -aether-sdk-typescript: 0.2.1 # @scitrera/aether-client +aether-sdk-typescript: 0.2.2 # @scitrera/aether-client -aether-sdk-python: 0.2.1 # scitrera-aether-client -aether-sdk-python-ag2: 0.0.1 # scitrera-aether-ag2 +aether-sdk-python: 0.2.2 # scitrera-aether-client +aether-sdk-python-ag2: 0.0.2 # scitrera-aether-ag2 go_toolchain: - go: "1.25.10" + go: "1.25.12" # set explicit versions where possible to protect against supply chain attacks preferred_versions: go: "google.golang.org/grpc": "1.81.1" "google.golang.org/protobuf": "1.36.11" + "github.com/scitrera/go-backpressure": "v0.1.1" python: - "grpcio": 1.80.0 - "grpcio-tools": 1.80.0 + "grpcio": 1.81.1 + "grpcio-tools": 1.81.1 "protobuf": ">6" # TODO: pin version that will be compatible with opentelemetry grpc "pytest": 9.0.3 typescript: @@ -40,6 +41,8 @@ project_rules: - { type: go_version, path: server/internal/version/version.go } - { type: gomod_require, path: server/go.mod, args: [ github.com/scitrera/aether/api ] } - { type: gomod_require, path: server/go.mod, args: [ github.com/scitrera/aether/sdk/go ] } + aether-api-go: + - { type: gomod_require, path: api/go.mod } aether-sdk-go: - { type: go_version, path: sdk/go/aether/version.go } - { type: gomod_require, path: sdk/go/go.mod, args: [ github.com/scitrera/aether/api ] }