From 1466315f5dcc63070805d58ac7bf0a83e1aa8e8a Mon Sep 17 00:00:00 2001 From: Wojtek Date: Tue, 28 Apr 2026 12:03:40 -0400 Subject: [PATCH] feat: stdio MCP wrapper for managed tool sidecars Add `claw-mcp-stdio` shared wrapper image that exposes a stdio MCP server behind a pod-internal Streamable HTTP /mcp endpoint compatible with the v0.11.0 cllama MCP client. New pod surfaces: - `x-claw.mcp-stdio: {command, args}` declares the child command; compose emission injects CLAW_MCP_STDIO_COMMAND/ARGS on the wrapper service. - `x-claw.describe-file` lets operators supply a deterministic v2 descriptor snapshot from the pod directory (highest-priority descriptor source ahead of image/build-context discovery). Wrapper handles initialize caching, MCP-Session-Id minting/validation, JSON-RPC id multiplexing, child restart with exponential backoff, and optional bearer auth. Stdio creds stay in wrapper env. Hermetic spike TestSpikeMCPStdio brings up wrapper + cllama + agent in Docker. cllama is unchanged. Closes #179 --- .github/workflows/claw-mcp-stdio-image.yml | 61 ++ README.md | 2 +- cmd/claw-mcp-stdio/main.go | 750 ++++++++++++++++++ cmd/claw-mcp-stdio/main_test.go | 306 +++++++ cmd/claw/compose_up.go | 31 +- cmd/claw/compose_up_test.go | 45 ++ cmd/claw/image_lifecycle.go | 6 +- cmd/claw/mcp_stdio_spike_test.go | 232 ++++++ cmd/claw/skill_data/SKILL.md | 15 +- dockerfiles/claw-mcp-stdio/Dockerfile | 18 + .../020-cllama-compiled-tool-mediation.md | 6 + .../2026-04-28-issue-179-stdio-mcp-wrapper.md | 489 ++++++++++++ examples/mcp-stdio/AGENTS.md | 4 + examples/mcp-stdio/Clawfile | 6 + examples/mcp-stdio/claw-pod.yml | 28 + examples/mcp-stdio/echo-server/server.js | 75 ++ examples/mcp-stdio/echo.claw-describe.json | 28 + examples/perplexity-stdio/AGENTS.md | 4 + examples/perplexity-stdio/claw-pod.yml | 28 + .../perplexity.claw-describe.json | 28 + internal/infraimages/release_manifest.go | 14 +- internal/infraimages/release_manifest_test.go | 1 + internal/pod/compose_emit.go | 34 +- internal/pod/compose_emit_mcp_stdio_test.go | 97 +++ internal/pod/parser.go | 100 ++- internal/pod/parser_mcp_stdio_test.go | 109 +++ internal/pod/types.go | 15 + site/changelog.md | 2 +- site/guide/pod-yaml.md | 21 + site/guide/tools.md | 49 ++ skills/clawdapus/SKILL.md | 15 +- skills/release/SKILL.md | 30 +- 32 files changed, 2594 insertions(+), 55 deletions(-) create mode 100644 .github/workflows/claw-mcp-stdio-image.yml create mode 100644 cmd/claw-mcp-stdio/main.go create mode 100644 cmd/claw-mcp-stdio/main_test.go create mode 100644 cmd/claw/mcp_stdio_spike_test.go create mode 100644 dockerfiles/claw-mcp-stdio/Dockerfile create mode 100644 docs/plans/2026-04-28-issue-179-stdio-mcp-wrapper.md create mode 100644 examples/mcp-stdio/AGENTS.md create mode 100644 examples/mcp-stdio/Clawfile create mode 100644 examples/mcp-stdio/claw-pod.yml create mode 100644 examples/mcp-stdio/echo-server/server.js create mode 100644 examples/mcp-stdio/echo.claw-describe.json create mode 100644 examples/perplexity-stdio/AGENTS.md create mode 100644 examples/perplexity-stdio/claw-pod.yml create mode 100644 examples/perplexity-stdio/perplexity.claw-describe.json create mode 100644 internal/pod/compose_emit_mcp_stdio_test.go create mode 100644 internal/pod/parser_mcp_stdio_test.go diff --git a/.github/workflows/claw-mcp-stdio-image.yml b/.github/workflows/claw-mcp-stdio-image.yml new file mode 100644 index 0000000..28690b9 --- /dev/null +++ b/.github/workflows/claw-mcp-stdio-image.yml @@ -0,0 +1,61 @@ +name: claw-mcp-stdio Image + +on: + push: + branches: + - master + tags: + - "v*" + pull_request: + paths: + - "cmd/claw-mcp-stdio/**" + - "dockerfiles/claw-mcp-stdio/**" + - "go.mod" + - "go.sum" + - ".github/workflows/claw-mcp-stdio-image.yml" + workflow_dispatch: + +permissions: + contents: read + packages: write + +jobs: + build-and-publish: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: docker/setup-qemu-action@v3 + + - uses: docker/setup-buildx-action@v3 + + - uses: docker/login-action@v3 + if: github.event_name != 'pull_request' + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GHCR_PAT }} + + - id: meta + uses: docker/metadata-action@v5 + with: + images: ghcr.io/${{ github.repository_owner }}/claw-mcp-stdio + tags: | + type=raw,value=latest,enable={{is_default_branch}} + type=ref,event=tag + type=sha,format=short + labels: | + claw.component=claw-mcp-stdio + claw.source=registry + claw.dirty=false + + - uses: docker/build-push-action@v6 + with: + context: . + file: dockerfiles/claw-mcp-stdio/Dockerfile + platforms: linux/amd64,linux/arm64 + push: ${{ github.event_name != 'pull_request' }} + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max diff --git a/README.md b/README.md index f9b861e..a1ccf1d 100644 --- a/README.md +++ b/README.md @@ -374,7 +374,7 @@ When a reasoning model tries to govern itself, the guardrails are part of the sa - **Identity resolution:** Single proxy serves an entire pod. Bearer tokens resolve which agent is calling. - **Cost accounting:** Extracts token usage from every response, multiplies by pricing table, tracks per agent/provider/model. - **Audit logging:** Structured JSON on stdout — timestamp, agent, model, latency, tokens, cost, intervention reason. -- **Managed tool mediation:** Services declare callable tools via `claw.describe` (MCP-shaped schemas). `claw up` compiles per-agent `tools.json`. cllama injects tools into LLM requests, intercepts `tool_call` responses, executes them against the service, and loops until terminal text — transparent to the runner. Both OpenAI-compatible and Anthropic formats are supported. +- **Managed tool mediation:** Services declare callable tools via `claw.describe` (MCP-shaped schemas). `claw up` compiles per-agent `tools.json`. cllama injects tools into LLM requests, intercepts `tool_call` responses, executes them against the service, and loops until terminal text — transparent to the runner. HTTP services, Streamable HTTP MCP sidecars, and stdio MCP servers wrapped by `claw-mcp-stdio` are supported. - **Ambient memory plane:** Services declare `recall`, `retain`, and `forget` endpoints via `claw.describe`. `claw up` compiles per-agent `memory.json`. cllama calls `/recall` before each inference turn and `/retain` after each successful response (async, non-blocking). Memory intelligence stays in swappable external services — the proxy owns orchestration only. - **Operator dashboard:** Real-time web UI at host port 8181 by default (container `:8081`) — agent activity, provider status, cost breakdown. diff --git a/cmd/claw-mcp-stdio/main.go b/cmd/claw-mcp-stdio/main.go new file mode 100644 index 0000000..31917d9 --- /dev/null +++ b/cmd/claw-mcp-stdio/main.go @@ -0,0 +1,750 @@ +package main + +import ( + "bufio" + "bytes" + "context" + "crypto/rand" + "encoding/hex" + "encoding/json" + "errors" + "flag" + "fmt" + "io" + "net" + "net/http" + "os" + "os/exec" + "os/signal" + "strconv" + "strings" + "sync" + "sync/atomic" + "syscall" + "time" +) + +const ( + defaultAddr = ":8080" + defaultMCPPath = "/mcp" + defaultRequestTimeoutMS = 60000 + defaultRestartBackoffMS = 1000 + defaultRestartMaxMS = 15000 + defaultMaxBodyBytes = 1 << 20 +) + +type config struct { + Command string + Args []string + Addr string + MCPPath string + AuthToken string + RequestTimeout time.Duration + RestartBackoff time.Duration + RestartMax time.Duration + MaxBodyBytes int64 +} + +type rpcRequest struct { + ID json.RawMessage `json:"id,omitempty"` + Method string `json:"method,omitempty"` +} + +type childResponse struct { + ID json.RawMessage `json:"id,omitempty"` +} + +type pendingCall struct { + originalID json.RawMessage + ch chan []byte +} + +type initCache struct { + response []byte +} + +type stdioBridge struct { + cfg config + + ctx context.Context + cancel context.CancelFunc + + mu sync.Mutex + stdin io.WriteCloser + available bool + generation int64 + pending map[string]pendingCall + sessions map[string]int64 + initialized bool + init *initCache + + initMu sync.Mutex + nextID atomic.Int64 +} + +func main() { + if err := run(os.Args[1:]); err != nil { + fmt.Fprintln(os.Stderr, err.Error()) + os.Exit(1) + } +} + +func run(args []string) error { + fs := flag.NewFlagSet("claw-mcp-stdio", flag.ContinueOnError) + fs.SetOutput(os.Stderr) + healthcheck := fs.Bool("healthcheck", false, "check HTTP health endpoint and exit") + if err := fs.Parse(args); err != nil { + return err + } + + cfg, err := loadConfig() + if err != nil { + return err + } + if *healthcheck { + return runHealthcheck(cfg.Addr) + } + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + bridge := newStdioBridge(ctx, cfg) + go bridge.Run() + + mux := http.NewServeMux() + mux.HandleFunc("/healthz", bridge.handleHealth) + mux.HandleFunc(cfg.MCPPath, bridge.handleMCP) + + server := &http.Server{ + Addr: cfg.Addr, + Handler: mux, + ReadHeaderTimeout: 10 * time.Second, + } + + errCh := make(chan error, 1) + go func() { + fmt.Fprintf(os.Stderr, "claw-mcp-stdio listening on %s path %s\n", cfg.Addr, cfg.MCPPath) + errCh <- server.ListenAndServe() + }() + + sigCh := make(chan os.Signal, 1) + signal.Notify(sigCh, syscall.SIGINT, syscall.SIGTERM) + + select { + case <-sigCh: + case err := <-errCh: + if errors.Is(err, http.ErrServerClosed) { + return nil + } + return err + } + + cancel() + shutdownCtx, shutdownCancel := context.WithTimeout(context.Background(), 5*time.Second) + defer shutdownCancel() + return server.Shutdown(shutdownCtx) +} + +func loadConfig() (config, error) { + command := strings.TrimSpace(os.Getenv("CLAW_MCP_STDIO_COMMAND")) + if command == "" { + return config{}, fmt.Errorf("claw-mcp-stdio: CLAW_MCP_STDIO_COMMAND is required") + } + + args, err := envJSONArray("CLAW_MCP_STDIO_ARGS") + if err != nil { + return config{}, err + } + addr := strings.TrimSpace(os.Getenv("CLAW_MCP_STDIO_ADDR")) + if addr == "" { + port := strings.TrimSpace(os.Getenv("CLAW_MCP_STDIO_PORT")) + if port == "" { + addr = defaultAddr + } else { + addr = ":" + strings.TrimPrefix(port, ":") + } + } + + mcpPath := strings.TrimSpace(os.Getenv("CLAW_MCP_STDIO_PATH")) + if mcpPath == "" { + mcpPath = defaultMCPPath + } + if !strings.HasPrefix(mcpPath, "/") { + return config{}, fmt.Errorf("claw-mcp-stdio: CLAW_MCP_STDIO_PATH must start with '/'") + } + + requestTimeout, err := envDurationMS("CLAW_MCP_STDIO_REQUEST_TIMEOUT_MS", defaultRequestTimeoutMS) + if err != nil { + return config{}, err + } + restartBackoff, err := envDurationMS("CLAW_MCP_STDIO_RESTART_BACKOFF_MS", defaultRestartBackoffMS) + if err != nil { + return config{}, err + } + restartMax, err := envDurationMS("CLAW_MCP_STDIO_RESTART_MAX_MS", defaultRestartMaxMS) + if err != nil { + return config{}, err + } + maxBodyBytes, err := envInt64("CLAW_MCP_STDIO_MAX_BODY_BYTES", defaultMaxBodyBytes) + if err != nil { + return config{}, err + } + if maxBodyBytes < 1 { + return config{}, fmt.Errorf("claw-mcp-stdio: CLAW_MCP_STDIO_MAX_BODY_BYTES must be at least 1") + } + + return config{ + Command: command, + Args: args, + Addr: addr, + MCPPath: mcpPath, + AuthToken: strings.TrimSpace(os.Getenv("CLAW_MCP_STDIO_AUTH_TOKEN")), + RequestTimeout: requestTimeout, + RestartBackoff: restartBackoff, + RestartMax: restartMax, + MaxBodyBytes: maxBodyBytes, + }, nil +} + +func envJSONArray(key string) ([]string, error) { + raw := strings.TrimSpace(os.Getenv(key)) + if raw == "" { + return []string{}, nil + } + var out []string + if err := json.Unmarshal([]byte(raw), &out); err != nil { + return nil, fmt.Errorf("claw-mcp-stdio: %s must be a JSON string array: %w", key, err) + } + return out, nil +} + +func envDurationMS(key string, fallback int) (time.Duration, error) { + value, err := envInt64(key, int64(fallback)) + if err != nil { + return 0, err + } + if value < 1 { + return 0, fmt.Errorf("claw-mcp-stdio: %s must be at least 1", key) + } + return time.Duration(value) * time.Millisecond, nil +} + +func envInt64(key string, fallback int64) (int64, error) { + raw := strings.TrimSpace(os.Getenv(key)) + if raw == "" { + return fallback, nil + } + value, err := strconv.ParseInt(raw, 10, 64) + if err != nil { + return 0, fmt.Errorf("claw-mcp-stdio: %s must be an integer: %w", key, err) + } + return value, nil +} + +func newStdioBridge(ctx context.Context, cfg config) *stdioBridge { + childCtx, cancel := context.WithCancel(ctx) + return &stdioBridge{ + cfg: cfg, + ctx: childCtx, + cancel: cancel, + pending: make(map[string]pendingCall), + sessions: make(map[string]int64), + } +} + +func (b *stdioBridge) Run() { + backoff := b.cfg.RestartBackoff + for { + if err := b.ctx.Err(); err != nil { + return + } + if backoff <= 0 { + backoff = time.Second + } + if b.cfg.RestartMax > 0 && backoff > b.cfg.RestartMax { + backoff = b.cfg.RestartMax + } + + err := b.runChild() + if err != nil && b.ctx.Err() == nil { + fmt.Fprintf(os.Stderr, "claw-mcp-stdio child stopped: %v\n", err) + } + b.markUnavailable("stdio child restarting") + + timer := time.NewTimer(backoff) + select { + case <-b.ctx.Done(): + timer.Stop() + return + case <-timer.C: + } + if b.cfg.RestartMax > 0 && backoff < b.cfg.RestartMax { + backoff *= 2 + if backoff > b.cfg.RestartMax { + backoff = b.cfg.RestartMax + } + } + } +} + +func (b *stdioBridge) runChild() error { + cmd := exec.CommandContext(b.ctx, b.cfg.Command, b.cfg.Args...) + stdin, err := cmd.StdinPipe() + if err != nil { + return err + } + stdout, err := cmd.StdoutPipe() + if err != nil { + return err + } + stderr, err := cmd.StderrPipe() + if err != nil { + return err + } + if err := cmd.Start(); err != nil { + return err + } + + generation := b.markAvailable(stdin) + fmt.Fprintf(os.Stderr, "claw-mcp-stdio spawned %q generation=%d\n", b.cfg.Command, generation) + + var wg sync.WaitGroup + wg.Add(2) + go func() { + defer wg.Done() + b.readStdout(generation, stdout) + }() + go func() { + defer wg.Done() + copyPrefixed(os.Stderr, "mcp-stdio stderr: ", stderr) + }() + + waitErr := cmd.Wait() + wg.Wait() + return waitErr +} + +func (b *stdioBridge) markAvailable(stdin io.WriteCloser) int64 { + b.mu.Lock() + defer b.mu.Unlock() + b.generation++ + b.stdin = stdin + b.available = true + b.pending = make(map[string]pendingCall) + b.sessions = make(map[string]int64) + b.initialized = false + b.init = nil + return b.generation +} + +func (b *stdioBridge) markUnavailable(message string) { + b.mu.Lock() + defer b.mu.Unlock() + for _, pending := range b.pending { + pending.ch <- rpcErrorResponse(pending.originalID, -32000, message) + } + b.pending = make(map[string]pendingCall) + b.sessions = make(map[string]int64) + b.initialized = false + b.init = nil + b.available = false + b.stdin = nil +} + +func (b *stdioBridge) readStdout(generation int64, r io.Reader) { + scanner := bufio.NewScanner(r) + scanner.Buffer(make([]byte, 0, 64*1024), 4*1024*1024) + for scanner.Scan() { + line := bytes.TrimSpace(scanner.Bytes()) + if len(line) == 0 { + continue + } + var resp childResponse + if err := json.Unmarshal(line, &resp); err != nil { + fmt.Fprintf(os.Stderr, "claw-mcp-stdio ignored non-JSON child output: %s\n", string(line)) + continue + } + key := idKey(resp.ID) + if key == "" { + fmt.Fprintf(os.Stderr, "claw-mcp-stdio child notification: %s\n", string(line)) + continue + } + b.mu.Lock() + pending, ok := b.pending[key] + if ok { + delete(b.pending, key) + } + b.mu.Unlock() + if !ok { + fmt.Fprintf(os.Stderr, "claw-mcp-stdio ignored response for unknown id %s\n", key) + continue + } + pending.ch <- restoreResponseID(line, pending.originalID) + } + if err := scanner.Err(); err != nil && b.ctx.Err() == nil { + fmt.Fprintf(os.Stderr, "claw-mcp-stdio stdout reader error: %v\n", err) + } + b.failGeneration(generation, "stdio child exited") +} + +func (b *stdioBridge) failGeneration(generation int64, message string) { + b.mu.Lock() + defer b.mu.Unlock() + if generation != b.generation { + return + } + for _, pending := range b.pending { + pending.ch <- rpcErrorResponse(pending.originalID, -32000, message) + } + b.pending = make(map[string]pendingCall) + b.available = false + b.stdin = nil + b.sessions = make(map[string]int64) + b.initialized = false + b.init = nil +} + +func (b *stdioBridge) handleHealth(w http.ResponseWriter, _ *http.Request) { + if b.isAvailable() { + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("ok\n")) + return + } + http.Error(w, "stdio child unavailable", http.StatusServiceUnavailable) +} + +func (b *stdioBridge) handleMCP(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + http.Error(w, "method not allowed", http.StatusMethodNotAllowed) + return + } + if !b.authorized(r) { + http.Error(w, "unauthorized", http.StatusUnauthorized) + return + } + + body, err := readLimited(r.Body, b.cfg.MaxBodyBytes) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + var req rpcRequest + if err := json.Unmarshal(body, &req); err != nil { + writeRPCError(w, nil, http.StatusBadRequest, -32700, "parse error") + return + } + method := strings.TrimSpace(req.Method) + if method == "" { + writeRPCError(w, req.ID, http.StatusBadRequest, -32600, "missing method") + return + } + + if method == "initialize" { + b.handleInitialize(w, body, req.ID) + return + } + + sessionID := strings.TrimSpace(r.Header.Get("MCP-Session-Id")) + if !b.validSession(sessionID) { + writeRPCError(w, req.ID, http.StatusNotFound, -32001, "unknown MCP session") + return + } + w.Header().Set("MCP-Session-Id", sessionID) + + if idKey(req.ID) == "" { + if err := b.forwardNotification(body, method); err != nil { + writeRPCError(w, nil, http.StatusServiceUnavailable, -32000, err.Error()) + return + } + w.WriteHeader(http.StatusAccepted) + return + } + + resp, err := b.forwardRequest(r.Context(), body, req.ID) + if err != nil { + writeRPCError(w, req.ID, http.StatusServiceUnavailable, -32000, err.Error()) + return + } + writeJSON(w, http.StatusOK, resp) +} + +func (b *stdioBridge) handleInitialize(w http.ResponseWriter, body []byte, originalID json.RawMessage) { + b.initMu.Lock() + defer b.initMu.Unlock() + + if b.init != nil { + sessionID, err := b.newSession() + if err != nil { + writeRPCError(w, originalID, http.StatusInternalServerError, -32000, err.Error()) + return + } + w.Header().Set("MCP-Session-Id", sessionID) + writeJSON(w, http.StatusOK, restoreResponseID(b.init.response, originalID)) + return + } + + resp, err := b.forwardRequest(context.Background(), body, originalID) + if err != nil { + writeRPCError(w, originalID, http.StatusServiceUnavailable, -32000, err.Error()) + return + } + if responseHasError(resp) { + writeJSON(w, http.StatusOK, resp) + return + } + b.init = &initCache{response: append([]byte(nil), resp...)} + + sessionID, err := b.newSession() + if err != nil { + writeRPCError(w, originalID, http.StatusInternalServerError, -32000, err.Error()) + return + } + w.Header().Set("MCP-Session-Id", sessionID) + writeJSON(w, http.StatusOK, resp) +} + +func (b *stdioBridge) authorized(r *http.Request) bool { + if b.cfg.AuthToken == "" { + return true + } + return strings.TrimSpace(r.Header.Get("Authorization")) == "Bearer "+b.cfg.AuthToken +} + +func (b *stdioBridge) isAvailable() bool { + b.mu.Lock() + defer b.mu.Unlock() + return b.available +} + +func (b *stdioBridge) validSession(sessionID string) bool { + if sessionID == "" { + return false + } + b.mu.Lock() + defer b.mu.Unlock() + _, ok := b.sessions[sessionID] + return ok +} + +func (b *stdioBridge) newSession() (string, error) { + token, err := randomToken() + if err != nil { + return "", err + } + b.mu.Lock() + defer b.mu.Unlock() + b.sessions[token] = b.generation + return token, nil +} + +func (b *stdioBridge) forwardNotification(body []byte, method string) error { + if method == "notifications/initialized" { + b.mu.Lock() + if b.initialized { + b.mu.Unlock() + return nil + } + b.initialized = true + b.mu.Unlock() + } + _, err := b.writeToChild(body, nil) + return err +} + +func (b *stdioBridge) forwardRequest(ctx context.Context, body []byte, originalID json.RawMessage) ([]byte, error) { + childID := b.nextID.Add(1) + childIDRaw := json.RawMessage(strconv.FormatInt(childID, 10)) + rewritten, err := rewriteRequestID(body, childIDRaw) + if err != nil { + return nil, err + } + + ch := make(chan []byte, 1) + key, err := b.writeToChild(rewritten, &pendingCall{originalID: append([]byte(nil), originalID...), ch: ch}) + if err != nil { + return nil, err + } + + timeout := b.cfg.RequestTimeout + if timeout <= 0 { + timeout = time.Minute + } + timer := time.NewTimer(timeout) + defer timer.Stop() + + select { + case resp := <-ch: + return resp, nil + case <-ctx.Done(): + b.removePending(key) + return nil, ctx.Err() + case <-timer.C: + b.removePending(key) + return nil, fmt.Errorf("stdio child request timed out after %s", timeout) + } +} + +func (b *stdioBridge) writeToChild(body []byte, pending *pendingCall) (string, error) { + var key string + if pending != nil { + var req rpcRequest + if err := json.Unmarshal(body, &req); err != nil { + return "", err + } + key = idKey(req.ID) + if key == "" { + return "", fmt.Errorf("request id is required") + } + } + + b.mu.Lock() + defer b.mu.Unlock() + if !b.available || b.stdin == nil { + return "", fmt.Errorf("stdio child unavailable") + } + if pending != nil { + b.pending[key] = *pending + } + if _, err := b.stdin.Write(append(body, '\n')); err != nil { + if pending != nil { + delete(b.pending, key) + } + return "", err + } + return key, nil +} + +func (b *stdioBridge) removePending(key string) { + if key == "" { + return + } + b.mu.Lock() + delete(b.pending, key) + b.mu.Unlock() +} + +func rewriteRequestID(body []byte, id json.RawMessage) ([]byte, error) { + var obj map[string]json.RawMessage + if err := json.Unmarshal(body, &obj); err != nil { + return nil, err + } + obj["id"] = id + return json.Marshal(obj) +} + +func restoreResponseID(body []byte, id json.RawMessage) []byte { + var obj map[string]json.RawMessage + if err := json.Unmarshal(body, &obj); err != nil { + return append([]byte(nil), body...) + } + if idKey(id) == "" { + delete(obj, "id") + } else { + obj["id"] = id + } + out, err := json.Marshal(obj) + if err != nil { + return append([]byte(nil), body...) + } + return out +} + +func responseHasError(body []byte) bool { + var obj map[string]json.RawMessage + if err := json.Unmarshal(body, &obj); err != nil { + return false + } + errRaw := bytes.TrimSpace(obj["error"]) + return len(errRaw) > 0 && string(errRaw) != "null" +} + +func idKey(raw json.RawMessage) string { + raw = bytes.TrimSpace(raw) + if len(raw) == 0 || bytes.Equal(raw, []byte("null")) { + return "" + } + return string(raw) +} + +func rpcErrorResponse(id json.RawMessage, code int, message string) []byte { + if idKey(id) == "" { + id = json.RawMessage("null") + } + body := map[string]any{ + "jsonrpc": "2.0", + "id": json.RawMessage(id), + "error": map[string]any{ + "code": code, + "message": message, + }, + } + data, _ := json.Marshal(body) + return data +} + +func writeRPCError(w http.ResponseWriter, id json.RawMessage, status int, code int, message string) { + writeJSON(w, status, rpcErrorResponse(id, code, message)) +} + +func writeJSON(w http.ResponseWriter, status int, body []byte) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(status) + _, _ = w.Write(body) + _, _ = w.Write([]byte("\n")) +} + +func readLimited(r io.Reader, limit int64) ([]byte, error) { + data, err := io.ReadAll(io.LimitReader(r, limit+1)) + if err != nil { + return nil, err + } + if int64(len(data)) > limit { + return nil, fmt.Errorf("request body exceeds %d bytes", limit) + } + return data, nil +} + +func randomToken() (string, error) { + var raw [16]byte + if _, err := rand.Read(raw[:]); err != nil { + return "", err + } + return hex.EncodeToString(raw[:]), nil +} + +func copyPrefixed(w io.Writer, prefix string, r io.Reader) { + scanner := bufio.NewScanner(r) + for scanner.Scan() { + fmt.Fprintln(w, prefix+scanner.Text()) + } +} + +func runHealthcheck(addr string) error { + client := &http.Client{Timeout: 3 * time.Second} + resp, err := client.Get(healthcheckURL(addr)) + if err != nil { + return err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("health endpoint returned %s", resp.Status) + } + return nil +} + +func healthcheckURL(addr string) string { + if addr == "" { + addr = defaultAddr + } + if strings.HasPrefix(addr, ":") { + return "http://127.0.0.1" + addr + "/healthz" + } + host, port, err := net.SplitHostPort(addr) + if err != nil { + return "http://127.0.0.1:8080/healthz" + } + if host == "" || host == "0.0.0.0" || host == "::" { + host = "127.0.0.1" + } + return "http://" + host + ":" + port + "/healthz" +} diff --git a/cmd/claw-mcp-stdio/main_test.go b/cmd/claw-mcp-stdio/main_test.go new file mode 100644 index 0000000..feabd4e --- /dev/null +++ b/cmd/claw-mcp-stdio/main_test.go @@ -0,0 +1,306 @@ +package main + +import ( + "bufio" + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/http/httptest" + "os" + "strings" + "sync" + "testing" + "time" +) + +func TestBridgeForwardsInitializeNotificationAndToolCall(t *testing.T) { + server, cleanup := newTestWrapper(t, "") + defer cleanup() + + initResp, sessionID := postRPC(t, server.URL, "", `{"jsonrpc":"2.0","id":1,"method":"initialize","params":{"protocolVersion":"2025-11-25"}}`) + if sessionID == "" { + t.Fatal("initialize did not return MCP-Session-Id") + } + assertJSONField(t, initResp, "id", float64(1)) + assertJSONField(t, initResp, "result.serverInfo.name", "helper") + + status := postRPCStatus(t, server.URL, sessionID, `{"jsonrpc":"2.0","method":"notifications/initialized"}`) + if status != http.StatusAccepted { + t.Fatalf("initialized notification status = %d, want %d", status, http.StatusAccepted) + } + + toolResp, _ := postRPC(t, server.URL, sessionID, `{"jsonrpc":"2.0","id":2,"method":"tools/call","params":{"name":"echo","arguments":{"message":"hello"}}}`) + assertJSONField(t, toolResp, "id", float64(2)) + assertJSONField(t, toolResp, "result.content.0.text", "hello") +} + +func TestBridgeRoutesConcurrentDuplicateClientIDs(t *testing.T) { + server, cleanup := newTestWrapper(t, "") + defer cleanup() + _, sessionID := postRPC(t, server.URL, "", `{"jsonrpc":"2.0","id":1,"method":"initialize","params":{}}`) + + var wg sync.WaitGroup + errCh := make(chan error, 2) + for _, message := range []string{"slow", "fast"} { + wg.Add(1) + go func(message string) { + defer wg.Done() + body := fmt.Sprintf(`{"jsonrpc":"2.0","id":2,"method":"tools/call","params":{"name":"echo","arguments":{"message":%q}}}`, message) + resp, _ := postRPC(t, server.URL, sessionID, body) + if got := jsonPath(resp, "result.content.0.text"); got != message { + errCh <- fmt.Errorf("message %q routed to %v", message, got) + } + if got := jsonPath(resp, "id"); got != float64(2) { + errCh <- fmt.Errorf("message %q response id = %v", message, got) + } + }(message) + } + wg.Wait() + close(errCh) + for err := range errCh { + t.Error(err) + } +} + +func TestBridgeRejectsUnknownSession(t *testing.T) { + server, cleanup := newTestWrapper(t, "") + defer cleanup() + + status := postRPCStatus(t, server.URL, "missing", `{"jsonrpc":"2.0","id":2,"method":"tools/call","params":{}}`) + if status != http.StatusNotFound { + t.Fatalf("status = %d, want %d", status, http.StatusNotFound) + } +} + +func TestBridgeOptionalBearerAuth(t *testing.T) { + server, cleanup := newTestWrapper(t, "secret") + defer cleanup() + + req, err := http.NewRequest(http.MethodPost, mcpURL(server.URL), strings.NewReader(`{"jsonrpc":"2.0","id":1,"method":"initialize","params":{}}`)) + if err != nil { + t.Fatal(err) + } + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + resp.Body.Close() + if resp.StatusCode != http.StatusUnauthorized { + t.Fatalf("status without auth = %d, want %d", resp.StatusCode, http.StatusUnauthorized) + } + + req, err = http.NewRequest(http.MethodPost, mcpURL(server.URL), strings.NewReader(`{"jsonrpc":"2.0","id":1,"method":"initialize","params":{}}`)) + if err != nil { + t.Fatal(err) + } + req.Header.Set("Authorization", "Bearer secret") + resp, err = http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + resp.Body.Close() + if resp.StatusCode != http.StatusOK { + t.Fatalf("status with auth = %d, want %d", resp.StatusCode, http.StatusOK) + } +} + +func newTestWrapper(t *testing.T, authToken string) (*httptest.Server, func()) { + t.Helper() + cfg := config{ + Command: os.Args[0], + Args: []string{"-test.run=TestHelperProcess", "--", "stdio-helper"}, + MCPPath: "/mcp", + AuthToken: authToken, + RequestTimeout: 5 * time.Second, + RestartBackoff: time.Hour, + RestartMax: time.Hour, + MaxBodyBytes: defaultMaxBodyBytes, + } + t.Setenv("GO_WANT_HELPER_PROCESS", "1") + ctx := testContext(t) + bridge := newStdioBridge(ctx, cfg) + go bridge.Run() + waitForAvailable(t, bridge) + + mux := http.NewServeMux() + mux.HandleFunc("/mcp", bridge.handleMCP) + mux.HandleFunc("/healthz", bridge.handleHealth) + server := httptest.NewServer(mux) + cleanup := func() { + server.Close() + bridge.cancel() + } + return server, cleanup +} + +func testContext(t *testing.T) context.Context { + t.Helper() + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + return ctx +} + +func waitForAvailable(t *testing.T, bridge *stdioBridge) { + t.Helper() + deadline := time.Now().Add(5 * time.Second) + for time.Now().Before(deadline) { + if bridge.isAvailable() { + return + } + time.Sleep(10 * time.Millisecond) + } + t.Fatal("bridge did not become available") +} + +func postRPC(t *testing.T, url string, sessionID string, body string) ([]byte, string) { + t.Helper() + req, err := http.NewRequest(http.MethodPost, mcpURL(url), strings.NewReader(body)) + if err != nil { + t.Fatal(err) + } + req.Header.Set("Content-Type", "application/json") + if sessionID != "" { + req.Header.Set("MCP-Session-Id", sessionID) + } + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + data, err := io.ReadAll(resp.Body) + if err != nil { + t.Fatal(err) + } + if resp.StatusCode != http.StatusOK { + t.Fatalf("status = %d, body = %s", resp.StatusCode, data) + } + return data, resp.Header.Get("MCP-Session-Id") +} + +func postRPCStatus(t *testing.T, url string, sessionID string, body string) int { + t.Helper() + req, err := http.NewRequest(http.MethodPost, mcpURL(url), strings.NewReader(body)) + if err != nil { + t.Fatal(err) + } + req.Header.Set("Content-Type", "application/json") + if sessionID != "" { + req.Header.Set("MCP-Session-Id", sessionID) + } + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + return resp.StatusCode +} + +func mcpURL(base string) string { + return strings.TrimRight(base, "/") + "/mcp" +} + +func assertJSONField(t *testing.T, data []byte, path string, want any) { + t.Helper() + if got := jsonPath(data, path); got != want { + t.Fatalf("%s = %v, want %v\n%s", path, got, want, data) + } +} + +func jsonPath(data []byte, path string) any { + var value any + if err := json.Unmarshal(data, &value); err != nil { + return nil + } + current := value + for _, part := range strings.Split(path, ".") { + switch node := current.(type) { + case map[string]any: + current = node[part] + case []any: + var idx int + fmt.Sscanf(part, "%d", &idx) + if idx < 0 || idx >= len(node) { + return nil + } + current = node[idx] + default: + return nil + } + } + return current +} + +func TestHelperProcess(t *testing.T) { + if os.Getenv("GO_WANT_HELPER_PROCESS") != "1" { + return + } + if len(os.Args) == 0 || os.Args[len(os.Args)-1] != "stdio-helper" { + os.Exit(2) + } + + scanner := bufio.NewScanner(os.Stdin) + var writeMu sync.Mutex + for scanner.Scan() { + line := append([]byte(nil), scanner.Bytes()...) + var req struct { + ID json.RawMessage `json:"id,omitempty"` + Method string `json:"method"` + Params struct { + Arguments map[string]any `json:"arguments"` + } `json:"params"` + } + if err := json.Unmarshal(line, &req); err != nil { + continue + } + if len(bytes.TrimSpace(req.ID)) == 0 { + continue + } + go func(req struct { + ID json.RawMessage `json:"id,omitempty"` + Method string `json:"method"` + Params struct { + Arguments map[string]any `json:"arguments"` + } `json:"params"` + }) { + message, _ := req.Params.Arguments["message"].(string) + if message == "slow" { + time.Sleep(100 * time.Millisecond) + } + var resp map[string]any + switch req.Method { + case "initialize": + resp = map[string]any{ + "jsonrpc": "2.0", + "id": json.RawMessage(req.ID), + "result": map[string]any{ + "protocolVersion": "2025-11-25", + "serverInfo": map[string]any{"name": "helper", "version": "test"}, + "capabilities": map[string]any{"tools": map[string]any{}}, + }, + } + case "tools/call": + resp = map[string]any{ + "jsonrpc": "2.0", + "id": json.RawMessage(req.ID), + "result": map[string]any{ + "content": []map[string]any{{"type": "text", "text": message}}, + }, + } + default: + resp = map[string]any{ + "jsonrpc": "2.0", + "id": json.RawMessage(req.ID), + "error": map[string]any{"code": -32601, "message": "method not found"}, + } + } + data, _ := json.Marshal(resp) + writeMu.Lock() + fmt.Println(string(data)) + writeMu.Unlock() + }(req) + } + os.Exit(0) +} diff --git a/cmd/claw/compose_up.go b/cmd/claw/compose_up.go index fce39f8..9c88058 100644 --- a/cmd/claw/compose_up.go +++ b/cmd/claw/compose_up.go @@ -62,6 +62,7 @@ var ( findClawdapusRepoRoot = findRepoRoot runInfraDockerCommand = runInfraDockerCommandDefault runComposeDockerCommand = runComposeDockerCommandDefault + loadDescriptorFromFile = describe.LoadFromFile loadDescriptorFromImage = describe.LoadFromImage loadDescriptorFromBuildCtx = describe.LoadFromBuildContext resolveBuildContextFile = describe.ResolveBuildContextFile @@ -188,13 +189,13 @@ func runComposeUp(podFile string) (err error) { // This is a cheap pass over the already-parsed pod YAML — no image inspection needed. podHandles := make(map[string]map[string]*driver.HandleInfo) // service → platform → HandleInfo for name, svc := range p.Services { - if svc.Claw != nil && len(svc.Claw.Handles) > 0 { + if svc.IsAgentManaged() && len(svc.Claw.Handles) > 0 { podHandles[name] = svc.Claw.Handles } } for name, svc := range p.Services { - if svc.Claw == nil { + if !svc.IsAgentManaged() { continue } @@ -3376,6 +3377,18 @@ func resolveServiceMetadata(podDir string, p *pod.Pod, serviceName string, svc * return "", nil, descriptor, nil } + if descriptorPath := explicitDescribeFile(podDir, svc); descriptorPath != "" { + descriptor, err := loadDescriptorFromFile(descriptorPath) + if err != nil { + return "", nil, nil, fmt.Errorf("load descriptor from describe-file: %w", err) + } + if svc != nil && strings.TrimSpace(svc.Image) != "" { + imageRefs[serviceName] = strings.TrimSpace(svc.Image) + } + descriptors[serviceName] = descriptor + return strings.TrimSpace(svc.Image), infos[serviceName], descriptor, nil + } + imageRef, info, err := inspectServiceMetadata(podDir, p, serviceName, svc, imageRefs, infos) if err != nil { return "", nil, nil, err @@ -3412,6 +3425,20 @@ func resolveServiceMetadata(podDir string, p *pod.Pod, serviceName string, svc * return imageRef, info, descriptor, nil } +func explicitDescribeFile(podDir string, svc *pod.Service) string { + if svc == nil || svc.Claw == nil { + return "" + } + path := strings.TrimSpace(svc.Claw.DescribeFile) + if path == "" { + return "" + } + if filepath.IsAbs(path) { + return filepath.Clean(path) + } + return filepath.Clean(filepath.Join(podDir, path)) +} + func resolvedImageDescriptorPath(info *inspect.ClawInfo) (string, bool) { if info != nil && strings.TrimSpace(info.DescribePath) != "" { return strings.TrimSpace(info.DescribePath), false diff --git a/cmd/claw/compose_up_test.go b/cmd/claw/compose_up_test.go index d1016fd..85d023f 100644 --- a/cmd/claw/compose_up_test.go +++ b/cmd/claw/compose_up_test.go @@ -2024,6 +2024,51 @@ LABEL maintainer=ops@example.com } } +func TestResolveServiceMetadataUsesExplicitDescribeFile(t *testing.T) { + podDir := t.TempDir() + if err := os.WriteFile(filepath.Join(podDir, "perplexity.claw-describe.json"), []byte(`{ + "version": 2, + "description": "Perplexity MCP", + "mcp": { "transport": "streamable_http", "path": "/mcp" }, + "tools": [ + { + "name": "search", + "description": "Search the web", + "inputSchema": { "type": "object" } + } + ] +}`), 0o644); err != nil { + t.Fatalf("write descriptor: %v", err) + } + + p := &pod.Pod{ + Services: map[string]*pod.Service{ + "perplexity": { + Image: "ghcr.io/mostlydev/claw-mcp-stdio:v0.12.0", + Claw: &pod.ClawBlock{ + DescribeFile: "./perplexity.claw-describe.json", + MCPStdio: &pod.MCPStdioBlock{Command: "npx", Args: []string{"-y", "perplexity-mcp"}}, + }, + }, + }, + } + + _, _, descriptor, err := resolveServiceMetadata(podDir, p, "perplexity", p.Services["perplexity"], map[string]string{}, map[string]*inspect.ClawInfo{}, map[string]*describe.ServiceDescriptor{}) + if err != nil { + t.Fatalf("resolve metadata: %v", err) + } + if descriptor == nil || descriptor.MCP == nil { + t.Fatalf("expected MCP descriptor, got %+v", descriptor) + } + registry, err := describe.BuildToolRegistry(map[string]*describe.ServiceDescriptor{"perplexity": descriptor}) + if err != nil { + t.Fatalf("BuildToolRegistry: %v", err) + } + if got := registry["perplexity"][0].MCP.Path; got != "/mcp" { + t.Fatalf("MCP path = %q, want /mcp", got) + } +} + func TestResolveServiceMetadataUsesDefaultImageDescriptorPathWhenLabelMissing(t *testing.T) { prevExists := imageExistsLocally prevInspect := inspectClawImage diff --git a/cmd/claw/image_lifecycle.go b/cmd/claw/image_lifecycle.go index 71a9546..69263e6 100644 --- a/cmd/claw/image_lifecycle.go +++ b/cmd/claw/image_lifecycle.go @@ -299,7 +299,7 @@ func requiredPodPullInfraSpecs(podDir string, p *pod.Pod, plans []plannedService hasManagedServices := false for _, svc := range p.Services { - if svc != nil && svc.Claw != nil { + if svc.IsAgentManaged() { hasManagedServices = true break } @@ -314,7 +314,7 @@ func requiredPodPullInfraSpecs(podDir string, p *pod.Pod, plans []plannedService needsConversationWall := false for _, plan := range plans { svc := p.Services[plan.ServiceName] - if svc == nil || svc.Claw == nil { + if !svc.IsAgentManaged() { continue } @@ -348,7 +348,7 @@ func requiredPodPullInfraSpecs(podDir string, p *pod.Pod, plans []plannedService } func serviceUsesProxyTypeForPull(podDir string, svc *pod.Service, plan plannedServiceImage, proxyType string) (bool, error) { - if svc == nil || svc.Claw == nil { + if !svc.IsAgentManaged() { return false, nil } if proxyListContains(svc.Claw.Cllama, proxyType) { diff --git a/cmd/claw/mcp_stdio_spike_test.go b/cmd/claw/mcp_stdio_spike_test.go new file mode 100644 index 0000000..7b01f43 --- /dev/null +++ b/cmd/claw/mcp_stdio_spike_test.go @@ -0,0 +1,232 @@ +//go:build spike + +package main + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "net/http" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "testing" + "time" +) + +func TestSpikeMCPStdio(t *testing.T) { + if err := exec.Command("docker", "info").Run(); err != nil { + t.Skipf("docker not available: %v", err) + } + + _, thisFile, _, ok := runtime.Caller(0) + if !ok { + t.Fatal("runtime.Caller failed") + } + repoRoot := filepath.Join(filepath.Dir(thisFile), "..", "..") + + wrapperTag := fmt.Sprintf("claw-spike-mcp-stdio:%d", time.Now().UnixNano()) + agentTag := fmt.Sprintf("claw-spike-mcp-agent:%d", time.Now().UnixNano()) + spikeBuildImage(t, repoRoot, wrapperTag, "dockerfiles/claw-mcp-stdio/Dockerfile") + spikeBuildImage(t, filepath.Join(repoRoot, "testdata", "openclaw-stub"), agentTag, "Clawfile") + t.Cleanup(func() { + exec.Command("docker", "image", "rm", "-f", wrapperTag, agentTag).CombinedOutput() + }) + + spikeEnsureRepoInfraImages(t, repoRoot, infraComponentClawdash) + spikeEnsureCllamaPassthroughImage(t, repoRoot) + + workDir := t.TempDir() + echoDir := filepath.Join(workDir, "echo-server") + if err := os.MkdirAll(echoDir, 0o755); err != nil { + t.Fatalf("mkdir echo-server: %v", err) + } + copyFile(t, filepath.Join(repoRoot, "examples", "mcp-stdio", "echo-server", "server.js"), filepath.Join(echoDir, "server.js")) + copyFile(t, filepath.Join(repoRoot, "examples", "mcp-stdio", "echo.claw-describe.json"), filepath.Join(workDir, "echo.claw-describe.json")) + if err := os.WriteFile(filepath.Join(workDir, "AGENTS.md"), []byte("# Agent\n\nUse the echo tool.\n"), 0o644); err != nil { + t.Fatalf("write AGENTS.md: %v", err) + } + + wrapperPort := spikeFreePort(t) + podPath := filepath.Join(workDir, "claw-pod.yml") + podYAML := fmt.Sprintf(`x-claw: + pod: mcp-stdio-spike + +services: + agent: + image: %s + x-claw: + agent: ./AGENTS.md + cllama: passthrough + cllama-env: + XAI_API_KEY: sk-spike-fake-not-real + tools: + - service: echo + allow: [echo] + + echo: + image: %s + volumes: + - ./echo-server:/srv/echo-server:ro + expose: + - "8080" + ports: + - "127.0.0.1:%s:8080" + x-claw: + describe-file: ./echo.claw-describe.json + mcp-stdio: + command: node + args: ["/srv/echo-server/server.js"] +`, agentTag, wrapperTag, wrapperPort) + if err := os.WriteFile(podPath, []byte(podYAML), 0o644); err != nil { + t.Fatalf("write pod: %v", err) + } + + prevDetach := composeUpDetach + composeUpDetach = true + defer func() { composeUpDetach = prevDetach }() + t.Setenv("CLLAMA_UI_PORT", spikeFreePort(t)) + t.Setenv("CLAWDASH_ADDR", ":"+spikeFreePort(t)) + + if err := runComposeUp(podPath); err != nil { + t.Fatalf("runComposeUp: %v", err) + } + + composePath := filepath.Join(workDir, "compose.generated.yml") + t.Cleanup(func() { + exec.Command("docker", "compose", "-f", composePath, "down", "--volumes", "--remove-orphans").CombinedOutput() + }) + + echoID := composeContainerID(t, composePath, "echo") + spikeWaitRunning(t, echoID, 30*time.Second) + spikeWaitHealthy(t, echoID, 60*time.Second) + + toolsPath := filepath.Join(workDir, ".claw-runtime", "context", "agent", "tools.json") + toolsRaw, err := os.ReadFile(toolsPath) + if err != nil { + t.Fatalf("read tools.json: %v", err) + } + var manifest struct { + Tools []struct { + Name string `json:"name"` + Execution struct { + Transport string `json:"transport"` + Path string `json:"path"` + ToolName string `json:"tool_name"` + } `json:"execution"` + } `json:"tools"` + } + if err := json.Unmarshal(toolsRaw, &manifest); err != nil { + t.Fatalf("parse tools.json: %v\n%s", err, toolsRaw) + } + if len(manifest.Tools) != 1 { + t.Fatalf("tools count = %d, want 1\n%s", len(manifest.Tools), toolsRaw) + } + tool := manifest.Tools[0] + if tool.Name != "echo.echo" || tool.Execution.Transport != "mcp" || tool.Execution.Path != "/mcp" || tool.Execution.ToolName != "echo" { + t.Fatalf("unexpected tool manifest: %+v\n%s", tool, toolsRaw) + } + + endpoint := "http://127.0.0.1:" + wrapperPort + "/mcp" + sessionID := mcpInitialize(t, endpoint) + mcpNotifyInitialized(t, endpoint, sessionID) + got := mcpCallEcho(t, endpoint, sessionID, "wrapped") + if got != "wrapped" { + t.Fatalf("echo result = %q, want wrapped", got) + } +} + +func copyFile(t *testing.T, src, dst string) { + t.Helper() + data, err := os.ReadFile(src) + if err != nil { + t.Fatalf("read %s: %v", src, err) + } + if err := os.WriteFile(dst, data, 0o644); err != nil { + t.Fatalf("write %s: %v", dst, err) + } +} + +func composeContainerID(t *testing.T, composePath, service string) string { + t.Helper() + out, err := exec.Command("docker", "compose", "-f", composePath, "ps", "-q", service).Output() + if err != nil { + t.Fatalf("docker compose ps %s: %v", service, err) + } + id := strings.TrimSpace(string(out)) + if id == "" { + t.Fatalf("no container id for %s", service) + } + return id +} + +func mcpInitialize(t *testing.T, endpoint string) string { + t.Helper() + req := []byte(`{"jsonrpc":"2.0","id":1,"method":"initialize","params":{"protocolVersion":"2025-11-25","capabilities":{},"clientInfo":{"name":"spike","version":"test"}}}`) + resp := mcpPost(t, endpoint, "", req, http.StatusOK) + defer resp.Body.Close() + sessionID := resp.Header.Get("MCP-Session-Id") + if sessionID == "" { + t.Fatal("initialize response missing MCP-Session-Id") + } + return sessionID +} + +func mcpNotifyInitialized(t *testing.T, endpoint, sessionID string) { + t.Helper() + mcpPost(t, endpoint, sessionID, []byte(`{"jsonrpc":"2.0","method":"notifications/initialized"}`), http.StatusAccepted) +} + +func mcpCallEcho(t *testing.T, endpoint, sessionID, message string) string { + t.Helper() + body := fmt.Sprintf(`{"jsonrpc":"2.0","id":2,"method":"tools/call","params":{"name":"echo","arguments":{"message":%q}}}`, message) + resp := mcpPost(t, endpoint, sessionID, []byte(body), http.StatusOK) + defer resp.Body.Close() + data, err := io.ReadAll(resp.Body) + if err != nil { + t.Fatal(err) + } + var parsed struct { + Result struct { + Content []struct { + Text string `json:"text"` + } `json:"content"` + } `json:"result"` + } + if err := json.Unmarshal(data, &parsed); err != nil { + t.Fatalf("parse MCP response: %v\n%s", err, data) + } + if len(parsed.Result.Content) == 0 { + t.Fatalf("MCP response missing content: %s", data) + } + return parsed.Result.Content[0].Text +} + +func mcpPost(t *testing.T, endpoint, sessionID string, body []byte, wantStatus int) *http.Response { + t.Helper() + req, err := http.NewRequest(http.MethodPost, endpoint, bytes.NewReader(body)) + if err != nil { + t.Fatal(err) + } + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Accept", "application/json, text/event-stream") + if sessionID != "" { + req.Header.Set("MCP-Session-Id", sessionID) + } + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + if resp.StatusCode != wantStatus { + data, _ := io.ReadAll(resp.Body) + resp.Body.Close() + t.Fatalf("MCP status = %d, want %d\n%s", resp.StatusCode, wantStatus, data) + } + if wantStatus == http.StatusAccepted { + resp.Body.Close() + } + return resp +} diff --git a/cmd/claw/skill_data/SKILL.md b/cmd/claw/skill_data/SKILL.md index bc5b20c..3b6f789 100644 --- a/cmd/claw/skill_data/SKILL.md +++ b/cmd/claw/skill_data/SKILL.md @@ -212,6 +212,18 @@ services: to: trading-floor environment: # standard compose — credentials go HERE DISCORD_BOT_TOKEN: "${DISCORD_BOT_TOKEN}" + + perplexity: + image: ghcr.io/mostlydev/claw-mcp-stdio:v0.12.0 + environment: + PERPLEXITY_API_KEY: "${PERPLEXITY_KEY}" + expose: + - "8080" + x-claw: + describe-file: ./perplexity.claw-describe.json + mcp-stdio: + command: npx + args: ["-y", "perplexity-mcp"] ``` ### Key rules @@ -221,12 +233,13 @@ services: - **`handles`**: Discord bot IDs, usernames, guilds. Clawdapus auto-generates native Discord `mentionPatterns`, `allowBots: true`, peer `users[]` allowlist. - **`surfaces`**: String form (`"channel://discord"`) = simple enable. Map form (`channel://discord: {dm: {...}}`) = routing config. - **`tools`**: Requires `cllama` on the consuming service. Services must publish tools via `claw.describe` descriptor v2. `allow: all` (implicit for scalar form) passes every tool; named lists are validated against the tool registry. +- **`mcp-stdio`**: Sidecar-only block for the shared `claw-mcp-stdio` wrapper. `command` is required, `args` is a list, and credentials stay in the sidecar's regular `environment:`. Pair with `describe-file` when the descriptor is supplied by the pod instead of baked into the image. - **`memory`**: Requires `cllama` on the consuming service. Target service must declare `memory` in its `claw.describe` descriptor v2. - **Pod defaults**: `*-defaults` at pod level are inherited by all services. Declaring the field at service level replaces the default. Use `...` spread token to extend list-type defaults (surfaces, feeds, skills, tools). Memory defaults are object-form (no spread — presence of `memory:` at service level replaces entirely; `memory: null` suppresses). ## Service Self-Description (claw.describe) -Services declare capabilities via a `.claw-describe.json` file (embedded in the image or discovered from Dockerfile labels). `claw up` extracts descriptors and compiles them into pod-global registries. +Services declare capabilities via a `.claw-describe.json` file (embedded in the image, discovered from Dockerfile labels, or supplied with service-level `x-claw.describe-file`). `claw up` extracts descriptors and compiles them into pod-global registries. ### Descriptor v2 diff --git a/dockerfiles/claw-mcp-stdio/Dockerfile b/dockerfiles/claw-mcp-stdio/Dockerfile new file mode 100644 index 0000000..a283919 --- /dev/null +++ b/dockerfiles/claw-mcp-stdio/Dockerfile @@ -0,0 +1,18 @@ +FROM golang:1.24-alpine AS build +WORKDIR /src +COPY go.mod go.sum ./ +RUN go mod download +COPY . . +RUN CGO_ENABLED=0 go build -o /claw-mcp-stdio ./cmd/claw-mcp-stdio + +FROM node:20-bookworm-slim +RUN apt-get update && apt-get install -y --no-install-recommends \ + ca-certificates tini python3 \ + && rm -rf /var/lib/apt/lists/* +ENV HOME=/tmp \ + NPM_CONFIG_CACHE=/tmp/.npm +COPY --from=build /claw-mcp-stdio /usr/local/bin/claw-mcp-stdio +EXPOSE 8080 +HEALTHCHECK --interval=15s --timeout=5s --retries=3 \ + CMD ["/usr/local/bin/claw-mcp-stdio", "-healthcheck"] +ENTRYPOINT ["tini", "--", "/usr/local/bin/claw-mcp-stdio"] diff --git a/docs/decisions/020-cllama-compiled-tool-mediation.md b/docs/decisions/020-cllama-compiled-tool-mediation.md index ff3e5cf..1dd1227 100644 --- a/docs/decisions/020-cllama-compiled-tool-mediation.md +++ b/docs/decisions/020-cllama-compiled-tool-mediation.md @@ -588,6 +588,12 @@ The capability-evolution wave (this ADR + ADR-021) landed together. Current stat - Managed-tool dispatch in cllama switches by transport (`http` → existing path, `mcp` → new MCP client) inside `executeManaged{OpenAI,Anthropic}Tool`; `tool_trace`, namespacing, audit, session-history, policy budgets, and credential-starvation boundaries are unchanged - Existing HTTP managed tools and descriptors continue to work without modification; descriptors without `mcp:` still require `tools[].http` +**Shipped (Phase 5b — stdio MCP wrapper, 2026-04-28, issue #179):** +- `claw-mcp-stdio` wraps stdio MCP commands behind the same Streamable HTTP `/mcp` surface that cllama v0.5.0 already mediates +- `x-claw.mcp-stdio` wires wrapper child command/args into the sidecar environment without turning the sidecar into an agent-managed runner +- `x-claw.describe-file` lets operators provide a deterministic host-side v2 descriptor snapshot for wrapper services +- No cllama transport changes are required; compiled `tools.json` still uses `execution.transport = "mcp"` + **Not yet shipped:** - Phase 2: native projection / runner-side MCP config generation - Phase 5 follow-up: `claw discover` (live `tools/list` snapshot into the build context) and baked `.claw-tools.json` artifact support diff --git a/docs/plans/2026-04-28-issue-179-stdio-mcp-wrapper.md b/docs/plans/2026-04-28-issue-179-stdio-mcp-wrapper.md new file mode 100644 index 0000000..728a846 --- /dev/null +++ b/docs/plans/2026-04-28-issue-179-stdio-mcp-wrapper.md @@ -0,0 +1,489 @@ +# Issue #179 — First-class stdio MCP wrapper for managed MCP sidecars + +**Issue:** https://github.com/mostlydev/clawdapus/issues/179 +**Related:** #177 (Streamable HTTP MCP transport landed in v0.11.0 / cllama v0.5.0) +**Workflow:** Claude drafts → Codex reviews + implements → Claude tests + releases. +**Target release:** clawdapus v0.12.0 (cllama is **not** touched in this scope). + +## 1. Problem and goal + +After #177, agents can call MCP tools through cllama as long as the MCP server +speaks **Streamable HTTP** at a pod-internal endpoint. Many useful MCP servers +(perplexity-mcp, filesystem, sqlite, the npx-installable ecosystem) are +distributed as **stdio commands**. Today, dropping one into a Clawdapus pod +requires hand-rolling a stdio→HTTP adapter per pod. + +**Goal:** make stdio MCP wrapping first-class. Operators declare a stdio +command and Clawdapus mediates it through the existing v0.11.0 cllama MCP +transport — no bespoke per-pod glue. + +**Non-goal in this scope:** +- Live `tools/list` discovery during `claw up` (stays compile-time hermetic + per ADR-020). Operators provide a baked descriptor. +- Touching cllama. The wrapper exposes the *same* Streamable HTTP MCP surface + cllama already speaks; cllama needs zero changes. +- Any change to the existing `tools[].http` HTTP managed-tool path. + +## 2. Operator surface (target shape) + +```yaml +services: + perplexity: + image: ghcr.io/mostlydev/claw-mcp-stdio:v0.12.0 # the new shared wrapper image + environment: + PERPLEXITY_API_KEY: ${PERPLEXITY_KEY} # stdio creds stay here + volumes: + - ./perplexity.claw-describe.json:/.claw-describe.json:ro + x-claw: + mcp-stdio: + command: npx + args: ["-y", "perplexity-mcp"] + + allen: + image: ghcr.io/mostlydev/hermes-base:v2026.3.17 + x-claw: + agent: allen + cllama: [openai] + tools: + - service: perplexity + allow: [search] +``` + +Why this shape: + +- **One shared wrapper image** (`ghcr.io/mostlydev/claw-mcp-stdio`), versioned + in lockstep with clawdapus releases (same `DefaultClawInfraTag`). No + per-package image to maintain. +- **`x-claw.mcp-stdio` is the *only* new pod surface** — a pure declarative + block that names the child command + args. `claw up` translates it into + env vars on the wrapper container. Nothing else in the pod parser changes. +- **The descriptor is operator-supplied via volume mount or COPY**, exactly + like any other v2 `claw.describe`. `mcp.transport: streamable_http` + + `path: /mcp` already work in v0.11.0; the wrapper image ships zero + descriptor by default so operators must supply tools[]. This makes + `claw up` deterministic — there's no live tools/list probe. +- **No new agent-side syntax.** Subscribing agents use the existing + `x-claw.tools: [{ service, allow }]` block from #177. +- **Stdio creds stay in the wrapper service env.** Agent containers never + see them. + +Operators write *one* image ref + *two* fields (`command`, `args`) + their +own descriptor file. That's the entire delta vs a normal MCP HTTP sidecar. + +## 3. Architecture: the `claw-mcp-stdio` wrapper image + +### 3.1 Composition + +A small Go binary (`cmd/claw-mcp-stdio/`) packaged in a Node-base image so +`npx`-style commands work out of the box. Single static binary as ENTRYPOINT. + +```dockerfile +FROM golang:1.24-alpine AS build +WORKDIR /src +COPY go.mod go.sum ./ +RUN go mod download +COPY . . +RUN CGO_ENABLED=0 go build -o /claw-mcp-stdio ./cmd/claw-mcp-stdio + +FROM node:20-bookworm-slim +RUN apt-get update && apt-get install -y --no-install-recommends \ + ca-certificates tini python3 \ + && rm -rf /var/lib/apt/lists/* +COPY --from=build /claw-mcp-stdio /usr/local/bin/claw-mcp-stdio +EXPOSE 8080 +HEALTHCHECK --interval=15s --timeout=5s --retries=3 \ + CMD ["/usr/local/bin/claw-mcp-stdio", "-healthcheck"] +ENTRYPOINT ["tini", "--", "/usr/local/bin/claw-mcp-stdio"] +``` + +**Base choice rationale:** `node:20-bookworm-slim` (~150 MB) covers the +dominant MCP distribution channel (npm + npx). `python3` is added for the +small but real subset of stdio MCP servers shipped via `uvx` / `python -m`. +Operators who need a heavier base build their own image; the shared image +covers the common case. + +### 3.2 Configuration (env-driven) + +Read at process startup: + +| Env var | Default | Meaning | +|--------------------------------------|-----------------|---------| +| `CLAW_MCP_STDIO_COMMAND` | (required) | Executable to spawn (e.g. `npx`). | +| `CLAW_MCP_STDIO_ARGS` | `[]` | JSON-encoded array of args. | +| `CLAW_MCP_STDIO_PORT` | `8080` | HTTP listen port. | +| `CLAW_MCP_STDIO_PATH` | `/mcp` | HTTP path that exposes the MCP endpoint. | +| `CLAW_MCP_STDIO_READY_TIMEOUT_MS` | `30000` | How long to wait for child `initialize` reply before failing readiness. | +| `CLAW_MCP_STDIO_RESTART_BACKOFF_MS` | `1000` | Initial restart backoff after child exit. | +| `CLAW_MCP_STDIO_RESTART_MAX_MS` | `15000` | Max restart backoff cap. | +| `CLAW_MCP_STDIO_AUTH_TOKEN` | (empty = open) | If set, requires `Authorization: Bearer ` on requests. | +| `CLAW_MCP_STDIO_REQUEST_TIMEOUT_MS` | `60000` | Per-request roundtrip timeout. | + +Child process inherits the wrapper container's full env (so +`PERPLEXITY_API_KEY` etc. flow through). + +### 3.3 Lifecycle + +**Single shared child.** One stdio MCP process. The wrapper multiplexes HTTP +JSON-RPC requests by their `id` field — every server in the wild already +handles concurrent IDs since this is how MCP works. + +``` + ┌────────────────────────┐ + cllama ──HTTP/JSON-RPC──▶ wrapper │ spawn(command, args) │ + │ ↕ stdin/stdout NDJSON │ + │ stdio MCP child │ + └────────────────────────┘ +``` + +Startup sequence: + +1. Read env, parse args. +2. `exec.Command(...)` with stdin/stdout pipes; stderr → wrapper stdout + (so `claw logs ` shows everything). +3. Start a stdout reader goroutine: parse newline-delimited JSON-RPC frames, + route by `id` to a pending-request map. +4. Send `initialize` (forwarded later from the HTTP side, or pre-warmed — + see §3.4) and wait for the initialized-ack. +5. Listen on `:PORT`, accept `POST /mcp` requests. + +If the child exits, restart it with exponential backoff up to +`CLAW_MCP_STDIO_RESTART_MAX_MS`. Pending requests during a restart get a +JSON-RPC error response (`code: -32000`, `message: "stdio child restarting"`) +so cllama returns a clean tool-error envelope instead of timing out. + +### 3.4 HTTP ↔ stdio bridge + +For each `POST /mcp`: + +1. Authenticate (bearer if `CLAW_MCP_STDIO_AUTH_TOKEN` set). +2. Read the JSON-RPC request body. Capture its `id`. +3. Create a pending-channel keyed on `id`. +4. Write the request as one newline-terminated JSON line to child stdin. +5. Wait on the channel (with `CLAW_MCP_STDIO_REQUEST_TIMEOUT_MS` deadline). +6. Write the response back as `application/json`. (No SSE in v1; cllama's + client already accepts JSON — see `cllama/internal/mcp/client.go`.) + +**Session handling.** The cllama client already manages MCP session IDs and +sends `Mcp-Session-Id` headers; the wrapper just needs to **mint and echo** +session IDs on `initialize`, store them in memory, and validate them on +subsequent calls. A single shared child means all sessions share the same +tool universe — that's correct, since the descriptor is one snapshot of one +MCP server. Session IDs are still useful for the cllama-side retry-on-expiry +behavior (forces a re-initialize when the wrapper restarts). + +For the v1 implementation a single in-memory session map is fine. Wrapper +restart drops the map — cllama's "retry once on session expiry" path +(`cllama/internal/mcp/client.go`) already handles this exact case from the +#177 work. + +### 3.5 Health and readiness + +`-healthcheck` flag returns 0 if a separate companion HTTP probe at +`GET /healthz` returns 200. Readiness flips green only after the wrapper has +successfully completed one round-trip with the child (lazy, on first +incoming request) **or** after a configurable warmup `initialize` succeeds. +Default: lazy — keeps cold start cheap and avoids racing npx package fetch. + +## 4. Pod parser additions (`internal/pod`) + +Minimal additions, all isolated to the service-level x-claw block. + +### 4.1 New types + +```go +// internal/pod/types.go +type ClawBlock struct { + // ...existing fields... + MCPStdio *MCPStdioBlock // nil unless x-claw.mcp-stdio is declared +} + +type MCPStdioBlock struct { + Command string + Args []string +} +``` + +### 4.2 Raw parser extension + +```go +// internal/pod/parser.go +type rawClawBlock struct { + // ...existing fields... + MCPStdio *rawMCPStdio `yaml:"mcp-stdio"` +} + +type rawMCPStdio struct { + Command string `yaml:"command"` + Args []string `yaml:"args"` +} +``` + +In the existing service-loop (around `parser.go:185+`), parse and validate: + +- `command` is required, must be non-empty after trim. +- `args` defaults to `[]`. +- Reject `mcp-stdio` if the service ALSO declares `x-claw.agent` (these are + agent-runner directives — mcp-stdio services are tool sidecars, not + agents). This prevents accidental conflation. +- Reject `mcp-stdio` if `x-claw.cllama` is set (same reason). +- `count > 1` is rejected for mcp-stdio services in v1 (single shared child + per service; horizontal scaling of stdio MCP is out of scope and has + ambiguous semantics). + +### 4.3 Tests to add (`internal/pod/parser_mcp_stdio_test.go`) + +- happy-path: `command + args` parses into `Service.Claw.MCPStdio`. +- empty `command` → error. +- mcp-stdio + agent → error. +- mcp-stdio + cllama → error. +- mcp-stdio + count > 1 → error. + +## 5. Compose emission (`internal/pod/compose_emit.go`) + +When `Service.Claw.MCPStdio != nil`, inject env vars on the compose service +output: + +- `CLAW_MCP_STDIO_COMMAND=` +- `CLAW_MCP_STDIO_ARGS=` + +Implementation hook: existing emission already merges `Service.Environment` +into the compose `environment:` map. Add a small helper +`mcpStdioEnv(*MCPStdioBlock) map[string]string` and merge it after user env +(so user env overrides if they really want to). + +**Don't** force the wrapper image. Operators choose their image — they may +extend the shared base with extra deps (e.g. their own corp CA). The pod +parser is concerned with *behavior* (env wiring), not *image identity*. + +### 5.1 Tests + +`internal/pod/compose_emit_mcp_stdio_test.go`: + +- mcp-stdio block → emitted env contains `CLAW_MCP_STDIO_COMMAND` and + `CLAW_MCP_STDIO_ARGS` JSON-encoded. +- args with shell metacharacters round-trip safely (JSON-encoded, not + shell-interpolated). +- user-provided `CLAW_MCP_STDIO_*` env wins (last write wins), so escape + hatches stay open. + +## 6. Compile path (already done in v0.11.0 — verify no regression) + +No changes needed in `cmd/claw/compose_up.go` for the manifest path. The +existing `buildToolManifestEntries` already emits +`execution.transport = "mcp"` when a `claw.describe` v2 descriptor declares +`mcp:` and tools without `http:`. The wrapper service's mounted +`.claw-describe.json` flows through `loadDescriptorFromImage` (image label +flow) or `loadDescriptorFromBuildCtx` (build-context flow) — both work +unchanged. + +**Verify (no code change):** the descriptor extraction path picks up the +mounted `/.claw-describe.json` at runtime. Today, `LoadFromImage` reads +labels + image fs. A bind-mounted descriptor file is *runtime*, not image +metadata, so `claw up` won't see it through the image-inspect path. + +**This is the one design subtlety**: how does `claw up` learn the wrapper +service's descriptor? + +Three options, in order of preference: + +a) **Explicit `x-claw.describe-file` pod field** pointing at a host file: + ```yaml + x-claw: + mcp-stdio: { command: npx, args: ["-y", "perplexity-mcp"] } + describe-file: ./perplexity.claw-describe.json + ``` + `claw up` reads it during descriptor collection. Build-context-style + resolution. Simple, explicit, no Docker plumbing needed. + +b) **Convention: descriptor co-located with pod** — look for + `/.claw-describe..json` automatically. Magic, but + zero new pod surface. + +c) **Bake into a per-package wrapper image** — operators publish their own + image extending `claw-mcp-stdio` with the descriptor baked in via COPY. + Heavyweight, defeats the "just drop in" goal. + +**Recommendation: (a).** Add `x-claw.describe-file: ` to the +service block. In `cmd/claw/compose_up.go:resolveServiceMetadata`, before +falling through to image/build-context inspection, check for the +service-level describe-file and `LoadFromBuildContext`-equivalent it. This +keeps it deterministic, visible in pod YAML, and reuses existing descriptor +extraction logic. + +This is a small extension to `internal/pod` (one new field, one parser +line) and one new code path in `compose_up.go`. + +## 7. Hermetic spike test + +### 7.1 Test fixture: `mcp-echo-stdio` + +Tiny stdio MCP server in `examples/mcp-stdio/echo-server/server.js` (or +`.py` — whichever is shorter). ~50 lines. Implements: + +- `initialize` → returns minimal capabilities, echoes protocol version. +- `tools/list` → returns one tool `echo` with `{message: string}` schema. +- `tools/call` `echo` → returns content blocks with the input message. + +No network, no external deps beyond the runtime present in the wrapper +base image. This means it can run in `TestSpikeMCPStdio` without secrets. + +### 7.2 Spike test: `cmd/claw/mcp_stdio_spike_test.go` + +`go test -tags spike -run TestSpikeMCPStdio ./cmd/claw/...` + +Mirrors `TestSpikeRollCall`'s structure: + +1. `claw build && claw up -d` on `examples/mcp-stdio/`. +2. Trigger the agent to make one tool call. +3. Assert the cllama session-history line for that turn carries + `tool_trace[].transport == "mcp"`. +4. Assert `claw audit` JSON shows the tool call attributed to the right + agent. +5. `claw down`. + +This proves end-to-end: stdio child → wrapper HTTP → cllama MCP client → +agent → tool call → audit trail. If this passes, perplexity-mcp will work. + +### 7.3 Manual real-world example + +`examples/perplexity-stdio/claw-pod.yml` with a comment block at the top +documenting the `PERPLEXITY_KEY` env var. Not run in CI, but a copy-pasteable +template for operators. + +## 8. Documentation + +- **`site/guide/tools.md` (or new `site/guide/mcp-stdio.md`):** add a + "Wrapping a stdio MCP server" section. Show the perplexity example and + the descriptor file shape. +- **`site/guide/cli.md`:** mention the new `mcp-stdio` x-claw field in the + pod-yml reference. +- **`skills/clawdapus/SKILL.md`** (and regenerated mirror at + `cmd/claw/skill_data/SKILL.md`): document `x-claw.mcp-stdio` + the + `claw-mcp-stdio` image so agents using the skill can guide operators. +- **`README.md`:** one-line callout in the feature list. +- **`docs/decisions/020-cllama-compiled-tool-mediation.md`:** add a Phase 5b + status section noting stdio wrapper added in v0.12.0 (issue #179). + +## 9. Release impact + +This is a **minor** release (0.12.0): + +- New shared infra image `ghcr.io/mostlydev/claw-mcp-stdio` versioned at + `DefaultClawInfraTag`. Add to `internal/infraimages/release_manifest.go`: + ```go + DefaultClawMCPStdioTag = DefaultClawInfraTag + // ReleaseRefs: + fmt.Sprintf("ghcr.io/mostlydev/claw-mcp-stdio:%s", releaseTag), + ``` +- New CI workflow `.github/workflows/claw-mcp-stdio-image.yml` mirroring + `claw-wall-image.yml` (same trigger pattern, same labels, same buildx + matrix). +- Update `.claude/skills/clawdapus-release/SKILL.md` with the new image in + Step 10 (prepublish list) and Step 12 (verify). +- New CLI surface change → bump SKILL.md as in Step 6 of the release skill. + +cllama is **not** touched. The submodule pointer doesn't move. + +## 10. Build sequence (for codex) + +Recommended commit/PR slicing — small, reviewable steps. Each step ends +green. + +**Step 1 — wrapper binary + image** (`cmd/claw-mcp-stdio/`, +`dockerfiles/claw-mcp-stdio/Dockerfile`, `.github/workflows/claw-mcp-stdio-image.yml`): +- `cmd/claw-mcp-stdio/main.go` — env parsing, child spawn, HTTP server, + request multiplexer, restart loop. +- Unit tests for the multiplexer (no Docker needed): feed mock stdio frames + in, drive HTTP requests, assert correct response routing. +- `Dockerfile` per §3.1. +- CI workflow. +- `internal/infraimages/release_manifest.go` adds the new ref. + +**Step 2 — pod parser** (`internal/pod/types.go`, `internal/pod/parser.go`, +`internal/pod/parser_mcp_stdio_test.go`): +- Add `MCPStdio` field, raw type, parse + validate per §4. + +**Step 3 — describe-file plumbing** (`internal/pod/types.go`, +`internal/pod/parser.go`, `cmd/claw/compose_up.go:resolveServiceMetadata`): +- Add `x-claw.describe-file` parsing (one new string field on `ClawBlock`). +- In `resolveServiceMetadata`, if describe-file is set, resolve the path + relative to `podDir` and load via existing descriptor parser. Falls + through to current image/build-context path when unset. +- Tests in `cmd/claw/compose_up_descriptor_test.go` (or wherever the + current descriptor tests live). + +**Step 4 — compose emission** (`internal/pod/compose_emit.go`, +`internal/pod/compose_emit_mcp_stdio_test.go`): +- Inject `CLAW_MCP_STDIO_COMMAND` + `CLAW_MCP_STDIO_ARGS` env per §5. + +**Step 5 — example + spike** (`examples/mcp-stdio/`, +`cmd/claw/mcp_stdio_spike_test.go`): +- Build the echo fixture, write the pod YAML + descriptor. +- Add the spike test. + +**Step 6 — docs sweep** (per §8) — last so it reflects what actually shipped. + +## 11. Testing matrix Codex must run before handoff + +```bash +unset GOROOT # if mise/homebrew Go drift bites again +go vet ./... +go test ./... +go test -tags integration ./... +go test -tags spike -run TestSpikeMCPStdio ./cmd/claw/... +# regression for #177: +go test -tags spike -run TestSpikeRollCall ./cmd/claw/... +``` + +Also `docker buildx build` the wrapper image locally (single arch is fine +for the dev loop) and run the spike against it. + +## 12. Open questions for codex review + +1. **Wrapper auth in v1.** Default is open (no token). Should we instead + default to "auto-mint a bearer at `claw up` time" (analogous to + `claw-api: self`) so even pod-internal traffic is authenticated? Argues + for: defense in depth, parity with claw-api. Argues against: extra + compile-time wiring just to gate a localhost-only call. Recommendation: + ship v1 with optional bearer (env-driven), revisit if a real attacker + model justifies it. + +2. **Single shared child vs per-session child.** §3.3 picks single shared. + Are there real-world MCP servers (perplexity-mcp included) that require + per-session isolation? Quick survey before committing. + +3. **`x-claw.describe-file` location: service-level or pod-level?** Plan + has it on the service block. Could also be a pod-level map keyed by + service name. Service-level is more local but adds one more service-block + field; the trade-off is mild. + +4. **Should the wrapper bundle Python alongside Node?** Adds ~30 MB to the + image but covers the `uvx`/`python -m` MCP servers without operators + needing a separate base. Plan says yes — challenge if you disagree. + +5. **Single image or matrix?** Should we publish `claw-mcp-stdio:node-only` + and `claw-mcp-stdio:full` (with Python)? v1 plan: one image, full. Easy + to split later; hard to merge if we start split. + +## 13. Acceptance checklist (against issue #179) + +- [ ] A pod can wrap a stdio MCP server and expose it at a pod-internal + Streamable HTTP `/mcp` endpoint. *(§3, §4, §5)* +- [ ] A `claw.describe` v2 descriptor with top-level + `mcp: { transport: "streamable_http", path: "/mcp" }` can point at + the wrapper service. *(unchanged from v0.11.0; verified in §6 + §7.2 + assertions)* +- [ ] `claw up` compiles the allowed tools into `tools.json` with + `execution.transport = "mcp"`. *(unchanged from v0.11.0; verified + via spike)* +- [ ] cllama can call the wrapped stdio tool through the existing MCP + transport path. *(spike test §7.2)* +- [ ] Sample/fixture demonstrates an npm stdio MCP package. *(echo fixture + in §7.1; perplexity example in §7.3)* + +## 14. Out of scope (file as follow-ups if pressure builds) + +- `claw discover` to snapshot live `tools/list` into a baked descriptor. +- Per-session child isolation. +- Header-style auth on the cllama MCP client side. +- Wrapper packaging beyond `node + python3` (e.g. Ruby/Elixir MCP servers). diff --git a/examples/mcp-stdio/AGENTS.md b/examples/mcp-stdio/AGENTS.md new file mode 100644 index 0000000..6d9df28 --- /dev/null +++ b/examples/mcp-stdio/AGENTS.md @@ -0,0 +1,4 @@ +# MCP Stdio Example Agent + +Use the managed `echo.echo` tool when asked to echo text. Do not call the echo +service directly. diff --git a/examples/mcp-stdio/Clawfile b/examples/mcp-stdio/Clawfile new file mode 100644 index 0000000..10a07fc --- /dev/null +++ b/examples/mcp-stdio/Clawfile @@ -0,0 +1,6 @@ +FROM openclaw:latest + +CLAW_TYPE openclaw +AGENT AGENTS.md +MODEL primary openrouter/openai/gpt-4o-mini +CLLAMA passthrough diff --git a/examples/mcp-stdio/claw-pod.yml b/examples/mcp-stdio/claw-pod.yml new file mode 100644 index 0000000..30feb45 --- /dev/null +++ b/examples/mcp-stdio/claw-pod.yml @@ -0,0 +1,28 @@ +x-claw: + pod: mcp-stdio-example + +services: + assistant: + build: + context: . + dockerfile: Clawfile + x-claw: + agent: ./AGENTS.md + cllama: passthrough + cllama-env: + OPENROUTER_API_KEY: ${OPENROUTER_API_KEY} + tools: + - service: echo + allow: [echo] + + echo: + image: ghcr.io/mostlydev/claw-mcp-stdio:v0.12.0 + volumes: + - ./echo-server:/srv/echo-server:ro + expose: + - "8080" + x-claw: + describe-file: ./echo.claw-describe.json + mcp-stdio: + command: node + args: ["/srv/echo-server/server.js"] diff --git a/examples/mcp-stdio/echo-server/server.js b/examples/mcp-stdio/echo-server/server.js new file mode 100644 index 0000000..a659144 --- /dev/null +++ b/examples/mcp-stdio/echo-server/server.js @@ -0,0 +1,75 @@ +#!/usr/bin/env node + +const readline = require("node:readline"); + +const rl = readline.createInterface({ + input: process.stdin, + crlfDelay: Infinity, +}); + +function send(message) { + process.stdout.write(`${JSON.stringify(message)}\n`); +} + +rl.on("line", (line) => { + if (!line.trim()) return; + let req; + try { + req = JSON.parse(line); + } catch { + return; + } + if (req.id === undefined || req.id === null) return; + + if (req.method === "initialize") { + send({ + jsonrpc: "2.0", + id: req.id, + result: { + protocolVersion: req.params?.protocolVersion || "2025-11-25", + serverInfo: { name: "mcp-echo-stdio", version: "0.1.0" }, + capabilities: { tools: {} }, + }, + }); + return; + } + + if (req.method === "tools/list") { + send({ + jsonrpc: "2.0", + id: req.id, + result: { + tools: [ + { + name: "echo", + description: "Echo a message back as MCP text content.", + inputSchema: { + type: "object", + properties: { message: { type: "string" } }, + required: ["message"], + }, + annotations: { readOnly: true }, + }, + ], + }, + }); + return; + } + + if (req.method === "tools/call" && req.params?.name === "echo") { + send({ + jsonrpc: "2.0", + id: req.id, + result: { + content: [{ type: "text", text: String(req.params?.arguments?.message || "") }], + }, + }); + return; + } + + send({ + jsonrpc: "2.0", + id: req.id, + error: { code: -32601, message: `unknown method ${req.method}` }, + }); +}); diff --git a/examples/mcp-stdio/echo.claw-describe.json b/examples/mcp-stdio/echo.claw-describe.json new file mode 100644 index 0000000..91170b6 --- /dev/null +++ b/examples/mcp-stdio/echo.claw-describe.json @@ -0,0 +1,28 @@ +{ + "version": 2, + "description": "Echo MCP stdio server wrapped by claw-mcp-stdio.", + "mcp": { + "transport": "streamable_http", + "path": "/mcp" + }, + "tools": [ + { + "name": "echo", + "description": "Echo a message back as text content.", + "inputSchema": { + "type": "object", + "properties": { + "message": { + "type": "string" + } + }, + "required": [ + "message" + ] + }, + "annotations": { + "readOnly": true + } + } + ] +} diff --git a/examples/perplexity-stdio/AGENTS.md b/examples/perplexity-stdio/AGENTS.md new file mode 100644 index 0000000..e05876c --- /dev/null +++ b/examples/perplexity-stdio/AGENTS.md @@ -0,0 +1,4 @@ +# Perplexity Stdio Example Agent + +Use the managed `perplexity.search` tool for fresh web search. Do not ask for +or expose the Perplexity API key. diff --git a/examples/perplexity-stdio/claw-pod.yml b/examples/perplexity-stdio/claw-pod.yml new file mode 100644 index 0000000..063769d --- /dev/null +++ b/examples/perplexity-stdio/claw-pod.yml @@ -0,0 +1,28 @@ +# Set PERPLEXITY_KEY in your shell or .env before running `claw up -d`. +# The API key stays on the stdio wrapper sidecar; agent containers only see the +# managed `perplexity.search` tool through cllama. + +x-claw: + pod: perplexity-stdio-example + +services: + perplexity: + image: ghcr.io/mostlydev/claw-mcp-stdio:v0.12.0 + environment: + PERPLEXITY_API_KEY: ${PERPLEXITY_KEY} + expose: + - "8080" + x-claw: + describe-file: ./perplexity.claw-describe.json + mcp-stdio: + command: npx + args: ["-y", "perplexity-mcp"] + + allen: + image: ghcr.io/mostlydev/hermes-base:v2026.3.17 + x-claw: + agent: ./AGENTS.md + cllama: passthrough + tools: + - service: perplexity + allow: [search] diff --git a/examples/perplexity-stdio/perplexity.claw-describe.json b/examples/perplexity-stdio/perplexity.claw-describe.json new file mode 100644 index 0000000..37154fc --- /dev/null +++ b/examples/perplexity-stdio/perplexity.claw-describe.json @@ -0,0 +1,28 @@ +{ + "version": 2, + "description": "Perplexity MCP search exposed through the shared stdio wrapper.", + "mcp": { + "transport": "streamable_http", + "path": "/mcp" + }, + "tools": [ + { + "name": "search", + "description": "Search the web with Perplexity.", + "inputSchema": { + "type": "object", + "properties": { + "query": { + "type": "string" + } + }, + "required": [ + "query" + ] + }, + "annotations": { + "readOnly": true + } + } + ] +} diff --git a/internal/infraimages/release_manifest.go b/internal/infraimages/release_manifest.go index 3aec080..41723e8 100644 --- a/internal/infraimages/release_manifest.go +++ b/internal/infraimages/release_manifest.go @@ -3,12 +3,13 @@ package infraimages import "fmt" const ( - DefaultClawInfraTag = "v0.11.0" - DefaultClawAPITag = DefaultClawInfraTag - DefaultClawdashTag = DefaultClawInfraTag - DefaultClawWallTag = DefaultClawInfraTag - DefaultCllamaTag = "v0.5.0" - DefaultHermesBaseTag = "v2026.3.17" + DefaultClawInfraTag = "v0.11.0" + DefaultClawAPITag = DefaultClawInfraTag + DefaultClawdashTag = DefaultClawInfraTag + DefaultClawWallTag = DefaultClawInfraTag + DefaultClawMCPStdioTag = DefaultClawInfraTag + DefaultCllamaTag = "v0.5.0" + DefaultHermesBaseTag = "v2026.3.17" ) func ReleaseRefs(releaseTag string) []string { @@ -16,6 +17,7 @@ func ReleaseRefs(releaseTag string) []string { fmt.Sprintf("ghcr.io/mostlydev/claw-api:%s", releaseTag), fmt.Sprintf("ghcr.io/mostlydev/clawdash:%s", releaseTag), fmt.Sprintf("ghcr.io/mostlydev/claw-wall:%s", releaseTag), + fmt.Sprintf("ghcr.io/mostlydev/claw-mcp-stdio:%s", releaseTag), fmt.Sprintf("ghcr.io/mostlydev/cllama:%s", DefaultCllamaTag), fmt.Sprintf("ghcr.io/mostlydev/hermes-base:%s", DefaultHermesBaseTag), } diff --git a/internal/infraimages/release_manifest_test.go b/internal/infraimages/release_manifest_test.go index ea71590..be04d7f 100644 --- a/internal/infraimages/release_manifest_test.go +++ b/internal/infraimages/release_manifest_test.go @@ -8,6 +8,7 @@ func TestReleaseRefs(t *testing.T) { "ghcr.io/mostlydev/claw-api:v1.2.3", "ghcr.io/mostlydev/clawdash:v1.2.3", "ghcr.io/mostlydev/claw-wall:v1.2.3", + "ghcr.io/mostlydev/claw-mcp-stdio:v1.2.3", "ghcr.io/mostlydev/cllama:" + DefaultCllamaTag, "ghcr.io/mostlydev/hermes-base:" + DefaultHermesBaseTag, } diff --git a/internal/pod/compose_emit.go b/internal/pod/compose_emit.go index ec6fb76..b00c8f3 100644 --- a/internal/pod/compose_emit.go +++ b/internal/pod/compose_emit.go @@ -86,7 +86,11 @@ func EmitCompose(p *Pod, results map[string]*driver.MaterializeResult, proxies . explicitResult := result != nil if result == nil { // Fail-closed defaults apply only to Claw-managed services. - if isClaw { + if svc.IsMCPStdioSidecar() { + result = &driver.MaterializeResult{ + Restart: "on-failure", + } + } else if isClaw { result = &driver.MaterializeResult{ ReadOnly: true, Restart: "on-failure", @@ -249,6 +253,18 @@ func EmitCompose(p *Pod, results map[string]*driver.MaterializeResult, proxies . } serviceEnv[key] = value } + if svc.IsMCPStdioSidecar() { + mcpEnv, err := mcpStdioEnv(svc.Claw.MCPStdio) + if err != nil { + return "", fmt.Errorf("service %q: mcp-stdio environment: %w", serviceName, err) + } + for key, value := range mcpEnv { + if _, exists := baseEnv[key]; exists { + continue + } + serviceEnv[key] = value + } + } // Environment: preserved compose env (lowest) < handle envs < service env fallback < driver env (highest). env, err := mergedEnvironment(serviceOut["environment"], handleEnvs, serviceEnv, result.Environment) @@ -266,7 +282,7 @@ func EmitCompose(p *Pod, results map[string]*driver.MaterializeResult, proxies . serviceOut["environment"] = env } - if isClaw || explicitResult { + if explicitResult || (isClaw && !svc.IsMCPStdioSidecar()) { serviceOut["read_only"] = result.ReadOnly } if result.Restart != "" { @@ -504,6 +520,20 @@ func sortedServiceNames(services map[string]*Service) []string { return names } +func mcpStdioEnv(block *MCPStdioBlock) (map[string]string, error) { + if block == nil { + return nil, nil + } + args, err := json.Marshal(block.Args) + if err != nil { + return nil, err + } + return map[string]string{ + "CLAW_MCP_STDIO_COMMAND": block.Command, + "CLAW_MCP_STDIO_ARGS": string(args), + }, nil +} + // computeHandleEnvs collects handles from all claw services and builds the // pod-wide CLAW_HANDLE___* env var map. func computeHandleEnvs(services map[string]*Service) map[string]string { diff --git a/internal/pod/compose_emit_mcp_stdio_test.go b/internal/pod/compose_emit_mcp_stdio_test.go new file mode 100644 index 0000000..f087cdf --- /dev/null +++ b/internal/pod/compose_emit_mcp_stdio_test.go @@ -0,0 +1,97 @@ +package pod + +import ( + "strings" + "testing" + + "gopkg.in/yaml.v3" +) + +func TestEmitComposeMCPStdioEnv(t *testing.T) { + p, err := Parse(strings.NewReader(` +x-claw: + pod: mcp-stdio-test +services: + search: + image: ghcr.io/mostlydev/claw-mcp-stdio:v0.12.0 + x-claw: + mcp-stdio: + command: npx + args: ["-y", "perplexity-mcp", "--flag=value with spaces"] +`)) + if err != nil { + t.Fatalf("Parse: %v", err) + } + + out, err := EmitCompose(p, nil) + if err != nil { + t.Fatalf("EmitCompose: %v", err) + } + + var cf struct { + Services map[string]struct { + Environment map[string]string `yaml:"environment"` + Networks []string `yaml:"networks"` + Restart string `yaml:"restart"` + ReadOnly *bool `yaml:"read_only"` + } `yaml:"services"` + } + if err := yaml.Unmarshal([]byte(out), &cf); err != nil { + t.Fatalf("unmarshal compose: %v", err) + } + + search := cf.Services["search"] + if search.Environment["CLAW_MCP_STDIO_COMMAND"] != "npx" { + t.Fatalf("CLAW_MCP_STDIO_COMMAND = %q", search.Environment["CLAW_MCP_STDIO_COMMAND"]) + } + if search.Environment["CLAW_MCP_STDIO_ARGS"] != `["-y","perplexity-mcp","--flag=value with spaces"]` { + t.Fatalf("CLAW_MCP_STDIO_ARGS = %q", search.Environment["CLAW_MCP_STDIO_ARGS"]) + } + if search.Restart != "on-failure" { + t.Fatalf("restart = %q, want on-failure", search.Restart) + } + if search.ReadOnly != nil { + t.Fatalf("mcp-stdio sidecar should preserve writable default rootfs, read_only=%v", *search.ReadOnly) + } + if len(search.Networks) != 1 || search.Networks[0] != "claw-internal" { + t.Fatalf("networks = %v, want claw-internal", search.Networks) + } +} + +func TestEmitComposeMCPStdioUserEnvWins(t *testing.T) { + p, err := Parse(strings.NewReader(` +services: + search: + image: wrapper:latest + environment: + CLAW_MCP_STDIO_COMMAND: custom + CLAW_MCP_STDIO_ARGS: '["manual"]' + x-claw: + mcp-stdio: + command: npx + args: ["-y", "perplexity-mcp"] +`)) + if err != nil { + t.Fatalf("Parse: %v", err) + } + + out, err := EmitCompose(p, nil) + if err != nil { + t.Fatalf("EmitCompose: %v", err) + } + var cf struct { + Services map[string]struct { + Environment map[string]string `yaml:"environment"` + } `yaml:"services"` + } + if err := yaml.Unmarshal([]byte(out), &cf); err != nil { + t.Fatalf("unmarshal compose: %v", err) + } + env := cf.Services["search"].Environment + if env["CLAW_MCP_STDIO_COMMAND"] != "custom" { + t.Fatalf("CLAW_MCP_STDIO_COMMAND = %q, want custom", env["CLAW_MCP_STDIO_COMMAND"]) + } + if env["CLAW_MCP_STDIO_ARGS"] != `["manual"]` { + t.Fatalf("CLAW_MCP_STDIO_ARGS = %q, want manual", env["CLAW_MCP_STDIO_ARGS"]) + } +} diff --git a/internal/pod/parser.go b/internal/pod/parser.go index a709450..5b60b70 100644 --- a/internal/pod/parser.go +++ b/internal/pod/parser.go @@ -58,21 +58,28 @@ type rawInvokeEntry struct { } type rawClawBlock struct { - Agent string `yaml:"agent"` - Persona string `yaml:"persona"` - Cllama interface{} `yaml:"cllama"` - Models map[string]string `yaml:"models"` - CllamaEnv map[string]string `yaml:"cllama-env"` - Count int `yaml:"count"` - Handles map[string]interface{} `yaml:"handles"` - Feeds []rawFeedEntry `yaml:"feeds"` - Tools []rawToolPolicyEntry `yaml:"tools"` - Memory *rawMemoryEntry `yaml:"memory"` - Include []rawIncludeEntry `yaml:"include"` - Surfaces []interface{} `yaml:"surfaces"` - Skills []string `yaml:"skills"` - Invoke []rawInvokeEntry `yaml:"invoke"` - ClawAPI interface{} `yaml:"claw-api"` + Agent string `yaml:"agent"` + Persona string `yaml:"persona"` + DescribeFile string `yaml:"describe-file"` + Cllama interface{} `yaml:"cllama"` + Models map[string]string `yaml:"models"` + CllamaEnv map[string]string `yaml:"cllama-env"` + Count int `yaml:"count"` + Handles map[string]interface{} `yaml:"handles"` + Feeds []rawFeedEntry `yaml:"feeds"` + Tools []rawToolPolicyEntry `yaml:"tools"` + Memory *rawMemoryEntry `yaml:"memory"` + Include []rawIncludeEntry `yaml:"include"` + Surfaces []interface{} `yaml:"surfaces"` + Skills []string `yaml:"skills"` + Invoke []rawInvokeEntry `yaml:"invoke"` + ClawAPI interface{} `yaml:"claw-api"` + MCPStdio *rawMCPStdioBlock `yaml:"mcp-stdio"` +} + +type rawMCPStdioBlock struct { + Command string `yaml:"command"` + Args []string `yaml:"args"` } type rawFeedEntry struct { @@ -234,6 +241,10 @@ func Parse(r io.Reader) (*Pod, error) { if err != nil { return nil, fmt.Errorf("service %q: parse memory: %w", name, err) } + mcpStdio, err := parseMCPStdio(name, svc.XClaw.MCPStdio, svc.XClaw.Agent, cllama, count) + if err != nil { + return nil, err + } invoke := make([]InvokeEntry, 0, len(svc.XClaw.Invoke)) for _, rawInv := range svc.XClaw.Invoke { if rawInv.Schedule == "" || rawInv.Message == "" { @@ -256,21 +267,23 @@ func Parse(r io.Reader) (*Pod, error) { return nil, fmt.Errorf("service %q: claw-api: %w", name, err) } service.Claw = &ClawBlock{ - Agent: svc.XClaw.Agent, - Persona: svc.XClaw.Persona, - Cllama: cllama, - Models: svc.XClaw.Models, - CllamaEnv: svc.XClaw.CllamaEnv, - Count: count, - Handles: handles, - Feeds: feeds, - Tools: tools, - Memory: memory, - Include: include, - Surfaces: parsedSurfaces, - Skills: skills, - Invoke: invoke, - ClawAPIMode: clawAPIMode, + Agent: svc.XClaw.Agent, + Persona: svc.XClaw.Persona, + DescribeFile: strings.TrimSpace(svc.XClaw.DescribeFile), + Cllama: cllama, + Models: svc.XClaw.Models, + CllamaEnv: svc.XClaw.CllamaEnv, + Count: count, + Handles: handles, + Feeds: feeds, + Tools: tools, + Memory: memory, + Include: include, + Surfaces: parsedSurfaces, + Skills: skills, + Invoke: invoke, + ClawAPIMode: clawAPIMode, + MCPStdio: mcpStdio, } } pod.Services[name] = service @@ -581,6 +594,33 @@ func parseMemory(raw *rawMemoryEntry) (*MemoryEntry, error) { }, nil } +func parseMCPStdio(serviceName string, raw *rawMCPStdioBlock, agent string, cllama []string, count int) (*MCPStdioBlock, error) { + if raw == nil { + return nil, nil + } + command := strings.TrimSpace(raw.Command) + if command == "" { + return nil, fmt.Errorf("service %q: mcp-stdio command is required", serviceName) + } + if strings.TrimSpace(agent) != "" { + return nil, fmt.Errorf("service %q: mcp-stdio cannot be combined with agent", serviceName) + } + if len(cllama) > 0 { + return nil, fmt.Errorf("service %q: mcp-stdio cannot be combined with cllama", serviceName) + } + if count > 1 { + return nil, fmt.Errorf("service %q: mcp-stdio does not support count > 1", serviceName) + } + args := raw.Args + if args == nil { + args = []string{} + } + return &MCPStdioBlock{ + Command: command, + Args: append([]string(nil), args...), + }, nil +} + func (r *rawFeedEntry) UnmarshalYAML(node *yaml.Node) error { switch node.Kind { case yaml.ScalarNode: diff --git a/internal/pod/parser_mcp_stdio_test.go b/internal/pod/parser_mcp_stdio_test.go new file mode 100644 index 0000000..79b2ed9 --- /dev/null +++ b/internal/pod/parser_mcp_stdio_test.go @@ -0,0 +1,109 @@ +package pod + +import ( + "strings" + "testing" +) + +func TestParseMCPStdio(t *testing.T) { + p, err := Parse(strings.NewReader(` +services: + perplexity: + image: ghcr.io/mostlydev/claw-mcp-stdio:v0.12.0 + x-claw: + mcp-stdio: + command: npx + args: ["-y", "perplexity-mcp"] + describe-file: ./perplexity.claw-describe.json +`)) + if err != nil { + t.Fatalf("Parse: %v", err) + } + svc := p.Services["perplexity"] + if svc == nil || svc.Claw == nil || svc.Claw.MCPStdio == nil { + t.Fatalf("expected mcp-stdio block, got %+v", svc) + } + if !svc.IsMCPStdioSidecar() { + t.Fatal("expected service to be classified as mcp-stdio sidecar") + } + if svc.IsAgentManaged() { + t.Fatal("mcp-stdio sidecar must not be classified as agent-managed") + } + if svc.Claw.MCPStdio.Command != "npx" { + t.Fatalf("command = %q, want npx", svc.Claw.MCPStdio.Command) + } + if got := strings.Join(svc.Claw.MCPStdio.Args, " "); got != "-y perplexity-mcp" { + t.Fatalf("args = %q", got) + } + if svc.Claw.DescribeFile != "./perplexity.claw-describe.json" { + t.Fatalf("describe-file = %q", svc.Claw.DescribeFile) + } +} + +func TestParseMCPStdioValidation(t *testing.T) { + tests := []struct { + name string + yaml string + want string + }{ + { + name: "empty command", + yaml: ` +services: + sidecar: + image: wrapper:latest + x-claw: + mcp-stdio: + command: " " +`, + want: "mcp-stdio command is required", + }, + { + name: "agent", + yaml: ` +services: + sidecar: + image: wrapper:latest + x-claw: + agent: ./AGENTS.md + mcp-stdio: + command: npx +`, + want: "mcp-stdio cannot be combined with agent", + }, + { + name: "cllama", + yaml: ` +services: + sidecar: + image: wrapper:latest + x-claw: + cllama: passthrough + mcp-stdio: + command: npx +`, + want: "mcp-stdio cannot be combined with cllama", + }, + { + name: "count", + yaml: ` +services: + sidecar: + image: wrapper:latest + x-claw: + count: 2 + mcp-stdio: + command: npx +`, + want: "mcp-stdio does not support count > 1", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + _, err := Parse(strings.NewReader(tt.yaml)) + if err == nil || !strings.Contains(err.Error(), tt.want) { + t.Fatalf("Parse error = %v, want containing %q", err, tt.want) + } + }) + } +} diff --git a/internal/pod/types.go b/internal/pod/types.go index 5cc8388..14a7ac0 100644 --- a/internal/pod/types.go +++ b/internal/pod/types.go @@ -29,6 +29,14 @@ type Service struct { Ports []string // container-side ports from compose ports: (host:container or plain container) } +func (s *Service) IsMCPStdioSidecar() bool { + return s != nil && s.Claw != nil && s.Claw.MCPStdio != nil +} + +func (s *Service) IsAgentManaged() bool { + return s != nil && s.Claw != nil && s.Claw.MCPStdio == nil +} + // InvokeEntry is a scheduled agent task declared in the pod x-claw.invoke block. type InvokeEntry struct { Schedule string // 5-field cron expression @@ -42,6 +50,7 @@ type InvokeEntry struct { type ClawBlock struct { Agent string Persona string + DescribeFile string Cllama []string Models map[string]string CllamaEnv map[string]string @@ -56,6 +65,12 @@ type ClawBlock struct { Skills []string Invoke []InvokeEntry ClawAPIMode string // "self" when claw-api: self is declared; empty otherwise + MCPStdio *MCPStdioBlock +} + +type MCPStdioBlock struct { + Command string + Args []string } // PodPrincipal is an explicit principal declared in the pod-level x-claw.principals list. diff --git a/site/changelog.md b/site/changelog.md index d903b4e..3addab8 100644 --- a/site/changelog.md +++ b/site/changelog.md @@ -29,7 +29,7 @@ outline: deep ## Unreleased - +- **Stdio MCP wrapper for managed tools** ([#179](https://github.com/mostlydev/clawdapus/issues/179)) — adds the shared `ghcr.io/mostlydev/claw-mcp-stdio` image and `x-claw.mcp-stdio` sidecar block so npm-style stdio MCP servers can be exposed as pod-internal Streamable HTTP `/mcp` endpoints. Operators provide a deterministic v2 descriptor snapshot with `x-claw.describe-file`; `claw up` still compiles subscribing agents to `tools.json` with `execution.transport = "mcp"`, so cllama's v0.5.0 mediation, audit, session history, budgets, and credential-starvation boundaries remain unchanged. ## v0.11.0 {#v0-11-0} diff --git a/site/guide/pod-yaml.md b/site/guide/pod-yaml.md index af311ce..8344747 100644 --- a/site/guide/pod-yaml.md +++ b/site/guide/pod-yaml.md @@ -187,6 +187,27 @@ services: This expands into ordinal-named compose services: `crusher-0`, `crusher-1`, `crusher-2`. Each instance gets its own bearer token and cllama context when the governance proxy is enabled. +## Stdio MCP Sidecars + +Use `x-claw.mcp-stdio` on a sidecar service to run a stdio MCP command behind the shared Streamable HTTP wrapper image: + +```yaml +services: + search: + image: ghcr.io/mostlydev/claw-mcp-stdio:v0.12.0 + environment: + PERPLEXITY_API_KEY: ${PERPLEXITY_KEY} + expose: + - "8080" + x-claw: + describe-file: ./perplexity.claw-describe.json + mcp-stdio: + command: npx + args: ["-y", "perplexity-mcp"] +``` + +`command` is required and `args` is a JSON-safe list; no shell interpolation is used. `describe-file` is a host path relative to the pod file and should contain a v2 descriptor with `mcp: { "transport": "streamable_http", "path": "/mcp" }` plus the baked `tools[]` snapshot. + ## Generated Output `claw up` reads the pod YAML, inspects images, runs driver enforcement, generates per-agent configs, wires the cllama proxy, and calls `docker compose`. The output is `compose.generated.yml` -- a standard compose file written next to the pod file. Inspect it freely, but do not hand-edit it; it is regenerated on every `claw up`. diff --git a/site/guide/tools.md b/site/guide/tools.md index ed633f5..8140c82 100644 --- a/site/guide/tools.md +++ b/site/guide/tools.md @@ -162,6 +162,55 @@ Tool names are namespaced as `.` to prevent collisions across ser `CLAWDAPUS.md` gains a `## Tools` section listing available tool names and descriptions so the agent's behavioral contract reflects what it can call. +## Wrapping Stdio MCP Servers + +Many MCP servers are distributed as stdio commands instead of HTTP services. Use the shared `ghcr.io/mostlydev/claw-mcp-stdio` image to expose those commands as a pod-internal Streamable HTTP MCP endpoint while keeping their credentials on the sidecar. + +```yaml +services: + perplexity: + image: ghcr.io/mostlydev/claw-mcp-stdio:v0.12.0 + environment: + PERPLEXITY_API_KEY: ${PERPLEXITY_KEY} + expose: + - "8080" + x-claw: + describe-file: ./perplexity.claw-describe.json + mcp-stdio: + command: npx + args: ["-y", "perplexity-mcp"] + + analyst: + image: analyst:latest + x-claw: + cllama: passthrough + tools: + - service: perplexity + allow: [search] +``` + +The descriptor remains the compile-time source of truth: + +```json +{ + "version": 2, + "mcp": { "transport": "streamable_http", "path": "/mcp" }, + "tools": [ + { + "name": "search", + "description": "Search the web with Perplexity.", + "inputSchema": { + "type": "object", + "properties": { "query": { "type": "string" } }, + "required": ["query"] + } + } + ] +} +``` + +`x-claw.describe-file` points `claw up` at that host-side descriptor snapshot. `x-claw.mcp-stdio` only configures the child process; the wrapper exposes `/mcp`, handles MCP session initialization, restarts the child on exit, and logs stderr through the sidecar container logs. + ## Runtime Behavior ### Tool Injection diff --git a/skills/clawdapus/SKILL.md b/skills/clawdapus/SKILL.md index bc5b20c..3b6f789 100644 --- a/skills/clawdapus/SKILL.md +++ b/skills/clawdapus/SKILL.md @@ -212,6 +212,18 @@ services: to: trading-floor environment: # standard compose — credentials go HERE DISCORD_BOT_TOKEN: "${DISCORD_BOT_TOKEN}" + + perplexity: + image: ghcr.io/mostlydev/claw-mcp-stdio:v0.12.0 + environment: + PERPLEXITY_API_KEY: "${PERPLEXITY_KEY}" + expose: + - "8080" + x-claw: + describe-file: ./perplexity.claw-describe.json + mcp-stdio: + command: npx + args: ["-y", "perplexity-mcp"] ``` ### Key rules @@ -221,12 +233,13 @@ services: - **`handles`**: Discord bot IDs, usernames, guilds. Clawdapus auto-generates native Discord `mentionPatterns`, `allowBots: true`, peer `users[]` allowlist. - **`surfaces`**: String form (`"channel://discord"`) = simple enable. Map form (`channel://discord: {dm: {...}}`) = routing config. - **`tools`**: Requires `cllama` on the consuming service. Services must publish tools via `claw.describe` descriptor v2. `allow: all` (implicit for scalar form) passes every tool; named lists are validated against the tool registry. +- **`mcp-stdio`**: Sidecar-only block for the shared `claw-mcp-stdio` wrapper. `command` is required, `args` is a list, and credentials stay in the sidecar's regular `environment:`. Pair with `describe-file` when the descriptor is supplied by the pod instead of baked into the image. - **`memory`**: Requires `cllama` on the consuming service. Target service must declare `memory` in its `claw.describe` descriptor v2. - **Pod defaults**: `*-defaults` at pod level are inherited by all services. Declaring the field at service level replaces the default. Use `...` spread token to extend list-type defaults (surfaces, feeds, skills, tools). Memory defaults are object-form (no spread — presence of `memory:` at service level replaces entirely; `memory: null` suppresses). ## Service Self-Description (claw.describe) -Services declare capabilities via a `.claw-describe.json` file (embedded in the image or discovered from Dockerfile labels). `claw up` extracts descriptors and compiles them into pod-global registries. +Services declare capabilities via a `.claw-describe.json` file (embedded in the image, discovered from Dockerfile labels, or supplied with service-level `x-claw.describe-file`). `claw up` extracts descriptors and compiles them into pod-global registries. ### Descriptor v2 diff --git a/skills/release/SKILL.md b/skills/release/SKILL.md index d74a01d..827a6da 100644 --- a/skills/release/SKILL.md +++ b/skills/release/SKILL.md @@ -328,10 +328,11 @@ those images are invisible. ```go const ( - DefaultClawInfraTag = "v0.X.Y" // clawdash + claw-api + claw-wall (lockstep) + DefaultClawInfraTag = "v0.X.Y" // clawdash + claw-api + claw-wall + claw-mcp-stdio (lockstep) DefaultClawAPITag = DefaultClawInfraTag DefaultClawdashTag = DefaultClawInfraTag DefaultClawWallTag = DefaultClawInfraTag + DefaultClawMCPStdioTag = DefaultClawInfraTag DefaultCllamaTag = "v0.A.B" DefaultHermesBaseTag = "v" ) @@ -339,9 +340,9 @@ const ( Decide which pins to move based on what the release actually touches: -- **Release includes new code in `cmd/clawdash/`, `cmd/claw-api/`, or - `cmd/claw-wall/`** → bump `DefaultClawInfraTag` to the new clawdapus release - tag. All three images publish in lockstep from the clawdapus tag, so this is +- **Release includes new code in `cmd/clawdash/`, `cmd/claw-api/`, + `cmd/claw-wall/`, or `cmd/claw-mcp-stdio/`** → bump `DefaultClawInfraTag` to the new clawdapus release + tag. These images publish in lockstep from the clawdapus tag, so this is a single value. - **Release includes new cllama code (submodule moved)** → bump `DefaultCllamaTag` to the cllama release you just cut in Step 2. @@ -355,7 +356,7 @@ Even if the image code didn't change, the corresponding published tag still needs to exist at the new release version for anything covered by `DefaultClawInfraTag`, because the release verifier in `scripts/check-release-infra-tags/` checks -`ghcr.io/mostlydev/{claw-api,clawdash,claw-wall}:` exist. That +`ghcr.io/mostlydev/{claw-api,clawdash,claw-wall,claw-mcp-stdio}:` exist. That prepublish happens in Step 10. The point of Step 7 is to make the released `claw` binary actually *use* those tags. @@ -439,11 +440,12 @@ The release workflow checks these refs before goreleaser runs: - `ghcr.io/mostlydev/claw-api:v0.X.Y` - `ghcr.io/mostlydev/clawdash:v0.X.Y` - `ghcr.io/mostlydev/claw-wall:v0.X.Y` +- `ghcr.io/mostlydev/claw-mcp-stdio:v0.X.Y` - whatever fixed refs are currently pinned in `internal/infraimages/release_manifest.go` (today: `cllama` and `hermes-base`) -The tag-triggered image workflows for `claw-api`, `clawdash`, and `claw-wall` +The tag-triggered image workflows for `claw-api`, `clawdash`, `claw-wall`, and `claw-mcp-stdio` run too late to satisfy that verifier. If the versioned refs do not already exist, publish them manually before creating the release tag. @@ -494,6 +496,17 @@ docker buildx build \ -f dockerfiles/claw-api/Dockerfile . ``` +### claw-mcp-stdio + +```bash +docker buildx build \ + --platform linux/amd64,linux/arm64 \ + -t ghcr.io/mostlydev/claw-mcp-stdio:latest \ + -t ghcr.io/mostlydev/claw-mcp-stdio:v0.X.Y \ + --push \ + -f dockerfiles/claw-mcp-stdio/Dockerfile . +``` + ### hermes-base Tagged per upstream Hermes version, not per clawdapus release: @@ -523,6 +536,7 @@ Check package visibility explicitly for anything newly pushed: gh api /users/mostlydev/packages/container/claw-api gh api /users/mostlydev/packages/container/clawdash gh api /users/mostlydev/packages/container/claw-wall +gh api /users/mostlydev/packages/container/claw-mcp-stdio gh api /users/mostlydev/packages/container/hermes-base ``` @@ -575,8 +589,8 @@ Then check the GitHub release object and assets: gh release view v0.X.Y ``` -Also make sure the tag-triggered `claw-api Image`, `clawdash Image`, and -`claw-wall Image` workflows finished green. They are confirmation that the CI +Also make sure the tag-triggered `claw-api Image`, `clawdash Image`, +`claw-wall Image`, and `claw-mcp-stdio Image` workflows finished green. They are confirmation that the CI publishing path still works, but they are not the thing the release job depended on.