Skip to content

Add --anthropic-path-prefix flag and auto-probe for Envoy AI Gateway #5174

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
yrobla wants to merge 1 commit into
base: main
Choose a base branch
from
Open

Add --anthropic-path-prefix flag and auto-probe for Envoy AI Gateway #5174

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 15 additions & 6 deletions cmd/thv/app/llm.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -214,9 +214,10 @@ func buildLLMTokenSource(cfg *llm.Config, interactive bool) (*llm.TokenSource, e

func newLLMSetupCommand() *cobra.Command {
var (
opts llm.SetOptions
tlsSkipVerify bool
targetClient string
opts llm.SetOptions
tlsSkipVerify bool
targetClient string
anthropicPathPrefix string
)

cmd := &cobra.Command{
Expand Down Expand Up @@ -250,7 +251,8 @@ Run "thv llm teardown" to revert all changes.`,
}
return runLLMSetup(
cmd.Context(), cmd.OutOrStdout(), cmd.ErrOrStderr(),
cm, config.NewDefaultProvider(), oidcLogin, opts, targetClient,
cm, config.NewDefaultProvider(), oidcLogin, opts,
anthropicPathPrefix, cmd.Flags().Changed("anthropic-path-prefix"), targetClient,
)
},
}
Expand All @@ -266,6 +268,9 @@ Run "thv llm teardown" to revert all changes.`,
"For direct-mode tools (Claude Code, Gemini CLI) this sets NODE_TLS_REJECT_UNAUTHORIZED=0, "+
"disabling TLS for ALL of that tool's outbound connections. "+
"For proxy-mode tools only the proxy-to-gateway connection is affected.")
cmd.Flags().StringVar(&anthropicPathPrefix, "anthropic-path-prefix", "",
"Path prefix appended to the gateway URL when writing ANTHROPIC_BASE_URL for direct-mode tools "+
"(e.g. /anthropic). When omitted, the gateway is probed automatically.")
Comment thread
yrobla marked this conversation as resolved.
Comment thread
yrobla marked this conversation as resolved.
cmd.Flags().StringVar(&targetClient, "client", "",
"Configure only this AI tool by name (e.g. claude-code, cursor). Omit to configure all detected tools.")

Expand All @@ -286,9 +291,13 @@ func oidcLogin(ctx context.Context, cfg *llm.Config) error {
func runLLMSetup(
ctx context.Context, out, errOut io.Writer,
cm *client.ClientManager, provider config.Provider, login llm.LoginFunc,
inlineOpts llm.SetOptions, targetClient string,
inlineOpts llm.SetOptions, anthropicPathPrefix string, anthropicPathPrefixSet bool, targetClient string,
) error {
return llm.Setup(ctx, out, errOut, &clientManagerAdapter{cm}, &configUpdaterAdapter{provider}, login, inlineOpts, targetClient)
return llm.Setup(
ctx, out, errOut,
&clientManagerAdapter{cm}, &configUpdaterAdapter{provider}, login,
inlineOpts, anthropicPathPrefix, anthropicPathPrefixSet, targetClient,
)
}

func newLLMTeardownCommand() *cobra.Command {
Expand Down
16 changes: 8 additions & 8 deletions cmd/thv/app/llm_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,7 @@ func TestRunLLMSetup_NotConfigured(t *testing.T) {
provider := llmProvider(t, llm.Config{}) // no gateway URL

var stdout, stderr bytes.Buffer
err := runLLMSetup(context.Background(), &stdout, &stderr, cm, provider, noopLogin, llm.SetOptions{}, "")
err := runLLMSetup(context.Background(), &stdout, &stderr, cm, provider, noopLogin, llm.SetOptions{}, "", false, "")
require.Error(t, err)
assert.Contains(t, err.Error(), "not configured")
}
Expand All @@ -98,7 +98,7 @@ func TestRunLLMSetup_NoDetectedTools(t *testing.T) {
})

var stdout, stderr bytes.Buffer
err := runLLMSetup(context.Background(), &stdout, &stderr, cm, provider, noopLogin, llm.SetOptions{}, "")
err := runLLMSetup(context.Background(), &stdout, &stderr, cm, provider, noopLogin, llm.SetOptions{}, "", false, "")
require.NoError(t, err)
assert.Contains(t, stdout.String(), "No supported AI tools detected")
}
Expand Down Expand Up @@ -142,7 +142,7 @@ func TestRunLLMSetup_PartialFailure(t *testing.T) {
})

var stdout, stderr bytes.Buffer
err := runLLMSetup(context.Background(), &stdout, &stderr, cm, provider, noopLogin, llm.SetOptions{}, "")
err := runLLMSetup(context.Background(), &stdout, &stderr, cm, provider, noopLogin, llm.SetOptions{}, "", false, "")
require.NoError(t, err)
assert.Contains(t, stderr.String(), "Warning: failed to configure claude-code")
assert.Contains(t, stdout.String(), "Configured gemini-cli")
Expand Down Expand Up @@ -176,7 +176,7 @@ func TestRunLLMSetup_RollbackOnConfigUpdateFailure(t *testing.T) {
provider := &errOnUpdateProvider{cfg: c, updateErr: errors.New("disk full")}

var stdout, stderr bytes.Buffer
err := runLLMSetup(context.Background(), &stdout, &stderr, cm, provider, noopLogin, llm.SetOptions{}, "")
err := runLLMSetup(context.Background(), &stdout, &stderr, cm, provider, noopLogin, llm.SetOptions{}, "", false, "")
require.Error(t, err)
assert.Contains(t, err.Error(), "persisting tool configuration")

Expand Down Expand Up @@ -226,7 +226,7 @@ func TestRunLLMSetup_RollbackBothToolsOnConfigUpdateFailure(t *testing.T) {
provider := &errOnUpdateProvider{cfg: c, updateErr: errors.New("disk full")}

var stdout, stderr bytes.Buffer
err := runLLMSetup(context.Background(), &stdout, &stderr, cm, provider, noopLogin, llm.SetOptions{}, "")
err := runLLMSetup(context.Background(), &stdout, &stderr, cm, provider, noopLogin, llm.SetOptions{}, "", false, "")
require.Error(t, err)
assert.Contains(t, err.Error(), "persisting tool configuration")

Expand Down Expand Up @@ -273,7 +273,7 @@ func TestRunLLMSetup_LoginFailureLeavesNoState(t *testing.T) {
var stdout, stderr bytes.Buffer
err := runLLMSetup(context.Background(), &stdout, &stderr, cm, provider,
func(_ context.Context, _ *llm.Config) error { return loginErr },
llm.SetOptions{}, "",
llm.SetOptions{}, "", false, "",
)
require.Error(t, err)
assert.Contains(t, err.Error(), "OIDC login failed")
Expand Down Expand Up @@ -474,7 +474,7 @@ func TestRunLLMSetup_ClientFlag_ConfiguresSingleTool(t *testing.T) {
})

var stdout, stderr bytes.Buffer
err := runLLMSetup(context.Background(), &stdout, &stderr, cm, provider, noopLogin, llm.SetOptions{}, "claude-code")
err := runLLMSetup(context.Background(), &stdout, &stderr, cm, provider, noopLogin, llm.SetOptions{}, "", false, "claude-code")
require.NoError(t, err)
assert.Contains(t, stdout.String(), "Configured claude-code")
assert.NotContains(t, stdout.String(), "gemini-cli")
Expand Down Expand Up @@ -509,7 +509,7 @@ func TestRunLLMSetup_ClientFlag_NotInstalled(t *testing.T) {

var stdout, stderr bytes.Buffer
// cursor is not installed (no dir); expect an error.
err := runLLMSetup(context.Background(), &stdout, &stderr, cm, provider, noopLogin, llm.SetOptions{}, "cursor")
err := runLLMSetup(context.Background(), &stdout, &stderr, cm, provider, noopLogin, llm.SetOptions{}, "", false, "cursor")
require.Error(t, err)
assert.Contains(t, err.Error(), `"cursor" is not installed or not detected`)
}
Expand Down
8 changes: 4 additions & 4 deletions pkg/client/config.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -162,8 +162,8 @@ const (
//
// Exactly one of ValueField or Literal must be set:
// - ValueField names which ApplyConfig field to write. Valid values:
// "GatewayURL", "ProxyBaseURL", "ProxyOrigin", "TokenHelperCommand",
// "PlaceholderAPIKey", "NodeTLSRejectUnauthorized".
// "GatewayURL", "AnthropicBaseURL", "ProxyBaseURL", "ProxyOrigin",
// "TokenHelperCommand", "PlaceholderAPIKey", "NodeTLSRejectUnauthorized".
// An unrecognised ValueField is a programming error and causes
// ConfigureLLMGateway to return an error.
// - Literal is written verbatim into the settings key (e.g. a fixed auth
Expand All @@ -177,7 +177,7 @@ const (
// flag is cleared. Ignored when Literal is set (literals are never empty).
type LLMGatewayKeySpec struct {
JSONPointer string // RFC 6901 path
// ValueField: "GatewayURL" | "ProxyBaseURL" | "ProxyOrigin" |
// ValueField: "GatewayURL" | "AnthropicBaseURL" | "ProxyBaseURL" | "ProxyOrigin" |
// "TokenHelperCommand" | "PlaceholderAPIKey" | "NodeTLSRejectUnauthorized"
ValueField string
Literal string // constant value written verbatim; mutually exclusive with ValueField
Expand Down Expand Up @@ -481,7 +481,7 @@ var supportedClientIntegrations = []clientAppConfig{
LLMSettingsRelPath: []string{".claude"},
LLMGatewayKeys: []LLMGatewayKeySpec{
{JSONPointer: "/apiKeyHelper", ValueField: "TokenHelperCommand"},
{JSONPointer: "/env/ANTHROPIC_BASE_URL", ValueField: "GatewayURL"},
{JSONPointer: "/env/ANTHROPIC_BASE_URL", ValueField: "AnthropicBaseURL"},
// NODE_TLS_REJECT_UNAUTHORIZED is only written when --tls-skip-verify is set.
// ClearWhenEmpty ensures it is removed when the flag is later cleared.
{JSONPointer: "/env/NODE_TLS_REJECT_UNAUTHORIZED", ValueField: "NodeTLSRejectUnauthorized", ClearWhenEmpty: true},
Comment thread
yrobla marked this conversation as resolved.
Expand Down
7 changes: 7 additions & 0 deletions pkg/client/llm_gateway.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -290,6 +290,13 @@ func llmValueForSpec(spec LLMGatewayKeySpec, cfg llmgateway.ApplyConfig) (string
switch spec.ValueField {
case "GatewayURL":
return cfg.GatewayURL, nil
case "AnthropicBaseURL":
// Use the pre-computed Anthropic base URL when available; fall back to
// GatewayURL so existing configs continue to work without the prefix.
if cfg.AnthropicBaseURL != "" {
return cfg.AnthropicBaseURL, nil
}
return cfg.GatewayURL, nil
Comment on lines +293 to +299
case "ProxyBaseURL":
return cfg.ProxyBaseURL, nil
case "TokenHelperCommand":
Expand Down
159 changes: 135 additions & 24 deletions pkg/llm/setup.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,11 @@ package llm

import (
"context"
"crypto/tls"
"fmt"
"io"
"net/http"
"net/url"
"os"
"strings"
"time"
Expand Down Expand Up @@ -56,7 +59,7 @@ type ConfigUpdater interface {
func Setup(
ctx context.Context, out, errOut io.Writer,
gm GatewayManager, provider ConfigUpdater, login LoginFunc,
inlineOpts SetOptions, targetClient string,
inlineOpts SetOptions, anthropicPathPrefix string, anthropicPathPrefixSet bool, targetClient string,
) error {
llmCfg := provider.GetLLMConfig()

Expand All @@ -71,31 +74,12 @@ func Setup(
return fmt.Errorf("LLM gateway is not configured — run \"thv llm config set\" first")
}

self, err := os.Executable()
tokenHelperCommand, err := buildTokenHelperCommand()
if err != nil {
return fmt.Errorf("resolving thv executable path: %w", err)
}
// Reject paths that contain shell metacharacters: the token-helper command
// is written verbatim into long-lived tool config files and re-executed by
// the shell inside Claude Code / Gemini CLI. A path with '"', '\', ';',
// '$', '`', newline, or carriage-return would silently produce a broken or
// exploitable command. '$' and '`' are included because they trigger
// variable and command substitution even inside double-quoted strings.
//
// Note: backslashes are Windows path separators, so this check effectively
// makes "thv llm setup" unsupported on Windows — consistent with the rest
// of the LLM gateway feature (token-helper tools use POSIX-style shells).
const shellUnsafe = `"\;$` + "`\n\r"
if strings.ContainsAny(self, shellUnsafe) {
return fmt.Errorf(
"executable path %q contains shell-unsafe characters; "+
"move thv to a path without quotes, backslashes, semicolons, "+
"dollar signs, or backticks "+
"(Windows paths are not supported by thv llm setup)", self)
return err
}

proxyBaseURL := fmt.Sprintf("http://localhost:%d/v1", llmCfg.EffectiveProxyPort())
tokenHelperCommand := fmt.Sprintf(`"%s" llm token`, self)

// Detect tools before login so we skip the interactive browser flow when
// there is nothing to configure. Login still runs before any files are
Expand All @@ -115,8 +99,18 @@ func Setup(
}
_, _ = fmt.Fprintln(out, "Login successful.")

// Resolve the effective path prefix for ANTHROPIC_BASE_URL.
// If the caller supplied --anthropic-path-prefix, use it directly.
// Otherwise auto-probe: a HEAD request to <gateway>/anthropic/v1/messages
// that returns 401 (rather than 404) indicates the gateway uses the
// /anthropic prefix, so we apply it automatically.
// Only probe if at least one detected tool uses direct mode; proxy-mode
// tools ignore the Anthropic prefix entirely.
anthropicPrefix := resolveAnthropicPrefix(ctx, gm, detected, llmCfg, anthropicPathPrefix, anthropicPathPrefixSet)

configured, err := configureDetectedTools(
out, errOut, gm, detected, llmCfg.GatewayURL, proxyBaseURL, tokenHelperCommand, llmCfg.TLSSkipVerify,
out, errOut, gm, detected, llmCfg.GatewayURL, proxyBaseURL, tokenHelperCommand,
llmCfg.TLSSkipVerify, anthropicPrefix,
)
if err != nil {
return err
Expand Down Expand Up @@ -326,11 +320,26 @@ func configureDetectedTools(
detected []string,
gatewayURL, proxyBaseURL, tokenHelperCommand string,
tlsSkipVerify bool,
anthropicPathPrefix string,
) ([]ToolConfig, error) {
var configured []ToolConfig
for _, clientType := range detected {
mode := gm.LLMGatewayModeFor(clientType)

// Only apply the Anthropic path prefix for direct-mode tools.
// Proxy-mode tools (Cursor, VS Code, Xcode) do not use ANTHROPIC_BASE_URL.
anthropicBaseURL := ""
if mode == "direct" && anthropicPathPrefix != "" {
// Trim any leading slash: url.JoinPath docs say elements should not
// start with "/", and path.Join already handles the join correctly.
if joined, err := url.JoinPath(gatewayURL, strings.TrimLeft(anthropicPathPrefix, "/")); err == nil {
anthropicBaseURL = joined
}
}
Comment thread
yrobla marked this conversation as resolved.

configPath, err := gm.ConfigureLLMGateway(clientType, llmgateway.ApplyConfig{
GatewayURL: gatewayURL,
AnthropicBaseURL: anthropicBaseURL,
ProxyBaseURL: proxyBaseURL,
TokenHelperCommand: tokenHelperCommand,
TLSSkipVerify: tlsSkipVerify,
Expand All @@ -339,7 +348,6 @@ func configureDetectedTools(
_, _ = fmt.Fprintf(errOut, "Warning: failed to configure %s: %v\n", clientType, err)
continue
}
mode := gm.LLMGatewayModeFor(clientType)
configured = append(configured, ToolConfig{
Tool: clientType,
Mode: mode,
Expand All @@ -356,6 +364,109 @@ func configureDetectedTools(
return configured, nil
}

// resolveAnthropicPrefix returns the effective Anthropic path prefix. When the
// caller explicitly set the flag (anthropicPathPrefixSet), the provided value is
// returned as-is (including empty string, which disables the prefix). Otherwise
// the gateway is auto-probed when at least one direct-mode client is present.
func resolveAnthropicPrefix(
ctx context.Context, gm GatewayManager, detected []string,
llmCfg Config, anthropicPathPrefix string, anthropicPathPrefixSet bool,
) string {
if anthropicPathPrefixSet || !hasDirectModeClient(gm, detected) {
return anthropicPathPrefix
}
return probeAnthropicPrefix(ctx, llmCfg.GatewayURL, llmCfg.TLSSkipVerify)
}

// probeAnthropicPrefix performs a HEAD request to <gatewayURL>/anthropic/v1/messages.
// If the server responds with HTTP 401 (Unauthorized) — meaning the path exists but
// requires authentication — it returns "/anthropic" as the path prefix so that
// ANTHROPIC_BASE_URL is constructed as <gateway>/anthropic rather than <gateway>.
// Any other status code (including 404 Not Found) or any network error is treated
// as "no prefix needed" and the function returns "".
func probeAnthropicPrefix(ctx context.Context, gatewayURL string, tlsSkipVerify bool) string {
if gatewayURL == "" {
return ""
}
probeURL, err := url.JoinPath(gatewayURL, "anthropic/v1/messages")
if err != nil {
return ""
}

// Build an http.Client that honours --tls-skip-verify so the probe works
// against gateways with self-signed certificates (local dev). Clone
// http.DefaultTransport to preserve all production defaults (timeouts,
// ProxyFromEnvironment, HTTP/2, connection pooling) and only toggle
// InsecureSkipVerify.
//nolint:forcetypeassert // DefaultTransport is always *http.Transport
transport := http.DefaultTransport.(*http.Transport).Clone()
if tlsSkipVerify {
if transport.TLSClientConfig == nil {
transport.TLSClientConfig = &tls.Config{MinVersion: tls.VersionTLS12}
}
transport.TLSClientConfig.InsecureSkipVerify = true //nolint:gosec // G402: intentional for local dev with self-signed certs
}
httpClient := &http.Client{Transport: transport}

// Use a short timeout so setup is not significantly slowed by an unreachable gateway.
probeCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
defer cancel()

req, err := http.NewRequestWithContext(probeCtx, http.MethodHead, probeURL, nil)
if err != nil {
return ""
}
resp, err := httpClient.Do(req)
if err != nil {
return ""
}
_ = resp.Body.Close()

if resp.StatusCode == http.StatusUnauthorized {
return "/anthropic"
}
return ""
}

// buildTokenHelperCommand returns the shell command string used as the
// token-helper for direct-mode tools. It rejects executable paths that contain
// shell metacharacters, since the command is written verbatim into long-lived
// tool config files and re-executed by the shell inside Claude Code / Gemini CLI.
// A path with '"', '\', ';', '$', '`', newline, or carriage-return would
// silently produce a broken or exploitable command. '$' and '`' are included
// because they trigger variable/command substitution inside double-quoted strings.
//
// Note: backslashes are Windows path separators, so this effectively makes
// "thv llm setup" unsupported on Windows — consistent with the rest of the LLM
// gateway feature (token-helper tools use POSIX-style shells).
func buildTokenHelperCommand() (string, error) {
self, err := os.Executable()
if err != nil {
return "", fmt.Errorf("resolving thv executable path: %w", err)
}
const shellUnsafe = `"\;$` + "`\n\r"
if strings.ContainsAny(self, shellUnsafe) {
return "", fmt.Errorf(
"executable path %q contains shell-unsafe characters; "+
"move thv to a path without quotes, backslashes, semicolons, "+
"dollar signs, or backticks "+
"(Windows paths are not supported by thv llm setup)", self)
}
return fmt.Sprintf(`"%s" llm token`, self), nil
}

// hasDirectModeClient reports whether any client in the detected list uses
// direct mode. Used to skip the Anthropic-prefix probe when no direct-mode
// tools are present (proxy-mode tools ignore ANTHROPIC_BASE_URL entirely).
func hasDirectModeClient(gm GatewayManager, detected []string) bool {
for _, clientType := range detected {
if gm.LLMGatewayModeFor(clientType) == "direct" {
return true
}
}
return false
}

// hasProxyMode reports whether any of the given tool configs uses proxy mode.
func hasProxyMode(cfgs []ToolConfig) bool {
for _, t := range cfgs {
Expand Down
Loading
Loading