From b35c92547fa501d9e727a1036d62bdb73da9058a Mon Sep 17 00:00:00 2001 From: Dan Pupius Date: Fri, 6 Mar 2026 11:15:31 -0800 Subject: [PATCH 1/4] fix(proxy): rewrite proxy host for custom network gateways Containers on custom Apple/Docker networks (created for services or BuildKit) couldn't reach the credential proxy because proxy URLs used the default network gateway IP, which is unreachable from a different subnet. Add NetworkGateway() to the NetworkManager interface so the run manager can inspect the custom network's actual gateway after creation. When it differs from the default, rewrite all proxy-related env vars to use the correct gateway. Also extract inspectAppleNetworkGateway() shared helper to deduplicate the gateway inspection logic between probeDefaultGateway and NetworkGateway, and constrain replaceHostInEnv to only replace in env var values (not keys). --- internal/container/apple.go | 23 +-------- internal/container/apple_network.go | 31 ++++++++++++ internal/container/docker.go | 15 ++++++ internal/container/runtime.go | 4 ++ internal/run/manager.go | 41 ++++++++++++++-- internal/run/manager_test.go | 75 +++++++++++++++++++++++++++++ 6 files changed, 165 insertions(+), 24 deletions(-) diff --git a/internal/container/apple.go b/internal/container/apple.go index 195f05b8..ee61fa32 100644 --- a/internal/container/apple.go +++ b/internal/container/apple.go @@ -80,28 +80,9 @@ func probeDefaultGateway(containerBin string) string { ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() - cmd := exec.CommandContext(ctx, containerBin, "network", "inspect", "default") - out, err := cmd.Output() - if err != nil { - log.Debug("failed to inspect default network, using fallback gateway", - "error", err, "fallback", fallback) - return fallback - } - - var networks []struct { - Status struct { - IPv4Gateway string `json:"ipv4Gateway"` - } `json:"status"` - } - if err := json.Unmarshal(out, &networks); err != nil || len(networks) == 0 { - log.Debug("failed to parse network inspect output, using fallback gateway", - "error", err, "fallback", fallback) - return fallback - } - - gw := networks[0].Status.IPv4Gateway + gw := inspectAppleNetworkGateway(ctx, containerBin, "default") if gw == "" { - log.Debug("default network has no ipv4Gateway, using fallback", "fallback", fallback) + log.Debug("default network has no gateway, using fallback", "fallback", fallback) return fallback } diff --git a/internal/container/apple_network.go b/internal/container/apple_network.go index 7449c5c3..2df1fc29 100644 --- a/internal/container/apple_network.go +++ b/internal/container/apple_network.go @@ -2,6 +2,7 @@ package container import ( "context" + "encoding/json" "fmt" "os/exec" "strings" @@ -69,3 +70,33 @@ func (m *appleNetworkManager) ListNetworks(ctx context.Context) ([]NetworkInfo, } return result, nil } + +// NetworkGateway returns the IPv4 gateway for the named Apple container network. +func (m *appleNetworkManager) NetworkGateway(ctx context.Context, networkID string) string { + return inspectAppleNetworkGateway(ctx, m.containerBin, networkID) +} + +// inspectAppleNetworkGateway runs `container network inspect` and extracts the +// IPv4 gateway address from the JSON output. Returns empty string on failure. +// Used by both probeDefaultGateway (init-time default network) and +// NetworkGateway (per-run custom networks). +func inspectAppleNetworkGateway(ctx context.Context, containerBin, networkName string) string { + cmd := exec.CommandContext(ctx, containerBin, "network", "inspect", networkName) + out, err := cmd.Output() + if err != nil { + log.Debug("failed to inspect network for gateway", "network", networkName, "error", err) + return "" + } + + var networks []struct { + Status struct { + IPv4Gateway string `json:"ipv4Gateway"` + } `json:"status"` + } + if err := json.Unmarshal(out, &networks); err != nil || len(networks) == 0 { + log.Debug("failed to parse network inspect for gateway", "network", networkName, "error", err) + return "" + } + + return networks[0].Status.IPv4Gateway +} diff --git a/internal/container/docker.go b/internal/container/docker.go index 9cc7440a..72f29241 100644 --- a/internal/container/docker.go +++ b/internal/container/docker.go @@ -848,6 +848,21 @@ func (m *dockerNetworkManager) ListNetworks(ctx context.Context) ([]NetworkInfo, return result, nil } +// NetworkGateway returns the IPv4 gateway for the given Docker network. +func (m *dockerNetworkManager) NetworkGateway(ctx context.Context, networkID string) string { + inspect, err := m.cli.NetworkInspect(ctx, networkID, network.InspectOptions{}) + if err != nil { + log.Debug("failed to inspect network for gateway", "network", networkID, "error", err) + return "" + } + for _, cfg := range inspect.IPAM.Config { + if cfg.Gateway != "" { + return cfg.Gateway + } + } + return "" +} + // dockerSidecarManager methods // StartSidecar starts a sidecar container (pull, create, start). diff --git a/internal/container/runtime.go b/internal/container/runtime.go index fbeb42ec..1d863003 100644 --- a/internal/container/runtime.go +++ b/internal/container/runtime.go @@ -144,6 +144,10 @@ type NetworkManager interface { // ListNetworks returns all moat-managed networks. ListNetworks(ctx context.Context) ([]NetworkInfo, error) + + // NetworkGateway returns the IPv4 gateway address for the given network. + // Returns empty string if the gateway cannot be determined. + NetworkGateway(ctx context.Context, networkID string) string } // NetworkInfo contains information about a network. diff --git a/internal/run/manager.go b/internal/run/manager.go index 3628af55..f66a251d 100644 --- a/internal/run/manager.go +++ b/internal/run/manager.go @@ -418,6 +418,7 @@ func (m *Manager) Create(ctx context.Context, opts Options) (*Run, error) { // Proxy environment and mount configuration var proxyEnv []string var providerEnv []string // Provider-specific env vars (e.g., dummy ANTHROPIC_API_KEY) + var hostAddr string // Host address for proxy (may be rewritten for custom networks) var mounts []container.MountConfig // Always mount workspace @@ -657,7 +658,7 @@ func (m *Manager) Create(ctx context.Context, opts Options) (*Run, error) { } // Get proxy host address (needed for both proxy URL and firewall setup) - hostAddr := m.runtime.GetHostAddress() + hostAddr = m.runtime.GetHostAddress() // Store proxy details from daemon response r.ProxyAuthToken = regResp.AuthToken @@ -892,9 +893,10 @@ region = %s return nil, fmt.Errorf("starting SSH agent proxy (TCP): %w", err) } - // Get the actual TCP address after binding + // Get the actual TCP address after binding. + // hostAddr is set earlier from m.runtime.GetHostAddress() and may be + // rewritten later for custom networks (replaceHostInEnv). tcpAddr := sshServer.TCPAddr() - hostAddr := m.runtime.GetHostAddress() containerSSHDir := "/run/moat/ssh" // Extract port from TCP address (format is "host:port" or "[::]:port") @@ -1988,6 +1990,23 @@ region = %s networkMode = networkID } + // When a custom network is used (for services or BuildKit), the container + // is on a different subnet than the default network. The proxy host address + // (derived from the default network gateway) may be unreachable. Rewrite + // all env vars that reference the old host address to use the custom + // network's gateway instead. + if networkID != "" && hostAddr != "" { + netMgr := m.runtime.NetworkManager() + if netMgr != nil { + if gw := netMgr.NetworkGateway(ctx, networkID); gw != "" && gw != hostAddr { + log.Debug("rewriting proxy host for custom network", + "old", hostAddr, "new", gw, "network", networkID) + proxyEnv = replaceHostInEnv(proxyEnv, hostAddr, gw) + r.ProxyHost = gw + } + } + } + // Add BuildKit env vars if enabled buildkitEnv := computeBuildKitEnv(buildkitCfg.Enabled) proxyEnv = append(proxyEnv, buildkitEnv...) @@ -2217,6 +2236,22 @@ region = %s // StartOptions configures how a run is started. type StartOptions struct{} +// replaceHostInEnv replaces all occurrences of oldHost with newHost in the +// value portion of env vars (after the first '='). This is used to rewrite +// proxy URLs when a container is placed on a custom network whose gateway +// differs from the default network gateway. +func replaceHostInEnv(env []string, oldHost, newHost string) []string { + result := make([]string, len(env)) + for i, e := range env { + if idx := strings.IndexByte(e, '='); idx >= 0 { + result[i] = e[:idx+1] + strings.ReplaceAll(e[idx+1:], oldHost, newHost) + } else { + result[i] = e + } + } + return result +} + // setLogContext configures the structured logger with run-specific fields // so all subsequent log entries in this goroutine are correlated to the run. func setLogContext(r *Run) { diff --git a/internal/run/manager_test.go b/internal/run/manager_test.go index 4de51991..4b29fc86 100644 --- a/internal/run/manager_test.go +++ b/internal/run/manager_test.go @@ -1117,3 +1117,78 @@ func TestHostGitIdentity(t *testing.T) { } }) } + +// TestReplaceHostInEnv verifies that replaceHostInEnv correctly rewrites +// the proxy host address in environment variables when a custom network +// is used. This is critical for Apple containers where custom networks +// (for service dependencies) have a different gateway than the default network. +func TestReplaceHostInEnv(t *testing.T) { + env := []string{ + "HTTP_PROXY=http://moat:token@192.168.64.1:19080", + "HTTPS_PROXY=http://moat:token@192.168.64.1:19080", + "NO_PROXY=192.168.64.1,localhost,127.0.0.1", + "ANTHROPIC_BASE_URL=http://192.168.64.1:19080/relay/anthropic", + "MOAT_SSH_TCP_ADDR=192.168.64.1:62098", + "SOME_UNRELATED_VAR=hello", + } + + result := replaceHostInEnv(env, "192.168.64.1", "192.168.72.1") + + want := []string{ + "HTTP_PROXY=http://moat:token@192.168.72.1:19080", + "HTTPS_PROXY=http://moat:token@192.168.72.1:19080", + "NO_PROXY=192.168.72.1,localhost,127.0.0.1", + "ANTHROPIC_BASE_URL=http://192.168.72.1:19080/relay/anthropic", + "MOAT_SSH_TCP_ADDR=192.168.72.1:62098", + "SOME_UNRELATED_VAR=hello", + } + + if len(result) != len(want) { + t.Fatalf("got %d env vars, want %d", len(result), len(want)) + } + for i := range want { + if result[i] != want[i] { + t.Errorf("env[%d]:\n got %q\n want %q", i, result[i], want[i]) + } + } +} + +func TestReplaceHostInEnv_NoChange(t *testing.T) { + env := []string{ + "HTTP_PROXY=http://moat:token@192.168.64.1:19080", + "FOO=bar", + } + + // Same old and new — no changes expected + result := replaceHostInEnv(env, "192.168.64.1", "192.168.64.1") + for i := range env { + if result[i] != env[i] { + t.Errorf("env[%d] changed unexpectedly: %q -> %q", i, env[i], result[i]) + } + } +} + +func TestReplaceHostInEnv_Empty(t *testing.T) { + result := replaceHostInEnv(nil, "192.168.64.1", "192.168.72.1") + if len(result) != 0 { + t.Errorf("expected empty result for nil input, got %d items", len(result)) + } +} + +func TestReplaceHostInEnv_KeyNotReplaced(t *testing.T) { + // Env var key contains the old host — only the value should be replaced. + env := []string{ + "ADDR_192.168.64.1=http://192.168.64.1:8080", + "NO_EQUALS_SIGN", + } + result := replaceHostInEnv(env, "192.168.64.1", "192.168.72.1") + want := []string{ + "ADDR_192.168.64.1=http://192.168.72.1:8080", + "NO_EQUALS_SIGN", + } + for i := range want { + if result[i] != want[i] { + t.Errorf("env[%d]:\n got %q\n want %q", i, result[i], want[i]) + } + } +} From 3df99cf9c4f0185b97c4873bbf630a09fc7cfa29 Mon Sep 17 00:00:00 2001 From: Dan Pupius Date: Fri, 6 Mar 2026 11:22:42 -0800 Subject: [PATCH 2/4] fix(proxy): address code review feedback Filter Docker NetworkGateway to IPv4-only gateways (skip IPv6 from dual-stack networks that would produce malformed proxy URLs), and update stale doc comment that claimed Apple containers return nil for NetworkManager. --- internal/container/docker.go | 4 +++- internal/container/runtime.go | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/internal/container/docker.go b/internal/container/docker.go index 72f29241..87104bdf 100644 --- a/internal/container/docker.go +++ b/internal/container/docker.go @@ -857,7 +857,9 @@ func (m *dockerNetworkManager) NetworkGateway(ctx context.Context, networkID str } for _, cfg := range inspect.IPAM.Config { if cfg.Gateway != "" { - return cfg.Gateway + if ip := net.ParseIP(cfg.Gateway); ip != nil && ip.To4() != nil { + return cfg.Gateway + } } } return "" diff --git a/internal/container/runtime.go b/internal/container/runtime.go index 1d863003..87e68ae6 100644 --- a/internal/container/runtime.go +++ b/internal/container/runtime.go @@ -78,7 +78,7 @@ type Runtime interface { SupportsHostNetwork() bool // NetworkManager returns the network manager if supported, nil otherwise. - // Docker provides this, Apple containers return nil. + // Both Docker and Apple runtimes provide this. NetworkManager() NetworkManager // SidecarManager returns the sidecar manager if supported, nil otherwise. From 8660ed70cf3d2aeb781b625d20f4e9a80490a795 Mon Sep 17 00:00:00 2001 From: Dan Pupius Date: Fri, 6 Mar 2026 11:29:11 -0800 Subject: [PATCH 3/4] fix(proxy): skip rewrite for hostname-based proxy hosts Only rewrite proxy env vars when hostAddr is an IP address (Apple containers). Docker on macOS uses host.docker.internal which already resolves correctly from any bridge network. Also fix stale ServiceManager doc comment (Apple provides it too). --- internal/container/runtime.go | 2 +- internal/run/manager.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/container/runtime.go b/internal/container/runtime.go index 87e68ae6..3a735e2a 100644 --- a/internal/container/runtime.go +++ b/internal/container/runtime.go @@ -90,7 +90,7 @@ type Runtime interface { BuildManager() BuildManager // ServiceManager returns the service manager if supported, nil otherwise. - // Docker provides this, Apple containers return nil. + // Both Docker and Apple runtimes provide this. ServiceManager() ServiceManager // Close releases runtime resources. diff --git a/internal/run/manager.go b/internal/run/manager.go index f66a251d..cbaed8dd 100644 --- a/internal/run/manager.go +++ b/internal/run/manager.go @@ -1995,7 +1995,7 @@ region = %s // (derived from the default network gateway) may be unreachable. Rewrite // all env vars that reference the old host address to use the custom // network's gateway instead. - if networkID != "" && hostAddr != "" { + if networkID != "" && net.ParseIP(hostAddr) != nil { netMgr := m.runtime.NetworkManager() if netMgr != nil { if gw := netMgr.NetworkGateway(ctx, networkID); gw != "" && gw != hostAddr { From de4064a3e130c7ffd4cb16855968a9c6592e883f Mon Sep 17 00:00:00 2001 From: Dan Pupius Date: Fri, 6 Mar 2026 11:48:41 -0800 Subject: [PATCH 4/4] fix(e2e): make TestServiceCleanup work on Apple containers The test hardcoded `docker ps` to verify service container cleanup, which fails on Apple containers where Docker isn't running. Extract a serviceContainerExists helper that tries Docker first, then falls back to the Apple container CLI. --- internal/e2e/services_test.go | 44 +++++++++++++++++++++++++++-------- 1 file changed, 34 insertions(+), 10 deletions(-) diff --git a/internal/e2e/services_test.go b/internal/e2e/services_test.go index ba3b74b2..41b185af 100644 --- a/internal/e2e/services_test.go +++ b/internal/e2e/services_test.go @@ -370,9 +370,7 @@ func TestServiceCleanup(t *testing.T) { // Verify a service container exists before cleanup serviceContainerName := "moat-postgres-" + runID - checkBefore := exec.CommandContext(ctx, "docker", "ps", "-a", "-q", "-f", "name="+serviceContainerName) - beforeOutput, _ := checkBefore.Output() - if len(strings.TrimSpace(string(beforeOutput))) == 0 { + if found := serviceContainerExists(ctx, t, serviceContainerName); !found { t.Logf("Note: service container %s not found before destroy (may have been cleaned up already)", serviceContainerName) } @@ -382,15 +380,41 @@ func TestServiceCleanup(t *testing.T) { } // Verify the service container no longer exists - checkAfter := exec.CommandContext(ctx, "docker", "ps", "-a", "-q", "-f", "name="+serviceContainerName) - afterOutput, err := checkAfter.Output() - if err != nil { - t.Fatalf("docker ps check: %v", err) - } - - if len(strings.TrimSpace(string(afterOutput))) > 0 { + if serviceContainerExists(ctx, t, serviceContainerName) { t.Errorf("Service container %s still exists after Destroy", serviceContainerName) } else { t.Logf("Service container %s correctly removed after Destroy", serviceContainerName) } } + +// serviceContainerExists checks whether a container with the given name exists +// using whichever CLI is available (docker or Apple container). +func serviceContainerExists(ctx context.Context, t *testing.T, name string) bool { + t.Helper() + + // Try Docker first + if _, err := exec.LookPath("docker"); err == nil { + cmd := exec.CommandContext(ctx, "docker", "ps", "-a", "-q", "-f", "name="+name) + out, err := cmd.Output() + if err != nil { + // Docker CLI exists but daemon may not be running — fall through to Apple + t.Logf("docker ps failed (trying Apple container CLI): %v", err) + } else { + return len(strings.TrimSpace(string(out))) > 0 + } + } + + // Try Apple container CLI + if _, err := exec.LookPath("container"); err == nil { + cmd := exec.CommandContext(ctx, "container", "list", "--all", "--format", "json") + out, err := cmd.Output() + if err != nil { + t.Logf("container list failed: %v", err) + return false + } + return strings.Contains(string(out), name) + } + + t.Log("No container CLI available to check container existence") + return false +}