add support for GOPROXY setting on the Go SDK

This is implemented in such a way that it's a bit more generic but
hidden from external clients for now.

The overall plumbing is:
1. A hidden (via `__` prefix) API on Container for specifying env vars
   that should be inherited from the engine container, where they must
   be prefixed with `_DAGGER_ENGINE_SYSTEMENV_` (e.g.
   `_DAGGER_ENGINE_SYSTEMENV_GOPROXY`).
2. The Container implementation sets metadata that the Executor needs
   via LLB *metadata*. Importantly, this is very different from the
   previous `ftp_proxy` hack in that this is just metadata and thus
   doesn't impact LLB digests. We specifically use the `Description` LLB
   metadata, which is explicitly doc'd as never being interpreted by the
   solver and thus safe to use for this.
3. The Worker's ResolveOp method registers the solver.Vertex (which
   includes that metadata) so that it can be read by the Executor as
   needed.
4. The Executor uses that metadata to set any requested env vars from
   the system.

Once we've fleshed out the design of support for these system env vars
we can open up the API.

This plumbing of executorMetadata is also intended to be easy to re-use
for any other needs that arise in the future similar to this.

Integ tests are also included here with a pretty nice GOPROXY
implementation usable a library.

Signed-off-by: Erik Sipsma <erik@sipsma.dev>

core/container.go

@@ -93,6 +93,10 @@ type Container struct {
 
 	// The args to invoke when using the terminal api on this container.
 	DefaultTerminalCmd DefaultTerminalCmdOpts `json:"defaultTerminalCmd,omitempty"`
+
+	// (Internal-only for now) Environment variables from the engine container, prefixed
+	// with a special value, that will be inherited by this container if set.
+	SystemEnvNames []string `json:"system_envs,omitempty"`
 }
 
 func (*Container) Type() *ast.Type {
@@ -157,6 +161,7 @@ func (container *Container) Clone() *Container {
 	cp.Sockets = cloneSlice(cp.Sockets)
 	cp.Ports = cloneSlice(cp.Ports)
 	cp.Services = cloneSlice(cp.Services)
+	cp.SystemEnvNames = cloneSlice(cp.SystemEnvNames)
 	return &cp
 }
 
@@ -1189,14 +1194,25 @@ func (container *Container) WithExec(ctx context.Context, opts ContainerExecOpts
 
 	execSt := fsSt.Run(runOpts...)
 
-	execDef, err := execSt.Root().Marshal(ctx, llb.Platform(platform.Spec()))
+	execMD := buildkit.ExecutionMetadata{
+		SystemEnvNames: container.SystemEnvNames,
+	}
+	execMDOpt, err := execMD.AsConstraintsOpt()
+	if err != nil {
+		return nil, fmt.Errorf("execution metadata: %w", err)
+	}
+	marshalOpts := []llb.ConstraintsOpt{
+		llb.Platform(platform.Spec()),
+		execMDOpt,
+	}
+	execDef, err := execSt.Root().Marshal(ctx, marshalOpts...)
 	if err != nil {
 		return nil, fmt.Errorf("marshal root: %w", err)
 	}
 
 	container.FS = execDef.ToPB()
 
-	metaDef, err := execSt.GetMount(buildkit.MetaMountDestPath).Marshal(ctx, llb.Platform(platform.Spec()))
+	metaDef, err := execSt.GetMount(buildkit.MetaMountDestPath).Marshal(ctx, marshalOpts...)
 	if err != nil {
 		return nil, fmt.Errorf("get meta mount: %w", err)
 	}
@@ -1211,7 +1227,7 @@ func (container *Container) WithExec(ctx context.Context, opts ContainerExecOpts
 		mountSt := execSt.GetMount(mnt.Target)
 
 		// propagate any changes to regular mounts to subsequent containers
-		execMountDef, err := mountSt.Marshal(ctx, llb.Platform(platform.Spec()))
+		execMountDef, err := mountSt.Marshal(ctx, marshalOpts...)
 		if err != nil {
 			return nil, fmt.Errorf("propagate %s: %w", mnt.Target, err)
 		}

core/integration/proxy_test.go

@@ -3,15 +3,21 @@ package core
 import (
 	"context"
 	"fmt"
+	"io"
 	"net"
+	"net/http"
 	"net/url"
 	"os"
 	"path/filepath"
 	"strings"
+	"sync"
 	"testing"
+	"time"
 
-	"dagger.io/dagger"
+	"github.com/goproxy/goproxy"
 	"github.com/stretchr/testify/require"
+
+	"dagger.io/dagger"
 )
 
 type proxyTest struct {
@@ -291,7 +297,7 @@ func TestContainerSystemProxies(t *testing.T) {
 					WithEnvVariable("HTTP_PROXY", u.String()).
 					WithExec([]string{"curl", "-v", f.httpServerURL.String()}).
 					Stderr(ctx)
-				// curl won't exit 0 if it gets a 407 on plain HTTP, so don't expect an error
+				// curl will exit 0 if it gets a 407 on plain HTTP, so don't expect an error
 				require.NoError(t, err)
 				require.Contains(t, out, "< HTTP/1.1 407 Proxy Authentication Required")
 			}},
@@ -316,9 +322,130 @@ func TestContainerSystemProxies(t *testing.T) {
 					WithEnvVariable("HTTPS_PROXY", u.String()).
 					WithExec([]string{"curl", "-v", f.httpsServerURL.String()}).
 					Stderr(ctx)
-				// curl WILL exit 0 if it gets a 407 when using TLS, so DO expect an error
+				// curl WON'T exit 0 if it gets a 407 when using TLS, so DO expect an error
 				require.ErrorContains(t, err, "< HTTP/1.1 407 Proxy Authentication Required")
 			}},
 		)
 	})
 }
+
+func TestContainerSystemGoProxy(t *testing.T) {
+	t.Parallel()
+	c, ctx := connect(t)
+
+	// just a subset of modules we expect to be downloaded since trying to go one to one would
+	// be too fragile whenever the SDK changes
+	expectedGoModDownloads := []string{
+		"github.com/99designs/gqlgen",
+		"github.com/Khan/genqlient",
+		"go.opentelemetry.io/otel/exporters/otlp/otlptrace",
+		"golang.org/x/sync",
+	}
+
+	executeTestEnvName := fmt.Sprintf("DAGGER_TEST_%s", strings.ToUpper(t.Name()))
+	if os.Getenv(executeTestEnvName) == "" {
+		const netID = 103
+		const goProxyAlias = "goproxy"
+		const goProxyPort = 8080
+		goProxySetting := fmt.Sprintf("http://%s:%d", goProxyAlias, goProxyPort)
+
+		fetcher := &goProxyFetcher{dlPaths: make(map[string]struct{})}
+		proxy := &goproxy.Goproxy{Fetcher: fetcher}
+
+		l, err := net.Listen("tcp", "127.0.0.1:0")
+		require.NoError(t, err)
+		t.Cleanup(func() {
+			l.Close()
+		})
+		port := l.Addr().(*net.TCPAddr).Port
+
+		goProxyCtx, cancelGoProxy := context.WithCancel(ctx)
+		t.Cleanup(cancelGoProxy)
+		srv := http.Server{
+			Handler:           proxy,
+			ReadHeaderTimeout: 30 * time.Second,
+			BaseContext: func(net.Listener) context.Context {
+				return goProxyCtx
+			},
+		}
+		t.Cleanup(func() {
+			srv.Shutdown(context.Background())
+		})
+
+		goProxyDone := make(chan error, 1)
+		go func() {
+			goProxyDone <- srv.Serve(l)
+		}()
+
+		goProxySvc := c.Host().Service([]dagger.PortForward{{
+			Backend:  port,
+			Frontend: goProxyPort,
+		}})
+
+		devEngine := devEngineContainer(c, netID, func(ctr *dagger.Container) *dagger.Container {
+			return ctr.
+				WithServiceBinding(goProxyAlias, goProxySvc).
+				WithEnvVariable("_DAGGER_ENGINE_SYSTEMENV_GOPROXY", goProxySetting)
+		})
+
+		thisRepoPath, err := filepath.Abs("../..")
+		require.NoError(t, err)
+		thisRepo := c.Host().Directory(thisRepoPath)
+
+		_, err = c.Container().From(golangImage).
+			With(goCache(c)).
+			WithMountedDirectory("/src", thisRepo).
+			WithWorkdir("/src").
+			WithServiceBinding("engine", devEngine.AsService()).
+			WithMountedFile("/bin/dagger", daggerCliFile(t, c)).
+			WithEnvVariable("_EXPERIMENTAL_DAGGER_CLI_BIN", "/bin/dagger").
+			WithEnvVariable("_EXPERIMENTAL_DAGGER_RUNNER_HOST", "tcp://engine:1234").
+			WithEnvVariable(executeTestEnvName, "ya").
+			WithExec([]string{"go", "test",
+				"-v",
+				"-timeout", "20m",
+				"-count", "1",
+				"-run", fmt.Sprintf("^%s$", t.Name()),
+				"./core/integration",
+			}).Sync(ctx)
+		require.NoError(t, err)
+
+		select {
+		case err := <-goProxyDone:
+			require.NoError(t, err)
+		default:
+		}
+
+		fetcher.mu.Lock()
+		defer fetcher.mu.Unlock()
+		require.NotEmpty(t, fetcher.dlPaths)
+		for _, expectedPath := range expectedGoModDownloads {
+			require.Contains(t, fetcher.dlPaths, expectedPath)
+		}
+
+		return
+	}
+
+	// we're in the container depending on the custom engine, run the actual tests
+	ctr := goGitBase(t, c).
+		WithMountedFile(testCLIBinPath, daggerCliFile(t, c))
+
+	out, err := ctr.
+		With(daggerCallAt(testGitModuleRef("top-level"), "fn")).
+		Stdout(ctx)
+	require.NoError(t, err)
+	require.Equal(t, "hi from top level hi from dep hi from dep2", strings.TrimSpace(out))
+}
+
+type goProxyFetcher struct {
+	goproxy.GoFetcher
+	mu      sync.Mutex
+	dlPaths map[string]struct{}
+}
+
+func (f *goProxyFetcher) Download(ctx context.Context, path, version string) (info, mod, zip io.ReadSeekCloser, err error) {
+	f.mu.Lock()
+	f.dlPaths[path] = struct{}{}
+	f.mu.Unlock()
+	return f.GoFetcher.Download(ctx, path, version)
+}

core/schema/container.go

@@ -122,6 +122,13 @@ func (s *containerSchema) Install() {
 					`environment variables defined in the container (e.g.,
 				"/opt/bin:$PATH").`),
 
+		// NOTE: this is internal-only for now (hidden from codegen via the __ prefix) as we
+		// currently only want to use it for allowing the Go SDK to inherit custom GOPROXY
+		// settings from the engine container. It may be made public in the future with more
+		// refined design.
+		dagql.Func("__withSystemEnvVariable", s.withSystemEnvVariable).
+			Doc(`(Internal-only) Inherit this environment variable from the engine container if set there with a special prefix.`),
+
 		dagql.Func("withSecretVariable", s.withSecretVariable).
 			Doc(`Retrieves this container plus an env variable containing the given secret.`).
 			ArgDoc("name", `The name of the secret variable (e.g., "API_SECRET").`).
@@ -747,6 +754,16 @@ func (s *containerSchema) withEnvVariable(ctx context.Context, parent *core.Cont
 	})
 }
 
+type containerWithSystemEnvArgs struct {
+	Name string
+}
+
+func (s *containerSchema) withSystemEnvVariable(ctx context.Context, parent *core.Container, args containerWithSystemEnvArgs) (*core.Container, error) {
+	ctr := parent.Clone()
+	ctr.SystemEnvNames = append(ctr.SystemEnvNames, args.Name)
+	return ctr, nil
+}
+
 type containerWithoutVariableArgs struct {
 	Name string
 }

core/schema/sdk.go

@@ -559,47 +559,60 @@ func (sdk *goSDK) base(ctx context.Context) (dagql.Instance[*core.Container], er
 	}
 
 	var ctr dagql.Instance[*core.Container]
-	if err := sdk.dag.Select(ctx, sdk.dag.Root(), &ctr, dagql.Selector{
-		Field: "builtinContainer",
-		Args: []dagql.NamedInput{
-			{
-				Name:  "digest",
-				Value: dagql.String(os.Getenv(distconsts.GoSDKManifestDigestEnvName)),
+	if err := sdk.dag.Select(ctx, sdk.dag.Root(), &ctr,
+		dagql.Selector{
+			Field: "builtinContainer",
+			Args: []dagql.NamedInput{
+				{
+					Name:  "digest",
+					Value: dagql.String(os.Getenv(distconsts.GoSDKManifestDigestEnvName)),
+				},
 			},
 		},
-	}, dagql.Selector{
-		Field: "withMountedCache",
-		Args: []dagql.NamedInput{
-			{
-				Name:  "path",
-				Value: dagql.String("/go/pkg/mod"),
-			},
-			{
-				Name:  "cache",
-				Value: dagql.NewID[*core.CacheVolume](modCache.ID()),
-			},
-			{
-				Name:  "sharing",
-				Value: core.CacheSharingModeShared,
+		dagql.Selector{
+			Field: "withMountedCache",
+			Args: []dagql.NamedInput{
+				{
+					Name:  "path",
+					Value: dagql.String("/go/pkg/mod"),
+				},
+				{
+					Name:  "cache",
+					Value: dagql.NewID[*core.CacheVolume](modCache.ID()),
+				},
+				{
+					Name:  "sharing",
+					Value: core.CacheSharingModeShared,
+				},
 			},
 		},
-	}, dagql.Selector{
-		Field: "withMountedCache",
-		Args: []dagql.NamedInput{
-			{
-				Name:  "path",
-				Value: dagql.String("/root/.cache/go-build"),
-			},
-			{
-				Name:  "cache",
-				Value: dagql.NewID[*core.CacheVolume](buildCache.ID()),
+		dagql.Selector{
+			Field: "withMountedCache",
+			Args: []dagql.NamedInput{
+				{
+					Name:  "path",
+					Value: dagql.String("/root/.cache/go-build"),
+				},
+				{
+					Name:  "cache",
+					Value: dagql.NewID[*core.CacheVolume](buildCache.ID()),
+				},
+				{
+					Name:  "sharing",
+					Value: core.CacheSharingModeShared,
+				},
 			},
-			{
-				Name:  "sharing",
-				Value: core.CacheSharingModeShared,
+		},
+		dagql.Selector{
+			Field: "__withSystemEnvVariable",
+			Args: []dagql.NamedInput{
+				{
+					Name:  "name",
+					Value: dagql.String("GOPROXY"),
+				},
 			},
 		},
-	}); err != nil {
+	); err != nil {
 		return inst, fmt.Errorf("failed to get container from go module sdk tarball: %w", err)
 	}
 	return ctr, nil