From e2c5e6e7470ac0b3afd73f33efc9e8657322ffd5 Mon Sep 17 00:00:00 2001 From: Erik Sipsma Date: Wed, 20 Mar 2024 20:29:22 -0700 Subject: [PATCH 1/4] consolidate module caller digest to be client id Signed-off-by: Erik Sipsma --- cmd/shim/main.go | 34 ++++++------- core/container.go | 43 ++++++---------- core/modfunc.go | 42 +++++++++------- core/query.go | 36 +++++++------- core/schema/host.go | 2 +- engine/buildkit/client.go | 7 ++- engine/client/client.go | 77 ++++++++++++++--------------- engine/opts.go | 11 ++--- engine/server/buildkitcontroller.go | 1 - engine/server/server.go | 24 +++++---- 10 files changed, 129 insertions(+), 148 deletions(-) diff --git a/cmd/shim/main.go b/cmd/shim/main.go index 496db0b9d1..f011e18d9c 100644 --- a/cmd/shim/main.go +++ b/cmd/shim/main.go @@ -28,7 +28,6 @@ import ( "github.com/dagger/dagger/engine/client" "github.com/dagger/dagger/network" "github.com/google/uuid" - "github.com/opencontainers/go-digest" "github.com/opencontainers/runtime-spec/specs-go" "github.com/vito/progrock" "golang.org/x/sys/unix" @@ -452,7 +451,7 @@ func setupBundle() int { keepEnv := []string{} for _, env := range spec.Process.Env { switch { - case strings.HasPrefix(env, "_DAGGER_ENABLE_NESTING="): + case strings.HasPrefix(env, "_DAGGER_NESTED_CLIENT_ID="): // keep the env var; we use it at runtime keepEnv = append(keepEnv, env) @@ -484,7 +483,9 @@ func setupBundle() int { Options: []string{"rbind"}, Source: execMetadata.ProgSockPath, }) - case strings.HasPrefix(env, "_DAGGER_SERVER_ID="): + case strings.HasPrefix(env, "_DAGGER_ENGINE_VERSION="): + // don't need this at runtime, it is just for invalidating cache, which + // has already happened by now case strings.HasPrefix(env, aliasPrefix): // NB: don't keep this env var, it's only for the bundling step // keepEnv = append(keepEnv, env) @@ -611,7 +612,8 @@ func internalEnv(name string) (string, bool) { } func runWithNesting(ctx context.Context, cmd *exec.Cmd) error { - if _, found := internalEnv("_DAGGER_ENABLE_NESTING"); !found { + clientID, ok := internalEnv("_DAGGER_NESTED_CLIENT_ID") + if !ok { // no nesting; run as normal return execProcess(cmd, true) } @@ -629,27 +631,25 @@ func runWithNesting(ctx context.Context, cmd *exec.Cmd) error { } sessionPort := l.Addr().(*net.TCPAddr).Port + serverID, ok := internalEnv("_DAGGER_SERVER_ID") + if !ok { + return errors.New("missing nested client server ID") + } + if clientID == "" { + // if this is a new client, it gets it own isolated server + serverID = "" + } + parentClientIDsVal, _ := internalEnv("_DAGGER_PARENT_CLIENT_IDS") clientParams := client.Params{ + ID: clientID, + ServerID: serverID, SecretToken: sessionToken.String(), RunnerHost: "unix:///.runner.sock", ParentClientIDs: strings.Fields(parentClientIDsVal), } - if _, ok := internalEnv("_DAGGER_ENABLE_NESTING_IN_SAME_SESSION"); ok { - serverID, ok := internalEnv("_DAGGER_SERVER_ID") - if !ok { - return fmt.Errorf("missing _DAGGER_SERVER_ID") - } - clientParams.ServerID = serverID - } - - moduleCallerDigest, ok := internalEnv("_DAGGER_MODULE_CALLER_DIGEST") - if ok { - clientParams.ModuleCallerDigest = digest.Digest(moduleCallerDigest) - } - progW, err := progrock.DialRPC(ctx, "unix:///.progrock.sock") if err != nil { return fmt.Errorf("error connecting to progrock: %w", err) diff --git a/core/container.go b/core/container.go index 4b97e9dfca..55f4f03364 100644 --- a/core/container.go +++ b/core/container.go @@ -1021,16 +1021,15 @@ func (container *Container) WithExec(ctx context.Context, opts ContainerExecOpts // this allows executed containers to communicate back to this API if opts.ExperimentalPrivilegedNesting { - // include the engine version so that these execs get invalidated if the engine/API change - runOpts = append(runOpts, llb.AddEnv("_DAGGER_ENABLE_NESTING", engine.Version)) - } - - if opts.ModuleCallerDigest != "" { - runOpts = append(runOpts, llb.AddEnv("_DAGGER_MODULE_CALLER_DIGEST", opts.ModuleCallerDigest.String())) - } - - if opts.NestedInSameSession { - runOpts = append(runOpts, llb.AddEnv("_DAGGER_ENABLE_NESTING_IN_SAME_SESSION", "")) + // clientID may be an empty string, in which case it will get its own + // new isolated server (currently only happens for nested execs besides + // function calls) + clientID := opts.NestedClientID + runOpts = append(runOpts, + llb.AddEnv("_DAGGER_NESTED_CLIENT_ID", clientID), + // include the engine version so that these execs get invalidated if the engine/API change + llb.AddEnv("_DAGGER_ENGINE_VERSION", engine.Version), + ) } metaSt, metaSourcePath := metaMount(opts.Stdin) @@ -1071,13 +1070,7 @@ func (container *Container) WithExec(ctx context.Context, opts ContainerExecOpts } // don't pass these through to the container when manually set, they are internal only - if name == "_DAGGER_ENABLE_NESTING" && !opts.ExperimentalPrivilegedNesting { - continue - } - if name == "_DAGGER_MODULE_CALLER_DIGEST" && opts.ModuleCallerDigest == "" { - continue - } - if name == "_DAGGER_ENABLE_NESTING_IN_SAME_SESSION" && !opts.NestedInSameSession { + if name == "_DAGGER_NESTED_CLIENT_ID" && !opts.ExperimentalPrivilegedNesting { continue } @@ -1783,22 +1776,16 @@ type ContainerExecOpts struct { // Redirect the command's standard error to a file in the container RedirectStderr string `default:""` - // Provide dagger access to the executed command - // Do not use this option unless you trust the command being executed. - // The command being executed WILL BE GRANTED FULL ACCESS TO YOUR HOST FILESYSTEM + // Provide the executed command access back to the Dagger API ExperimentalPrivilegedNesting bool `default:"false"` // Grant the process all root capabilities InsecureRootCapabilities bool `default:"false"` - // (Internal-only) If this exec is for a module function, this digest will be set in the - // grpc context metadata for any api requests back to the engine. It's used by the API - // server to determine which schema to serve and other module context metadata. - ModuleCallerDigest digest.Digest `name:"-"` - - // (Internal-only) Used for module function execs to trigger the nested api client to - // be connected back to the same session. - NestedInSameSession bool `name:"-"` + // (Internal-only) If this exec is nested, this sets the client ID used to connect back + // to the API. It's used by the API server to determine which schema to serve and other + // module context metadata. + NestedClientID string `name:"-"` } type BuildArg struct { diff --git a/core/modfunc.go b/core/modfunc.go index a131036a04..cf05f88245 100644 --- a/core/modfunc.go +++ b/core/modfunc.go @@ -189,24 +189,6 @@ func (fn *ModuleFunction) Call(ctx context.Context, caller *call.ID, opts *CallO } }() - ctr := fn.runtime - - metaDir := NewScratchDirectory(mod.Query, mod.Query.Platform) - ctr, err := ctr.WithMountedDirectory(ctx, modMetaDirPath, metaDir, "", false) - if err != nil { - return nil, fmt.Errorf("failed to mount mod metadata directory: %w", err) - } - - // Setup the Exec for the Function call and evaluate it - ctr, err = ctr.WithExec(ctx, ContainerExecOpts{ - ModuleCallerDigest: callerDigest, - ExperimentalPrivilegedNesting: true, - NestedInSameSession: true, - }) - if err != nil { - return nil, fmt.Errorf("failed to exec function: %w", err) - } - parentJSON, err := json.Marshal(opts.ParentVal) if err != nil { return nil, fmt.Errorf("failed to marshal parent value: %w", err) @@ -233,12 +215,34 @@ func (fn *ModuleFunction) Call(ctx context.Context, caller *call.ID, opts *CallO deps = mod.Deps.Prepend(mod) } - err = mod.Query.RegisterFunctionCall(ctx, callerDigest, deps, fn.mod, callMeta, + // only use encoded part of digest because this ID ends up becoming a buildkit Session ID + // and buildkit has some ancient internal logic that splits on a colon to support some + // dev mode logic: https://github.com/moby/buildkit/pull/290 + // also trim it to 25 chars as it ends up becoming part of service URLs + clientID := callerDigest.Encoded()[:25] + err = mod.Query.RegisterFunctionCall(ctx, clientID, deps, fn.mod, callMeta, progrock.FromContext(ctx).Parent) if err != nil { return nil, fmt.Errorf("failed to register function call: %w", err) } + ctr := fn.runtime + + metaDir := NewScratchDirectory(mod.Query, mod.Query.Platform) + ctr, err = ctr.WithMountedDirectory(ctx, modMetaDirPath, metaDir, "", false) + if err != nil { + return nil, fmt.Errorf("failed to mount mod metadata directory: %w", err) + } + + // Setup the Exec for the Function call and evaluate it + ctr, err = ctr.WithExec(ctx, ContainerExecOpts{ + ExperimentalPrivilegedNesting: true, + NestedClientID: clientID, + }) + if err != nil { + return nil, fmt.Errorf("failed to exec function: %w", err) + } + _, err = ctr.Evaluate(ctx) if err != nil { if fn.metadata.OriginalName == "" { diff --git a/core/query.go b/core/query.go index 4470a68838..47c4d44c1e 100644 --- a/core/query.go +++ b/core/query.go @@ -14,7 +14,6 @@ import ( "github.com/dagger/dagger/engine" "github.com/dagger/dagger/engine/buildkit" "github.com/moby/buildkit/util/leaseutil" - "github.com/opencontainers/go-digest" "github.com/vektah/gqlparser/v2/ast" "github.com/vito/progrock" ) @@ -60,8 +59,9 @@ type QueryOpts struct { // For the special case of the main client caller, the key is just empty string. // This is never explicitly deleted from; instead it will just be garbage collected // when this server for the session shuts down - ClientCallContext map[digest.Digest]*ClientCallContext - ClientCallMu *sync.RWMutex + ClientCallContext map[string]*ClientCallContext + ClientCallMu *sync.RWMutex + MainClientCallerID string // the http endpoints being served (as a map since APIs like shellEndpoint can add more) Endpoints map[string]http.Handler @@ -125,7 +125,7 @@ func (q *Query) ServeModuleToMainClient(ctx context.Context, modMeta dagql.Insta if err != nil { return err } - if clientMetadata.ModuleCallerDigest != "" { + if clientMetadata.ClientID != q.MainClientCallerID { return fmt.Errorf("cannot serve module to client %s", clientMetadata.ClientID) } @@ -133,7 +133,7 @@ func (q *Query) ServeModuleToMainClient(ctx context.Context, modMeta dagql.Insta q.ClientCallMu.Lock() defer q.ClientCallMu.Unlock() - callCtx, ok := q.ClientCallContext[""] + callCtx, ok := q.ClientCallContext[clientMetadata.ClientID] if !ok { return fmt.Errorf("client call not found") } @@ -143,19 +143,19 @@ func (q *Query) ServeModuleToMainClient(ctx context.Context, modMeta dagql.Insta func (q *Query) RegisterFunctionCall( ctx context.Context, - dgst digest.Digest, + clientID string, deps *ModDeps, mod *Module, call *FunctionCall, progrockParent string, ) error { - if dgst == "" { - return fmt.Errorf("cannot register function call with empty digest") + if clientID == "" { + return fmt.Errorf("cannot register function call with empty clientID") } q.ClientCallMu.Lock() defer q.ClientCallMu.Unlock() - _, ok := q.ClientCallContext[dgst] + _, ok := q.ClientCallContext[clientID] if ok { return nil } @@ -163,7 +163,7 @@ func (q *Query) RegisterFunctionCall( if err != nil { return err } - q.ClientCallContext[dgst] = &ClientCallContext{ + q.ClientCallContext[clientID] = &ClientCallContext{ Root: newRoot, Deps: deps, Module: mod, @@ -178,15 +178,15 @@ func (q *Query) CurrentModule(ctx context.Context) (*Module, error) { if err != nil { return nil, err } - if clientMetadata.ModuleCallerDigest == "" { + if clientMetadata.ClientID == q.MainClientCallerID { return nil, fmt.Errorf("%w: main client caller has no module", ErrNoCurrentModule) } q.ClientCallMu.RLock() defer q.ClientCallMu.RUnlock() - callCtx, ok := q.ClientCallContext[clientMetadata.ModuleCallerDigest] + callCtx, ok := q.ClientCallContext[clientMetadata.ClientID] if !ok { - return nil, fmt.Errorf("client call %s not found", clientMetadata.ModuleCallerDigest) + return nil, fmt.Errorf("client call %s not found", clientMetadata.ClientID) } return callCtx.Module, nil } @@ -196,15 +196,15 @@ func (q *Query) CurrentFunctionCall(ctx context.Context) (*FunctionCall, error) if err != nil { return nil, err } - if clientMetadata.ModuleCallerDigest == "" { + if clientMetadata.ClientID == q.MainClientCallerID { return nil, fmt.Errorf("%w: main client caller has no function", ErrNoCurrentModule) } q.ClientCallMu.RLock() defer q.ClientCallMu.RUnlock() - callCtx, ok := q.ClientCallContext[clientMetadata.ModuleCallerDigest] + callCtx, ok := q.ClientCallContext[clientMetadata.ClientID] if !ok { - return nil, fmt.Errorf("client call %s not found", clientMetadata.ModuleCallerDigest) + return nil, fmt.Errorf("client call %s not found", clientMetadata.ClientID) } return callCtx.FnCall, nil @@ -215,9 +215,9 @@ func (q *Query) CurrentServedDeps(ctx context.Context) (*ModDeps, error) { if err != nil { return nil, err } - callCtx, ok := q.ClientCallContext[clientMetadata.ModuleCallerDigest] + callCtx, ok := q.ClientCallContext[clientMetadata.ClientID] if !ok { - return nil, fmt.Errorf("client call %s not found", clientMetadata.ModuleCallerDigest) + return nil, fmt.Errorf("client call %s not found", clientMetadata.ClientID) } return callCtx.Deps, nil } diff --git a/core/schema/host.go b/core/schema/host.go index a30b1c8618..03e0d7d956 100644 --- a/core/schema/host.go +++ b/core/schema/host.go @@ -206,7 +206,7 @@ func (s *hostSchema) socket(ctx context.Context, host *core.Host, args hostSocke if err != nil { return nil, fmt.Errorf("failed to get client metadata: %w", err) } - if clientMetadata.ClientID != host.Query.Buildkit.MainClientCallerID { + if clientMetadata.ClientID != host.Query.MainClientCallerID { return nil, fmt.Errorf("only the main client can access the host's unix sockets") } diff --git a/engine/buildkit/client.go b/engine/buildkit/client.go index 50aa824342..f7e8a9702d 100644 --- a/engine/buildkit/client.go +++ b/engine/buildkit/client.go @@ -59,10 +59,9 @@ type Opts struct { // client. It is special in that when it shuts down, the client will be closed and // that registry auth and sockets are currently only ever sourced from this caller, // not any nested clients (may change in future). - MainClientCaller bksession.Caller - MainClientCallerID string - DNSConfig *oci.DNSConfig - Frontends map[string]bkfrontend.Frontend + MainClientCaller bksession.Caller + DNSConfig *oci.DNSConfig + Frontends map[string]bkfrontend.Frontend sharedClientState } diff --git a/engine/client/client.go b/engine/client/client.go index a6b278920b..3d483840b3 100644 --- a/engine/client/client.go +++ b/engine/client/client.go @@ -32,7 +32,6 @@ import ( "github.com/moby/buildkit/session/filesync" "github.com/moby/buildkit/session/grpchijack" "github.com/moby/buildkit/util/grpcerrors" - "github.com/opencontainers/go-digest" "github.com/tonistiigi/fsutil" fstypes "github.com/tonistiigi/fsutil/types" "github.com/vito/progrock" @@ -50,8 +49,13 @@ import ( const ProgrockParentHeader = "X-Progrock-Parent" type Params struct { - // The id of the server to connect to, or if blank a new one - // should be started. + // The id to connect to the API server with. If blank, will be set to a + // new random value. + ID string + + // The id of the server to connect to, or if blank a new one should be started. + // Needed separately from the client ID as that ID is a digest which could + // be reused across multiple servers. ServerID string // Parent client IDs of this Dagger client. @@ -72,11 +76,6 @@ type Params struct { ProgrockParent string EngineNameCallback func(string) CloudURLCallback func(string) - - // If this client is for a module function, this digest will be set in the - // grpc context metadata for any api requests back to the engine. It's used by the API - // server to determine which schema to serve and other module context metadata. - ModuleCallerDigest digest.Digest } type Client struct { @@ -111,12 +110,15 @@ type Client struct { func Connect(ctx context.Context, params Params) (_ *Client, _ context.Context, rerr error) { c := &Client{Params: params} - if c.SecretToken == "" { - c.SecretToken = uuid.New().String() + if c.ID == "" { + c.ID = identity.NewID() } if c.ServerID == "" { c.ServerID = identity.NewID() } + if c.SecretToken == "" { + c.SecretToken = uuid.New().String() + } c.internalCtx, c.internalCancel = context.WithCancel(context.Background()) c.eg, c.internalCtx = errgroup.WithContext(c.internalCtx) @@ -180,19 +182,15 @@ func Connect(ctx context.Context, params Params) (_ *Client, _ context.Context, return c, ctx, nil } - // sneakily using ModuleCallerDigest here because it seems nicer than just + // sneakily using the client ID here because it seems nicer than just // making something up, and should be pretty much 1:1 I think (even // non-cached things will have a different caller digest each time) - connectDigest := params.ModuleCallerDigest + connectDigest := params.ID var opts []progrock.VertexOpt - if connectDigest == "" { - connectDigest = digest.FromString("_root") // arbitrary - } else { - opts = append(opts, progrock.Internal()) - } + opts = append(opts, progrock.Internal()) // NB: don't propagate this ctx, we don't want everything tucked beneath connect - _, loader := progrock.Span(ctx, connectDigest.String(), "connect", opts...) + _, loader := progrock.Span(ctx, connectDigest, "connect", opts...) defer func() { loader.Done(rerr) }() // Check if any of the upstream cache importers/exporters are enabled. @@ -260,13 +258,11 @@ func Connect(ctx context.Context, params Params) (_ *Client, _ context.Context, c.labels = labels c.internalCtx = engine.ContextWithClientMetadata(c.internalCtx, &engine.ClientMetadata{ - ClientID: c.ID(), - ClientSecretToken: c.SecretToken, - ServerID: c.ServerID, - ClientHostname: c.hostname, - Labels: c.labels, - ParentClientIDs: c.ParentClientIDs, - ModuleCallerDigest: c.ModuleCallerDigest, + ClientID: c.ID, + ClientSecretToken: c.SecretToken, + ClientHostname: c.hostname, + Labels: c.labels, + ParentClientIDs: c.ParentClientIDs, }) // progress @@ -294,17 +290,25 @@ func Connect(ctx context.Context, params Params) (_ *Client, _ context.Context, // already started c.eg.Go(func() error { return bkSession.Run(c.internalCtx, func(ctx context.Context, proto string, meta map[string][]string) (net.Conn, error) { + // overwrite the session ID to be our client ID + const sessionIDHeader = "X-Docker-Expose-Session-Uuid" + if _, ok := meta[sessionIDHeader]; !ok { + // should never happen unless upstream changes the value of the header key, + // in which case we want to know + return nil, fmt.Errorf("missing header %s", sessionIDHeader) + } + meta[sessionIDHeader] = []string{c.ID} + return grpchijack.Dialer(c.bkClient.ControlClient())(ctx, proto, engine.ClientMetadata{ RegisterClient: true, - ClientID: c.ID(), - ClientSecretToken: c.SecretToken, + ClientID: c.ID, ServerID: c.ServerID, + ClientSecretToken: c.SecretToken, ParentClientIDs: c.ParentClientIDs, ClientHostname: hostname, UpstreamCacheImportConfig: c.upstreamCacheImportOptions, UpstreamCacheExportConfig: c.upstreamCacheExportOptions, Labels: c.labels, - ModuleCallerDigest: c.ModuleCallerDigest, CloudToken: os.Getenv("DAGGER_CLOUD_TOKEN"), DoNotTrack: analytics.DoNotTrack(), }.AppendToMD(meta)) @@ -461,10 +465,6 @@ func (c *Client) withClientCloseCancel(ctx context.Context) (context.Context, co return ctx, cancel, nil } -func (c *Client) ID() string { - return c.bkSession.ID() -} - func (c *Client) DialContext(ctx context.Context, _, _ string) (conn net.Conn, err error) { // NOTE: the context given to grpchijack.Dialer is for the lifetime of the stream. // If http connection re-use is enabled, that can be far past this DialContext call. @@ -481,13 +481,12 @@ func (c *Client) DialContext(ctx context.Context, _, _ string) (conn net.Conn, e }).Dial("tcp", "127.0.0.1:"+strconv.Itoa(c.nestedSessionPort)) } else { conn, err = grpchijack.Dialer(c.bkClient.ControlClient())(ctx, "", engine.ClientMetadata{ - ClientID: c.ID(), - ClientSecretToken: c.SecretToken, - ServerID: c.ServerID, - ClientHostname: c.hostname, - ParentClientIDs: c.ParentClientIDs, - Labels: c.labels, - ModuleCallerDigest: c.ModuleCallerDigest, + ClientID: c.ID, + ServerID: c.ServerID, + ClientSecretToken: c.SecretToken, + ClientHostname: c.hostname, + ParentClientIDs: c.ParentClientIDs, + Labels: c.labels, }.ToGRPCMD()) } if err != nil { diff --git a/engine/opts.go b/engine/opts.go index b8a14fc6c5..22f27b1d4f 100644 --- a/engine/opts.go +++ b/engine/opts.go @@ -9,7 +9,6 @@ import ( "github.com/dagger/dagger/core/pipeline" controlapi "github.com/moby/buildkit/api/services/control" - "github.com/opencontainers/go-digest" "google.golang.org/grpc/metadata" ) @@ -28,11 +27,12 @@ const ( ) type ClientMetadata struct { - // ClientID is unique to every session created by every client + // ClientID is unique to each client. The main client's ID is the empty string, + // any module and/or nested exec client's ID is a unique digest. ClientID string `json:"client_id"` // ClientSecretToken is a secret token that is unique to every client. It's - // initially provided to the server in the controller.Solve request. Every + // initially provided to the server in the controller.Session request. Every // other request w/ that client ID must also include the same token. ClientSecretToken string `json:"client_secret_token"` @@ -61,11 +61,6 @@ type ClientMetadata struct { // parent of the parent, and so on. ParentClientIDs []string `json:"parent_client_ids"` - // If this client is for a module function, this digest will be set in the - // grpc context metadata for any api requests back to the engine. It's used by the API - // server to determine which schema to serve and other module context metadata. - ModuleCallerDigest digest.Digest `json:"module_caller_digest"` - // Import configuration for Buildkit's remote cache UpstreamCacheImportConfig []*controlapi.CacheOptionsEntry diff --git a/engine/server/buildkitcontroller.go b/engine/server/buildkitcontroller.go index 4353b6905b..67b4fb7d0f 100644 --- a/engine/server/buildkitcontroller.go +++ b/engine/server/buildkitcontroller.go @@ -153,7 +153,6 @@ func (e *BuildkitController) Session(stream controlapi.Control_SessionServer) (r ctx = bklog.WithLogger(ctx, bklog.G(ctx). WithField("client_id", opts.ClientID). WithField("client_hostname", opts.ClientHostname). - WithField("client_call_digest", opts.ModuleCallerDigest). WithField("server_id", opts.ServerID)) bklog.G(ctx).WithField("register_client", opts.RegisterClient).Debug("handling session call") defer func() { diff --git a/engine/server/server.go b/engine/server/server.go index 4fdec56127..4b05e220f2 100644 --- a/engine/server/server.go +++ b/engine/server/server.go @@ -32,7 +32,6 @@ import ( "github.com/moby/buildkit/identity" "github.com/moby/buildkit/session" "github.com/moby/buildkit/util/bklog" - "github.com/opencontainers/go-digest" "github.com/sirupsen/logrus" "github.com/vektah/gqlparser/v2/gqlerror" "github.com/vito/progrock" @@ -46,10 +45,9 @@ type DaggerServer struct { clientIDMu sync.RWMutex // The metadata of client calls. - // For the special case of the main client caller, the key is just empty string. // This is never explicitly deleted from; instead it will just be garbage collected // when this server for the session shuts down - clientCallContext map[digest.Digest]*core.ClientCallContext + clientCallContext map[string]*core.ClientCallContext clientCallMu *sync.RWMutex // the http endpoints being served (as a map since APIs like shellEndpoint can add more) @@ -75,7 +73,7 @@ func (e *BuildkitController) newDaggerServer(ctx context.Context, clientMetadata serverID: clientMetadata.ServerID, clientIDToSecretToken: map[string]string{}, - clientCallContext: map[digest.Digest]*core.ClientCallContext{}, + clientCallContext: map[string]*core.ClientCallContext{}, clientCallMu: &sync.RWMutex{}, endpoints: map[string]http.Handler{}, endpointMu: &sync.RWMutex{}, @@ -170,7 +168,6 @@ func (e *BuildkitController) newDaggerServer(ctx context.Context, clientMetadata UpstreamCacheImports: cacheImporterCfgs, ProgSockPath: progSockPath, MainClientCaller: sessionCaller, - MainClientCallerID: s.mainClientCallerID, DNSConfig: e.DNSConfig, Frontends: e.Frontends, }, @@ -183,6 +180,7 @@ func (e *BuildkitController) newDaggerServer(ctx context.Context, clientMetadata Auth: authProvider, ClientCallContext: s.clientCallContext, ClientCallMu: s.clientCallMu, + MainClientCallerID: s.mainClientCallerID, Endpoints: s.endpoints, EndpointMu: s.endpointMu, Recorder: s.recorder, @@ -205,7 +203,7 @@ func (e *BuildkitController) newDaggerServer(ctx context.Context, clientMetadata } // the main client caller starts out with the core API loaded - s.clientCallContext[""] = &core.ClientCallContext{ + s.clientCallContext[s.mainClientCallerID] = &core.ClientCallContext{ Deps: root.DefaultDeps, Root: root, } @@ -268,9 +266,9 @@ func (s *DaggerServer) ServeHTTP(w http.ResponseWriter, r *http.Request) { return } - callContext, ok := s.ClientCallContext(clientMetadata.ModuleCallerDigest) + callContext, ok := s.ClientCallContext(clientMetadata.ClientID) if !ok { - errorOut(fmt.Errorf("client call %s not found", clientMetadata.ModuleCallerDigest), http.StatusInternalServerError) + errorOut(fmt.Errorf("client call for %s not found", clientMetadata.ClientID), http.StatusInternalServerError) return } @@ -345,7 +343,7 @@ func (s *DaggerServer) ServeHTTP(w http.ResponseWriter, r *http.Request) { } } s.clientCallMu.RLock() - bk := s.clientCallContext[""].Root.Buildkit + bk := s.clientCallContext[s.mainClientCallerID].Root.Buildkit s.clientCallMu.RUnlock() err := bk.UpstreamCacheExport(ctx, cacheExporterFuncs) if err != nil { @@ -399,10 +397,10 @@ func (s *DaggerServer) VerifyClient(clientID, secretToken string) error { return nil } -func (s *DaggerServer) ClientCallContext(clientDigest digest.Digest) (*core.ClientCallContext, bool) { +func (s *DaggerServer) ClientCallContext(clientID string) (*core.ClientCallContext, bool) { s.clientCallMu.RLock() defer s.clientCallMu.RUnlock() - ctx, ok := s.clientCallContext[clientDigest] + ctx, ok := s.clientCallContext[clientID] return ctx, ok } @@ -411,9 +409,9 @@ func (s *DaggerServer) CurrentServedDeps(ctx context.Context) (*core.ModDeps, er if err != nil { return nil, err } - callCtx, ok := s.ClientCallContext(clientMetadata.ModuleCallerDigest) + callCtx, ok := s.ClientCallContext(clientMetadata.ClientID) if !ok { - return nil, fmt.Errorf("client call %s not found", clientMetadata.ModuleCallerDigest) + return nil, fmt.Errorf("client call for %s not found", clientMetadata.ClientID) } return callCtx.Deps, nil } From 416292f03613b8126eb449cc69f0135946f110b8 Mon Sep 17 00:00:00 2001 From: Erik Sipsma Date: Thu, 21 Mar 2024 15:28:56 -0700 Subject: [PATCH 2/4] reuse server for nested execs; register them as calls too Signed-off-by: Erik Sipsma --- cmd/shim/main.go | 4 -- core/container.go | 15 +++-- core/interface.go | 2 +- core/modfunc.go | 56 +++--------------- core/module.go | 2 +- core/object.go | 9 ++- core/query.go | 88 ++++++++++++++++++----------- core/schema/module.go | 2 +- core/typedef.go | 11 ++++ engine/buildkit/client.go | 8 +-- engine/buildkit/containerimage.go | 2 +- engine/buildkit/filesync.go | 46 +++++---------- engine/buildkit/session.go | 17 +++++- engine/client/client.go | 9 --- engine/opts.go | 10 ++++ engine/server/buildkitcontroller.go | 9 +++ engine/server/server.go | 5 +- 17 files changed, 142 insertions(+), 153 deletions(-) diff --git a/cmd/shim/main.go b/cmd/shim/main.go index f011e18d9c..dc7759f7bd 100644 --- a/cmd/shim/main.go +++ b/cmd/shim/main.go @@ -635,10 +635,6 @@ func runWithNesting(ctx context.Context, cmd *exec.Cmd) error { if !ok { return errors.New("missing nested client server ID") } - if clientID == "" { - // if this is a new client, it gets it own isolated server - serverID = "" - } parentClientIDsVal, _ := internalEnv("_DAGGER_PARENT_CLIENT_IDS") diff --git a/core/container.go b/core/container.go index 55f4f03364..93d314f79f 100644 --- a/core/container.go +++ b/core/container.go @@ -1021,10 +1021,10 @@ func (container *Container) WithExec(ctx context.Context, opts ContainerExecOpts // this allows executed containers to communicate back to this API if opts.ExperimentalPrivilegedNesting { - // clientID may be an empty string, in which case it will get its own - // new isolated server (currently only happens for nested execs besides - // function calls) - clientID := opts.NestedClientID + clientID, err := container.Query.RegisterCaller(ctx, opts.NestedExecFunctionCall) + if err != nil { + return nil, fmt.Errorf("register caller: %w", err) + } runOpts = append(runOpts, llb.AddEnv("_DAGGER_NESTED_CLIENT_ID", clientID), // include the engine version so that these execs get invalidated if the engine/API change @@ -1782,10 +1782,9 @@ type ContainerExecOpts struct { // Grant the process all root capabilities InsecureRootCapabilities bool `default:"false"` - // (Internal-only) If this exec is nested, this sets the client ID used to connect back - // to the API. It's used by the API server to determine which schema to serve and other - // module context metadata. - NestedClientID string `name:"-"` + // (Internal-only) If this is a nested exec for a Function call, this should be set + // with the metadata for that call + NestedExecFunctionCall *FunctionCall `name:"-"` } type BuildArg struct { diff --git a/core/interface.go b/core/interface.go index 83ea883601..345a04e5b9 100644 --- a/core/interface.go +++ b/core/interface.go @@ -221,7 +221,7 @@ func (iface *InterfaceType) Install(ctx context.Context, dag *dagql.Server) erro }) } - res, err := callable.Call(ctx, dagql.CurrentID(ctx), &CallOpts{ + res, err := callable.Call(ctx, &CallOpts{ Inputs: callInputs, ParentVal: runtimeVal.Fields, }) diff --git a/core/modfunc.go b/core/modfunc.go index cf05f88245..9e4902fa7b 100644 --- a/core/modfunc.go +++ b/core/modfunc.go @@ -10,13 +10,11 @@ import ( "github.com/dagger/dagger/core/pipeline" "github.com/dagger/dagger/dagql" "github.com/dagger/dagger/dagql/call" - "github.com/dagger/dagger/engine" "github.com/dagger/dagger/engine/buildkit" bkgw "github.com/moby/buildkit/frontend/gateway/client" "github.com/moby/buildkit/util/bklog" "github.com/opencontainers/go-digest" ocispecs "github.com/opencontainers/image-spec/specs-go/v1" - "github.com/vito/progrock" ) type ModuleFunction struct { @@ -111,7 +109,7 @@ func (fn *ModuleFunction) recordCall(ctx context.Context) { analytics.Ctx(ctx).Capture(ctx, "module_call", props) } -func (fn *ModuleFunction) Call(ctx context.Context, caller *call.ID, opts *CallOpts) (t dagql.Typed, rerr error) { +func (fn *ModuleFunction) Call(ctx context.Context, opts *CallOpts) (t dagql.Typed, rerr error) { mod := fn.mod lg := bklog.G(ctx).WithField("module", mod.Name()).WithField("function", fn.metadata.Name) @@ -164,23 +162,6 @@ func (fn *ModuleFunction) Call(ctx context.Context, caller *call.ID, opts *CallO }) } - callerDigestInputs := []string{} - { - callerIDDigest := caller.Digest() // FIXME(vito) canonicalize, once all that's implemented - callerDigestInputs = append(callerDigestInputs, callerIDDigest.String()) - } - if !opts.Cache { - // use the ServerID so that we bust cache once-per-session - clientMetadata, err := engine.ClientMetadataFromContext(ctx) - if err != nil { - return nil, fmt.Errorf("failed to get client metadata: %w", err) - } - callerDigestInputs = append(callerDigestInputs, clientMetadata.ServerID) - } - - callerDigest := digest.FromString(strings.Join(callerDigestInputs, " ")) - - ctx = bklog.WithLogger(ctx, bklog.G(ctx).WithField("caller_digest", callerDigest.String())) bklog.G(ctx).Debug("function call") defer func() { bklog.G(ctx).Debug("function call done") @@ -195,37 +176,18 @@ func (fn *ModuleFunction) Call(ctx context.Context, caller *call.ID, opts *CallO } callMeta := &FunctionCall{ - Query: fn.root, - Name: fn.metadata.OriginalName, - Parent: parentJSON, - InputArgs: callInputs, + Query: fn.root, + Name: fn.metadata.OriginalName, + Parent: parentJSON, + InputArgs: callInputs, + Module: mod, + Cache: opts.Cache, + SkipSelfSchema: opts.SkipSelfSchema, } if fn.objDef != nil { callMeta.ParentName = fn.objDef.OriginalName } - var deps *ModDeps - if opts.SkipSelfSchema { - // Only serve the APIs of the deps of this module. This is currently only needed for the special - // case of the function used to get the definition of the module itself (which can't obviously - // be served the API its returning the definition of). - deps = mod.Deps - } else { - // by default, serve both deps and the module's own API to itself - deps = mod.Deps.Prepend(mod) - } - - // only use encoded part of digest because this ID ends up becoming a buildkit Session ID - // and buildkit has some ancient internal logic that splits on a colon to support some - // dev mode logic: https://github.com/moby/buildkit/pull/290 - // also trim it to 25 chars as it ends up becoming part of service URLs - clientID := callerDigest.Encoded()[:25] - err = mod.Query.RegisterFunctionCall(ctx, clientID, deps, fn.mod, callMeta, - progrock.FromContext(ctx).Parent) - if err != nil { - return nil, fmt.Errorf("failed to register function call: %w", err) - } - ctr := fn.runtime metaDir := NewScratchDirectory(mod.Query, mod.Query.Platform) @@ -237,7 +199,7 @@ func (fn *ModuleFunction) Call(ctx context.Context, caller *call.ID, opts *CallO // Setup the Exec for the Function call and evaluate it ctr, err = ctr.WithExec(ctx, ContainerExecOpts{ ExperimentalPrivilegedNesting: true, - NestedClientID: clientID, + NestedExecFunctionCall: callMeta, }) if err != nil { return nil, fmt.Errorf("failed to exec function: %w", err) diff --git a/core/module.go b/core/module.go index 3734011893..6bbf7b5f26 100644 --- a/core/module.go +++ b/core/module.go @@ -137,7 +137,7 @@ func (mod *Module) Initialize(ctx context.Context, oldSelf dagql.Instance[*Modul return nil, fmt.Errorf("failed to create module definition function for module %q: %w", mod.Name(), err) } - result, err := getModDefFn.Call(ctx, newID, &CallOpts{Cache: true, SkipSelfSchema: true}) + result, err := getModDefFn.Call(ctx, &CallOpts{Cache: true, SkipSelfSchema: true}) if err != nil { return nil, fmt.Errorf("failed to call module %q to get functions: %w", mod.Name(), err) } diff --git a/core/object.go b/core/object.go index 7222457dca..a9562d9328 100644 --- a/core/object.go +++ b/core/object.go @@ -7,7 +7,6 @@ import ( "sort" "github.com/dagger/dagger/dagql" - "github.com/dagger/dagger/dagql/call" "github.com/moby/buildkit/solver/pb" "github.com/vektah/gqlparser/v2/ast" ) @@ -84,7 +83,7 @@ func (t *ModuleObjectType) TypeDef() *TypeDef { } type Callable interface { - Call(context.Context, *call.ID, *CallOpts) (dagql.Typed, error) + Call(context.Context, *CallOpts) (dagql.Typed, error) ReturnType() (ModType, error) ArgType(argName string) (ModType, error) } @@ -254,7 +253,7 @@ func (obj *ModuleObject) installConstructor(ctx context.Context, dag *dagql.Serv Value: v, }) } - return fn.Call(ctx, dagql.CurrentID(ctx), &CallOpts{ + return fn.Call(ctx, &CallOpts{ Inputs: callInput, ParentVal: nil, }) @@ -351,7 +350,7 @@ func objFun(ctx context.Context, mod *Module, objDef *ObjectTypeDef, fun *Functi sort.Slice(opts.Inputs, func(i, j int) bool { return opts.Inputs[i].Name < opts.Inputs[j].Name }) - return modFun.Call(ctx, dagql.CurrentID(ctx), opts) + return modFun.Call(ctx, opts) }, }, nil } @@ -362,7 +361,7 @@ type CallableField struct { Return ModType } -func (f *CallableField) Call(ctx context.Context, id *call.ID, opts *CallOpts) (dagql.Typed, error) { +func (f *CallableField) Call(ctx context.Context, opts *CallOpts) (dagql.Typed, error) { val, ok := opts.ParentVal[f.Field.OriginalName] if !ok { return nil, fmt.Errorf("field %q not found on object %q", f.Field.Name, opts.ParentVal) diff --git a/core/query.go b/core/query.go index 47c4d44c1e..74f9b955b1 100644 --- a/core/query.go +++ b/core/query.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "net/http" + "strings" "sync" "github.com/containerd/containerd/content" @@ -14,6 +15,7 @@ import ( "github.com/dagger/dagger/engine" "github.com/dagger/dagger/engine/buildkit" "github.com/moby/buildkit/util/leaseutil" + "github.com/opencontainers/go-digest" "github.com/vektah/gqlparser/v2/ast" "github.com/vito/progrock" ) @@ -114,22 +116,16 @@ type ClientCallContext struct { // If the client is itself from a function call in a user module, these are set with the // metadata of that ongoing function call - Module *Module FnCall *FunctionCall ProgrockParent string } -func (q *Query) ServeModuleToMainClient(ctx context.Context, modMeta dagql.Instance[*Module]) error { +func (q *Query) ServeModule(ctx context.Context, mod *Module) error { clientMetadata, err := engine.ClientMetadataFromContext(ctx) if err != nil { return err } - if clientMetadata.ClientID != q.MainClientCallerID { - return fmt.Errorf("cannot serve module to client %s", clientMetadata.ClientID) - } - - mod := modMeta.Self q.ClientCallMu.Lock() defer q.ClientCallMu.Unlock() @@ -141,36 +137,64 @@ func (q *Query) ServeModuleToMainClient(ctx context.Context, modMeta dagql.Insta return nil } -func (q *Query) RegisterFunctionCall( - ctx context.Context, - clientID string, - deps *ModDeps, - mod *Module, - call *FunctionCall, - progrockParent string, -) error { - if clientID == "" { - return fmt.Errorf("cannot register function call with empty clientID") +func (q *Query) RegisterCaller(ctx context.Context, call *FunctionCall) (string, error) { + if call == nil { + call = &FunctionCall{} + } + callCtx := &ClientCallContext{ + FnCall: call, + ProgrockParent: progrock.FromContext(ctx).Parent, + } + + currentID := dagql.CurrentID(ctx) + clientIDInputs := []string{currentID.Digest().String()} + if !call.Cache { + // use the ServerID so that we bust cache once-per-session + clientMetadata, err := engine.ClientMetadataFromContext(ctx) + if err != nil { + return "", err + } + clientIDInputs = append(clientIDInputs, clientMetadata.ServerID) + } + clientIDDigest := digest.FromString(strings.Join(clientIDInputs, " ")) + + // only use encoded part of digest because this ID ends up becoming a buildkit Session ID + // and buildkit has some ancient internal logic that splits on a colon to support some + // dev mode logic: https://github.com/moby/buildkit/pull/290 + // also trim it to 25 chars as it ends up becoming part of service URLs + clientID := clientIDDigest.Encoded()[:25] + + // break glass for debugging which client is which operation + // bklog.G(ctx).Debugf("CLIENT ID %s = %s", clientID, currentID.Display()) + + if call.Module == nil { + callCtx.Deps = q.DefaultDeps + } else { + callCtx.Deps = call.Module.Deps + // By default, serve both deps and the module's own API to itself. But if SkipSelfSchema is set, + // only serve the APIs of the deps of this module. This is currently only needed for the special + // case of the function used to get the definition of the module itself (which can't obviously + // be served the API its returning the definition of). + if !call.SkipSelfSchema { + callCtx.Deps = callCtx.Deps.Append(call.Module) + } } q.ClientCallMu.Lock() defer q.ClientCallMu.Unlock() _, ok := q.ClientCallContext[clientID] if ok { - return nil + return clientID, nil } - newRoot, err := NewRoot(ctx, q.QueryOpts) + + var err error + callCtx.Root, err = NewRoot(ctx, q.QueryOpts) if err != nil { - return err + return "", err } - q.ClientCallContext[clientID] = &ClientCallContext{ - Root: newRoot, - Deps: deps, - Module: mod, - FnCall: call, - ProgrockParent: progrockParent, - } - return nil + + q.ClientCallContext[clientID] = callCtx + return clientID, nil } func (q *Query) CurrentModule(ctx context.Context) (*Module, error) { @@ -178,9 +202,6 @@ func (q *Query) CurrentModule(ctx context.Context) (*Module, error) { if err != nil { return nil, err } - if clientMetadata.ClientID == q.MainClientCallerID { - return nil, fmt.Errorf("%w: main client caller has no module", ErrNoCurrentModule) - } q.ClientCallMu.RLock() defer q.ClientCallMu.RUnlock() @@ -188,7 +209,10 @@ func (q *Query) CurrentModule(ctx context.Context) (*Module, error) { if !ok { return nil, fmt.Errorf("client call %s not found", clientMetadata.ClientID) } - return callCtx.Module, nil + if callCtx.FnCall.Module == nil { + return nil, ErrNoCurrentModule + } + return callCtx.FnCall.Module, nil } func (q *Query) CurrentFunctionCall(ctx context.Context) (*FunctionCall, error) { diff --git a/core/schema/module.go b/core/schema/module.go index 55d114a2fb..f54c9ef8a8 100644 --- a/core/schema/module.go +++ b/core/schema/module.go @@ -464,7 +464,7 @@ func (s *moduleSchema) currentFunctionCall(ctx context.Context, self *core.Query } func (s *moduleSchema) moduleServe(ctx context.Context, modMeta dagql.Instance[*core.Module], _ struct{}) (dagql.Nullable[core.Void], error) { - return dagql.Null[core.Void](), modMeta.Self.Query.ServeModuleToMainClient(ctx, modMeta) + return dagql.Null[core.Void](), modMeta.Self.Query.ServeModule(ctx, modMeta.Self) } func (s *moduleSchema) currentTypeDefs(ctx context.Context, self *core.Query, _ struct{}) ([]*core.TypeDef, error) { diff --git a/core/typedef.go b/core/typedef.go index c01d1b6730..795a4ee6b0 100644 --- a/core/typedef.go +++ b/core/typedef.go @@ -842,6 +842,17 @@ type FunctionCall struct { ParentName string `field:"true" doc:"The name of the parent object of the function being called. If the function is top-level to the module, this is the name of the module."` Parent JSON `field:"true" doc:"The value of the parent object of the function being called. If the function is top-level to the module, this is always an empty object."` InputArgs []*FunctionCallArgValue `field:"true" doc:"The argument values the function is being invoked with."` + + // Below are not in public API + + // The module that the function is being called from + Module *Module + + // Whether the function call should be cached across different servers + Cache bool + + // Whether to serve the schema for the function's own module to it or not + SkipSelfSchema bool } func (*FunctionCall) Type() *ast.Type { diff --git a/engine/buildkit/client.go b/engine/buildkit/client.go index f7e8a9702d..6a7abe56be 100644 --- a/engine/buildkit/client.go +++ b/engine/buildkit/client.go @@ -615,13 +615,7 @@ func (c *Client) ListenHostToContainer( return nil, nil, err } - clientMetadata, err := engine.ClientMetadataFromContext(ctx) - if err != nil { - cancel() - return nil, nil, fmt.Errorf("failed to get requester session ID: %s", err) - } - - clientCaller, err := c.SessionManager.Get(ctx, clientMetadata.ClientID, false) + clientCaller, err := c.GetSessionCaller(ctx, false) if err != nil { cancel() return nil, nil, fmt.Errorf("failed to get requester session: %s", err) diff --git a/engine/buildkit/containerimage.go b/engine/buildkit/containerimage.go index 7e6c877848..fa886f0310 100644 --- a/engine/buildkit/containerimage.go +++ b/engine/buildkit/containerimage.go @@ -108,7 +108,7 @@ func (c *Client) ExportContainerImage( IsFileStream: true, }.AppendToOutgoingContext(ctx) - resp, descRef, err := expInstance.Export(ctx, combinedResult, nil, clientMetadata.ClientID) + resp, descRef, err := expInstance.Export(ctx, combinedResult, nil, clientMetadata.BuildkitSessionID()) if err != nil { return nil, fmt.Errorf("failed to export: %s", err) } diff --git a/engine/buildkit/filesync.go b/engine/buildkit/filesync.go index 096f71c226..2af3bb87b5 100644 --- a/engine/buildkit/filesync.go +++ b/engine/buildkit/filesync.go @@ -46,7 +46,7 @@ func (c *Client) LocalImport( } localOpts := []llb.LocalOption{ - llb.SessionID(clientMetadata.ClientID), + llb.SessionID(clientMetadata.BuildkitSessionID()), llb.SharedKeyHint(strings.Join([]string{clientMetadata.ClientHostname, srcPath}, " ")), } @@ -112,18 +112,13 @@ func (c *Client) ReadCallerHostFile(ctx context.Context, path string) ([]byte, e } defer cancel() - clientMetadata, err := engine.ClientMetadataFromContext(ctx) - if err != nil { - return nil, fmt.Errorf("failed to get requester session ID: %s", err) - } - ctx = engine.LocalImportOpts{ Path: path, ReadSingleFileOnly: true, MaxFileSize: MaxFileContentsChunkSize, }.AppendToOutgoingContext(ctx) - clientCaller, err := c.SessionManager.Get(ctx, clientMetadata.ClientID, false) + clientCaller, err := c.GetSessionCaller(ctx, true) if err != nil { return nil, fmt.Errorf("failed to get requester session: %s", err) } @@ -147,18 +142,13 @@ func (c *Client) StatCallerHostPath(ctx context.Context, path string, returnAbsP } defer cancel() - clientMetadata, err := engine.ClientMetadataFromContext(ctx) - if err != nil { - return nil, fmt.Errorf("failed to get requester session ID: %s", err) - } - ctx = engine.LocalImportOpts{ Path: path, StatPathOnly: true, StatReturnAbsPath: returnAbsPath, }.AppendToOutgoingContext(ctx) - clientCaller, err := c.SessionManager.Get(ctx, clientMetadata.ClientID, false) + clientCaller, err := c.GetSessionCaller(ctx, true) if err != nil { return nil, fmt.Errorf("failed to get requester session: %s", err) } @@ -229,7 +219,7 @@ func (c *Client) LocalDirExport( Path: destPath, }.AppendToOutgoingContext(ctx) - _, descRef, err := expInstance.Export(ctx, cacheRes, nil, clientMetadata.ClientID) + _, descRef, err := expInstance.Export(ctx, cacheRes, nil, clientMetadata.BuildkitSessionID()) if err != nil { return fmt.Errorf("failed to export: %s", err) } @@ -304,11 +294,6 @@ func (c *Client) LocalFileExport( return fmt.Errorf("failed to stat file: %s", err) } - clientMetadata, err := engine.ClientMetadataFromContext(ctx) - if err != nil { - return fmt.Errorf("failed to get requester session ID: %s", err) - } - ctx = engine.LocalExportOpts{ Path: destPath, IsFileStream: true, @@ -317,7 +302,7 @@ func (c *Client) LocalFileExport( FileMode: stat.Mode().Perm(), }.AppendToOutgoingContext(ctx) - clientCaller, err := c.SessionManager.Get(ctx, clientMetadata.ClientID, false) + clientCaller, err := c.GetSessionCaller(ctx, true) if err != nil { return fmt.Errorf("failed to get requester session: %s", err) } @@ -373,11 +358,6 @@ func (c *Client) IOReaderExport(ctx context.Context, r io.Reader, destPath strin lg.Debug("finished exporting bytes") }() - clientMetadata, err := engine.ClientMetadataFromContext(ctx) - if err != nil { - return fmt.Errorf("failed to get requester session ID: %s", err) - } - ctx = engine.LocalExportOpts{ Path: destPath, IsFileStream: true, @@ -385,13 +365,13 @@ func (c *Client) IOReaderExport(ctx context.Context, r io.Reader, destPath strin FileMode: destMode, }.AppendToOutgoingContext(ctx) - clientCaller, err := c.SessionManager.Get(ctx, clientMetadata.ClientID, false) + clientCaller, err := c.GetSessionCaller(ctx, true) if err != nil { - return fmt.Errorf("failed to get requester session: %s", err) + return fmt.Errorf("failed to get requester session: %w", err) } diffCopyClient, err := filesync.NewFileSendClient(clientCaller.Conn()).DiffCopy(ctx) if err != nil { - return fmt.Errorf("failed to create diff copy client: %s", err) + return fmt.Errorf("failed to create diff copy client: %w", err) } defer diffCopyClient.CloseSend() @@ -405,25 +385,25 @@ func (c *Client) IOReaderExport(ctx context.Context, r io.Reader, destPath strin err = nil } if err != nil { - return fmt.Errorf("failed to read file: %s", err) + return fmt.Errorf("failed to read file: %w", err) } err = diffCopyClient.SendMsg(&filesync.BytesMessage{Data: buf.Bytes()}) if errors.Is(err, io.EOF) { err := diffCopyClient.RecvMsg(struct{}{}) if err != nil { - return fmt.Errorf("diff copy client error: %s", err) + return fmt.Errorf("diff copy client error: %w", err) } } else if err != nil { - return fmt.Errorf("failed to send file chunk: %s", err) + return fmt.Errorf("failed to send file chunk: %w", err) } } if err := diffCopyClient.CloseSend(); err != nil { - return fmt.Errorf("failed to close send: %s", err) + return fmt.Errorf("failed to close send: %w", err) } // wait for receiver to finish var msg filesync.BytesMessage if err := diffCopyClient.RecvMsg(&msg); err != io.EOF { - return fmt.Errorf("unexpected closing recv msg: %s", err) + return fmt.Errorf("unexpected closing recv msg: %w", err) } return nil } diff --git a/engine/buildkit/session.go b/engine/buildkit/session.go index a71be56c14..f993738360 100644 --- a/engine/buildkit/session.go +++ b/engine/buildkit/session.go @@ -9,6 +9,7 @@ import ( "github.com/containerd/containerd/content" "github.com/containerd/containerd/content/local" + "github.com/dagger/dagger/engine" "github.com/dagger/dagger/engine/client" "github.com/dagger/dagger/internal/distconsts" "github.com/moby/buildkit/identity" @@ -84,7 +85,17 @@ func (c *Client) newSession() (*bksession.Session, error) { return sess, nil } -func (c *Client) GetSessionCaller(ctx context.Context, clientID string) (bksession.Caller, error) { - waitForSession := true - return c.SessionManager.Get(ctx, clientID, !waitForSession) +func (c *Client) GetSessionCaller(ctx context.Context, wait bool) (bksession.Caller, error) { + clientMetadata, err := engine.ClientMetadataFromContext(ctx) + if err != nil { + return nil, err + } + caller, err := c.SessionManager.Get(ctx, clientMetadata.BuildkitSessionID(), !wait) + if err != nil { + return nil, err + } + if caller == nil { + return nil, fmt.Errorf("session for %q not found", clientMetadata.BuildkitSessionID()) + } + return caller, nil } diff --git a/engine/client/client.go b/engine/client/client.go index 3d483840b3..fe0b39cfd0 100644 --- a/engine/client/client.go +++ b/engine/client/client.go @@ -290,15 +290,6 @@ func Connect(ctx context.Context, params Params) (_ *Client, _ context.Context, // already started c.eg.Go(func() error { return bkSession.Run(c.internalCtx, func(ctx context.Context, proto string, meta map[string][]string) (net.Conn, error) { - // overwrite the session ID to be our client ID - const sessionIDHeader = "X-Docker-Expose-Session-Uuid" - if _, ok := meta[sessionIDHeader]; !ok { - // should never happen unless upstream changes the value of the header key, - // in which case we want to know - return nil, fmt.Errorf("missing header %s", sessionIDHeader) - } - meta[sessionIDHeader] = []string{c.ID} - return grpchijack.Dialer(c.bkClient.ControlClient())(ctx, proto, engine.ClientMetadata{ RegisterClient: true, ClientID: c.ID, diff --git a/engine/opts.go b/engine/opts.go index 22f27b1d4f..3f2f4f1f94 100644 --- a/engine/opts.go +++ b/engine/opts.go @@ -6,6 +6,7 @@ import ( "encoding/json" "fmt" "os" + "strings" "github.com/dagger/dagger/core/pipeline" controlapi "github.com/moby/buildkit/api/services/control" @@ -90,6 +91,15 @@ func (m ClientMetadata) AppendToMD(md metadata.MD) metadata.MD { return md } +// The ID to use for this client's buildkit session. It's a combination of both +// the client and the server IDs to account for the fact that the client ID is +// a content digest for functions/nested-execs, meaning it can reoccur across +// different servers; that doesn't work because buildkit's SessionManager is +// global to the whole process. +func (m ClientMetadata) BuildkitSessionID() string { + return strings.Join([]string{m.ClientID, m.ServerID}, "-") +} + func ContextWithClientMetadata(ctx context.Context, clientMetadata *ClientMetadata) context.Context { return contextWithMD(ctx, clientMetadata.ToGRPCMD()) } diff --git a/engine/server/buildkitcontroller.go b/engine/server/buildkitcontroller.go index 67b4fb7d0f..123c7d16b4 100644 --- a/engine/server/buildkitcontroller.go +++ b/engine/server/buildkitcontroller.go @@ -184,6 +184,15 @@ func (e *BuildkitController) Session(stream controlapi.Control_SessionServer) (r eg, egctx := errgroup.WithContext(ctx) eg.Go(func() error { + // overwrite the session ID to be our client ID + server ID + const sessionIDHeader = "x-docker-expose-session-uuid" + if _, ok := hijackmd[sessionIDHeader]; !ok { + // should never happen unless upstream changes the value of the header key, + // in which case we want to know + panic(fmt.Errorf("missing header %s", sessionIDHeader)) + } + hijackmd[sessionIDHeader] = []string{opts.BuildkitSessionID()} + bklog.G(ctx).Debug("session manager handling conn") err := e.SessionManager.HandleConn(egctx, conn, hijackmd) bklog.G(ctx).WithError(err).Debug("session manager handle conn done") diff --git a/engine/server/server.go b/engine/server/server.go index 4b05e220f2..97de4c550b 100644 --- a/engine/server/server.go +++ b/engine/server/server.go @@ -97,7 +97,7 @@ func (e *BuildkitController) newDaggerServer(ctx context.Context, clientMetadata getSessionCtx, getSessionCancel := context.WithTimeout(ctx, 10*time.Second) defer getSessionCancel() - sessionCaller, err := e.SessionManager.Get(getSessionCtx, clientMetadata.ClientID, false) + sessionCaller, err := e.SessionManager.Get(getSessionCtx, clientMetadata.BuildkitSessionID(), false) if err != nil { return nil, fmt.Errorf("get session: %w", err) } @@ -280,12 +280,15 @@ func (s *DaggerServer) ServeHTTP(w http.ResponseWriter, r *http.Request) { } ctx = progrock.ToContext(ctx, rec) + s.clientCallMu.RLock() schema, err := callContext.Deps.Schema(ctx) if err != nil { + s.clientCallMu.RUnlock() // TODO: technically this is not *always* bad request, should ideally be more specific and differentiate errorOut(err, http.StatusBadRequest) return } + s.clientCallMu.RUnlock() defer func() { if v := recover(); v != nil { From e0d9376deb574b031bf00886cd9aa79d9faa05c1 Mon Sep 17 00:00:00 2001 From: Erik Sipsma Date: Mon, 25 Mar 2024 10:59:45 -0700 Subject: [PATCH 3/4] disentangle dagql server from ModDeps, tie to Query instead Signed-off-by: Erik Sipsma --- core/container.go | 4 +- core/integration/module_config_test.go | 2 +- core/interface.go | 26 +- core/moddeps.go | 158 +++-------- core/module.go | 36 +-- core/object.go | 13 +- core/query.go | 224 ++------------- core/schema/coremod.go | 313 ++++++++++----------- core/schema/module.go | 74 ++--- core/schema/modulesource.go | 114 ++++++++ core/schema/sdk.go | 122 ++++----- core/schema/util.go | 15 - core/typedef.go | 14 + dagql/objects.go | 23 +- dagql/server.go | 30 ++ dagql/types.go | 4 + engine/server/server.go | 363 ++++++++++++++++++------- 17 files changed, 750 insertions(+), 785 deletions(-) diff --git a/core/container.go b/core/container.go index 93d314f79f..ca6cc339e2 100644 --- a/core/container.go +++ b/core/container.go @@ -1021,12 +1021,12 @@ func (container *Container) WithExec(ctx context.Context, opts ContainerExecOpts // this allows executed containers to communicate back to this API if opts.ExperimentalPrivilegedNesting { - clientID, err := container.Query.RegisterCaller(ctx, opts.NestedExecFunctionCall) + newRoot, err := container.Query.NewRootForCurrentCall(ctx, opts.NestedExecFunctionCall) if err != nil { return nil, fmt.Errorf("register caller: %w", err) } runOpts = append(runOpts, - llb.AddEnv("_DAGGER_NESTED_CLIENT_ID", clientID), + llb.AddEnv("_DAGGER_NESTED_CLIENT_ID", newRoot.ClientID), // include the engine version so that these execs get invalidated if the engine/API change llb.AddEnv("_DAGGER_ENGINE_VERSION", engine.Version), ) diff --git a/core/integration/module_config_test.go b/core/integration/module_config_test.go index cef63fbdd1..a6474577f3 100644 --- a/core/integration/module_config_test.go +++ b/core/integration/module_config_test.go @@ -861,7 +861,7 @@ func (m *Test) Fn(ctx context.Context) (string, error) { With(daggerExec("init", "--name=test", "--sdk=go", "--source=.")). With(daggerExec("install", testGitModuleRef("this/just/does/not/exist"))). Sync(ctx) - require.ErrorContains(t, err, `module "test" dependency "" with source root path "this/just/does/not/exist" does not exist or does not have a configuration file`) + require.ErrorContains(t, err, `dependency "" with source root path "this/just/does/not/exist" does not exist or does not have a configuration file`) }) t.Run("unpinned gets pinned", func(t *testing.T) { diff --git a/core/interface.go b/core/interface.go index 345a04e5b9..92ca2f4f2e 100644 --- a/core/interface.go +++ b/core/interface.go @@ -30,30 +30,31 @@ func (iface *InterfaceType) ConvertFromSDKResult(ctx context.Context, value any) // TODO: this seems expensive fromID := func(id *call.ID) (dagql.Typed, error) { - deps, err := iface.mod.Query.IDDeps(ctx, id) + dynamicRoot, err := iface.mod.Query.NewRootForDynamicID(ctx, id) if err != nil { - return nil, fmt.Errorf("schema: %w", err) + return nil, fmt.Errorf("failed to get root for DynamicID: %w", err) } - dag, err := deps.Schema(ctx) - if err != nil { - return nil, fmt.Errorf("schema: %w", err) - } - val, err := dag.Load(ctx, id) + val, err := dynamicRoot.Dag.Load(ctx, id) if err != nil { return nil, fmt.Errorf("load interface ID %s: %w", id.Display(), err) } typeName := val.ObjectType().TypeName() + // Verify that the object provided actually implements the interface. This + // is also enforced by only adding "As*" fields to objects in a schema once + // they implement the interface, but in theory an SDK could provide + // arbitrary IDs of objects here, so we need to check again to be fully + // robust. var checkType *TypeDef - if objType, found, err := deps.ModTypeFor(ctx, &TypeDef{ + if objType, found, err := dynamicRoot.Deps.ModTypeFor(ctx, &TypeDef{ Kind: TypeDefKindObject, AsObject: dagql.NonNull(&ObjectTypeDef{ Name: typeName, }), }); err == nil && found { checkType = objType.TypeDef() - } else if ifaceType, found, err := deps.ModTypeFor(ctx, &TypeDef{ + } else if ifaceType, found, err := dynamicRoot.Deps.ModTypeFor(ctx, &TypeDef{ Kind: TypeDefKindInterface, AsInterface: dagql.NonNull(&InterfaceTypeDef{ Name: typeName, @@ -63,12 +64,6 @@ func (iface *InterfaceType) ConvertFromSDKResult(ctx context.Context, value any) } else { return nil, fmt.Errorf("could not find object or interface type for %q", typeName) } - - // Verify that the object provided actually implements the interface. This - // is also enforced by only adding "As*" fields to objects in a schema once - // they implement the interface, but in theory an SDK could provide - // arbitrary IDs of objects here, so we need to check again to be fully - // robust. if ok := checkType.IsSubtypeOf(iface.TypeDef()); !ok { return nil, fmt.Errorf("type %s does not implement interface %s", typeName, iface.typeDef.Name) } @@ -124,6 +119,7 @@ func (iface *InterfaceType) Install(ctx context.Context, dag *dagql.Server) erro TypeDef: iface.typeDef, IfaceType: iface, }, + SourceModule: iface.mod.IDModule(), }) dag.InstallObject(class) diff --git a/core/moddeps.go b/core/moddeps.go index 60a814e8e5..5382717775 100644 --- a/core/moddeps.go +++ b/core/moddeps.go @@ -2,14 +2,12 @@ package core import ( "context" - "encoding/json" "fmt" - "sync" + "strings" - "github.com/dagger/dagger/cmd/codegen/introspection" "github.com/dagger/dagger/dagql" - dagintro "github.com/dagger/dagger/dagql/introspection" - "github.com/dagger/dagger/tracing" + "github.com/opencontainers/go-digest" + "golang.org/x/exp/slices" ) const ( @@ -19,146 +17,47 @@ const ( ModuleName = "daggercore" ) -// We don't expose these types to modules SDK codegen, but -// we still want their graphql schemas to be available for -// internal usage. So we use this list to scrub them from -// the introspection JSON that module SDKs use for codegen. -var typesHiddenFromModuleSDKs = []dagql.Typed{ - &Host{}, -} - /* ModDeps represents a set of dependencies for a module or for a caller depending on a particular set of modules to be served. */ type ModDeps struct { - root *Query Mods []Mod // TODO hide - - // should not be read directly, call Schema and SchemaIntrospectionJSON instead - lazilyLoadedSchema *dagql.Server - lazilyLoadedIntrospectionJSON string - loadSchemaErr error - loadSchemaLock sync.Mutex } -func NewModDeps(root *Query, mods []Mod) *ModDeps { +func NewModDeps(mods []Mod) *ModDeps { + slices.SortFunc(mods, func(a, b Mod) int { + return strings.Compare(a.Digest().String(), b.Digest().String()) + }) + mods = slices.CompactFunc(mods, func(a, b Mod) bool { + return a.Digest() == b.Digest() + }) return &ModDeps{ - root: root, Mods: mods, } } -func (d *ModDeps) Prepend(mods ...Mod) *ModDeps { - deps := append([]Mod{}, mods...) - deps = append(deps, d.Mods...) - return NewModDeps(d.root, deps) -} - func (d *ModDeps) Append(mods ...Mod) *ModDeps { deps := append([]Mod{}, d.Mods...) deps = append(deps, mods...) - return NewModDeps(d.root, deps) + return NewModDeps(deps) } // The combined schema exposed by each mod in this set of dependencies -func (d *ModDeps) Schema(ctx context.Context) (*dagql.Server, error) { - schema, _, err := d.lazilyLoadSchema(ctx) - if err != nil { - return nil, err - } - return schema, nil -} - -// The introspection json for combined schema exposed by each mod in this set of dependencies -func (d *ModDeps) SchemaIntrospectionJSON(ctx context.Context, forModule bool) (string, error) { - _, introspectionJSON, err := d.lazilyLoadSchema(ctx) - if err != nil { - return "", err - } - - if !forModule { - return introspectionJSON, nil - } - - var introspection introspection.Response - if err := json.Unmarshal([]byte(introspectionJSON), &introspection); err != nil { - return "", fmt.Errorf("failed to unmarshal introspection JSON: %w", err) - } - - for _, typed := range typesHiddenFromModuleSDKs { - introspection.Schema.ScrubType(typed.Type().Name()) - introspection.Schema.ScrubType(dagql.IDTypeNameFor(typed)) - } - bs, err := json.Marshal(introspection) - if err != nil { - return "", fmt.Errorf("failed to marshal introspection JSON: %w", err) - } - return string(bs), nil -} - -// All the TypeDefs exposed by this set of dependencies -func (d *ModDeps) TypeDefs(ctx context.Context) ([]*TypeDef, error) { - var typeDefs []*TypeDef - for _, mod := range d.Mods { - modTypeDefs, err := mod.TypeDefs(ctx) - if err != nil { - return nil, fmt.Errorf("failed to get objects from mod %q: %w", mod.Name(), err) - } - typeDefs = append(typeDefs, modTypeDefs...) - } - return typeDefs, nil -} - -func schemaIntrospectionJSON(ctx context.Context, dag *dagql.Server) (json.RawMessage, error) { - data, err := dag.Query(ctx, introspection.Query, nil) - if err != nil { - return nil, fmt.Errorf("introspection query failed: %w", err) - } - jsonBytes, err := json.Marshal(data) - if err != nil { - return nil, fmt.Errorf("failed to marshal introspection result: %w", err) - } - return json.RawMessage(jsonBytes), nil -} - -func (d *ModDeps) lazilyLoadSchema(ctx context.Context) (loadedSchema *dagql.Server, loadedIntrospectionJSON string, rerr error) { - d.loadSchemaLock.Lock() - defer d.loadSchemaLock.Unlock() - if d.lazilyLoadedSchema != nil { - return d.lazilyLoadedSchema, d.lazilyLoadedIntrospectionJSON, nil - } - if d.loadSchemaErr != nil { - return nil, "", d.loadSchemaErr - } - defer func() { - d.lazilyLoadedSchema = loadedSchema - d.lazilyLoadedIntrospectionJSON = loadedIntrospectionJSON - d.loadSchemaErr = rerr - }() - - dag := dagql.NewServer[*Query](d.root) - - dag.Around(tracing.AroundFunc) - - // share the same cache session-wide - dag.Cache = d.root.Cache - - dagintro.Install[*Query](dag) - +func (d *ModDeps) Install(ctx context.Context, dag *dagql.Server) error { var objects []*ModuleObjectType var ifaces []*InterfaceType for _, mod := range d.Mods { err := mod.Install(ctx, dag) if err != nil { - return nil, "", fmt.Errorf("failed to get schema for module %q: %w", mod.Name(), err) + return fmt.Errorf("failed to get schema for module %q: %w", mod.Name(), err) } // TODO support core interfaces types if userMod, ok := mod.(*Module); ok { defs, err := mod.TypeDefs(ctx) if err != nil { - return nil, "", fmt.Errorf("failed to get type defs for module %q: %w", mod.Name(), err) + return fmt.Errorf("failed to get type defs for module %q: %w", mod.Name(), err) } for _, def := range defs { switch def.Kind { @@ -182,7 +81,7 @@ func (d *ModDeps) lazilyLoadSchema(ctx context.Context) (loadedSchema *dagql.Ser obj := objType.typeDef class, found := dag.ObjectType(obj.Name) if !found { - return nil, "", fmt.Errorf("failed to find object %q in schema", obj.Name) + return fmt.Errorf("failed to find object %q in schema", obj.Name) } for _, ifaceType := range ifaces { iface := ifaceType.typeDef @@ -215,12 +114,20 @@ func (d *ModDeps) lazilyLoadSchema(ctx context.Context) (loadedSchema *dagql.Ser } } - introspectionJSON, err := schemaIntrospectionJSON(ctx, dag) - if err != nil { - return nil, "", fmt.Errorf("failed to get schema introspection JSON: %w", err) - } + return nil +} - return dag, string(introspectionJSON), nil +// All the TypeDefs exposed by this set of dependencies +func (d *ModDeps) TypeDefs(ctx context.Context) ([]*TypeDef, error) { + var typeDefs []*TypeDef + for _, mod := range d.Mods { + modTypeDefs, err := mod.TypeDefs(ctx) + if err != nil { + return nil, fmt.Errorf("failed to get objects from mod %q: %w", mod.Name(), err) + } + typeDefs = append(typeDefs, modTypeDefs...) + } + return typeDefs, nil } // Search the deps for the given type def, returning the ModType if found. This does not recurse @@ -239,3 +146,12 @@ func (d *ModDeps) ModTypeFor(ctx context.Context, typeDef *TypeDef) (ModType, bo } return nil, false, nil } + +// Return a unique digest for this set of dependencies, based on the module digests +func (d *ModDeps) Digest() digest.Digest { + dgsts := make([]string, len(d.Mods)) + for i, mod := range d.Mods { + dgsts[i] = mod.Digest().String() + } + return digest.FromString(strings.Join(dgsts, " ")) +} diff --git a/core/module.go b/core/module.go index 6bbf7b5f26..b5ad90d3d3 100644 --- a/core/module.go +++ b/core/module.go @@ -10,6 +10,7 @@ import ( "github.com/dagger/dagger/dagql" "github.com/dagger/dagger/dagql/call" "github.com/moby/buildkit/solver/pb" + "github.com/opencontainers/go-digest" "github.com/vektah/gqlparser/v2/ast" ) @@ -41,7 +42,7 @@ type Module struct { DependencyConfig []*ModuleDependency `field:"true" doc:"The dependencies as configured by the module."` // The module's loaded dependencies, not yet initialized - DependenciesField []dagql.Instance[*Module] `field:"true" name:"dependencies" doc:"Modules used by this module."` + DependenciesField []*Module `field:"true" name:"dependencies" doc:"Modules used by this module."` // Deps contains the module's dependency DAG. Deps *ModDeps @@ -100,12 +101,8 @@ func (mod *Module) Name() string { return mod.NameField } -func (mod *Module) Dependencies() []Mod { - mods := make([]Mod, len(mod.DependenciesField)) - for i, dep := range mod.DependenciesField { - mods[i] = dep.Self - } - return mods +func (mod *Module) Digest() digest.Digest { + return mod.Source.ID().Digest() } func (mod *Module) IDModule() *call.Module { @@ -236,10 +233,6 @@ func (mod *Module) TypeDefs(ctx context.Context) ([]*TypeDef, error) { return typeDefs, nil } -func (mod *Module) DependencySchemaIntrospectionJSON(ctx context.Context, forModule bool) (string, error) { - return mod.Deps.SchemaIntrospectionJSON(ctx, forModule) -} - func (mod *Module) ModTypeFor(ctx context.Context, typeDef *TypeDef, checkDirectDeps bool) (ModType, bool, error) { var modType ModType switch typeDef.Kind { @@ -533,9 +526,6 @@ type Mod interface { // The name of the module Name() string - // The direct dependencies of this module - Dependencies() []Mod - // TODO describe Install(context.Context, *dagql.Server) error @@ -546,6 +536,9 @@ type Mod interface { // All the TypeDefs exposed by this module (does not include dependencies) TypeDefs(ctx context.Context) ([]*TypeDef, error) + + // A unique digest for the module + Digest() digest.Digest } /* @@ -571,16 +564,11 @@ type SDK interface { The Code field of the returned GeneratedCode object should be the generated contents of the module sourceDirSubpath, in the case where that's different than the root of the sourceDir. - - The provided Module is not fully initialized; the Runtime field will not be set yet. */ - Codegen(context.Context, *ModDeps, dagql.Instance[*ModuleSource]) (*GeneratedCode, error) + Codegen(context.Context, dagql.Instance[*ModuleSource]) (*GeneratedCode, error) - /* Runtime returns a container that is used to execute module code at runtime in the Dagger engine. - - The provided Module is not fully initialized; the Runtime field will not be set yet. - */ - Runtime(context.Context, *ModDeps, dagql.Instance[*ModuleSource]) (*Container, error) + // Runtime returns a container that is used to execute module code at runtime in the Dagger engine. + Runtime(context.Context, dagql.Instance[*ModuleSource]) (*Container, error) // Paths that should always be loaded from module sources using this SDK. Ensures that e.g. main.go // in the Go SDK is always loaded even if dagger.json has include settings that don't include it. @@ -613,9 +601,9 @@ func (mod Module) Clone() *Module { cp.DependencyConfig[i] = dep.Clone() } - cp.DependenciesField = make([]dagql.Instance[*Module], len(mod.DependenciesField)) + cp.DependenciesField = make([]*Module, len(mod.DependenciesField)) for i, dep := range mod.DependenciesField { - cp.DependenciesField[i].Self = dep.Self.Clone() + cp.DependenciesField[i] = dep.Clone() } cp.ObjectDefs = make([]*TypeDef, len(mod.ObjectDefs)) diff --git a/core/object.go b/core/object.go index a9562d9328..b53884de54 100644 --- a/core/object.go +++ b/core/object.go @@ -50,15 +50,11 @@ func (t *ModuleObjectType) ConvertToSDKInput(ctx context.Context, value dagql.Ty // needing to make calls to their own API). switch x := value.(type) { case DynamicID: - deps, err := t.mod.Query.IDDeps(ctx, x.ID()) + dynamicRoot, err := t.mod.Query.NewRootForDynamicID(ctx, x.ID()) if err != nil { - return nil, fmt.Errorf("failed to get deps for DynamicID: %w", err) + return nil, fmt.Errorf("failed to get root for DynamicID: %w", err) } - dag, err := deps.Schema(ctx) - if err != nil { - return nil, fmt.Errorf("schema: %w", err) - } - val, err := dag.Load(ctx, x.ID()) + val, err := dynamicRoot.Dag.Load(ctx, x.ID()) if err != nil { return nil, fmt.Errorf("load DynamicID: %w", err) } @@ -172,7 +168,8 @@ func (obj *ModuleObject) Install(ctx context.Context, dag *dagql.Server) error { return fmt.Errorf("installing object %q too early", obj.TypeDef.Name) } class := dagql.NewClass(dagql.ClassOpts[*ModuleObject]{ - Typed: obj, + Typed: obj, + SourceModule: obj.Module.IDModule(), }) objDef := obj.TypeDef mod := obj.Module diff --git a/core/query.go b/core/query.go index 74f9b955b1..7eb97a1aad 100644 --- a/core/query.go +++ b/core/query.go @@ -4,28 +4,36 @@ import ( "context" "fmt" "net/http" - "strings" - "sync" "github.com/containerd/containerd/content" "github.com/dagger/dagger/auth" "github.com/dagger/dagger/core/pipeline" "github.com/dagger/dagger/dagql" "github.com/dagger/dagger/dagql/call" - "github.com/dagger/dagger/engine" "github.com/dagger/dagger/engine/buildkit" "github.com/moby/buildkit/util/leaseutil" - "github.com/opencontainers/go-digest" "github.com/vektah/gqlparser/v2/ast" - "github.com/vito/progrock" ) // Query forms the root of the DAG and houses all necessary state and // dependencies for evaluating queries. type Query struct { QueryOpts + + Dag *dagql.Server + Buildkit *buildkit.Client + ClientID string + + SecretToken string + + Deps *ModDeps + + FnCall *FunctionCall + + ProgrockParent string + // The current pipeline. Pipeline pipeline.Path } @@ -34,6 +42,8 @@ var ErrNoCurrentModule = fmt.Errorf("no current module") // Settings for Query that are shared across all instances for a given DaggerServer type QueryOpts struct { + DaggerServer + ProgrockSocketPath string Services *Services @@ -48,42 +58,17 @@ type QueryOpts struct { // The default platform. Platform Platform - // The default deps of every user module (currently just core) - DefaultDeps *ModDeps - - // The DagQL query cache. - Cache dagql.Cache - - BuildkitOpts *buildkit.Opts - Recorder *progrock.Recorder - - // The metadata of client calls. - // For the special case of the main client caller, the key is just empty string. - // This is never explicitly deleted from; instead it will just be garbage collected - // when this server for the session shuts down - ClientCallContext map[string]*ClientCallContext - ClientCallMu *sync.RWMutex MainClientCallerID string - - // the http endpoints being served (as a map since APIs like shellEndpoint can add more) - Endpoints map[string]http.Handler - EndpointMu *sync.RWMutex } -func NewRoot(ctx context.Context, opts QueryOpts) (*Query, error) { - bk, err := buildkit.NewClient(ctx, opts.BuildkitOpts) - if err != nil { - return nil, fmt.Errorf("buildkit client: %w", err) - } - - // NOTE: context.WithoutCancel is used because if the provided context is canceled, buildkit can - // leave internal progress contexts open and leak goroutines. - bk.WriteStatusesTo(context.WithoutCancel(ctx), opts.Recorder) - - return &Query{ - QueryOpts: opts, - Buildkit: bk, - }, nil +// Methods that Query needs from the over-arching DaggerServer which involve mutating and/or creating +// Query instances +type DaggerServer interface { + MuxEndpoint(ctx context.Context, path string, handler http.Handler) error + ServeModule(ctx context.Context, mod *Module) error + NewRootForCurrentCall(ctx context.Context, call *FunctionCall) (*Query, error) + NewRootForDependencies(ctx context.Context, deps *ModDeps) (*Query, error) + NewRootForDynamicID(ctx context.Context, id *call.ID) (*Query, error) } func (*Query) Type() *ast.Type { @@ -101,149 +86,20 @@ func (q Query) Clone() *Query { return &q } -func (q *Query) MuxEndpoint(ctx context.Context, path string, handler http.Handler) error { - q.EndpointMu.Lock() - defer q.EndpointMu.Unlock() - q.Endpoints[path] = handler - return nil -} - -type ClientCallContext struct { - Root *Query - - // the DAG of modules being served to this client - Deps *ModDeps - - // If the client is itself from a function call in a user module, these are set with the - // metadata of that ongoing function call - FnCall *FunctionCall - - ProgrockParent string -} - -func (q *Query) ServeModule(ctx context.Context, mod *Module) error { - clientMetadata, err := engine.ClientMetadataFromContext(ctx) - if err != nil { - return err - } - - q.ClientCallMu.Lock() - defer q.ClientCallMu.Unlock() - callCtx, ok := q.ClientCallContext[clientMetadata.ClientID] - if !ok { - return fmt.Errorf("client call not found") - } - callCtx.Deps = callCtx.Deps.Append(mod) - return nil -} - -func (q *Query) RegisterCaller(ctx context.Context, call *FunctionCall) (string, error) { - if call == nil { - call = &FunctionCall{} - } - callCtx := &ClientCallContext{ - FnCall: call, - ProgrockParent: progrock.FromContext(ctx).Parent, - } - - currentID := dagql.CurrentID(ctx) - clientIDInputs := []string{currentID.Digest().String()} - if !call.Cache { - // use the ServerID so that we bust cache once-per-session - clientMetadata, err := engine.ClientMetadataFromContext(ctx) - if err != nil { - return "", err - } - clientIDInputs = append(clientIDInputs, clientMetadata.ServerID) - } - clientIDDigest := digest.FromString(strings.Join(clientIDInputs, " ")) - - // only use encoded part of digest because this ID ends up becoming a buildkit Session ID - // and buildkit has some ancient internal logic that splits on a colon to support some - // dev mode logic: https://github.com/moby/buildkit/pull/290 - // also trim it to 25 chars as it ends up becoming part of service URLs - clientID := clientIDDigest.Encoded()[:25] - - // break glass for debugging which client is which operation - // bklog.G(ctx).Debugf("CLIENT ID %s = %s", clientID, currentID.Display()) - - if call.Module == nil { - callCtx.Deps = q.DefaultDeps - } else { - callCtx.Deps = call.Module.Deps - // By default, serve both deps and the module's own API to itself. But if SkipSelfSchema is set, - // only serve the APIs of the deps of this module. This is currently only needed for the special - // case of the function used to get the definition of the module itself (which can't obviously - // be served the API its returning the definition of). - if !call.SkipSelfSchema { - callCtx.Deps = callCtx.Deps.Append(call.Module) - } - } - - q.ClientCallMu.Lock() - defer q.ClientCallMu.Unlock() - _, ok := q.ClientCallContext[clientID] - if ok { - return clientID, nil - } - - var err error - callCtx.Root, err = NewRoot(ctx, q.QueryOpts) - if err != nil { - return "", err - } - - q.ClientCallContext[clientID] = callCtx - return clientID, nil -} - func (q *Query) CurrentModule(ctx context.Context) (*Module, error) { - clientMetadata, err := engine.ClientMetadataFromContext(ctx) - if err != nil { - return nil, err - } - - q.ClientCallMu.RLock() - defer q.ClientCallMu.RUnlock() - callCtx, ok := q.ClientCallContext[clientMetadata.ClientID] - if !ok { - return nil, fmt.Errorf("client call %s not found", clientMetadata.ClientID) - } - if callCtx.FnCall.Module == nil { + mod := q.FnCall.Module + if mod == nil { return nil, ErrNoCurrentModule } - return callCtx.FnCall.Module, nil + return mod, nil } func (q *Query) CurrentFunctionCall(ctx context.Context) (*FunctionCall, error) { - clientMetadata, err := engine.ClientMetadataFromContext(ctx) - if err != nil { - return nil, err - } - if clientMetadata.ClientID == q.MainClientCallerID { - return nil, fmt.Errorf("%w: main client caller has no function", ErrNoCurrentModule) - } - - q.ClientCallMu.RLock() - defer q.ClientCallMu.RUnlock() - callCtx, ok := q.ClientCallContext[clientMetadata.ClientID] - if !ok { - return nil, fmt.Errorf("client call %s not found", clientMetadata.ClientID) - } - - return callCtx.FnCall, nil -} - -func (q *Query) CurrentServedDeps(ctx context.Context) (*ModDeps, error) { - clientMetadata, err := engine.ClientMetadataFromContext(ctx) - if err != nil { - return nil, err - } - callCtx, ok := q.ClientCallContext[clientMetadata.ClientID] - if !ok { - return nil, fmt.Errorf("client call %s not found", clientMetadata.ClientID) + fnCall := q.FnCall + if fnCall == nil { + return nil, ErrNoCurrentModule } - return callCtx.Deps, nil + return fnCall, nil } func (q *Query) WithPipeline(name, desc string, labels []pipeline.Label) *Query { @@ -305,23 +161,3 @@ func (q *Query) NewHostService(upstream string, ports []PortForward) *Service { HostPorts: ports, } } - -// IDDeps loads the module dependencies of a given ID. -// -// The returned ModDeps extends the inner DefaultDeps with all modules found in -// the ID, loaded by using the DefaultDeps schema. -func (q *Query) IDDeps(ctx context.Context, id *call.ID) (*ModDeps, error) { - bootstrap, err := q.DefaultDeps.Schema(ctx) - if err != nil { - return nil, fmt.Errorf("bootstrap schema: %w", err) - } - deps := q.DefaultDeps - for _, modID := range id.Modules() { - mod, err := dagql.NewID[*Module](modID.ID()).Load(ctx, bootstrap) - if err != nil { - return nil, fmt.Errorf("load source mod: %w", err) - } - deps = deps.Append(mod.Self) - } - return deps, nil -} diff --git a/core/schema/coremod.go b/core/schema/coremod.go index e001909419..ee0ff62bfc 100644 --- a/core/schema/coremod.go +++ b/core/schema/coremod.go @@ -5,12 +5,14 @@ import ( "encoding/json" "fmt" "log/slog" + "slices" "strings" - "github.com/dagger/dagger/cmd/codegen/introspection" "github.com/dagger/dagger/core" "github.com/dagger/dagger/dagql" "github.com/dagger/dagger/dagql/call" + "github.com/opencontainers/go-digest" + "github.com/vektah/gqlparser/v2/ast" ) // CoreMod is a special implementation of Mod for our core API, which is not *technically* a true module yet @@ -26,8 +28,8 @@ func (m *CoreMod) Name() string { return core.ModuleName } -func (m *CoreMod) Dependencies() []core.Mod { - return nil +func (m *CoreMod) Digest() digest.Digest { + return digest.FromString(m.Name()) } func (m *CoreMod) Install(ctx context.Context, dag *dagql.Server) error { @@ -72,8 +74,9 @@ func (m *CoreMod) ModTypeFor(ctx context.Context, typeDef *core.TypeDef, checkDi } case core.TypeDefKindObject: - _, ok := m.Dag.ObjectType(typeDef.AsObject.Value.Name) - if !ok { + objType, ok := m.Dag.ObjectType(typeDef.AsObject.Value.Name) + if !ok || objType.SourceModule() != nil { + // SourceModule is nil for core types return nil, false, nil } modType = &CoreModObject{coreMod: m} @@ -97,111 +100,158 @@ func (m *CoreMod) ModTypeFor(ctx context.Context, typeDef *core.TypeDef, checkDi } func (m *CoreMod) TypeDefs(ctx context.Context) ([]*core.TypeDef, error) { - introspectionJSON, err := schemaIntrospectionJSON(ctx, m.Dag) - if err != nil { - return nil, fmt.Errorf("failed to get schema introspection JSON: %w", err) - } - var schemaResp introspection.Response - if err := json.Unmarshal([]byte(introspectionJSON), &schemaResp); err != nil { - return nil, fmt.Errorf("failed to unmarshal introspection JSON: %w", err) - } - schema := schemaResp.Schema - - typeDefs := make([]*core.TypeDef, 0, len(schema.Types)) - for _, introspectionType := range schema.Types { - switch introspectionType.Kind { - case introspection.TypeKindObject: - typeDef := &core.ObjectTypeDef{ - Name: introspectionType.Name, - Description: introspectionType.Description, + typeDefs := make(map[string]*core.TypeDef) + typeDefStubs := make(map[string]*core.TypeDef) + + var getStub func(*ast.Type, bool) *core.TypeDef + getStub = func(t *ast.Type, isArg bool) *core.TypeDef { + switch { + case t.NamedType != "": + stubName := t.String() // includes ! when non-null + if isArg && strings.HasSuffix(t.NamedType, "ID") { + // convert ID arguments to the actual object + stubName = strings.TrimSuffix(t.NamedType, "ID") + if t.NonNull { + stubName += "!" + } } - isIdable := false - for _, introspectionField := range introspectionType.Fields { - if introspectionField.Name == "id" { - isIdable = true - continue + stub, ok := typeDefStubs[stubName] + if !ok { + stub = &core.TypeDef{ + Optional: !t.NonNull, } + typeDefStubs[stubName] = stub + } + return stub + case t.Elem != nil: + return &core.TypeDef{ + Kind: core.TypeDefKindList, + Optional: !t.NonNull, + AsList: dagql.NonNull(&core.ListTypeDef{ + ElementTypeDef: getStub(t.Elem, isArg), + }), + } + } + return nil + } - fn := &core.Function{ - Name: introspectionField.Name, - Description: introspectionField.Description, - } + objTypes := m.Dag.ObjectTypes() + for _, objType := range objTypes { + if objType.SourceModule() != nil { + // not a core type + continue + } + astDef := objType.TypeDefinition() + objTypeDef := core.NewObjectTypeDef(astDef.Name, astDef.Description) + + // check that this is an idable object quick first so we don't recurse to stubs for e.g. introspection types + isIdable := false + for _, field := range astDef.Fields { + if field.Name == "id" { + isIdable = true + break + } + } + if !isIdable { + continue + } - rtType, ok, err := introspectionRefToTypeDef(introspectionField.TypeRef, false, false) - if err != nil { - return nil, fmt.Errorf("failed to convert return type: %w", err) - } - if !ok { - continue + for _, field := range astDef.Fields { + if field.Name == "id" { + continue + } + fn := &core.Function{ + Name: field.Name, + Description: field.Description, + ReturnType: getStub(field.Type, false), + } + for _, arg := range field.Arguments { + fnArg := &core.FunctionArg{ + Name: arg.Name, + Description: arg.Description, + TypeDef: getStub(arg.Type, true), } - fn.ReturnType = rtType - - for _, introspectionArg := range introspectionField.Args { - fnArg := &core.FunctionArg{ - Name: introspectionArg.Name, - Description: introspectionArg.Description, - } - - if introspectionArg.DefaultValue != nil { - fnArg.DefaultValue = core.JSON(*introspectionArg.DefaultValue) - } - - argType, ok, err := introspectionRefToTypeDef(introspectionArg.TypeRef, false, true) + if arg.DefaultValue != nil { + anyVal, err := arg.DefaultValue.Value(nil) if err != nil { - return nil, fmt.Errorf("failed to convert argument type: %w", err) + return nil, fmt.Errorf("failed to get default value: %w", err) } - if !ok { - continue + bs, err := json.Marshal(anyVal) + if err != nil { + return nil, fmt.Errorf("failed to marshal default value: %w", err) } - fnArg.TypeDef = argType - - fn.Args = append(fn.Args, fnArg) + fnArg.DefaultValue = core.JSON(bs) } - - typeDef.Functions = append(typeDef.Functions, fn) + fn.Args = append(fn.Args, fnArg) } + objTypeDef.Functions = append(objTypeDef.Functions, fn) + } + typeDefs[objTypeDef.Name] = &core.TypeDef{ + Kind: core.TypeDefKindObject, + AsObject: dagql.NonNull(objTypeDef), + } + } - if !isIdable { - continue + inputTypes := m.Dag.TypeDefs() + for _, inputType := range inputTypes { + astDef := inputType.TypeDefinition() + inputTypeDef := &core.InputTypeDef{ + Name: astDef.Name, + Fields: make([]*core.FieldTypeDef, 0, len(astDef.Fields)), + } + for _, field := range astDef.Fields { + fieldDef := &core.FieldTypeDef{ + Name: field.Name, + Description: field.Description, + TypeDef: getStub(field.Type, false), } + inputTypeDef.Fields = append(inputTypeDef.Fields, fieldDef) + } + typeDefs[inputTypeDef.Name] = &core.TypeDef{ + Kind: core.TypeDefKindInput, + AsInput: dagql.NonNull(inputTypeDef), + } + } - typeDefs = append(typeDefs, &core.TypeDef{ - Kind: core.TypeDefKindObject, - AsObject: dagql.NonNull(typeDef), - }) - - case introspection.TypeKindInputObject: - typeDef := &core.InputTypeDef{ - Name: introspectionType.Name, + scalarTypes := m.Dag.ScalarTypes() + for _, scalarType := range scalarTypes { + switch scalarType.(type) { + case dagql.Int: + typeDefs[scalarType.TypeName()] = &core.TypeDef{ + Kind: core.TypeDefKindInteger, } - - for _, introspectionField := range introspectionType.InputFields { - field := &core.FieldTypeDef{ - Name: introspectionField.Name, - Description: introspectionField.Description, - } - fieldType, ok, err := introspectionRefToTypeDef(introspectionField.TypeRef, false, false) - if err != nil { - return nil, fmt.Errorf("failed to convert return type: %w", err) - } - if !ok { - continue - } - field.TypeDef = fieldType - typeDef.Fields = append(typeDef.Fields, field) + case dagql.Boolean: + typeDefs[scalarType.TypeName()] = &core.TypeDef{ + Kind: core.TypeDefKindBoolean, } - - typeDefs = append(typeDefs, &core.TypeDef{ - Kind: core.TypeDefKindInput, - AsInput: dagql.NonNull(typeDef), - }) - default: - continue + // default everything else to string for now + typeDefs[scalarType.TypeName()] = &core.TypeDef{ + Kind: core.TypeDefKindString, + } } } - return typeDefs, nil + + // fill in stubs with kinds + for name, stub := range typeDefStubs { + typeDef, ok := typeDefs[strings.TrimSuffix(name, "!")] + if !ok { + return nil, fmt.Errorf("failed to find type %q", name) + } + stub.Kind = typeDef.Kind + stub.AsObject = typeDef.AsObject + stub.AsInput = typeDef.AsInput + } + + sortedTypeDefs := make([]*core.TypeDef, 0, len(typeDefs)) + for _, typeDef := range typeDefs { + sortedTypeDefs = append(sortedTypeDefs, typeDef) + } + slices.SortFunc(sortedTypeDefs, func(a, b *core.TypeDef) int { + return strings.Compare(a.Name(), b.Name()) + }) + return sortedTypeDefs, nil } // CoreModObject represents objects from core (Container, Directory, etc.) @@ -261,84 +311,3 @@ func (obj *CoreModObject) TypeDef() *core.TypeDef { }), } } - -func introspectionRefToTypeDef(introspectionType *introspection.TypeRef, nonNull, isInput bool) (*core.TypeDef, bool, error) { - switch introspectionType.Kind { - case introspection.TypeKindNonNull: - return introspectionRefToTypeDef(introspectionType.OfType, true, isInput) - - case introspection.TypeKindScalar: - if isInput && strings.HasSuffix(introspectionType.Name, "ID") { - // convert ID inputs to the actual object - objName := strings.TrimSuffix(introspectionType.Name, "ID") - return &core.TypeDef{ - Kind: core.TypeDefKindObject, - Optional: !nonNull, - AsObject: dagql.NonNull(&core.ObjectTypeDef{ - Name: objName, - }), - }, true, nil - } - - typeDef := &core.TypeDef{ - Optional: !nonNull, - } - switch introspectionType.Name { - case string(introspection.ScalarString): - typeDef.Kind = core.TypeDefKindString - case string(introspection.ScalarInt): - typeDef.Kind = core.TypeDefKindInteger - case string(introspection.ScalarBoolean): - typeDef.Kind = core.TypeDefKindBoolean - default: - // default to saying it's a string for now - typeDef.Kind = core.TypeDefKindString - } - - return typeDef, true, nil - - case introspection.TypeKindEnum: - return &core.TypeDef{ - // just call it a string for now - Kind: core.TypeDefKindString, - Optional: !nonNull, - }, true, nil - - case introspection.TypeKindList: - elementTypeDef, ok, err := introspectionRefToTypeDef(introspectionType.OfType, false, isInput) - if err != nil { - return nil, false, fmt.Errorf("failed to convert list element type: %w", err) - } - if !ok { - return nil, false, nil - } - return &core.TypeDef{ - Kind: core.TypeDefKindList, - Optional: !nonNull, - AsList: dagql.NonNull(&core.ListTypeDef{ - ElementTypeDef: elementTypeDef, - }), - }, true, nil - - case introspection.TypeKindObject: - return &core.TypeDef{ - Kind: core.TypeDefKindObject, - Optional: !nonNull, - AsObject: dagql.NonNull(&core.ObjectTypeDef{ - Name: introspectionType.Name, - }), - }, true, nil - - case introspection.TypeKindInputObject: - return &core.TypeDef{ - Kind: core.TypeDefKindInput, - Optional: !nonNull, - AsInput: dagql.NonNull(&core.InputTypeDef{ - Name: introspectionType.Name, - }), - }, true, nil - - default: - return nil, false, fmt.Errorf("unexpected type kind %s", introspectionType.Kind) - } -} diff --git a/core/schema/module.go b/core/schema/module.go index f54c9ef8a8..4a6d257cc2 100644 --- a/core/schema/module.go +++ b/core/schema/module.go @@ -11,7 +11,6 @@ import ( "github.com/dagger/dagger/core/modules" "github.com/dagger/dagger/dagql" "github.com/dagger/dagger/engine" - "golang.org/x/sync/errgroup" ) type moduleSchema struct { @@ -161,6 +160,11 @@ func (s *moduleSchema) Install() { ArgDoc("name", `The name of the view to set.`). ArgDoc("patterns", `The patterns to set as the view filters.`). Doc(`Update the module source with a new named view.`), + + // have a func for returning this as a JSON string for SDKs, but a separate field + // for returning the actual introspection type could be added if ever needed + dagql.NodeFunc("schemaIntrospectionJSON", s.moduleSourceSchemaIntrospectionJSON). + Doc(`The graphql schema that will be presented to the module when invoked (excluding it's own self schema), formatted as JSON.`), }.Install(s.dag) dagql.Fields[*core.ModuleSourceView]{ @@ -468,11 +472,7 @@ func (s *moduleSchema) moduleServe(ctx context.Context, modMeta dagql.Instance[* } func (s *moduleSchema) currentTypeDefs(ctx context.Context, self *core.Query, _ struct{}) ([]*core.TypeDef, error) { - deps, err := self.CurrentServedDeps(ctx) - if err != nil { - return nil, fmt.Errorf("failed to get current module: %w", err) - } - return deps.TypeDefs(ctx) + return self.Deps.TypeDefs(ctx) } func (s *moduleSchema) functionCallReturnValue(ctx context.Context, fnCall *core.FunctionCall, args struct { @@ -707,60 +707,20 @@ func (s *moduleSchema) updateDeps( mod *core.Module, src dagql.Instance[*core.ModuleSource], ) error { - var deps []dagql.Instance[*core.ModuleDependency] - err := s.dag.Select(ctx, src, &deps, dagql.Selector{Field: "dependencies"}) + depCfgs, deps, err := s.loadModuleDependencies(ctx, src) if err != nil { return fmt.Errorf("failed to load module dependencies: %w", err) } - mod.DependencyConfig = make([]*core.ModuleDependency, len(deps)) - for i, dep := range deps { - // verify that the dependency config actually exists - _, cfgExists, err := dep.Self.Source.Self.ModuleConfig(ctx) - if err != nil { - return fmt.Errorf("failed to load module %q dependency %q config: %w", mod.NameField, dep.Self.Name, err) - } - if !cfgExists { - // best effort for err message, ignore err - sourceRootPath, _ := dep.Self.Source.Self.SourceRootSubpath() - return fmt.Errorf("module %q dependency %q with source root path %q does not exist or does not have a configuration file", mod.NameField, dep.Self.Name, sourceRootPath) - } - mod.DependencyConfig[i] = dep.Self - } - mod.DependenciesField = make([]dagql.Instance[*core.Module], len(deps)) - var eg errgroup.Group - for i, dep := range deps { - i, dep := i, dep - eg.Go(func() error { - err := s.dag.Select(ctx, dep.Self.Source, &mod.DependenciesField[i], - dagql.Selector{ - Field: "withName", - Args: []dagql.NamedInput{ - {Name: "name", Value: dagql.String(dep.Self.Name)}, - }, - }, - dagql.Selector{ - Field: "asModule", - }, - dagql.Selector{ - Field: "initialize", - }, - ) - if err != nil { - return fmt.Errorf("failed to initialize dependency module: %w", err) - } - return nil - }) - } - if err := eg.Wait(); err != nil { - return fmt.Errorf("failed to initialize dependency modules: %w", err) - } + mod.DependencyConfig = depCfgs - mod.Deps = core.NewModDeps(src.Self.Query, src.Self.Query.DefaultDeps.Mods) - for _, dep := range mod.DependenciesField { - mod.Deps = mod.Deps.Append(dep.Self) + depMods := make([]core.Mod, len(deps)) + mod.DependenciesField = make([]*core.Module, len(deps)) + for i, dep := range deps { + mod.DependenciesField[i] = dep.Self + depMods[i] = dep.Self } - + mod.Deps = core.NewModDeps(depMods).Append(&CoreMod{Dag: s.dag}) return nil } @@ -790,7 +750,7 @@ func (s *moduleSchema) updateCodegenAndRuntime( return fmt.Errorf("failed to load sdk for module: %w", err) } - generatedCode, err := sdk.Codegen(ctx, mod.Deps, src) + generatedCode, err := sdk.Codegen(ctx, mod.Source) if err != nil { return fmt.Errorf("failed to generate code: %w", err) } @@ -910,7 +870,7 @@ func (s *moduleSchema) updateCodegenAndRuntime( } } - mod.Runtime, err = sdk.Runtime(ctx, mod.Deps, src) + mod.Runtime, err = sdk.Runtime(ctx, mod.Source) if err != nil { return fmt.Errorf("failed to get module runtime: %w", err) } @@ -989,7 +949,7 @@ func (s *moduleSchema) updateDaggerConfig( depName := dep.Name if dep.Name == "" { // fill in default dep names if missing with the name of the module - depName = mod.DependenciesField[i].Self.Name() + depName = mod.DependenciesField[i].Name() } modCfg.Dependencies[i] = &modules.ModuleConfigDependency{ diff --git a/core/schema/modulesource.go b/core/schema/modulesource.go index df94e9c3b3..433a789013 100644 --- a/core/schema/modulesource.go +++ b/core/schema/modulesource.go @@ -9,6 +9,7 @@ import ( "sort" "strings" + "github.com/dagger/dagger/cmd/codegen/introspection" "github.com/dagger/dagger/core" "github.com/dagger/dagger/core/modules" "github.com/dagger/dagger/dagql" @@ -17,6 +18,14 @@ import ( "golang.org/x/sync/errgroup" ) +// We don't expose these types to modules SDK codegen, but +// we still want their graphql schemas to be available for +// internal usage. So we use this list to scrub them from +// the introspection JSON that module SDKs use for codegen. +var typesHiddenFromModuleSDKs = []dagql.Typed{ + &core.Host{}, +} + type moduleSourceArgs struct { // avoiding name "ref" due to that being a reserved word in some SDKs (e.g. Rust) RefString string @@ -399,6 +408,111 @@ func (s *moduleSchema) moduleSourceDependencies( return finalDeps, nil } +func (s *moduleSchema) moduleSourceSchemaIntrospectionJSON( + ctx context.Context, + src dagql.Instance[*core.ModuleSource], + args struct{}, +) (core.JSON, error) { + _, deps, err := s.loadModuleDependencies(ctx, src) + if err != nil { + return nil, fmt.Errorf("failed to load module dependencies: %w", err) + } + + depMods := make([]core.Mod, len(deps)) + for i, dep := range deps { + depMods[i] = dep.Self + } + root, err := src.Self.Query.NewRootForDependencies(ctx, core.NewModDeps(depMods)) + if err != nil { + return nil, fmt.Errorf("failed to create root for dependencies: %w", err) + } + + data, err := root.Dag.Query(ctx, introspection.Query, nil) + if err != nil { + return nil, fmt.Errorf("introspection query failed: %w", err) + } + jsonBytes, err := json.Marshal(data) + if err != nil { + return nil, fmt.Errorf("failed to marshal introspection result: %w", err) + } + + var introspection introspection.Response + if err := json.Unmarshal(jsonBytes, &introspection); err != nil { + return nil, fmt.Errorf("failed to unmarshal introspection JSON: %w", err) + } + + for _, typed := range typesHiddenFromModuleSDKs { + introspection.Schema.ScrubType(typed.Type().Name()) + introspection.Schema.ScrubType(dagql.IDTypeNameFor(typed)) + } + jsonBytes, err = json.Marshal(introspection) + if err != nil { + return nil, fmt.Errorf("failed to marshal introspection JSON: %w", err) + } + return core.JSON(jsonBytes), nil +} + +// internal util, loads all the module source deps as actual modules +func (s *moduleSchema) loadModuleDependencies( + ctx context.Context, + src dagql.Instance[*core.ModuleSource], +) ([]*core.ModuleDependency, []dagql.Instance[*core.Module], error) { + var deps []dagql.Instance[*core.ModuleDependency] + err := s.dag.Select(ctx, src, &deps, dagql.Selector{Field: "dependencies"}) + if err != nil { + return nil, nil, fmt.Errorf("failed to load module dependencies: %w", err) + } + + depCfgs := make([]*core.ModuleDependency, len(deps)) + for i, dep := range deps { + // verify that the dependency config actually exists + _, cfgExists, err := dep.Self.Source.Self.ModuleConfig(ctx) + if err != nil { + return nil, nil, fmt.Errorf("failed to load dependency %q config: %w", dep.Self.Name, err) + } + if !cfgExists { + // best effort for err message, ignore err + sourceRootPath, _ := dep.Self.Source.Self.SourceRootSubpath() + return nil, nil, fmt.Errorf( + "dependency %q with source root path %q does not exist or does not have a configuration file", + dep.Self.Name, sourceRootPath, + ) + } + depCfgs[i] = dep.Self + } + + depMods := make([]dagql.Instance[*core.Module], len(deps)) + var eg errgroup.Group + for i, dep := range deps { + i, dep := i, dep + eg.Go(func() error { + err := s.dag.Select(ctx, dep.Self.Source, &depMods[i], + dagql.Selector{ + Field: "withName", + Args: []dagql.NamedInput{ + {Name: "name", Value: dagql.String(dep.Self.Name)}, + }, + }, + dagql.Selector{ + Field: "asModule", + }, + dagql.Selector{ + Field: "initialize", + }, + ) + if err != nil { + return fmt.Errorf("failed to initialize dependency module: %w", err) + } + return nil + }) + } + if err := eg.Wait(); err != nil { + return nil, nil, fmt.Errorf("failed to initialize dependency modules: %w", err) + } + + return depCfgs, depMods, nil +} + func (s *moduleSchema) moduleSourceWithDependencies( ctx context.Context, src *core.ModuleSource, diff --git a/core/schema/sdk.go b/core/schema/sdk.go index 77de2f54f9..03bcd62d94 100644 --- a/core/schema/sdk.go +++ b/core/schema/sdk.go @@ -86,7 +86,7 @@ var errUnknownBuiltinSDK = fmt.Errorf("unknown builtin sdk") func (s *moduleSchema) builtinSDK(ctx context.Context, root *core.Query, sdkName string) (core.SDK, error) { switch sdkName { case "go": - return &goSDK{root: root, dag: s.dag}, nil + return &goSDK{root: root}, nil case "python": return s.loadBuiltinSDK(ctx, root, sdkName, digest.Digest(os.Getenv(distconsts.PythonSDKManifestDigestEnvName))) case "typescript": @@ -98,12 +98,12 @@ func (s *moduleSchema) builtinSDK(ctx context.Context, root *core.Query, sdkName // moduleSDK is an SDK implemented as module; i.e. every module besides the special case go sdk. type moduleSDK struct { - // The module implementing this SDK. - mod dagql.Instance[*core.Module] - // A server that the SDK module has been installed to. - dag *dagql.Server + // A query root that the SDK module has been installed to. + root *core.Query // The SDK object retrieved from the server, for calling functions against. sdk dagql.Object + // name of this module + modName string } func (s *moduleSchema) newModuleSDK( @@ -112,15 +112,9 @@ func (s *moduleSchema) newModuleSDK( sdkModMeta dagql.Instance[*core.Module], optionalFullSDKSourceDir dagql.Instance[*core.Directory], ) (*moduleSDK, error) { - dag := dagql.NewServer(root) - dag.Cache = root.Cache - if err := sdkModMeta.Self.Install(ctx, dag); err != nil { - return nil, fmt.Errorf("failed to install sdk module %s: %w", sdkModMeta.Self.Name(), err) - } - for _, defaultDep := range sdkModMeta.Self.Query.DefaultDeps.Mods { - if err := defaultDep.Install(ctx, dag); err != nil { - return nil, fmt.Errorf("failed to install default dep %s for sdk module %s: %w", defaultDep.Name(), sdkModMeta.Self.Name(), err) - } + newRoot, err := root.NewRootForDependencies(ctx, core.NewModDeps([]core.Mod{sdkModMeta.Self})) + if err != nil { + return nil, fmt.Errorf("failed to create new root for sdk module %s: %w", sdkModMeta.Self.Name(), err) } var sdk dagql.Object @@ -130,7 +124,7 @@ func (s *moduleSchema) newModuleSDK( {Name: "sdkSourceDir", Value: dagql.Opt(dagql.NewID[*core.Directory](optionalFullSDKSourceDir.ID()))}, } } - if err := dag.Select(ctx, dag.Root(), &sdk, + if err := newRoot.Dag.Select(ctx, newRoot.Dag.Root(), &sdk, dagql.Selector{ Field: gqlFieldName(sdkModMeta.Self.Name()), Args: constructorArgs, @@ -139,51 +133,56 @@ func (s *moduleSchema) newModuleSDK( return nil, fmt.Errorf("failed to get sdk object for sdk module %s: %w", sdkModMeta.Self.Name(), err) } - return &moduleSDK{mod: sdkModMeta, dag: dag, sdk: sdk}, nil + return &moduleSDK{root: newRoot, sdk: sdk, modName: sdkModMeta.Self.Name()}, nil } // Codegen calls the Codegen function on the SDK Module -func (sdk *moduleSDK) Codegen(ctx context.Context, deps *core.ModDeps, source dagql.Instance[*core.ModuleSource]) (*core.GeneratedCode, error) { - introspectionJSON, err := deps.SchemaIntrospectionJSON(ctx, true) - if err != nil { - return nil, fmt.Errorf("failed to get schema introspection json during %s module sdk codegen: %w", sdk.mod.Self.Name(), err) +func (sdk *moduleSDK) Codegen(ctx context.Context, modSrc dagql.Instance[*core.ModuleSource]) (*core.GeneratedCode, error) { + var introspectionJSON core.JSON + if err := sdk.root.Dag.Select(ctx, modSrc, &introspectionJSON, + dagql.Selector{Field: "schemaIntrospectionJSON"}, + ); err != nil { + modName, _ := modSrc.Self.ModuleName(ctx) + return nil, fmt.Errorf("failed to get schema introspection json for module %s during %s module sdk codegen: %w", modName, sdk.modName, err) } var inst dagql.Instance[*core.GeneratedCode] - err = sdk.dag.Select(ctx, sdk.sdk, &inst, dagql.Selector{ + if err := sdk.root.Dag.Select(ctx, sdk.sdk, &inst, dagql.Selector{ Field: "codegen", Args: []dagql.NamedInput{ { Name: "modSource", - Value: dagql.NewID[*core.ModuleSource](source.ID()), + Value: dagql.NewID[*core.ModuleSource](modSrc.ID()), }, { Name: "introspectionJson", Value: dagql.String(introspectionJSON), }, }, - }) - if err != nil { + }); err != nil { return nil, fmt.Errorf("failed to call sdk module codegen: %w", err) } return inst.Self, nil } // Runtime calls the Runtime function on the SDK Module -func (sdk *moduleSDK) Runtime(ctx context.Context, deps *core.ModDeps, source dagql.Instance[*core.ModuleSource]) (*core.Container, error) { - introspectionJSON, err := deps.SchemaIntrospectionJSON(ctx, true) - if err != nil { - return nil, fmt.Errorf("failed to get schema introspection json during %s module sdk runtime: %w", sdk.mod.Self.Name(), err) +func (sdk *moduleSDK) Runtime(ctx context.Context, modSrc dagql.Instance[*core.ModuleSource]) (*core.Container, error) { + var introspectionJSON core.JSON + if err := sdk.root.Dag.Select(ctx, modSrc, &introspectionJSON, + dagql.Selector{Field: "schemaIntrospectionJSON"}, + ); err != nil { + modName, _ := modSrc.Self.ModuleName(ctx) + return nil, fmt.Errorf("failed to get schema introspection json for module %s during %s module sdk codegen: %w", modName, sdk.modName, err) } var inst dagql.Instance[*core.Container] - err = sdk.dag.Select(ctx, sdk.sdk, &inst, + if err := sdk.root.Dag.Select(ctx, sdk.sdk, &inst, dagql.Selector{ Field: "moduleRuntime", Args: []dagql.NamedInput{ { Name: "modSource", - Value: dagql.NewID[*core.ModuleSource](source.ID()), + Value: dagql.NewID[*core.ModuleSource](modSrc.ID()), }, { Name: "introspectionJson", @@ -200,8 +199,7 @@ func (sdk *moduleSDK) Runtime(ctx context.Context, deps *core.ModDeps, source da }, }, }, - ) - if err != nil { + ); err != nil { return nil, fmt.Errorf("failed to call sdk module moduleRuntime: %w", err) } return inst.Self, nil @@ -209,7 +207,7 @@ func (sdk *moduleSDK) Runtime(ctx context.Context, deps *core.ModDeps, source da func (sdk *moduleSDK) RequiredPaths(ctx context.Context) ([]string, error) { var paths []string - err := sdk.dag.Select(ctx, sdk.sdk, &paths, + err := sdk.root.Dag.Select(ctx, sdk.sdk, &paths, dagql.Selector{ Field: "requiredPaths", }, @@ -295,21 +293,16 @@ it with the resulting /runtime binary. */ type goSDK struct { root *core.Query - dag *dagql.Server } -func (sdk *goSDK) Codegen( - ctx context.Context, - deps *core.ModDeps, - source dagql.Instance[*core.ModuleSource], -) (*core.GeneratedCode, error) { - ctr, err := sdk.baseWithCodegen(ctx, deps, source) +func (sdk *goSDK) Codegen(ctx context.Context, modSrc dagql.Instance[*core.ModuleSource]) (*core.GeneratedCode, error) { + ctr, err := sdk.baseWithCodegen(ctx, modSrc) if err != nil { return nil, err } var modifiedSrcDir dagql.Instance[*core.Directory] - if err := sdk.dag.Select(ctx, ctr, &modifiedSrcDir, dagql.Selector{ + if err := sdk.root.Dag.Select(ctx, ctr, &modifiedSrcDir, dagql.Selector{ Field: "directory", Args: []dagql.NamedInput{ { @@ -336,16 +329,12 @@ func (sdk *goSDK) Codegen( }, nil } -func (sdk *goSDK) Runtime( - ctx context.Context, - deps *core.ModDeps, - source dagql.Instance[*core.ModuleSource], -) (*core.Container, error) { - ctr, err := sdk.baseWithCodegen(ctx, deps, source) +func (sdk *goSDK) Runtime(ctx context.Context, modSrc dagql.Instance[*core.ModuleSource]) (*core.Container, error) { + ctr, err := sdk.baseWithCodegen(ctx, modSrc) if err != nil { return nil, err } - if err := sdk.dag.Select(ctx, ctr, &ctr, + if err := sdk.root.Dag.Select(ctx, ctr, &ctr, dagql.Selector{ Field: "withExec", Args: []dagql.NamedInput{ @@ -404,28 +393,29 @@ func (sdk *goSDK) RequiredPaths(_ context.Context) ([]string, error) { }, nil } -func (sdk *goSDK) baseWithCodegen( - ctx context.Context, - deps *core.ModDeps, - src dagql.Instance[*core.ModuleSource], -) (dagql.Instance[*core.Container], error) { +func (sdk *goSDK) baseWithCodegen(ctx context.Context, modSrc dagql.Instance[*core.ModuleSource]) (dagql.Instance[*core.Container], error) { var ctr dagql.Instance[*core.Container] - introspectionJSON, err := deps.SchemaIntrospectionJSON(ctx, true) - if err != nil { - return ctr, fmt.Errorf("failed to get schema introspection json during module sdk codegen: %w", err) + var introspectionJSON core.JSON + if err := sdk.root.Dag.Select(ctx, modSrc, &introspectionJSON, + dagql.Selector{Field: "schemaIntrospectionJSON"}, + ); err != nil { + modName, _ := modSrc.Self.ModuleName(ctx) + return ctr, fmt.Errorf("failed to get schema introspection json for module %s during go sdk codegen: %w", + modName, err, + ) } - modName, err := src.Self.ModuleOriginalName(ctx) + modName, err := modSrc.Self.ModuleOriginalName(ctx) if err != nil { return ctr, fmt.Errorf("failed to get module name for go module sdk codegen: %w", err) } - contextDir, err := src.Self.ContextDirectory() + contextDir, err := modSrc.Self.ContextDirectory() if err != nil { return ctr, fmt.Errorf("failed to get context directory for go module sdk codegen: %w", err) } - srcSubpath, err := src.Self.SourceSubpathWithDefault(ctx) + srcSubpath, err := modSrc.Self.SourceSubpathWithDefault(ctx) if err != nil { return ctr, fmt.Errorf("failed to get subpath for go module sdk codegen: %w", err) } @@ -440,12 +430,12 @@ func (sdk *goSDK) baseWithCodegen( // anyways. If it doesn't exist, we ignore not found in the implementation of // `withoutFile` so it will be a no-op. var emptyDir dagql.Instance[*core.Directory] - if err := sdk.dag.Select(ctx, sdk.dag.Root(), &emptyDir, dagql.Selector{Field: "directory"}); err != nil { + if err := sdk.root.Dag.Select(ctx, sdk.root.Dag.Root(), &emptyDir, dagql.Selector{Field: "directory"}); err != nil { return ctr, fmt.Errorf("failed to create empty directory for go module sdk codegen: %w", err) } var updatedContextDir dagql.Instance[*core.Directory] - if err := sdk.dag.Select(ctx, contextDir, &updatedContextDir, + if err := sdk.root.Dag.Select(ctx, contextDir, &updatedContextDir, dagql.Selector{ Field: "withDirectory", Args: []dagql.NamedInput{ @@ -466,7 +456,7 @@ func (sdk *goSDK) baseWithCodegen( return ctr, fmt.Errorf("failed to remove dagger.gen.go from source directory: %w", err) } - if err := sdk.dag.Select(ctx, ctr, &ctr, dagql.Selector{ + if err := sdk.root.Dag.Select(ctx, ctr, &ctr, dagql.Selector{ Field: "withNewFile", Args: []dagql.NamedInput{ { @@ -475,7 +465,7 @@ func (sdk *goSDK) baseWithCodegen( }, { Name: "contents", - Value: dagql.NewString(introspectionJSON), + Value: dagql.String(introspectionJSON), }, { Name: "permissions", @@ -532,7 +522,7 @@ func (sdk *goSDK) base(ctx context.Context) (dagql.Instance[*core.Container], er var inst dagql.Instance[*core.Container] var modCache dagql.Instance[*core.CacheVolume] - if err := sdk.dag.Select(ctx, sdk.dag.Root(), &modCache, dagql.Selector{ + if err := sdk.root.Dag.Select(ctx, sdk.root.Dag.Root(), &modCache, dagql.Selector{ Field: "cacheVolume", Args: []dagql.NamedInput{ { @@ -544,7 +534,7 @@ func (sdk *goSDK) base(ctx context.Context) (dagql.Instance[*core.Container], er return inst, fmt.Errorf("failed to get mod cache from go module sdk tarball: %w", err) } var buildCache dagql.Instance[*core.CacheVolume] - if err := sdk.dag.Select(ctx, sdk.dag.Root(), &buildCache, dagql.Selector{ + if err := sdk.root.Dag.Select(ctx, sdk.root.Dag.Root(), &buildCache, dagql.Selector{ Field: "cacheVolume", Args: []dagql.NamedInput{ { @@ -557,7 +547,7 @@ func (sdk *goSDK) base(ctx context.Context) (dagql.Instance[*core.Container], er } var ctr dagql.Instance[*core.Container] - if err := sdk.dag.Select(ctx, sdk.dag.Root(), &ctr, dagql.Selector{ + if err := sdk.root.Dag.Select(ctx, sdk.root.Dag.Root(), &ctr, dagql.Selector{ Field: "builtinContainer", Args: []dagql.NamedInput{ { diff --git a/core/schema/util.go b/core/schema/util.go index bca52015cf..78079a645b 100644 --- a/core/schema/util.go +++ b/core/schema/util.go @@ -2,11 +2,8 @@ package schema import ( "context" - "encoding/json" - "fmt" "github.com/dagger/dagger/dagql" - "github.com/dagger/dagger/dagql/introspection" "github.com/dagger/dagger/engine/buildkit" "github.com/iancoleman/strcase" ) @@ -70,18 +67,6 @@ func asArrayInput[T any, I dagql.Input](ts []T, conv func(T) I) dagql.ArrayInput return ins } -func schemaIntrospectionJSON(ctx context.Context, dag *dagql.Server) (json.RawMessage, error) { - data, err := dag.Query(ctx, introspection.Query, nil) - if err != nil { - return nil, fmt.Errorf("introspection query failed: %w", err) - } - jsonBytes, err := json.Marshal(data) - if err != nil { - return nil, fmt.Errorf("failed to marshal introspection result: %w", err) - } - return json.RawMessage(jsonBytes), nil -} - func gqlFieldName(name string) string { // gql field name is uncapitalized camel case return strcase.ToLowerCamel(name) diff --git a/core/typedef.go b/core/typedef.go index 795a4ee6b0..d756905d2a 100644 --- a/core/typedef.go +++ b/core/typedef.go @@ -376,6 +376,20 @@ func (typeDef *TypeDef) Underlying() *TypeDef { } } +// if the type is an object, input or interface, returns the name of the type +func (typeDef *TypeDef) Name() string { + switch typeDef.Kind { + case TypeDefKindObject: + return typeDef.AsObject.Value.Name + case TypeDefKindInterface: + return typeDef.AsInterface.Value.Name + case TypeDefKindInput: + return typeDef.AsInput.Value.Name + default: + return "" + } +} + func (typeDef *TypeDef) WithKind(kind TypeDefKind) *TypeDef { typeDef = typeDef.Clone() typeDef.Kind = kind diff --git a/dagql/objects.go b/dagql/objects.go index 3704f3f42a..d4312af9dc 100644 --- a/dagql/objects.go +++ b/dagql/objects.go @@ -19,10 +19,11 @@ import ( // The class is defined by a set of fields, which are installed into the class // dynamically at runtime. type Class[T Typed] struct { - inner T - idable bool - fields map[string]*Field[T] - fieldsL *sync.Mutex + inner T + idable bool + fields map[string]*Field[T] + fieldsL *sync.Mutex + sourceModule *call.Module } type ClassOpts[T Typed] struct { @@ -34,6 +35,9 @@ type ClassOpts[T Typed] struct { // In the simple case, we can just use a zero-value, but it is also allowed // to use a dynamic Typed value. Typed T + + // If this object is defined by a user module, this is the module. Otherwise nil. + SourceModule *call.Module } // NewClass returns a new empty class for a given type. @@ -43,9 +47,10 @@ func NewClass[T Typed](opts_ ...ClassOpts[T]) Class[T] { opts = opts_[0] } class := Class[T]{ - inner: opts.Typed, - fields: map[string]*Field[T]{}, - fieldsL: new(sync.Mutex), + inner: opts.Typed, + fields: map[string]*Field[T]{}, + fieldsL: new(sync.Mutex), + sourceModule: opts.SourceModule, } if !opts.NoIDs { class.Install( @@ -102,6 +107,10 @@ func (cls Class[T]) TypeName() string { return cls.inner.Type().Name() } +func (class Class[T]) SourceModule() *call.Module { + return class.sourceModule +} + func (cls Class[T]) Extend(spec FieldSpec, fun FieldFunc) { cls.fieldsL.Lock() defer cls.fieldsL.Unlock() diff --git a/dagql/server.go b/dagql/server.go index b504f84271..dd2f410823 100644 --- a/dagql/server.go +++ b/dagql/server.go @@ -244,6 +244,16 @@ func (s *Server) ObjectType(name string) (ObjectType, bool) { return t, ok } +func (s *Server) ObjectTypes() []ObjectType { + s.installLock.Lock() + defer s.installLock.Unlock() + types := make([]ObjectType, 0, len(s.objects)) + for _, t := range s.objects { + types = append(types, t) + } + return types +} + // ScalarType returns the ScalarType with the given name, if it exists. func (s *Server) ScalarType(name string) (ScalarType, bool) { s.installLock.Lock() @@ -252,6 +262,16 @@ func (s *Server) ScalarType(name string) (ScalarType, bool) { return t, ok } +func (s *Server) ScalarTypes() []ScalarType { + s.installLock.Lock() + defer s.installLock.Unlock() + types := make([]ScalarType, 0, len(s.scalars)) + for _, t := range s.scalars { + types = append(types, t) + } + return types +} + // InputType returns the InputType with the given name, if it exists. func (s *Server) TypeDef(name string) (TypeDef, bool) { s.installLock.Lock() @@ -260,6 +280,16 @@ func (s *Server) TypeDef(name string) (TypeDef, bool) { return t, ok } +func (s *Server) TypeDefs() []TypeDef { + s.installLock.Lock() + defer s.installLock.Unlock() + types := make([]TypeDef, 0, len(s.typeDefs)) + for _, t := range s.typeDefs { + types = append(types, t) + } + return types +} + // Around installs a function to be called around every non-cached selection. func (s *Server) Around(rec AroundFunc) { s.telemetry = rec diff --git a/dagql/types.go b/dagql/types.go index 08422a65b5..dc39894912 100644 --- a/dagql/types.go +++ b/dagql/types.go @@ -47,6 +47,10 @@ type ObjectType interface { // Unlike natively added fields, the extended func is limited to the external // Object interface. Extend(FieldSpec, FieldFunc) + // If this object is defined by a user module, that module. Otherwise nil. + SourceModule() *call.Module + // The underlying ast definition for the object. + TypeDefinition() *ast.Definition } type IDType interface { diff --git a/engine/server/server.go b/engine/server/server.go index 97de4c550b..6cdbc7bc00 100644 --- a/engine/server/server.go +++ b/engine/server/server.go @@ -10,6 +10,7 @@ import ( "net/http" "runtime" "runtime/debug" + "strings" "sync" "time" @@ -22,6 +23,8 @@ import ( "github.com/dagger/dagger/core/pipeline" "github.com/dagger/dagger/core/schema" "github.com/dagger/dagger/dagql" + "github.com/dagger/dagger/dagql/call" + dagintro "github.com/dagger/dagger/dagql/introspection" "github.com/dagger/dagger/engine" "github.com/dagger/dagger/engine/buildkit" "github.com/dagger/dagger/engine/cache" @@ -32,6 +35,8 @@ import ( "github.com/moby/buildkit/identity" "github.com/moby/buildkit/session" "github.com/moby/buildkit/util/bklog" + "github.com/moby/locker" + "github.com/opencontainers/go-digest" "github.com/sirupsen/logrus" "github.com/vektah/gqlparser/v2/gqlerror" "github.com/vito/progrock" @@ -40,19 +45,21 @@ import ( type DaggerServer struct { serverID string - clientIDToSecretToken map[string]string - connectedClients int - clientIDMu sync.RWMutex - - // The metadata of client calls. + // The root for each client, keyed by client ID // This is never explicitly deleted from; instead it will just be garbage collected // when this server for the session shuts down - clientCallContext map[string]*core.ClientCallContext - clientCallMu *sync.RWMutex + clientRoots map[string]*core.Query + clientRootMu sync.RWMutex + perClientMu *locker.Locker + connectedClients int // the http endpoints being served (as a map since APIs like shellEndpoint can add more) endpoints map[string]http.Handler - endpointMu *sync.RWMutex + endpointMu sync.RWMutex + + dagqlCache dagql.Cache + queryOpts core.QueryOpts + buildkitOpts *buildkit.Opts services *core.Services @@ -72,11 +79,11 @@ func (e *BuildkitController) newDaggerServer(ctx context.Context, clientMetadata s := &DaggerServer{ serverID: clientMetadata.ServerID, - clientIDToSecretToken: map[string]string{}, - clientCallContext: map[string]*core.ClientCallContext{}, - clientCallMu: &sync.RWMutex{}, - endpoints: map[string]http.Handler{}, - endpointMu: &sync.RWMutex{}, + clientRoots: map[string]*core.Query{}, + perClientMu: locker.New(), + endpoints: map[string]http.Handler{}, + + dagqlCache: dagql.NewCache(), doneCh: make(chan struct{}, 1), @@ -156,21 +163,23 @@ func (e *BuildkitController) newDaggerServer(ctx context.Context, clientMetadata }) } - root, err := core.NewRoot(ctx, core.QueryOpts{ - BuildkitOpts: &buildkit.Opts{ - Worker: e.worker, - SessionManager: e.SessionManager, - LLBSolver: e.llbSolver, - GenericSolver: e.genericSolver, - SecretStore: secretStore, - AuthProvider: authProvider, - PrivilegedExecEnabled: e.privilegedExecEnabled, - UpstreamCacheImports: cacheImporterCfgs, - ProgSockPath: progSockPath, - MainClientCaller: sessionCaller, - DNSConfig: e.DNSConfig, - Frontends: e.Frontends, - }, + s.buildkitOpts = &buildkit.Opts{ + Worker: e.worker, + SessionManager: e.SessionManager, + LLBSolver: e.llbSolver, + GenericSolver: e.genericSolver, + SecretStore: secretStore, + AuthProvider: authProvider, + PrivilegedExecEnabled: e.privilegedExecEnabled, + UpstreamCacheImports: cacheImporterCfgs, + ProgSockPath: progSockPath, + MainClientCaller: sessionCaller, + DNSConfig: e.DNSConfig, + Frontends: e.Frontends, + } + + s.queryOpts = core.QueryOpts{ + DaggerServer: s, ProgrockSocketPath: progSockPath, Services: s.services, Platform: core.Platform(e.worker.Platforms(true)[0]), @@ -178,36 +187,15 @@ func (e *BuildkitController) newDaggerServer(ctx context.Context, clientMetadata OCIStore: e.worker.ContentStore(), LeaseManager: e.worker.LeaseManager(), Auth: authProvider, - ClientCallContext: s.clientCallContext, - ClientCallMu: s.clientCallMu, MainClientCallerID: s.mainClientCallerID, - Endpoints: s.endpoints, - EndpointMu: s.endpointMu, - Recorder: s.recorder, - }) - if err != nil { - return nil, err } - dag := dagql.NewServer(root) - - // stash away the cache so we can share it between other servers - root.Cache = dag.Cache - - dag.Around(tracing.AroundFunc) - - coreMod := &schema.CoreMod{Dag: dag} - root.DefaultDeps = core.NewModDeps(root, []core.Mod{coreMod}) - if err := coreMod.Install(ctx, dag); err != nil { + // register the main client caller + newRoot, err := s.newRootForClient(ctx, s.mainClientCallerID, nil) + if err != nil { return nil, err } - - // the main client caller starts out with the core API loaded - s.clientCallContext[s.mainClientCallerID] = &core.ClientCallContext{ - Deps: root.DefaultDeps, - Root: root, - } - + s.clientRoots[newRoot.ClientID] = newRoot return s, nil } @@ -222,14 +210,14 @@ func (s *DaggerServer) ServeClientConn( return fmt.Errorf("failed to verify client: %w", err) } - s.clientIDMu.Lock() + s.clientRootMu.Lock() s.connectedClients++ + s.clientRootMu.Unlock() defer func() { - s.clientIDMu.Lock() + s.clientRootMu.Lock() s.connectedClients-- - s.clientIDMu.Unlock() + s.clientRootMu.Unlock() }() - s.clientIDMu.Unlock() conn = newLogicalDeadlineConn(splitWriteConn{nopCloserConn{conn}, defaults.DefaultMaxSendMsgSize * 95 / 100}) l := &singleConnListener{conn: conn, closeCh: make(chan struct{})} @@ -266,7 +254,7 @@ func (s *DaggerServer) ServeHTTP(w http.ResponseWriter, r *http.Request) { return } - callContext, ok := s.ClientCallContext(clientMetadata.ClientID) + root, ok := s.rootFor(clientMetadata.ClientID) if !ok { errorOut(fmt.Errorf("client call for %s not found", clientMetadata.ClientID), http.StatusInternalServerError) return @@ -275,21 +263,11 @@ func (s *DaggerServer) ServeHTTP(w http.ResponseWriter, r *http.Request) { rec := progrock.FromContext(ctx) if header := r.Header.Get(client.ProgrockParentHeader); header != "" { rec = rec.WithParent(header) - } else if callContext.ProgrockParent != "" { - rec = rec.WithParent(callContext.ProgrockParent) + } else if root.ProgrockParent != "" { + rec = rec.WithParent(root.ProgrockParent) } ctx = progrock.ToContext(ctx, rec) - s.clientCallMu.RLock() - schema, err := callContext.Deps.Schema(ctx) - if err != nil { - s.clientCallMu.RUnlock() - // TODO: technically this is not *always* bad request, should ideally be more specific and differentiate - errorOut(err, http.StatusBadRequest) - return - } - s.clientCallMu.RUnlock() - defer func() { if v := recover(); v != nil { bklog.G(context.TODO()).Errorf("panic serving schema: %v %s", v, string(debug.Stack())) @@ -320,14 +298,15 @@ func (s *DaggerServer) ServeHTTP(w http.ResponseWriter, r *http.Request) { } }() - srv := handler.NewDefaultServer(schema) - // NB: break glass when needed: - // srv.AroundResponses(func(ctx context.Context, next graphql.ResponseHandler) *graphql.Response { - // res := next(ctx) - // pl, err := json.Marshal(res) - // slog.Debug("graphql response", "response", string(pl), "error", err) - // return res - // }) + srv := handler.NewDefaultServer(root.Dag) + /* NB: break glass when needed: + srv.AroundResponses(func(ctx context.Context, next graphql.ResponseHandler) *graphql.Response { + res := next(ctx) + pl, err := json.Marshal(res) + slog.Debug("graphql response", "response", string(pl), "error", err) + return res + }) + */ mux := http.NewServeMux() mux.Handle("/query", srv) mux.Handle("/shutdown", http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { @@ -345,9 +324,9 @@ func (s *DaggerServer) ServeHTTP(w http.ResponseWriter, r *http.Request) { return exporterFunc(ctx, sessionGroup, cacheExportCfg.Attrs) } } - s.clientCallMu.RLock() - bk := s.clientCallContext[s.mainClientCallerID].Root.Buildkit - s.clientCallMu.RUnlock() + s.clientRootMu.RLock() + bk := s.clientRoots[s.mainClientCallerID].Buildkit + s.clientRootMu.RUnlock() err := bk.UpstreamCacheExport(ctx, cacheExporterFuncs) if err != nil { bklog.G(ctx).WithError(err).Errorf("error running cache export for client %s", clientMetadata.ClientID) @@ -369,16 +348,19 @@ func (s *DaggerServer) ServeHTTP(w http.ResponseWriter, r *http.Request) { } func (s *DaggerServer) RegisterClient(clientID, clientHostname, secretToken string) error { - s.clientIDMu.Lock() - defer s.clientIDMu.Unlock() - existingToken, ok := s.clientIDToSecretToken[clientID] - if ok { + s.clientRootMu.Lock() + defer s.clientRootMu.Unlock() + existingRoot, ok := s.clientRoots[clientID] + if !ok { + return fmt.Errorf("client ID %q not found", clientID) + } + if existingToken := existingRoot.SecretToken; existingToken != "" { if existingToken != secretToken { return fmt.Errorf("client ID %q already registered with different secret token", clientID) } return nil } - s.clientIDToSecretToken[clientID] = secretToken + existingRoot.SecretToken = secretToken // NOTE: we purposely don't delete the secret token, it should never be reused and will be released // from memory once the dagger server instance corresponding to this buildkit client shuts down. // Deleting it would make it easier to create race conditions around using the client's session @@ -388,22 +370,197 @@ func (s *DaggerServer) RegisterClient(clientID, clientHostname, secretToken stri } func (s *DaggerServer) VerifyClient(clientID, secretToken string) error { - s.clientIDMu.RLock() - defer s.clientIDMu.RUnlock() - existingToken, ok := s.clientIDToSecretToken[clientID] + s.clientRootMu.RLock() + defer s.clientRootMu.RUnlock() + existingRoot, ok := s.clientRoots[clientID] if !ok { - return fmt.Errorf("client ID %q not registered", clientID) + return fmt.Errorf("client ID %q not found", clientID) } - if existingToken != secretToken { + if existingRoot.SecretToken != secretToken { return fmt.Errorf("client ID %q registered with different secret token", clientID) } return nil } -func (s *DaggerServer) ClientCallContext(clientID string) (*core.ClientCallContext, bool) { - s.clientCallMu.RLock() - defer s.clientCallMu.RUnlock() - ctx, ok := s.clientCallContext[clientID] +func (s *DaggerServer) MuxEndpoint(ctx context.Context, path string, handler http.Handler) error { + s.endpointMu.Lock() + defer s.endpointMu.Unlock() + s.endpoints[path] = handler + return nil +} + +func (s *DaggerServer) ServeModule(ctx context.Context, mod *core.Module) error { + clientMetadata, err := engine.ClientMetadataFromContext(ctx) + if err != nil { + return err + } + + s.clientRootMu.Lock() + defer s.clientRootMu.Unlock() + root, ok := s.clientRoots[clientMetadata.ClientID] + if !ok { + return fmt.Errorf("client call for %s not found", clientMetadata.ClientID) + } + root.Deps = root.Deps.Append(mod) + if err := mod.Install(ctx, root.Dag); err != nil { + return fmt.Errorf("install module: %w", err) + } + return nil +} + +// Initialize a new root Query for the current dagql call (or return an existing one if it already exists for the current call's digest). +func (s *DaggerServer) NewRootForCurrentCall(ctx context.Context, fnCall *core.FunctionCall) (*core.Query, error) { + if fnCall == nil { + fnCall = &core.FunctionCall{} + } + + clientID := s.digestToClientID(dagql.CurrentID(ctx).Digest(), fnCall.Cache) + + s.perClientMu.Lock(clientID) + defer s.perClientMu.Unlock(clientID) + + s.clientRootMu.Lock() + if root, ok := s.clientRoots[clientID]; ok { + s.clientRootMu.Unlock() + return root, nil + } + s.clientRootMu.Unlock() + + newRoot, err := s.newRootForClient(ctx, clientID, fnCall) + if err != nil { + return nil, fmt.Errorf("new root: %w", err) + } + s.clientRoots[newRoot.ClientID] = newRoot + return newRoot, nil +} + +// Initialize a new root Query for the given set of dependency mods (or return an existing one if it already exists for the dep's digest). +func (s *DaggerServer) NewRootForDependencies(ctx context.Context, deps *core.ModDeps) (*core.Query, error) { + clientID := "deps" + s.digestToClientID(deps.Digest(), false) + + s.perClientMu.Lock(clientID) + defer s.perClientMu.Unlock(clientID) + + s.clientRootMu.Lock() + if root, ok := s.clientRoots[clientID]; ok { + s.clientRootMu.Unlock() + return root, nil + } + s.clientRootMu.Unlock() + + newRoot, err := s.newRootForClient(ctx, clientID, nil) + if err != nil { + return nil, fmt.Errorf("new root: %w", err) + } + if err := deps.Install(ctx, newRoot.Dag); err != nil { + return nil, fmt.Errorf("install dep: %w", err) + } + newRoot.Deps = newRoot.Deps.Append(deps.Mods...) + + s.clientRoots[newRoot.ClientID] = newRoot + return newRoot, nil +} + +// Initialize a new root Query for the given ID based on the modules it needs to be loaded (or return an existing one if it already exists for the ID's digest). +func (s *DaggerServer) NewRootForDynamicID(ctx context.Context, id *call.ID) (*core.Query, error) { + clientID := "dynamicid" + s.digestToClientID(id.Digest(), false) + + s.perClientMu.Lock(clientID) + defer s.perClientMu.Unlock(clientID) + + s.clientRootMu.Lock() + if root, ok := s.clientRoots[clientID]; ok { + s.clientRootMu.Unlock() + return root, nil + } + s.clientRootMu.Unlock() + + newRoot, err := s.newRootForClient(ctx, clientID, nil) + if err != nil { + return nil, fmt.Errorf("new root: %w", err) + } + + deps := core.NewModDeps(nil) + for _, modID := range id.Modules() { + mod, err := dagql.NewID[*core.Module](modID.ID()).Load(ctx, newRoot.Dag) + if err != nil { + return nil, fmt.Errorf("load source mod: %w", err) + } + deps = deps.Append(mod.Self) + } + if err := deps.Install(ctx, newRoot.Dag); err != nil { + return nil, fmt.Errorf("install deps for dynamic id: %w", err) + } + newRoot.Deps = newRoot.Deps.Append(deps.Mods...) + + s.clientRoots[newRoot.ClientID] = newRoot + return newRoot, nil +} + +func (s *DaggerServer) digestToClientID(dgst digest.Digest, cache bool) string { + clientIDInputs := []string{dgst.String()} + if !cache { + // use the ServerID so that we bust cache once-per-session + clientIDInputs = append(clientIDInputs, s.serverID) + } + clientIDDigest := digest.FromString(strings.Join(clientIDInputs, " ")) + + // only use encoded part of digest because this ID ends up becoming a buildkit Session ID + // and buildkit has some ancient internal logic that splits on a colon to support some + // dev mode logic: https://github.com/moby/buildkit/pull/290 + // also trim it to 25 chars as it ends up becoming part of service URLs + return clientIDDigest.Encoded()[:25] +} + +func (s *DaggerServer) newRootForClient(ctx context.Context, clientID string, fnCall *core.FunctionCall) (*core.Query, error) { + if fnCall == nil { + fnCall = &core.FunctionCall{} + } + root := &core.Query{ + QueryOpts: s.queryOpts, + ClientID: clientID, + FnCall: fnCall, + ProgrockParent: progrock.FromContext(ctx).Parent, + } + + var err error + root.Buildkit, err = buildkit.NewClient(ctx, s.buildkitOpts) + if err != nil { + return nil, fmt.Errorf("buildkit client: %w", err) + } + + // NOTE: context.WithoutCancel is used because if the provided context is canceled, buildkit can + // leave internal progress contexts open and leak goroutines. + root.Buildkit.WriteStatusesTo(context.WithoutCancel(ctx), s.recorder) + + root.Dag = dagql.NewServer(root) + root.Dag.Around(tracing.AroundFunc) + root.Dag.Cache = s.dagqlCache + dagintro.Install[*core.Query](root.Dag) + + coreMod := &schema.CoreMod{Dag: root.Dag} + root.Deps = core.NewModDeps([]core.Mod{coreMod}) + if fnCall.Module != nil { + root.Deps = root.Deps.Append(fnCall.Module.Deps.Mods...) + // By default, serve both deps and the module's own API to itself. But if SkipSelfSchema is set, + // only serve the APIs of the deps of this module. This is currently only needed for the special + // case of the function used to get the definition of the module itself (which can't obviously + // be served the API its returning the definition of). + if !fnCall.SkipSelfSchema { + root.Deps = root.Deps.Append(fnCall.Module) + } + } + if err := root.Deps.Install(ctx, root.Dag); err != nil { + return nil, fmt.Errorf("install deps: %w", err) + } + + return root, nil +} + +func (s *DaggerServer) rootFor(clientID string) (*core.Query, bool) { + s.clientRootMu.RLock() + defer s.clientRootMu.RUnlock() + ctx, ok := s.clientRoots[clientID] return ctx, ok } @@ -412,16 +569,16 @@ func (s *DaggerServer) CurrentServedDeps(ctx context.Context) (*core.ModDeps, er if err != nil { return nil, err } - callCtx, ok := s.ClientCallContext(clientMetadata.ClientID) + root, ok := s.rootFor(clientMetadata.ClientID) if !ok { return nil, fmt.Errorf("client call for %s not found", clientMetadata.ClientID) } - return callCtx.Deps, nil + return root.Deps, nil } func (s *DaggerServer) LogMetrics(l *logrus.Entry) *logrus.Entry { - s.clientIDMu.RLock() - defer s.clientIDMu.RUnlock() + s.clientRootMu.RLock() + defer s.clientRootMu.RUnlock() return l.WithField(fmt.Sprintf("server-%s-client-count", s.serverID), s.connectedClients) } @@ -436,11 +593,11 @@ func (s *DaggerServer) Close(ctx context.Context) error { slog.Error("failed to stop client services", "error", err) } - s.clientCallMu.RLock() - for _, callCtx := range s.clientCallContext { - err = errors.Join(err, callCtx.Root.Buildkit.Close()) + s.clientRootMu.RLock() + for _, root := range s.clientRoots { + err = errors.Join(err, root.Buildkit.Close()) } - s.clientCallMu.RUnlock() + s.clientRootMu.RUnlock() // mark all groups completed s.recorder.Complete() From 698fd710fc63566dfb3ed275ece6ae47a8b96097 Mon Sep 17 00:00:00 2001 From: Erik Sipsma Date: Tue, 26 Mar 2024 16:58:04 -0700 Subject: [PATCH 4/4] wip Signed-off-by: Erik Sipsma --- core/container.go | 91 +++++++++++------------ core/host.go | 11 +-- core/integration/module_test.go | 2 - core/interface.go | 15 ++-- core/modfunc.go | 22 +++++- core/object.go | 15 ++-- core/query.go | 17 +++-- core/schema/container.go | 10 +-- core/schema/directory.go | 2 +- core/schema/host.go | 2 +- core/schema/secret.go | 42 +++++------ core/secret.go | 27 ------- core/util.go | 65 +++++++++++++++++ dagql/referencedTypes.go | 106 +++++++++++++++++++++++++++ dagql/server.go | 15 ++++ engine/buildkit/client.go | 124 ++------------------------------ engine/buildkit/session.go | 5 +- engine/server/server.go | 93 +++++++++++++++++------- 18 files changed, 372 insertions(+), 292 deletions(-) create mode 100644 dagql/referencedTypes.go diff --git a/core/container.go b/core/container.go index ca6cc339e2..b61a25416e 100644 --- a/core/container.go +++ b/core/container.go @@ -55,8 +55,6 @@ type DefaultTerminalCmdOpts struct { // Container is a content-addressed container. type Container struct { - Query *Query - // The container's root filesystem. FS *pb.Definition `json:"fs"` @@ -135,12 +133,8 @@ func (container *Container) PBDefinitions(ctx context.Context) ([]*pb.Definition return defs, nil } -func NewContainer(root *Query, platform Platform) (*Container, error) { - if root == nil { - panic("query must be non-nil") - } +func NewContainer(platform Platform) (*Container, error) { return &Container{ - Query: root, Platform: platform, }, nil } @@ -167,7 +161,9 @@ var _ pipeline.Pipelineable = (*Container)(nil) // PipelinePath returns the container's pipeline path. func (container *Container) PipelinePath() pipeline.Path { - return container.Query.Pipeline + //TODO: + return nil + // return container.Query.Pipeline } // Ownership contains a UID/GID pair resolved from a user/group name or ID pair @@ -281,7 +277,7 @@ func (mnts ContainerMounts) With(newMnt ContainerMount) ContainerMounts { } func (container *Container) From(ctx context.Context, addr string) (*Container, error) { - bk := container.Query.Buildkit + bk := CurrentQuery(ctx).Buildkit container = container.Clone() @@ -361,8 +357,8 @@ func (container *Container) Build( // set image ref to empty string container.ImageRef = "" - svcs := container.Query.Services - bk := container.Query.Buildkit + svcs := CurrentQuery(ctx).Services + bk := CurrentQuery(ctx).Buildkit // add a weak group for the docker build vertices ctx, subRecorder := progrock.WithGroup(ctx, "docker build", progrock.Weak()) @@ -399,13 +395,7 @@ func (container *Container) Build( dockerui.DefaultLocalNameDockerfile: contextDir.LLB, } - // FIXME: ew, this is a terrible way to pass this around - //nolint:staticcheck - solveCtx := context.WithValue(ctx, "secret-translator", func(name string) (string, error) { - return GetLocalSecretAccessor(ctx, container.Query, name) - }) - - res, err := bk.Solve(solveCtx, bkgw.SolveRequest{ + res, err := bk.Solve(ctx, bkgw.SolveRequest{ Frontend: "dockerfile.v0", FrontendOpt: opts, FrontendInputs: inputs, @@ -455,7 +445,7 @@ func (container *Container) Build( func (container *Container) RootFS(ctx context.Context) (*Directory, error) { return &Directory{ - Query: container.Query, + Query: CurrentQuery(ctx), LLB: container.FS, Dir: "/", Platform: container.Platform, @@ -743,8 +733,8 @@ func (container *Container) Directory(ctx context.Context, dirPath string) (*Dir return nil, err } - svcs := container.Query.Services - bk := container.Query.Buildkit + svcs := CurrentQuery(ctx).Services + bk := CurrentQuery(ctx).Buildkit // check that the directory actually exists so the user gets an error earlier // rather than when the dir is used @@ -811,7 +801,7 @@ func locatePath[T *File | *Directory]( } return init( - container.Query, + CurrentQuery(ctx), mnt.Source, sub, container.Platform, @@ -822,7 +812,7 @@ func locatePath[T *File | *Directory]( // Not found in a mount return init( - container.Query, + CurrentQuery(ctx), container.FS, containerPath, container.Platform, @@ -899,7 +889,7 @@ func (container *Container) chown( return nil, "", err } - ref, err := bkRef(ctx, container.Query.Buildkit, def.ToPB()) + ref, err := bkRef(ctx, CurrentQuery(ctx).Buildkit, def.ToPB()) if err != nil { return nil, "", err } @@ -978,7 +968,8 @@ func (container *Container) UpdateImageConfig(ctx context.Context, updateFn func func (container *Container) WithPipeline(ctx context.Context, name, description string, labels []pipeline.Label) (*Container, error) { container = container.Clone() - container.Query = container.Query.WithPipeline(name, description, labels) + // TODO: + // container.Query = container.Query.WithPipeline(name, description, labels) return container, nil } @@ -999,7 +990,7 @@ func (container *Container) WithExec(ctx context.Context, opts ContainerExecOpts mounts := container.Mounts platform := container.Platform if platform.OS == "" { - platform = container.Query.Platform + platform = CurrentQuery(ctx).Platform } args, err := container.command(opts) @@ -1021,9 +1012,13 @@ func (container *Container) WithExec(ctx context.Context, opts ContainerExecOpts // this allows executed containers to communicate back to this API if opts.ExperimentalPrivilegedNesting { - newRoot, err := container.Query.NewRootForCurrentCall(ctx, opts.NestedExecFunctionCall) - if err != nil { - return nil, fmt.Errorf("register caller: %w", err) + newRoot := opts.NestedExecRoot + if newRoot == nil { + var err error + newRoot, err = CurrentQuery(ctx).NewRootForCurrentCall(ctx, nil) + if err != nil { + return nil, fmt.Errorf("register caller: %w", err) + } } runOpts = append(runOpts, llb.AddEnv("_DAGGER_NESTED_CLIENT_ID", newRoot.ClientID), @@ -1087,7 +1082,7 @@ func (container *Container) WithExec(ctx context.Context, opts ContainerExecOpts secretsToScrub := SecretToScrubInfo{} for i, secret := range container.Secrets { - secretOpts := []llb.SecretOption{llb.SecretID(secret.Secret.Accessor)} + secretOpts := []llb.SecretOption{llb.SecretID(secret.Secret.Name)} var secretDest string switch { @@ -1240,7 +1235,7 @@ func (container Container) Evaluate(ctx context.Context) (*buildkit.Result, erro return nil, nil } - root := container.Query + root := CurrentQuery(ctx) detach, _, err := root.Services.StartBindings(ctx, container.Services) if err != nil { @@ -1274,7 +1269,7 @@ func (container *Container) MetaFileContents(ctx context.Context, filePath strin } file := NewFile( - container.Query, + CurrentQuery(ctx), container.Meta, path.Join(buildkit.MetaSourcePath, filePath), container.Platform, @@ -1344,8 +1339,8 @@ func (container *Container) Publish( opts[string(exptypes.OptKeyForceCompression)] = strconv.FormatBool(true) } - svcs := container.Query.Services - bk := container.Query.Buildkit + svcs := CurrentQuery(ctx).Services + bk := CurrentQuery(ctx).Buildkit detach, _, err := svcs.StartBindings(ctx, services) if err != nil { @@ -1388,8 +1383,8 @@ func (container *Container) Export( forcedCompression ImageLayerCompression, mediaTypes ImageMediaTypes, ) error { - svcs := container.Query.Services - bk := container.Query.Buildkit + svcs := CurrentQuery(ctx).Services + bk := CurrentQuery(ctx).Buildkit if mediaTypes == "" { // Modern registry implementations support oci types and docker daemons @@ -1455,9 +1450,9 @@ func (container *Container) AsTarball( forcedCompression ImageLayerCompression, mediaTypes ImageMediaTypes, ) (*File, error) { - bk := container.Query.Buildkit - svcs := container.Query.Services - engineHostPlatform := container.Query.Platform + bk := CurrentQuery(ctx).Buildkit + svcs := CurrentQuery(ctx).Services + engineHostPlatform := CurrentQuery(ctx).Platform if mediaTypes == "" { mediaTypes = OCIMediaTypes @@ -1513,7 +1508,7 @@ func (container *Container) AsTarball( if err != nil { return nil, fmt.Errorf("container image to tarball file conversion failed: %w", err) } - return NewFile(container.Query, pbDef, fileName, engineHostPlatform, nil), nil + return NewFile(CurrentQuery(ctx), pbDef, fileName, engineHostPlatform, nil), nil } func (container *Container) Import( @@ -1521,9 +1516,9 @@ func (container *Container) Import( source *File, tag string, ) (*Container, error) { - bk := container.Query.Buildkit - store := container.Query.OCIStore - lm := container.Query.LeaseManager + bk := CurrentQuery(ctx).Buildkit + store := CurrentQuery(ctx).OCIStore + lm := CurrentQuery(ctx).LeaseManager container = container.Clone() @@ -1705,7 +1700,7 @@ func (container *Container) Service(ctx context.Context) (*Service, error) { return nil, err } } - return container.Query.NewContainerService(container), nil + return CurrentQuery(ctx).NewContainerService(container), nil } func (container *Container) ownership(ctx context.Context, owner string) (*Ownership, error) { @@ -1719,7 +1714,7 @@ func (container *Container) ownership(ctx context.Context, owner string) (*Owner return nil, err } - return resolveUIDGID(ctx, fsSt, container.Query.Buildkit, container.Platform, owner) + return resolveUIDGID(ctx, fsSt, CurrentQuery(ctx).Buildkit, container.Platform, owner) } func (container *Container) command(opts ContainerExecOpts) ([]string, error) { @@ -1782,9 +1777,9 @@ type ContainerExecOpts struct { // Grant the process all root capabilities InsecureRootCapabilities bool `default:"false"` - // (Internal-only) If this is a nested exec for a Function call, this should be set - // with the metadata for that call - NestedExecFunctionCall *FunctionCall `name:"-"` + // (Internal-only) If this is a nested exec, use this as the root Query. + // Otherwise, nested execs will get a new root based on the current ID. + NestedExecRoot *Query `name:"-"` } type BuildArg struct { diff --git a/core/host.go b/core/host.go index 75e702a336..a3c1b03558 100644 --- a/core/host.go +++ b/core/host.go @@ -120,17 +120,12 @@ func (host *Host) File(ctx context.Context, srv *dagql.Server, filePath string) } func (host *Host) SetSecretFile(ctx context.Context, srv *dagql.Server, secretName string, path string) (i dagql.Instance[*Secret], err error) { - accessor, err := GetLocalSecretAccessor(ctx, host.Query, secretName) - if err != nil { - return i, err - } - secretFileContent, err := host.Query.Buildkit.ReadCallerHostFile(ctx, path) if err != nil { return i, fmt.Errorf("read secret file: %w", err) } - if err := host.Query.Secrets.AddSecret(ctx, accessor, secretFileContent); err != nil { + if err := host.Query.Secrets.AddSecret(ctx, secretName, secretFileContent); err != nil { return i, err } err = srv.Select(ctx, srv.Root(), &i, dagql.Selector{ @@ -140,10 +135,6 @@ func (host *Host) SetSecretFile(ctx context.Context, srv *dagql.Server, secretNa Name: "name", Value: dagql.NewString(secretName), }, - { - Name: "accessor", - Value: dagql.Opt(dagql.NewString(accessor)), - }, }, }) return diff --git a/core/integration/module_test.go b/core/integration/module_test.go index 507cd03e73..f840f31300 100644 --- a/core/integration/module_test.go +++ b/core/integration/module_test.go @@ -5410,8 +5410,6 @@ func (t *Toplevel) Attempt(ctx context.Context, uniq string) error { // even when we know the underlying IDs t.Parallel() - t.Skip("this protection is not yet implemented") - var logs safeBuffer c, ctx := connect(t, dagger.WithLogOutput(io.MultiWriter(os.Stderr, &logs))) diff --git a/core/interface.go b/core/interface.go index 92ca2f4f2e..8150e5913a 100644 --- a/core/interface.go +++ b/core/interface.go @@ -7,7 +7,6 @@ import ( "github.com/dagger/dagger/dagql" "github.com/dagger/dagger/dagql/call" - "github.com/moby/buildkit/solver/pb" "github.com/moby/buildkit/util/bklog" "github.com/vektah/gqlparser/v2/ast" ) @@ -351,10 +350,10 @@ func (iface *InterfaceAnnotatedValue) TypeDescription() string { return iface.TypeDef.Description } -var _ HasPBDefinitions = (*InterfaceAnnotatedValue)(nil) +var _ HasFieldValues = (*InterfaceAnnotatedValue)(nil) -func (iface *InterfaceAnnotatedValue) PBDefinitions(ctx context.Context) ([]*pb.Definition, error) { - defs := []*pb.Definition{} +func (iface *InterfaceAnnotatedValue) FieldValues(ctx context.Context) ([]dagql.Typed, error) { + values := make([]dagql.Typed, 0, len(iface.Fields)) objDef := iface.UnderlyingType.TypeDef().AsObject.Value for name, val := range iface.Fields { fieldDef, ok := objDef.FieldByOriginalName(name) @@ -373,11 +372,7 @@ func (iface *InterfaceAnnotatedValue) PBDefinitions(ctx context.Context) ([]*pb. if err != nil { return nil, fmt.Errorf("failed to convert arg %q: %w", name, err) } - fieldDefs, err := collectPBDefinitions(ctx, converted) - if err != nil { - return nil, err - } - defs = append(defs, fieldDefs...) + values = append(values, converted) } - return defs, nil + return values, nil } diff --git a/core/modfunc.go b/core/modfunc.go index 9e4902fa7b..2061d9282f 100644 --- a/core/modfunc.go +++ b/core/modfunc.go @@ -188,6 +188,10 @@ func (fn *ModuleFunction) Call(ctx context.Context, opts *CallOpts) (t dagql.Typ callMeta.ParentName = fn.objDef.OriginalName } + newRoot, err := fn.root.NewRootForCurrentCall(ctx, callMeta) + if err != nil { + return nil, fmt.Errorf("register caller: %w", err) + } ctr := fn.runtime metaDir := NewScratchDirectory(mod.Query, mod.Query.Platform) @@ -199,7 +203,7 @@ func (fn *ModuleFunction) Call(ctx context.Context, opts *CallOpts) (t dagql.Typ // Setup the Exec for the Function call and evaluate it ctr, err = ctr.WithExec(ctx, ContainerExecOpts{ ExperimentalPrivilegedNesting: true, - NestedExecFunctionCall: callMeta, + NestedExecRoot: newRoot, }) if err != nil { return nil, fmt.Errorf("failed to exec function: %w", err) @@ -251,6 +255,22 @@ func (fn *ModuleFunction) Call(ctx context.Context, opts *CallOpts) (t dagql.Typ return nil, fmt.Errorf("failed to convert return value: %w", err) } + // TODO: split out + secrets, err := referencedTypes[*Secret](ctx, returnValueTyped, newRoot.Dag) + if err != nil { + return nil, fmt.Errorf("failed to link secret dependency blobs: %w", err) + } + for _, secret := range secrets { + plaintext, err := newRoot.Secrets.GetSecret(ctx, secret.Name) + if err != nil { + return nil, fmt.Errorf("failed to get secret %q: %w", secret.Name, err) + } + + if err := CurrentQuery(ctx).Secrets.AddSecret(ctx, secret.Name, plaintext); err != nil { + return nil, fmt.Errorf("failed to add secret %q: %w", secret.Name, err) + } + } + if err := fn.linkDependencyBlobs(ctx, result, returnValueTyped); err != nil { return nil, fmt.Errorf("failed to link dependency blobs: %w", err) } diff --git a/core/object.go b/core/object.go index b53884de54..17722c861f 100644 --- a/core/object.go +++ b/core/object.go @@ -7,7 +7,6 @@ import ( "sort" "github.com/dagger/dagger/dagql" - "github.com/moby/buildkit/solver/pb" "github.com/vektah/gqlparser/v2/ast" ) @@ -127,10 +126,10 @@ func (obj *ModuleObject) Type() *ast.Type { } } -var _ HasPBDefinitions = (*ModuleObject)(nil) +var _ HasFieldValues = (*ModuleObject)(nil) -func (obj *ModuleObject) PBDefinitions(ctx context.Context) ([]*pb.Definition, error) { - defs := []*pb.Definition{} +func (obj *ModuleObject) FieldValues(ctx context.Context) ([]dagql.Typed, error) { + values := []dagql.Typed{} objDef := obj.TypeDef for name, val := range obj.Fields { fieldDef, ok := objDef.FieldByOriginalName(name) @@ -150,13 +149,9 @@ func (obj *ModuleObject) PBDefinitions(ctx context.Context) ([]*pb.Definition, e if err != nil { return nil, fmt.Errorf("failed to convert field %q: %w", name, err) } - fieldDefs, err := collectPBDefinitions(ctx, converted) - if err != nil { - return nil, err - } - defs = append(defs, fieldDefs...) + values = append(values, converted) } - return defs, nil + return values, nil } func (obj *ModuleObject) TypeDescription() string { diff --git a/core/query.go b/core/query.go index 7eb97a1aad..abcf8f7b12 100644 --- a/core/query.go +++ b/core/query.go @@ -28,6 +28,8 @@ type Query struct { SecretToken string + Secrets *SecretStore + Deps *ModDeps FnCall *FunctionCall @@ -48,8 +50,6 @@ type QueryOpts struct { Services *Services - Secrets *SecretStore - Auth *auth.RegistryAuthProvider OCIStore content.Store @@ -71,6 +71,11 @@ type DaggerServer interface { NewRootForDynamicID(ctx context.Context, id *call.ID) (*Query, error) } +func CurrentQuery(ctx context.Context) *Query { + srv := dagql.CurrentServer(ctx) + return srv.Root().(dagql.Instance[*Query]).Self +} + func (*Query) Type() *ast.Type { return &ast.Type{ NamedType: "Query", @@ -114,16 +119,14 @@ func (q *Query) WithPipeline(name, desc string, labels []pipeline.Label) *Query func (q *Query) NewContainer(platform Platform) *Container { return &Container{ - Query: q, Platform: platform, } } -func (q *Query) NewSecret(name string, accessor string) *Secret { +func (q *Query) NewSecret(name string) *Secret { return &Secret{ - Query: q, - Name: name, - Accessor: accessor, + Query: q, + Name: name, } } diff --git a/core/schema/container.go b/core/schema/container.go index fdc1c642fc..6530b7f1ac 100644 --- a/core/schema/container.go +++ b/core/schema/container.go @@ -1223,12 +1223,12 @@ func (s *containerSchema) withRegistryAuth(ctx context.Context, parent *core.Con return nil, err } - secretBytes, err := parent.Query.Secrets.GetSecret(ctx, secret.Self.Accessor) + secretBytes, err := core.CurrentQuery(ctx).Secrets.GetSecret(ctx, secret.Self.Name) if err != nil { return nil, err } - if err := parent.Query.Auth.AddCredential(args.Address, args.Username, string(secretBytes)); err != nil { + if err := core.CurrentQuery(ctx).Auth.AddCredential(args.Address, args.Username, string(secretBytes)); err != nil { return nil, err } @@ -1239,8 +1239,8 @@ type containerWithoutRegistryAuthArgs struct { Address string } -func (s *containerSchema) withoutRegistryAuth(_ context.Context, parent *core.Container, args containerWithoutRegistryAuthArgs) (*core.Container, error) { - if err := parent.Query.Auth.RemoveCredential(args.Address); err != nil { +func (s *containerSchema) withoutRegistryAuth(ctx context.Context, parent *core.Container, args containerWithoutRegistryAuthArgs) (*core.Container, error) { + if err := core.CurrentQuery(ctx).Auth.RemoveCredential(args.Address); err != nil { return nil, err } @@ -1384,7 +1384,7 @@ func (s *containerSchema) terminal( return nil, err } - if err := ctr.Self.Query.MuxEndpoint(ctx, path.Join("/", term.Endpoint), handler); err != nil { + if err := core.CurrentQuery(ctx).MuxEndpoint(ctx, path.Join("/", term.Endpoint), handler); err != nil { return nil, err } diff --git a/core/schema/directory.go b/core/schema/directory.go index f9db9e185c..e6aad79f1f 100644 --- a/core/schema/directory.go +++ b/core/schema/directory.go @@ -286,7 +286,7 @@ func (s *directorySchema) dockerBuild(ctx context.Context, parent *core.Director if args.Platform.Valid { platform = args.Platform.Value } - ctr, err := core.NewContainer(parent.Query, platform) + ctr, err := core.NewContainer(platform) if err != nil { return nil, err } diff --git a/core/schema/host.go b/core/schema/host.go index 03e0d7d956..b8f8307492 100644 --- a/core/schema/host.go +++ b/core/schema/host.go @@ -76,7 +76,7 @@ func (s *hostSchema) Install() { return nil, fmt.Errorf("marshal root: %w", err) } - container, err := core.NewContainer(parent, parent.Platform) + container, err := core.NewContainer(parent.Platform) if err != nil { return nil, fmt.Errorf("new container: %w", err) } diff --git a/core/schema/secret.go b/core/schema/secret.go index 11adcca994..8653b7070e 100644 --- a/core/schema/secret.go +++ b/core/schema/secret.go @@ -2,9 +2,11 @@ package schema import ( "context" + "fmt" "github.com/dagger/dagger/core" "github.com/dagger/dagger/dagql" + "github.com/dagger/dagger/engine" "github.com/moby/buildkit/session/secrets" "github.com/pkg/errors" ) @@ -39,22 +41,10 @@ func (s *secretSchema) Install() { type secretArgs struct { Name string - - // Accessor is the scoped per-module name, which should guarantee uniqueness. - Accessor dagql.Optional[dagql.String] } func (s *secretSchema) secret(ctx context.Context, parent *core.Query, args secretArgs) (*core.Secret, error) { - accessor := string(args.Accessor.GetOr("")) - if accessor == "" { - var err error - accessor, err = core.GetLocalSecretAccessor(ctx, parent, args.Name) - if err != nil { - return nil, err - } - } - - return parent.NewSecret(args.Name, accessor), nil + return parent.NewSecret(args.Name), nil } type setSecretArgs struct { @@ -62,12 +52,8 @@ type setSecretArgs struct { Plaintext string `sensitive:"true"` // NB: redundant with ArgSensitive above } -func (s *secretSchema) setSecret(ctx context.Context, parent *core.Query, args setSecretArgs) (i dagql.Instance[*core.Secret], err error) { - accessor, err := core.GetLocalSecretAccessor(ctx, parent, args.Name) - if err != nil { - return i, err - } - if err := parent.Secrets.AddSecret(ctx, accessor, []byte(args.Plaintext)); err != nil { +func (s *secretSchema) setSecret(ctx context.Context, _ *core.Query, args setSecretArgs) (i dagql.Instance[*core.Secret], err error) { + if err := core.CurrentQuery(ctx).Secrets.AddSecret(ctx, args.Name, []byte(args.Plaintext)); err != nil { return i, err } @@ -80,10 +66,6 @@ func (s *secretSchema) setSecret(ctx context.Context, parent *core.Query, args s Name: "name", Value: dagql.NewString(args.Name), }, - { - Name: "accessor", - Value: dagql.Opt(dagql.NewString(accessor)), - }, }, }); err != nil { return i, err @@ -93,8 +75,20 @@ func (s *secretSchema) setSecret(ctx context.Context, parent *core.Query, args s } func (s *secretSchema) plaintext(ctx context.Context, secret *core.Secret, args struct{}) (dagql.String, error) { - bytes, err := secret.Query.Secrets.GetSecret(ctx, secret.Accessor) + bytes, err := core.CurrentQuery(ctx).Secrets.GetSecret(ctx, secret.Name) + // bytes, err := secret.Query.Secrets.GetSecret(ctx, secret.Name) if err != nil { + // TODO: + // TODO: + // TODO: + // TODO: + // TODO: + clientMetadata, err := engine.ClientMetadataFromContext(ctx) + if err != nil { + return "", err + } + fmt.Printf("PLAINTEXT CLIENT %s %s %s\n", secret.Query.ClientID, secret.Name, clientMetadata.ClientID) + if errors.Is(err, secrets.ErrNotFound) { return "", errors.Wrapf(secrets.ErrNotFound, "secret %s", secret.Name) } diff --git a/core/secret.go b/core/secret.go index b22281a214..3225832912 100644 --- a/core/secret.go +++ b/core/secret.go @@ -2,12 +2,9 @@ package core import ( "context" - "crypto/hmac" - "encoding/hex" "sync" "github.com/moby/buildkit/session/secrets" - "github.com/opencontainers/go-digest" "github.com/pkg/errors" "github.com/vektah/gqlparser/v2/ast" ) @@ -18,30 +15,6 @@ type Secret struct { // Name specifies the name of the secret. Name string `json:"name,omitempty"` - - // Accessor specifies the accessor key for the secret. - Accessor string `json:"accessor,omitempty"` -} - -func GetLocalSecretAccessor(ctx context.Context, parent *Query, name string) (string, error) { - m, err := parent.CurrentModule(ctx) - if err != nil && !errors.Is(err, ErrNoCurrentModule) { - return "", err - } - var d digest.Digest - if m != nil { - d = m.Source.ID().Digest() - } - return NewSecretAccessor(name, d.String()), nil -} - -func NewSecretAccessor(name string, scope string) string { - // Use an HMAC, which allows us to keep the scope secret - // This also protects from length-extension attacks (where if we had - // access to secret FOO in scope X, we could derive access to FOOBAR). - h := hmac.New(digest.SHA256.Hash, []byte(scope)) - dt := h.Sum([]byte(name)) - return hex.EncodeToString(dt) } func (*Secret) Type() *ast.Type { diff --git a/core/util.go b/core/util.go index 76eefa1e9c..c39f443d7a 100644 --- a/core/util.go +++ b/core/util.go @@ -27,6 +27,10 @@ type HasPBDefinitions interface { PBDefinitions(context.Context) ([]*pb.Definition, error) } +type HasFieldValues interface { + FieldValues(context.Context) ([]dagql.Typed, error) +} + func collectPBDefinitions(ctx context.Context, value dagql.Typed) ([]*pb.Definition, error) { switch x := value.(type) { case dagql.String, dagql.Int, dagql.Boolean, dagql.Float: @@ -54,6 +58,20 @@ func collectPBDefinitions(ctx context.Context, value dagql.Typed) ([]*pb.Definit } case dagql.Wrapper: // dagql.Instance return collectPBDefinitions(ctx, x.Unwrap()) + case HasFieldValues: + fieldValues, err := x.FieldValues(ctx) + if err != nil { + return nil, fmt.Errorf("failed to get field values: %w", err) + } + defs := []*pb.Definition{} + for _, val := range fieldValues { + fieldDefs, err := collectPBDefinitions(ctx, val) + if err != nil { + return nil, err + } + defs = append(defs, fieldDefs...) + } + return defs, nil case HasPBDefinitions: return x.PBDefinitions(ctx) default: @@ -65,6 +83,53 @@ func collectPBDefinitions(ctx context.Context, value dagql.Typed) ([]*pb.Definit } } +func referencedTypes[T dagql.Typed](ctx context.Context, value dagql.Typed, dag *dagql.Server) ([]T, error) { + switch x := value.(type) { + case dagql.String, dagql.Int, dagql.Boolean, dagql.Float: + // nothing to do + return nil, nil + case dagql.Enumerable: // dagql.Array + values := []T{} + for i := 1; i < x.Len(); i++ { + val, err := x.Nth(i) + if err != nil { + return nil, fmt.Errorf("failed to get nth value: %w", err) + } + elemValues, err := referencedTypes[T](ctx, val, dag) + if err != nil { + return nil, fmt.Errorf("failed to link nth value dependency blobs: %w", err) + } + values = append(values, elemValues...) + } + return values, nil + case dagql.Derefable: // dagql.Nullable + if inner, ok := x.Deref(); ok { + return referencedTypes[T](ctx, inner, dag) + } else { + return nil, nil + } + case HasFieldValues: + fieldValues, err := x.FieldValues(ctx) + if err != nil { + return nil, fmt.Errorf("failed to get field values: %w", err) + } + values := []T{} + for _, val := range fieldValues { + fieldValues, err := referencedTypes[T](ctx, val, dag) + if err != nil { + return nil, err + } + values = append(values, fieldValues...) + } + return values, nil + case dagql.IDable: + return dagql.ReferencedTypes[T](ctx, x.ID(), dag, true) + default: + slog.Warn("referencedTypes: unhandled type", "type", fmt.Sprintf("%T", value)) + return nil, nil + } +} + func absPath(workDir string, containerPath string) string { if path.IsAbs(containerPath) { return containerPath diff --git a/dagql/referencedTypes.go b/dagql/referencedTypes.go new file mode 100644 index 0000000000..39676dd46d --- /dev/null +++ b/dagql/referencedTypes.go @@ -0,0 +1,106 @@ +package dagql + +import ( + "context" + + "github.com/dagger/dagger/dagql/call" + "github.com/opencontainers/go-digest" +) + +// TODO: doc, note that it excludes return of ID itself +func ReferencedTypes[T Typed](ctx context.Context, id *call.ID, srv *Server, includeSelf bool) ([]T, error) { + ids := idWalker{ + typeNameToIDs: map[string][]*call.ID{}, + memo: map[digest.Digest]struct{}{}, + } + if err := ids.walkID(id); err != nil { + return nil, err + } + + var implementingTypes []T + srv.installLock.Lock() + for _, objType := range srv.objects { + innerType := objType.Typed() + if t, ok := innerType.(T); ok { + implementingTypes = append(implementingTypes, t) + } + } + srv.installLock.Unlock() + + if len(implementingTypes) == 0 { + return nil, nil + } + + var typedIDs []ID[T] + for _, t := range implementingTypes { + typeName := t.Type().Name() + idProtos, ok := ids.typeNameToIDs[typeName] + if !ok { + return nil, nil + } + for _, idProto := range idProtos { + if !includeSelf && idProto.Digest() == id.Digest() { + // skip self + continue + } + typedIDs = append(typedIDs, NewID[T](idProto)) + } + } + + return LoadIDs[T](ctx, srv, typedIDs) +} + +type idWalker struct { + typeNameToIDs map[string][]*call.ID + memo map[digest.Digest]struct{} +} + +func (idWalker *idWalker) walkID(id *call.ID) error { + dgst := id.Digest() + if _, ok := idWalker.memo[dgst]; ok { + return nil + } + idWalker.memo[dgst] = struct{}{} + + namedType := id.Type().NamedType() + if namedType != "" { + idWalker.typeNameToIDs[namedType] = append(idWalker.typeNameToIDs[namedType], id) + } + + if id.Base != nil { + if err := idWalker.walkID(id); err != nil { + return err + } + } + + for _, arg := range id.Args() { + if err := idWalker.walkLiteral(arg.Value()); err != nil { + return err + } + } + + return nil +} + +func (idWalker *idWalker) walkLiteral(lit call.Literal) error { + switch x := lit.(type) { + case *call.LiteralID: + return idWalker.walkID(x.Value()) + case *call.LiteralList: + if err := x.Range(func(_ int, v call.Literal) error { + return idWalker.walkLiteral(v) + }); err != nil { + return err + } + case *call.LiteralObject: + if err := x.Range(func(_ int, _ string, v call.Literal) error { + return idWalker.walkLiteral(v) + }); err != nil { + return err + } + default: + // NOTE: not handling any primitive types right now, could be added + // if needed for some reason + } + return nil +} diff --git a/dagql/server.go b/dagql/server.go index dd2f410823..288b978e6e 100644 --- a/dagql/server.go +++ b/dagql/server.go @@ -598,12 +598,27 @@ func CurrentID(ctx context.Context) *call.ID { return val.(*call.ID) } +type srvCtx struct{} + +func srvToContext(ctx context.Context, srv *Server) context.Context { + return context.WithValue(ctx, srvCtx{}, srv) +} + +func CurrentServer(ctx context.Context) *Server { + val := ctx.Value(srvCtx{}) + if val == nil { + return nil + } + return val.(*Server) +} + func (s *Server) cachedSelect(ctx context.Context, self Object, sel Selector) (res Typed, chained *call.ID, rerr error) { chainedID, err := self.IDFor(ctx, sel) if err != nil { return nil, nil, err } ctx = idToContext(ctx, chainedID) + ctx = srvToContext(ctx, s) doSelect := func(ctx context.Context) (Typed, error) { return self.Select(ctx, sel) } diff --git a/engine/buildkit/client.go b/engine/buildkit/client.go index 6a7abe56be..5809640ee7 100644 --- a/engine/buildkit/client.go +++ b/engine/buildkit/client.go @@ -50,7 +50,6 @@ type Opts struct { SessionManager *bksession.Manager LLBSolver *llbsolver.Solver GenericSolver *bksolver.Solver - SecretStore bksecrets.SecretStore AuthProvider *auth.RegistryAuthProvider PrivilegedExecEnabled bool UpstreamCacheImports []bkgw.CacheOptionsEntry @@ -95,7 +94,7 @@ type Client struct { closeMu sync.RWMutex } -func NewClient(ctx context.Context, opts *Opts) (*Client, error) { +func NewClient(ctx context.Context, opts *Opts, secretStore bksecrets.SecretStore) (*Client, error) { // override the outer cancel, we will manage cancellation ourselves here ctx, cancel := context.WithCancel(context.WithoutCancel(ctx)) client := &Client{ @@ -117,7 +116,7 @@ func NewClient(ctx context.Context, opts *Opts) (*Client, error) { } client.execMetadataMu.Unlock() - session, err := client.newSession() + session, err := client.newSession(secretStore) if err != nil { return nil, err } @@ -256,9 +255,7 @@ func (c *Client) Solve(ctx context.Context, req bkgw.SolveRequest) (_ *Result, r req.CacheImports = c.UpstreamCacheImports // include exec metadata that isn't included in the cache key - var llbRes *bkfrontend.Result - switch { - case req.Definition != nil && req.Definition.Def != nil: + if req.Definition != nil && req.Definition.Def != nil { clientMetadata, err := engine.ClientMetadataFromContext(ctx) if err != nil { return nil, err @@ -311,43 +308,12 @@ func (c *Client) Solve(ctx context.Context, req bkgw.SolveRequest) (_ *Result, r return nil, err } req.Definition = newDef - - llbRes, err = c.llbBridge.Solve(ctx, req, c.ID()) - if err != nil { - return nil, wrapError(ctx, err, c.ID()) - } - case req.Frontend != "": - // HACK: don't force evaluation like this, we can write custom frontend - // wrappers (for dockerfile.v0 and gateway.v0) that read from ctx to - // replace the llbBridge it knows about. - // This current implementation may be limited when it comes to - // implement provenance/etc. - - f, ok := c.Frontends[req.Frontend] - if !ok { - return nil, fmt.Errorf("invalid frontend: %s", req.Frontend) - } - - gw := newFilterGateway(c.llbBridge, req) - gw.secretTranslator = ctx.Value("secret-translator").(func(string) (string, error)) - - llbRes, err = f.Solve(ctx, gw, c.llbExec, req.FrontendOpt, req.FrontendInputs, c.ID(), c.SessionManager) - if err != nil { - return nil, err - } - if req.Evaluate { - err = llbRes.EachRef(func(ref bksolver.ResultProxy) error { - _, err := ref.Result(ctx) - return err - }) - if err != nil { - return nil, err - } - } - default: - llbRes = &bkfrontend.Result{} } + llbRes, err := c.llbBridge.Solve(ctx, req, c.ID()) + if err != nil { + return nil, wrapError(ctx, err, c.ID()) + } res, err := solverresult.ConvertResult(llbRes, func(rp bksolver.ResultProxy) (*ref, error) { return newRef(rp, c), nil }) @@ -780,79 +746,3 @@ func (md *ContainerExecUncachedMetadata) FromEnv(envKV string) (bool, error) { } return true, nil } - -// filteringGateway is a helper gateway that filters+converts various -// operations for the frontend -type filteringGateway struct { - bkfrontend.FrontendLLBBridge - - // secretTranslator is a function to convert secret ids. Frontends may - // attempt to access secrets by raw IDs, but they may be keyed differently - // in the secret store. - secretTranslator func(string) (string, error) - - // skipInputs specifies op digests that were part of the request inputs and - // so shouldn't be processed. - skipInputs map[digest.Digest]struct{} -} - -func newFilterGateway(bridge bkfrontend.FrontendLLBBridge, req bkgw.SolveRequest) *filteringGateway { - inputs := map[digest.Digest]struct{}{} - for _, inp := range req.FrontendInputs { - for _, def := range inp.Def { - inputs[digest.FromBytes(def)] = struct{}{} - } - } - - return &filteringGateway{ - FrontendLLBBridge: bridge, - skipInputs: inputs, - } -} - -func (gw *filteringGateway) Solve(ctx context.Context, req bkfrontend.SolveRequest, sid string) (*bkfrontend.Result, error) { - if req.Definition != nil && req.Definition.Def != nil { - dag, err := DefToDAG(req.Definition) - if err != nil { - return nil, err - } - if err := dag.Walk(func(dag *OpDAG) error { - if _, ok := gw.skipInputs[*dag.OpDigest]; ok { - return SkipInputs - } - - execOp, ok := dag.AsExec() - if !ok { - return nil - } - - for _, secret := range execOp.ExecOp.GetSecretenv() { - secret.ID, err = gw.secretTranslator(secret.ID) - if err != nil { - return err - } - } - for _, mount := range execOp.ExecOp.GetMounts() { - if mount.MountType != bksolverpb.MountType_SECRET { - continue - } - secret := mount.SecretOpt - secret.ID, err = gw.secretTranslator(secret.ID) - if err != nil { - return err - } - } - return nil - }); err != nil { - return nil, err - } - - newDef, err := dag.Marshal() - if err != nil { - return nil, err - } - req.Definition = newDef - } - - return gw.FrontendLLBBridge.Solve(ctx, req, sid) -} diff --git a/engine/buildkit/session.go b/engine/buildkit/session.go index f993738360..cd83fa088c 100644 --- a/engine/buildkit/session.go +++ b/engine/buildkit/session.go @@ -15,6 +15,7 @@ import ( "github.com/moby/buildkit/identity" bksession "github.com/moby/buildkit/session" sessioncontent "github.com/moby/buildkit/session/content" + bksecrets "github.com/moby/buildkit/session/secrets" "github.com/moby/buildkit/session/secrets/secretsprovider" "github.com/moby/buildkit/util/bklog" ) @@ -30,7 +31,7 @@ const ( BuiltinContentOCIStoreName = "dagger-builtin-content" ) -func (c *Client) newSession() (*bksession.Session, error) { +func (c *Client) newSession(secretStore bksecrets.SecretStore) (*bksession.Session, error) { sess, err := bksession.NewSession(c.closeCtx, identity.NewID(), "") if err != nil { return nil, err @@ -41,7 +42,7 @@ func (c *Client) newSession() (*bksession.Session, error) { return nil, fmt.Errorf("failed to create go sdk content store: %w", err) } - sess.Allow(secretsprovider.NewSecretProvider(c.SecretStore)) + sess.Allow(secretsprovider.NewSecretProvider(secretStore)) sess.Allow(&socketProxy{c}) sess.Allow(&authProxy{c}) sess.Allow(&client.AnyDirSource{}) diff --git a/engine/server/server.go b/engine/server/server.go index 6cdbc7bc00..746a89bb19 100644 --- a/engine/server/server.go +++ b/engine/server/server.go @@ -138,7 +138,6 @@ func (e *BuildkitController) newDaggerServer(ctx context.Context, clientMetadata } s.recorder = progrock.NewRecorder(progWriter, progrock.WithLabels(progrockLabels...)) - secretStore := core.NewSecretStore() authProvider := auth.NewRegistryAuthProvider() cacheImporterCfgs := make([]bkgw.CacheOptionsEntry, 0, len(clientMetadata.UpstreamCacheImportConfig)) @@ -168,7 +167,6 @@ func (e *BuildkitController) newDaggerServer(ctx context.Context, clientMetadata SessionManager: e.SessionManager, LLBSolver: e.llbSolver, GenericSolver: e.genericSolver, - SecretStore: secretStore, AuthProvider: authProvider, PrivilegedExecEnabled: e.privilegedExecEnabled, UpstreamCacheImports: cacheImporterCfgs, @@ -183,7 +181,6 @@ func (e *BuildkitController) newDaggerServer(ctx context.Context, clientMetadata ProgrockSocketPath: progSockPath, Services: s.services, Platform: core.Platform(e.worker.Platforms(true)[0]), - Secrets: secretStore, OCIStore: e.worker.ContentStore(), LeaseManager: e.worker.LeaseManager(), Auth: authProvider, @@ -191,7 +188,7 @@ func (e *BuildkitController) newDaggerServer(ctx context.Context, clientMetadata } // register the main client caller - newRoot, err := s.newRootForClient(ctx, s.mainClientCallerID, nil) + newRoot, err := s.newRootForClient(ctx, s.mainClientCallerID, nil, nil, nil) if err != nil { return nil, err } @@ -414,22 +411,44 @@ func (s *DaggerServer) NewRootForCurrentCall(ctx context.Context, fnCall *core.F fnCall = &core.FunctionCall{} } - clientID := s.digestToClientID(dagql.CurrentID(ctx).Digest(), fnCall.Cache) + currentID := dagql.CurrentID(ctx) + clientID := s.digestToClientID(currentID.Digest(), fnCall.Cache) + + // TODO: + // TODO: + // TODO: + // TODO: + // TODO: + // TODO: + fmt.Printf("NewRootForCurrentCall: %s %s\n", clientID, currentID.Display()) s.perClientMu.Lock(clientID) defer s.perClientMu.Unlock(clientID) - - s.clientRootMu.Lock() - if root, ok := s.clientRoots[clientID]; ok { - s.clientRootMu.Unlock() + if root, ok := s.rootFor(clientID); ok { return root, nil } - s.clientRootMu.Unlock() - newRoot, err := s.newRootForClient(ctx, clientID, fnCall) + clientMetadata, err := engine.ClientMetadataFromContext(ctx) + if err != nil { + return nil, err + } + currentRoot, ok := s.rootFor(clientMetadata.ClientID) + if !ok { + return nil, fmt.Errorf("client call for %s not found", clientMetadata.ClientID) + } + secrets, err := dagql.ReferencedTypes[*core.Secret](ctx, currentID, currentRoot.Dag, false) + if err != nil { + return nil, fmt.Errorf("referenced secrets: %w", err) + } + currentSecretStore := currentRoot.Secrets + + newRoot, err := s.newRootForClient(ctx, clientID, fnCall, currentSecretStore, secrets) if err != nil { return nil, fmt.Errorf("new root: %w", err) } + + s.clientRootMu.Lock() + defer s.clientRootMu.Unlock() s.clientRoots[newRoot.ClientID] = newRoot return newRoot, nil } @@ -440,15 +459,11 @@ func (s *DaggerServer) NewRootForDependencies(ctx context.Context, deps *core.Mo s.perClientMu.Lock(clientID) defer s.perClientMu.Unlock(clientID) - - s.clientRootMu.Lock() - if root, ok := s.clientRoots[clientID]; ok { - s.clientRootMu.Unlock() + if root, ok := s.rootFor(clientID); ok { return root, nil } - s.clientRootMu.Unlock() - newRoot, err := s.newRootForClient(ctx, clientID, nil) + newRoot, err := s.newRootForClient(ctx, clientID, nil, nil, nil) if err != nil { return nil, fmt.Errorf("new root: %w", err) } @@ -457,6 +472,8 @@ func (s *DaggerServer) NewRootForDependencies(ctx context.Context, deps *core.Mo } newRoot.Deps = newRoot.Deps.Append(deps.Mods...) + s.clientRootMu.Lock() + defer s.clientRootMu.Unlock() s.clientRoots[newRoot.ClientID] = newRoot return newRoot, nil } @@ -467,15 +484,17 @@ func (s *DaggerServer) NewRootForDynamicID(ctx context.Context, id *call.ID) (*c s.perClientMu.Lock(clientID) defer s.perClientMu.Unlock(clientID) - - s.clientRootMu.Lock() - if root, ok := s.clientRoots[clientID]; ok { - s.clientRootMu.Unlock() + if root, ok := s.rootFor(clientID); ok { return root, nil } - s.clientRootMu.Unlock() - newRoot, err := s.newRootForClient(ctx, clientID, nil) + // TODO: + // TODO: + // TODO: + // TODO: + // TODO: include secrets here? + + newRoot, err := s.newRootForClient(ctx, clientID, nil, nil, nil) if err != nil { return nil, fmt.Errorf("new root: %w", err) } @@ -493,6 +512,8 @@ func (s *DaggerServer) NewRootForDynamicID(ctx context.Context, id *call.ID) (*c } newRoot.Deps = newRoot.Deps.Append(deps.Mods...) + s.clientRootMu.Lock() + defer s.clientRootMu.Unlock() s.clientRoots[newRoot.ClientID] = newRoot return newRoot, nil } @@ -512,7 +533,14 @@ func (s *DaggerServer) digestToClientID(dgst digest.Digest, cache bool) string { return clientIDDigest.Encoded()[:25] } -func (s *DaggerServer) newRootForClient(ctx context.Context, clientID string, fnCall *core.FunctionCall) (*core.Query, error) { +func (s *DaggerServer) newRootForClient( + ctx context.Context, + clientID string, + fnCall *core.FunctionCall, + // TODO: cleanup + currentSecretStore *core.SecretStore, + secrets []*core.Secret, +) (*core.Query, error) { if fnCall == nil { fnCall = &core.FunctionCall{} } @@ -523,8 +551,19 @@ func (s *DaggerServer) newRootForClient(ctx context.Context, clientID string, fn ProgrockParent: progrock.FromContext(ctx).Parent, } + root.Secrets = core.NewSecretStore() + for _, secret := range secrets { + plaintext, err := currentSecretStore.GetSecret(ctx, secret.Name) + if err != nil { + return nil, fmt.Errorf("get secret: %w", err) + } + if err := root.Secrets.AddSecret(ctx, secret.Name, plaintext); err != nil { + return nil, fmt.Errorf("add secret: %w", err) + } + } + var err error - root.Buildkit, err = buildkit.NewClient(ctx, s.buildkitOpts) + root.Buildkit, err = buildkit.NewClient(ctx, s.buildkitOpts, root.Secrets) if err != nil { return nil, fmt.Errorf("buildkit client: %w", err) } @@ -560,8 +599,8 @@ func (s *DaggerServer) newRootForClient(ctx context.Context, clientID string, fn func (s *DaggerServer) rootFor(clientID string) (*core.Query, bool) { s.clientRootMu.RLock() defer s.clientRootMu.RUnlock() - ctx, ok := s.clientRoots[clientID] - return ctx, ok + root, ok := s.clientRoots[clientID] + return root, ok } func (s *DaggerServer) CurrentServedDeps(ctx context.Context) (*core.ModDeps, error) {