Skip to content

Commit 1cb5624

Browse files
authored
engine-wide dagql caching (#9682)
* share dagql cache across sessions Signed-off-by: Erik Sipsma <erik@sipsma.dev> * various tangential git fixes, module cache key fix, more tests Signed-off-by: Erik Sipsma <erik@sipsma.dev> * secrets refactor Signed-off-by: Erik Sipsma <erik@sipsma.dev> * handle postcall secret transfer across sessions Signed-off-by: Erik Sipsma <erik@sipsma.dev> * services wip Signed-off-by: Erik Sipsma <erik@sipsma.dev> * hacky fix to cross-session telemetry Signed-off-by: Erik Sipsma <erik@sipsma.dev> * rebase fixes Signed-off-by: Erik Sipsma <erik@sipsma.dev> * support cached telemetry from dagql cache hits Signed-off-by: Erik Sipsma <erik@sipsma.dev> * fixup Signed-off-by: Erik Sipsma <erik@sipsma.dev> * send telemetry once per session for given key Signed-off-by: Erik Sipsma <erik@sipsma.dev> * basic cleanup Signed-off-by: Erik Sipsma <erik@sipsma.dev> * fix client init w/ module containing secrets Signed-off-by: Erik Sipsma <erik@sipsma.dev> * fix cmd/introspect compile error Signed-off-by: Erik Sipsma <erik@sipsma.dev> * fix dagql_test compile errors Signed-off-by: Erik Sipsma <erik@sipsma.dev> * more unit test fixes; rm extraneous secret unit test Signed-off-by: Erik Sipsma <erik@sipsma.dev> * fix panic from setSecretFile Signed-off-by: Erik Sipsma <erik@sipsma.dev> * regen Signed-off-by: Erik Sipsma <erik@sipsma.dev> * rough fix for dep module load cache hit corner case Signed-off-by: Erik Sipsma <erik@sipsma.dev> * fix TestDaggerGitRefs Signed-off-by: Erik Sipsma <erik@sipsma.dev> * check auth in gitdns source always Signed-off-by: Erik Sipsma <erik@sipsma.dev> * remove now invalidated separate secret stores test Signed-off-by: Erik Sipsma <erik@sipsma.dev> * fix TestClientGenerator issue w/ stable client id Signed-off-by: Erik Sipsma <erik@sipsma.dev> * fix git services + git tree content hash + context dir load Signed-off-by: Erik Sipsma <erik@sipsma.dev> * add cross-session secret uri test Signed-off-by: Erik Sipsma <erik@sipsma.dev> * include ctxargs as input resources Signed-off-by: Erik Sipsma <erik@sipsma.dev> * ensure git authSock in DAG; rm stray SetSecret per-session cache key Signed-off-by: Erik Sipsma <erik@sipsma.dev> * fix panic from unset client.dag after merge Signed-off-by: Erik Sipsma <erik@sipsma.dev> * update dagql unit tests after merge Signed-off-by: Erik Sipsma <erik@sipsma.dev> * fix concurrent map write panic Signed-off-by: Erik Sipsma <erik@sipsma.dev> * fix nil panic w/ local git repos Signed-off-by: Erik Sipsma <erik@sipsma.dev> * misc cleanup Signed-off-by: Erik Sipsma <erik@sipsma.dev> * regen Signed-off-by: Erik Sipsma <erik@sipsma.dev> * update telemetry tests Signed-off-by: Erik Sipsma <erik@sipsma.dev> * address lints Signed-off-by: Erik Sipsma <erik@sipsma.dev> * first batch of fixes from feedback Signed-off-by: Erik Sipsma <erik@sipsma.dev> * update TODO comment Signed-off-by: Erik Sipsma <erik@sipsma.dev> * adjust llm tests to account for more caching Signed-off-by: Erik Sipsma <erik@sipsma.dev> * remove unixSocket dep on __internalSocket Signed-off-by: Erik Sipsma <erik@sipsma.dev> * update llm unit tests Signed-off-by: Erik Sipsma <erik@sipsma.dev> * deprecate setSecretFile Signed-off-by: Erik Sipsma <erik@sipsma.dev> * cleanup stray TODO comments/add a couple extra tests Signed-off-by: Erik Sipsma <erik@sipsma.dev> * regen Signed-off-by: Erik Sipsma <erik@sipsma.dev> * fix dotnet sdk check failure Signed-off-by: Erik Sipsma <erik@sipsma.dev> * regen Signed-off-by: Erik Sipsma <erik@sipsma.dev> --------- Signed-off-by: Erik Sipsma <erik@sipsma.dev>
1 parent 959cefa commit 1cb5624

65 files changed

Lines changed: 1957 additions & 1097 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

cmd/codegen/generator/go/templates/modules.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -361,7 +361,7 @@ func dispatch(ctx context.Context) (rerr error) {
361361
defer func() {
362362
if rerr != nil {
363363
if ` + voidRet + ` := fnCall.ReturnError(ctx, convertError(rerr)); err != nil {
364-
fmt.Println("failed to return error:", err)
364+
fmt.Println("failed to return error:", err, "\noriginal error:", rerr)
365365
}
366366
}
367367
}()

cmd/introspect/main.go

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,13 +10,15 @@ import (
1010
"github.com/dagger/dagger/core"
1111
"github.com/dagger/dagger/core/schema"
1212
"github.com/dagger/dagger/dagql"
13+
"github.com/dagger/dagger/engine/cache"
14+
"github.com/opencontainers/go-digest"
1315
)
1416

1517
func main() {
1618
ctx := context.Background()
1719

1820
root := &core.Query{}
19-
dag := dagql.NewServer(root)
21+
dag := dagql.NewServer(root, dagql.NewSessionCache(cache.NewCache[digest.Digest, dagql.Typed]()))
2022
coreMod := &schema.CoreMod{Dag: dag}
2123
if err := coreMod.Install(ctx, dag); err != nil {
2224
panic(err)

core/container.go

Lines changed: 28 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ import (
3030
"github.com/pkg/errors"
3131
"github.com/vektah/gqlparser/v2/ast"
3232
"go.opentelemetry.io/otel/propagation"
33+
"go.opentelemetry.io/otel/trace"
3334

3435
"github.com/dagger/dagger/dagql"
3536
"github.com/dagger/dagger/dagql/call"
@@ -185,7 +186,7 @@ func (owner Ownership) Opt() llb.ChownOption {
185186
// ContainerSecret configures a secret to expose, either as an environment
186187
// variable or mounted to a file path.
187188
type ContainerSecret struct {
188-
Secret *Secret
189+
Secret dagql.Instance[*Secret]
189190
EnvName string
190191
MountPath string
191192
Owner *Ownership
@@ -388,7 +389,7 @@ func (container *Container) Build(
388389
dockerfile string,
389390
buildArgs []BuildArg,
390391
target string,
391-
secrets []*Secret,
392+
secrets []dagql.Instance[*Secret],
392393
secretStore *SecretStore,
393394
noInit bool,
394395
) (*Container, error) {
@@ -399,15 +400,15 @@ func (container *Container) Build(
399400

400401
secretNameToLLBID := make(map[string]string)
401402
for _, secret := range secrets {
402-
secretName, ok := secretStore.GetSecretName(secret.IDDigest)
403+
secretName, ok := secretStore.GetSecretName(secret.ID().Digest())
403404
if !ok {
404-
return nil, fmt.Errorf("secret not found: %s", secret.IDDigest)
405+
return nil, fmt.Errorf("secret not found: %s", secret.ID().Digest())
405406
}
406407
container.Secrets = append(container.Secrets, ContainerSecret{
407408
Secret: secret,
408409
MountPath: fmt.Sprintf("/run/secrets/%s", secretName),
409410
})
410-
secretNameToLLBID[secretName] = secret.IDDigest.String()
411+
secretNameToLLBID[secretName] = secret.ID().Digest().String()
411412
}
412413

413414
// set image ref to empty string
@@ -730,7 +731,13 @@ func (container *Container) WithMountedTemp(ctx context.Context, target string,
730731
return container, nil
731732
}
732733

733-
func (container *Container) WithMountedSecret(ctx context.Context, target string, source *Secret, owner string, mode fs.FileMode) (*Container, error) {
734+
func (container *Container) WithMountedSecret(
735+
ctx context.Context,
736+
target string,
737+
source dagql.Instance[*Secret],
738+
owner string,
739+
mode fs.FileMode,
740+
) (*Container, error) {
734741
container = container.Clone()
735742

736743
target = absPath(container.Config.WorkingDir, target)
@@ -840,7 +847,11 @@ func (container *Container) WithoutUnixSocket(ctx context.Context, target string
840847
return container, nil
841848
}
842849

843-
func (container *Container) WithSecretVariable(ctx context.Context, name string, secret *Secret) (*Container, error) {
850+
func (container *Container) WithSecretVariable(
851+
ctx context.Context,
852+
name string,
853+
secret dagql.Instance[*Secret],
854+
) (*Container, error) {
844855
container = container.Clone()
845856

846857
container.Secrets = append(container.Secrets, ContainerSecret{
@@ -1623,7 +1634,11 @@ func (container *Container) AsServiceLegacy(ctx context.Context) (*Service, erro
16231634
return nil, err
16241635
}
16251636
}
1626-
return container.Query.NewContainerService(ctx, container), nil
1637+
return &Service{
1638+
Creator: trace.SpanContextFromContext(ctx),
1639+
Query: container.Query,
1640+
Container: container,
1641+
}, nil
16271642
}
16281643

16291644
func (container *Container) AsService(ctx context.Context, args ContainerAsServiceArgs) (*Service, error) {
@@ -1658,7 +1673,11 @@ func (container *Container) AsService(ctx context.Context, args ContainerAsServi
16581673
return nil, err
16591674
}
16601675

1661-
return container.Query.NewContainerService(ctx, container), nil
1676+
return &Service{
1677+
Creator: trace.SpanContextFromContext(ctx),
1678+
Query: container.Query,
1679+
Container: container,
1680+
}, nil
16621681
}
16631682

16641683
func (container *Container) ownership(ctx context.Context, owner string) (*Ownership, error) {

core/container_exec.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -194,7 +194,7 @@ func (container *Container) WithExec(ctx context.Context, opts ContainerExecOpts
194194
}
195195

196196
for i, secret := range container.Secrets {
197-
secretOpts := []llb.SecretOption{llb.SecretID(secret.Secret.LLBID())}
197+
secretOpts := []llb.SecretOption{llb.SecretID(secret.Secret.ID().Digest().String())}
198198

199199
var secretDest string
200200
switch {

core/contenthash.go

Lines changed: 19 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -30,24 +30,37 @@ func MakeDirectoryContentHashed(
3030
bk *buildkit.Client,
3131
dirInst dagql.Instance[*Directory],
3232
) (retInst dagql.Instance[*Directory], err error) {
33+
dgst, err := GetContentHashFromDirectory(ctx, bk, dirInst)
34+
if err != nil {
35+
return retInst, fmt.Errorf("failed to get content hash: %w", err)
36+
}
37+
38+
return dirInst.WithDigest(dgst), nil
39+
}
40+
41+
func GetContentHashFromDirectory(
42+
ctx context.Context,
43+
bk *buildkit.Client,
44+
dirInst dagql.Instance[*Directory],
45+
) (digest.Digest, error) {
3346
if dirInst.Self == nil {
34-
return retInst, fmt.Errorf("directory instance is nil")
47+
return "", fmt.Errorf("directory instance is nil")
3548
}
3649

3750
st, err := dirInst.Self.State()
3851
if err != nil {
39-
return retInst, fmt.Errorf("failed to get state: %w", err)
52+
return "", fmt.Errorf("failed to get state: %w", err)
4053
}
4154
def, err := st.Marshal(ctx, llb.Platform(dirInst.Self.Platform.Spec()))
4255
if err != nil {
43-
return retInst, fmt.Errorf("failed to marshal state: %w", err)
56+
return "", fmt.Errorf("failed to marshal state: %w", err)
4457
}
4558
dgst, err := GetContentHashFromDef(ctx, bk, def.ToPB(), dirInst.Self.Dir)
4659
if err != nil {
47-
return retInst, fmt.Errorf("failed to get content hash: %w", err)
60+
return "", fmt.Errorf("failed to get content hash: %w", err)
4861
}
4962

50-
return dirInst.WithDigest(dgst), nil
63+
return dgst, nil
5164
}
5265

5366
func GetContentHashFromDef(
@@ -100,7 +113,7 @@ func GetContentHashFromDef(
100113
}
101114

102115
ctx, span := Tracer(ctx).Start(ctx,
103-
fmt.Sprintf("checksum def: %s", ref.ID()),
116+
fmt.Sprintf("checksum def: %s", key),
104117
telemetry.Internal(),
105118
)
106119
defer telemetry.End(span, func() error { return rerr })

core/git.go

Lines changed: 19 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -131,8 +131,8 @@ type RemoteGitRepository struct {
131131
Services ServiceBindings
132132
Platform Platform
133133

134-
AuthToken *Secret
135-
AuthHeader *Secret
134+
AuthToken dagql.Instance[*Secret]
135+
AuthHeader dagql.Instance[*Secret]
136136
}
137137

138138
var _ GitRepositoryBackend = (*RemoteGitRepository)(nil)
@@ -271,6 +271,19 @@ func (ref *RemoteGitRef) Commit(ctx context.Context) (string, error) {
271271
if err != nil {
272272
return "", err
273273
}
274+
275+
if len(ref.Repo.Services) > 0 {
276+
svcs, err := ref.Query.Services(ctx)
277+
if err != nil {
278+
return "", err
279+
}
280+
detach, _, err := svcs.StartBindings(ctx, ref.Repo.Services)
281+
if err != nil {
282+
return "", err
283+
}
284+
defer detach()
285+
}
286+
274287
p, err := resolveProvenance(ctx, bk, st)
275288
if err != nil {
276289
return "", err
@@ -293,11 +306,11 @@ func (ref *RemoteGitRef) getState(ctx context.Context, discardGitDir bool) (llb.
293306
if ref.Repo.SSHAuthSocket != nil {
294307
opts = append(opts, llb.MountSSHSock(ref.Repo.SSHAuthSocket.LLBID()))
295308
}
296-
if ref.Repo.AuthToken != nil {
297-
opts = append(opts, llb.AuthTokenSecret(ref.Repo.AuthToken.LLBID()))
309+
if ref.Repo.AuthToken.Self != nil {
310+
opts = append(opts, llb.AuthTokenSecret(ref.Repo.AuthToken.ID().Digest().String()))
298311
}
299-
if ref.Repo.AuthHeader != nil {
300-
opts = append(opts, llb.AuthHeaderSecret(ref.Repo.AuthHeader.LLBID()))
312+
if ref.Repo.AuthHeader.Self != nil {
313+
opts = append(opts, llb.AuthHeaderSecret(ref.Repo.AuthHeader.ID().Digest().String()))
301314
}
302315

303316
clientMetadata, err := engine.ClientMetadataFromContext(ctx)

core/host.go

Lines changed: 0 additions & 50 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,7 @@
11
package core
22

33
import (
4-
"context"
5-
"fmt"
6-
74
"github.com/vektah/gqlparser/v2/ast"
8-
9-
"github.com/dagger/dagger/dagql"
105
)
116

127
type Host struct {
@@ -23,48 +18,3 @@ func (*Host) Type() *ast.Type {
2318
func (*Host) TypeDescription() string {
2419
return "Information about the host environment."
2520
}
26-
27-
func (host *Host) SetSecretFile(ctx context.Context, srv *dagql.Server, secretName string, path string) (i dagql.Instance[*Secret], err error) {
28-
secretStore, err := host.Query.Secrets(ctx)
29-
if err != nil {
30-
return i, fmt.Errorf("failed to get secrets: %w", err)
31-
}
32-
33-
accessor, err := GetClientResourceAccessor(ctx, host.Query, secretName)
34-
if err != nil {
35-
return i, err
36-
}
37-
38-
bk, err := host.Query.Buildkit(ctx)
39-
if err != nil {
40-
return i, fmt.Errorf("failed to get buildkit client: %w", err)
41-
}
42-
43-
secretFileContent, err := bk.ReadCallerHostFile(ctx, path)
44-
if err != nil {
45-
return i, fmt.Errorf("read secret file: %w", err)
46-
}
47-
48-
err = srv.Select(ctx, srv.Root(), &i, dagql.Selector{
49-
Field: "loadSecretFromName",
50-
Args: []dagql.NamedInput{
51-
{
52-
Name: "name",
53-
Value: dagql.NewString(secretName),
54-
},
55-
{
56-
Name: "accessor",
57-
Value: dagql.Opt(dagql.NewString(accessor)),
58-
},
59-
},
60-
})
61-
if err != nil {
62-
return i, fmt.Errorf("failed to select secret: %w", err)
63-
}
64-
65-
if err := secretStore.AddSecret(i.Self, secretName, secretFileContent); err != nil {
66-
return i, fmt.Errorf("failed to add secret: %w", err)
67-
}
68-
69-
return i, nil
70-
}

core/integration/container_test.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4792,6 +4792,7 @@ func (ContainerSuite) TestEnvExpand(ctx context.Context, t *testctx.T) {
47924792
dir := t.TempDir()
47934793
require.NoError(t, os.WriteFile(filepath.Join(dir, "some-file"), data, 0o600))
47944794

4795+
//nolint:staticcheck // SA1019 deprecated
47954796
secret := c.Host().SetSecretFile("mysecret", filepath.Join(dir, "some-file"))
47964797
output, err := c.Container().
47974798
From("alpine:latest").

0 commit comments

Comments
 (0)