diff --git a/docs/build-repro.md b/docs/build-repro.md index 5294ecbbb5f7..c1058ecfa7c5 100644 --- a/docs/build-repro.md +++ b/docs/build-repro.md @@ -51,6 +51,13 @@ ARG SOURCE_DATE_EPOCH=1704067200 FROM alpine ``` +The Dockerfile frontend also supports the special value `SOURCE_DATE_EPOCH=context`. +This resolves the main build context to a numeric Unix timestamp before the build: + +- git context: commit time +- HTTP context: `Last-Modified`, or the newest archive entry mtime when building from an archive without `Last-Modified` +- local context: ignored, leaving `SOURCE_DATE_EPOCH` unset + ```console buildctl build --frontend dockerfile.v0 --opt build-arg:SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct) ... ``` diff --git a/exporter/util/epoch/parse.go b/exporter/util/epoch/parse.go index aa32b16b8e0e..efd0e978220e 100644 --- a/exporter/util/epoch/parse.go +++ b/exporter/util/epoch/parse.go @@ -21,7 +21,13 @@ type Epoch struct { func ParseBuildArgs(opt map[string]string) (string, bool) { v, ok := opt[frontendSourceDateEpochArg] - return v, ok + if !ok { + return "", false + } + if _, err := parseTime(frontendSourceDateEpochArg, v); err != nil { + return "", false + } + return v, true } func ParseExporterAttrs(opt map[string]string) (*Epoch, map[string]string, error) { diff --git a/exporter/util/epoch/parse_test.go b/exporter/util/epoch/parse_test.go new file mode 100644 index 000000000000..9b128d0c0a4a --- /dev/null +++ b/exporter/util/epoch/parse_test.go @@ -0,0 +1,19 @@ +package epoch + +import "testing" + +func TestParseBuildArgs(t *testing.T) { + t.Parallel() + + if v, ok := ParseBuildArgs(map[string]string{frontendSourceDateEpochArg: "1700000601"}); !ok || v != "1700000601" { + t.Fatalf("expected numeric SOURCE_DATE_EPOCH to be forwarded, got %q %v", v, ok) + } + + if _, ok := ParseBuildArgs(map[string]string{frontendSourceDateEpochArg: "context"}); ok { + t.Fatal("expected SOURCE_DATE_EPOCH=context to stay frontend-only") + } + + if v, ok := ParseBuildArgs(map[string]string{frontendSourceDateEpochArg: ""}); !ok || v != "" { + t.Fatalf("expected empty SOURCE_DATE_EPOCH to remain a valid exporter override, got %q %v", v, ok) + } +} diff --git a/frontend/dockerfile/builder/build.go b/frontend/dockerfile/builder/build.go index 33a8c9294102..ae3a734bfee6 100644 --- a/frontend/dockerfile/builder/build.go +++ b/frontend/dockerfile/builder/build.go @@ -2,6 +2,7 @@ package builder import ( "context" + "maps" "strings" "github.com/containerd/platforms" @@ -131,6 +132,7 @@ func Build(ctx context.Context, c client.Client) (_ *client.Result, err error) { rb, err := bc.Build(ctx, func(ctx context.Context, platform *ocispecs.Platform, idx int) (*dockerui.BuildResult, error) { opt := convertOpt + opt.BuildArgs = maps.Clone(opt.BuildArgs) opt.TargetPlatform = platform if idx != 0 { opt.Warn = nil diff --git a/frontend/dockerfile/dockerfile2llb/convert.go b/frontend/dockerfile/dockerfile2llb/convert.go index 3220046f112f..cb14676a73af 100644 --- a/frontend/dockerfile/dockerfile2llb/convert.go +++ b/frontend/dockerfile/dockerfile2llb/convert.go @@ -208,6 +208,7 @@ type dispatchContext struct { opt ConvertOpt platformOpt *platformOpt globalArgs *llb.EnvList + epoch *time.Time shlex *shell.Lex outline outlineCapture lint *linter.Linter @@ -300,9 +301,13 @@ func toDispatchState(ctx context.Context, dt []byte, opt ConvertOpt) (*dispatchS return nil, err } - opt.Epoch, err = resolveSourceDateEpoch(opt.Epoch, globalArgs) - if err != nil { - return nil, err + var resolvedEpoch *time.Time + if sourceDateEpoch, ok := getBuildArgValue(opt.BuildArgs, globalArgs, "SOURCE_DATE_EPOCH"); ok { + resolvedEpoch, err = resolveSourceDateEpochValue(ctx, sourceDateEpoch, opt, stages, globalArgs, shlex) + if err != nil { + return nil, err + } + globalArgs = setBuildArgValue(opt.BuildArgs, globalArgs, "SOURCE_DATE_EPOCH", formatSourceDateEpochValue(resolvedEpoch)) } metaResolver := opt.MetaResolver @@ -314,6 +319,7 @@ func toDispatchState(ctx context.Context, dt []byte, opt ConvertOpt) (*dispatchS opt: opt, platformOpt: platformOpt, globalArgs: globalArgs, + epoch: resolvedEpoch, shlex: shlex, outline: outline, lint: lint, @@ -353,25 +359,39 @@ func toDispatchState(ctx context.Context, dt []byte, opt ConvertOpt) (*dispatchS return target, nil } -func resolveSourceDateEpoch(explicit *time.Time, globalArgs *llb.EnvList) (*time.Time, error) { - if explicit != nil { - return explicit, nil +func getBuildArgValue(buildArgs map[string]string, globalArgs *llb.EnvList, key string) (string, bool) { + if v, ok := buildArgs[key]; ok { + return v, true } if globalArgs == nil { - return nil, nil + return "", false } - - v, ok := globalArgs.Get("SOURCE_DATE_EPOCH") + v, ok := globalArgs.Get(key) if !ok || v == "" { - return nil, nil + return "", false } + return v, true +} - sde, err := strconv.ParseInt(v, 10, 64) - if err != nil { - return nil, errors.Wrapf(err, "invalid SOURCE_DATE_EPOCH: %s", v) +func setBuildArgValue(buildArgs map[string]string, globalArgs *llb.EnvList, key, value string) *llb.EnvList { + if _, ok := buildArgs[key]; ok { + if value == "" { + delete(buildArgs, key) + } else { + buildArgs[key] = value + } + } + if globalArgs != nil { + if _, ok := globalArgs.Get(key); ok { + if value == "" { + updated := globalArgs.Delete(key) + globalArgs = &updated + } else { + globalArgs = globalArgs.AddOrReplace(key, value) + } + } } - tm := time.Unix(sde, 0).UTC() - return &tm, nil + return globalArgs } func (dctx *dispatchContext) buildDispatchStates(stages []instructions.Stage) error { @@ -402,7 +422,7 @@ func (dctx *dispatchContext) buildDispatchStates(stages []instructions.Stage) er stageName: st.Name, prefixPlatform: dctx.opt.MultiPlatformRequested, outline: dctx.outline.clone(), - epoch: dctx.opt.Epoch, + epoch: dctx.epoch, } if v := st.Platform; v != "" { diff --git a/frontend/dockerfile/dockerfile2llb/convert_test.go b/frontend/dockerfile/dockerfile2llb/convert_test.go index bfb9d0117e3c..b013ae82f380 100644 --- a/frontend/dockerfile/dockerfile2llb/convert_test.go +++ b/frontend/dockerfile/dockerfile2llb/convert_test.go @@ -1,13 +1,16 @@ package dockerfile2llb import ( + "bytes" "context" + "maps" "testing" "time" "github.com/moby/buildkit/client/llb" "github.com/moby/buildkit/frontend/dockerfile/instructions" "github.com/moby/buildkit/frontend/dockerfile/linter" + "github.com/moby/buildkit/frontend/dockerfile/parser" "github.com/moby/buildkit/frontend/dockerfile/shell" "github.com/moby/buildkit/frontend/dockerui" "github.com/moby/buildkit/util/appcontext" @@ -276,3 +279,129 @@ func TestDispatchHealthcheckHistory(t *testing.T) { want := `HEALTHCHECK {Test:[bin -c exit 0] Interval:1s Timeout:10s StartPeriod:3s StartInterval:100ms Retries:5}` require.Equal(t, want, d.image.History[0].CreatedBy) } + +func TestResolveSourceDateEpochValue(t *testing.T) { + t.Parallel() + + globalArgs := &llb.EnvList{} + shlex := shell.NewLex('\\') + + tm, err := resolveSourceDateEpochValue(context.Background(), "1700000501", ConvertOpt{}, nil, globalArgs, shlex) + require.NoError(t, err) + require.NotNil(t, tm) + assert.Equal(t, time.Unix(1700000501, 0).UTC(), *tm) + assert.Equal(t, "1700000501", formatSourceDateEpochValue(tm)) + + tm, err = resolveSourceDateEpochValue(context.Background(), "context", ConvertOpt{}, nil, globalArgs, shlex) + require.NoError(t, err) + assert.Nil(t, tm) + assert.Empty(t, formatSourceDateEpochValue(tm)) + + _, err = resolveSourceDateEpochValue(context.Background(), "not-a-timestamp", ConvertOpt{}, nil, globalArgs, shlex) + require.ErrorContains(t, err, "invalid SOURCE_DATE_EPOCH") +} + +func TestResolveSourceDateEpochValueStageInvalid(t *testing.T) { + t.Parallel() + + df := []byte(` +ARG SOURCE_DATE_EPOCH=mysource +FROM scratch AS mysource +COPY Dockerfile /Dockerfile +FROM scratch +`) + + parsed, err := parser.Parse(bytes.NewReader(df)) + require.NoError(t, err) + + stages, _, err := instructions.Parse(parsed.AST, nil) + require.NoError(t, err) + + globalArgs := (&llb.EnvList{}).AddOrReplace("SOURCE_DATE_EPOCH", "mysource") + _, err = resolveSourceDateEpochValue(context.Background(), "mysource", ConvertOpt{}, stages, globalArgs, shell.NewLex('\\')) + require.ErrorContains(t, err, "SOURCE_DATE_EPOCH stage does not meet source-only requirements") +} + +func TestSourceDateEpochStageSourceHTTP(t *testing.T) { + t.Parallel() + + df := []byte(` +FROM scratch AS mysource +ARG URL=https://example.com/src.tar +ADD $URL / +`) + + parsed, err := parser.Parse(bytes.NewReader(df)) + require.NoError(t, err) + + stages, _, err := instructions.Parse(parsed.AST, nil) + require.NoError(t, err) + require.Len(t, stages, 1) + + state, err := sourceDateEpochStageSource(stages[0], nil, &llb.EnvList{}, shell.NewLex('\\')) + require.NoError(t, err) + require.NotNil(t, state) + sourceOp, err := sourceOpFromState(context.Background(), state) + require.NoError(t, err) + require.NotNil(t, sourceOp) + assert.Equal(t, "src.tar", sourceOp.Attrs["http.filename"]) +} + +func TestSourceDateEpochStageSourceRequiresScratch(t *testing.T) { + t.Parallel() + + df := []byte(` +FROM busybox AS mysource +ADD https://example.com/src.tar / +`) + + parsed, err := parser.Parse(bytes.NewReader(df)) + require.NoError(t, err) + + stages, _, err := instructions.Parse(parsed.AST, nil) + require.NoError(t, err) + require.Len(t, stages, 1) + + _, err = sourceDateEpochStageSource(stages[0], nil, &llb.EnvList{}, shell.NewLex('\\')) + require.ErrorContains(t, err, "SOURCE_DATE_EPOCH stage must use FROM scratch") +} + +func TestSourceOpFromStateWrappedCopy(t *testing.T) { + t.Parallel() + + st := llb.Scratch().File(llb.Copy(llb.HTTP("https://example.com/src.tar"), "src.tar", "/foo")) + + sourceOp, err := sourceOpFromState(context.Background(), &st) + require.NoError(t, err) + require.NotNil(t, sourceOp) + assert.Equal(t, "https://example.com/src.tar", sourceOp.Identifier) +} + +func TestSourceOpFromStateMultipleSourcesIgnored(t *testing.T) { + t.Parallel() + + st := llb.Scratch(). + File(llb.Copy(llb.HTTP("https://example.com/src1.tar"), "src1.tar", "/foo")). + File(llb.Copy(llb.HTTP("https://example.com/src2.tar"), "src2.tar", "/bar")) + + sourceOp, err := sourceOpFromState(context.Background(), &st) + require.NoError(t, err) + assert.Nil(t, sourceOp) +} + +func TestSourceStateFromSourceOpWrappedCopy(t *testing.T) { + t.Parallel() + + st := llb.Scratch().File(llb.Copy(llb.HTTP("https://example.com/src.tar", llb.Filename("src.tar")), "src.tar", "/foo")) + + sourceOp, err := sourceOpFromState(context.Background(), &st) + require.NoError(t, err) + require.NotNil(t, sourceOp) + + sourceState := llb.NewState(llb.NewSource(sourceOp.Identifier, maps.Clone(sourceOp.Attrs), llb.Constraints{}).Output()) + rewrittenSourceOp, err := sourceOpFromState(context.Background(), &sourceState) + require.NoError(t, err) + require.NotNil(t, rewrittenSourceOp) + assert.Equal(t, sourceOp.Identifier, rewrittenSourceOp.Identifier) + assert.Equal(t, sourceOp.Attrs, rewrittenSourceOp.Attrs) +} diff --git a/frontend/dockerfile/dockerfile2llb/epoch.go b/frontend/dockerfile/dockerfile2llb/epoch.go new file mode 100644 index 000000000000..03e7d667d310 --- /dev/null +++ b/frontend/dockerfile/dockerfile2llb/epoch.go @@ -0,0 +1,381 @@ +package dockerfile2llb + +import ( + "archive/tar" + "bytes" + "context" + "io" + "maps" + "net/url" + "path" + "strconv" + "strings" + "time" + + "github.com/moby/buildkit/client/llb" + "github.com/moby/buildkit/client/llb/sourceresolver" + "github.com/moby/buildkit/frontend/dockerfile/dfgitutil" + "github.com/moby/buildkit/frontend/dockerfile/instructions" + "github.com/moby/buildkit/frontend/dockerfile/parser" + "github.com/moby/buildkit/frontend/dockerfile/shell" + "github.com/moby/buildkit/frontend/dockerui" + gwclient "github.com/moby/buildkit/frontend/gateway/client" + "github.com/moby/buildkit/solver/pb" + "github.com/moby/buildkit/util/gitutil/gitobject" + archivecompression "github.com/moby/go-archive/compression" + digest "github.com/opencontainers/go-digest" + "github.com/pkg/errors" +) + +type sourceDateEpochStateOpt struct { + LogName string +} + +func resolveSourceDateEpochValue(ctx context.Context, v string, opt ConvertOpt, stages []instructions.Stage, globalArgs *llb.EnvList, shlex *shell.Lex) (*time.Time, error) { + if v == "" { + return nil, nil + } + if sde, err := strconv.ParseInt(v, 10, 64); err == nil { + tm := time.Unix(sde, 0).UTC() + return &tm, nil + } + + state, stateOpt, err := resolveSourceDateEpochState(ctx, v, opt, stages, globalArgs, shlex) + if err != nil { + return nil, err + } + if state == nil || opt.Client == nil { + return nil, nil + } + return resolveSourceDateEpochFromState(ctx, *state, opt.Client, stateOpt) +} + +func formatSourceDateEpochValue(tm *time.Time) string { + if tm == nil { + return "" + } + return strconv.FormatInt(tm.Unix(), 10) +} + +func resolveSourceDateEpochState(ctx context.Context, value string, opt ConvertOpt, stages []instructions.Stage, globalArgs *llb.EnvList, shlex *shell.Lex) (*llb.State, sourceDateEpochStateOpt, error) { + if value == "context" { + if opt.Client == nil { + return nil, sourceDateEpochStateOpt{}, nil + } + mainContextState, err := opt.Client.MainContext(ctx) + if err != nil { + return nil, sourceDateEpochStateOpt{}, err + } + return mainContextState, sourceDateEpochStateOpt{ + LogName: "[internal] resolve main build context metadata", + }, nil + } + + if opt.Client != nil { + nc, err := opt.Client.NamedContext(value, dockerui.ContextOpt{}) + if err != nil { + return nil, sourceDateEpochStateOpt{}, err + } + if nc != nil { + st, _, err := nc.Load(ctx) + if err != nil { + return nil, sourceDateEpochStateOpt{}, err + } + return st, sourceDateEpochStateOpt{ + LogName: "[internal] resolve SOURCE_DATE_EPOCH named context " + value, + }, nil + } + } + + for i := range stages { + if !strings.EqualFold(stages[i].Name, value) { + continue + } + + args := globalArgs + if globalArgs != nil { + updated := globalArgs.Delete("SOURCE_DATE_EPOCH") + args = &updated + } + + sourceState, err := sourceDateEpochStageSource(stages[i], opt.BuildArgs, args, shlex) + if err != nil { + return nil, sourceDateEpochStateOpt{}, parser.WithLocation(err, stages[i].Location) + } + + return sourceState, sourceDateEpochStateOpt{ + LogName: "[internal] resolve SOURCE_DATE_EPOCH source stage " + stages[i].Name, + }, nil + } + return nil, sourceDateEpochStateOpt{}, errors.Errorf("invalid SOURCE_DATE_EPOCH: %s", value) +} + +func sourceDateEpochStageSource(stage instructions.Stage, buildArgs map[string]string, globalArgs *llb.EnvList, shlex *shell.Lex) (*llb.State, error) { + stageBaseName, _, err := shlex.ProcessWord(stage.BaseName, globalArgs) + if err != nil { + return nil, errors.Wrapf(err, "failed to process source stage base name %q", stage.BaseName) + } + if stageBaseName != emptyImageName { + return nil, errors.New("SOURCE_DATE_EPOCH stage must use FROM scratch") + } + + env := globalArgs + var sourceState *llb.State + + for _, cmd := range stage.Commands { + switch c := cmd.(type) { + case *instructions.ArgCommand: + env, err = applySourceDateEpochStageArgs(c.Args, env, buildArgs, shlex) + if err != nil { + return nil, err + } + case *instructions.AddCommand: + if sourceState != nil { + return nil, errors.New("SOURCE_DATE_EPOCH stage must contain exactly one remote ADD") + } + sourceState, err = sourceDateEpochAddSource(c, env, shlex) + if err != nil { + return nil, err + } + default: + return nil, errors.Errorf("SOURCE_DATE_EPOCH stage does not meet source-only requirements: unsupported %s instruction", cmd.Name()) + } + } + + if sourceState == nil { + return nil, errors.New("SOURCE_DATE_EPOCH stage must contain exactly one remote ADD") + } + + return sourceState, nil +} + +func applySourceDateEpochStageArgs(args []instructions.KeyValuePairOptional, env *llb.EnvList, buildArgs map[string]string, shlex *shell.Lex) (*llb.EnvList, error) { + for _, arg := range args { + if v, ok := buildArgs[arg.Key]; ok { + env = env.AddOrReplace(arg.Key, v) + continue + } + if arg.Value == nil { + continue + } + v, _, err := shlex.ProcessWord(*arg.Value, env) + if err != nil { + return nil, err + } + env = env.AddOrReplace(arg.Key, v) + } + return env, nil +} + +func sourceDateEpochAddSource(cmd *instructions.AddCommand, env *llb.EnvList, shlex *shell.Lex) (*llb.State, error) { + if len(cmd.SourceContents) != 0 || len(cmd.SourcePaths) != 1 { + return nil, errors.New("SOURCE_DATE_EPOCH stage must contain exactly one remote ADD source") + } + + src, _, err := shlex.ProcessWord(cmd.SourcePaths[0], env) + if err != nil { + return nil, err + } + + if isHTTPSource(src) { + var checksum digest.Digest + if cmd.Checksum != "" { + expandedChecksum, _, err := shlex.ProcessWord(cmd.Checksum, env) + if err != nil { + return nil, err + } + checksum, err = digest.Parse(expandedChecksum) + if err != nil { + return nil, err + } + } + st := llb.HTTP(src, llb.Filename(sourceDateEpochHTTPFilename(src)), llb.Checksum(checksum)) + return &st, nil + } + + gitRef, isGit, gitRefErr := dfgitutil.ParseGitRef(src) + if gitRefErr != nil && isGit { + return nil, gitRefErr + } + if gitRefErr == nil && !gitRef.IndistinguishableFromLocal { + gitOptions := []llb.GitOption{ + llb.GitRef(gitRef.Ref), + } + if cmd.KeepGitDir != nil && *cmd.KeepGitDir { + gitOptions = append(gitOptions, llb.KeepGitDir()) + } + if gitRef.KeepGitDir != nil && *gitRef.KeepGitDir { + gitOptions = append(gitOptions, llb.KeepGitDir()) + } + if cmd.Checksum != "" { + expandedChecksum, _, err := shlex.ProcessWord(cmd.Checksum, env) + if err != nil { + return nil, err + } + gitOptions = append(gitOptions, llb.GitChecksum(expandedChecksum)) + } else if gitRef.Checksum != "" { + gitOptions = append(gitOptions, llb.GitChecksum(gitRef.Checksum)) + } + if gitRef.SubDir != "" { + gitOptions = append(gitOptions, llb.GitSubDir(gitRef.SubDir)) + } + if gitRef.Submodules != nil && !*gitRef.Submodules { + gitOptions = append(gitOptions, llb.GitSkipSubmodules()) + } + st := llb.Git(gitRef.Remote, "", gitOptions...) + return &st, nil + } + + return nil, errors.New("SOURCE_DATE_EPOCH stage source must be a single HTTP(S) or Git ADD") +} + +func sourceDateEpochHTTPFilename(src string) string { + u, err := url.Parse(src) + if err == nil { + if base := path.Base(u.Path); base != "." && base != "/" { + return base + } + } + return "__unnamed__" +} + +func resolveSourceDateEpochFromState(ctx context.Context, st llb.State, client *dockerui.Client, opt sourceDateEpochStateOpt) (*time.Time, error) { + sourceOp, err := sourceOpFromState(ctx, &st, llb.WithCaps(client.BuildOpts().Caps)) + if err != nil { + return nil, err + } + if sourceOp == nil { + return nil, nil + } + + metaOpt := sourceresolver.Opt{ + LogName: opt.LogName, + } + if strings.HasPrefix(sourceOp.Identifier, "git://") { + metaOpt.GitOpt = &sourceresolver.ResolveGitOpt{ReturnObject: true} + } + isHTTP := strings.HasPrefix(sourceOp.Identifier, "http://") || strings.HasPrefix(sourceOp.Identifier, "https://") + md, err := client.GatewayClient().ResolveSourceMetadata(ctx, sourceOp, metaOpt) + if err != nil { + return nil, err + } + if tm, ok, err := sourceDateEpochFromMetadata(md); ok || err != nil { + return tm, err + } + filename := sourceOp.Attrs[pb.AttrHTTPFilename] + if !isHTTP || filename == "" { + return nil, nil + } + + sourceState := llb.NewState(llb.NewSource(sourceOp.Identifier, maps.Clone(sourceOp.Attrs), llb.Constraints{}).Output()) + def, err := sourceState.Marshal(ctx, llb.WithCaps(client.BuildOpts().Caps)) + if err != nil { + return nil, err + } + res, err := client.GatewayClient().Solve(ctx, gwclient.SolveRequest{ + Definition: def.ToPB(), + }) + if err != nil { + return nil, err + } + ref, err := res.SingleRef() + if err != nil { + return nil, err + } + + return archiveMaxTimeFromRef(ctx, ref, filename, true) +} + +func sourceOpFromState(ctx context.Context, st *llb.State, opts ...llb.ConstraintsOpt) (*pb.SourceOp, error) { + if st == nil { + return nil, nil + } + def, err := st.Marshal(ctx, opts...) + if err != nil { + return nil, err + } + dt := def.ToPB().Def + var src *pb.SourceOp + for _, d := range dt { + var op pb.Op + if err := op.Unmarshal(d); err != nil { + return nil, err + } + opSrc := op.GetSource() + if opSrc == nil { + continue + } + if src != nil { + return nil, nil + } + src = opSrc + } + return cloneSourceOp(src), nil +} + +func cloneSourceOp(op *pb.SourceOp) *pb.SourceOp { + if op == nil { + return nil + } + return &pb.SourceOp{ + Identifier: op.Identifier, + Attrs: maps.Clone(op.Attrs), + } +} + +func sourceDateEpochFromMetadata(md *sourceresolver.MetaResponse) (*time.Time, bool, error) { + if md.Git != nil && len(md.Git.CommitObject) > 0 { + obj, err := gitobject.Parse(md.Git.CommitObject) + if err != nil { + return nil, false, err + } + commit, err := obj.ToCommit() + if err != nil { + return nil, false, err + } + return commit.Committer.When, true, nil + } + if md.HTTP != nil && md.HTTP.LastModified != nil { + return md.HTTP.LastModified, true, nil + } + return nil, false, nil +} + +func archiveMaxTimeFromRef(ctx context.Context, ref gwclient.Reference, filename string, allowNonArchive bool) (*time.Time, error) { + dt, err := ref.ReadFile(ctx, gwclient.ReadRequest{ + Filename: filename, + }) + if err != nil { + return nil, err + } + rc, err := archivecompression.DecompressStream(bytes.NewReader(dt)) + if err != nil { + if allowNonArchive { + return nil, nil + } + return nil, err + } + defer rc.Close() + + tr := tar.NewReader(rc) + var maxTime *time.Time + for { + hdr, err := tr.Next() + if err != nil { + if errors.Is(err, io.EOF) { + return maxTime, nil + } + if allowNonArchive { + return nil, nil + } + return nil, err + } + if !hdr.FileInfo().Mode().IsRegular() { + continue + } + tm := hdr.ModTime.UTC() + if maxTime == nil || tm.After(*maxTime) { + maxTime = &tm + } + } +} diff --git a/frontend/dockerfile/dockerfile_test.go b/frontend/dockerfile/dockerfile_test.go index da3e713a37f2..a80efbfb49f3 100644 --- a/frontend/dockerfile/dockerfile_test.go +++ b/frontend/dockerfile/dockerfile_test.go @@ -234,6 +234,15 @@ var reproTests = integration.TestFuncs( testSourceDateEpochDockerfileDefaultOverride, testSourceDateEpochDockerfileDefaultReset, testSourceDateEpochDockerfileDefaultInvalid, + testSourceDateEpochContextGit, + testSourceDateEpochContextHTTPLastModified, + testSourceDateEpochContextHTTPArchive, + testSourceDateEpochContextLocalUnset, + testSourceDateEpochStageHTTPArchive, + testSourceDateEpochStageOverride, + testSourceDateEpochStageInvalid, + testSourceDateEpochNamedContextHTTPLastModified, + testSourceDateEpochNamedContextHTTPArchive, ) var ( @@ -1031,6 +1040,61 @@ WORKDIR /mydir require.Equal(t, index1, index2) } +type tarContextFile struct { + name string + data []byte + modTime time.Time +} + +func makeTarContext(t *testing.T, files ...tarContextFile) []byte { + t.Helper() + + buf := bytes.NewBuffer(nil) + tw := tar.NewWriter(buf) + for _, file := range files { + require.NoError(t, tw.WriteHeader(&tar.Header{ + Name: file.name, + Mode: 0600, + Size: int64(len(file.data)), + Typeflag: tar.TypeReg, + ModTime: file.modTime, + })) + _, err := tw.Write(file.data) + require.NoError(t, err) + } + require.NoError(t, tw.Close()) + return buf.Bytes() +} + +func readOCIImage(t *testing.T, dt []byte) ocispecs.Image { + t.Helper() + + m, err := testutil.ReadTarToMap(dt, false) + require.NoError(t, err) + + var idx ocispecs.Index + err = json.Unmarshal(m[ocispecs.ImageIndexFile].Data, &idx) + require.NoError(t, err) + + var mfst ocispecs.Manifest + err = json.Unmarshal(m[ocispecs.ImageBlobsDir+"/sha256/"+idx.Manifests[0].Digest.Hex()].Data, &mfst) + require.NoError(t, err) + + var img ocispecs.Image + err = json.Unmarshal(m[ocispecs.ImageBlobsDir+"/sha256/"+mfst.Config.Digest.Hex()].Data, &img) + require.NoError(t, err) + + return img +} + +func readOCIImageCreated(t *testing.T, dt []byte) time.Time { + t.Helper() + + img := readOCIImage(t, dt) + require.NotNil(t, img.Created) + return *img.Created +} + func readOCIManifest(t *testing.T, dt []byte) ocispecs.Manifest { t.Helper() @@ -1048,6 +1112,18 @@ func readOCIManifest(t *testing.T, dt []byte) ocispecs.Manifest { return mfst } +func readOCILayerMap(t *testing.T, dt []byte, layer ocispecs.Descriptor) map[string]*testutil.TarItem { + t.Helper() + + m, err := testutil.ReadTarToMap(dt, false) + require.NoError(t, err) + + layerMap, err := testutil.ReadTarToMap(m[ocispecs.ImageBlobsDir+"/sha256/"+layer.Digest.Hex()].Data, true) + require.NoError(t, err) + + return layerMap +} + func testCacheReleased(t *testing.T, sb integration.Sandbox) { f := getFrontend(t, sb) @@ -9183,17 +9259,516 @@ COPY Dockerfile . dockerui.DefaultLocalNameDockerfile: dir, dockerui.DefaultLocalNameContext: dir, }, + Exports: []client.ExportEntry{ + { + Type: client.ExporterOCI, + Output: fixedWriteCloser(outW), + }, + }, + }, nil) + require.ErrorContains(t, err, "invalid SOURCE_DATE_EPOCH: not-a-timestamp") +} + +func testSourceDateEpochContextGit(t *testing.T, sb integration.Sandbox) { + integration.SkipOnPlatform(t, "windows") + workers.CheckFeatureCompat(t, sb, workers.FeatureOCIExporter, workers.FeatureSourceDateEpoch) + f := getFrontend(t, sb) + + gitDir := t.TempDir() + dockerfile := []byte("FROM scratch\nCOPY Dockerfile /\n") + require.NoError(t, os.WriteFile(filepath.Join(gitDir, "Dockerfile"), dockerfile, 0600)) + + commitTime := time.Unix(1700000101, 0).UTC() + require.NoError(t, runShell(gitDir, + "git init", + "git config --local user.email test@example.com", + "git config --local user.name test", + "git add Dockerfile", + fmt.Sprintf("GIT_AUTHOR_DATE=%q GIT_COMMITTER_DATE=%q git commit -m msg", commitTime.Format(time.RFC3339), commitTime.Format(time.RFC3339)), + "git update-server-info", + )) + + server := httptest.NewServer(http.FileServer(http.Dir(filepath.Clean(gitDir)))) + defer server.Close() + + c, err := client.New(sb.Context(), sb.Address()) + require.NoError(t, err) + defer c.Close() + + out := filepath.Join(t.TempDir(), "out.tar") + outW, err := os.Create(out) + require.NoError(t, err) + + _, err = f.Solve(sb.Context(), c, client.SolveOpt{ + FrontendAttrs: map[string]string{ + "context": server.URL + "/.git", + "build-arg:SOURCE_DATE_EPOCH": "context", + }, + Exports: []client.ExportEntry{ + { + Type: client.ExporterOCI, + Output: fixedWriteCloser(outW), + }, + }, + }, nil) + require.NoError(t, err) + + dt, err := os.ReadFile(out) + require.NoError(t, err) + require.Equal(t, commitTime.Unix(), readOCIImageCreated(t, dt).Unix()) +} + +func testSourceDateEpochContextHTTPLastModified(t *testing.T, sb integration.Sandbox) { + integration.SkipOnPlatform(t, "windows") + workers.CheckFeatureCompat(t, sb, workers.FeatureOCIExporter, workers.FeatureSourceDateEpoch) + f := getFrontend(t, sb) + + lastModified := time.Unix(1700000201, 0).UTC() + archiveMaxTime := time.Unix(1700000202, 0).UTC() + resp := &httpserver.Response{ + Etag: identity.NewID(), + LastModified: &lastModified, + Content: makeTarContext(t, + tarContextFile{name: "Dockerfile", data: []byte("FROM scratch\nCOPY foo /\n"), modTime: archiveMaxTime.Add(-time.Hour)}, + tarContextFile{name: "foo", data: []byte("bar"), modTime: archiveMaxTime}, + ), + } + server := httpserver.NewTestServer(map[string]*httpserver.Response{ + "/context.tar": resp, + }) + defer server.Close() + + c, err := client.New(sb.Context(), sb.Address()) + require.NoError(t, err) + defer c.Close() + + out := filepath.Join(t.TempDir(), "out.tar") + outW, err := os.Create(out) + require.NoError(t, err) + + _, err = f.Solve(sb.Context(), c, client.SolveOpt{ + FrontendAttrs: map[string]string{ + "context": server.URL + "/context.tar", + "build-arg:SOURCE_DATE_EPOCH": "context", + }, + Exports: []client.ExportEntry{ + { + Type: client.ExporterOCI, + Output: fixedWriteCloser(outW), + }, + }, + }, nil) + require.NoError(t, err) + + dt, err := os.ReadFile(out) + require.NoError(t, err) + require.Equal(t, lastModified.Unix(), readOCIImageCreated(t, dt).Unix()) +} + +func testSourceDateEpochContextHTTPArchive(t *testing.T, sb integration.Sandbox) { + integration.SkipOnPlatform(t, "windows") + workers.CheckFeatureCompat(t, sb, workers.FeatureOCIExporter, workers.FeatureSourceDateEpoch) + f := getFrontend(t, sb) + + archiveMaxTime := time.Unix(1700000302, 0).UTC() + resp := &httpserver.Response{ + Etag: identity.NewID(), + Content: makeTarContext(t, + tarContextFile{name: "Dockerfile", data: []byte("FROM busybox\nARG SOURCE_DATE_EPOCH\nRUN echo -n \"$SOURCE_DATE_EPOCH\" >/epoch\n"), modTime: archiveMaxTime.Add(-time.Hour)}, + tarContextFile{name: "foo", data: []byte("bar"), modTime: archiveMaxTime}, + ), + } + server := httpserver.NewTestServer(map[string]*httpserver.Response{ + "/context.tar": resp, + }) + defer server.Close() + + c, err := client.New(sb.Context(), sb.Address()) + require.NoError(t, err) + defer c.Close() + + out := filepath.Join(t.TempDir(), "out.tar") + outW, err := os.Create(out) + require.NoError(t, err) + + _, err = f.Solve(sb.Context(), c, client.SolveOpt{ + FrontendAttrs: map[string]string{ + "context": server.URL + "/context.tar", + "build-arg:SOURCE_DATE_EPOCH": "context", + }, Exports: []client.ExportEntry{ { Type: client.ExporterOCI, Attrs: map[string]string{ - "source-date-epoch": "", + "rewrite-timestamp": "true", }, Output: fixedWriteCloser(outW), }, }, }, nil) - require.ErrorContains(t, err, "invalid SOURCE_DATE_EPOCH: not-a-timestamp") + require.NoError(t, err) + + dt, err := os.ReadFile(out) + require.NoError(t, err) + + mfst := readOCIManifest(t, dt) + require.NotEmpty(t, mfst.Layers) + require.Equal(t, fmt.Sprintf("%d", archiveMaxTime.Unix()), mfst.Layers[len(mfst.Layers)-1].Annotations["buildkit/rewritten-timestamp"]) + + layerMap := readOCILayerMap(t, dt, mfst.Layers[len(mfst.Layers)-1]) + require.Equal(t, fmt.Sprintf("%d", archiveMaxTime.Unix()), string(layerMap["epoch"].Data)) + + require.Equal(t, archiveMaxTime.Unix(), readOCIImageCreated(t, dt).Unix()) +} + +func testSourceDateEpochContextLocalUnset(t *testing.T, sb integration.Sandbox) { + integration.SkipOnPlatform(t, "windows") + workers.CheckFeatureCompat(t, sb, workers.FeatureOCIExporter, workers.FeatureSourceDateEpoch) + f := getFrontend(t, sb) + + dockerfile := []byte("FROM scratch\nCOPY Dockerfile /\n") + dir := integration.Tmpdir( + t, + fstest.CreateFile("Dockerfile", dockerfile, 0600), + ) + + c, err := client.New(sb.Context(), sb.Address()) + require.NoError(t, err) + defer c.Close() + + out := filepath.Join(t.TempDir(), "out.tar") + outW, err := os.Create(out) + require.NoError(t, err) + + _, err = f.Solve(sb.Context(), c, client.SolveOpt{ + FrontendAttrs: map[string]string{ + "build-arg:SOURCE_DATE_EPOCH": "context", + }, + LocalMounts: map[string]fsutil.FS{ + dockerui.DefaultLocalNameDockerfile: dir, + dockerui.DefaultLocalNameContext: dir, + }, + Exports: []client.ExportEntry{ + { + Type: client.ExporterOCI, + Attrs: map[string]string{ + "rewrite-timestamp": "true", + }, + Output: fixedWriteCloser(outW), + }, + }, + }, nil) + require.NoError(t, err) + + dt, err := os.ReadFile(out) + require.NoError(t, err) + mfst := readOCIManifest(t, dt) + require.Len(t, mfst.Layers, 1) + require.Empty(t, mfst.Layers[0].Annotations["buildkit/rewritten-timestamp"]) +} + +func testSourceDateEpochStageHTTPArchive(t *testing.T, sb integration.Sandbox) { + integration.SkipOnPlatform(t, "windows") + workers.CheckFeatureCompat(t, sb, workers.FeatureOCIExporter, workers.FeatureSourceDateEpoch) + f := getFrontend(t, sb) + + archiveMaxTime := time.Unix(1700000402, 0).UTC() + server := httpserver.NewTestServer(map[string]*httpserver.Response{ + "/src.tar": { + Etag: identity.NewID(), + Content: makeTarContext(t, + tarContextFile{name: "foo", data: []byte("bar"), modTime: archiveMaxTime}, + ), + }, + }) + defer server.Close() + + dockerfile := fmt.Appendf(nil, ` +ARG SOURCE_DATE_EPOCH=mysource +FROM scratch AS mysource +ADD %s/src.tar / +FROM busybox +ARG SOURCE_DATE_EPOCH +RUN echo -n "$SOURCE_DATE_EPOCH" >/epoch +`, server.URL) + + dir := integration.Tmpdir( + t, + fstest.CreateFile("Dockerfile", dockerfile, 0600), + ) + + c, err := client.New(sb.Context(), sb.Address()) + require.NoError(t, err) + defer c.Close() + + out := filepath.Join(t.TempDir(), "out.tar") + outW, err := os.Create(out) + require.NoError(t, err) + + _, err = f.Solve(sb.Context(), c, client.SolveOpt{ + LocalMounts: map[string]fsutil.FS{ + dockerui.DefaultLocalNameDockerfile: dir, + dockerui.DefaultLocalNameContext: dir, + }, + Exports: []client.ExportEntry{ + { + Type: client.ExporterOCI, + Attrs: map[string]string{ + "rewrite-timestamp": "true", + }, + Output: fixedWriteCloser(outW), + }, + }, + }, nil) + require.NoError(t, err) + + dt, err := os.ReadFile(out) + require.NoError(t, err) + + mfst := readOCIManifest(t, dt) + require.NotEmpty(t, mfst.Layers) + require.Equal(t, fmt.Sprintf("%d", archiveMaxTime.Unix()), mfst.Layers[len(mfst.Layers)-1].Annotations["buildkit/rewritten-timestamp"]) + + layerMap := readOCILayerMap(t, dt, mfst.Layers[len(mfst.Layers)-1]) + require.Equal(t, fmt.Sprintf("%d", archiveMaxTime.Unix()), string(layerMap["epoch"].Data)) + require.Equal(t, archiveMaxTime.Unix(), readOCIImageCreated(t, dt).Unix()) +} + +func testSourceDateEpochStageOverride(t *testing.T, sb integration.Sandbox) { + integration.SkipOnPlatform(t, "windows") + workers.CheckFeatureCompat(t, sb, workers.FeatureOCIExporter, workers.FeatureSourceDateEpoch) + f := getFrontend(t, sb) + + defaultTime := time.Unix(1700000501, 0).UTC() + overrideTime := time.Unix(1700000502, 0).UTC() + server := httpserver.NewTestServer(map[string]*httpserver.Response{ + "/src.tar": { + Etag: identity.NewID(), + Content: makeTarContext(t, + tarContextFile{name: "foo", data: []byte("bar"), modTime: defaultTime}, + ), + }, + }) + defer server.Close() + + dockerfile := fmt.Appendf(nil, ` +ARG SOURCE_DATE_EPOCH=mysource +FROM scratch AS mysource +ADD %s/src.tar / +FROM scratch +COPY Dockerfile . +`, server.URL) + + dir := integration.Tmpdir( + t, + fstest.CreateFile("Dockerfile", dockerfile, 0600), + ) + + c, err := client.New(sb.Context(), sb.Address()) + require.NoError(t, err) + defer c.Close() + + out := filepath.Join(t.TempDir(), "out.tar") + outW, err := os.Create(out) + require.NoError(t, err) + + _, err = f.Solve(sb.Context(), c, client.SolveOpt{ + FrontendAttrs: map[string]string{ + "build-arg:SOURCE_DATE_EPOCH": fmt.Sprintf("%d", overrideTime.Unix()), + }, + LocalMounts: map[string]fsutil.FS{ + dockerui.DefaultLocalNameDockerfile: dir, + dockerui.DefaultLocalNameContext: dir, + }, + Exports: []client.ExportEntry{ + { + Type: client.ExporterOCI, + Attrs: map[string]string{ + "rewrite-timestamp": "true", + }, + Output: fixedWriteCloser(outW), + }, + }, + }, nil) + require.NoError(t, err) + + dt, err := os.ReadFile(out) + require.NoError(t, err) + mfst := readOCIManifest(t, dt) + require.Len(t, mfst.Layers, 1) + require.Equal(t, fmt.Sprintf("%d", overrideTime.Unix()), mfst.Layers[0].Annotations["buildkit/rewritten-timestamp"]) +} + +func testSourceDateEpochStageInvalid(t *testing.T, sb integration.Sandbox) { + integration.SkipOnPlatform(t, "windows") + workers.CheckFeatureCompat(t, sb, workers.FeatureOCIExporter, workers.FeatureSourceDateEpoch) + f := getFrontend(t, sb) + + dockerfile := []byte(` +ARG SOURCE_DATE_EPOCH=mysource +FROM scratch AS mysource +COPY Dockerfile /Dockerfile +FROM scratch +`) + + dir := integration.Tmpdir( + t, + fstest.CreateFile("Dockerfile", dockerfile, 0600), + ) + + c, err := client.New(sb.Context(), sb.Address()) + require.NoError(t, err) + defer c.Close() + + out := filepath.Join(t.TempDir(), "out.tar") + outW, err := os.Create(out) + require.NoError(t, err) + + _, err = f.Solve(sb.Context(), c, client.SolveOpt{ + LocalMounts: map[string]fsutil.FS{ + dockerui.DefaultLocalNameDockerfile: dir, + dockerui.DefaultLocalNameContext: dir, + }, + Exports: []client.ExportEntry{ + { + Type: client.ExporterOCI, + Output: fixedWriteCloser(outW), + }, + }, + }, nil) + require.ErrorContains(t, err, "SOURCE_DATE_EPOCH stage does not meet source-only requirements") +} + +func testSourceDateEpochNamedContextHTTPLastModified(t *testing.T, sb integration.Sandbox) { + integration.SkipOnPlatform(t, "windows") + workers.CheckFeatureCompat(t, sb, workers.FeatureOCIExporter, workers.FeatureSourceDateEpoch) + f := getFrontend(t, sb) + + lastModified := time.Unix(1700000601, 0).UTC() + server := httpserver.NewTestServer(map[string]*httpserver.Response{ + "/src.tar": { + Etag: identity.NewID(), + LastModified: &lastModified, + Content: makeTarContext(t, + tarContextFile{name: "foo", data: []byte("bar"), modTime: lastModified.Add(time.Hour)}, + ), + }, + }) + defer server.Close() + + dockerfile := []byte(` +ARG SOURCE_DATE_EPOCH=mysource +FROM scratch +COPY Dockerfile . +`) + + dir := integration.Tmpdir( + t, + fstest.CreateFile("Dockerfile", dockerfile, 0600), + ) + + c, err := client.New(sb.Context(), sb.Address()) + require.NoError(t, err) + defer c.Close() + + out := filepath.Join(t.TempDir(), "out.tar") + outW, err := os.Create(out) + require.NoError(t, err) + + _, err = f.Solve(sb.Context(), c, client.SolveOpt{ + FrontendAttrs: map[string]string{ + "context:mysource": server.URL + "/src.tar", + }, + LocalMounts: map[string]fsutil.FS{ + dockerui.DefaultLocalNameDockerfile: dir, + dockerui.DefaultLocalNameContext: dir, + }, + Exports: []client.ExportEntry{ + { + Type: client.ExporterOCI, + Attrs: map[string]string{ + "rewrite-timestamp": "true", + }, + Output: fixedWriteCloser(outW), + }, + }, + }, nil) + require.NoError(t, err) + + dt, err := os.ReadFile(out) + require.NoError(t, err) + mfst := readOCIManifest(t, dt) + require.Len(t, mfst.Layers, 1) + require.Equal(t, fmt.Sprintf("%d", lastModified.Unix()), mfst.Layers[0].Annotations["buildkit/rewritten-timestamp"]) + require.Equal(t, lastModified.Unix(), readOCIImageCreated(t, dt).Unix()) +} + +func testSourceDateEpochNamedContextHTTPArchive(t *testing.T, sb integration.Sandbox) { + integration.SkipOnPlatform(t, "windows") + workers.CheckFeatureCompat(t, sb, workers.FeatureOCIExporter, workers.FeatureSourceDateEpoch) + f := getFrontend(t, sb) + + archiveMaxTime := time.Unix(1700000602, 0).UTC() + server := httpserver.NewTestServer(map[string]*httpserver.Response{ + "/src.tar": { + Etag: identity.NewID(), + Content: makeTarContext(t, + tarContextFile{name: "foo", data: []byte("bar"), modTime: archiveMaxTime}, + ), + }, + }) + defer server.Close() + + dockerfile := []byte(` +ARG SOURCE_DATE_EPOCH=mysource +FROM busybox +ARG SOURCE_DATE_EPOCH +RUN echo -n "$SOURCE_DATE_EPOCH" >/epoch +`) + + dir := integration.Tmpdir( + t, + fstest.CreateFile("Dockerfile", dockerfile, 0600), + ) + + c, err := client.New(sb.Context(), sb.Address()) + require.NoError(t, err) + defer c.Close() + + out := filepath.Join(t.TempDir(), "out.tar") + outW, err := os.Create(out) + require.NoError(t, err) + + _, err = f.Solve(sb.Context(), c, client.SolveOpt{ + FrontendAttrs: map[string]string{ + "context:mysource": server.URL + "/src.tar", + }, + LocalMounts: map[string]fsutil.FS{ + dockerui.DefaultLocalNameDockerfile: dir, + dockerui.DefaultLocalNameContext: dir, + }, + Exports: []client.ExportEntry{ + { + Type: client.ExporterOCI, + Attrs: map[string]string{ + "rewrite-timestamp": "true", + }, + Output: fixedWriteCloser(outW), + }, + }, + }, nil) + require.NoError(t, err) + + dt, err := os.ReadFile(out) + require.NoError(t, err) + + mfst := readOCIManifest(t, dt) + require.NotEmpty(t, mfst.Layers) + require.Equal(t, fmt.Sprintf("%d", archiveMaxTime.Unix()), mfst.Layers[len(mfst.Layers)-1].Annotations["buildkit/rewritten-timestamp"]) + + layerMap := readOCILayerMap(t, dt, mfst.Layers[len(mfst.Layers)-1]) + require.Equal(t, fmt.Sprintf("%d", archiveMaxTime.Unix()), string(layerMap["epoch"].Data)) + require.Equal(t, archiveMaxTime.Unix(), readOCIImageCreated(t, dt).Unix()) } func testSBOMScannerImage(t *testing.T, sb integration.Sandbox) { diff --git a/frontend/dockerui/attr.go b/frontend/dockerui/attr.go index ffb618cb23a5..a6f717d04d5a 100644 --- a/frontend/dockerui/attr.go +++ b/frontend/dockerui/attr.go @@ -4,7 +4,6 @@ import ( "net" "strconv" "strings" - "time" "github.com/containerd/platforms" "github.com/docker/go-units" @@ -113,18 +112,6 @@ func parseNetMode(v string) (pb.NetMode, error) { } } -func parseSourceDateEpoch(v string) (*time.Time, error) { - if v == "" { - return nil, nil - } - sde, err := strconv.ParseInt(v, 10, 64) - if err != nil { - return nil, errors.Wrapf(err, "invalid SOURCE_DATE_EPOCH: %s", v) - } - tm := time.Unix(sde, 0).UTC() - return &tm, nil -} - func parseLocalSessionIDs(opt map[string]string) map[string]string { m := map[string]string{} for k, v := range opt { diff --git a/frontend/dockerui/config.go b/frontend/dockerui/config.go index 619b8345730f..27ccc6f16b41 100644 --- a/frontend/dockerui/config.go +++ b/frontend/dockerui/config.go @@ -8,7 +8,6 @@ import ( "strconv" "strings" "sync" - "time" "github.com/containerd/platforms" "github.com/distribution/reference" @@ -51,14 +50,12 @@ const ( keyHostnameArg = "build-arg:BUILDKIT_SANDBOX_HOSTNAME" keyDockerfileLintArg = "build-arg:BUILDKIT_DOCKERFILE_CHECK" keyContextKeepGitDirArg = "build-arg:BUILDKIT_CONTEXT_KEEP_GIT_DIR" - keySourceDateEpoch = "build-arg:SOURCE_DATE_EPOCH" ) type Config struct { BuildArgs map[string]string CacheIDNamespace string CgroupParent string - Epoch *time.Time ExtraHosts []llb.HostIP Hostname string ImageResolveMode llb.ResolveMode @@ -147,6 +144,10 @@ func (bc *Client) BuildOpts() client.BuildOpts { return bc.bopts } +func (bc *Client) GatewayClient() client.Client { + return bc.client +} + func (bc *Client) init() error { opts := bc.bopts.Opts @@ -251,12 +252,6 @@ func (bc *Client) init() error { } bc.CacheImports = cacheImports - epoch, err := parseSourceDateEpoch(opts[keySourceDateEpoch]) - if err != nil { - return err - } - bc.Epoch = epoch - attests, err := attestations.Parse(opts) if err != nil { return err diff --git a/frontend/dockerui/context.go b/frontend/dockerui/context.go index 3bd5094f6118..a250b31bd66d 100644 --- a/frontend/dockerui/context.go +++ b/frontend/dockerui/context.go @@ -4,15 +4,23 @@ import ( "archive/tar" "bytes" "context" + "io" + "maps" "path/filepath" "regexp" "slices" "strconv" + "strings" + "time" "github.com/moby/buildkit/client/llb" + "github.com/moby/buildkit/client/llb/sourceresolver" "github.com/moby/buildkit/frontend/dockerfile/dfgitutil" "github.com/moby/buildkit/frontend/gateway/client" gwpb "github.com/moby/buildkit/frontend/gateway/pb" + "github.com/moby/buildkit/solver/pb" + "github.com/moby/buildkit/util/gitutil/gitobject" + archivecompression "github.com/moby/go-archive/compression" "github.com/pkg/errors" ) @@ -36,10 +44,14 @@ var httpPrefix = regexp.MustCompile(`^https?://`) type buildContext struct { context *llb.State // set if not local dockerfile *llb.State // override remoteContext if set + contextRef client.Reference contextLocalName string dockerfileLocalName string filename string forceLocalDockerfile bool + sourceOp *pb.SourceOp + httpContextIsArchive bool + httpContextFilename string } func (bc *Client) marshalOpts() []llb.ConstraintsOpt { @@ -75,16 +87,25 @@ func (bc *Client) initContext(ctx context.Context) (*buildContext, error) { keepGit = &v } var extraGitOpts []llb.GitOption - if opts[keySourceDateEpoch] != "" { + if opts[buildArgPrefix+"SOURCE_DATE_EPOCH"] != "" { extraGitOpts = append(extraGitOpts, llb.GitMTimeCommit()) } if st, ok, err := DetectGitContext(opts[localNameContext], keepGit, extraGitOpts...); ok { if err != nil { return nil, err } + sourceOp, err := sourceOpFromState(ctx, st, bc.marshalOpts()...) + if err != nil { + return nil, errors.Wrapf(err, "failed to derive git source op") + } bctx.context = st bctx.dockerfile = st + bctx.sourceOp = sourceOp } else if st, filename, ok := DetectHTTPContext(opts[localNameContext]); ok { + sourceOp, err := sourceOpFromState(ctx, st, bc.marshalOpts()...) + if err != nil { + return nil, errors.Wrapf(err, "failed to derive http source op") + } def, err := st.Marshal(ctx, bc.marshalOpts()...) if err != nil { return nil, errors.Wrapf(err, "failed to marshal httpcontext") @@ -115,11 +136,15 @@ func (bc *Client) initContext(ctx context.Context) (*buildContext, error) { AttemptUnpack: true, })) bctx.context = &bc + bctx.httpContextIsArchive = true } else { bctx.filename = filename bctx.context = st } + bctx.contextRef = ref bctx.dockerfile = bctx.context + bctx.sourceOp = sourceOp + bctx.httpContextFilename = filename } else if (&gwcaps).Supports(gwpb.CapFrontendInputs) == nil { inputs, err := bc.client.Inputs(ctx) if err != nil { @@ -148,6 +173,123 @@ func (bc *Client) initContext(ctx context.Context) (*buildContext, error) { return bctx, nil } +func (bc *Client) ResolveMainContextSourceDateEpoch(ctx context.Context) (*time.Time, error) { + bctx, err := bc.buildContext(ctx) + if err != nil { + return nil, err + } + if bctx.sourceOp == nil { + return nil, nil + } + + opt := sourceresolver.Opt{ + LogName: "[internal] resolve main build context metadata", + } + if strings.HasPrefix(bctx.sourceOp.Identifier, "git://") { + opt.GitOpt = &sourceresolver.ResolveGitOpt{ReturnObject: true} + } + md, err := bc.client.ResolveSourceMetadata(ctx, cloneSourceOp(bctx.sourceOp), opt) + if err != nil { + return nil, err + } + if md.Git != nil && len(md.Git.CommitObject) > 0 { + obj, err := gitobject.Parse(md.Git.CommitObject) + if err != nil { + return nil, err + } + commit, err := obj.ToCommit() + if err != nil { + return nil, err + } + return commit.Committer.When, nil + } + if md.HTTP != nil { + if md.HTTP.LastModified != nil { + return md.HTTP.LastModified, nil + } + if bctx.httpContextIsArchive { + return archiveMaxTimeFromHTTPArchive(ctx, bctx) + } + } + return nil, nil +} + +func archiveMaxTimeFromHTTPArchive(ctx context.Context, bctx *buildContext) (*time.Time, error) { + if bctx.contextRef == nil || bctx.httpContextFilename == "" { + return nil, nil + } + dt, err := bctx.contextRef.ReadFile(ctx, client.ReadRequest{ + Filename: bctx.httpContextFilename, + }) + if err != nil { + return nil, err + } + rc, err := archivecompression.DecompressStream(bytes.NewReader(dt)) + if err != nil { + return nil, err + } + defer rc.Close() + + tr := tar.NewReader(rc) + var maxTime *time.Time + for { + hdr, err := tr.Next() + if err != nil { + if errors.Is(err, io.EOF) { + return maxTime, nil + } + return nil, err + } + if !hdr.FileInfo().Mode().IsRegular() { + continue + } + tm := hdr.ModTime.UTC() + if maxTime == nil || tm.After(*maxTime) { + maxTime = &tm + } + } +} + +func cloneSourceOp(op *pb.SourceOp) *pb.SourceOp { + if op == nil { + return nil + } + return &pb.SourceOp{ + Identifier: op.Identifier, + Attrs: maps.Clone(op.Attrs), + } +} + +func sourceOpFromState(ctx context.Context, st *llb.State, opts ...llb.ConstraintsOpt) (*pb.SourceOp, error) { + if st == nil { + return nil, nil + } + def, err := st.Marshal(ctx, opts...) + if err != nil { + return nil, err + } + dt := def.ToPB().Def + var src *pb.SourceOp + for _, d := range dt { + var op pb.Op + if err := op.Unmarshal(d); err != nil { + return nil, err + } + opSrc := op.GetSource() + if opSrc == nil { + continue + } + if src != nil { + return nil, errors.New("state marshaled to multiple source ops") + } + src = opSrc + } + if src == nil { + return nil, errors.New("state did not marshal to a source op") + } + return cloneSourceOp(src), nil +} + func DetectGitContext(ref string, keepGit *bool, opts ...llb.GitOption) (*llb.State, bool, error) { g, isGit, err := dfgitutil.ParseGitRef(ref) if err != nil { diff --git a/frontend/dockerui/namedcontext.go b/frontend/dockerui/namedcontext.go index ccda59c0ec09..06e396e05ef7 100644 --- a/frontend/dockerui/namedcontext.go +++ b/frontend/dockerui/namedcontext.go @@ -151,14 +151,14 @@ func (nc *NamedContext) load(ctx context.Context, count int) (*llb.State, *docke return st, nil, nil case "http", "https": st, ok, err := DetectGitContext(nc.input, nil) - if !ok { - httpst := llb.HTTP(nc.input, llb.WithCustomName("[context "+nc.nameWithPlatform+"] "+nc.input)) - st = &httpst - } - if err != nil { - return nil, nil, err + if ok { + if err != nil { + return nil, nil, err + } + return st, nil, nil } - return st, nil, nil + httpst := llb.HTTP(nc.input, llb.Filename("context"), llb.WithCustomName("[context "+nc.nameWithPlatform+"] "+nc.input)) + return &httpst, nil, nil case "oci-layout": refSpec := strings.TrimPrefix(vv[1], "//") ref, err := reference.Parse(refSpec) diff --git a/source/git/source.go b/source/git/source.go index c3a20ca60dc0..6b2042b94d8e 100644 --- a/source/git/source.go +++ b/source/git/source.go @@ -287,13 +287,13 @@ func (gs *Source) ResolveMetadata(ctx context.Context, id *GitIdentifier, sm *se return nil, err } + gsh.cacheCommit = md.Checksum + gsh.sha256 = len(md.Checksum) == 64 + if !opt.ReturnObject && id.VerifySignature == nil { return md, nil } - gsh.cacheCommit = md.Checksum - gsh.sha256 = len(md.Checksum) == 64 - if err := gsh.addGitObjectsToMetadata(ctx, jobCtx, md); err != nil { return nil, err }