diff --git a/frontend/dockerfile/builder/build.go b/frontend/dockerfile/builder/build.go index fe63cc914c17..d9c3dec8922a 100644 --- a/frontend/dockerfile/builder/build.go +++ b/frontend/dockerfile/builder/build.go @@ -27,6 +27,7 @@ import ( "github.com/moby/buildkit/solver/errdefs" "github.com/moby/buildkit/solver/pb" binfotypes "github.com/moby/buildkit/util/buildinfo/types" + "github.com/moby/buildkit/util/gitutil" ocispecs "github.com/opencontainers/image-spec/specs-go/v1" "github.com/pkg/errors" "golang.org/x/sync/errgroup" @@ -69,7 +70,6 @@ const ( ) var httpPrefix = regexp.MustCompile(`^https?://`) -var gitURLPathWithFragmentSuffix = regexp.MustCompile(`\.git(?:#.+)?$`) func Build(ctx context.Context, c client.Client) (*client.Result, error) { opts := c.BuildOpts().Opts @@ -180,7 +180,11 @@ func Build(ctx context.Context, c client.Client) (*client.Result, error) { var buildContext *llb.State isNotLocalContext := false - if st, ok := detectGitContext(opts[localNameContext], opts[keyContextKeepGitDirArg]); ok { + keepGit := false + if v, err := strconv.ParseBool(opts[keyContextKeepGitDirArg]); err == nil { + keepGit = v + } + if st, ok := detectGitContext(opts[localNameContext], keepGit); ok { if !forceLocalDockerfile { src = *st } @@ -598,40 +602,21 @@ func filter(opt map[string]string, key string) map[string]string { return m } -func detectGitContext(ref, gitContext string) (*llb.State, bool) { - found := false - if httpPrefix.MatchString(ref) && gitURLPathWithFragmentSuffix.MatchString(ref) { - found = true - } - - keepGit := false - if gitContext != "" { - if v, err := strconv.ParseBool(gitContext); err == nil { - keepGit = v - } - } - - for _, prefix := range []string{"git://", "github.com/", "git@"} { - if strings.HasPrefix(ref, prefix) { - found = true - break - } - } - if !found { +func detectGitContext(ref string, keepGit bool) (*llb.State, bool) { + g, err := gitutil.ParseGitRef(ref) + if err != nil { return nil, false } - - parts := strings.SplitN(ref, "#", 2) - branch := "" - if len(parts) > 1 { - branch = parts[1] + if g.SubDir != "" { + // "https://github.com/docker/docker.git#:dir" form is not implemented yet + return nil, false } gitOpts := []llb.GitOption{dockerfile2llb.WithInternalName("load git source " + ref)} if keepGit { gitOpts = append(gitOpts, llb.KeepGitDir()) } - st := llb.Git(parts[0], branch, gitOpts...) + st := llb.Git(g.Remote, g.Commit, gitOpts...) return &st, true } @@ -837,13 +822,13 @@ func contextByName(ctx context.Context, c client.Client, name string, platform * st := llb.Image(ref, imgOpt...) return &st, nil, nil, nil case "git": - st, ok := detectGitContext(v, "1") + st, ok := detectGitContext(v, true) if !ok { return nil, nil, nil, errors.Errorf("invalid git context %s", v) } return st, nil, nil, nil case "http", "https": - st, ok := detectGitContext(v, "1") + st, ok := detectGitContext(v, true) if !ok { httpst := llb.HTTP(v, llb.WithCustomName("[context "+name+"] "+v)) st = &httpst diff --git a/frontend/dockerfile/dockerfile2llb/convert.go b/frontend/dockerfile/dockerfile2llb/convert.go index 3f934e28d6cc..e18a849ee0b9 100644 --- a/frontend/dockerfile/dockerfile2llb/convert.go +++ b/frontend/dockerfile/dockerfile2llb/convert.go @@ -26,6 +26,7 @@ import ( "github.com/moby/buildkit/solver/pb" "github.com/moby/buildkit/util/apicaps" binfotypes "github.com/moby/buildkit/util/buildinfo/types" + "github.com/moby/buildkit/util/gitutil" "github.com/moby/buildkit/util/suggest" "github.com/moby/buildkit/util/system" "github.com/moby/sys/signal" @@ -1001,7 +1002,28 @@ func dispatchCopy(d *dispatchState, cfg copyConfig) error { for _, src := range cfg.params.SourcePaths { commitMessage.WriteString(" " + src) - if strings.HasPrefix(src, "http://") || strings.HasPrefix(src, "https://") { + gitRef, gitRefErr := gitutil.ParseGitRef(src) + if gitRefErr == nil && !gitRef.IndistinguishableFromLocal { + if !cfg.isAddCommand { + return errors.New("source can't be a git ref for COPY") + } + if gitRef.UnencryptedTCP { + return errors.New("unencrypted TCP is not supported for ADD ") + } + if gitRef.SubDir != "" { + return errors.New("currently ADD does not support specifying the work dir") + } + st := llb.Git(gitRef.Remote, gitRef.Commit, llb.KeepGitDir()) + opts := append([]llb.CopyOption{&llb.CopyInfo{ + Mode: mode, + CreateDestPath: true, + }}, copyOpt...) + if a == nil { + a = llb.Copy(st, "/", dest, opts...) + } else { + a = a.Copy(st, "/", dest, opts...) + } + } else if strings.HasPrefix(src, "http://") || strings.HasPrefix(src, "https://") { if !cfg.isAddCommand { return errors.New("source can't be a URL for COPY") } diff --git a/frontend/dockerfile/dockerfile_test.go b/frontend/dockerfile/dockerfile_test.go index 9e6ae4704a4f..b5a33dbd62d9 100644 --- a/frontend/dockerfile/dockerfile_test.go +++ b/frontend/dockerfile/dockerfile_test.go @@ -130,6 +130,7 @@ var allTests = integration.TestFuncs( testCopyVarSubstitution, testCopyWildcards, testCopyRelative, + testAddGit, testAddURLChmod, testTarContext, testTarContextExternalDockerfile, @@ -3463,6 +3464,51 @@ COPY --from=build /dest /dest require.Equal(t, []byte("0644\n0755\n0413\n"), dt) } +func testAddGit(t *testing.T, sb integration.Sandbox) { + f := getFrontend(t, sb) + + dockerfile := []byte(` +FROM alpine + +# Basic case +ADD https://github.com/moby/buildkit.git#v0.4.0 /buildkit +RUN cd /buildkit && \ + [ -f "README.md" ] && \ + apk add git && \ + [ -z "$(git status -s)" ] && \ + [ "$(git rev-parse HEAD)" = "c35410878ab9070498c66f6c67d3e8bc3b92241f" ] + +# Complicated case +ARG REPO="git@github.com/moby/buildkit.git" +ARG TAG="v0.5.0" +ADD --chown=4242:8484 ${REPO}#${TAG} /buildkit-chowned +USER 4242 +RUN cd /buildkit-chowned && \ + [ "$(stat -c %u README.md)" = "4242" ] && \ + [ "$(stat -c %g README.md)" = "8484" ] && \ + [ -z "$(git status -s)" ] && \ + [ "$(git rev-parse HEAD)" = "8c0fa8fdec187d8f259a349d2da16dc2dc5f144a" ] +`) + + dir, err := tmpdir( + fstest.CreateFile("Dockerfile", dockerfile, 0600), + ) + require.NoError(t, err) + defer os.RemoveAll(dir) + + c, err := client.New(sb.Context(), sb.Address()) + require.NoError(t, err) + defer c.Close() + + _, err = f.Solve(sb.Context(), c, client.SolveOpt{ + LocalDirs: map[string]string{ + builder.DefaultLocalNameDockerfile: dir, + builder.DefaultLocalNameContext: dir, + }, + }, nil) + require.NoError(t, err) +} + func testDockerfileFromGit(t *testing.T, sb integration.Sandbox) { f := getFrontend(t, sb) diff --git a/frontend/dockerfile/docs/syntax.md b/frontend/dockerfile/docs/syntax.md index 33ff1cfbb9a6..27c37bee5853 100644 --- a/frontend/dockerfile/docs/syntax.md +++ b/frontend/dockerfile/docs/syntax.md @@ -25,6 +25,39 @@ incrementing the major component of a version and you may want to pin the image change in between releases on labs channel, the old versions are guaranteed to be backward compatible. +## Adding a git repository `ADD ` + + +To use this instruction set Dockerfile version to `upstream-master`: + +```dockerfile +# syntax=docker/dockerfile:upstream-master` +``` + +This instruction allows you to add a git repository to an image directly, without using the `git` command inside the image. + +```dockerfile +# syntax=docker/dockerfile:upstream-master +FROM alpine +ADD https://github.com/moby/buildkit.git#v0.10.1 /buildkit +``` + + +To add a private repo via SSH: +```dockerfile +# syntax = docker/dockerfile:upstream-master +FROM alpine +ADD git@git.example.com:foo/bar.git /bar +``` + +```console +$ docker build --ssh default +``` + +```console +$ buildctl build --frontend=dockerfile.v0 --local context=. --local dockerfile=. --ssh default +``` + ## Linked copies `COPY --link`, `ADD --link` To use this flag set Dockerfile version to at least `1.4`. diff --git a/util/gitutil/git_ref.go b/util/gitutil/git_ref.go new file mode 100644 index 000000000000..6a12b38d58af --- /dev/null +++ b/util/gitutil/git_ref.go @@ -0,0 +1,96 @@ +package gitutil + +import ( + "regexp" + "strings" + + "github.com/containerd/containerd/errdefs" +) + +// GitRef represents a git ref. +// +// Examples: +// - "https://github.com/foo/bar.git#baz/qux:quux/quuz" is parsed into: +// {Remote: "https://github.com/foo/bar.git", ShortName: "bar", Commit:"baz/qux", SubDir: "quux/quuz"}. +type GitRef struct { + // Remote is the remote repository path. + Remote string + + // ShortName is the directory name of the repo. + // e.g., "bar" for "https://github.com/foo/bar.git" + ShortName string + + // Commit is a commit hash, a tag, or branch name. + // Commit is optional. + Commit string + + // SubDir is a directory path inside the repo. + // SubDir is optional. + SubDir string + + // IndistinguishableFromLocal is true for a ref that is indistinguishable from a local file path, + // e.g., "github.com/foo/bar". + // + // Deprecated. Use a distinguishable form such as "https://github.com/foo/bar.git". + // + // The dockerfile frontend still accepts this form only for build contexts. + IndistinguishableFromLocal bool + + // Unencrypted TCP is true for a ref that needs an unencrypted TCP connection, + // e.g., "git://..." and "http://..." . + // + // Deprecated. Use an encrypted TCP connection such as "git@github.com/foo/bar.git" or "https://github.com/foo/bar.git". + // + // The dockerfile frontend still accepts this form only for build contexts. + UnencryptedTCP bool +} + +var httpPrefix = regexp.MustCompile(`^https?://`) +var gitURLPathWithFragmentSuffix = regexp.MustCompile(`\.git(?:#.+)?$`) + +func ParseGitRef(ref string) (*GitRef, error) { + res := &GitRef{} + var valid bool + + if httpPrefix.MatchString(ref) && gitURLPathWithFragmentSuffix.MatchString(ref) { + valid = true + if strings.HasPrefix(ref, "http://") { + res.UnencryptedTCP = true + } + } else { + if strings.HasPrefix(ref, "git://") { + valid = true + res.UnencryptedTCP = true + } + if strings.HasPrefix(ref, "github.com/") { + valid = true + res.IndistinguishableFromLocal = true + } + if strings.HasPrefix(ref, "git@") { + valid = true + } + } + if !valid { + return res, errdefs.ErrInvalidArgument + } + + refSplitBySharp := strings.SplitN(ref, "#", 2) + res.Remote = refSplitBySharp[0] + if len(res.Remote) == 0 { + return res, errdefs.ErrInvalidArgument + } + + if len(refSplitBySharp) > 1 { + refSplitBySharpSplitByColon := strings.SplitN(refSplitBySharp[1], ":", 2) + res.Commit = refSplitBySharpSplitByColon[0] + if len(res.Commit) == 0 { + return res, errdefs.ErrInvalidArgument + } + if len(refSplitBySharpSplitByColon) > 1 { + res.SubDir = refSplitBySharpSplitByColon[1] + } + } + repoSplitBySlash := strings.Split(res.Remote, "/") + res.ShortName = strings.TrimSuffix(repoSplitBySlash[len(repoSplitBySlash)-1], ".git") + return res, nil +} diff --git a/util/gitutil/git_ref_test.go b/util/gitutil/git_ref_test.go new file mode 100644 index 000000000000..2c096cbe56fd --- /dev/null +++ b/util/gitutil/git_ref_test.go @@ -0,0 +1,77 @@ +package gitutil + +import ( + "reflect" + "testing" +) + +func TestParseGitRef(t *testing.T) { + cases := map[string]*GitRef{ + "https://example.com/": nil, + "https://example.com/foo": nil, + "https://example.com/foo.git": { + Remote: "https://example.com/foo.git", + ShortName: "foo", + }, + "https://example.com/foo.git#deadbeef": { + Remote: "https://example.com/foo.git", + ShortName: "foo", + Commit: "deadbeef", + }, + "https://example.com/foo.git#release/1.2": { + Remote: "https://example.com/foo.git", + ShortName: "foo", + Commit: "release/1.2", + }, + "https://example.com/foo.git/": nil, + "https://example.com/foo.git.bar": nil, + "git://example.com/foo": { + Remote: "git://example.com/foo", + ShortName: "foo", + UnencryptedTCP: true, + }, + "github.com/moby/buildkit": { + Remote: "github.com/moby/buildkit", ShortName: "buildkit", + IndistinguishableFromLocal: true, + }, + "https://github.com/moby/buildkit": nil, + "https://github.com/moby/buildkit.git": { + Remote: "https://github.com/moby/buildkit.git", + ShortName: "buildkit", + }, + "git@github.com:moby/buildkit": { + Remote: "git@github.com:moby/buildkit", + ShortName: "buildkit", + }, + "git@github.com:moby/buildkit.git": { + Remote: "git@github.com:moby/buildkit.git", + ShortName: "buildkit", + }, + "git@bitbucket.org:atlassianlabs/atlassian-docker.git": { + Remote: "git@bitbucket.org:atlassianlabs/atlassian-docker.git", + ShortName: "atlassian-docker", + }, + "https://github.com/foo/bar.git#baz/qux:quux/quuz": { + Remote: "https://github.com/foo/bar.git", + ShortName: "bar", + Commit: "baz/qux", + SubDir: "quux/quuz", + }, + "http://github.com/docker/docker.git:#branch": nil, + } + for ref, expected := range cases { + got, err := ParseGitRef(ref) + if expected == nil { + if err == nil { + t.Errorf("expected an error for ParseGitRef(%q)", ref) + } + } else { + if err != nil { + t.Errorf("got an unexpected error: ParseGitRef(%q): %v", ref, err) + } + if !reflect.DeepEqual(got, expected) { + t.Errorf("expected ParseGitRef(%q) to return %#v, got %#v", ref, expected, got) + } + } + } +}