dockerfile: implement ADD <git ref>

e.g.,
```dockerfile
FROM alpine
ADD https://github.com/moby/buildkit.git#v0.10.1 /buildkit
```

Close issue 775

Signed-off-by: Akihiro Suda <akihiro.suda.cz@hco.ntt.co.jp>

frontend/dockerfile/builder/build.go

@@ -27,6 +27,7 @@ import (
 	"github.com/moby/buildkit/solver/errdefs"
 	"github.com/moby/buildkit/solver/pb"
 	binfotypes "github.com/moby/buildkit/util/buildinfo/types"
+	"github.com/moby/buildkit/util/gitutil"
 	ocispecs "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/pkg/errors"
 	"golang.org/x/sync/errgroup"
@@ -69,7 +70,6 @@ const (
 )
 
 var httpPrefix = regexp.MustCompile(`^https?://`)
-var gitURLPathWithFragmentSuffix = regexp.MustCompile(`\.git(?:#.+)?$`)
 
 func Build(ctx context.Context, c client.Client) (*client.Result, error) {
 	opts := c.BuildOpts().Opts
@@ -180,7 +180,11 @@ func Build(ctx context.Context, c client.Client) (*client.Result, error) {
 
 	var buildContext *llb.State
 	isNotLocalContext := false
-	if st, ok := detectGitContext(opts[localNameContext], opts[keyContextKeepGitDirArg]); ok {
+	keepGit := false
+	if v, err := strconv.ParseBool(opts[keyContextKeepGitDirArg]); err == nil {
+		keepGit = v
+	}
+	if st, ok := detectGitContext(opts[localNameContext], keepGit); ok {
 		if !forceLocalDockerfile {
 			src = *st
 		}
@@ -598,40 +602,17 @@ func filter(opt map[string]string, key string) map[string]string {
 	return m
 }
 
-func detectGitContext(ref, gitContext string) (*llb.State, bool) {
-	found := false
-	if httpPrefix.MatchString(ref) && gitURLPathWithFragmentSuffix.MatchString(ref) {
-		found = true
-	}
-
-	keepGit := false
-	if gitContext != "" {
-		if v, err := strconv.ParseBool(gitContext); err == nil {
-			keepGit = v
-		}
-	}
-
-	for _, prefix := range []string{"git://", "github.com/", "git@"} {
-		if strings.HasPrefix(ref, prefix) {
-			found = true
-			break
-		}
-	}
-	if !found {
+func detectGitContext(ref string, keepGit bool) (*llb.State, bool) {
+	g, err := gitutil.ParseGitRef(ref)
+	if err != nil {
 		return nil, false
 	}
-
-	parts := strings.SplitN(ref, "#", 2)
-	branch := ""
-	if len(parts) > 1 {
-		branch = parts[1]
-	}
 	gitOpts := []llb.GitOption{dockerfile2llb.WithInternalName("load git source " + ref)}
 	if keepGit {
 		gitOpts = append(gitOpts, llb.KeepGitDir())
 	}
 
-	st := llb.Git(parts[0], branch, gitOpts...)
+	st := llb.Git(g.Repo, g.Commit, gitOpts...)
 	return &st, true
 }
 
@@ -837,13 +818,13 @@ func contextByName(ctx context.Context, c client.Client, name string, platform *
 		st := llb.Image(ref, imgOpt...)
 		return &st, nil, nil, nil
 	case "git":
-		st, ok := detectGitContext(v, "1")
+		st, ok := detectGitContext(v, true)
 		if !ok {
 			return nil, nil, nil, errors.Errorf("invalid git context %s", v)
 		}
 		return st, nil, nil, nil
 	case "http", "https":
-		st, ok := detectGitContext(v, "1")
+		st, ok := detectGitContext(v, true)
 		if !ok {
 			httpst := llb.HTTP(v, llb.WithCustomName("[context "+name+"] "+v))
 			st = &httpst

frontend/dockerfile/dockerfile2llb/convert.go

@@ -26,6 +26,7 @@ import (
 	"github.com/moby/buildkit/solver/pb"
 	"github.com/moby/buildkit/util/apicaps"
 	binfotypes "github.com/moby/buildkit/util/buildinfo/types"
+	"github.com/moby/buildkit/util/gitutil"
 	"github.com/moby/buildkit/util/suggest"
 	"github.com/moby/buildkit/util/system"
 	"github.com/moby/sys/signal"
@@ -1001,7 +1002,25 @@ func dispatchCopy(d *dispatchState, cfg copyConfig) error {
 
 	for _, src := range cfg.params.SourcePaths {
 		commitMessage.WriteString(" " + src)
-		if strings.HasPrefix(src, "http://") || strings.HasPrefix(src, "https://") {
+		if gitutil.IsGitRef(src) {
+			if !cfg.isAddCommand {
+				return errors.New("source can't be a git ref for COPY")
+			}
+			g, err := gitutil.ParseGitRef(src)
+			if err != nil {
+				return errors.Wrapf(err, "failed to parse a git ref %q", src)
+			}
+			st := llb.Git(g.Repo, g.Commit, llb.KeepGitDir())
+			opts := append([]llb.CopyOption{&llb.CopyInfo{
+				Mode:           mode,
+				CreateDestPath: true,
+			}}, copyOpt...)
+			if a == nil {
+				a = llb.Copy(st, "/", dest, opts...)
+			} else {
+				a = a.Copy(st, "/", dest, opts...)
+			}
+		} else if strings.HasPrefix(src, "http://") || strings.HasPrefix(src, "https://") {
 			if !cfg.isAddCommand {
 				return errors.New("source can't be a URL for COPY")
 			}

frontend/dockerfile/dockerfile_test.go

@@ -130,6 +130,7 @@ var allTests = integration.TestFuncs(
 	testCopyVarSubstitution,
 	testCopyWildcards,
 	testCopyRelative,
+	testAddGit,
 	testAddURLChmod,
 	testTarContext,
 	testTarContextExternalDockerfile,
@@ -3463,6 +3464,51 @@ COPY --from=build /dest /dest
 	require.Equal(t, []byte("0644\n0755\n0413\n"), dt)
 }
 
+func testAddGit(t *testing.T, sb integration.Sandbox) {
+	f := getFrontend(t, sb)
+
+	dockerfile := []byte(`
+FROM alpine
+
+# Basic case
+ADD https://github.com/moby/buildkit.git#v0.4.0 /buildkit
+RUN cd /buildkit && \
+  [ -f "README.md" ] && \
+  apk add git && \
+  [ -z "$(git status -s)" ] && \
+  [ "$(git rev-parse HEAD)" = "c35410878ab9070498c66f6c67d3e8bc3b92241f" ]
+
+# Complicated case
+ARG REPO="git@github.com/moby/buildkit.git"
+ARG TAG="v0.5.0"
+ADD --chown=4242:8484 ${REPO}#${TAG} /buildkit-chowned
+USER 4242
+RUN cd /buildkit-chowned && \
+  [ "$(stat -c %u README.md)" = "4242" ] && \
+  [ "$(stat -c %g README.md)" = "8484" ] && \
+  [ -z "$(git status -s)" ] && \
+  [ "$(git rev-parse HEAD)" = "8c0fa8fdec187d8f259a349d2da16dc2dc5f144a" ]
+`)
+
+	dir, err := tmpdir(
+		fstest.CreateFile("Dockerfile", dockerfile, 0600),
+	)
+	require.NoError(t, err)
+	defer os.RemoveAll(dir)
+
+	c, err := client.New(sb.Context(), sb.Address())
+	require.NoError(t, err)
+	defer c.Close()
+
+	_, err = f.Solve(sb.Context(), c, client.SolveOpt{
+		LocalDirs: map[string]string{
+			builder.DefaultLocalNameDockerfile: dir,
+			builder.DefaultLocalNameContext:    dir,
+		},
+	}, nil)
+	require.NoError(t, err)
+}
+
 func testDockerfileFromGit(t *testing.T, sb integration.Sandbox) {
 	f := getFrontend(t, sb)
 

frontend/dockerfile/docs/syntax.md

@@ -25,6 +25,22 @@ incrementing the major component of a version and you may want to pin the image
 change in between releases on labs channel, the old versions are guaranteed to be backward compatible.
 
 
+## Adding a git repository `ADD <git ref> <dir>`
+
+To use this instruction set Dockerfile version to at least `1.5`.
+
+```dockerfile
+# syntax=docker/dockerfile:1.5
+```
+
+This instruction allows you to add a git repository to an image directly, without using the `git` command inside the image.
+
+```dockerfile
+# syntax=docker/dockerfile:1.5
+FROM alpine
+ADD https://github.com/moby/buildkit.git#v0.10.1 /buildkit
+```
+
 ## Linked copies `COPY --link`, `ADD --link`
 
 To use this flag set Dockerfile version to at least `1.4`.

util/gitutil/git_ref.go

@@ -0,0 +1,48 @@
+package gitutil
+
+import (
+	"regexp"
+	"strings"
+
+	"github.com/containerd/containerd/errdefs"
+)
+
+type GitRef struct {
+	Repo   string
+	Commit string // Optional
+	Dir    string
+}
+
+var httpPrefix = regexp.MustCompile(`^https?://`)
+var gitURLPathWithFragmentSuffix = regexp.MustCompile(`\.git(?:#.+)?$`)
+
+func IsGitRef(ref string) bool {
+	if httpPrefix.MatchString(ref) && gitURLPathWithFragmentSuffix.MatchString(ref) {
+		return true
+	}
+	for _, prefix := range []string{"git://", "github.com/", "git@"} {
+		if strings.HasPrefix(ref, prefix) {
+			return true
+		}
+	}
+	return false
+}
+
+func ParseGitRef(ref string) (*GitRef, error) {
+	if !IsGitRef(ref) {
+		return nil, errdefs.ErrInvalidArgument
+	}
+	refSplitBySharp := strings.SplitN(ref, "#", 2)
+	repo := refSplitBySharp[0]
+	commit := ""
+	if len(refSplitBySharp) > 1 {
+		commit = refSplitBySharp[1]
+	}
+	repoSplitBySlash := strings.Split(ref, "/")
+	dir := strings.TrimSuffix(repoSplitBySlash[len(repoSplitBySlash)-1], ".git")
+	return &GitRef{
+		Repo:   repo,
+		Commit: commit,
+		Dir:    dir,
+	}, nil
+}

util/gitutil/git_ref_test.go

@@ -0,0 +1,26 @@
+package gitutil
+
+import "testing"
+
+func TestIsGitRef(t *testing.T) {
+	cases := map[string]bool{
+		"https://example.com/":                 false,
+		"https://example.com/foo":              false,
+		"https://example.com/foo.git":          true,
+		"https://example.com/foo.git#deadbeef": true,
+		"https://example.com/foo.git/":         false,
+		"https://example.com/foo.git.bar":      false,
+		"git://example.com/foo":                true,
+		"github.com/moby/buildkit":             true,
+		"https://github.com/moby/buildkit":     false,
+		"https://github.com/moby/buildkit.git": true,
+		"git@github.com/moby/buildkit":         true,
+		"git@github.com/moby/buildkit.git":     true,
+	}
+	for ref, expected := range cases {
+		got := IsGitRef(ref)
+		if got != expected {
+			t.Errorf("expected IsGitRef(%q) to be %v, got %v", ref, expected, got)
+		}
+	}
+}