Skip to content

Commit

Permalink
dockerfile: implement ADD <git ref>
Browse files Browse the repository at this point in the history
e.g.,
```dockerfile
FROM alpine
ADD https://github.com/moby/buildkit.git#v0.10.1 /buildkit
```

Close issue 775

Signed-off-by: Akihiro Suda <akihiro.suda.cz@hco.ntt.co.jp>
  • Loading branch information
AkihiroSuda committed Apr 27, 2022
1 parent 3eed7fd commit 1e14e51
Show file tree
Hide file tree
Showing 6 changed files with 290 additions and 31 deletions.
45 changes: 15 additions & 30 deletions frontend/dockerfile/builder/build.go
Expand Up @@ -27,6 +27,7 @@ import (
"github.com/moby/buildkit/solver/errdefs"
"github.com/moby/buildkit/solver/pb"
binfotypes "github.com/moby/buildkit/util/buildinfo/types"
"github.com/moby/buildkit/util/gitutil"
ocispecs "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/pkg/errors"
"golang.org/x/sync/errgroup"
Expand Down Expand Up @@ -69,7 +70,6 @@ const (
)

var httpPrefix = regexp.MustCompile(`^https?://`)
var gitURLPathWithFragmentSuffix = regexp.MustCompile(`\.git(?:#.+)?$`)

func Build(ctx context.Context, c client.Client) (*client.Result, error) {
opts := c.BuildOpts().Opts
Expand Down Expand Up @@ -180,7 +180,11 @@ func Build(ctx context.Context, c client.Client) (*client.Result, error) {

var buildContext *llb.State
isNotLocalContext := false
if st, ok := detectGitContext(opts[localNameContext], opts[keyContextKeepGitDirArg]); ok {
keepGit := false
if v, err := strconv.ParseBool(opts[keyContextKeepGitDirArg]); err == nil {
keepGit = v
}
if st, ok := detectGitContext(opts[localNameContext], keepGit); ok {
if !forceLocalDockerfile {
src = *st
}
Expand Down Expand Up @@ -598,40 +602,21 @@ func filter(opt map[string]string, key string) map[string]string {
return m
}

func detectGitContext(ref, gitContext string) (*llb.State, bool) {
found := false
if httpPrefix.MatchString(ref) && gitURLPathWithFragmentSuffix.MatchString(ref) {
found = true
}

keepGit := false
if gitContext != "" {
if v, err := strconv.ParseBool(gitContext); err == nil {
keepGit = v
}
}

for _, prefix := range []string{"git://", "github.com/", "git@"} {
if strings.HasPrefix(ref, prefix) {
found = true
break
}
}
if !found {
func detectGitContext(ref string, keepGit bool) (*llb.State, bool) {
g, err := gitutil.ParseGitRef(ref)
if err != nil {
return nil, false
}

parts := strings.SplitN(ref, "#", 2)
branch := ""
if len(parts) > 1 {
branch = parts[1]
if g.SubDir != "" {
// "https://github.com/docker/docker.git#:dir" form is not implemented yet
return nil, false
}
gitOpts := []llb.GitOption{dockerfile2llb.WithInternalName("load git source " + ref)}
if keepGit {
gitOpts = append(gitOpts, llb.KeepGitDir())
}

st := llb.Git(parts[0], branch, gitOpts...)
st := llb.Git(g.Remote, g.Commit, gitOpts...)
return &st, true
}

Expand Down Expand Up @@ -837,13 +822,13 @@ func contextByName(ctx context.Context, c client.Client, name string, platform *
st := llb.Image(ref, imgOpt...)
return &st, nil, nil, nil
case "git":
st, ok := detectGitContext(v, "1")
st, ok := detectGitContext(v, true)
if !ok {
return nil, nil, nil, errors.Errorf("invalid git context %s", v)
}
return st, nil, nil, nil
case "http", "https":
st, ok := detectGitContext(v, "1")
st, ok := detectGitContext(v, true)
if !ok {
httpst := llb.HTTP(v, llb.WithCustomName("[context "+name+"] "+v))
st = &httpst
Expand Down
24 changes: 23 additions & 1 deletion frontend/dockerfile/dockerfile2llb/convert.go
Expand Up @@ -26,6 +26,7 @@ import (
"github.com/moby/buildkit/solver/pb"
"github.com/moby/buildkit/util/apicaps"
binfotypes "github.com/moby/buildkit/util/buildinfo/types"
"github.com/moby/buildkit/util/gitutil"
"github.com/moby/buildkit/util/suggest"
"github.com/moby/buildkit/util/system"
"github.com/moby/sys/signal"
Expand Down Expand Up @@ -1001,7 +1002,28 @@ func dispatchCopy(d *dispatchState, cfg copyConfig) error {

for _, src := range cfg.params.SourcePaths {
commitMessage.WriteString(" " + src)
if strings.HasPrefix(src, "http://") || strings.HasPrefix(src, "https://") {
gitRef, gitRefErr := gitutil.ParseGitRef(src)
if gitRefErr == nil && !gitRef.IndistinguishableFromLocal {
if !cfg.isAddCommand {
return errors.New("source can't be a git ref for COPY")
}
if gitRef.UnencryptedTCP {
return errors.New("unencrypted TCP is not supported for ADD <git ref>")
}
if gitRef.SubDir != "" {
return errors.New("currently ADD <git ref> does not support specifying the work dir")
}
st := llb.Git(gitRef.Remote, gitRef.Commit, llb.KeepGitDir())
opts := append([]llb.CopyOption{&llb.CopyInfo{
Mode: mode,
CreateDestPath: true,
}}, copyOpt...)
if a == nil {
a = llb.Copy(st, "/", dest, opts...)
} else {
a = a.Copy(st, "/", dest, opts...)
}
} else if strings.HasPrefix(src, "http://") || strings.HasPrefix(src, "https://") {
if !cfg.isAddCommand {
return errors.New("source can't be a URL for COPY")
}
Expand Down
46 changes: 46 additions & 0 deletions frontend/dockerfile/dockerfile_test.go
Expand Up @@ -130,6 +130,7 @@ var allTests = integration.TestFuncs(
testCopyVarSubstitution,
testCopyWildcards,
testCopyRelative,
testAddGit,
testAddURLChmod,
testTarContext,
testTarContextExternalDockerfile,
Expand Down Expand Up @@ -3463,6 +3464,51 @@ COPY --from=build /dest /dest
require.Equal(t, []byte("0644\n0755\n0413\n"), dt)
}

func testAddGit(t *testing.T, sb integration.Sandbox) {
f := getFrontend(t, sb)

dockerfile := []byte(`
FROM alpine
# Basic case
ADD https://github.com/moby/buildkit.git#v0.4.0 /buildkit
RUN cd /buildkit && \
[ -f "README.md" ] && \
apk add git && \
[ -z "$(git status -s)" ] && \
[ "$(git rev-parse HEAD)" = "c35410878ab9070498c66f6c67d3e8bc3b92241f" ]
# Complicated case
ARG REPO="git@github.com/moby/buildkit.git"
ARG TAG="v0.5.0"
ADD --chown=4242:8484 ${REPO}#${TAG} /buildkit-chowned
USER 4242
RUN cd /buildkit-chowned && \
[ "$(stat -c %u README.md)" = "4242" ] && \
[ "$(stat -c %g README.md)" = "8484" ] && \
[ -z "$(git status -s)" ] && \
[ "$(git rev-parse HEAD)" = "8c0fa8fdec187d8f259a349d2da16dc2dc5f144a" ]
`)

dir, err := tmpdir(
fstest.CreateFile("Dockerfile", dockerfile, 0600),
)
require.NoError(t, err)
defer os.RemoveAll(dir)

c, err := client.New(sb.Context(), sb.Address())
require.NoError(t, err)
defer c.Close()

_, err = f.Solve(sb.Context(), c, client.SolveOpt{
LocalDirs: map[string]string{
builder.DefaultLocalNameDockerfile: dir,
builder.DefaultLocalNameContext: dir,
},
}, nil)
require.NoError(t, err)
}

func testDockerfileFromGit(t *testing.T, sb integration.Sandbox) {
f := getFrontend(t, sb)

Expand Down
33 changes: 33 additions & 0 deletions frontend/dockerfile/docs/syntax.md
Expand Up @@ -25,6 +25,39 @@ incrementing the major component of a version and you may want to pin the image
change in between releases on labs channel, the old versions are guaranteed to be backward compatible.


## Adding a git repository `ADD <git ref> <dir>`

<!-- TODO: s/upstream-master/1.5/g after the release of 1.5 -->
To use this instruction set Dockerfile version to `upstream-master`:

```dockerfile
# syntax=docker/dockerfile:upstream-master`
```

This instruction allows you to add a git repository to an image directly, without using the `git` command inside the image.

```dockerfile
# syntax=docker/dockerfile:upstream-master
FROM alpine
ADD https://github.com/moby/buildkit.git#v0.10.1 /buildkit
```


To add a private repo via SSH:
```dockerfile
# syntax = docker/dockerfile:upstream-master
FROM alpine
ADD git@git.example.com:foo/bar.git /bar
```

```console
$ docker build --ssh default
```

```console
$ buildctl build --frontend=dockerfile.v0 --local context=. --local dockerfile=. --ssh default
```

## Linked copies `COPY --link`, `ADD --link`

To use this flag set Dockerfile version to at least `1.4`.
Expand Down
96 changes: 96 additions & 0 deletions util/gitutil/git_ref.go
@@ -0,0 +1,96 @@
package gitutil

import (
"regexp"
"strings"

"github.com/containerd/containerd/errdefs"
)

// GitRef represents a git ref.
//
// Examples:
// - "https://github.com/foo/bar.git#baz/qux:quux/quuz" is parsed into:
// {Remote: "https://github.com/foo/bar.git", ShortName: "bar", Commit:"baz/qux", SubDir: "quux/quuz"}.
type GitRef struct {
// Remote is the remote repository path.
Remote string

// ShortName is the directory name of the repo.
// e.g., "bar" for "https://github.com/foo/bar.git"
ShortName string

// Commit is a commit hash, a tag, or branch name.
// Commit is optional.
Commit string

// SubDir is a directory path inside the repo.
// SubDir is optional.
SubDir string

// IndistinguishableFromLocal is true for a ref that is indistinguishable from a local file path,
// e.g., "github.com/foo/bar".
//
// Deprecated. Use a distinguishable form such as "https://github.com/foo/bar.git".
//
// The dockerfile frontend still accepts this form only for build contexts.
IndistinguishableFromLocal bool

// Unencrypted TCP is true for a ref that needs an unencrypted TCP connection,
// e.g., "git://..." and "http://..." .
//
// Deprecated. Use an encrypted TCP connection such as "git@github.com/foo/bar.git" or "https://github.com/foo/bar.git".
//
// The dockerfile frontend still accepts this form only for build contexts.
UnencryptedTCP bool
}

var httpPrefix = regexp.MustCompile(`^https?://`)
var gitURLPathWithFragmentSuffix = regexp.MustCompile(`\.git(?:#.+)?$`)

func ParseGitRef(ref string) (*GitRef, error) {
res := &GitRef{}
var valid bool

if httpPrefix.MatchString(ref) && gitURLPathWithFragmentSuffix.MatchString(ref) {
valid = true
if strings.HasPrefix(ref, "http://") {
res.UnencryptedTCP = true
}
} else {
if strings.HasPrefix(ref, "git://") {
valid = true
res.UnencryptedTCP = true
}
if strings.HasPrefix(ref, "github.com/") {
valid = true
res.IndistinguishableFromLocal = true
}
if strings.HasPrefix(ref, "git@") {
valid = true
}
}
if !valid {
return res, errdefs.ErrInvalidArgument
}

refSplitBySharp := strings.SplitN(ref, "#", 2)
res.Remote = refSplitBySharp[0]
if len(res.Remote) == 0 {
return res, errdefs.ErrInvalidArgument
}

if len(refSplitBySharp) > 1 {
refSplitBySharpSplitByColon := strings.SplitN(refSplitBySharp[1], ":", 2)
res.Commit = refSplitBySharpSplitByColon[0]
if len(res.Commit) == 0 {
return res, errdefs.ErrInvalidArgument
}
if len(refSplitBySharpSplitByColon) > 1 {
res.SubDir = refSplitBySharpSplitByColon[1]
}
}
repoSplitBySlash := strings.Split(res.Remote, "/")
res.ShortName = strings.TrimSuffix(repoSplitBySlash[len(repoSplitBySlash)-1], ".git")
return res, nil
}

0 comments on commit 1e14e51

Please sign in to comment.