build: implement build secrets with buildkit

This patch implements `docker build --secret id=mysecret,src=/secret/file`
for buildkit frontends that request the mysecret secret.

It is currently implemented in the tonistiigi/dockerfile:secrets20180808
frontend via RUN --mount=type=secret,id=mysecret

Signed-off-by: Tibor Vass <tibor@docker.com>

cli/command/image/build.go

@@ -72,6 +72,7 @@ type buildOptions struct {
 	stream         bool
 	platform       string
 	untrusted      bool
+	secrets        []string
 }
 
 // dockerfileFromStdin returns true when the user specified that the Dockerfile
@@ -156,6 +157,10 @@ func NewBuildCommand(dockerCli command.Cli) *cobra.Command {
 	flags.StringVar(&options.progress, "progress", "auto", "Set type of progress output (only if BuildKit enabled) (auto, plain, tty). Use plain to show container output")
 	flags.SetAnnotation("progress", "experimental", nil)
 	flags.SetAnnotation("progress", "version", []string{"1.38"})
+
+	flags.StringArrayVar(&options.secrets, "secret", []string{}, "Secret file to expose to the build (only if BuildKit enabled): id=mysecret,src=/local/secret")
+	flags.SetAnnotation("secret", "experimental", nil)
+	flags.SetAnnotation("secret", "version", []string{"1.39"})
 	return cmd
 }
 

cli/command/image/build_buildkit.go

@@ -3,6 +3,7 @@ package image
 import (
 	"bytes"
 	"context"
+	"encoding/csv"
 	"encoding/json"
 	"fmt"
 	"io"
@@ -22,8 +23,10 @@ import (
 	"github.com/docker/docker/pkg/urlutil"
 	controlapi "github.com/moby/buildkit/api/services/control"
 	"github.com/moby/buildkit/client"
+	"github.com/moby/buildkit/session"
 	"github.com/moby/buildkit/session/auth/authprovider"
 	"github.com/moby/buildkit/session/filesync"
+	"github.com/moby/buildkit/session/secrets/secretsprovider"
 	"github.com/moby/buildkit/util/appcontext"
 	"github.com/moby/buildkit/util/progress/progressui"
 	"github.com/pkg/errors"
@@ -128,6 +131,13 @@ func runBuildBuildKit(dockerCli command.Cli, options buildOptions) error {
 	}
 
 	s.Allow(authprovider.NewDockerAuthProvider())
+	if len(options.secrets) > 0 {
+		sp, err := parseSecretSpecs(options.secrets)
+		if err != nil {
+			return errors.Wrapf(err, "could not parse secrets: %v", options.secrets)
+		}
+		s.Allow(sp)
+	}
 
 	eg, ctx := errgroup.WithContext(ctx)
 
@@ -204,7 +214,7 @@ func doBuild(ctx context.Context, eg *errgroup.Group, dockerCli command.Cli, opt
 		}
 		// not using shared context to not disrupt display but let is finish reporting errors
 		eg.Go(func() error {
-			return progressui.DisplaySolveStatus(context.TODO(), c, out, displayCh)
+			return progressui.DisplaySolveStatus(context.TODO(), "", c, out, displayCh)
 		})
 	}
 
@@ -348,3 +358,53 @@ func (t *tracer) write(msg jsonmessage.JSONMessage) {
 
 	t.displayCh <- &s
 }
+
+func parseSecretSpecs(sl []string) (session.Attachable, error) {
+	fs := make([]secretsprovider.FileSource, 0, len(sl))
+	for _, v := range sl {
+		s, err := parseSecret(v)
+		if err != nil {
+			return nil, err
+		}
+		fs = append(fs, *s)
+	}
+	store, err := secretsprovider.NewFileStore(fs)
+	if err != nil {
+		return nil, err
+	}
+	return secretsprovider.NewSecretProvider(store), nil
+}
+
+func parseSecret(value string) (*secretsprovider.FileSource, error) {
+	csvReader := csv.NewReader(strings.NewReader(value))
+	fields, err := csvReader.Read()
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to parse csv secret")
+	}
+
+	fs := secretsprovider.FileSource{}
+
+	for _, field := range fields {
+		parts := strings.SplitN(field, "=", 2)
+		key := strings.ToLower(parts[0])
+
+		if len(parts) != 2 {
+			return nil, errors.Errorf("invalid field '%s' must be a key=value pair", field)
+		}
+
+		value := parts[1]
+		switch key {
+		case "type":
+			if value != "file" {
+				return nil, errors.Errorf("unsupported secret type %q", value)
+			}
+		case "id":
+			fs.ID = value
+		case "source", "src":
+			fs.FilePath = value
+		default:
+			return nil, errors.Errorf("unexpected key '%s' in '%s'", key, field)
+		}
+	}
+	return &fs, nil
+}

cli/command/image/build_test.go

@@ -5,6 +5,7 @@ import (
 	"bytes"
 	"compress/gzip"
 	"context"
+	"fmt"
 	"io"
 	"io/ioutil"
 	"os"
@@ -17,6 +18,7 @@ import (
 	"github.com/docker/docker/api/types"
 	"github.com/docker/docker/pkg/archive"
 	"github.com/google/go-cmp/cmp"
+	"github.com/moby/buildkit/session/secrets/secretsprovider"
 	"gotest.tools/assert"
 	"gotest.tools/fs"
 	"gotest.tools/skip"
@@ -173,6 +175,66 @@ RUN echo hello world
 	assert.DeepEqual(t, fakeBuild.filenames(t), []string{"Dockerfile"})
 }
 
+func TestParseSecret(t *testing.T) {
+	type testcase struct {
+		value       string
+		errExpected bool
+		errMatch    string
+		filesource  *secretsprovider.FileSource
+	}
+	var testcases = []testcase{
+		{
+			value:       "",
+			errExpected: true,
+		}, {
+			value:       "foobar",
+			errExpected: true,
+			errMatch:    "must be a key=value pair",
+		}, {
+			value:       "foo,bar",
+			errExpected: true,
+			errMatch:    "must be a key=value pair",
+		}, {
+			value:       "foo=bar",
+			errExpected: true,
+			errMatch:    "unexpected key",
+		}, {
+			value:      "src=somefile",
+			filesource: &secretsprovider.FileSource{FilePath: "somefile"},
+		}, {
+			value:      "source=somefile",
+			filesource: &secretsprovider.FileSource{FilePath: "somefile"},
+		}, {
+			value:      "id=mysecret",
+			filesource: &secretsprovider.FileSource{ID: "mysecret"},
+		}, {
+			value:      "id=mysecret,src=somefile",
+			filesource: &secretsprovider.FileSource{ID: "mysecret", FilePath: "somefile"},
+		}, {
+			value:      "id=mysecret,source=somefile,type=file",
+			filesource: &secretsprovider.FileSource{ID: "mysecret", FilePath: "somefile"},
+		}, {
+			value:      "id=mysecret,src=somefile,src=othersecretfile",
+			filesource: &secretsprovider.FileSource{ID: "mysecret", FilePath: "othersecretfile"},
+		}, {
+			value:       "type=invalid",
+			errExpected: true,
+			errMatch:    "unsupported secret type",
+		},
+	}
+
+	for _, tc := range testcases {
+		t.Run(tc.value, func(t *testing.T) {
+			secret, err := parseSecret(tc.value)
+			assert.Equal(t, err != nil, tc.errExpected, fmt.Sprintf("err=%v errExpected=%t", err, tc.errExpected))
+			if tc.errMatch != "" {
+				assert.ErrorContains(t, err, tc.errMatch)
+			}
+			assert.DeepEqual(t, secret, tc.filesource)
+		})
+	}
+}
+
 type fakeBuild struct {
 	context *tar.Reader
 	options types.ImageBuildOptions

vendor.conf

@@ -1,8 +1,8 @@
 github.com/agl/ed25519 5312a61534124124185d41f09206b9fef1d88403
 github.com/Azure/go-ansiterm d6e3b3328b783f23731bc4d058875b0371ff8109
 github.com/beorn7/perks 3a771d992973f24aa725d07868b467d1ddfceafb
-github.com/containerd/console cb7008ab3d8359b78c5f464cb7cf160107ad5925
-github.com/containerd/containerd 08f7ee9828af1783dc98cc5cc1739e915697c667
+github.com/containerd/console 5d1b48d6114b8c9666f0c8b916f871af97b0a761
+github.com/containerd/containerd a88b6319614de846458750ff882723479ca7b1a1
 github.com/containerd/continuity d8fb8589b0e8e85b8c8bbaa8840226d0dfeb7371
 github.com/coreos/etcd v3.3.9
 github.com/cpuguy83/go-md2man v1.0.8
@@ -42,7 +42,7 @@ github.com/matttproud/golang_protobuf_extensions v1.0.1
 github.com/Microsoft/go-winio v0.4.9
 github.com/miekg/pkcs11 287d9350987cc9334667882061e202e96cdfb4d0
 github.com/mitchellh/mapstructure f15292f7a699fcc1a38a80977f80a046874ba8ac
-github.com/moby/buildkit 9acf51e49185b348608e0096b2903dd72907adcb
+github.com/moby/buildkit 785436a312230fcc79b41aa044c1643528c91913
 github.com/modern-go/concurrent bacd9c7ef1dd9b15be4a9909b8ac7a4e313eec94 # 1.0.3
 github.com/modern-go/reflect2 4b7aa43c6742a2c18fdef89dd197aaae7dac7ccd # 1.0.1
 github.com/morikuni/aec 39771216ff4c63d11f5e604076f9c45e8be1067b
@@ -63,15 +63,15 @@ github.com/sirupsen/logrus v1.0.6
 github.com/spf13/cobra v0.0.3
 github.com/spf13/pflag v1.0.1
 github.com/theupdateframework/notary v0.6.1
-github.com/tonistiigi/fsutil 8abad97ee3969cdf5e9c367f46adba2c212b3ddb
-github.com/tonistiigi/units 29de085e9400559bd68aea2e7bc21566e7b8281d
+github.com/tonistiigi/fsutil b19464cd1b6a00773b4f2eb7acf9c30426f9df42
+github.com/tonistiigi/units 6950e57a87eaf136bbe44ef2ec8e75b9e3569de2
 github.com/xeipuuv/gojsonpointer 4e3ac2762d5f479393488629ee9370b50873b3a6
 github.com/xeipuuv/gojsonreference bd5ef7bd5415a7ac448318e64f11a24cd21e594b
 github.com/xeipuuv/gojsonschema 93e72a773fade158921402d6a24c819b48aba29d
 golang.org/x/crypto a2144134853fc9a27a7b1e3eb4f19f1a76df13c9
 golang.org/x/net a680a1efc54dd51c040b3b5ce4939ea3cf2ea0d1
 golang.org/x/sync 1d60e4601c6fd243af51cc01ddf169918a5407ca
-golang.org/x/sys ac767d655b305d4e9612f5f6e33120b9176c4ad4
+golang.org/x/sys 1b2967e3c290b7c545b3db0deeda16e9be4f98a2
 golang.org/x/text f21a4dfb5e38f5895301dc265a8def02365cc3d0 # v0.3.0
 golang.org/x/time fbb02b2291d28baffd63558aa44b4b56f178d650
 google.golang.org/genproto 02b4e95473316948020af0b7a4f0f22c73929b0e