Skip to content
Permalink
Browse files

Merge pull request #1438 from thaJeztah/18.09_backport_hide-buildkit-…

…flags-if-not-enabled

[18.09 backport] builder / buildkit updates
  • Loading branch information...
tiborvass committed Oct 11, 2018
2 parents 4280972 + 3dfacb5 commit 39f1110308a06ec45d763b581e9b82da0c064c53
Showing with 20,308 additions and 547 deletions.
  1. +15 −0 cli/command/cli.go
  2. +16 −12 cli/command/image/build.go
  3. +33 −2 cli/command/image/build_buildkit.go
  4. +15 −3 cmd/docker/docker.go
  5. +1 −0 docs/reference/commandline/build.md
  6. +3 −3 vendor.conf
  7. +334 −19 vendor/github.com/moby/buildkit/api/types/worker.pb.go
  8. +8 −0 vendor/github.com/moby/buildkit/api/types/worker.proto
  9. +10 −0 vendor/github.com/moby/buildkit/client/build.go
  10. +5 −7 vendor/github.com/moby/buildkit/client/client.go
  11. +101 −4 vendor/github.com/moby/buildkit/client/llb/exec.go
  12. +7 −4 vendor/github.com/moby/buildkit/client/llb/meta.go
  13. +2 −0 vendor/github.com/moby/buildkit/client/llb/resolver.go
  14. +10 −3 vendor/github.com/moby/buildkit/client/llb/state.go
  15. +2 −2 vendor/github.com/moby/buildkit/client/solve.go
  16. +21 −0 vendor/github.com/moby/buildkit/client/workers.go
  17. +15 −2 vendor/github.com/moby/buildkit/frontend/gateway/client/client.go
  18. +29 −13 vendor/github.com/moby/buildkit/frontend/gateway/grpcclient/client.go
  19. +15 −0 vendor/github.com/moby/buildkit/frontend/gateway/pb/caps.go
  20. +829 −66 vendor/github.com/moby/buildkit/frontend/gateway/pb/gateway.pb.go
  21. +24 −0 vendor/github.com/moby/buildkit/frontend/gateway/pb/gateway.proto
  22. +3 −2 vendor/github.com/moby/buildkit/session/filesync/diffcopy.go
  23. +2 −1 vendor/github.com/moby/buildkit/session/filesync/filesync.go
  24. +1 −0 vendor/github.com/moby/buildkit/session/grpchijack/hijack.go
  25. +61 −0 vendor/github.com/moby/buildkit/session/sshforward/copy.go
  26. +3 −0 vendor/github.com/moby/buildkit/session/sshforward/generate.go
  27. +113 −0 vendor/github.com/moby/buildkit/session/sshforward/ssh.go
  28. +816 −0 vendor/github.com/moby/buildkit/session/sshforward/ssh.pb.go
  29. +22 −0 vendor/github.com/moby/buildkit/session/sshforward/ssh.proto
  30. +198 −0 vendor/github.com/moby/buildkit/session/sshforward/sshprovider/agentprovider.go
  31. +31 −10 vendor/github.com/moby/buildkit/solver/pb/caps.go
  32. +13 −0 vendor/github.com/moby/buildkit/solver/pb/const.go
  33. +490 −115 vendor/github.com/moby/buildkit/solver/pb/ops.pb.go
  34. +18 −0 vendor/github.com/moby/buildkit/solver/pb/ops.proto
  35. +11 −9 vendor/github.com/moby/buildkit/vendor.conf
  36. +3 −1 vendor/github.com/tonistiigi/fsutil/diff.go
  37. +4 −3 vendor/github.com/tonistiigi/fsutil/diff_containerd.go
  38. +4 −3 vendor/github.com/tonistiigi/fsutil/diskwriter.go
  39. +3 −2 vendor/github.com/tonistiigi/fsutil/diskwriter_unix.go
  40. +3 −2 vendor/github.com/tonistiigi/fsutil/diskwriter_windows.go
  41. +4 −3 vendor/github.com/tonistiigi/fsutil/fs.go
  42. +2 −1 vendor/github.com/tonistiigi/fsutil/hardlinks.go
  43. +12 −11 vendor/github.com/tonistiigi/fsutil/receive.go
  44. +13 −12 vendor/github.com/tonistiigi/fsutil/send.go
  45. +61 −0 vendor/github.com/tonistiigi/fsutil/stat.go
  46. +13 −7 vendor/github.com/tonistiigi/fsutil/{walker_unix.go → stat_unix.go}
  47. +16 −0 vendor/github.com/tonistiigi/fsutil/stat_windows.go
  48. +1 −1 vendor/github.com/tonistiigi/fsutil/{ → types}/generate.go
  49. +101 −123 vendor/github.com/tonistiigi/fsutil/{ → types}/stat.pb.go
  50. +3 −1 vendor/github.com/tonistiigi/fsutil/{ → types}/stat.proto
  51. +26 −51 vendor/github.com/tonistiigi/fsutil/{ → types}/wire.pb.go
  52. +3 −1 vendor/github.com/tonistiigi/fsutil/{ → types}/wire.proto
  53. +6 −34 vendor/github.com/tonistiigi/fsutil/walker.go
  54. +0 −14 vendor/github.com/tonistiigi/fsutil/walker_windows.go
  55. +217 −0 vendor/golang.org/x/crypto/ed25519/ed25519.go
  56. +1,422 −0 vendor/golang.org/x/crypto/ed25519/internal/edwards25519/const.go
  57. +1,793 −0 vendor/golang.org/x/crypto/ed25519/internal/edwards25519/edwards25519.go
  58. +264 −0 vendor/golang.org/x/crypto/internal/chacha20/chacha_generic.go
  59. +16 −0 vendor/golang.org/x/crypto/internal/chacha20/chacha_noasm.go
  60. +30 −0 vendor/golang.org/x/crypto/internal/chacha20/chacha_s390x.go
  61. +283 −0 vendor/golang.org/x/crypto/internal/chacha20/chacha_s390x.s
  62. +43 −0 vendor/golang.org/x/crypto/internal/chacha20/xor.go
  63. +32 −0 vendor/golang.org/x/crypto/internal/subtle/aliasing.go
  64. +35 −0 vendor/golang.org/x/crypto/internal/subtle/aliasing_appengine.go
  65. +33 −0 vendor/golang.org/x/crypto/poly1305/poly1305.go
  66. +22 −0 vendor/golang.org/x/crypto/poly1305/sum_amd64.go
  67. +125 −0 vendor/golang.org/x/crypto/poly1305/sum_amd64.s
  68. +22 −0 vendor/golang.org/x/crypto/poly1305/sum_arm.go
  69. +427 −0 vendor/golang.org/x/crypto/poly1305/sum_arm.s
  70. +14 −0 vendor/golang.org/x/crypto/poly1305/sum_noasm.go
  71. +139 −0 vendor/golang.org/x/crypto/poly1305/sum_ref.go
  72. +49 −0 vendor/golang.org/x/crypto/poly1305/sum_s390x.go
  73. +400 −0 vendor/golang.org/x/crypto/poly1305/sum_s390x.s
  74. +931 −0 vendor/golang.org/x/crypto/poly1305/sum_vmsl_s390x.s
  75. +683 −0 vendor/golang.org/x/crypto/ssh/agent/client.go
  76. +103 −0 vendor/golang.org/x/crypto/ssh/agent/forward.go
  77. +215 −0 vendor/golang.org/x/crypto/ssh/agent/keyring.go
  78. +523 −0 vendor/golang.org/x/crypto/ssh/agent/server.go
  79. +97 −0 vendor/golang.org/x/crypto/ssh/buffer.go
  80. +521 −0 vendor/golang.org/x/crypto/ssh/certs.go
  81. +633 −0 vendor/golang.org/x/crypto/ssh/channel.go
  82. +770 −0 vendor/golang.org/x/crypto/ssh/cipher.go
  83. +278 −0 vendor/golang.org/x/crypto/ssh/client.go
  84. +525 −0 vendor/golang.org/x/crypto/ssh/client_auth.go
  85. +383 −0 vendor/golang.org/x/crypto/ssh/common.go
  86. +143 −0 vendor/golang.org/x/crypto/ssh/connection.go
  87. +21 −0 vendor/golang.org/x/crypto/ssh/doc.go
  88. +646 −0 vendor/golang.org/x/crypto/ssh/handshake.go
  89. +540 −0 vendor/golang.org/x/crypto/ssh/kex.go
  90. +1,035 −0 vendor/golang.org/x/crypto/ssh/keys.go
  91. +61 −0 vendor/golang.org/x/crypto/ssh/mac.go
  92. +766 −0 vendor/golang.org/x/crypto/ssh/messages.go
  93. +330 −0 vendor/golang.org/x/crypto/ssh/mux.go
  94. +593 −0 vendor/golang.org/x/crypto/ssh/server.go
  95. +647 −0 vendor/golang.org/x/crypto/ssh/session.go
  96. +116 −0 vendor/golang.org/x/crypto/ssh/streamlocal.go
  97. +474 −0 vendor/golang.org/x/crypto/ssh/tcpip.go
  98. +353 −0 vendor/golang.org/x/crypto/ssh/transport.go
@@ -8,6 +8,7 @@ import (
"os"
"path/filepath"
"runtime"
"strconv"
"time"

"github.com/docker/cli/cli"
@@ -133,6 +134,20 @@ func (cli *DockerCli) ContentTrustEnabled() bool {
return cli.contentTrust
}

// BuildKitEnabled returns whether buildkit is enabled either through a daemon setting
// or otherwise the client-side DOCKER_BUILDKIT environment variable
func BuildKitEnabled(si ServerInfo) (bool, error) {
buildkitEnabled := si.BuildkitVersion == types.BuilderBuildKit
if buildkitEnv := os.Getenv("DOCKER_BUILDKIT"); buildkitEnv != "" {
var err error
buildkitEnabled, err = strconv.ParseBool(buildkitEnv)
if err != nil {
return false, errors.Wrap(err, "DOCKER_BUILDKIT environment variable expects boolean value")
}
}
return buildkitEnabled, nil
}

// ManifestStore returns a store for local manifests
func (cli *DockerCli) ManifestStore() manifeststore.Store {
// TODO: support override default location from config file
@@ -13,7 +13,6 @@ import (
"path/filepath"
"regexp"
"runtime"
"strconv"
"strings"

"github.com/docker/cli/cli"
@@ -73,6 +72,7 @@ type buildOptions struct {
platform string
untrusted bool
secrets []string
ssh []string
}

// dockerfileFromStdin returns true when the user specified that the Dockerfile
@@ -136,6 +136,8 @@ func NewBuildCommand(dockerCli command.Cli) *cobra.Command {
flags.BoolVar(&options.pull, "pull", false, "Always attempt to pull a newer version of the image")
flags.StringSliceVar(&options.cacheFrom, "cache-from", []string{}, "Images to consider as cache sources")
flags.BoolVar(&options.compress, "compress", false, "Compress the build context using gzip")
flags.SetAnnotation("compress", "no-buildkit", nil)

flags.StringSliceVar(&options.securityOpt, "security-opt", []string{}, "Security options")
flags.StringVar(&options.networkMode, "network", "default", "Set the networking mode for the RUN instructions during build")
flags.SetAnnotation("network", "version", []string{"1.25"})
@@ -153,11 +155,18 @@ func NewBuildCommand(dockerCli command.Cli) *cobra.Command {
flags.BoolVar(&options.stream, "stream", false, "Stream attaches to server to negotiate build context")
flags.SetAnnotation("stream", "experimental", nil)
flags.SetAnnotation("stream", "version", []string{"1.31"})
flags.SetAnnotation("stream", "no-buildkit", nil)

flags.StringVar(&options.progress, "progress", "auto", "Set type of progress output (only if BuildKit enabled) (auto, plain, tty). Use plain to show container output")
flags.StringVar(&options.progress, "progress", "auto", "Set type of progress output (auto, plain, tty). Use plain to show container output")
flags.SetAnnotation("progress", "buildkit", nil)

flags.StringArrayVar(&options.secrets, "secret", []string{}, "Secret file to expose to the build (only if BuildKit enabled): id=mysecret,src=/local/secret")
flags.SetAnnotation("secret", "version", []string{"1.39"})
flags.SetAnnotation("secret", "buildkit", nil)

flags.StringArrayVar(&options.ssh, "ssh", []string{}, "SSH agent socket or keys to expose to the build (only if BuildKit enabled) (format: default|<id>[=<socket>|<key>[,<key>]])")
flags.SetAnnotation("ssh", "version", []string{"1.39"})
flags.SetAnnotation("ssh", "buildkit", nil)
return cmd
}

@@ -179,22 +188,17 @@ func (out *lastProgressOutput) WriteProgress(prog progress.Progress) error {

// nolint: gocyclo
func runBuild(dockerCli command.Cli, options buildOptions) error {
if buildkitEnv := os.Getenv("DOCKER_BUILDKIT"); buildkitEnv != "" {
enableBuildkit, err := strconv.ParseBool(buildkitEnv)
if err != nil {
return errors.Wrap(err, "DOCKER_BUILDKIT environment variable expects boolean value")
}
if enableBuildkit {
return runBuildBuildKit(dockerCli, options)
}
} else if dockerCli.ServerInfo().BuildkitVersion == types.BuilderBuildKit {
buildkitEnabled, err := command.BuildKitEnabled(dockerCli.ServerInfo())
if err != nil {
return err
}
if buildkitEnabled {
return runBuildBuildKit(dockerCli, options)
}

var (
buildCtx io.ReadCloser
dockerfileCtx io.ReadCloser
err error
contextDir string
tempDir string
relDockerfile string
@@ -27,10 +27,11 @@ import (
"github.com/moby/buildkit/session/auth/authprovider"
"github.com/moby/buildkit/session/filesync"
"github.com/moby/buildkit/session/secrets/secretsprovider"
"github.com/moby/buildkit/session/sshforward/sshprovider"
"github.com/moby/buildkit/util/appcontext"
"github.com/moby/buildkit/util/progress/progressui"
"github.com/pkg/errors"
"github.com/tonistiigi/fsutil"
fsutiltypes "github.com/tonistiigi/fsutil/types"
"golang.org/x/sync/errgroup"
)

@@ -138,6 +139,13 @@ func runBuildBuildKit(dockerCli command.Cli, options buildOptions) error {
}
s.Allow(sp)
}
if len(options.ssh) > 0 {
sshp, err := parseSSHSpecs(options.ssh)
if err != nil {
return errors.Wrapf(err, "could not parse ssh: %v", options.ssh)
}
s.Allow(sshp)
}

eg, ctx := errgroup.WithContext(ctx)

@@ -291,7 +299,7 @@ func doBuild(ctx context.Context, eg *errgroup.Group, dockerCli command.Cli, opt
return err
}

func resetUIDAndGID(s *fsutil.Stat) bool {
func resetUIDAndGID(s *fsutiltypes.Stat) bool {
s.Uid = 0
s.Gid = 0
return true
@@ -408,3 +416,26 @@ func parseSecret(value string) (*secretsprovider.FileSource, error) {
}
return &fs, nil
}

func parseSSHSpecs(sl []string) (session.Attachable, error) {
configs := make([]sshprovider.AgentConfig, 0, len(sl))
for _, v := range sl {
c, err := parseSSH(v)
if err != nil {
return nil, err
}
configs = append(configs, *c)
}
return sshprovider.NewSSHAgentProvider(configs)
}

func parseSSH(value string) (*sshprovider.AgentConfig, error) {
parts := strings.SplitN(value, "=", 2)
cfg := sshprovider.AgentConfig{
ID: parts[0],
}
if len(parts) > 1 {
cfg.Paths = strings.Split(parts[1], ",")
}
return &cfg, nil
}
@@ -100,8 +100,10 @@ func setHelpFunc(dockerCli *command.DockerCli, cmd *cobra.Command, flags *pflag.
ccmd.Println(err)
return
}

hideUnsupportedFeatures(ccmd, dockerCli)
if err := hideUnsupportedFeatures(ccmd, dockerCli); err != nil {
ccmd.Println(err)
return
}
defaultHelpFunc(ccmd, args)
})
}
@@ -235,15 +237,21 @@ func hideFeatureSubCommand(subcmd *cobra.Command, hasFeature bool, annotation st
}
}

func hideUnsupportedFeatures(cmd *cobra.Command, details versionDetails) {
func hideUnsupportedFeatures(cmd *cobra.Command, details versionDetails) error {
clientVersion := details.Client().ClientVersion()
osType := details.ServerInfo().OSType
hasExperimental := details.ServerInfo().HasExperimental
hasExperimentalCLI := details.ClientInfo().HasExperimental
hasBuildKit, err := command.BuildKitEnabled(details.ServerInfo())
if err != nil {
return err
}

cmd.Flags().VisitAll(func(f *pflag.Flag) {
hideFeatureFlag(f, hasExperimental, "experimental")
hideFeatureFlag(f, hasExperimentalCLI, "experimentalCLI")
hideFeatureFlag(f, hasBuildKit, "buildkit")
hideFeatureFlag(f, !hasBuildKit, "no-buildkit")
// hide flags not supported by the server
if !isOSTypeSupported(f, osType) || !isVersionSupported(f, clientVersion) {
f.Hidden = true
@@ -259,6 +267,8 @@ func hideUnsupportedFeatures(cmd *cobra.Command, details versionDetails) {
for _, subcmd := range cmd.Commands() {
hideFeatureSubCommand(subcmd, hasExperimental, "experimental")
hideFeatureSubCommand(subcmd, hasExperimentalCLI, "experimentalCLI")
hideFeatureSubCommand(subcmd, hasBuildKit, "buildkit")
hideFeatureSubCommand(subcmd, !hasBuildKit, "no-buildkit")
// hide subcommands not supported by the server
if subcmdVersion, ok := subcmd.Annotations["version"]; ok && versions.LessThan(clientVersion, subcmdVersion) {
subcmd.Hidden = true
@@ -267,6 +277,7 @@ func hideUnsupportedFeatures(cmd *cobra.Command, details versionDetails) {
subcmd.Hidden = true
}
}
return nil
}

// Checks if a command or one of its ancestors is in the list
@@ -313,6 +324,7 @@ func areFlagsSupported(cmd *cobra.Command, details versionDetails) error {
if _, ok := f.Annotations["experimentalCLI"]; ok && !hasExperimentalCLI {
errs = append(errs, fmt.Sprintf("\"--%s\" is on a Docker cli with experimental cli features enabled", f.Name))
}
// buildkit-specific flags are noop when buildkit is not enabled, so we do not add an error in that case
}
})
if len(errs) > 0 {
@@ -59,6 +59,7 @@ Options:
Unit is optional and can be `b` (bytes), `k` (kilobytes), `m` (megabytes),
or `g` (gigabytes). If you omit the unit, the system uses bytes.
--squash Squash newly built layers into a single new layer (**Experimental Only**)
--ssh SSH agent socket or keys to expose to the build (only if BuildKit enabled) (format: default|<id>[=<socket>|<key>[,<key>]])
-t, --tag value Name and optionally a tag in the 'name:tag' format (default [])
--target string Set the target build stage to build.
--ulimit value Ulimit options (default [])
@@ -51,7 +51,7 @@ github.com/Microsoft/hcsshim 44c060121b68e8bdc40b411beba551f3b4ee9e55
github.com/Microsoft/go-winio v0.4.10
github.com/miekg/pkcs11 287d9350987cc9334667882061e202e96cdfb4d0
github.com/mitchellh/mapstructure f15292f7a699fcc1a38a80977f80a046874ba8ac
github.com/moby/buildkit 6812dac65e0440bb75affce1fb2175e640edc15d
github.com/moby/buildkit 520201006c9dc676da9cf9655337ac711f7f127d
github.com/modern-go/concurrent bacd9c7ef1dd9b15be4a9909b8ac7a4e313eec94 # 1.0.3
github.com/modern-go/reflect2 4b7aa43c6742a2c18fdef89dd197aaae7dac7ccd # 1.0.1
github.com/morikuni/aec 39771216ff4c63d11f5e604076f9c45e8be1067b
@@ -76,12 +76,12 @@ github.com/spf13/cobra v0.0.3
github.com/spf13/pflag 4cb166e4f25ac4e8016a3595bbf7ea2e9aa85a2c https://github.com/thaJeztah/pflag.git
github.com/syndtr/gocapability 2c00daeb6c3b45114c80ac44119e7b8801fdd852
github.com/theupdateframework/notary v0.6.1
github.com/tonistiigi/fsutil b19464cd1b6a00773b4f2eb7acf9c30426f9df42
github.com/tonistiigi/fsutil f567071bed2416e4d87d260d3162722651182317
github.com/tonistiigi/units 6950e57a87eaf136bbe44ef2ec8e75b9e3569de2
github.com/xeipuuv/gojsonpointer 4e3ac2762d5f479393488629ee9370b50873b3a6
github.com/xeipuuv/gojsonreference bd5ef7bd5415a7ac448318e64f11a24cd21e594b
github.com/xeipuuv/gojsonschema 93e72a773fade158921402d6a24c819b48aba29d
golang.org/x/crypto a2144134853fc9a27a7b1e3eb4f19f1a76df13c9
golang.org/x/crypto 0709b304e793a5edb4a2c0145f281ecdc20838a4
golang.org/x/net a680a1efc54dd51c040b3b5ce4939ea3cf2ea0d1
golang.org/x/sync 1d60e4601c6fd243af51cc01ddf169918a5407ca
golang.org/x/sys 1b2967e3c290b7c545b3db0deeda16e9be4f98a2
Oops, something went wrong.

0 comments on commit 39f1110

Please sign in to comment.
You can’t perform that action at this time.