Adding recent changes, --devices cli, Argon container policy for desktop

CONTRIBUTING.md

@@ -8,7 +8,7 @@ process](docs/contributing/).
 
 This page contains information about reporting issues as well as some tips and
 guidelines useful to experienced open source contributors. Finally, make sure
-you read our [community guidelines](#docker-community-guidelines) before you
+you read our [community guidelines](#moby-community-guidelines) before you
 start participating.
 
 ## Topics
@@ -17,7 +17,7 @@ start participating.
 * [Design and Cleanup Proposals](#design-and-cleanup-proposals)
 * [Reporting Issues](#reporting-other-issues)
 * [Quick Contribution Tips and Guidelines](#quick-contribution-tips-and-guidelines)
-* [Community Guidelines](#docker-community-guidelines)
+* [Community Guidelines](#moby-community-guidelines)
 
 ## Reporting security issues
 

Dockerfile

@@ -24,10 +24,10 @@
 # the case. Therefore, you don't have to disable it anymore.
 #
 
-FROM golang:1.10.2 AS base
+FROM golang:1.10.3 AS base
 # FIXME(vdemeester) this is kept for other script depending on it to not fail right away
 # Remove this once the other scripts uses something else to detect the version
-ENV GO_VERSION 1.10.2
+ENV GO_VERSION 1.10.3
 # allow replacing httpredir or deb mirror
 ARG APT_MIRROR=deb.debian.org
 RUN sed -ri "s/(httpredir|deb).debian.org/$APT_MIRROR/g" /etc/apt/sources.list
@@ -170,6 +170,8 @@ RUN PREFIX=/build/ ./install.sh $INSTALL_BINARY_NAME
 FROM runtime-dev AS dev
 RUN groupadd -r docker
 RUN useradd --create-home --gid docker unprivilegeduser
+# Let us use a .bashrc file
+RUN ln -sfv /go/src/github.com/docker/docker/.bashrc ~/.bashrc
 # Activate bash completion and include Docker's completion if mounted with DOCKER_BASH_COMPLETION_PATH
 RUN echo "source /usr/share/bash-completion/bash_completion" >> /etc/bash.bashrc
 RUN ln -s /usr/local/completion/bash/docker /etc/bash_completion.d/docker

Dockerfile.e2e

@@ -1,5 +1,5 @@
 ## Step 1: Build tests
-FROM golang:1.10.2-alpine3.7 as builder
+FROM golang:1.10.3-alpine3.7 as builder
 
 RUN apk add --update \
     bash \

Dockerfile.simple

@@ -42,7 +42,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
 #            will need updating, to avoid errors. Ping #docker-maintainers on IRC
 #            with a heads-up.
 # IMPORTANT: When updating this please note that stdlib archive/tar pkg is vendored
-ENV GO_VERSION 1.10.2
+ENV GO_VERSION 1.10.3
 RUN curl -fsSL "https://golang.org/dl/go${GO_VERSION}.linux-amd64.tar.gz" \
 	| tar -xzC /usr/local
 ENV PATH /go/bin:/usr/local/go/bin:$PATH

Dockerfile.windows

@@ -161,7 +161,7 @@ SHELL ["powershell", "-Command", "$ErrorActionPreference = 'Stop'; $ProgressPref
 # Environment variable notes:
 #  - GO_VERSION must be consistent with 'Dockerfile' used by Linux.
 #  - FROM_DOCKERFILE is used for detection of building within a container.
-ENV GO_VERSION=1.10.2 `
+ENV GO_VERSION=1.10.3 `
     GIT_VERSION=2.11.1 `
     GOPATH=C:\go `
     FROM_DOCKERFILE=1

Makefile

@@ -31,6 +31,7 @@ DOCKER_ENVS := \
 	-e DOCKER_BUILD_ARGS \
 	-e DOCKER_BUILD_GOGC \
 	-e DOCKER_BUILD_PKGS \
+	-e DOCKER_BUILDKIT \
 	-e DOCKER_BASH_COMPLETION_PATH \
 	-e DOCKER_CLI_PATH \
 	-e DOCKER_DEBUG \
@@ -55,7 +56,8 @@ DOCKER_ENVS := \
 	-e https_proxy \
 	-e no_proxy \
 	-e VERSION \
-	-e PLATFORM
+	-e PLATFORM \
+	-e PRODUCT
 # note: we _cannot_ add "-e DOCKER_BUILDTAGS" here because even if it's unset in the shell, that would shadow the "ENV DOCKER_BUILDTAGS" set in our Dockerfile, which is very important for our official builds
 
 # to allow `make BIND_DIR=. shell` or `make BIND_DIR= test`

TESTING.md

@@ -8,11 +8,11 @@ questions you may have as an aspiring Moby contributor.
 Moby has two test suites (and one legacy test suite):
 
 * Unit tests - use standard `go test` and
-  [gotestyourself/assert](https://godoc.org/github.com/gotestyourself/gotestyourself/assert) assertions. They are located in
+  [gotest.tools/assert](https://godoc.org/gotest.tools/assert) assertions. They are located in
   the package they test. Unit tests should be fast and test only their own 
   package.
 * API integration tests - use standard `go test` and
-  [gotestyourself/assert](https://godoc.org/github.com/gotestyourself/gotestyourself/assert) assertions. They are located in
+  [gotest.tools/assert](https://godoc.org/gotest.tools/assert) assertions. They are located in
   `./integration/<component>` directories, where `component` is: container,
   image, volume, etc. These tests perform HTTP requests to an API endpoint and
   check the HTTP response and daemon state after the call.
@@ -47,6 +47,24 @@ Bugs fixes should include a unit test case which exercises the bug.
 A bug fix may also include new assertions in an existing integration tests for the
 API endpoint.
 
+### Integration tests environment considerations
+
+When adding new tests or modifying existing test under `integration/`, testing 
+environment should be properly considered. `skip.If` from 
+[gotest.tools/skip](https://godoc.org/gotest.tools/skip) can be used to make the 
+test run conditionally. Full testing environment conditions can be found at 
+[environment.go](https://github.com/moby/moby/blob/cb37987ee11655ed6bbef663d245e55922354c68/internal/test/environment/environment.go)
+
+Here is a quick example. If the test needs to interact with a docker daemon on 
+the same host, the following condition should be checked within the test code
+
+```go
+skip.If(t, testEnv.IsRemoteDaemon())
+// your integration test code
+```
+
+If a remote daemon is detected, the test will be skipped.
+
 ## Running tests
 
 To run the unit test suite:

api/server/backend/build/backend.go

@@ -8,10 +8,12 @@ import (
 	"github.com/docker/docker/api/types"
 	"github.com/docker/docker/api/types/backend"
 	"github.com/docker/docker/builder"
+	buildkit "github.com/docker/docker/builder/builder-next"
 	"github.com/docker/docker/builder/fscache"
 	"github.com/docker/docker/image"
 	"github.com/docker/docker/pkg/stringid"
 	"github.com/pkg/errors"
+	"golang.org/x/sync/errgroup"
 )
 
 // ImageComponent provides an interface for working with images
@@ -30,24 +32,39 @@ type Backend struct {
 	builder        Builder
 	fsCache        *fscache.FSCache
 	imageComponent ImageComponent
+	buildkit       *buildkit.Builder
 }
 
 // NewBackend creates a new build backend from components
-func NewBackend(components ImageComponent, builder Builder, fsCache *fscache.FSCache) (*Backend, error) {
-	return &Backend{imageComponent: components, builder: builder, fsCache: fsCache}, nil
+func NewBackend(components ImageComponent, builder Builder, fsCache *fscache.FSCache, buildkit *buildkit.Builder) (*Backend, error) {
+	return &Backend{imageComponent: components, builder: builder, fsCache: fsCache, buildkit: buildkit}, nil
 }
 
 // Build builds an image from a Source
 func (b *Backend) Build(ctx context.Context, config backend.BuildConfig) (string, error) {
 	options := config.Options
+	useBuildKit := options.Version == types.BuilderBuildKit
+
 	tagger, err := NewTagger(b.imageComponent, config.ProgressWriter.StdoutFormatter, options.Tags)
 	if err != nil {
 		return "", err
 	}
 
-	build, err := b.builder.Build(ctx, config)
-	if err != nil {
-		return "", err
+	var build *builder.Result
+	if useBuildKit {
+		build, err = b.buildkit.Build(ctx, config)
+		if err != nil {
+			return "", err
+		}
+	} else {
+		build, err = b.builder.Build(ctx, config)
+		if err != nil {
+			return "", err
+		}
+	}
+
+	if build == nil {
+		return "", nil
 	}
 
 	var imageID = build.ImageID
@@ -56,25 +73,54 @@ func (b *Backend) Build(ctx context.Context, config backend.BuildConfig) (string
 			return "", err
 		}
 		if config.ProgressWriter.AuxFormatter != nil {
-			if err = config.ProgressWriter.AuxFormatter.Emit(types.BuildResult{ID: imageID}); err != nil {
+			if err = config.ProgressWriter.AuxFormatter.Emit("moby.image.id", types.BuildResult{ID: imageID}); err != nil {
 				return "", err
 			}
 		}
 	}
 
-	stdout := config.ProgressWriter.StdoutFormatter
-	fmt.Fprintf(stdout, "Successfully built %s\n", stringid.TruncateID(imageID))
-	err = tagger.TagImages(image.ID(imageID))
+	if !useBuildKit {
+		stdout := config.ProgressWriter.StdoutFormatter
+		fmt.Fprintf(stdout, "Successfully built %s\n", stringid.TruncateID(imageID))
+		err = tagger.TagImages(image.ID(imageID))
+	}
 	return imageID, err
 }
 
 // PruneCache removes all cached build sources
 func (b *Backend) PruneCache(ctx context.Context) (*types.BuildCachePruneReport, error) {
-	size, err := b.fsCache.Prune(ctx)
-	if err != nil {
-		return nil, errors.Wrap(err, "failed to prune build cache")
+	eg, ctx := errgroup.WithContext(ctx)
+
+	var fsCacheSize uint64
+	eg.Go(func() error {
+		var err error
+		fsCacheSize, err = b.fsCache.Prune(ctx)
+		if err != nil {
+			return errors.Wrap(err, "failed to prune fscache")
+		}
+		return nil
+	})
+
+	var buildCacheSize int64
+	eg.Go(func() error {
+		var err error
+		buildCacheSize, err = b.buildkit.Prune(ctx)
+		if err != nil {
+			return errors.Wrap(err, "failed to prune build cache")
+		}
+		return nil
+	})
+
+	if err := eg.Wait(); err != nil {
+		return nil, err
 	}
-	return &types.BuildCachePruneReport{SpaceReclaimed: size}, nil
+
+	return &types.BuildCachePruneReport{SpaceReclaimed: fsCacheSize + uint64(buildCacheSize)}, nil
+}
+
+// Cancel cancels the build by ID
+func (b *Backend) Cancel(ctx context.Context, id string) error {
+	return b.buildkit.Cancel(ctx, id)
 }
 
 func squashBuild(build *builder.Result, imageComponent ImageComponent) (string, error) {

api/server/middleware/debug_test.go

@@ -3,8 +3,8 @@ package middleware // import "github.com/docker/docker/api/server/middleware"
 import (
 	"testing"
 
-	"github.com/gotestyourself/gotestyourself/assert"
-	is "github.com/gotestyourself/gotestyourself/assert/cmp"
+	"gotest.tools/assert"
+	is "gotest.tools/assert/cmp"
 )
 
 func TestMaskSecretKeys(t *testing.T) {

api/server/middleware/version_test.go

@@ -8,8 +8,8 @@ import (
 	"testing"
 
 	"github.com/docker/docker/api/server/httputils"
-	"github.com/gotestyourself/gotestyourself/assert"
-	is "github.com/gotestyourself/gotestyourself/assert/cmp"
+	"gotest.tools/assert"
+	is "gotest.tools/assert/cmp"
 )
 
 func TestVersionMiddlewareVersion(t *testing.T) {

api/server/router/build/backend.go

@@ -15,6 +15,8 @@ type Backend interface {
 
 	// Prune build cache
 	PruneCache(context.Context) (*types.BuildCachePruneReport, error)
+
+	Cancel(context.Context, string) error
 }
 
 type experimentalProvider interface {

api/server/router/build/build.go

@@ -25,5 +25,6 @@ func (r *buildRouter) initRoutes() {
 	r.routes = []router.Route{
 		router.NewPostRoute("/build", r.postBuild, router.WithCancel),
 		router.NewPostRoute("/build/prune", r.postPrune, router.WithCancel),
+		router.NewPostRoute("/build/cancel", r.postCancel),
 	}
 }