Merge pull request #32677 from tonistiigi/builder-remote-context-4

Implement long running interactive session and sending build context incrementally

api/server/backend/build/backend.go

@@ -4,11 +4,11 @@ import (
 	"fmt"
 
 	"github.com/docker/distribution/reference"
+	"github.com/docker/docker/api/types"
 	"github.com/docker/docker/api/types/backend"
 	"github.com/docker/docker/builder"
-	"github.com/docker/docker/builder/dockerfile"
+	"github.com/docker/docker/builder/fscache"
 	"github.com/docker/docker/image"
-	"github.com/docker/docker/pkg/idtools"
 	"github.com/docker/docker/pkg/stringid"
 	"github.com/pkg/errors"
 	"golang.org/x/net/context"
@@ -20,16 +20,21 @@ type ImageComponent interface {
 	TagImageWithReference(image.ID, string, reference.Named) error
 }
 
+// Builder defines interface for running a build
+type Builder interface {
+	Build(context.Context, backend.BuildConfig) (*builder.Result, error)
+}
+
 // Backend provides build functionality to the API router
 type Backend struct {
-	manager        *dockerfile.BuildManager
+	builder        Builder
+	fsCache        *fscache.FSCache
 	imageComponent ImageComponent
 }
 
 // NewBackend creates a new build backend from components
-func NewBackend(components ImageComponent, builderBackend builder.Backend, idMappings *idtools.IDMappings) *Backend {
-	manager := dockerfile.NewBuildManager(builderBackend, idMappings)
-	return &Backend{imageComponent: components, manager: manager}
+func NewBackend(components ImageComponent, builder Builder, fsCache *fscache.FSCache) (*Backend, error) {
+	return &Backend{imageComponent: components, builder: builder, fsCache: fsCache}, nil
 }
 
 // Build builds an image from a Source
@@ -40,7 +45,7 @@ func (b *Backend) Build(ctx context.Context, config backend.BuildConfig) (string
 		return "", err
 	}
 
-	build, err := b.manager.Build(ctx, config)
+	build, err := b.builder.Build(ctx, config)
 	if err != nil {
 		return "", err
 	}
@@ -58,6 +63,15 @@ func (b *Backend) Build(ctx context.Context, config backend.BuildConfig) (string
 	return imageID, err
 }
 
+// PruneCache removes all cached build sources
+func (b *Backend) PruneCache(ctx context.Context) (*types.BuildCachePruneReport, error) {
+	size, err := b.fsCache.Prune()
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to prune build cache")
+	}
+	return &types.BuildCachePruneReport{SpaceReclaimed: size}, nil
+}
+
 func squashBuild(build *builder.Result, imageComponent ImageComponent) (string, error) {
 	var fromID string
 	if build.FromImage != nil {

api/server/router/build/backend.go

@@ -1,6 +1,7 @@
 package build
 
 import (
+	"github.com/docker/docker/api/types"
 	"github.com/docker/docker/api/types/backend"
 	"golang.org/x/net/context"
 )
@@ -10,6 +11,9 @@ type Backend interface {
 	// Build a Docker image returning the id of the image
 	// TODO: make this return a reference instead of string
 	Build(context.Context, backend.BuildConfig) (string, error)
+
+	// Prune build cache
+	PruneCache(context.Context) (*types.BuildCachePruneReport, error)
 }
 
 type experimentalProvider interface {

api/server/router/build/build.go

@@ -24,5 +24,6 @@ func (r *buildRouter) Routes() []router.Route {
 func (r *buildRouter) initRoutes() {
 	r.routes = []router.Route{
 		router.NewPostRoute("/build", r.postBuild, router.WithCancel),
+		router.NewPostRoute("/build/prune", r.postPrune, router.WithCancel),
 	}
 }

api/server/router/build/build_routes.go

@@ -127,10 +127,19 @@ func newImageBuildOptions(ctx context.Context, r *http.Request) (*types.ImageBui
 		}
 		options.CacheFrom = cacheFrom
 	}
+	options.SessionID = r.FormValue("session")
 
 	return options, nil
 }
 
+func (br *buildRouter) postPrune(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error {
+	report, err := br.backend.PruneCache(ctx)
+	if err != nil {
+		return err
+	}
+	return httputils.WriteJSON(w, http.StatusOK, report)
+}
+
 func (br *buildRouter) postBuild(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error {
 	var (
 		notVerboseBuffer = bytes.NewBuffer(nil)

api/server/router/session/backend.go

@@ -0,0 +1,12 @@
+package session
+
+import (
+	"net/http"
+
+	"golang.org/x/net/context"
+)
+
+// Backend abstracts an session receiver from an http request.
+type Backend interface {
+	HandleHTTPRequest(ctx context.Context, w http.ResponseWriter, r *http.Request) error
+}

api/server/router/session/session.go

@@ -0,0 +1,29 @@
+package session
+
+import "github.com/docker/docker/api/server/router"
+
+// sessionRouter is a router to talk with the session controller
+type sessionRouter struct {
+	backend Backend
+	routes  []router.Route
+}
+
+// NewRouter initializes a new session router
+func NewRouter(b Backend) router.Router {
+	r := &sessionRouter{
+		backend: b,
+	}
+	r.initRoutes()
+	return r
+}
+
+// Routes returns the available routers to the session controller
+func (r *sessionRouter) Routes() []router.Route {
+	return r.routes
+}
+
+func (r *sessionRouter) initRoutes() {
+	r.routes = []router.Route{
+		router.Experimental(router.NewPostRoute("/session", r.startSession)),
+	}
+}

api/server/router/session/session_routes.go

@@ -0,0 +1,16 @@
+package session
+
+import (
+	"net/http"
+
+	apierrors "github.com/docker/docker/api/errors"
+	"golang.org/x/net/context"
+)
+
+func (sr *sessionRouter) startSession(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error {
+	err := sr.backend.HandleHTTPRequest(ctx, w, r)
+	if err != nil {
+		return apierrors.NewBadRequestError(err)
+	}
+	return nil
+}

api/server/router/system/system.go

@@ -2,6 +2,7 @@ package system
 
 import (
 	"github.com/docker/docker/api/server/router"
+	"github.com/docker/docker/builder/fscache"
 	"github.com/docker/docker/daemon/cluster"
 )
 
@@ -11,13 +12,15 @@ type systemRouter struct {
 	backend Backend
 	cluster *cluster.Cluster
 	routes  []router.Route
+	builder *fscache.FSCache
 }
 
 // NewRouter initializes a new system router
-func NewRouter(b Backend, c *cluster.Cluster) router.Router {
+func NewRouter(b Backend, c *cluster.Cluster, fscache *fscache.FSCache) router.Router {
 	r := &systemRouter{
 		backend: b,
 		cluster: c,
+		builder: fscache,
 	}
 
 	r.routes = []router.Route{

api/server/router/system/system_routes.go

@@ -17,6 +17,7 @@ import (
 	timetypes "github.com/docker/docker/api/types/time"
 	"github.com/docker/docker/api/types/versions"
 	"github.com/docker/docker/pkg/ioutils"
+	pkgerrors "github.com/pkg/errors"
 	"golang.org/x/net/context"
 )
 
@@ -75,6 +76,11 @@ func (s *systemRouter) getDiskUsage(ctx context.Context, w http.ResponseWriter,
 	if err != nil {
 		return err
 	}
+	builderSize, err := s.builder.DiskUsage()
+	if err != nil {
+		return pkgerrors.Wrap(err, "error getting build cache usage")
+	}
+	du.BuilderSize = builderSize
 
 	return httputils.WriteJSON(w, http.StatusOK, du)
 }

api/swagger.yaml

@@ -4745,6 +4745,27 @@ paths:
           schema:
             $ref: "#/definitions/ErrorResponse"
       tags: ["Image"]
+  /build/prune:
+    post:
+      summary: "Delete builder cache"
+      produces:
+        - "application/json"
+      operationId: "BuildPrune"
+      responses:
+        200:
+          description: "No error"
+          schema:
+            type: "object"
+            properties:
+              SpaceReclaimed:
+                description: "Disk space reclaimed in bytes"
+                type: "integer"
+                format: "int64"
+        500:
+          description: "Server error"
+          schema:
+            $ref: "#/definitions/ErrorResponse"
+      tags: ["Image"]
   /images/create:
     post:
       summary: "Create an image"
@@ -8481,3 +8502,43 @@ paths:
           type: "string"
           required: true
       tags: ["Distribution"]
+  /session:
+    post:
+      summary: "Initialize interactive session"
+      description: |
+        Start a new interactive session with a server. Session allows server to call back to the client for advanced capabilities.
+
+        ### Hijacking
+
+        This endpoint hijacks the HTTP connection to HTTP2 transport that allows the client to expose gPRC services on that connection.
+
+        For example, the client sends this request to upgrade the connection:
+
+        ```
+        POST /session HTTP/1.1
+        Upgrade: h2c
+        Connection: Upgrade
+        ```
+
+        The Docker daemon will respond with a `101 UPGRADED` response follow with the raw stream:
+
+        ```
+        HTTP/1.1 101 UPGRADED
+        Connection: Upgrade
+        Upgrade: h2c
+        ```
+      operationId: "Session"
+      produces:
+        - "application/vnd.docker.raw-stream"
+      responses:
+        101:
+          description: "no error, hijacking successful"
+        400:
+          description: "bad parameter"
+          schema:
+            $ref: "#/definitions/ErrorResponse"
+        500:
+          description: "server error"
+          schema:
+            $ref: "#/definitions/ErrorResponse"
+      tags: ["Session"]

api/types/client.go

@@ -7,7 +7,7 @@ import (
 
 	"github.com/docker/docker/api/types/container"
 	"github.com/docker/docker/api/types/filters"
-	"github.com/docker/go-units"
+	units "github.com/docker/go-units"
 )
 
 // CheckpointCreateOptions holds parameters to create a checkpoint from a container
@@ -178,6 +178,7 @@ type ImageBuildOptions struct {
 	SecurityOpt []string
 	ExtraHosts  []string // List of extra hosts
 	Target      string
+	SessionID   string
 
 	// TODO @jhowardmsft LCOW Support: This will require extending to include
 	// `Platform string`, but is ommited for now as it's hard-coded temporarily

api/types/types.go

@@ -489,10 +489,11 @@ type Runtime struct {
 // DiskUsage contains response of Engine API:
 // GET "/system/df"
 type DiskUsage struct {
-	LayersSize int64
-	Images     []*ImageSummary
-	Containers []*Container
-	Volumes    []*Volume
+	LayersSize  int64
+	Images      []*ImageSummary
+	Containers  []*Container
+	Volumes     []*Volume
+	BuilderSize int64
 }
 
 // ContainersPruneReport contains the response for Engine API:
@@ -516,6 +517,12 @@ type ImagesPruneReport struct {
 	SpaceReclaimed uint64
 }
 
+// BuildCachePruneReport contains the response for Engine API:
+// POST "/build/prune"
+type BuildCachePruneReport struct {
+	SpaceReclaimed uint64
+}
+
 // NetworksPruneReport contains the response for Engine API:
 // POST "/networks/prune"
 type NetworksPruneReport struct {