Implement content addressability for plugins

Move plugins to shared distribution stack with images.

Create immutable plugin config that matches schema2 requirements.

Ensure data being pushed is same as pulled/created.

Store distribution artifacts in a blobstore.

Run init layer setup for every plugin start.

Fix breakouts from unsafe file accesses.

Add support for `docker plugin install --alias`

Uses normalized references for default names to avoid collisions when using default hosts/tags.

Some refactoring of the plugin manager to support the change, like removing the singleton manager and adding manager config struct.

Signed-off-by: Tonis Tiigi <tonistiigi@gmail.com>
Signed-off-by: Derek McGowan <derek@mcgstyle.net>

api/server/router/plugin/backend.go

@@ -5,6 +5,7 @@ import (
 	"net/http"
 
 	enginetypes "github.com/docker/docker/api/types"
+	"github.com/docker/docker/reference"
 	"golang.org/x/net/context"
 )
 
@@ -13,11 +14,11 @@ type Backend interface {
 	Disable(name string, config *enginetypes.PluginDisableConfig) error
 	Enable(name string, config *enginetypes.PluginEnableConfig) error
 	List() ([]enginetypes.Plugin, error)
-	Inspect(name string) (enginetypes.Plugin, error)
+	Inspect(name string) (*enginetypes.Plugin, error)
 	Remove(name string, config *enginetypes.PluginRmConfig) error
 	Set(name string, args []string) error
-	Privileges(name string, metaHeaders http.Header, authConfig *enginetypes.AuthConfig) (enginetypes.PluginPrivileges, error)
-	Pull(name string, metaHeaders http.Header, authConfig *enginetypes.AuthConfig, privileges enginetypes.PluginPrivileges) error
-	Push(name string, metaHeaders http.Header, authConfig *enginetypes.AuthConfig) error
-	CreateFromContext(ctx context.Context, tarCtx io.Reader, options *enginetypes.PluginCreateOptions) error
+	Privileges(ctx context.Context, ref reference.Named, metaHeaders http.Header, authConfig *enginetypes.AuthConfig) (enginetypes.PluginPrivileges, error)
+	Pull(ctx context.Context, ref reference.Named, name string, metaHeaders http.Header, authConfig *enginetypes.AuthConfig, privileges enginetypes.PluginPrivileges, outStream io.Writer) error
+	Push(ctx context.Context, name string, metaHeaders http.Header, authConfig *enginetypes.AuthConfig, outStream io.Writer) error
+	CreateFromContext(ctx context.Context, tarCtx io.ReadCloser, options *enginetypes.PluginCreateOptions) error
 }

api/server/router/plugin/plugin.go

@@ -30,8 +30,8 @@ func (r *pluginRouter) initRoutes() {
 		router.NewDeleteRoute("/plugins/{name:.*}", r.removePlugin),
 		router.NewPostRoute("/plugins/{name:.*}/enable", r.enablePlugin), // PATCH?
 		router.NewPostRoute("/plugins/{name:.*}/disable", r.disablePlugin),
-		router.NewPostRoute("/plugins/pull", r.pullPlugin),
-		router.NewPostRoute("/plugins/{name:.*}/push", r.pushPlugin),
+		router.Cancellable(router.NewPostRoute("/plugins/pull", r.pullPlugin)),
+		router.Cancellable(router.NewPostRoute("/plugins/{name:.*}/push", r.pushPlugin)),
 		router.NewPostRoute("/plugins/{name:.*}/set", r.setPlugin),
 		router.NewPostRoute("/plugins/create", r.createPlugin),
 	}

api/server/router/plugin/plugin_routes.go

@@ -7,8 +7,13 @@ import (
 	"strconv"
 	"strings"
 
+	distreference "github.com/docker/distribution/reference"
 	"github.com/docker/docker/api/server/httputils"
 	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/pkg/ioutils"
+	"github.com/docker/docker/pkg/streamformatter"
+	"github.com/docker/docker/reference"
+	"github.com/pkg/errors"
 	"golang.org/x/net/context"
 )
 
@@ -34,14 +39,61 @@ func parseHeaders(headers http.Header) (map[string][]string, *types.AuthConfig)
 	return metaHeaders, authConfig
 }
 
+// parseRemoteRef parses the remote reference into a reference.Named
+// returning the tag associated with the reference. In the case the
+// given reference string includes both digest and tag, the returned
+// reference will have the digest without the tag, but the tag will
+// be returned.
+func parseRemoteRef(remote string) (reference.Named, string, error) {
+	// Parse remote reference, supporting remotes with name and tag
+	// NOTE: Using distribution reference to handle references
+	// containing both a name and digest
+	remoteRef, err := distreference.ParseNamed(remote)
+	if err != nil {
+		return nil, "", err
+	}
+
+	var tag string
+	if t, ok := remoteRef.(distreference.Tagged); ok {
+		tag = t.Tag()
+	}
+
+	// Convert distribution reference to docker reference
+	// TODO: remove when docker reference changes reconciled upstream
+	ref, err := reference.WithName(remoteRef.Name())
+	if err != nil {
+		return nil, "", err
+	}
+	if d, ok := remoteRef.(distreference.Digested); ok {
+		ref, err = reference.WithDigest(ref, d.Digest())
+		if err != nil {
+			return nil, "", err
+		}
+	} else if tag != "" {
+		ref, err = reference.WithTag(ref, tag)
+		if err != nil {
+			return nil, "", err
+		}
+	} else {
+		ref = reference.WithDefaultTag(ref)
+	}
+
+	return ref, tag, nil
+}
+
 func (pr *pluginRouter) getPrivileges(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error {
 	if err := httputils.ParseForm(r); err != nil {
 		return err
 	}
 
 	metaHeaders, authConfig := parseHeaders(r.Header)
 
-	privileges, err := pr.backend.Privileges(r.FormValue("name"), metaHeaders, authConfig)
+	ref, _, err := parseRemoteRef(r.FormValue("remote"))
+	if err != nil {
+		return err
+	}
+
+	privileges, err := pr.backend.Privileges(ctx, ref, metaHeaders, authConfig)
 	if err != nil {
 		return err
 	}
@@ -50,20 +102,66 @@ func (pr *pluginRouter) getPrivileges(ctx context.Context, w http.ResponseWriter
 
 func (pr *pluginRouter) pullPlugin(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error {
 	if err := httputils.ParseForm(r); err != nil {
-		return err
+		return errors.Wrap(err, "failed to parse form")
 	}
 
 	var privileges types.PluginPrivileges
-	if err := json.NewDecoder(r.Body).Decode(&privileges); err != nil {
-		return err
+	dec := json.NewDecoder(r.Body)
+	if err := dec.Decode(&privileges); err != nil {
+		return errors.Wrap(err, "failed to parse privileges")
+	}
+	if dec.More() {
+		return errors.New("invalid privileges")
 	}
 
 	metaHeaders, authConfig := parseHeaders(r.Header)
 
-	if err := pr.backend.Pull(r.FormValue("name"), metaHeaders, authConfig, privileges); err != nil {
+	ref, tag, err := parseRemoteRef(r.FormValue("remote"))
+	if err != nil {
 		return err
 	}
-	w.WriteHeader(http.StatusCreated)
+
+	name := r.FormValue("name")
+	if name == "" {
+		if _, ok := ref.(reference.Canonical); ok {
+			trimmed := reference.TrimNamed(ref)
+			if tag != "" {
+				nt, err := reference.WithTag(trimmed, tag)
+				if err != nil {
+					return err
+				}
+				name = nt.String()
+			} else {
+				name = reference.WithDefaultTag(trimmed).String()
+			}
+		} else {
+			name = ref.String()
+		}
+	} else {
+		localRef, err := reference.ParseNamed(name)
+		if err != nil {
+			return err
+		}
+		if _, ok := localRef.(reference.Canonical); ok {
+			return errors.New("cannot use digest in plugin tag")
+		}
+		if distreference.IsNameOnly(localRef) {
+			// TODO: log change in name to out stream
+			name = reference.WithDefaultTag(localRef).String()
+		}
+	}
+	w.Header().Set("Docker-Plugin-Name", name)
+
+	w.Header().Set("Content-Type", "application/json")
+	output := ioutils.NewWriteFlusher(w)
+
+	if err := pr.backend.Pull(ctx, ref, name, metaHeaders, authConfig, privileges, output); err != nil {
+		if !output.Flushed() {
+			return err
+		}
+		output.Write(streamformatter.NewJSONStreamFormatter().FormatError(err))
+	}
+
 	return nil
 }
 
@@ -125,12 +223,21 @@ func (pr *pluginRouter) removePlugin(ctx context.Context, w http.ResponseWriter,
 
 func (pr *pluginRouter) pushPlugin(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error {
 	if err := httputils.ParseForm(r); err != nil {
-		return err
+		return errors.Wrap(err, "failed to parse form")
 	}
 
 	metaHeaders, authConfig := parseHeaders(r.Header)
 
-	return pr.backend.Push(vars["name"], metaHeaders, authConfig)
+	w.Header().Set("Content-Type", "application/json")
+	output := ioutils.NewWriteFlusher(w)
+
+	if err := pr.backend.Push(ctx, vars["name"], metaHeaders, authConfig, output); err != nil {
+		if !output.Flushed() {
+			return err
+		}
+		output.Write(streamformatter.NewJSONStreamFormatter().FormatError(err))
+	}
+	return nil
 }
 
 func (pr *pluginRouter) setPlugin(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error {

api/swagger.yaml

@@ -1347,16 +1347,13 @@ definitions:
   Plugin:
     description: "A plugin for the Engine API"
     type: "object"
-    required: [Settings, Enabled, Config, Name, Tag]
+    required: [Settings, Enabled, Config, Name]
     properties:
       Id:
         type: "string"
       Name:
         type: "string"
         x-nullable: false
-      Tag:
-        type: "string"
-        x-nullable: false
       Enabled:
         description: "True when the plugin is running. False when the plugin is not running, only installed."
         type: "boolean"
@@ -1392,7 +1389,7 @@ definitions:
           - Documentation
           - Interface
           - Entrypoint
-          - Workdir
+          - WorkDir
           - Network
           - Linux
           - PropagatedMount
@@ -1423,7 +1420,7 @@ definitions:
             type: "array"
             items:
               type: "string"
-          Workdir:
+          WorkDir:
             type: "string"
             x-nullable: false
           User:
@@ -1490,6 +1487,15 @@ definitions:
                 type: "array"
                 items:
                   type: "string"
+          rootfs:
+            type: "object"
+            properties:
+              type:
+                type: "string"
+              diff_ids:
+                type: "array"
+                items:
+                  type: "string"
     example:
       Id: "5724e2c8652da337ab2eedd19fc6fc0ec908e4bd907c7421bf6a8dfc70c4c078"
       Name: "tiborvass/no-remove"
@@ -1528,7 +1534,7 @@ definitions:
         Entrypoint:
           - "plugin-no-remove"
           - "/data"
-        Workdir: ""
+        WorkDir: ""
         User: {}
         Network:
           Type: "host"
@@ -6397,7 +6403,7 @@ paths:
                   Entrypoint:
                     - "plugin-no-remove"
                     - "/data"
-                  Workdir: ""
+                  WorkDir: ""
                   User: {}
                   Network:
                     Type: "host"
@@ -6503,14 +6509,22 @@ paths:
           schema:
             $ref: "#/definitions/ErrorResponse"
       parameters:
-        - name: "name"
+        - name: "remote"
           in: "query"
           description: |
-            The plugin to install.
+            Remote reference for plugin to install.
 
             The `:latest` tag is optional, and is used as the default if omitted.
           required: true
           type: "string"
+        - name: "name"
+          in: "query"
+          description: |
+            Local name for the pulled plugin.
+
+            The `:latest` tag is optional, and is used as the default if omitted.
+          required: false
+          type: "string"
         - name: "X-Registry-Auth"
           in: "header"
           description: "A base64-encoded auth configuration to use when pulling a plugin from a registry. [See the authentication section for details.](#section/Authentication)"

api/types/client.go

@@ -350,6 +350,7 @@ type PluginInstallOptions struct {
 	Disabled              bool
 	AcceptAllPermissions  bool
 	RegistryAuth          string // RegistryAuth is the base64 encoded credentials for the registry
+	RemoteRef             string // RemoteRef is the plugin name on the registry
 	PrivilegeFunc         RequestPrivilegeFunc
 	AcceptPermissionsFunc func(PluginPrivileges) (bool, error)
 	Args                  []string

api/types/plugin.go

@@ -25,10 +25,6 @@ type Plugin struct {
 	// settings
 	// Required: true
 	Settings PluginSettings `json:"Settings"`
-
-	// tag
-	// Required: true
-	Tag string `json:"Tag"`
 }
 
 // PluginConfig The config of a plugin.
@@ -78,9 +74,12 @@ type PluginConfig struct {
 	// user
 	User PluginConfigUser `json:"User,omitempty"`
 
-	// workdir
+	// work dir
 	// Required: true
-	Workdir string `json:"Workdir"`
+	WorkDir string `json:"WorkDir"`
+
+	// rootfs
+	Rootfs *PluginConfigRootfs `json:"rootfs,omitempty"`
 }
 
 // PluginConfigArgs plugin config args
@@ -143,6 +142,17 @@ type PluginConfigNetwork struct {
 	Type string `json:"Type"`
 }
 
+// PluginConfigRootfs plugin config rootfs
+// swagger:model PluginConfigRootfs
+type PluginConfigRootfs struct {
+
+	// diff ids
+	DiffIds []string `json:"diff_ids"`
+
+	// type
+	Type string `json:"type,omitempty"`
+}
+
 // PluginConfigUser plugin config user
 // swagger:model PluginConfigUser
 type PluginConfigUser struct {

cli/command/plugin/create.go

@@ -64,8 +64,8 @@ func newCreateCommand(dockerCli *command.DockerCli) *cobra.Command {
 	options := pluginCreateOptions{}
 
 	cmd := &cobra.Command{
-		Use:   "create [OPTIONS] PLUGIN[:tag] PATH-TO-ROOTFS(rootfs + config.json)",
-		Short: "Create a plugin from a rootfs and config",
+		Use:   "create [OPTIONS] PLUGIN PLUGIN-DATA-DIR",
+		Short: "Create a plugin from a rootfs and configuration. Plugin data directory must contain config.json and rootfs directory.",
 		Args:  cli.RequiresMinArgs(2),
 		RunE: func(cmd *cobra.Command, args []string) error {
 			options.repoName = args[0]

cli/command/plugin/disable.go

@@ -6,7 +6,6 @@ import (
 	"github.com/docker/docker/api/types"
 	"github.com/docker/docker/cli"
 	"github.com/docker/docker/cli/command"
-	"github.com/docker/docker/reference"
 	"github.com/spf13/cobra"
 	"golang.org/x/net/context"
 )
@@ -29,18 +28,7 @@ func newDisableCommand(dockerCli *command.DockerCli) *cobra.Command {
 }
 
 func runDisable(dockerCli *command.DockerCli, name string, force bool) error {
-	named, err := reference.ParseNamed(name) // FIXME: validate
-	if err != nil {
-		return err
-	}
-	if reference.IsNameOnly(named) {
-		named = reference.WithDefaultTag(named)
-	}
-	ref, ok := named.(reference.NamedTagged)
-	if !ok {
-		return fmt.Errorf("invalid name: %s", named.String())
-	}
-	if err := dockerCli.Client().PluginDisable(context.Background(), ref.String(), types.PluginDisableOptions{Force: force}); err != nil {
+	if err := dockerCli.Client().PluginDisable(context.Background(), name, types.PluginDisableOptions{Force: force}); err != nil {
 		return err
 	}
 	fmt.Fprintln(dockerCli.Out(), name)